From d14845c5d5314ceeb6fe336caf9ed10f3d7f3127 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 14 Apr 2023 20:02:45 +1000 Subject: [PATCH 01/25] feat(elsa): add htsget-elsa crate --- .gitignore | 11 +- Cargo.lock | 3435 ++++++++++++++++++++++++++++++ Cargo.toml | 5 + htsget-elsa/Cargo.lock | 7 + htsget-elsa/Cargo.toml | 14 + htsget-elsa/src/dynamodb.rs | 32 + htsget-elsa/src/elsa_endpoint.rs | 64 + htsget-elsa/src/lib.rs | 19 + 8 files changed, 3577 insertions(+), 10 deletions(-) create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 htsget-elsa/Cargo.lock create mode 100644 htsget-elsa/Cargo.toml create mode 100644 htsget-elsa/src/dynamodb.rs create mode 100644 htsget-elsa/src/elsa_endpoint.rs create mode 100644 htsget-elsa/src/lib.rs diff --git a/.gitignore b/.gitignore index 088ba6b..ea8c4bf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1 @@ -# Generated by Cargo -# will have compiled files and executables -/target/ - -# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries -# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html -Cargo.lock - -# These are backup files generated by rustfmt -**/*.rs.bk +/target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..925858c --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3435 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "assert-json-diff" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4259cbe96513d2f1073027a259fc2ca917feb3026a5a8d984e3628e490255cc0" +dependencies = [ + "extend", + "serde", + "serde_json", +] + +[[package]] +name = "async-compression" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-stream" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "async-trait" +version = "0.1.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ea188f25f0255d8f92797797c97ebf5631fa88178beb1a46fdf5622c9a00e4" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.8", +] + +[[package]] +name = "atomic" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b88d82667eca772c4aa12f0f1348b3ae643424c8876448f3f7bd5787032e234c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "aws-config" +version = "0.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3d1e2a1f1ab3ac6c4b884e37413eaa03eb9d901e4fc68ee8f5c1d49721680e" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-sdk-sso", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "hex", + "http", + "hyper", + "ring", + "time", + "tokio", + "tower", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "0.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0696a0523a39a19087747e4dafda0362dc867531e3d72a3f195564c84e5e08" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-endpoint" +version = "0.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80a4f935ab6a1919fbfd6102a80c4fccd9ff5f47f94ba154074afe1051903261" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "aws-types", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-http" +version = "0.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82976ca4e426ee9ca3ffcf919d9b2c8d14d0cd80d43cc02173737a8f07f28d4d" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "http-body", + "lazy_static", + "percent-encoding", + "pin-project-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-dynamodb" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34fc8efd8ed65312b05dc84bb0d7ba79c31802873c4b5676403e220724af39b7" +dependencies = [ + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http", + "regex", + "tokio-stream", + "tower", + "tracing", +] + +[[package]] +name = "aws-sdk-s3" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1533be023eeac69668eb718b1c48af7bd5e26305ed770553d2877ab1f7507b68" +dependencies = [ + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-client", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http", + "http-body", + "once_cell", + "percent-encoding", + "regex", + "tokio-stream", + "tower", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca0119bacf0c42f587506769390983223ba834e605f049babe514b2bd646dbb2" +dependencies = [ + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "regex", + "tokio-stream", + "tower", +] + +[[package]] +name = "aws-sdk-sts" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "270b6a33969ebfcb193512fbd5e8ee5306888ad6c6d5d775cdbfb2d50d94de26" +dependencies = [ + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "regex", + "tower", + "tracing", +] + +[[package]] +name = "aws-sig-auth" +version = "0.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "660a02a98ab1af83bd8d714afbab2d502ba9b18c49e7e4cddd6bf8837ff778cb" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-types", + "http", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "0.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdaf11005b7444e6cd66f600d09861a3aeb6eb89a0f003c7c9820dbab2d15297" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-http", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http", + "once_cell", + "percent-encoding", + "regex", + "sha2", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63c712a28a4f2f2139759235c08bf98aca99d4fdf1b13c78c5f95613df0a5db9" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", + "tokio-stream", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3875fb4b28606a5368a048016a28c15707f2b21238d5b2e4a23198f590e92c4" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http", + "http-body", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-client" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "104ca17f56cde00a10207169697dfe9c6810db339d52fb352707e64875b30a44" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-protocol-test", + "aws-smithy-types", + "bytes", + "fastrand", + "http", + "http-body", + "hyper", + "hyper-rustls", + "lazy_static", + "pin-project-lite", + "serde", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac250d8c0e42af0097a6837ffc5a6fb9f8ba4107bb53124c047c91bc2a58878f" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873f316f1833add0d3aa54ed1b0cd252ddd88c792a0cf839886400099971e844" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "hyper", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "aws-smithy-http-tower" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f38231d3f5dac9ac7976f44e12803add1385119ffca9e5f050d8e980733d164" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "http", + "http-body", + "pin-project-lite", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd83ff2b79e9f729746fcc8ad798676b68fe6ea72986571569a5306a277a182" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-protocol-test" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d1c9bcb35ce11055ec128dab2c66a7ed47e2dfff99883e32c21a1ab6d6bee6" +dependencies = [ + "assert-json-diff", + "http", + "pretty_assertions", + "regex", + "roxmltree", + "serde_json", + "thiserror", +] + +[[package]] +name = "aws-smithy-query" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2f0445dafe9d2cd50b44339ae3c3ed46549aad8ac696c52ad660b3e7ae8682b" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-types" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8161232eda10290f5136610a1eb9de56aceaccd70c963a26a260af20ac24794f" +dependencies = [ + "base64-simd", + "itoa", + "num-integer", + "ryu", + "time", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.54.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343ffe9a9bb3f542675f4df0e0d5933513d6ad038ca3907ad1767ba690a99684" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "0.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8f15b34253b68cde08e39b0627cc6101bcca64351229484b4743392c035d057" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-types", + "http", + "rustc_version", + "tracing", +] + +[[package]] +name = "aws_lambda_events" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d376097ccf49a2699d554558267b2735f62ce94ee1840a9f73253ad7c26fbaa7" +dependencies = [ + "base64 0.13.1", + "bytes", + "chrono", + "http", + "http-body", + "http-serde", + "query_map", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "axum" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f8ccfd9221ee7d1f3d4b33e1f8319b3a81ed8f61f2ea40b37b859794b4491" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2f958c80c248b34b9a877a643811be8dbca03ca5ba827f2b63baf3a81e5fc4e" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-extra" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51227033e4d3acad15c879092ac8a228532707b5db5ff2628f638334f63e1b7a" +dependencies = [ + "axum", + "bytes", + "futures-util", + "http", + "mime", + "pin-project-lite", + "tokio", + "tower", + "tower-http", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +dependencies = [ + "serde", +] + +[[package]] +name = "bytes-utils" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" +dependencies = [ + "bytes", + "either", +] + +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +dependencies = [ + "iana-time-zone", + "num-integer", + "num-traits", + "serde", + "winapi", +] + +[[package]] +name = "clap" +version = "4.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c911b090850d79fc64fe9ea01e28e465f65e821e08813ced95bced72f7a8a9b" +dependencies = [ + "bitflags", + "clap_derive", + "clap_lex", + "is-terminal", + "once_cell", + "strsim", + "termcolor", +] + +[[package]] +name = "clap_derive" +version = "4.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a932373bab67b984c790ddf2c9ca295d8e3af3b7ef92de5a5bacdccdee4b09b" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.8", +] + +[[package]] +name = "clap_lex" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "033f6b7a4acb1f358c742aaca805c939ee73b4c6209ae4318ec7aca81c42e646" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "cpufeatures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32c" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfea2db42e9927a3845fb268a10a72faed6d416065f77873f05e411457c363e" +dependencies = [ + "rustc_version", +] + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctor" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "cxx" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c00419335c41018365ddf7e4d5f1c12ee3659ddcf3e01974650ba1de73d038" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb8307ad413a98fff033c8545ecf133e3257747b3bae935e7602aab8aa92d4ca" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.8", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc52e2eb08915cb12596d29d55f0b5384f00d697a646dbd269b6ecb0fbd9d31" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631569015d0d8d54e6c241733f944042623ab6df7bc3be7466874b05fcdb1c5f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.8", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "digest" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "encoding_rs" +version = "0.8.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "extend" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f47da3a72ec598d9c8937a7ebca8962a5c7a1f28444e38c2b33c771ba3f55f05" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "figment" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e56602b469b2201400dec66a66aec5a9b8761ee97cd1b8c96ab2483fcc16cc9" +dependencies = [ + "atomic", + "pear", + "serde", + "toml 0.5.11", + "uncased", + "version_check", +] + +[[package]] +name = "flate2" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531ac96c6ff5fd7c62263c5e3c67a603af4fcaee2e1a0ae5565ba3a11e69e549" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" + +[[package]] +name = "futures-executor" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1997dd9df74cdac935c76252744c1ed5794fac083242ea4fe77ef3ed60ba0f83" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" + +[[package]] +name = "futures-macro" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "futures-sink" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" + +[[package]] +name = "futures-task" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" + +[[package]] +name = "futures-util" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "h2" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "htsget-config" +version = "0.1.4" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#d2d9aa7378c8cffa1873b9c17c57ce63febd9813" +dependencies = [ + "async-trait", + "clap", + "figment", + "http", + "http-serde", + "noodles", + "regex", + "serde", + "serde_regex", + "serde_with", + "thiserror", + "toml 0.7.3", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "htsget-elsa" +version = "0.1.0" +dependencies = [ + "async-trait", + "aws-config", + "aws-sdk-dynamodb", + "htsget-config", + "http", + "reqwest", + "thiserror", +] + +[[package]] +name = "htsget-elsa-lambda" +version = "0.1.0" +dependencies = [ + "htsget-config", + "htsget-lambda", + "http", + "http-serde", + "lambda_http", + "serde", + "serde_json", + "tokio", + "tracing", +] + +[[package]] +name = "htsget-http" +version = "0.1.4" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#d2d9aa7378c8cffa1873b9c17c57ce63febd9813" +dependencies = [ + "futures", + "htsget-config", + "htsget-search", + "htsget-test", + "http", + "serde", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "htsget-lambda" +version = "0.1.4" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#d2d9aa7378c8cffa1873b9c17c57ce63febd9813" +dependencies = [ + "bytes", + "htsget-config", + "htsget-http", + "htsget-search", + "htsget-test", + "lambda_http", + "lambda_runtime", + "mime", + "regex", + "serde", + "serde_json", + "tokio", + "tower-http", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "htsget-search" +version = "0.1.4" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#d2d9aa7378c8cffa1873b9c17c57ce63febd9813" +dependencies = [ + "async-trait", + "aws-config", + "aws-sdk-s3", + "axum", + "axum-extra", + "base64 0.21.0", + "bytes", + "futures", + "futures-util", + "htsget-config", + "htsget-test", + "http", + "hyper", + "noodles", + "rustls-pemfile", + "serde", + "thiserror", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tracing", +] + +[[package]] +name = "htsget-test" +version = "0.1.4" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#d2d9aa7378c8cffa1873b9c17c57ce63febd9813" +dependencies = [ + "async-trait", + "base64 0.21.0", + "futures", + "htsget-config", + "http", + "mime", + "noodles-bgzf", + "noodles-vcf", + "rcgen", + "reqwest", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + +[[package]] +name = "http-serde" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e272971f774ba29341db2f686255ff8a979365a26fb9e4277f6b6d9ec0cdd5e" +dependencies = [ + "http", + "serde", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +dependencies = [ + "http", + "hyper", + "log", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +dependencies = [ + "autocfg", + "hashbrown", + "serde", +] + +[[package]] +name = "inlinable_string" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys 0.45.0", +] + +[[package]] +name = "ipnet" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" + +[[package]] +name = "is-terminal" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8687c819457e979cc940d09cb16e42a1bf70aa6b60a549de6d3a62a0ee90c69e" +dependencies = [ + "hermit-abi 0.3.1", + "io-lifetimes", + "rustix", + "windows-sys 0.45.0", +] + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lambda_http" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16fd842ce9fc6908f1688030cf8b6841e2009bd874eb21244f124570ac06264f" +dependencies = [ + "aws_lambda_events", + "base64 0.13.1", + "bytes", + "encoding_rs", + "http", + "http-body", + "hyper", + "lambda_runtime", + "mime", + "percent-encoding", + "serde", + "serde_json", + "serde_urlencoded", + "url", +] + +[[package]] +name = "lambda_runtime" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd32d5799db2155ae4d47116bb3e169b59f531ced4d5762a10c2125bdd2bf134" +dependencies = [ + "async-stream", + "bytes", + "futures", + "http", + "hyper", + "lambda_runtime_api_client", + "serde", + "serde_json", + "tokio", + "tokio-stream", + "tower", + "tracing", +] + +[[package]] +name = "lambda_runtime_api_client" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7210012be904051520f0dc502140ba599bae3042b65b3737b87727f1aa88a7d6" +dependencies = [ + "http", + "hyper", + "tokio", + "tower-service", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" + +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" + +[[package]] +name = "md-5" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +dependencies = [ + "digest", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.45.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "noodles" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34d8a52c91a7fb516482ebae88106109900f65308915900a95049c994e58deb5" +dependencies = [ + "noodles-bam", + "noodles-bcf", + "noodles-bgzf", + "noodles-core", + "noodles-cram", + "noodles-csi", + "noodles-fasta", + "noodles-fastq", + "noodles-sam", + "noodles-tabix", + "noodles-vcf", +] + +[[package]] +name = "noodles-bam" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a5d3990c4cde117936769f9b3ea3f0beaf13e0b7d8850247c931e90af26955" +dependencies = [ + "bit-vec", + "byteorder", + "bytes", + "futures", + "noodles-bgzf", + "noodles-core", + "noodles-csi", + "noodles-fasta", + "noodles-sam", + "tokio", +] + +[[package]] +name = "noodles-bcf" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b5979fec9d28d2d4552e85a04f289e26daad63dd6a4693d9c128d2a457c1c7" +dependencies = [ + "byteorder", + "futures", + "indexmap", + "noodles-bgzf", + "noodles-core", + "noodles-csi", + "noodles-vcf", + "tokio", +] + +[[package]] +name = "noodles-bgzf" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d6971da2c55bdd0ed57058348ba52b11dedf0c33180ac6d0853c9ecde5bbdf" +dependencies = [ + "byteorder", + "bytes", + "crossbeam-channel", + "flate2", + "futures", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "noodles-core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f52f0dbc792f4d76e94e3ccaee57237b649ad1e4b64a97ad037d6b3b8f6909" + +[[package]] +name = "noodles-cram" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d83ba5b9eb0161c7fe4385aaa99aac720f216b7115a8856d8dca075244ad803" +dependencies = [ + "async-compression", + "bitflags", + "byteorder", + "bytes", + "bzip2", + "flate2", + "futures", + "md-5", + "noodles-bam", + "noodles-core", + "noodles-fasta", + "noodles-sam", + "pin-project-lite", + "tokio", + "xz2", +] + +[[package]] +name = "noodles-csi" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4797afdb8fc80e7ddc0d6a3fa33163c2473fec431c4838e67d7f957f68ede527" +dependencies = [ + "bit-vec", + "byteorder", + "noodles-bgzf", + "noodles-core", + "tokio", +] + +[[package]] +name = "noodles-fasta" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3264d8ce30f2959a84339366548afca3fb5d4043a9f8853ca448411f305958" +dependencies = [ + "bytes", + "memchr", + "noodles-bgzf", + "noodles-core", + "tokio", +] + +[[package]] +name = "noodles-fastq" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c7d065ab7c0814ea9df05624a1be1410d2039b091213ac1cb5281866c3fdfb" +dependencies = [ + "futures", + "tokio", +] + +[[package]] +name = "noodles-sam" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "647b9f1b0b3815ee21aef82823bd9062c40706c6d9148935aef3e2b7a9c572f6" +dependencies = [ + "bitflags", + "futures", + "indexmap", + "lexical-core", + "memchr", + "noodles-bgzf", + "noodles-core", + "noodles-csi", + "noodles-fasta", + "tokio", +] + +[[package]] +name = "noodles-tabix" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c74c56094217a32748db1d8457f39f07b6695b5c23e71919fba9784bd6f929f4" +dependencies = [ + "bit-vec", + "byteorder", + "indexmap", + "noodles-bgzf", + "noodles-core", + "noodles-csi", + "tokio", +] + +[[package]] +name = "noodles-vcf" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "008dc218516a5363d7f55add28866c20463f14ffcb661cbfc4863343e1486a20" +dependencies = [ + "futures", + "indexmap", + "memchr", + "nom", + "noodles-bgzf", + "noodles-core", + "noodles-csi", + "noodles-tabix", + "percent-encoding", + "tokio", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi 0.2.6", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "openssl" +version = "0.10.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "518915b97df115dd36109bfa429a48b8f737bd05508cf9588977b599648926d2" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "666416d899cf077260dac8698d60a60b435a46d57e82acb1be3d0dad87284e5b" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "os_str_bytes" +version = "6.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" + +[[package]] +name = "output_vt100" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66" +dependencies = [ + "winapi", +] + +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "pear" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec95680a7087503575284e5063e14b694b7a9c0b065e5dceec661e0497127e8" +dependencies = [ + "inlinable_string", + "pear_codegen", + "yansi", +] + +[[package]] +name = "pear_codegen" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9661a3a53f93f09f2ea882018e4d7c88f6ff2956d809a276060476fd8c879d3c" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.8", +] + +[[package]] +name = "pem" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +dependencies = [ + "base64 0.13.1", +] + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + +[[package]] +name = "pretty_assertions" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a25e9bcb20aa780fd0bb16b72403a9064d6b3f22f026946029acb941a50af755" +dependencies = [ + "ctor", + "diff", + "output_vt100", + "yansi", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "606c4ba35817e2922a308af55ad51bab3645b59eae5c570d4a6cf07e36bd493b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.8", + "version_check", + "yansi", +] + +[[package]] +name = "query_map" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4465aacac3bebc9484cf7a56dc8b2d7feacb657da6002a9198b4f7af4247a204" +dependencies = [ + "form_urlencoded", + "serde", + "serde_derive", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rcgen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" +dependencies = [ + "pem", + "ring", + "time", + "yasna", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cce168fea28d3e05f158bda4576cf0c844d5045bc2cc3620fa0292ed5bb5814c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "reqwest" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ba30cc2c0cd02af1222ed216ba659cdb2f879dfe3181852fe7c50b1d0005949" +dependencies = [ + "base64 0.21.0", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tokio-rustls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "roxmltree" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.36.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4165c9963ab29e422d6c26fbc1d37f15bace6b2810221f9d925023480fcf0e" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.45.0", +] + +[[package]] +name = "rustls" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +dependencies = [ + "base64 0.21.0", +] + +[[package]] +name = "rustversion" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "schannel" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +dependencies = [ + "windows-sys 0.42.0", +] + +[[package]] +name = "scratch" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" + +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "security-framework" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" + +[[package]] +name = "serde" +version = "1.0.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.8", +] + +[[package]] +name = "serde_json" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7f05c1d5476066defcdfacce1f52fc3cae3af1d3089727100c02ae92e5abbe0" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_regex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8136f1a4ea815d7eac4101cfd0b16dc0cb5e1fe1b8609dfd728058656b7badf" +dependencies = [ + "regex", + "serde", +] + +[[package]] +name = "serde_spanned" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85456ffac572dc8826334164f2fb6fb40a7c766aebe195a2a21ee69ee2885ecf" +dependencies = [ + "base64 0.13.1", + "chrono", + "hex", + "indexmap", + "serde", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cbcd6104f8a4ab6af7f6be2a0da6be86b9de3c401f6e86bb856ab2af739232f" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc02725fd69ab9f26eab07fad303e2497fad6fb9eba4f96c4d1687bdf704ad9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "tempfile" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys 0.42.0", +] + +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.8", +] + +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "time" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + +[[package]] +name = "time-macros" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +dependencies = [ + "time-core", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" +dependencies = [ + "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.45.0", +] + +[[package]] +name = "tokio-macros" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-stream" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "toml" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + +[[package]] +name = "uncased" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b01702b0fd0b3fadcf98e098780badda8742d4f4a7676615cad90e8ac73622" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "urlencoding" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + +[[package]] +name = "web-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "winnow" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deac0939bd6e4f24ab5919fbf751c97a8cfc8543bb083a305ed5c0c10bb241d1" +dependencies = [ + "memchr", +] + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + +[[package]] +name = "xmlparser" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd" + +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + +[[package]] +name = "yasna" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aed2e7a52e3744ab4d0c05c20aa065258e84c49fd4226f5191b2ed29712710b4" +dependencies = [ + "time", +] + +[[package]] +name = "zeroize" +version = "1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..8a39b97 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,5 @@ +[workspace] +members = [ + "htsget-elsa", + "htsget-elsa-lambda" +] diff --git a/htsget-elsa/Cargo.lock b/htsget-elsa/Cargo.lock new file mode 100644 index 0000000..e408e85 --- /dev/null +++ b/htsget-elsa/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "htsget-elsa" +version = "0.1.0" diff --git a/htsget-elsa/Cargo.toml b/htsget-elsa/Cargo.toml new file mode 100644 index 0000000..b280c00 --- /dev/null +++ b/htsget-elsa/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "htsget-elsa" +version = "0.1.0" +edition = "2021" + +[dependencies] +async-trait = "0.1" +http = "0.2" +reqwest = { version = "0.11", features = ["rustls-tls"] } +thiserror = "1.0" +aws-config = "0.54" +aws-sdk-dynamodb = "0.24" + +htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa"} \ No newline at end of file diff --git a/htsget-elsa/src/dynamodb.rs b/htsget-elsa/src/dynamodb.rs new file mode 100644 index 0000000..4a3f6f7 --- /dev/null +++ b/htsget-elsa/src/dynamodb.rs @@ -0,0 +1,32 @@ +use crate::Cache; +use async_trait::async_trait; +use aws_sdk_dynamodb::Client; + +#[derive(Debug)] +pub struct DynamoDBCache { + client: Client, + table_name: String +} + +impl DynamoDBCache { + pub fn new(client: Client, table_name: String) -> Self { + Self { client, table_name } + } + + pub async fn new_with_default_config(table_name: String) -> Self { + DynamoDBCache::new(Client::new(&aws_config::load_from_env().await), table_name) + } +} + +#[async_trait] +impl Cache for DynamoDBCache { + type Item = (); + + async fn get + Send>(&self, key: K) -> Self::Item { + todo!() + } + + async fn put + Send>(&self, key: K, item: Self::Item) { + todo!() + } +} \ No newline at end of file diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs new file mode 100644 index 0000000..5f51092 --- /dev/null +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -0,0 +1,64 @@ +use std::result; +use htsget_config::resolver::Resolver; +use http::uri::Authority; +use crate::{Cache, ResolverFromElsa}; +use async_trait::async_trait; +use reqwest::{Certificate, Client, ClientBuilder, Identity}; +use thiserror::Error; + +pub type Result = result::Result; + +#[derive(Error, Debug)] +pub enum Error { + #[error("invalid certificate: `{0}`")] + InvalidCertificate(reqwest::Error), + #[error("invalid identity: `{0}`")] + InvalidIdentity(reqwest::Error), + #[error("invalid client: `{0}`")] + InvalidClient(reqwest::Error), +} + +#[derive(Debug)] +pub struct ElsaEndpoint { + endpoint: Authority, + client: Client, + cache: C +} + +#[async_trait] +impl ResolverFromElsa for ElsaEndpoint +where C: Cache + Send + Sync { + async fn get(&self) -> Resolver { + todo!() + } +} + +impl ElsaEndpoint + where C: Cache { + pub fn new(endpoint: Authority, cache: C, root_certificate: String, identity: String) -> Result { + Ok(Self { endpoint, client: Self::create_mtls_client(root_certificate, identity)?, cache }) + } + + fn create_mtls_client(root_certificate: String, identity: String) -> Result { + Client::builder() + .use_rustls_tls() + .tls_built_in_root_certs(false) + .add_root_certificate(Self::create_cert(root_certificate)?) + .identity(Self::create_identity(identity)?) + .https_only(true) + .build() + .map_err(|err| Error::InvalidClient(err)) + } + + fn create_cert(certificate: String) -> Result { + Certificate::from_pem(&certificate.into_bytes()).map_err(|err| Error::InvalidCertificate(err)) + } + + fn create_identity(certificate: String) -> Result { + Identity::from_pem(&certificate.into_bytes()).map_err(|err| Error::InvalidIdentity(err)) + } + + pub fn get_manifest(&self) -> String { + todo!() + } +} \ No newline at end of file diff --git a/htsget-elsa/src/lib.rs b/htsget-elsa/src/lib.rs new file mode 100644 index 0000000..9b660de --- /dev/null +++ b/htsget-elsa/src/lib.rs @@ -0,0 +1,19 @@ +pub mod elsa_endpoint; +pub mod dynamodb; + +use std::str::FromStr; +use async_trait::async_trait; +use htsget_config::resolver::Resolver; + +#[async_trait] +pub trait Cache { + type Item; + + async fn get + Send>(&self, key: K) -> Self::Item; + async fn put + Send>(&self, key: K, item: Self::Item); +} + +#[async_trait] +pub trait ResolverFromElsa { + async fn get(&self) -> Resolver; +} \ No newline at end of file From d8f0739a02f21c89b1e8961703c8a0ea6066bf82 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 14 Apr 2023 20:03:09 +1000 Subject: [PATCH 02/25] feat(elsa-lambda): add htsget-elsa-lambda crate --- htsget-elsa-lambda/Cargo.lock | 7 +++++++ htsget-elsa-lambda/Cargo.toml | 16 ++++++++++++++++ htsget-elsa-lambda/src/config.rs | 24 ++++++++++++++++++++++++ htsget-elsa-lambda/src/lib.rs | 26 ++++++++++++++++++++++++++ htsget-elsa-lambda/src/main.rs | 23 +++++++++++++++++++++++ 5 files changed, 96 insertions(+) create mode 100644 htsget-elsa-lambda/Cargo.lock create mode 100644 htsget-elsa-lambda/Cargo.toml create mode 100644 htsget-elsa-lambda/src/config.rs create mode 100644 htsget-elsa-lambda/src/lib.rs create mode 100644 htsget-elsa-lambda/src/main.rs diff --git a/htsget-elsa-lambda/Cargo.lock b/htsget-elsa-lambda/Cargo.lock new file mode 100644 index 0000000..89d81b1 --- /dev/null +++ b/htsget-elsa-lambda/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "htsget-elsa-lambda" +version = "0.1.0" diff --git a/htsget-elsa-lambda/Cargo.toml b/htsget-elsa-lambda/Cargo.toml new file mode 100644 index 0000000..f3bd202 --- /dev/null +++ b/htsget-elsa-lambda/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "htsget-elsa-lambda" +version = "0.1.0" +edition = "2021" + +[dependencies] +tokio = { version = "1.25", features = ["macros", "rt-multi-thread"] } +lambda_http = { version = "0.7" } +serde = { version = "1.0" } +serde_json = "1.0" +tracing = "0.1" +http-serde = "1.1" +http = "0.2" + +htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa"} +htsget-lambda = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa"} \ No newline at end of file diff --git a/htsget-elsa-lambda/src/config.rs b/htsget-elsa-lambda/src/config.rs new file mode 100644 index 0000000..66403b1 --- /dev/null +++ b/htsget-elsa-lambda/src/config.rs @@ -0,0 +1,24 @@ +use std::io; +use std::path::Path; +use htsget_config::config::Config as HtsGetConfig; +use htsget_config::config::parser::from_path; +use http::uri::Authority; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone)] +struct Config { + #[serde(flatten)] + htsget_config: HtsGetConfig, + root_certificate_id: String, + identity_id: String, + parameters_secrets_extension_http_port: u16, + dynamodb_table_name: String, + #[serde(with = "http_serde::authority")] + elsa_endpoint: Authority +} + +impl Config { + pub fn from_path(path: &Path) -> io::Result { + from_path(path) + } +} \ No newline at end of file diff --git a/htsget-elsa-lambda/src/lib.rs b/htsget-elsa-lambda/src/lib.rs new file mode 100644 index 0000000..dfa3482 --- /dev/null +++ b/htsget-elsa-lambda/src/lib.rs @@ -0,0 +1,26 @@ +use std::sync::Arc; +use htsget_config::config::cors::CorsConfig; +use htsget_config::config::ServiceInfo; +use htsget_config::resolver::Resolver; +use htsget_lambda::{handle_request_service_fn, Router}; +use lambda_http::{Error, Request}; +use tracing::info; + +pub mod config; + +pub async fn handle_request(cors: CorsConfig, service_info: &ServiceInfo) -> Result<(), Error> +{ + handle_request_service_fn(cors, |event: Request| async move { + info!(event = ?event, "received request"); + + let resolver = get_resolvers(&event).await; + let router = Router::new(Arc::new(resolver), &service_info); + + router.route_request(event).await + }) + .await +} + +pub async fn get_resolvers(event: &Request) -> Vec { + todo!(); +} \ No newline at end of file diff --git a/htsget-elsa-lambda/src/main.rs b/htsget-elsa-lambda/src/main.rs new file mode 100644 index 0000000..5e39c1b --- /dev/null +++ b/htsget-elsa-lambda/src/main.rs @@ -0,0 +1,23 @@ +use std::sync::Arc; +use htsget_config::config::cors::CorsConfig; +use htsget_config::config::parser::from_path; +use htsget_config::config::ServiceInfo; + +use lambda_http::Error; + +use htsget_lambda::Config; +use htsget_elsa_lambda::handle_request; + +#[tokio::main] +async fn main() -> Result<(), Error> { + Config::setup_tracing()?; + + if let Some(path) = Config::parse_args() { + let cors: CorsConfig = from_path(&path)?; + let service_info: ServiceInfo = from_path(&path)?; + + handle_request(cors, &service_info).await + } else { + Ok(()) + } +} \ No newline at end of file From cf26d255daf3683085f0d78c927d187f1a698598 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 14 Apr 2023 20:03:21 +1000 Subject: [PATCH 03/25] feat: implement get manifest function --- Cargo.lock | 1 + htsget-elsa-lambda/src/config.rs | 10 ++-- htsget-elsa-lambda/src/lib.rs | 9 ++-- htsget-elsa-lambda/src/main.rs | 6 +-- htsget-elsa/Cargo.toml | 3 +- htsget-elsa/src/dynamodb.rs | 4 +- htsget-elsa/src/elsa_endpoint.rs | 88 ++++++++++++++++++++++---------- htsget-elsa/src/lib.rs | 6 +-- 8 files changed, 81 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 925858c..a822ae7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1232,6 +1232,7 @@ dependencies = [ "htsget-config", "http", "reqwest", + "serde", "thiserror", ] diff --git a/htsget-elsa-lambda/src/config.rs b/htsget-elsa-lambda/src/config.rs index 66403b1..908c0b6 100644 --- a/htsget-elsa-lambda/src/config.rs +++ b/htsget-elsa-lambda/src/config.rs @@ -1,9 +1,9 @@ -use std::io; -use std::path::Path; -use htsget_config::config::Config as HtsGetConfig; use htsget_config::config::parser::from_path; +use htsget_config::config::Config as HtsGetConfig; use http::uri::Authority; use serde::{Deserialize, Serialize}; +use std::io; +use std::path::Path; #[derive(Serialize, Deserialize, Debug, Clone)] struct Config { @@ -14,11 +14,11 @@ struct Config { parameters_secrets_extension_http_port: u16, dynamodb_table_name: String, #[serde(with = "http_serde::authority")] - elsa_endpoint: Authority + elsa_endpoint: Authority, } impl Config { pub fn from_path(path: &Path) -> io::Result { from_path(path) } -} \ No newline at end of file +} diff --git a/htsget-elsa-lambda/src/lib.rs b/htsget-elsa-lambda/src/lib.rs index dfa3482..de189d5 100644 --- a/htsget-elsa-lambda/src/lib.rs +++ b/htsget-elsa-lambda/src/lib.rs @@ -1,15 +1,14 @@ -use std::sync::Arc; use htsget_config::config::cors::CorsConfig; use htsget_config::config::ServiceInfo; use htsget_config::resolver::Resolver; use htsget_lambda::{handle_request_service_fn, Router}; use lambda_http::{Error, Request}; +use std::sync::Arc; use tracing::info; pub mod config; -pub async fn handle_request(cors: CorsConfig, service_info: &ServiceInfo) -> Result<(), Error> -{ +pub async fn handle_request(cors: CorsConfig, service_info: &ServiceInfo) -> Result<(), Error> { handle_request_service_fn(cors, |event: Request| async move { info!(event = ?event, "received request"); @@ -18,9 +17,9 @@ pub async fn handle_request(cors: CorsConfig, service_info: &ServiceInfo) -> Res router.route_request(event).await }) - .await + .await } pub async fn get_resolvers(event: &Request) -> Vec { todo!(); -} \ No newline at end of file +} diff --git a/htsget-elsa-lambda/src/main.rs b/htsget-elsa-lambda/src/main.rs index 5e39c1b..0621013 100644 --- a/htsget-elsa-lambda/src/main.rs +++ b/htsget-elsa-lambda/src/main.rs @@ -1,12 +1,12 @@ -use std::sync::Arc; use htsget_config::config::cors::CorsConfig; use htsget_config::config::parser::from_path; use htsget_config::config::ServiceInfo; +use std::sync::Arc; use lambda_http::Error; -use htsget_lambda::Config; use htsget_elsa_lambda::handle_request; +use htsget_lambda::Config; #[tokio::main] async fn main() -> Result<(), Error> { @@ -20,4 +20,4 @@ async fn main() -> Result<(), Error> { } else { Ok(()) } -} \ No newline at end of file +} diff --git a/htsget-elsa/Cargo.toml b/htsget-elsa/Cargo.toml index b280c00..d2ef1c0 100644 --- a/htsget-elsa/Cargo.toml +++ b/htsget-elsa/Cargo.toml @@ -6,9 +6,10 @@ edition = "2021" [dependencies] async-trait = "0.1" http = "0.2" -reqwest = { version = "0.11", features = ["rustls-tls"] } +reqwest = { version = "0.11", features = ["rustls-tls", "json"] } thiserror = "1.0" aws-config = "0.54" aws-sdk-dynamodb = "0.24" +serde = { version = "1.0", features = ["derive"] } htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa"} \ No newline at end of file diff --git a/htsget-elsa/src/dynamodb.rs b/htsget-elsa/src/dynamodb.rs index 4a3f6f7..4fca243 100644 --- a/htsget-elsa/src/dynamodb.rs +++ b/htsget-elsa/src/dynamodb.rs @@ -5,7 +5,7 @@ use aws_sdk_dynamodb::Client; #[derive(Debug)] pub struct DynamoDBCache { client: Client, - table_name: String + table_name: String, } impl DynamoDBCache { @@ -29,4 +29,4 @@ impl Cache for DynamoDBCache { async fn put + Send>(&self, key: K, item: Self::Item) { todo!() } -} \ No newline at end of file +} diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index 5f51092..ea995ea 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -1,64 +1,98 @@ -use std::result; -use htsget_config::resolver::Resolver; -use http::uri::Authority; +use crate::elsa_endpoint::Error::{DeserializeResponseError, GetError, InvalidUri}; use crate::{Cache, ResolverFromElsa}; use async_trait::async_trait; -use reqwest::{Certificate, Client, ClientBuilder, Identity}; +use htsget_config::resolver::Resolver; +use http::uri::Authority; +use reqwest::{Client, Url}; +use serde::Deserialize; +use std::result; +use std::sync::mpsc::SendError; use thiserror::Error; +const ENDPOINT_PATH: &str = "/manifest/htsget"; + pub type Result = result::Result; #[derive(Error, Debug)] pub enum Error { - #[error("invalid certificate: `{0}`")] - InvalidCertificate(reqwest::Error), - #[error("invalid identity: `{0}`")] - InvalidIdentity(reqwest::Error), #[error("invalid client: `{0}`")] InvalidClient(reqwest::Error), + #[error("invalid uri constructed from release key: `{0}`")] + InvalidUri(String), + #[error("failed to get uri: `{0}`")] + GetError(reqwest::Error), + #[error("failed to deserialize response: `{0}`")] + DeserializeResponseError(reqwest::Error), +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ElsaLocation { + bucket: String, + key: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ElsaResponse { + location: ElsaLocation, + max_age: u64, } #[derive(Debug)] pub struct ElsaEndpoint { endpoint: Authority, client: Client, - cache: C + cache: C, } #[async_trait] impl ResolverFromElsa for ElsaEndpoint -where C: Cache + Send + Sync { +where + C: Cache + Send + Sync, +{ async fn get(&self) -> Resolver { todo!() } } impl ElsaEndpoint - where C: Cache { - pub fn new(endpoint: Authority, cache: C, root_certificate: String, identity: String) -> Result { - Ok(Self { endpoint, client: Self::create_mtls_client(root_certificate, identity)?, cache }) +where + C: Cache, +{ + pub fn new(endpoint: Authority, cache: C) -> Result { + Ok(Self { + endpoint, + client: Self::create_client()?, + cache, + }) } - fn create_mtls_client(root_certificate: String, identity: String) -> Result { + fn create_client() -> Result { Client::builder() .use_rustls_tls() - .tls_built_in_root_certs(false) - .add_root_certificate(Self::create_cert(root_certificate)?) - .identity(Self::create_identity(identity)?) .https_only(true) .build() .map_err(|err| Error::InvalidClient(err)) } - fn create_cert(certificate: String) -> Result { - Certificate::from_pem(&certificate.into_bytes()).map_err(|err| Error::InvalidCertificate(err)) - } - - fn create_identity(certificate: String) -> Result { - Identity::from_pem(&certificate.into_bytes()).map_err(|err| Error::InvalidIdentity(err)) - } + pub async fn get_manifest(&self, release_key: String) -> Result { + let uri = http::Uri::builder() + .scheme("https") + .authority(self.endpoint.as_str()) + .path_and_query(format!("/manifest/htsget/{}?type=S3", release_key)) + .build() + .map(|uri| Url::parse(&uri.to_string())) + .map_err(|_| InvalidUri(release_key.to_string()))? + .map_err(|_| InvalidUri(release_key.to_string()))?; - pub fn get_manifest(&self) -> String { - todo!() + self.client + .get(uri) + .send() + .await + .map_err(|err| GetError(err))? + .json() + .await + .map_err(|err| DeserializeResponseError(err)) } -} \ No newline at end of file +} diff --git a/htsget-elsa/src/lib.rs b/htsget-elsa/src/lib.rs index 9b660de..a6faa30 100644 --- a/htsget-elsa/src/lib.rs +++ b/htsget-elsa/src/lib.rs @@ -1,9 +1,9 @@ -pub mod elsa_endpoint; pub mod dynamodb; +pub mod elsa_endpoint; -use std::str::FromStr; use async_trait::async_trait; use htsget_config::resolver::Resolver; +use std::str::FromStr; #[async_trait] pub trait Cache { @@ -16,4 +16,4 @@ pub trait Cache { #[async_trait] pub trait ResolverFromElsa { async fn get(&self) -> Resolver; -} \ No newline at end of file +} From dcbdaa1227a96160c02f6f7e2dc648e4a66498ad Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 14 Apr 2023 20:03:29 +1000 Subject: [PATCH 04/25] feat(elsa): add s3 get object implementation --- Cargo.lock | 2 + htsget-elsa/Cargo.toml | 2 + htsget-elsa/src/dynamodb.rs | 2 +- htsget-elsa/src/elsa_endpoint.rs | 77 +++++++++++++++++++++----------- htsget-elsa/src/lib.rs | 38 ++++++++++++++-- htsget-elsa/src/s3.rs | 38 ++++++++++++++++ 6 files changed, 128 insertions(+), 31 deletions(-) create mode 100644 htsget-elsa/src/s3.rs diff --git a/Cargo.lock b/Cargo.lock index a822ae7..e0f9d58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1229,10 +1229,12 @@ dependencies = [ "async-trait", "aws-config", "aws-sdk-dynamodb", + "aws-sdk-s3", "htsget-config", "http", "reqwest", "serde", + "serde_json", "thiserror", ] diff --git a/htsget-elsa/Cargo.toml b/htsget-elsa/Cargo.toml index d2ef1c0..b5e9830 100644 --- a/htsget-elsa/Cargo.toml +++ b/htsget-elsa/Cargo.toml @@ -10,6 +10,8 @@ reqwest = { version = "0.11", features = ["rustls-tls", "json"] } thiserror = "1.0" aws-config = "0.54" aws-sdk-dynamodb = "0.24" +aws-sdk-s3 = "0.24" serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa"} \ No newline at end of file diff --git a/htsget-elsa/src/dynamodb.rs b/htsget-elsa/src/dynamodb.rs index 4fca243..6fd4be1 100644 --- a/htsget-elsa/src/dynamodb.rs +++ b/htsget-elsa/src/dynamodb.rs @@ -26,7 +26,7 @@ impl Cache for DynamoDBCache { todo!() } - async fn put + Send>(&self, key: K, item: Self::Item) { + async fn put + Send>(&self, key: K, item: Self::Item, expirey: u64) { todo!() } } diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index ea995ea..6fdb81c 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -1,30 +1,14 @@ -use crate::elsa_endpoint::Error::{DeserializeResponseError, GetError, InvalidUri}; -use crate::{Cache, ResolverFromElsa}; +use crate::Error::{DeserializeError, GetGetManifest, InvalidUri}; +use crate::{Cache, Error, GetObject, ResolverFromElsa, Result}; use async_trait::async_trait; use htsget_config::resolver::Resolver; use http::uri::Authority; use reqwest::{Client, Url}; use serde::Deserialize; -use std::result; -use std::sync::mpsc::SendError; -use thiserror::Error; +use std::collections::HashMap; const ENDPOINT_PATH: &str = "/manifest/htsget"; -pub type Result = result::Result; - -#[derive(Error, Debug)] -pub enum Error { - #[error("invalid client: `{0}`")] - InvalidClient(reqwest::Error), - #[error("invalid uri constructed from release key: `{0}`")] - InvalidUri(String), - #[error("failed to get uri: `{0}`")] - GetError(reqwest::Error), - #[error("failed to deserialize response: `{0}`")] - DeserializeResponseError(reqwest::Error), -} - #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ElsaLocation { @@ -39,32 +23,67 @@ pub struct ElsaResponse { max_age: u64, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ElsaReadsManifest { + url: String, + restriction: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ElsaVariantsManifest { + url: String, + variant_sample_id: String, + restriction: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ElsaRestrictionsManifest {} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ElsaManifest { + #[serde(alias = "id")] + release_key: String, + reads: HashMap, + variants: HashMap, + restrictions: ElsaRestrictionsManifest, +} + #[derive(Debug)] -pub struct ElsaEndpoint { +pub struct ElsaEndpoint { endpoint: Authority, client: Client, cache: C, + get_object: S, } #[async_trait] -impl ResolverFromElsa for ElsaEndpoint +impl ResolverFromElsa for ElsaEndpoint where C: Cache + Send + Sync, + S: GetObject + Send + Sync, { - async fn get(&self) -> Resolver { + type Error = Error; + + async fn try_get(&self, release_key: String) -> Result { todo!() } } -impl ElsaEndpoint +impl ElsaEndpoint where C: Cache, + S: GetObject, { - pub fn new(endpoint: Authority, cache: C) -> Result { + pub fn new(endpoint: Authority, cache: C, get_object: S) -> Result { Ok(Self { endpoint, client: Self::create_client()?, cache, + get_object, }) } @@ -76,7 +95,7 @@ where .map_err(|err| Error::InvalidClient(err)) } - pub async fn get_manifest(&self, release_key: String) -> Result { + pub async fn get_response(&self, release_key: String) -> Result { let uri = http::Uri::builder() .scheme("https") .authority(self.endpoint.as_str()) @@ -90,9 +109,13 @@ where .get(uri) .send() .await - .map_err(|err| GetError(err))? + .map_err(|err| GetGetManifest(err))? .json() .await - .map_err(|err| DeserializeResponseError(err)) + .map_err(|err| DeserializeError(err.to_string())) + } + + pub async fn get_manifest(&self, response: ElsaResponse) -> Result { + todo!() } } diff --git a/htsget-elsa/src/lib.rs b/htsget-elsa/src/lib.rs index a6faa30..529f802 100644 --- a/htsget-elsa/src/lib.rs +++ b/htsget-elsa/src/lib.rs @@ -1,19 +1,51 @@ pub mod dynamodb; pub mod elsa_endpoint; +pub mod s3; use async_trait::async_trait; use htsget_config::resolver::Resolver; -use std::str::FromStr; +use serde::Deserialize; +use std::result; +use thiserror::Error; + +pub type Result = result::Result; + +#[derive(Error, Debug)] +pub enum Error { + #[error("invalid client: `{0}`")] + InvalidClient(reqwest::Error), + #[error("invalid uri constructed from release key: `{0}`")] + InvalidUri(String), + #[error("failed to get manifest from Elsa: `{0}`")] + GetGetManifest(reqwest::Error), + #[error("failed to deserialize type: `{0}")] + DeserializeError(String), + #[error("failed to get object from storage: `{0}`")] + GetObjectError(String), +} #[async_trait] pub trait Cache { type Item; async fn get + Send>(&self, key: K) -> Self::Item; - async fn put + Send>(&self, key: K, item: Self::Item); + async fn put + Send>(&self, key: K, item: Self::Item, expirey: u64); +} + +#[async_trait] +pub trait GetObject { + type Error; + + async fn get_object Deserialize<'de>>( + &self, + bucket: impl Into + Send, + key: impl Into + Send, + ) -> result::Result; } #[async_trait] pub trait ResolverFromElsa { - async fn get(&self) -> Resolver; + type Error; + + async fn try_get(&self, release_key: String) -> result::Result; } diff --git a/htsget-elsa/src/s3.rs b/htsget-elsa/src/s3.rs new file mode 100644 index 0000000..050809b --- /dev/null +++ b/htsget-elsa/src/s3.rs @@ -0,0 +1,38 @@ +use crate::Error::{DeserializeError, GetObjectError}; +use crate::{Error, GetObject, Result}; +use async_trait::async_trait; +use aws_sdk_s3::Client; +use serde::Deserialize; +use serde_json::from_slice; + +#[derive(Debug)] +pub struct S3 { + s3_client: Client, +} + +#[async_trait] +impl GetObject for S3 { + type Error = Error; + + async fn get_object Deserialize<'de>>( + &self, + bucket: impl Into + Send, + key: impl Into + Send, + ) -> Result { + let output = self + .s3_client + .get_object() + .bucket(bucket) + .key(key) + .send() + .await + .map_err(|err| GetObjectError(err.to_string()))? + .body + .collect() + .await + .map_err(|err| GetObjectError(err.to_string()))? + .into_bytes(); + + from_slice(output.as_ref()).map_err(|err| DeserializeError(err.to_string())) + } +} From eb6d4e9dd283aeb6d656b329494eaf0418e8e2a6 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 14 Apr 2023 20:03:36 +1000 Subject: [PATCH 05/25] feat(elsa): implement resolver from manifest --- Cargo.lock | 410 ++++++++++++++++++++----------- htsget-elsa-lambda/Cargo.toml | 4 +- htsget-elsa/Cargo.toml | 3 +- htsget-elsa/src/dynamodb.rs | 7 +- htsget-elsa/src/elsa_endpoint.rs | 130 ++++++++-- htsget-elsa/src/lib.rs | 15 +- 6 files changed, 399 insertions(+), 170 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e0f9d58..26eef0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,46 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "342258dd14006105c2b75ab1bd7543a03bdf0cfc94383303ac212a04939dff6f" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-wincon", + "concolor-override", + "concolor-query", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2" + +[[package]] +name = "anstyle-parse" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7d1bb534e9efed14f3e5f44e7dd1a4f709384023a4165199a4241e18dff0116" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-wincon" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3127af6145b149f3287bb9a0d10ad9c5692dba8c53ad48285e5bec4063834fa" +dependencies = [ + "anstyle", + "windows-sys 0.45.0", +] + [[package]] name = "assert-json-diff" version = "1.1.0" @@ -74,13 +114,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.67" +version = "0.1.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ea188f25f0255d8f92797797c97ebf5631fa88178beb1a46fdf5622c9a00e4" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.8", + "syn 2.0.13", ] [[package]] @@ -708,39 +748,45 @@ dependencies = [ [[package]] name = "clap" -version = "4.1.13" +version = "4.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c911b090850d79fc64fe9ea01e28e465f65e821e08813ced95bced72f7a8a9b" +checksum = "046ae530c528f252094e4a77886ee1374437744b2bff1497aa898bbddbbb29b3" dependencies = [ - "bitflags", + "clap_builder", "clap_derive", - "clap_lex", - "is-terminal", "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "223163f58c9a40c3b0a43e1c4b50a9ce09f007ea2cb1ec258a687945b4b7929f" +dependencies = [ + "anstream", + "anstyle", + "bitflags", + "clap_lex", "strsim", - "termcolor", ] [[package]] name = "clap_derive" -version = "4.1.12" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a932373bab67b984c790ddf2c9ca295d8e3af3b7ef92de5a5bacdccdee4b09b" +checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.8", + "syn 2.0.13", ] [[package]] name = "clap_lex" -version = "0.3.3" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "033f6b7a4acb1f358c742aaca805c939ee73b4c6209ae4318ec7aca81c42e646" -dependencies = [ - "os_str_bytes", -] +checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" [[package]] name = "codespan-reporting" @@ -752,6 +798,21 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "concolor-override" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a855d4a1978dc52fb0536a04d384c2c0c1aa273597f08b77c8c4d3b2eec6037f" + +[[package]] +name = "concolor-query" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d11d52c3d7ca2e6d0040212be9e4dbbcd78b6447f535b6b561f449427944cf" +dependencies = [ + "windows-sys 0.45.0", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -764,15 +825,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" dependencies = [ "libc", ] @@ -836,9 +897,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c00419335c41018365ddf7e4d5f1c12ee3659ddcf3e01974650ba1de73d038" +checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" dependencies = [ "cc", "cxxbridge-flags", @@ -848,9 +909,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb8307ad413a98fff033c8545ecf133e3257747b3bae935e7602aab8aa92d4ca" +checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" dependencies = [ "cc", "codespan-reporting", @@ -858,24 +919,24 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.8", + "syn 2.0.13", ] [[package]] name = "cxxbridge-flags" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc52e2eb08915cb12596d29d55f0b5384f00d697a646dbd269b6ecb0fbd9d31" +checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" [[package]] name = "cxxbridge-macro" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "631569015d0d8d54e6c241733f944042623ab6df7bc3be7466874b05fcdb1c5f" +checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.8", + "syn 2.0.13", ] [[package]] @@ -947,13 +1008,13 @@ dependencies = [ [[package]] name = "errno" -version = "0.2.8" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" dependencies = [ "errno-dragonfly", "libc", - "winapi", + "windows-sys 0.45.0", ] [[package]] @@ -1043,9 +1104,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531ac96c6ff5fd7c62263c5e3c67a603af4fcaee2e1a0ae5565ba3a11e69e549" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" dependencies = [ "futures-channel", "futures-core", @@ -1058,9 +1119,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", "futures-sink", @@ -1068,15 +1129,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" [[package]] name = "futures-executor" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1997dd9df74cdac935c76252744c1ed5794fac083242ea4fe77ef3ed60ba0f83" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" dependencies = [ "futures-core", "futures-task", @@ -1085,38 +1146,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" [[package]] name = "futures-macro" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.13", ] [[package]] name = "futures-sink" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" [[package]] name = "futures-task" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" [[package]] name = "futures-util" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-channel", "futures-core", @@ -1132,9 +1193,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -1204,7 +1265,7 @@ dependencies = [ [[package]] name = "htsget-config" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#d2d9aa7378c8cffa1873b9c17c57ce63febd9813" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#a040432d131c83be7d5460930c0879f5c13dfbda" dependencies = [ "async-trait", "clap", @@ -1232,6 +1293,7 @@ dependencies = [ "aws-sdk-s3", "htsget-config", "http", + "regex", "reqwest", "serde", "serde_json", @@ -1256,7 +1318,7 @@ dependencies = [ [[package]] name = "htsget-http" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#d2d9aa7378c8cffa1873b9c17c57ce63febd9813" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#a040432d131c83be7d5460930c0879f5c13dfbda" dependencies = [ "futures", "htsget-config", @@ -1272,7 +1334,7 @@ dependencies = [ [[package]] name = "htsget-lambda" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#d2d9aa7378c8cffa1873b9c17c57ce63febd9813" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#a040432d131c83be7d5460930c0879f5c13dfbda" dependencies = [ "bytes", "htsget-config", @@ -1294,7 +1356,7 @@ dependencies = [ [[package]] name = "htsget-search" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#d2d9aa7378c8cffa1873b9c17c57ce63febd9813" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#a040432d131c83be7d5460930c0879f5c13dfbda" dependencies = [ "async-trait", "aws-config", @@ -1324,7 +1386,7 @@ dependencies = [ [[package]] name = "htsget-test" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#d2d9aa7378c8cffa1873b9c17c57ce63febd9813" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#a040432d131c83be7d5460930c0879f5c13dfbda" dependencies = [ "async-trait", "base64 0.21.0", @@ -1445,9 +1507,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.54" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" +checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1485,9 +1547,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown", @@ -1511,26 +1573,26 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" dependencies = [ "hermit-abi 0.3.1", "libc", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "ipnet" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" +checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" [[package]] name = "is-terminal" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8687c819457e979cc940d09cb16e42a1bf70aa6b60a549de6d3a62a0ee90c69e" +checksum = "256017f749ab3117e93acb91063009e1f1bb56d03965b14c2c8df4eb02c524d8" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", @@ -1679,9 +1741,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.140" +version = "0.2.141" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" +checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" [[package]] name = "link-cplusplus" @@ -1694,9 +1756,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.1.4" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" [[package]] name = "log" @@ -2051,9 +2113,9 @@ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "openssl" -version = "0.10.48" +version = "0.10.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518915b97df115dd36109bfa429a48b8f737bd05508cf9588977b599648926d2" +checksum = "4d2f106ab837a24e03672c59b1239669a0596406ff657c3c0835b6b7f0f35a33" dependencies = [ "bitflags", "cfg-if", @@ -2066,13 +2128,13 @@ dependencies = [ [[package]] name = "openssl-macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.13", ] [[package]] @@ -2083,23 +2145,16 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.83" +version = "0.9.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "666416d899cf077260dac8698d60a60b435a46d57e82acb1be3d0dad87284e5b" +checksum = "3a20eace9dc2d82904039cb76dcf50fb1a0bba071cfd1629720b5d6f1ddba0fa" dependencies = [ - "autocfg", "cc", "libc", "pkg-config", "vcpkg", ] -[[package]] -name = "os_str_bytes" -version = "6.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" - [[package]] name = "output_vt100" version = "0.1.3" @@ -2141,7 +2196,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.8", + "syn 2.0.13", ] [[package]] @@ -2235,9 +2290,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.53" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba466839c78239c09faf015484e5cc04860f88242cff4d03eb038f04b4699b73" +checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" dependencies = [ "unicode-ident", ] @@ -2250,7 +2305,7 @@ checksum = "606c4ba35817e2922a308af55ad51bab3645b59eae5c570d4a6cf07e36bd493b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.8", + "syn 2.0.13", "version_check", "yansi", ] @@ -2289,18 +2344,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ "bitflags", ] [[package]] name = "regex" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cce168fea28d3e05f158bda4576cf0c844d5045bc2cc3620fa0292ed5bb5814c" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" dependencies = [ "aho-corasick", "memchr", @@ -2324,9 +2379,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "reqwest" -version = "0.11.15" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ba30cc2c0cd02af1222ed216ba659cdb2f879dfe3181852fe7c50b1d0005949" +checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" dependencies = [ "base64 0.21.0", "bytes", @@ -2399,9 +2454,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.11" +version = "0.37.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4165c9963ab29e422d6c26fbc1d37f15bace6b2810221f9d925023480fcf0e" +checksum = "2aae838e49b3d63e9274e1c01833cc8139d3fec468c3b84688c628f44b1ae11d" dependencies = [ "bitflags", "errno", @@ -2512,29 +2567,29 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.158" +version = "1.0.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" +checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.158" +version = "1.0.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" +checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" dependencies = [ "proc-macro2", "quote", - "syn 2.0.8", + "syn 2.0.13", ] [[package]] name = "serde_json" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" +checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" dependencies = [ "indexmap", "itoa", @@ -2703,9 +2758,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.8" +version = "2.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc02725fd69ab9f26eab07fad303e2497fad6fb9eba4f96c4d1687bdf704ad9" +checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec" dependencies = [ "proc-macro2", "quote", @@ -2720,15 +2775,15 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "tempfile" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" +checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ "cfg-if", "fastrand", "redox_syscall", "rustix", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -2757,7 +2812,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.8", + "syn 2.0.13", ] [[package]] @@ -2814,14 +2869,13 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.26.0" +version = "1.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" +checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" dependencies = [ "autocfg", "bytes", "libc", - "memchr", "mio", "num_cpus", "pin-project-lite", @@ -2832,13 +2886,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.13", ] [[package]] @@ -3127,6 +3181,12 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "valuable" version = "0.1.0" @@ -3295,11 +3355,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.46.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets", + "windows-targets 0.48.0", ] [[package]] @@ -3308,13 +3368,13 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -3323,7 +3383,16 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets", + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", ] [[package]] @@ -3332,13 +3401,28 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] [[package]] @@ -3347,47 +3431,89 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + [[package]] name = "windows_i686_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + [[package]] name = "windows_i686_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + [[package]] name = "winnow" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deac0939bd6e4f24ab5919fbf751c97a8cfc8543bb083a305ed5c0c10bb241d1" +checksum = "ae8970b36c66498d8ff1d66685dc86b91b29db0c7739899012f63a63814b4b28" dependencies = [ "memchr", ] @@ -3433,6 +3559,6 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f" +checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" diff --git a/htsget-elsa-lambda/Cargo.toml b/htsget-elsa-lambda/Cargo.toml index f3bd202..4287a7b 100644 --- a/htsget-elsa-lambda/Cargo.toml +++ b/htsget-elsa-lambda/Cargo.toml @@ -12,5 +12,5 @@ tracing = "0.1" http-serde = "1.1" http = "0.2" -htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa"} -htsget-lambda = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa"} \ No newline at end of file +htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } +htsget-lambda = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } \ No newline at end of file diff --git a/htsget-elsa/Cargo.toml b/htsget-elsa/Cargo.toml index b5e9830..e557feb 100644 --- a/htsget-elsa/Cargo.toml +++ b/htsget-elsa/Cargo.toml @@ -13,5 +13,6 @@ aws-sdk-dynamodb = "0.24" aws-sdk-s3 = "0.24" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +regex = "1.7" -htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa"} \ No newline at end of file +htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } \ No newline at end of file diff --git a/htsget-elsa/src/dynamodb.rs b/htsget-elsa/src/dynamodb.rs index 6fd4be1..83bddc4 100644 --- a/htsget-elsa/src/dynamodb.rs +++ b/htsget-elsa/src/dynamodb.rs @@ -1,4 +1,4 @@ -use crate::Cache; +use crate::{Cache, Error, Result}; use async_trait::async_trait; use aws_sdk_dynamodb::Client; @@ -20,13 +20,14 @@ impl DynamoDBCache { #[async_trait] impl Cache for DynamoDBCache { + type Error = Error; type Item = (); - async fn get + Send>(&self, key: K) -> Self::Item { + async fn get + Send>(&self, key: K) -> Result { todo!() } - async fn put + Send>(&self, key: K, item: Self::Item, expirey: u64) { + async fn put + Send>(&self, key: K, item: Self::Item, expiry: u64) { todo!() } } diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index 6fdb81c..1cac100 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -1,11 +1,19 @@ -use crate::Error::{DeserializeError, GetGetManifest, InvalidUri}; -use crate::{Cache, Error, GetObject, ResolverFromElsa, Result}; +use crate::Error::{ + DeserializeError, GetGetManifest, InvalidManifestUri, InvalidReleaseUri, + UnsupportedManifestFeature, +}; +use crate::{Cache, Error, GetObject, ResolversFromElsa, Result}; use async_trait::async_trait; -use htsget_config::resolver::Resolver; -use http::uri::Authority; +use htsget_config::resolver::{AllowGuard, Resolver}; +use htsget_config::storage::s3::S3Storage; +use htsget_config::storage::Storage; +use htsget_config::types::Format; +use http::uri::{Authority, Parts, Scheme}; +use http::Uri; use reqwest::{Client, Url}; use serde::Deserialize; use std::collections::HashMap; +use std::str::FromStr; const ENDPOINT_PATH: &str = "/manifest/htsget"; @@ -27,6 +35,7 @@ pub struct ElsaResponse { #[serde(rename_all = "camelCase")] pub struct ElsaReadsManifest { url: String, + format: Option, restriction: Option, } @@ -34,6 +43,7 @@ pub struct ElsaReadsManifest { #[serde(rename_all = "camelCase")] pub struct ElsaVariantsManifest { url: String, + format: Option, variant_sample_id: String, restriction: Option, } @@ -52,6 +62,82 @@ pub struct ElsaManifest { restrictions: ElsaRestrictionsManifest, } +impl ElsaManifest { + pub fn resolver_from_manifest_parts( + release_key: &str, + url: &str, + id: &str, + format: Format, + ) -> Result { + let uri = Uri::from_str(url) + .map_err(|err| InvalidManifestUri(err.to_string()))? + .into_parts(); + + match uri.scheme { + Some(scheme) if scheme.as_str() == "s3" || scheme.as_str() == "S3" => Ok(()), + _ => Err(UnsupportedManifestFeature( + "only S3 manifest uris are supported".to_string(), + )), + }?; + + let bucket = match uri.authority { + Some(bucket) => Ok(bucket.to_string()), + None => Err(InvalidManifestUri("missing bucket from uri".to_string())), + }?; + + let key = match uri.path_and_query { + Some(path) => match path.to_string().strip_suffix(format.file_ending()) { + None => Ok(path.to_string()), + Some(key) => Ok(key.to_string()), + }, + None => Err(InvalidManifestUri( + "missing object path from uri".to_string(), + )), + }?; + + Resolver::new( + Storage::S3 { + s3_storage: S3Storage::new(bucket), + }, + &format!("^{}/{}$", release_key, id), + &key, + AllowGuard::default().with_allow_formats(vec![format]), + ) + .map_err(|err| { + InvalidManifestUri(format!("failed to construct regex: {}", err.to_string())) + }) + } +} + +impl TryFrom for Vec { + type Error = Error; + + fn try_from(manifest: ElsaManifest) -> Result { + let release_key = manifest.release_key; + + manifest + .reads + .into_iter() + .map(|(id, reads_manifest)| { + ElsaManifest::resolver_from_manifest_parts( + &release_key, + &reads_manifest.url, + &id, + reads_manifest.format.unwrap_or(Format::Bam), + ) + }) + .chain(manifest.variants.into_iter().map(|(id, variants_manifest)| { + ElsaManifest::resolver_from_manifest_parts( + &release_key, + &variants_manifest.url, + &id, + variants_manifest.format.unwrap_or(Format::Bam), + ) + })) + .collect() + } +} + #[derive(Debug)] pub struct ElsaEndpoint { endpoint: Authority, @@ -61,22 +147,30 @@ pub struct ElsaEndpoint { } #[async_trait] -impl ResolverFromElsa for ElsaEndpoint +impl ResolversFromElsa for ElsaEndpoint where - C: Cache + Send + Sync, - S: GetObject + Send + Sync, + C: Cache, Error = Error> + Send + Sync, + S: GetObject + Send + Sync, { type Error = Error; - async fn try_get(&self, release_key: String) -> Result { - todo!() + async fn try_get(&self, release_key: String) -> Result> { + match self.cache.get(&release_key).await { + Ok(cached) => Ok(cached), + Err(_) => { + let response = self.get_response(&release_key).await?; + let manifest = self.get_manifest(response).await?; + + manifest.try_into() + } + } } } impl ElsaEndpoint where - C: Cache, - S: GetObject, + C: Cache, Error = Error>, + S: GetObject, { pub fn new(endpoint: Authority, cache: C, get_object: S) -> Result { Ok(Self { @@ -95,15 +189,15 @@ where .map_err(|err| Error::InvalidClient(err)) } - pub async fn get_response(&self, release_key: String) -> Result { + pub async fn get_response(&self, release_key: &str) -> Result { let uri = http::Uri::builder() .scheme("https") .authority(self.endpoint.as_str()) - .path_and_query(format!("/manifest/htsget/{}?type=S3", release_key)) + .path_and_query(format!("{}/{}?type=S3", ENDPOINT_PATH, release_key)) .build() .map(|uri| Url::parse(&uri.to_string())) - .map_err(|_| InvalidUri(release_key.to_string()))? - .map_err(|_| InvalidUri(release_key.to_string()))?; + .map_err(|_| InvalidReleaseUri(release_key.to_string()))? + .map_err(|_| InvalidReleaseUri(release_key.to_string()))?; self.client .get(uri) @@ -115,7 +209,9 @@ where .map_err(|err| DeserializeError(err.to_string())) } - pub async fn get_manifest(&self, response: ElsaResponse) -> Result { - todo!() + pub async fn get_manifest(&self, response: ElsaResponse) -> Result { + self.get_object + .get_object(response.location.bucket, response.location.key) + .await } } diff --git a/htsget-elsa/src/lib.rs b/htsget-elsa/src/lib.rs index 529f802..4681e80 100644 --- a/htsget-elsa/src/lib.rs +++ b/htsget-elsa/src/lib.rs @@ -15,21 +15,26 @@ pub enum Error { #[error("invalid client: `{0}`")] InvalidClient(reqwest::Error), #[error("invalid uri constructed from release key: `{0}`")] - InvalidUri(String), + InvalidReleaseUri(String), #[error("failed to get manifest from Elsa: `{0}`")] GetGetManifest(reqwest::Error), #[error("failed to deserialize type: `{0}")] DeserializeError(String), #[error("failed to get object from storage: `{0}`")] GetObjectError(String), + #[error("invalid uri received from manifest: `{0}`")] + InvalidManifestUri(String), + #[error("unsupported component of manifest: `{0}`")] + UnsupportedManifestFeature(String), } #[async_trait] pub trait Cache { + type Error; type Item; - async fn get + Send>(&self, key: K) -> Self::Item; - async fn put + Send>(&self, key: K, item: Self::Item, expirey: u64); + async fn get + Send>(&self, key: K) -> result::Result; + async fn put + Send>(&self, key: K, item: Self::Item, expiry: u64); } #[async_trait] @@ -44,8 +49,8 @@ pub trait GetObject { } #[async_trait] -pub trait ResolverFromElsa { +pub trait ResolversFromElsa { type Error; - async fn try_get(&self, release_key: String) -> result::Result; + async fn try_get(&self, release_key: String) -> result::Result, Self::Error>; } From d547576e3c5439fe3561260e6aa4a05d38cf7040 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 14 Apr 2023 20:03:47 +1000 Subject: [PATCH 06/25] feat(elsa): add s3 cache --- Cargo.lock | 551 +++++++++++++++++++++++++------ htsget-elsa/Cargo.toml | 6 +- htsget-elsa/src/dynamodb.rs | 33 -- htsget-elsa/src/elsa_endpoint.rs | 35 +- htsget-elsa/src/lib.rs | 21 +- htsget-elsa/src/s3.rs | 111 ++++++- 6 files changed, 587 insertions(+), 170 deletions(-) delete mode 100644 htsget-elsa/src/dynamodb.rs diff --git a/Cargo.lock b/Cargo.lock index 26eef0b..f39787b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,17 +144,17 @@ version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c3d1e2a1f1ab3ac6c4b884e37413eaa03eb9d901e4fc68ee8f5c1d49721680e" dependencies = [ - "aws-credential-types", - "aws-http", - "aws-sdk-sso", - "aws-sdk-sts", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-types", + "aws-credential-types 0.54.1", + "aws-http 0.54.1", + "aws-sdk-sso 0.24.0", + "aws-sdk-sts 0.24.0", + "aws-smithy-async 0.54.4", + "aws-smithy-client 0.54.4", + "aws-smithy-http 0.54.4", + "aws-smithy-http-tower 0.54.4", + "aws-smithy-json 0.54.4", + "aws-smithy-types 0.54.4", + "aws-types 0.54.1", "bytes", "hex", "http", @@ -167,14 +167,58 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-config" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1854be4730cc87602316707045a5c0585287419d54f293bbb52a82c895d9086a" +dependencies = [ + "aws-credential-types 0.55.0", + "aws-http 0.55.0", + "aws-sdk-sso 0.25.0", + "aws-sdk-sts 0.25.0", + "aws-smithy-async 0.55.0", + "aws-smithy-client 0.55.0", + "aws-smithy-http 0.55.0", + "aws-smithy-http-tower 0.55.0", + "aws-smithy-json 0.55.0", + "aws-smithy-types 0.55.0", + "aws-types 0.55.0", + "bytes", + "fastrand", + "hex", + "http", + "hyper", + "ring", + "time", + "tokio", + "tower", + "tracing", + "zeroize", +] + [[package]] name = "aws-credential-types" version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0696a0523a39a19087747e4dafda0362dc867531e3d72a3f195564c84e5e08" dependencies = [ - "aws-smithy-async", - "aws-smithy-types", + "aws-smithy-async 0.54.4", + "aws-smithy-types 0.54.4", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77e37e62f59cf3284067337da7467d842df8cfe3f5e5c06487ac7521819cf16d" +dependencies = [ + "aws-smithy-async 0.55.0", + "aws-smithy-types 0.55.0", + "fastrand", "tokio", "tracing", "zeroize", @@ -186,9 +230,23 @@ version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80a4f935ab6a1919fbfd6102a80c4fccd9ff5f47f94ba154074afe1051903261" dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "aws-types", + "aws-smithy-http 0.54.4", + "aws-smithy-types 0.54.4", + "aws-types 0.54.1", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-endpoint" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f38276d5875b19a9bb2b4ae049fd776c932fcc62068f78b71ce475093ccb4c8" +dependencies = [ + "aws-smithy-http 0.55.0", + "aws-smithy-types 0.55.0", + "aws-types 0.55.0", "http", "regex", "tracing", @@ -200,10 +258,10 @@ version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82976ca4e426ee9ca3ffcf919d9b2c8d14d0cd80d43cc02173737a8f07f28d4d" dependencies = [ - "aws-credential-types", - "aws-smithy-http", - "aws-smithy-types", - "aws-types", + "aws-credential-types 0.54.1", + "aws-smithy-http 0.54.4", + "aws-smithy-types 0.54.4", + "aws-types 0.54.1", "bytes", "http", "http-body", @@ -214,28 +272,21 @@ dependencies = [ ] [[package]] -name = "aws-sdk-dynamodb" -version = "0.24.0" +name = "aws-http" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34fc8efd8ed65312b05dc84bb0d7ba79c31802873c4b5676403e220724af39b7" -dependencies = [ - "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-types", +checksum = "67224bfece71e21a63ae82b1ebbfda05be28678a0fab06def03229c7a445d3fb" +dependencies = [ + "aws-credential-types 0.55.0", + "aws-smithy-http 0.55.0", + "aws-smithy-types 0.55.0", + "aws-types 0.55.0", "bytes", - "fastrand", "http", - "regex", - "tokio-stream", - "tower", + "http-body", + "lazy_static", + "percent-encoding", + "pin-project-lite", "tracing", ] @@ -245,21 +296,21 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1533be023eeac69668eb718b1c48af7bd5e26305ed770553d2877ab1f7507b68" dependencies = [ - "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-checksums", - "aws-smithy-client", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", + "aws-credential-types 0.54.1", + "aws-endpoint 0.54.1", + "aws-http 0.54.1", + "aws-sig-auth 0.54.1", + "aws-sigv4 0.54.1", + "aws-smithy-async 0.54.4", + "aws-smithy-checksums 0.54.4", + "aws-smithy-client 0.54.4", + "aws-smithy-eventstream 0.54.4", + "aws-smithy-http 0.54.4", + "aws-smithy-http-tower 0.54.4", + "aws-smithy-json 0.54.4", + "aws-smithy-types 0.54.4", + "aws-smithy-xml 0.54.4", + "aws-types 0.54.1", "bytes", "bytes-utils", "fastrand", @@ -274,23 +325,56 @@ dependencies = [ "url", ] +[[package]] +name = "aws-sdk-s3" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f51a65bdefc7a7fbc3c232663fea5e50c6ca5f6c4a88d6ea766f8e078a945d4" +dependencies = [ + "aws-credential-types 0.55.0", + "aws-endpoint 0.55.0", + "aws-http 0.55.0", + "aws-sig-auth 0.55.0", + "aws-sigv4 0.55.0", + "aws-smithy-async 0.55.0", + "aws-smithy-checksums 0.55.0", + "aws-smithy-client 0.55.0", + "aws-smithy-eventstream 0.55.0", + "aws-smithy-http 0.55.0", + "aws-smithy-http-tower 0.55.0", + "aws-smithy-json 0.55.0", + "aws-smithy-types 0.55.0", + "aws-smithy-xml 0.55.0", + "aws-types 0.55.0", + "bytes", + "http", + "http-body", + "once_cell", + "percent-encoding", + "regex", + "tokio-stream", + "tower", + "tracing", + "url", +] + [[package]] name = "aws-sdk-sso" version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca0119bacf0c42f587506769390983223ba834e605f049babe514b2bd646dbb2" dependencies = [ - "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-types", + "aws-credential-types 0.54.1", + "aws-endpoint 0.54.1", + "aws-http 0.54.1", + "aws-sig-auth 0.54.1", + "aws-smithy-async 0.54.4", + "aws-smithy-client 0.54.4", + "aws-smithy-http 0.54.4", + "aws-smithy-http-tower 0.54.4", + "aws-smithy-json 0.54.4", + "aws-smithy-types 0.54.4", + "aws-types 0.54.1", "bytes", "http", "regex", @@ -298,25 +382,76 @@ dependencies = [ "tower", ] +[[package]] +name = "aws-sdk-sso" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c10657158e12163d6b3fb1e0c9154e43656843794a3071d9ee63ec82a4de2d" +dependencies = [ + "aws-credential-types 0.55.0", + "aws-endpoint 0.55.0", + "aws-http 0.55.0", + "aws-sig-auth 0.55.0", + "aws-smithy-async 0.55.0", + "aws-smithy-client 0.55.0", + "aws-smithy-http 0.55.0", + "aws-smithy-http-tower 0.55.0", + "aws-smithy-json 0.55.0", + "aws-smithy-types 0.55.0", + "aws-types 0.55.0", + "bytes", + "http", + "regex", + "tokio-stream", + "tower", + "tracing", +] + [[package]] name = "aws-sdk-sts" version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "270b6a33969ebfcb193512fbd5e8ee5306888ad6c6d5d775cdbfb2d50d94de26" dependencies = [ - "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-query", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", + "aws-credential-types 0.54.1", + "aws-endpoint 0.54.1", + "aws-http 0.54.1", + "aws-sig-auth 0.54.1", + "aws-smithy-async 0.54.4", + "aws-smithy-client 0.54.4", + "aws-smithy-http 0.54.4", + "aws-smithy-http-tower 0.54.4", + "aws-smithy-json 0.54.4", + "aws-smithy-query 0.54.4", + "aws-smithy-types 0.54.4", + "aws-smithy-xml 0.54.4", + "aws-types 0.54.1", + "bytes", + "http", + "regex", + "tower", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "706a308b7277ac9aac78ab37933509659c6f978a8473e95d8e7a8103093133c3" +dependencies = [ + "aws-credential-types 0.55.0", + "aws-endpoint 0.55.0", + "aws-http 0.55.0", + "aws-sig-auth 0.55.0", + "aws-smithy-async 0.55.0", + "aws-smithy-client 0.55.0", + "aws-smithy-http 0.55.0", + "aws-smithy-http-tower 0.55.0", + "aws-smithy-json 0.55.0", + "aws-smithy-query 0.55.0", + "aws-smithy-types 0.55.0", + "aws-smithy-xml 0.55.0", + "aws-types 0.55.0", "bytes", "http", "regex", @@ -330,11 +465,26 @@ version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "660a02a98ab1af83bd8d714afbab2d502ba9b18c49e7e4cddd6bf8837ff778cb" dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-types", + "aws-credential-types 0.54.1", + "aws-sigv4 0.54.1", + "aws-smithy-eventstream 0.54.4", + "aws-smithy-http 0.54.4", + "aws-types 0.54.1", + "http", + "tracing", +] + +[[package]] +name = "aws-sig-auth" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cebfa0f118afd7197185d5e097bfcdfca9f8410dca50435d67784405f1fd5a05" +dependencies = [ + "aws-credential-types 0.55.0", + "aws-sigv4 0.55.0", + "aws-smithy-eventstream 0.55.0", + "aws-smithy-http 0.55.0", + "aws-types 0.55.0", "http", "tracing", ] @@ -345,8 +495,29 @@ version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdaf11005b7444e6cd66f600d09861a3aeb6eb89a0f003c7c9820dbab2d15297" dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-http", + "aws-smithy-eventstream 0.54.4", + "aws-smithy-http 0.54.4", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http", + "once_cell", + "percent-encoding", + "regex", + "sha2", + "time", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a4f5c05c8646d12b7bb3f18c04edc5ac5e8928ab80e1649e568190f2bc7b79" +dependencies = [ + "aws-smithy-eventstream 0.55.0", + "aws-smithy-http 0.55.0", "bytes", "form_urlencoded", "hex", @@ -372,14 +543,47 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "aws-smithy-async" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cd4b9b7d99263f75304fc1fcd752361cbc4cbf068b832acd8daeaaff44267eb" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", + "tokio-stream", +] + [[package]] name = "aws-smithy-checksums" version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3875fb4b28606a5368a048016a28c15707f2b21238d5b2e4a23198f590e92c4" dependencies = [ - "aws-smithy-http", - "aws-smithy-types", + "aws-smithy-http 0.54.4", + "aws-smithy-types 0.54.4", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http", + "http-body", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa50ea8cc39f5efc78c35f1a2e1eefcd5b82016a38cf0fceba5528d269df823b" +dependencies = [ + "aws-smithy-http 0.55.0", + "aws-smithy-types 0.55.0", "bytes", "crc32c", "crc32fast", @@ -399,11 +603,11 @@ version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "104ca17f56cde00a10207169697dfe9c6810db339d52fb352707e64875b30a44" dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-tower", + "aws-smithy-async 0.54.4", + "aws-smithy-http 0.54.4", + "aws-smithy-http-tower 0.54.4", "aws-smithy-protocol-test", - "aws-smithy-types", + "aws-smithy-types 0.54.4", "bytes", "fastrand", "http", @@ -418,13 +622,48 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-smithy-client" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "748298b60bbd0594223ea136ceed2ed4b6d50970bcefa69a5ff6d710ce593854" +dependencies = [ + "aws-smithy-async 0.55.0", + "aws-smithy-http 0.55.0", + "aws-smithy-http-tower 0.55.0", + "aws-smithy-types 0.55.0", + "bytes", + "fastrand", + "http", + "http-body", + "hyper", + "hyper-rustls", + "lazy_static", + "pin-project-lite", + "rustls", + "tokio", + "tower", + "tracing", +] + [[package]] name = "aws-smithy-eventstream" version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac250d8c0e42af0097a6837ffc5a6fb9f8ba4107bb53124c047c91bc2a58878f" dependencies = [ - "aws-smithy-types", + "aws-smithy-types 0.54.4", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7638e9fe2e97d2c4b173a44d9002c7cf29da7be7bab25c7837cfdce7945b2ff8" +dependencies = [ + "aws-smithy-types 0.55.0", "bytes", "crc32fast", ] @@ -435,8 +674,31 @@ version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "873f316f1833add0d3aa54ed1b0cd252ddd88c792a0cf839886400099971e844" dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-types", + "aws-smithy-eventstream 0.54.4", + "aws-smithy-types 0.54.4", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "hyper", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "aws-smithy-http" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78510732b81040689dc146e3693bfbcf388ab88cbda667d3ef67f8869b0744a" +dependencies = [ + "aws-smithy-eventstream 0.55.0", + "aws-smithy-types 0.55.0", "bytes", "bytes-utils", "futures-core", @@ -458,8 +720,24 @@ version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f38231d3f5dac9ac7976f44e12803add1385119ffca9e5f050d8e980733d164" dependencies = [ - "aws-smithy-http", - "aws-smithy-types", + "aws-smithy-http 0.54.4", + "aws-smithy-types 0.54.4", + "bytes", + "http", + "http-body", + "pin-project-lite", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-http-tower" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33689c27bbd8184412b45c4d1ab795d9a35402562d9fde6c53695a90969740" +dependencies = [ + "aws-smithy-http 0.55.0", + "aws-smithy-types 0.55.0", "bytes", "http", "http-body", @@ -474,7 +752,16 @@ version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bd83ff2b79e9f729746fcc8ad798676b68fe6ea72986571569a5306a277a182" dependencies = [ - "aws-smithy-types", + "aws-smithy-types 0.54.4", +] + +[[package]] +name = "aws-smithy-json" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada31cab1b1d1f0abc5c4d1183de5b278597704851aa703801b82feabf19aa74" +dependencies = [ + "aws-smithy-types 0.55.0", ] [[package]] @@ -498,7 +785,17 @@ version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2f0445dafe9d2cd50b44339ae3c3ed46549aad8ac696c52ad660b3e7ae8682b" dependencies = [ - "aws-smithy-types", + "aws-smithy-types 0.54.4", + "urlencoding", +] + +[[package]] +name = "aws-smithy-query" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55358401b657d192f70f093927f01d73cc4859e2907956b20c4043c76624006" +dependencies = [ + "aws-smithy-types 0.55.0", "urlencoding", ] @@ -515,6 +812,19 @@ dependencies = [ "time", ] +[[package]] +name = "aws-smithy-types" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "474d145c2e0f82892841d2502bd546ca0dbc1e4e242c3563d96e7061054c268f" +dependencies = [ + "base64-simd", + "itoa", + "num-integer", + "ryu", + "time", +] + [[package]] name = "aws-smithy-xml" version = "0.54.4" @@ -524,17 +834,42 @@ dependencies = [ "xmlparser", ] +[[package]] +name = "aws-smithy-xml" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb159921734d090b01c586a4fef73964f42fcb7eb53a8184b2db374bd6a6fc99" +dependencies = [ + "xmlparser", +] + [[package]] name = "aws-types" version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8f15b34253b68cde08e39b0627cc6101bcca64351229484b4743392c035d057" dependencies = [ - "aws-credential-types", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-types", + "aws-credential-types 0.54.1", + "aws-smithy-async 0.54.4", + "aws-smithy-client 0.54.4", + "aws-smithy-http 0.54.4", + "aws-smithy-types 0.54.4", + "http", + "rustc_version", + "tracing", +] + +[[package]] +name = "aws-types" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fb02591b5075d318e0083dcb76df0e151db4ce48f987ecd00e5b53c7a6ba59" +dependencies = [ + "aws-credential-types 0.55.0", + "aws-smithy-async 0.55.0", + "aws-smithy-client 0.55.0", + "aws-smithy-http 0.55.0", + "aws-smithy-types 0.55.0", "http", "rustc_version", "tracing", @@ -1288,9 +1623,9 @@ name = "htsget-elsa" version = "0.1.0" dependencies = [ "async-trait", - "aws-config", - "aws-sdk-dynamodb", - "aws-sdk-s3", + "aws-config 0.55.0", + "aws-sdk-s3 0.25.0", + "bytes", "htsget-config", "http", "regex", @@ -1359,8 +1694,8 @@ version = "0.1.4" source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#a040432d131c83be7d5460930c0879f5c13dfbda" dependencies = [ "async-trait", - "aws-config", - "aws-sdk-s3", + "aws-config 0.54.1", + "aws-sdk-s3 0.24.0", "axum", "axum-extra", "base64 0.21.0", diff --git a/htsget-elsa/Cargo.toml b/htsget-elsa/Cargo.toml index e557feb..c43787b 100644 --- a/htsget-elsa/Cargo.toml +++ b/htsget-elsa/Cargo.toml @@ -8,11 +8,11 @@ async-trait = "0.1" http = "0.2" reqwest = { version = "0.11", features = ["rustls-tls", "json"] } thiserror = "1.0" -aws-config = "0.54" -aws-sdk-dynamodb = "0.24" -aws-sdk-s3 = "0.24" +aws-config = "0.55" +aws-sdk-s3 = "0.25" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" regex = "1.7" +bytes = "1.4" htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } \ No newline at end of file diff --git a/htsget-elsa/src/dynamodb.rs b/htsget-elsa/src/dynamodb.rs deleted file mode 100644 index 83bddc4..0000000 --- a/htsget-elsa/src/dynamodb.rs +++ /dev/null @@ -1,33 +0,0 @@ -use crate::{Cache, Error, Result}; -use async_trait::async_trait; -use aws_sdk_dynamodb::Client; - -#[derive(Debug)] -pub struct DynamoDBCache { - client: Client, - table_name: String, -} - -impl DynamoDBCache { - pub fn new(client: Client, table_name: String) -> Self { - Self { client, table_name } - } - - pub async fn new_with_default_config(table_name: String) -> Self { - DynamoDBCache::new(Client::new(&aws_config::load_from_env().await), table_name) - } -} - -#[async_trait] -impl Cache for DynamoDBCache { - type Error = Error; - type Item = (); - - async fn get + Send>(&self, key: K) -> Result { - todo!() - } - - async fn put + Send>(&self, key: K, item: Self::Item, expiry: u64) { - todo!() - } -} diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index 1cac100..347399f 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -126,14 +126,19 @@ impl TryFrom for Vec { reads_manifest.format.unwrap_or(Format::Bam), ) }) - .chain(manifest.variants.into_iter().map(|(id, variants_manifest)| { - ElsaManifest::resolver_from_manifest_parts( - &release_key, - &variants_manifest.url, - &id, - variants_manifest.format.unwrap_or(Format::Bam), - ) - })) + .chain( + manifest + .variants + .into_iter() + .map(|(id, variants_manifest)| { + ElsaManifest::resolver_from_manifest_parts( + &release_key, + &variants_manifest.url, + &id, + variants_manifest.format.unwrap_or(Format::Bam), + ) + }), + ) .collect() } } @@ -156,12 +161,18 @@ where async fn try_get(&self, release_key: String) -> Result> { match self.cache.get(&release_key).await { - Ok(cached) => Ok(cached), - Err(_) => { + Ok(Some(cached)) => Ok(cached), + _ => { let response = self.get_response(&release_key).await?; - let manifest = self.get_manifest(response).await?; + let max_age = response.max_age; + + let resolvers: Vec = self.get_manifest(response).await?.try_into()?; + + self.cache + .put(&release_key, resolvers.clone(), max_age) + .await?; - manifest.try_into() + Ok(resolvers) } } } diff --git a/htsget-elsa/src/lib.rs b/htsget-elsa/src/lib.rs index 4681e80..fa2ad09 100644 --- a/htsget-elsa/src/lib.rs +++ b/htsget-elsa/src/lib.rs @@ -1,4 +1,3 @@ -pub mod dynamodb; pub mod elsa_endpoint; pub mod s3; @@ -18,14 +17,20 @@ pub enum Error { InvalidReleaseUri(String), #[error("failed to get manifest from Elsa: `{0}`")] GetGetManifest(reqwest::Error), - #[error("failed to deserialize type: `{0}")] + #[error("failed to deserialize: `{0}")] DeserializeError(String), + #[error("failed to serialize: `{0}")] + SerializeError(String), #[error("failed to get object from storage: `{0}`")] GetObjectError(String), + #[error("failed to put object into storage: `{0}`")] + PutObjectError(String), #[error("invalid uri received from manifest: `{0}`")] InvalidManifestUri(String), #[error("unsupported component of manifest: `{0}`")] UnsupportedManifestFeature(String), + #[error("system error: `{0}`")] + SystemError(String), } #[async_trait] @@ -33,8 +38,16 @@ pub trait Cache { type Error; type Item; - async fn get + Send>(&self, key: K) -> result::Result; - async fn put + Send>(&self, key: K, item: Self::Item, expiry: u64); + async fn get + Send + Sync>( + &self, + key: K, + ) -> result::Result, Self::Error>; + async fn put + Send + Sync>( + &self, + key: K, + item: Self::Item, + max_age: u64, + ) -> result::Result<(), Self::Error>; } #[async_trait] diff --git a/htsget-elsa/src/s3.rs b/htsget-elsa/src/s3.rs index 050809b..b248eac 100644 --- a/htsget-elsa/src/s3.rs +++ b/htsget-elsa/src/s3.rs @@ -1,24 +1,49 @@ -use crate::Error::{DeserializeError, GetObjectError}; -use crate::{Error, GetObject, Result}; +use crate::Error::{DeserializeError, GetObjectError, PutObjectError, SerializeError}; +use crate::{Cache, Error, GetObject, Result}; use async_trait::async_trait; +use aws_sdk_s3::primitives::{ByteStream, DateTime}; use aws_sdk_s3::Client; -use serde::Deserialize; -use serde_json::from_slice; +use bytes::Bytes; +use htsget_config::resolver::Resolver; +use serde::{Deserialize, Serialize}; +use serde_json::{from_slice, to_string, to_vec}; +use std::ops::Add; +use std::time::{Duration, SystemTime}; #[derive(Debug)] pub struct S3 { s3_client: Client, + cache_bucket: String, } -#[async_trait] -impl GetObject for S3 { - type Error = Error; +#[derive(Debug, Deserialize, Serialize)] +pub struct CacheItem { + item: Vec, + max_age: u64, +} + +impl S3 { + pub fn new(s3_client: Client, cache_bucket: String) -> Self { + Self { + s3_client, + cache_bucket, + } + } + pub async fn new_with_default_config(cache_bucket: String) -> Self { + Self::new( + Client::new(&aws_config::load_from_env().await), + cache_bucket, + ) + } +} + +impl S3 { async fn get_object Deserialize<'de>>( &self, bucket: impl Into + Send, key: impl Into + Send, - ) -> Result { + ) -> Result<(T, Option)> { let output = self .s3_client .get_object() @@ -26,13 +51,79 @@ impl GetObject for S3 { .key(key) .send() .await - .map_err(|err| GetObjectError(err.to_string()))? + .map_err(|err| GetObjectError(err.to_string()))?; + + let last_modified = output.last_modified().copied(); + + let output = output .body .collect() .await .map_err(|err| GetObjectError(err.to_string()))? .into_bytes(); - from_slice(output.as_ref()).map_err(|err| DeserializeError(err.to_string())) + Ok(( + from_slice(output.as_ref()).map_err(|err| DeserializeError(err.to_string()))?, + last_modified, + )) + } +} + +#[async_trait] +impl GetObject for S3 { + type Error = Error; + + async fn get_object Deserialize<'de>>( + &self, + bucket: impl Into + Send, + key: impl Into + Send, + ) -> Result { + Ok(self.get_object(bucket, key).await?.0) + } +} + +#[async_trait] +impl Cache for S3 { + type Error = Error; + type Item = Vec; + + async fn get + Send + Sync>(&self, key: K) -> Result> { + let (object, last_modified): (CacheItem, _) = self + .get_object(self.cache_bucket.clone(), key.as_ref()) + .await?; + + match last_modified { + Some(last_modified) + if last_modified.as_nanos() + <= DateTime::from( + SystemTime::now().add(Duration::from_secs(object.max_age)), + ) + .as_nanos() => + { + Ok(Some(object.item)) + } + _ => Ok(None), + } + } + + async fn put + Send + Sync>( + &self, + key: K, + item: Self::Item, + max_age: u64, + ) -> Result<()> { + self.s3_client + .put_object() + .bucket(self.cache_bucket.clone()) + .key(key.as_ref()) + .body(ByteStream::from(Bytes::from( + to_vec(&CacheItem { item, max_age }) + .map_err(|err| SerializeError(err.to_string()))?, + ))) + .send() + .await + .map_err(|err| PutObjectError(err.to_string()))?; + + Ok(()) } } From 4d9e0d220fdd474eea77694e6a9e99283a72ac94 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 14 Apr 2023 20:03:55 +1000 Subject: [PATCH 07/25] feat(elsa-lambda): use elsa endpoint for lambda function --- Cargo.lock | 52 +++++++++++----------- htsget-elsa-lambda/Cargo.toml | 4 +- htsget-elsa-lambda/src/config.rs | 21 ++++++--- htsget-elsa-lambda/src/lib.rs | 75 +++++++++++++++++++++++++++----- htsget-elsa-lambda/src/main.rs | 12 ++--- htsget-elsa/Cargo.toml | 1 + htsget-elsa/src/elsa_endpoint.rs | 22 +++++++--- htsget-elsa/src/lib.rs | 1 + htsget-elsa/src/s3.rs | 3 ++ 9 files changed, 135 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f39787b..5b2b54d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,9 +92,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" dependencies = [ "async-stream-impl", "futures-core", @@ -103,13 +103,13 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.13", ] [[package]] @@ -175,8 +175,8 @@ checksum = "1854be4730cc87602316707045a5c0585287419d54f293bbb52a82c895d9086a" dependencies = [ "aws-credential-types 0.55.0", "aws-http 0.55.0", - "aws-sdk-sso 0.25.0", - "aws-sdk-sts 0.25.0", + "aws-sdk-sso 0.25.1", + "aws-sdk-sts 0.25.1", "aws-smithy-async 0.55.0", "aws-smithy-client 0.55.0", "aws-smithy-http 0.55.0", @@ -327,9 +327,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "0.25.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f51a65bdefc7a7fbc3c232663fea5e50c6ca5f6c4a88d6ea766f8e078a945d4" +checksum = "392b9811ca489747ac84349790e49deaa1f16631949e7dd4156000251c260eae" dependencies = [ "aws-credential-types 0.55.0", "aws-endpoint 0.55.0", @@ -384,9 +384,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "0.25.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c10657158e12163d6b3fb1e0c9154e43656843794a3071d9ee63ec82a4de2d" +checksum = "aad0ad03c4ba802906340847aaea796d4158317864276864acbb281155bdcf4d" dependencies = [ "aws-credential-types 0.55.0", "aws-endpoint 0.55.0", @@ -435,9 +435,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "0.25.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "706a308b7277ac9aac78ab37933509659c6f978a8473e95d8e7a8103093133c3" +checksum = "2493efc64aa1c6469a252c221c335ccdcebfcc556c81b1efaee051c6ddc3dd99" dependencies = [ "aws-credential-types 0.55.0", "aws-endpoint 0.55.0", @@ -1600,7 +1600,7 @@ dependencies = [ [[package]] name = "htsget-config" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#a040432d131c83be7d5460930c0879f5c13dfbda" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#70ca723440e9331099d2bb77cbe4bab82c07b5b4" dependencies = [ "async-trait", "clap", @@ -1624,7 +1624,7 @@ version = "0.1.0" dependencies = [ "async-trait", "aws-config 0.55.0", - "aws-sdk-s3 0.25.0", + "aws-sdk-s3 0.25.1", "bytes", "htsget-config", "http", @@ -1633,6 +1633,7 @@ dependencies = [ "serde", "serde_json", "thiserror", + "tracing", ] [[package]] @@ -1640,6 +1641,7 @@ name = "htsget-elsa-lambda" version = "0.1.0" dependencies = [ "htsget-config", + "htsget-elsa", "htsget-lambda", "http", "http-serde", @@ -1653,7 +1655,7 @@ dependencies = [ [[package]] name = "htsget-http" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#a040432d131c83be7d5460930c0879f5c13dfbda" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#70ca723440e9331099d2bb77cbe4bab82c07b5b4" dependencies = [ "futures", "htsget-config", @@ -1669,7 +1671,7 @@ dependencies = [ [[package]] name = "htsget-lambda" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#a040432d131c83be7d5460930c0879f5c13dfbda" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#70ca723440e9331099d2bb77cbe4bab82c07b5b4" dependencies = [ "bytes", "htsget-config", @@ -1691,7 +1693,7 @@ dependencies = [ [[package]] name = "htsget-search" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#a040432d131c83be7d5460930c0879f5c13dfbda" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#70ca723440e9331099d2bb77cbe4bab82c07b5b4" dependencies = [ "async-trait", "aws-config 0.54.1", @@ -1721,7 +1723,7 @@ dependencies = [ [[package]] name = "htsget-test" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#a040432d131c83be7d5460930c0879f5c13dfbda" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#70ca723440e9331099d2bb77cbe4bab82c07b5b4" dependencies = [ "async-trait", "base64 0.21.0", @@ -2974,9 +2976,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85456ffac572dc8826334164f2fb6fb40a7c766aebe195a2a21ee69ee2885ecf" +checksum = "331bb8c3bf9b92457ab7abecf07078c13f7d270ba490103e84e8b014490cd0b0" dependencies = [ "base64 0.13.1", "chrono", @@ -2990,9 +2992,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cbcd6104f8a4ab6af7f6be2a0da6be86b9de3c401f6e86bb856ab2af739232f" +checksum = "859011bddcc11f289f07f467cc1fe01c7a941daa4d8f6c40d4d1c92eb6d9319c" dependencies = [ "darling", "proc-macro2", @@ -3885,9 +3887,9 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "yasna" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aed2e7a52e3744ab4d0c05c20aa065258e84c49fd4226f5191b2ed29712710b4" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ "time", ] diff --git a/htsget-elsa-lambda/Cargo.toml b/htsget-elsa-lambda/Cargo.toml index 4287a7b..c405a26 100644 --- a/htsget-elsa-lambda/Cargo.toml +++ b/htsget-elsa-lambda/Cargo.toml @@ -13,4 +13,6 @@ http-serde = "1.1" http = "0.2" htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } -htsget-lambda = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } \ No newline at end of file +htsget-lambda = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } + +htsget-elsa = { version = "0.1", path = "../htsget-elsa" } \ No newline at end of file diff --git a/htsget-elsa-lambda/src/config.rs b/htsget-elsa-lambda/src/config.rs index 908c0b6..59d900e 100644 --- a/htsget-elsa-lambda/src/config.rs +++ b/htsget-elsa-lambda/src/config.rs @@ -4,21 +4,32 @@ use http::uri::Authority; use serde::{Deserialize, Serialize}; use std::io; use std::path::Path; +use tracing::instrument; #[derive(Serialize, Deserialize, Debug, Clone)] -struct Config { +pub struct Config { #[serde(flatten)] htsget_config: HtsGetConfig, - root_certificate_id: String, - identity_id: String, - parameters_secrets_extension_http_port: u16, - dynamodb_table_name: String, #[serde(with = "http_serde::authority")] elsa_endpoint: Authority, + cache_location: String, } impl Config { + #[instrument(level = "debug", ret)] pub fn from_path(path: &Path) -> io::Result { from_path(path) } + + pub fn htsget_config(&self) -> &HtsGetConfig { + &self.htsget_config + } + + pub fn elsa_endpoint(&self) -> &Authority { + &self.elsa_endpoint + } + + pub fn cache_location(&self) -> &str { + &self.cache_location + } } diff --git a/htsget-elsa-lambda/src/lib.rs b/htsget-elsa-lambda/src/lib.rs index de189d5..4412fa4 100644 --- a/htsget-elsa-lambda/src/lib.rs +++ b/htsget-elsa-lambda/src/lib.rs @@ -1,25 +1,76 @@ +use crate::config::Config; use htsget_config::config::cors::CorsConfig; use htsget_config::config::ServiceInfo; use htsget_config::resolver::Resolver; -use htsget_lambda::{handle_request_service_fn, Router}; -use lambda_http::{Error, Request}; +use htsget_elsa::elsa_endpoint::ElsaEndpoint; +use htsget_elsa::s3::S3; +use htsget_elsa::{Cache, GetObject, ResolversFromElsa}; +use htsget_lambda::handlers::FormatJson; +use htsget_lambda::RouteType::Id; +use htsget_lambda::{handle_request_service_fn, Route, Router}; +use http::{Response, StatusCode}; +use lambda_http::{Body, Error, Request}; use std::sync::Arc; -use tracing::info; +use tracing::{info, instrument, warn}; pub mod config; -pub async fn handle_request(cors: CorsConfig, service_info: &ServiceInfo) -> Result<(), Error> { - handle_request_service_fn(cors, |event: Request| async move { - info!(event = ?event, "received request"); +pub async fn handle_request(config: Config) -> Result<(), Error> { + handle_request_service_fn( + config.htsget_config().ticket_server().cors().clone(), + |event: Request| async { + info!(event = ?event, "received request"); - let resolver = get_resolvers(&event).await; - let router = Router::new(Arc::new(resolver), &service_info); + match Route::try_from(&event) { + Ok(route) => { + let s3 = S3::new_with_default_config(config.cache_location().to_string()).await; + let elsa_endpoint = + match ElsaEndpoint::new(config.elsa_endpoint().clone(), &s3, &s3) { + Ok(elsa_endpoint) => elsa_endpoint, + Err(err) => { + return Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::from(err.to_string())); + } + }; - router.route_request(event).await - }) + let resolver = get_resolvers(&config, &route, &elsa_endpoint).await?; + let router = Router::new( + Arc::new(resolver), + &config.htsget_config().ticket_server().service_info(), + ); + router.route_request_with_route(event, route).await + } + Err(err) => err, + } + }, + ) .await } -pub async fn get_resolvers(event: &Request) -> Vec { - todo!(); +#[instrument(level = "debug", skip(elsa_endpoint), ret)] +pub async fn get_resolvers<'a, C, S>( + config: &Config, + route: &Route, + elsa_endpoint: &'a ElsaEndpoint<'a, C, S>, +) -> http::Result> +where + C: Cache, Error = htsget_elsa::Error> + Send + Sync, + S: GetObject + Send + Sync, +{ + if let Id(id) = route.route_type() { + if let Some(release_key) = id.split("/").collect::>().first() { + if let Ok(mut resolvers) = elsa_endpoint.try_get(release_key.to_string()).await { + resolvers.append(&mut config.htsget_config().resolvers().to_vec()); + + return Ok(resolvers); + } + } + } + + warn!( + "failed to get resolvers from elsa endpoint, attempting to use only resolvers from config" + ); + + Ok(config.htsget_config().resolvers().to_vec()) } diff --git a/htsget-elsa-lambda/src/main.rs b/htsget-elsa-lambda/src/main.rs index 0621013..e3189eb 100644 --- a/htsget-elsa-lambda/src/main.rs +++ b/htsget-elsa-lambda/src/main.rs @@ -5,18 +5,18 @@ use std::sync::Arc; use lambda_http::Error; +use htsget_elsa_lambda::config::Config; use htsget_elsa_lambda::handle_request; -use htsget_lambda::Config; +use htsget_lambda::Config as HtsgetConfig; #[tokio::main] async fn main() -> Result<(), Error> { - Config::setup_tracing()?; + HtsgetConfig::setup_tracing()?; - if let Some(path) = Config::parse_args() { - let cors: CorsConfig = from_path(&path)?; - let service_info: ServiceInfo = from_path(&path)?; + if let Some(path) = HtsgetConfig::parse_args() { + let config: Config = from_path(&path)?; - handle_request(cors, &service_info).await + handle_request(config).await } else { Ok(()) } diff --git a/htsget-elsa/Cargo.toml b/htsget-elsa/Cargo.toml index c43787b..d90b629 100644 --- a/htsget-elsa/Cargo.toml +++ b/htsget-elsa/Cargo.toml @@ -14,5 +14,6 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" regex = "1.7" bytes = "1.4" +tracing = "0.1" htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } \ No newline at end of file diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index 347399f..8c09606 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -14,6 +14,7 @@ use reqwest::{Client, Url}; use serde::Deserialize; use std::collections::HashMap; use std::str::FromStr; +use tracing::{debug, instrument}; const ENDPOINT_PATH: &str = "/manifest/htsget"; @@ -63,6 +64,7 @@ pub struct ElsaManifest { } impl ElsaManifest { + #[instrument(level = "trace", ret)] pub fn resolver_from_manifest_parts( release_key: &str, url: &str, @@ -112,6 +114,7 @@ impl ElsaManifest { impl TryFrom for Vec { type Error = Error; + #[instrument(level = "trace", ret)] fn try_from(manifest: ElsaManifest) -> Result { let release_key = manifest.release_key; @@ -144,25 +147,28 @@ impl TryFrom for Vec { } #[derive(Debug)] -pub struct ElsaEndpoint { +pub struct ElsaEndpoint<'a, C, S> { endpoint: Authority, client: Client, - cache: C, - get_object: S, + cache: &'a C, + get_object: &'a S, } #[async_trait] -impl ResolversFromElsa for ElsaEndpoint +impl<'a, C, S> ResolversFromElsa for ElsaEndpoint<'a, C, S> where C: Cache, Error = Error> + Send + Sync, S: GetObject + Send + Sync, { type Error = Error; + #[instrument(level = "debug", skip_all)] async fn try_get(&self, release_key: String) -> Result> { match self.cache.get(&release_key).await { Ok(Some(cached)) => Ok(cached), _ => { + debug!("no cached response, fetching from elsa"); + let response = self.get_response(&release_key).await?; let max_age = response.max_age; @@ -178,12 +184,12 @@ where } } -impl ElsaEndpoint +impl<'a, C, S> ElsaEndpoint<'a, C, S> where C: Cache, Error = Error>, S: GetObject, { - pub fn new(endpoint: Authority, cache: C, get_object: S) -> Result { + pub fn new(endpoint: Authority, cache: &'a C, get_object: &'a S) -> Result { Ok(Self { endpoint, client: Self::create_client()?, @@ -200,8 +206,9 @@ where .map_err(|err| Error::InvalidClient(err)) } + #[instrument(level = "debug", skip(self), ret)] pub async fn get_response(&self, release_key: &str) -> Result { - let uri = http::Uri::builder() + let uri = Uri::builder() .scheme("https") .authority(self.endpoint.as_str()) .path_and_query(format!("{}/{}?type=S3", ENDPOINT_PATH, release_key)) @@ -220,6 +227,7 @@ where .map_err(|err| DeserializeError(err.to_string())) } + #[instrument(level = "debug", skip(self), ret)] pub async fn get_manifest(&self, response: ElsaResponse) -> Result { self.get_object .get_object(response.location.bucket, response.location.key) diff --git a/htsget-elsa/src/lib.rs b/htsget-elsa/src/lib.rs index fa2ad09..3905ffc 100644 --- a/htsget-elsa/src/lib.rs +++ b/htsget-elsa/src/lib.rs @@ -3,6 +3,7 @@ pub mod s3; use async_trait::async_trait; use htsget_config::resolver::Resolver; +use http::Response; use serde::Deserialize; use std::result; use thiserror::Error; diff --git a/htsget-elsa/src/s3.rs b/htsget-elsa/src/s3.rs index b248eac..4901411 100644 --- a/htsget-elsa/src/s3.rs +++ b/htsget-elsa/src/s3.rs @@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize}; use serde_json::{from_slice, to_string, to_vec}; use std::ops::Add; use std::time::{Duration, SystemTime}; +use tracing::instrument; #[derive(Debug)] pub struct S3 { @@ -87,6 +88,7 @@ impl Cache for S3 { type Error = Error; type Item = Vec; + #[instrument(level = "trace", skip_all, ret)] async fn get + Send + Sync>(&self, key: K) -> Result> { let (object, last_modified): (CacheItem, _) = self .get_object(self.cache_bucket.clone(), key.as_ref()) @@ -106,6 +108,7 @@ impl Cache for S3 { } } + #[instrument(level = "trace", skip_all, ret)] async fn put + Send + Sync>( &self, key: K, From 82abb02f8658dcee927e47777b18021da9914fad Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 14 Apr 2023 20:04:01 +1000 Subject: [PATCH 08/25] feat(elsa-lambda): create route request function --- htsget-elsa-lambda/src/lib.rs | 43 +++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/htsget-elsa-lambda/src/lib.rs b/htsget-elsa-lambda/src/lib.rs index 4412fa4..19359e3 100644 --- a/htsget-elsa-lambda/src/lib.rs +++ b/htsget-elsa-lambda/src/lib.rs @@ -22,25 +22,7 @@ pub async fn handle_request(config: Config) -> Result<(), Error> { info!(event = ?event, "received request"); match Route::try_from(&event) { - Ok(route) => { - let s3 = S3::new_with_default_config(config.cache_location().to_string()).await; - let elsa_endpoint = - match ElsaEndpoint::new(config.elsa_endpoint().clone(), &s3, &s3) { - Ok(elsa_endpoint) => elsa_endpoint, - Err(err) => { - return Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(Body::from(err.to_string())); - } - }; - - let resolver = get_resolvers(&config, &route, &elsa_endpoint).await?; - let router = Router::new( - Arc::new(resolver), - &config.htsget_config().ticket_server().service_info(), - ); - router.route_request_with_route(event, route).await - } + Ok(route) => route_request(&config, event, route).await, Err(err) => err, } }, @@ -48,6 +30,29 @@ pub async fn handle_request(config: Config) -> Result<(), Error> { .await } +async fn route_request( + config: &Config, + event: Request, + route: Route, +) -> http::Result> { + let s3 = S3::new_with_default_config(config.cache_location().to_string()).await; + let elsa_endpoint = match ElsaEndpoint::new(config.elsa_endpoint().clone(), &s3, &s3) { + Ok(elsa_endpoint) => elsa_endpoint, + Err(err) => { + return Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::from(err.to_string())); + } + }; + + let resolver = get_resolvers(&config, &route, &elsa_endpoint).await?; + let router = Router::new( + Arc::new(resolver), + &config.htsget_config().ticket_server().service_info(), + ); + router.route_request_with_route(event, route).await +} + #[instrument(level = "debug", skip(elsa_endpoint), ret)] pub async fn get_resolvers<'a, C, S>( config: &Config, From 5ee745e011e1b17d024962a87561d8c6f795f6d4 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 14 Apr 2023 20:04:09 +1000 Subject: [PATCH 09/25] test(elsa): add get response test --- Cargo.lock | 998 ++++++++++++++++--------------- htsget-elsa/Cargo.toml | 12 +- htsget-elsa/src/elsa_endpoint.rs | 101 +++- htsget-elsa/src/lib.rs | 2 +- htsget-elsa/src/s3.rs | 4 +- 5 files changed, 601 insertions(+), 516 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b2b54d..066692d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -66,6 +66,18 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + [[package]] name = "assert-json-diff" version = "1.1.0" @@ -77,6 +89,16 @@ dependencies = [ "serde_json", ] +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "async-compression" version = "0.3.15" @@ -109,7 +131,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.14", ] [[package]] @@ -120,7 +142,16 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.14", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", ] [[package]] @@ -132,6 +163,17 @@ dependencies = [ "autocfg", ] +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -144,17 +186,17 @@ version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c3d1e2a1f1ab3ac6c4b884e37413eaa03eb9d901e4fc68ee8f5c1d49721680e" dependencies = [ - "aws-credential-types 0.54.1", - "aws-http 0.54.1", - "aws-sdk-sso 0.24.0", - "aws-sdk-sts 0.24.0", - "aws-smithy-async 0.54.4", - "aws-smithy-client 0.54.4", - "aws-smithy-http 0.54.4", - "aws-smithy-http-tower 0.54.4", - "aws-smithy-json 0.54.4", - "aws-smithy-types 0.54.4", - "aws-types 0.54.1", + "aws-credential-types", + "aws-http", + "aws-sdk-sso", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", "bytes", "hex", "http", @@ -167,58 +209,14 @@ dependencies = [ "zeroize", ] -[[package]] -name = "aws-config" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1854be4730cc87602316707045a5c0585287419d54f293bbb52a82c895d9086a" -dependencies = [ - "aws-credential-types 0.55.0", - "aws-http 0.55.0", - "aws-sdk-sso 0.25.1", - "aws-sdk-sts 0.25.1", - "aws-smithy-async 0.55.0", - "aws-smithy-client 0.55.0", - "aws-smithy-http 0.55.0", - "aws-smithy-http-tower 0.55.0", - "aws-smithy-json 0.55.0", - "aws-smithy-types 0.55.0", - "aws-types 0.55.0", - "bytes", - "fastrand", - "hex", - "http", - "hyper", - "ring", - "time", - "tokio", - "tower", - "tracing", - "zeroize", -] - [[package]] name = "aws-credential-types" version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0696a0523a39a19087747e4dafda0362dc867531e3d72a3f195564c84e5e08" dependencies = [ - "aws-smithy-async 0.54.4", - "aws-smithy-types 0.54.4", - "tokio", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-credential-types" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e37e62f59cf3284067337da7467d842df8cfe3f5e5c06487ac7521819cf16d" -dependencies = [ - "aws-smithy-async 0.55.0", - "aws-smithy-types 0.55.0", - "fastrand", + "aws-smithy-async", + "aws-smithy-types", "tokio", "tracing", "zeroize", @@ -230,23 +228,9 @@ version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80a4f935ab6a1919fbfd6102a80c4fccd9ff5f47f94ba154074afe1051903261" dependencies = [ - "aws-smithy-http 0.54.4", - "aws-smithy-types 0.54.4", - "aws-types 0.54.1", - "http", - "regex", - "tracing", -] - -[[package]] -name = "aws-endpoint" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f38276d5875b19a9bb2b4ae049fd776c932fcc62068f78b71ce475093ccb4c8" -dependencies = [ - "aws-smithy-http 0.55.0", - "aws-smithy-types 0.55.0", - "aws-types 0.55.0", + "aws-smithy-http", + "aws-smithy-types", + "aws-types", "http", "regex", "tracing", @@ -258,29 +242,10 @@ version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82976ca4e426ee9ca3ffcf919d9b2c8d14d0cd80d43cc02173737a8f07f28d4d" dependencies = [ - "aws-credential-types 0.54.1", - "aws-smithy-http 0.54.4", - "aws-smithy-types 0.54.4", - "aws-types 0.54.1", - "bytes", - "http", - "http-body", - "lazy_static", - "percent-encoding", - "pin-project-lite", - "tracing", -] - -[[package]] -name = "aws-http" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67224bfece71e21a63ae82b1ebbfda05be28678a0fab06def03229c7a445d3fb" -dependencies = [ - "aws-credential-types 0.55.0", - "aws-smithy-http 0.55.0", - "aws-smithy-types 0.55.0", - "aws-types 0.55.0", + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-types", + "aws-types", "bytes", "http", "http-body", @@ -296,21 +261,21 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1533be023eeac69668eb718b1c48af7bd5e26305ed770553d2877ab1f7507b68" dependencies = [ - "aws-credential-types 0.54.1", - "aws-endpoint 0.54.1", - "aws-http 0.54.1", - "aws-sig-auth 0.54.1", - "aws-sigv4 0.54.1", - "aws-smithy-async 0.54.4", - "aws-smithy-checksums 0.54.4", - "aws-smithy-client 0.54.4", - "aws-smithy-eventstream 0.54.4", - "aws-smithy-http 0.54.4", - "aws-smithy-http-tower 0.54.4", - "aws-smithy-json 0.54.4", - "aws-smithy-types 0.54.4", - "aws-smithy-xml 0.54.4", - "aws-types 0.54.1", + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-client", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", "bytes", "bytes-utils", "fastrand", @@ -325,56 +290,23 @@ dependencies = [ "url", ] -[[package]] -name = "aws-sdk-s3" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "392b9811ca489747ac84349790e49deaa1f16631949e7dd4156000251c260eae" -dependencies = [ - "aws-credential-types 0.55.0", - "aws-endpoint 0.55.0", - "aws-http 0.55.0", - "aws-sig-auth 0.55.0", - "aws-sigv4 0.55.0", - "aws-smithy-async 0.55.0", - "aws-smithy-checksums 0.55.0", - "aws-smithy-client 0.55.0", - "aws-smithy-eventstream 0.55.0", - "aws-smithy-http 0.55.0", - "aws-smithy-http-tower 0.55.0", - "aws-smithy-json 0.55.0", - "aws-smithy-types 0.55.0", - "aws-smithy-xml 0.55.0", - "aws-types 0.55.0", - "bytes", - "http", - "http-body", - "once_cell", - "percent-encoding", - "regex", - "tokio-stream", - "tower", - "tracing", - "url", -] - [[package]] name = "aws-sdk-sso" version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca0119bacf0c42f587506769390983223ba834e605f049babe514b2bd646dbb2" dependencies = [ - "aws-credential-types 0.54.1", - "aws-endpoint 0.54.1", - "aws-http 0.54.1", - "aws-sig-auth 0.54.1", - "aws-smithy-async 0.54.4", - "aws-smithy-client 0.54.4", - "aws-smithy-http 0.54.4", - "aws-smithy-http-tower 0.54.4", - "aws-smithy-json 0.54.4", - "aws-smithy-types 0.54.4", - "aws-types 0.54.1", + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", "bytes", "http", "regex", @@ -382,76 +314,25 @@ dependencies = [ "tower", ] -[[package]] -name = "aws-sdk-sso" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aad0ad03c4ba802906340847aaea796d4158317864276864acbb281155bdcf4d" -dependencies = [ - "aws-credential-types 0.55.0", - "aws-endpoint 0.55.0", - "aws-http 0.55.0", - "aws-sig-auth 0.55.0", - "aws-smithy-async 0.55.0", - "aws-smithy-client 0.55.0", - "aws-smithy-http 0.55.0", - "aws-smithy-http-tower 0.55.0", - "aws-smithy-json 0.55.0", - "aws-smithy-types 0.55.0", - "aws-types 0.55.0", - "bytes", - "http", - "regex", - "tokio-stream", - "tower", - "tracing", -] - [[package]] name = "aws-sdk-sts" version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "270b6a33969ebfcb193512fbd5e8ee5306888ad6c6d5d775cdbfb2d50d94de26" dependencies = [ - "aws-credential-types 0.54.1", - "aws-endpoint 0.54.1", - "aws-http 0.54.1", - "aws-sig-auth 0.54.1", - "aws-smithy-async 0.54.4", - "aws-smithy-client 0.54.4", - "aws-smithy-http 0.54.4", - "aws-smithy-http-tower 0.54.4", - "aws-smithy-json 0.54.4", - "aws-smithy-query 0.54.4", - "aws-smithy-types 0.54.4", - "aws-smithy-xml 0.54.4", - "aws-types 0.54.1", - "bytes", - "http", - "regex", - "tower", - "tracing", -] - -[[package]] -name = "aws-sdk-sts" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2493efc64aa1c6469a252c221c335ccdcebfcc556c81b1efaee051c6ddc3dd99" -dependencies = [ - "aws-credential-types 0.55.0", - "aws-endpoint 0.55.0", - "aws-http 0.55.0", - "aws-sig-auth 0.55.0", - "aws-smithy-async 0.55.0", - "aws-smithy-client 0.55.0", - "aws-smithy-http 0.55.0", - "aws-smithy-http-tower 0.55.0", - "aws-smithy-json 0.55.0", - "aws-smithy-query 0.55.0", - "aws-smithy-types 0.55.0", - "aws-smithy-xml 0.55.0", - "aws-types 0.55.0", + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", "bytes", "http", "regex", @@ -465,59 +346,23 @@ version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "660a02a98ab1af83bd8d714afbab2d502ba9b18c49e7e4cddd6bf8837ff778cb" dependencies = [ - "aws-credential-types 0.54.1", - "aws-sigv4 0.54.1", - "aws-smithy-eventstream 0.54.4", - "aws-smithy-http 0.54.4", - "aws-types 0.54.1", - "http", - "tracing", -] - -[[package]] -name = "aws-sig-auth" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cebfa0f118afd7197185d5e097bfcdfca9f8410dca50435d67784405f1fd5a05" -dependencies = [ - "aws-credential-types 0.55.0", - "aws-sigv4 0.55.0", - "aws-smithy-eventstream 0.55.0", - "aws-smithy-http 0.55.0", - "aws-types 0.55.0", + "aws-credential-types", + "aws-sigv4", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-types", "http", "tracing", ] [[package]] name = "aws-sigv4" -version = "0.54.1" +version = "0.54.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdaf11005b7444e6cd66f600d09861a3aeb6eb89a0f003c7c9820dbab2d15297" +checksum = "86529e7b64d902efea8fff52c1b2529368d04f90305cf632729e3713f6b57dc0" dependencies = [ - "aws-smithy-eventstream 0.54.4", - "aws-smithy-http 0.54.4", - "bytes", - "form_urlencoded", - "hex", - "hmac", - "http", - "once_cell", - "percent-encoding", - "regex", - "sha2", - "time", - "tracing", -] - -[[package]] -name = "aws-sigv4" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19a4f5c05c8646d12b7bb3f18c04edc5ac5e8928ab80e1649e568190f2bc7b79" -dependencies = [ - "aws-smithy-eventstream 0.55.0", - "aws-smithy-http 0.55.0", + "aws-smithy-eventstream", + "aws-smithy-http", "bytes", "form_urlencoded", "hex", @@ -543,47 +388,14 @@ dependencies = [ "tokio-stream", ] -[[package]] -name = "aws-smithy-async" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd4b9b7d99263f75304fc1fcd752361cbc4cbf068b832acd8daeaaff44267eb" -dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", - "tokio-stream", -] - [[package]] name = "aws-smithy-checksums" version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3875fb4b28606a5368a048016a28c15707f2b21238d5b2e4a23198f590e92c4" dependencies = [ - "aws-smithy-http 0.54.4", - "aws-smithy-types 0.54.4", - "bytes", - "crc32c", - "crc32fast", - "hex", - "http", - "http-body", - "md-5", - "pin-project-lite", - "sha1", - "sha2", - "tracing", -] - -[[package]] -name = "aws-smithy-checksums" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa50ea8cc39f5efc78c35f1a2e1eefcd5b82016a38cf0fceba5528d269df823b" -dependencies = [ - "aws-smithy-http 0.55.0", - "aws-smithy-types 0.55.0", + "aws-smithy-http", + "aws-smithy-types", "bytes", "crc32c", "crc32fast", @@ -603,11 +415,11 @@ version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "104ca17f56cde00a10207169697dfe9c6810db339d52fb352707e64875b30a44" dependencies = [ - "aws-smithy-async 0.54.4", - "aws-smithy-http 0.54.4", - "aws-smithy-http-tower 0.54.4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-tower", "aws-smithy-protocol-test", - "aws-smithy-types 0.54.4", + "aws-smithy-types", "bytes", "fastrand", "http", @@ -622,48 +434,13 @@ dependencies = [ "tracing", ] -[[package]] -name = "aws-smithy-client" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "748298b60bbd0594223ea136ceed2ed4b6d50970bcefa69a5ff6d710ce593854" -dependencies = [ - "aws-smithy-async 0.55.0", - "aws-smithy-http 0.55.0", - "aws-smithy-http-tower 0.55.0", - "aws-smithy-types 0.55.0", - "bytes", - "fastrand", - "http", - "http-body", - "hyper", - "hyper-rustls", - "lazy_static", - "pin-project-lite", - "rustls", - "tokio", - "tower", - "tracing", -] - [[package]] name = "aws-smithy-eventstream" version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac250d8c0e42af0097a6837ffc5a6fb9f8ba4107bb53124c047c91bc2a58878f" dependencies = [ - "aws-smithy-types 0.54.4", - "bytes", - "crc32fast", -] - -[[package]] -name = "aws-smithy-eventstream" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7638e9fe2e97d2c4b173a44d9002c7cf29da7be7bab25c7837cfdce7945b2ff8" -dependencies = [ - "aws-smithy-types 0.55.0", + "aws-smithy-types", "bytes", "crc32fast", ] @@ -674,31 +451,8 @@ version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "873f316f1833add0d3aa54ed1b0cd252ddd88c792a0cf839886400099971e844" dependencies = [ - "aws-smithy-eventstream 0.54.4", - "aws-smithy-types 0.54.4", - "bytes", - "bytes-utils", - "futures-core", - "http", - "http-body", - "hyper", - "once_cell", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "aws-smithy-http" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78510732b81040689dc146e3693bfbcf388ab88cbda667d3ef67f8869b0744a" -dependencies = [ - "aws-smithy-eventstream 0.55.0", - "aws-smithy-types 0.55.0", + "aws-smithy-eventstream", + "aws-smithy-types", "bytes", "bytes-utils", "futures-core", @@ -720,24 +474,8 @@ version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f38231d3f5dac9ac7976f44e12803add1385119ffca9e5f050d8e980733d164" dependencies = [ - "aws-smithy-http 0.54.4", - "aws-smithy-types 0.54.4", - "bytes", - "http", - "http-body", - "pin-project-lite", - "tower", - "tracing", -] - -[[package]] -name = "aws-smithy-http-tower" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33689c27bbd8184412b45c4d1ab795d9a35402562d9fde6c53695a90969740" -dependencies = [ - "aws-smithy-http 0.55.0", - "aws-smithy-types 0.55.0", + "aws-smithy-http", + "aws-smithy-types", "bytes", "http", "http-body", @@ -752,16 +490,7 @@ version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bd83ff2b79e9f729746fcc8ad798676b68fe6ea72986571569a5306a277a182" dependencies = [ - "aws-smithy-types 0.54.4", -] - -[[package]] -name = "aws-smithy-json" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ada31cab1b1d1f0abc5c4d1183de5b278597704851aa703801b82feabf19aa74" -dependencies = [ - "aws-smithy-types 0.55.0", + "aws-smithy-types", ] [[package]] @@ -770,7 +499,7 @@ version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4d1c9bcb35ce11055ec128dab2c66a7ed47e2dfff99883e32c21a1ab6d6bee6" dependencies = [ - "assert-json-diff", + "assert-json-diff 1.1.0", "http", "pretty_assertions", "regex", @@ -785,17 +514,7 @@ version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2f0445dafe9d2cd50b44339ae3c3ed46549aad8ac696c52ad660b3e7ae8682b" dependencies = [ - "aws-smithy-types 0.54.4", - "urlencoding", -] - -[[package]] -name = "aws-smithy-query" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b55358401b657d192f70f093927f01d73cc4859e2907956b20c4043c76624006" -dependencies = [ - "aws-smithy-types 0.55.0", + "aws-smithy-types", "urlencoding", ] @@ -813,15 +532,12 @@ dependencies = [ ] [[package]] -name = "aws-smithy-types" -version = "0.55.0" +name = "aws-smithy-types-convert" +version = "0.54.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "474d145c2e0f82892841d2502bd546ca0dbc1e4e242c3563d96e7061054c268f" +checksum = "e2c209718c72f778b0e866e59be9cbbe0e62e835db73ceca48383b1ffafd5340" dependencies = [ - "base64-simd", - "itoa", - "num-integer", - "ryu", + "aws-smithy-types", "time", ] @@ -834,42 +550,17 @@ dependencies = [ "xmlparser", ] -[[package]] -name = "aws-smithy-xml" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb159921734d090b01c586a4fef73964f42fcb7eb53a8184b2db374bd6a6fc99" -dependencies = [ - "xmlparser", -] - [[package]] name = "aws-types" version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8f15b34253b68cde08e39b0627cc6101bcca64351229484b4743392c035d057" dependencies = [ - "aws-credential-types 0.54.1", - "aws-smithy-async 0.54.4", - "aws-smithy-client 0.54.4", - "aws-smithy-http 0.54.4", - "aws-smithy-types 0.54.4", - "http", - "rustc_version", - "tracing", -] - -[[package]] -name = "aws-types" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fb02591b5075d318e0083dcb76df0e151db4ce48f987ecd00e5b53c7a6ba59" -dependencies = [ - "aws-credential-types 0.55.0", - "aws-smithy-async 0.55.0", - "aws-smithy-client 0.55.0", - "aws-smithy-http 0.55.0", - "aws-smithy-types 0.55.0", + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-types", "http", "rustc_version", "tracing", @@ -895,9 +586,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.6.12" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f8ccfd9221ee7d1f3d4b33e1f8319b3a81ed8f61f2ea40b37b859794b4491" +checksum = "6e5abc76e6ef57bf438c067d2898481475994102eeb5d662a0b2162d0c2fdcf1" dependencies = [ "async-trait", "axum-core", @@ -927,9 +618,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2f958c80c248b34b9a877a643811be8dbca03ca5ba827f2b63baf3a81e5fc4e" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", "bytes", @@ -1035,6 +726,15 @@ dependencies = [ "either", ] +[[package]] +name = "bytestring" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae" +dependencies = [ + "bytes", +] + [[package]] name = "bzip2" version = "0.4.4" @@ -1114,7 +814,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.14", ] [[package]] @@ -1133,6 +833,17 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "colored" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd" +dependencies = [ + "atty", + "lazy_static", + "winapi", +] + [[package]] name = "concolor-override" version = "1.0.0" @@ -1193,9 +904,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1254,7 +965,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.13", + "syn 2.0.14", ] [[package]] @@ -1271,7 +982,7 @@ checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.14", ] [[package]] @@ -1343,13 +1054,13 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -1493,7 +1204,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.14", ] [[package]] @@ -1536,6 +1247,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "h2" version = "0.3.16" @@ -1567,6 +1289,15 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.2.6" @@ -1588,6 +1319,16 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hex-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7685beb53fc20efc2605f32f5d51e9ba18b8ef237961d1760169d2290d3bee" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "hmac" version = "0.12.1" @@ -1600,7 +1341,7 @@ dependencies = [ [[package]] name = "htsget-config" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#70ca723440e9331099d2bb77cbe4bab82c07b5b4" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#ca86a31e59922a02bbbc90857fa5336ad9be02d2" dependencies = [ "async-trait", "clap", @@ -1623,16 +1364,19 @@ name = "htsget-elsa" version = "0.1.0" dependencies = [ "async-trait", - "aws-config 0.55.0", - "aws-sdk-s3 0.25.1", + "aws-config", + "aws-sdk-s3", "bytes", "htsget-config", + "htsget-test", "http", + "mockito", "regex", "reqwest", "serde", "serde_json", "thiserror", + "tokio", "tracing", ] @@ -1655,7 +1399,7 @@ dependencies = [ [[package]] name = "htsget-http" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#70ca723440e9331099d2bb77cbe4bab82c07b5b4" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#ca86a31e59922a02bbbc90857fa5336ad9be02d2" dependencies = [ "futures", "htsget-config", @@ -1671,7 +1415,7 @@ dependencies = [ [[package]] name = "htsget-lambda" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#70ca723440e9331099d2bb77cbe4bab82c07b5b4" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#ca86a31e59922a02bbbc90857fa5336ad9be02d2" dependencies = [ "bytes", "htsget-config", @@ -1693,11 +1437,11 @@ dependencies = [ [[package]] name = "htsget-search" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#70ca723440e9331099d2bb77cbe4bab82c07b5b4" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#ca86a31e59922a02bbbc90857fa5336ad9be02d2" dependencies = [ "async-trait", - "aws-config 0.54.1", - "aws-sdk-s3 0.24.0", + "aws-config", + "aws-sdk-s3", "axum", "axum-extra", "base64 0.21.0", @@ -1723,9 +1467,12 @@ dependencies = [ [[package]] name = "htsget-test" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#70ca723440e9331099d2bb77cbe4bab82c07b5b4" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#ca86a31e59922a02bbbc90857fa5336ad9be02d2" dependencies = [ "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-s3", "base64 0.21.0", "futures", "htsget-config", @@ -1735,8 +1482,12 @@ dependencies = [ "noodles-vcf", "rcgen", "reqwest", + "s3s", + "s3s-aws", + "s3s-fs", "serde", "serde_json", + "tempfile", "tokio", ] @@ -1927,14 +1678,14 @@ checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" [[package]] name = "is-terminal" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256017f749ab3117e93acb91063009e1f1bb56d03965b14c2c8df4eb02c524d8" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", "rustix", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -2097,6 +1848,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.17" @@ -2190,6 +1951,26 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "mockito" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea57936ab3bf56156f135f20ee24b840e5a8ad97a8e1710eace33ac078f8f537" +dependencies = [ + "assert-json-diff 2.0.2", + "colored", + "futures", + "hyper", + "lazy_static", + "log", + "rand", + "regex", + "serde_json", + "serde_urlencoded", + "similar", + "tokio", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -2442,6 +2223,12 @@ dependencies = [ "libc", ] +[[package]] +name = "numeric_cast" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf70ee2d9b1737d1836c20d9f8f96ec3901b2bf92128439db13237ddce9173a5" + [[package]] name = "once_cell" version = "1.17.1" @@ -2450,9 +2237,9 @@ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "openssl" -version = "0.10.49" +version = "0.10.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d2f106ab837a24e03672c59b1239669a0596406ff657c3c0835b6b7f0f35a33" +checksum = "7e30d8bc91859781f0a943411186324d580f2bbeb71b452fe91ae344806af3f1" dependencies = [ "bitflags", "cfg-if", @@ -2471,7 +2258,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.14", ] [[package]] @@ -2482,9 +2269,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.84" +version = "0.9.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a20eace9dc2d82904039cb76dcf50fb1a0bba071cfd1629720b5d6f1ddba0fa" +checksum = "0d3d193fb1488ad46ffe3aaabc912cc931d02ee8518fe2959aea8ef52718b0c0" dependencies = [ "cc", "libc", @@ -2513,6 +2300,47 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "windows-sys 0.45.0", +] + +[[package]] +name = "path-absolutize" +version = "3.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f1d4993b16f7325d90c18c3c6a3327db7808752db8d208cea0acee0abd52c52" +dependencies = [ + "path-dedot", +] + +[[package]] +name = "path-dedot" +version = "3.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a81540d94551664b72b72829b12bd167c73c9d25fbac0e04fafa8023f7e4901" +dependencies = [ + "once_cell", +] + [[package]] name = "pear" version = "0.2.4" @@ -2533,7 +2361,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.13", + "syn 2.0.14", ] [[package]] @@ -2589,6 +2417,12 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + [[package]] name = "pretty_assertions" version = "1.3.0" @@ -2642,7 +2476,7 @@ checksum = "606c4ba35817e2922a308af55ad51bab3645b59eae5c570d4a6cf07e36bd493b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.14", "version_check", "yansi", ] @@ -2658,6 +2492,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "quick-xml" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc053f057dd768a56f62cd7e434c42c831d296968997e9ac1f76ea7c2d14c41" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.26" @@ -2667,6 +2511,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + [[package]] name = "rcgen" version = "0.10.0" @@ -2679,6 +2553,15 @@ dependencies = [ "yasna", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_syscall" version = "0.3.5" @@ -2791,16 +2674,16 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.7" +version = "0.37.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aae838e49b3d63e9274e1c01833cc8139d3fec468c3b84688c628f44b1ae11d" +checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" dependencies = [ "bitflags", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -2848,6 +2731,92 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "s3s" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4020158a5c1250280240d327dc0d0f2b0ccf62a7938bbf245a1cd2637d64bcfe" +dependencies = [ + "arrayvec", + "ascii", + "async-trait", + "atoi", + "base64-simd", + "bytes", + "bytestring", + "chrono", + "crc32fast", + "futures", + "hex-simd", + "hmac", + "http-body", + "httparse", + "hyper", + "memchr", + "mime", + "nom", + "pin-project-lite", + "quick-xml", + "serde", + "serde_urlencoded", + "sha1", + "sha2", + "smallvec", + "thiserror", + "time", + "tracing", + "transform-stream", + "urlencoding", +] + +[[package]] +name = "s3s-aws" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79f4605deb4310ac2dcdc7d4ba6a0fa50d0427deb157088522b912b7dcc964c" +dependencies = [ + "async-trait", + "aws-sdk-s3", + "aws-smithy-http", + "aws-smithy-types", + "aws-smithy-types-convert", + "bytes", + "futures", + "hyper", + "s3s", + "sync_wrapper", + "tracing", + "transform-stream", +] + +[[package]] +name = "s3s-fs" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4000a09d36e9dfc997faf146c7e41243dbdb873344bac6d073235b5ed664dc7" +dependencies = [ + "async-trait", + "base64-simd", + "bytes", + "chrono", + "futures", + "hex-simd", + "md-5", + "mime", + "numeric_cast", + "path-absolutize", + "s3s", + "serde_json", + "thiserror", + "time", + "tokio", + "tokio-util", + "tracing", + "tracing-error", + "transform-stream", + "uuid", +] + [[package]] name = "schannel" version = "0.1.21" @@ -2857,6 +2826,12 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "scratch" version = "1.0.5" @@ -2904,22 +2879,22 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.159" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" +checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.159" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" +checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.14", ] [[package]] @@ -3033,6 +3008,21 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "similar" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" + [[package]] name = "slab" version = "0.4.8" @@ -3095,9 +3085,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.13" +version = "2.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec" +checksum = "fcf316d5356ed6847742d036f8a39c3b8435cac10bd528a4bd461928a6ab34d5" dependencies = [ "proc-macro2", "quote", @@ -3118,7 +3108,7 @@ checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", + "redox_syscall 0.3.5", "rustix", "windows-sys 0.45.0", ] @@ -3149,7 +3139,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.14", ] [[package]] @@ -3215,7 +3205,9 @@ dependencies = [ "libc", "mio", "num_cpus", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.45.0", @@ -3229,7 +3221,7 @@ checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.14", ] [[package]] @@ -3409,6 +3401,16 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-error" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +dependencies = [ + "tracing", + "tracing-subscriber", +] + [[package]] name = "tracing-log" version = "0.1.3" @@ -3438,6 +3440,15 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "transform-stream" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05034de7a8fcb11796a36478a2a8b16dca6772644dec5f49f709d5c66a38d359" +dependencies = [ + "futures-core", +] + [[package]] name = "try-lock" version = "0.2.4" @@ -3524,6 +3535,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "uuid" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b55a3fef2a1e3b3a00ce878640918820d3c51081576ac657d23af9fc7928fdb" +dependencies = [ + "getrandom", +] + [[package]] name = "valuable" version = "0.1.0" diff --git a/htsget-elsa/Cargo.toml b/htsget-elsa/Cargo.toml index d90b629..45023ab 100644 --- a/htsget-elsa/Cargo.toml +++ b/htsget-elsa/Cargo.toml @@ -8,12 +8,18 @@ async-trait = "0.1" http = "0.2" reqwest = { version = "0.11", features = ["rustls-tls", "json"] } thiserror = "1.0" -aws-config = "0.55" -aws-sdk-s3 = "0.25" +aws-config = "0.54" +aws-sdk-s3 = "0.24" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" regex = "1.7" bytes = "1.4" tracing = "0.1" -htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } \ No newline at end of file +htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } + +[dev-dependencies] +htsget-test = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa", features = ["aws-mocks"] } + +tokio = { version = "1.25", features = ["macros", "rt-multi-thread"] } +mockito = "1.0" \ No newline at end of file diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index 8c09606..f7794dd 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -1,5 +1,5 @@ use crate::Error::{ - DeserializeError, GetGetManifest, InvalidManifestUri, InvalidReleaseUri, + DeserializeError, GetManifest, InvalidManifestUri, InvalidReleaseUri, UnsupportedManifestFeature, }; use crate::{Cache, Error, GetObject, ResolversFromElsa, Result}; @@ -16,23 +16,24 @@ use std::collections::HashMap; use std::str::FromStr; use tracing::{debug, instrument}; -const ENDPOINT_PATH: &str = "/manifest/htsget"; +const ENDPOINT_PATH: &str = "/api/manifest/htsget"; +const CACHE_PATH: &str = "htsget-manifest-cache"; -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ElsaLocation { bucket: String, key: String, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ElsaResponse { location: ElsaLocation, max_age: u64, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ElsaReadsManifest { url: String, @@ -40,7 +41,7 @@ pub struct ElsaReadsManifest { restriction: Option, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ElsaVariantsManifest { url: String, @@ -49,11 +50,11 @@ pub struct ElsaVariantsManifest { restriction: Option, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ElsaRestrictionsManifest {} -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ElsaManifest { #[serde(alias = "id")] @@ -175,7 +176,7 @@ where let resolvers: Vec = self.get_manifest(response).await?.try_into()?; self.cache - .put(&release_key, resolvers.clone(), max_age) + .put(format!("{CACHE_PATH}/{release_key}"), resolvers.clone(), max_age) .await?; Ok(resolvers) @@ -190,12 +191,16 @@ where S: GetObject, { pub fn new(endpoint: Authority, cache: &'a C, get_object: &'a S) -> Result { - Ok(Self { + Ok(Self::new_with_client(Self::create_client()?, endpoint, cache, get_object)) + } + + fn new_with_client(client: Client, endpoint: Authority, cache: &'a C, get_object: &'a S) -> Self { + Self { endpoint, - client: Self::create_client()?, + client, cache, get_object, - }) + } } fn create_client() -> Result { @@ -206,25 +211,32 @@ where .map_err(|err| Error::InvalidClient(err)) } - #[instrument(level = "debug", skip(self), ret)] - pub async fn get_response(&self, release_key: &str) -> Result { + async fn get_response_with_scheme(&self, release_key: &str, scheme: &str) -> Result { let uri = Uri::builder() - .scheme("https") + .scheme(scheme) .authority(self.endpoint.as_str()) - .path_and_query(format!("{}/{}?type=S3", ENDPOINT_PATH, release_key)) + .path_and_query(format!("{ENDPOINT_PATH}/{release_key}?type=S3")) .build() .map(|uri| Url::parse(&uri.to_string())) .map_err(|_| InvalidReleaseUri(release_key.to_string()))? .map_err(|_| InvalidReleaseUri(release_key.to_string()))?; - self.client + let response = self.client .get(uri) .send() .await - .map_err(|err| GetGetManifest(err))? - .json() - .await - .map_err(|err| DeserializeError(err.to_string())) + .map_err(|err| GetManifest(err.to_string()))?; + + if response.status().is_success() { + response.json().await.map_err(|err| DeserializeError(err.to_string())) + } else { + Err(GetManifest(format!("status code {}", response.status()))) + } + } + + #[instrument(level = "debug", skip(self), ret)] + pub async fn get_response(&self, release_key: &str) -> Result { + self.get_response_with_scheme(release_key, "https").await } #[instrument(level = "debug", skip(self), ret)] @@ -234,3 +246,50 @@ where .await } } + +#[cfg(test)] +mod tests { + use std::future::Future; + use std::str::FromStr; + use aws_sdk_s3::Client; + use http::uri::Authority; + use mockito::Server; + use htsget_test::aws_mocks::with_s3_test_server_tmp; + use crate::elsa_endpoint::{ElsaEndpoint, ElsaLocation, ElsaResponse, ENDPOINT_PATH}; + use crate::s3::S3; + + #[tokio::test] + async fn get_response() { + with_test_mocks(|endpoint, s3_client, reqwest_client| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let endpoint = ElsaEndpoint::new_with_client(reqwest_client, Authority::from_str(&endpoint).unwrap(), &s3, &s3); + + let response = endpoint.get_response_with_scheme("R001", "http").await.unwrap(); + assert_eq!(response, ElsaResponse { + location: ElsaLocation { + bucket: "elsa-data-tmp".to_string(), + key: "R001".to_string(), + }, + max_age: 86400, + }); + }).await; + } + + async fn with_test_mocks(test: F) + where F: FnOnce(String, Client, reqwest::Client) -> Fut, + Fut: Future, { + with_s3_test_server_tmp(|client| async move { + let mut server = Server::new_async().await; + + let mock = server.mock("GET", format!("{ENDPOINT_PATH}/R001?type=S3").as_str()) + .with_status(200) + .with_body(r#"{"location":{"bucket":"elsa-data-tmp","key":"R001"},"maxAge":86400}"#) + .create(); + + test(server.host_with_port(), client, reqwest::Client::builder() + .build().unwrap()).await; + + mock.assert_async().await; + }).await; + } +} diff --git a/htsget-elsa/src/lib.rs b/htsget-elsa/src/lib.rs index 3905ffc..36cef75 100644 --- a/htsget-elsa/src/lib.rs +++ b/htsget-elsa/src/lib.rs @@ -17,7 +17,7 @@ pub enum Error { #[error("invalid uri constructed from release key: `{0}`")] InvalidReleaseUri(String), #[error("failed to get manifest from Elsa: `{0}`")] - GetGetManifest(reqwest::Error), + GetManifest(String), #[error("failed to deserialize: `{0}")] DeserializeError(String), #[error("failed to serialize: `{0}")] diff --git a/htsget-elsa/src/s3.rs b/htsget-elsa/src/s3.rs index 4901411..3e4d252 100644 --- a/htsget-elsa/src/s3.rs +++ b/htsget-elsa/src/s3.rs @@ -1,14 +1,14 @@ use crate::Error::{DeserializeError, GetObjectError, PutObjectError, SerializeError}; use crate::{Cache, Error, GetObject, Result}; use async_trait::async_trait; -use aws_sdk_s3::primitives::{ByteStream, DateTime}; -use aws_sdk_s3::Client; use bytes::Bytes; use htsget_config::resolver::Resolver; use serde::{Deserialize, Serialize}; use serde_json::{from_slice, to_string, to_vec}; use std::ops::Add; use std::time::{Duration, SystemTime}; +use aws_sdk_s3::Client; +use aws_sdk_s3::types::{ByteStream, DateTime}; use tracing::instrument; #[derive(Debug)] From fab590415268f93dc3fe3b7b9c3630697e20d297 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 14 Apr 2023 20:04:16 +1000 Subject: [PATCH 10/25] test(elsa): add get manifest test --- Cargo.lock | 244 ++++++++++++++++++++++++++++--- htsget-elsa/Cargo.toml | 3 +- htsget-elsa/src/elsa_endpoint.rs | 154 ++++++++++++++++--- 3 files changed, 362 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 066692d..c62cadd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -66,6 +66,12 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "anyhow" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" + [[package]] name = "arrayvec" version = "0.7.2" @@ -99,6 +105,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-channel" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + [[package]] name = "async-compression" version = "0.3.15" @@ -586,9 +603,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.6.14" +version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e5abc76e6ef57bf438c067d2898481475994102eeb5d662a0b2162d0c2fdcf1" +checksum = "3b32c5ea3aabaf4deb5f5ced2d688ec0844c881c9e6c696a8b769a05fc691e62" dependencies = [ "async-trait", "axum-core", @@ -859,6 +876,15 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "concurrent-queue" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62ec6771ecfa0762d24683ee5a32ad78487a3d3afdc0fb8cae19d2c5deb50b7c" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -1020,6 +1046,25 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "deadpool" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "421fe0f90f2ab22016f32a9881be5134fdd71c65298917084b0c7477cbc3856e" +dependencies = [ + "async-trait", + "deadpool-runtime", + "num_cpus", + "retain_mut", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaa37046cc0f6c3cc6090fbdbf73ef0b8ef4cfcc37f6befc0020f63e8cf121e1" + [[package]] name = "diff" version = "0.1.13" @@ -1073,6 +1118,12 @@ dependencies = [ "libc", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "extend" version = "0.1.2" @@ -1196,6 +1247,21 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-macro" version = "0.3.28" @@ -1219,6 +1285,12 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + [[package]] name = "futures-util" version = "0.3.28" @@ -1247,6 +1319,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.9" @@ -1255,7 +1338,7 @@ checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] @@ -1341,7 +1424,7 @@ dependencies = [ [[package]] name = "htsget-config" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#ca86a31e59922a02bbbc90857fa5336ad9be02d2" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#c44aea26ab9ead617bbcf8747010ad7b3ec10d8e" dependencies = [ "async-trait", "clap", @@ -1378,6 +1461,7 @@ dependencies = [ "thiserror", "tokio", "tracing", + "wiremock", ] [[package]] @@ -1399,7 +1483,7 @@ dependencies = [ [[package]] name = "htsget-http" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#ca86a31e59922a02bbbc90857fa5336ad9be02d2" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#c44aea26ab9ead617bbcf8747010ad7b3ec10d8e" dependencies = [ "futures", "htsget-config", @@ -1415,7 +1499,7 @@ dependencies = [ [[package]] name = "htsget-lambda" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#ca86a31e59922a02bbbc90857fa5336ad9be02d2" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#c44aea26ab9ead617bbcf8747010ad7b3ec10d8e" dependencies = [ "bytes", "htsget-config", @@ -1437,7 +1521,7 @@ dependencies = [ [[package]] name = "htsget-search" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#ca86a31e59922a02bbbc90857fa5336ad9be02d2" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#c44aea26ab9ead617bbcf8747010ad7b3ec10d8e" dependencies = [ "async-trait", "aws-config", @@ -1467,7 +1551,7 @@ dependencies = [ [[package]] name = "htsget-test" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#ca86a31e59922a02bbbc90857fa5336ad9be02d2" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#c44aea26ab9ead617bbcf8747010ad7b3ec10d8e" dependencies = [ "async-trait", "aws-config", @@ -1529,6 +1613,27 @@ dependencies = [ "serde", ] +[[package]] +name = "http-types" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" +dependencies = [ + "anyhow", + "async-channel", + "base64 0.13.1", + "futures-lite", + "http", + "infer", + "pin-project-lite", + "rand 0.7.3", + "serde", + "serde_json", + "serde_qs", + "serde_urlencoded", + "url", +] + [[package]] name = "httparse" version = "1.8.0" @@ -1644,6 +1749,12 @@ dependencies = [ "serde", ] +[[package]] +name = "infer" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" + [[package]] name = "inlinable_string" version = "0.1.15" @@ -1947,7 +2058,7 @@ checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.45.0", ] @@ -1963,7 +2074,7 @@ dependencies = [ "hyper", "lazy_static", "log", - "rand", + "rand 0.8.5", "regex", "serde_json", "serde_urlencoded", @@ -2300,6 +2411,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "parking" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" + [[package]] name = "parking_lot" version = "0.12.1" @@ -2511,6 +2628,19 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" @@ -2518,8 +2648,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", ] [[package]] @@ -2529,7 +2669,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", ] [[package]] @@ -2538,7 +2687,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.9", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", ] [[package]] @@ -2639,6 +2797,12 @@ dependencies = [ "winreg", ] +[[package]] +name = "retain_mut" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0" + [[package]] name = "ring" version = "0.16.20" @@ -2899,9 +3063,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "indexmap", "itoa", @@ -2918,6 +3082,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_qs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" +dependencies = [ + "percent-encoding", + "serde", + "thiserror", +] + [[package]] name = "serde_regex" version = "1.1.0" @@ -3521,6 +3696,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -3541,7 +3717,7 @@ version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b55a3fef2a1e3b3a00ce878640918820d3c51081576ac657d23af9fc7928fdb" dependencies = [ - "getrandom", + "getrandom 0.2.9", ] [[package]] @@ -3568,6 +3744,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + [[package]] name = "want" version = "0.3.0" @@ -3578,6 +3760,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -3884,6 +4072,28 @@ dependencies = [ "winapi", ] +[[package]] +name = "wiremock" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd7b0b5b253ebc0240d6aac6dd671c495c467420577bf634d3064ae7e6fa2b4c" +dependencies = [ + "assert-json-diff 2.0.2", + "async-trait", + "base64 0.21.0", + "deadpool", + "futures", + "futures-timer", + "http-types", + "hyper", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "xmlparser" version = "0.13.5" diff --git a/htsget-elsa/Cargo.toml b/htsget-elsa/Cargo.toml index 45023ab..9f48e9d 100644 --- a/htsget-elsa/Cargo.toml +++ b/htsget-elsa/Cargo.toml @@ -22,4 +22,5 @@ htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat htsget-test = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa", features = ["aws-mocks"] } tokio = { version = "1.25", features = ["macros", "rt-multi-thread"] } -mockito = "1.0" \ No newline at end of file +mockito = "1.0" +wiremock = "0.5" \ No newline at end of file diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index f7794dd..8c64979 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -153,6 +153,7 @@ pub struct ElsaEndpoint<'a, C, S> { client: Client, cache: &'a C, get_object: &'a S, + scheme: &'a str, } #[async_trait] @@ -191,15 +192,16 @@ where S: GetObject, { pub fn new(endpoint: Authority, cache: &'a C, get_object: &'a S) -> Result { - Ok(Self::new_with_client(Self::create_client()?, endpoint, cache, get_object)) + Ok(Self::new_with_client(Self::create_client()?, endpoint, cache, get_object, "https")) } - fn new_with_client(client: Client, endpoint: Authority, cache: &'a C, get_object: &'a S) -> Self { + fn new_with_client(client: Client, endpoint: Authority, cache: &'a C, get_object: &'a S, scheme: &'a str) -> Self { Self { endpoint, client, cache, get_object, + scheme } } @@ -230,13 +232,13 @@ where if response.status().is_success() { response.json().await.map_err(|err| DeserializeError(err.to_string())) } else { - Err(GetManifest(format!("status code {}", response.status()))) + Err(GetManifest(response.status().to_string())) } } #[instrument(level = "debug", skip(self), ret)] pub async fn get_response(&self, release_key: &str) -> Result { - self.get_response_with_scheme(release_key, "https").await + self.get_response_with_scheme(release_key, self.scheme).await } #[instrument(level = "debug", skip(self), ret)] @@ -249,47 +251,157 @@ where #[cfg(test)] mod tests { + use std::fs; use std::future::Future; use std::str::FromStr; use aws_sdk_s3::Client; use http::uri::Authority; - use mockito::Server; use htsget_test::aws_mocks::with_s3_test_server_tmp; + use serde_json::from_str; + use wiremock::{Mock, MockServer, Request, ResponseTemplate, Times}; + use wiremock::matchers::{method, path, query_param}; use crate::elsa_endpoint::{ElsaEndpoint, ElsaLocation, ElsaResponse, ENDPOINT_PATH}; use crate::s3::S3; + use crate::Error::{GetManifest, GetObjectError}; + use crate::ResolversFromElsa; #[tokio::test] async fn get_response() { with_test_mocks(|endpoint, s3_client, reqwest_client| async move { let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); - let endpoint = ElsaEndpoint::new_with_client(reqwest_client, Authority::from_str(&endpoint).unwrap(), &s3, &s3); + let endpoint = ElsaEndpoint::new_with_client(reqwest_client, Authority::from_str(&endpoint).unwrap(), &s3, &s3, "http"); - let response = endpoint.get_response_with_scheme("R001", "http").await.unwrap(); + let response = endpoint.get_response_with_scheme("R004", "http").await.unwrap(); assert_eq!(response, ElsaResponse { location: ElsaLocation { bucket: "elsa-data-tmp".to_string(), - key: "R001".to_string(), + key: "htsget-manifests/R004".to_string(), }, max_age: 86400, }); - }).await; + }, 1).await; } - async fn with_test_mocks(test: F) - where F: FnOnce(String, Client, reqwest::Client) -> Fut, - Fut: Future, { - with_s3_test_server_tmp(|client| async move { - let mut server = Server::new_async().await; + #[tokio::test] + async fn get_manifest() { + with_test_mocks(|endpoint, s3_client, reqwest_client| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let endpoint = ElsaEndpoint::new_with_client(reqwest_client, Authority::from_str(&endpoint).unwrap(), &s3, &s3, "http"); - let mock = server.mock("GET", format!("{ENDPOINT_PATH}/R001?type=S3").as_str()) - .with_status(200) - .with_body(r#"{"location":{"bucket":"elsa-data-tmp","key":"R001"},"maxAge":86400}"#) - .create(); + let response = endpoint.get_response_with_scheme("R004", "http").await.unwrap(); + let manifest = endpoint.get_manifest(response).await.unwrap(); - test(server.host_with_port(), client, reqwest::Client::builder() - .build().unwrap()).await; + assert_eq!(manifest, from_str(&example_elsa_manifest()).unwrap()); + }, 1).await; + } + + #[tokio::test] + async fn get_manifest_not_present() { + with_test_mocks(|endpoint, s3_client, reqwest_client| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let endpoint = ElsaEndpoint::new_with_client(reqwest_client, Authority::from_str(&endpoint).unwrap(), &s3, &s3, "http"); + + let manifest = endpoint.get_manifest(from_str(&example_elsa_response()).unwrap()).await; + + assert!(matches!(manifest, Err(GetObjectError(_)))); + }, 0).await; + } - mock.assert_async().await; + fn example_elsa_manifest() -> String { + r#" + { + "id": "R004", + "reads": { + "30F9F3FED8F711ED8C35DBEF59E9F537": { + "url": "s3://umccr-10g-data-dev/HG00097/HG00097.bam" + }, + "30F9FFD4D8F711ED8C353BBCB8861211": { + "url": "s3://umccr-10g-data-dev/HG00096/HG00096.bam" + } + }, + "variants": { + "30F9F3FED8F711ED8C35DBEF59E9F537": { + "url": "s3://umccr-10g-data-dev/HG00097/HG00097.hard-filtered.vcf.gz", + "variantSampleId": "" + }, + "30F9FFD4D8F711ED8C353BBCB8861211": { + "url": "s3://umccr-10g-data-dev/HG00096/HG00096.hard-filtered.vcf.gz", + "variantSampleId": "" + } + }, + "restrictions": {}, + "cases": [ + { + "ids": { "": "SINGLETONCHARLES" }, + "patients": [ + { + "ids": { "": "CHARLES" }, + "specimens": [ + { + "htsgetId": "30F9FFD4D8F711ED8C353BBCB8861211", + "ids": { "": "HG00096" } + } + ] + } + ] + }, + { + "ids": { "": "SINGLETONMARY" }, + "patients": [ + { + "ids": { "": "MARY" }, + "specimens": [ + { + "htsgetId": "30F9F3FED8F711ED8C35DBEF59E9F537", + "ids": { "": "HG00097" } + } + ] + } + ] + } + ] + } + "#.to_string() + } + + fn example_elsa_response() -> String { + r#" + { + "location": { + "bucket": "elsa-data-tmp", + "key": "htsget-manifests/R004" + }, + "maxAge": 86400 + } + "#.to_string() + } + + async fn with_test_mocks(test: F, expect_times: T) + where + T: Into, + F: FnOnce(String, Client, reqwest::Client) -> Fut, + Fut: Future, { + with_s3_test_server_tmp(|client, server_base_path| async move { + let mock_server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path(format!("{ENDPOINT_PATH}/R004"))) + .and(query_param("type", "S3")) + .respond_with(move |_: &Request| { + let manifest_path = server_base_path.join("elsa-data-tmp/htsget-manifests"); + + fs::create_dir_all(&manifest_path).unwrap(); + fs::write(&manifest_path.join("R004"), example_elsa_manifest()).unwrap(); + + ResponseTemplate::new(200) + .set_body_string(example_elsa_response()) + }) + .expect(expect_times) + .mount(&mock_server) + .await; + + test(mock_server.address().to_string(), client, reqwest::Client::builder() + .build().unwrap()).await; }).await; } } From ebe4089982625207a9737e14cea1e8aa127fc279 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 14 Apr 2023 20:04:24 +1000 Subject: [PATCH 11/25] test(elsa): add s3 tests --- htsget-elsa/src/elsa_endpoint.rs | 111 ++------------------------- htsget-elsa/src/lib.rs | 115 ++++++++++++++++++++++++++++ htsget-elsa/src/s3.rs | 125 ++++++++++++++++++++++++------- 3 files changed, 220 insertions(+), 131 deletions(-) diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index 8c64979..262ac31 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -16,8 +16,8 @@ use std::collections::HashMap; use std::str::FromStr; use tracing::{debug, instrument}; -const ENDPOINT_PATH: &str = "/api/manifest/htsget"; -const CACHE_PATH: &str = "htsget-manifest-cache"; +pub const ENDPOINT_PATH: &str = "/api/manifest/htsget"; +pub const CACHE_PATH: &str = "htsget-manifest-cache"; #[derive(Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] @@ -261,13 +261,14 @@ mod tests { use wiremock::{Mock, MockServer, Request, ResponseTemplate, Times}; use wiremock::matchers::{method, path, query_param}; use crate::elsa_endpoint::{ElsaEndpoint, ElsaLocation, ElsaResponse, ENDPOINT_PATH}; - use crate::s3::S3; use crate::Error::{GetManifest, GetObjectError}; use crate::ResolversFromElsa; + use crate::s3::S3; + use crate::tests::{example_elsa_manifest, example_elsa_response, with_test_mocks}; #[tokio::test] async fn get_response() { - with_test_mocks(|endpoint, s3_client, reqwest_client| async move { + with_test_mocks(|endpoint, s3_client, reqwest_client, _| async move { let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); let endpoint = ElsaEndpoint::new_with_client(reqwest_client, Authority::from_str(&endpoint).unwrap(), &s3, &s3, "http"); @@ -284,7 +285,7 @@ mod tests { #[tokio::test] async fn get_manifest() { - with_test_mocks(|endpoint, s3_client, reqwest_client| async move { + with_test_mocks(|endpoint, s3_client, reqwest_client, _| async move { let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); let endpoint = ElsaEndpoint::new_with_client(reqwest_client, Authority::from_str(&endpoint).unwrap(), &s3, &s3, "http"); @@ -297,7 +298,7 @@ mod tests { #[tokio::test] async fn get_manifest_not_present() { - with_test_mocks(|endpoint, s3_client, reqwest_client| async move { + with_test_mocks(|endpoint, s3_client, reqwest_client, _| async move { let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); let endpoint = ElsaEndpoint::new_with_client(reqwest_client, Authority::from_str(&endpoint).unwrap(), &s3, &s3, "http"); @@ -306,102 +307,4 @@ mod tests { assert!(matches!(manifest, Err(GetObjectError(_)))); }, 0).await; } - - fn example_elsa_manifest() -> String { - r#" - { - "id": "R004", - "reads": { - "30F9F3FED8F711ED8C35DBEF59E9F537": { - "url": "s3://umccr-10g-data-dev/HG00097/HG00097.bam" - }, - "30F9FFD4D8F711ED8C353BBCB8861211": { - "url": "s3://umccr-10g-data-dev/HG00096/HG00096.bam" - } - }, - "variants": { - "30F9F3FED8F711ED8C35DBEF59E9F537": { - "url": "s3://umccr-10g-data-dev/HG00097/HG00097.hard-filtered.vcf.gz", - "variantSampleId": "" - }, - "30F9FFD4D8F711ED8C353BBCB8861211": { - "url": "s3://umccr-10g-data-dev/HG00096/HG00096.hard-filtered.vcf.gz", - "variantSampleId": "" - } - }, - "restrictions": {}, - "cases": [ - { - "ids": { "": "SINGLETONCHARLES" }, - "patients": [ - { - "ids": { "": "CHARLES" }, - "specimens": [ - { - "htsgetId": "30F9FFD4D8F711ED8C353BBCB8861211", - "ids": { "": "HG00096" } - } - ] - } - ] - }, - { - "ids": { "": "SINGLETONMARY" }, - "patients": [ - { - "ids": { "": "MARY" }, - "specimens": [ - { - "htsgetId": "30F9F3FED8F711ED8C35DBEF59E9F537", - "ids": { "": "HG00097" } - } - ] - } - ] - } - ] - } - "#.to_string() - } - - fn example_elsa_response() -> String { - r#" - { - "location": { - "bucket": "elsa-data-tmp", - "key": "htsget-manifests/R004" - }, - "maxAge": 86400 - } - "#.to_string() - } - - async fn with_test_mocks(test: F, expect_times: T) - where - T: Into, - F: FnOnce(String, Client, reqwest::Client) -> Fut, - Fut: Future, { - with_s3_test_server_tmp(|client, server_base_path| async move { - let mock_server = MockServer::start().await; - - Mock::given(method("GET")) - .and(path(format!("{ENDPOINT_PATH}/R004"))) - .and(query_param("type", "S3")) - .respond_with(move |_: &Request| { - let manifest_path = server_base_path.join("elsa-data-tmp/htsget-manifests"); - - fs::create_dir_all(&manifest_path).unwrap(); - fs::write(&manifest_path.join("R004"), example_elsa_manifest()).unwrap(); - - ResponseTemplate::new(200) - .set_body_string(example_elsa_response()) - }) - .expect(expect_times) - .mount(&mock_server) - .await; - - test(mock_server.address().to_string(), client, reqwest::Client::builder() - .build().unwrap()).await; - }).await; - } } diff --git a/htsget-elsa/src/lib.rs b/htsget-elsa/src/lib.rs index 36cef75..6fbbc70 100644 --- a/htsget-elsa/src/lib.rs +++ b/htsget-elsa/src/lib.rs @@ -68,3 +68,118 @@ pub trait ResolversFromElsa { async fn try_get(&self, release_key: String) -> result::Result, Self::Error>; } + +#[cfg(test)] +pub(crate) mod tests { + use std::fs; + use std::future::Future; + use std::path::{Path, PathBuf}; + use aws_sdk_s3::Client; + use htsget_test::aws_mocks::with_s3_test_server_tmp; + use wiremock::{Mock, MockServer, Request, ResponseTemplate, Times}; + use wiremock::matchers::{method, path, query_param}; + use crate::elsa_endpoint::ENDPOINT_PATH; + + pub(crate) fn example_elsa_manifest() -> String { + r#" + { + "id": "R004", + "reads": { + "30F9F3FED8F711ED8C35DBEF59E9F537": { + "url": "s3://umccr-10g-data-dev/HG00097/HG00097.bam" + }, + "30F9FFD4D8F711ED8C353BBCB8861211": { + "url": "s3://umccr-10g-data-dev/HG00096/HG00096.bam" + } + }, + "variants": { + "30F9F3FED8F711ED8C35DBEF59E9F537": { + "url": "s3://umccr-10g-data-dev/HG00097/HG00097.hard-filtered.vcf.gz", + "variantSampleId": "" + }, + "30F9FFD4D8F711ED8C353BBCB8861211": { + "url": "s3://umccr-10g-data-dev/HG00096/HG00096.hard-filtered.vcf.gz", + "variantSampleId": "" + } + }, + "restrictions": {}, + "cases": [ + { + "ids": { "": "SINGLETONCHARLES" }, + "patients": [ + { + "ids": { "": "CHARLES" }, + "specimens": [ + { + "htsgetId": "30F9FFD4D8F711ED8C353BBCB8861211", + "ids": { "": "HG00096" } + } + ] + } + ] + }, + { + "ids": { "": "SINGLETONMARY" }, + "patients": [ + { + "ids": { "": "MARY" }, + "specimens": [ + { + "htsgetId": "30F9F3FED8F711ED8C35DBEF59E9F537", + "ids": { "": "HG00097" } + } + ] + } + ] + } + ] + } + "#.to_string() + } + + pub(crate) fn example_elsa_response() -> String { + r#" + { + "location": { + "bucket": "elsa-data-tmp", + "key": "htsget-manifests/R004" + }, + "maxAge": 86400 + } + "#.to_string() + } + + pub(crate) fn write_example_manifest(manifest_path: &Path) { + fs::create_dir_all(&manifest_path).unwrap(); + fs::write(&manifest_path.join("R004"), example_elsa_manifest()).unwrap(); + } + + pub(crate) async fn with_test_mocks(test: F, expect_times: T) + where + T: Into, + F: FnOnce(String, Client, reqwest::Client, PathBuf) -> Fut, + Fut: Future, { + with_s3_test_server_tmp(|client, server_base_path| async move { + let mock_server = MockServer::start().await; + + let base_path = server_base_path.clone(); + Mock::given(method("GET")) + .and(path(format!("{ENDPOINT_PATH}/R004"))) + .and(query_param("type", "S3")) + .respond_with(move |_: &Request| { + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + + write_example_manifest(&manifest_path); + + ResponseTemplate::new(200) + .set_body_string(example_elsa_response()) + }) + .expect(expect_times) + .mount(&mock_server) + .await; + + test(mock_server.address().to_string(), client, reqwest::Client::builder() + .build().unwrap(), server_base_path).await; + }).await; + } +} \ No newline at end of file diff --git a/htsget-elsa/src/s3.rs b/htsget-elsa/src/s3.rs index 3e4d252..7d9e12f 100644 --- a/htsget-elsa/src/s3.rs +++ b/htsget-elsa/src/s3.rs @@ -5,10 +5,14 @@ use bytes::Bytes; use htsget_config::resolver::Resolver; use serde::{Deserialize, Serialize}; use serde_json::{from_slice, to_string, to_vec}; -use std::ops::Add; +use std::ops::{Add, Sub}; +use std::result; use std::time::{Duration, SystemTime}; use aws_sdk_s3::Client; -use aws_sdk_s3::types::{ByteStream, DateTime}; +use aws_sdk_s3::error::HeadObjectError; +use aws_sdk_s3::Error::NotFound; +use aws_sdk_s3::output::HeadObjectOutput; +use aws_sdk_s3::types::{ByteStream, DateTime, SdkError}; use tracing::instrument; #[derive(Debug)] @@ -40,11 +44,28 @@ impl S3 { } impl S3 { + /// Get the last modified date of the object. + async fn last_modified( + &self, + bucket: impl Into + Send, + key: impl Into + Send, + ) -> Option { + self + .s3_client + .head_object() + .bucket(bucket) + .key(key) + .send() + .await + .ok() + .and_then(|output| output.last_modified) + } + async fn get_object Deserialize<'de>>( &self, bucket: impl Into + Send, key: impl Into + Send, - ) -> Result<(T, Option)> { + ) -> Result { let output = self .s3_client .get_object() @@ -52,21 +73,14 @@ impl S3 { .key(key) .send() .await - .map_err(|err| GetObjectError(err.to_string()))?; - - let last_modified = output.last_modified().copied(); - - let output = output + .map_err(|err| GetObjectError(err.to_string()))? .body .collect() .await .map_err(|err| GetObjectError(err.to_string()))? .into_bytes(); - Ok(( - from_slice(output.as_ref()).map_err(|err| DeserializeError(err.to_string()))?, - last_modified, - )) + Ok(from_slice(output.as_ref()).map_err(|err| DeserializeError(err.to_string()))?) } } @@ -79,7 +93,7 @@ impl GetObject for S3 { bucket: impl Into + Send, key: impl Into + Send, ) -> Result { - Ok(self.get_object(bucket, key).await?.0) + Ok(self.get_object(bucket, key).await?) } } @@ -90,22 +104,18 @@ impl Cache for S3 { #[instrument(level = "trace", skip_all, ret)] async fn get + Send + Sync>(&self, key: K) -> Result> { - let (object, last_modified): (CacheItem, _) = self - .get_object(self.cache_bucket.clone(), key.as_ref()) - .await?; - - match last_modified { - Some(last_modified) - if last_modified.as_nanos() - <= DateTime::from( - SystemTime::now().add(Duration::from_secs(object.max_age)), - ) - .as_nanos() => - { - Ok(Some(object.item)) + if let Some(last_modified) = self.last_modified(self.cache_bucket.clone(), key.as_ref()).await { + let object: CacheItem = self.get_object(self.cache_bucket.clone(), key.as_ref()).await?; + + if last_modified.as_nanos() + > DateTime::from( + SystemTime::now().sub(Duration::from_secs(object.max_age)), + ).as_nanos() { + return Ok(Some(object.item)); } - _ => Ok(None), } + + Ok(None) } #[instrument(level = "trace", skip_all, ret)] @@ -130,3 +140,64 @@ impl Cache for S3 { Ok(()) } } + +#[cfg(test)] +mod tests { + use std::str::FromStr; + use http::uri::Authority; + use serde_json::from_str; + use crate::elsa_endpoint::{ElsaEndpoint, ElsaManifest}; + use crate::s3::S3; + use crate::tests::{with_test_mocks, write_example_manifest, example_elsa_manifest}; + + #[tokio::test] + async fn last_modified() { + with_test_mocks(|_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + write_example_manifest(&manifest_path); + + let result = s3.last_modified("elsa-data-tmp", "htsget-manifests/R004").await; + assert!(matches!(result, Some(_))); + }, 0).await; + } + + #[tokio::test] + async fn last_modified_not_found() { + with_test_mocks(|_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + write_example_manifest(&manifest_path); + + let result = s3.last_modified("elsa-data-tmp", "htsget-manifests/R005").await; + assert!(matches!(result, None)); + }, 0).await; + } + + #[tokio::test] + async fn get_object() { + with_test_mocks(|_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + write_example_manifest(&manifest_path); + + let result: ElsaManifest = s3.get_object("elsa-data-tmp", "htsget-manifests/R004").await.unwrap(); + assert_eq!(result, from_str(&example_elsa_manifest()).unwrap()); + }, 0).await; + } + + #[tokio::test] + async fn get_object_not_found() { + with_test_mocks(|_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + write_example_manifest(&manifest_path); + + assert!(s3.get_object::("elsa-data-tmp", "htsget-manifests/R005").await.is_err()); + }, 0).await; + } +} From ff8d3e725f5a4f0dbfc18690686b72a611c103ef Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 14 Apr 2023 20:04:30 +1000 Subject: [PATCH 12/25] test(elsa): add get and put tests --- htsget-elsa/src/s3.rs | 73 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/htsget-elsa/src/s3.rs b/htsget-elsa/src/s3.rs index 7d9e12f..a4819b3 100644 --- a/htsget-elsa/src/s3.rs +++ b/htsget-elsa/src/s3.rs @@ -143,11 +143,15 @@ impl Cache for S3 { #[cfg(test)] mod tests { + use std::fs; use std::str::FromStr; + use std::time::Duration; use http::uri::Authority; - use serde_json::from_str; + use serde_json::{from_str, to_string}; + use tokio::time::sleep; + use crate::Cache; use crate::elsa_endpoint::{ElsaEndpoint, ElsaManifest}; - use crate::s3::S3; + use crate::s3::{CacheItem, S3}; use crate::tests::{with_test_mocks, write_example_manifest, example_elsa_manifest}; #[tokio::test] @@ -200,4 +204,69 @@ mod tests { assert!(s3.get_object::("elsa-data-tmp", "htsget-manifests/R005").await.is_err()); }, 0).await; } + + #[tokio::test] + async fn get_not_found() { + with_test_mocks(|_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + fs::create_dir_all(&manifest_path).unwrap(); + fs::write(&manifest_path.join("R004"), to_string( + &CacheItem { item: vec![], max_age: 1000 } + ).unwrap()).unwrap(); + + let result = s3.get("htsget-manifests/R005").await; + assert!(matches!(result, Ok(None))); + }, 0).await; + } + + #[tokio::test] + async fn get_cache_expired() { + with_test_mocks(|_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + fs::create_dir_all(&manifest_path).unwrap(); + fs::write(&manifest_path.join("R004"), to_string( + &CacheItem { item: vec![], max_age: 0 } + ).unwrap()).unwrap(); + + let result = s3.get("htsget-manifests/R004").await; + assert!(matches!(result, Ok(None))); + }, 0).await; + } + + #[tokio::test] + async fn get() { + with_test_mocks(|_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + fs::create_dir_all(&manifest_path).unwrap(); + fs::write(&manifest_path.join("R004"), to_string( + &CacheItem { item: vec![], max_age: 1000 } + ).unwrap()).unwrap(); + + let result = s3.get("htsget-manifests/R004").await.unwrap().unwrap(); + assert!(result.is_empty()); + }, 0).await; + } + + #[tokio::test] + async fn put() { + with_test_mocks(|_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + + let manifest_path = base_path.join("elsa-data-tmp"); + fs::create_dir_all(&manifest_path).unwrap(); + + s3.put("htsget-manifests/R004", vec![], 1000).await.unwrap(); + + let result: CacheItem = from_str(&fs::read_to_string(&manifest_path.join("htsget-manifests/R004")).unwrap()).unwrap(); + + assert!(result.item.is_empty()); + assert_eq!(result.max_age, 1000); + }, 0).await; + } } From be27632d90a1861739f939cec9f05c6b06fe4346 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 14 Apr 2023 20:04:39 +1000 Subject: [PATCH 13/25] test(elsa): add remaining tests for manifest functions --- Cargo.lock | 1 + htsget-elsa/Cargo.toml | 1 + htsget-elsa/src/elsa_endpoint.rs | 363 ++++++++++++++++++++++++------- htsget-elsa/src/lib.rs | 42 ++-- htsget-elsa/src/s3.rs | 263 +++++++++++++--------- 5 files changed, 482 insertions(+), 188 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c62cadd..e22fb86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1453,6 +1453,7 @@ dependencies = [ "htsget-config", "htsget-test", "http", + "indexmap", "mockito", "regex", "reqwest", diff --git a/htsget-elsa/Cargo.toml b/htsget-elsa/Cargo.toml index 9f48e9d..5e8b89a 100644 --- a/htsget-elsa/Cargo.toml +++ b/htsget-elsa/Cargo.toml @@ -15,6 +15,7 @@ serde_json = "1.0" regex = "1.7" bytes = "1.4" tracing = "0.1" +indexmap = "1.9" htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index 262ac31..b46620a 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -1,6 +1,5 @@ use crate::Error::{ - DeserializeError, GetManifest, InvalidManifestUri, InvalidReleaseUri, - UnsupportedManifestFeature, + DeserializeError, GetManifest, InvalidManifest, InvalidReleaseUri, UnsupportedManifestFeature, }; use crate::{Cache, Error, GetObject, ResolversFromElsa, Result}; use async_trait::async_trait; @@ -72,43 +71,38 @@ impl ElsaManifest { id: &str, format: Format, ) -> Result { - let uri = Uri::from_str(url) - .map_err(|err| InvalidManifestUri(err.to_string()))? - .into_parts(); - - match uri.scheme { - Some(scheme) if scheme.as_str() == "s3" || scheme.as_str() == "S3" => Ok(()), + let url = match (url.strip_prefix("s3://"), url.strip_prefix("S3://")) { + (Some(url), _) | (_, Some(url)) => Ok(url), _ => Err(UnsupportedManifestFeature( "only S3 manifest uris are supported".to_string(), )), }?; - let bucket = match uri.authority { - Some(bucket) => Ok(bucket.to_string()), - None => Err(InvalidManifestUri("missing bucket from uri".to_string())), - }?; - - let key = match uri.path_and_query { - Some(path) => match path.to_string().strip_suffix(format.file_ending()) { - None => Ok(path.to_string()), - Some(key) => Ok(key.to_string()), - }, - None => Err(InvalidManifestUri( - "missing object path from uri".to_string(), + let (bucket, key) = match url.split_once("/") { + Some(split) => Ok(split), + None => Err(InvalidManifest( + "could not split url into bucket and object key".to_string(), )), }?; + if bucket == "" || key == "" { + return Err(InvalidManifest("bucket or key is empty".to_string())); + } + + let key = match key.to_string().strip_suffix(format.file_ending()) { + None => key.to_string(), + Some(key) => key.to_string(), + }; + Resolver::new( Storage::S3 { - s3_storage: S3Storage::new(bucket), + s3_storage: S3Storage::new(bucket.to_string()), }, &format!("^{}/{}$", release_key, id), &key, AllowGuard::default().with_allow_formats(vec![format]), ) - .map_err(|err| { - InvalidManifestUri(format!("failed to construct regex: {}", err.to_string())) - }) + .map_err(|err| InvalidManifest(format!("failed to construct regex: {}", err.to_string()))) } } @@ -139,7 +133,7 @@ impl TryFrom for Vec { &release_key, &variants_manifest.url, &id, - variants_manifest.format.unwrap_or(Format::Bam), + variants_manifest.format.unwrap_or(Format::Vcf), ) }), ) @@ -166,7 +160,9 @@ where #[instrument(level = "debug", skip_all)] async fn try_get(&self, release_key: String) -> Result> { - match self.cache.get(&release_key).await { + let cache_key = format!("{CACHE_PATH}/{release_key}"); + + match self.cache.get(&cache_key).await { Ok(Some(cached)) => Ok(cached), _ => { debug!("no cached response, fetching from elsa"); @@ -177,7 +173,7 @@ where let resolvers: Vec = self.get_manifest(response).await?.try_into()?; self.cache - .put(format!("{CACHE_PATH}/{release_key}"), resolvers.clone(), max_age) + .put(cache_key, resolvers.clone(), max_age) .await?; Ok(resolvers) @@ -192,16 +188,28 @@ where S: GetObject, { pub fn new(endpoint: Authority, cache: &'a C, get_object: &'a S) -> Result { - Ok(Self::new_with_client(Self::create_client()?, endpoint, cache, get_object, "https")) + Ok(Self::new_with_client( + Self::create_client()?, + endpoint, + cache, + get_object, + "https", + )) } - fn new_with_client(client: Client, endpoint: Authority, cache: &'a C, get_object: &'a S, scheme: &'a str) -> Self { + fn new_with_client( + client: Client, + endpoint: Authority, + cache: &'a C, + get_object: &'a S, + scheme: &'a str, + ) -> Self { Self { endpoint, client, cache, get_object, - scheme + scheme, } } @@ -213,7 +221,11 @@ where .map_err(|err| Error::InvalidClient(err)) } - async fn get_response_with_scheme(&self, release_key: &str, scheme: &str) -> Result { + async fn get_response_with_scheme( + &self, + release_key: &str, + scheme: &str, + ) -> Result { let uri = Uri::builder() .scheme(scheme) .authority(self.endpoint.as_str()) @@ -223,14 +235,18 @@ where .map_err(|_| InvalidReleaseUri(release_key.to_string()))? .map_err(|_| InvalidReleaseUri(release_key.to_string()))?; - let response = self.client + let response = self + .client .get(uri) .send() .await .map_err(|err| GetManifest(err.to_string()))?; if response.status().is_success() { - response.json().await.map_err(|err| DeserializeError(err.to_string())) + response + .json() + .await + .map_err(|err| DeserializeError(err.to_string())) } else { Err(GetManifest(response.status().to_string())) } @@ -238,7 +254,8 @@ where #[instrument(level = "debug", skip(self), ret)] pub async fn get_response(&self, release_key: &str) -> Result { - self.get_response_with_scheme(release_key, self.scheme).await + self.get_response_with_scheme(release_key, self.scheme) + .await } #[instrument(level = "debug", skip(self), ret)] @@ -251,60 +268,260 @@ where #[cfg(test)] mod tests { - use std::fs; - use std::future::Future; - use std::str::FromStr; - use aws_sdk_s3::Client; + use crate::elsa_endpoint::{ + ElsaEndpoint, ElsaLocation, ElsaManifest, ElsaResponse, CACHE_PATH, ENDPOINT_PATH, + }; + use crate::s3::S3; + use crate::tests::{ + example_elsa_manifest, example_elsa_response, with_test_mocks, write_example_manifest, + }; + use crate::Error::{GetObjectError, InvalidManifest, UnsupportedManifestFeature}; + use crate::{Cache, ResolversFromElsa}; + use htsget_config::resolver::Resolver; + use htsget_config::storage; + use htsget_config::types::Format; + use http::response; use http::uri::Authority; - use htsget_test::aws_mocks::with_s3_test_server_tmp; use serde_json::from_str; - use wiremock::{Mock, MockServer, Request, ResponseTemplate, Times}; - use wiremock::matchers::{method, path, query_param}; - use crate::elsa_endpoint::{ElsaEndpoint, ElsaLocation, ElsaResponse, ENDPOINT_PATH}; - use crate::Error::{GetManifest, GetObjectError}; - use crate::ResolversFromElsa; - use crate::s3::S3; - use crate::tests::{example_elsa_manifest, example_elsa_response, with_test_mocks}; + use std::iter::Iterator; + use std::str::FromStr; + use wiremock::matchers::{path, query_param}; + use wiremock::{MockServer, Request, ResponseTemplate, Times}; #[tokio::test] async fn get_response() { - with_test_mocks(|endpoint, s3_client, reqwest_client, _| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); - let endpoint = ElsaEndpoint::new_with_client(reqwest_client, Authority::from_str(&endpoint).unwrap(), &s3, &s3, "http"); - - let response = endpoint.get_response_with_scheme("R004", "http").await.unwrap(); - assert_eq!(response, ElsaResponse { - location: ElsaLocation { - bucket: "elsa-data-tmp".to_string(), - key: "htsget-manifests/R004".to_string(), - }, - max_age: 86400, - }); - }, 1).await; + with_test_mocks( + |endpoint, s3_client, reqwest_client, _| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let endpoint = ElsaEndpoint::new_with_client( + reqwest_client, + Authority::from_str(&endpoint).unwrap(), + &s3, + &s3, + "http", + ); + + let response = endpoint.get_response("R004").await.unwrap(); + assert_eq!( + response, + ElsaResponse { + location: ElsaLocation { + bucket: "elsa-data-tmp".to_string(), + key: "htsget-manifests/R004".to_string(), + }, + max_age: 86400, + } + ); + }, + 1, + ) + .await; } #[tokio::test] async fn get_manifest() { - with_test_mocks(|endpoint, s3_client, reqwest_client, _| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); - let endpoint = ElsaEndpoint::new_with_client(reqwest_client, Authority::from_str(&endpoint).unwrap(), &s3, &s3, "http"); + with_test_mocks( + |endpoint, s3_client, reqwest_client, _| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let endpoint = ElsaEndpoint::new_with_client( + reqwest_client, + Authority::from_str(&endpoint).unwrap(), + &s3, + &s3, + "http", + ); + + let response = endpoint.get_response("R004").await.unwrap(); + let manifest = endpoint.get_manifest(response).await.unwrap(); + + assert_eq!(manifest, from_str(&example_elsa_manifest()).unwrap()); + }, + 1, + ) + .await; + } - let response = endpoint.get_response_with_scheme("R004", "http").await.unwrap(); - let manifest = endpoint.get_manifest(response).await.unwrap(); + #[tokio::test] + async fn get_manifest_not_present() { + with_test_mocks( + |endpoint, s3_client, reqwest_client, _| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let endpoint = ElsaEndpoint::new_with_client( + reqwest_client, + Authority::from_str(&endpoint).unwrap(), + &s3, + &s3, + "http", + ); + + let manifest = endpoint + .get_manifest(from_str(&example_elsa_response()).unwrap()) + .await; + + assert!(matches!(manifest, Err(GetObjectError(_)))); + }, + 0, + ) + .await; + } - assert_eq!(manifest, from_str(&example_elsa_manifest()).unwrap()); - }, 1).await; + #[tokio::test] + async fn try_get_cached() { + with_test_mocks( + |endpoint, s3_client, reqwest_client, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let endpoint = ElsaEndpoint::new_with_client( + reqwest_client, + Authority::from_str(&endpoint).unwrap(), + &s3, + &s3, + "http", + ); + + s3.put(format!("{CACHE_PATH}/R004"), vec![], 1000) + .await + .unwrap(); + + let resolvers = endpoint.try_get("R004".to_string()).await.unwrap(); + assert!(resolvers.is_empty()); + }, + 0, + ) + .await; } #[tokio::test] - async fn get_manifest_not_present() { - with_test_mocks(|endpoint, s3_client, reqwest_client, _| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); - let endpoint = ElsaEndpoint::new_with_client(reqwest_client, Authority::from_str(&endpoint).unwrap(), &s3, &s3, "http"); + async fn try_get_not_cached() { + with_test_mocks( + |endpoint, s3_client, reqwest_client, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let endpoint = ElsaEndpoint::new_with_client( + reqwest_client, + Authority::from_str(&endpoint).unwrap(), + &s3, + &s3, + "http", + ); + + assert!(!base_path + .join(format!("elsa-data-tmp/{CACHE_PATH}/R004")) + .exists()); + let resolvers = endpoint.try_get("R004".to_string()).await.unwrap(); + + assert_manifest_resolvers(resolvers); + assert!(base_path + .join(format!("elsa-data-tmp/{CACHE_PATH}/R004")) + .exists()); + }, + 1, + ) + .await; + } + + #[test] + fn resolvers_from_manifest() { + let manifest: ElsaManifest = from_str(&example_elsa_manifest()).unwrap(); + let resolvers: Vec = manifest.try_into().unwrap(); + assert_manifest_resolvers(resolvers); + } + + #[test] + fn resolver_from_parts() { + let response = ElsaManifest::resolver_from_manifest_parts( + "R004", + "s3://umccr-10g-data-dev/HG00097/HG00097.bam", + "30F9F3FED8F711ED8C35DBEF59E9F537", + Format::Bam, + ) + .unwrap(); + assert!(is_resolver_from_parts(&response)); + } - let manifest = endpoint.get_manifest(from_str(&example_elsa_response()).unwrap()).await; + #[test] + fn resolver_from_parts_uppercase() { + let response = ElsaManifest::resolver_from_manifest_parts( + "R004", + "S3://umccr-10g-data-dev/HG00097/HG00097.bam", + "30F9F3FED8F711ED8C35DBEF59E9F537", + Format::Bam, + ) + .unwrap(); + assert!(is_resolver_from_parts(&response)); + } + + #[test] + fn resolver_from_parts_no_file_ending() { + let response = ElsaManifest::resolver_from_manifest_parts( + "R004", + "s3://umccr-10g-data-dev/HG00097/HG00097", + "30F9F3FED8F711ED8C35DBEF59E9F537", + Format::Bam, + ) + .unwrap(); + assert!(is_resolver_from_parts(&response)); + } + + #[test] + fn resolver_from_parts_invalid_scheme() { + let response = ElsaManifest::resolver_from_manifest_parts( + "R004", + "gcp://umccr-10g-data-dev/HG00097/HG00097.bam", + "30F9F3FED8F711ED8C35DBEF59E9F537", + Format::Bam, + ); + assert!(matches!(response, Err(UnsupportedManifestFeature(_)))); + } + + #[test] + fn resolver_from_parts_no_object_key() { + let response = ElsaManifest::resolver_from_manifest_parts( + "R004", + "s3://umccr-10g-data-dev", + "30F9F3FED8F711ED8C35DBEF59E9F537", + Format::Bam, + ); + assert!(matches!(response, Err(InvalidManifest(_)))); + } + + #[test] + fn resolver_from_parts_no_bucket() { + let response = ElsaManifest::resolver_from_manifest_parts( + "R004", + "s3:///HG00097/HG00097.bam", + "30F9F3FED8F711ED8C35DBEF59E9F537", + Format::Bam, + ); + assert!(matches!(response, Err(InvalidManifest(_)))); + } + + fn is_resolver_from_parts(resolver: &Resolver) -> bool { + resolver.regex().to_string() == "^R004/30F9F3FED8F711ED8C35DBEF59E9F537$" + && resolver.substitution_string().to_string() == "HG00097/HG00097" + && matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") + && resolver.allow_formats() == [Format::Bam] + } - assert!(matches!(manifest, Err(GetObjectError(_)))); - }, 0).await; + fn assert_manifest_resolvers(resolvers: Vec) { + assert!(resolvers.iter().any(|resolver| { + resolver.regex().to_string() == "^R004/30F9FFD4D8F711ED8C353BBCB8861211$" && + resolver.substitution_string().to_string() == "HG00096/HG00096" && + matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && + resolver.allow_formats() == [Format::Bam] + })); + assert!(resolvers + .iter() + .any(|resolver| is_resolver_from_parts(resolver))); + assert!(resolvers.iter().any(|resolver| { + resolver.regex().to_string() == "^R004/30F9FFD4D8F711ED8C353BBCB8861211$" && + resolver.substitution_string().to_string() == "HG00096/HG00096.hard-filtered" && + matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && + resolver.allow_formats() == [Format::Vcf] + })); + assert!(resolvers.iter().any(|resolver| { + resolver.regex().to_string() == "^R004/30F9F3FED8F711ED8C35DBEF59E9F537$" && + resolver.substitution_string().to_string() == "HG00097/HG00097.hard-filtered" && + matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && + resolver.allow_formats() == [Format::Vcf] + })); } } diff --git a/htsget-elsa/src/lib.rs b/htsget-elsa/src/lib.rs index 6fbbc70..c38c126 100644 --- a/htsget-elsa/src/lib.rs +++ b/htsget-elsa/src/lib.rs @@ -27,7 +27,7 @@ pub enum Error { #[error("failed to put object into storage: `{0}`")] PutObjectError(String), #[error("invalid uri received from manifest: `{0}`")] - InvalidManifestUri(String), + InvalidManifest(String), #[error("unsupported component of manifest: `{0}`")] UnsupportedManifestFeature(String), #[error("system error: `{0}`")] @@ -71,14 +71,14 @@ pub trait ResolversFromElsa { #[cfg(test)] pub(crate) mod tests { + use crate::elsa_endpoint::ENDPOINT_PATH; + use aws_sdk_s3::Client; + use htsget_test::aws_mocks::with_s3_test_server_tmp; use std::fs; use std::future::Future; use std::path::{Path, PathBuf}; - use aws_sdk_s3::Client; - use htsget_test::aws_mocks::with_s3_test_server_tmp; - use wiremock::{Mock, MockServer, Request, ResponseTemplate, Times}; use wiremock::matchers::{method, path, query_param}; - use crate::elsa_endpoint::ENDPOINT_PATH; + use wiremock::{Mock, MockServer, Request, ResponseTemplate, Times}; pub(crate) fn example_elsa_manifest() -> String { r#" @@ -134,7 +134,8 @@ pub(crate) mod tests { } ] } - "#.to_string() + "# + .to_string() } pub(crate) fn example_elsa_response() -> String { @@ -146,7 +147,8 @@ pub(crate) mod tests { }, "maxAge": 86400 } - "#.to_string() + "# + .to_string() } pub(crate) fn write_example_manifest(manifest_path: &Path) { @@ -155,10 +157,11 @@ pub(crate) mod tests { } pub(crate) async fn with_test_mocks(test: F, expect_times: T) - where - T: Into, - F: FnOnce(String, Client, reqwest::Client, PathBuf) -> Fut, - Fut: Future, { + where + T: Into, + F: FnOnce(String, Client, reqwest::Client, PathBuf) -> Fut, + Fut: Future, + { with_s3_test_server_tmp(|client, server_base_path| async move { let mock_server = MockServer::start().await; @@ -171,15 +174,20 @@ pub(crate) mod tests { write_example_manifest(&manifest_path); - ResponseTemplate::new(200) - .set_body_string(example_elsa_response()) + ResponseTemplate::new(200).set_body_string(example_elsa_response()) }) .expect(expect_times) .mount(&mock_server) .await; - test(mock_server.address().to_string(), client, reqwest::Client::builder() - .build().unwrap(), server_base_path).await; - }).await; + test( + mock_server.address().to_string(), + client, + reqwest::Client::builder().build().unwrap(), + server_base_path, + ) + .await; + }) + .await; } -} \ No newline at end of file +} diff --git a/htsget-elsa/src/s3.rs b/htsget-elsa/src/s3.rs index a4819b3..60e0e59 100644 --- a/htsget-elsa/src/s3.rs +++ b/htsget-elsa/src/s3.rs @@ -1,6 +1,11 @@ use crate::Error::{DeserializeError, GetObjectError, PutObjectError, SerializeError}; use crate::{Cache, Error, GetObject, Result}; use async_trait::async_trait; +use aws_sdk_s3::error::HeadObjectError; +use aws_sdk_s3::output::HeadObjectOutput; +use aws_sdk_s3::types::{ByteStream, DateTime, SdkError}; +use aws_sdk_s3::Client; +use aws_sdk_s3::Error::NotFound; use bytes::Bytes; use htsget_config::resolver::Resolver; use serde::{Deserialize, Serialize}; @@ -8,11 +13,6 @@ use serde_json::{from_slice, to_string, to_vec}; use std::ops::{Add, Sub}; use std::result; use std::time::{Duration, SystemTime}; -use aws_sdk_s3::Client; -use aws_sdk_s3::error::HeadObjectError; -use aws_sdk_s3::Error::NotFound; -use aws_sdk_s3::output::HeadObjectOutput; -use aws_sdk_s3::types::{ByteStream, DateTime, SdkError}; use tracing::instrument; #[derive(Debug)] @@ -50,8 +50,7 @@ impl S3 { bucket: impl Into + Send, key: impl Into + Send, ) -> Option { - self - .s3_client + self.s3_client .head_object() .bucket(bucket) .key(key) @@ -104,13 +103,18 @@ impl Cache for S3 { #[instrument(level = "trace", skip_all, ret)] async fn get + Send + Sync>(&self, key: K) -> Result> { - if let Some(last_modified) = self.last_modified(self.cache_bucket.clone(), key.as_ref()).await { - let object: CacheItem = self.get_object(self.cache_bucket.clone(), key.as_ref()).await?; + if let Some(last_modified) = self + .last_modified(self.cache_bucket.clone(), key.as_ref()) + .await + { + let object: CacheItem = self + .get_object(self.cache_bucket.clone(), key.as_ref()) + .await?; if last_modified.as_nanos() - > DateTime::from( - SystemTime::now().sub(Duration::from_secs(object.max_age)), - ).as_nanos() { + > DateTime::from(SystemTime::now().sub(Duration::from_secs(object.max_age))) + .as_nanos() + { return Ok(Some(object.item)); } } @@ -143,130 +147,193 @@ impl Cache for S3 { #[cfg(test)] mod tests { + use crate::elsa_endpoint::{ElsaEndpoint, ElsaManifest}; + use crate::s3::{CacheItem, S3}; + use crate::tests::{example_elsa_manifest, with_test_mocks, write_example_manifest}; + use crate::Cache; + use http::uri::Authority; + use serde_json::{from_str, to_string}; use std::fs; use std::str::FromStr; use std::time::Duration; - use http::uri::Authority; - use serde_json::{from_str, to_string}; use tokio::time::sleep; - use crate::Cache; - use crate::elsa_endpoint::{ElsaEndpoint, ElsaManifest}; - use crate::s3::{CacheItem, S3}; - use crate::tests::{with_test_mocks, write_example_manifest, example_elsa_manifest}; #[tokio::test] async fn last_modified() { - with_test_mocks(|_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); - - let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); - write_example_manifest(&manifest_path); - - let result = s3.last_modified("elsa-data-tmp", "htsget-manifests/R004").await; - assert!(matches!(result, Some(_))); - }, 0).await; + with_test_mocks( + |_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + write_example_manifest(&manifest_path); + + let result = s3 + .last_modified("elsa-data-tmp", "htsget-manifests/R004") + .await; + assert!(matches!(result, Some(_))); + }, + 0, + ) + .await; } #[tokio::test] async fn last_modified_not_found() { - with_test_mocks(|_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); - - let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); - write_example_manifest(&manifest_path); - - let result = s3.last_modified("elsa-data-tmp", "htsget-manifests/R005").await; - assert!(matches!(result, None)); - }, 0).await; + with_test_mocks( + |_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + write_example_manifest(&manifest_path); + + let result = s3 + .last_modified("elsa-data-tmp", "htsget-manifests/R005") + .await; + assert!(matches!(result, None)); + }, + 0, + ) + .await; } #[tokio::test] async fn get_object() { - with_test_mocks(|_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); - - let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); - write_example_manifest(&manifest_path); - - let result: ElsaManifest = s3.get_object("elsa-data-tmp", "htsget-manifests/R004").await.unwrap(); - assert_eq!(result, from_str(&example_elsa_manifest()).unwrap()); - }, 0).await; + with_test_mocks( + |_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + write_example_manifest(&manifest_path); + + let result: ElsaManifest = s3 + .get_object("elsa-data-tmp", "htsget-manifests/R004") + .await + .unwrap(); + assert_eq!(result, from_str(&example_elsa_manifest()).unwrap()); + }, + 0, + ) + .await; } #[tokio::test] async fn get_object_not_found() { - with_test_mocks(|_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); - - let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); - write_example_manifest(&manifest_path); - - assert!(s3.get_object::("elsa-data-tmp", "htsget-manifests/R005").await.is_err()); - }, 0).await; + with_test_mocks( + |_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + write_example_manifest(&manifest_path); + + assert!(s3 + .get_object::("elsa-data-tmp", "htsget-manifests/R005") + .await + .is_err()); + }, + 0, + ) + .await; } #[tokio::test] async fn get_not_found() { - with_test_mocks(|_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); - - let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); - fs::create_dir_all(&manifest_path).unwrap(); - fs::write(&manifest_path.join("R004"), to_string( - &CacheItem { item: vec![], max_age: 1000 } - ).unwrap()).unwrap(); - - let result = s3.get("htsget-manifests/R005").await; - assert!(matches!(result, Ok(None))); - }, 0).await; + with_test_mocks( + |_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + fs::create_dir_all(&manifest_path).unwrap(); + fs::write( + &manifest_path.join("R004"), + to_string(&CacheItem { + item: vec![], + max_age: 1000, + }) + .unwrap(), + ) + .unwrap(); + + let result = s3.get("htsget-manifests/R005").await; + assert!(matches!(result, Ok(None))); + }, + 0, + ) + .await; } #[tokio::test] async fn get_cache_expired() { - with_test_mocks(|_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); - - let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); - fs::create_dir_all(&manifest_path).unwrap(); - fs::write(&manifest_path.join("R004"), to_string( - &CacheItem { item: vec![], max_age: 0 } - ).unwrap()).unwrap(); - - let result = s3.get("htsget-manifests/R004").await; - assert!(matches!(result, Ok(None))); - }, 0).await; + with_test_mocks( + |_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + fs::create_dir_all(&manifest_path).unwrap(); + fs::write( + &manifest_path.join("R004"), + to_string(&CacheItem { + item: vec![], + max_age: 0, + }) + .unwrap(), + ) + .unwrap(); + + let result = s3.get("htsget-manifests/R004").await; + assert!(matches!(result, Ok(None))); + }, + 0, + ) + .await; } #[tokio::test] async fn get() { - with_test_mocks(|_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); - - let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); - fs::create_dir_all(&manifest_path).unwrap(); - fs::write(&manifest_path.join("R004"), to_string( - &CacheItem { item: vec![], max_age: 1000 } - ).unwrap()).unwrap(); - - let result = s3.get("htsget-manifests/R004").await.unwrap().unwrap(); - assert!(result.is_empty()); - }, 0).await; + with_test_mocks( + |_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + fs::create_dir_all(&manifest_path).unwrap(); + fs::write( + &manifest_path.join("R004"), + to_string(&CacheItem { + item: vec![], + max_age: 1000, + }) + .unwrap(), + ) + .unwrap(); + + let result = s3.get("htsget-manifests/R004").await.unwrap().unwrap(); + assert!(result.is_empty()); + }, + 0, + ) + .await; } #[tokio::test] async fn put() { - with_test_mocks(|_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + with_test_mocks( + |_, s3_client, _, base_path| async move { + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); - let manifest_path = base_path.join("elsa-data-tmp"); - fs::create_dir_all(&manifest_path).unwrap(); + let manifest_path = base_path.join("elsa-data-tmp"); + fs::create_dir_all(&manifest_path).unwrap(); - s3.put("htsget-manifests/R004", vec![], 1000).await.unwrap(); + s3.put("htsget-manifests/R004", vec![], 1000).await.unwrap(); - let result: CacheItem = from_str(&fs::read_to_string(&manifest_path.join("htsget-manifests/R004")).unwrap()).unwrap(); + let result: CacheItem = from_str( + &fs::read_to_string(&manifest_path.join("htsget-manifests/R004")).unwrap(), + ) + .unwrap(); - assert!(result.item.is_empty()); - assert_eq!(result.max_age, 1000); - }, 0).await; + assert!(result.item.is_empty()); + assert_eq!(result.max_age, 1000); + }, + 0, + ) + .await; } } From 9db63199d717d6ebdf3841d56563880180b64729 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 14 Apr 2023 20:04:48 +1000 Subject: [PATCH 14/25] style: tidy with clippy and fmt --- Cargo.lock | 160 ++----------------------------- data/events/event_get.json | 102 ++++++++++++++++++++ htsget-elsa-lambda/src/config.rs | 5 +- htsget-elsa-lambda/src/lib.rs | 22 ++--- htsget-elsa-lambda/src/main.rs | 6 +- htsget-elsa/Cargo.toml | 2 - htsget-elsa/src/elsa_endpoint.rs | 61 ++++++------ htsget-elsa/src/lib.rs | 19 ++-- htsget-elsa/src/s3.rs | 39 ++++---- 9 files changed, 178 insertions(+), 238 deletions(-) create mode 100644 data/events/event_get.json diff --git a/Cargo.lock b/Cargo.lock index e22fb86..b8e282d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -180,17 +180,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -850,17 +839,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "colored" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd" -dependencies = [ - "atty", - "lazy_static", - "winapi", -] - [[package]] name = "concolor-override" version = "1.0.0" @@ -1372,15 +1350,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.2.6" @@ -1453,8 +1422,6 @@ dependencies = [ "htsget-config", "htsget-test", "http", - "indexmap", - "mockito", "regex", "reqwest", "serde", @@ -1627,7 +1594,7 @@ dependencies = [ "http", "infer", "pin-project-lite", - "rand 0.7.3", + "rand", "serde", "serde_json", "serde_qs", @@ -1960,16 +1927,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" -[[package]] -name = "lock_api" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" -dependencies = [ - "autocfg", - "scopeguard", -] - [[package]] name = "log" version = "0.4.17" @@ -2063,26 +2020,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "mockito" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea57936ab3bf56156f135f20ee24b840e5a8ad97a8e1710eace33ac078f8f537" -dependencies = [ - "assert-json-diff 2.0.2", - "colored", - "futures", - "hyper", - "lazy_static", - "log", - "rand 0.8.5", - "regex", - "serde_json", - "serde_urlencoded", - "similar", - "tokio", -] - [[package]] name = "native-tls" version = "0.2.11" @@ -2418,29 +2355,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "windows-sys 0.45.0", -] - [[package]] name = "path-absolutize" version = "3.0.14" @@ -2637,22 +2551,11 @@ checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ "getrandom 0.1.16", "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", + "rand_chacha", + "rand_core", "rand_hc", ] -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - [[package]] name = "rand_chacha" version = "0.2.2" @@ -2660,17 +2563,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" dependencies = [ "ppv-lite86", - "rand_core 0.5.1", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", + "rand_core", ] [[package]] @@ -2682,22 +2575,13 @@ dependencies = [ "getrandom 0.1.16", ] -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.9", -] - [[package]] name = "rand_hc" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" dependencies = [ - "rand_core 0.5.1", + "rand_core", ] [[package]] @@ -2712,15 +2596,6 @@ dependencies = [ "yasna", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - [[package]] name = "redox_syscall" version = "0.3.5" @@ -2991,12 +2866,6 @@ dependencies = [ "windows-sys 0.42.0", ] -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - [[package]] name = "scratch" version = "1.0.5" @@ -3184,21 +3053,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "signal-hook-registry" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" -dependencies = [ - "libc", -] - -[[package]] -name = "similar" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" - [[package]] name = "slab" version = "0.4.8" @@ -3284,7 +3138,7 @@ checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ "cfg-if", "fastrand", - "redox_syscall 0.3.5", + "redox_syscall", "rustix", "windows-sys 0.45.0", ] @@ -3381,9 +3235,7 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot", "pin-project-lite", - "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.45.0", diff --git a/data/events/event_get.json b/data/events/event_get.json new file mode 100644 index 0000000..2b65f35 --- /dev/null +++ b/data/events/event_get.json @@ -0,0 +1,102 @@ +{ + "httpMethod": "GET", + "path": "/variants/1-vcf/sample1-bcbio-cancer", + "body": null, + + "resource": "/{proxy+}", + "headers": { + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "accept-encoding": "gzip, deflate, br", + "accept-language": "en-US,en;q=0.9", + "cookie": "s_fid=7AAB6XMPLAFD9BBF-0643XMPL09956DE2; regStatus=pre-register", + "Host": "70ixmpl4fl.execute-api.us-east-2.amazonaws.com", + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "none", + "upgrade-insecure-requests": "1", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "X-Amzn-Trace-Id": "Root=1-5e66d96f-7491f09xmpl79d18acf3d050", + "X-Forwarded-For": "52.255.255.12", + "X-Forwarded-Port": "443", + "X-Forwarded-Proto": "https" + }, + "multiValueHeaders": { + "accept": [ + "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" + ], + "accept-encoding": [ + "gzip, deflate, br" + ], + "accept-language": [ + "en-US,en;q=0.9" + ], + "cookie": [ + "s_fid=7AABXMPL1AFD9BBF-0643XMPL09956DE2; regStatus=pre-register;" + ], + "Host": [ + "70ixmpl4fl.execute-api.ca-central-1.amazonaws.com" + ], + "sec-fetch-dest": [ + "document" + ], + "sec-fetch-mode": [ + "navigate" + ], + "sec-fetch-site": [ + "none" + ], + "upgrade-insecure-requests": [ + "1" + ], + "User-Agent": [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36" + ], + "X-Amzn-Trace-Id": [ + "Root=1-5e66d96f-7491f09xmpl79d18acf3d050" + ], + "X-Forwarded-For": [ + "52.255.255.12" + ], + "X-Forwarded-Port": [ + "443" + ], + "X-Forwarded-Proto": [ + "https" + ] + }, + "queryStringParameters": null, + "multiValueQueryStringParameters": null, + "pathParameters": null, + "stageVariables": null, + "requestContext": { + "resourceId": "2gxmpl", + "resourcePath": "/", + "httpMethod": "GET", + "extendedRequestId": "JJbxmplHYosFVYQ=", + "requestTime": "10/Mar/2020:00:03:59 +0000", + "path": "/variants/vcf/sample1-bcbio-cancer", + "accountId": "123456789012", + "protocol": "HTTP/1.1", + "stage": "Prod", + "domainPrefix": "70ixmpl4fl", + "requestTimeEpoch": 1583798639428, + "requestId": "77375676-xmpl-4b79-853a-f982474efe18", + "identity": { + "cognitoIdentityPoolId": null, + "accountId": null, + "cognitoIdentityId": null, + "caller": null, + "sourceIp": "52.255.255.12", + "principalOrgId": null, + "accessKey": null, + "cognitoAuthenticationType": null, + "cognitoAuthenticationProvider": null, + "userArn": null, + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "user": null + }, + "domainName": "70ixmpl4fl.execute-api.us-east-2.amazonaws.com", + "apiId": "70ixmpl4fl" + }, + "isBase64Encoded": false +} \ No newline at end of file diff --git a/htsget-elsa-lambda/src/config.rs b/htsget-elsa-lambda/src/config.rs index 59d900e..251822f 100644 --- a/htsget-elsa-lambda/src/config.rs +++ b/htsget-elsa-lambda/src/config.rs @@ -1,9 +1,10 @@ +use std::io; +use std::path::Path; + use htsget_config::config::parser::from_path; use htsget_config::config::Config as HtsGetConfig; use http::uri::Authority; use serde::{Deserialize, Serialize}; -use std::io; -use std::path::Path; use tracing::instrument; #[derive(Serialize, Deserialize, Debug, Clone)] diff --git a/htsget-elsa-lambda/src/lib.rs b/htsget-elsa-lambda/src/lib.rs index 19359e3..ab7aeb5 100644 --- a/htsget-elsa-lambda/src/lib.rs +++ b/htsget-elsa-lambda/src/lib.rs @@ -1,18 +1,18 @@ -use crate::config::Config; -use htsget_config::config::cors::CorsConfig; -use htsget_config::config::ServiceInfo; +use std::sync::Arc; + use htsget_config::resolver::Resolver; -use htsget_elsa::elsa_endpoint::ElsaEndpoint; -use htsget_elsa::s3::S3; -use htsget_elsa::{Cache, GetObject, ResolversFromElsa}; -use htsget_lambda::handlers::FormatJson; use htsget_lambda::RouteType::Id; use htsget_lambda::{handle_request_service_fn, Route, Router}; use http::{Response, StatusCode}; use lambda_http::{Body, Error, Request}; -use std::sync::Arc; use tracing::{info, instrument, warn}; +use htsget_elsa::elsa_endpoint::ElsaEndpoint; +use htsget_elsa::s3::S3; +use htsget_elsa::{Cache, GetObject, ResolversFromElsa}; + +use crate::config::Config; + pub mod config; pub async fn handle_request(config: Config) -> Result<(), Error> { @@ -45,10 +45,10 @@ async fn route_request( } }; - let resolver = get_resolvers(&config, &route, &elsa_endpoint).await?; + let resolver = get_resolvers(config, &route, &elsa_endpoint).await?; let router = Router::new( Arc::new(resolver), - &config.htsget_config().ticket_server().service_info(), + config.htsget_config().ticket_server().service_info(), ); router.route_request_with_route(event, route).await } @@ -64,7 +64,7 @@ where S: GetObject + Send + Sync, { if let Id(id) = route.route_type() { - if let Some(release_key) = id.split("/").collect::>().first() { + if let Some(release_key) = id.split('/').collect::>().first() { if let Ok(mut resolvers) = elsa_endpoint.try_get(release_key.to_string()).await { resolvers.append(&mut config.htsget_config().resolvers().to_vec()); diff --git a/htsget-elsa-lambda/src/main.rs b/htsget-elsa-lambda/src/main.rs index e3189eb..01f815f 100644 --- a/htsget-elsa-lambda/src/main.rs +++ b/htsget-elsa-lambda/src/main.rs @@ -1,13 +1,9 @@ -use htsget_config::config::cors::CorsConfig; use htsget_config::config::parser::from_path; -use htsget_config::config::ServiceInfo; -use std::sync::Arc; - +use htsget_lambda::Config as HtsgetConfig; use lambda_http::Error; use htsget_elsa_lambda::config::Config; use htsget_elsa_lambda::handle_request; -use htsget_lambda::Config as HtsgetConfig; #[tokio::main] async fn main() -> Result<(), Error> { diff --git a/htsget-elsa/Cargo.toml b/htsget-elsa/Cargo.toml index 5e8b89a..5b5e9de 100644 --- a/htsget-elsa/Cargo.toml +++ b/htsget-elsa/Cargo.toml @@ -15,7 +15,6 @@ serde_json = "1.0" regex = "1.7" bytes = "1.4" tracing = "0.1" -indexmap = "1.9" htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } @@ -23,5 +22,4 @@ htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat htsget-test = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa", features = ["aws-mocks"] } tokio = { version = "1.25", features = ["macros", "rt-multi-thread"] } -mockito = "1.0" wiremock = "0.5" \ No newline at end of file diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index b46620a..2f91235 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -1,20 +1,21 @@ -use crate::Error::{ - DeserializeError, GetManifest, InvalidManifest, InvalidReleaseUri, UnsupportedManifestFeature, -}; -use crate::{Cache, Error, GetObject, ResolversFromElsa, Result}; +use std::collections::HashMap; + use async_trait::async_trait; use htsget_config::resolver::{AllowGuard, Resolver}; use htsget_config::storage::s3::S3Storage; use htsget_config::storage::Storage; use htsget_config::types::Format; -use http::uri::{Authority, Parts, Scheme}; +use http::uri::Authority; use http::Uri; use reqwest::{Client, Url}; use serde::Deserialize; -use std::collections::HashMap; -use std::str::FromStr; use tracing::{debug, instrument}; +use crate::Error::{ + DeserializeError, GetManifest, InvalidManifest, InvalidReleaseUri, UnsupportedManifestFeature, +}; +use crate::{Cache, Error, GetObject, ResolversFromElsa, Result}; + pub const ENDPOINT_PATH: &str = "/api/manifest/htsget"; pub const CACHE_PATH: &str = "htsget-manifest-cache"; @@ -78,14 +79,14 @@ impl ElsaManifest { )), }?; - let (bucket, key) = match url.split_once("/") { + let (bucket, key) = match url.split_once('/') { Some(split) => Ok(split), None => Err(InvalidManifest( "could not split url into bucket and object key".to_string(), )), }?; - if bucket == "" || key == "" { + if bucket.is_empty() || key.is_empty() { return Err(InvalidManifest("bucket or key is empty".to_string())); } @@ -102,7 +103,7 @@ impl ElsaManifest { &key, AllowGuard::default().with_allow_formats(vec![format]), ) - .map_err(|err| InvalidManifest(format!("failed to construct regex: {}", err.to_string()))) + .map_err(|err| InvalidManifest(format!("failed to construct regex: {}", err))) } } @@ -218,7 +219,7 @@ where .use_rustls_tls() .https_only(true) .build() - .map_err(|err| Error::InvalidClient(err)) + .map_err(Error::InvalidClient) } async fn get_response_with_scheme( @@ -268,25 +269,21 @@ where #[cfg(test)] mod tests { - use crate::elsa_endpoint::{ - ElsaEndpoint, ElsaLocation, ElsaManifest, ElsaResponse, CACHE_PATH, ENDPOINT_PATH, - }; - use crate::s3::S3; - use crate::tests::{ - example_elsa_manifest, example_elsa_response, with_test_mocks, write_example_manifest, - }; - use crate::Error::{GetObjectError, InvalidManifest, UnsupportedManifestFeature}; - use crate::{Cache, ResolversFromElsa}; + use std::str::FromStr; + use htsget_config::resolver::Resolver; use htsget_config::storage; use htsget_config::types::Format; - use http::response; use http::uri::Authority; use serde_json::from_str; - use std::iter::Iterator; - use std::str::FromStr; - use wiremock::matchers::{path, query_param}; - use wiremock::{MockServer, Request, ResponseTemplate, Times}; + + use crate::elsa_endpoint::{ + ElsaEndpoint, ElsaLocation, ElsaManifest, ElsaResponse, CACHE_PATH, + }; + use crate::s3::S3; + use crate::tests::{example_elsa_manifest, example_elsa_response, with_test_mocks}; + use crate::Error::{GetObjectError, InvalidManifest, UnsupportedManifestFeature}; + use crate::{Cache, ResolversFromElsa}; #[tokio::test] async fn get_response() { @@ -368,7 +365,7 @@ mod tests { #[tokio::test] async fn try_get_cached() { with_test_mocks( - |endpoint, s3_client, reqwest_client, base_path| async move { + |endpoint, s3_client, reqwest_client, _| async move { let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); let endpoint = ElsaEndpoint::new_with_client( reqwest_client, @@ -496,7 +493,7 @@ mod tests { fn is_resolver_from_parts(resolver: &Resolver) -> bool { resolver.regex().to_string() == "^R004/30F9F3FED8F711ED8C35DBEF59E9F537$" - && resolver.substitution_string().to_string() == "HG00097/HG00097" + && resolver.substitution_string() == "HG00097/HG00097" && matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && resolver.allow_formats() == [Format::Bam] } @@ -504,22 +501,20 @@ mod tests { fn assert_manifest_resolvers(resolvers: Vec) { assert!(resolvers.iter().any(|resolver| { resolver.regex().to_string() == "^R004/30F9FFD4D8F711ED8C353BBCB8861211$" && - resolver.substitution_string().to_string() == "HG00096/HG00096" && + resolver.substitution_string() == "HG00096/HG00096" && matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && resolver.allow_formats() == [Format::Bam] })); - assert!(resolvers - .iter() - .any(|resolver| is_resolver_from_parts(resolver))); + assert!(resolvers.iter().any(is_resolver_from_parts)); assert!(resolvers.iter().any(|resolver| { resolver.regex().to_string() == "^R004/30F9FFD4D8F711ED8C353BBCB8861211$" && - resolver.substitution_string().to_string() == "HG00096/HG00096.hard-filtered" && + resolver.substitution_string() == "HG00096/HG00096.hard-filtered" && matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && resolver.allow_formats() == [Format::Vcf] })); assert!(resolvers.iter().any(|resolver| { resolver.regex().to_string() == "^R004/30F9F3FED8F711ED8C35DBEF59E9F537$" && - resolver.substitution_string().to_string() == "HG00097/HG00097.hard-filtered" && + resolver.substitution_string() == "HG00097/HG00097.hard-filtered" && matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && resolver.allow_formats() == [Format::Vcf] })); diff --git a/htsget-elsa/src/lib.rs b/htsget-elsa/src/lib.rs index c38c126..a39507d 100644 --- a/htsget-elsa/src/lib.rs +++ b/htsget-elsa/src/lib.rs @@ -1,13 +1,13 @@ -pub mod elsa_endpoint; -pub mod s3; +use std::result; use async_trait::async_trait; use htsget_config::resolver::Resolver; -use http::Response; use serde::Deserialize; -use std::result; use thiserror::Error; +pub mod elsa_endpoint; +pub mod s3; + pub type Result = result::Result; #[derive(Error, Debug)] @@ -71,12 +71,13 @@ pub trait ResolversFromElsa { #[cfg(test)] pub(crate) mod tests { - use crate::elsa_endpoint::ENDPOINT_PATH; - use aws_sdk_s3::Client; - use htsget_test::aws_mocks::with_s3_test_server_tmp; use std::fs; use std::future::Future; use std::path::{Path, PathBuf}; + + use crate::elsa_endpoint::ENDPOINT_PATH; + use aws_sdk_s3::Client; + use htsget_test::aws_mocks::with_s3_test_server_tmp; use wiremock::matchers::{method, path, query_param}; use wiremock::{Mock, MockServer, Request, ResponseTemplate, Times}; @@ -152,8 +153,8 @@ pub(crate) mod tests { } pub(crate) fn write_example_manifest(manifest_path: &Path) { - fs::create_dir_all(&manifest_path).unwrap(); - fs::write(&manifest_path.join("R004"), example_elsa_manifest()).unwrap(); + fs::create_dir_all(manifest_path).unwrap(); + fs::write(manifest_path.join("R004"), example_elsa_manifest()).unwrap(); } pub(crate) async fn with_test_mocks(test: F, expect_times: T) diff --git a/htsget-elsa/src/s3.rs b/htsget-elsa/src/s3.rs index 60e0e59..8c3cd72 100644 --- a/htsget-elsa/src/s3.rs +++ b/htsget-elsa/src/s3.rs @@ -1,20 +1,18 @@ -use crate::Error::{DeserializeError, GetObjectError, PutObjectError, SerializeError}; -use crate::{Cache, Error, GetObject, Result}; +use std::ops::Sub; +use std::time::{Duration, SystemTime}; + use async_trait::async_trait; -use aws_sdk_s3::error::HeadObjectError; -use aws_sdk_s3::output::HeadObjectOutput; -use aws_sdk_s3::types::{ByteStream, DateTime, SdkError}; +use aws_sdk_s3::types::{ByteStream, DateTime}; use aws_sdk_s3::Client; -use aws_sdk_s3::Error::NotFound; use bytes::Bytes; use htsget_config::resolver::Resolver; use serde::{Deserialize, Serialize}; -use serde_json::{from_slice, to_string, to_vec}; -use std::ops::{Add, Sub}; -use std::result; -use std::time::{Duration, SystemTime}; +use serde_json::{from_slice, to_vec}; use tracing::instrument; +use crate::Error::{DeserializeError, GetObjectError, PutObjectError, SerializeError}; +use crate::{Cache, Error, GetObject, Result}; + #[derive(Debug)] pub struct S3 { s3_client: Client, @@ -79,7 +77,7 @@ impl S3 { .map_err(|err| GetObjectError(err.to_string()))? .into_bytes(); - Ok(from_slice(output.as_ref()).map_err(|err| DeserializeError(err.to_string()))?) + from_slice(output.as_ref()).map_err(|err| DeserializeError(err.to_string())) } } @@ -147,16 +145,13 @@ impl Cache for S3 { #[cfg(test)] mod tests { - use crate::elsa_endpoint::{ElsaEndpoint, ElsaManifest}; + use serde_json::{from_str, to_string}; + use std::fs; + + use crate::elsa_endpoint::ElsaManifest; use crate::s3::{CacheItem, S3}; use crate::tests::{example_elsa_manifest, with_test_mocks, write_example_manifest}; use crate::Cache; - use http::uri::Authority; - use serde_json::{from_str, to_string}; - use std::fs; - use std::str::FromStr; - use std::time::Duration; - use tokio::time::sleep; #[tokio::test] async fn last_modified() { @@ -244,7 +239,7 @@ mod tests { let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); fs::create_dir_all(&manifest_path).unwrap(); fs::write( - &manifest_path.join("R004"), + manifest_path.join("R004"), to_string(&CacheItem { item: vec![], max_age: 1000, @@ -270,7 +265,7 @@ mod tests { let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); fs::create_dir_all(&manifest_path).unwrap(); fs::write( - &manifest_path.join("R004"), + manifest_path.join("R004"), to_string(&CacheItem { item: vec![], max_age: 0, @@ -296,7 +291,7 @@ mod tests { let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); fs::create_dir_all(&manifest_path).unwrap(); fs::write( - &manifest_path.join("R004"), + manifest_path.join("R004"), to_string(&CacheItem { item: vec![], max_age: 1000, @@ -325,7 +320,7 @@ mod tests { s3.put("htsget-manifests/R004", vec![], 1000).await.unwrap(); let result: CacheItem = from_str( - &fs::read_to_string(&manifest_path.join("htsget-manifests/R004")).unwrap(), + &fs::read_to_string(manifest_path.join("htsget-manifests/R004")).unwrap(), ) .unwrap(); From 8de3bf0f30851724dcc0849686653b98b5133c59 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 18 Apr 2023 15:44:16 +1000 Subject: [PATCH 15/25] test: additional tests for routing a mock event --- Cargo.lock | 106 +-------------------- data/events/event_elsa.json | 102 +++++++++++++++++++++ htsget-elsa-lambda/Cargo.toml | 8 +- htsget-elsa-lambda/src/config.rs | 26 ++++-- htsget-elsa-lambda/src/lib.rs | 129 +++++++++++++++++++++++--- htsget-elsa/Cargo.toml | 16 +++- htsget-elsa/src/elsa_endpoint.rs | 42 ++------- htsget-elsa/src/lib.rs | 126 +------------------------ htsget-elsa/src/s3.rs | 5 +- htsget-elsa/src/test_utils.rs | 152 +++++++++++++++++++++++++++++++ 10 files changed, 424 insertions(+), 288 deletions(-) create mode 100644 data/events/event_elsa.json create mode 100644 htsget-elsa/src/test_utils.rs diff --git a/Cargo.lock b/Cargo.lock index b8e282d..0d274f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1153,21 +1153,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.1.0" @@ -1420,6 +1405,7 @@ dependencies = [ "aws-sdk-s3", "bytes", "htsget-config", + "htsget-elsa", "htsget-test", "http", "regex", @@ -1439,6 +1425,8 @@ dependencies = [ "htsget-config", "htsget-elsa", "htsget-lambda", + "htsget-search", + "htsget-test", "http", "http-serde", "lambda_http", @@ -1653,19 +1641,6 @@ dependencies = [ "tokio-rustls", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "iana-time-zone" version = "0.1.56" @@ -2020,24 +1995,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nom" version = "7.1.3" @@ -2284,50 +2241,12 @@ version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" -[[package]] -name = "openssl" -version = "0.10.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e30d8bc91859781f0a943411186324d580f2bbeb71b452fe91ae344806af3f1" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.14", -] - [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-sys" -version = "0.9.85" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d3d193fb1488ad46ffe3aaabc912cc931d02ee8518fe2959aea8ef52718b0c0" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "output_vt100" version = "0.1.3" @@ -2647,12 +2566,10 @@ dependencies = [ "http-body", "hyper", "hyper-rustls", - "hyper-tls", "ipnet", "js-sys", "log", "mime", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -2662,7 +2579,6 @@ dependencies = [ "serde_json", "serde_urlencoded", "tokio", - "tokio-native-tls", "tokio-rustls", "tower-service", "url", @@ -3252,16 +3168,6 @@ dependencies = [ "syn 2.0.14", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.23.4" @@ -3579,12 +3485,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.4" diff --git a/data/events/event_elsa.json b/data/events/event_elsa.json new file mode 100644 index 0000000..4edfdb0 --- /dev/null +++ b/data/events/event_elsa.json @@ -0,0 +1,102 @@ +{ + "httpMethod": "GET", + "path": "/variants/R004/30F9F3FED8F711ED8C35DBEF59E9F537", + "body": null, + + "resource": "/{proxy+}", + "headers": { + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", + "accept-encoding": "gzip, deflate, br", + "accept-language": "en-US,en;q=0.9", + "cookie": "s_fid=7AAB6XMPLAFD9BBF-0643XMPL09956DE2; regStatus=pre-register", + "Host": "70ixmpl4fl.execute-api.us-east-2.amazonaws.com", + "sec-fetch-dest": "document", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "none", + "upgrade-insecure-requests": "1", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "X-Amzn-Trace-Id": "Root=1-5e66d96f-7491f09xmpl79d18acf3d050", + "X-Forwarded-For": "52.255.255.12", + "X-Forwarded-Port": "443", + "X-Forwarded-Proto": "https" + }, + "multiValueHeaders": { + "accept": [ + "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9" + ], + "accept-encoding": [ + "gzip, deflate, br" + ], + "accept-language": [ + "en-US,en;q=0.9" + ], + "cookie": [ + "s_fid=7AABXMPL1AFD9BBF-0643XMPL09956DE2; regStatus=pre-register;" + ], + "Host": [ + "70ixmpl4fl.execute-api.ca-central-1.amazonaws.com" + ], + "sec-fetch-dest": [ + "document" + ], + "sec-fetch-mode": [ + "navigate" + ], + "sec-fetch-site": [ + "none" + ], + "upgrade-insecure-requests": [ + "1" + ], + "User-Agent": [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36" + ], + "X-Amzn-Trace-Id": [ + "Root=1-5e66d96f-7491f09xmpl79d18acf3d050" + ], + "X-Forwarded-For": [ + "52.255.255.12" + ], + "X-Forwarded-Port": [ + "443" + ], + "X-Forwarded-Proto": [ + "https" + ] + }, + "queryStringParameters": null, + "multiValueQueryStringParameters": null, + "pathParameters": null, + "stageVariables": null, + "requestContext": { + "resourceId": "2gxmpl", + "resourcePath": "/", + "httpMethod": "GET", + "extendedRequestId": "JJbxmplHYosFVYQ=", + "requestTime": "10/Mar/2020:00:03:59 +0000", + "path": "/variants/vcf/sample1-bcbio-cancer", + "accountId": "123456789012", + "protocol": "HTTP/1.1", + "stage": "Prod", + "domainPrefix": "70ixmpl4fl", + "requestTimeEpoch": 1583798639428, + "requestId": "77375676-xmpl-4b79-853a-f982474efe18", + "identity": { + "cognitoIdentityPoolId": null, + "accountId": null, + "cognitoIdentityId": null, + "caller": null, + "sourceIp": "52.255.255.12", + "principalOrgId": null, + "accessKey": null, + "cognitoAuthenticationType": null, + "cognitoAuthenticationProvider": null, + "userArn": null, + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36", + "user": null + }, + "domainName": "70ixmpl4fl.execute-api.us-east-2.amazonaws.com", + "apiId": "70ixmpl4fl" + }, + "isBase64Encoded": false +} \ No newline at end of file diff --git a/htsget-elsa-lambda/Cargo.toml b/htsget-elsa-lambda/Cargo.toml index c405a26..68c8865 100644 --- a/htsget-elsa-lambda/Cargo.toml +++ b/htsget-elsa-lambda/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "htsget-elsa-lambda" +authors = ["Marko Malenic "] version = "0.1.0" edition = "2021" @@ -15,4 +16,9 @@ http = "0.2" htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } htsget-lambda = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } -htsget-elsa = { version = "0.1", path = "../htsget-elsa" } \ No newline at end of file +htsget-elsa = { version = "0.1", path = "../htsget-elsa" } + +[dev-dependencies] +htsget-elsa = { version = "0.1", path = "../htsget-elsa", features = ["test-utils"] } +htsget-search = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } +htsget-test = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa", default-features = false, features = ["http-tests"] } \ No newline at end of file diff --git a/htsget-elsa-lambda/src/config.rs b/htsget-elsa-lambda/src/config.rs index 251822f..c2cbbee 100644 --- a/htsget-elsa-lambda/src/config.rs +++ b/htsget-elsa-lambda/src/config.rs @@ -9,28 +9,40 @@ use tracing::instrument; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Config { - #[serde(flatten)] + #[serde(flatten, default)] htsget_config: HtsGetConfig, #[serde(with = "http_serde::authority")] - elsa_endpoint: Authority, + elsa_endpoint_authority: Authority, cache_location: String, } impl Config { - #[instrument(level = "debug", ret)] - pub fn from_path(path: &Path) -> io::Result { - from_path(path) + pub fn new( + htsget_config: HtsGetConfig, + elsa_endpoint_authority: Authority, + cache_location: String, + ) -> Self { + Self { + htsget_config, + elsa_endpoint_authority, + cache_location, + } } pub fn htsget_config(&self) -> &HtsGetConfig { &self.htsget_config } - pub fn elsa_endpoint(&self) -> &Authority { - &self.elsa_endpoint + pub fn elsa_endpoint_authority(&self) -> &Authority { + &self.elsa_endpoint_authority } pub fn cache_location(&self) -> &str { &self.cache_location } + + #[instrument(level = "debug", ret)] + pub fn from_path(path: &Path) -> io::Result { + from_path(path) + } } diff --git a/htsget-elsa-lambda/src/lib.rs b/htsget-elsa-lambda/src/lib.rs index ab7aeb5..3b262a9 100644 --- a/htsget-elsa-lambda/src/lib.rs +++ b/htsget-elsa-lambda/src/lib.rs @@ -22,7 +22,21 @@ pub async fn handle_request(config: Config) -> Result<(), Error> { info!(event = ?event, "received request"); match Route::try_from(&event) { - Ok(route) => route_request(&config, event, route).await, + Ok(route) => { + let s3 = S3::new_with_default_config(config.cache_location().to_string()).await; + let elsa_endpoint = + match ElsaEndpoint::new(config.elsa_endpoint_authority().clone(), &s3, &s3) + { + Ok(elsa_endpoint) => elsa_endpoint, + Err(err) => { + return Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::from(err.to_string())); + } + }; + + route_request(&config, event, route, &elsa_endpoint).await + } Err(err) => err, } }, @@ -30,26 +44,18 @@ pub async fn handle_request(config: Config) -> Result<(), Error> { .await } -async fn route_request( +pub async fn route_request<'a>( config: &Config, event: Request, route: Route, + elsa_endpoint: &ElsaEndpoint<'a, S3, S3>, ) -> http::Result> { - let s3 = S3::new_with_default_config(config.cache_location().to_string()).await; - let elsa_endpoint = match ElsaEndpoint::new(config.elsa_endpoint().clone(), &s3, &s3) { - Ok(elsa_endpoint) => elsa_endpoint, - Err(err) => { - return Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(Body::from(err.to_string())); - } - }; - - let resolver = get_resolvers(config, &route, &elsa_endpoint).await?; + let resolver = get_resolvers(config, &route, elsa_endpoint).await?; let router = Router::new( Arc::new(resolver), config.htsget_config().ticket_server().service_info(), ); + router.route_request_with_route(event, route).await } @@ -79,3 +85,100 @@ where Ok(config.htsget_config().resolvers().to_vec()) } + +#[cfg(test)] +mod tests { + use std::fs; + use std::path::PathBuf; + use std::str::FromStr; + + use htsget_config::resolver::Resolver; + use htsget_lambda::Route; + use htsget_test::http_tests::default_test_config; + use http::uri::Authority; + use lambda_http::request::from_str; + + use htsget_elsa::elsa_endpoint::ElsaEndpoint; + use htsget_elsa::s3::S3; + use htsget_elsa::test_utils::{is_manifest_resolvers, with_test_mocks}; + + use crate::config::Config; + use crate::get_resolvers; + + #[tokio::test] + async fn test_route_request() { + with_test_mocks( + |endpoint, s3_client, reqwest_client, _| async move { + let config = Config::new( + default_test_config(), + Authority::from_str(&endpoint).unwrap(), + "cache".to_string(), + ); + + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let endpoint = ElsaEndpoint::new_with_client( + reqwest_client, + config.elsa_endpoint_authority().clone(), + &s3, + &s3, + "http", + ); + + let response = + resolvers_from_endpoint(&config, &endpoint, "data/events/event_get.json").await; + + assert!(!is_manifest_resolvers(response)); + }, + 0, + ) + .await; + } + + #[tokio::test] + async fn route_request_elsa_endpoint() { + with_test_mocks( + |endpoint, s3_client, reqwest_client, _| async move { + let config = Config::new( + default_test_config(), + Authority::from_str(&endpoint).unwrap(), + "cache".to_string(), + ); + + let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let endpoint = ElsaEndpoint::new_with_client( + reqwest_client, + config.elsa_endpoint_authority().clone(), + &s3, + &s3, + "http", + ); + + let response = + resolvers_from_endpoint(&config, &endpoint, "data/events/event_elsa.json") + .await; + + assert!(is_manifest_resolvers(response)); + }, + 1, + ) + .await; + } + + async fn resolvers_from_endpoint<'a>( + config: &Config, + endpoint: &ElsaEndpoint<'a, S3, S3>, + path: &str, + ) -> Vec { + let path = PathBuf::from_str(env!("CARGO_MANIFEST_DIR")) + .unwrap() + .parent() + .unwrap() + .join(path); + let event = fs::read_to_string(path).unwrap(); + let event = from_str(&event).unwrap(); + + let route = Route::try_from(&event).unwrap(); + + get_resolvers(config, &route, endpoint).await.unwrap() + } +} diff --git a/htsget-elsa/Cargo.toml b/htsget-elsa/Cargo.toml index 5b5e9de..d7cfa61 100644 --- a/htsget-elsa/Cargo.toml +++ b/htsget-elsa/Cargo.toml @@ -1,12 +1,18 @@ [package] name = "htsget-elsa" +authors = ["Marko Malenic "] version = "0.1.0" edition = "2021" +[features] + +test-utils = ["htsget-test", "wiremock"] +default = [] + [dependencies] async-trait = "0.1" http = "0.2" -reqwest = { version = "0.11", features = ["rustls-tls", "json"] } +reqwest = { version = "0.11", default-features = false, features = ["rustls-tls", "json"] } thiserror = "1.0" aws-config = "0.54" aws-sdk-s3 = "0.24" @@ -18,8 +24,10 @@ tracing = "0.1" htsget-config = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } -[dev-dependencies] -htsget-test = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa", features = ["aws-mocks"] } +htsget-test = { git = "https://github.com/umccr/htsget-rs.git", optional = true, branch = "feat/htsget-elsa", features = ["aws-mocks"] } +wiremock = { version = "0.5", optional = true } +[dev-dependencies] tokio = { version = "1.25", features = ["macros", "rt-multi-thread"] } -wiremock = "0.5" \ No newline at end of file + +htsget-elsa = { path = ".", features = ["test-utils"] } \ No newline at end of file diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index 2f91235..ff703b2 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -198,7 +198,8 @@ where )) } - fn new_with_client( + #[cfg(feature = "test-utils")] + pub fn new_with_client( client: Client, endpoint: Authority, cache: &'a C, @@ -272,7 +273,6 @@ mod tests { use std::str::FromStr; use htsget_config::resolver::Resolver; - use htsget_config::storage; use htsget_config::types::Format; use http::uri::Authority; use serde_json::from_str; @@ -281,7 +281,10 @@ mod tests { ElsaEndpoint, ElsaLocation, ElsaManifest, ElsaResponse, CACHE_PATH, }; use crate::s3::S3; - use crate::tests::{example_elsa_manifest, example_elsa_response, with_test_mocks}; + use crate::test_utils::{ + example_elsa_manifest, example_elsa_response, is_manifest_resolvers, + is_resolver_from_parts, with_test_mocks, + }; use crate::Error::{GetObjectError, InvalidManifest, UnsupportedManifestFeature}; use crate::{Cache, ResolversFromElsa}; @@ -405,7 +408,7 @@ mod tests { .exists()); let resolvers = endpoint.try_get("R004".to_string()).await.unwrap(); - assert_manifest_resolvers(resolvers); + assert!(is_manifest_resolvers(resolvers)); assert!(base_path .join(format!("elsa-data-tmp/{CACHE_PATH}/R004")) .exists()); @@ -419,7 +422,7 @@ mod tests { fn resolvers_from_manifest() { let manifest: ElsaManifest = from_str(&example_elsa_manifest()).unwrap(); let resolvers: Vec = manifest.try_into().unwrap(); - assert_manifest_resolvers(resolvers); + assert!(is_manifest_resolvers(resolvers)); } #[test] @@ -490,33 +493,4 @@ mod tests { ); assert!(matches!(response, Err(InvalidManifest(_)))); } - - fn is_resolver_from_parts(resolver: &Resolver) -> bool { - resolver.regex().to_string() == "^R004/30F9F3FED8F711ED8C35DBEF59E9F537$" - && resolver.substitution_string() == "HG00097/HG00097" - && matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") - && resolver.allow_formats() == [Format::Bam] - } - - fn assert_manifest_resolvers(resolvers: Vec) { - assert!(resolvers.iter().any(|resolver| { - resolver.regex().to_string() == "^R004/30F9FFD4D8F711ED8C353BBCB8861211$" && - resolver.substitution_string() == "HG00096/HG00096" && - matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && - resolver.allow_formats() == [Format::Bam] - })); - assert!(resolvers.iter().any(is_resolver_from_parts)); - assert!(resolvers.iter().any(|resolver| { - resolver.regex().to_string() == "^R004/30F9FFD4D8F711ED8C353BBCB8861211$" && - resolver.substitution_string() == "HG00096/HG00096.hard-filtered" && - matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && - resolver.allow_formats() == [Format::Vcf] - })); - assert!(resolvers.iter().any(|resolver| { - resolver.regex().to_string() == "^R004/30F9F3FED8F711ED8C35DBEF59E9F537$" && - resolver.substitution_string() == "HG00097/HG00097.hard-filtered" && - matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && - resolver.allow_formats() == [Format::Vcf] - })); - } } diff --git a/htsget-elsa/src/lib.rs b/htsget-elsa/src/lib.rs index a39507d..452704b 100644 --- a/htsget-elsa/src/lib.rs +++ b/htsget-elsa/src/lib.rs @@ -7,6 +7,8 @@ use thiserror::Error; pub mod elsa_endpoint; pub mod s3; +#[cfg(feature = "test-utils")] +pub mod test_utils; pub type Result = result::Result; @@ -68,127 +70,3 @@ pub trait ResolversFromElsa { async fn try_get(&self, release_key: String) -> result::Result, Self::Error>; } - -#[cfg(test)] -pub(crate) mod tests { - use std::fs; - use std::future::Future; - use std::path::{Path, PathBuf}; - - use crate::elsa_endpoint::ENDPOINT_PATH; - use aws_sdk_s3::Client; - use htsget_test::aws_mocks::with_s3_test_server_tmp; - use wiremock::matchers::{method, path, query_param}; - use wiremock::{Mock, MockServer, Request, ResponseTemplate, Times}; - - pub(crate) fn example_elsa_manifest() -> String { - r#" - { - "id": "R004", - "reads": { - "30F9F3FED8F711ED8C35DBEF59E9F537": { - "url": "s3://umccr-10g-data-dev/HG00097/HG00097.bam" - }, - "30F9FFD4D8F711ED8C353BBCB8861211": { - "url": "s3://umccr-10g-data-dev/HG00096/HG00096.bam" - } - }, - "variants": { - "30F9F3FED8F711ED8C35DBEF59E9F537": { - "url": "s3://umccr-10g-data-dev/HG00097/HG00097.hard-filtered.vcf.gz", - "variantSampleId": "" - }, - "30F9FFD4D8F711ED8C353BBCB8861211": { - "url": "s3://umccr-10g-data-dev/HG00096/HG00096.hard-filtered.vcf.gz", - "variantSampleId": "" - } - }, - "restrictions": {}, - "cases": [ - { - "ids": { "": "SINGLETONCHARLES" }, - "patients": [ - { - "ids": { "": "CHARLES" }, - "specimens": [ - { - "htsgetId": "30F9FFD4D8F711ED8C353BBCB8861211", - "ids": { "": "HG00096" } - } - ] - } - ] - }, - { - "ids": { "": "SINGLETONMARY" }, - "patients": [ - { - "ids": { "": "MARY" }, - "specimens": [ - { - "htsgetId": "30F9F3FED8F711ED8C35DBEF59E9F537", - "ids": { "": "HG00097" } - } - ] - } - ] - } - ] - } - "# - .to_string() - } - - pub(crate) fn example_elsa_response() -> String { - r#" - { - "location": { - "bucket": "elsa-data-tmp", - "key": "htsget-manifests/R004" - }, - "maxAge": 86400 - } - "# - .to_string() - } - - pub(crate) fn write_example_manifest(manifest_path: &Path) { - fs::create_dir_all(manifest_path).unwrap(); - fs::write(manifest_path.join("R004"), example_elsa_manifest()).unwrap(); - } - - pub(crate) async fn with_test_mocks(test: F, expect_times: T) - where - T: Into, - F: FnOnce(String, Client, reqwest::Client, PathBuf) -> Fut, - Fut: Future, - { - with_s3_test_server_tmp(|client, server_base_path| async move { - let mock_server = MockServer::start().await; - - let base_path = server_base_path.clone(); - Mock::given(method("GET")) - .and(path(format!("{ENDPOINT_PATH}/R004"))) - .and(query_param("type", "S3")) - .respond_with(move |_: &Request| { - let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); - - write_example_manifest(&manifest_path); - - ResponseTemplate::new(200).set_body_string(example_elsa_response()) - }) - .expect(expect_times) - .mount(&mock_server) - .await; - - test( - mock_server.address().to_string(), - client, - reqwest::Client::builder().build().unwrap(), - server_base_path, - ) - .await; - }) - .await; - } -} diff --git a/htsget-elsa/src/s3.rs b/htsget-elsa/src/s3.rs index 8c3cd72..752e723 100644 --- a/htsget-elsa/src/s3.rs +++ b/htsget-elsa/src/s3.rs @@ -145,12 +145,13 @@ impl Cache for S3 { #[cfg(test)] mod tests { - use serde_json::{from_str, to_string}; use std::fs; + use serde_json::{from_str, to_string}; + use crate::elsa_endpoint::ElsaManifest; use crate::s3::{CacheItem, S3}; - use crate::tests::{example_elsa_manifest, with_test_mocks, write_example_manifest}; + use crate::test_utils::{example_elsa_manifest, with_test_mocks, write_example_manifest}; use crate::Cache; #[tokio::test] diff --git a/htsget-elsa/src/test_utils.rs b/htsget-elsa/src/test_utils.rs new file mode 100644 index 0000000..a093ca6 --- /dev/null +++ b/htsget-elsa/src/test_utils.rs @@ -0,0 +1,152 @@ +use std::fs; +use std::future::Future; +use std::path::{Path, PathBuf}; + +use crate::elsa_endpoint::ENDPOINT_PATH; +use aws_sdk_s3::Client; +use htsget_config::resolver::Resolver; +use htsget_config::storage; +use htsget_config::types::Format; +use htsget_test::aws_mocks::with_s3_test_server_tmp; +use wiremock::matchers::{method, path, query_param}; +use wiremock::{Mock, MockServer, Request, ResponseTemplate, Times}; + +pub fn example_elsa_manifest() -> String { + r#" + { + "id": "R004", + "reads": { + "30F9F3FED8F711ED8C35DBEF59E9F537": { + "url": "s3://umccr-10g-data-dev/HG00097/HG00097.bam" + }, + "30F9FFD4D8F711ED8C353BBCB8861211": { + "url": "s3://umccr-10g-data-dev/HG00096/HG00096.bam" + } + }, + "variants": { + "30F9F3FED8F711ED8C35DBEF59E9F537": { + "url": "s3://umccr-10g-data-dev/HG00097/HG00097.hard-filtered.vcf.gz", + "variantSampleId": "" + }, + "30F9FFD4D8F711ED8C353BBCB8861211": { + "url": "s3://umccr-10g-data-dev/HG00096/HG00096.hard-filtered.vcf.gz", + "variantSampleId": "" + } + }, + "restrictions": {}, + "cases": [ + { + "ids": { "": "SINGLETONCHARLES" }, + "patients": [ + { + "ids": { "": "CHARLES" }, + "specimens": [ + { + "htsgetId": "30F9FFD4D8F711ED8C353BBCB8861211", + "ids": { "": "HG00096" } + } + ] + } + ] + }, + { + "ids": { "": "SINGLETONMARY" }, + "patients": [ + { + "ids": { "": "MARY" }, + "specimens": [ + { + "htsgetId": "30F9F3FED8F711ED8C35DBEF59E9F537", + "ids": { "": "HG00097" } + } + ] + } + ] + } + ] + } + "# + .to_string() +} + +pub(crate) fn example_elsa_response() -> String { + r#" + { + "location": { + "bucket": "elsa-data-tmp", + "key": "htsget-manifests/R004" + }, + "maxAge": 86400 + } + "# + .to_string() +} + +pub fn write_example_manifest(manifest_path: &Path) { + fs::create_dir_all(manifest_path).unwrap(); + fs::write(manifest_path.join("R004"), example_elsa_manifest()).unwrap(); +} + +pub async fn with_test_mocks(test: F, expect_times: T) +where + T: Into, + F: FnOnce(String, Client, reqwest::Client, PathBuf) -> Fut, + Fut: Future, +{ + with_s3_test_server_tmp(|client, server_base_path| async move { + let mock_server = MockServer::start().await; + + let base_path = server_base_path.clone(); + Mock::given(method("GET")) + .and(path(format!("{ENDPOINT_PATH}/R004"))) + .and(query_param("type", "S3")) + .respond_with(move |_: &Request| { + let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); + + write_example_manifest(&manifest_path); + + ResponseTemplate::new(200).set_body_string(example_elsa_response()) + }) + .expect(expect_times) + .mount(&mock_server) + .await; + + test( + mock_server.address().to_string(), + client, + reqwest::Client::builder().build().unwrap(), + server_base_path, + ) + .await; + }) + .await; +} + +pub fn is_resolver_from_parts(resolver: &Resolver) -> bool { + resolver.regex().to_string() == "^R004/30F9F3FED8F711ED8C35DBEF59E9F537$" + && resolver.substitution_string() == "HG00097/HG00097" + && matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") + && resolver.allow_formats() == [Format::Bam] +} + +pub fn is_manifest_resolvers(resolvers: Vec) -> bool { + resolvers.iter().any(|resolver| { + resolver.regex().to_string() == "^R004/30F9FFD4D8F711ED8C353BBCB8861211$" && + resolver.substitution_string() == "HG00096/HG00096" && + matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && + resolver.allow_formats() == [Format::Bam] + }) && + resolvers.iter().any(is_resolver_from_parts) && + resolvers.iter().any(|resolver| { + resolver.regex().to_string() == "^R004/30F9FFD4D8F711ED8C353BBCB8861211$" && + resolver.substitution_string() == "HG00096/HG00096.hard-filtered" && + matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && + resolver.allow_formats() == [Format::Vcf] + }) && + resolvers.iter().any(|resolver| { + resolver.regex().to_string() == "^R004/30F9F3FED8F711ED8C35DBEF59E9F537$" && + resolver.substitution_string() == "HG00097/HG00097.hard-filtered" && + matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && + resolver.allow_formats() == [Format::Vcf] + }) +} From 52f1889aaebc9bfb20398bdc558aa1fe161b2e78 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Tue, 18 Apr 2023 15:49:22 +1000 Subject: [PATCH 16/25] wip: deployment --- deploy/.gitignore | 7 + deploy/.npmignore | 6 + deploy/.prettierrc.json | 1 + deploy/README.md | 164 +++ deploy/bin/htsget-elsa-lambda.ts | 21 + deploy/cdk.json | 44 + deploy/config/config_dev_umccr.toml | 49 + deploy/lib/htsget-elsa-lambda-stack.ts | 288 ++++ deploy/package-lock.json | 1698 ++++++++++++++++++++++++ deploy/package.json | 28 + deploy/tsconfig.json | 23 + 11 files changed, 2329 insertions(+) create mode 100644 deploy/.gitignore create mode 100644 deploy/.npmignore create mode 100644 deploy/.prettierrc.json create mode 100644 deploy/README.md create mode 100644 deploy/bin/htsget-elsa-lambda.ts create mode 100644 deploy/cdk.json create mode 100644 deploy/config/config_dev_umccr.toml create mode 100644 deploy/lib/htsget-elsa-lambda-stack.ts create mode 100644 deploy/package-lock.json create mode 100644 deploy/package.json create mode 100644 deploy/tsconfig.json diff --git a/deploy/.gitignore b/deploy/.gitignore new file mode 100644 index 0000000..a633556 --- /dev/null +++ b/deploy/.gitignore @@ -0,0 +1,7 @@ +*.js +*.d.ts +node_modules + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/deploy/.npmignore b/deploy/.npmignore new file mode 100644 index 0000000..c1d6d45 --- /dev/null +++ b/deploy/.npmignore @@ -0,0 +1,6 @@ +*.ts +!*.d.ts + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/deploy/.prettierrc.json b/deploy/.prettierrc.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/deploy/.prettierrc.json @@ -0,0 +1 @@ +{} diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..bf537b8 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,164 @@ +# Deployment of htsget-lambda + +The [htsget-lambda] crate is a cloud-based implementation of [htsget-rs]. It uses AWS Lambda as the ticket server, and AWS S3 as the data block server. + +This is an example that deploys [htsget-lambda] using [aws-cdk]. It is deployed as an AWS HTTP [API Gateway Lambda proxy +integration][aws-api-gateway]. The stack uses [RustFunction][rust-function] in order to integrate [htsget-lambda] +with API Gateway. It uses a [JWT authorizer][jwt-authorizer] with [AWS Cognito][aws-cognito] as the issuer, and routes +the htsget-rs server with [AWS Route 53][route-53]. + +## Configuration + +To configure the deployment change the config files in the [`config`][config] directory. There are two configuration files +corresponding to the deployed environment. The config file used for deployment can be controlled by passing `--context "env=dev"` or +`--context "env=prod"` to cdk. When no context parameter is supplied, the default context is `dev`. + +These config files configure [htsget-lambda]. See [htsget-config] for a list of available configuration options. + +There is additional config that modifies the AWS infrastructure surrounding [htsget-lambda]. This config is sourced +from [AWS System Manager Parameter Store][aws-ssm]. Parameters are fetched using [`StringParemeter.valueFromLookup`][cdk-lookup-value] +which takes a parameter name. The properties in [`cdk.json`][cdk-json] `parameter_store_names` hold these parameter names, +which correspond to the following values: + +| Property storing SSM parameter name | Description of SSM parameter | +| ----------------------------------- | ------------------------------------------------------ | +| `arn_cert` | The ARN for the ACM certificate of the htsget domain. | +| `jwt_aud` | The JWT [audience][jwt-audience] for the token. | +| `cog_user_pool_id` | The Cognito user pool id which controls authorization. | +| `htsget_domain` | The domain name for the htsget server. | +| `hosted_zone_id` | The Route 53 hosted zone id for the htsget server. | +| `hosted_zone_name` | The Route 53 hosted zone name for the htsget server. | + +Modify these properties to change which parameter the values come from. SSM parameters can also be cached locally using +the [CDK runtime context][cdk-context]. + +[htsget-rs]: https://github.com/umccr/htsget-rs +[htsget-lambda]: ../htsget-lambda +[htsget-config]: ../htsget-config +[config]: config +[aws-cdk]: https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html +[cdk-context]: https://docs.aws.amazon.com/cdk/v2/guide/context.html +[cdk-lookup-value]: https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ssm.StringParameter.html#static-valuewbrfromwbrlookupscope-parametername +[cdk-json]: cdk.json +[aws-ssm]: https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html +[aws-api-gateway]: https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html +[aws-cognito]: https://docs.aws.amazon.com/cognito/latest/developerguide/cognito-user-identity-pools.html +[jwt-authorizer]: https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-jwt-authorizer.html +[jwt-audience]: https://docs.aws.amazon.com/apigatewayv2/latest/api-reference/apis-apiid-authorizers-authorizerid.html#apis-apiid-authorizers-authorizerid-model-jwtconfiguration +[route-53]: https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/Welcome.html +[rust-function]: https://www.npmjs.com/package/rust.aws-cdk-lambda + +## Deploying + +### Prerequisites + +- [aws-cli] should be installed and authenticated in the shell. +- Node.js and [npm] should be installed. +- [Rust][rust] should be installed. + +After installing the basic dependencies, complete the following steps: + +1. Add the arm cross-compilation target to rust. +2. Install [Zig][zig] using one of the methods show in [getting started][zig-getting-started], or by running the commands below and following the prompts. Zig is used by cargo-lambda for cross-compilation. +3. Install [cargo-lambda], as it is used to compile artifacts that are uploaded to aws lambda. +4. Install packages from this directory and compile [htsget-lambda]. This should place artifacts compiled for arm64 under the `target/lambda` directory which can be deployed to AWS. + +Below is a summary of commands to run in this directory: + +```sh +rustup target add aarch64-unknown-linux-gnu +cargo install cargo-lambda +npm install + +cd .. +cargo lambda build --release --arm64 --bin htsget-lambda +cd deploy +``` + +[aws-cdk]: https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html +[aws-cli]: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html +[npm]: https://docs.npmjs.com/downloading-and-installing-node-js-and-npm +[rust]: https://www.rust-lang.org/tools/install +[zig]: https://ziglang.org/ +[zig-getting-started]: https://ziglang.org/learn/getting-started/ + +### Deploy to AWS + +CDK will run many of the commands above again. However, it is recommended to run them once before trying the commands below, +to ensure that prerequisites are met. + +CDK should be bootstrapped once, if this hasn't been done before. + +```sh +npx cdk bootstrap +``` + +In order to deploy, check that the +stack synthesizes correctly and then deploy. + +```sh +npx cdk synth +npx cdk deploy +``` + +### Testing the endpoint + +When the deployment is finished, the htsget endpoint can be tested by querying it. Since a JWT authorizer is used, +a valid JWT token must be obtained in order to access the endpoint. This token should be obtained from AWS Cognito using +the configured user pool id and audience parameters. Then `curl` can be used to query the endpoint: + +```sh +curl -H "Authorization: " "https:///reads/service-info" +``` + +With a possible output: + +```json +{ + "id": "", + "name": "", + "version": "", + "organization": { + "name": "", + "url": "" + }, + "type": { + "group": "", + "artifact": "", + "version": "" + }, + "htsget": { + "datatype": "reads", + "formats": ["BAM", "CRAM"], + "fieldsParametersEffective": false, + "TagsParametersEffective": false + }, + "contactUrl": "", + "documentationUrl": "", + "createdAt": "", + "UpdatedAt": "", + "environment": "" +} +``` + +[awscurl]: https://github.com/okigan/awscurl + +### Local testing + +The [Lambda][htsget-lambda] function can also be run locally using [cargo-lambda]. From the root project directory, execute the following command. + +```sh +cargo lambda watch +``` + +Then in a **separate terminal session** run. + +```sh +cargo lambda invoke htsget-lambda --data-file data/events/event_elsa.json +``` + +Examples of different Lambda events are located in the [`data/events`][data-events] directory. + +[htsget-lambda]: ../htsget-lambda +[cargo-lambda]: https://github.com/cargo-lambda/cargo-lambda +[data-events]: ../data/events diff --git a/deploy/bin/htsget-elsa-lambda.ts b/deploy/bin/htsget-elsa-lambda.ts new file mode 100644 index 0000000..5c13209 --- /dev/null +++ b/deploy/bin/htsget-elsa-lambda.ts @@ -0,0 +1,21 @@ +#!/usr/bin/env node +import "source-map-support/register"; +import * as cdk from "aws-cdk-lib"; +import { HtsgetElsaLambdaStack } from "../lib/htsget-elsa-lambda-stack"; + +export const STACK_NAME = "HtsgetElsaLambdaStack"; +const STACK_DESCRIPTION = + "A stack deploying htsget-elsa-lambda with API gateway."; + +const app = new cdk.App(); +new HtsgetElsaLambdaStack(app, STACK_NAME, { + stackName: STACK_NAME, + description: STACK_DESCRIPTION, + tags: { + Stack: STACK_NAME, + }, + env: { + account: process.env.CDK_DEFAULT_ACCOUNT, + region: process.env.CDK_DEFAULT_REGION, + }, +}); diff --git a/deploy/cdk.json b/deploy/cdk.json new file mode 100644 index 0000000..01bae12 --- /dev/null +++ b/deploy/cdk.json @@ -0,0 +1,44 @@ +{ + "app": "npx ts-node --prefer-ts-exts bin/htsget-elsa-lambda.ts", + "watch": { + "include": ["**"], + "exclude": [ + "README.md", + "cdk*.json", + "**/*.d.ts", + "**/*.js", + "tsconfig.json", + "package*.json", + "yarn.lock", + "node_modules", + "test" + ] + }, + "context": { + "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, + "@aws-cdk/core:stackRelativeExports": true, + "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, + "@aws-cdk/aws-lambda:recognizeVersionProps": true, + "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, + "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, + "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/core:target-partitions": ["aws", "aws-cn"], + "dev": { + "config": "config/config_dev_umccr.toml", + "parameter_store_names": { + "arn_cert": "/htsget/acm/apse2_arn", + "jwt_aud": [ + "/data_portal/client/cog_app_client_id_local", + "/data_portal/client/cog_app_client_id_stage", + "/data_portal/client/data2/cog_app_client_id_stage" + ], + "cog_user_pool_id": "/data_portal/client/cog_user_pool_id", + "htsget_domain": "/htsget/elsa/domain", + "hosted_zone_id": "hosted_zone_id", + "hosted_zone_name": "hosted_zone_name" + } + } + } +} diff --git a/deploy/config/config_dev_umccr.toml b/deploy/config/config_dev_umccr.toml new file mode 100644 index 0000000..4d2a512 --- /dev/null +++ b/deploy/config/config_dev_umccr.toml @@ -0,0 +1,49 @@ +elsa_endpoint_authority = "elsa.dev.umccr.org" +cache_location = "elsa-tmp-bucket" + +ticket_server_cors_allow_headers = "All" +ticket_server_cors_allow_origins = [ + "https://elsa.dev.umccr.org", + "https://elsa.umccr.org", +] +ticket_server_cors_allow_methods = "All" +ticket_server_cors_allow_credentials = true +ticket_server_cors_max_age = 300 + +data_server_enabled = false + +name = "umccr-htsget-elsa-rs" +version = "0.1" +organization_name = "UMCCR" +organization_url = "https://umccr.org/" +contact_url = "https://umccr.org/" +documentation_url = "https://github.com/umccr/htsget-elsa-rs" +environment = "dev" + +[[resolvers]] +regex = '^(umccr-10c-data-dev)/(?P.*)$' +substitution_string = '$key' + +[[resolvers]] +regex = '^(umccr-10f-data-dev)/(?P.*)$' +substitution_string = '$key' + +[[resolvers]] +regex = '^(umccr-10g-data-dev)/(?P.*)$' +substitution_string = '$key' + +[[resolvers]] +regex = '^(umccr-agha-test-dev)/(?P.*)$' +substitution_string = '$key' + +[[resolvers]] +regex = '^(umccr-research-dev)/(?P.*)$' +substitution_string = '$key' + +[[resolvers]] +regex = '^(umccr-primary-data-dev)/(?P.*)$' +substitution_string = '$key' + +[[resolvers]] +regex = '^(umccr-validation-prod)/(?P.*)$' +substitution_string = '$key' diff --git a/deploy/lib/htsget-elsa-lambda-stack.ts b/deploy/lib/htsget-elsa-lambda-stack.ts new file mode 100644 index 0000000..b28c1c5 --- /dev/null +++ b/deploy/lib/htsget-elsa-lambda-stack.ts @@ -0,0 +1,288 @@ +import { Duration, Stack, StackProps, Tags } from "aws-cdk-lib"; +import { Construct } from "constructs"; +import * as iam from "aws-cdk-lib/aws-iam"; +import { RustFunction, Settings } from "rust.aws-cdk-lambda"; +import { Architecture } from "aws-cdk-lib/aws-lambda"; +import * as apigwv2 from "@aws-cdk/aws-apigatewayv2-alpha"; +import { STACK_NAME } from "../bin/htsget-elsa-lambda"; +import { HttpLambdaIntegration } from "@aws-cdk/aws-apigatewayv2-integrations-alpha"; +import { StringParameter } from "aws-cdk-lib/aws-ssm"; +import { HttpJwtAuthorizer } from "@aws-cdk/aws-apigatewayv2-authorizers-alpha"; +import { ARecord, HostedZone, RecordTarget } from "aws-cdk-lib/aws-route53"; +import { ApiGatewayv2DomainProperties } from "aws-cdk-lib/aws-route53-targets"; +import { Certificate } from "aws-cdk-lib/aws-certificatemanager"; +import * as fs from "fs"; +import * as TOML from "@iarna/toml"; + +/** + * Configuration for HtsgetLambdaStack. + */ +export type Config = { + environment: string; + htsgetConfig: { [key: string]: any }; + allowCredentials?: boolean; + allowHeaders?: string[]; + allowMethods?: apigwv2.CorsHttpMethod[]; + allowOrigins?: string[]; + exposeHeaders?: string[]; + maxAge?: Duration; + parameterStoreConfig: ParameterStoreConfig; +}; + +/** + * Configuration values obtained from AWS System Manager Parameter Store. + */ +export type ParameterStoreConfig = { + arnCert: string; + hostedZoneId: string; + hostedZoneName: string; + htsgetDomain: string; + cogUserPoolId: string; + jwtAud: string[]; +}; + +/** + * Stack used to deploy htsget-lambda. + */ +export class HtsgetElsaLambdaStack extends Stack { + constructor(scope: Construct, id: string, props?: StackProps) { + super(scope, id, props); + + Tags.of(this).add("Stack", STACK_NAME); + + const lambdaRole = new iam.Role(this, id + "Role", { + assumedBy: new iam.ServicePrincipal("lambda.amazonaws.com"), + description: "Lambda execution role for " + id, + }); + + const s3BucketPolicy = new iam.PolicyStatement({ + actions: ["s3:List*", "s3:Get*", "s3:PutObject"], + resources: ["arn:aws:s3:::*"], + }); + + lambdaRole.addManagedPolicy( + iam.ManagedPolicy.fromAwsManagedPolicyName( + "service-role/AWSLambdaBasicExecutionRole" + ) + ); + lambdaRole.addToPolicy(s3BucketPolicy); + + // Set the workspace directory of htsget. + Settings.WORKSPACE_DIR = "../"; + // Don't build htsget packages other than htsget-elsa-lambda. + Settings.BUILD_INDIVIDUALLY = true; + + const config = this.getConfig(); + let htsgetLambda = new RustFunction(this, id + "Function", { + // Build htsget-elsa-lambda only. + package: "htsget-elsa-lambda", + target: "aarch64-unknown-linux-gnu", + + memorySize: 128, + timeout: Duration.seconds(28), + environment: { + ...config.htsgetConfig, + RUST_LOG: + "info,htsget_http_lambda=trace,htsget_config=trace,htsget_http_core=trace,htsget_search=trace", + }, + buildEnvironment: { + RUSTFLAGS: "-C target-cpu=neoverse-n1", + CARGO_PROFILE_RELEASE_LTO: "true", + CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1", + }, + architecture: Architecture.ARM_64, + role: lambdaRole, + }); + + const parameterStoreConfig = config.parameterStoreConfig; + const httpIntegration = new HttpLambdaIntegration( + id + "HtsgetElsaIntegration", + htsgetLambda + ); + const authorizer = new HttpJwtAuthorizer( + id + "HtsgetElsaAuthorizer", + `https://cognito-idp.${this.region}.amazonaws.com/${parameterStoreConfig.cogUserPoolId}`, + { + identitySource: ["$request.header.Authorization"], + jwtAudience: parameterStoreConfig.jwtAud, + } + ); + + const domainName = new apigwv2.DomainName( + this, + id + "HtsgetElsaDomainName", + { + certificate: Certificate.fromCertificateArn( + this, + id + "HtsgetElsaDomainCert", + parameterStoreConfig.arnCert + ), + domainName: parameterStoreConfig.htsgetDomain, + } + ); + const hostedZone = HostedZone.fromHostedZoneAttributes( + this, + id + "HtsgetElsaHostedZone", + { + hostedZoneId: parameterStoreConfig.hostedZoneId, + zoneName: parameterStoreConfig.hostedZoneName, + } + ); + new ARecord(this, id + "HtsgetElsaARecord", { + zone: hostedZone, + recordName: "htsget.elsa", + target: RecordTarget.fromAlias( + new ApiGatewayv2DomainProperties( + domainName.regionalDomainName, + domainName.regionalHostedZoneId + ) + ), + }); + + const httpApi = new apigwv2.HttpApi(this, id + "ApiGw", { + // Use explicit routes GET, POST with {proxy+} path + // defaultIntegration: httpIntegration, + defaultAuthorizer: authorizer, + defaultDomainMapping: { + domainName: domainName, + }, + corsPreflight: { + allowCredentials: config.allowCredentials, + allowHeaders: config.allowHeaders, + allowMethods: config.allowMethods, + allowOrigins: config.allowOrigins, + exposeHeaders: config.exposeHeaders, + maxAge: config.maxAge, + }, + }); + + httpApi.addRoutes({ + path: "/{proxy+}", + methods: [apigwv2.HttpMethod.GET, apigwv2.HttpMethod.POST], + integration: httpIntegration, + }); + } + + /** + * Get config values from the Parameter Store. + */ + getParameterStoreConfig(config: any): ParameterStoreConfig { + const parameterStoreNames = config.parameter_store_names; + return { + arnCert: StringParameter.valueFromLookup( + this, + parameterStoreNames.arn_cert + ), + jwtAud: parameterStoreNames.jwt_aud.map((jwtAud: string) => + StringParameter.valueFromLookup(this, jwtAud) + ), + cogUserPoolId: StringParameter.valueFromLookup( + this, + parameterStoreNames.cog_user_pool_id + ), + htsgetDomain: StringParameter.valueFromLookup( + this, + parameterStoreNames.htsget_domain + ), + hostedZoneId: StringParameter.valueFromLookup( + this, + parameterStoreNames.hosted_zone_id + ), + hostedZoneName: StringParameter.valueFromLookup( + this, + parameterStoreNames.hosted_zone_name + ), + }; + } + + /** + * Convert JSON config to htsget-rs env representation. + */ + static configToEnv(config: any): { [key: string]: string } { + const out: { [key: string]: string } = {}; + for (const key in config) { + out[`HTSGET_${key.toUpperCase()}`] = TOML.stringify.value(config[key]); + } + return out; + } + + /** + * Convert htsget-rs CORS option to CORS options for API Gateway. + */ + static convertCors(configToml: any, corsValue: string): string[] | undefined { + const value = configToml[corsValue]; + + if ( + value !== undefined && + (value.toString().toLowerCase() === "all" || + value.toString().toLowerCase() === "mirror") + ) { + return ["*"]; + } else if (Array.isArray(value)) { + return value; + } + + return undefined; + } + + /** + * Convert a string CORS allowMethod option to CorsHttpMethod. + */ + static corsAllowMethodToHttpMethod( + corsAllowMethod?: string[] + ): apigwv2.CorsHttpMethod[] | undefined { + if (corsAllowMethod?.length === 1 && corsAllowMethod.includes("*")) { + return [apigwv2.CorsHttpMethod.ANY]; + } else { + return corsAllowMethod?.map( + (element) => + apigwv2.CorsHttpMethod[element as keyof typeof apigwv2.CorsHttpMethod] + ); + } + } + + /** + * Get the environment configuration from cdk.json. Pass `--context "env=dev"` or `--context "env=prod"` to + * control the environment. + */ + getConfig(): Config { + let env: string = this.node.tryGetContext("env"); + if (!env) { + console.log("No environment supplied, using `dev` environment config"); + env = "dev"; + } + + const config = this.node.tryGetContext(env); + const configToml = TOML.parse(fs.readFileSync(config.config).toString()); + + return { + environment: env, + htsgetConfig: HtsgetElsaLambdaStack.configToEnv(configToml), + allowCredentials: + configToml.ticket_server_cors_allow_credentials as boolean, + allowHeaders: HtsgetElsaLambdaStack.convertCors( + configToml, + "ticket_server_cors_allow_headers" + ), + allowMethods: HtsgetElsaLambdaStack.corsAllowMethodToHttpMethod( + HtsgetElsaLambdaStack.convertCors( + configToml, + "ticket_server_cors_allow_methods" + ) + ), + allowOrigins: HtsgetElsaLambdaStack.convertCors( + configToml, + "ticket_server_cors_allow_origins" + ), + exposeHeaders: HtsgetElsaLambdaStack.convertCors( + configToml, + "ticket_server_cors_expose_headers" + ), + maxAge: + configToml.ticket_server_cors_max_age !== undefined + ? Duration.seconds(configToml.ticket_server_cors_max_age as number) + : undefined, + parameterStoreConfig: this.getParameterStoreConfig(config), + }; + } +} diff --git a/deploy/package-lock.json b/deploy/package-lock.json new file mode 100644 index 0000000..b008d0a --- /dev/null +++ b/deploy/package-lock.json @@ -0,0 +1,1698 @@ +{ + "name": "htsget-lambda", + "version": "0.1", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "htsget-lambda", + "version": "0.1", + "dependencies": { + "@aws-cdk/aws-apigatewayv2-alpha": "^2.59.0-alpha.0", + "@aws-cdk/aws-apigatewayv2-authorizers-alpha": "^2.59.0-alpha.0", + "@aws-cdk/aws-apigatewayv2-integrations-alpha": "^2.59.0-alpha.0", + "@iarna/toml": "^3.0.0", + "aws-cdk-lib": "^2.64.0", + "constructs": "^10.1.148", + "rust.aws-cdk-lambda": "^1.1.3", + "source-map-support": "^0.5.21" + }, + "bin": { + "htsget_app": "bin/htsget-lambda.js" + }, + "devDependencies": { + "@types/node": "^18.11.18", + "aws-cdk": "^2.64.0", + "prettier": "2.8.3", + "typescript": "^4.9.4" + } + }, + "node_modules/@aws-cdk/asset-awscli-v1": { + "version": "2.2.63", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.63.tgz", + "integrity": "sha512-LP+RSwH6JCx2T/AJyaveRg8VRvrM612TxGnec8SVuf9pQ6ah82LcW+4abuMhizeXDcx2ZufwNf9g/G2rp47bjA==" + }, + "node_modules/@aws-cdk/asset-kubectl-v20": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.1.tgz", + "integrity": "sha512-U1ntiX8XiMRRRH5J1IdC+1t5CE89015cwyt5U63Cpk0GnMlN5+h9WsWMlKlPXZR4rdq/m806JRlBMRpBUB2Dhw==" + }, + "node_modules/@aws-cdk/asset-node-proxy-agent-v5": { + "version": "2.0.52", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v5/-/asset-node-proxy-agent-v5-2.0.52.tgz", + "integrity": "sha512-sQhGP8QhAeP3ZsXxoOQrRFW/0VMH3HvI8NjZe/9ORtKwh8Gyjv2BJWIxlXLvRcc2o26FEl1oHIKRbaK/z/4iMw==" + }, + "node_modules/@aws-cdk/aws-apigatewayv2-alpha": { + "version": "2.59.0-alpha.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigatewayv2-alpha/-/aws-apigatewayv2-alpha-2.59.0-alpha.0.tgz", + "integrity": "sha512-dChzTZt8Otm/5iTm76qHs/1xa5PXZyS3G7s8WyaWhH8plgZGeC/q6LdYvJ/FkuDYnD0a3Yp1t6wqJAxb9R0zxA==", + "engines": { + "node": ">= 14.15.0" + }, + "peerDependencies": { + "aws-cdk-lib": "^2.59.0", + "constructs": "^10.0.0" + } + }, + "node_modules/@aws-cdk/aws-apigatewayv2-authorizers-alpha": { + "version": "2.59.0-alpha.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigatewayv2-authorizers-alpha/-/aws-apigatewayv2-authorizers-alpha-2.59.0-alpha.0.tgz", + "integrity": "sha512-J6EdrjbU/riYDW18WYnHs7KuUXuuWQwmbQLaAsbh6fVgrc8c8DGG+He3PhETTYScChkmOUXXxz2y45R0E5g7dA==", + "engines": { + "node": ">= 14.15.0" + }, + "peerDependencies": { + "@aws-cdk/aws-apigatewayv2-alpha": "2.59.0-alpha.0", + "aws-cdk-lib": "^2.59.0", + "constructs": "^10.0.0" + } + }, + "node_modules/@aws-cdk/aws-apigatewayv2-integrations-alpha": { + "version": "2.59.0-alpha.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigatewayv2-integrations-alpha/-/aws-apigatewayv2-integrations-alpha-2.59.0-alpha.0.tgz", + "integrity": "sha512-HSGcmRhnqfASOhP7ucX+RrVgaIgLWsdlzWIN+fQlwX2sSAh/K9oQ9ObOWqAhQix48TNlvnr2nr1yfyEeRX/Dbw==", + "engines": { + "node": ">= 14.15.0" + }, + "peerDependencies": { + "@aws-cdk/aws-apigatewayv2-alpha": "2.59.0-alpha.0", + "aws-cdk-lib": "^2.59.0", + "constructs": "^10.0.0" + } + }, + "node_modules/@iarna/toml": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@iarna/toml/-/toml-3.0.0.tgz", + "integrity": "sha512-td6ZUkz2oS3VeleBcN+m//Q6HlCFCPrnI0FZhrt/h4XqLEdOyYp2u21nd8MdsR+WJy5r9PTDaHTDDfhf4H4l6Q==" + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "optional": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "optional": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "optional": true, + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@types/fs-extra": { + "version": "9.0.13", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.13.tgz", + "integrity": "sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA==", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/minimist": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz", + "integrity": "sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==", + "optional": true + }, + "node_modules/@types/node": { + "version": "18.13.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.13.0.tgz", + "integrity": "sha512-gC3TazRzGoOnoKAhUx+Q0t8S9Tzs74z7m0ipwGpSqQrleP14hKxP4/JUeEQcD3W1/aIpnWl8pHowI7WokuZpXg==", + "devOptional": true + }, + "node_modules/@types/ps-tree": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@types/ps-tree/-/ps-tree-1.1.2.tgz", + "integrity": "sha512-ZREFYlpUmPQJ0esjxoG1fMvB2HNaD3z+mjqdSosZvd3RalncI9NEur73P8ZJz4YQdL64CmV1w0RuqoRUlhQRBw==", + "optional": true + }, + "node_modules/@types/which": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@types/which/-/which-2.0.1.tgz", + "integrity": "sha512-Jjakcv8Roqtio6w1gr0D7y6twbhx6gGgFGF5BLwajPpnOIOxFkakFhCq+LmyyeAz7BX6ULrjBOxdKaCDy+4+dQ==", + "optional": true + }, + "node_modules/@ziglang/cli": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/@ziglang/cli/-/cli-0.0.11.tgz", + "integrity": "sha512-PCeyxchKF1nG9tr4sHUJlJiDjSFP0D+YS7jS0qREaAsa0cjdXYCqGXltFrEEbSMgWWkxJr9KnWviJgnHnnSFeg==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "zx": "^6.0.7" + }, + "bin": { + "zig": "bin/zig", + "zig-cli": "zig.js", + "zig-install": "install.js", + "zig-postinstall": "postinstall.js", + "zig-reinstall": "reinstall.js", + "zig-uninstall": "uninstall.js" + } + }, + "node_modules/aws-cdk": { + "version": "2.64.0", + "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.64.0.tgz", + "integrity": "sha512-iXkvVeYKt6Glboeicrb3QxC6K6o25+zitM/UTfgVzDlKEvC4hwQp1KqXy/caN7SfA6X2N0LJmXfC99T4cvIH0A==", + "dev": true, + "bin": { + "cdk": "bin/cdk" + }, + "engines": { + "node": ">= 14.15.0" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/aws-cdk-lib": { + "version": "2.64.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.64.0.tgz", + "integrity": "sha512-IrgL7thb6TeOyHgyR/qKWTdA9FBb9lv7Z9QPDzCNJlkKI+0ANjYHy3RYV8Gd+1+kc6l8DG9Z1elij40YCr/Ptg==", + "bundleDependencies": [ + "@balena/dockerignore", + "case", + "fs-extra", + "ignore", + "jsonschema", + "minimatch", + "punycode", + "semver", + "yaml" + ], + "dependencies": { + "@aws-cdk/asset-awscli-v1": "^2.2.52", + "@aws-cdk/asset-kubectl-v20": "^2.1.1", + "@aws-cdk/asset-node-proxy-agent-v5": "^2.0.42", + "@balena/dockerignore": "^1.0.2", + "case": "1.6.3", + "fs-extra": "^9.1.0", + "ignore": "^5.2.4", + "jsonschema": "^1.4.1", + "minimatch": "^3.1.2", + "punycode": "^2.3.0", + "semver": "^7.3.8", + "yaml": "1.10.2" + }, + "engines": { + "node": ">= 14.15.0" + }, + "peerDependencies": { + "constructs": "^10.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/@balena/dockerignore": { + "version": "1.0.2", + "inBundle": true, + "license": "Apache-2.0" + }, + "node_modules/aws-cdk-lib/node_modules/at-least-node": { + "version": "1.0.0", + "inBundle": true, + "license": "ISC", + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/balanced-match": { + "version": "1.0.2", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/brace-expansion": { + "version": "1.1.11", + "inBundle": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/aws-cdk-lib/node_modules/case": { + "version": "1.6.3", + "inBundle": true, + "license": "(MIT OR GPL-3.0-or-later)", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/concat-map": { + "version": "0.0.1", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/fs-extra": { + "version": "9.1.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/aws-cdk-lib/node_modules/graceful-fs": { + "version": "4.2.10", + "inBundle": true, + "license": "ISC" + }, + "node_modules/aws-cdk-lib/node_modules/ignore": { + "version": "5.2.4", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/aws-cdk-lib/node_modules/jsonfile": { + "version": "6.1.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/jsonschema": { + "version": "1.4.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/aws-cdk-lib/node_modules/lru-cache": { + "version": "6.0.0", + "inBundle": true, + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/aws-cdk-lib/node_modules/minimatch": { + "version": "3.1.2", + "inBundle": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/aws-cdk-lib/node_modules/punycode": { + "version": "2.3.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/aws-cdk-lib/node_modules/semver": { + "version": "7.3.8", + "inBundle": true, + "license": "ISC", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/aws-cdk-lib/node_modules/universalify": { + "version": "2.0.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/yallist": { + "version": "4.0.0", + "inBundle": true, + "license": "ISC" + }, + "node_modules/aws-cdk-lib/node_modules/yaml": { + "version": "1.10.2", + "inBundle": true, + "license": "ISC", + "engines": { + "node": ">= 6" + } + }, + "node_modules/braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "optional": true, + "dependencies": { + "fill-range": "^7.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + }, + "node_modules/chalk": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.2.0.tgz", + "integrity": "sha512-ree3Gqw/nazQAPuJJEy+avdl7QfZMcUvmHIKgEZkGL+xOBzRvup5Hxo6LHuMceSxOabuJLJm5Yp/92R9eMmMvA==", + "optional": true, + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/constructs": { + "version": "10.1.246", + "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.1.246.tgz", + "integrity": "sha512-2U2hnAuA4tCGGjHk/TulZfSlPobTyokEh+Azuch9nivv2yGI7/5nXDHC14i2MU/K7HFnnkQOHRSrwKSmOZkT/w==", + "engines": { + "node": ">= 14.17.0" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "optional": true, + "engines": { + "node": ">= 12" + } + }, + "node_modules/dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "optional": true, + "dependencies": { + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", + "optional": true + }, + "node_modules/event-stream": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz", + "integrity": "sha512-QHpkERcGsR0T7Qm3HNJSyXKEEj8AHNxkY3PK8TS2KJvQ7NiSHe3DDpwVKKtoYprL/AreyzFBeIkBIWChAqn60g==", + "optional": true, + "dependencies": { + "duplexer": "~0.1.1", + "from": "~0", + "map-stream": "~0.1.0", + "pause-stream": "0.0.11", + "split": "0.3", + "stream-combiner": "~0.0.4", + "through": "~2.3.1" + } + }, + "node_modules/fast-glob": { + "version": "3.2.12", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", + "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", + "optional": true, + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fastq": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", + "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", + "optional": true, + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "optional": true, + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "optional": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "optional": true, + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==", + "optional": true + }, + "node_modules/fs-extra": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", + "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "optional": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "optional": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/globby": { + "version": "13.1.3", + "resolved": "https://registry.npmjs.org/globby/-/globby-13.1.3.tgz", + "integrity": "sha512-8krCNHXvlCgHDpegPzleMq07yMYTO2sXKASmZmquEYWEmCx6J5UTRbp5RwMJkTJGtcQ44YpiUYUiN0b9mzy8Bw==", + "optional": true, + "dependencies": { + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.11", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "optional": true + }, + "node_modules/ignore": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", + "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==", + "optional": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "optional": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "optional": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "optional": true + }, + "node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "optional": true, + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/map-stream": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.1.0.tgz", + "integrity": "sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==", + "optional": true + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "optional": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "optional": true, + "dependencies": { + "braces": "^3.0.2", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "optional": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "optional": true, + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.0.tgz", + "integrity": "sha512-BKwRP/O0UvoMKp7GNdwPlObhYGB5DQqwhEDQlNKuoqwVYSxkSZCSbHjnFFmUEtwSKRPU4kNK8PbDYYitwaE3QA==", + "optional": true, + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "optional": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/pause-stream": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==", + "optional": true, + "dependencies": { + "through": "~2.3" + } + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "optional": true, + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/prettier": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.3.tgz", + "integrity": "sha512-tJ/oJ4amDihPoufT5sM0Z1SKEuKay8LfVAMlbbhnnkvt6BUserZylqo2PN+p9KeljLr0OHa2rXHU1T8reeoTrw==", + "dev": true, + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/ps-tree": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/ps-tree/-/ps-tree-1.2.0.tgz", + "integrity": "sha512-0VnamPPYHl4uaU/nSFeZZpR21QAWRz+sRv4iW9+v/GS/J5U5iZB5BNN6J0RMoOvdx2gWM2+ZFMIm58q24e4UYA==", + "optional": true, + "dependencies": { + "event-stream": "=3.3.4" + }, + "bin": { + "ps-tree": "bin/ps-tree.js" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true + }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "optional": true, + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true, + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/rust.aws-cdk-lambda": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/rust.aws-cdk-lambda/-/rust.aws-cdk-lambda-1.2.1.tgz", + "integrity": "sha512-BlkV7rVRQUkLVQuwP1A89C8f9yuDA1St7B/+EsckxBu/fUER8KKLydoe/hMfskCt6LQaIIAQqGqZ6HUr1DHjnw==", + "dependencies": { + "aws-cdk-lib": "^2.*", + "constructs": "^10.*", + "toml": "~3.0.0" + }, + "engines": { + "node": ">= 10.3.0" + }, + "optionalDependencies": { + "@ziglang/cli": "^0.0.11" + }, + "peerDependencies": { + "aws-cdk-lib": "^2.*", + "constructs": "^10.*" + } + }, + "node_modules/slash": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + "optional": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/split": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz", + "integrity": "sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==", + "optional": true, + "dependencies": { + "through": "2" + }, + "engines": { + "node": "*" + } + }, + "node_modules/stream-combiner": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz", + "integrity": "sha512-rT00SPnTVyRsaSz5zgSPma/aHSOic5U1prhYdRy5HS2kTZviFpmDgzilbtsJsxiroqACmayynDN/9VzIbX5DOw==", + "optional": true, + "dependencies": { + "duplexer": "~0.1.1" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "optional": true + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "optional": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toml": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/toml/-/toml-3.0.0.tgz", + "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==" + }, + "node_modules/typescript": { + "version": "4.9.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", + "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, + "node_modules/universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "optional": true, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", + "optional": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "optional": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/yaml": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.1.tgz", + "integrity": "sha512-e0WHiYql7+9wr4cWMx3TVQrNwejKaEe7/rHNmQmqRjazfOP5W8PB6Jpebb5o6fIapbz9o9+2ipcaTM2ZwDI6lw==", + "optional": true, + "engines": { + "node": ">= 14" + } + }, + "node_modules/zx": { + "version": "6.2.5", + "resolved": "https://registry.npmjs.org/zx/-/zx-6.2.5.tgz", + "integrity": "sha512-aTEZSL8sp1IhMUhxlXRTPH6MwzcuTBA3G4LLuH68NIj8kENfcaVGV7cn8LQXXHwG1M9LDI6ST6Qcw0EOrhRAVA==", + "optional": true, + "dependencies": { + "@types/fs-extra": "^9.0.13", + "@types/minimist": "^1.2.2", + "@types/node": "^17.0", + "@types/ps-tree": "^1.1.2", + "@types/which": "^2.0.1", + "chalk": "^5.0.1", + "fs-extra": "^10.1.0", + "globby": "^13.1.1", + "ignore": "^5.2.0", + "minimist": "^1.2.6", + "node-fetch": "^3.2.5", + "ps-tree": "^1.2.0", + "which": "^2.0.2", + "yaml": "^2.1.1" + }, + "bin": { + "zx": "build/cli.js" + }, + "engines": { + "node": ">= 16.0.0" + } + }, + "node_modules/zx/node_modules/@types/node": { + "version": "17.0.45", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.45.tgz", + "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==", + "optional": true + } + }, + "dependencies": { + "@aws-cdk/asset-awscli-v1": { + "version": "2.2.63", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.63.tgz", + "integrity": "sha512-LP+RSwH6JCx2T/AJyaveRg8VRvrM612TxGnec8SVuf9pQ6ah82LcW+4abuMhizeXDcx2ZufwNf9g/G2rp47bjA==" + }, + "@aws-cdk/asset-kubectl-v20": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.1.tgz", + "integrity": "sha512-U1ntiX8XiMRRRH5J1IdC+1t5CE89015cwyt5U63Cpk0GnMlN5+h9WsWMlKlPXZR4rdq/m806JRlBMRpBUB2Dhw==" + }, + "@aws-cdk/asset-node-proxy-agent-v5": { + "version": "2.0.52", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v5/-/asset-node-proxy-agent-v5-2.0.52.tgz", + "integrity": "sha512-sQhGP8QhAeP3ZsXxoOQrRFW/0VMH3HvI8NjZe/9ORtKwh8Gyjv2BJWIxlXLvRcc2o26FEl1oHIKRbaK/z/4iMw==" + }, + "@aws-cdk/aws-apigatewayv2-alpha": { + "version": "2.59.0-alpha.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigatewayv2-alpha/-/aws-apigatewayv2-alpha-2.59.0-alpha.0.tgz", + "integrity": "sha512-dChzTZt8Otm/5iTm76qHs/1xa5PXZyS3G7s8WyaWhH8plgZGeC/q6LdYvJ/FkuDYnD0a3Yp1t6wqJAxb9R0zxA==", + "requires": {} + }, + "@aws-cdk/aws-apigatewayv2-authorizers-alpha": { + "version": "2.59.0-alpha.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigatewayv2-authorizers-alpha/-/aws-apigatewayv2-authorizers-alpha-2.59.0-alpha.0.tgz", + "integrity": "sha512-J6EdrjbU/riYDW18WYnHs7KuUXuuWQwmbQLaAsbh6fVgrc8c8DGG+He3PhETTYScChkmOUXXxz2y45R0E5g7dA==", + "requires": {} + }, + "@aws-cdk/aws-apigatewayv2-integrations-alpha": { + "version": "2.59.0-alpha.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/aws-apigatewayv2-integrations-alpha/-/aws-apigatewayv2-integrations-alpha-2.59.0-alpha.0.tgz", + "integrity": "sha512-HSGcmRhnqfASOhP7ucX+RrVgaIgLWsdlzWIN+fQlwX2sSAh/K9oQ9ObOWqAhQix48TNlvnr2nr1yfyEeRX/Dbw==", + "requires": {} + }, + "@iarna/toml": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@iarna/toml/-/toml-3.0.0.tgz", + "integrity": "sha512-td6ZUkz2oS3VeleBcN+m//Q6HlCFCPrnI0FZhrt/h4XqLEdOyYp2u21nd8MdsR+WJy5r9PTDaHTDDfhf4H4l6Q==" + }, + "@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "optional": true, + "requires": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + } + }, + "@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "optional": true + }, + "@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "optional": true, + "requires": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + } + }, + "@types/fs-extra": { + "version": "9.0.13", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.13.tgz", + "integrity": "sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA==", + "optional": true, + "requires": { + "@types/node": "*" + } + }, + "@types/minimist": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz", + "integrity": "sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==", + "optional": true + }, + "@types/node": { + "version": "18.13.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.13.0.tgz", + "integrity": "sha512-gC3TazRzGoOnoKAhUx+Q0t8S9Tzs74z7m0ipwGpSqQrleP14hKxP4/JUeEQcD3W1/aIpnWl8pHowI7WokuZpXg==", + "devOptional": true + }, + "@types/ps-tree": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@types/ps-tree/-/ps-tree-1.1.2.tgz", + "integrity": "sha512-ZREFYlpUmPQJ0esjxoG1fMvB2HNaD3z+mjqdSosZvd3RalncI9NEur73P8ZJz4YQdL64CmV1w0RuqoRUlhQRBw==", + "optional": true + }, + "@types/which": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@types/which/-/which-2.0.1.tgz", + "integrity": "sha512-Jjakcv8Roqtio6w1gr0D7y6twbhx6gGgFGF5BLwajPpnOIOxFkakFhCq+LmyyeAz7BX6ULrjBOxdKaCDy+4+dQ==", + "optional": true + }, + "@ziglang/cli": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/@ziglang/cli/-/cli-0.0.11.tgz", + "integrity": "sha512-PCeyxchKF1nG9tr4sHUJlJiDjSFP0D+YS7jS0qREaAsa0cjdXYCqGXltFrEEbSMgWWkxJr9KnWviJgnHnnSFeg==", + "optional": true, + "requires": { + "zx": "^6.0.7" + } + }, + "aws-cdk": { + "version": "2.64.0", + "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.64.0.tgz", + "integrity": "sha512-iXkvVeYKt6Glboeicrb3QxC6K6o25+zitM/UTfgVzDlKEvC4hwQp1KqXy/caN7SfA6X2N0LJmXfC99T4cvIH0A==", + "dev": true, + "requires": { + "fsevents": "2.3.2" + } + }, + "aws-cdk-lib": { + "version": "2.64.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.64.0.tgz", + "integrity": "sha512-IrgL7thb6TeOyHgyR/qKWTdA9FBb9lv7Z9QPDzCNJlkKI+0ANjYHy3RYV8Gd+1+kc6l8DG9Z1elij40YCr/Ptg==", + "requires": { + "@aws-cdk/asset-awscli-v1": "^2.2.52", + "@aws-cdk/asset-kubectl-v20": "^2.1.1", + "@aws-cdk/asset-node-proxy-agent-v5": "^2.0.42", + "@balena/dockerignore": "^1.0.2", + "case": "1.6.3", + "fs-extra": "^9.1.0", + "ignore": "^5.2.4", + "jsonschema": "^1.4.1", + "minimatch": "^3.1.2", + "punycode": "^2.3.0", + "semver": "^7.3.8", + "yaml": "1.10.2" + }, + "dependencies": { + "@balena/dockerignore": { + "version": "1.0.2", + "bundled": true + }, + "at-least-node": { + "version": "1.0.0", + "bundled": true + }, + "balanced-match": { + "version": "1.0.2", + "bundled": true + }, + "brace-expansion": { + "version": "1.1.11", + "bundled": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "case": { + "version": "1.6.3", + "bundled": true + }, + "concat-map": { + "version": "0.0.1", + "bundled": true + }, + "fs-extra": { + "version": "9.1.0", + "bundled": true, + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "graceful-fs": { + "version": "4.2.10", + "bundled": true + }, + "ignore": { + "version": "5.2.4", + "bundled": true + }, + "jsonfile": { + "version": "6.1.0", + "bundled": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "jsonschema": { + "version": "1.4.1", + "bundled": true + }, + "lru-cache": { + "version": "6.0.0", + "bundled": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "minimatch": { + "version": "3.1.2", + "bundled": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "punycode": { + "version": "2.3.0", + "bundled": true + }, + "semver": { + "version": "7.3.8", + "bundled": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "universalify": { + "version": "2.0.0", + "bundled": true + }, + "yallist": { + "version": "4.0.0", + "bundled": true + }, + "yaml": { + "version": "1.10.2", + "bundled": true + } + } + }, + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "optional": true, + "requires": { + "fill-range": "^7.0.1" + } + }, + "buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + }, + "chalk": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.2.0.tgz", + "integrity": "sha512-ree3Gqw/nazQAPuJJEy+avdl7QfZMcUvmHIKgEZkGL+xOBzRvup5Hxo6LHuMceSxOabuJLJm5Yp/92R9eMmMvA==", + "optional": true + }, + "constructs": { + "version": "10.1.246", + "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.1.246.tgz", + "integrity": "sha512-2U2hnAuA4tCGGjHk/TulZfSlPobTyokEh+Azuch9nivv2yGI7/5nXDHC14i2MU/K7HFnnkQOHRSrwKSmOZkT/w==" + }, + "data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "optional": true + }, + "dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "optional": true, + "requires": { + "path-type": "^4.0.0" + } + }, + "duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", + "optional": true + }, + "event-stream": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz", + "integrity": "sha512-QHpkERcGsR0T7Qm3HNJSyXKEEj8AHNxkY3PK8TS2KJvQ7NiSHe3DDpwVKKtoYprL/AreyzFBeIkBIWChAqn60g==", + "optional": true, + "requires": { + "duplexer": "~0.1.1", + "from": "~0", + "map-stream": "~0.1.0", + "pause-stream": "0.0.11", + "split": "0.3", + "stream-combiner": "~0.0.4", + "through": "~2.3.1" + } + }, + "fast-glob": { + "version": "3.2.12", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", + "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", + "optional": true, + "requires": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + } + }, + "fastq": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", + "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", + "optional": true, + "requires": { + "reusify": "^1.0.4" + } + }, + "fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "optional": true, + "requires": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + } + }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "optional": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "optional": true, + "requires": { + "fetch-blob": "^3.1.2" + } + }, + "from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==", + "optional": true + }, + "fs-extra": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", + "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "optional": true, + "requires": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, + "fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "optional": true + }, + "glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "optional": true, + "requires": { + "is-glob": "^4.0.1" + } + }, + "globby": { + "version": "13.1.3", + "resolved": "https://registry.npmjs.org/globby/-/globby-13.1.3.tgz", + "integrity": "sha512-8krCNHXvlCgHDpegPzleMq07yMYTO2sXKASmZmquEYWEmCx6J5UTRbp5RwMJkTJGtcQ44YpiUYUiN0b9mzy8Bw==", + "optional": true, + "requires": { + "dir-glob": "^3.0.1", + "fast-glob": "^3.2.11", + "ignore": "^5.2.0", + "merge2": "^1.4.1", + "slash": "^4.0.0" + } + }, + "graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "optional": true + }, + "ignore": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", + "integrity": "sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==", + "optional": true + }, + "is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "optional": true + }, + "is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "optional": true, + "requires": { + "is-extglob": "^2.1.1" + } + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "optional": true + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "optional": true + }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "optional": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, + "map-stream": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.1.0.tgz", + "integrity": "sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==", + "optional": true + }, + "merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "optional": true + }, + "micromatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "optional": true, + "requires": { + "braces": "^3.0.2", + "picomatch": "^2.3.1" + } + }, + "minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "optional": true + }, + "node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "optional": true + }, + "node-fetch": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.0.tgz", + "integrity": "sha512-BKwRP/O0UvoMKp7GNdwPlObhYGB5DQqwhEDQlNKuoqwVYSxkSZCSbHjnFFmUEtwSKRPU4kNK8PbDYYitwaE3QA==", + "optional": true, + "requires": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + } + }, + "path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "optional": true + }, + "pause-stream": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==", + "optional": true, + "requires": { + "through": "~2.3" + } + }, + "picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "optional": true + }, + "prettier": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.3.tgz", + "integrity": "sha512-tJ/oJ4amDihPoufT5sM0Z1SKEuKay8LfVAMlbbhnnkvt6BUserZylqo2PN+p9KeljLr0OHa2rXHU1T8reeoTrw==", + "dev": true + }, + "ps-tree": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/ps-tree/-/ps-tree-1.2.0.tgz", + "integrity": "sha512-0VnamPPYHl4uaU/nSFeZZpR21QAWRz+sRv4iW9+v/GS/J5U5iZB5BNN6J0RMoOvdx2gWM2+ZFMIm58q24e4UYA==", + "optional": true, + "requires": { + "event-stream": "=3.3.4" + } + }, + "queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "optional": true + }, + "reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "optional": true + }, + "run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "optional": true, + "requires": { + "queue-microtask": "^1.2.2" + } + }, + "rust.aws-cdk-lambda": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/rust.aws-cdk-lambda/-/rust.aws-cdk-lambda-1.2.1.tgz", + "integrity": "sha512-BlkV7rVRQUkLVQuwP1A89C8f9yuDA1St7B/+EsckxBu/fUER8KKLydoe/hMfskCt6LQaIIAQqGqZ6HUr1DHjnw==", + "requires": { + "@ziglang/cli": "^0.0.11", + "aws-cdk-lib": "^2.*", + "constructs": "^10.*", + "toml": "~3.0.0" + } + }, + "slash": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + "optional": true + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + }, + "source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "split": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz", + "integrity": "sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==", + "optional": true, + "requires": { + "through": "2" + } + }, + "stream-combiner": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz", + "integrity": "sha512-rT00SPnTVyRsaSz5zgSPma/aHSOic5U1prhYdRy5HS2kTZviFpmDgzilbtsJsxiroqACmayynDN/9VzIbX5DOw==", + "optional": true, + "requires": { + "duplexer": "~0.1.1" + } + }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "optional": true + }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "optional": true, + "requires": { + "is-number": "^7.0.0" + } + }, + "toml": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/toml/-/toml-3.0.0.tgz", + "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==" + }, + "typescript": { + "version": "4.9.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz", + "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==", + "dev": true + }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "optional": true + }, + "web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", + "optional": true + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "optional": true, + "requires": { + "isexe": "^2.0.0" + } + }, + "yaml": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.1.tgz", + "integrity": "sha512-e0WHiYql7+9wr4cWMx3TVQrNwejKaEe7/rHNmQmqRjazfOP5W8PB6Jpebb5o6fIapbz9o9+2ipcaTM2ZwDI6lw==", + "optional": true + }, + "zx": { + "version": "6.2.5", + "resolved": "https://registry.npmjs.org/zx/-/zx-6.2.5.tgz", + "integrity": "sha512-aTEZSL8sp1IhMUhxlXRTPH6MwzcuTBA3G4LLuH68NIj8kENfcaVGV7cn8LQXXHwG1M9LDI6ST6Qcw0EOrhRAVA==", + "optional": true, + "requires": { + "@types/fs-extra": "^9.0.13", + "@types/minimist": "^1.2.2", + "@types/node": "^17.0", + "@types/ps-tree": "^1.1.2", + "@types/which": "^2.0.1", + "chalk": "^5.0.1", + "fs-extra": "^10.1.0", + "globby": "^13.1.1", + "ignore": "^5.2.0", + "minimist": "^1.2.6", + "node-fetch": "^3.2.5", + "ps-tree": "^1.2.0", + "which": "^2.0.2", + "yaml": "^2.1.1" + }, + "dependencies": { + "@types/node": { + "version": "17.0.45", + "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.45.tgz", + "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==", + "optional": true + } + } + } + } +} diff --git a/deploy/package.json b/deploy/package.json new file mode 100644 index 0000000..9fa6fa9 --- /dev/null +++ b/deploy/package.json @@ -0,0 +1,28 @@ +{ + "name": "htsget-elsa-lambda", + "version": "0.1", + "bin": { + "htsget_app": "bin/htsget-elsa-lambda.js" + }, + "scripts": { + "build": "tsc", + "watch": "tsc -w", + "cdk": "cdk" + }, + "devDependencies": { + "@types/node": "^18.11.18", + "aws-cdk": "^2.64.0", + "prettier": "2.8.3", + "typescript": "^4.9.4" + }, + "dependencies": { + "@aws-cdk/aws-apigatewayv2-alpha": "^2.59.0-alpha.0", + "@aws-cdk/aws-apigatewayv2-authorizers-alpha": "^2.59.0-alpha.0", + "@aws-cdk/aws-apigatewayv2-integrations-alpha": "^2.59.0-alpha.0", + "@iarna/toml": "^3.0.0", + "aws-cdk-lib": "^2.64.0", + "constructs": "^10.1.148", + "rust.aws-cdk-lambda": "^1.1.3", + "source-map-support": "^0.5.21" + } +} diff --git a/deploy/tsconfig.json b/deploy/tsconfig.json new file mode 100644 index 0000000..810149e --- /dev/null +++ b/deploy/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2021", + "module": "commonjs", + "lib": ["ES2021"], + "declaration": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": false, + "inlineSourceMap": true, + "inlineSources": true, + "experimentalDecorators": true, + "strictPropertyInitialization": false, + "typeRoots": ["./node_modules/@types"] + }, + "exclude": ["node_modules", "cdk.out"] +} From b73b27fea56cfade8012584bedcaec450b8701fc Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 21 Jun 2023 14:59:55 +1000 Subject: [PATCH 17/25] style: clippy and fmt --- Cargo.lock | 673 +++++++++++-------------- Cargo.toml | 2 + deploy/lib/htsget-elsa-lambda-stack.ts | 4 +- htsget-elsa/src/s3.rs | 4 +- 4 files changed, 304 insertions(+), 379 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d274f3..8abf25e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,13 +10,19 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -28,55 +34,64 @@ dependencies = [ [[package]] name = "anstream" -version = "0.2.6" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "342258dd14006105c2b75ab1bd7543a03bdf0cfc94383303ac212a04939dff6f" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" dependencies = [ "anstyle", "anstyle-parse", + "anstyle-query", "anstyle-wincon", - "concolor-override", - "concolor-query", + "colorchoice", "is-terminal", "utf8parse", ] [[package]] name = "anstyle" -version = "0.3.5" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" [[package]] name = "anstyle-parse" -version = "0.1.1" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7d1bb534e9efed14f3e5f44e7dd1a4f709384023a4165199a4241e18dff0116" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" dependencies = [ "utf8parse", ] +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "anstyle-wincon" -version = "0.2.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3127af6145b149f3287bb9a0d10ad9c5692dba8c53ad48285e5bec4063834fa" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" dependencies = [ "anstyle", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "anyhow" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" [[package]] name = "arrayvec" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "ascii" @@ -148,7 +163,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] @@ -159,7 +174,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] @@ -173,12 +188,9 @@ dependencies = [ [[package]] name = "atomic" -version = "0.5.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b88d82667eca772c4aa12f0f1348b3ae643424c8876448f3f7bd5787032e234c" -dependencies = [ - "autocfg", -] +checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" [[package]] name = "autocfg" @@ -431,7 +443,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls", + "hyper-rustls 0.23.2", "lazy_static", "pin-project-lite", "serde", @@ -592,9 +604,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.6.15" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b32c5ea3aabaf4deb5f5ced2d688ec0844c881c9e6c696a8b769a05fc691e62" +checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" dependencies = [ "async-trait", "axum-core", @@ -666,9 +678,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.0" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" [[package]] name = "base64-simd" @@ -703,9 +715,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byteorder" @@ -776,12 +788,12 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.24" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" dependencies = [ + "android-tzdata", "iana-time-zone", - "num-integer", "num-traits", "serde", "winapi", @@ -789,9 +801,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.2.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046ae530c528f252094e4a77886ee1374437744b2bff1497aa898bbddbbb29b3" +checksum = "2686c4115cb0810d9a984776e197823d08ec94f176549a89a9efded477c456dc" dependencies = [ "clap_builder", "clap_derive", @@ -800,9 +812,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.2.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223163f58c9a40c3b0a43e1c4b50a9ce09f007ea2cb1ec258a687945b4b7929f" +checksum = "2e53afce1efce6ed1f633cf0e57612fe51db54a1ee4fd8f8503d078fe02d69ae" dependencies = [ "anstream", "anstyle", @@ -813,46 +825,27 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.2.0" +version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" +checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] name = "clap_lex" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" - -[[package]] -name = "codespan-reporting" -version = "0.11.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" -dependencies = [ - "termcolor", - "unicode-width", -] +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" [[package]] -name = "concolor-override" +name = "colorchoice" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a855d4a1978dc52fb0536a04d384c2c0c1aa273597f08b77c8c4d3b2eec6037f" - -[[package]] -name = "concolor-query" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d11d52c3d7ca2e6d0040212be9e4dbbcd78b6447f535b6b561f449427944cf" -dependencies = [ - "windows-sys 0.45.0", -] +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "concurrent-queue" @@ -881,9 +874,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" +checksum = "03e69e28e9f7f77debdedbaafa2866e1de9ba56df55a8bd7cfc724c25a09987c" dependencies = [ "libc", ] @@ -918,9 +911,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ "cfg-if", ] @@ -945,55 +938,11 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "cxx" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" -dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn 2.0.14", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.14", -] - [[package]] name = "darling" -version = "0.14.4" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +checksum = "0558d22a7b463ed0241e993f76f09f30b126687447751a8638587b864e4b3944" dependencies = [ "darling_core", "darling_macro", @@ -1001,27 +950,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.14.4" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +checksum = "ab8bfa2e259f8ee1ce5e97824a3c55ec4404a0d772ca7fa96bf19f0752a046eb" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn 1.0.109", + "syn 2.0.18", ] [[package]] name = "darling_macro" -version = "0.14.4" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" dependencies = [ "darling_core", "quote", - "syn 1.0.109", + "syn 2.0.18", ] [[package]] @@ -1051,9 +1000,9 @@ checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -1125,23 +1074,23 @@ dependencies = [ [[package]] name = "figment" -version = "0.10.8" +version = "0.10.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e56602b469b2201400dec66a66aec5a9b8761ee97cd1b8c96ab2483fcc16cc9" +checksum = "4547e226f4c9ab860571e070a9034192b3175580ecea38da34fcdb53a018c9a5" dependencies = [ "atomic", "pear", "serde", - "toml 0.5.11", + "toml", "uncased", "version_check", ] [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", "miniz_oxide", @@ -1155,9 +1104,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ "percent-encoding", ] @@ -1233,7 +1182,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] @@ -1295,9 +1244,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", @@ -1306,9 +1255,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.16" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" +checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" dependencies = [ "bytes", "fnv", @@ -1378,7 +1327,7 @@ dependencies = [ [[package]] name = "htsget-config" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#c44aea26ab9ead617bbcf8747010ad7b3ec10d8e" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#34685bccf8b5d95ebcf7c059b5caa1cd3bbe9e96" dependencies = [ "async-trait", "clap", @@ -1391,7 +1340,7 @@ dependencies = [ "serde_regex", "serde_with", "thiserror", - "toml 0.7.3", + "toml", "tracing", "tracing-subscriber", ] @@ -1439,7 +1388,7 @@ dependencies = [ [[package]] name = "htsget-http" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#c44aea26ab9ead617bbcf8747010ad7b3ec10d8e" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#34685bccf8b5d95ebcf7c059b5caa1cd3bbe9e96" dependencies = [ "futures", "htsget-config", @@ -1455,7 +1404,7 @@ dependencies = [ [[package]] name = "htsget-lambda" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#c44aea26ab9ead617bbcf8747010ad7b3ec10d8e" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#34685bccf8b5d95ebcf7c059b5caa1cd3bbe9e96" dependencies = [ "bytes", "htsget-config", @@ -1477,14 +1426,14 @@ dependencies = [ [[package]] name = "htsget-search" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#c44aea26ab9ead617bbcf8747010ad7b3ec10d8e" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#34685bccf8b5d95ebcf7c059b5caa1cd3bbe9e96" dependencies = [ "async-trait", "aws-config", "aws-sdk-s3", "axum", "axum-extra", - "base64 0.21.0", + "base64 0.21.2", "bytes", "futures", "futures-util", @@ -1497,7 +1446,7 @@ dependencies = [ "serde", "thiserror", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.4", "tokio-util", "tower", "tower-http", @@ -1507,13 +1456,13 @@ dependencies = [ [[package]] name = "htsget-test" version = "0.1.4" -source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#c44aea26ab9ead617bbcf8747010ad7b3ec10d8e" +source = "git+https://github.com/umccr/htsget-rs.git?branch=feat/htsget-elsa#34685bccf8b5d95ebcf7c059b5caa1cd3bbe9e96" dependencies = [ "async-trait", "aws-config", "aws-credential-types", "aws-sdk-s3", - "base64 0.21.0", + "base64 0.21.2", "futures", "htsget-config", "http", @@ -1604,9 +1553,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.25" +version = "0.14.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" dependencies = [ "bytes", "futures-channel", @@ -1635,17 +1584,30 @@ dependencies = [ "http", "hyper", "log", - "rustls", + "rustls 0.20.8", "rustls-native-certs", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.4", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" +dependencies = [ + "http", + "hyper", + "rustls 0.21.2", + "tokio", + "tokio-rustls 0.24.1", ] [[package]] name = "iana-time-zone" -version = "0.1.56" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1657,12 +1619,11 @@ dependencies = [ [[package]] name = "iana-time-zone-haiku" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "cxx", - "cxx-build", + "cc", ] [[package]] @@ -1673,9 +1634,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1715,9 +1676,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi 0.3.1", "libc", @@ -1750,9 +1711,9 @@ checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -1883,33 +1844,21 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.141" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" - -[[package]] -name = "link-cplusplus" -version = "1.0.8" +version = "0.2.146" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] +checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "log" -version = "0.4.17" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" [[package]] name = "lzma-sys" @@ -1976,23 +1925,22 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -2237,9 +2185,9 @@ checksum = "cf70ee2d9b1737d1836c20d9f8f96ec3901b2bf92128439db13237ddce9173a5" [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "openssl-probe" @@ -2276,18 +2224,18 @@ checksum = "14f2252c834a40ed9bb5422029649578e63aa341ac401f74e719dd1afda8394e" [[package]] name = "path-absolutize" -version = "3.0.14" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f1d4993b16f7325d90c18c3c6a3327db7808752db8d208cea0acee0abd52c52" +checksum = "43eb3595c63a214e1b37b44f44b0a84900ef7ae0b4c5efce59e123d246d7a0de" dependencies = [ "path-dedot", ] [[package]] name = "path-dedot" -version = "3.0.18" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a81540d94551664b72b72829b12bd167c73c9d25fbac0e04fafa8023f7e4901" +checksum = "9d55e486337acb9973cdea3ec5638c1b3bcb22e573b2b7b41969e0c744d5a15e" dependencies = [ "once_cell", ] @@ -2312,7 +2260,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] @@ -2326,28 +2274,28 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", ] [[package]] @@ -2364,9 +2312,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "ppv-lite86" @@ -2412,9 +2360,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" dependencies = [ "unicode-ident", ] @@ -2427,7 +2375,7 @@ checksum = "606c4ba35817e2922a308af55ad51bab3645b59eae5c570d4a6cf07e36bd493b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", "version_check", "yansi", ] @@ -2455,9 +2403,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ "proc-macro2", ] @@ -2526,13 +2474,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.2", ] [[package]] @@ -2541,7 +2489,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.29", ] [[package]] @@ -2550,13 +2498,19 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" + [[package]] name = "reqwest" -version = "0.11.16" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.0", + "base64 0.21.2", "bytes", "encoding_rs", "futures-core", @@ -2565,7 +2519,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls", + "hyper-rustls 0.24.0", "ipnet", "js-sys", "log", @@ -2573,13 +2527,13 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", + "rustls 0.21.2", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", "tower-service", "url", "wasm-bindgen", @@ -2630,9 +2584,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.11" +version = "0.37.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" +checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" dependencies = [ "bitflags", "errno", @@ -2654,11 +2608,23 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + [[package]] name = "rustls-native-certs" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", "rustls-pemfile", @@ -2672,7 +2638,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64 0.21.0", + "base64 0.21.2", +] + +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", ] [[package]] @@ -2782,12 +2758,6 @@ dependencies = [ "windows-sys 0.42.0", ] -[[package]] -name = "scratch" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" - [[package]] name = "sct" version = "0.7.0" @@ -2800,9 +2770,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.8.2" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" dependencies = [ "bitflags", "core-foundation", @@ -2813,9 +2783,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" dependencies = [ "core-foundation-sys", "libc", @@ -2829,29 +2799,29 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.160" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a" dependencies = [ "indexmap", "itoa", @@ -2891,9 +2861,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" +checksum = "93107647184f6027e3b7dcb2e11034cf95ffa1e3a682c67951963ac69c1c007d" dependencies = [ "serde", ] @@ -2912,9 +2882,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331bb8c3bf9b92457ab7abecf07078c13f7d270ba490103e84e8b014490cd0b0" +checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" dependencies = [ "base64 0.13.1", "chrono", @@ -2928,14 +2898,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859011bddcc11f289f07f467cc1fe01c7a941daa4d8f6c40d4d1c92eb6d9319c" +checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" dependencies = [ "darling", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", ] [[package]] @@ -2951,9 +2921,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ "cfg-if", "cpufeatures", @@ -3014,9 +2984,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" @@ -3031,9 +3001,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.14" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf316d5356ed6847742d036f8a39c3b8435cac10bd528a4bd461928a6ab34d5" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ "proc-macro2", "quote", @@ -3048,24 +3018,16 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "tempfile" -version = "3.5.0" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" dependencies = [ + "autocfg", "cfg-if", "fastrand", "redox_syscall", "rustix", - "windows-sys 0.45.0", -] - -[[package]] -name = "termcolor" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", + "windows-sys 0.48.0", ] [[package]] @@ -3085,7 +3047,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] @@ -3100,9 +3062,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.20" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" dependencies = [ "itoa", "serde", @@ -3112,15 +3074,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" dependencies = [ "time-core", ] @@ -3142,9 +3104,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.27.0" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg", "bytes", @@ -3154,18 +3116,18 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.14", + "syn 2.0.18", ] [[package]] @@ -3174,16 +3136,26 @@ version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.8", "tokio", "webpki", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.2", + "tokio", +] + [[package]] name = "tokio-stream" -version = "0.1.12" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", @@ -3192,9 +3164,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ "bytes", "futures-core", @@ -3207,18 +3179,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.11" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - -[[package]] -name = "toml" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21" +checksum = "d6135d499e69981f9ff0ef2167955a5333c35e36f6937d382974566b3d5b94ec" dependencies = [ "serde", "serde_spanned", @@ -3228,18 +3191,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.8" +version = "0.19.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13" +checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" dependencies = [ "indexmap", "serde", @@ -3316,20 +3279,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "8803eee176538f94ae9a14b55b2804eb7e1441f8210b1c31290b3bccdccff73b" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", "valuable", @@ -3358,9 +3321,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ "matchers", "nu-ansi-term", @@ -3397,9 +3360,9 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "uncased" -version = "0.9.7" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b01702b0fd0b3fadcf98e098780badda8742d4f4a7676615cad90e8ac73622" +checksum = "9b9bc53168a4be7402ab86c3aad243a84dd7381d09be0eddc81280c1da95ca68" dependencies = [ "version_check", ] @@ -3421,9 +3384,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unicode-normalization" @@ -3434,12 +3397,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-width" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" - [[package]] name = "untrusted" version = "0.7.1" @@ -3448,9 +3405,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" dependencies = [ "form_urlencoded", "idna", @@ -3472,11 +3429,11 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.3.1" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b55a3fef2a1e3b3a00ce878640918820d3c51081576ac657d23af9fc7928fdb" +checksum = "0fa2982af2eec27de306107c027578ff7f423d65f7250e40ce0fea8f45248b81" dependencies = [ - "getrandom 0.2.9", + "getrandom 0.2.10", ] [[package]] @@ -3505,11 +3462,10 @@ checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] @@ -3527,9 +3483,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3537,24 +3493,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.34" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ "cfg-if", "js-sys", @@ -3564,9 +3520,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3574,28 +3530,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", @@ -3636,15 +3592,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -[[package]] -name = "winapi-util" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" -dependencies = [ - "winapi", -] - [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -3657,7 +3604,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets 0.48.0", + "windows-targets", ] [[package]] @@ -3675,37 +3622,13 @@ dependencies = [ "windows_x86_64_msvc 0.42.2", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows-targets", ] [[package]] @@ -3809,9 +3732,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.1" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8970b36c66498d8ff1d66685dc86b91b29db0c7739899012f63a63814b4b28" +checksum = "ca0ace3845f0d96209f0375e6d367e3eb87eb65d27d445bdc9f1843a26f39448" dependencies = [ "memchr", ] @@ -3827,13 +3750,13 @@ dependencies = [ [[package]] name = "wiremock" -version = "0.5.18" +version = "0.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7b0b5b253ebc0240d6aac6dd671c495c467420577bf634d3064ae7e6fa2b4c" +checksum = "c6f71803d3a1c80377a06221e0530be02035d5b3e854af56c6ece7ac20ac441d" dependencies = [ "assert-json-diff 2.0.2", "async-trait", - "base64 0.21.0", + "base64 0.21.2", "deadpool", "futures", "futures-timer", diff --git a/Cargo.toml b/Cargo.toml index 8a39b97..e909290 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,6 @@ [workspace] +resolver = "2" + members = [ "htsget-elsa", "htsget-elsa-lambda" diff --git a/deploy/lib/htsget-elsa-lambda-stack.ts b/deploy/lib/htsget-elsa-lambda-stack.ts index b28c1c5..1c99133 100644 --- a/deploy/lib/htsget-elsa-lambda-stack.ts +++ b/deploy/lib/htsget-elsa-lambda-stack.ts @@ -83,7 +83,7 @@ export class HtsgetElsaLambdaStack extends Stack { environment: { ...config.htsgetConfig, RUST_LOG: - "info,htsget_http_lambda=trace,htsget_config=trace,htsget_http_core=trace,htsget_search=trace", + "info,htsget_elsa=trace,htsget_elsa_lambda=trace,htsget_lambda=trace,htsget_config=trace,htsget_http=trace,htsget_search=trace", }, buildEnvironment: { RUSTFLAGS: "-C target-cpu=neoverse-n1", @@ -130,7 +130,7 @@ export class HtsgetElsaLambdaStack extends Stack { ); new ARecord(this, id + "HtsgetElsaARecord", { zone: hostedZone, - recordName: "htsget.elsa", + recordName: "htsget-elsa", target: RecordTarget.fromAlias( new ApiGatewayv2DomainProperties( domainName.regionalDomainName, diff --git a/htsget-elsa/src/s3.rs b/htsget-elsa/src/s3.rs index 752e723..37dbf3e 100644 --- a/htsget-elsa/src/s3.rs +++ b/htsget-elsa/src/s3.rs @@ -166,7 +166,7 @@ mod tests { let result = s3 .last_modified("elsa-data-tmp", "htsget-manifests/R004") .await; - assert!(matches!(result, Some(_))); + assert!(result.is_some()); }, 0, ) @@ -185,7 +185,7 @@ mod tests { let result = s3 .last_modified("elsa-data-tmp", "htsget-manifests/R005") .await; - assert!(matches!(result, None)); + assert!(result.is_none()); }, 0, ) From c90fa3267e1e131fc06618b0d5aeba17dfa01fdc Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 21 Jun 2023 15:05:20 +1000 Subject: [PATCH 18/25] build: update config --- deploy/config/config_dev_umccr.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/config/config_dev_umccr.toml b/deploy/config/config_dev_umccr.toml index 4d2a512..f22d5fe 100644 --- a/deploy/config/config_dev_umccr.toml +++ b/deploy/config/config_dev_umccr.toml @@ -1,10 +1,10 @@ -elsa_endpoint_authority = "elsa.dev.umccr.org" +elsa_endpoint_authority = "elsa-data.dev.umccr.org" cache_location = "elsa-tmp-bucket" ticket_server_cors_allow_headers = "All" ticket_server_cors_allow_origins = [ - "https://elsa.dev.umccr.org", - "https://elsa.umccr.org", + "https://elsa-data.dev.umccr.org", + "https://elsa-data.dev.umccr.org", ] ticket_server_cors_allow_methods = "All" ticket_server_cors_allow_credentials = true From 9f3be81478301285109101c6b6770c69b5d254d6 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 21 Jun 2023 15:16:42 +1000 Subject: [PATCH 19/25] fix: feature flag errors --- deploy/README.md | 2 +- deploy/config/config_dev_umccr.toml | 2 +- deploy/package-lock.json | 6 +++--- htsget-elsa/Cargo.toml | 1 - htsget-elsa/src/elsa_endpoint.rs | 8 ++++---- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index bf537b8..6d1808f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -71,7 +71,7 @@ cargo install cargo-lambda npm install cd .. -cargo lambda build --release --arm64 --bin htsget-lambda +cargo lambda build --release --arm64 --bin htsget-elsa-lambda cd deploy ``` diff --git a/deploy/config/config_dev_umccr.toml b/deploy/config/config_dev_umccr.toml index f22d5fe..c9c67e3 100644 --- a/deploy/config/config_dev_umccr.toml +++ b/deploy/config/config_dev_umccr.toml @@ -4,7 +4,7 @@ cache_location = "elsa-tmp-bucket" ticket_server_cors_allow_headers = "All" ticket_server_cors_allow_origins = [ "https://elsa-data.dev.umccr.org", - "https://elsa-data.dev.umccr.org", + "https://elsa-data.umccr.org", ] ticket_server_cors_allow_methods = "All" ticket_server_cors_allow_credentials = true diff --git a/deploy/package-lock.json b/deploy/package-lock.json index b008d0a..fe2ba83 100644 --- a/deploy/package-lock.json +++ b/deploy/package-lock.json @@ -1,11 +1,11 @@ { - "name": "htsget-lambda", + "name": "htsget-elsa-lambda", "version": "0.1", "lockfileVersion": 2, "requires": true, "packages": { "": { - "name": "htsget-lambda", + "name": "htsget-elsa-lambda", "version": "0.1", "dependencies": { "@aws-cdk/aws-apigatewayv2-alpha": "^2.59.0-alpha.0", @@ -18,7 +18,7 @@ "source-map-support": "^0.5.21" }, "bin": { - "htsget_app": "bin/htsget-lambda.js" + "htsget_app": "bin/htsget-elsa-lambda.js" }, "devDependencies": { "@types/node": "^18.11.18", diff --git a/htsget-elsa/Cargo.toml b/htsget-elsa/Cargo.toml index d7cfa61..e3c4e56 100644 --- a/htsget-elsa/Cargo.toml +++ b/htsget-elsa/Cargo.toml @@ -5,7 +5,6 @@ version = "0.1.0" edition = "2021" [features] - test-utils = ["htsget-test", "wiremock"] default = [] diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index ff703b2..8277eef 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -189,13 +189,13 @@ where S: GetObject, { pub fn new(endpoint: Authority, cache: &'a C, get_object: &'a S) -> Result { - Ok(Self::new_with_client( - Self::create_client()?, + Ok(Self { + client: Self::create_client()?, endpoint, cache, get_object, - "https", - )) + scheme: "https", + }) } #[cfg(feature = "test-utils")] From 3ec0385fc66df8acb9a9eb7c8c5df138aad1e013 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 26 Jun 2023 08:48:54 +1000 Subject: [PATCH 20/25] feat: add restrictions --- deploy/config/config_dev_umccr.toml | 2 +- htsget-elsa/src/elsa_endpoint.rs | 72 +++++++++++++++++++++++------ htsget-elsa/src/s3.rs | 13 +++++- htsget-elsa/src/test_utils.rs | 65 ++++++++++++++++++++++---- 4 files changed, 124 insertions(+), 28 deletions(-) diff --git a/deploy/config/config_dev_umccr.toml b/deploy/config/config_dev_umccr.toml index c9c67e3..1ea3e63 100644 --- a/deploy/config/config_dev_umccr.toml +++ b/deploy/config/config_dev_umccr.toml @@ -1,5 +1,5 @@ elsa_endpoint_authority = "elsa-data.dev.umccr.org" -cache_location = "elsa-tmp-bucket" +cache_location = "elsadatalocaldevtestinfr-tempprivatebucket3b80855-p9qoue91haoy" ticket_server_cors_allow_headers = "All" ticket_server_cors_allow_origins = [ diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index 8277eef..4ffd83a 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -1,10 +1,10 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use async_trait::async_trait; -use htsget_config::resolver::{AllowGuard, Resolver}; +use htsget_config::resolver::{AllowGuard, ReferenceNames, Resolver}; use htsget_config::storage::s3::S3Storage; use htsget_config::storage::Storage; -use htsget_config::types::Format; +use htsget_config::types::{Format, Interval}; use http::uri::Authority; use http::Uri; use reqwest::{Client, Url}; @@ -38,7 +38,7 @@ pub struct ElsaResponse { pub struct ElsaReadsManifest { url: String, format: Option, - restriction: Option, + restrictions: Vec, } #[derive(Debug, Deserialize, PartialEq, Eq)] @@ -47,12 +47,16 @@ pub struct ElsaVariantsManifest { url: String, format: Option, variant_sample_id: String, - restriction: Option, + restrictions: Vec, } #[derive(Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct ElsaRestrictionsManifest {} +pub struct ElsaRestrictionManifest { + chromosome: u8, + start: Option, + end: Option, +} #[derive(Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] @@ -61,7 +65,6 @@ pub struct ElsaManifest { release_key: String, reads: HashMap, variants: HashMap, - restrictions: ElsaRestrictionsManifest, } impl ElsaManifest { @@ -71,6 +74,7 @@ impl ElsaManifest { url: &str, id: &str, format: Format, + restriction: &ElsaRestrictionManifest, ) -> Result { let url = match (url.strip_prefix("s3://"), url.strip_prefix("S3://")) { (Some(url), _) | (_, Some(url)) => Ok(url), @@ -101,7 +105,12 @@ impl ElsaManifest { }, &format!("^{}/{}$", release_key, id), &key, - AllowGuard::default().with_allow_formats(vec![format]), + AllowGuard::default() + .with_allow_formats(vec![format]) + .with_allow_reference_names(ReferenceNames::List(HashSet::from_iter(vec![ + restriction.chromosome.to_string(), + ]))) + .with_allow_interval(Interval::new(restriction.start, restriction.end)), ) .map_err(|err| InvalidManifest(format!("failed to construct regex: {}", err))) } @@ -114,15 +123,31 @@ impl TryFrom for Vec { fn try_from(manifest: ElsaManifest) -> Result { let release_key = manifest.release_key; - manifest + let get_resolvers = + |url: &str, id: &str, format: Format, restrictions: Vec| { + restrictions + .iter() + .map(|restriction| { + ElsaManifest::resolver_from_manifest_parts( + &release_key, + url, + id, + format, + restriction, + ) + }) + .collect() + }; + + Ok(manifest .reads .into_iter() .map(|(id, reads_manifest)| { - ElsaManifest::resolver_from_manifest_parts( - &release_key, + get_resolvers( &reads_manifest.url, &id, reads_manifest.format.unwrap_or(Format::Bam), + reads_manifest.restrictions, ) }) .chain( @@ -130,15 +155,18 @@ impl TryFrom for Vec { .variants .into_iter() .map(|(id, variants_manifest)| { - ElsaManifest::resolver_from_manifest_parts( - &release_key, + get_resolvers( &variants_manifest.url, &id, variants_manifest.format.unwrap_or(Format::Vcf), + variants_manifest.restrictions, ) }), ) - .collect() + .collect::>>>()? + .into_iter() + .flatten() + .collect()) } } @@ -278,7 +306,7 @@ mod tests { use serde_json::from_str; use crate::elsa_endpoint::{ - ElsaEndpoint, ElsaLocation, ElsaManifest, ElsaResponse, CACHE_PATH, + ElsaEndpoint, ElsaLocation, ElsaManifest, ElsaResponse, ElsaRestrictionManifest, CACHE_PATH, }; use crate::s3::S3; use crate::test_utils::{ @@ -432,6 +460,7 @@ mod tests { "s3://umccr-10g-data-dev/HG00097/HG00097.bam", "30F9F3FED8F711ED8C35DBEF59E9F537", Format::Bam, + &example_restrictions_manifest(), ) .unwrap(); assert!(is_resolver_from_parts(&response)); @@ -444,6 +473,7 @@ mod tests { "S3://umccr-10g-data-dev/HG00097/HG00097.bam", "30F9F3FED8F711ED8C35DBEF59E9F537", Format::Bam, + &example_restrictions_manifest(), ) .unwrap(); assert!(is_resolver_from_parts(&response)); @@ -456,6 +486,7 @@ mod tests { "s3://umccr-10g-data-dev/HG00097/HG00097", "30F9F3FED8F711ED8C35DBEF59E9F537", Format::Bam, + &example_restrictions_manifest(), ) .unwrap(); assert!(is_resolver_from_parts(&response)); @@ -468,6 +499,7 @@ mod tests { "gcp://umccr-10g-data-dev/HG00097/HG00097.bam", "30F9F3FED8F711ED8C35DBEF59E9F537", Format::Bam, + &example_restrictions_manifest(), ); assert!(matches!(response, Err(UnsupportedManifestFeature(_)))); } @@ -479,6 +511,7 @@ mod tests { "s3://umccr-10g-data-dev", "30F9F3FED8F711ED8C35DBEF59E9F537", Format::Bam, + &example_restrictions_manifest(), ); assert!(matches!(response, Err(InvalidManifest(_)))); } @@ -490,7 +523,16 @@ mod tests { "s3:///HG00097/HG00097.bam", "30F9F3FED8F711ED8C35DBEF59E9F537", Format::Bam, + &example_restrictions_manifest(), ); assert!(matches!(response, Err(InvalidManifest(_)))); } + + fn example_restrictions_manifest() -> ElsaRestrictionManifest { + ElsaRestrictionManifest { + chromosome: 1, + start: Some(1), + end: Some(10), + } + } } diff --git a/htsget-elsa/src/s3.rs b/htsget-elsa/src/s3.rs index 37dbf3e..49e578e 100644 --- a/htsget-elsa/src/s3.rs +++ b/htsget-elsa/src/s3.rs @@ -8,7 +8,7 @@ use bytes::Bytes; use htsget_config::resolver::Resolver; use serde::{Deserialize, Serialize}; use serde_json::{from_slice, to_vec}; -use tracing::instrument; +use tracing::{instrument, trace}; use crate::Error::{DeserializeError, GetObjectError, PutObjectError, SerializeError}; use crate::{Cache, Error, GetObject, Result}; @@ -101,6 +101,8 @@ impl Cache for S3 { #[instrument(level = "trace", skip_all, ret)] async fn get + Send + Sync>(&self, key: K) -> Result> { + trace!(key = key.as_ref(), "getting key"); + if let Some(last_modified) = self .last_modified(self.cache_bucket.clone(), key.as_ref()) .await @@ -127,6 +129,8 @@ impl Cache for S3 { item: Self::Item, max_age: u64, ) -> Result<()> { + trace!(key = key.as_ref(), "putting key"); + self.s3_client .put_object() .bucket(self.cache_bucket.clone()) @@ -137,7 +141,12 @@ impl Cache for S3 { ))) .send() .await - .map_err(|err| PutObjectError(err.to_string()))?; + .map_err(|err| { + let err = err.into_service_error(); + trace!(err = err.message(), "put object error"); + + PutObjectError(err.to_string()) + })?; Ok(()) } diff --git a/htsget-elsa/src/test_utils.rs b/htsget-elsa/src/test_utils.rs index a093ca6..1628b7b 100644 --- a/htsget-elsa/src/test_utils.rs +++ b/htsget-elsa/src/test_utils.rs @@ -1,12 +1,14 @@ +use std::collections::HashSet; use std::fs; use std::future::Future; use std::path::{Path, PathBuf}; use crate::elsa_endpoint::ENDPOINT_PATH; use aws_sdk_s3::Client; +use htsget_config::resolver::ReferenceNames::List; use htsget_config::resolver::Resolver; use htsget_config::storage; -use htsget_config::types::Format; +use htsget_config::types::{Format, Interval}; use htsget_test::aws_mocks::with_s3_test_server_tmp; use wiremock::matchers::{method, path, query_param}; use wiremock::{Mock, MockServer, Request, ResponseTemplate, Times}; @@ -17,47 +19,82 @@ pub fn example_elsa_manifest() -> String { "id": "R004", "reads": { "30F9F3FED8F711ED8C35DBEF59E9F537": { - "url": "s3://umccr-10g-data-dev/HG00097/HG00097.bam" + "url": "s3://umccr-10g-data-dev/HG00097/HG00097.bam", + "restrictions": [ + { + "chromosome": 1, + "start": 1, + "end": 10 + } + ] }, "30F9FFD4D8F711ED8C353BBCB8861211": { - "url": "s3://umccr-10g-data-dev/HG00096/HG00096.bam" + "url": "s3://umccr-10g-data-dev/HG00096/HG00096.bam", + "restrictions": [ + { + "chromosome": 2, + "end": 10 + } + ] } }, "variants": { "30F9F3FED8F711ED8C35DBEF59E9F537": { "url": "s3://umccr-10g-data-dev/HG00097/HG00097.hard-filtered.vcf.gz", + "restrictions": [ + { + "chromosome": 3, + "start": 10 + } + ], "variantSampleId": "" }, "30F9FFD4D8F711ED8C353BBCB8861211": { "url": "s3://umccr-10g-data-dev/HG00096/HG00096.hard-filtered.vcf.gz", + "restrictions": [ + { + "chromosome": 4 + } + ], "variantSampleId": "" } }, - "restrictions": {}, "cases": [ { - "ids": { "": "SINGLETONCHARLES" }, + "ids": { + "": "SINGLETONCHARLES" + }, "patients": [ { - "ids": { "": "CHARLES" }, + "ids": { + "": "CHARLES" + }, "specimens": [ { "htsgetId": "30F9FFD4D8F711ED8C353BBCB8861211", - "ids": { "": "HG00096" } + "ids": { + "": "HG00096" + } } ] } ] }, { - "ids": { "": "SINGLETONMARY" }, + "ids": { + "": "SINGLETONMARY" + }, "patients": [ { - "ids": { "": "MARY" }, + "ids": { + "": "MARY" + }, "specimens": [ { "htsgetId": "30F9F3FED8F711ED8C35DBEF59E9F537", - "ids": { "": "HG00097" } + "ids": { + "": "HG00097" + } } ] } @@ -127,6 +164,8 @@ pub fn is_resolver_from_parts(resolver: &Resolver) -> bool { && resolver.substitution_string() == "HG00097/HG00097" && matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && resolver.allow_formats() == [Format::Bam] + && resolver.allow_reference_names() == &List(HashSet::from_iter(vec!["1".to_string()])) + && resolver.allow_interval() == Interval::new(Some(1), Some(10)) } pub fn is_manifest_resolvers(resolvers: Vec) -> bool { @@ -135,6 +174,8 @@ pub fn is_manifest_resolvers(resolvers: Vec) -> bool { resolver.substitution_string() == "HG00096/HG00096" && matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && resolver.allow_formats() == [Format::Bam] + && resolver.allow_reference_names() == &List(HashSet::from_iter(vec!["2".to_string()])) + && resolver.allow_interval() == Interval::new(None, Some(10)) }) && resolvers.iter().any(is_resolver_from_parts) && resolvers.iter().any(|resolver| { @@ -142,11 +183,15 @@ pub fn is_manifest_resolvers(resolvers: Vec) -> bool { resolver.substitution_string() == "HG00096/HG00096.hard-filtered" && matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && resolver.allow_formats() == [Format::Vcf] + && resolver.allow_reference_names() == &List(HashSet::from_iter(vec!["4".to_string()])) + && resolver.allow_interval() == Interval::new(None, None) }) && resolvers.iter().any(|resolver| { resolver.regex().to_string() == "^R004/30F9F3FED8F711ED8C35DBEF59E9F537$" && resolver.substitution_string() == "HG00097/HG00097.hard-filtered" && matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") && resolver.allow_formats() == [Format::Vcf] + && resolver.allow_reference_names() == &List(HashSet::from_iter(vec!["3".to_string()])) + && resolver.allow_interval() == Interval::new(Some(10), None) }) } From adbc3174ecc255df5eff017f85cc19e9de527bbc Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 1 Sep 2023 20:13:30 +1000 Subject: [PATCH 21/25] docs: add READMEs and update comments --- deploy/README.md | 166 +------------------------------ htsget-elsa-lambda/Cargo.toml | 1 + htsget-elsa-lambda/LICENSE | 21 ++++ htsget-elsa-lambda/README.md | 37 +++++++ htsget-elsa/Cargo.toml | 1 + htsget-elsa/LICENSE | 21 ++++ htsget-elsa/README.md | 78 +++++++++++++++ htsget-elsa/src/elsa_endpoint.rs | 1 + htsget-elsa/src/lib.rs | 8 ++ 9 files changed, 172 insertions(+), 162 deletions(-) create mode 100644 htsget-elsa-lambda/LICENSE create mode 100644 htsget-elsa-lambda/README.md create mode 100644 htsget-elsa/LICENSE create mode 100644 htsget-elsa/README.md diff --git a/deploy/README.md b/deploy/README.md index 6d1808f..aba26fc 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,164 +1,6 @@ -# Deployment of htsget-lambda +# Deployment of htsget-elsa-lambda -The [htsget-lambda] crate is a cloud-based implementation of [htsget-rs]. It uses AWS Lambda as the ticket server, and AWS S3 as the data block server. +This folder contains CDK deployment code for htsget-elsa-lambda. For now this is basically just a copy of the +main htsget-rs [deployment][deployment], and those instructions apply here as well. -This is an example that deploys [htsget-lambda] using [aws-cdk]. It is deployed as an AWS HTTP [API Gateway Lambda proxy -integration][aws-api-gateway]. The stack uses [RustFunction][rust-function] in order to integrate [htsget-lambda] -with API Gateway. It uses a [JWT authorizer][jwt-authorizer] with [AWS Cognito][aws-cognito] as the issuer, and routes -the htsget-rs server with [AWS Route 53][route-53]. - -## Configuration - -To configure the deployment change the config files in the [`config`][config] directory. There are two configuration files -corresponding to the deployed environment. The config file used for deployment can be controlled by passing `--context "env=dev"` or -`--context "env=prod"` to cdk. When no context parameter is supplied, the default context is `dev`. - -These config files configure [htsget-lambda]. See [htsget-config] for a list of available configuration options. - -There is additional config that modifies the AWS infrastructure surrounding [htsget-lambda]. This config is sourced -from [AWS System Manager Parameter Store][aws-ssm]. Parameters are fetched using [`StringParemeter.valueFromLookup`][cdk-lookup-value] -which takes a parameter name. The properties in [`cdk.json`][cdk-json] `parameter_store_names` hold these parameter names, -which correspond to the following values: - -| Property storing SSM parameter name | Description of SSM parameter | -| ----------------------------------- | ------------------------------------------------------ | -| `arn_cert` | The ARN for the ACM certificate of the htsget domain. | -| `jwt_aud` | The JWT [audience][jwt-audience] for the token. | -| `cog_user_pool_id` | The Cognito user pool id which controls authorization. | -| `htsget_domain` | The domain name for the htsget server. | -| `hosted_zone_id` | The Route 53 hosted zone id for the htsget server. | -| `hosted_zone_name` | The Route 53 hosted zone name for the htsget server. | - -Modify these properties to change which parameter the values come from. SSM parameters can also be cached locally using -the [CDK runtime context][cdk-context]. - -[htsget-rs]: https://github.com/umccr/htsget-rs -[htsget-lambda]: ../htsget-lambda -[htsget-config]: ../htsget-config -[config]: config -[aws-cdk]: https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html -[cdk-context]: https://docs.aws.amazon.com/cdk/v2/guide/context.html -[cdk-lookup-value]: https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ssm.StringParameter.html#static-valuewbrfromwbrlookupscope-parametername -[cdk-json]: cdk.json -[aws-ssm]: https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html -[aws-api-gateway]: https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html -[aws-cognito]: https://docs.aws.amazon.com/cognito/latest/developerguide/cognito-user-identity-pools.html -[jwt-authorizer]: https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-jwt-authorizer.html -[jwt-audience]: https://docs.aws.amazon.com/apigatewayv2/latest/api-reference/apis-apiid-authorizers-authorizerid.html#apis-apiid-authorizers-authorizerid-model-jwtconfiguration -[route-53]: https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/Welcome.html -[rust-function]: https://www.npmjs.com/package/rust.aws-cdk-lambda - -## Deploying - -### Prerequisites - -- [aws-cli] should be installed and authenticated in the shell. -- Node.js and [npm] should be installed. -- [Rust][rust] should be installed. - -After installing the basic dependencies, complete the following steps: - -1. Add the arm cross-compilation target to rust. -2. Install [Zig][zig] using one of the methods show in [getting started][zig-getting-started], or by running the commands below and following the prompts. Zig is used by cargo-lambda for cross-compilation. -3. Install [cargo-lambda], as it is used to compile artifacts that are uploaded to aws lambda. -4. Install packages from this directory and compile [htsget-lambda]. This should place artifacts compiled for arm64 under the `target/lambda` directory which can be deployed to AWS. - -Below is a summary of commands to run in this directory: - -```sh -rustup target add aarch64-unknown-linux-gnu -cargo install cargo-lambda -npm install - -cd .. -cargo lambda build --release --arm64 --bin htsget-elsa-lambda -cd deploy -``` - -[aws-cdk]: https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html -[aws-cli]: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html -[npm]: https://docs.npmjs.com/downloading-and-installing-node-js-and-npm -[rust]: https://www.rust-lang.org/tools/install -[zig]: https://ziglang.org/ -[zig-getting-started]: https://ziglang.org/learn/getting-started/ - -### Deploy to AWS - -CDK will run many of the commands above again. However, it is recommended to run them once before trying the commands below, -to ensure that prerequisites are met. - -CDK should be bootstrapped once, if this hasn't been done before. - -```sh -npx cdk bootstrap -``` - -In order to deploy, check that the -stack synthesizes correctly and then deploy. - -```sh -npx cdk synth -npx cdk deploy -``` - -### Testing the endpoint - -When the deployment is finished, the htsget endpoint can be tested by querying it. Since a JWT authorizer is used, -a valid JWT token must be obtained in order to access the endpoint. This token should be obtained from AWS Cognito using -the configured user pool id and audience parameters. Then `curl` can be used to query the endpoint: - -```sh -curl -H "Authorization: " "https:///reads/service-info" -``` - -With a possible output: - -```json -{ - "id": "", - "name": "", - "version": "", - "organization": { - "name": "", - "url": "" - }, - "type": { - "group": "", - "artifact": "", - "version": "" - }, - "htsget": { - "datatype": "reads", - "formats": ["BAM", "CRAM"], - "fieldsParametersEffective": false, - "TagsParametersEffective": false - }, - "contactUrl": "", - "documentationUrl": "", - "createdAt": "", - "UpdatedAt": "", - "environment": "" -} -``` - -[awscurl]: https://github.com/okigan/awscurl - -### Local testing - -The [Lambda][htsget-lambda] function can also be run locally using [cargo-lambda]. From the root project directory, execute the following command. - -```sh -cargo lambda watch -``` - -Then in a **separate terminal session** run. - -```sh -cargo lambda invoke htsget-lambda --data-file data/events/event_elsa.json -``` - -Examples of different Lambda events are located in the [`data/events`][data-events] directory. - -[htsget-lambda]: ../htsget-lambda -[cargo-lambda]: https://github.com/cargo-lambda/cargo-lambda -[data-events]: ../data/events +[deployment]: https://github.com/umccr/htsget-rs/tree/main/deploy \ No newline at end of file diff --git a/htsget-elsa-lambda/Cargo.toml b/htsget-elsa-lambda/Cargo.toml index 68c8865..84a1b6a 100644 --- a/htsget-elsa-lambda/Cargo.toml +++ b/htsget-elsa-lambda/Cargo.toml @@ -1,6 +1,7 @@ [package] name = "htsget-elsa-lambda" authors = ["Marko Malenic "] +description = "A lambda function for htsget-rs to communicate with Elsa" version = "0.1.0" edition = "2021" diff --git a/htsget-elsa-lambda/LICENSE b/htsget-elsa-lambda/LICENSE new file mode 100644 index 0000000..97dd6a8 --- /dev/null +++ b/htsget-elsa-lambda/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 UMCCR + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/htsget-elsa-lambda/README.md b/htsget-elsa-lambda/README.md new file mode 100644 index 0000000..61b36d7 --- /dev/null +++ b/htsget-elsa-lambda/README.md @@ -0,0 +1,37 @@ +# htsget-elsa-rs +A htsget-rs lambda function for interacting with Elsa + +[![MIT licensed][mit-badge]][mit-url] +[![Build Status][actions-badge]][actions-url] + +[mit-badge]: https://img.shields.io/badge/license-MIT-blue.svg +[mit-url]: https://github.com/umccr/htsget-rs/blob/main/LICENSE +[actions-badge]: https://github.com/umccr/htsget-rs/actions/workflows/action.yml/badge.svg +[actions-url]: https://github.com/umccr/htsget-rs/actions?query=workflow%3Atests+branch%3Amain + +This crate is deployed as a lambda function that wraps htsget-rs and allows it to interact with Elsa. + +The lambda function first attempts to resolve the query using the manifest files from Elsa where the first path segment +in the query matches the release key in the manifest file. For example, the following matches the release key and file name: + +``` +GET https:///reads//?format=BAM&referenceName=1&start=0&end=1000 +``` + +This function supports all the regular htsget-rs configuration options, and also includes the following to configure the +Elsa endpoint and cache location: + +| Option | Description | Type | Default | Example | +|---------------------------|------------------------------------------------------|---------------|---------------------------|-----------------------------| +| `elsa_endpoint_authority` | The URL authority of the Elsa endpoint. | URL Authority | Not specified, required. | `'elsa-data.dev.umccr.org'` | +| `cache_location` | The name of the bucket where resolvers are cached. | String | Not specified, required. | `'cache_bucket'` | + +To deploy this function, see the [deploy][deploy] folder. + +[deploy]: ../deploy + +## License + +This project is licensed under the [MIT license][license]. + +[license]: LICENSE \ No newline at end of file diff --git a/htsget-elsa/Cargo.toml b/htsget-elsa/Cargo.toml index e3c4e56..b51035b 100644 --- a/htsget-elsa/Cargo.toml +++ b/htsget-elsa/Cargo.toml @@ -1,6 +1,7 @@ [package] name = "htsget-elsa" authors = ["Marko Malenic "] +description = "The htsget-rs library code used to interact with Elsa" version = "0.1.0" edition = "2021" diff --git a/htsget-elsa/LICENSE b/htsget-elsa/LICENSE new file mode 100644 index 0000000..97dd6a8 --- /dev/null +++ b/htsget-elsa/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 UMCCR + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/htsget-elsa/README.md b/htsget-elsa/README.md new file mode 100644 index 0000000..e7abbef --- /dev/null +++ b/htsget-elsa/README.md @@ -0,0 +1,78 @@ +# htsget-elsa + +[![MIT licensed][mit-badge]][mit-url] +[![Build Status][actions-badge]][actions-url] + +[mit-badge]: https://img.shields.io/badge/license-MIT-blue.svg +[mit-url]: https://github.com/umccr/htsget-rs/blob/main/LICENSE +[actions-badge]: https://github.com/umccr/htsget-rs/actions/workflows/action.yml/badge.svg +[actions-url]: https://github.com/umccr/htsget-rs/actions?query=workflow%3Atests+branch%3Amain + +This crate is library code that allows htsget-rs to interact with Elsa. + +Elsa is a genomic data sharing support software that allows access to genomic datasets using various sharing mechanisms. +A sharing mechanism that Elsa can use to define which users get access to genomic data is htsget. Elsa can restrict access based on +files, formats, genomic regions and start and end positions using htsget. + +## Mechanism + +Elsa and htsget-rs interact in the following way: + +* Elsa has a public endpoint which returns information for a manifest file that htsget-elsa can use. +* htsget-elsa calls this endpoint with a GET request at: `/api/manifest/htsget/?type=S3`. +* The response is a JSON object with the following structure: +```json +{ + "location": { + "bucket": "bucket", + "key": "key" + }, + "maxAge": 800 +} +``` +* This object can be used by htsget-elsa to fetch the manifest file from S3, which has the following structure: +```json +{ + "id": "R001", + "reads": { + "C618CDEE119511EE8A2DFFDE5D69679D": { + "url": "s3://url/to/file.bam", + "restrictions": [ + { "chromosome": 1, "start": 0, "end": 1000 } + ] + } + }, + "variants": { + "C618CDEE119511EE8A2DFFDE5D69679D": { + "url": "s3://url/to/file.vcf.gz", + "variantSampleId": "", + "restrictions": [ + { "chromosome": 9, "start": 130713043, "end": 130887675 } + ] + } + } +} +``` +* The manifest file is used by htsget-elsa to create resolvers, which match the restrictions on the urls and are used by + htsget-rs when resolving queries. +* It is also cached by htsget-elsa in S3 based on the `maxAge`. + +## Layout + +This crate has a few components which form the basis of the htsget-rs and Elsa interaction: +* The `GetObject` trait is used by htsget-elsa to get request to cloud storage. For now this is only S3, but it could be + extend to other cloud providers. +* The `Cache` trait is used to cache the resolvers once they have been processed. This only caches back to S3, but it + also use other mechanisms, including databases such as DynamoDB. +* The `ResolversFromElsa` represents the whole mechanism as described above, and is implemented by the `ElsaEndpoint` struct. + +#### Feature flags + +This crate has the following features: +* `test-utils`: used to enable test dependencies. + +## License + +This project is licensed under the [MIT license][license]. + +[license]: LICENSE \ No newline at end of file diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index 4ffd83a..44e4c6e 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -170,6 +170,7 @@ impl TryFrom for Vec { } } +/// Implements the mechanism which fetches manifests from Elsa. #[derive(Debug)] pub struct ElsaEndpoint<'a, C, S> { endpoint: Authority, diff --git a/htsget-elsa/src/lib.rs b/htsget-elsa/src/lib.rs index 452704b..c6a1395 100644 --- a/htsget-elsa/src/lib.rs +++ b/htsget-elsa/src/lib.rs @@ -36,15 +36,19 @@ pub enum Error { SystemError(String), } +/// Cache resolver objects to a cache. #[async_trait] pub trait Cache { type Error; type Item; + /// Get the resolvers from the cache. async fn get + Send + Sync>( &self, key: K, ) -> result::Result, Self::Error>; + + /// Put the resolvers in the cache. async fn put + Send + Sync>( &self, key: K, @@ -53,10 +57,12 @@ pub trait Cache { ) -> result::Result<(), Self::Error>; } +/// Get objects from cloud storage. #[async_trait] pub trait GetObject { type Error; + /// Get the object. async fn get_object Deserialize<'de>>( &self, bucket: impl Into + Send, @@ -64,9 +70,11 @@ pub trait GetObject { ) -> result::Result; } +/// Get resolvers from Elsa. #[async_trait] pub trait ResolversFromElsa { type Error; + /// Get the resolvers from Elsa using the release key. async fn try_get(&self, release_key: String) -> result::Result, Self::Error>; } From c94c9a46993a5a4c6fe7d32bcf2aee79f93aad2e Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Fri, 1 Sep 2023 21:43:56 +1000 Subject: [PATCH 22/25] ci: add test workflow --- .github/workflows/tests.yml | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..6f0b92a --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,45 @@ +name: tests + +on: + push: + branches: + - main + pull_request: + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + rust: [stable] + os: [ubuntu-latest] + steps: + - name: Check out + uses: actions/checkout@v3 + - name: Install Rust + uses: dtolnay/rust-toolchain@master + id: toolchain + with: + toolchain: ${{ matrix.rust }} + components: rustfmt, clippy + - run: rustup override set ${{ steps.toolchain.outputs.name }} + - name: Cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: build-cache + save-if: false + - name: Cargo fmt + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --all -- --check + - name: Cargo clippy + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --all-targets --all-features -- -D warnings + - name: Run cargo tests + uses: actions-rs/cargo@v1 + with: + command: test + args: --all-features \ No newline at end of file From 08bc251326d483f13bfd664e35e5dbc44efa9c1a Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 4 Sep 2023 08:28:43 +1000 Subject: [PATCH 23/25] style: formatting changes and comments --- .github/workflows/tests.yml | 2 +- .gitignore | 11 +- data/events/event_elsa.json | 2 +- data/events/event_get.json | 2 +- deploy/README.md | 4 +- deploy/package-lock.json | 366 ++++++++++++++++++++++++++----- htsget-elsa-lambda/Cargo.toml | 2 +- htsget-elsa-lambda/src/config.rs | 13 +- htsget-elsa-lambda/src/lib.rs | 4 + htsget-elsa/Cargo.toml | 2 +- htsget-elsa/README.md | 10 +- htsget-elsa/src/elsa_endpoint.rs | 22 +- htsget-elsa/src/lib.rs | 1 + htsget-elsa/src/s3.rs | 7 + htsget-elsa/src/test_utils.rs | 12 +- 15 files changed, 378 insertions(+), 82 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6f0b92a..3209754 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,4 +42,4 @@ jobs: uses: actions-rs/cargo@v1 with: command: test - args: --all-features \ No newline at end of file + args: --all-features diff --git a/.gitignore b/.gitignore index ea8c4bf..73fab07 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,10 @@ -/target +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb diff --git a/data/events/event_elsa.json b/data/events/event_elsa.json index 4edfdb0..c816f21 100644 --- a/data/events/event_elsa.json +++ b/data/events/event_elsa.json @@ -99,4 +99,4 @@ "apiId": "70ixmpl4fl" }, "isBase64Encoded": false -} \ No newline at end of file +} diff --git a/data/events/event_get.json b/data/events/event_get.json index 2b65f35..3a367f4 100644 --- a/data/events/event_get.json +++ b/data/events/event_get.json @@ -99,4 +99,4 @@ "apiId": "70ixmpl4fl" }, "isBase64Encoded": false -} \ No newline at end of file +} diff --git a/deploy/README.md b/deploy/README.md index aba26fc..155dab3 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,6 +1,6 @@ # Deployment of htsget-elsa-lambda -This folder contains CDK deployment code for htsget-elsa-lambda. For now this is basically just a copy of the +This folder contains CDK deployment code for htsget-elsa-lambda. For now this is just a copy of the main htsget-rs [deployment][deployment], and those instructions apply here as well. -[deployment]: https://github.com/umccr/htsget-rs/tree/main/deploy \ No newline at end of file +[deployment]: https://github.com/umccr/htsget-rs/tree/main/deploy diff --git a/deploy/package-lock.json b/deploy/package-lock.json index fe2ba83..780c4c4 100644 --- a/deploy/package-lock.json +++ b/deploy/package-lock.json @@ -28,19 +28,19 @@ } }, "node_modules/@aws-cdk/asset-awscli-v1": { - "version": "2.2.63", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.63.tgz", - "integrity": "sha512-LP+RSwH6JCx2T/AJyaveRg8VRvrM612TxGnec8SVuf9pQ6ah82LcW+4abuMhizeXDcx2ZufwNf9g/G2rp47bjA==" + "version": "2.2.200", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.200.tgz", + "integrity": "sha512-Kf5J8DfJK4wZFWT2Myca0lhwke7LwHcHBo+4TvWOGJrFVVKVuuiLCkzPPRBQQVDj0Vtn2NBokZAz8pfMpAqAKg==" }, "node_modules/@aws-cdk/asset-kubectl-v20": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.1.tgz", - "integrity": "sha512-U1ntiX8XiMRRRH5J1IdC+1t5CE89015cwyt5U63Cpk0GnMlN5+h9WsWMlKlPXZR4rdq/m806JRlBMRpBUB2Dhw==" + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.2.tgz", + "integrity": "sha512-3M2tELJOxQv0apCIiuKQ4pAbncz9GuLwnKFqxifWfe77wuMxyTRPmxssYHs42ePqzap1LT6GDcPygGs+hHstLg==" }, - "node_modules/@aws-cdk/asset-node-proxy-agent-v5": { - "version": "2.0.52", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v5/-/asset-node-proxy-agent-v5-2.0.52.tgz", - "integrity": "sha512-sQhGP8QhAeP3ZsXxoOQrRFW/0VMH3HvI8NjZe/9ORtKwh8Gyjv2BJWIxlXLvRcc2o26FEl1oHIKRbaK/z/4iMw==" + "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.0.1.tgz", + "integrity": "sha512-DDt4SLdLOwWCjGtltH4VCST7hpOI5DzieuhGZsBpZ+AgJdSI2GCjklCXm0GCTwJG/SolkL5dtQXyUKgg9luBDg==" }, "node_modules/@aws-cdk/aws-apigatewayv2-alpha": { "version": "2.59.0-alpha.0", @@ -187,9 +187,9 @@ } }, "node_modules/aws-cdk-lib": { - "version": "2.64.0", - "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.64.0.tgz", - "integrity": "sha512-IrgL7thb6TeOyHgyR/qKWTdA9FBb9lv7Z9QPDzCNJlkKI+0ANjYHy3RYV8Gd+1+kc6l8DG9Z1elij40YCr/Ptg==", + "version": "2.94.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.94.0.tgz", + "integrity": "sha512-pB/UzKeM+p/wY9WuFYkEewOFUh2r8qwaML63is4vUChXY2G2Bj3pGyfJ97Xir2Q5KIhgJPJz5igdouI4+F9A+g==", "bundleDependencies": [ "@balena/dockerignore", "case", @@ -199,20 +199,22 @@ "minimatch", "punycode", "semver", + "table", "yaml" ], "dependencies": { - "@aws-cdk/asset-awscli-v1": "^2.2.52", - "@aws-cdk/asset-kubectl-v20": "^2.1.1", - "@aws-cdk/asset-node-proxy-agent-v5": "^2.0.42", + "@aws-cdk/asset-awscli-v1": "^2.2.200", + "@aws-cdk/asset-kubectl-v20": "^2.1.2", + "@aws-cdk/asset-node-proxy-agent-v6": "^2.0.1", "@balena/dockerignore": "^1.0.2", "case": "1.6.3", - "fs-extra": "^9.1.0", + "fs-extra": "^11.1.1", "ignore": "^5.2.4", "jsonschema": "^1.4.1", "minimatch": "^3.1.2", "punycode": "^2.3.0", - "semver": "^7.3.8", + "semver": "^7.5.4", + "table": "^6.8.1", "yaml": "1.10.2" }, "engines": { @@ -227,12 +229,49 @@ "inBundle": true, "license": "Apache-2.0" }, - "node_modules/aws-cdk-lib/node_modules/at-least-node": { - "version": "1.0.0", + "node_modules/aws-cdk-lib/node_modules/ajv": { + "version": "8.12.0", "inBundle": true, - "license": "ISC", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/aws-cdk-lib/node_modules/ansi-regex": { + "version": "5.0.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/ansi-styles": { + "version": "4.3.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, "engines": { - "node": ">= 4.0.0" + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/aws-cdk-lib/node_modules/astral-regex": { + "version": "2.0.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=8" } }, "node_modules/aws-cdk-lib/node_modules/balanced-match": { @@ -257,27 +296,52 @@ "node": ">= 0.8.0" } }, + "node_modules/aws-cdk-lib/node_modules/color-convert": { + "version": "2.0.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/color-name": { + "version": "1.1.4", + "inBundle": true, + "license": "MIT" + }, "node_modules/aws-cdk-lib/node_modules/concat-map": { "version": "0.0.1", "inBundle": true, "license": "MIT" }, + "node_modules/aws-cdk-lib/node_modules/emoji-regex": { + "version": "8.0.0", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/fast-deep-equal": { + "version": "3.1.3", + "inBundle": true, + "license": "MIT" + }, "node_modules/aws-cdk-lib/node_modules/fs-extra": { - "version": "9.1.0", + "version": "11.1.1", "inBundle": true, "license": "MIT", "dependencies": { - "at-least-node": "^1.0.0", "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" }, "engines": { - "node": ">=10" + "node": ">=14.14" } }, "node_modules/aws-cdk-lib/node_modules/graceful-fs": { - "version": "4.2.10", + "version": "4.2.11", "inBundle": true, "license": "ISC" }, @@ -289,6 +353,19 @@ "node": ">= 4" } }, + "node_modules/aws-cdk-lib/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/json-schema-traverse": { + "version": "1.0.0", + "inBundle": true, + "license": "MIT" + }, "node_modules/aws-cdk-lib/node_modules/jsonfile": { "version": "6.1.0", "inBundle": true, @@ -308,6 +385,11 @@ "node": "*" } }, + "node_modules/aws-cdk-lib/node_modules/lodash.truncate": { + "version": "4.4.2", + "inBundle": true, + "license": "MIT" + }, "node_modules/aws-cdk-lib/node_modules/lru-cache": { "version": "6.0.0", "inBundle": true, @@ -338,8 +420,16 @@ "node": ">=6" } }, + "node_modules/aws-cdk-lib/node_modules/require-from-string": { + "version": "2.0.2", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/aws-cdk-lib/node_modules/semver": { - "version": "7.3.8", + "version": "7.5.4", "inBundle": true, "license": "ISC", "dependencies": { @@ -352,6 +442,61 @@ "node": ">=10" } }, + "node_modules/aws-cdk-lib/node_modules/slice-ansi": { + "version": "4.0.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "astral-regex": "^2.0.0", + "is-fullwidth-code-point": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/aws-cdk-lib/node_modules/string-width": { + "version": "4.2.3", + "inBundle": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/strip-ansi": { + "version": "6.0.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/table": { + "version": "6.8.1", + "inBundle": true, + "license": "BSD-3-Clause", + "dependencies": { + "ajv": "^8.0.1", + "lodash.truncate": "^4.4.2", + "slice-ansi": "^4.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/aws-cdk-lib/node_modules/universalify": { "version": "2.0.0", "inBundle": true, @@ -360,6 +505,14 @@ "node": ">= 10.0.0" } }, + "node_modules/aws-cdk-lib/node_modules/uri-js": { + "version": "4.4.1", + "inBundle": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, "node_modules/aws-cdk-lib/node_modules/yallist": { "version": "4.0.0", "inBundle": true, @@ -979,9 +1132,9 @@ } }, "node_modules/yaml": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.1.tgz", - "integrity": "sha512-e0WHiYql7+9wr4cWMx3TVQrNwejKaEe7/rHNmQmqRjazfOP5W8PB6Jpebb5o6fIapbz9o9+2ipcaTM2ZwDI6lw==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.2.tgz", + "integrity": "sha512-N/lyzTPaJasoDmfV7YTrYCI0G/3ivm/9wdG0aHuheKowWQwGTsK0Eoiw6utmzAnI6pkJa0DUVygvp3spqqEKXg==", "optional": true, "engines": { "node": ">= 14" @@ -1024,19 +1177,19 @@ }, "dependencies": { "@aws-cdk/asset-awscli-v1": { - "version": "2.2.63", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.63.tgz", - "integrity": "sha512-LP+RSwH6JCx2T/AJyaveRg8VRvrM612TxGnec8SVuf9pQ6ah82LcW+4abuMhizeXDcx2ZufwNf9g/G2rp47bjA==" + "version": "2.2.200", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.200.tgz", + "integrity": "sha512-Kf5J8DfJK4wZFWT2Myca0lhwke7LwHcHBo+4TvWOGJrFVVKVuuiLCkzPPRBQQVDj0Vtn2NBokZAz8pfMpAqAKg==" }, "@aws-cdk/asset-kubectl-v20": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.1.tgz", - "integrity": "sha512-U1ntiX8XiMRRRH5J1IdC+1t5CE89015cwyt5U63Cpk0GnMlN5+h9WsWMlKlPXZR4rdq/m806JRlBMRpBUB2Dhw==" + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-kubectl-v20/-/asset-kubectl-v20-2.1.2.tgz", + "integrity": "sha512-3M2tELJOxQv0apCIiuKQ4pAbncz9GuLwnKFqxifWfe77wuMxyTRPmxssYHs42ePqzap1LT6GDcPygGs+hHstLg==" }, - "@aws-cdk/asset-node-proxy-agent-v5": { - "version": "2.0.52", - "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v5/-/asset-node-proxy-agent-v5-2.0.52.tgz", - "integrity": "sha512-sQhGP8QhAeP3ZsXxoOQrRFW/0VMH3HvI8NjZe/9ORtKwh8Gyjv2BJWIxlXLvRcc2o26FEl1oHIKRbaK/z/4iMw==" + "@aws-cdk/asset-node-proxy-agent-v6": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.0.1.tgz", + "integrity": "sha512-DDt4SLdLOwWCjGtltH4VCST7hpOI5DzieuhGZsBpZ+AgJdSI2GCjklCXm0GCTwJG/SolkL5dtQXyUKgg9luBDg==" }, "@aws-cdk/aws-apigatewayv2-alpha": { "version": "2.59.0-alpha.0", @@ -1139,21 +1292,22 @@ } }, "aws-cdk-lib": { - "version": "2.64.0", - "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.64.0.tgz", - "integrity": "sha512-IrgL7thb6TeOyHgyR/qKWTdA9FBb9lv7Z9QPDzCNJlkKI+0ANjYHy3RYV8Gd+1+kc6l8DG9Z1elij40YCr/Ptg==", + "version": "2.94.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.94.0.tgz", + "integrity": "sha512-pB/UzKeM+p/wY9WuFYkEewOFUh2r8qwaML63is4vUChXY2G2Bj3pGyfJ97Xir2Q5KIhgJPJz5igdouI4+F9A+g==", "requires": { - "@aws-cdk/asset-awscli-v1": "^2.2.52", - "@aws-cdk/asset-kubectl-v20": "^2.1.1", - "@aws-cdk/asset-node-proxy-agent-v5": "^2.0.42", + "@aws-cdk/asset-awscli-v1": "^2.2.200", + "@aws-cdk/asset-kubectl-v20": "^2.1.2", + "@aws-cdk/asset-node-proxy-agent-v6": "^2.0.1", "@balena/dockerignore": "^1.0.2", "case": "1.6.3", - "fs-extra": "^9.1.0", + "fs-extra": "^11.1.1", "ignore": "^5.2.4", "jsonschema": "^1.4.1", "minimatch": "^3.1.2", "punycode": "^2.3.0", - "semver": "^7.3.8", + "semver": "^7.5.4", + "table": "^6.8.1", "yaml": "1.10.2" }, "dependencies": { @@ -1161,8 +1315,29 @@ "version": "1.0.2", "bundled": true }, - "at-least-node": { - "version": "1.0.0", + "ajv": { + "version": "8.12.0", + "bundled": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + } + }, + "ansi-regex": { + "version": "5.0.1", + "bundled": true + }, + "ansi-styles": { + "version": "4.3.0", + "bundled": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "astral-regex": { + "version": "2.0.0", "bundled": true }, "balanced-match": { @@ -1181,28 +1356,54 @@ "version": "1.6.3", "bundled": true }, + "color-convert": { + "version": "2.0.1", + "bundled": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "bundled": true + }, "concat-map": { "version": "0.0.1", "bundled": true }, + "emoji-regex": { + "version": "8.0.0", + "bundled": true + }, + "fast-deep-equal": { + "version": "3.1.3", + "bundled": true + }, "fs-extra": { - "version": "9.1.0", + "version": "11.1.1", "bundled": true, "requires": { - "at-least-node": "^1.0.0", "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", "universalify": "^2.0.0" } }, "graceful-fs": { - "version": "4.2.10", + "version": "4.2.11", "bundled": true }, "ignore": { "version": "5.2.4", "bundled": true }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "bundled": true + }, + "json-schema-traverse": { + "version": "1.0.0", + "bundled": true + }, "jsonfile": { "version": "6.1.0", "bundled": true, @@ -1215,6 +1416,10 @@ "version": "1.4.1", "bundled": true }, + "lodash.truncate": { + "version": "4.4.2", + "bundled": true + }, "lru-cache": { "version": "6.0.0", "bundled": true, @@ -1233,17 +1438,64 @@ "version": "2.3.0", "bundled": true }, + "require-from-string": { + "version": "2.0.2", + "bundled": true + }, "semver": { - "version": "7.3.8", + "version": "7.5.4", "bundled": true, "requires": { "lru-cache": "^6.0.0" } }, + "slice-ansi": { + "version": "4.0.0", + "bundled": true, + "requires": { + "ansi-styles": "^4.0.0", + "astral-regex": "^2.0.0", + "is-fullwidth-code-point": "^3.0.0" + } + }, + "string-width": { + "version": "4.2.3", + "bundled": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "bundled": true, + "requires": { + "ansi-regex": "^5.0.1" + } + }, + "table": { + "version": "6.8.1", + "bundled": true, + "requires": { + "ajv": "^8.0.1", + "lodash.truncate": "^4.4.2", + "slice-ansi": "^4.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1" + } + }, "universalify": { "version": "2.0.0", "bundled": true }, + "uri-js": { + "version": "4.4.1", + "bundled": true, + "requires": { + "punycode": "^2.1.0" + } + }, "yallist": { "version": "4.0.0", "bundled": true @@ -1659,9 +1911,9 @@ } }, "yaml": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.1.tgz", - "integrity": "sha512-e0WHiYql7+9wr4cWMx3TVQrNwejKaEe7/rHNmQmqRjazfOP5W8PB6Jpebb5o6fIapbz9o9+2ipcaTM2ZwDI6lw==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.2.tgz", + "integrity": "sha512-N/lyzTPaJasoDmfV7YTrYCI0G/3ivm/9wdG0aHuheKowWQwGTsK0Eoiw6utmzAnI6pkJa0DUVygvp3spqqEKXg==", "optional": true }, "zx": { diff --git a/htsget-elsa-lambda/Cargo.toml b/htsget-elsa-lambda/Cargo.toml index 84a1b6a..f54f46e 100644 --- a/htsget-elsa-lambda/Cargo.toml +++ b/htsget-elsa-lambda/Cargo.toml @@ -22,4 +22,4 @@ htsget-elsa = { version = "0.1", path = "../htsget-elsa" } [dev-dependencies] htsget-elsa = { version = "0.1", path = "../htsget-elsa", features = ["test-utils"] } htsget-search = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa" } -htsget-test = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa", default-features = false, features = ["http-tests"] } \ No newline at end of file +htsget-test = { git = "https://github.com/umccr/htsget-rs.git", branch = "feat/htsget-elsa", default-features = false, features = ["http-tests"] } diff --git a/htsget-elsa-lambda/src/config.rs b/htsget-elsa-lambda/src/config.rs index c2cbbee..75e4617 100644 --- a/htsget-elsa-lambda/src/config.rs +++ b/htsget-elsa-lambda/src/config.rs @@ -5,8 +5,8 @@ use htsget_config::config::parser::from_path; use htsget_config::config::Config as HtsGetConfig; use http::uri::Authority; use serde::{Deserialize, Serialize}; -use tracing::instrument; +/// Configuration for htsget-elsa. Includes the standard HtsGetConfig. #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Config { #[serde(flatten, default)] @@ -17,6 +17,7 @@ pub struct Config { } impl Config { + /// Create a new config. pub fn new( htsget_config: HtsGetConfig, elsa_endpoint_authority: Authority, @@ -29,20 +30,26 @@ impl Config { } } + /// Get the standard htsget config. pub fn htsget_config(&self) -> &HtsGetConfig { &self.htsget_config } + /// Get the endpoint authority. pub fn elsa_endpoint_authority(&self) -> &Authority { &self.elsa_endpoint_authority } + /// Get the cache location. pub fn cache_location(&self) -> &str { &self.cache_location } +} + +impl TryFrom<&Path> for Config { + type Error = io::Error; - #[instrument(level = "debug", ret)] - pub fn from_path(path: &Path) -> io::Result { + fn try_from(path: &Path) -> io::Result { from_path(path) } } diff --git a/htsget-elsa-lambda/src/lib.rs b/htsget-elsa-lambda/src/lib.rs index 3b262a9..93af842 100644 --- a/htsget-elsa-lambda/src/lib.rs +++ b/htsget-elsa-lambda/src/lib.rs @@ -15,6 +15,7 @@ use crate::config::Config; pub mod config; +/// The request handler. pub async fn handle_request(config: Config) -> Result<(), Error> { handle_request_service_fn( config.htsget_config().ticket_server().cors().clone(), @@ -44,6 +45,7 @@ pub async fn handle_request(config: Config) -> Result<(), Error> { .await } +/// Route the request with the Elsa endpoint. pub async fn route_request<'a>( config: &Config, event: Request, @@ -59,6 +61,8 @@ pub async fn route_request<'a>( router.route_request_with_route(event, route).await } +/// Get the resolvers for this route. First tries htsget-elsa, then falls back to the +/// standard htsget-rs resolvers. #[instrument(level = "debug", skip(elsa_endpoint), ret)] pub async fn get_resolvers<'a, C, S>( config: &Config, diff --git a/htsget-elsa/Cargo.toml b/htsget-elsa/Cargo.toml index b51035b..eb574f6 100644 --- a/htsget-elsa/Cargo.toml +++ b/htsget-elsa/Cargo.toml @@ -30,4 +30,4 @@ wiremock = { version = "0.5", optional = true } [dev-dependencies] tokio = { version = "1.25", features = ["macros", "rt-multi-thread"] } -htsget-elsa = { path = ".", features = ["test-utils"] } \ No newline at end of file +htsget-elsa = { path = ".", features = ["test-utils"] } diff --git a/htsget-elsa/README.md b/htsget-elsa/README.md index e7abbef..edad8e4 100644 --- a/htsget-elsa/README.md +++ b/htsget-elsa/README.md @@ -59,17 +59,17 @@ Elsa and htsget-rs interact in the following way: ## Layout -This crate has a few components which form the basis of the htsget-rs and Elsa interaction: -* The `GetObject` trait is used by htsget-elsa to get request to cloud storage. For now this is only S3, but it could be - extend to other cloud providers. -* The `Cache` trait is used to cache the resolvers once they have been processed. This only caches back to S3, but it +This crate has a few components implement the htsget-rs and Elsa interaction: +* The `GetObject` trait is used by htsget-elsa to request data from cloud storage. For now this is only S3, but it could be + extended to other cloud providers. +* The `Cache` trait is used to cache the resolvers once they have been processed. This only caches back to S3, but it could also use other mechanisms, including databases such as DynamoDB. * The `ResolversFromElsa` represents the whole mechanism as described above, and is implemented by the `ElsaEndpoint` struct. #### Feature flags This crate has the following features: -* `test-utils`: used to enable test dependencies. +* `test-utils`: used to enable test dependencies and functions. ## License diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index 44e4c6e..367dcda 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -19,6 +19,7 @@ use crate::{Cache, Error, GetObject, ResolversFromElsa, Result}; pub const ENDPOINT_PATH: &str = "/api/manifest/htsget"; pub const CACHE_PATH: &str = "htsget-manifest-cache"; +/// The location of the manifest. #[derive(Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ElsaLocation { @@ -26,6 +27,7 @@ pub struct ElsaLocation { key: String, } +/// Defines the response that htsget-elsa expects from Elsa when requesting the manifest. #[derive(Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ElsaResponse { @@ -33,6 +35,7 @@ pub struct ElsaResponse { max_age: u64, } +/// The format of reads in the manifest. #[derive(Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ElsaReadsManifest { @@ -41,6 +44,7 @@ pub struct ElsaReadsManifest { restrictions: Vec, } +/// The format of variants in the manifest. #[derive(Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ElsaVariantsManifest { @@ -50,6 +54,7 @@ pub struct ElsaVariantsManifest { restrictions: Vec, } +/// The format of the restrictions in the manifest. #[derive(Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ElsaRestrictionManifest { @@ -58,6 +63,7 @@ pub struct ElsaRestrictionManifest { end: Option, } +/// This defines the manifest format that htsget-elsa expects from Elsa. #[derive(Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ElsaManifest { @@ -68,6 +74,7 @@ pub struct ElsaManifest { } impl ElsaManifest { + /// Creates a resolver from the different parts of the manifest. #[instrument(level = "trace", ret)] pub fn resolver_from_manifest_parts( release_key: &str, @@ -170,7 +177,7 @@ impl TryFrom for Vec { } } -/// Implements the mechanism which fetches manifests from Elsa. +/// Implements fetching manifests from Elsa. #[derive(Debug)] pub struct ElsaEndpoint<'a, C, S> { endpoint: Authority, @@ -217,6 +224,7 @@ where C: Cache, Error = Error>, S: GetObject, { + /// Create an ElsaEndpoint. pub fn new(endpoint: Authority, cache: &'a C, get_object: &'a S) -> Result { Ok(Self { client: Self::create_client()?, @@ -227,6 +235,7 @@ where }) } + /// Create an ElsaEndpoint with a client. Used for testing where TLS is not required. #[cfg(feature = "test-utils")] pub fn new_with_client( client: Client, @@ -283,13 +292,14 @@ where } } + /// Call the Elsa endpoint and get the response. #[instrument(level = "debug", skip(self), ret)] pub async fn get_response(&self, release_key: &str) -> Result { self.get_response_with_scheme(release_key, self.scheme) .await } - #[instrument(level = "debug", skip(self), ret)] + /// Convert a response to a manifest by fetching it from the storage. pub async fn get_manifest(&self, response: ElsaResponse) -> Result { self.get_object .get_object(response.location.bucket, response.location.key) @@ -312,7 +322,7 @@ mod tests { use crate::s3::S3; use crate::test_utils::{ example_elsa_manifest, example_elsa_response, is_manifest_resolvers, - is_resolver_from_parts, with_test_mocks, + is_reads_resolver_from_parts, with_test_mocks, }; use crate::Error::{GetObjectError, InvalidManifest, UnsupportedManifestFeature}; use crate::{Cache, ResolversFromElsa}; @@ -464,7 +474,7 @@ mod tests { &example_restrictions_manifest(), ) .unwrap(); - assert!(is_resolver_from_parts(&response)); + assert!(is_reads_resolver_from_parts(&response)); } #[test] @@ -477,7 +487,7 @@ mod tests { &example_restrictions_manifest(), ) .unwrap(); - assert!(is_resolver_from_parts(&response)); + assert!(is_reads_resolver_from_parts(&response)); } #[test] @@ -490,7 +500,7 @@ mod tests { &example_restrictions_manifest(), ) .unwrap(); - assert!(is_resolver_from_parts(&response)); + assert!(is_reads_resolver_from_parts(&response)); } #[test] diff --git a/htsget-elsa/src/lib.rs b/htsget-elsa/src/lib.rs index c6a1395..15f6bf6 100644 --- a/htsget-elsa/src/lib.rs +++ b/htsget-elsa/src/lib.rs @@ -12,6 +12,7 @@ pub mod test_utils; pub type Result = result::Result; +/// Main error type for this crate. #[derive(Error, Debug)] pub enum Error { #[error("invalid client: `{0}`")] diff --git a/htsget-elsa/src/s3.rs b/htsget-elsa/src/s3.rs index 49e578e..2083b2b 100644 --- a/htsget-elsa/src/s3.rs +++ b/htsget-elsa/src/s3.rs @@ -13,12 +13,14 @@ use tracing::{instrument, trace}; use crate::Error::{DeserializeError, GetObjectError, PutObjectError, SerializeError}; use crate::{Cache, Error, GetObject, Result}; +/// S3 storage implementation. #[derive(Debug)] pub struct S3 { s3_client: Client, cache_bucket: String, } +/// The shape of the item to cache. #[derive(Debug, Deserialize, Serialize)] pub struct CacheItem { item: Vec, @@ -26,6 +28,7 @@ pub struct CacheItem { } impl S3 { + /// Create a new S3 storage. pub fn new(s3_client: Client, cache_bucket: String) -> Self { Self { s3_client, @@ -33,6 +36,7 @@ impl S3 { } } + /// Create a new S3 storage with default AWS config. pub async fn new_with_default_config(cache_bucket: String) -> Self { Self::new( Client::new(&aws_config::load_from_env().await), @@ -43,6 +47,7 @@ impl S3 { impl S3 { /// Get the last modified date of the object. + #[instrument(level = "trace", skip_all)] async fn last_modified( &self, bucket: impl Into + Send, @@ -58,6 +63,8 @@ impl S3 { .and_then(|output| output.last_modified) } + /// Execute a get object request. + #[instrument(level = "trace", skip_all)] async fn get_object Deserialize<'de>>( &self, bucket: impl Into + Send, diff --git a/htsget-elsa/src/test_utils.rs b/htsget-elsa/src/test_utils.rs index 1628b7b..a8b7092 100644 --- a/htsget-elsa/src/test_utils.rs +++ b/htsget-elsa/src/test_utils.rs @@ -13,6 +13,7 @@ use htsget_test::aws_mocks::with_s3_test_server_tmp; use wiremock::matchers::{method, path, query_param}; use wiremock::{Mock, MockServer, Request, ResponseTemplate, Times}; +/// An example Elsa manifest file. pub fn example_elsa_manifest() -> String { r#" { @@ -106,7 +107,8 @@ pub fn example_elsa_manifest() -> String { .to_string() } -pub(crate) fn example_elsa_response() -> String { +/// An example GET response from Elsa. +pub fn example_elsa_response() -> String { r#" { "location": { @@ -119,11 +121,13 @@ pub(crate) fn example_elsa_response() -> String { .to_string() } +/// Writes the example manifest to the path. pub fn write_example_manifest(manifest_path: &Path) { fs::create_dir_all(manifest_path).unwrap(); fs::write(manifest_path.join("R004"), example_elsa_manifest()).unwrap(); } +/// Runs a test with a mock Elsa server that accepts GET requests for the htsget manifest. pub async fn with_test_mocks(test: F, expect_times: T) where T: Into, @@ -159,7 +163,8 @@ where .await; } -pub fn is_resolver_from_parts(resolver: &Resolver) -> bool { +/// Check if the resolver correctly matches the 30F9F3FED8F711ED8C35DBEF59E9F537 reads example. +pub fn is_reads_resolver_from_parts(resolver: &Resolver) -> bool { resolver.regex().to_string() == "^R004/30F9F3FED8F711ED8C35DBEF59E9F537$" && resolver.substitution_string() == "HG00097/HG00097" && matches!(resolver.storage(), storage::Storage::S3 { s3_storage } if s3_storage.bucket() == "umccr-10g-data-dev") @@ -168,6 +173,7 @@ pub fn is_resolver_from_parts(resolver: &Resolver) -> bool { && resolver.allow_interval() == Interval::new(Some(1), Some(10)) } +/// Check if the resolvers correct match the whole example manifest. pub fn is_manifest_resolvers(resolvers: Vec) -> bool { resolvers.iter().any(|resolver| { resolver.regex().to_string() == "^R004/30F9FFD4D8F711ED8C353BBCB8861211$" && @@ -177,7 +183,7 @@ pub fn is_manifest_resolvers(resolvers: Vec) -> bool { && resolver.allow_reference_names() == &List(HashSet::from_iter(vec!["2".to_string()])) && resolver.allow_interval() == Interval::new(None, Some(10)) }) && - resolvers.iter().any(is_resolver_from_parts) && + resolvers.iter().any(is_reads_resolver_from_parts) && resolvers.iter().any(|resolver| { resolver.regex().to_string() == "^R004/30F9FFD4D8F711ED8C353BBCB8861211$" && resolver.substitution_string() == "HG00096/HG00096.hard-filtered" && From 75ac556a059270773bb816e57103670637d88250 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 4 Sep 2023 08:30:41 +1000 Subject: [PATCH 24/25] build: update .gitignore --- .gitignore | 4 ++++ deploy/.gitignore | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 73fab07..53c6643 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ target/ # MSVC Windows builds of rustc generate these, which store debugging information *.pdb + +*.code-workspace +.vscode +.idea diff --git a/deploy/.gitignore b/deploy/.gitignore index a633556..1b60dba 100644 --- a/deploy/.gitignore +++ b/deploy/.gitignore @@ -5,3 +5,5 @@ node_modules # CDK asset staging directory .cdk.staging cdk.out + +.build From 9be1d8106f472141e7a17a20d54a551b49fed8bf Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Mon, 4 Sep 2023 09:07:56 +1000 Subject: [PATCH 25/25] feat: make caching optional --- htsget-elsa-lambda/README.md | 11 +++-- htsget-elsa-lambda/src/config.rs | 8 +-- htsget-elsa-lambda/src/lib.rs | 12 +++-- htsget-elsa/src/elsa_endpoint.rs | 10 ++-- htsget-elsa/src/s3.rs | 83 +++++++++++++++++--------------- 5 files changed, 65 insertions(+), 59 deletions(-) diff --git a/htsget-elsa-lambda/README.md b/htsget-elsa-lambda/README.md index 61b36d7..609ba2a 100644 --- a/htsget-elsa-lambda/README.md +++ b/htsget-elsa-lambda/README.md @@ -18,13 +18,14 @@ in the query matches the release key in the manifest file. For example, the foll GET https:///reads//?format=BAM&referenceName=1&start=0&end=1000 ``` -This function supports all the regular htsget-rs configuration options, and also includes the following to configure the +This function supports all the regular htsget-rs configuration options, and if a query fails to match a release key, it +will fall back to the resolvers defined in the config. This crate also includes the following options to configure the Elsa endpoint and cache location: -| Option | Description | Type | Default | Example | -|---------------------------|------------------------------------------------------|---------------|---------------------------|-----------------------------| -| `elsa_endpoint_authority` | The URL authority of the Elsa endpoint. | URL Authority | Not specified, required. | `'elsa-data.dev.umccr.org'` | -| `cache_location` | The name of the bucket where resolvers are cached. | String | Not specified, required. | `'cache_bucket'` | +| Option | Description | Type | Default | Example | +|---------------------------|--------------------------------------------------------------------------------------------------------|---------------|---------------------|-----------------------------| +| `elsa_endpoint_authority` | The URL authority of the Elsa endpoint. | URL Authority | Not specified, required. | `'elsa-data.dev.umccr.org'` | +| `cache_location` | The name of the bucket where resolvers are cached. If this is not specified, no caching is performed. | String | Not specified. | `'cache_bucket'` | To deploy this function, see the [deploy][deploy] folder. diff --git a/htsget-elsa-lambda/src/config.rs b/htsget-elsa-lambda/src/config.rs index 75e4617..5669782 100644 --- a/htsget-elsa-lambda/src/config.rs +++ b/htsget-elsa-lambda/src/config.rs @@ -13,7 +13,7 @@ pub struct Config { htsget_config: HtsGetConfig, #[serde(with = "http_serde::authority")] elsa_endpoint_authority: Authority, - cache_location: String, + cache_location: Option, } impl Config { @@ -21,7 +21,7 @@ impl Config { pub fn new( htsget_config: HtsGetConfig, elsa_endpoint_authority: Authority, - cache_location: String, + cache_location: Option, ) -> Self { Self { htsget_config, @@ -41,8 +41,8 @@ impl Config { } /// Get the cache location. - pub fn cache_location(&self) -> &str { - &self.cache_location + pub fn cache_location(&self) -> Option<&str> { + self.cache_location.as_deref() } } diff --git a/htsget-elsa-lambda/src/lib.rs b/htsget-elsa-lambda/src/lib.rs index 93af842..d4090c2 100644 --- a/htsget-elsa-lambda/src/lib.rs +++ b/htsget-elsa-lambda/src/lib.rs @@ -24,7 +24,9 @@ pub async fn handle_request(config: Config) -> Result<(), Error> { match Route::try_from(&event) { Ok(route) => { - let s3 = S3::new_with_default_config(config.cache_location().to_string()).await; + let s3 = + S3::new_with_default_config(config.cache_location().map(str::to_string)) + .await; let elsa_endpoint = match ElsaEndpoint::new(config.elsa_endpoint_authority().clone(), &s3, &s3) { @@ -116,10 +118,10 @@ mod tests { let config = Config::new( default_test_config(), Authority::from_str(&endpoint).unwrap(), - "cache".to_string(), + Some("cache".to_string()), ); - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string())); let endpoint = ElsaEndpoint::new_with_client( reqwest_client, config.elsa_endpoint_authority().clone(), @@ -145,10 +147,10 @@ mod tests { let config = Config::new( default_test_config(), Authority::from_str(&endpoint).unwrap(), - "cache".to_string(), + Some("cache".to_string()), ); - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string())); let endpoint = ElsaEndpoint::new_with_client( reqwest_client, config.elsa_endpoint_authority().clone(), diff --git a/htsget-elsa/src/elsa_endpoint.rs b/htsget-elsa/src/elsa_endpoint.rs index 367dcda..d8035f2 100644 --- a/htsget-elsa/src/elsa_endpoint.rs +++ b/htsget-elsa/src/elsa_endpoint.rs @@ -331,7 +331,7 @@ mod tests { async fn get_response() { with_test_mocks( |endpoint, s3_client, reqwest_client, _| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string())); let endpoint = ElsaEndpoint::new_with_client( reqwest_client, Authority::from_str(&endpoint).unwrap(), @@ -361,7 +361,7 @@ mod tests { async fn get_manifest() { with_test_mocks( |endpoint, s3_client, reqwest_client, _| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string())); let endpoint = ElsaEndpoint::new_with_client( reqwest_client, Authority::from_str(&endpoint).unwrap(), @@ -384,7 +384,7 @@ mod tests { async fn get_manifest_not_present() { with_test_mocks( |endpoint, s3_client, reqwest_client, _| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string())); let endpoint = ElsaEndpoint::new_with_client( reqwest_client, Authority::from_str(&endpoint).unwrap(), @@ -408,7 +408,7 @@ mod tests { async fn try_get_cached() { with_test_mocks( |endpoint, s3_client, reqwest_client, _| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string())); let endpoint = ElsaEndpoint::new_with_client( reqwest_client, Authority::from_str(&endpoint).unwrap(), @@ -433,7 +433,7 @@ mod tests { async fn try_get_not_cached() { with_test_mocks( |endpoint, s3_client, reqwest_client, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string())); let endpoint = ElsaEndpoint::new_with_client( reqwest_client, Authority::from_str(&endpoint).unwrap(), diff --git a/htsget-elsa/src/s3.rs b/htsget-elsa/src/s3.rs index 2083b2b..5388116 100644 --- a/htsget-elsa/src/s3.rs +++ b/htsget-elsa/src/s3.rs @@ -17,7 +17,7 @@ use crate::{Cache, Error, GetObject, Result}; #[derive(Debug)] pub struct S3 { s3_client: Client, - cache_bucket: String, + cache_bucket: Option, } /// The shape of the item to cache. @@ -29,7 +29,7 @@ pub struct CacheItem { impl S3 { /// Create a new S3 storage. - pub fn new(s3_client: Client, cache_bucket: String) -> Self { + pub fn new(s3_client: Client, cache_bucket: Option) -> Self { Self { s3_client, cache_bucket, @@ -37,7 +37,7 @@ impl S3 { } /// Create a new S3 storage with default AWS config. - pub async fn new_with_default_config(cache_bucket: String) -> Self { + pub async fn new_with_default_config(cache_bucket: Option) -> Self { Self::new( Client::new(&aws_config::load_from_env().await), cache_bucket, @@ -110,20 +110,19 @@ impl Cache for S3 { async fn get + Send + Sync>(&self, key: K) -> Result> { trace!(key = key.as_ref(), "getting key"); - if let Some(last_modified) = self - .last_modified(self.cache_bucket.clone(), key.as_ref()) - .await - { - let object: CacheItem = self - .get_object(self.cache_bucket.clone(), key.as_ref()) - .await?; - - if last_modified.as_nanos() - > DateTime::from(SystemTime::now().sub(Duration::from_secs(object.max_age))) - .as_nanos() - { - return Ok(Some(object.item)); + if let Some(cache_bucket) = &self.cache_bucket { + if let Some(last_modified) = self.last_modified(cache_bucket, key.as_ref()).await { + let object: CacheItem = self.get_object(cache_bucket, key.as_ref()).await?; + + if last_modified.as_nanos() + > DateTime::from(SystemTime::now().sub(Duration::from_secs(object.max_age))) + .as_nanos() + { + return Ok(Some(object.item)); + } } + } else { + trace!("no caching bucket configured"); } Ok(None) @@ -138,22 +137,26 @@ impl Cache for S3 { ) -> Result<()> { trace!(key = key.as_ref(), "putting key"); - self.s3_client - .put_object() - .bucket(self.cache_bucket.clone()) - .key(key.as_ref()) - .body(ByteStream::from(Bytes::from( - to_vec(&CacheItem { item, max_age }) - .map_err(|err| SerializeError(err.to_string()))?, - ))) - .send() - .await - .map_err(|err| { - let err = err.into_service_error(); - trace!(err = err.message(), "put object error"); - - PutObjectError(err.to_string()) - })?; + if let Some(cache_bucket) = &self.cache_bucket { + self.s3_client + .put_object() + .bucket(cache_bucket) + .key(key.as_ref()) + .body(ByteStream::from(Bytes::from( + to_vec(&CacheItem { item, max_age }) + .map_err(|err| SerializeError(err.to_string()))?, + ))) + .send() + .await + .map_err(|err| { + let err = err.into_service_error(); + trace!(err = err.message(), "put object error"); + + PutObjectError(err.to_string()) + })?; + } else { + trace!("no caching bucket configured"); + } Ok(()) } @@ -174,7 +177,7 @@ mod tests { async fn last_modified() { with_test_mocks( |_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string())); let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); write_example_manifest(&manifest_path); @@ -193,7 +196,7 @@ mod tests { async fn last_modified_not_found() { with_test_mocks( |_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string())); let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); write_example_manifest(&manifest_path); @@ -212,7 +215,7 @@ mod tests { async fn get_object() { with_test_mocks( |_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string())); let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); write_example_manifest(&manifest_path); @@ -232,7 +235,7 @@ mod tests { async fn get_object_not_found() { with_test_mocks( |_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string())); let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); write_example_manifest(&manifest_path); @@ -251,7 +254,7 @@ mod tests { async fn get_not_found() { with_test_mocks( |_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string())); let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); fs::create_dir_all(&manifest_path).unwrap(); @@ -277,7 +280,7 @@ mod tests { async fn get_cache_expired() { with_test_mocks( |_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string())); let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); fs::create_dir_all(&manifest_path).unwrap(); @@ -303,7 +306,7 @@ mod tests { async fn get() { with_test_mocks( |_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string())); let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests"); fs::create_dir_all(&manifest_path).unwrap(); @@ -329,7 +332,7 @@ mod tests { async fn put() { with_test_mocks( |_, s3_client, _, base_path| async move { - let s3 = S3::new(s3_client, "elsa-data-tmp".to_string()); + let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string())); let manifest_path = base_path.join("elsa-data-tmp"); fs::create_dir_all(&manifest_path).unwrap();