Skip to content

Commit

Permalink
feat: make caching optional
Browse files Browse the repository at this point in the history
  • Loading branch information
@mmalenic
mmalenic committed Sep 3, 2023
1 parent 75ac556 commit 9be1d81
Show file tree
Hide file tree
Showing 5 changed files with 65 additions and 59 deletions.
11 changes: 6 additions & 5 deletions htsget-elsa-lambda/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,14 @@ in the query matches the release key in the manifest file. For example, the foll
GET https://<htsget_domain>/reads/<release_key>/<file>?format=BAM&referenceName=1&start=0&end=1000
```

This function supports all the regular htsget-rs configuration options, and also includes the following to configure the
This function supports all the regular htsget-rs configuration options, and if a query fails to match a release key, it
will fall back to the resolvers defined in the config. This crate also includes the following options to configure the
Elsa endpoint and cache location:

| Option | Description | Type | Default | Example |
|---------------------------|------------------------------------------------------|---------------|---------------------------|-----------------------------|
| `elsa_endpoint_authority` | The URL authority of the Elsa endpoint. | URL Authority | Not specified, required. | `'elsa-data.dev.umccr.org'` |
| `cache_location` | The name of the bucket where resolvers are cached. | String | Not specified, required. | `'cache_bucket'` |
| Option | Description | Type | Default | Example |
|---------------------------|--------------------------------------------------------------------------------------------------------|---------------|---------------------|-----------------------------|
| `elsa_endpoint_authority` | The URL authority of the Elsa endpoint. | URL Authority | Not specified, required. | `'elsa-data.dev.umccr.org'` |
| `cache_location` | The name of the bucket where resolvers are cached. If this is not specified, no caching is performed. | String | Not specified. | `'cache_bucket'` |

To deploy this function, see the [deploy][deploy] folder.

Expand Down
8 changes: 4 additions & 4 deletions htsget-elsa-lambda/src/config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ pub struct Config {
htsget_config: HtsGetConfig,
#[serde(with = "http_serde::authority")]
elsa_endpoint_authority: Authority,
cache_location: String,
cache_location: Option<String>,
}

impl Config {
/// Create a new config.
pub fn new(
htsget_config: HtsGetConfig,
elsa_endpoint_authority: Authority,
cache_location: String,
cache_location: Option<String>,
) -> Self {
Self {
htsget_config,
Expand All @@ -41,8 +41,8 @@ impl Config {
}

/// Get the cache location.
pub fn cache_location(&self) -> &str {
&self.cache_location
pub fn cache_location(&self) -> Option<&str> {
self.cache_location.as_deref()
}
}

Expand Down
12 changes: 7 additions & 5 deletions htsget-elsa-lambda/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ pub async fn handle_request(config: Config) -> Result<(), Error> {

match Route::try_from(&event) {
Ok(route) => {
let s3 = S3::new_with_default_config(config.cache_location().to_string()).await;
let s3 =
S3::new_with_default_config(config.cache_location().map(str::to_string))
.await;
let elsa_endpoint =
match ElsaEndpoint::new(config.elsa_endpoint_authority().clone(), &s3, &s3)
{
Expand Down Expand Up @@ -116,10 +118,10 @@ mod tests {
let config = Config::new(
default_test_config(),
Authority::from_str(&endpoint).unwrap(),
"cache".to_string(),
Some("cache".to_string()),
);

let s3 = S3::new(s3_client, "elsa-data-tmp".to_string());
let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string()));
let endpoint = ElsaEndpoint::new_with_client(
reqwest_client,
config.elsa_endpoint_authority().clone(),
Expand All @@ -145,10 +147,10 @@ mod tests {
let config = Config::new(
default_test_config(),
Authority::from_str(&endpoint).unwrap(),
"cache".to_string(),
Some("cache".to_string()),
);

let s3 = S3::new(s3_client, "elsa-data-tmp".to_string());
let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string()));
let endpoint = ElsaEndpoint::new_with_client(
reqwest_client,
config.elsa_endpoint_authority().clone(),
Expand Down
10 changes: 5 additions & 5 deletions htsget-elsa/src/elsa_endpoint.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,7 @@ mod tests {
async fn get_response() {
with_test_mocks(
|endpoint, s3_client, reqwest_client, _| async move {
let s3 = S3::new(s3_client, "elsa-data-tmp".to_string());
let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string()));
let endpoint = ElsaEndpoint::new_with_client(
reqwest_client,
Authority::from_str(&endpoint).unwrap(),
Expand Down Expand Up @@ -361,7 +361,7 @@ mod tests {
async fn get_manifest() {
with_test_mocks(
|endpoint, s3_client, reqwest_client, _| async move {
let s3 = S3::new(s3_client, "elsa-data-tmp".to_string());
let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string()));
let endpoint = ElsaEndpoint::new_with_client(
reqwest_client,
Authority::from_str(&endpoint).unwrap(),
Expand All @@ -384,7 +384,7 @@ mod tests {
async fn get_manifest_not_present() {
with_test_mocks(
|endpoint, s3_client, reqwest_client, _| async move {
let s3 = S3::new(s3_client, "elsa-data-tmp".to_string());
let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string()));
let endpoint = ElsaEndpoint::new_with_client(
reqwest_client,
Authority::from_str(&endpoint).unwrap(),
Expand All @@ -408,7 +408,7 @@ mod tests {
async fn try_get_cached() {
with_test_mocks(
|endpoint, s3_client, reqwest_client, _| async move {
let s3 = S3::new(s3_client, "elsa-data-tmp".to_string());
let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string()));
let endpoint = ElsaEndpoint::new_with_client(
reqwest_client,
Authority::from_str(&endpoint).unwrap(),
Expand All @@ -433,7 +433,7 @@ mod tests {
async fn try_get_not_cached() {
with_test_mocks(
|endpoint, s3_client, reqwest_client, base_path| async move {
let s3 = S3::new(s3_client, "elsa-data-tmp".to_string());
let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string()));
let endpoint = ElsaEndpoint::new_with_client(
reqwest_client,
Authority::from_str(&endpoint).unwrap(),
Expand Down
83 changes: 43 additions & 40 deletions htsget-elsa/src/s3.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use crate::{Cache, Error, GetObject, Result};
#[derive(Debug)]
pub struct S3 {
s3_client: Client,
cache_bucket: String,
cache_bucket: Option<String>,
}

/// The shape of the item to cache.
Expand All @@ -29,15 +29,15 @@ pub struct CacheItem {

impl S3 {
/// Create a new S3 storage.
pub fn new(s3_client: Client, cache_bucket: String) -> Self {
pub fn new(s3_client: Client, cache_bucket: Option<String>) -> Self {
Self {
s3_client,
cache_bucket,
}
}

/// Create a new S3 storage with default AWS config.
pub async fn new_with_default_config(cache_bucket: String) -> Self {
pub async fn new_with_default_config(cache_bucket: Option<String>) -> Self {
Self::new(
Client::new(&aws_config::load_from_env().await),
cache_bucket,
Expand Down Expand Up @@ -110,20 +110,19 @@ impl Cache for S3 {
async fn get<K: AsRef<str> + Send + Sync>(&self, key: K) -> Result<Option<Self::Item>> {
trace!(key = key.as_ref(), "getting key");

if let Some(last_modified) = self
.last_modified(self.cache_bucket.clone(), key.as_ref())
.await
{
let object: CacheItem = self
.get_object(self.cache_bucket.clone(), key.as_ref())
.await?;

if last_modified.as_nanos()
> DateTime::from(SystemTime::now().sub(Duration::from_secs(object.max_age)))
.as_nanos()
{
return Ok(Some(object.item));
if let Some(cache_bucket) = &self.cache_bucket {
if let Some(last_modified) = self.last_modified(cache_bucket, key.as_ref()).await {
let object: CacheItem = self.get_object(cache_bucket, key.as_ref()).await?;

if last_modified.as_nanos()
> DateTime::from(SystemTime::now().sub(Duration::from_secs(object.max_age)))
.as_nanos()
{
return Ok(Some(object.item));
}
}
} else {
trace!("no caching bucket configured");
}

Ok(None)
Expand All @@ -138,22 +137,26 @@ impl Cache for S3 {
) -> Result<()> {
trace!(key = key.as_ref(), "putting key");

self.s3_client
.put_object()
.bucket(self.cache_bucket.clone())
.key(key.as_ref())
.body(ByteStream::from(Bytes::from(
to_vec(&CacheItem { item, max_age })
.map_err(|err| SerializeError(err.to_string()))?,
)))
.send()
.await
.map_err(|err| {
let err = err.into_service_error();
trace!(err = err.message(), "put object error");

PutObjectError(err.to_string())
})?;
if let Some(cache_bucket) = &self.cache_bucket {
self.s3_client
.put_object()
.bucket(cache_bucket)
.key(key.as_ref())
.body(ByteStream::from(Bytes::from(
to_vec(&CacheItem { item, max_age })
.map_err(|err| SerializeError(err.to_string()))?,
)))
.send()
.await
.map_err(|err| {
let err = err.into_service_error();
trace!(err = err.message(), "put object error");

PutObjectError(err.to_string())
})?;
} else {
trace!("no caching bucket configured");
}

Ok(())
}
Expand All @@ -174,7 +177,7 @@ mod tests {
async fn last_modified() {
with_test_mocks(
|_, s3_client, _, base_path| async move {
let s3 = S3::new(s3_client, "elsa-data-tmp".to_string());
let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string()));

let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests");
write_example_manifest(&manifest_path);
Expand All @@ -193,7 +196,7 @@ mod tests {
async fn last_modified_not_found() {
with_test_mocks(
|_, s3_client, _, base_path| async move {
let s3 = S3::new(s3_client, "elsa-data-tmp".to_string());
let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string()));

let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests");
write_example_manifest(&manifest_path);
Expand All @@ -212,7 +215,7 @@ mod tests {
async fn get_object() {
with_test_mocks(
|_, s3_client, _, base_path| async move {
let s3 = S3::new(s3_client, "elsa-data-tmp".to_string());
let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string()));

let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests");
write_example_manifest(&manifest_path);
Expand All @@ -232,7 +235,7 @@ mod tests {
async fn get_object_not_found() {
with_test_mocks(
|_, s3_client, _, base_path| async move {
let s3 = S3::new(s3_client, "elsa-data-tmp".to_string());
let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string()));

let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests");
write_example_manifest(&manifest_path);
Expand All @@ -251,7 +254,7 @@ mod tests {
async fn get_not_found() {
with_test_mocks(
|_, s3_client, _, base_path| async move {
let s3 = S3::new(s3_client, "elsa-data-tmp".to_string());
let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string()));

let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests");
fs::create_dir_all(&manifest_path).unwrap();
Expand All @@ -277,7 +280,7 @@ mod tests {
async fn get_cache_expired() {
with_test_mocks(
|_, s3_client, _, base_path| async move {
let s3 = S3::new(s3_client, "elsa-data-tmp".to_string());
let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string()));

let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests");
fs::create_dir_all(&manifest_path).unwrap();
Expand All @@ -303,7 +306,7 @@ mod tests {
async fn get() {
with_test_mocks(
|_, s3_client, _, base_path| async move {
let s3 = S3::new(s3_client, "elsa-data-tmp".to_string());
let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string()));

let manifest_path = base_path.join("elsa-data-tmp/htsget-manifests");
fs::create_dir_all(&manifest_path).unwrap();
Expand All @@ -329,7 +332,7 @@ mod tests {
async fn put() {
with_test_mocks(
|_, s3_client, _, base_path| async move {
let s3 = S3::new(s3_client, "elsa-data-tmp".to_string());
let s3 = S3::new(s3_client, Some("elsa-data-tmp".to_string()));

let manifest_path = base_path.join("elsa-data-tmp");
fs::create_dir_all(&manifest_path).unwrap();
Expand Down

0 comments on commit 9be1d81

Please sign in to comment.