feat: add mirroring to S3

Signed-off-by: Guillaume Hivert <[email protected]>

apps/backend/src/api/hex_repo.gleam

@@ -6,6 +6,7 @@ import gleam/httpc
 import gleam/json
 import gleam/package_interface
 import gleam/result
+import s3
 import simplifile
 import tom
 import wisp
@@ -40,6 +41,7 @@ fn read_archive(archives_path: String, name: String, version: String) {
   let slug = package_slug(name, version) <> ".tar"
   let filepath = archives_path <> "/" <> name <> "/" <> slug
   use content <- result.map(simplifile.read_bits(filepath))
+  let _ = put_s3(name, slug, content)
   wisp.log_debug("Using filesystem for " <> slug)
   content
 }
@@ -55,13 +57,29 @@ fn create_archive(
   let _ = simplifile.create_directory_all(package_path)
   let filepath = package_path <> "/" <> slug
   let _ = simplifile.write_bits(filepath, archive)
+  let _ = put_s3(name, slug, archive)
+  archive
+}
+
+fn read_s3(name: String, slug: String) {
+  let full_slug = name <> "/" <> slug
+  use archive <- result.map(s3.get(full_slug))
+  wisp.log_debug("Using S3 for " <> slug)
+  archive
+}
+
+fn put_s3(name: String, slug: String, archive: BitArray) {
+  let full_slug = name <> "/" <> slug
+  use _ <- result.map(s3.put(full_slug, archive))
+  wisp.log_debug("Put on S3 for " <> slug)
   archive
 }
 
 fn get_tarball(name: String, version: String) {
   let slug = package_slug(name, version) <> ".tar"
   use archives_path <- result.try(create_archives_directory())
   use _ <- result.try_recover(read_archive(archives_path, name, version))
+  use _ <- result.try_recover(read_s3(name, slug))
   wisp.log_debug("Querying hex for " <> slug)
   request.new()
   |> request.set_host("repo.hex.pm")

apps/backend/src/backend/config.gleam

@@ -23,3 +23,14 @@ pub fn get_secret_key_base() {
 pub fn is_dev() {
   os.get_env("GLEAM_ENV") == Ok("development")
 }
+
+pub fn bucket_uri() {
+  let assert Ok(content) = os.get_env("BUCKET_URI")
+  content
+}
+
+pub fn scaleway_keys() {
+  let assert Ok(access_key) = os.get_env("SCALEWAY_ACCESS_KEY")
+  let assert Ok(secret_key) = os.get_env("SCALEWAY_SECRET_KEY")
+  #(access_key, secret_key)
+}

apps/backend/src/backend/error.gleam

@@ -17,6 +17,7 @@ pub type Error {
   UnknownError(String)
   ParseTomlError(tom.ParseError)
   GetTomlError(tom.GetError)
+  EmptyError
 }
 
 pub fn log_dynamic_error(error: dynamic.DecodeError) {
@@ -49,6 +50,7 @@ pub fn log_decode_error(error: json.DecodeError) {
 
 pub fn log_error(error: Error) {
   case error {
+    EmptyError -> Nil
     FetchError(_dyn) -> wisp.log_warning("Fetch error")
     DatabaseError(error) -> {
       wisp.log_warning("Query error")

apps/backend/src/s3.gleam

@@ -0,0 +1,40 @@
+import aws4_request
+import backend/config
+import birl
+import gleam/http
+import gleam/http/request
+import gleam/httpc
+import gleam/option.{type Option, None, Some}
+import gleam/result
+
+fn request(url: String, method: http.Method, body: Option(BitArray)) {
+  let bucket_uri = config.bucket_uri()
+  let date = birl.to_erlang_universal_datetime(birl.now())
+  let #(access_key, secret_key) = config.scaleway_keys()
+  request.new()
+  |> request.set_method(method)
+  |> request.set_path(url)
+  |> request.set_body(option.unwrap(body, <<>>))
+  |> request.set_host(bucket_uri)
+  |> request.set_scheme(http.Https)
+  |> request.set_header("content-type", "application/octet-stream")
+  |> aws4_request.sign(date, access_key, secret_key, "fr-par", "s3")
+  |> httpc.send_bits()
+  |> result.nil_error()
+}
+
+pub fn get(name: String) {
+  use res <- result.try(request("/" <> name, http.Get, None))
+  case res.status {
+    200 -> Ok(res.body)
+    _ -> Error(Nil)
+  }
+}
+
+pub fn put(name: String, content: BitArray) {
+  use res <- result.try(request("/" <> name, http.Put, Some(content)))
+  case res.status {
+    200 -> Ok(res.body)
+    _ -> Error(Nil)
+  }
+}