From 0f1abadba72b2b9822406effe7ba5ad6b9b0432e Mon Sep 17 00:00:00 2001 From: Guillaume Hivert Date: Sat, 4 May 2024 15:34:15 +0200 Subject: [PATCH] refactor: improve separation of concerns Signed-off-by: Guillaume Hivert --- apps/backend/src/api/hex.gleam | 2 +- apps/backend/src/api/hex_repo.gleam | 4 +- apps/backend/src/api/signatures.gleam | 506 ++++-------------- apps/backend/src/backend/data/hex_read.gleam | 10 + apps/backend/src/backend/data/hex_user.gleam | 14 + .../backend/{log_error.gleam => error.gleam} | 50 +- apps/backend/src/backend/gleam/context.gleam | 27 + .../src/backend/gleam/generate/metadata.gleam | 22 + .../gleam/generate/sources.gleam | 0 .../{ => backend}/gleam/generate/types.gleam | 163 ++---- apps/backend/src/backend/gleam/toml.gleam | 26 + apps/backend/src/backend/index/decoders.gleam | 24 - apps/backend/src/backend/index/error.gleam | 14 - .../connect.gleam => postgres/postgres.gleam} | 0 .../backend/{index => postgres}/queries.gleam | 169 +++++- .../src/{backend/index => }/helpers.gleam | 0 apps/backend/src/periodic.gleam | 4 +- apps/backend/src/retrier.gleam | 5 +- apps/backend/src/tasks/hex.gleam | 23 +- 19 files changed, 467 insertions(+), 596 deletions(-) rename apps/backend/src/backend/{log_error.gleam => error.gleam} (83%) create mode 100644 apps/backend/src/backend/gleam/context.gleam create mode 100644 apps/backend/src/backend/gleam/generate/metadata.gleam rename apps/backend/src/{ => backend}/gleam/generate/sources.gleam (100%) rename apps/backend/src/{ => backend}/gleam/generate/types.gleam (69%) create mode 100644 apps/backend/src/backend/gleam/toml.gleam delete mode 100644 apps/backend/src/backend/index/decoders.gleam delete mode 100644 apps/backend/src/backend/index/error.gleam rename apps/backend/src/backend/{index/connect.gleam => postgres/postgres.gleam} (100%) rename apps/backend/src/backend/{index => postgres}/queries.gleam (53%) rename apps/backend/src/{backend/index => }/helpers.gleam (100%) diff --git a/apps/backend/src/api/hex.gleam b/apps/backend/src/api/hex.gleam index ed9daad..62af644 100644 --- a/apps/backend/src/api/hex.gleam +++ b/apps/backend/src/api/hex.gleam @@ -1,4 +1,4 @@ -import backend/index/error +import backend/error import gleam/dynamic import gleam/hexpm import gleam/http/request diff --git a/apps/backend/src/api/hex_repo.gleam b/apps/backend/src/api/hex_repo.gleam index 2775816..5f6fb91 100644 --- a/apps/backend/src/api/hex_repo.gleam +++ b/apps/backend/src/api/hex_repo.gleam @@ -1,4 +1,4 @@ -import backend/index/error +import backend/error import gleam/bit_array import gleam/http import gleam/http/request @@ -63,7 +63,7 @@ fn read_package_interface(blob: option.Option(String)) { fn read_gleam_toml(blob: String) { blob |> tom.parse() - |> result.map_error(error.TomlError) + |> result.map_error(error.ParseTomlError) } fn extract_package_infos(name: String, version: String) { diff --git a/apps/backend/src/api/signatures.gleam b/apps/backend/src/api/signatures.gleam index 43708a4..a75a05b 100644 --- a/apps/backend/src/api/signatures.gleam +++ b/apps/backend/src/api/signatures.gleam @@ -1,439 +1,151 @@ -import backend/index/error -import gleam/bool -import gleam/dict.{type Dict} -import gleam/dynamic -import gleam/function -import gleam/generate/sources.{ +import backend/gleam/context.{type Context} +import backend/gleam/generate/metadata +import backend/gleam/generate/sources.{ constant_to_string, function_to_string, type_alias_to_string, type_definition_to_string, } -import gleam/generate/types.{ - constant_to_json, function_to_json, implementations_to_json, - type_alias_to_json, type_definition_to_json, +import backend/gleam/generate/types.{ + constant_to_json, function_to_json, type_alias_to_json, + type_definition_to_json, } -import gleam/json +import backend/postgres/queries +import gleam/bool +import gleam/dict import gleam/list import gleam/option.{None, Some} -import gleam/package_interface.{type Module, type Package} -import gleam/pgo +import gleam/package_interface import gleam/result -import gleam/string -import tom.{type Toml} import wisp -fn add_gleam_constraint(db: pgo.Connection, package: Package, release_id: Int) { - case package.gleam_version_constraint { +fn add_gleam_constraint(ctx: Context, release_id: Int) { + case ctx.package_interface.gleam_version_constraint { + Some(c) -> queries.add_package_gleam_constraint(ctx.db, c, release_id) None -> Ok(Nil) - Some(c) -> { - "UPDATE package_release SET gleam_constraint = $1 WHERE id = $2" - |> pgo.execute(db, [pgo.text(c), pgo.int(release_id)], dynamic.dynamic) - |> result.replace(Nil) - |> result.map_error(error.DatabaseError) - } } } -fn get_package_release_ids(db: pgo.Connection, package: Package) { - use response <- result.try({ - let args = [pgo.text(package.name), pgo.text(package.version)] - "SELECT - package.id package_id, - package_release.id package_release_id - FROM package - JOIN package_release - ON package_release.package_id = package.id - WHERE package.name = $1 - AND package_release.version = $2" - |> pgo.execute(db, args, dynamic.tuple2(dynamic.int, dynamic.int)) - |> result.map_error(error.DatabaseError) - }) - response.rows - |> list.first() - |> result.replace_error(error.UnknownError( - "No release found for " <> package.name <> "@" <> package.version, - )) -} - -fn upsert_package_module( - db: pgo.Connection, - module_name: String, - module: Module, - release_id: Int, -) { - let documentation = - module.documentation - |> string.join("\n") - |> pgo.text() - use response <- result.try({ - let args = [pgo.text(module_name), documentation, pgo.int(release_id)] - "INSERT INTO package_module (name, documentation, package_release_id) - VALUES ($1, $2, $3) - ON CONFLICT (name, package_release_id) DO UPDATE - SET documentation = $2 - RETURNING id" - |> pgo.execute(db, args, dynamic.element(0, dynamic.int)) - |> result.map_error(error.DatabaseError) - }) - response.rows - |> list.first() - |> result.replace_error(error.UnknownError( - "No module found for " <> module_name, - )) -} - -fn upsert_type_definitions( - db: pgo.Connection, - package_interface: Package, - module_id: Int, - module: Module, - gleam_toml: Dict(String, Toml), -) { - let all_types = dict.to_list(module.types) +fn upsert_type_definitions(ctx: Context, module: context.Module) { + let name = context.qualified_name(ctx, module) + wisp.log_info("Extracting " <> name <> " type definitions") + let all_types = dict.to_list(module.module.types) result.all({ use #(type_name, type_def) <- list.map(all_types) - let documentation = string.trim(option.unwrap(type_def.documentation, "")) - let deprecation = option.map(type_def.deprecation, fn(d) { d.message }) - let metadata = - json.object([#("deprecation", json.nullable(deprecation, json.string))]) - |> json.to_string() - let signature = type_definition_to_string(type_name, type_def) - let type_def_json = - type_definition_to_json( - db, - package_interface, - type_name, - type_def, - gleam_toml, - ) - use #(json_signature, parameters) <- result.try(type_def_json) - - "INSERT INTO package_type_fun_signature ( - name, - documentation, - signature_, - json_signature, - nature, - parameters, - metadata, - package_module_id, - deprecation - ) VALUES ($1, $2, $3, $4, 'type_definition', $5, $6, $7, $8) - ON CONFLICT (package_module_id, name) DO UPDATE - SET - documentation = $2, - signature_ = $3, - json_signature = $4, - nature = 'type_definition', - parameters = $5, - metadata = $6, - deprecation = $8" - |> pgo.execute( - db, - [ - pgo.text(type_name), - pgo.text(documentation), - pgo.text(signature), - json_signature - |> json.to_string() - |> pgo.text(), - dynamic.unsafe_coerce(dynamic.from(parameters)), - pgo.text(metadata), - pgo.int(module_id), - type_def.deprecation - |> option.map(fn(d) { d.message }) - |> pgo.nullable(pgo.text, _), - ], - dynamic.dynamic, + use gen <- result.try(type_definition_to_json(ctx, type_name, type_def)) + queries.upsert_package_type_fun_signature( + db: ctx.db, + nature: queries.TypeDefinition, + name: type_name, + documentation: type_def.documentation, + metadata: metadata.generate(type_def.deprecation, None), + signature: type_definition_to_string(type_name, type_def), + json_signature: gen.0, + parameters: gen.1, + module_id: module.id, + deprecation: type_def.deprecation, + implementations: None, ) - |> result.map_error(error.DatabaseError) - |> result.replace(Nil) }) } -fn upsert_type_aliases( - db: pgo.Connection, - package_interface: Package, - module_id: Int, - module: Module, - gleam_toml: Dict(String, Toml), -) { - let all_types = dict.to_list(module.type_aliases) +fn upsert_type_aliases(ctx: Context, module: context.Module) { + let name = context.qualified_name(ctx, module) + wisp.log_info("Extracting " <> name <> " type aliases") + let all_types = dict.to_list(module.module.type_aliases) result.all({ use #(type_name, type_alias) <- list.map(all_types) - let documentation = string.trim(option.unwrap(type_alias.documentation, "")) - let deprecation = option.map(type_alias.deprecation, fn(d) { d.message }) - let metadata = - json.object([#("deprecation", json.nullable(deprecation, json.string))]) - |> json.to_string() - let signature = type_alias_to_string(type_name, type_alias) - let type_alias_json = - type_alias_to_json( - db, - package_interface, - type_name, - type_alias, - gleam_toml, - ) - use #(json_signature, parameters) <- result.try(type_alias_json) - - "INSERT INTO package_type_fun_signature ( - name, - documentation, - signature_, - json_signature, - nature, - parameters, - metadata, - package_module_id, - deprecation - ) VALUES ($1, $2, $3, $4, 'type_alias', $5, $6, $7, $8) - ON CONFLICT (package_module_id, name) DO UPDATE - SET - documentation = $2, - signature_ = $3, - json_signature = $4, - nature = 'type_alias', - parameters = $5, - metadata = $6, - deprecation = $8" - |> pgo.execute( - db, - [ - pgo.text(type_name), - pgo.text(documentation), - pgo.text(signature), - json_signature - |> json.to_string() - |> pgo.text(), - dynamic.unsafe_coerce(dynamic.from(parameters)), - pgo.text(metadata), - pgo.int(module_id), - type_alias.deprecation - |> option.map(fn(d) { d.message }) - |> pgo.nullable(pgo.text, _), - ], - dynamic.dynamic, + use gen <- result.try(type_alias_to_json(ctx, type_name, type_alias)) + queries.upsert_package_type_fun_signature( + db: ctx.db, + name: type_name, + nature: queries.TypeAlias, + documentation: type_alias.documentation, + metadata: metadata.generate(type_alias.deprecation, None), + signature: type_alias_to_string(type_name, type_alias), + json_signature: gen.0, + parameters: gen.1, + module_id: module.id, + deprecation: type_alias.deprecation, + implementations: None, ) - |> result.map_error(error.DatabaseError) - |> result.replace(Nil) }) } -fn implementations_pgo(implementations: package_interface.Implementations) { - [ - #("gleam", implementations.gleam), - #("erlang", implementations.uses_erlang_externals), - #("javascript", implementations.uses_javascript_externals), - ] - |> list.filter(fn(t) { t.1 }) - |> list.map(fn(t) { t.0 }) - |> string.join(",") -} - -fn upsert_constants( - db: pgo.Connection, - package_interface: Package, - module_id: Int, - module: Module, - gleam_toml: Dict(String, Toml), -) { - let all_constants = dict.to_list(module.constants) +fn upsert_constants(ctx: Context, module: context.Module) { + let name = context.qualified_name(ctx, module) + wisp.log_info("Extracting " <> name <> " constants") + let all_constants = dict.to_list(module.module.constants) result.all({ use #(constant_name, constant) <- list.map(all_constants) - let documentation = string.trim(option.unwrap(constant.documentation, "")) - let deprecation = option.map(constant.deprecation, fn(d) { d.message }) - let impl = constant.implementations - let metadata = - json.object([ - #("deprecation", json.nullable(deprecation, json.string)), - #("implementations", implementations_to_json(impl)), - ]) - |> json.to_string() - let signature = constant_to_string(constant_name, constant) - let constant_json = - constant_to_json( - db, - package_interface, - constant_name, - constant, - gleam_toml, - ) - use #(json_signature, parameters) <- result.try(constant_json) - - "INSERT INTO package_type_fun_signature ( - name, - documentation, - signature_, - json_signature, - nature, - parameters, - metadata, - package_module_id, - deprecation, - implementations - ) VALUES ($1, $2, $3, $4, 'constant', $5, $6, $7, $8, $9) - ON CONFLICT (package_module_id, name) DO UPDATE - SET - documentation = $2, - signature_ = $3, - json_signature = $4, - nature = 'constant', - parameters = $5, - metadata = $6, - deprecation = $8, - implementations = $9" - |> pgo.execute( - db, - [ - pgo.text(constant_name), - pgo.text(documentation), - pgo.text(signature), - json_signature - |> json.to_string() - |> pgo.text(), - dynamic.unsafe_coerce(dynamic.from(parameters)), - pgo.text(metadata), - pgo.int(module_id), - constant.deprecation - |> option.map(fn(d) { d.message }) - |> pgo.nullable(pgo.text, _), - constant.implementations - |> implementations_pgo() - |> pgo.text(), - ], - dynamic.dynamic, + use gen <- result.try(constant_to_json(ctx, constant_name, constant)) + queries.upsert_package_type_fun_signature( + db: ctx.db, + name: constant_name, + nature: queries.Constant, + documentation: constant.documentation, + metadata: Some(constant.implementations) + |> metadata.generate(constant.deprecation, _), + signature: constant_to_string(constant_name, constant), + json_signature: gen.0, + parameters: gen.1, + module_id: module.id, + deprecation: constant.deprecation, + implementations: Some(constant.implementations), ) - |> result.map_error(error.DatabaseError) - |> result.replace(Nil) }) } -fn upsert_functions( - db: pgo.Connection, - package_interface: Package, - module_id: Int, - module: Module, - gleam_toml: Dict(String, Toml), -) { - let all_functions = dict.to_list(module.functions) +fn upsert_functions(ctx: Context, module: context.Module) { + let name = context.qualified_name(ctx, module) + wisp.log_info("Extracting " <> name <> " functions") + let all_functions = dict.to_list(module.module.functions) result.all({ use #(function_name, function) <- list.map(all_functions) - let documentation = string.trim(option.unwrap(function.documentation, "")) - let deprecation = option.map(function.deprecation, fn(d) { d.message }) - let impl = function.implementations - let metadata = - json.object([ - #("deprecation", json.nullable(deprecation, json.string)), - #("implementations", implementations_to_json(impl)), - ]) - |> json.to_string() - let signature = function_to_string(function_name, function) - let function_json = - function_to_json( - db, - package_interface, - function_name, - function, - gleam_toml, - ) - use #(json_signature, parameters) <- result.try(function_json) - - "INSERT INTO package_type_fun_signature ( - name, - documentation, - signature_, - json_signature, - nature, - parameters, - metadata, - package_module_id, - deprecation, - implementations - ) VALUES ($1, $2, $3, $4, 'function', $5, $6, $7, $8, $9) - ON CONFLICT (package_module_id, name) DO UPDATE - SET - documentation = $2, - signature_ = $3, - json_signature = $4, - nature = 'function', - parameters = $5, - metadata = $6, - deprecation = $8, - implementations = $9" - |> pgo.execute( - db, - [ - pgo.text(function_name), - pgo.text(documentation), - pgo.text(signature), - json_signature - |> json.to_string() - |> pgo.text(), - dynamic.unsafe_coerce(dynamic.from(parameters)), - pgo.text(metadata), - pgo.int(module_id), - function.deprecation - |> option.map(fn(d) { d.message }) - |> pgo.nullable(pgo.text, _), - function.implementations - |> implementations_pgo() - |> pgo.text(), - ], - dynamic.dynamic, + use gen <- result.try(function_to_json(ctx, function_name, function)) + queries.upsert_package_type_fun_signature( + db: ctx.db, + name: function_name, + nature: queries.Function, + documentation: function.documentation, + metadata: Some(function.implementations) + |> metadata.generate(function.deprecation, _), + signature: function_to_string(function_name, function), + json_signature: gen.0, + parameters: gen.1, + module_id: module.id, + deprecation: function.deprecation, + implementations: Some(function.implementations), ) - |> result.map_error(error.DatabaseError) - |> result.replace(Nil) }) } -pub fn extract_signatures( - db: pgo.Connection, - package: Package, - gleam_toml: Dict(String, Toml), +fn extract_module_signatures( + ctx: Context, + release_id: Int, + module: #(String, package_interface.Module), ) { - wisp.log_info( - "Extracting signatures for " <> package.name <> "@" <> package.version, - ) - use #(_pid, rid) <- result.try(get_package_release_ids(db, package)) - use _ <- result.try(add_gleam_constraint(db, package, rid)) + let module = context.Module(module.1, -1, module.0, release_id) + let name = context.qualified_name(ctx, module) + wisp.log_info("Extracting " <> name <> " signatures") + use module_id <- result.try(queries.upsert_package_module(ctx.db, module)) + let module = context.Module(..module, id: module_id) + use _ <- result.try(upsert_type_definitions(ctx, module)) + use _ <- result.try(upsert_type_aliases(ctx, module)) + use _ <- result.try(upsert_constants(ctx, module)) + let res = upsert_functions(ctx, module) + use <- bool.guard(when: result.is_error(res), return: res) + wisp.log_info("Extracting " <> name <> " finished") + res +} + +pub fn extract_signatures(ctx: Context) { + let package = ctx.package_interface + let package_slug = package.name <> "@" <> package.version + wisp.log_info("Extracting signatures for " <> package_slug) + let res = queries.get_package_release_ids(ctx.db, ctx.package_interface) + use #(_pid, release_id) <- result.try(res) + use _ <- result.try(add_gleam_constraint(ctx, release_id)) package.modules |> dict.to_list() - |> list.map(fn(mod) { - let #(mod_name, module) = mod - let qualified_name = - package.name <> "/" <> mod_name <> "@" <> package.version - wisp.log_info("Extracting signatures for " <> qualified_name) - use module_id <- result.try(upsert_package_module(db, mod_name, module, rid)) - wisp.log_info("Extracting " <> qualified_name <> " type definitions") - use _ <- result.try(upsert_type_definitions( - db, - package, - module_id, - module, - gleam_toml, - )) - wisp.log_info("Extracting " <> qualified_name <> " type aliases") - use _ <- result.try(upsert_type_aliases( - db, - package, - module_id, - module, - gleam_toml, - )) - wisp.log_info("Extracting " <> qualified_name <> " constants") - use _ <- result.try(upsert_constants( - db, - package, - module_id, - module, - gleam_toml, - )) - wisp.log_info("Extracting " <> qualified_name <> " functions") - upsert_functions(db, package, module_id, module, gleam_toml) - |> function.tap(fn(r) { - use <- bool.guard(when: result.is_error(r), return: Nil) - wisp.log_info("Extracting " <> qualified_name <> " finished") - }) - }) + |> list.map(extract_module_signatures(ctx, release_id, _)) |> result.all() } diff --git a/apps/backend/src/backend/data/hex_read.gleam b/apps/backend/src/backend/data/hex_read.gleam index 738fe7b..d4caa20 100644 --- a/apps/backend/src/backend/data/hex_read.gleam +++ b/apps/backend/src/backend/data/hex_read.gleam @@ -1,5 +1,15 @@ import birl.{type Time} +import gleam/dynamic +import helpers pub type HexRead { HexRead(id: Int, last_check: Time) } + +pub fn decode(data) { + dynamic.decode2( + HexRead, + dynamic.element(0, dynamic.int), + dynamic.element(1, helpers.decode_time), + )(data) +} diff --git a/apps/backend/src/backend/data/hex_user.gleam b/apps/backend/src/backend/data/hex_user.gleam index 9fdd93f..04c5a2d 100644 --- a/apps/backend/src/backend/data/hex_user.gleam +++ b/apps/backend/src/backend/data/hex_user.gleam @@ -1,5 +1,7 @@ import birl.{type Time} +import gleam/dynamic import gleam/option.{type Option} +import helpers pub type HexUser { HexUser( @@ -11,3 +13,15 @@ pub type HexUser { updated_at: Time, ) } + +pub fn decode(data) { + dynamic.decode6( + HexUser, + dynamic.element(0, dynamic.int), + dynamic.element(1, dynamic.string), + dynamic.element(2, dynamic.optional(dynamic.string)), + dynamic.element(3, dynamic.string), + dynamic.element(4, helpers.decode_time), + dynamic.element(5, helpers.decode_time), + )(data) +} diff --git a/apps/backend/src/backend/log_error.gleam b/apps/backend/src/backend/error.gleam similarity index 83% rename from apps/backend/src/backend/log_error.gleam rename to apps/backend/src/backend/error.gleam index 260ac73..bccb7b5 100644 --- a/apps/backend/src/backend/log_error.gleam +++ b/apps/backend/src/backend/error.gleam @@ -1,4 +1,3 @@ -import backend/index/error import gleam/dynamic import gleam/int import gleam/json @@ -10,6 +9,16 @@ import simplifile import tom import wisp +pub type Error { + DatabaseError(pgo.QueryError) + FetchError(dynamic.Dynamic) + JsonError(json.DecodeError) + SimplifileError(simplifile.FileError, String) + UnknownError(String) + ParseTomlError(tom.ParseError) + GetTomlError(tom.GetError) +} + pub fn log_dynamic_error(error: dynamic.DecodeError) { wisp.log_warning("Dynamic Decode Error") wisp.log_warning(" expected: " <> error.expected) @@ -38,34 +47,38 @@ pub fn log_decode_error(error: json.DecodeError) { } } -pub fn log_error(error: error.Error) { +pub fn log(error: Error) { case error { - error.FetchError(_dyn) -> wisp.log_warning("Fetch error") - error.DatabaseError(error) -> { + FetchError(_dyn) -> wisp.log_warning("Fetch error") + DatabaseError(error) -> { wisp.log_warning("Query error") log_pgo_error(error) } - error.JsonError(error) -> { + JsonError(error) -> { wisp.log_warning("JSON error") log_decode_error(error) } - error.SimplifileError(error, filepath) -> { + SimplifileError(error, filepath) -> { wisp.log_warning("Simplifile error") wisp.log_warning(" filepath: " <> filepath) log_simplifile(error) } - error.UnknownError(error) -> { + UnknownError(error) -> { wisp.log_warning("Unknown error") wisp.log_warning(" error: " <> error) } - error.TomlError(error) -> { - wisp.log_warning("Toml Error") - log_tom_error(error) + ParseTomlError(error) -> { + wisp.log_warning("Parse Toml Error") + log_parse_tom_error(error) + } + GetTomlError(error) -> { + wisp.log_warning("Get Toml Error") + log_get_tom_error(error) } } } -pub fn log_tom_error(error: tom.ParseError) { +pub fn log_parse_tom_error(error: tom.ParseError) { case error { tom.Unexpected(got, expected) -> { wisp.log_warning("Unexpected TOML error") @@ -79,6 +92,21 @@ pub fn log_tom_error(error: tom.ParseError) { } } +pub fn log_get_tom_error(error: tom.GetError) { + case error { + tom.NotFound(key) -> { + wisp.log_warning("Key not found") + wisp.log_warning(" key: " <> string.join(key, "/")) + } + tom.WrongType(key, expected, got) -> { + wisp.log_warning("Wrong type") + wisp.log_warning(" key: " <> string.join(key, "/")) + wisp.log_warning(" got: " <> got) + wisp.log_warning(" expected: " <> expected) + } + } +} + pub fn log_simplifile(error: simplifile.FileError) { case error { simplifile.Eacces -> wisp.log_warning("Eacces") diff --git a/apps/backend/src/backend/gleam/context.gleam b/apps/backend/src/backend/gleam/context.gleam new file mode 100644 index 0000000..80534df --- /dev/null +++ b/apps/backend/src/backend/gleam/context.gleam @@ -0,0 +1,27 @@ +import gleam/dict.{type Dict} +import gleam/package_interface +import gleam/pgo +import tom + +pub type Context { + Context( + db: pgo.Connection, + package_interface: package_interface.Package, + gleam_toml: Dict(String, tom.Toml), + ) +} + +pub type Module { + Module( + module: package_interface.Module, + id: Int, + name: String, + release_id: Int, + ) +} + +pub fn qualified_name(ctx: Context, module: Module) { + let package = ctx.package_interface + let module_slug = module.name <> "@" <> package.version + package.name <> "/" <> module_slug +} diff --git a/apps/backend/src/backend/gleam/generate/metadata.gleam b/apps/backend/src/backend/gleam/generate/metadata.gleam new file mode 100644 index 0000000..e216a1b --- /dev/null +++ b/apps/backend/src/backend/gleam/generate/metadata.gleam @@ -0,0 +1,22 @@ +import backend/gleam/generate/types +import gleam/json +import gleam/list +import gleam/option.{type Option, Some} +import gleam/package_interface +import gleam/pair + +pub fn generate( + deprecation: Option(package_interface.Deprecation), + impl: Option(package_interface.Implementations), +) { + let deprecation = + deprecation + |> option.map(fn(d) { d.message }) + |> json.nullable(json.string) + |> pair.new("deprecation", _) + impl + |> option.map(fn(i) { #("implementations", types.implementations_to_json(i)) }) + |> list.prepend([Some(deprecation)], _) + |> option.values() + |> json.object() +} diff --git a/apps/backend/src/gleam/generate/sources.gleam b/apps/backend/src/backend/gleam/generate/sources.gleam similarity index 100% rename from apps/backend/src/gleam/generate/sources.gleam rename to apps/backend/src/backend/gleam/generate/sources.gleam diff --git a/apps/backend/src/gleam/generate/types.gleam b/apps/backend/src/backend/gleam/generate/types.gleam similarity index 69% rename from apps/backend/src/gleam/generate/types.gleam rename to apps/backend/src/backend/gleam/generate/types.gleam index eb6b2e6..c623108 100644 --- a/apps/backend/src/gleam/generate/types.gleam +++ b/apps/backend/src/backend/gleam/generate/types.gleam @@ -1,26 +1,23 @@ -import backend/index/error +import backend/error +import backend/gleam/context.{type Context} +import backend/gleam/toml import gleam/bit_array import gleam/bool -import gleam/dict.{type Dict} +import gleam/dict import gleam/dynamic import gleam/json.{type Json} import gleam/list import gleam/option import gleam/order import gleam/package_interface.{ - type Constant, type Function, type Implementations, type Package, - type Parameter, type Type, type TypeAlias, type TypeConstructor, - type TypeDefinition, + type Constant, type Function, type Implementations, type Parameter, type Type, + type TypeAlias, type TypeConstructor, type TypeDefinition, } import gleam/pair import gleam/pgo import gleam/result import gleam/set.{type Set} import gleam/verl -import tom.{type Toml} - -type GleamToml = - Dict(String, Toml) fn reduce_components( components: List(a), @@ -35,13 +32,11 @@ fn reduce_components( } pub fn type_definition_to_json( - db: pgo.Connection, - package_interface: Package, + ctx: Context, type_name: String, type_def: TypeDefinition, - toml: GleamToml, ) -> Result(#(Json, List(Int)), error.Error) { - let mapper = type_constructor_to_json(db, package_interface, toml, _) + let mapper = type_constructor_to_json(ctx, _) use gen <- result.map(reduce_components(type_def.constructors, mapper)) use constructors <- pair.map_first(pair.map_second(gen, set.to_list)) json.object([ @@ -54,13 +49,8 @@ pub fn type_definition_to_json( ]) } -fn type_constructor_to_json( - db: pgo.Connection, - package_interface: Package, - gleam_toml: GleamToml, - constructor: TypeConstructor, -) { - let mapper = parameters_to_json(db, package_interface, gleam_toml, _) +fn type_constructor_to_json(ctx: Context, constructor: TypeConstructor) { + let mapper = parameters_to_json(ctx, _) use gen <- result.map(reduce_components(constructor.parameters, mapper)) use parameters <- pair.map_first(gen) json.object([ @@ -71,18 +61,8 @@ fn type_constructor_to_json( ]) } -fn parameters_to_json( - db: pgo.Connection, - package_interface: Package, - gleam_toml: GleamToml, - parameter: Parameter, -) { - use gen <- result.map(type_to_json( - db, - package_interface, - gleam_toml, - parameter.type_, - )) +fn parameters_to_json(ctx: Context, parameter: Parameter) { + use gen <- result.map(type_to_json(ctx, parameter.type_)) use type_ <- pair.map_first(gen) json.object([ #("type", json.string("parameter")), @@ -91,15 +71,10 @@ fn parameters_to_json( ]) } -fn type_to_json( - db: pgo.Connection, - package_interface: Package, - gleam_toml: GleamToml, - type_: Type, -) { +fn type_to_json(ctx: Context, type_: Type) { case type_ { package_interface.Tuple(elements) -> { - let mapper = type_to_json(db, package_interface, gleam_toml, _) + let mapper = type_to_json(ctx, _) use gen <- result.map(reduce_components(elements, mapper)) use elements <- pair.map_first(gen) json.object([ @@ -108,14 +83,9 @@ fn type_to_json( ]) } package_interface.Fn(params, return) -> { - let mapper = type_to_json(db, package_interface, gleam_toml, _) + let mapper = type_to_json(ctx, _) use #(elements, params) <- result.try(reduce_components(params, mapper)) - use gen <- result.map(type_to_json( - db, - package_interface, - gleam_toml, - return, - )) + use gen <- result.map(type_to_json(ctx, return)) let new_params = set.union(of: params, and: gen.1) json.object([ #("type", json.string("fn")), @@ -130,16 +100,10 @@ fn type_to_json( Ok(#(json, set.new())) } package_interface.Named(name, package, module, parameters) -> { - let mapper = type_to_json(db, package_interface, gleam_toml, _) + let mapper = type_to_json(ctx, _) use gen <- result.try(reduce_components(parameters, mapper)) - use ref <- result.map(extract_parameters_relation( - db, - package_interface, - gleam_toml, - name, - package, - module, - )) + let res = extract_parameters_relation(ctx, name, package, module) + use ref <- result.map(res) let new_ids = case ref { option.None -> gen.1 option.Some(ref) -> set.insert(gen.1, ref) @@ -157,11 +121,7 @@ fn type_to_json( } } -fn find_package_release( - db: pgo.Connection, - package: String, - requirement: String, -) { +fn find_package_release(ctx: Context, package: String, requirement: String) { let decoder = dynamic.tuple2(dynamic.int, dynamic.string) use response <- result.try({ "SELECT package_release.id, package_release.version @@ -169,7 +129,7 @@ fn find_package_release( JOIN package_release ON package.id = package_release.package_id WHERE package.name = $1" - |> pgo.execute(db, [pgo.text(package)], decoder) + |> pgo.execute(ctx.db, [pgo.text(package)], decoder) |> result.map_error(error.DatabaseError) }) response.rows @@ -197,8 +157,7 @@ fn keep_matching_releases(rows: List(#(Int, String)), requirement: String) { } fn find_type_signature( - db: pgo.Connection, - package_interface: Package, + ctx: Context, name: String, package: String, module: String, @@ -216,7 +175,7 @@ fn find_type_signature( AND package_module.name = $2 AND package_module.package_release_id = $3" |> pgo.execute( - db, + ctx.db, [pgo.text(name), pgo.text(module), pgo.int(release)], dynamic.element(0, dynamic.int), ) @@ -231,10 +190,10 @@ fn find_type_signature( }) { option.None -> { - case package_interface.name == package { + case ctx.package_interface.name == package { False -> Error(error.UnknownError("No release found")) True -> - case dict.get(package_interface.modules, module) { + case dict.get(ctx.package_interface.modules, module) { Error(_) -> Error(error.UnknownError("No module found")) Ok(mod) -> case dict.get(mod.type_aliases, name) { @@ -253,30 +212,15 @@ fn find_type_signature( } fn extract_parameters_relation( - db: pgo.Connection, - package_interface: Package, - gleam_toml: GleamToml, + ctx: Context, name: String, package: String, module: String, ) -> Result(option.Option(Int), error.Error) { use <- bool.guard(when: is_prelude(package, module), return: Ok(option.None)) - use requirement <- result.try(get_toml_requirement(gleam_toml, package)) - use releases <- result.try(find_package_release(db, package, requirement)) - find_type_signature(db, package_interface, name, package, module, releases) -} - -fn get_toml_requirement(gleam_toml: GleamToml, package: String) { - tom.get_string(gleam_toml, ["name"]) - |> result.try(fn(package_name) { - let not_same_package = package_name != package - use <- bool.guard(when: not_same_package, return: Error(tom.NotFound([]))) - tom.get_string(gleam_toml, ["version"]) - }) - |> result.try_recover(fn(_) { - tom.get_string(gleam_toml, ["dependencies", package]) - }) - |> result.replace_error(error.UnknownError("No dep found for " <> package)) + use requirement <- result.try(toml.find_package_requirement(ctx, package)) + use releases <- result.try(find_package_release(ctx, package, requirement)) + find_type_signature(ctx, name, package, module, releases) } fn is_prelude(package: String, module: String) { @@ -284,18 +228,11 @@ fn is_prelude(package: String, module: String) { } pub fn type_alias_to_json( - db: pgo.Connection, - package_interface: Package, + ctx: Context, type_name: String, type_alias: TypeAlias, - gleam_toml: GleamToml, ) { - use gen <- result.map(type_to_json( - db, - package_interface, - gleam_toml, - type_alias.alias, - )) + use gen <- result.map(type_to_json(ctx, type_alias.alias)) use alias <- pair.map_first(pair.map_second(gen, set.to_list)) json.object([ #("type", json.string("type-alias")), @@ -308,29 +245,16 @@ pub fn type_alias_to_json( } pub fn implementations_to_json(implementations: Implementations) { + let uses_js = json.bool(implementations.uses_javascript_externals) json.object([ #("gleam", json.bool(implementations.gleam)), #("uses_erlang_externals", json.bool(implementations.uses_erlang_externals)), - #( - "uses_javascript_externals", - json.bool(implementations.uses_javascript_externals), - ), + #("uses_javascript_externals", uses_js), ]) } -pub fn constant_to_json( - db: pgo.Connection, - package_interface: Package, - constant_name: String, - constant: Constant, - gleam_toml: GleamToml, -) { - use gen <- result.map(type_to_json( - db, - package_interface, - gleam_toml, - constant.type_, - )) +pub fn constant_to_json(ctx: Context, constant_name: String, constant: Constant) { + use gen <- result.map(type_to_json(ctx, constant.type_)) use type_ <- pair.map_first(pair.map_second(gen, set.to_list)) json.object([ #("type", json.string("constant")), @@ -342,21 +266,10 @@ pub fn constant_to_json( ]) } -pub fn function_to_json( - db: pgo.Connection, - package_interface: Package, - function_name: String, - function: Function, - gleam_toml: GleamToml, -) { - let mapper = parameters_to_json(db, package_interface, gleam_toml, _) +pub fn function_to_json(ctx: Context, function_name: String, function: Function) { + let mapper = parameters_to_json(ctx, _) use gen <- result.try(reduce_components(function.parameters, mapper)) - use ret <- result.map(type_to_json( - db, - package_interface, - gleam_toml, - function.return, - )) + use ret <- result.map(type_to_json(ctx, function.return)) gen |> pair.map_second(fn(s) { set.to_list(set.union(s, ret.1)) }) |> pair.map_first(fn(parameters) { diff --git a/apps/backend/src/backend/gleam/toml.gleam b/apps/backend/src/backend/gleam/toml.gleam new file mode 100644 index 0000000..f8ee3a0 --- /dev/null +++ b/apps/backend/src/backend/gleam/toml.gleam @@ -0,0 +1,26 @@ +import backend/error +import backend/gleam/context.{type Context} +import gleam/bool +import gleam/result +import tom + +fn is_dependency(ctx: Context, package_name: String) { + use name <- result.map(tom.get_string(ctx.gleam_toml, ["name"])) + name != package_name +} + +pub fn extract_dep_version(ctx: Context, package_name: String) { + tom.get_string(ctx.gleam_toml, ["dependencies", package_name]) +} + +fn extract_package_version(ctx: Context, package_name: String) { + use is_dep <- result.try(is_dependency(ctx, package_name)) + use <- bool.guard(when: is_dep, return: Error(tom.NotFound([]))) + tom.get_string(ctx.gleam_toml, ["version"]) +} + +pub fn find_package_requirement(ctx: Context, package_name: String) { + extract_package_version(ctx, package_name) + |> result.try_recover(fn(_) { extract_dep_version(ctx, package_name) }) + |> result.map_error(error.GetTomlError) +} diff --git a/apps/backend/src/backend/index/decoders.gleam b/apps/backend/src/backend/index/decoders.gleam deleted file mode 100644 index 9f4199e..0000000 --- a/apps/backend/src/backend/index/decoders.gleam +++ /dev/null @@ -1,24 +0,0 @@ -import backend/data/hex_read -import backend/data/hex_user -import backend/index/helpers -import gleam/dynamic - -pub fn hex_user(data) { - dynamic.decode6( - hex_user.HexUser, - dynamic.element(0, dynamic.int), - dynamic.element(1, dynamic.string), - dynamic.element(2, dynamic.optional(dynamic.string)), - dynamic.element(3, dynamic.string), - dynamic.element(4, helpers.decode_time), - dynamic.element(5, helpers.decode_time), - )(data) -} - -pub fn hex_read(data) { - dynamic.decode2( - hex_read.HexRead, - dynamic.element(0, dynamic.int), - dynamic.element(1, helpers.decode_time), - )(data) -} diff --git a/apps/backend/src/backend/index/error.gleam b/apps/backend/src/backend/index/error.gleam deleted file mode 100644 index e860085..0000000 --- a/apps/backend/src/backend/index/error.gleam +++ /dev/null @@ -1,14 +0,0 @@ -import gleam/dynamic -import gleam/json -import gleam/pgo -import simplifile -import tom - -pub type Error { - DatabaseError(pgo.QueryError) - FetchError(dynamic.Dynamic) - JsonError(json.DecodeError) - SimplifileError(simplifile.FileError, String) - UnknownError(String) - TomlError(tom.ParseError) -} diff --git a/apps/backend/src/backend/index/connect.gleam b/apps/backend/src/backend/postgres/postgres.gleam similarity index 100% rename from apps/backend/src/backend/index/connect.gleam rename to apps/backend/src/backend/postgres/postgres.gleam diff --git a/apps/backend/src/backend/index/queries.gleam b/apps/backend/src/backend/postgres/queries.gleam similarity index 53% rename from apps/backend/src/backend/index/queries.gleam rename to apps/backend/src/backend/postgres/queries.gleam index 9664847..067d7bb 100644 --- a/apps/backend/src/backend/index/queries.gleam +++ b/apps/backend/src/backend/postgres/queries.gleam @@ -1,22 +1,32 @@ import backend/data/hex_read.{type HexRead, HexRead} import backend/data/hex_user.{type HexUser} -import backend/index/decoders -import backend/index/error -import backend/index/helpers +import backend/error +import backend/gleam/context import birl.{type Time} import gleam/bool import gleam/dict import gleam/dynamic import gleam/hexpm import gleam/io +import gleam/json import gleam/list -import gleam/option +import gleam/option.{type Option} +import gleam/package_interface import gleam/pgo import gleam/result +import gleam/string +import helpers + +pub type SignatureNature { + TypeAlias + TypeDefinition + Constant + Function +} pub fn get_last_hex_date(db: pgo.Connection) { "SELECT id, last_check FROM hex_read ORDER BY last_check DESC LIMIT 1" - |> pgo.execute(db, [], decoders.hex_read) + |> pgo.execute(db, [], hex_read.decode) |> result.map_error(error.DatabaseError) |> result.map(fn(response) { response.rows @@ -33,7 +43,7 @@ pub fn upsert_most_recent_hex_timestamp(db: pgo.Connection, latest: Time) { ON CONFLICT (id) DO UPDATE SET last_check = $1 RETURNING *" - |> pgo.execute(db, [timestamp], decoders.hex_read) + |> pgo.execute(db, [timestamp], hex_read.decode) |> result.map_error(io.debug) |> result.map_error(error.DatabaseError) |> result.try(fn(response) { @@ -53,7 +63,7 @@ pub fn upsert_hex_user(db: pgo.Connection, owner: hexpm.PackageOwner) { ON CONFLICT (username) DO UPDATE SET email = $2, url = $3 RETURNING id, username, email, url, created_at, updated_at" - |> pgo.execute(db, [username, email, url], decoders.hex_user) + |> pgo.execute(db, [username, email, url], hex_user.decode) |> result.map(fn(r) { r.rows }) |> result.map_error(error.DatabaseError) } @@ -184,3 +194,148 @@ pub fn upsert_release( |> pgo.execute(db, [package_id, version, url], dynamic.dynamic) |> result.map_error(error.DatabaseError) } + +pub fn add_package_gleam_constraint( + db: pgo.Connection, + constraint: String, + release_id: Int, +) { + let constraint = pgo.text(constraint) + let release_id = pgo.int(release_id) + "UPDATE package_release SET gleam_constraint = $1 WHERE id = $2" + |> pgo.execute(db, [constraint, release_id], dynamic.dynamic) + |> result.replace(Nil) + |> result.map_error(error.DatabaseError) +} + +pub fn get_package_release_ids( + db: pgo.Connection, + package: package_interface.Package, +) { + use response <- result.try({ + let args = [pgo.text(package.name), pgo.text(package.version)] + "SELECT + package.id package_id, + package_release.id package_release_id + FROM package + JOIN package_release + ON package_release.package_id = package.id + WHERE package.name = $1 + AND package_release.version = $2" + |> pgo.execute(db, args, dynamic.tuple2(dynamic.int, dynamic.int)) + |> result.map_error(error.DatabaseError) + }) + response.rows + |> list.first() + |> result.replace_error(error.UnknownError( + "No release found for " <> package.name <> "@" <> package.version, + )) +} + +pub fn upsert_package_module(db: pgo.Connection, module: context.Module) { + use response <- result.try({ + let args = [ + pgo.text(module.name), + module.module.documentation + |> string.join("\n") + |> pgo.text(), + pgo.int(module.release_id), + ] + "INSERT INTO package_module (name, documentation, package_release_id) + VALUES ($1, $2, $3) + ON CONFLICT (name, package_release_id) DO UPDATE + SET documentation = $2 + RETURNING id" + |> pgo.execute(db, args, dynamic.element(0, dynamic.int)) + |> result.map_error(error.DatabaseError) + }) + response.rows + |> list.first() + |> result.replace_error(error.UnknownError( + "No module found for " <> module.name, + )) +} + +fn implementations_pgo(implementations: package_interface.Implementations) { + [ + #("gleam", implementations.gleam), + #("erlang", implementations.uses_erlang_externals), + #("javascript", implementations.uses_javascript_externals), + ] + |> list.filter(fn(t) { t.1 }) + |> list.map(fn(t) { t.0 }) + |> string.join(",") +} + +pub fn upsert_package_type_fun_signature( + db db: pgo.Connection, + nature nature: SignatureNature, + name name: String, + documentation documentation: Option(String), + metadata metadata: json.Json, + signature signature: String, + json_signature json_signature: json.Json, + parameters parameters: List(Int), + module_id module_id: Int, + deprecation deprecation: Option(package_interface.Deprecation), + implementations implementations: Option(package_interface.Implementations), +) { + let nature = case nature { + Function -> "function" + TypeAlias -> "type_alias" + TypeDefinition -> "type_definition" + Constant -> "constant" + } + "INSERT INTO package_type_fun_signature ( + name, + documentation, + signature_, + json_signature, + nature, + parameters, + metadata, + package_module_id, + deprecation, + implementations + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (package_module_id, name) DO UPDATE + SET + documentation = $2, + signature_ = $3, + json_signature = $4, + nature = $5, + parameters = $6, + metadata = $7, + deprecation = $9, + implementations = $10" + |> pgo.execute( + db, + [ + pgo.text(name), + documentation + |> option.unwrap("") + |> string.trim() + |> pgo.text(), + pgo.text(signature), + json_signature + |> json.to_string() + |> pgo.text(), + pgo.text(nature), + dynamic.unsafe_coerce(dynamic.from(parameters)), + metadata + |> json.to_string() + |> pgo.text(), + pgo.int(module_id), + deprecation + |> option.map(fn(d) { d.message }) + |> pgo.nullable(pgo.text, _), + implementations + |> option.map(implementations_pgo) + |> option.map(pgo.text) + |> option.unwrap(pgo.null()), + ], + dynamic.dynamic, + ) + |> result.map_error(error.DatabaseError) + |> result.replace(Nil) +} diff --git a/apps/backend/src/backend/index/helpers.gleam b/apps/backend/src/helpers.gleam similarity index 100% rename from apps/backend/src/backend/index/helpers.gleam rename to apps/backend/src/helpers.gleam diff --git a/apps/backend/src/periodic.gleam b/apps/backend/src/periodic.gleam index 9a508f5..fd44392 100644 --- a/apps/backend/src/periodic.gleam +++ b/apps/backend/src/periodic.gleam @@ -1,9 +1,9 @@ +import backend/error.{type Error} import gleam/erlang/process.{type Subject} -import gleam/io import gleam/function +import gleam/io import gleam/otp/actor import gleam/result -import backend/index/error.{type Error} pub opaque type Message { Rerun diff --git a/apps/backend/src/retrier.gleam b/apps/backend/src/retrier.gleam index 0cb9f81..0844e0f 100644 --- a/apps/backend/src/retrier.gleam +++ b/apps/backend/src/retrier.gleam @@ -1,5 +1,4 @@ -import backend/index/error.{type Error} -import backend/log_error +import backend/error.{type Error} import gleam/erlang/process.{type Subject} import gleam/function import gleam/otp/actor @@ -44,7 +43,7 @@ fn loop(message: Message, state: State(a)) -> actor.Next(Message, State(a)) { case state.work() { Ok(_) -> actor.Stop(process.Normal) Error(e) -> { - log_error.log_error(e) + error.log(e) enqueue_next_rerun(state) actor.continue(state) } diff --git a/apps/backend/src/tasks/hex.gleam b/apps/backend/src/tasks/hex.gleam index a2ec893..6903e82 100644 --- a/apps/backend/src/tasks/hex.gleam +++ b/apps/backend/src/tasks/hex.gleam @@ -3,9 +3,10 @@ import api/hex_repo import api/signatures import backend/config.{type Config} import backend/data/hex_read.{type HexRead} -import backend/index/connect as postgres -import backend/index/error.{type Error} -import backend/index/queries as index +import backend/error.{type Error} +import backend/gleam/context +import backend/postgres/postgres +import backend/postgres/queries import birl.{type Time} import birl/duration import gleam/hexpm.{type Package} @@ -36,7 +37,7 @@ pub fn sync_new_gleam_releases( ) -> Result(HexRead, Error) { let ctx = postgres.connect(cnf) wisp.log_info("Syncing new releases from Hex") - use limit <- result.try(index.get_last_hex_date(ctx.connection)) + use limit <- result.try(queries.get_last_hex_date(ctx.connection)) use latest <- result.try(sync_packages( State( page: 1, @@ -48,7 +49,7 @@ pub fn sync_new_gleam_releases( ), children, )) - let latest = index.upsert_most_recent_hex_timestamp(ctx.connection, latest) + let latest = queries.upsert_most_recent_hex_timestamp(ctx.connection, latest) wisp.log_info("\nUp to date!") latest } @@ -119,13 +120,13 @@ fn insert_package_and_releases( |> list.map(fn(release) { release.version }) |> string.join(", v") wisp.log_info("Saving " <> package.name <> " v" <> versions) - use id <- result.try(index.upsert_package(state.db, package)) + use id <- result.try(queries.upsert_package(state.db, package)) wisp.log_info("Saving owners for " <> package.name) use owners <- result.try(api.get_package_owners(package.name, secret: secret)) - use _ <- result.try(index.sync_package_owners(state.db, id, owners)) + use _ <- result.try(queries.sync_package_owners(state.db, id, owners)) wisp.log_info("Saving releases for " <> package.name) list.try_each(releases, fn(r) { - use _ <- result.map(index.upsert_release(state.db, id, r)) + use _ <- result.map(queries.upsert_release(state.db, id, r)) supervisor.add(children, { use _ <- supervisor.worker() retrier.retry(fn() { @@ -133,8 +134,10 @@ fn insert_package_and_releases( use #(package, gleam_toml) <- result.try(infos) case package { option.None -> Ok([]) - option.Some(package) -> - signatures.extract_signatures(state.db, package, gleam_toml) + option.Some(package) -> { + let ctx = context.Context(state.db, package, gleam_toml) + signatures.extract_signatures(ctx) + } } }) })