From 85fe37ddcb3848f7bdad3588e5044970239e47f1 Mon Sep 17 00:00:00 2001 From: Victor Song Date: Thu, 29 Feb 2024 02:56:22 -0600 Subject: [PATCH 1/3] fix(topbar): support teams in topbar owner links --- ...f3f5addfba6ed45f46afdc523b324da2cbab.json} | 10 ++++- ...7903829961a7b50ae4bfbdb9b34c38f374932.json | 23 ----------- ...a84edd7d17e6171523ebbd57f1d9dd7c9b46c.json | 24 +++++++++++ ...dacc3289dcadf82061774866fec3a62669c5.json} | 10 ++++- ...e5e9c7016a1b8b35b031cb05b4d0802b6be1.json} | 10 ++++- .../20240227040753_add_owner_kind.down.sql | 2 + .../20240227040753_add_owner_kind.up.sql | 4 ++ src/db/add_package.rs | 21 +++++++--- src/db/delete.rs | 2 + src/registry_api.rs | 4 ++ src/web/crate_details.rs | 41 +++++++++++++++---- src/web/releases.rs | 1 + templates/rustdoc/topbar.html | 2 +- 13 files changed, 110 insertions(+), 44 deletions(-) rename .sqlx/{query-d9fdce61d807d32b2c700c29e0b8100b5abf2d283016f48f468d823bd85da551.json => query-24c697ed817d33dbdfb0e5126feaf3f5addfba6ed45f46afdc523b324da2cbab.json} (52%) delete mode 100644 .sqlx/query-5deb5bb52b993cc54f7b48714c77903829961a7b50ae4bfbdb9b34c38f374932.json create mode 100644 .sqlx/query-87952bd450ed2c13b99bd502a73a84edd7d17e6171523ebbd57f1d9dd7c9b46c.json rename .sqlx/{query-073b7016109d65c3d7907ce2b32b47018ce25577d6d9267da38edb53828ada41.json => query-b6273727360630c958b856427cb9dacc3289dcadf82061774866fec3a62669c5.json} (60%) rename .sqlx/{query-8e1cb8355b3586b849494ae1cbde9b034ca01090469a84df2d9af4446f9d9451.json => query-ee9dddd0c37c0e89eab1feaddd27e5e9c7016a1b8b35b031cb05b4d0802b6be1.json} (58%) create mode 100644 migrations/20240227040753_add_owner_kind.down.sql create mode 100644 migrations/20240227040753_add_owner_kind.up.sql diff --git a/.sqlx/query-d9fdce61d807d32b2c700c29e0b8100b5abf2d283016f48f468d823bd85da551.json b/.sqlx/query-24c697ed817d33dbdfb0e5126feaf3f5addfba6ed45f46afdc523b324da2cbab.json similarity index 52% rename from .sqlx/query-d9fdce61d807d32b2c700c29e0b8100b5abf2d283016f48f468d823bd85da551.json rename to .sqlx/query-24c697ed817d33dbdfb0e5126feaf3f5addfba6ed45f46afdc523b324da2cbab.json index 709cac8c7..8ec492e59 100644 --- a/.sqlx/query-d9fdce61d807d32b2c700c29e0b8100b5abf2d283016f48f468d823bd85da551.json +++ b/.sqlx/query-24c697ed817d33dbdfb0e5126feaf3f5addfba6ed45f46afdc523b324da2cbab.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT login, avatar\n FROM owners\n INNER JOIN owner_rels ON owner_rels.oid = owners.id\n WHERE cid = $1", + "query": "SELECT login, avatar, kind\n FROM owners\n INNER JOIN owner_rels ON owner_rels.oid = owners.id\n WHERE cid = $1", "describe": { "columns": [ { @@ -12,6 +12,11 @@ "ordinal": 1, "name": "avatar", "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "kind", + "type_info": "Text" } ], "parameters": { @@ -20,9 +25,10 @@ ] }, "nullable": [ + false, false, false ] }, - "hash": "d9fdce61d807d32b2c700c29e0b8100b5abf2d283016f48f468d823bd85da551" + "hash": "24c697ed817d33dbdfb0e5126feaf3f5addfba6ed45f46afdc523b324da2cbab" } diff --git a/.sqlx/query-5deb5bb52b993cc54f7b48714c77903829961a7b50ae4bfbdb9b34c38f374932.json b/.sqlx/query-5deb5bb52b993cc54f7b48714c77903829961a7b50ae4bfbdb9b34c38f374932.json deleted file mode 100644 index 6200d4eb0..000000000 --- a/.sqlx/query-5deb5bb52b993cc54f7b48714c77903829961a7b50ae4bfbdb9b34c38f374932.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO owners (login, avatar)\n VALUES ($1, $2)\n ON CONFLICT (login) DO UPDATE\n SET\n avatar = EXCLUDED.avatar\n RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Varchar" - ] - }, - "nullable": [ - false - ] - }, - "hash": "5deb5bb52b993cc54f7b48714c77903829961a7b50ae4bfbdb9b34c38f374932" -} diff --git a/.sqlx/query-87952bd450ed2c13b99bd502a73a84edd7d17e6171523ebbd57f1d9dd7c9b46c.json b/.sqlx/query-87952bd450ed2c13b99bd502a73a84edd7d17e6171523ebbd57f1d9dd7c9b46c.json new file mode 100644 index 000000000..0b8f8417f --- /dev/null +++ b/.sqlx/query-87952bd450ed2c13b99bd502a73a84edd7d17e6171523ebbd57f1d9dd7c9b46c.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO owners (login, avatar, kind)\n VALUES ($1, $2, $3)\n ON CONFLICT (login) DO UPDATE\n SET\n avatar = EXCLUDED.avatar,\n kind = EXCLUDED.kind\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "87952bd450ed2c13b99bd502a73a84edd7d17e6171523ebbd57f1d9dd7c9b46c" +} diff --git a/.sqlx/query-073b7016109d65c3d7907ce2b32b47018ce25577d6d9267da38edb53828ada41.json b/.sqlx/query-b6273727360630c958b856427cb9dacc3289dcadf82061774866fec3a62669c5.json similarity index 60% rename from .sqlx/query-073b7016109d65c3d7907ce2b32b47018ce25577d6d9267da38edb53828ada41.json rename to .sqlx/query-b6273727360630c958b856427cb9dacc3289dcadf82061774866fec3a62669c5.json index 2dcdec9a3..9aea6ce4e 100644 --- a/.sqlx/query-073b7016109d65c3d7907ce2b32b47018ce25577d6d9267da38edb53828ada41.json +++ b/.sqlx/query-b6273727360630c958b856427cb9dacc3289dcadf82061774866fec3a62669c5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT login, avatar\n FROM owners", + "query": "SELECT login, avatar, kind FROM owners", "describe": { "columns": [ { @@ -12,15 +12,21 @@ "ordinal": 1, "name": "avatar", "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "kind", + "type_info": "Text" } ], "parameters": { "Left": [] }, "nullable": [ + false, false, false ] }, - "hash": "073b7016109d65c3d7907ce2b32b47018ce25577d6d9267da38edb53828ada41" + "hash": "b6273727360630c958b856427cb9dacc3289dcadf82061774866fec3a62669c5" } diff --git a/.sqlx/query-8e1cb8355b3586b849494ae1cbde9b034ca01090469a84df2d9af4446f9d9451.json b/.sqlx/query-ee9dddd0c37c0e89eab1feaddd27e5e9c7016a1b8b35b031cb05b4d0802b6be1.json similarity index 58% rename from .sqlx/query-8e1cb8355b3586b849494ae1cbde9b034ca01090469a84df2d9af4446f9d9451.json rename to .sqlx/query-ee9dddd0c37c0e89eab1feaddd27e5e9c7016a1b8b35b031cb05b4d0802b6be1.json index 84aff1df2..c8d901e06 100644 --- a/.sqlx/query-8e1cb8355b3586b849494ae1cbde9b034ca01090469a84df2d9af4446f9d9451.json +++ b/.sqlx/query-ee9dddd0c37c0e89eab1feaddd27e5e9c7016a1b8b35b031cb05b4d0802b6be1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT login, avatar FROM owners", + "query": "SELECT login, avatar, kind\n FROM owners", "describe": { "columns": [ { @@ -12,15 +12,21 @@ "ordinal": 1, "name": "avatar", "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "kind", + "type_info": "Text" } ], "parameters": { "Left": [] }, "nullable": [ + false, false, false ] }, - "hash": "8e1cb8355b3586b849494ae1cbde9b034ca01090469a84df2d9af4446f9d9451" + "hash": "ee9dddd0c37c0e89eab1feaddd27e5e9c7016a1b8b35b031cb05b4d0802b6be1" } diff --git a/migrations/20240227040753_add_owner_kind.down.sql b/migrations/20240227040753_add_owner_kind.down.sql new file mode 100644 index 000000000..f897ec835 --- /dev/null +++ b/migrations/20240227040753_add_owner_kind.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE owners +DROP COLUMN IF EXISTS kind; diff --git a/migrations/20240227040753_add_owner_kind.up.sql b/migrations/20240227040753_add_owner_kind.up.sql new file mode 100644 index 000000000..5c182918d --- /dev/null +++ b/migrations/20240227040753_add_owner_kind.up.sql @@ -0,0 +1,4 @@ +ALTER TABLE owners +ADD COLUMN IF NOT EXISTS kind TEXT NOT NULL CHECK ( + kind IN ('user', 'team') +) DEFAULT 'user'; diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 5cc5a87f3..7302be584 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -454,14 +454,16 @@ async fn update_owners_in_database( for owner in owners { oids.push( sqlx::query_scalar!( - "INSERT INTO owners (login, avatar) - VALUES ($1, $2) + "INSERT INTO owners (login, avatar, kind) + VALUES ($1, $2, $3) ON CONFLICT (login) DO UPDATE SET - avatar = EXCLUDED.avatar + avatar = EXCLUDED.avatar, + kind = EXCLUDED.kind RETURNING id", owner.login, - owner.avatar + owner.avatar, + owner.kind, ) .fetch_one(&mut *conn) .await?, @@ -647,18 +649,20 @@ mod test { let owner1 = CrateOwner { avatar: "avatar".into(), login: "login".into(), + kind: "user".into(), }; update_owners_in_database(&mut conn, &[owner1.clone()], crate_id).await?; let owner_def = sqlx::query!( - "SELECT login, avatar + "SELECT login, avatar, kind FROM owners" ) .fetch_one(&mut *conn) .await?; assert_eq!(owner_def.login, owner1.login); assert_eq!(owner_def.avatar, owner1.avatar); + assert_eq!(owner_def.kind, owner1.kind); let owner_rel = sqlx::query!( "SELECT o.login @@ -688,6 +692,7 @@ mod test { &[CrateOwner { login: "login".into(), avatar: "avatar".into(), + kind: "user".into(), }], crate_id, ) @@ -696,14 +701,16 @@ mod test { let updated_owner = CrateOwner { login: "login".into(), avatar: "avatar2".into(), + kind: "team".into(), }; update_owners_in_database(&mut conn, &[updated_owner.clone()], crate_id).await?; - let owner_def = sqlx::query!("SELECT login, avatar FROM owners") + let owner_def = sqlx::query!("SELECT login, avatar, kind FROM owners") .fetch_one(&mut *conn) .await?; assert_eq!(owner_def.login, updated_owner.login); assert_eq!(owner_def.avatar, updated_owner.avatar); + assert_eq!(owner_def.kind, updated_owner.kind); let owner_rel = sqlx::query!( "SELECT o.login @@ -733,6 +740,7 @@ mod test { &[CrateOwner { login: "login".into(), avatar: "avatar".into(), + kind: "user".into(), }], crate_id, ) @@ -742,6 +750,7 @@ mod test { .map(|i| CrateOwner { login: format!("login{i}"), avatar: format!("avatar{i}"), + kind: "user".into(), }) .collect(); diff --git a/src/db/delete.rs b/src/db/delete.rs index 7f9c778f6..49de88b50 100644 --- a/src/db/delete.rs +++ b/src/db/delete.rs @@ -328,6 +328,7 @@ mod tests { .add_owner(CrateOwner { login: "malicious actor".into(), avatar: "https://example.org/malicious".into(), + kind: "user".into(), }) .create()?; assert!(release_exists(&mut db.conn(), v1)?); @@ -358,6 +359,7 @@ mod tests { .add_owner(CrateOwner { login: "Peter Rabbit".into(), avatar: "https://example.org/peter".into(), + kind: "user".into(), }) .create()?; assert!(release_exists(&mut db.conn(), v2)?); diff --git a/src/registry_api.rs b/src/registry_api.rs index fb0e1ca5d..f067bd847 100644 --- a/src/registry_api.rs +++ b/src/registry_api.rs @@ -46,6 +46,7 @@ impl Default for ReleaseData { pub struct CrateOwner { pub(crate) avatar: String, pub(crate) login: String, + pub(crate) kind: String, } impl RegistryApi { @@ -168,6 +169,8 @@ impl RegistryApi { avatar: Option, #[serde(default)] login: Option, + #[serde(default)] + kind: Option, } let response: Response = retry_async( @@ -198,6 +201,7 @@ impl RegistryApi { .map(|data| CrateOwner { avatar: data.avatar.unwrap_or_default(), login: data.login.unwrap_or_default(), + kind: data.kind.unwrap_or("user".into()), }) .collect(); diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index be079ff41..119470bcb 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -35,7 +35,7 @@ pub(crate) struct CrateDetails { name: String, pub version: Version, description: Option, - owners: Vec<(String, String)>, + owners: Vec<(String, String, String)>, dependencies: Option, #[serde(serialize_with = "optional_markdown")] readme: Option, @@ -264,14 +264,14 @@ impl CrateDetails { // get owners crate_details.owners = sqlx::query!( - "SELECT login, avatar + "SELECT login, avatar, kind FROM owners INNER JOIN owner_rels ON owner_rels.oid = owners.id WHERE cid = $1", krate.crate_id, ) .fetch(&mut *conn) - .map_ok(|row| (row.login, row.avatar)) + .map_ok(|row| (row.login, row.avatar, row.kind)) .try_collect() .await?; @@ -1253,6 +1253,7 @@ mod tests { .add_owner(CrateOwner { login: "foobar".into(), avatar: "https://example.org/foobar".into(), + kind: "user".into(), }) .create()?; @@ -1262,7 +1263,11 @@ mod tests { }); assert_eq!( details.owners, - vec![("foobar".into(), "https://example.org/foobar".into())] + vec![( + "foobar".into(), + "https://example.org/foobar".into(), + "user".into() + )] ); // Adding a new owner, and changing details on an existing owner @@ -1272,10 +1277,12 @@ mod tests { .add_owner(CrateOwner { login: "foobar".into(), avatar: "https://example.org/foobarv2".into(), + kind: "user".into(), }) .add_owner(CrateOwner { login: "barfoo".into(), avatar: "https://example.org/barfoo".into(), + kind: "user".into(), }) .create()?; @@ -1288,8 +1295,16 @@ mod tests { assert_eq!( owners, vec![ - ("barfoo".into(), "https://example.org/barfoo".into()), - ("foobar".into(), "https://example.org/foobarv2".into()) + ( + "barfoo".into(), + "https://example.org/barfoo".into(), + "user".into() + ), + ( + "foobar".into(), + "https://example.org/foobarv2".into(), + "user".into() + ) ] ); @@ -1300,6 +1315,7 @@ mod tests { .add_owner(CrateOwner { login: "barfoo".into(), avatar: "https://example.org/barfoo".into(), + kind: "user".into(), }) .create()?; @@ -1309,7 +1325,11 @@ mod tests { }); assert_eq!( details.owners, - vec![("barfoo".into(), "https://example.org/barfoo".into())] + vec![( + "barfoo".into(), + "https://example.org/barfoo".into(), + "user".into() + )] ); // Changing owner details on another of their crates applies the change to both @@ -1319,6 +1339,7 @@ mod tests { .add_owner(CrateOwner { login: "barfoo".into(), avatar: "https://example.org/barfoov2".into(), + kind: "user".into(), }) .create()?; @@ -1328,7 +1349,11 @@ mod tests { }); assert_eq!( details.owners, - vec![("barfoo".into(), "https://example.org/barfoov2".into())] + vec![( + "barfoo".into(), + "https://example.org/barfoov2".into(), + "user".into() + )] ); Ok(()) diff --git a/src/web/releases.rs b/src/web/releases.rs index e7486813a..9b931c66d 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -1642,6 +1642,7 @@ mod tests { .add_owner(CrateOwner { login: "foobar".into(), avatar: "https://example.org/foobar".into(), + kind: "user".into(), }) .create()?; diff --git a/templates/rustdoc/topbar.html b/templates/rustdoc/topbar.html index a5b6e1bfd..26dc7138c 100644 --- a/templates/rustdoc/topbar.html +++ b/templates/rustdoc/topbar.html @@ -112,7 +112,7 @@ {%- for owner in krate.owners -%}
  • - + {{ "user" | fas }} {{ owner[0] }}
    • From 4addf4c311d4c5eb2f4acedd3f5fbcfc69f3fd60 Mon Sep 17 00:00:00 2001 From: Victor Song Date: Thu, 29 Feb 2024 03:58:00 -0600 Subject: [PATCH 2/3] fix(owners): add command to update all crate data --- src/bin/cratesfyi.rs | 35 +++++++++++++++++++++++++++++++++++ src/db/add_package.rs | 10 ++++++++++ src/db/mod.rs | 4 +++- 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 0d8b46ca6..42e66c4f7 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -25,6 +25,7 @@ use humantime::Duration; use once_cell::sync::OnceCell; use sentry::TransactionContext; use tokio::runtime::{Builder, Runtime}; +use tracing::info; use tracing_log::LogTracer; use tracing_subscriber::{filter::Directive, prelude::*, EnvFilter}; @@ -515,6 +516,9 @@ enum DatabaseSubcommand { /// Backfill GitHub/Gitlab stats for crates. BackfillRepositoryStats, + /// Backfill crate owner kind from crates.io API + BackfillCrateOwnerKind, + /// Updates info for a crate from the registry's API UpdateCrateRegistryFields { #[arg(name = "CRATE")] @@ -601,6 +605,37 @@ impl DatabaseSubcommand { .block_on(ctx.repository_stats_updater()?.backfill_repositories())?; } + Self::BackfillCrateOwnerKind => { + let pool = ctx.pool()?; + ctx.runtime()? + .block_on(async { + let mut list_crates_conn = pool.get_async().await?; + let mut update_crates_conn = pool.get_async().await?; + + let mut result_stream = + sqlx::query!("SELECT id, name FROM crates ORDER BY name") + .fetch(&mut *list_crates_conn); + + while let Some(row) = result_stream.next().await { + let row = row?; + let registry_data = + ctx.registry_api()?.get_crate_data(&row.name).await?; + + info!("Updating crate {}", row.name); + + db::update_crate_data_in_db_by_id( + &mut update_crates_conn, + row.id, + ®istry_data, + ) + .await?; + } + + Ok::<(), anyhow::Error>(()) + }) + .context("Failed to backfill crate owner kind")? + } + Self::UpdateCrateRegistryFields { name } => ctx.runtime()?.block_on(async move { let mut conn = ctx.pool()?.get_async().await?; let registry_data = ctx.registry_api()?.get_crate_data(&name).await?; diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 7302be584..7f18576f8 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -435,6 +435,16 @@ pub async fn update_crate_data_in_database( .fetch_one(&mut *conn) .await?; + update_crate_data_in_db_by_id(conn, crate_id, registry_data).await?; + + Ok(()) +} + +pub async fn update_crate_data_in_db_by_id( + conn: &mut sqlx::PgConnection, + crate_id: i32, + registry_data: &CrateData, +) -> Result<()> { update_owners_in_database(conn, ®istry_data.owners, crate_id).await?; Ok(()) diff --git a/src/db/mod.rs b/src/db/mod.rs index 0085ad564..9916a2e48 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -7,7 +7,9 @@ pub(crate) use self::add_package::{ add_build_into_database, add_doc_coverage, add_package_into_database, }; pub use self::{ - add_package::{update_build_status, update_crate_data_in_database}, + add_package::{ + update_build_status, update_crate_data_in_database, update_crate_data_in_db_by_id, + }, delete::{delete_crate, delete_version}, file::{add_path_into_database, add_path_into_remote_archive}, overrides::Overrides, From 097c989834313725607cb3161e68e0440f35f1ff Mon Sep 17 00:00:00 2001 From: Victor Song Date: Thu, 2 May 2024 01:12:20 -0500 Subject: [PATCH 3/3] Make `owner_kind` an enum, change migration --- ...387c8bfa4790b9e8d317159b37397e42b7ceb.json | 42 ++++++++++++++++++ ...12c14ed13f652419b1467c21e1829f31cd573.json | 42 ++++++++++++++++++ ...af3f5addfba6ed45f46afdc523b324da2cbab.json | 34 -------------- ...a84edd7d17e6171523ebbd57f1d9dd7c9b46c.json | 12 ++++- ...9dacc3289dcadf82061774866fec3a62669c5.json | 32 -------------- ...58ab7883bf28f216d5b5fd720c56fd8889eed.json | 44 +++++++++++++++++++ ...7e5e9c7016a1b8b35b031cb05b4d0802b6be1.json | 32 -------------- .../20240227040753_add_owner_kind.down.sql | 2 + .../20240227040753_add_owner_kind.up.sql | 16 +++++-- src/bin/cratesfyi.rs | 35 --------------- src/db/add_package.rs | 34 ++++++-------- src/db/delete.rs | 6 +-- src/db/mod.rs | 4 +- src/registry_api.rs | 18 ++++++-- src/web/crate_details.rs | 27 ++++++------ src/web/releases.rs | 4 +- 16 files changed, 201 insertions(+), 183 deletions(-) create mode 100644 .sqlx/query-16dfb0d87266568fb8a84585614387c8bfa4790b9e8d317159b37397e42b7ceb.json create mode 100644 .sqlx/query-1e48486ba272a4e2ea8793114e412c14ed13f652419b1467c21e1829f31cd573.json delete mode 100644 .sqlx/query-24c697ed817d33dbdfb0e5126feaf3f5addfba6ed45f46afdc523b324da2cbab.json delete mode 100644 .sqlx/query-b6273727360630c958b856427cb9dacc3289dcadf82061774866fec3a62669c5.json create mode 100644 .sqlx/query-d87220d3f4503e99fa17815db0058ab7883bf28f216d5b5fd720c56fd8889eed.json delete mode 100644 .sqlx/query-ee9dddd0c37c0e89eab1feaddd27e5e9c7016a1b8b35b031cb05b4d0802b6be1.json diff --git a/.sqlx/query-16dfb0d87266568fb8a84585614387c8bfa4790b9e8d317159b37397e42b7ceb.json b/.sqlx/query-16dfb0d87266568fb8a84585614387c8bfa4790b9e8d317159b37397e42b7ceb.json new file mode 100644 index 000000000..a3ae9ba9b --- /dev/null +++ b/.sqlx/query-16dfb0d87266568fb8a84585614387c8bfa4790b9e8d317159b37397e42b7ceb.json @@ -0,0 +1,42 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT login, avatar, kind as \"kind: OwnerKind\"\n FROM owners", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "login", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "avatar", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "kind: OwnerKind", + "type_info": { + "Custom": { + "name": "owner_kind", + "kind": { + "Enum": [ + "user", + "team" + ] + } + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "16dfb0d87266568fb8a84585614387c8bfa4790b9e8d317159b37397e42b7ceb" +} diff --git a/.sqlx/query-1e48486ba272a4e2ea8793114e412c14ed13f652419b1467c21e1829f31cd573.json b/.sqlx/query-1e48486ba272a4e2ea8793114e412c14ed13f652419b1467c21e1829f31cd573.json new file mode 100644 index 000000000..90d8b6731 --- /dev/null +++ b/.sqlx/query-1e48486ba272a4e2ea8793114e412c14ed13f652419b1467c21e1829f31cd573.json @@ -0,0 +1,42 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT login, avatar, kind as \"kind: OwnerKind\" FROM owners", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "login", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "avatar", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "kind: OwnerKind", + "type_info": { + "Custom": { + "name": "owner_kind", + "kind": { + "Enum": [ + "user", + "team" + ] + } + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "1e48486ba272a4e2ea8793114e412c14ed13f652419b1467c21e1829f31cd573" +} diff --git a/.sqlx/query-24c697ed817d33dbdfb0e5126feaf3f5addfba6ed45f46afdc523b324da2cbab.json b/.sqlx/query-24c697ed817d33dbdfb0e5126feaf3f5addfba6ed45f46afdc523b324da2cbab.json deleted file mode 100644 index 8ec492e59..000000000 --- a/.sqlx/query-24c697ed817d33dbdfb0e5126feaf3f5addfba6ed45f46afdc523b324da2cbab.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT login, avatar, kind\n FROM owners\n INNER JOIN owner_rels ON owner_rels.oid = owners.id\n WHERE cid = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "login", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "avatar", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "kind", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Int4" - ] - }, - "nullable": [ - false, - false, - false - ] - }, - "hash": "24c697ed817d33dbdfb0e5126feaf3f5addfba6ed45f46afdc523b324da2cbab" -} diff --git a/.sqlx/query-87952bd450ed2c13b99bd502a73a84edd7d17e6171523ebbd57f1d9dd7c9b46c.json b/.sqlx/query-87952bd450ed2c13b99bd502a73a84edd7d17e6171523ebbd57f1d9dd7c9b46c.json index 0b8f8417f..9b7cbc9f7 100644 --- a/.sqlx/query-87952bd450ed2c13b99bd502a73a84edd7d17e6171523ebbd57f1d9dd7c9b46c.json +++ b/.sqlx/query-87952bd450ed2c13b99bd502a73a84edd7d17e6171523ebbd57f1d9dd7c9b46c.json @@ -13,7 +13,17 @@ "Left": [ "Varchar", "Varchar", - "Text" + { + "Custom": { + "name": "owner_kind", + "kind": { + "Enum": [ + "user", + "team" + ] + } + } + } ] }, "nullable": [ diff --git a/.sqlx/query-b6273727360630c958b856427cb9dacc3289dcadf82061774866fec3a62669c5.json b/.sqlx/query-b6273727360630c958b856427cb9dacc3289dcadf82061774866fec3a62669c5.json deleted file mode 100644 index 9aea6ce4e..000000000 --- a/.sqlx/query-b6273727360630c958b856427cb9dacc3289dcadf82061774866fec3a62669c5.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT login, avatar, kind FROM owners", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "login", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "avatar", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "kind", - "type_info": "Text" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false - ] - }, - "hash": "b6273727360630c958b856427cb9dacc3289dcadf82061774866fec3a62669c5" -} diff --git a/.sqlx/query-d87220d3f4503e99fa17815db0058ab7883bf28f216d5b5fd720c56fd8889eed.json b/.sqlx/query-d87220d3f4503e99fa17815db0058ab7883bf28f216d5b5fd720c56fd8889eed.json new file mode 100644 index 000000000..7feaf3ddd --- /dev/null +++ b/.sqlx/query-d87220d3f4503e99fa17815db0058ab7883bf28f216d5b5fd720c56fd8889eed.json @@ -0,0 +1,44 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT login, avatar, kind as \"kind: OwnerKind\"\n FROM owners\n INNER JOIN owner_rels ON owner_rels.oid = owners.id\n WHERE cid = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "login", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "avatar", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "kind: OwnerKind", + "type_info": { + "Custom": { + "name": "owner_kind", + "kind": { + "Enum": [ + "user", + "team" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "d87220d3f4503e99fa17815db0058ab7883bf28f216d5b5fd720c56fd8889eed" +} diff --git a/.sqlx/query-ee9dddd0c37c0e89eab1feaddd27e5e9c7016a1b8b35b031cb05b4d0802b6be1.json b/.sqlx/query-ee9dddd0c37c0e89eab1feaddd27e5e9c7016a1b8b35b031cb05b4d0802b6be1.json deleted file mode 100644 index c8d901e06..000000000 --- a/.sqlx/query-ee9dddd0c37c0e89eab1feaddd27e5e9c7016a1b8b35b031cb05b4d0802b6be1.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT login, avatar, kind\n FROM owners", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "login", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "avatar", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "kind", - "type_info": "Text" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false - ] - }, - "hash": "ee9dddd0c37c0e89eab1feaddd27e5e9c7016a1b8b35b031cb05b4d0802b6be1" -} diff --git a/migrations/20240227040753_add_owner_kind.down.sql b/migrations/20240227040753_add_owner_kind.down.sql index f897ec835..859b40c10 100644 --- a/migrations/20240227040753_add_owner_kind.down.sql +++ b/migrations/20240227040753_add_owner_kind.down.sql @@ -1,2 +1,4 @@ ALTER TABLE owners DROP COLUMN IF EXISTS kind; + +DROP TYPE IF EXISTS owner_kind; diff --git a/migrations/20240227040753_add_owner_kind.up.sql b/migrations/20240227040753_add_owner_kind.up.sql index 5c182918d..64bbaef7e 100644 --- a/migrations/20240227040753_add_owner_kind.up.sql +++ b/migrations/20240227040753_add_owner_kind.up.sql @@ -1,4 +1,14 @@ +CREATE TYPE owner_kind AS ENUM ( + 'user', + 'team' +); + ALTER TABLE owners -ADD COLUMN IF NOT EXISTS kind TEXT NOT NULL CHECK ( - kind IN ('user', 'team') -) DEFAULT 'user'; +ADD COLUMN IF NOT EXISTS kind owner_kind NOT NULL DEFAULT 'user'; + +UPDATE owners +SET + kind = CASE + WHEN login LIKE 'github:%' THEN 'team'::owner_kind + ELSE 'user'::owner_kind + END; diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 42e66c4f7..0d8b46ca6 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -25,7 +25,6 @@ use humantime::Duration; use once_cell::sync::OnceCell; use sentry::TransactionContext; use tokio::runtime::{Builder, Runtime}; -use tracing::info; use tracing_log::LogTracer; use tracing_subscriber::{filter::Directive, prelude::*, EnvFilter}; @@ -516,9 +515,6 @@ enum DatabaseSubcommand { /// Backfill GitHub/Gitlab stats for crates. BackfillRepositoryStats, - /// Backfill crate owner kind from crates.io API - BackfillCrateOwnerKind, - /// Updates info for a crate from the registry's API UpdateCrateRegistryFields { #[arg(name = "CRATE")] @@ -605,37 +601,6 @@ impl DatabaseSubcommand { .block_on(ctx.repository_stats_updater()?.backfill_repositories())?; } - Self::BackfillCrateOwnerKind => { - let pool = ctx.pool()?; - ctx.runtime()? - .block_on(async { - let mut list_crates_conn = pool.get_async().await?; - let mut update_crates_conn = pool.get_async().await?; - - let mut result_stream = - sqlx::query!("SELECT id, name FROM crates ORDER BY name") - .fetch(&mut *list_crates_conn); - - while let Some(row) = result_stream.next().await { - let row = row?; - let registry_data = - ctx.registry_api()?.get_crate_data(&row.name).await?; - - info!("Updating crate {}", row.name); - - db::update_crate_data_in_db_by_id( - &mut update_crates_conn, - row.id, - ®istry_data, - ) - .await?; - } - - Ok::<(), anyhow::Error>(()) - }) - .context("Failed to backfill crate owner kind")? - } - Self::UpdateCrateRegistryFields { name } => ctx.runtime()?.block_on(async move { let mut conn = ctx.pool()?.get_async().await?; let registry_data = ctx.registry_api()?.get_crate_data(&name).await?; diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 7f18576f8..3bbcff263 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -435,16 +435,6 @@ pub async fn update_crate_data_in_database( .fetch_one(&mut *conn) .await?; - update_crate_data_in_db_by_id(conn, crate_id, registry_data).await?; - - Ok(()) -} - -pub async fn update_crate_data_in_db_by_id( - conn: &mut sqlx::PgConnection, - crate_id: i32, - registry_data: &CrateData, -) -> Result<()> { update_owners_in_database(conn, ®istry_data.owners, crate_id).await?; Ok(()) @@ -473,7 +463,7 @@ async fn update_owners_in_database( RETURNING id", owner.login, owner.avatar, - owner.kind, + owner.kind as _, ) .fetch_one(&mut *conn) .await?, @@ -532,6 +522,7 @@ where #[cfg(test)] mod test { use super::*; + use crate::registry_api::OwnerKind; use crate::test::*; use crate::utils::CargoMetadata; use test_case::test_case; @@ -659,14 +650,14 @@ mod test { let owner1 = CrateOwner { avatar: "avatar".into(), login: "login".into(), - kind: "user".into(), + kind: OwnerKind::User, }; update_owners_in_database(&mut conn, &[owner1.clone()], crate_id).await?; let owner_def = sqlx::query!( - "SELECT login, avatar, kind - FROM owners" + r#"SELECT login, avatar, kind as "kind: OwnerKind" + FROM owners"# ) .fetch_one(&mut *conn) .await?; @@ -702,7 +693,7 @@ mod test { &[CrateOwner { login: "login".into(), avatar: "avatar".into(), - kind: "user".into(), + kind: OwnerKind::User, }], crate_id, ) @@ -711,13 +702,14 @@ mod test { let updated_owner = CrateOwner { login: "login".into(), avatar: "avatar2".into(), - kind: "team".into(), + kind: OwnerKind::Team, }; update_owners_in_database(&mut conn, &[updated_owner.clone()], crate_id).await?; - let owner_def = sqlx::query!("SELECT login, avatar, kind FROM owners") - .fetch_one(&mut *conn) - .await?; + let owner_def = + sqlx::query!(r#"SELECT login, avatar, kind as "kind: OwnerKind" FROM owners"#) + .fetch_one(&mut *conn) + .await?; assert_eq!(owner_def.login, updated_owner.login); assert_eq!(owner_def.avatar, updated_owner.avatar); assert_eq!(owner_def.kind, updated_owner.kind); @@ -750,7 +742,7 @@ mod test { &[CrateOwner { login: "login".into(), avatar: "avatar".into(), - kind: "user".into(), + kind: OwnerKind::User, }], crate_id, ) @@ -760,7 +752,7 @@ mod test { .map(|i| CrateOwner { login: format!("login{i}"), avatar: format!("avatar{i}"), - kind: "user".into(), + kind: OwnerKind::User, }) .collect(); diff --git a/src/db/delete.rs b/src/db/delete.rs index 49de88b50..8f96c4bd4 100644 --- a/src/db/delete.rs +++ b/src/db/delete.rs @@ -196,7 +196,7 @@ fn delete_crate_from_database(conn: &mut Client, name: &str, crate_id: i32) -> R #[cfg(test)] mod tests { use super::*; - use crate::registry_api::CrateOwner; + use crate::registry_api::{CrateOwner, OwnerKind}; use crate::test::{assert_success, wrapper}; use test_case::test_case; @@ -328,7 +328,7 @@ mod tests { .add_owner(CrateOwner { login: "malicious actor".into(), avatar: "https://example.org/malicious".into(), - kind: "user".into(), + kind: OwnerKind::User, }) .create()?; assert!(release_exists(&mut db.conn(), v1)?); @@ -359,7 +359,7 @@ mod tests { .add_owner(CrateOwner { login: "Peter Rabbit".into(), avatar: "https://example.org/peter".into(), - kind: "user".into(), + kind: OwnerKind::User, }) .create()?; assert!(release_exists(&mut db.conn(), v2)?); diff --git a/src/db/mod.rs b/src/db/mod.rs index 9916a2e48..0085ad564 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -7,9 +7,7 @@ pub(crate) use self::add_package::{ add_build_into_database, add_doc_coverage, add_package_into_database, }; pub use self::{ - add_package::{ - update_build_status, update_crate_data_in_database, update_crate_data_in_db_by_id, - }, + add_package::{update_build_status, update_crate_data_in_database}, delete::{delete_crate, delete_version}, file::{add_path_into_database, add_path_into_remote_archive}, overrides::Overrides, diff --git a/src/registry_api.rs b/src/registry_api.rs index f067bd847..47d403179 100644 --- a/src/registry_api.rs +++ b/src/registry_api.rs @@ -3,7 +3,7 @@ use anyhow::{anyhow, Context}; use chrono::{DateTime, Utc}; use reqwest::header::{HeaderValue, ACCEPT, USER_AGENT}; use semver::Version; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tracing::instrument; use url::Url; @@ -46,7 +46,17 @@ impl Default for ReleaseData { pub struct CrateOwner { pub(crate) avatar: String, pub(crate) login: String, - pub(crate) kind: String, + pub(crate) kind: OwnerKind, +} + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, sqlx::Type, +)] +#[sqlx(type_name = "owner_kind", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum OwnerKind { + User, + Team, } impl RegistryApi { @@ -170,7 +180,7 @@ impl RegistryApi { #[serde(default)] login: Option, #[serde(default)] - kind: Option, + kind: Option, } let response: Response = retry_async( @@ -201,7 +211,7 @@ impl RegistryApi { .map(|data| CrateOwner { avatar: data.avatar.unwrap_or_default(), login: data.login.unwrap_or_default(), - kind: data.kind.unwrap_or("user".into()), + kind: data.kind.unwrap_or(OwnerKind::User), }) .collect(); diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index 119470bcb..fb562c60d 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -1,4 +1,5 @@ use super::{markdown, match_version, MetaData}; +use crate::registry_api::OwnerKind; use crate::utils::{get_correct_docsrs_style_file, report_error}; use crate::web::rustdoc::RustdocHtmlParams; use crate::{ @@ -35,7 +36,7 @@ pub(crate) struct CrateDetails { name: String, pub version: Version, description: Option, - owners: Vec<(String, String, String)>, + owners: Vec<(String, String, OwnerKind)>, dependencies: Option, #[serde(serialize_with = "optional_markdown")] readme: Option, @@ -264,10 +265,10 @@ impl CrateDetails { // get owners crate_details.owners = sqlx::query!( - "SELECT login, avatar, kind + r#"SELECT login, avatar, kind as "kind: OwnerKind" FROM owners INNER JOIN owner_rels ON owner_rels.oid = owners.id - WHERE cid = $1", + WHERE cid = $1"#, krate.crate_id, ) .fetch(&mut *conn) @@ -1253,7 +1254,7 @@ mod tests { .add_owner(CrateOwner { login: "foobar".into(), avatar: "https://example.org/foobar".into(), - kind: "user".into(), + kind: OwnerKind::User, }) .create()?; @@ -1266,7 +1267,7 @@ mod tests { vec![( "foobar".into(), "https://example.org/foobar".into(), - "user".into() + OwnerKind::User )] ); @@ -1277,12 +1278,12 @@ mod tests { .add_owner(CrateOwner { login: "foobar".into(), avatar: "https://example.org/foobarv2".into(), - kind: "user".into(), + kind: OwnerKind::User, }) .add_owner(CrateOwner { login: "barfoo".into(), avatar: "https://example.org/barfoo".into(), - kind: "user".into(), + kind: OwnerKind::User, }) .create()?; @@ -1298,12 +1299,12 @@ mod tests { ( "barfoo".into(), "https://example.org/barfoo".into(), - "user".into() + OwnerKind::User ), ( "foobar".into(), "https://example.org/foobarv2".into(), - "user".into() + OwnerKind::User ) ] ); @@ -1315,7 +1316,7 @@ mod tests { .add_owner(CrateOwner { login: "barfoo".into(), avatar: "https://example.org/barfoo".into(), - kind: "user".into(), + kind: OwnerKind::User, }) .create()?; @@ -1328,7 +1329,7 @@ mod tests { vec![( "barfoo".into(), "https://example.org/barfoo".into(), - "user".into() + OwnerKind::User )] ); @@ -1339,7 +1340,7 @@ mod tests { .add_owner(CrateOwner { login: "barfoo".into(), avatar: "https://example.org/barfoov2".into(), - kind: "user".into(), + kind: OwnerKind::User, }) .create()?; @@ -1352,7 +1353,7 @@ mod tests { vec![( "barfoo".into(), "https://example.org/barfoov2".into(), - "user".into() + OwnerKind::User )] ); diff --git a/src/web/releases.rs b/src/web/releases.rs index 9b931c66d..f6fe8ea00 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -778,7 +778,7 @@ pub(crate) async fn build_queue_handler( #[cfg(test)] mod tests { use super::*; - use crate::registry_api::CrateOwner; + use crate::registry_api::{CrateOwner, OwnerKind}; use crate::test::{ assert_cache_control, assert_redirect, assert_redirect_unchecked, assert_success, wrapper, TestFrontend, @@ -1642,7 +1642,7 @@ mod tests { .add_owner(CrateOwner { login: "foobar".into(), avatar: "https://example.org/foobar".into(), - kind: "user".into(), + kind: OwnerKind::User, }) .create()?;