From 0eab7aad905417561860f314e237e28fae00f378 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 12 Dec 2024 14:04:14 +0000 Subject: [PATCH 01/27] Make possible set searching scope to both orgs and repos and pull repositories in both scopes, adjust tests. --- DESCRIPTION | 2 +- R/GitHost.R | 117 ++++++++++-------- R/GitHostGitHub.R | 2 +- R/GitHostGitLab.R | 2 +- tests/testthat/_snaps/01-get_repos-GitHub.md | 7 -- tests/testthat/_snaps/helpers.md | 10 -- tests/testthat/_snaps/set_host.md | 21 +--- tests/testthat/helper-fixtures.R | 88 ++++++------- tests/testthat/test-01-get_repos-GitHub.R | 56 +++++++-- .../test-get_files_structure-GitHub.R | 10 +- .../test-get_files_structure-GitStats.R | 2 +- tests/testthat/test-get_release-GitHub.R | 17 ++- tests/testthat/test-helpers.R | 10 -- tests/testthat/test-set_host.R | 17 --- 14 files changed, 184 insertions(+), 177 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index cdfba9aa..b6e5fb16 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Standardized Git Repository Data -Version: 2.1.2.9003 +Version: 2.1.2.9004 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), diff --git a/R/GitHost.R b/R/GitHost.R index 0fb6981c..ec600e58 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -367,9 +367,10 @@ GitHost <- R6::R6Class( if (is.null(repos) && is.null(orgs)) { if (private$is_public) { cli::cli_abort(c( - "You need to specify `orgs` for public Git Host.", + "You need to specify `orgs` or/and `repos` for public Git Host.", "x" = "Host will not be added.", - "i" = "Add organizations to your `orgs` parameter." + "i" = "Add organizations to your `orgs` and/or repositories to + `repos` parameter." ), call = NULL) } else { @@ -385,25 +386,11 @@ GitHost <- R6::R6Class( private$scan_all <- TRUE } } - if (!is.null(repos) && is.null(orgs)) { - if (verbose) { - cli::cli_alert_info(cli::col_grey("Searching scope set to [repo].")) - } - private$searching_scope <- "repo" - } - if (is.null(repos) && !is.null(orgs)) { - if (verbose) { - cli::cli_alert_info(cli::col_grey("Searching scope set to [org].")) - } - private$searching_scope <- "org" + if (!is.null(repos)) { + private$searching_scope <- c(private$searching_scope, "repo") } - if (!is.null(repos) && !is.null(orgs)) { - cli::cli_abort(c( - "Do not specify `orgs` while specifing `repos`.", - "x" = "Host will not be added.", - "i" = "Specify `orgs` or `repos`." - ), - call = NULL) + if (!is.null(orgs)) { + private$searching_scope <- c(private$searching_scope, "org") } }, @@ -422,8 +409,8 @@ GitHost <- R6::R6Class( verbose = verbose ) private$repos_fullnames <- repos - orgs_repos <- private$extract_repos_and_orgs(repos) - private$orgs <- private$set_owner_type( + orgs_repos <- private$extract_repos_and_orgs(private$repos_fullnames) + orgs <- private$set_owner_type( owners = names(orgs_repos) ) private$repos <- unname(unlist(orgs_repos)) @@ -589,7 +576,7 @@ GitHost <- R6::R6Class( # Set repositories set_repos = function(org) { - if (private$searching_scope == "repo") { + if ("repo" %in% private$searching_scope) { repos <- private$orgs_repos[[org]] } else { repos <- NULL @@ -605,7 +592,6 @@ GitHost <- R6::R6Class( ) }, - #' Retrieve all repositories for an organization in a table format. get_all_repos = function(verbose = TRUE, progress = TRUE) { if (private$scan_all && is.null(private$orgs)) { if (verbose) { @@ -617,34 +603,67 @@ GitHost <- R6::R6Class( } private$orgs <- private$engines$graphql$get_orgs() } - graphql_engine <- private$engines$graphql - repos_table <- purrr::map(private$orgs, function(org) { - type <- attr(org, "type") %||% "organization" - org <- utils::URLdecode(org) - if (!private$scan_all && verbose) { - show_message( - host = private$host_name, - engine = "graphql", - scope = org, - information = "Pulling repositories" - ) - } - repos <- private$set_repos(org) - repos_table <- graphql_engine$get_repos_from_org( - org = org, - type = type - ) %>% - graphql_engine$prepare_repos_table() - if (!is.null(repos)) { - repos_table <- repos_table %>% - dplyr::filter(repo_name %in% repos) - } - return(repos_table) - }, .progress = progress) %>% - purrr::list_rbind() + repos_table <- purrr::list_rbind( + list( + private$get_repos_from_orgs(verbose, progress), + private$get_repos_individual(verbose, progress) + ) + ) return(repos_table) }, + get_repos_from_orgs = function(verbose, progress) { + if ("org" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + purrr::map(private$orgs, function(org) { + type <- attr(org, "type") %||% "organization" + org <- utils::URLdecode(org) + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = "Pulling repositories" + ) + } + repos_table <- graphql_engine$get_repos_from_org( + org = org, + type = type + ) |> + graphql_engine$prepare_repos_table() + return(repos_table) + }, .progress = progress) |> + purrr::list_rbind() + } + }, + + get_repos_individual = function(verbose, progress) { + if ("repo" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + orgs <- names(private$orgs_repos) + purrr::map(orgs, function(org) { + type <- attr(org, "type") %||% "organization" + org <- utils::URLdecode(org) + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = "Pulling repositories" + ) + } + repos_table <- graphql_engine$get_repos_from_org( + org = org, + type = type + ) |> + graphql_engine$prepare_repos_table() |> + dplyr::filter(repo_name == private$orgs_repos[[org]]) + return(repos_table) + }, .progress = progress) |> + purrr::list_rbind() + } + }, + # Pull repositories with specific code get_repos_with_code = function(code, in_files = NULL, diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index 4b0089c3..ab38aa91 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -203,7 +203,7 @@ GitHostGitHub <- R6::R6Class( # Use repositories either from parameter or, if not set, pull them from API set_repositories = function(org) { - if (private$searching_scope == "repo") { + if ("repo" %in% private$searching_scope) { repos_names <- private$orgs_repos[[org]] } else { repos_table <- private$get_all_repos( diff --git a/R/GitHostGitLab.R b/R/GitHostGitLab.R index 61e40006..d8d129e3 100644 --- a/R/GitHostGitLab.R +++ b/R/GitHostGitLab.R @@ -266,7 +266,7 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", # Use repositories either from parameter or, if not set, pull them from API set_repositories = function(org, settings) { - if (private$searching_scope == "repo") { + if ("repo" %in% private$searching_scope) { repos <- private$orgs_repos[[org]] repos_names <- paste0(utils::URLencode(org, reserved = TRUE), "%2f", repos) } else { diff --git a/tests/testthat/_snaps/01-get_repos-GitHub.md b/tests/testthat/_snaps/01-get_repos-GitHub.md index e51ab31c..2edba7cf 100644 --- a/tests/testthat/_snaps/01-get_repos-GitHub.md +++ b/tests/testthat/_snaps/01-get_repos-GitHub.md @@ -5,10 +5,3 @@ Output [1] "\n query GetReposByOrg($login: String!) {\n repositoryOwner(login: $login) {\n ... on Organization {\n \n repositories(first: 100) {\n totalCount\n pageInfo {\n endCursor\n hasNextPage\n }\n nodes {\n repo_id: id\n repo_name: name\n default_branch: defaultBranchRef {\n name\n }\n stars: stargazerCount\n forks: forkCount\n created_at: createdAt\n last_activity_at: pushedAt\n languages (first: 5) { nodes {name} }\n issues_open: issues (first: 100 states: [OPEN]) {\n totalCount\n }\n issues_closed: issues (first: 100 states: [CLOSED]) {\n totalCount\n }\n organization: owner {\n login\n }\n repo_url: url\n }\n }\n \n }\n }\n }" -# `get_all_repos()` prints proper message - - Code - gh_repos_table <- github_testhost_priv$get_all_repos(verbose = TRUE) - Message - i [Host:GitHub][Engine:GraphQl][Scope:test-org] Pulling repositories... - diff --git a/tests/testthat/_snaps/helpers.md b/tests/testthat/_snaps/helpers.md index aec3df67..d1167c04 100644 --- a/tests/testthat/_snaps/helpers.md +++ b/tests/testthat/_snaps/helpers.md @@ -3,16 +3,12 @@ Code gitlab_testhost_priv$set_searching_scope(orgs = "mbtests", repos = NULL, verbose = TRUE) - Message - i Searching scope set to [org]. --- Code gitlab_testhost_priv$set_searching_scope(orgs = NULL, repos = "mbtests/GitStatsTesting", verbose = TRUE) - Message - i Searching scope set to [repo]. # When token is empty throw error @@ -50,9 +46,3 @@ Message i Using PAT from GITLAB_PAT envar. -# `set_searching_scope` throws error when both `orgs` and `repos` are defined - - Do not specify `orgs` while specifing `repos`. - x Host will not be added. - i Specify `orgs` or `repos`. - diff --git a/tests/testthat/_snaps/set_host.md b/tests/testthat/_snaps/set_host.md index 19211cfd..0cbab877 100644 --- a/tests/testthat/_snaps/set_host.md +++ b/tests/testthat/_snaps/set_host.md @@ -4,7 +4,6 @@ set_github_host(gitstats = test_gitstats, token = Sys.getenv("GITHUB_PAT"), orgs = c("openpharma", "r-world-devs")) Message - i Searching scope set to [org]. i Checking organizations... v Set connection to GitHub. @@ -14,7 +13,6 @@ test_gitstats %>% set_gitlab_host(token = Sys.getenv("GITLAB_PAT_PUBLIC"), orgs = c("mbtests")) Message - i Searching scope set to [org]. i Checking organizations... v Set connection to GitLab. @@ -25,7 +23,6 @@ "r-world-devs")) Message i Using PAT from GITHUB_PAT envar. - i Searching scope set to [org]. i Checking organizations... v Set connection to GitHub. @@ -37,7 +34,6 @@ }) Message i Using PAT from GITLAB_PAT envar. - i Searching scope set to [org]. i Checking organizations... v Set connection to GitLab. @@ -48,7 +44,6 @@ "r-world-devs/GitStats", "r-world-devs/shinyCohortBuilder", "openpharma/GithubMetrics", "openpharma/DataFakeR")) Message - i Searching scope set to [repo]. i Checking repositories... v Set connection to GitHub. @@ -58,21 +53,14 @@ test_gitstats %>% set_gitlab_host(token = Sys.getenv("GITLAB_PAT_PUBLIC"), repos = c("mbtests/gitstatstesting", "mbtests/gitstats-testing-2")) Message - i Searching scope set to [repo]. i Checking repositories... v Set connection to GitLab. -# Set host prints error when repos and orgs are defined and host is not passed to GitStats - - Do not specify `orgs` while specifing `repos`. - x Host will not be added. - i Specify `orgs` or `repos`. - # Error shows if organizations are not specified and host is not passed - You need to specify `orgs` for public Git Host. + You need to specify `orgs` or/and `repos` for public Git Host. x Host will not be added. - i Add organizations to your `orgs` parameter. + i Add organizations to your `orgs` and/or repositories to `repos` parameter. # Error shows, when wrong input is passed when setting connection and host is not passed @@ -86,10 +74,8 @@ test_gitstats %>% set_github_host(token = Sys.getenv("GITHUB_PAT"), orgs = "pharmaverse") %>% set_github_host(token = Sys.getenv("GITHUB_PAT"), orgs = "openpharma") Message - i Searching scope set to [org]. i Checking organizations... v Set connection to GitHub. - i Searching scope set to [org]. i Checking organizations... v Set connection to GitHub. Condition @@ -102,7 +88,6 @@ test_gitstats <- create_gitstats() %>% set_github_host(token = Sys.getenv( "GITHUB_PAT"), orgs = c("openparma")) Message - i Searching scope set to [org]. i Checking organizations... Condition Error in `purrr::map()`: @@ -118,7 +103,6 @@ test_gitstats <- create_gitstats() %>% set_gitlab_host(token = Sys.getenv( "GITLAB_PAT_PUBLIC"), orgs = c("openparma", "mbtests")) Message - i Searching scope set to [org]. i Checking organizations... Condition Error in `purrr::map()`: @@ -134,7 +118,6 @@ test_gitstats <- create_gitstats() %>% set_github_host(token = Sys.getenv( "GITHUB_PAT"), orgs = c("openpharma", "r_world_devs")) Message - i Searching scope set to [org]. i Checking organizations... Condition Error in `purrr::map()`: diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index 873d30e2..e1773d7f 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -96,40 +96,42 @@ test_fixtures$gitlab_repositories_rest_response <- list( ) ) -github_repository_node <- list( - "repo_id" = "xyz", - "repo_name" = "TestRepo", - "default_branch" = list( - "name" = "main" - ), - "stars" = 10, - "forks" = 2, - "created_at" = "2022-04-20T00:00:00Z", - "last_activity_at" = "2023-04-20T00:00:00Z", - "languages" = list( - "nodes" = list( - list( - "name" = "R" - ), - list( - "name" = "CSS" - ), - list( - "name" = "JavaScript" +github_repository_node <- function(repo_name) { + list( + "repo_id" = "xyz", + "repo_name" = repo_name, + "default_branch" = list( + "name" = "main" + ), + "stars" = 10, + "forks" = 2, + "created_at" = "2022-04-20T00:00:00Z", + "last_activity_at" = "2023-04-20T00:00:00Z", + "languages" = list( + "nodes" = list( + list( + "name" = "R" + ), + list( + "name" = "CSS" + ), + list( + "name" = "JavaScript" + ) ) - ) - ), - "issues_open" = list( - "totalCount" = 10 - ), - "issues_closed" = list( - "totalCount" = 5 - ), - "organization" = list( - "login" = "test_org" - ), - "repo_url" = "https://test_url" -) + ), + "issues_open" = list( + "totalCount" = 10 + ), + "issues_closed" = list( + "totalCount" = 5 + ), + "organization" = list( + "login" = "test_org" + ), + "repo_url" = "https://test_url" + ) +} test_fixtures$github_repos_by_org_response <- list( "data" = list( @@ -141,11 +143,11 @@ test_fixtures$github_repos_by_org_response <- list( "hasNextPage" = FALSE ), "nodes" = list( - github_repository_node, - github_repository_node, - github_repository_node, - github_repository_node, - github_repository_node + github_repository_node("TestRepo"), + github_repository_node("TestRepo1"), + github_repository_node("TestRepo2"), + github_repository_node("TestRepo3"), + github_repository_node("TestRepo4") ) ) ) @@ -162,11 +164,11 @@ test_fixtures$github_repos_by_user_response <- list( "hasNextPage" = FALSE ), "nodes" = list( - github_repository_node, - github_repository_node, - github_repository_node, - github_repository_node, - github_repository_node + github_repository_node("TestRepo"), + github_repository_node("TestRepo1"), + github_repository_node("TestRepo2"), + github_repository_node("TestRepo3"), + github_repository_node("TestRepo4") ) ) ) diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index 6fca8d8c..43c50f81 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -391,32 +391,64 @@ test_that("GitHub prepares repos table from repositories response", { test_mocker$cache(gh_repos_table) }) -test_that("`get_all_repos()` works as expected", { +test_that("get_repos_from_org works", { mockery::stub( - github_testhost_priv$get_all_repos, + github_testhost_priv$get_repos_from_orgs, "graphql_engine$prepare_repos_table", test_mocker$use("gh_repos_table") ) - gh_repos_table <- github_testhost_priv$get_all_repos( - verbose = FALSE + gh_repos_from_orgs <- github_testhost_priv$get_repos_from_orgs( + verbose = FALSE, + progress = FALSE ) expect_repos_table( - gh_repos_table + gh_repos_from_orgs ) - test_mocker$cache(gh_repos_table) + test_mocker$cache(gh_repos_from_orgs) }) -test_that("`get_all_repos()` prints proper message", { +test_that("get_repos_individual works", { mockery::stub( - github_testhost_priv$get_all_repos, + github_testhost_priv$get_repos_individual, "graphql_engine$prepare_repos_table", test_mocker$use("gh_repos_table") ) - expect_snapshot( - gh_repos_table <- github_testhost_priv$get_all_repos( - verbose = TRUE - ) + github_testhost_priv$searching_scope <- c("org", "repo") + github_testhost_priv$repos_fullnames <- c("test_org/TestRepo") + github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + test_org <- "test_org" + attr(test_org, "type") <- "organization" + github_testhost_priv$orgs <- test_org + + gh_repos_individual <- github_testhost_priv$get_repos_individual( + verbose = FALSE, + progress = FALSE + ) + expect_repos_table( + gh_repos_individual ) + test_mocker$cache(gh_repos_individual) +}) + +test_that("`get_all_repos()` works as expected", { + mockery::stub( + github_testhost_priv$get_all_repos, + "private$get_repos_from_orgs", + test_mocker$use("gh_repos_from_orgs") + ) + mockery::stub( + github_testhost_priv$get_all_repos, + "private$get_repos_individual", + test_mocker$use("gh_repos_individual") + ) + gh_repos_table <- github_testhost_priv$get_all_repos( + verbose = FALSE, + progress = FALSE + ) + expect_repos_table( + gh_repos_table + ) + test_mocker$cache(gh_repos_table) }) test_that("GitHost adds `repo_api_url` column to GitHub repos table", { diff --git a/tests/testthat/test-get_files_structure-GitHub.R b/tests/testthat/test-get_files_structure-GitHub.R index 0c63da20..3c632d55 100644 --- a/tests/testthat/test-get_files_structure-GitHub.R +++ b/tests/testthat/test-get_files_structure-GitHub.R @@ -127,12 +127,12 @@ test_that("GitHub GraphQL Engine pulls files structure from repositories", { ) gh_files_structure <- test_graphql_github$get_files_structure_from_org( org = "test_org", - repos = rep("TestRepo", 5) + repos = c("TestRepo", "TestRepo1", "TestRepo2", "TestRepo3", "TestRepo4") ) purrr::walk(gh_files_structure, ~ expect_true(length(.) > 0)) expect_equal( names(gh_files_structure), - rep("TestRepo", 5) + c("TestRepo", "TestRepo1", "TestRepo2", "TestRepo3", "TestRepo4") ) test_mocker$cache(gh_files_structure) }) @@ -174,7 +174,7 @@ test_that("get_files_structure_from_orgs", { ) expect_equal( names(gh_files_structure_from_orgs), - "test-org" + "test_org" ) test_mocker$cache(gh_files_structure_from_orgs) }) @@ -219,7 +219,7 @@ test_that("get_path_from_files_structure gets file path from files structure", { file_path <- test_graphql_github$get_path_from_files_structure( host_files_structure = test_mocker$use("gh_files_structure_from_orgs"), only_text_files = FALSE, - org = "test-org", + org = "test_org", repo = "TestRepo" ) expect_equal(typeof(file_path), "character") @@ -239,7 +239,7 @@ test_that("get_files_structure pulls files structure for repositories in orgs", ) expect_equal( names(gh_files_structure_from_orgs), - "test-org" + "test_org" ) purrr::walk(gh_files_structure_from_orgs[[1]], function(repo_files) { expect_true(any(grepl("\\.md|\\.Rmd", repo_files))) diff --git a/tests/testthat/test-get_files_structure-GitStats.R b/tests/testthat/test-get_files_structure-GitStats.R index 280f5873..50d6c46c 100644 --- a/tests/testthat/test-get_files_structure-GitStats.R +++ b/tests/testthat/test-get_files_structure-GitStats.R @@ -11,7 +11,7 @@ test_that("get_files_structure_from_hosts works as expected", { ) expect_equal(names(files_structure_from_hosts), c("github.com", "gitlab.com")) - expect_equal(names(files_structure_from_hosts[[1]]), c("test-org")) + expect_equal(names(files_structure_from_hosts[[1]]), c("test_org")) files_structure_from_hosts[[2]] <- test_mocker$use("gl_files_structure_from_orgs") test_mocker$cache(files_structure_from_hosts) }) diff --git a/tests/testthat/test-get_release-GitHub.R b/tests/testthat/test-get_release-GitHub.R index 60a0e56b..88fcb586 100644 --- a/tests/testthat/test-get_release-GitHub.R +++ b/tests/testthat/test-get_release-GitHub.R @@ -33,12 +33,27 @@ test_that("`prepare_releases_table()` prepares releases table", { test_mocker$cache(releases_table) }) -test_that("`set_repositories` works", { +test_that("`set_repositories` works when searching scope set to repo", { mockery::stub( github_testhost_priv$set_repositories, "private$get_all_repos", test_mocker$use("gh_repos_table") ) + github_testhost_priv$searching_scope <- "repo" + github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + repos_names <- github_testhost_priv$set_repositories(org = "test_org") + expect_type(repos_names, "character") + expect_gt(length(repos_names), 0) +}) + +test_that("`set_repositories` works for whole orgs", { + mockery::stub( + github_testhost_priv$set_repositories, + "private$get_all_repos", + test_mocker$use("gh_repos_table") + ) + github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + github_testhost_priv$searching_scope <- "org" repos_names <- github_testhost_priv$set_repositories() expect_type(repos_names, "character") expect_gt(length(repos_names), 0) diff --git a/tests/testthat/test-helpers.R b/tests/testthat/test-helpers.R index 09bb8488..d95052ac 100644 --- a/tests/testthat/test-helpers.R +++ b/tests/testthat/test-helpers.R @@ -192,13 +192,3 @@ test_that("`set_default_token` sets default token for GitLab", { 200 ) }) - -test_that("`set_searching_scope` throws error when both `orgs` and `repos` are defined", { - expect_snapshot_error( - gitlab_testhost_priv$set_searching_scope( - orgs = "mbtests", - repos = "mbtests/GitStatsTesting", - verbose = TRUE - ) - ) -}) diff --git a/tests/testthat/test-set_host.R b/tests/testthat/test-set_host.R index 956d28c5..c26f8308 100644 --- a/tests/testthat/test-set_host.R +++ b/tests/testthat/test-set_host.R @@ -71,23 +71,6 @@ test_that("Set GitLab host with particular repos vector instead of orgs", { ) }) -test_that("Set host prints error when repos and orgs are defined and host is not passed to GitStats", { - skip_on_cran() - test_gitstats <- create_gitstats() - expect_snapshot_error( - test_gitstats %>% - set_github_host( - token = Sys.getenv("GITHUB_PAT"), - orgs = c('r-world-devs', "openpharma"), - repos = c("r-world-devs/GitStats", "r-world-devs/shinyCohortBuilder", "openpharma/GithubMetrics", "openpharma/DataFakeR") - ) - ) - expect_length( - test_gitstats$.__enclos_env__$private$hosts, - 0 - ) -}) - test_that("Error shows if organizations are not specified and host is not passed", { skip_on_cran() test_gitstats <- create_gitstats() From 4b3c5a59465029edd5b050d91281619f0721dc53 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 12 Dec 2024 14:06:49 +0000 Subject: [PATCH 02/27] Fix/adjust workflows. --- inst/example_workflow.R | 77 +++++++++++++++++++++++++++++++++++++ inst/get_files_workflow.R | 8 ++-- inst/get_storage_workflow.R | 6 +-- 3 files changed, 84 insertions(+), 7 deletions(-) create mode 100644 inst/example_workflow.R diff --git a/inst/example_workflow.R b/inst/example_workflow.R new file mode 100644 index 00000000..2b17f83f --- /dev/null +++ b/inst/example_workflow.R @@ -0,0 +1,77 @@ +git_stats <- create_gitstats() |> + set_github_host( + orgs = "r-world-devs", + repos = "openpharma/DataFakeR" + ) + +get_repos(git_stats) + +get_repos_urls(git_stats, + with_files = "project_metadata.yaml", + progress = FALSE) + +get_repos_urls(git_stats, + with_files = "project_metadata.yaml", + cache = FALSE, + verbose = FALSE, + progress = TRUE) + +get_repos_urls(git_stats, + with_code = "Shiny", + in_files = "DESCRIPTION", + cache = FALSE, + verbose = FALSE) + +get_repos(git_stats) + +get_repos(git_stats, + cache = FALSE, + verbose = FALSE, + progress = TRUE) + +get_repos(git_stats, + cache = FALSE, + verbose = FALSE) + +get_repos(git_stats, + with_code = "Shiny") + +get_repos(git_stats, + with_code = "Shiny", + cache = FALSE, + verbose = FALSE) + +get_repos(git_stats, + with_code = "Shiny", + cache = FALSE, + verbose = FALSE, + progress = TRUE) + +get_repos(git_stats, + with_code = "Shiny", + in_files = "DESCRIPTION", + cache = FALSE) + +get_repos(git_stats, + with_code = c("shiny", "purrr"), + in_files = c("DESCRIPTION", "NAMESPACE"), + verbose = FALSE) + +get_commits(git_stats, since = "2024-06-01") + +get_commits(git_stats, + since = "2024-06-02", + verbose = FALSE, + progress = TRUE) + +get_release_logs( + gitstats = git_stats, + since = "2024-06-02", + verbose = FALSE +) + +get_release_logs( + gitstats = git_stats, + sinces = "2024-06-01", + verbose = TRUE +) diff --git a/inst/get_files_workflow.R b/inst/get_files_workflow.R index 960e7427..8b7d2eff 100644 --- a/inst/get_files_workflow.R +++ b/inst/get_files_workflow.R @@ -9,12 +9,12 @@ test_gitstats <- create_gitstats() |> ) get_files_content( - gitstats_obj = test_gitstats, + gitstats = test_gitstats, file_path = c("LICENSE", "DESCRIPTION") ) md_files_structure <- get_files_structure( - gitstats_obj = test_gitstats, + gitstats = test_gitstats, pattern = "\\.md|.R", depth = 2L ) @@ -22,8 +22,8 @@ md_files_structure <- get_files_structure( get_files_content(test_gitstats) md_files_structure <- get_files_structure( - gitstats_obj = test_gitstats, - pattern = "\\.md|\\.qmd|\\.Rmd", + gitstats = test_gitstats, + pattern = "DESCRIPTION|\\.md|\\.qmd|\\.Rmd", depth = 2L, verbose = FALSE ) diff --git a/inst/get_storage_workflow.R b/inst/get_storage_workflow.R index 10bc2221..c8ebb4ce 100644 --- a/inst/get_storage_workflow.R +++ b/inst/get_storage_workflow.R @@ -7,17 +7,17 @@ git_stats <- create_gitstats() %>% ) release_logs <- get_release_logs( - gitstats_object = git_stats, + gitstats = git_stats, since = "2024-01-01" ) repos_urls <- get_repos_urls( - gitstats_object = git_stats, + gitstats = git_stats, with_code = "shiny" ) files_structure <- get_files_structure( - gitstats_object = git_stats, + gitstats = git_stats, pattern = "\\.md", depth = 1L ) From 138f2aa02b4025c0a52ef55c5e161e2cc11f0e55 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 12 Dec 2024 15:26:44 +0000 Subject: [PATCH 03/27] Make possible pulling commits with new approach to setting scope. --- R/GitHost.R | 13 ++++- R/GitHostGitHub.R | 115 ++++++++++++++++++++++++++-------------- R/GitHostGitLab.R | 130 ++++++++++++++++++++++++++++++---------------- 3 files changed, 172 insertions(+), 86 deletions(-) diff --git a/R/GitHost.R b/R/GitHost.R index ec600e58..d851b6c8 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -133,12 +133,23 @@ GitHost <- R6::R6Class( cli::cli_alert_info("[{private$host_name}][Engine:{cli::col_yellow('GraphQL')}] Pulling all organizations...") private$orgs <- private$engines$graphql$get_orgs() } - commits_table <- private$get_commits_from_orgs( + commits_from_orgs <- private$get_commits_from_orgs( since = since, until = until, verbose = verbose, progress = progress ) + commits_from_repos <- private$get_commits_from_repos( + since = since, + until = until, + verbose = verbose, + progress = progress + ) + commits_table <- list( + commits_from_orgs, + commits_from_repos + ) |> + purrr::list_rbind() return(commits_table) }, diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index ab38aa91..0b8abdbc 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -167,50 +167,85 @@ GitHostGitHub <- R6::R6Class( # Pull commits from GitHub get_commits_from_orgs = function(since, until, verbose, progress) { - graphql_engine <- private$engines$graphql - commits_table <- purrr::map(private$orgs, function(org) { - commits_table_org <- NULL - if (!private$scan_all && verbose) { - show_message( - host = private$host_name, - engine = "graphql", - scope = org, - information = "Pulling commits" - ) - } - repos_names <- private$set_repositories( - org = org - ) - commits_table_org <- graphql_engine$get_commits_from_repos( - org = org, - repos_names = repos_names, - since = since, - until = until, - progress = progress - ) %>% - graphql_engine$prepare_commits_table( - org = org - ) - return(commits_table_org) - }, .progress = if (private$scan_all && progress) { - "[GitHost:GitHub] Pulling commits..." - } else { - FALSE - }) %>% - purrr::list_rbind() - return(commits_table) + if ("org" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + commits_table <- purrr::map(private$orgs, function(org) { + commits_table_org <- NULL + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = "Pulling commits" + ) + } + commits_table_org <- graphql_engine$get_commits_from_repos( + org = org, + repos_names = private$get_repos_names(org), + since = since, + until = until, + progress = progress + ) %>% + graphql_engine$prepare_commits_table( + org = org + ) + return(commits_table_org) + }, .progress = if (private$scan_all && progress) { + "[GitHost:GitHub] Pulling commits..." + } else { + FALSE + }) %>% + purrr::list_rbind() + return(commits_table) + } }, - # Use repositories either from parameter or, if not set, pull them from API - set_repositories = function(org) { + # Pull commits from GitHub + get_commits_from_repos = function(since, until, verbose, progress) { if ("repo" %in% private$searching_scope) { - repos_names <- private$orgs_repos[[org]] - } else { - repos_table <- private$get_all_repos( - verbose = FALSE - ) - repos_names <- repos_table$repo_name + graphql_engine <- private$engines$graphql + orgs <- names(private$orgs_repos) + commits_table <- purrr::map(orgs, function(org) { + commits_table_org <- NULL + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = "Pulling commits" + ) + } + commits_table_org <- graphql_engine$get_commits_from_repos( + org = org, + repos_names = private$orgs_repos[[org]], + since = since, + until = until, + progress = progress + ) %>% + graphql_engine$prepare_commits_table( + org = org + ) + return(commits_table_org) + }, .progress = if (private$scan_all && progress) { + "[GitHost:GitHub] Pulling commits..." + } else { + FALSE + }) %>% + purrr::list_rbind() + return(commits_table) } + }, + + # Use repositories either from parameter or, if not set, pull them from API + get_repos_names = function(org) { + type <- attr(org, "type") %||% "organization" + org <- utils::URLdecode(org) + graphql_engine <- private$engines$graphql + repos_names <- graphql_engine$get_repos_from_org( + org = org, + type = type + ) |> + purrr::map_vec(~ .$repo_name) return(repos_names) }, diff --git a/R/GitHostGitLab.R b/R/GitHostGitLab.R index d8d129e3..05884359 100644 --- a/R/GitHostGitLab.R +++ b/R/GitHostGitLab.R @@ -223,61 +223,101 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", }) }, - # Pull commits from GitHub get_commits_from_orgs = function(since, until, verbose = TRUE, progress = verbose) { - rest_engine <- private$engines$rest - commits_table <- purrr::map(private$orgs, function(org) { - commits_table_org <- NULL - if (!private$scan_all && verbose) { - show_message( - host = private$host_name, - engine = "rest", - scope = utils::URLdecode(org), - information = "Pulling commits" + if ("org" %in% private$searching_scope) { + rest_engine <- private$engines$rest + commits_table <- purrr::map(private$orgs, function(org) { + commits_table_org <- NULL + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "rest", + scope = utils::URLdecode(org), + information = "Pulling commits" + ) + } + repos_names <- private$get_repos_names( + org = org ) - } - repos_names <- private$set_repositories( - org = org - ) - commits_table_org <- rest_engine$get_commits_from_repos( - repos_names = repos_names, - since = since, - until = until, - progress = progress - ) %>% - rest_engine$tailor_commits_info(org = org) %>% - rest_engine$prepare_commits_table() %>% - rest_engine$get_commits_authors_handles_and_names( - verbose = verbose, + commits_table_org <- rest_engine$get_commits_from_repos( + repos_names = repos_names, + since = since, + until = until, progress = progress - ) - return(commits_table_org) - }, .progress = if (private$scan_all && progress) { - "[GitHost:GitLab] Pulling commits..." - } else { - FALSE - }) %>% - purrr::list_rbind() - return(commits_table) + ) %>% + rest_engine$tailor_commits_info(org = org) %>% + rest_engine$prepare_commits_table() %>% + rest_engine$get_commits_authors_handles_and_names( + verbose = verbose, + progress = progress + ) + return(commits_table_org) + }, .progress = if (private$scan_all && progress) { + "[GitHost:GitLab] Pulling commits..." + } else { + FALSE + }) %>% + purrr::list_rbind() + return(commits_table) + } }, - # Use repositories either from parameter or, if not set, pull them from API - set_repositories = function(org, settings) { + get_commits_from_repos = function(since, + until, + verbose = TRUE, + progress = verbose) { if ("repo" %in% private$searching_scope) { - repos <- private$orgs_repos[[org]] - repos_names <- paste0(utils::URLencode(org, reserved = TRUE), "%2f", repos) - } else { - repos_table <- private$get_all_repos( - verbose = FALSE - ) - gitlab_web_url <- stringr::str_extract(private$api_url, "^.*?(?=api)") - repos <- stringr::str_remove(repos_table$repo_url, gitlab_web_url) - repos_names <- utils::URLencode(repos, reserved = TRUE) + rest_engine <- private$engines$rest + orgs <- names(private$orgs_repos) + commits_table <- purrr::map(orgs, function(org) { + commits_table_org <- NULL + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "rest", + scope = utils::URLdecode(org), + information = "Pulling commits" + ) + } + repos <- private$orgs_repos[[org]] + repos_names <- paste0(utils::URLencode(org, reserved = TRUE), "%2f", repos) + commits_table_org <- rest_engine$get_commits_from_repos( + repos_names = repos_names, + since = since, + until = until, + progress = progress + ) %>% + rest_engine$tailor_commits_info(org = org) %>% + rest_engine$prepare_commits_table() %>% + rest_engine$get_commits_authors_handles_and_names( + verbose = verbose, + progress = progress + ) + return(commits_table_org) + }, .progress = if (private$scan_all && progress) { + "[GitHost:GitLab] Pulling commits..." + } else { + FALSE + }) %>% + purrr::list_rbind() + return(commits_table) } - return(repos_names) + }, + + # Use repositories either from parameter or, if not set, pull them from API + get_repos_names = function(org) { + graphql_engine <- private$engines$graphql + type <- attr(org, "type") %||% "organization" + repos_names <- graphql_engine$get_repos_from_org( + org = utils::URLdecode(org), + type = type + ) |> + purrr::map_vec(~ .$node$repo_path) + org <- utils::URLencode(org, reserved = TRUE) + return(paste0(org, "%2f", repos_names)) }, are_non_text_files = function(file_path, host_files_structure) { From 39e90245cfc280ccde14596fe59163e13895d1ea Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 13 Dec 2024 10:11:28 +0000 Subject: [PATCH 04/27] Adjust releases to pull from both scopes, adjust tests. --- R/GitHost.R | 122 +++++++++++++++----- tests/testthat/_snaps/get_release-GitHub.md | 8 -- tests/testthat/test-01-get_commits-GitHub.R | 50 ++++++-- tests/testthat/test-get_release-GitHub.R | 84 ++++++++------ tests/testthat/test-get_release-GitLab.R | 65 ++++++++++- 5 files changed, 239 insertions(+), 90 deletions(-) diff --git a/R/GitHost.R b/R/GitHost.R index d851b6c8..5b3647e9 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -218,36 +218,22 @@ GitHost <- R6::R6Class( private$orgs <- private$engines$graphql$get_orgs() } until <- until %||% Sys.time() - release_logs_table <- purrr::map(private$orgs, function(org) { - org <- utils::URLdecode(org) - release_logs_table_org <- NULL - if (!private$scan_all && verbose) { - show_message( - host = private$host_name, - engine = "graphql", - scope = org, - information = "Pulling release logs" - ) - } - repos_names <- private$set_repositories( - org = org - ) - graphql_engine <- private$engines$graphql - if (length(repos_names) > 0) { - release_logs_table_org <- graphql_engine$get_release_logs_from_org( - org = org, - repos_names = repos_names - ) %>% - graphql_engine$prepare_releases_table(org, since, until) - } else { - releases_logs_table_org <- NULL - } - return(release_logs_table_org) - }, .progress = if (progress) { - glue::glue("[GitHost:{private$host_name}] Pulling release logs...") - } else { - FALSE - }) %>% + release_logs_from_orgs <- private$get_release_logs_from_orgs( + since = since, + until = until, + verbose = verbose, + progress= progress + ) + release_logs_from_repos <- private$get_release_logs_from_repos( + since = since, + until = until, + verbose = verbose, + progress= progress + ) + release_logs_table <- list( + release_logs_from_orgs, + release_logs_from_repos + ) |> purrr::list_rbind() return(release_logs_table) } @@ -1020,6 +1006,82 @@ GitHost <- R6::R6Class( private$prepare_files_table_from_rest() %>% private$add_repo_api_url() return(files_table) + }, + + get_release_logs_from_orgs = function(since, until, verbose, progress) { + if ("org" %in% private$searching_scope) { + release_logs_table <- purrr::map(private$orgs, function(org) { + org <- utils::URLdecode(org) + release_logs_table_org <- NULL + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = "Pulling release logs" + ) + } + repos_names <- private$get_repos_names( + org = org + ) + graphql_engine <- private$engines$graphql + if (length(repos_names) > 0) { + release_logs_table_org <- graphql_engine$get_release_logs_from_org( + org = org, + repos_names = repos_names + ) %>% + graphql_engine$prepare_releases_table( + org = org, + since = since, + until = until + ) + } else { + releases_logs_table_org <- NULL + } + return(release_logs_table_org) + }, .progress = if (progress) { + glue::glue("[GitHost:{private$host_name}] Pulling release logs...") + } else { + FALSE + }) %>% + purrr::list_rbind() + return(release_logs_table) + } + }, + + get_release_logs_from_repos = function(since, until, verbose, progress) { + if ("repo" %in% private$searching_scope) { + orgs <- names(private$orgs_repos) + release_logs_table <- purrr::map(orgs, function(org) { + org <- utils::URLdecode(org) + release_logs_table_org <- NULL + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = "Pulling release logs" + ) + } + graphql_engine <- private$engines$graphql + release_logs_table_org <- graphql_engine$get_release_logs_from_org( + org = org, + repos_names = private$orgs_repos[[org]] + ) %>% + graphql_engine$prepare_releases_table( + org = org, + since = since, + until = until + ) + return(release_logs_table_org) + }, .progress = if (progress) { + glue::glue("[GitHost:{private$host_name}] Pulling release logs...") + } else { + FALSE + }) %>% + purrr::list_rbind() + return(release_logs_table) + } } ) ) diff --git a/tests/testthat/_snaps/get_release-GitHub.md b/tests/testthat/_snaps/get_release-GitHub.md index a46f4267..116312b0 100644 --- a/tests/testthat/_snaps/get_release-GitHub.md +++ b/tests/testthat/_snaps/get_release-GitHub.md @@ -5,11 +5,3 @@ Output [1] "query GetReleasesFromRepo ($org: String!, $repo: String!) {\n repository(owner:$org, name:$repo){\n name\n url\n releases (last: 100) {\n nodes {\n name\n tagName\n publishedAt\n url\n description\n }\n }\n }\n }" -# `get_release_logs()` prints proper message when running - - Code - releases_table <- github_testhost$get_release_logs(since = "2023-05-01", until = "2023-09-30", - verbose = TRUE, progress = FALSE) - Message - i [Host:GitHub][Engine:GraphQl][Scope:test-org] Pulling release logs... - diff --git a/tests/testthat/test-01-get_commits-GitHub.R b/tests/testthat/test-01-get_commits-GitHub.R index 91140a8f..7765ae92 100644 --- a/tests/testthat/test-01-get_commits-GitHub.R +++ b/tests/testthat/test-01-get_commits-GitHub.R @@ -104,28 +104,53 @@ test_that("get_commits_from_orgs for GitHub works", { "graphql_engine$prepare_commits_table", test_mocker$use("gh_commits_table") ) - suppressMessages( - gh_commits_table <- github_testhost_repos_priv$get_commits_from_orgs( - since = "2023-03-01", - until = "2023-04-01", - verbose = FALSE, - progress = FALSE - ) + github_testhost_repos_priv$searching_scope <- "org" + gh_commits_from_orgs <- github_testhost_repos_priv$get_commits_from_orgs( + since = "2023-03-01", + until = "2023-04-01", + verbose = FALSE, + progress = FALSE ) expect_commits_table( - gh_commits_table + gh_commits_from_orgs ) - test_mocker$cache(gh_commits_table) + test_mocker$cache(gh_commits_from_orgs) +}) + + +test_that("get_commits_from_repos for GitHub works", { + mockery::stub( + github_testhost_repos_priv$get_commits_from_repos, + "graphql_engine$prepare_commits_table", + test_mocker$use("gh_commits_table") + ) + github_testhost_repos_priv$searching_scope <- "repo" + github_testhost_repos_priv$orgs_repos <- list("test_org" = "TestRepo") + gh_commits_from_repos <- github_testhost_repos_priv$get_commits_from_repos( + since = "2023-03-01", + until = "2023-04-01", + verbose = FALSE, + progress = FALSE + ) + expect_commits_table( + gh_commits_from_repos + ) + test_mocker$cache(gh_commits_from_repos) }) test_that("`get_commits()` retrieves commits in the table format", { mockery::stub( github_testhost$get_commits, "private$get_commits_from_orgs", - test_mocker$use("gh_commits_table") + test_mocker$use("gh_commits_from_orgs") + ) + mockery::stub( + github_testhost$get_commits, + "private$get_commits_from_repos", + test_mocker$use("gh_commits_from_repos") ) suppressMessages( - commits_table <- github_testhost$get_commits( + gh_commits_table <- github_testhost$get_commits( since = "2023-01-01", until = "2023-02-28", verbose = FALSE, @@ -133,8 +158,9 @@ test_that("`get_commits()` retrieves commits in the table format", { ) ) expect_commits_table( - commits_table + gh_commits_table ) + test_mocker$cache(gh_commits_table) }) test_that("get_commits for GitHub repositories works", { diff --git a/tests/testthat/test-get_release-GitHub.R b/tests/testthat/test-get_release-GitHub.R index 88fcb586..b23c8f56 100644 --- a/tests/testthat/test-get_release-GitHub.R +++ b/tests/testthat/test-get_release-GitHub.R @@ -33,73 +33,81 @@ test_that("`prepare_releases_table()` prepares releases table", { test_mocker$cache(releases_table) }) -test_that("`set_repositories` works when searching scope set to repo", { +test_that("`get_repos_names` works", { mockery::stub( - github_testhost_priv$set_repositories, - "private$get_all_repos", - test_mocker$use("gh_repos_table") - ) - github_testhost_priv$searching_scope <- "repo" - github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") - repos_names <- github_testhost_priv$set_repositories(org = "test_org") - expect_type(repos_names, "character") - expect_gt(length(repos_names), 0) -}) - -test_that("`set_repositories` works for whole orgs", { - mockery::stub( - github_testhost_priv$set_repositories, - "private$get_all_repos", - test_mocker$use("gh_repos_table") + github_testhost_priv$get_repos_names, + "graphql_engine$get_repos_from_org", + test_mocker$use("gh_repos_from_org") ) github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") github_testhost_priv$searching_scope <- "org" - repos_names <- github_testhost_priv$set_repositories() + repos_names <- github_testhost_priv$get_repos_names( + org = "test_org" + ) expect_type(repos_names, "character") expect_gt(length(repos_names), 0) test_mocker$cache(repos_names) }) -test_that("`get_release_logs()` pulls release logs in the table format", { +test_that("`get_release_logs_from_orgs()` works", { mockery::stub( - github_testhost$get_release_logs, + github_testhost_priv$get_release_logs_from_orgs, "graphql_engine$prepare_releases_table", test_mocker$use("releases_table") ) mockery::stub( - github_testhost$get_release_logs, - "private$set_repositories", + github_testhost_priv$get_release_logs_from_orgs, + "private$get_repos_names", test_mocker$use("repos_names") ) - releases_table <- github_testhost$get_release_logs( + github_testhost_priv$searching_scope <- "org" + releases_from_orgs <- github_testhost_priv$get_release_logs_from_orgs( since = "2023-05-01", until = "2023-09-30", verbose = FALSE, progress = FALSE ) - expect_releases_table(releases_table) - expect_gt(min(releases_table$published_at), as.POSIXct("2023-05-01")) - expect_lt(max(releases_table$published_at), as.POSIXct("2023-09-30")) - test_mocker$cache(releases_table) + expect_releases_table(releases_from_orgs) + test_mocker$cache(releases_from_orgs) }) -test_that("`get_release_logs()` prints proper message when running", { +test_that("`get_release_logs_from_repos()` works", { mockery::stub( - github_testhost$get_release_logs, + github_testhost_priv$get_release_logs_from_repos, "graphql_engine$prepare_releases_table", test_mocker$use("releases_table") ) + github_testhost_priv$searching_scope <- "repo" + github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + releases_from_repos <- github_testhost_priv$get_release_logs_from_repos( + since = "2023-05-01", + until = "2023-09-30", + verbose = FALSE, + progress = FALSE + ) + expect_releases_table(releases_from_repos) + test_mocker$cache(releases_from_repos) +}) + +test_that("`get_release_logs()` pulls release logs in the table format", { mockery::stub( github_testhost$get_release_logs, - "private$set_repositories", - test_mocker$use("repos_names") + "private$get_release_logs_from_repos", + test_mocker$use("releases_from_repos") ) - expect_snapshot( - releases_table <- github_testhost$get_release_logs( - since = "2023-05-01", - until = "2023-09-30", - verbose = TRUE, - progress = FALSE - ) + mockery::stub( + github_testhost$get_release_logs, + "private$get_release_logs_from_orgs", + test_mocker$use("releases_from_orgs") ) + releases_table <- github_testhost$get_release_logs( + since = "2023-05-01", + until = "2023-09-30", + verbose = FALSE, + progress = FALSE + ) + expect_releases_table(releases_table) + expect_gt(min(releases_table$published_at), as.POSIXct("2023-05-01")) + expect_lt(max(releases_table$published_at), as.POSIXct("2023-09-30")) + test_mocker$cache(releases_table) }) diff --git a/tests/testthat/test-get_release-GitLab.R b/tests/testthat/test-get_release-GitLab.R index fe79a434..97f64ed3 100644 --- a/tests/testthat/test-get_release-GitLab.R +++ b/tests/testthat/test-get_release-GitLab.R @@ -34,12 +34,73 @@ test_that("`prepare_releases_table()` prepares releases table", { test_mocker$cache(releases_table) }) -test_that("`get_release_logs()` pulls release logs in the table format", { +test_that("`get_repos_names` works", { mockery::stub( - gitlab_testhost$get_release_logs, + gitlab_testhost_priv$get_repos_names, + "graphql_engine$get_repos_from_org", + test_mocker$use("gl_repos_from_org") + ) + gitlab_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + gitlab_testhost_priv$searching_scope <- "org" + repos_names <- gitlab_testhost_priv$get_repos_names( + org = "test_org" + ) + expect_type(repos_names, "character") + expect_gt(length(repos_names), 0) + test_mocker$cache(repos_names) +}) + +test_that("`get_release_logs_from_orgs()` works", { + mockery::stub( + gitlab_testhost_priv$get_release_logs_from_orgs, + "graphql_engine$prepare_releases_table", + test_mocker$use("releases_table") + ) + mockery::stub( + gitlab_testhost_priv$get_release_logs_from_orgs, + "private$get_repos_names", + test_mocker$use("repos_names") + ) + gitlab_testhost_priv$searching_scope <- "org" + releases_from_orgs <- gitlab_testhost_priv$get_release_logs_from_orgs( + since = "2023-05-01", + until = "2023-09-30", + verbose = FALSE, + progress = FALSE + ) + expect_releases_table(releases_from_orgs) + test_mocker$cache(releases_from_orgs) +}) + +test_that("`get_release_logs_from_repos()` works", { + mockery::stub( + gitlab_testhost_priv$get_release_logs_from_repos, "graphql_engine$prepare_releases_table", test_mocker$use("releases_table") ) + gitlab_testhost_priv$searching_scope <- "repo" + gitlab_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + releases_from_repos <- gitlab_testhost_priv$get_release_logs_from_repos( + since = "2023-05-01", + until = "2023-09-30", + verbose = FALSE, + progress = FALSE + ) + expect_releases_table(releases_from_repos) + test_mocker$cache(releases_from_repos) +}) + +test_that("`get_release_logs()` pulls release logs in the table format", { + mockery::stub( + gitlab_testhost$get_release_logs, + "private$get_release_logs_from_repos", + test_mocker$use("releases_from_repos") + ) + mockery::stub( + gitlab_testhost$get_release_logs, + "private$get_release_logs_from_orgs", + test_mocker$use("releases_from_orgs") + ) releases_table <- gitlab_testhost$get_release_logs( since = "2023-08-01", until = "2024-06-30", From abc6d7ba1090d6c13d90ca9899117a3dd35f05aa Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 13 Dec 2024 13:46:08 +0000 Subject: [PATCH 05/27] Fixes in function parameters and in passing properly repos names to GraphQL requests, fixing and adjusting tests. In tests there was a need to change order of test calls: first should go repos, then commits, as get_repos_names was moved to tests for commits (from releases). --- R/EngineGraphQLGitHub.R | 12 +++++----- R/EngineGraphQLGitLab.R | 10 ++++---- R/GitHost.R | 10 ++++---- R/GitHostGitHub.R | 5 ++-- R/GitHostGitLab.R | 11 ++++----- ...its-GitHub.md => 02-get_commits-GitHub.md} | 0 ...GitStats.md => 02-get_commits-GitStats.md} | 0 ...-GitHub.R => test-02-get_commits-GitHub.R} | 21 ++++++++++++++++ ...-GitLab.R => test-02-get_commits-GitLab.R} | 0 ...Stats.R => test-02-get_commits-GitStats.R} | 0 tests/testthat/test-get_release-GitHub.R | 24 ++++--------------- tests/testthat/test-get_release-GitLab.R | 6 ++--- 12 files changed, 52 insertions(+), 47 deletions(-) rename tests/testthat/_snaps/{01-get_commits-GitHub.md => 02-get_commits-GitHub.md} (100%) rename tests/testthat/_snaps/{01-get_commits-GitStats.md => 02-get_commits-GitStats.md} (100%) rename tests/testthat/{test-01-get_commits-GitHub.R => test-02-get_commits-GitHub.R} (88%) rename tests/testthat/{test-01-get_commits-GitLab.R => test-02-get_commits-GitLab.R} (100%) rename tests/testthat/{test-01-get_commits-GitStats.R => test-02-get_commits-GitStats.R} (100%) diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index 5ce4ddae..b944ae05 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -297,7 +297,7 @@ EngineGraphQLGitHub <- R6::R6Class( }, # Prepare releases table. - prepare_releases_table = function(releases_response, org, date_from, date_until) { + prepare_releases_table = function(releases_response, org, since, until) { if (!is.null(releases_response)) { releases_table <- purrr::map(releases_response, function(release) { @@ -310,7 +310,7 @@ EngineGraphQLGitHub <- R6::R6Class( release_log = node$description ) }) %>% - purrr::list_rbind() %>% + purrr::list_rbind() |> dplyr::mutate( repo_name = release$data$repository$name, repo_url = release$data$repository$url @@ -321,14 +321,14 @@ EngineGraphQLGitHub <- R6::R6Class( ) return(release_table) }) %>% - purrr::list_rbind() %>% + purrr::list_rbind() |> dplyr::filter( - published_at <= as.POSIXct(date_until) + published_at <= as.POSIXct(until) ) - if (!is.null(date_from)) { + if (!is.null(since)) { releases_table <- releases_table %>% dplyr::filter( - published_at >= as.POSIXct(date_from) + published_at >= as.POSIXct(since) ) } } else { diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index 3f75ef19..13667721 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -368,7 +368,7 @@ EngineGraphQLGitLab <- R6::R6Class( response <- self$gql_response( gql_query = releases_from_repo_query, vars = list( - "project_path" = utils::URLdecode(repository) + "project_path" = paste0(org, "/", utils::URLdecode(repository)) ) ) return(response) @@ -378,7 +378,7 @@ EngineGraphQLGitLab <- R6::R6Class( }, # Prepare releases table. - prepare_releases_table = function(releases_response, org, date_from, date_until) { + prepare_releases_table = function(releases_response, org, since, until) { if (length(releases_response) > 0) { releases_table <- purrr::map(releases_response, function(release) { @@ -404,12 +404,12 @@ EngineGraphQLGitLab <- R6::R6Class( }) %>% purrr::list_rbind() %>% dplyr::filter( - published_at <= as.POSIXct(date_until) + published_at <= as.POSIXct(until) ) - if (!is.null(date_from)) { + if (!is.null(since)) { releases_table <- releases_table %>% dplyr::filter( - published_at >= as.POSIXct(date_from) + published_at >= as.POSIXct(since) ) } } else { diff --git a/R/GitHost.R b/R/GitHost.R index 5b3647e9..153ac788 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -1027,8 +1027,8 @@ GitHost <- R6::R6Class( graphql_engine <- private$engines$graphql if (length(repos_names) > 0) { release_logs_table_org <- graphql_engine$get_release_logs_from_org( - org = org, - repos_names = repos_names + repos_names = repos_names, + org = org ) %>% graphql_engine$prepare_releases_table( org = org, @@ -1059,14 +1059,14 @@ GitHost <- R6::R6Class( show_message( host = private$host_name, engine = "graphql", - scope = org, + scope = paste0(org, "/", private$orgs_repos[[org]]), information = "Pulling release logs" ) } graphql_engine <- private$engines$graphql release_logs_table_org <- graphql_engine$get_release_logs_from_org( - org = org, - repos_names = private$orgs_repos[[org]] + repos_names = private$orgs_repos[[org]], + org = org ) %>% graphql_engine$prepare_releases_table( org = org, diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index 0b8abdbc..34998ccf 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -179,9 +179,10 @@ GitHostGitHub <- R6::R6Class( information = "Pulling commits" ) } + repos_names <- private$get_repos_names(org) commits_table_org <- graphql_engine$get_commits_from_repos( org = org, - repos_names = private$get_repos_names(org), + repos_names = repos_names, since = since, until = until, progress = progress @@ -211,7 +212,7 @@ GitHostGitHub <- R6::R6Class( show_message( host = private$host_name, engine = "graphql", - scope = org, + scope = paste0(org, "/", private$orgs_repos[[org]], collapse = ", "), information = "Pulling commits" ) } diff --git a/R/GitHostGitLab.R b/R/GitHostGitLab.R index 05884359..11c48118 100644 --- a/R/GitHostGitLab.R +++ b/R/GitHostGitLab.R @@ -243,7 +243,7 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", org = org ) commits_table_org <- rest_engine$get_commits_from_repos( - repos_names = repos_names, + repos_names = paste0(org, "%2f", repos_names), since = since, until = until, progress = progress @@ -274,16 +274,16 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", orgs <- names(private$orgs_repos) commits_table <- purrr::map(orgs, function(org) { commits_table_org <- NULL + repos <- private$orgs_repos[[org]] + repos_names <- paste0(utils::URLencode(org, reserved = TRUE), "%2f", repos) if (!private$scan_all && verbose) { show_message( host = private$host_name, engine = "rest", - scope = utils::URLdecode(org), + scope = utils::URLdecode(paste0(repos_names, collapse = "|")), information = "Pulling commits" ) } - repos <- private$orgs_repos[[org]] - repos_names <- paste0(utils::URLencode(org, reserved = TRUE), "%2f", repos) commits_table_org <- rest_engine$get_commits_from_repos( repos_names = repos_names, since = since, @@ -316,8 +316,7 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", type = type ) |> purrr::map_vec(~ .$node$repo_path) - org <- utils::URLencode(org, reserved = TRUE) - return(paste0(org, "%2f", repos_names)) + return(repos_names) }, are_non_text_files = function(file_path, host_files_structure) { diff --git a/tests/testthat/_snaps/01-get_commits-GitHub.md b/tests/testthat/_snaps/02-get_commits-GitHub.md similarity index 100% rename from tests/testthat/_snaps/01-get_commits-GitHub.md rename to tests/testthat/_snaps/02-get_commits-GitHub.md diff --git a/tests/testthat/_snaps/01-get_commits-GitStats.md b/tests/testthat/_snaps/02-get_commits-GitStats.md similarity index 100% rename from tests/testthat/_snaps/01-get_commits-GitStats.md rename to tests/testthat/_snaps/02-get_commits-GitStats.md diff --git a/tests/testthat/test-01-get_commits-GitHub.R b/tests/testthat/test-02-get_commits-GitHub.R similarity index 88% rename from tests/testthat/test-01-get_commits-GitHub.R rename to tests/testthat/test-02-get_commits-GitHub.R index 7765ae92..e0ba88e6 100644 --- a/tests/testthat/test-01-get_commits-GitHub.R +++ b/tests/testthat/test-02-get_commits-GitHub.R @@ -98,12 +98,33 @@ test_that("fill_empty_authors() works as expected", { ) }) +test_that("`get_repos_names` works", { + mockery::stub( + github_testhost_priv$get_repos_names, + "graphql_engine$get_repos_from_org", + test_mocker$use("gh_repos_from_org") + ) + github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + github_testhost_priv$searching_scope <- "org" + gh_repos_names <- github_testhost_priv$get_repos_names( + org = "test_org" + ) + expect_type(gh_repos_names, "character") + expect_gt(length(gh_repos_names), 0) + test_mocker$cache(gh_repos_names) +}) + test_that("get_commits_from_orgs for GitHub works", { mockery::stub( github_testhost_repos_priv$get_commits_from_orgs, "graphql_engine$prepare_commits_table", test_mocker$use("gh_commits_table") ) + mockery::stub( + github_testhost_repos_priv$get_commits_from_orgs, + "private$get_repos_names", + test_mocker$use("gh_repos_names") + ) github_testhost_repos_priv$searching_scope <- "org" gh_commits_from_orgs <- github_testhost_repos_priv$get_commits_from_orgs( since = "2023-03-01", diff --git a/tests/testthat/test-01-get_commits-GitLab.R b/tests/testthat/test-02-get_commits-GitLab.R similarity index 100% rename from tests/testthat/test-01-get_commits-GitLab.R rename to tests/testthat/test-02-get_commits-GitLab.R diff --git a/tests/testthat/test-01-get_commits-GitStats.R b/tests/testthat/test-02-get_commits-GitStats.R similarity index 100% rename from tests/testthat/test-01-get_commits-GitStats.R rename to tests/testthat/test-02-get_commits-GitStats.R diff --git a/tests/testthat/test-get_release-GitHub.R b/tests/testthat/test-get_release-GitHub.R index b23c8f56..3596902d 100644 --- a/tests/testthat/test-get_release-GitHub.R +++ b/tests/testthat/test-get_release-GitHub.R @@ -23,9 +23,9 @@ test_that("`get_releases_from_org()` pulls releases from the repositories", { test_that("`prepare_releases_table()` prepares releases table", { releases_table <- test_graphql_github$prepare_releases_table( releases_response = test_mocker$use("releases_from_repos"), - org = "r-world-devs", - date_from = "2023-05-01", - date_until = "2023-09-30" + org = "r-world-devs", + since = "2023-05-01", + until = "2023-09-30" ) expect_releases_table(releases_table) expect_gt(min(releases_table$published_at), as.POSIXct("2023-05-01")) @@ -33,22 +33,6 @@ test_that("`prepare_releases_table()` prepares releases table", { test_mocker$cache(releases_table) }) -test_that("`get_repos_names` works", { - mockery::stub( - github_testhost_priv$get_repos_names, - "graphql_engine$get_repos_from_org", - test_mocker$use("gh_repos_from_org") - ) - github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") - github_testhost_priv$searching_scope <- "org" - repos_names <- github_testhost_priv$get_repos_names( - org = "test_org" - ) - expect_type(repos_names, "character") - expect_gt(length(repos_names), 0) - test_mocker$cache(repos_names) -}) - test_that("`get_release_logs_from_orgs()` works", { mockery::stub( github_testhost_priv$get_release_logs_from_orgs, @@ -58,7 +42,7 @@ test_that("`get_release_logs_from_orgs()` works", { mockery::stub( github_testhost_priv$get_release_logs_from_orgs, "private$get_repos_names", - test_mocker$use("repos_names") + test_mocker$use("gh_repos_names") ) github_testhost_priv$searching_scope <- "org" releases_from_orgs <- github_testhost_priv$get_release_logs_from_orgs( diff --git a/tests/testthat/test-get_release-GitLab.R b/tests/testthat/test-get_release-GitLab.R index 97f64ed3..06e73b5d 100644 --- a/tests/testthat/test-get_release-GitLab.R +++ b/tests/testthat/test-get_release-GitLab.R @@ -24,9 +24,9 @@ test_that("`get_releases_from_org()` pulls releases from the repositories", { test_that("`prepare_releases_table()` prepares releases table", { releases_table <- test_graphql_gitlab$prepare_releases_table( releases_response = test_mocker$use("releases_from_repos"), - org = "test_org", - date_from = "2023-08-01", - date_until = "2024-06-30" + org = "test_org", + since = "2023-08-01", + until = "2024-06-30" ) expect_releases_table(releases_table) expect_gt(min(releases_table$published_at), as.POSIXct("2023-08-01")) From 93ad9dd75aec36aca34edae6188924719e995f4f Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 13 Dec 2024 13:46:56 +0000 Subject: [PATCH 06/27] Fix lint. --- R/GitHost.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/GitHost.R b/R/GitHost.R index 153ac788..e9ffc4e2 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -222,13 +222,13 @@ GitHost <- R6::R6Class( since = since, until = until, verbose = verbose, - progress= progress + progress = progress ) release_logs_from_repos <- private$get_release_logs_from_repos( since = since, until = until, verbose = verbose, - progress= progress + progress = progress ) release_logs_table <- list( release_logs_from_orgs, From 469573a2a76faf61a50797dc2a5ffd2379022793 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 13 Dec 2024 16:23:56 +0000 Subject: [PATCH 07/27] Adjust get_repos_urls to new scope approach, fix setting orgs when pulling data from repos (when org is a user), change name of get_repos_individual to get_repos_from_repos, fix pulling releases when empty. --- R/EngineGraphQLGitHub.R | 2 +- R/EngineRestGitLab.R | 6 +- R/GitHost.R | 97 ++++++++++++++--------- R/GitHostGitHub.R | 4 +- R/GitHostGitLab.R | 4 +- tests/testthat/test-01-get_repos-GitHub.R | 8 +- 6 files changed, 76 insertions(+), 45 deletions(-) diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index b944ae05..9c2a653f 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -298,7 +298,7 @@ EngineGraphQLGitHub <- R6::R6Class( # Prepare releases table. prepare_releases_table = function(releases_response, org, since, until) { - if (!is.null(releases_response)) { + if (length(releases_response) > 0) { releases_table <- purrr::map(releases_response, function(release) { release_table <- purrr::map(release$data$repository$releases$nodes, function(node) { diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index 2ce2455d..c7887a14 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -168,8 +168,10 @@ EngineRestGitLab <- R6::R6Class( # Pull all repositories URLs from organization get_repos_urls = function(type, org, repos) { - repos_response <- self$response( - endpoint = paste0(private$endpoints[["organizations"]], utils::URLencode(org, reserved = TRUE), "/projects") + repos_response <- private$paginate_results( + endpoint = paste0(private$endpoints[["organizations"]], + utils::URLencode(org, reserved = TRUE), + "/projects") ) if (!is.null(repos)) { repos_response <- repos_response %>% diff --git a/R/GitHost.R b/R/GitHost.R index e9ffc4e2..ffad9966 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -407,9 +407,6 @@ GitHost <- R6::R6Class( ) private$repos_fullnames <- repos orgs_repos <- private$extract_repos_and_orgs(private$repos_fullnames) - orgs <- private$set_owner_type( - owners = names(orgs_repos) - ) private$repos <- unname(unlist(orgs_repos)) private$orgs_repos <- orgs_repos } @@ -571,16 +568,6 @@ GitHost <- R6::R6Class( return(orgs_repo_list) }, - # Set repositories - set_repos = function(org) { - if ("repo" %in% private$searching_scope) { - repos <- private$orgs_repos[[org]] - } else { - repos <- NULL - } - return(repos) - }, - # Filter repositories table by host filter_repos_by_host = function(repos_table) { dplyr::filter( @@ -603,7 +590,7 @@ GitHost <- R6::R6Class( repos_table <- purrr::list_rbind( list( private$get_repos_from_orgs(verbose, progress), - private$get_repos_individual(verbose, progress) + private$get_repos_from_repos(verbose, progress) ) ) return(repos_table) @@ -634,10 +621,12 @@ GitHost <- R6::R6Class( } }, - get_repos_individual = function(verbose, progress) { + get_repos_from_repos = function(verbose, progress) { if ("repo" %in% private$searching_scope) { graphql_engine <- private$engines$graphql - orgs <- names(private$orgs_repos) + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) + ) purrr::map(orgs, function(org) { type <- attr(org, "type") %||% "organization" org <- utils::URLdecode(org) @@ -703,28 +692,62 @@ GitHost <- R6::R6Class( } private$orgs <- private$engines$graphql$get_orgs() } - rest_engine <- private$engines$rest - repos_vector <- purrr::map(private$orgs, function(org) { - org <- utils::URLdecode(org) - if (!private$scan_all && verbose) { - show_message( - host = private$host_name, - engine = "rest", - scope = org, - information = "Pulling repositories (URLS)" - ) - } - repos_urls <- rest_engine$get_repos_urls( - type = type, - org = org, - repos = private$set_repos(org) - ) - return(repos_urls) - }, .progress = progress) %>% - unlist() + repos_vector <- c( + private$get_repos_urls_from_orgs(type, verbose, progress), + private$get_repos_urls_from_repos(type, verbose, progress) + ) return(repos_vector) }, + get_repos_urls_from_orgs = function(type, verbose, progress) { + if ("org" %in% private$searching_scope) { + rest_engine <- private$engines$rest + repos_vector <- purrr::map(private$orgs, function(org) { + org <- utils::URLdecode(org) + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "rest", + scope = org, + information = "Pulling repositories (URLS)" + ) + } + repos_urls <- rest_engine$get_repos_urls( + type = type, + org = org, + repos = NULL + ) + return(repos_urls) + }, .progress = progress) %>% + unlist() + } + }, + + get_repos_urls_from_repos = function(type, verbose, progress) { + if ("repo" %in% private$searching_scope) { + rest_engine <- private$engines$rest + orgs <- names(private$orgs_repos) + repos_vector <- purrr::map(orgs, function(org) { + org <- utils::URLdecode(org) + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "rest", + scope = org, + information = "Pulling repositories (URLS)" + ) + } + repos_urls <- rest_engine$get_repos_urls( + type = type, + org = org, + repos = private$orgs_repos[[org]] + ) + return(repos_urls) + }, .progress = progress) %>% + unlist() + } + }, + # Pull repositories with code from whole Git Host get_repos_with_code_from_host = function(code, in_files = NULL, @@ -1051,7 +1074,9 @@ GitHost <- R6::R6Class( get_release_logs_from_repos = function(since, until, verbose, progress) { if ("repo" %in% private$searching_scope) { - orgs <- names(private$orgs_repos) + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) + ) release_logs_table <- purrr::map(orgs, function(org) { org <- utils::URLdecode(org) release_logs_table_org <- NULL diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index 34998ccf..71e1c42a 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -205,7 +205,9 @@ GitHostGitHub <- R6::R6Class( get_commits_from_repos = function(since, until, verbose, progress) { if ("repo" %in% private$searching_scope) { graphql_engine <- private$engines$graphql - orgs <- names(private$orgs_repos) + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) + ) commits_table <- purrr::map(orgs, function(org) { commits_table_org <- NULL if (!private$scan_all && verbose) { diff --git a/R/GitHostGitLab.R b/R/GitHostGitLab.R index 11c48118..8ebbdcc5 100644 --- a/R/GitHostGitLab.R +++ b/R/GitHostGitLab.R @@ -271,7 +271,9 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", progress = verbose) { if ("repo" %in% private$searching_scope) { rest_engine <- private$engines$rest - orgs <- names(private$orgs_repos) + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) + ) commits_table <- purrr::map(orgs, function(org) { commits_table_org <- NULL repos <- private$orgs_repos[[org]] diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index 43c50f81..af001cb9 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -407,9 +407,9 @@ test_that("get_repos_from_org works", { test_mocker$cache(gh_repos_from_orgs) }) -test_that("get_repos_individual works", { +test_that("get_repos_from_repos works", { mockery::stub( - github_testhost_priv$get_repos_individual, + github_testhost_priv$get_repos_from_repos, "graphql_engine$prepare_repos_table", test_mocker$use("gh_repos_table") ) @@ -420,7 +420,7 @@ test_that("get_repos_individual works", { attr(test_org, "type") <- "organization" github_testhost_priv$orgs <- test_org - gh_repos_individual <- github_testhost_priv$get_repos_individual( + gh_repos_individual <- github_testhost_priv$get_repos_from_repos( verbose = FALSE, progress = FALSE ) @@ -438,7 +438,7 @@ test_that("`get_all_repos()` works as expected", { ) mockery::stub( github_testhost_priv$get_all_repos, - "private$get_repos_individual", + "private$get_repos_from_repos", test_mocker$use("gh_repos_individual") ) gh_repos_table <- github_testhost_priv$get_all_repos( From 2fa819bec6e8be59e7b7b8131e2535ac5df5372f Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 13 Dec 2024 16:24:11 +0000 Subject: [PATCH 08/27] Add NEWS. --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 64459297..eaa5c2d8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,7 @@ ## Features: +- From now on it is possible to pass `orgs` and `repos` in `set_*_host()` functions ([#400](https://github.com/r-world-devs/GitStats/issues/400)). - Improved `get_commits_stats()` function ([#556](https://github.com/r-world-devs/GitStats/issues/556), [#557](https://github.com/r-world-devs/GitStats/issues/557)) with: - giving possibility to customize grouping variable by passing it with the `group_var` parameter, - changing name of the `time_interval` parameter to `time_aggregation`, From 06c713f14ba6fedd1087930fd15ae4c7d7430b5f Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 16 Dec 2024 08:04:42 +0000 Subject: [PATCH 09/27] Typo. --- R/GitHost.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/GitHost.R b/R/GitHost.R index ffad9966..bb85651a 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -709,7 +709,7 @@ GitHost <- R6::R6Class( host = private$host_name, engine = "rest", scope = org, - information = "Pulling repositories (URLS)" + information = "Pulling repositories (URLs)" ) } repos_urls <- rest_engine$get_repos_urls( @@ -734,7 +734,7 @@ GitHost <- R6::R6Class( host = private$host_name, engine = "rest", scope = org, - information = "Pulling repositories (URLS)" + information = "Pulling repositories (URLs)" ) } repos_urls <- rest_engine$get_repos_urls( From 23b10ecd94b333b1714681363c9548837b70700b Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Mon, 16 Dec 2024 10:58:27 +0000 Subject: [PATCH 10/27] Fix tests. --- R/GitHost.R | 14 ++++- tests/testthat/helper-fixtures.R | 6 +- tests/testthat/setup.R | 18 +----- tests/testthat/test-01-get_repos-GitHub.R | 12 ++-- tests/testthat/test-02-get_commits-GitHub.R | 42 +++++--------- tests/testthat/test-02-get_commits-GitLab.R | 16 ------ .../test-get_files_structure-GitHub.R | 10 ++-- tests/testthat/test-get_release-GitHub.R | 7 +++ tests/testthat/test-get_urls_repos-GitHub.R | 56 ++++++++++++++++++- tests/testthat/test-get_urls_repos-GitLab.R | 54 +++++++++++------- 10 files changed, 139 insertions(+), 96 deletions(-) diff --git a/R/GitHost.R b/R/GitHost.R index bb85651a..fd81880c 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -692,9 +692,19 @@ GitHost <- R6::R6Class( } private$orgs <- private$engines$graphql$get_orgs() } + repos_urls_from_orgs <- private$get_repos_urls_from_orgs( + type = type, + verbose = verbose, + progress = progress + ) + repos_urls_from_repos <- private$get_repos_urls_from_repos( + type = type, + verbose = verbose, + progress = progress + ) repos_vector <- c( - private$get_repos_urls_from_orgs(type, verbose, progress), - private$get_repos_urls_from_repos(type, verbose, progress) + repos_urls_from_orgs, + repos_urls_from_repos ) return(repos_vector) }, diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index e1773d7f..e80d3e7f 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -34,10 +34,10 @@ test_fixtures$github_repository_rest_response <- list( "id" = 627452680, "node_id" = "R_kgDOJWYrCA", "name" = "testRepo", - "full_name" = "test-org/TestRepo", + "full_name" = "test_org/TestRepo", "private" = FALSE, "owner" = list( - "login" = "test-org", + "login" = "test_org", "id" = 103638913, "node_id" = "O_kgDOBi1ngQ", "avatar_url" = "https://avatars.githubusercontent.com/u/103638913?v=4" @@ -529,7 +529,7 @@ test_fixtures$github_files_tree_response <- list( "repository" = list( "id" = "R_kgD0Ivtxsg", "name" = "TestRepo", - "url" = "https://github.com/test-org/TestRepo", + "url" = "https://github.com/test_org/TestRepo", "object" = list( "entries" = list( list( diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R index 9704d42a..69811c67 100644 --- a/tests/testthat/setup.R +++ b/tests/testthat/setup.R @@ -30,24 +30,10 @@ test_graphql_gitlab <- EngineGraphQLGitLab$new( ) test_graphql_gitlab_priv <- environment(test_graphql_gitlab$initialize)$private -github_testhost <- create_github_testhost(orgs = "test-org") +github_testhost <- create_github_testhost(orgs = "test_org") -github_testhost_priv <- create_github_testhost(orgs = "test-org", mode = "private") - -github_testhost_repos <- create_github_testhost( - repos = c("openpharma/DataFakeR", "r-world-devs/GitStats", "r-world-devs/cohortBuilder") -) - -github_testhost_repos_priv <- create_github_testhost( - repos = c("openpharma/DataFakeR", "r-world-devs/GitStats", "r-world-devs/cohortBuilder"), - mode = "private" -) +github_testhost_priv <- create_github_testhost(orgs = "test_org", mode = "private") gitlab_testhost <- create_gitlab_testhost(orgs = "mbtests") gitlab_testhost_priv <- create_gitlab_testhost(orgs = "mbtests", mode = "private") - -gitlab_testhost_repos <- create_gitlab_testhost( - repos = c("mbtests/gitstatstesting", "mbtests/gitstats-testing-2") -) - diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index af001cb9..6c01c398 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -408,18 +408,20 @@ test_that("get_repos_from_org works", { }) test_that("get_repos_from_repos works", { + test_org <- "test_org" + attr(test_org, "type") <- "organization" + mockery::stub( + github_testhost_priv$get_repos_from_repos, + "private$set_owner_type", + test_org + ) mockery::stub( github_testhost_priv$get_repos_from_repos, "graphql_engine$prepare_repos_table", test_mocker$use("gh_repos_table") ) github_testhost_priv$searching_scope <- c("org", "repo") - github_testhost_priv$repos_fullnames <- c("test_org/TestRepo") github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") - test_org <- "test_org" - attr(test_org, "type") <- "organization" - github_testhost_priv$orgs <- test_org - gh_repos_individual <- github_testhost_priv$get_repos_from_repos( verbose = FALSE, progress = FALSE diff --git a/tests/testthat/test-02-get_commits-GitHub.R b/tests/testthat/test-02-get_commits-GitHub.R index e0ba88e6..a42025e5 100644 --- a/tests/testthat/test-02-get_commits-GitHub.R +++ b/tests/testthat/test-02-get_commits-GitHub.R @@ -116,17 +116,17 @@ test_that("`get_repos_names` works", { test_that("get_commits_from_orgs for GitHub works", { mockery::stub( - github_testhost_repos_priv$get_commits_from_orgs, + github_testhost_priv$get_commits_from_orgs, "graphql_engine$prepare_commits_table", test_mocker$use("gh_commits_table") ) mockery::stub( - github_testhost_repos_priv$get_commits_from_orgs, + github_testhost_priv$get_commits_from_orgs, "private$get_repos_names", test_mocker$use("gh_repos_names") ) - github_testhost_repos_priv$searching_scope <- "org" - gh_commits_from_orgs <- github_testhost_repos_priv$get_commits_from_orgs( + github_testhost_priv$searching_scope <- "org" + gh_commits_from_orgs <- github_testhost_priv$get_commits_from_orgs( since = "2023-03-01", until = "2023-04-01", verbose = FALSE, @@ -141,13 +141,20 @@ test_that("get_commits_from_orgs for GitHub works", { test_that("get_commits_from_repos for GitHub works", { mockery::stub( - github_testhost_repos_priv$get_commits_from_repos, + github_testhost_priv$get_commits_from_repos, "graphql_engine$prepare_commits_table", test_mocker$use("gh_commits_table") ) - github_testhost_repos_priv$searching_scope <- "repo" - github_testhost_repos_priv$orgs_repos <- list("test_org" = "TestRepo") - gh_commits_from_repos <- github_testhost_repos_priv$get_commits_from_repos( + github_testhost_priv$searching_scope <- "repo" + github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + test_org <- "test_org" + attr(test_org, "type") <- "organization" + mockery::stub( + github_testhost_priv$get_commits_from_repos, + "private$set_owner_type", + test_org + ) + gh_commits_from_repos <- github_testhost_priv$get_commits_from_repos( since = "2023-03-01", until = "2023-04-01", verbose = FALSE, @@ -183,22 +190,3 @@ test_that("`get_commits()` retrieves commits in the table format", { ) test_mocker$cache(gh_commits_table) }) - -test_that("get_commits for GitHub repositories works", { - mockery::stub( - github_testhost_repos$get_commits, - "private$get_commits_from_orgs", - test_mocker$use("gh_commits_table") - ) - suppressMessages( - gh_commits_table <- github_testhost_repos$get_commits( - since = "2023-03-01", - until = "2023-04-01", - verbose = FALSE, - progress = FALSE - ) - ) - expect_commits_table( - gh_commits_table - ) -}) diff --git a/tests/testthat/test-02-get_commits-GitLab.R b/tests/testthat/test-02-get_commits-GitLab.R index 5f904133..e3e45d37 100644 --- a/tests/testthat/test-02-get_commits-GitLab.R +++ b/tests/testthat/test-02-get_commits-GitLab.R @@ -139,19 +139,3 @@ test_that("get_commits_from_orgs works", { ) test_mocker$cache(gl_commits_table) }) - -test_that("get_commits for GitLab works with repos implied", { - mockery::stub( - gitlab_testhost_repos$get_commits, - "private$get_commits_from_orgs", - test_mocker$use("gl_commits_table") - ) - gl_commits_table <- gitlab_testhost_repos$get_commits( - since = "2023-01-01", - until = "2023-06-01", - verbose = FALSE - ) - expect_commits_table( - gl_commits_table - ) -}) diff --git a/tests/testthat/test-get_files_structure-GitHub.R b/tests/testthat/test-get_files_structure-GitHub.R index 3c632d55..51dfad59 100644 --- a/tests/testthat/test-get_files_structure-GitHub.R +++ b/tests/testthat/test-get_files_structure-GitHub.R @@ -14,7 +14,7 @@ test_that("get_file_response works", { test_fixtures$github_files_tree_response ) gh_files_tree_response <- test_graphql_github_priv$get_file_response( - org = "test-org", + org = "test_org", repo = "TestRepo", def_branch = "master", file_path = "", @@ -55,7 +55,7 @@ test_that("get_files_structure_from_repo returns list with files and dirs vector files_and_dirs ) files_structure <- test_graphql_github_priv$get_files_structure_from_repo( - org = "test-org", + org = "test_org", repo = "TestRepo", def_branch = "master" ) @@ -77,13 +77,13 @@ test_that("get_files_structure_from_repo returns list of files up to 2 tier of d files_and_dirs <- test_mocker$use("files_and_dirs_list") ) files_structure_very_shallow <- test_graphql_github_priv$get_files_structure_from_repo( - org = "test-org", + org = "test_org", repo = "TestRepo", def_branch = "master", depth = 1L ) files_structure_shallow <- test_graphql_github_priv$get_files_structure_from_repo( - org = "test-org", + org = "test_org", repo = "TestRepo", def_branch = "master", depth = 2L @@ -149,7 +149,7 @@ test_that("GitHub GraphQL Engine pulls files structure with pattern from reposit test_mocker$use("md_files_structure") ) gh_md_files_structure <- test_graphql_github$get_files_structure_from_org( - org = "test-org", + org = "test_org", repos = "TestRepo", pattern = "\\.md|\\.qmd|\\.Rmd" ) diff --git a/tests/testthat/test-get_release-GitHub.R b/tests/testthat/test-get_release-GitHub.R index 3596902d..a576cfd6 100644 --- a/tests/testthat/test-get_release-GitHub.R +++ b/tests/testthat/test-get_release-GitHub.R @@ -61,6 +61,13 @@ test_that("`get_release_logs_from_repos()` works", { "graphql_engine$prepare_releases_table", test_mocker$use("releases_table") ) + test_org <- "test_org" + attr(test_org, "type") <- "organization" + mockery::stub( + github_testhost_priv$get_release_logs_from_repos, + "private$set_owner_type", + test_org + ) github_testhost_priv$searching_scope <- "repo" github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") releases_from_repos <- github_testhost_priv$get_release_logs_from_repos( diff --git a/tests/testthat/test-get_urls_repos-GitHub.R b/tests/testthat/test-get_urls_repos-GitHub.R index fd0c2e75..a0f379b3 100644 --- a/tests/testthat/test-get_urls_repos-GitHub.R +++ b/tests/testthat/test-get_urls_repos-GitHub.R @@ -34,7 +34,7 @@ test_that("get_repos_urls() works for individual repos", { test_mocker$cache(gh_repos_urls) }) -test_that("get_all_repos_urls prepares api repo_urls vector", { +test_that("get_repos_urls prepares api repo_urls vector", { github_testhost_priv <- create_github_testhost(orgs = "test-org", mode = "private") mockery::stub( @@ -52,13 +52,63 @@ test_that("get_all_repos_urls prepares api repo_urls vector", { test_mocker$cache(gh_api_repos_urls) }) -test_that("get_all_repos_urls prepares web repo_urls vector", { +test_that("get_repos_urls_from_orgs prepares web repo_urls vector", { mockery::stub( - github_testhost_priv$get_all_repos_urls, + github_testhost_priv$get_repos_urls_from_orgs, + "rest_engine$get_repos_urls", + test_mocker$use("gh_repos_urls") + ) + github_testhost_priv$searching_scope <- "org" + github_testhost_priv$orgs <- "test_org" + gh_repos_urls_from_orgs <- github_testhost_priv$get_repos_urls_from_orgs( + type = "web", + verbose = FALSE, + progress = FALSE + ) + expect_gt(length(gh_repos_urls_from_orgs), 0) + expect_true(any(grepl("test-org", gh_repos_urls_from_orgs))) + expect_true(all(grepl("https://testhost.com/", gh_repos_urls_from_orgs))) + test_mocker$cache(gh_repos_urls_from_orgs) +}) + +test_that("get_repos_urls_from_repos prepares web repo_urls vector", { + test_org <- "test_org" + attr(test_org, "type") <- "organization" + mockery::stub( + github_testhost_priv$get_repos_urls_from_repos, + "private$set_owner_type", + test_org + ) + mockery::stub( + github_testhost_priv$get_repos_urls_from_repos, "rest_engine$get_repos_urls", test_mocker$use("gh_repos_urls"), depth = 2L ) + github_testhost_priv$searching_scope <- c("repo") + github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + gh_repos_urls <- github_testhost_priv$get_repos_urls_from_repos( + type = "web", + verbose = FALSE, + progress = FALSE + ) + expect_gt(length(gh_repos_urls), 0) + expect_true(any(grepl("test-org", gh_repos_urls))) + expect_true(all(grepl("https://testhost.com/", gh_repos_urls))) + test_mocker$cache(gh_repos_urls) +}) + +test_that("get_all_repos_urls prepares web repo_urls vector", { + mockery::stub( + github_testhost_priv$get_all_repos_urls, + "private$get_repos_urls_from_orgs", + test_mocker$use("gh_repos_urls_from_orgs") + ) + mockery::stub( + github_testhost_priv$get_all_repos_urls, + "private$get_repos_urls_from_repos", + test_mocker$use("gh_repos_urls") + ) gh_repos_urls <- github_testhost_priv$get_all_repos_urls( type = "web", verbose = FALSE diff --git a/tests/testthat/test-get_urls_repos-GitLab.R b/tests/testthat/test-get_urls_repos-GitLab.R index 75f28141..17d4589b 100644 --- a/tests/testthat/test-get_urls_repos-GitLab.R +++ b/tests/testthat/test-get_urls_repos-GitLab.R @@ -1,12 +1,12 @@ test_that("get_repos_urls() works for org", { mockery::stub( test_rest_gitlab$get_repos_urls, - "self$response", + "private$paginate_results", test_fixtures$gitlab_repositories_rest_response ) gl_api_repos_urls <- test_rest_gitlab$get_repos_urls( type = "api", - org = "mbtests", + org = "test_org", repos = NULL ) expect_length( @@ -16,7 +16,7 @@ test_that("get_repos_urls() works for org", { test_mocker$cache(gl_api_repos_urls) gl_web_repos_urls <- test_rest_gitlab$get_repos_urls( type = "web", - org = "mbtests", + org = "test_org", repos = NULL ) expect_length( @@ -29,12 +29,12 @@ test_that("get_repos_urls() works for org", { test_that("get_repos_urls() works for individual repos", { mockery::stub( test_rest_gitlab$get_repos_urls, - "self$response", + "private$paginate_results", test_fixtures$gitlab_repositories_rest_response ) gl_api_repos_urls <- test_rest_gitlab$get_repos_urls( type = "api", - org = "mbtests", + org = "test_org", repos = c("testRepo1", "testRepo2") ) expect_length( @@ -54,44 +54,60 @@ test_that("get_repos_urls() works for individual repos", { test_mocker$cache(gl_web_repos_urls) }) -test_that("get_all_repos_urls prepares api repo_urls vector", { +test_that("get_repos_urls_from_orgs prepares api repo_urls vector", { mockery::stub( - gitlab_testhost_priv$get_all_repos_urls, + gitlab_testhost_priv$get_repos_urls_from_orgs, "rest_engine$get_repos_urls", test_mocker$use("gl_api_repos_urls") ) - gl_api_repos_urls <- gitlab_testhost_priv$get_all_repos_urls( + gitlab_testhost_priv$searching_scope <- "org" + gitlab_testhost_priv$orgs <- "test_org" + gl_api_repos_urls <- gitlab_testhost_priv$get_repos_urls_from_orgs( type = "api", - verbose = FALSE + verbose = FALSE, + progress = FALSE ) expect_gt(length(gl_api_repos_urls), 0) expect_true(all(grepl("api", gl_api_repos_urls))) test_mocker$cache(gl_api_repos_urls) +}) + +test_that("get_repos_urls_from_repos prepares api repo_urls vector", { mockery::stub( - gitlab_testhost_priv$get_all_repos_urls, + gitlab_testhost_priv$get_repos_urls_from_repos, "rest_engine$get_repos_urls", test_mocker$use("gl_web_repos_urls") ) - gl_web_repos_urls <- gitlab_testhost_priv$get_all_repos_urls( + gitlab_testhost_priv$searching_scope <- c("repo") + gitlab_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + gl_web_repos_urls <- gitlab_testhost_priv$get_repos_urls_from_repos( type = "web", - verbose = FALSE + verbose = FALSE, + progress = FALSE ) expect_gt(length(gl_web_repos_urls), 0) test_mocker$cache(gl_web_repos_urls) }) -test_that("get_all_repos_urls prepares web repo_urls vector", { + +test_that("get_all_repos_urls prepares api repo_urls vector", { mockery::stub( gitlab_testhost_priv$get_all_repos_urls, - "rest_engine$get_repos_urls", - test_mocker$use("gl_web_repos_urls") + "private$get_repos_urls_from_orgs", + test_mocker$use("gl_api_repos_urls") ) - gl_repos_urls <- gitlab_testhost_priv$get_all_repos_urls( - type = "web", + mockery::stub( + gitlab_testhost_priv$get_all_repos_urls, + "private$get_repos_urls_from_repos", + NULL + ) + gl_api_repos_urls <- gitlab_testhost_priv$get_all_repos_urls( + type = "api", verbose = FALSE ) - expect_gt(length(gl_repos_urls), 0) - expect_true(all(!grepl("api", gl_repos_urls))) + expect_true(all(grepl("api", gl_api_repos_urls))) + expect_gt(length(gl_api_repos_urls), 0) + test_mocker$cache(gl_api_repos_urls) }) test_that("`get_repo_url_from_response()` works", { From e470ace8ebd2917877ac31dd6bd783f1fdbfefbf Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 17 Dec 2024 12:16:33 +0000 Subject: [PATCH 11/27] Adjust get_files_content() to changes in scanning scope, remove possibility to pull non-text files, fix pagination when pulling GitLab files from org, some changes in style. In case of pagination NULL values were assinged to hasNextPage and endCursor. In case of removing possibility to pull content of non-text files - it seemed superfluous, made code very complex with custom methods for GitLab nota bene new REST method only for pulling content of non-text files (!). Pulling content should not end in pulling no content. --- R/EngineGraphQL.R | 2 +- R/EngineGraphQLGitHub.R | 20 +- R/EngineGraphQLGitLab.R | 26 +- R/GitHost.R | 302 +++++++++++++----- R/GitHostGitLab.R | 162 ++++------ R/GitStats.R | 17 +- R/gitstats_functions.R | 50 ++- R/global.R | 5 +- inst/get_files_workflow.R | 26 ++ man/get_files_content.Rd | 14 +- .../_snaps/get_files_structure-GitHub.md | 6 +- .../_snaps/get_files_structure-GitLab.md | 2 +- tests/testthat/helper-fixtures.R | 14 - .../testthat/test-get_files_content-GitHub.R | 88 +++-- .../testthat/test-get_files_content-GitLab.R | 37 +-- .../test-get_files_structure-GitHub.R | 11 +- .../test-get_files_structure-GitLab.R | 5 +- 17 files changed, 438 insertions(+), 349 deletions(-) diff --git a/R/EngineGraphQL.R b/R/EngineGraphQL.R index 7036b23a..c462180d 100644 --- a/R/EngineGraphQL.R +++ b/R/EngineGraphQL.R @@ -86,7 +86,7 @@ EngineGraphQL <- R6::R6Class( file_path <- host_files_structure[[org]][[repo]] } if (only_text_files) { - file_path <- file_path[!grepl(non_text_files_pattern, file_path)] + file_path <- file_path[grepl(text_files_pattern, file_path)] } return(file_path) } diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index 9c2a653f..fd596d15 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -175,9 +175,8 @@ EngineGraphQLGitHub <- R6::R6Class( get_files_from_org = function(org, type, repos, - file_paths, - host_files_structure, - only_text_files, + file_paths = NULL, + host_files_structure = NULL, verbose = TRUE, progress = TRUE) { repo_data <- private$get_repos_data( @@ -193,7 +192,6 @@ EngineGraphQLGitHub <- R6::R6Class( org = org, file_paths = file_paths, host_files_structure = host_files_structure, - only_text_files = only_text_files, progress = progress ) names(org_files_list) <- repositories @@ -230,10 +228,10 @@ EngineGraphQLGitHub <- R6::R6Class( # Pull all files from all repositories of an organization. get_files_structure_from_org = function(org, type, - repos, - pattern = NULL, - depth = Inf, - verbose = FALSE, + repos = NULL, + pattern = NULL, + depth = Inf, + verbose = FALSE, progress = TRUE) { repo_data <- private$get_repos_data( org = org, @@ -453,19 +451,17 @@ EngineGraphQLGitHub <- R6::R6Class( def_branches, org, host_files_structure, - only_text_files, file_paths, progress) { purrr::map2(repositories, def_branches, function(repo, def_branch) { if (!is.null(host_files_structure)) { file_paths <- private$get_path_from_files_structure( host_files_structure = host_files_structure, - only_text_files = only_text_files, org = org, repo = repo ) - } else if (is.null(host_files_structure) && only_text_files) { - file_paths <- file_paths[!grepl(non_text_files_pattern, file_paths)] + } else if (is.null(host_files_structure)) { + file_paths <- file_paths[grepl(text_files_pattern, file_paths)] } repo_files_list <- purrr::map(file_paths, function(file_path) { private$get_file_response( diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index 13667721..7b23c6e6 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -124,8 +124,8 @@ EngineGraphQLGitLab <- R6::R6Class( get_files_from_org = function(org, type, repos, - file_paths, - host_files_structure, + file_paths = NULL, + host_files_structure = NULL, only_text_files, verbose = FALSE, progress = FALSE) { @@ -139,8 +139,8 @@ EngineGraphQLGitLab <- R6::R6Class( only_text_files = only_text_files, org = org ) - } else if (is.null(host_files_structure) && only_text_files) { - file_paths <- file_paths[!grepl(non_text_files_pattern, file_paths)] + } else if (only_text_files) { + file_paths <- file_paths[grepl(text_files_pattern, file_paths)] } if (type == "organization") { while (next_page) { @@ -194,13 +194,13 @@ EngineGraphQLGitLab <- R6::R6Class( purrr::discard(~ length(.$repository$blobs$nodes) == 0) if (is.null(files_list)) files_list <- list() if (length(files_list) > 0) { - next_page <- files_response$pageInfo$hasNextPage + next_page <- projects$pageInfo$hasNextPage } else { next_page <- FALSE } if (is.null(next_page)) next_page <- FALSE if (next_page) { - end_cursor <- files_response$pageInfo$endCursor + end_cursor <- projects$pageInfo$endCursor } else { end_cursor <- "" } @@ -250,16 +250,16 @@ EngineGraphQLGitLab <- R6::R6Class( if (!is.null(host_files_structure)) { file_paths <- private$get_path_from_files_structure( host_files_structure = host_files_structure, - only_text_files = only_text_files, - org = org, - repo = repo + only_text_files = only_text_files, + org = org, + repo = repo ) } files_response <- tryCatch( { private$get_file_blobs_response( - org = org, - repo = repo, + org = org, + repo = repo, file_paths = file_paths ) }, @@ -272,7 +272,7 @@ EngineGraphQLGitLab <- R6::R6Class( }, # Prepare files table. - prepare_files_table = function(files_response, org, file_path) { + prepare_files_table = function(files_response, org) { if (!is.null(files_response)) { if (private$response_prepared_by_iteration(files_response)) { files_table <- purrr::map(files_response, function(response_data) { @@ -315,7 +315,7 @@ EngineGraphQLGitLab <- R6::R6Class( get_files_structure_from_org = function(org, type, - repos, + repos = NULL, pattern = NULL, depth = Inf, verbose = TRUE, diff --git a/R/GitHost.R b/R/GitHost.R index fd81880c..023f2a99 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -168,20 +168,35 @@ GitHost <- R6::R6Class( #' a table format. get_files_content = function(file_path, host_files_structure = NULL, - only_text_files = TRUE, - verbose = TRUE, - progress = TRUE) { - files_table <- if (!private$scan_all) { - private$get_files_content_from_orgs( - file_path = file_path, + verbose = TRUE, + progress = TRUE) { + if (is.null(host_files_structure)) { + if (!private$scan_all) { + files_content_from_orgs <- private$get_files_content_from_orgs( + file_path = file_path, + verbose = verbose, + progress = progress + ) + files_content_from_repos <- private$get_files_content_from_repos( + file_path = file_path, + verbose = verbose, + progress = progress + ) + files_table <- rbind( + files_content_from_orgs, + files_content_from_repos + ) + } else { + files_table <- private$get_files_content_from_host( + file_path = file_path, + verbose = verbose, + progress = progress + ) + } + } + if (!is.null(host_files_structure)) { + files_table <- private$get_files_content_from_files_structure( host_files_structure = host_files_structure, - only_text_files = only_text_files, - verbose = verbose, - progress = progress - ) - } else { - private$get_files_content_from_host( - file_path = file_path, verbose = verbose, progress = progress ) @@ -200,12 +215,22 @@ GitHost <- R6::R6Class( "i" = "Set `orgs` or `repos` arguments in `set_*_host()` if you wish to run this function." ), call = NULL) } - files_structure <- private$get_files_structure_from_orgs( - pattern = pattern, - depth = depth, - verbose = verbose, + files_structure_from_orgs <- private$get_files_structure_from_orgs( + pattern = pattern, + depth = depth, + verbose = verbose, + progress = progress + ) + files_structure_from_repos <- private$get_files_structure_from_repos( + pattern = pattern, + depth = depth, + verbose = verbose, progress = progress ) + files_structure <- append( + files_structure_from_orgs %||% list(), + files_structure_from_repos %||% list() + ) return(files_structure) }, @@ -473,7 +498,7 @@ GitHost <- R6::R6Class( cli::cli_abort( c( "x" = "{type} you provided does not exist or its name was passed - in a wrong way: {cli::col_red({endpoint})}", + in a wrong way: {cli::col_red({utils::URLdecode(endpoint)})}", "!" = "Please type your {tolower(type)} name as you see it in web URL.", "i" = "E.g. do not use spaces. {type} names as you see on the @@ -915,55 +940,115 @@ GitHost <- R6::R6Class( # Pull files content from organizations get_files_content_from_orgs = function(file_path, - host_files_structure = NULL, - only_text_files = TRUE, - verbose = TRUE, - progress = TRUE) { + verbose = TRUE, + progress = TRUE) { + if ("org" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + files_table <- purrr::map(private$orgs, function(org) { + if (verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = glue::glue("Pulling files content: [{paste0(file_path, collapse = ', ')}]") + ) + } + type <- attr(org, "type") %||% "organization" + graphql_engine$get_files_from_org( + org = org, + type = type, + repos = NULL, + file_paths = file_path, + verbose = verbose, + progress = progress + ) |> + graphql_engine$prepare_files_table( + org = org, + file_path = file_path + ) + }) |> + purrr::list_rbind() |> + private$add_repo_api_url() + return(files_table) + } + }, + + # Pull files content from organizations + get_files_content_from_repos = function(file_path, + verbose = TRUE, + progress = TRUE) { + if ("repo" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) + ) + files_table <- purrr::map(orgs, function(org) { + if (verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = glue::glue("Pulling files content: [{paste0(file_path, collapse = ', ')}]") + ) + } + type <- attr(org, "type") %||% "organization" + graphql_engine$get_files_from_org( + org = org, + type = type, + repos = private$orgs_repos[[org]], + file_paths = file_path, + verbose = verbose, + progress = progress + ) |> + graphql_engine$prepare_files_table( + org = org, + file_path = file_path + ) + }) |> + purrr::list_rbind() |> + private$add_repo_api_url() + return(files_table) + } + }, + + get_files_content_from_files_structure = function(host_files_structure, + verbose = TRUE, + progress = TRUE) { graphql_engine <- private$engines$graphql - if (!is.null(host_files_structure)) { - if (verbose) { - cli::cli_alert_info(cli::col_green("I will make use of files structure stored in GitStats.")) - } - result <- private$get_orgs_and_repos_from_files_structure( - host_files_structure = host_files_structure + if (verbose) { + cli::cli_alert_info( + cli::col_green("I will make use of files structure stored in GitStats.") ) - orgs <- result$orgs - repos <- result$repos - } else { - orgs <- private$orgs - repos <- private$repos } + result <- private$get_orgs_and_repos_from_files_structure( + host_files_structure = host_files_structure + ) + orgs <- result$orgs + repos <- result$repos files_table <- purrr::map(orgs, function(org) { if (verbose) { - user_msg <- if (!is.null(host_files_structure)) { - "Pulling files from files structure" - } else { - glue::glue("Pulling files content: [{paste0(file_path, collapse = ', ')}]") - } show_message( - host = private$host_name, - engine = "graphql", - scope = org, - information = user_msg + host = private$host_name, + engine = "graphql", + scope = org, + information = "Pulling files from files structure" ) } type <- attr(org, "type") %||% "organization" graphql_engine$get_files_from_org( - org = org, - type = type, - repos = repos, - file_paths = file_path, + org = org, + type = type, + repos = repos, host_files_structure = host_files_structure, - only_text_files = only_text_files, - verbose = verbose, - progress = progress - ) %>% + verbose = verbose, + progress = progress + ) |> graphql_engine$prepare_files_table( - org = org, + org = org, file_path = file_path ) - }) %>% - purrr::list_rbind() %>% + }) |> + purrr::list_rbind() |> private$add_repo_api_url() return(files_table) }, @@ -980,43 +1065,92 @@ GitHost <- R6::R6Class( depth, verbose = TRUE, progress = TRUE) { - graphql_engine <- private$engines$graphql - files_structure_list <- purrr::map(private$orgs, function(org) { - if (verbose) { - user_info <- if (!is.null(pattern)) { - glue::glue("Pulling files structure...[files matching pattern: '{pattern}']") - } else { - glue::glue("Pulling files structure...") + if ("org" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + files_structure_list <- purrr::map(private$orgs, function(org) { + if (verbose) { + user_info <- if (!is.null(pattern)) { + glue::glue("Pulling files structure...[files matching pattern: '{pattern}']") + } else { + glue::glue("Pulling files structure...") + } + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = user_info + ) } - show_message( - host = private$host_name, - engine = "graphql", - scope = org, - information = user_info + type <- attr(org, "type") %||% "organization" + graphql_engine$get_files_structure_from_org( + org = org, + type = type, + pattern = pattern, + depth = depth, + verbose = verbose, + progress = progress + ) + }) + names(files_structure_list) <- private$orgs + files_structure_list <- files_structure_list %>% + purrr::discard(~ length(.) == 0) + if (length(files_structure_list) == 0 && verbose) { + cli::cli_alert_warning( + cli::col_yellow( + "For {private$host_name} no files structure found." + ) ) } - type <- attr(org, "type") %||% "organization" - graphql_engine$get_files_structure_from_org( - org = org, - type = type, - repos = private$repos, - pattern = pattern, - depth = depth, - verbose = verbose, - progress = progress + return(files_structure_list) + } + }, + + get_files_structure_from_repos = function(pattern, + depth, + verbose = TRUE, + progress = TRUE) { + if ("repo" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) ) - }) - names(files_structure_list) <- private$orgs - files_structure_list <- files_structure_list %>% - purrr::discard(~ length(.) == 0) - if (length(files_structure_list) == 0 && verbose) { - cli::cli_alert_warning( - cli::col_yellow( - "For {private$host_name} no files structure found." + files_structure_list <- purrr::map(orgs, function(org) { + if (verbose) { + user_info <- if (!is.null(pattern)) { + glue::glue("Pulling files structure...[files matching pattern: '{pattern}']") + } else { + glue::glue("Pulling files structure...") + } + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = user_info + ) + } + type <- attr(org, "type") %||% "organization" + graphql_engine$get_files_structure_from_org( + org = org, + type = type, + repos = private$repos, + pattern = pattern, + depth = depth, + verbose = verbose, + progress = progress ) - ) + }) + names(files_structure_list) <- orgs + files_structure_list <- files_structure_list %>% + purrr::discard(~ length(.) == 0) + if (length(files_structure_list) == 0 && verbose) { + cli::cli_alert_warning( + cli::col_yellow( + "For {private$host_name} no files structure found." + ) + ) + } + return(files_structure_list) } - return(files_structure_list) }, # Pull files from host diff --git a/R/GitHostGitLab.R b/R/GitHostGitLab.R index 8ebbdcc5..e5370724 100644 --- a/R/GitHostGitLab.R +++ b/R/GitHostGitLab.R @@ -21,56 +21,6 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", if (verbose) { cli::cli_alert_success("Set connection to GitLab.") } - }, - - # Retrieve content of given text files from all repositories for a host in - # a table format. - get_files_content = function(file_path, - host_files_structure = NULL, - only_text_files = TRUE, - verbose = TRUE, - progress = TRUE) { - if (!private$scan_all && private$are_non_text_files(file_path, host_files_structure)) { - if (only_text_files) { - files_table <- private$get_files_content_from_orgs( - file_path = file_path, - host_files_structure = host_files_structure, - only_text_files = only_text_files, - verbose = verbose, - progress = progress - ) - } else { - text_files_table <- private$get_files_content_from_orgs( - file_path = file_path, - host_files_structure = host_files_structure, - only_text_files = TRUE, - verbose = verbose, - progress = progress - ) - non_text_files_table <- private$get_files_content_from_orgs_via_rest( - file_path = file_path, - host_files_structure = host_files_structure, - clean_files_content = FALSE, - only_non_text_files = TRUE, - verbose = verbose, - progress = progress - ) - files_table <- purrr::list_rbind( - list( - text_files_table, - non_text_files_table - ) - ) - } - } else { - files_table <- super$get_files_content( - file_path = file_path, - host_files_structure = host_files_structure, - verbose = verbose, - progress = progress - ) - } - return(files_table) } ), private = list( @@ -331,56 +281,82 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", } }, - # Pull files from orgs via rest - get_files_content_from_orgs_via_rest = function(file_path, - host_files_structure, - only_non_text_files, - clean_files_content, - verbose, - progress) { - rest_engine <- private$engines$rest - if (!is.null(host_files_structure)) { - if (verbose) { - cli::cli_alert_info(cli::col_green("I will make use of files structure stored in GitStats.")) - } - result <- private$get_orgs_and_repos_from_files_structure( - host_files_structure = host_files_structure + # Pull files content from organizations + get_files_content_from_repos = function(file_path, + verbose = TRUE, + progress = TRUE) { + if ("repo" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) ) - orgs <- result$orgs - repos <- result$repos - } else { - orgs <- private$orgs - repos <- private$repos + files_table <- purrr::map(orgs, function(org) { + if (verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = glue::glue("Pulling files content: [{paste0(file_path, collapse = ', ')}]") + ) + } + type <- attr(org, "type") %||% "organization" + graphql_engine$get_files_from_org_per_repo( + org = org, + type = type, + repos = private$orgs_repos[[org]], + file_paths = file_path, + verbose = verbose, + progress = progress + ) |> + graphql_engine$prepare_files_table( + org = org, + file_path = file_path + ) + }) |> + purrr::list_rbind() |> + private$add_repo_api_url() + return(files_table) } + }, + + get_files_content_from_files_structure = function(host_files_structure, + verbose = TRUE, + progress = TRUE) { + graphql_engine <- private$engines$graphql if (verbose) { - user_msg <- if (!is.null(host_files_structure)) { - "Pulling files from files structure" - } else { - glue::glue("Pulling files content: [{paste0(file_path, collapse = ', ')}]") - } - show_message( - host = private$host_name, - engine = "rest", - information = user_msg + cli::cli_alert_info( + cli::col_green("I will make use of files structure stored in GitStats.") ) } + result <- private$get_orgs_and_repos_from_files_structure( + host_files_structure = host_files_structure + ) + orgs <- result$orgs + repos <- result$repos files_table <- purrr::map(orgs, function(org) { - if (!is.null(host_files_structure)) { - file_path <- host_files_structure[[org]] %>% unlist(use.names = FALSE) %>% unique() - } - if (only_non_text_files) { - file_path <- file_path[grepl(non_text_files_pattern, file_path)] + if (verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = "Pulling files from files structure" + ) } - files_table <- rest_engine$get_files( - file_paths = file_path, - clean_files_content = clean_files_content, - org = org, - verbose = FALSE, - progress = progress - ) %>% - rest_engine$prepare_files_table() - }, .progress = progress) %>% - purrr::list_rbind() %>% + type <- attr(org, "type") %||% "organization" + graphql_engine$get_files_from_org_per_repo( + org = org, + type = type, + repos = repos, + host_files_structure = host_files_structure, + verbose = verbose, + progress = progress + ) |> + graphql_engine$prepare_files_table( + org = org, + file_path = file_path + ) + }) |> + purrr::list_rbind() |> private$add_repo_api_url() return(files_table) } diff --git a/R/GitStats.R b/R/GitStats.R index a58518b8..58dd8070 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -194,14 +194,12 @@ GitStats <- R6::R6Class( get_files_content = function(file_path = NULL, use_files_structure = TRUE, - only_text_files = TRUE, cache = TRUE, verbose = TRUE, progress = verbose) { private$check_for_host() args_list <- list("file_path" = file_path, - "use_files_structure" = use_files_structure, - "only_text_files" = only_text_files) + "use_files_structure" = use_files_structure) trigger <- private$trigger_pulling( cache = cache, storage = "files", @@ -212,7 +210,6 @@ GitStats <- R6::R6Class( files <- private$get_files_content_from_hosts( file_path = file_path, use_files_structure = use_files_structure, - only_text_files = only_text_files, verbose = verbose, progress = progress ) %>% @@ -707,7 +704,6 @@ GitStats <- R6::R6Class( # Pull content of a text file in a table form get_files_content_from_hosts = function(file_path, use_files_structure, - only_text_files, verbose, progress) { purrr::map(private$hosts, function(host) { @@ -729,11 +725,10 @@ GitStats <- R6::R6Class( NULL } else { host$get_files_content( - file_path = file_path, + file_path = file_path, host_files_structure = host_files_structure, - only_text_files = only_text_files, - verbose = verbose, - progress = progress + verbose = verbose, + progress = progress ) } }) %>% @@ -1021,13 +1016,13 @@ GitStats <- R6::R6Class( print_orgs_and_repos = function() { orgs <- purrr::map(private$hosts, function(host) { host_priv <- environment(host$initialize)$private - if (host_priv$searching_scope == "org") { + if ("org" %in% host_priv$searching_scope) { orgs <- host_priv$orgs } }) repos <- purrr::map(private$hosts, function(host) { host_priv <- environment(host$initialize)$private - if (host_priv$searching_scope == "repo") { + if ("repo" %in% host_priv$searching_scope) { repos <- host_priv$repos_fullnames } }) diff --git a/R/gitstats_functions.R b/R/gitstats_functions.R index fb95e945..f858f05d 100644 --- a/R/gitstats_functions.R +++ b/R/gitstats_functions.R @@ -283,16 +283,15 @@ get_users <- function(gitstats, cache = TRUE, verbose = is_verbose(gitstats)) { gitstats$get_users( - logins = logins, - cache = cache, + logins = logins, + cache = cache, verbose = verbose ) } #' @title Get content of files #' @name get_files_content -#' @description Pull text files content for a given scope (orgs, repos or whole -#' git hosts). +#' @description Pulls content of text files. #' @param gitstats A GitStats object. #' @param file_path Optional. A standardized path to file(s) in repositories. #' May be a character vector if multiple files are to be pulled. If set to @@ -303,9 +302,6 @@ get_users <- function(gitstats, #' `get_files_structure()` function and kept in storage. If there is no #' `files_structure` in storage, an error will be returned. If `file_path` is #' defined, it will override `use_files_structure` parameter. -#' @param only_text_files A logical, `TRUE` by default. If set to `FALSE`, apart -#' from files with text content shows in table output also non-text files with -#' `NA` value for text content. #' @param cache A logical, if set to `TRUE` GitStats will retrieve the last #' result from its storage. #' @param verbose A logical, `TRUE` by default. If `FALSE` messages and printing @@ -324,37 +320,35 @@ get_users <- function(gitstats, #' orgs = "mbtests" #' ) #' get_files_content( -#' gitstats_obj = my_gitstats, +#' gitstats = my_gitstats, #' file_path = c("LICENSE", "DESCRIPTION") #' ) #' #' # example with files structure #' files_structure <- get_files_structure( -#' gitstats_obj = my_gitstats, +#' gitstats = my_gitstats, #' pattern = "\\.Rmd", #' depth = 2L #' ) #' # get_files_content() will make use of pulled earlier files structure #' files_content <- get_files_content( -#' gitstats_obj = my_gitstats +#' gitstats = my_gitstats #' ) #' } #' @return A data.frame. #' @export get_files_content <- function(gitstats, - file_path = NULL, + file_path = NULL, use_files_structure = TRUE, - only_text_files = TRUE, - cache = TRUE, - verbose = is_verbose(gitstats), - progress = verbose) { + cache = TRUE, + verbose = is_verbose(gitstats), + progress = verbose) { gitstats$get_files_content( - file_path = file_path, + file_path = file_path, use_files_structure = use_files_structure, - only_text_files = only_text_files, - cache = cache, - verbose = verbose, - progress = progress + cache = cache, + verbose = verbose, + progress = progress ) } @@ -393,16 +387,16 @@ get_files_content <- function(gitstats, #' @return A list of vectors. #' @export get_files_structure <- function(gitstats, - pattern = NULL, - depth = Inf, - cache = TRUE, - verbose = is_verbose(gitstats), + pattern = NULL, + depth = Inf, + cache = TRUE, + verbose = is_verbose(gitstats), progress = verbose) { gitstats$get_files_structure( - pattern = pattern, - depth = depth, - cache = cache, - verbose = verbose, + pattern = pattern, + depth = depth, + cache = cache, + verbose = verbose, progress = progress ) } diff --git a/R/global.R b/R/global.R index df3db824..26726851 100644 --- a/R/global.R +++ b/R/global.R @@ -7,4 +7,7 @@ globalVariables(c( "contributors_n", "githost" )) -non_text_files_pattern <- "\\.(png||.jpg||.jpeg||.bmp||.gif||.tiff)$" +text_ext_files <- "\\.(txt|md|qmd|Rmd|markdown|yaml|yml|csv|json|xml|html|htm|css|js|r|py|sh|bat|ini|conf|log|sql|tsv|mdx)$" +no_ext_files <- "^[^\\.]+$" + +text_files_pattern <- paste0("(", text_ext_files, "|", no_ext_files, ")") diff --git a/inst/get_files_workflow.R b/inst/get_files_workflow.R index 8b7d2eff..6aa09d4d 100644 --- a/inst/get_files_workflow.R +++ b/inst/get_files_workflow.R @@ -13,6 +13,32 @@ get_files_content( file_path = c("LICENSE", "DESCRIPTION") ) +github_stats <- create_gitstats() |> + set_github_host( + orgs = c("r-world-devs"), + repos = "openpharma/DataFakeR" + ) |> + set_gitlab_host( + repos = "mbtests/graphql_tests" + ) + +github_stats + +get_files_content( + gitstats = github_stats, + file_path = "DESCRIPTION" +) + +datafaker_stats <- create_gitstats() |> + set_github_host( + repos = "openpharma/DataFakeR" + ) + +get_files_content( + gitstats = datafaker_stats, + file_path = "DESCRIPTION" +) + md_files_structure <- get_files_structure( gitstats = test_gitstats, pattern = "\\.md|.R", diff --git a/man/get_files_content.Rd b/man/get_files_content.Rd index 1aa1a830..f2acc20e 100644 --- a/man/get_files_content.Rd +++ b/man/get_files_content.Rd @@ -8,7 +8,6 @@ get_files_content( gitstats, file_path = NULL, use_files_structure = TRUE, - only_text_files = TRUE, cache = TRUE, verbose = is_verbose(gitstats), progress = verbose @@ -28,10 +27,6 @@ will try to pull data from \code{files_structure} (see below).} \code{files_structure} in storage, an error will be returned. If \code{file_path} is defined, it will override \code{use_files_structure} parameter.} -\item{only_text_files}{A logical, \code{TRUE} by default. If set to \code{FALSE}, apart -from files with text content shows in table output also non-text files with -\code{NA} value for text content.} - \item{cache}{A logical, if set to \code{TRUE} GitStats will retrieve the last result from its storage.} @@ -45,8 +40,7 @@ output is switched off.} A data.frame. } \description{ -Pull text files content for a given scope (orgs, repos or whole -git hosts). +Pulls content of text files. } \examples{ \dontrun{ @@ -60,19 +54,19 @@ git hosts). orgs = "mbtests" ) get_files_content( - gitstats_obj = my_gitstats, + gitstats = my_gitstats, file_path = c("LICENSE", "DESCRIPTION") ) # example with files structure files_structure <- get_files_structure( - gitstats_obj = my_gitstats, + gitstats = my_gitstats, pattern = "\\\\.Rmd", depth = 2L ) # get_files_content() will make use of pulled earlier files structure files_content <- get_files_content( - gitstats_obj = my_gitstats + gitstats = my_gitstats ) } } diff --git a/tests/testthat/_snaps/get_files_structure-GitHub.md b/tests/testthat/_snaps/get_files_structure-GitHub.md index 9a747432..1a29ddd2 100644 --- a/tests/testthat/_snaps/get_files_structure-GitHub.md +++ b/tests/testthat/_snaps/get_files_structure-GitHub.md @@ -8,11 +8,9 @@ # when files_structure is empty, appropriate message is returned Code - github_testhost_priv$get_files_structure_from_orgs(pattern = "\\.png", depth = 1L, + github_testhost_priv$get_files_structure_from_repos(pattern = "\\.png", depth = 1L, verbose = TRUE) Message - i [Host:GitHub][Engine:GraphQl][Scope:r-world-devs] Pulling files structure...[files matching pattern: '\.png']... - i [Host:GitHub][Engine:GraphQl][Scope:openpharma] Pulling files structure...[files matching pattern: '\.png']... ! For GitHub no files structure found. Output named list() @@ -20,7 +18,7 @@ # get_files_content makes use of files_structure Code - files_content <- github_testhost_priv$get_files_content_from_orgs(file_path = NULL, + files_content <- github_testhost_priv$get_files_content_from_files_structure( host_files_structure = test_mocker$use("gh_files_structure_from_orgs")) Message i I will make use of files structure stored in GitStats. diff --git a/tests/testthat/_snaps/get_files_structure-GitLab.md b/tests/testthat/_snaps/get_files_structure-GitLab.md index 342d0ead..6f4c26ad 100644 --- a/tests/testthat/_snaps/get_files_structure-GitLab.md +++ b/tests/testthat/_snaps/get_files_structure-GitLab.md @@ -8,7 +8,7 @@ # get_files_content makes use of files_structure Code - files_content <- gitlab_testhost_priv$get_files_content_from_orgs(file_path = NULL, + files_content <- gitlab_testhost_priv$get_files_content_from_files_structure( host_files_structure = test_mocker$use("gl_files_structure_from_orgs")) Message i I will make use of files structure stored in GitStats. diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index e80d3e7f..8d7e1ee8 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -419,20 +419,6 @@ test_fixtures$gitlab_file_repo_response <- list( ) ) -test_fixtures$github_png_file_response <- list( - "data" = list( - "repository" = list( - "repo_id" = "01010101", - "repo_name" = "TestProject", - "repo_url" = "https://github.com/r-world-devs/GitStats", - "file" = list( - "text" = NULL, - "byteSize" = 50L - ) - ) - ) -) - test_fixtures$gitlab_search_response <- list( list( "basename" = "test", diff --git a/tests/testthat/test-get_files_content-GitHub.R b/tests/testthat/test-get_files_content-GitHub.R index ec9183c8..886eb3bb 100644 --- a/tests/testthat/test-get_files_content-GitHub.R +++ b/tests/testthat/test-get_files_content-GitHub.R @@ -47,23 +47,6 @@ test_that("GitHub GraphQL Engine pulls file response", { test_mocker$cache(github_file_response) }) -test_that("GitHub GraphQL Engine pulls png file response", { - mockery::stub( - test_graphql_github_priv$get_file_response, - "self$gql_response", - test_fixtures$github_png_file_response - ) - github_png_file_response <- test_graphql_github_priv$get_file_response( - org = "r-world-devs", - repo = "GitStats", - def_branch = "master", - file_path = "man/figures/logo.png", - files_query = test_mocker$use("gh_file_blobs_from_repo_query") - ) - expect_github_files_response(github_png_file_response) - test_mocker$cache(github_png_file_response) -}) - test_that("get_repositories_with_files works", { mockery::stub( test_graphql_github_priv$get_repositories_with_files, @@ -73,11 +56,10 @@ test_that("get_repositories_with_files works", { gh_repositories_with_files <- test_graphql_github_priv$get_repositories_with_files( repositories = c("test_repo_1", "test_repo_2", "test_repo_3", "test_repo_4", "test_repo_5"), def_branches = c("test_branch_1", "test_branch_2", "test_branch_3", "test_branch_4", "test_branch_5"), - org = "test-org", - file_paths = "test_files", - only_text_files = TRUE, + org = "test_org", + file_paths = "test_files.txt", host_files_structure = NULL, - progress = FALSE + progress = FALSE ) expect_type(gh_repositories_with_files, "list") test_mocker$cache(gh_repositories_with_files) @@ -95,13 +77,12 @@ test_that("GitHub GraphQL Engine pulls files from organization", { test_mocker$use("gh_repositories_with_files") ) github_files_response <- test_graphql_github$get_files_from_org( - org = "test_org", - repos = NULL, - file_paths = "test_files", - only_text_files = TRUE, + org = "test_org", + repos = NULL, + file_paths = "test_files.txt", host_files_structure = NULL, - verbose = FALSE, - progress = FALSE + verbose = FALSE, + progress = FALSE ) expect_github_files_response(github_files_response) test_mocker$cache(github_files_response) @@ -117,44 +98,59 @@ test_that("GitHubHost prepares table from files response", { test_mocker$cache(gh_files_table) }) -test_that("GitHubHost prepares table from files with no content", { - empty_files_response <- test_mocker$use("github_files_response") %>% - purrr::map(function(test_repo) { - test_repo$test_files$file$text <- NULL - return(test_repo) - }) - gh_empty_files_table <- test_graphql_github$prepare_files_table( - files_response = empty_files_response, - org = "test_org", - file_path = "test_files" +test_that("get_files_content_from_orgs for GitHub works", { + mockery::stub( + github_testhost_priv$get_files_content_from_orgs, + "graphql_engine$prepare_files_table", + test_mocker$use("gh_files_table") + ) + github_testhost_priv$searching_scope <- "org" + gh_files_from_orgs <- github_testhost_priv$get_files_content_from_orgs( + file_path = "DESCRIPTION", + verbose = FALSE ) - expect_files_table(gh_empty_files_table) - expect_true(all(is.na(gh_empty_files_table$file_content))) - test_mocker$cache(gh_empty_files_table) + expect_files_table( + gh_files_from_orgs, + with_cols = "api_url" + ) + test_mocker$cache(gh_files_from_orgs) }) -test_that("get_files_content_from_orgs for GitHub works", { +test_that("get_files_content_from_repos for GitHub works", { + test_org <- "test_org" + attr(test_org, "type") <- "organization" mockery::stub( - github_testhost_priv$get_files_content_from_orgs, + github_testhost_priv$get_files_content_from_repos, + "private$set_owner_type", + test_org + ) + mockery::stub( + github_testhost_priv$get_files_content_from_repos, "graphql_engine$prepare_files_table", test_mocker$use("gh_files_table") ) - gh_files_table <- github_testhost_priv$get_files_content_from_orgs( + github_testhost_priv$searching_scope <- "repo" + gh_files_from_repos <- github_testhost_priv$get_files_content_from_repos( file_path = "DESCRIPTION", verbose = FALSE ) expect_files_table( - gh_files_table, + gh_files_from_repos, with_cols = "api_url" ) - test_mocker$cache(gh_files_table) + test_mocker$cache(gh_files_from_repos) }) test_that("`get_files_content()` pulls files in the table format", { mockery::stub( github_testhost$get_files_content, "private$get_files_content_from_orgs", - test_mocker$use("gh_files_table") + test_mocker$use("gh_files_from_orgs") + ) + mockery::stub( + github_testhost$get_files_content, + "private$get_files_content_from_repos", + test_mocker$use("gh_files_from_repos") ) gh_files_table <- github_testhost$get_files_content( file_path = "DESCRIPTION" diff --git a/tests/testthat/test-get_files_content-GitLab.R b/tests/testthat/test-get_files_content-GitLab.R index 1c84606a..3aeeff65 100644 --- a/tests/testthat/test-get_files_content-GitLab.R +++ b/tests/testthat/test-get_files_content-GitLab.R @@ -78,15 +78,16 @@ test_that("GitLab GraphQL Engine pulls files from org by iterating over repos", "private$get_file_blobs_response", test_mocker$use("gl_file_blobs_response") ) - gl_files_from_org <- test_graphql_gitlab$get_files_from_org_per_repo( - org = "mbtests", - repos = "graphql_tests", + gl_files_from_org_per_repo <- test_graphql_gitlab$get_files_from_org_per_repo( + org = "test_org", + repos = "TestProject", file_paths = c("project_metadata.yaml", "README.md") ) expect_gitlab_files_from_org_by_repos_response( - response = gl_files_from_org, + response = gl_files_from_org_per_repo, expected_files = c("project_metadata.yaml", "README.md") ) + test_mocker$cache(gl_files_from_org_per_repo) }) test_that("is query error is FALSE when response is empty (non query error)", { @@ -111,11 +112,16 @@ test_that("Gitlab GraphQL switches to pulling files per repositories when query "private$is_complexity_error", TRUE ) + mockery::stub( + test_graphql_gitlab$get_files_from_org, + "self$get_files_from_org_per_repo", + test_mocker$use("gl_files_from_org_per_repo") + ) gitlab_files_response_by_repos <- test_graphql_gitlab$get_files_from_org( org = "mbtests", type = "organization", repos = NULL, - file_paths = c("DESCRIPTION", "project_metadata.yaml", "README.md"), + file_paths = c("project_metadata.yaml", "README.md"), host_files_structure = NULL, only_text_files = TRUE, verbose = FALSE, @@ -123,7 +129,7 @@ test_that("Gitlab GraphQL switches to pulling files per repositories when query ) expect_gitlab_files_from_org_by_repos_response( response = gitlab_files_response_by_repos, - expected_files = c("DESCRIPTION", "project_metadata.yaml", "README.md") + expected_files = c("project_metadata.yaml", "README.md") ) test_mocker$cache(gitlab_files_response_by_repos) }) @@ -144,8 +150,7 @@ test_that("checker properly identifies gitlab files responses", { test_that("GitLab prepares table from files response", { gl_files_table <- test_graphql_gitlab$prepare_files_table( files_response = test_mocker$use("gitlab_files_response"), - org = "mbtests", - file_path = "meta_data.yaml" + org = "mbtests" ) expect_files_table(gl_files_table) test_mocker$cache(gl_files_table) @@ -154,8 +159,7 @@ test_that("GitLab prepares table from files response", { test_that("GitLab prepares table from files response prepared in alternative way", { gl_files_table <- test_graphql_gitlab$prepare_files_table( files_response = test_mocker$use("gitlab_files_response_by_repos"), - org = "mbtests", - file_path = "meta_data.yaml" + org = "mbtests" ) expect_files_table(gl_files_table) }) @@ -177,16 +181,3 @@ test_that("get_files_content_from_orgs for GitLab works", { ) test_mocker$cache(gl_files_table) }) - -test_that("`get_files_content()` pulls files in the table format", { - mockery::stub( - gitlab_testhost$get_files_content, - "super$get_files_content", - test_mocker$use("gl_files_table") - ) - gl_files_table <- gitlab_testhost$get_files_content( - file_path = "README.md" - ) - expect_files_table(gl_files_table, with_cols = "api_url") - test_mocker$cache(gl_files_table) -}) diff --git a/tests/testthat/test-get_files_structure-GitHub.R b/tests/testthat/test-get_files_structure-GitHub.R index 51dfad59..ed0ba9f8 100644 --- a/tests/testthat/test-get_files_structure-GitHub.R +++ b/tests/testthat/test-get_files_structure-GitHub.R @@ -163,6 +163,7 @@ test_that("get_files_structure_from_orgs", { "graphql_engine$get_files_structure_from_org", test_mocker$use("gh_md_files_structure") ) + github_testhost_priv$searching_scope <- "org" gh_files_structure_from_orgs <- github_testhost_priv$get_files_structure_from_orgs( pattern = "\\.md|\\.qmd|\\.Rmd", depth = Inf, @@ -196,13 +197,14 @@ test_that("when files_structure is empty, appropriate message is returned", { mode = "private" ) mockery::stub( - github_testhost_priv$get_files_structure_from_orgs, + github_testhost_priv$get_files_structure_from_repos, "graphql_engine$get_files_structure_from_org", list() |> purrr::set_names() ) + github_testhost_priv$searching_scope <- "repo" expect_snapshot( - github_testhost_priv$get_files_structure_from_orgs( + github_testhost_priv$get_files_structure_from_repos( pattern = "\\.png", depth = 1L, verbose = TRUE @@ -249,13 +251,12 @@ test_that("get_files_structure pulls files structure for repositories in orgs", test_that("get_files_content makes use of files_structure", { mockery::stub( - github_testhost_priv$get_files_content_from_orgs, + github_testhost_priv$get_files_content_from_files_structure, "private$add_repo_api_url", test_mocker$use("gh_files_table") ) expect_snapshot( - files_content <- github_testhost_priv$get_files_content_from_orgs( - file_path = NULL, + files_content <- github_testhost_priv$get_files_content_from_files_structure( host_files_structure = test_mocker$use("gh_files_structure_from_orgs") ) ) diff --git a/tests/testthat/test-get_files_structure-GitLab.R b/tests/testthat/test-get_files_structure-GitLab.R index e2bf6450..1a8e3401 100644 --- a/tests/testthat/test-get_files_structure-GitLab.R +++ b/tests/testthat/test-get_files_structure-GitLab.R @@ -229,13 +229,12 @@ test_that("get_files_structure pulls files structure for repositories in orgs", test_that("get_files_content makes use of files_structure", { mockery::stub( - gitlab_testhost_priv$get_files_content_from_orgs, + gitlab_testhost_priv$get_files_content_from_files_structure, "private$add_repo_api_url", test_mocker$use("gl_files_table") ) expect_snapshot( - files_content <- gitlab_testhost_priv$get_files_content_from_orgs( - file_path = NULL, + files_content <- gitlab_testhost_priv$get_files_content_from_files_structure( host_files_structure = test_mocker$use("gl_files_structure_from_orgs") ) ) From 8eb451650aee0b4c15f2d246be9ff59fe57a02d6 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 17 Dec 2024 13:25:34 +0000 Subject: [PATCH 12/27] Fixes. --- R/EngineGraphQL.R | 5 +--- R/EngineGraphQLGitHub.R | 2 +- R/EngineGraphQLGitLab.R | 2 -- R/EngineRestGitHub.R | 29 +++++++++++++++++++ R/GitHost.R | 11 +++---- R/GitHostGitHub.R | 29 ------------------- R/GitHostGitLab.R | 6 ++-- R/gitstats_functions.R | 2 +- man/get_files_structure.Rd | 2 +- .../testthat/test-get_files_content-GitHub.R | 3 +- .../test-get_files_structure-GitHub.R | 1 - .../test-get_files_structure-GitLab.R | 1 - 12 files changed, 40 insertions(+), 53 deletions(-) diff --git a/R/EngineGraphQL.R b/R/EngineGraphQL.R index c462180d..99d7f7aa 100644 --- a/R/EngineGraphQL.R +++ b/R/EngineGraphQL.R @@ -75,7 +75,6 @@ EngineGraphQL <- R6::R6Class( }, get_path_from_files_structure = function(host_files_structure, - only_text_files, org, repo = NULL) { if (is.null(repo)) { @@ -85,9 +84,7 @@ EngineGraphQL <- R6::R6Class( } else { file_path <- host_files_structure[[org]][[repo]] } - if (only_text_files) { - file_path <- file_path[grepl(text_files_pattern, file_path)] - } + file_path <- file_path[grepl(text_files_pattern, file_path)] return(file_path) } ) diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index fd596d15..19220e49 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -202,7 +202,7 @@ EngineGraphQLGitHub <- R6::R6Class( }, # Prepare files table. - prepare_files_table = function(files_response, org, file_path) { + prepare_files_table = function(files_response, org) { if (!is.null(files_response)) { files_table <- purrr::map(files_response, function(repository) { purrr::imap(repository, function(file_data, file_name) { diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index 7b23c6e6..1c3e60f4 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -136,7 +136,6 @@ EngineGraphQLGitLab <- R6::R6Class( if (!is.null(host_files_structure)) { file_paths <- private$get_path_from_files_structure( host_files_structure = host_files_structure, - only_text_files = only_text_files, org = org ) } else if (only_text_files) { @@ -250,7 +249,6 @@ EngineGraphQLGitLab <- R6::R6Class( if (!is.null(host_files_structure)) { file_paths <- private$get_path_from_files_structure( host_files_structure = host_files_structure, - only_text_files = only_text_files, org = org, repo = repo ) diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R index 86ec5e91..2b07e4eb 100644 --- a/R/EngineRestGitHub.R +++ b/R/EngineRestGitHub.R @@ -24,6 +24,28 @@ EngineRestGitHub <- R6::R6Class( return(files_list) }, + # Prepare files table from REST API. + prepare_files_table = function(files_list) { + files_table <- NULL + if (!is.null(files_list)) { + files_table <- purrr::map(files_list, function(file_data) { + repo_fullname <- private$get_repo_fullname(file_data$url) + org_repo <- stringr::str_split_1(repo_fullname, "/") + data.frame( + "repo_name" = org_repo[2], + "repo_id" = NA_character_, + "organization" = org_repo[1], + "file_path" = file_data$path, + "file_content" = file_data$content, + "file_size" = file_data$size, + "repo_url" = private$set_repo_url(file_data$url) + ) + }) %>% + purrr::list_rbind() + } + return(files_table) + }, + # Pulling repositories where code appears get_repos_by_code = function(code, org = NULL, @@ -302,6 +324,13 @@ EngineRestGitHub <- R6::R6Class( purrr::map(search_result, ~ self$response(.$url), .progress = glue::glue("Adding file [{filename}] info...")) %>% unique() + }, + + # Get repository full name + get_repo_fullname = function(file_url) { + stringr::str_remove_all(file_url, + paste0(private$endpoints$repositories, "/")) %>% + stringr::str_replace_all("/contents.*", "") } ) ) diff --git a/R/GitHost.R b/R/GitHost.R index 023f2a99..493bb4f5 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -963,8 +963,7 @@ GitHost <- R6::R6Class( progress = progress ) |> graphql_engine$prepare_files_table( - org = org, - file_path = file_path + org = org ) }) |> purrr::list_rbind() |> @@ -1001,8 +1000,7 @@ GitHost <- R6::R6Class( progress = progress ) |> graphql_engine$prepare_files_table( - org = org, - file_path = file_path + org = org ) }) |> purrr::list_rbind() |> @@ -1044,8 +1042,7 @@ GitHost <- R6::R6Class( progress = progress ) |> graphql_engine$prepare_files_table( - org = org, - file_path = file_path + org = org ) }) |> purrr::list_rbind() |> @@ -1170,7 +1167,7 @@ GitHost <- R6::R6Class( verbose = verbose, progress = progress ) %>% - private$prepare_files_table_from_rest() %>% + rest_engine$prepare_files_table() %>% private$add_repo_api_url() return(files_table) }, diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index 71e1c42a..84e0b1d7 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -252,35 +252,6 @@ GitHostGitHub <- R6::R6Class( return(repos_names) }, - # Prepare files table from REST API. - prepare_files_table_from_rest = function(files_list) { - files_table <- NULL - if (!is.null(files_list)) { - files_table <- purrr::map(files_list, function(file_data) { - repo_fullname <- private$get_repo_fullname(file_data$url) - org_repo <- stringr::str_split_1(repo_fullname, "/") - data.frame( - "repo_name" = org_repo[2], - "repo_id" = NA_character_, - "organization" = org_repo[1], - "file_path" = file_data$path, - "file_content" = file_data$content, - "file_size" = file_data$size, - "repo_url" = private$set_repo_url(file_data$url) - ) - }) %>% - purrr::list_rbind() - } - return(files_table) - }, - - # Get repository full name - get_repo_fullname = function(file_url) { - stringr::str_remove_all(file_url, - paste0(private$endpoints$repositories, "/")) %>% - stringr::str_replace_all("/contents.*", "") - }, - # Get repository url set_repo_url = function(repo_fullname) { paste0(private$endpoints$repositories, "/", repo_fullname) diff --git a/R/GitHostGitLab.R b/R/GitHostGitLab.R index e5370724..ff4f041d 100644 --- a/R/GitHostGitLab.R +++ b/R/GitHostGitLab.R @@ -309,8 +309,7 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", progress = progress ) |> graphql_engine$prepare_files_table( - org = org, - file_path = file_path + org = org ) }) |> purrr::list_rbind() |> @@ -352,8 +351,7 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", progress = progress ) |> graphql_engine$prepare_files_table( - org = org, - file_path = file_path + org = org ) }) |> purrr::list_rbind() |> diff --git a/R/gitstats_functions.R b/R/gitstats_functions.R index f858f05d..e38fce10 100644 --- a/R/gitstats_functions.R +++ b/R/gitstats_functions.R @@ -380,7 +380,7 @@ get_files_content <- function(gitstats, #' orgs = "mbtests" #' ) #' get_files_structure( -#' gitstats_obj = my_gitstats, +#' gitstats = my_gitstats, #' pattern = "\\.md" #' ) #' } diff --git a/man/get_files_structure.Rd b/man/get_files_structure.Rd index f02be215..db33b9a4 100644 --- a/man/get_files_structure.Rd +++ b/man/get_files_structure.Rd @@ -51,7 +51,7 @@ Pulls file structure for a given repository. orgs = "mbtests" ) get_files_structure( - gitstats_obj = my_gitstats, + gitstats = my_gitstats, pattern = "\\\\.md" ) } diff --git a/tests/testthat/test-get_files_content-GitHub.R b/tests/testthat/test-get_files_content-GitHub.R index 886eb3bb..54c896ce 100644 --- a/tests/testthat/test-get_files_content-GitHub.R +++ b/tests/testthat/test-get_files_content-GitHub.R @@ -91,8 +91,7 @@ test_that("GitHub GraphQL Engine pulls files from organization", { test_that("GitHubHost prepares table from files response", { gh_files_table <- test_graphql_github$prepare_files_table( files_response = test_mocker$use("github_files_response"), - org = "r-world-devs", - file_path = "LICENSE" + org = "r-world-devs" ) expect_files_table(gh_files_table) test_mocker$cache(gh_files_table) diff --git a/tests/testthat/test-get_files_structure-GitHub.R b/tests/testthat/test-get_files_structure-GitHub.R index ed0ba9f8..0c300350 100644 --- a/tests/testthat/test-get_files_structure-GitHub.R +++ b/tests/testthat/test-get_files_structure-GitHub.R @@ -220,7 +220,6 @@ test_that("get_path_from_files_structure gets file path from files structure", { test_graphql_github <- environment(test_graphql_github$initialize)$private file_path <- test_graphql_github$get_path_from_files_structure( host_files_structure = test_mocker$use("gh_files_structure_from_orgs"), - only_text_files = FALSE, org = "test_org", repo = "TestRepo" ) diff --git a/tests/testthat/test-get_files_structure-GitLab.R b/tests/testthat/test-get_files_structure-GitLab.R index 1a8e3401..64f08825 100644 --- a/tests/testthat/test-get_files_structure-GitLab.R +++ b/tests/testthat/test-get_files_structure-GitLab.R @@ -198,7 +198,6 @@ test_that("get_path_from_files_structure gets file path from files structure", { test_graphql_gitlab <- environment(test_graphql_gitlab$initialize)$private file_path <- test_graphql_gitlab$get_path_from_files_structure( host_files_structure = test_mocker$use("gl_files_structure_from_orgs"), - only_text_files = TRUE, org = "mbtests" # this will need fixing and repo parameter must come back ) expect_equal(typeof(file_path), "character") From 831b15bb5678617fa96dadff27ff3e52140b4d49 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 19 Dec 2024 08:37:12 +0000 Subject: [PATCH 13/27] Lint. --- R/EngineGraphQLGitHub.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index 19220e49..64a52c1b 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -49,7 +49,7 @@ EngineGraphQLGitHub <- R6::R6Class( while (next_page) { repos_response <- private$get_repos_page( login = org, - type = type, + type = type, repo_cursor = repo_cursor ) repositories <- if (type == "organization") { From d6f271b05e6d850b412c417d78d416cbe49a7360 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 19 Dec 2024 08:38:33 +0000 Subject: [PATCH 14/27] Fix printing ID of the GitLab repositories in table, style to make code prettier. --- R/EngineGraphQLGitLab.R | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index 1c3e60f4..a4010946 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -82,23 +82,24 @@ EngineGraphQLGitLab <- R6::R6Class( if (length(repos_list) > 0) { repos_table <- purrr::map(repos_list, function(repo) { repo <- repo$node - repo$default_branch <- repo$repository$rootRef %||% "" + repo[["repo_id"]] <- sub(".*/(\\d+)$", "\\1", repo$repo_id) + repo[["default_branch"]] <- repo$repository$rootRef %||% "" repo$repository <- NULL - repo$languages <- if (length(repo$languages) > 0) { + repo[["languages"]] <- if (length(repo$languages) > 0) { purrr::map_chr(repo$languages, ~ .$name) %>% paste0(collapse = ", ") } else { "" } - repo$created_at <- gts_to_posixt(repo$created_at) - repo$issues_open <- repo$issues$opened - repo$issues_closed <- repo$issues$closed + repo[["created_at"]] <- gts_to_posixt(repo$created_at) + repo[["issues_open"]] <- repo$issues$opened + repo[["issues_closed"]] <- repo$issues$closed repo$issues <- NULL - repo$last_activity_at <- as.POSIXct(repo$last_activity_at) - repo$organization <- repo$namespace$path + repo[["last_activity_at"]] <- as.POSIXct(repo$last_activity_at) + repo[["organization"]] <- repo$namespace$path repo$namespace <- NULL repo$repo_path <- NULL # temporary to close issue 338 - data.frame(repo) + return(data.frame(repo)) }) %>% purrr::list_rbind() %>% dplyr::relocate( From 22f077b9110bee381ba7b21a3831ce6e6c2b4023 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 19 Dec 2024 08:40:37 +0000 Subject: [PATCH 15/27] Apply searching for code both by orgs and repositories. --- R/EngineRestGitHub.R | 131 ++++++++++++++++----- R/EngineRestGitLab.R | 100 ++++++++++++---- R/GitHost.R | 263 +++++++++++++++++++++++++++++++------------ R/GitHostGitHub.R | 6 +- R/GitHostGitLab.R | 5 +- R/GitStats.R | 17 ++- 6 files changed, 396 insertions(+), 126 deletions(-) diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R index 2b07e4eb..0add30e4 100644 --- a/R/EngineRestGitHub.R +++ b/R/EngineRestGitHub.R @@ -48,33 +48,34 @@ EngineRestGitHub <- R6::R6Class( # Pulling repositories where code appears get_repos_by_code = function(code, - org = NULL, - filename = NULL, - in_path = FALSE, - output = "table_full", - verbose = TRUE, - progress = TRUE) { - user_query <- if (!is.null(org)) { - paste0('+user:', org) - } else { - '' - } - query <- if (!in_path) { - paste0('"', code, '"', user_query) - } else { - paste0('"', code, '"+in:path', user_query) - } - if (!is.null(filename)) { - query <- paste0(query, '+in:file+filename:', filename) + org = NULL, + repos = NULL, + filename = NULL, + in_path = FALSE, + output = "table_full", + verbose = TRUE, + progress = TRUE) { + if (!is.null(org)) { + search_result <- private$search_for_code( + code = code, + org = org, + filename = filename, + in_path = in_path, + verbose = verbose, + progress = progress + ) } - search_endpoint <- paste0(private$endpoints[["search"]], query) - if (verbose) cli::cli_alert_info("Searching for code [{code}]...") - total_n <- self$response(search_endpoint)[["total_count"]] - if (length(total_n) > 0) { - search_result <- private$search_response( - search_endpoint = search_endpoint, - total_n = total_n + if (!is.null(repos)) { + search_result <- private$search_repos_for_code( + code = code, + repos = repos, + filename = filename, + in_path = in_path, + verbose = verbose, + progress = progress ) + } + if (length(search_result) > 0) { if (output == "table_full" || output == "table_min") { search_output <- private$map_search_into_repos( search_response = search_result, @@ -129,8 +130,14 @@ EngineRestGitHub <- R6::R6Class( #' Pull all repositories URLS from organization get_repos_urls = function(type, org, repos) { - repos_response <- self$response( - endpoint = paste0(private$endpoints[["organizations"]], org, "/repos") + owner_type <- attr(org, "type") %||% "organization" + if (owner_type == "user") { + repo_endpoint <- paste0(private$endpoints[["users"]], org, "/repos") + } else { + repo_endpoint <- paste0(private$endpoints[["organizations"]], org, "/repos") + } + repos_response <- private$paginate_results( + endpoint = repo_endpoint ) if (!is.null(repos)) { repos_response <- repos_response %>% @@ -216,12 +223,82 @@ EngineRestGitHub <- R6::R6Class( self$rest_api_url, "/orgs/" ) + private$endpoints[["users"]] <- paste0( + self$rest_api_url, + "/users/" + ) private$endpoints[["repositories"]] <- paste0( self$rest_api_url, "/repos/" ) }, + search_for_code = function(code, + org, + filename, + in_path, + verbose, + progress) { + user_query <- if (!is.null(org)) { + paste0('+user:', org) + } else { + '' + } + query <- if (!in_path) { + paste0('"', code, '"', user_query) + } else { + paste0('"', code, '"+in:path', user_query) + } + if (!is.null(filename)) { + query <- paste0(query, '+in:file+filename:', filename) + } + search_endpoint <- paste0(private$endpoints[["search"]], query) + if (verbose) cli::cli_alert_info("Searching for code [{code}]...") + total_n <- self$response(search_endpoint)[["total_count"]] + search_result <- if (length(total_n) > 0) { + private$search_response( + search_endpoint = search_endpoint, + total_n = total_n + ) + } else { + list() + } + return(search_result) + }, + + search_repos_for_code = function(code, + repos, + filename, + in_path, + verbose, + progress) { + if (verbose) cli::cli_alert_info("Searching for code [{code}]...") + search_result <- purrr::map(repos, function(repo) { + repo_query <- paste0('+repo:', repo) + query <- if (!in_path) { + paste0('"', code, '"', repo_query) + } else { + paste0('"', code, '"+in:path', repo_query) + } + if (!is.null(filename)) { + query <- paste0(query, '+in:file+filename:', filename) + } + search_endpoint <- paste0(private$endpoints[["search"]], query) + total_n <- self$response(search_endpoint)[["total_count"]] + result <- if (length(total_n) > 0) { + private$search_response( + search_endpoint = search_endpoint, + total_n = total_n + ) + } else { + list() + } + return(result) + }) |> + purrr::list_flatten() + return(search_result) + }, + # A wrapper for proper pagination of GitHub search REST API # @param search_endpoint A character, a search endpoint # @param total_n Number of results diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index c7887a14..791e7b1e 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -62,19 +62,31 @@ EngineRestGitLab <- R6::R6Class( # filtering by language. For more information look here: # https://gitlab.com/gitlab-org/gitlab/-/issues/340333 get_repos_by_code = function(code, - org = NULL, + org = NULL, + repos = NULL, filename = NULL, - in_path = FALSE, - output = "table_full", - verbose = TRUE, + in_path = FALSE, + output = "table_full", + verbose = TRUE, progress = TRUE) { - search_response <- private$search_for_code( - code = code, - filename = filename, - in_path = in_path, - org = org, - verbose = verbose - ) + if (!is.null(org)) { + search_response <- private$search_for_code( + code = code, + filename = filename, + in_path = in_path, + org = utils::URLencode(org, reserved = TRUE), + verbose = verbose + ) + } + if (!is.null(repos)) { + search_response <- private$search_repos_for_code( + code = code, + filename = filename, + in_path = in_path, + repos = repos, + verbose = verbose + ) + } if (output == "raw") { search_output <- search_response } else if (output == "table_full" || output == "table_min") { @@ -319,21 +331,25 @@ EngineRestGitLab <- R6::R6Class( # Set search endpoint set_search_endpoint = function(org = NULL) { - groups_search <- if (!private$scan_all) { - private$set_groups_search_endpoint(org) + scope_endpoint <- if (!is.null(org)) { + paste0("/groups/", private$get_group_id(org)) } else { "" } - private$endpoints[["search"]] <- paste0( + paste0( self$rest_api_url, - groups_search, + scope_endpoint, "/search?scope=blobs&search=" ) }, - # set groups search endpoint - set_groups_search_endpoint = function(org) { - paste0("/groups/", private$get_group_id(org)) + set_projects_search_endpoint = function(repo) { + paste0( + self$rest_api_url, + "/projects/", + utils::URLencode(repo, reserved = TRUE), + "/search?scope=blobs&search=" + ) }, # Iterator over pulling pages of repositories. @@ -359,7 +375,7 @@ EngineRestGitLab <- R6::R6Class( page <- 1 still_more_hits <- TRUE full_repos_list <- list() - private$set_search_endpoint(org) + search_endpoint <- private$set_search_endpoint(org) if (verbose) cli::cli_alert_info("Searching for code [{code}]...") if (!in_path) { query <- paste0("%22", code, "%22") @@ -372,7 +388,7 @@ EngineRestGitLab <- R6::R6Class( while (still_more_hits | page < page_max) { search_result <- self$response( paste0( - private$endpoints[["search"]], + search_endpoint, query, "&per_page=100&page=", page @@ -389,11 +405,53 @@ EngineRestGitLab <- R6::R6Class( return(full_repos_list) }, + search_repos_for_code = function(code, + repos, + filename = NULL, + in_path = FALSE, + page_max = 1e6, + verbose = TRUE) { + if (verbose) cli::cli_alert_info("Searching for code [{code}]...") + if (!in_path) { + query <- paste0("%22", code, "%22") + } else { + query <- paste0("path:", code) + } + if (!is.null(filename)) { + query <- paste0(query, "%20filename:", filename) + } + search_response <- purrr::map(repos, function(repo) { + page <- 1 + still_more_hits <- TRUE + full_repos_list <- list() + search_endpoint <- private$set_projects_search_endpoint(repo) + while (still_more_hits | page < page_max) { + search_result <- self$response( + paste0( + search_endpoint, + query, + "&per_page=100&page=", + page + ) + ) + if (length(search_result) == 0) { + still_more_hits <- FALSE + break() + } else { + full_repos_list <- append(full_repos_list, search_result) + page <- page + 1 + } + } + return(full_repos_list) + }) |> + purrr::list_flatten() + return(search_response) + }, + # Parse search response into repositories output map_search_into_repos = function(search_response, progress) { repos_ids <- purrr::map_chr(search_response, ~ as.character(.$project_id)) %>% unique() - repos_list <- purrr::map(repos_ids, function(repo_id) { content <- self$response( endpoint = paste0(private$endpoints[["projects"]], repo_id) diff --git a/R/GitHost.R b/R/GitHost.R index 493bb4f5..3f6a496e 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -86,38 +86,32 @@ GitHost <- R6::R6Class( }, # Get repositories URLS from the Git host - get_repos_urls = function(type = "web", + get_repos_urls = function(type = "web", with_code = NULL, - in_files = NULL, + in_files = NULL, with_file = NULL, - verbose = TRUE, - progress = TRUE) { + verbose = TRUE, + progress = TRUE) { if (!is.null(with_code)) { - repo_urls <- private$get_repos_with_code( - code = with_code, + repo_urls <- private$get_repos_urls_with_code( + type = type, + code = with_code, in_files = in_files, - output = "raw", - verbose = verbose - ) %>% - private$get_repo_url_from_response( - type = type, - progress = progress - ) + verbose = verbose, + progress = progress + ) } else if (!is.null(with_file)) { - repo_urls <- private$get_repos_with_code( - code = with_file, + repo_urls <- private$get_repos_urls_with_code( + type = type, + code = with_file, in_path = TRUE, - output = "raw", - verbose = verbose - ) %>% - private$get_repo_url_from_response( - type = type, - progress = progress - ) + verbose = verbose, + progress = progress + ) } else { repo_urls <- private$get_all_repos_urls( - type = type, - verbose = verbose, + type = type, + verbose = verbose, progress = progress ) } @@ -453,7 +447,7 @@ GitHost <- R6::R6Class( repo <- NULL } return(repo) - }) %>% + }, .progress = verbose) %>% purrr::keep(~ length(.) > 0) %>% unlist() if (length(repos) == 0) { @@ -668,7 +662,7 @@ GitHost <- R6::R6Class( type = type ) |> graphql_engine$prepare_repos_table() |> - dplyr::filter(repo_name == private$orgs_repos[[org]]) + dplyr::filter(repo_name %in% private$orgs_repos[[org]]) return(repos_table) }, .progress = progress) |> purrr::list_rbind() @@ -677,11 +671,11 @@ GitHost <- R6::R6Class( # Pull repositories with specific code get_repos_with_code = function(code, - in_files = NULL, - in_path = FALSE, - output = "table_full", - verbose = TRUE, - progress = TRUE) { + in_files = NULL, + in_path = FALSE, + output = "table_full", + verbose = TRUE, + progress = TRUE) { if (private$scan_all) { repos_table <- private$get_repos_with_code_from_host( code = code, @@ -693,7 +687,15 @@ GitHost <- R6::R6Class( ) } if (!private$scan_all) { - repos_table <- private$get_repos_with_code_from_orgs( + repos_from_org <- private$get_repos_with_code_from_orgs( + code = code, + in_files = in_files, + in_path = in_path, + output = output, + verbose = verbose, + progress = progress + ) + repos_from_repos <- private$get_repos_with_code_from_repos( code = code, in_files = in_files, in_path = in_path, @@ -701,6 +703,7 @@ GitHost <- R6::R6Class( verbose = verbose, progress = progress ) + repos_table <- rbind(repos_from_org, repos_from_repos) } return(repos_table) }, @@ -710,8 +713,8 @@ GitHost <- R6::R6Class( if (private$scan_all && is.null(private$orgs)) { if (verbose) { show_message( - host = private$host_name, - engine = "graphql", + host = private$host_name, + engine = "graphql", information = "Pulling all organizations" ) } @@ -741,15 +744,15 @@ GitHost <- R6::R6Class( org <- utils::URLdecode(org) if (!private$scan_all && verbose) { show_message( - host = private$host_name, - engine = "rest", - scope = org, + host = private$host_name, + engine = "rest", + scope = org, information = "Pulling repositories (URLs)" ) } repos_urls <- rest_engine$get_repos_urls( type = type, - org = org, + org = org, repos = NULL ) return(repos_urls) @@ -761,20 +764,21 @@ GitHost <- R6::R6Class( get_repos_urls_from_repos = function(type, verbose, progress) { if ("repo" %in% private$searching_scope) { rest_engine <- private$engines$rest - orgs <- names(private$orgs_repos) + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) + ) repos_vector <- purrr::map(orgs, function(org) { - org <- utils::URLdecode(org) if (!private$scan_all && verbose) { show_message( - host = private$host_name, - engine = "rest", - scope = org, + host = private$host_name, + engine = "rest", + scope = org, information = "Pulling repositories (URLs)" ) } repos_urls <- rest_engine$get_repos_urls( type = type, - org = org, + org = org, repos = private$orgs_repos[[org]] ) return(repos_urls) @@ -847,66 +851,185 @@ GitHost <- R6::R6Class( output = "table_full", verbose = TRUE, progress = TRUE) { - repos_list <- purrr::map(private$orgs, function(org) { + if ("org" %in% private$searching_scope) { + repos_list <- purrr::map(private$orgs, function(org) { + if (verbose) { + show_message( + host = private$host_name, + engine = "rest", + scope = utils::URLdecode(org), + code = code, + information = "Pulling repositories" + ) + } + rest_engine <- private$engines$rest + if (is.null(in_files)) { + repos_response <- rest_engine$get_repos_by_code( + org = org, + code = code, + in_path = in_path, + output = output, + verbose = verbose, + progress = progress + ) + } else { + repos_response <- purrr::map(in_files, function(filename) { + rest_engine$get_repos_by_code( + org = org, + code = code, + filename = filename, + in_path = in_path, + output = output, + verbose = verbose, + progress = progress + ) + }) %>% + purrr::list_flatten() + } + if (output != "raw") { + repos_table <- repos_response %>% + rest_engine$tailor_repos_response( + output = output + ) %>% + rest_engine$prepare_repos_table( + output = output, + verbose = verbose + ) + if (output == "table_full") { + repos_table <- repos_table %>% + rest_engine$get_repos_issues( + progress = progress + ) + } + return(repos_table) + } else { + return(repos_response) + } + }, .progress = progress) + if (output != "raw") { + repos_output <- purrr::list_rbind(repos_list) + } else { + repos_output <- purrr::list_flatten(repos_list) + } + return(repos_output) + } + }, + + # Pull repositories with code from given organizations + get_repos_with_code_from_repos = function(code, + in_files = NULL, + in_path = FALSE, + output = "table_full", + verbose = TRUE, + progress = TRUE) { + orgs <- names(private$orgs_repos) + if ("repo" %in% private$searching_scope) { if (verbose) { show_message( host = private$host_name, engine = "rest", - scope = utils::URLdecode(org), + scope = utils::URLdecode(paste0(orgs, collapse = "|")), code = code, information = "Pulling repositories" ) } rest_engine <- private$engines$rest if (is.null(in_files)) { - repos_response <- rest_engine$get_repos_by_code( - org = org, - code = code, - in_path = in_path, - output = output, - verbose = verbose, + repos_output <- rest_engine$get_repos_by_code( + repos = private$repos_fullnames, + code = code, + in_path = in_path, + output = output, + verbose = verbose, progress = progress ) } else { - repos_response <- purrr::map(in_files, function(filename) { + repos_output <- purrr::map(in_files, function(filename) { rest_engine$get_repos_by_code( - org = org, - code = code, + repos = private$repos_fullnames, + code = code, filename = filename, - in_path = in_path, - output = output, - verbose = verbose, + in_path = in_path, + output = output, + verbose = verbose, progress = progress ) }) %>% purrr::list_flatten() } if (output != "raw") { - repos_table <- repos_response %>% + repos_output <- repos_output %>% rest_engine$tailor_repos_response( output = output ) %>% rest_engine$prepare_repos_table( - output = output, + output = output, verbose = verbose ) if (output == "table_full") { - repos_table <- repos_table %>% + repos_output <- repos_output %>% rest_engine$get_repos_issues( progress = progress ) } - return(repos_table) - } else { - return(repos_response) } - }, .progress = progress) - if (output != "raw") { - repos_output <- purrr::list_rbind(repos_list) - } else { - repos_output <- purrr::list_flatten(repos_list) + return(repos_output) } - return(repos_output) + }, + + get_repos_urls_with_code = function(type, + code, + in_files = NULL, + in_path = FALSE, + verbose, + progress) { + repos_urls_from_orgs <- private$get_repos_urls_with_code_from_orgs( + type = type, + code = code, + in_files = in_files, + in_path = in_path, + verbose = verbose, + progress = progress + ) + repos_urls_from_repos <- private$get_repos_urls_with_code_from_repos( + type = type, + code = code, + in_files = in_files, + in_path = in_path, + verbose = verbose, + progress = progress + ) + repos_urls <- c(repos_urls_from_orgs, repos_urls_from_repos) + return(repos_urls) + }, + + get_repos_urls_with_code_from_orgs = function(type, code, in_files, in_path, verbose, progress) { + private$get_repos_with_code_from_orgs( + code = code, + in_files = in_files, + in_path = in_path, + output = "raw", + verbose = verbose + ) |> + private$get_repo_url_from_response( + type = type, + progress = progress + ) + }, + + get_repos_urls_with_code_from_repos = function(type, code, in_files, in_path, verbose, progress) { + private$get_repos_with_code_from_repos( + code = code, + in_files = in_files, + in_path = in_path, + output = "raw", + verbose = verbose + ) |> + private$get_repo_url_from_response( + type = type, + repos_fullnames = private$repos_fullnames, + progress = progress + ) }, add_platform = function(repos_table) { diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index 84e0b1d7..f0ac14fc 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -155,7 +155,11 @@ GitHostGitHub <- R6::R6Class( }, # Get projects URL from search response - get_repo_url_from_response = function(search_response, type, progress = TRUE) { + get_repo_url_from_response = function(search_response, repos_fullnames = NULL, type, progress = TRUE) { + if (!is.null(repos_fullnames)) { + search_response <- search_response |> + purrr::keep(~ .$repository$full_name %in% repos_fullnames) + } purrr::map_vec(search_response, function(project) { if (type == "api") { project$repository$url diff --git a/R/GitHostGitLab.R b/R/GitHostGitLab.R index ff4f041d..4f2f0957 100644 --- a/R/GitHostGitLab.R +++ b/R/GitHostGitLab.R @@ -153,8 +153,8 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", }, # Get projects API URL from search response - get_repo_url_from_response = function(search_response, type, progress = TRUE) { - purrr::map_vec(search_response, function(response) { + get_repo_url_from_response = function(search_response, type, repos_fullnames = NULL, progress = TRUE) { + repo_urls <- purrr::map_vec(search_response, function(response) { api_url <- paste0(private$api_url, "/projects/", response$project_id) if (type == "api") { return(api_url) @@ -171,6 +171,7 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", } else { FALSE }) + return(repo_urls) }, get_commits_from_orgs = function(since, diff --git a/R/GitStats.R b/R/GitStats.R index 58dd8070..b4fa288e 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -114,14 +114,21 @@ GitStats <- R6::R6Class( with_files = with_files, verbose = verbose, progress = progress - ) %>% - private$set_object_class( + ) + if (!is.null(repos_urls)) { + repos_urls <- private$set_object_class( + object = repos_urls, class = "repos_urls", attr_list = args_list ) - private$save_to_storage( - table = repos_urls - ) + private$save_to_storage( + table = repos_urls + ) + } else if (verbose) { + cli::cli_alert_warning( + cli::col_yellow("No findings.") + ) + } } else { repos_urls <- private$get_from_storage( table = "repos_urls", From 99fded804a63d30b78a0d7e86342da8dc442a167 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 19 Dec 2024 08:41:27 +0000 Subject: [PATCH 16/27] Fix printing GitLab org in repo table when it is a subgroup. Earlier it was printing only the parent group. --- R/GQLQueryGitLab.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/GQLQueryGitLab.R b/R/GQLQueryGitLab.R index 6f6270bf..fa3def62 100644 --- a/R/GQLQueryGitLab.R +++ b/R/GQLQueryGitLab.R @@ -238,7 +238,7 @@ GQLQueryGitLab <- R6::R6Class("GQLQueryGitLab", opened } namespace { - path + path: fullPath } repo_url: webUrl } From 58540b02458ba2784a8c44ddbb18990d50c1bf3f Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 19 Dec 2024 09:00:38 +0000 Subject: [PATCH 17/27] Update vignettes. --- vignettes/set_hosts.Rmd | 49 +++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/vignettes/set_hosts.Rmd b/vignettes/set_hosts.Rmd index 2e4d35c9..ec7386e8 100644 --- a/vignettes/set_hosts.Rmd +++ b/vignettes/set_hosts.Rmd @@ -21,40 +21,40 @@ To make `GitStats` work you need to set hosts after creating `gitstats`. You can set GitLab host with `set_gitlab_host()` and GitHub host with `set_github_host()` or both. -When setting hosts you need to take into account: +When setting hosts you need to consider: * Do you wish to connect to `private` or `public` hosts? -* What `scanning scope` do you wish to set? Do you want to scan specific `organizations` or `repositories` or maybe whole git platforms? +* What `scanning scope` do you wish to set? Do you want to scan specific `organizations` and/or `repositories` or maybe whole git platforms? * Do you have `tokens` set up and stored in your environment variables that grant you access to APIs? ## Public and private hosts -If you connect to public hosts you simply call `set_github_host()` or `set_gitlab_host()` function without specifying `host` parameter. +If you connect to **public** hosts you simply call `set_github_host()` or `set_gitlab_host()` function without specifying `host` parameter. ```{r, eval = FALSE} library(GitStats) -git_stats <- create_gitstats() %>% +git_stats <- create_gitstats() |> set_github_host( orgs = c("r-world-devs", "openpharma"), token = Sys.getenv("GITHUB_PAT") - ) %>% + ) |> set_gitlab_host( orgs = c("mbtests"), token = Sys.getenv("GITLAB_PAT_PUBLIC") ) ``` -If you wish to connect to internal GitHub or GitLab, you need to pass names of the hosts to `host` parameter. Remember also to have tokens set up properly for these hosts (on tokens read below). +If you wish to connect to **internal** GitHub or GitLab, you need to pass names of the hosts to `host` parameter. Remember also to have tokens set up properly for these hosts (on tokens read below). ```r -git_stats <- create_gitstats() %>% +git_stats <- create_gitstats() |> set_github_host( host = "github.internal.com", orgs = c("org_1", "org_2", "org_3"), token = Sys.getenv("YOUR_GITHUB_PAT") - ) %>% + ) |> set_gitlab_host( host = "internal.host.com", orgs = c("internal_org"), @@ -64,20 +64,20 @@ git_stats <- create_gitstats() %>% ### API versions -GitStats is configured to connect to GitHub API (version 3) and GitLab API (version 4). +`GitStats` is configured to connect to GitHub API (version 3) and GitLab API (version 4). ## Scanning scope -When setting hosts you choose what scanning scope of your GitStats will be: +When setting hosts you choose what scanning scope of your `GitStats` will be: * `organizations/groups` - in this case you need to pass character arguments (names of organizations (in case of GitHub) or groups (in case of GitLab)) to `orgs` parameter. ```{r, eval = FALSE} -git_stats <- create_gitstats() %>% +git_stats <- create_gitstats() |> set_github_host( orgs = c("r-world-devs", "openpharma"), token = Sys.getenv("GITHUB_PAT") - ) %>% + ) |> set_gitlab_host( orgs = c("mbtests"), token = Sys.getenv("GITLAB_PAT_PUBLIC") @@ -87,25 +87,36 @@ git_stats <- create_gitstats() %>% * `repositories` - in this case you need to pass full names of repositories (`{org_name}/{repo_name}`) to the `repos` parameter. ```{r, eval = FALSE} -git_stats <- create_gitstats() %>% +git_stats <- create_gitstats() |> set_github_host( repos = c("r-world-devs/GitStats", "r-world-devs/shinyCohortBuilder", "openpharma/DataFakeR"), token = Sys.getenv("GITHUB_PAT") - ) %>% + ) |> set_gitlab_host( repos = "mbtests/gitstatstesting", token = Sys.getenv("GITLAB_PAT_PUBLIC") ) ``` -* `whole hosts` - this is possible for the time being only in case of private hosts, as public ones are deemed to be too large. To set whole Git platform to be scanned just set hosts without specifying `orgs` or `repos`. On the other hand, remember that to connect with internal host, you need to pass argument to `host` parameter. +* `organizations/groups` and `repositories` - you can define both at the same time: + +```{r, eval = FALSE} +git_stats <- create_gitstats() |> + set_github_host( + orgs = "openpharma", + repos = c("r-world-devs/GitStats", "r-world-devs/shinyCohortBuilder"), + token = Sys.getenv("GITHUB_PAT") + ) +``` + +* `whole hosts` - this is possible for the time being only in case of private hosts, as public ones are deemed to be too large. To set whole Git platform to be scanned just set hosts **without specifying** `orgs` or `repos`. On the other hand, remember that to connect with internal host, you need to pass argument to `host` parameter. ```r -git_stats <- create_gitstats() %>% +git_stats <- create_gitstats() |> set_github_host( host = "github.internal.com", token = Sys.getenv("YOUR_GITHUB_PAT") - ) %>% + ) |> set_gitlab_host( host = "internal.host.com", token = Sys.getenv("YOUR_GITLAB_PAT") @@ -128,10 +139,10 @@ When creating tokens you will be asked to set access scopes of the tokens. For ` If you have your access tokens stored in environment variables with such names as `GITHUB_PAT` or `GITHUB_PAT_*` and `GITLAB_PAT` or `GITLAB_PAT_*` you do not need to specify them in `set_*_host()` functions, `GitStats` will automatically find them. ```{r, eval = FALSE} -git_stats <- create_gitstats() %>% +git_stats <- create_gitstats() |> set_github_host( orgs = c("r-world-devs", "openpharma") - ) %>% + ) |> set_gitlab_host( orgs = c("mbtests") ) From ef59a17f9e17331e5fbd36dd3c9aa661bf14ca0f Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 19 Dec 2024 10:19:11 +0000 Subject: [PATCH 18/27] Update workflow. --- inst/package_usage_workflow.R | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/inst/package_usage_workflow.R b/inst/package_usage_workflow.R index d7cf27e8..15db6f50 100644 --- a/inst/package_usage_workflow.R +++ b/inst/package_usage_workflow.R @@ -1,19 +1,36 @@ devtools::load_all(".") -test_gitstats <- create_gitstats() %>% +openpharma_stats <- create_gitstats() %>% set_github_host( orgs = "openpharma" ) -get_R_package_usage(test_gitstats, packages = "no_such_package") +get_R_package_usage(openpharma_stats, packages = "no_such_package") get_R_package_usage( - test_gitstats, + openpharma_stats, packages = c("purrr", "shiny") ) get_R_package_usage( - test_gitstats, + openpharma_stats, packages = c("dplyr", "shiny"), split_output = TRUE ) + +pharmaverse_stats <- create_gitstats() %>% + set_github_host( + orgs = "pharmaverse" + ) + +get_R_package_usage(pharmaverse_stats, + packages = c("purrr", "shiny")) + +rwd_stats <- create_gitstats() %>% + set_github_host( + repos = "openpharma/DataFakeR", + orgs = "r-world-devs" + ) + +get_R_package_usage(rwd_stats, + packages = c("purrr", "shiny")) From dd6321a1af6c7bed6c63eeb7c409742bb314ba89 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 19 Dec 2024 10:19:44 +0000 Subject: [PATCH 19/27] Fix tests. --- tests/testthat/_snaps/01-get_repos-GitLab.md | 2 +- tests/testthat/test-01-get_repos-GitHub.R | 75 ++++++++++++++++---- tests/testthat/test-get_urls_repos-GitHub.R | 68 ++++++++++++++---- tests/testthat/test-get_urls_repos-GitLab.R | 55 ++++++++++++-- 4 files changed, 170 insertions(+), 30 deletions(-) diff --git a/tests/testthat/_snaps/01-get_repos-GitLab.md b/tests/testthat/_snaps/01-get_repos-GitLab.md index 632c8019..8b4158e8 100644 --- a/tests/testthat/_snaps/01-get_repos-GitLab.md +++ b/tests/testthat/_snaps/01-get_repos-GitLab.md @@ -3,5 +3,5 @@ Code gl_repos_by_org_query Output - [1] "\n query GetReposByOrg($org: ID! $repo_cursor: String!) {\n group(fullPath: $org) {\n projects(first: 100 after: $repo_cursor) {\n \n count\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n repo_id: id\n repo_name: name\n repo_path: path\n ... on Project {\n repository {\n rootRef\n }\n }\n stars: starCount\n forks: forksCount\n created_at: createdAt\n last_activity_at: lastActivityAt\n languages {\n name\n }\n issues: issueStatusCounts {\n all\n closed\n opened\n }\n namespace {\n path\n }\n repo_url: webUrl\n }\n }\n }\n }\n }" + [1] "\n query GetReposByOrg($org: ID! $repo_cursor: String!) {\n group(fullPath: $org) {\n projects(first: 100 after: $repo_cursor) {\n \n count\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n repo_id: id\n repo_name: name\n repo_path: path\n ... on Project {\n repository {\n rootRef\n }\n }\n stars: starCount\n forks: forksCount\n created_at: createdAt\n last_activity_at: lastActivityAt\n languages {\n name\n }\n issues: issueStatusCounts {\n all\n closed\n opened\n }\n namespace {\n path: fullPath\n }\n repo_url: webUrl\n }\n }\n }\n }\n }" diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index 6c01c398..54afe647 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -128,48 +128,73 @@ test_that("Mapping search result to repositories works", { test_mocker$cache(gh_mapped_repos) }) -test_that("`get_repos_by_code()` returns repos output for code search in files", { +test_that("`search_for_code()` returns repos output for code search in files", { mockery::stub( - test_rest_github$get_repos_by_code, + test_rest_github_priv$search_for_code, "self$response", list("total_count" = 3L) ) mockery::stub( - test_rest_github$get_repos_by_code, + test_rest_github_priv$search_for_code, "private$search_response", test_fixtures$github_search_response ) mockery::stub( - test_rest_github$get_repos_by_code, + test_rest_github_priv$search_for_code, "private$map_search_into_repos", test_mocker$use("gh_mapped_repos") ) - gh_repos_by_code <- test_rest_github$get_repos_by_code( + gh_search_for_code <- test_rest_github_priv$search_for_code( code = "test_code", filename = "test_file", + in_path = FALSE, org = "test_org", - verbose = FALSE + verbose = FALSE, + progress = FALSE ) - expect_gh_repos_rest_response(gh_repos_by_code) - test_mocker$cache(gh_repos_by_code) + expect_gh_search_response(gh_search_for_code$items) + test_mocker$cache(gh_search_for_code) }) -test_that("`get_repos_by_code()` for GitHub prepares a raw search response", { +test_that("`search_repos_for_code()` returns repos output for code search in files", { mockery::stub( - test_rest_github$get_repos_by_code, + test_rest_github_priv$search_repos_for_code, "self$response", list("total_count" = 3L) ) mockery::stub( - test_rest_github$get_repos_by_code, + test_rest_github_priv$search_repos_for_code, "private$search_response", test_fixtures$github_search_response ) mockery::stub( - test_rest_github$get_repos_by_code, + test_rest_github_priv$search_repos_for_code, "private$map_search_into_repos", test_mocker$use("gh_mapped_repos") ) + gh_search_repos_for_code <- test_rest_github_priv$search_repos_for_code( + code = "test_code", + filename = "test_file", + in_path = FALSE, + repos = c("TestRepo", "TestRepo1"), + verbose = FALSE, + progress = FALSE + ) + expect_gh_search_response(gh_search_repos_for_code$items) + test_mocker$cache(gh_search_repos_for_code) +}) + +test_that("`get_repos_by_code()` for GitHub prepares a raw search response", { + mockery::stub( + test_rest_github$get_repos_by_code, + "private$search_for_code", + test_mocker$use("gh_search_for_code") + ) + mockery::stub( + test_rest_github$get_repos_by_code, + "private$search_repos_for_code", + test_mocker$use("gh_search_repos_for_code") + ) gh_repos_by_code_raw <- test_rest_github$get_repos_by_code( code = "test_code", org = "test_org", @@ -180,6 +205,32 @@ test_that("`get_repos_by_code()` for GitHub prepares a raw search response", { test_mocker$cache(gh_repos_by_code_raw) }) +test_that("`get_repos_by_code()` for GitHub prepares a repository output", { + mockery::stub( + test_rest_github$get_repos_by_code, + "private$search_for_code", + test_mocker$use("gh_search_for_code") + ) + mockery::stub( + test_rest_github$get_repos_by_code, + "private$search_repos_for_code", + test_mocker$use("gh_search_repos_for_code") + ) + mockery::stub( + test_rest_github$get_repos_by_code, + "private$map_search_into_repos", + test_mocker$use("gh_mapped_repos") + ) + gh_repos_by_code <- test_rest_github$get_repos_by_code( + code = "test_code", + org = "test_org", + output = "table_min", + verbose = FALSE + ) + expect_gh_repos_rest_response(gh_repos_by_code) + test_mocker$cache(gh_repos_by_code) +}) + test_that("GitHub tailors precisely `repos_list`", { gh_repos_by_code <- test_mocker$use("gh_repos_by_code") gh_repos_by_code_tailored <- diff --git a/tests/testthat/test-get_urls_repos-GitHub.R b/tests/testthat/test-get_urls_repos-GitHub.R index a0f379b3..72c0a3ab 100644 --- a/tests/testthat/test-get_urls_repos-GitHub.R +++ b/tests/testthat/test-get_urls_repos-GitHub.R @@ -1,12 +1,12 @@ test_that("get_repos_urls() works for whole orgs", { mockery::stub( test_rest_github$get_repos_urls, - "self$response", + "private$paginate_results", test_fixtures$github_repositories_rest_response ) gh_repos_urls <- test_rest_github$get_repos_urls( type = "web", - org = "test-org", + org = "test_org", repos = NULL ) expect_length( @@ -19,12 +19,12 @@ test_that("get_repos_urls() works for whole orgs", { test_that("get_repos_urls() works for individual repos", { mockery::stub( test_rest_github$get_repos_urls, - "self$response", + "private$paginate_results", test_fixtures$github_repositories_rest_response ) gh_repos_urls <- test_rest_github$get_repos_urls( type = "web", - org = "test-org", + org = "test_org", repos = c("testRepo", "testRepo2") ) expect_length( @@ -35,14 +35,13 @@ test_that("get_repos_urls() works for individual repos", { }) test_that("get_repos_urls prepares api repo_urls vector", { - github_testhost_priv <- create_github_testhost(orgs = "test-org", - mode = "private") mockery::stub( test_rest_github$get_repos_urls, - "self$response", + "private$paginate_results", test_fixtures$github_repositories_rest_response ) gh_api_repos_urls <- test_rest_github$get_repos_urls( + org = "test_org", repos = NULL, type = "api" ) @@ -136,17 +135,62 @@ test_that("get_repo_url_from_response retrieves repositories URLS", { test_mocker$cache(gh_repo_web_urls) }) -test_that("get_repos_urls returns repositories URLS", { +test_that("get_repos_urls_with_code_from_orgs returns repositories URLS", { mockery::stub( - github_testhost$get_repos_urls, + github_testhost_priv$get_repos_urls_with_code_from_orgs, "private$get_repo_url_from_response", test_mocker$use("gh_repo_web_urls") ) - gh_repos_urls_with_code_in_files <- github_testhost$get_repos_urls( + gh_repos_urls_with_code_from_orgs <- github_testhost_priv$get_repos_urls_with_code_from_orgs( type = "web", - with_code = "shiny", + code = "shiny", in_files = "DESCRIPTION", - verbose = FALSE + in_path = FALSE, + verbose = FALSE, + progress= FALSE + ) + expect_type(gh_repos_urls_with_code_from_orgs, "character") + expect_gt(length(gh_repos_urls_with_code_from_orgs), 0) + test_mocker$cache(gh_repos_urls_with_code_from_orgs) +}) + +test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { + mockery::stub( + github_testhost_priv$get_repos_urls_with_code_from_repos, + "private$get_repo_url_from_response", + test_mocker$use("gh_repo_web_urls") + ) + gh_repos_urls_with_code_from_repos <- github_testhost_priv$get_repos_urls_with_code_from_repos( + type = "web", + code = "shiny", + in_files = "DESCRIPTION", + in_path = FALSE, + verbose = FALSE, + progress = FALSE + ) + expect_type(gh_repos_urls_with_code_from_repos, "character") + expect_gt(length(gh_repos_urls_with_code_from_repos), 0) + test_mocker$cache(gh_repos_urls_with_code_from_repos) +}) + +test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { + mockery::stub( + github_testhost_priv$get_repos_urls_with_code, + "private$get_repos_urls_with_code_from_orgs", + test_mocker$use("gh_repos_urls_with_code_from_orgs") + ) + mockery::stub( + github_testhost_priv$get_repos_urls_with_code, + "private$get_repos_urls_with_code_from_repos", + test_mocker$use("gh_repos_urls_with_code_from_repos") + ) + gh_repos_urls_with_code_in_files <- github_testhost_priv$get_repos_urls_with_code( + type = "web", + code = "shiny", + in_files = "DESCRIPTION", + in_path = FALSE, + verbose = FALSE, + progress = FALSE ) expect_type(gh_repos_urls_with_code_in_files, "character") expect_gt(length(gh_repos_urls_with_code_in_files), 0) diff --git a/tests/testthat/test-get_urls_repos-GitLab.R b/tests/testthat/test-get_urls_repos-GitLab.R index 17d4589b..91fb72dc 100644 --- a/tests/testthat/test-get_urls_repos-GitLab.R +++ b/tests/testthat/test-get_urls_repos-GitLab.R @@ -121,17 +121,62 @@ test_that("`get_repo_url_from_response()` works", { test_mocker$cache(gl_repo_web_urls) }) -test_that("get_repos_urls returns repositories URLS", { +test_that("get_repos_urls_with_code_from_orgs returns repositories URLS", { mockery::stub( - gitlab_testhost$get_repos_urls, + gitlab_testhost_priv$get_repos_urls_with_code_from_orgs, "private$get_repo_url_from_response", test_mocker$use("gl_repo_web_urls") ) - gl_repos_urls_with_code_in_files <- gitlab_testhost$get_repos_urls( + gl_repos_urls_with_code_from_orgs <- gitlab_testhost_priv$get_repos_urls_with_code_from_orgs( type = "web", - with_code = "shiny", + code = "shiny", in_files = "DESCRIPTION", - verbose = FALSE + in_path = FALSE, + verbose = FALSE, + progress = FALSE + ) + expect_type(gl_repos_urls_with_code_from_orgs, "character") + expect_gt(length(gl_repos_urls_with_code_from_orgs), 0) + test_mocker$cache(gl_repos_urls_with_code_from_orgs) +}) + +test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { + mockery::stub( + gitlab_testhost_priv$get_repos_urls_with_code_from_repos, + "private$get_repo_url_from_response", + test_mocker$use("gl_repo_web_urls") + ) + gl_repos_urls_with_code_from_repos <- gitlab_testhost_priv$get_repos_urls_with_code_from_repos( + type = "web", + code = "shiny", + in_files = "DESCRIPTION", + in_path = FALSE, + verbose = FALSE, + progress = FALSE + ) + expect_type(gl_repos_urls_with_code_from_repos, "character") + expect_gt(length(gl_repos_urls_with_code_from_repos), 0) + test_mocker$cache(gl_repos_urls_with_code_from_repos) +}) + +test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { + mockery::stub( + gitlab_testhost_priv$get_repos_urls_with_code, + "private$get_repos_urls_with_code_from_orgs", + test_mocker$use("gl_repos_urls_with_code_from_orgs") + ) + mockery::stub( + gitlab_testhost_priv$get_repos_urls_with_code, + "private$get_repos_urls_with_code_from_repos", + test_mocker$use("gl_repos_urls_with_code_from_repos") + ) + gl_repos_urls_with_code_in_files <- gitlab_testhost_priv$get_repos_urls_with_code( + type = "web", + code = "shiny", + in_files = "DESCRIPTION", + in_path = FALSE, + verbose = FALSE, + progress = FALSE ) expect_type(gl_repos_urls_with_code_in_files, "character") expect_gt(length(gl_repos_urls_with_code_in_files), 0) From 200669dfbd571e4314a895da9cb02c9d95b3656a Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 19 Dec 2024 11:06:24 +0000 Subject: [PATCH 20/27] Add test. --- tests/testthat/test-get_urls_repos-GitHub.R | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/testthat/test-get_urls_repos-GitHub.R b/tests/testthat/test-get_urls_repos-GitHub.R index 72c0a3ab..8d6e95f2 100644 --- a/tests/testthat/test-get_urls_repos-GitHub.R +++ b/tests/testthat/test-get_urls_repos-GitHub.R @@ -196,3 +196,22 @@ test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { expect_gt(length(gh_repos_urls_with_code_in_files), 0) test_mocker$cache(gh_repos_urls_with_code_in_files) }) + +test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { + mockery::stub( + github_testhost$get_repos_urls, + "private$get_repos_urls_with_code", + test_mocker$use("gh_repos_urls_with_code_in_files") + ) + gh_repos_urls_with_code_in_files <- github_testhost$get_repos_urls( + type = "web", + with_code = "shiny", + in_files = "DESCRIPTION", + with_file = NULL, + verbose = FALSE, + progress = FALSE + ) + expect_type(gh_repos_urls_with_code_in_files, "character") + expect_gt(length(gh_repos_urls_with_code_in_files), 0) + test_mocker$cache(gh_repos_urls_with_code_in_files) +}) From d2b243766e2d600085b97d32c2643baaab9e1c73 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 19 Dec 2024 11:24:45 +0000 Subject: [PATCH 21/27] Add test to fix coverage. --- tests/testthat/test-01-get_repos-GitHub.R | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index 54afe647..0e34136e 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -637,5 +637,16 @@ test_that("`get_repos()` pulls table in minimalist version", { repo_cols = repo_min_colnames, with_cols = c("api_url", "platform") ) + gh_repos_table_min <- github_testhost$get_repos( + add_contributors = FALSE, + with_file = "test_file", + output = "table_min", + verbose = FALSE + ) + expect_repos_table( + gh_repos_table_min, + repo_cols = repo_min_colnames, + with_cols = c("api_url", "platform") + ) test_mocker$cache(gh_repos_table_min) }) From 59a34d2309a744b4fba2afc66fd0d585e5dfc41a Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 19 Dec 2024 11:56:53 +0000 Subject: [PATCH 22/27] Add tests to meet coverage threshold. --- tests/testthat/_snaps/01-get_repos-GitLab.md | 14 +++++++ .../_snaps/get_files_structure-GitHub.md | 5 +++ tests/testthat/_snaps/helpers.md | 8 ++++ tests/testthat/test-01-get_repos-GitLab.R | 18 +++++++++ .../test-get_files_structure-GitHub.R | 11 ++++++ tests/testthat/test-get_urls_repos-GitHub.R | 37 ++++++++++++++++++- tests/testthat/test-helpers.R | 10 +++++ 7 files changed, 101 insertions(+), 2 deletions(-) diff --git a/tests/testthat/_snaps/01-get_repos-GitLab.md b/tests/testthat/_snaps/01-get_repos-GitLab.md index 8b4158e8..9e412a4f 100644 --- a/tests/testthat/_snaps/01-get_repos-GitLab.md +++ b/tests/testthat/_snaps/01-get_repos-GitLab.md @@ -5,3 +5,17 @@ Output [1] "\n query GetReposByOrg($org: ID! $repo_cursor: String!) {\n group(fullPath: $org) {\n projects(first: 100 after: $repo_cursor) {\n \n count\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n repo_id: id\n repo_name: name\n repo_path: path\n ... on Project {\n repository {\n rootRef\n }\n }\n stars: starCount\n forks: forksCount\n created_at: createdAt\n last_activity_at: lastActivityAt\n languages {\n name\n }\n issues: issueStatusCounts {\n all\n closed\n opened\n }\n namespace {\n path: fullPath\n }\n repo_url: webUrl\n }\n }\n }\n }\n }" +# get_repos_from_org prints proper message + + Code + gl_repos_from_orgs <- gitlab_testhost_priv$get_repos_from_orgs(verbose = TRUE, + progress = FALSE) + Message + i [Host:GitLab][Engine:GraphQl][Scope:mbtests] Pulling repositories... + +# get_repos_from_repos prints proper message + + Code + gl_repos_from_repos <- gitlab_testhost_priv$get_repos_from_repos(verbose = TRUE, + progress = FALSE) + diff --git a/tests/testthat/_snaps/get_files_structure-GitHub.md b/tests/testthat/_snaps/get_files_structure-GitHub.md index 1a29ddd2..e00cecfe 100644 --- a/tests/testthat/_snaps/get_files_structure-GitHub.md +++ b/tests/testthat/_snaps/get_files_structure-GitHub.md @@ -15,6 +15,11 @@ Output named list() +# get_files_structure aborts when scope to scan whole host + + x This feature is not applicable to scan whole Git Host (time consuming). + i Set `orgs` or `repos` arguments in `set_*_host()` if you wish to run this function. + # get_files_content makes use of files_structure Code diff --git a/tests/testthat/_snaps/helpers.md b/tests/testthat/_snaps/helpers.md index d1167c04..896199b3 100644 --- a/tests/testthat/_snaps/helpers.md +++ b/tests/testthat/_snaps/helpers.md @@ -10,6 +10,14 @@ gitlab_testhost_priv$set_searching_scope(orgs = NULL, repos = "mbtests/GitStatsTesting", verbose = TRUE) +# `set_searching_scope` sets scope to whole host + + Code + gitlab_testhost_priv$set_searching_scope(orgs = NULL, repos = NULL, verbose = TRUE) + Message + i No `orgs` nor `repos` specified. + i Searching scope set to [all]. + # When token is empty throw error Code diff --git a/tests/testthat/test-01-get_repos-GitLab.R b/tests/testthat/test-01-get_repos-GitLab.R index fffd5153..f4c40890 100644 --- a/tests/testthat/test-01-get_repos-GitLab.R +++ b/tests/testthat/test-01-get_repos-GitLab.R @@ -158,6 +158,24 @@ test_that("`prepare_repos_table()` prepares repos table", { test_mocker$cache(gl_repos_table) }) +test_that("get_repos_from_org prints proper message", { + mockery::stub( + gitlab_testhost_priv$get_repos_from_orgs, + "graphql_engine$prepare_repos_table", + test_mocker$use("gl_repos_table") + ) + expect_snapshot( + gl_repos_from_orgs <- gitlab_testhost_priv$get_repos_from_orgs( + verbose = TRUE, + progress = FALSE + ) + ) + expect_repos_table( + gl_repos_from_orgs + ) + test_mocker$cache(gl_repos_from_orgs) +}) + test_that("GitHost adds `repo_api_url` column to GitLab repos table", { repos_table <- test_mocker$use("gl_repos_table") gl_repos_table_with_api_url <- gitlab_testhost_priv$add_repo_api_url(repos_table) diff --git a/tests/testthat/test-get_files_structure-GitHub.R b/tests/testthat/test-get_files_structure-GitHub.R index 0c300350..cc6fc82c 100644 --- a/tests/testthat/test-get_files_structure-GitHub.R +++ b/tests/testthat/test-get_files_structure-GitHub.R @@ -248,6 +248,17 @@ test_that("get_files_structure pulls files structure for repositories in orgs", test_mocker$cache(gh_files_structure_from_orgs) }) +test_that("get_files_structure aborts when scope to scan whole host", { + github_testhost$.__enclos_env__$private$scan_all <- TRUE + expect_snapshot_error( + github_testhost$get_files_structure( + pattern = "\\.md|\\.qmd", + depth = 1L, + verbose = FALSE + ) + ) +}) + test_that("get_files_content makes use of files_structure", { mockery::stub( github_testhost_priv$get_files_content_from_files_structure, diff --git a/tests/testthat/test-get_urls_repos-GitHub.R b/tests/testthat/test-get_urls_repos-GitHub.R index 8d6e95f2..613fe956 100644 --- a/tests/testthat/test-get_urls_repos-GitHub.R +++ b/tests/testthat/test-get_urls_repos-GitHub.R @@ -173,7 +173,7 @@ test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { test_mocker$cache(gh_repos_urls_with_code_from_repos) }) -test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { +test_that("get_repos_urls_with_code returns repositories URLS", { mockery::stub( github_testhost_priv$get_repos_urls_with_code, "private$get_repos_urls_with_code_from_orgs", @@ -197,7 +197,7 @@ test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { test_mocker$cache(gh_repos_urls_with_code_in_files) }) -test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { +test_that("get_repos_urls returns repositories URLS", { mockery::stub( github_testhost$get_repos_urls, "private$get_repos_urls_with_code", @@ -215,3 +215,36 @@ test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { expect_gt(length(gh_repos_urls_with_code_in_files), 0) test_mocker$cache(gh_repos_urls_with_code_in_files) }) + +test_that("get_repos_urls returns repositories URLS", { + mockery::stub( + github_testhost$get_repos_urls, + "private$get_repos_urls_with_code", + test_mocker$use("gh_repos_urls_with_code_in_files") + ) + gh_repos_urls_with_code_in_files <- github_testhost$get_repos_urls( + type = "web", + with_file = "DESCRIPTION", + verbose = FALSE, + progress = FALSE + ) + expect_type(gh_repos_urls_with_code_in_files, "character") + expect_gt(length(gh_repos_urls_with_code_in_files), 0) + test_mocker$cache(gh_repos_urls_with_code_in_files) +}) + +test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { + mockery::stub( + github_testhost$get_repos_urls, + "private$get_all_repos_urls", + test_mocker$use("gh_repos_urls") + ) + gh_repos_urls <- github_testhost$get_repos_urls( + type = "web", + verbose = FALSE, + progress = FALSE + ) + expect_type(gh_repos_urls, "character") + expect_gt(length(gh_repos_urls), 0) + test_mocker$cache(gh_repos_urls) +}) diff --git a/tests/testthat/test-helpers.R b/tests/testthat/test-helpers.R index d95052ac..827bcd41 100644 --- a/tests/testthat/test-helpers.R +++ b/tests/testthat/test-helpers.R @@ -7,6 +7,16 @@ test_that("`set_searching_scope` does not throw error when `orgs` or `repos` are ) }) +test_that("`set_searching_scope` sets scope to whole host", { + gitlab_testhost_priv$is_public <- FALSE + expect_snapshot( + gitlab_testhost_priv$set_searching_scope(orgs = NULL, repos = NULL, verbose = TRUE) + ) + expect_true( + gitlab_testhost_priv$scan_all + ) +}) + test_that("`extract_repos_and_orgs` extracts fullnames vector into a list of GitLab organizations with assigned repositories", { repos_fullnames <- c( "mbtests/gitstatstesting", "mbtests/gitstats-testing-2", "mbtests/subgroup/test-project-in-subgroup" From b621cab388c2a18ddc73b3acb853d36ff1335ceb Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 19 Dec 2024 12:40:55 +0000 Subject: [PATCH 23/27] Add tests, adjust code. --- R/GitHost.R | 9 ++-- R/test_helpers.R | 22 +++++++++ tests/testthat/_snaps/01-get_repos-GitHub.md | 7 +++ tests/testthat/_snaps/01-get_repos-GitLab.md | 6 --- .../testthat/_snaps/02-get_commits-GitHub.md | 8 ++++ tests/testthat/_snaps/get_release-GitHub.md | 8 ++++ tests/testthat/test-01-get_repos-GitHub.R | 48 +++++++++++++++++++ tests/testthat/test-02-get_commits-GitHub.R | 39 ++++++++++++--- tests/testthat/test-get_release-GitHub.R | 27 +++++++++++ 9 files changed, 158 insertions(+), 16 deletions(-) diff --git a/R/GitHost.R b/R/GitHost.R index 3f6a496e..c7aa46b2 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -125,7 +125,8 @@ GitHost <- R6::R6Class( progress = TRUE) { if (private$scan_all && is.null(private$orgs) && verbose) { cli::cli_alert_info("[{private$host_name}][Engine:{cli::col_yellow('GraphQL')}] Pulling all organizations...") - private$orgs <- private$engines$graphql$get_orgs() + graphql_engine <- private$engines$graphql + private$orgs <- graphql_engine$get_orgs() } commits_from_orgs <- private$get_commits_from_orgs( since = since, @@ -234,7 +235,8 @@ GitHost <- R6::R6Class( if (verbose) { cli::cli_alert_info("[{private$host_name}][Engine:{cli::col_yellow('GraphQL')}] Pulling all organizations...") } - private$orgs <- private$engines$graphql$get_orgs() + graphql_engine <- private$engines$graphql + private$orgs <- graphql_engine$get_orgs() } until <- until %||% Sys.time() release_logs_from_orgs <- private$get_release_logs_from_orgs( @@ -604,7 +606,8 @@ GitHost <- R6::R6Class( information = "Pulling all organizations" ) } - private$orgs <- private$engines$graphql$get_orgs() + graphql_engine <- private$engines$graphql + private$orgs <- graphql_engine$get_orgs() } repos_table <- purrr::list_rbind( list( diff --git a/R/test_helpers.R b/R/test_helpers.R index 056e2a86..18afac35 100644 --- a/R/test_helpers.R +++ b/R/test_helpers.R @@ -127,6 +127,28 @@ create_github_testhost <- function(host = NULL, return(test_host) } +#' @noRd +create_github_testhost_all <- function(host = NULL, + orgs = NULL, + repos = NULL, + token = NULL, + mode = "") { + suppressMessages( + test_host <- GitHostGitHubTest$new( + host = NULL, + token = token, + orgs = orgs, + repos = repos + ) + ) + test_host$.__enclos_env__$private$orgs <- NULL + test_host$.__enclos_env__$private$scan_all <- TRUE + if (mode == "private") { + test_host <- environment(test_host$initialize)$private + } + return(test_host) +} + #' @noRd create_gitlab_testhost <- function(host = NULL, orgs = NULL, diff --git a/tests/testthat/_snaps/01-get_repos-GitHub.md b/tests/testthat/_snaps/01-get_repos-GitHub.md index 2edba7cf..bafe0009 100644 --- a/tests/testthat/_snaps/01-get_repos-GitHub.md +++ b/tests/testthat/_snaps/01-get_repos-GitHub.md @@ -5,3 +5,10 @@ Output [1] "\n query GetReposByOrg($login: String!) {\n repositoryOwner(login: $login) {\n ... on Organization {\n \n repositories(first: 100) {\n totalCount\n pageInfo {\n endCursor\n hasNextPage\n }\n nodes {\n repo_id: id\n repo_name: name\n default_branch: defaultBranchRef {\n name\n }\n stars: stargazerCount\n forks: forkCount\n created_at: createdAt\n last_activity_at: pushedAt\n languages (first: 5) { nodes {name} }\n issues_open: issues (first: 100 states: [OPEN]) {\n totalCount\n }\n issues_closed: issues (first: 100 states: [CLOSED]) {\n totalCount\n }\n organization: owner {\n login\n }\n repo_url: url\n }\n }\n \n }\n }\n }" +# `get_all_repos()` is set to scan whole git host + + Code + gh_repos <- github_testhost_all_priv$get_all_repos(verbose = TRUE, progress = FALSE) + Message + i [Host:GitHub][Engine:GraphQl] Pulling all organizations... + diff --git a/tests/testthat/_snaps/01-get_repos-GitLab.md b/tests/testthat/_snaps/01-get_repos-GitLab.md index 9e412a4f..d84ced03 100644 --- a/tests/testthat/_snaps/01-get_repos-GitLab.md +++ b/tests/testthat/_snaps/01-get_repos-GitLab.md @@ -13,9 +13,3 @@ Message i [Host:GitLab][Engine:GraphQl][Scope:mbtests] Pulling repositories... -# get_repos_from_repos prints proper message - - Code - gl_repos_from_repos <- gitlab_testhost_priv$get_repos_from_repos(verbose = TRUE, - progress = FALSE) - diff --git a/tests/testthat/_snaps/02-get_commits-GitHub.md b/tests/testthat/_snaps/02-get_commits-GitHub.md index 5fe49647..8c59addf 100644 --- a/tests/testthat/_snaps/02-get_commits-GitHub.md +++ b/tests/testthat/_snaps/02-get_commits-GitHub.md @@ -9,3 +9,11 @@ i GraphQL response error +# `get_commits()` is set to scan whole git host + + Code + gh_commits_table <- github_testhost_all$get_commits(since = "2023-01-01", + until = "2023-02-28", verbose = TRUE, progress = FALSE) + Message + i [GitHub][Engine:GraphQL] Pulling all organizations... + diff --git a/tests/testthat/_snaps/get_release-GitHub.md b/tests/testthat/_snaps/get_release-GitHub.md index 116312b0..94bc9e8e 100644 --- a/tests/testthat/_snaps/get_release-GitHub.md +++ b/tests/testthat/_snaps/get_release-GitHub.md @@ -5,3 +5,11 @@ Output [1] "query GetReleasesFromRepo ($org: String!, $repo: String!) {\n repository(owner:$org, name:$repo){\n name\n url\n releases (last: 100) {\n nodes {\n name\n tagName\n publishedAt\n url\n description\n }\n }\n }\n }" +# `get_release_logs()` is set to scan whole git host + + Code + gh_releases_table <- github_testhost_all$get_release_logs(since = "2023-01-01", + until = "2023-02-28", verbose = TRUE, progress = FALSE) + Message + i [GitHub][Engine:GraphQL] Pulling all organizations... + diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index 0e34136e..e86c31bf 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -395,6 +395,7 @@ test_that("`get_repos_with_code_from_host()` pulls raw response", { ) expect_type(repos_with_code_from_host_raw, "list") expect_gt(length(repos_with_code_from_host_raw), 0) + test_mocker$cache(repos_with_code_from_host_raw) }) test_that("get_repos_with_code() works", { @@ -432,6 +433,25 @@ test_that("get_repos_with_code() works", { test_mocker$cache(github_repos_with_code_min) }) + +test_that("get_repos_with_code() scans whole host", { + mockery::stub( + github_testhost_priv$get_repos_with_code, + "private$get_repos_with_code_from_host", + test_mocker$use("repos_with_code_from_host_raw") + ) + github_testhost_priv$scan_all <- TRUE + github_repos_with_code_raw <- github_testhost_priv$get_repos_with_code( + code = "test-code", + output = "raw", + verbose = FALSE, + progress = FALSE + ) + expect_type(github_repos_with_code_raw, "list") + expect_gt(length(github_repos_with_code_raw), 0) + github_testhost_priv$scan_all <- FALSE +}) + test_that("GitHub prepares repos table from repositories response", { gh_repos_table <- test_graphql_github$prepare_repos_table( repos_list = test_mocker$use("gh_repos_from_org") @@ -504,6 +524,34 @@ test_that("`get_all_repos()` works as expected", { test_mocker$cache(gh_repos_table) }) +test_that("`get_all_repos()` is set to scan whole git host", { + github_testhost_all_priv <- create_github_testhost_all( + orgs = "test_org", + mode = "private" + ) + mockery::stub( + github_testhost_all_priv$get_all_repos, + "graphql_engine$get_orgs", + "test_org" + ) + mockery::stub( + github_testhost_all_priv$get_all_repos, + "private$get_repos_from_orgs", + test_mocker$use("gh_repos_from_orgs") + ) + mockery::stub( + github_testhost_all_priv$get_all_repos, + "private$get_repos_from_repos", + test_mocker$use("gh_repos_individual") + ) + expect_snapshot( + gh_repos <- github_testhost_all_priv$get_all_repos( + verbose = TRUE, + progress = FALSE + ) + ) +}) + test_that("GitHost adds `repo_api_url` column to GitHub repos table", { repos_table <- test_mocker$use("gh_repos_table") gh_repos_table_with_api_url <- github_testhost_priv$add_repo_api_url(repos_table) diff --git a/tests/testthat/test-02-get_commits-GitHub.R b/tests/testthat/test-02-get_commits-GitHub.R index a42025e5..38f0c5e1 100644 --- a/tests/testthat/test-02-get_commits-GitHub.R +++ b/tests/testthat/test-02-get_commits-GitHub.R @@ -177,16 +177,41 @@ test_that("`get_commits()` retrieves commits in the table format", { "private$get_commits_from_repos", test_mocker$use("gh_commits_from_repos") ) - suppressMessages( - gh_commits_table <- github_testhost$get_commits( - since = "2023-01-01", - until = "2023-02-28", - verbose = FALSE, - progress = FALSE - ) + gh_commits_table <- github_testhost$get_commits( + since = "2023-01-01", + until = "2023-02-28", + verbose = FALSE, + progress = FALSE ) expect_commits_table( gh_commits_table ) test_mocker$cache(gh_commits_table) }) + +test_that("`get_commits()` is set to scan whole git host", { + github_testhost_all <- create_github_testhost_all(orgs = "test_org") + mockery::stub( + github_testhost_all$get_commits, + "graphql_engine$get_orgs", + "test_org" + ) + mockery::stub( + github_testhost_all$get_commits, + "private$get_commits_from_orgs", + test_mocker$use("gh_commits_from_orgs") + ) + mockery::stub( + github_testhost_all$get_commits, + "private$get_commits_from_repos", + test_mocker$use("gh_commits_from_repos") + ) + expect_snapshot( + gh_commits_table <- github_testhost_all$get_commits( + since = "2023-01-01", + until = "2023-02-28", + verbose = TRUE, + progress = FALSE + ) + ) +}) diff --git a/tests/testthat/test-get_release-GitHub.R b/tests/testthat/test-get_release-GitHub.R index a576cfd6..19b8f49a 100644 --- a/tests/testthat/test-get_release-GitHub.R +++ b/tests/testthat/test-get_release-GitHub.R @@ -102,3 +102,30 @@ test_that("`get_release_logs()` pulls release logs in the table format", { expect_lt(max(releases_table$published_at), as.POSIXct("2023-09-30")) test_mocker$cache(releases_table) }) + +test_that("`get_release_logs()` is set to scan whole git host", { + github_testhost_all <- create_github_testhost_all(orgs = "test_org") + mockery::stub( + github_testhost_all$get_release_logs, + "graphql_engine$get_orgs", + "test_org" + ) + mockery::stub( + github_testhost_all$get_release_logs, + "private$get_release_logs_from_repos", + test_mocker$use("releases_from_repos") + ) + mockery::stub( + github_testhost_all$get_release_logs, + "private$get_release_logs_from_orgs", + test_mocker$use("releases_from_orgs") + ) + expect_snapshot( + gh_releases_table <- github_testhost_all$get_release_logs( + since = "2023-01-01", + until = "2023-02-28", + verbose = TRUE, + progress = FALSE + ) + ) +}) From b85ed501a3f25acdf646372992f8c6f3fb149a6f Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 19 Dec 2024 15:34:42 +0000 Subject: [PATCH 24/27] Fix after code review. --- R/EngineGraphQLGitLab.R | 6 +----- tests/testthat/test-get_files_content-GitLab.R | 2 -- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index a4010946..5251b0f8 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -127,7 +127,6 @@ EngineGraphQLGitLab <- R6::R6Class( repos, file_paths = NULL, host_files_structure = NULL, - only_text_files, verbose = FALSE, progress = FALSE) { org <- URLdecode(org) @@ -139,7 +138,7 @@ EngineGraphQLGitLab <- R6::R6Class( host_files_structure = host_files_structure, org = org ) - } else if (only_text_files) { + } else { file_paths <- file_paths[grepl(text_files_pattern, file_paths)] } if (type == "organization") { @@ -177,7 +176,6 @@ EngineGraphQLGitLab <- R6::R6Class( repos = repos, file_paths = file_paths, host_files_structure = host_files_structure, - only_text_files = only_text_files, verbose = verbose, progress = progress ) @@ -219,7 +217,6 @@ EngineGraphQLGitLab <- R6::R6Class( repos = repos, file_paths = file_paths, host_files_structure = host_files_structure, - only_text_files = only_text_files, verbose = verbose, progress = progress ) @@ -235,7 +232,6 @@ EngineGraphQLGitLab <- R6::R6Class( repos, file_paths = NULL, host_files_structure = NULL, - only_text_files = TRUE, verbose = FALSE, progress = FALSE) { if (is.null(repos)) { diff --git a/tests/testthat/test-get_files_content-GitLab.R b/tests/testthat/test-get_files_content-GitLab.R index 3aeeff65..acacb91e 100644 --- a/tests/testthat/test-get_files_content-GitLab.R +++ b/tests/testthat/test-get_files_content-GitLab.R @@ -60,7 +60,6 @@ test_that("GitLab GraphQL Engine pulls files from a group", { type = "organization", repos = NULL, file_paths = "meta_data.yaml", - only_text_files = TRUE, host_files_structure = NULL ) expect_gitlab_files_from_org_response(gitlab_files_response) @@ -123,7 +122,6 @@ test_that("Gitlab GraphQL switches to pulling files per repositories when query repos = NULL, file_paths = c("project_metadata.yaml", "README.md"), host_files_structure = NULL, - only_text_files = TRUE, verbose = FALSE, progress = FALSE ) From 4f357b5233506a525dbc397db85af7db7360362f Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 20 Dec 2024 08:03:10 +0000 Subject: [PATCH 25/27] Add tests. --- R/GitHost.R | 7 +- tests/testthat/_snaps/01-get_repos-GitHub.md | 47 ++++++++ .../testthat/_snaps/get_urls_repos-GitHub.md | 8 ++ tests/testthat/test-01-get_repos-GitHub.R | 105 ++++++++++++++++-- tests/testthat/test-get_urls_repos-GitHub.R | 39 ++++++- 5 files changed, 189 insertions(+), 17 deletions(-) create mode 100644 tests/testthat/_snaps/get_urls_repos-GitHub.md diff --git a/R/GitHost.R b/R/GitHost.R index c7aa46b2..d373f1c4 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -721,7 +721,8 @@ GitHost <- R6::R6Class( information = "Pulling all organizations" ) } - private$orgs <- private$engines$graphql$get_orgs() + graphql_engine <- private$engines$graphql + private$orgs <- graphql_engine$get_orgs() } repos_urls_from_orgs <- private$get_repos_urls_from_orgs( type = type, @@ -823,14 +824,14 @@ GitHost <- R6::R6Class( verbose = verbose, progress = progress ) - }) %>% + }) |> purrr::list_flatten() } if (output != "raw") { repos_table <- repos_response %>% rest_engine$tailor_repos_response( output = output - ) %>% + ) |> rest_engine$prepare_repos_table( output = output, verbose = verbose diff --git a/tests/testthat/_snaps/01-get_repos-GitHub.md b/tests/testthat/_snaps/01-get_repos-GitHub.md index bafe0009..42efdf6e 100644 --- a/tests/testthat/_snaps/01-get_repos-GitHub.md +++ b/tests/testthat/_snaps/01-get_repos-GitHub.md @@ -5,6 +5,53 @@ Output [1] "\n query GetReposByOrg($login: String!) {\n repositoryOwner(login: $login) {\n ... on Organization {\n \n repositories(first: 100) {\n totalCount\n pageInfo {\n endCursor\n hasNextPage\n }\n nodes {\n repo_id: id\n repo_name: name\n default_branch: defaultBranchRef {\n name\n }\n stars: stargazerCount\n forks: forkCount\n created_at: createdAt\n last_activity_at: pushedAt\n languages (first: 5) { nodes {name} }\n issues_open: issues (first: 100 states: [OPEN]) {\n totalCount\n }\n issues_closed: issues (first: 100 states: [CLOSED]) {\n totalCount\n }\n organization: owner {\n login\n }\n repo_url: url\n }\n }\n \n }\n }\n }" +# `get_repos_with_code_from_orgs()` pulls raw response + + Code + repos_with_code_from_orgs_raw <- github_testhost_priv$ + get_repos_with_code_from_orgs(code = "shiny", in_files = c("DESCRIPTION", + "NAMESPACE"), output = "raw", verbose = TRUE) + Message + i [Host:GitHub][Engine:REST][Scope:test_org] Pulling repositories... + +# `get_repos_with_code_from_host()` pulls and parses output into table + + Code + repos_with_code_from_host_table <- github_testhost_priv$ + get_repos_with_code_from_host(code = "DESCRIPTION", in_path = TRUE, output = "table_full", + verbose = TRUE) + Message + i [Host:GitHub][Engine:REST] Pulling repositories... + +# `get_repos_with_code_from_repos()` works + + Code + repos_with_code_from_repos_full <- github_testhost_priv$ + get_repos_with_code_from_repos(code = "tests", output = "table_full", + verbose = TRUE) + Message + i [Host:GitHub][Engine:REST][Scope:] Pulling repositories... + i Preparing repositories table... + +# `get_repos_with_code_from_orgs()` pulls minimum version of table + + Code + repos_with_code_from_repos_min <- github_testhost_priv$ + get_repos_with_code_from_repos(code = "tests", in_files = "DESCRIPTION", + output = "table_min", verbose = TRUE) + Message + i [Host:GitHub][Engine:REST][Scope:] Pulling repositories... + i Preparing repositories table... + +# `get_repos_with_code_from_host()` pulls raw response + + Code + repos_with_code_from_host_raw <- github_testhost_priv$ + get_repos_with_code_from_host(code = "shiny", in_files = c("DESCRIPTION", + "NAMESPACE"), output = "raw", verbose = TRUE) + Message + i [Host:GitHub][Engine:REST] Pulling repositories... + # `get_all_repos()` is set to scan whole git host Code diff --git a/tests/testthat/_snaps/get_urls_repos-GitHub.md b/tests/testthat/_snaps/get_urls_repos-GitHub.md new file mode 100644 index 00000000..aeb16a31 --- /dev/null +++ b/tests/testthat/_snaps/get_urls_repos-GitHub.md @@ -0,0 +1,8 @@ +# get_repos_urls_from_orgs prepares web repo_urls vector + + Code + gh_repos_urls_from_orgs <- github_testhost_priv$get_repos_urls_from_orgs(type = "web", + verbose = TRUE, progress = FALSE) + Message + i [Host:GitHub][Engine:REST][Scope:test_org] Pulling repositories (URLs)... + diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index e86c31bf..3036fb20 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -371,27 +371,112 @@ test_that("`get_repos_with_code_from_orgs()` pulls raw response", { "rest_engine$get_repos_by_code", test_mocker$use("gh_repos_by_code_raw") ) - repos_with_code_from_orgs_raw <- github_testhost_priv$get_repos_with_code_from_orgs( - code = "shiny", - in_files = c("DESCRIPTION", "NAMESPACE"), - output = "raw", - verbose = FALSE + expect_snapshot( + repos_with_code_from_orgs_raw <- github_testhost_priv$get_repos_with_code_from_orgs( + code = "shiny", + in_files = c("DESCRIPTION", "NAMESPACE"), + output = "raw", + verbose = TRUE + ) ) expect_type(repos_with_code_from_orgs_raw, "list") expect_gt(length(repos_with_code_from_orgs_raw), 0) }) +test_that("`get_repos_with_code_from_host()` pulls and parses output into table", { + mockery::stub( + github_testhost_priv$get_repos_with_code_from_host, + "rest_engine$get_repos_by_code", + test_mocker$use("gh_repos_by_code_raw") + ) + mockery::stub( + github_testhost_priv$get_repos_with_code_from_host, + "rest_engine$prepare_repos_table", + test_mocker$use("gh_repos_by_code_table") + ) + mockery::stub( + github_testhost_priv$get_repos_with_code_from_host, + "rest_engine$get_repos_issues", + test_mocker$use("gh_repos_by_code_table") + ) + expect_snapshot( + repos_with_code_from_host_table <- github_testhost_priv$get_repos_with_code_from_host( + code = "DESCRIPTION", + in_path = TRUE, + output = "table_full", + verbose = TRUE + ) + ) + expect_repos_table(repos_with_code_from_host_table) +}) + +test_that("`get_repos_with_code_from_repos()` works", { + github_testhost_priv <- create_github_testhost( + repos = c("TestRepo1", "TestRepo2"), + mode = "private" + ) + mockery::stub( + github_testhost_priv$get_repos_with_code_from_repos, + "rest_engine$get_repos_by_code", + test_mocker$use("gh_repos_by_code") + ) + mockery::stub( + github_testhost_priv$get_repos_with_code_from_repos, + "rest_engine$get_repos_issues", + test_mocker$use("gh_repos_by_code_table") + ) + github_testhost_priv$searching_scope <- c("repo") + expect_snapshot( + repos_with_code_from_repos_full <- github_testhost_priv$get_repos_with_code_from_repos( + code = "tests", + output = "table_full", + verbose = TRUE + ) + ) + expect_repos_table(repos_with_code_from_repos_full) +}) + +test_that("`get_repos_with_code_from_repos()` pulls minimum version of table", { + github_testhost_priv <- create_github_testhost( + repos = c("TestRepo1", "TestRepo2"), + mode = "private" + ) + mockery::stub( + github_testhost_priv$get_repos_with_code_from_repos, + "rest_engine$get_repos_by_code", + test_mocker$use("gh_repos_by_code") + ) + mockery::stub( + github_testhost_priv$get_repos_with_code_from_repos, + "rest_engine$get_repos_issues", + test_mocker$use("gh_repos_by_code_table") + ) + github_testhost_priv$searching_scope <- c("org", "repo") + expect_snapshot( + repos_with_code_from_repos_min <- github_testhost_priv$get_repos_with_code_from_repos( + code = "tests", + in_files = "DESCRIPTION", + output = "table_min", + verbose = TRUE + ) + ) + expect_repos_table(repos_with_code_from_repos_min, + repo_cols = repo_min_colnames) +}) + test_that("`get_repos_with_code_from_host()` pulls raw response", { mockery::stub( github_testhost_priv$get_repos_with_code_from_host, "rest_engine$get_repos_by_code", test_mocker$use("gh_repos_by_code_raw") ) - repos_with_code_from_host_raw <- github_testhost_priv$get_repos_with_code_from_host( - code = "shiny", - in_files = c("DESCRIPTION", "NAMESPACE"), - output = "raw", - verbose = FALSE + expect_snapshot( + repos_with_code_from_host_raw <- github_testhost_priv$get_repos_with_code_from_host( + code = "shiny", + in_files = c("DESCRIPTION", "NAMESPACE"), + output = "raw", + verbose = TRUE + ) ) expect_type(repos_with_code_from_host_raw, "list") expect_gt(length(repos_with_code_from_host_raw), 0) diff --git a/tests/testthat/test-get_urls_repos-GitHub.R b/tests/testthat/test-get_urls_repos-GitHub.R index 613fe956..0a9c008e 100644 --- a/tests/testthat/test-get_urls_repos-GitHub.R +++ b/tests/testthat/test-get_urls_repos-GitHub.R @@ -59,10 +59,12 @@ test_that("get_repos_urls_from_orgs prepares web repo_urls vector", { ) github_testhost_priv$searching_scope <- "org" github_testhost_priv$orgs <- "test_org" - gh_repos_urls_from_orgs <- github_testhost_priv$get_repos_urls_from_orgs( - type = "web", - verbose = FALSE, - progress = FALSE + expect_snapshot( + gh_repos_urls_from_orgs <- github_testhost_priv$get_repos_urls_from_orgs( + type = "web", + verbose = TRUE, + progress = FALSE + ) ) expect_gt(length(gh_repos_urls_from_orgs), 0) expect_true(any(grepl("test-org", gh_repos_urls_from_orgs))) @@ -118,6 +120,35 @@ test_that("get_all_repos_urls prepares web repo_urls vector", { test_mocker$cache(gh_repos_urls) }) +test_that("get_all_repos_urls is set to scan whole host", { + github_testhost_all_priv <- create_github_testhost_all( + orgs = "test_org", + mode = "private" + ) + mockery::stub( + github_testhost_all_priv$get_all_repos_urls, + "graphql_engine$get_orgs", + "test_org" + ) + mockery::stub( + github_testhost_all_priv$get_all_repos_urls, + "private$get_repos_urls_from_orgs", + test_mocker$use("gh_repos_urls_from_orgs") + ) + mockery::stub( + github_testhost_all_priv$get_all_repos_urls, + "private$get_repos_urls_from_repos", + test_mocker$use("gh_repos_urls") + ) + gh_repos_urls <- github_testhost_all_priv$get_all_repos_urls( + type = "web", + verbose = FALSE + ) + expect_gt(length(gh_repos_urls), 0) + expect_true(any(grepl("test-org", gh_repos_urls))) + expect_true(all(grepl("https://testhost.com/", gh_repos_urls))) +}) + test_that("get_repo_url_from_response retrieves repositories URLS", { gh_repo_api_urls <- github_testhost_priv$get_repo_url_from_response( search_response = test_mocker$use("gh_search_repos_response"), From 065b6f6c8bdb1055f764d75819ef36590be1aa29 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 20 Dec 2024 12:04:28 +0000 Subject: [PATCH 26/27] Add tests, fix GitLab commits tests. --- tests/testthat/_snaps/01-get_repos-GitHub.md | 19 ++- tests/testthat/_snaps/01-get_repos-GitLab.md | 16 +++ .../testthat/_snaps/02-get_commits-GitLab.md | 16 +++ tests/testthat/_snaps/get_release-GitLab.md | 8 ++ .../testthat/_snaps/get_urls_repos-GitHub.md | 13 ++ tests/testthat/helper-expect-responses.R | 2 +- tests/testthat/helper-fixtures.R | 4 +- tests/testthat/test-01-get_repos-GitHub.R | 20 +-- tests/testthat/test-01-get_repos-GitLab.R | 118 +++++++++++++++++- tests/testthat/test-02-get_commits-GitLab.R | 61 ++++++++- tests/testthat/test-get_release-GitLab.R | 12 +- tests/testthat/test-get_urls_repos-GitHub.R | 18 +-- 12 files changed, 271 insertions(+), 36 deletions(-) create mode 100644 tests/testthat/_snaps/02-get_commits-GitLab.md diff --git a/tests/testthat/_snaps/01-get_repos-GitHub.md b/tests/testthat/_snaps/01-get_repos-GitHub.md index 42efdf6e..043a672d 100644 --- a/tests/testthat/_snaps/01-get_repos-GitHub.md +++ b/tests/testthat/_snaps/01-get_repos-GitHub.md @@ -33,7 +33,7 @@ i [Host:GitHub][Engine:REST][Scope:] Pulling repositories... i Preparing repositories table... -# `get_repos_with_code_from_orgs()` pulls minimum version of table +# `get_repos_with_code_from_repos()` pulls minimum version of table Code repos_with_code_from_repos_min <- github_testhost_priv$ @@ -52,6 +52,14 @@ Message i [Host:GitHub][Engine:REST] Pulling repositories... +# get_repos_from_repos works + + Code + gh_repos_individual <- github_testhost_priv$get_repos_from_repos(verbose = TRUE, + progress = FALSE) + Message + i [Host:GitHub][Engine:GraphQl][Scope:test_org] Pulling repositories... + # `get_all_repos()` is set to scan whole git host Code @@ -59,3 +67,12 @@ Message i [Host:GitHub][Engine:GraphQl] Pulling all organizations... +# `get_repos_contributors()` works on GitHost level + + Code + gh_repos_with_contributors <- github_testhost_priv$get_repos_contributors( + repos_table = test_mocker$use("gh_repos_table_with_platform"), verbose = TRUE, + progress = FALSE) + Message + i [Host:GitHub][Engine:REST] Pulling contributors... + diff --git a/tests/testthat/_snaps/01-get_repos-GitLab.md b/tests/testthat/_snaps/01-get_repos-GitLab.md index d84ced03..f13ca202 100644 --- a/tests/testthat/_snaps/01-get_repos-GitLab.md +++ b/tests/testthat/_snaps/01-get_repos-GitLab.md @@ -5,6 +5,22 @@ Output [1] "\n query GetReposByOrg($org: ID! $repo_cursor: String!) {\n group(fullPath: $org) {\n projects(first: 100 after: $repo_cursor) {\n \n count\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n repo_id: id\n repo_name: name\n repo_path: path\n ... on Project {\n repository {\n rootRef\n }\n }\n stars: starCount\n forks: forksCount\n created_at: createdAt\n last_activity_at: lastActivityAt\n languages {\n name\n }\n issues: issueStatusCounts {\n all\n closed\n opened\n }\n namespace {\n path: fullPath\n }\n repo_url: webUrl\n }\n }\n }\n }\n }" +# `search_for_code()` works + + Code + gl_search_repos_by_code <- test_rest_gitlab_priv$search_for_code(code = "test", + filename = "TESTFILE", verbose = TRUE, page_max = 2) + Message + i Searching for code [test]... + +# `search_repos_for_code()` works + + Code + gl_search_repos_by_code <- test_rest_gitlab_priv$search_repos_for_code(code = "test", + repos = "TestRepo", filename = "TESTFILE", verbose = TRUE, page_max = 2) + Message + i Searching for code [test]... + # get_repos_from_org prints proper message Code diff --git a/tests/testthat/_snaps/02-get_commits-GitLab.md b/tests/testthat/_snaps/02-get_commits-GitLab.md new file mode 100644 index 00000000..13418dda --- /dev/null +++ b/tests/testthat/_snaps/02-get_commits-GitLab.md @@ -0,0 +1,16 @@ +# get_commits_from_orgs works + + Code + gl_commits_table <- gitlab_testhost_priv$get_commits_from_orgs(since = "2023-03-01", + until = "2023-04-01", verbose = TRUE, progress = FALSE) + Message + i [Host:GitLab][Engine:REST][Scope:mbtests] Pulling commits... + +# get_commits_from_repos works + + Code + gl_commits_table <- gitlab_testhost_priv$get_commits_from_repos(since = "2023-03-01", + until = "2023-04-01", verbose = TRUE, progress = FALSE) + Message + i [Host:GitLab][Engine:REST][Scope:test_org/TestRepo] Pulling commits... + diff --git a/tests/testthat/_snaps/get_release-GitLab.md b/tests/testthat/_snaps/get_release-GitLab.md index 7c4fb7a8..b456d7ca 100644 --- a/tests/testthat/_snaps/get_release-GitLab.md +++ b/tests/testthat/_snaps/get_release-GitLab.md @@ -5,3 +5,11 @@ Output [1] "query GetReleasesFromRepo($project_path: ID!) {\n project(fullPath: $project_path) {\n name\n webUrl\n \t\t\t\t\t\treleases {\n nodes{\n name\n tagName\n releasedAt\n links {\n selfUrl\n }\n description\n }\n }\n }\n }" +# `get_release_logs_from_repos()` works + + Code + releases_from_repos <- gitlab_testhost_priv$get_release_logs_from_repos(since = "2023-05-01", + until = "2023-09-30", verbose = TRUE, progress = FALSE) + Message + i [Host:GitLab][Engine:GraphQl][Scope:test_org/TestRepo] Pulling release logs... + diff --git a/tests/testthat/_snaps/get_urls_repos-GitHub.md b/tests/testthat/_snaps/get_urls_repos-GitHub.md index aeb16a31..34365484 100644 --- a/tests/testthat/_snaps/get_urls_repos-GitHub.md +++ b/tests/testthat/_snaps/get_urls_repos-GitHub.md @@ -6,3 +6,16 @@ Message i [Host:GitHub][Engine:REST][Scope:test_org] Pulling repositories (URLs)... +# get_repos_urls_from_repos prepares web repo_urls vector + + Code + gh_repos_urls <- github_testhost_priv$get_repos_urls_from_repos(type = "web", + verbose = TRUE, progress = FALSE) + Message + i [Host:GitHub][Engine:REST][Scope:test_org] Pulling repositories (URLs)... + +# get_all_repos_urls prepares web repo_urls vector + + Code + gh_repos_urls <- github_testhost_priv$get_all_repos_urls(type = "web", verbose = TRUE) + diff --git a/tests/testthat/helper-expect-responses.R b/tests/testthat/helper-expect-responses.R index 32dba117..acffaad3 100644 --- a/tests/testthat/helper-expect-responses.R +++ b/tests/testthat/helper-expect-responses.R @@ -96,7 +96,7 @@ expect_gl_commit_rest_response <- function(object) { "list" ) expect_list_contains( - object[[1]], + object, c( "id", "short_id", "created_at", "parent_ids", "title", "message", "author_name", "author_email", "authored_date", "committer_name", diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index 8d7e1ee8..a464927c 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -311,9 +311,7 @@ gitlab_commit <- list( ) ) -test_fixtures$gitlab_commits_response <- list( - rep(gitlab_commit, 5) -) +test_fixtures$gitlab_commits_response <- rep(list(gitlab_commit), 5) test_fixtures$github_file_response <- list( "data" = list( diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index 3036fb20..ee7798f6 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -578,9 +578,11 @@ test_that("get_repos_from_repos works", { ) github_testhost_priv$searching_scope <- c("org", "repo") github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") - gh_repos_individual <- github_testhost_priv$get_repos_from_repos( - verbose = FALSE, - progress = FALSE + expect_snapshot( + gh_repos_individual <- github_testhost_priv$get_repos_from_repos( + verbose = TRUE, + progress = FALSE + ) ) expect_repos_table( gh_repos_individual @@ -679,7 +681,7 @@ test_that("`get_repos_contributors()` adds contributors to repos table", { ) gh_repos_with_contributors <- test_rest_github$get_repos_contributors( repos_table = test_mocker$use("gh_repos_table_with_platform"), - progress = FALSE + progress = FALSE ) expect_repos_table( gh_repos_with_contributors, @@ -698,10 +700,12 @@ test_that("`get_repos_contributors()` works on GitHost level", { "rest_engine$get_repos_contributors", test_mocker$use("gh_repos_with_contributors") ) - gh_repos_with_contributors <- github_testhost_priv$get_repos_contributors( - repos_table = test_mocker$use("gh_repos_table_with_platform"), - verbose = FALSE, - progress = FALSE + expect_snapshot( + gh_repos_with_contributors <- github_testhost_priv$get_repos_contributors( + repos_table = test_mocker$use("gh_repos_table_with_platform"), + verbose = TRUE, + progress = FALSE + ) ) expect_repos_table( gh_repos_with_contributors, diff --git a/tests/testthat/test-01-get_repos-GitLab.R b/tests/testthat/test-01-get_repos-GitLab.R index f4c40890..695eb38d 100644 --- a/tests/testthat/test-01-get_repos-GitLab.R +++ b/tests/testthat/test-01-get_repos-GitLab.R @@ -119,6 +119,39 @@ test_that("`get_repos_from_org()` does not fail when GraphQL response is not com ) }) +test_that("`search_for_code()` works", { + mockery::stub( + test_rest_gitlab_priv$search_for_code, + "self$response", + list() + ) + expect_snapshot( + gl_search_repos_by_code <- test_rest_gitlab_priv$search_for_code( + code = "test", + filename = "TESTFILE", + verbose = TRUE, + page_max = 2 + ) + ) +}) + +test_that("`search_repos_for_code()` works", { + mockery::stub( + test_rest_gitlab_priv$search_repos_for_code, + "self$response", + list() + ) + expect_snapshot( + gl_search_repos_by_code <- test_rest_gitlab_priv$search_repos_for_code( + code = "test", + repos = "TestRepo", + filename = "TESTFILE", + verbose = TRUE, + page_max = 2 + ) + ) +}) + test_that("`map_search_into_repos()` works", { gl_search_response <- test_fixtures$gitlab_search_response test_mocker$cache(gl_search_response) @@ -132,7 +165,7 @@ test_that("`map_search_into_repos()` works", { test_mocker$cache(gl_search_repos_by_code) }) -test_that("`get_repos_languages` works", { +test_that("`get_repos_languages()` works", { repos_list <- test_mocker$use("gl_search_repos_by_code") repos_list[[1]]$id <- "45300912" mockery::stub( @@ -140,12 +173,63 @@ test_that("`get_repos_languages` works", { "self$response", test_fixtures$gitlab_languages_response ) - repos_list_with_languages <- test_rest_gitlab_priv$get_repos_languages( + gl_repos_list_with_languages <- test_rest_gitlab_priv$get_repos_languages( repos_list = repos_list, progress = FALSE ) - purrr::walk(repos_list_with_languages, ~ expect_list_contains(., "languages")) - expect_equal(repos_list_with_languages[[1]]$languages, c("Python", "R")) + purrr::walk(gl_repos_list_with_languages, ~ expect_list_contains(., "languages")) + expect_equal(gl_repos_list_with_languages[[1]]$languages, c("Python", "R")) + test_mocker$cache(gl_repos_list_with_languages) +}) + +test_that("`get_repos_by_code()` works", { + mockery::stub( + test_rest_gitlab$get_repos_by_code, + "private$search_for_code", + test_fixtures$gitlab_search_response + ) + mockery::stub( + test_rest_gitlab$get_repos_by_code, + "private$map_search_into_repos", + test_mocker$use("gl_search_repos_by_code") + ) + mockery::stub( + test_rest_gitlab$get_repos_by_code, + "private$get_repos_languages", + test_mocker$use("gl_repos_list_with_languages") + ) + gl_repos_by_code <- test_rest_gitlab$get_repos_by_code( + code = "test", + org = "test_org" + ) + expect_gl_repos_rest_response( + gl_repos_by_code + ) +}) + +test_that("`get_repos_by_code()` works", { + mockery::stub( + test_rest_gitlab$get_repos_by_code, + "private$search_repos_for_code", + test_fixtures$gitlab_search_response + ) + mockery::stub( + test_rest_gitlab$get_repos_by_code, + "private$map_search_into_repos", + test_mocker$use("gl_search_repos_by_code") + ) + mockery::stub( + test_rest_gitlab$get_repos_by_code, + "private$get_repos_languages", + test_mocker$use("gl_repos_list_with_languages") + ) + gl_repos_by_code <- test_rest_gitlab$get_repos_by_code( + code = "test", + repos = c("TestRepo1", "TestRepo2") + ) + expect_gl_repos_rest_response( + gl_repos_by_code + ) }) test_that("`prepare_repos_table()` prepares repos table", { @@ -186,7 +270,10 @@ test_that("GitHost adds `repo_api_url` column to GitLab repos table", { test_that("`tailor_repos_response()` tailors precisely `repos_list`", { gl_repos_by_code <- test_mocker$use("gl_search_repos_by_code") gl_repos_by_code_tailored <- - test_rest_gitlab$tailor_repos_response(gl_repos_by_code) + test_rest_gitlab$tailor_repos_response( + repos_response = gl_repos_by_code, + output = "table_full" + ) gl_repos_by_code_tailored %>% expect_type("list") %>% expect_length(length(gl_repos_by_code)) @@ -206,6 +293,27 @@ test_that("`tailor_repos_response()` tailors precisely `repos_list`", { test_mocker$cache(gl_repos_by_code_tailored) }) +test_that("`tailor_repos_response()` tailors precisely to minimal `repos_list`", { + gl_repos_by_code <- test_mocker$use("gl_search_repos_by_code") + gl_repos_by_code_tailored <- + test_rest_gitlab$tailor_repos_response( + repos_response = gl_repos_by_code, + output = "table_min" + ) + gl_repos_by_code_tailored %>% + expect_type("list") %>% + expect_length(length(gl_repos_by_code)) + + expect_list_contains_only( + gl_repos_by_code_tailored[[1]], + c("repo_id", "repo_name", "created_at", "default_branch", "organization") + ) + expect_lt( + length(gl_repos_by_code_tailored[[1]]), + length(gl_repos_by_code[[1]]) + ) +}) + test_that("REST client prepares table from GitLab repositories response", { gl_repos_by_code_table <- test_rest_gitlab$prepare_repos_table( repos_list = test_mocker$use("gl_repos_by_code_tailored"), diff --git a/tests/testthat/test-02-get_commits-GitLab.R b/tests/testthat/test-02-get_commits-GitLab.R index e3e45d37..8ee44a3f 100644 --- a/tests/testthat/test-02-get_commits-GitLab.R +++ b/tests/testthat/test-02-get_commits-GitLab.R @@ -1,17 +1,34 @@ -test_that("`get_commits_from_repos()` pulls commits from repo", { +test_that("`get_commits_from_one_repo()` pulls commits from repository", { + mockery::stub( + test_rest_gitlab_priv$get_commits_from_one_repo, + "private$paginate_results", + test_fixtures$gitlab_commits_response + ) + gl_commits_repo <- test_rest_gitlab_priv$get_commits_from_one_repo( + repo_path = "TestRepo", + since = "2023-01-01", + until = "2023-04-20" + ) + expect_gt(length(gl_commits_repo), 1) + purrr::walk(gl_commits_repo, ~ expect_gl_commit_rest_response(.)) + test_mocker$cache(gl_commits_repo) +}) + +test_that("`get_commits_from_repos()` pulls commits from repositories", { mockery::stub( test_rest_gitlab$get_commits_from_repos, "private$get_commits_from_one_repo", - test_fixtures$gitlab_commits_response + test_mocker$use("gl_commits_repo") ) - repos_names <- c("mbtests%2Fgitstatstesting", "mbtests%2Fgitstats-testing-2") + repos_names <- c("test_org/TestRepo1", "test_org/TestRepo2") gl_commits_org <- test_rest_gitlab$get_commits_from_repos( repos_names = repos_names, since = "2023-01-01", until = "2023-04-20", progress = FALSE ) - purrr::walk(gl_commits_org, ~ expect_gl_commit_rest_response(.)) + expect_equal(names(gl_commits_org), c("test_org/TestRepo1", "test_org/TestRepo2")) + purrr::walk(gl_commits_org[[1]], ~ expect_gl_commit_rest_response(.)) test_mocker$cache(gl_commits_org) }) @@ -126,11 +143,11 @@ test_that("get_commits_from_orgs works", { "rest_engine$get_commits_authors_handles_and_names", test_mocker$use("gl_commits_table") ) - suppressMessages( + expect_snapshot( gl_commits_table <- gitlab_testhost_priv$get_commits_from_orgs( since = "2023-03-01", until = "2023-04-01", - verbose = FALSE, + verbose = TRUE, progress = FALSE ) ) @@ -139,3 +156,35 @@ test_that("get_commits_from_orgs works", { ) test_mocker$cache(gl_commits_table) }) + +test_that("get_commits_from_repos works", { + gitlab_testhost_priv <- create_gitlab_testhost( + repos = "TestRepo", + mode = "private" + ) + test_org <- "test_org" + attr(test_org, "type") <- "organization" + mockery::stub( + gitlab_testhost_priv$get_repos_from_repos, + "private$set_owner_type", + test_org + ) + mockery::stub( + gitlab_testhost_priv$get_commits_from_repos, + "rest_engine$get_commits_authors_handles_and_names", + test_mocker$use("gl_commits_table") + ) + gitlab_testhost_priv$searching_scope <- "repo" + gitlab_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + expect_snapshot( + gl_commits_table <- gitlab_testhost_priv$get_commits_from_repos( + since = "2023-03-01", + until = "2023-04-01", + verbose = TRUE, + progress = FALSE + ) + ) + expect_commits_table( + gl_commits_table + ) +}) diff --git a/tests/testthat/test-get_release-GitLab.R b/tests/testthat/test-get_release-GitLab.R index 06e73b5d..05d8eb4f 100644 --- a/tests/testthat/test-get_release-GitLab.R +++ b/tests/testthat/test-get_release-GitLab.R @@ -80,11 +80,13 @@ test_that("`get_release_logs_from_repos()` works", { ) gitlab_testhost_priv$searching_scope <- "repo" gitlab_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") - releases_from_repos <- gitlab_testhost_priv$get_release_logs_from_repos( - since = "2023-05-01", - until = "2023-09-30", - verbose = FALSE, - progress = FALSE + expect_snapshot( + releases_from_repos <- gitlab_testhost_priv$get_release_logs_from_repos( + since = "2023-05-01", + until = "2023-09-30", + verbose = TRUE, + progress = FALSE + ) ) expect_releases_table(releases_from_repos) test_mocker$cache(releases_from_repos) diff --git a/tests/testthat/test-get_urls_repos-GitHub.R b/tests/testthat/test-get_urls_repos-GitHub.R index 0a9c008e..e474b445 100644 --- a/tests/testthat/test-get_urls_repos-GitHub.R +++ b/tests/testthat/test-get_urls_repos-GitHub.R @@ -88,10 +88,12 @@ test_that("get_repos_urls_from_repos prepares web repo_urls vector", { ) github_testhost_priv$searching_scope <- c("repo") github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") - gh_repos_urls <- github_testhost_priv$get_repos_urls_from_repos( - type = "web", - verbose = FALSE, - progress = FALSE + expect_snapshot( + gh_repos_urls <- github_testhost_priv$get_repos_urls_from_repos( + type = "web", + verbose = TRUE, + progress = FALSE + ) ) expect_gt(length(gh_repos_urls), 0) expect_true(any(grepl("test-org", gh_repos_urls))) @@ -110,9 +112,11 @@ test_that("get_all_repos_urls prepares web repo_urls vector", { "private$get_repos_urls_from_repos", test_mocker$use("gh_repos_urls") ) - gh_repos_urls <- github_testhost_priv$get_all_repos_urls( - type = "web", - verbose = FALSE + expect_snapshot( + gh_repos_urls <- github_testhost_priv$get_all_repos_urls( + type = "web", + verbose = TRUE + ) ) expect_gt(length(gh_repos_urls), 0) expect_true(any(grepl("test-org", gh_repos_urls))) From 88ed0d6840e7f039f06ee55bcfc45744d45ba842 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 20 Dec 2024 12:07:14 +0000 Subject: [PATCH 27/27] Add NEWS. --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index eaa5c2d8..62178505 100644 --- a/NEWS.md +++ b/NEWS.md @@ -14,6 +14,7 @@ - Fixed pulling commits for GitLab subgroups when repositories are set as scope to scan ([#551](https://github.com/r-world-devs/GitStats/issues/551)). - Filled more information on `author_name` and `author_login` if it was missing in `commits_table` ([#550](https://github.com/r-world-devs/GitStats/issues/550)). - Handled a `GraphQL` response error when pulling repositories with R error. Earlier, `GitStats` just returned empty table with no clue on what has happened, as errors from `GraphQL` are returned as list outputs (they do not break code). +- Fixed getting R package usage when repositories are set ([#548](https://github.com/r-world-devs/GitStats/issues/548)). # GitStats 2.1.2