diff --git a/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt b/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt index 8b31110..86d48d0 100644 --- a/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt +++ b/src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt @@ -28,7 +28,11 @@ interface GitHubClient { ): GitHubAppResponse @GetMapping("/api/v3/app/installations") - fun getInstallations(@RequestHeader("Authorization") jwt: String): List + fun getInstallations( + @RequestHeader("Authorization") jwt: String, + @RequestParam("per_page", defaultValue = "30") perPage: Int, + @RequestParam("page", defaultValue = "1") page: Int + ): List @GetMapping("/api/v3/app/installations/{installationId}") fun getInstallation( @@ -43,7 +47,11 @@ interface GitHubClient { ): InstallationTokenResponse @GetMapping("/api/v3/organizations") - fun getOrganizations(@RequestHeader("Authorization") token: String): List + fun getOrganizations( + @RequestHeader("Authorization") jwt: String, + @RequestParam("per_page", defaultValue = "30") perPage: Int, + @RequestParam("since", defaultValue = "1") since: Int + ): List @GetMapping("/api/v3/orgs/{org}/repos") fun getRepositories( diff --git a/src/main/kotlin/net/leanix/githubagent/services/GitHubAPIService.kt b/src/main/kotlin/net/leanix/githubagent/services/GitHubAPIService.kt new file mode 100644 index 0000000..3e57035 --- /dev/null +++ b/src/main/kotlin/net/leanix/githubagent/services/GitHubAPIService.kt @@ -0,0 +1,42 @@ +package net.leanix.githubagent.services + +import net.leanix.githubagent.client.GitHubClient +import net.leanix.githubagent.dto.Installation +import net.leanix.githubagent.dto.Organization +import org.springframework.stereotype.Service + +@Service +class GitHubAPIService( + private val gitHubClient: GitHubClient, +) { + + companion object { + private const val PAGE_SIZE = 30 // Maximum allowed by GitHub API is 100 + } + + fun getPaginatedInstallations(jwtToken: String): List { + val installations = mutableListOf() + var page = 1 + var currentInstallations: List + + do { + currentInstallations = gitHubClient.getInstallations("Bearer $jwtToken", PAGE_SIZE, page) + if (currentInstallations.isNotEmpty()) installations.addAll(currentInstallations) else break + page++ + } while (currentInstallations.size == PAGE_SIZE) + return installations + } + + fun getPaginatedOrganizations(installationToken: String): List { + val organizations = mutableListOf() + var since = 1 + var currentOrganizations: List + + do { + currentOrganizations = gitHubClient.getOrganizations("Bearer $installationToken", PAGE_SIZE, since) + if (currentOrganizations.isNotEmpty()) organizations.addAll(currentOrganizations) else break + since = currentOrganizations.last().id + } while (currentOrganizations.size == PAGE_SIZE) + return organizations + } +} diff --git a/src/main/kotlin/net/leanix/githubagent/services/GitHubAuthenticationService.kt b/src/main/kotlin/net/leanix/githubagent/services/GitHubAuthenticationService.kt index 50b6391..a76395e 100644 --- a/src/main/kotlin/net/leanix/githubagent/services/GitHubAuthenticationService.kt +++ b/src/main/kotlin/net/leanix/githubagent/services/GitHubAuthenticationService.kt @@ -6,6 +6,7 @@ import net.leanix.githubagent.client.GitHubClient import net.leanix.githubagent.config.GitHubEnterpriseProperties import net.leanix.githubagent.dto.Installation import net.leanix.githubagent.exceptions.FailedToCreateJWTException +import net.leanix.githubagent.exceptions.JwtTokenNotFound import org.bouncycastle.jce.provider.BouncyCastleProvider import org.slf4j.LoggerFactory import org.springframework.core.io.ResourceLoader @@ -26,7 +27,8 @@ class GitHubAuthenticationService( private val githubEnterpriseProperties: GitHubEnterpriseProperties, private val resourceLoader: ResourceLoader, private val gitHubEnterpriseService: GitHubEnterpriseService, - private val gitHubClient: GitHubClient + private val gitHubClient: GitHubClient, + private val gitHubAPIService: GitHubAPIService, ) { companion object { @@ -38,11 +40,9 @@ class GitHubAuthenticationService( fun refreshTokens() { generateAndCacheJwtToken() - val jwtToken = cachingService.get("jwtToken") - generateAndCacheInstallationTokens( - gitHubClient.getInstallations("Bearer $jwtToken"), - jwtToken.toString() - ) + val jwtToken = cachingService.get("jwtToken") ?: throw JwtTokenNotFound() + val installations = gitHubAPIService.getPaginatedInstallations(jwtToken.toString()) + generateAndCacheInstallationTokens(installations, jwtToken.toString()) } fun generateAndCacheJwtToken() { diff --git a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt index de6a324..e51d24b 100644 --- a/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt +++ b/src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt @@ -29,6 +29,7 @@ class GitHubScanningService( private val syncLogService: SyncLogService, private val rateLimitHandler: RateLimitHandler, private val gitHubEnterpriseService: GitHubEnterpriseService, + private val gitHubAPIService: GitHubAPIService, ) { private val logger = LoggerFactory.getLogger(GitHubScanningService::class.java) @@ -67,7 +68,7 @@ class GitHubScanningService( } private fun getInstallations(jwtToken: String): List { - val installations = gitHubClient.getInstallations("Bearer $jwtToken") + val installations = gitHubAPIService.getPaginatedInstallations(jwtToken) gitHubAuthenticationService.generateAndCacheInstallationTokens(installations, jwtToken) return installations } @@ -81,7 +82,7 @@ class GitHubScanningService( return } val installationToken = cachingService.get("installationToken:${installations.first().id}") - val organizations = gitHubClient.getOrganizations("Bearer $installationToken") + val organizations = gitHubAPIService.getPaginatedOrganizations(installationToken.toString()) .map { organization -> if (installations.find { it.account.login == organization.login } != null) { OrganizationDto(organization.id, organization.login, true) diff --git a/src/test/kotlin/net/leanix/githubagent/services/GitHubAPIServiceTest.kt b/src/test/kotlin/net/leanix/githubagent/services/GitHubAPIServiceTest.kt new file mode 100644 index 0000000..d9bda62 --- /dev/null +++ b/src/test/kotlin/net/leanix/githubagent/services/GitHubAPIServiceTest.kt @@ -0,0 +1,81 @@ +package net.leanix.githubagent.services + +import io.mockk.every +import io.mockk.mockk +import net.leanix.githubagent.client.GitHubClient +import net.leanix.githubagent.dto.Account +import net.leanix.githubagent.dto.Installation +import net.leanix.githubagent.dto.Organization +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class GitHubAPIServiceTest { + + private val gitHubClient = mockk() + private val gitHubAPIService = GitHubAPIService(gitHubClient) + + private val permissions = mapOf("administration" to "read", "contents" to "read", "metadata" to "read") + private val events = listOf("label", "public", "repository", "push") + + @Test + fun `test getPaginatedInstallations with one page`() { + val jwtToken = "test-jwt-token" + val installationsPage1 = listOf( + Installation(1, Account("test-account"), permissions, events), + Installation(2, Account("test-account"), permissions, events) + ) + + every { gitHubClient.getInstallations(any(), any(), any()) } returns installationsPage1 + + val installations = gitHubAPIService.getPaginatedInstallations(jwtToken) + assertEquals(2, installations.size) + assertEquals(installationsPage1, installations) + } + + @Test + fun `test getPaginatedInstallations with multiple pages`() { + val jwtToken = "test-jwt-token" + val perPage = 30 + val totalInstallations = 100 + val installations = (1..totalInstallations).map { + Installation(it.toLong(), Account("test-account-$it"), permissions, events) + } + val pages = installations.chunked(perPage) + + every { gitHubClient.getInstallations(any(), any(), any()) } returnsMany pages + listOf(emptyList()) + + val result = gitHubAPIService.getPaginatedInstallations(jwtToken) + assertEquals(totalInstallations, result.size) + assertEquals(installations, result) + } + + @Test + fun `test getPaginatedOrganizations with one page`() { + val installationToken = "test-installation-token" + val organizationsPage1 = listOf( + Organization("org-1", 1), + Organization("org-2", 2) + ) + + every { gitHubClient.getOrganizations(any(), any(), any()) } returns organizationsPage1 + + val organizations = gitHubAPIService.getPaginatedOrganizations(installationToken) + assertEquals(2, organizations.size) + assertEquals(organizationsPage1, organizations) + } + + @Test + fun `test getPaginatedOrganizations with multiple pages`() { + val installationToken = "test-installation-token" + val perPage = 30 + val totalOrganizations = 100 + val organizations = (1..totalOrganizations).map { Organization("org-$it", it) } + val pages = organizations.chunked(perPage) + + every { gitHubClient.getOrganizations(any(), any(), any()) } returnsMany pages + listOf(emptyList()) + + val result = gitHubAPIService.getPaginatedOrganizations(installationToken) + assertEquals(totalOrganizations, result.size) + assertEquals(organizations, result) + } +} diff --git a/src/test/kotlin/net/leanix/githubagent/services/GitHubAuthenticationServiceTest.kt b/src/test/kotlin/net/leanix/githubagent/services/GitHubAuthenticationServiceTest.kt index cf694df..0471c36 100644 --- a/src/test/kotlin/net/leanix/githubagent/services/GitHubAuthenticationServiceTest.kt +++ b/src/test/kotlin/net/leanix/githubagent/services/GitHubAuthenticationServiceTest.kt @@ -21,12 +21,14 @@ class GitHubAuthenticationServiceTest { private val gitHubEnterpriseService = mockk() private val gitHubClient = mockk() private val syncLogService = mockk() + private val gitHubAPIService = mockk() private val githubAuthenticationService = GitHubAuthenticationService( cachingService, githubEnterpriseProperties, resourceLoader, gitHubEnterpriseService, - gitHubClient + gitHubClient, + gitHubAPIService ) @BeforeEach diff --git a/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt b/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt index df1a420..4e0fe61 100644 --- a/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt +++ b/src/test/kotlin/net/leanix/githubagent/services/GitHubScanningServiceTest.kt @@ -35,6 +35,7 @@ class GitHubScanningServiceTest { private val gitHubAuthenticationService = mockk() private val syncLogService = mockk(relaxUnitFun = true) private val rateLimitHandler = mockk(relaxUnitFun = true) + private val gitHubAPIService = mockk() private val gitHubEnterpriseService = GitHubEnterpriseService(gitHubClient, syncLogService) private val gitHubScanningService = GitHubScanningService( gitHubClient, @@ -45,6 +46,7 @@ class GitHubScanningServiceTest { syncLogService, rateLimitHandler, gitHubEnterpriseService, + gitHubAPIService ) private val runId = UUID.randomUUID() @@ -54,13 +56,13 @@ class GitHubScanningServiceTest { @BeforeEach fun setup() { every { cachingService.get(any()) } returns "value" - every { gitHubClient.getInstallations(any()) } returns listOf( + every { gitHubAPIService.getPaginatedInstallations(any()) } returns listOf( Installation(1, Account("testInstallation"), permissions, events) ) every { gitHubClient.createInstallationToken(1, any()) } returns InstallationTokenResponse("testToken", "2024-01-01T00:00:00Z", mapOf(), "all") every { cachingService.set(any(), any(), any()) } returns Unit - every { gitHubClient.getOrganizations(any()) } returns listOf(Organization("testOrganization", 1)) + every { gitHubAPIService.getPaginatedOrganizations(any()) } returns listOf(Organization("testOrganization", 1)) every { gitHubGraphQLService.getRepositories(any(), any()) } returns PagedRepositories( repositories = emptyList(), hasNextPage = false, @@ -92,7 +94,7 @@ class GitHubScanningServiceTest { val runId = UUID.randomUUID() every { cachingService.get("runId") } returns runId - every { gitHubClient.getInstallations(any()) } returns emptyList() + every { gitHubAPIService.getPaginatedInstallations(any()) } returns emptyList() gitHubScanningService.scanGitHubResources() @@ -236,7 +238,7 @@ class GitHubScanningServiceTest { @Test fun `scanGitHubResources should skip organizations without correct permissions and events`() { every { cachingService.get("runId") } returns runId - every { gitHubClient.getInstallations(any()) } returns listOf( + every { gitHubAPIService.getPaginatedInstallations(any()) } returns listOf( Installation(1, Account("testInstallation1"), mapOf(), listOf()), Installation(2, Account("testInstallation2"), permissions, events), Installation(3, Account("testInstallation3"), permissions, events) diff --git a/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt b/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt index 683b815..7e4bda2 100644 --- a/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt +++ b/src/test/kotlin/net/leanix/githubagent/services/WebhookEventServiceTest.kt @@ -37,11 +37,14 @@ class WebhookEventServiceTest { @MockkBean private lateinit var gitHubAuthenticationService: GitHubAuthenticationService + @Autowired + private lateinit var webhookEventService: WebhookEventService + @MockkBean private lateinit var gitHubClient: GitHubClient - @Autowired - private lateinit var webhookEventService: WebhookEventService + @MockkBean + private lateinit var gitHubAPIService: GitHubAPIService private val permissions = mapOf("administration" to "read", "contents" to "read", "metadata" to "read") private val events = listOf("label", "public", "repository", "push") @@ -53,7 +56,7 @@ class WebhookEventServiceTest { every { webSocketService.sendMessage(any(), any()) } returns Unit every { cachingService.get(any()) } returns "token" every { gitHubGraphQLService.getManifestFileContent(any(), any(), any(), any()) } returns "content" - every { gitHubClient.getInstallations(any()) } returns listOf(installation) + every { gitHubAPIService.getPaginatedInstallations(any()) } returns listOf(installation) every { gitHubClient.getInstallation(any(), any()) } returns installation } @@ -379,7 +382,8 @@ class WebhookEventServiceTest { every { cachingService.get("runId") } returnsMany listOf("value", null, runId) every { cachingService.set("runId", any(), any()) } just runs every { cachingService.remove("runId") } just runs - every { gitHubClient.getOrganizations(any()) } returns listOf(Organization("testOrganization", 1)) + every { gitHubAPIService.getPaginatedOrganizations(any()) } returns + listOf(Organization("testOrganization", 1)) val eventType = "INSTALLATION" val payload = """{