From c2437ce44ecc5eaee809eb709e3da33142080241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mindaugas=20Ni=C5=BEauskas?= Date: Thu, 4 Jul 2024 09:38:55 +0300 Subject: [PATCH 1/9] create CustomPropertiesStream class --- tap_github/repository_streams.py | 22 ++++++++++++++++++++++ tap_github/streams.py | 2 ++ 2 files changed, 24 insertions(+) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index de8371e8..2766665a 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2441,3 +2441,25 @@ class TrafficPageViewsStream(TrafficRestStream): th.Property("count", th.IntegerType), th.Property("uniques", th.IntegerType), ).to_dict() + + +class CustomPropertiesStream(RepositoryStream): + """Defines 'custom_properties' stream.""" + + name = "custom_properties" + path = "/repos/{org}/{repo}/properties/values" + primary_keys = ["property_name"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = False + state_partitioning_keys = ["repo", "org"] + selected_by_default = True + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Custom Property Keys + th.Property("property_name", th.StringType), + th.Property("value", th.StringType), + ).to_dict() \ No newline at end of file diff --git a/tap_github/streams.py b/tap_github/streams.py index e1b05e58..a0690b9b 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -17,6 +17,7 @@ CommitsStream, CommunityProfileStream, ContributorsStream, + CustomPropertiesStream, DependenciesStream, DependentsStream, EventsStream, @@ -74,6 +75,7 @@ def __init__(self, valid_queries: Set[str], streams: List[Type[Stream]]): CommitsStream, CommunityProfileStream, ContributorsStream, + CustomPropertiesStream, DependenciesStream, DependentsStream, EventsStream, From 65e002e4528d4853c4377a4bb08a184d7464d621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mindaugas=20Ni=C5=BEauskas?= Date: Thu, 4 Jul 2024 09:45:16 +0300 Subject: [PATCH 2/9] add empty line --- tap_github/repository_streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 2766665a..4dce9a0e 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2462,4 +2462,4 @@ class CustomPropertiesStream(RepositoryStream): # Custom Property Keys th.Property("property_name", th.StringType), th.Property("value", th.StringType), - ).to_dict() \ No newline at end of file + ).to_dict() From 14ec150025171a9107dc59c80bc3de61a24097b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mindaugas=20Ni=C5=BEauskas?= Date: Thu, 4 Jul 2024 09:47:10 +0300 Subject: [PATCH 3/9] remove selected_by_default --- tap_github/repository_streams.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 4dce9a0e..17915595 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2452,7 +2452,6 @@ class CustomPropertiesStream(RepositoryStream): parent_stream_type = RepositoryStream ignore_parent_replication_key = False state_partitioning_keys = ["repo", "org"] - selected_by_default = True schema = th.PropertiesList( # Parent Keys From 8ada871fa87c5bb40300405804564126b749849c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mindaugas=20Ni=C5=BEauskas?= Date: Thu, 4 Jul 2024 10:19:37 +0300 Subject: [PATCH 4/9] set replication_key = None --- tap_github/repository_streams.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 17915595..d2942695 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2443,14 +2443,15 @@ class TrafficPageViewsStream(TrafficRestStream): ).to_dict() -class CustomPropertiesStream(RepositoryStream): +class CustomPropertiesStream(GitHubRestStream): """Defines 'custom_properties' stream.""" name = "custom_properties" path = "/repos/{org}/{repo}/properties/values" - primary_keys = ["property_name"] + primary_keys = ["repo", "org", "property_name"] + replication_key = None parent_stream_type = RepositoryStream - ignore_parent_replication_key = False + ignore_parent_replication_key = True state_partitioning_keys = ["repo", "org"] schema = th.PropertiesList( From b8c5650c528c216f0de44f3148aee8954f6822c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mindaugas=20Ni=C5=BEauskas?= Date: Mon, 8 Jul 2024 18:05:16 +0300 Subject: [PATCH 5/9] move CustomPropertiesStream to ORGANIZATIONS --- tap_github/organization_streams.py | 23 +++++++++++++++++++++++ tap_github/repository_streams.py | 22 ---------------------- tap_github/streams.py | 5 ++--- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tap_github/organization_streams.py b/tap_github/organization_streams.py index b4222172..2b12054b 100644 --- a/tap_github/organization_streams.py +++ b/tap_github/organization_streams.py @@ -5,6 +5,7 @@ from singer_sdk import typing as th # JSON Schema typing helpers from tap_github.client import GitHubRestStream +from tap_github.repository_streams import RepositoryStream class OrganizationStream(GitHubRestStream): @@ -177,3 +178,25 @@ class TeamRolesStream(GitHubRestStream): th.Property("role", th.StringType), th.Property("state", th.StringType), ).to_dict() + + +class CustomPropertiesStream(GitHubRestStream): + """Defines 'custom_properties' stream.""" + + name = "custom_properties" + path = "/repos/{org}/{repo}/properties/values" + primary_keys = ["repo", "org", "property_name"] + replication_key = None + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys = ["repo", "org"] + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Custom Property Keys + th.Property("property_name", th.StringType), + th.Property("value", th.StringType), + ).to_dict() diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index d2942695..de8371e8 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2441,25 +2441,3 @@ class TrafficPageViewsStream(TrafficRestStream): th.Property("count", th.IntegerType), th.Property("uniques", th.IntegerType), ).to_dict() - - -class CustomPropertiesStream(GitHubRestStream): - """Defines 'custom_properties' stream.""" - - name = "custom_properties" - path = "/repos/{org}/{repo}/properties/values" - primary_keys = ["repo", "org", "property_name"] - replication_key = None - parent_stream_type = RepositoryStream - ignore_parent_replication_key = True - state_partitioning_keys = ["repo", "org"] - - schema = th.PropertiesList( - # Parent Keys - th.Property("repo", th.StringType), - th.Property("org", th.StringType), - th.Property("repo_id", th.IntegerType), - # Custom Property Keys - th.Property("property_name", th.StringType), - th.Property("value", th.StringType), - ).to_dict() diff --git a/tap_github/streams.py b/tap_github/streams.py index a0690b9b..e6c9eb3d 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -4,6 +4,7 @@ from singer_sdk.streams.core import Stream from tap_github.organization_streams import ( + CustomPropertiesStream, OrganizationStream, TeamMembersStream, TeamRolesStream, @@ -17,7 +18,6 @@ CommitsStream, CommunityProfileStream, ContributorsStream, - CustomPropertiesStream, DependenciesStream, DependentsStream, EventsStream, @@ -75,7 +75,6 @@ def __init__(self, valid_queries: Set[str], streams: List[Type[Stream]]): CommitsStream, CommunityProfileStream, ContributorsStream, - CustomPropertiesStream, DependenciesStream, DependentsStream, EventsStream, @@ -119,7 +118,7 @@ def __init__(self, valid_queries: Set[str], streams: List[Type[Stream]]): ) ORGANIZATIONS = ( {"organizations"}, - [OrganizationStream, TeamMembersStream, TeamRolesStream, TeamsStream], + [CustomPropertiesStream, OrganizationStream, TeamMembersStream, TeamRolesStream, TeamsStream], ) @classmethod From 27024df41833cd07025e3c8ba49d3574a359c10f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mindaugas=20Ni=C5=BEauskas?= Date: Tue, 9 Jul 2024 07:43:49 +0300 Subject: [PATCH 6/9] return CustomPropertiesStream to repository_streams --- tap_github/organization_streams.py | 23 ----------------------- tap_github/repository_streams.py | 22 ++++++++++++++++++++++ tap_github/streams.py | 3 ++- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/tap_github/organization_streams.py b/tap_github/organization_streams.py index 2b12054b..b4222172 100644 --- a/tap_github/organization_streams.py +++ b/tap_github/organization_streams.py @@ -5,7 +5,6 @@ from singer_sdk import typing as th # JSON Schema typing helpers from tap_github.client import GitHubRestStream -from tap_github.repository_streams import RepositoryStream class OrganizationStream(GitHubRestStream): @@ -178,25 +177,3 @@ class TeamRolesStream(GitHubRestStream): th.Property("role", th.StringType), th.Property("state", th.StringType), ).to_dict() - - -class CustomPropertiesStream(GitHubRestStream): - """Defines 'custom_properties' stream.""" - - name = "custom_properties" - path = "/repos/{org}/{repo}/properties/values" - primary_keys = ["repo", "org", "property_name"] - replication_key = None - parent_stream_type = RepositoryStream - ignore_parent_replication_key = True - state_partitioning_keys = ["repo", "org"] - - schema = th.PropertiesList( - # Parent Keys - th.Property("repo", th.StringType), - th.Property("org", th.StringType), - th.Property("repo_id", th.IntegerType), - # Custom Property Keys - th.Property("property_name", th.StringType), - th.Property("value", th.StringType), - ).to_dict() diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index de8371e8..d2942695 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2441,3 +2441,25 @@ class TrafficPageViewsStream(TrafficRestStream): th.Property("count", th.IntegerType), th.Property("uniques", th.IntegerType), ).to_dict() + + +class CustomPropertiesStream(GitHubRestStream): + """Defines 'custom_properties' stream.""" + + name = "custom_properties" + path = "/repos/{org}/{repo}/properties/values" + primary_keys = ["repo", "org", "property_name"] + replication_key = None + parent_stream_type = RepositoryStream + ignore_parent_replication_key = True + state_partitioning_keys = ["repo", "org"] + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Custom Property Keys + th.Property("property_name", th.StringType), + th.Property("value", th.StringType), + ).to_dict() diff --git a/tap_github/streams.py b/tap_github/streams.py index e6c9eb3d..88838ac6 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -4,7 +4,6 @@ from singer_sdk.streams.core import Stream from tap_github.organization_streams import ( - CustomPropertiesStream, OrganizationStream, TeamMembersStream, TeamRolesStream, @@ -18,6 +17,7 @@ CommitsStream, CommunityProfileStream, ContributorsStream, + CustomPropertiesStream, DependenciesStream, DependentsStream, EventsStream, @@ -75,6 +75,7 @@ def __init__(self, valid_queries: Set[str], streams: List[Type[Stream]]): CommitsStream, CommunityProfileStream, ContributorsStream, + CustomPropertiesStream, DependenciesStream, DependentsStream, EventsStream, From c36489e5d66a3cab769c3f59789398ed5a193a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mindaugas=20Ni=C5=BEauskas?= Date: Tue, 9 Jul 2024 12:39:58 +0300 Subject: [PATCH 7/9] add empty line --- tap_github/repository_streams.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index d2942695..92738ce6 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2463,3 +2463,4 @@ class CustomPropertiesStream(GitHubRestStream): th.Property("property_name", th.StringType), th.Property("value", th.StringType), ).to_dict() + From cca1be791e0c97b68e8bdebd89f6a832c15653c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mindaugas=20Ni=C5=BEauskas?= Date: Tue, 9 Jul 2024 12:42:24 +0300 Subject: [PATCH 8/9] rm newline --- tap_github/repository_streams.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 92738ce6..d2942695 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2463,4 +2463,3 @@ class CustomPropertiesStream(GitHubRestStream): th.Property("property_name", th.StringType), th.Property("value", th.StringType), ).to_dict() - From c46aeb51722f4e979fde4ceff33f5b37c346ea12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mindaugas=20Ni=C5=BEauskas?= Date: Tue, 9 Jul 2024 12:51:54 +0300 Subject: [PATCH 9/9] reformat ORGANIZATIONS --- tap_github/streams.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tap_github/streams.py b/tap_github/streams.py index 88838ac6..64ae392a 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -119,7 +119,13 @@ def __init__(self, valid_queries: Set[str], streams: List[Type[Stream]]): ) ORGANIZATIONS = ( {"organizations"}, - [CustomPropertiesStream, OrganizationStream, TeamMembersStream, TeamRolesStream, TeamsStream], + [ + CustomPropertiesStream, + OrganizationStream, + TeamMembersStream, + TeamRolesStream, + TeamsStream, + ], ) @classmethod