diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json index 5c0d4dc8c29a..36cfbcd0197a 100644 --- a/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json @@ -211,6 +211,36 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "merge_request_approvals", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "merge_request_notes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "merge_request_pipelines", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_approvals.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_approvals.json new file mode 100644 index 000000000000..5846bd97fbb6 --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_approvals.json @@ -0,0 +1,139 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "iid": { + "type": ["null", "integer"] + }, + "project_id": { + "type": ["null", "integer"] + }, + "merge_request_iid": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "merge_status": { + "type": ["null", "string"] + }, + "approved": { + "type": ["null", "boolean"] + }, + "approvals_required" : { + "type": ["null", "integer"] + }, + "approvals_left": { + "type": ["null", "integer"] + }, + "require_password_to_approve" : { + "type": ["null", "boolean"] + }, + "approved_by": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "user": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "username": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "web_url": { + "type": ["null", "string"] + } + } + } + } + } + }, + "suggested_approvers": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "username": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "web_url": { + "type": ["null", "string"] + } + } + } + }, + "approval_rules_left": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "rule_type": { + "type": ["null", "string"] + } + } + } + }, + "user_has_approved": { + "type": ["null", "boolean"] + }, + "user_can_approve": { + "type": ["null", "boolean"] + }, + "has_approval_rules": { + "type": ["null", "boolean"] + }, + "merge_request_approvers_available": { + "type": ["null", "boolean"] + }, + "multiple_approval_rules_available": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commit_details.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commit_details.json new file mode 100644 index 000000000000..dda71947ab0d --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commit_details.json @@ -0,0 +1,74 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "project_id": { + "type": ["null", "integer"] + }, + "merge_request_iid": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "short_id": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "parent_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "title": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "author_name": { + "type": ["null", "string"] + }, + "author_email": { + "type": ["null", "string"] + }, + "authored_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "committer_name": { + "type": ["null", "string"] + }, + "committer_email": { + "type": ["null", "string"] + }, + "committed_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "trailers": { + "type": ["null", "object"] + }, + "web_url": { + "type": ["null", "string"] + }, + "stats": { + "type": ["null", "object"], + "properties": { + "additions": { + "type": ["null", "integer"] + }, + "deletions": { + "type": ["null", "integer"] + }, + "total": { + "type": ["null", "integer"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commit_merge_details.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commit_merge_details.json new file mode 100644 index 000000000000..dda71947ab0d --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commit_merge_details.json @@ -0,0 +1,74 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "project_id": { + "type": ["null", "integer"] + }, + "merge_request_iid": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "short_id": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "parent_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "title": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "author_name": { + "type": ["null", "string"] + }, + "author_email": { + "type": ["null", "string"] + }, + "authored_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "committer_name": { + "type": ["null", "string"] + }, + "committer_email": { + "type": ["null", "string"] + }, + "committed_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "trailers": { + "type": ["null", "object"] + }, + "web_url": { + "type": ["null", "string"] + }, + "stats": { + "type": ["null", "object"], + "properties": { + "additions": { + "type": ["null", "integer"] + }, + "deletions": { + "type": ["null", "integer"] + }, + "total": { + "type": ["null", "integer"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commit_squash_details.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commit_squash_details.json new file mode 100644 index 000000000000..dda71947ab0d --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commit_squash_details.json @@ -0,0 +1,74 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "project_id": { + "type": ["null", "integer"] + }, + "merge_request_iid": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "short_id": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "parent_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "title": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "author_name": { + "type": ["null", "string"] + }, + "author_email": { + "type": ["null", "string"] + }, + "authored_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "committer_name": { + "type": ["null", "string"] + }, + "committer_email": { + "type": ["null", "string"] + }, + "committed_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "trailers": { + "type": ["null", "object"] + }, + "web_url": { + "type": ["null", "string"] + }, + "stats": { + "type": ["null", "object"], + "properties": { + "additions": { + "type": ["null", "integer"] + }, + "deletions": { + "type": ["null", "integer"] + }, + "total": { + "type": ["null", "integer"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commits.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commits.json index 694b5d7d729b..3e2c68df57ab 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commits.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commits.json @@ -5,194 +5,50 @@ "id": { "type": ["null", "integer"] }, - "iid": { - "type": ["null", "integer"] - }, "project_id": { "type": ["null", "integer"] }, - "title": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] + "merge_request_iid": { + "type": ["null", "integer"] }, - "state": { + "short_id": { "type": ["null", "string"] }, "created_at": { "type": ["null", "string"], "format": "date-time" }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "merged_by": { - "type": ["null", "object"] - }, - "merged_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "closed_by": { - "type": ["null", "object"] - }, - "closed_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "target_branch": { - "type": ["null", "string"] - }, - "source_branch": { - "type": ["null", "string"] - }, - "user_notes_count": { - "type": ["null", "integer"] - }, - "upvotes": { - "type": ["null", "integer"] - }, - "downvotes": { - "type": ["null", "integer"] - }, - "author": { - "type": ["null", "object"] - }, - "assignees": { - "type": ["null", "array"], - "items": { - "type": "object" - } - }, - "assignee": { - "type": ["null", "object"] - }, - "reviewers": { - "type": ["null", "array"], - "items": { - "type": "object" - } - }, - "source_project_id": { - "type": ["null", "integer"] - }, - "target_project_id": { - "type": ["null", "integer"] - }, - "labels": { - "type": ["null", "array"], - "items": { - "type": "string" - } + "parent_ids": { + "type": ["null", "array"] }, - "work_in_progress": { - "type": ["null", "boolean"] - }, - "milestone": { - "type": ["null", "object"] - }, - "merge_when_pipeline_succeeds": { - "type": ["null", "boolean"] - }, - "merge_status": { - "type": ["null", "string"] - }, - "sha": { + "title": { "type": ["null", "string"] }, - "merge_commit_sha": { + "message": { "type": ["null", "string"] }, - "squash_commit_sha": { + "author_email": { "type": ["null", "string"] }, - "discussion_locked": { - "type": ["null", "boolean"] - }, - "should_remove_source_branch": { - "type": ["null", "boolean"] - }, - "force_remove_source_branch": { - "type": ["null", "boolean"] + "authored_date": { + "type": ["null", "string"], + "format": "date-time" }, - "reference": { + "committer_name": { "type": ["null", "string"] }, - "references": { - "type": ["null", "object"] - }, - "web_url": { + "committer_email": { "type": ["null", "string"] }, - "time_stats": { - "type": ["null", "object"] - }, - "squash": { - "type": ["null", "boolean"] - }, - "task_completion_status": { - "type": ["null", "object"] - }, - "has_conflicts": { - "type": ["null", "boolean"] - }, - "blocking_discussions_resolved": { - "type": ["null", "boolean"] - }, - "approvals_before_merge": { - "type": ["null", "boolean", "string", "object"] - }, - "subscribed": { - "type": ["null", "boolean"] - }, - "changes_count": { - "type": ["null", "integer", "string"] - }, - "latest_build_started_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "latest_build_finished_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "first_deployed_to_production_at": { + "committed_date": { "type": ["null", "string"], "format": "date-time" }, - "pipeline": { + "trailers": { "type": ["null", "object"] }, - "head_pipeline": { - "type": ["null", "object", "string", "boolean", "integer"] - }, - "diff_refs": { - "type": ["null", "object"], - "properties": { - "base_sha": { - "type": ["null", "string"] - }, - "head_sha": { - "type": ["null", "string"] - }, - "start_sha": { - "type": ["null", "string"] - } - } - }, - "merge_error": { - "type": ["null", "boolean", "string"] - }, - "first_contribution": { - "type": ["null", "boolean"] - }, - "user": { - "type": ["null", "object"] - }, - "merge_request_iid": { - "type": ["null", "integer"] + "web_url": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_notes.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_notes.json new file mode 100644 index 000000000000..d8e7d43009e8 --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_notes.json @@ -0,0 +1,333 @@ +{ + "$schema":"http://json-schema.org/draft-07/schema#", + "type":"object", + "properties":{ + "id":{ + "type":[ + "null", + "integer" + ] + }, + "project_id":{ + "type":[ + "null", + "integer" + ] + }, + "merge_request_iid":{ + "type":[ + "null", + "integer" + ] + }, + "type":{ + "type":[ + "null", + "string" + ] + }, + "body":{ + "type":[ + "null", + "string" + ] + }, + "author":{ + "type":[ + "null", + "object" + ], + "properties":{ + "id":{ + "type":[ + "null", + "integer" + ] + }, + "name":{ + "type":[ + "null", + "string" + ] + }, + "username":{ + "type":[ + "null", + "string" + ] + }, + "state":{ + "type":[ + "null", + "string" + ] + }, + "avatar_url":{ + "type":[ + "null", + "string" + ] + }, + "web_url":{ + "type":[ + "null", + "string" + ] + } + } + }, + "created_at":{ + "type":[ + "null", + "string" + ], + "format":"date-time" + }, + "updated_at":{ + "type":[ + "null", + "string" + ], + "format":"date-time" + }, + "system":{ + "type":[ + "null", + "boolean" + ] + }, + "noteable_id":{ + "type":[ + "null", + "integer" + ] + }, + "noteable_type":{ + "type":[ + "null", + "string" + ] + }, + "resolvable":{ + "type":[ + "null", + "boolean" + ] + }, + "confidential":{ + "type":[ + "null", + "boolean" + ] + }, + "internal":{ + "type":[ + "null", + "boolean" + ] + }, + "noteable_iid":{ + "type":[ + "null", + "integer" + ] + }, + "commands_changes":{ + "type":[ + "null", + "object" + ] + }, + "commit_id":{ + "type":[ + "null", + "string" + ] + }, + "position":{ + "type":[ + "null", + "object" + ], + "properties":{ + "base_sha":{ + "type":[ + "null", + "string" + ] + }, + "start_sha":{ + "type":[ + "null", + "string" + ] + }, + "head_sha":{ + "type":[ + "null", + "string" + ] + }, + "old_path":{ + "type":[ + "null", + "string" + ] + }, + "new_path":{ + "type":[ + "null", + "string" + ] + }, + "position_type":{ + "type":[ + "null", + "string" + ] + }, + "old_line":{ + "type":[ + "null", + "integer" + ] + }, + "new_line":{ + "type":[ + "null", + "integer" + ] + }, + "line_range":{ + "type":[ + "null", + "object" + ], + "properties":{ + "start":{ + "type":[ + "null", + "object" + ], + "properties":{ + "line_code":{ + "type":[ + "null", + "string" + ] + }, + "type":{ + "type":[ + "null", + "string" + ] + }, + "old_line":{ + "type":[ + "null", + "integer" + ] + }, + "new_line":{ + "type":[ + "null", + "integer" + ] + } + } + }, + "end":{ + "type":[ + "null", + "object" + ], + "properties":{ + "line_code":{ + "type":[ + "null", + "string" + ] + }, + "type":{ + "type":[ + "null", + "string" + ] + }, + "old_line":{ + "type":[ + "null", + "integer" + ] + }, + "new_line":{ + "type":[ + "null", + "integer" + ] + } + } + } + } + } + } + }, + "resolved":{ + "type":[ + "null", + "boolean" + ] + }, + "resolved_by":{ + "type":[ + "null", + "object" + ], + "properties":{ + "id":{ + "type":[ + "null", + "integer" + ] + }, + "name":{ + "type":[ + "null", + "string" + ] + }, + "username":{ + "type":[ + "null", + "string" + ] + }, + "state":{ + "type":[ + "null", + "string" + ] + }, + "avatar_url":{ + "type":[ + "null", + "string" + ] + }, + "web_url":{ + "type":[ + "null", + "string" + ] + } + } + }, + "resolved_at":{ + "type":[ + "null", + "string" + ], + "format":"date-time" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_pipelines.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_pipelines.json new file mode 100644 index 000000000000..2ee1075ea640 --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_pipelines.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "iid": { + "type": ["null", "integer"] + }, + "project_id": { + "type": ["null", "integer"] + }, + "merge_request_iid": { + "type": ["null", "integer"] + }, + "sha": { + "type": ["null", "string"] + }, + "ref": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "web_url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py index ca162994301a..d6f51cf9ce3a 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py @@ -36,6 +36,12 @@ Releases, Tags, Users, + MergeRequestApprovals, + MergeRequestNotes, + MergeRequestPipelines, + MergeRequestCommitDetails, + MergeRequestCommitMergeDetails, + MergeRequestCommitSquashDetails, ) @@ -98,6 +104,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: pipelines = Pipelines(parent_stream=projects, start_date=config["start_date"], **auth_params) merge_requests = MergeRequests(parent_stream=projects, start_date=config["start_date"], **auth_params) epics = Epics(parent_stream=groups, **auth_params) + merge_request_commits = MergeRequestCommits(parent_stream=merge_requests, **auth_params) streams = [ groups, @@ -116,12 +123,18 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: ProjectLabels(parent_stream=projects, **auth_params), GroupLabels(parent_stream=groups, **auth_params), merge_requests, - MergeRequestCommits(parent_stream=merge_requests, **auth_params), + merge_request_commits, Releases(parent_stream=projects, **auth_params), Tags(parent_stream=projects, repository_part=True, **auth_params), pipelines, PipelinesExtended(parent_stream=pipelines, **auth_params), Users(parent_stream=projects, **auth_params), + MergeRequestApprovals(parent_stream=merge_requests, **auth_params), + MergeRequestNotes(parent_stream=merge_requests, **auth_params), + MergeRequestPipelines(parent_stream=merge_requests, **auth_params), + MergeRequestCommitDetails(parent_stream=merge_request_commits, **auth_params), + MergeRequestCommitMergeDetails(parent_stream=merge_requests, **auth_params), + MergeRequestCommitSquashDetails(parent_stream=merge_requests, **auth_params), ] return streams diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py index 43b3190ae9d1..a52fac312089 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py @@ -10,6 +10,7 @@ import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.core import StreamData class GitlabStream(HttpStream, ABC): @@ -18,7 +19,7 @@ class GitlabStream(HttpStream, ABC): flatten_id_keys = [] flatten_list_keys = [] page = 1 - per_page = 50 + per_page = 100 def __init__(self, api_url: str, **kwargs): super().__init__(**kwargs) @@ -74,6 +75,9 @@ def _flatten_id(self, record: Dict[str, Any], target: str): def _flatten_list(self, record: Dict[str, Any], target: str): record[target] = [target_data.get("id") for target_data in record.get(target, [])] + def backoff_time(self, response: requests.Response) -> Optional[float]: + return 61 + class GitlabChildStream(GitlabStream): path_list = ["id"] @@ -253,7 +257,53 @@ class MergeRequests(IncrementalGitlabChildStream): class MergeRequestCommits(GitlabChildStream): path_list = ["project_id", "iid"] - path_template = "projects/{project_id}/merge_requests/{iid}" + path_template = "projects/{project_id}/merge_requests/{iid}/commits" + + def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs): + super().transform(record, stream_slice, **kwargs) + record["project_id"] = stream_slice["project_id"] + record["merge_request_iid"] = stream_slice["iid"] + + return record + +class MergeRequestCommitDetails(GitlabChildStream): + path_list = ["project_id", "short_id", "merge_request_iid"] + path_template = "projects/{project_id}/repository/commits/{short_id}" + + def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs): + super().transform(record, stream_slice, **kwargs) + record["project_id"] = stream_slice["project_id"] + record["merge_request_iid"] = stream_slice["merge_request_iid"] + + return record + + +class MergeRequestCommitMergeDetails(GitlabChildStream): + path_list = ["project_id", "merge_commit_sha", "iid"] + path_template = "projects/{project_id}/repository/commits/{merge_commit_sha}" + + def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs): + super().transform(record, stream_slice, **kwargs) + record["project_id"] = stream_slice["project_id"] + record["merge_request_iid"] = stream_slice["iid"] + + return record + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[StreamData]: + if(stream_slice['merge_commit_sha'] == None): + return [] + else: + return super().read_records(sync_mode, cursor_field, stream_slice, stream_state) + +class MergeRequestCommitSquashDetails(GitlabChildStream): + path_list = ["project_id", "squash_commit_sha", "iid"] + path_template = "projects/{project_id}/repository/commits/{squash_commit_sha}" def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs): super().transform(record, stream_slice, **kwargs) @@ -262,6 +312,19 @@ def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs): return record + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[StreamData]: + if(stream_slice['squash_commit_sha'] == None): + return [] + else: + return super().read_records(sync_mode, cursor_field, stream_slice, stream_state) + + class Releases(GitlabChildStream): primary_key = "name" @@ -328,3 +391,38 @@ class EpicIssues(GitlabChildStream): flatten_id_keys = ["milestone", "assignee", "author"] flatten_list_keys = ["assignees"] path_template = "groups/{group_id}/epics/{id}/issues" + +class MergeRequestApprovals(GitlabChildStream): + path_list = ["project_id", "iid"] + path_template = "projects/{project_id}/merge_requests/{iid}/approvals" + + def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs): + super().transform(record, stream_slice, **kwargs) + record["project_id"] = stream_slice["project_id"] + record["merge_request_iid"] = stream_slice["iid"] + + return record + +class MergeRequestNotes(GitlabChildStream): + path_list = ["project_id", "iid"] + path_template = "projects/{project_id}/merge_requests/{iid}/notes" + + def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs): + super().transform(record, stream_slice, **kwargs) + record["project_id"] = stream_slice["project_id"] + record["merge_request_iid"] = stream_slice["iid"] + + return record + +class MergeRequestPipelines(GitlabChildStream): + path_list = ["project_id", "iid"] + path_template = "projects/{project_id}/merge_requests/{iid}/pipelines" + + def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs): + super().transform(record, stream_slice, **kwargs) + record["project_id"] = stream_slice["project_id"] + record["merge_request_iid"] = stream_slice["iid"] + + return record + +