Merge branch 'airbytehq:master' into master

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.50.3
+current_version = 0.50.4
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.github/workflows/connectors_tests.yml

@@ -20,6 +20,13 @@ jobs:
     timeout-minutes: 1440 # 24 hours
     runs-on: dev-medium-runner
     steps:
+      - name: Check if PR is from a fork
+        if: github.event_name == 'pull_request'
+        run: |
+          if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then
+            echo "PR is from a fork. Exiting workflow..."
+            exit 78
+          fi
       - name: Get start timestamp
         id: get-start-timestamp
         run: echo "::set-output name=start-timestamp::$(date +%s)"

.github/workflows/gradle.yml

@@ -95,11 +95,11 @@ jobs:
         with:
           ref: ${{ github.head_ref }}
 
-      - name: Cache Build Artifacts
-        uses: ./.github/actions/cache-build-artifacts
-        with:
-          cache-key: ${{ secrets.CACHE_VERSION }}
-          cache-python: "false"
+      # - name: Cache Build Artifacts
+      #   uses: ./.github/actions/cache-build-artifacts
+      #   with:
+      #     cache-key: ${{ secrets.CACHE_VERSION }}
+      #     cache-python: "false"
 
       - uses: actions/setup-java@v3
         with:
@@ -251,10 +251,10 @@ jobs:
         with:
           ref: ${{ github.head_ref }}
 
-      - name: Cache Build Artifacts
-        uses: ./.github/actions/cache-build-artifacts
-        with:
-          cache-key: ${{ secrets.CACHE_VERSION }}-connectors-gradle-build
+      # - name: Cache Build Artifacts
+      #   uses: ./.github/actions/cache-build-artifacts
+      #   with:
+      #     cache-key: ${{ secrets.CACHE_VERSION }}-connectors-gradle-build
 
       - uses: actions/setup-java@v3
         with:

airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -955,7 +955,7 @@ definitions:
         enum: [DpathExtractor]
       field_path:
         title: Field Path
-        description: List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array.
+        description: List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).
         type: array
         items:
           - type: string

airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -917,7 +917,7 @@ class DpathExtractor(BaseModel):
     type: Literal["DpathExtractor"]
     field_path: List[str] = Field(
         ...,
-        description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array.',
+        description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).',
         examples=[
             ["data"],
             ["data", "records"],

airbyte-integrations/connectors/destination-convex/metadata.yaml

@@ -14,7 +14,7 @@ data:
     oss:
       enabled: true
   releaseStage: alpha
-  documentationUrl: https://docs.airbyte.io/integrations/destinations/convex
+  documentationUrl: https://docs.airbyte.com/integrations/destinations/convex
   tags:
     - language:python
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-dev-null/metadata.yaml

@@ -14,7 +14,7 @@ data:
     oss:
       enabled: false
   releaseStage: alpha
-  documentationUrl: https://docs.airbyte.io/integrations/destinations/e2e-test
+  documentationUrl: https://docs.airbyte.com/integrations/destinations/e2e-test
   tags:
     - language:java
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-duckdb/metadata.yaml

@@ -14,7 +14,7 @@ data:
     oss:
       enabled: true
   releaseStage: alpha
-  documentationUrl: https://docs.airbyte.io/integrations/destinations/duckdb
+  documentationUrl: https://docs.airbyte.com/integrations/destinations/duckdb
   tags:
     - language:python
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/destination-snowflake/README.md

@@ -1,11 +1,14 @@
 # Snowflake Destination
 
 ## Documentation
-* [User Documentation](https://docs.airbyte.io/integrations/destinations/snowflake)
+
+- [User Documentation](https://docs.airbyte.io/integrations/destinations/snowflake)
 
 ## Community Contributor
+
 1. Look at the integration documentation to see how to create a warehouse/database/schema/user/role for Airbyte to sync into.
 1. Create a file at `secrets/config.json` with the following format:
+
 ```
 {
   "host": "testhost.snowflakecomputing.com",
@@ -25,7 +28,7 @@
 Put the contents of the following LastPass secrets into corresponding files under the `secrets` directory:
 
 | LastPass Secret                                                                                        | File                                     |
-|--------------------------------------------------------------------------------------------------------|------------------------------------------|
+| ------------------------------------------------------------------------------------------------------ | ---------------------------------------- |
 | `destination snowflake - test creds (secrets/config.json)`                                             | `secrets/config.json`                    |
 | `destination snowflake - insert test creds (secrets/insert_config.json)`                               | `secrets/insert_config.json`             |
 | `destination snowflake - internal staging test creds (secrets/internal_staging_config.json)`           | `secrets/internal_staging_config.json`   |
@@ -83,6 +86,7 @@ DESC STORAGE INTEGRATION GCS_AIRBYTE_INTEGRATION;
 That last query (`DESC STORAGE`) will show a `STORAGE_GCP_SERVICE_ACCOUNT` property with an email as the property value. Add read/write permissions to your bucket with that email if it's not already there.
 
 If you ever need to start over, use this:
+
 ```sql
 DROP DATABASE IF EXISTS INTEGRATION_TEST_DESTINATION;
 DROP USER IF EXISTS INTEGRATION_TEST_USER_DESTINATION;

airbyte-integrations/connectors/destination-teradata/metadata.yaml

@@ -14,7 +14,7 @@ data:
     oss:
       enabled: true
   releaseStage: alpha
-  documentationUrl: https://docs.airbyte.io/integrations/destinations/teradata
+  documentationUrl: https://docs.airbyte.com/integrations/destinations/teradata
   tags:
     - language:java
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/source-braze/metadata.yaml

@@ -14,7 +14,7 @@ data:
     oss:
       enabled: true
   releaseStage: alpha
-  documentationUrl: https://docs.airbyte.io/integrations/sources/braze
+  documentationUrl: https://docs.airbyte.com/integrations/sources/braze
   tags:
     - language:low-code
     - language:python

airbyte-integrations/connectors/source-e2e-test-cloud/metadata.yaml

@@ -14,7 +14,7 @@ data:
     oss:
       enabled: false
   releaseStage: alpha
-  documentationUrl: https://docs.airbyte.io/integrations/sources/e2e-test
+  documentationUrl: https://docs.airbyte.com/integrations/sources/e2e-test
   tags:
     - language:java
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/source-firebase-realtime-database/metadata.yaml

@@ -16,7 +16,7 @@ data:
     oss:
       enabled: true
   releaseStage: alpha
-  documentationUrl: https://docs.airbyte.io/integrations/sources/firebase-realtime-database
+  documentationUrl: https://docs.airbyte.com/integrations/sources/firebase-realtime-database
   tags:
     - language:python
 metadataSpecVersion: "1.0"

airbyte-integrations/connectors/source-gitlab/Dockerfile

@@ -13,5 +13,5 @@ COPY main.py ./
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=1.4.2
+LABEL io.airbyte.version=1.5.0
 LABEL io.airbyte.name=airbyte/source-gitlab

airbyte-integrations/connectors/source-gitlab/metadata.yaml

@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80
-  dockerImageTag: 1.4.2
+  dockerImageTag: 1.5.0
   dockerRepository: airbyte/source-gitlab
   githubIssueLabel: source-gitlab
   icon: gitlab.svg
@@ -16,7 +16,7 @@ data:
       enabled: true
     oss:
       enabled: true
-  releaseStage: beta
+  releaseStage: generally_available
   documentationUrl: https://docs.airbyte.com/integrations/sources/gitlab
   tags:
     - language:python

airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py

@@ -91,6 +91,11 @@ def __init__(self, *args, **kwargs):
         self.__groups_stream: Optional[GitlabStream] = None
         self.__projects_stream: Optional[GitlabStream] = None
 
+    @staticmethod
+    def _ensure_default_values(config: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
+        config["api_url"] = config.get("api_url") or "gitlab.com"
+        return config
+
     def _groups_stream(self, config: MutableMapping[str, Any]) -> Groups:
         if not self.__groups_stream:
             auth_params = self._auth_params(config)
@@ -136,6 +141,7 @@ def _is_http_allowed() -> bool:
         return os.environ.get("DEPLOYMENT_MODE", "").upper() != "CLOUD"
 
     def check_connection(self, logger, config) -> Tuple[bool, any]:
+        config = self._ensure_default_values(config)
         is_valid, scheme, _ = parse_url(config["api_url"])
         if not is_valid:
             return False, "Invalid API resource locator."
@@ -151,6 +157,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
             return False, f"Unable to connect to Gitlab API with the provided credentials - {repr(error)}"
 
     def streams(self, config: MutableMapping[str, Any]) -> List[Stream]:
+        config = self._ensure_default_values(config)
         auth_params = self._auth_params(config)
 
         groups, projects = self._groups_stream(config), self._projects_stream(config)

airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json

@@ -4,7 +4,7 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Source Gitlab Spec",
     "type": "object",
-    "required": ["api_url", "start_date", "credentials"],
+    "required": ["start_date", "credentials"],
     "additionalProperties": true,
     "properties": {
       "credentials": {
@@ -73,23 +73,23 @@
           }
         ]
       },
-      "api_url": {
-        "type": "string",
-        "examples": ["gitlab.com", "https://gitlab.com", "https://gitlab.company.org"],
-        "title": "API URL",
-        "default": "gitlab.com",
-        "description": "Please enter your basic URL from GitLab instance.",
-        "order": 1
-      },
       "start_date": {
         "type": "string",
         "title": "Start Date",
         "description": "The date from which you'd like to replicate data for GitLab API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.",
         "examples": ["2021-03-01T00:00:00Z"],
         "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
-        "order": 2,
+        "order": 1,
         "format": "date-time"
       },
+      "api_url": {
+        "type": "string",
+        "examples": ["gitlab.com", "https://gitlab.com", "https://gitlab.company.org"],
+        "title": "API URL",
+        "default": "gitlab.com",
+        "description": "Please enter your basic URL from GitLab instance.",
+        "order": 2
+      },
       "groups": {
         "type": "string",
         "examples": ["airbyte.io"],

airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml

@@ -19,6 +19,12 @@ data:
     oss:
       enabled: true
   releaseStage: generally_available
+  resourceRequirements:
+    jobSpecific:
+      - jobType: check_connection
+        resourceRequirements:
+          memory_limit: 500Mi
+          memory_request: 500Mi
   documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-data-api
   tags:
     - language:python

airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.35
+LABEL io.airbyte.version=0.1.36
 LABEL io.airbyte.name=airbyte/source-google-analytics-v4

airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml

@@ -8,7 +8,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: eff3616a-f9c3-11eb-9a03-0242ac130003
-  dockerImageTag: 0.1.35
+  dockerImageTag: 0.1.36
   dockerRepository: airbyte/source-google-analytics-v4
   githubIssueLabel: source-google-analytics-v4
   icon: google-analytics.svg

airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/custom_reports_validator.py

@@ -7,6 +7,9 @@
 from typing import Dict, List, Optional, Union
 
 from pydantic import BaseModel, Field, ValidationError, validator
+from pydantic.class_validators import root_validator
+
+ERROR_MSG_MISSING_SEGMENT_DIMENSION = "errors: `ga:segment` is required"
 
 
 class Model(BaseModel):
@@ -17,6 +20,7 @@ class Config:
     dimensions: list[str]
     metrics: list[str]
     filter: Optional[str]
+    segments: Optional[list[str]]
 
     @validator("dimensions", "metrics")
     def check_field_reference_forrmat(cls, value):
@@ -28,6 +32,16 @@ def check_field_reference_forrmat(cls, value):
             if "ga:" not in v:
                 raise ValueError(v)
 
+    @classmethod
+    @root_validator(pre=True)
+    def check_segment_included_in_dimension(cls, values):
+        dimensions = values.get("dimensions")
+        segments = values.get("segments")
+        print(segments, dimensions)
+        if segments and "ga:segment" not in dimensions:
+            raise ValueError(ERROR_MSG_MISSING_SEGMENT_DIMENSION)
+        return values
+
 
 class Explainer:
     """

airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py

@@ -10,12 +10,19 @@
 @pytest.mark.parametrize(
     "custom_reports, expected",
     (
-        ([{"name": "test", "dimensions": ["ga+test"], "metrics": ["ga!test"]}], "errors: incorrect field reference"),
-        ([{"name": [], "dimensions": ["ga:test"], "metrics": ["ga:test"]}], "errors: type errors"),
-        ([{"name": "test", "dimensions": ["ga:test"], "metrics": ["ga:test"], "added_field": "test"}], "errors: fields not permitted"),
-        ([{"missing_name": "test", "dimensions": ["ga:test"], "metrics": ["ga:test"]}], "errors: fields required"),
+            ([{"name": "test", "dimensions": ["ga+test"], "metrics": ["ga!test"]}], "errors: incorrect field reference"),
+            ([{"name": [], "dimensions": ["ga:test"], "metrics": ["ga:test"]}], "errors: type errors"),
+            ([{"name": "test", "dimensions": ["ga:test"], "metrics": ["ga:test"], "added_field": "test"}], "errors: fields not permitted"),
+            ([{"name": "missing_segment_dimension", "dimensions": ["ga:test"], "segments": ["another_segment"],"metrics": ["ga:test"]}], "errors: `ga:segment` is required"),
+            ([{"missing_name": "test", "dimensions": ["ga:test"], "metrics": ["ga:test"]}], "errors: fields required"),
     ),
-    ids=["incorrrect field reference", "type_error", "not_permitted", "missing"],
+    ids=[
+        "incorrrect field reference",
+        "type_error",
+        "not_permitted",
+        "missing",
+        "missing_segment_dimension"
+    ]
 )
 def test_custom_reports_validator(custom_reports, expected):
     try:

airbyte-integrations/connectors/source-mixpanel/Dockerfile

@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=0.1.34
+LABEL io.airbyte.version=0.1.35
 LABEL io.airbyte.name=airbyte/source-mixpanel