Merge branch 'main' into docs/dataproc-deployment

kedro-org · Feb 22, 2025 · 1492e84 · 1492e84
2 parents 0fd2036 + 9c70bae
commit 1492e84
Show file tree

Hide file tree

Showing 56 changed files with 1,661 additions and 6,048 deletions.
diff --git a/.github/styles/Kedro/ignore-names.txt b/.github/styles/Kedro/ignore-names.txt
@@ -94,6 +94,7 @@ Puneet
 Rashida
 Ravi
 Richard
+Schopp
 Schwarzmann
 Sorokin
 Stichbury

diff --git a/.github/styles/Kedro/ignore.txt b/.github/styles/Kedro/ignore.txt
@@ -1,5 +1,6 @@
 Kedro
 Kedro's
+Kedroids
 Kubeflow
 Databricks
 Conda

diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml
@@ -54,7 +54,6 @@ jobs:
       python-version: ${{ matrix.python-version }}
       branch: ${{ inputs.branch }}
 
-
   pip-compile:
     strategy:
       matrix:
@@ -65,3 +64,14 @@ jobs:
       os: ${{ matrix.os }}
       python-version: ${{ matrix.python-version }}
       branch: ${{ inputs.branch }}
+
+  detect-secrets:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.11" ]
+    uses: ./.github/workflows/detect-secrets.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+      branch: ${{ inputs.branch }}
diff --git a/.github/workflows/detect-secrets.yml b/.github/workflows/detect-secrets.yml
@@ -0,0 +1,38 @@
+name: Detect secrets on Kedro
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        type: string
+      python-version:
+        type: string
+      branch:
+        type: string
+        default: ''
+
+jobs:
+  detect-secrets:
+    runs-on: ${{ inputs.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+      - name: Set up Python ${{ inputs.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ inputs.python-version }}
+      - name: Cache python packages
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Install dependencies
+        run: |
+            make install-test-requirements
+            make install-pre-commit
+      - name: pip freeze
+        run: uv pip freeze --system
+      - name: Scan all tracked files
+        run: git ls-files -z | xargs -0 detect-secrets-hook --baseline .secrets.baseline
diff --git a/.github/workflows/docs-language-linter.yml b/.github/workflows/docs-language-linter.yml
@@ -15,3 +15,4 @@ jobs:
       - uses: errata-ai/vale-action@reviewdog
         with:
           reporter: github-pr-check
+          version: 3.9.2 # temp
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -40,11 +40,9 @@ repos:
             pass_filenames: false
             entry: lint-imports
 
-    - repo: local
+    - repo: https://github.com/Yelp/detect-secrets
+      rev: v1.5.0
       hooks:
-          - id: secret_scan
-            name: "Secret scan"
-            language: system
+          - id: detect-secrets
+            args: ['--baseline', '.secrets.baseline']
             exclude: ^features/steps/test_starter
-            pass_filenames: false
-            entry: make secret-scan
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -0,0 +1,223 @@
+{
+  "version": "1.5.0",
+  "plugins_used": [
+    {
+      "name": "ArtifactoryDetector"
+    },
+    {
+      "name": "AWSKeyDetector"
+    },
+    {
+      "name": "AzureStorageKeyDetector"
+    },
+    {
+      "name": "Base64HighEntropyString",
+      "limit": 4.5
+    },
+    {
+      "name": "BasicAuthDetector"
+    },
+    {
+      "name": "CloudantDetector"
+    },
+    {
+      "name": "DiscordBotTokenDetector"
+    },
+    {
+      "name": "GitHubTokenDetector"
+    },
+    {
+      "name": "GitLabTokenDetector"
+    },
+    {
+      "name": "HexHighEntropyString",
+      "limit": 3.0
+    },
+    {
+      "name": "IbmCloudIamDetector"
+    },
+    {
+      "name": "IbmCosHmacDetector"
+    },
+    {
+      "name": "IPPublicDetector"
+    },
+    {
+      "name": "JwtTokenDetector"
+    },
+    {
+      "name": "KeywordDetector",
+      "keyword_exclude": ""
+    },
+    {
+      "name": "MailchimpDetector"
+    },
+    {
+      "name": "NpmDetector"
+    },
+    {
+      "name": "OpenAIDetector"
+    },
+    {
+      "name": "PrivateKeyDetector"
+    },
+    {
+      "name": "PypiTokenDetector"
+    },
+    {
+      "name": "SendGridDetector"
+    },
+    {
+      "name": "SlackDetector"
+    },
+    {
+      "name": "SoftlayerDetector"
+    },
+    {
+      "name": "SquareOAuthDetector"
+    },
+    {
+      "name": "StripeDetector"
+    },
+    {
+      "name": "TelegramBotTokenDetector"
+    },
+    {
+      "name": "TwilioKeyDetector"
+    }
+  ],
+  "filters_used": [
+    {
+      "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_baseline_file",
+      "filename": ".secrets.baseline"
+    },
+    {
+      "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
+      "min_level": 2
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_indirect_reference"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_likely_id_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_lock_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_potential_uuid"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_sequential_string"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_swagger_file"
+    },
+    {
+      "path": "detect_secrets.filters.heuristic.is_templated_secret"
+    }
+  ],
+  "results": {
+    "features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/local/credentials.yml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/local/credentials.yml",
+        "hashed_secret": "a62f2225bf70bfaccbc7f1ef2a397836717377de",
+        "is_verified": false,
+        "line_number": 8
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "features/steps/test_starter/{{ cookiecutter.repo_name }}/conf/local/credentials.yml",
+        "hashed_secret": "d033e22ae348aeb5660fc2140aec35850c4da997",
+        "is_verified": false,
+        "line_number": 16
+      }
+    ],
+    "kedro/templates/project/{{ cookiecutter.repo_name }}/conf/local/credentials.yml": [
+      {
+        "type": "Secret Keyword",
+        "filename": "kedro/templates/project/{{ cookiecutter.repo_name }}/conf/local/credentials.yml",
+        "hashed_secret": "e5e9fa1ba31ecd1ae84f75caaa474f3a663f05f4",
+        "is_verified": false,
+        "line_number": 9
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "kedro/templates/project/{{ cookiecutter.repo_name }}/conf/local/credentials.yml",
+        "hashed_secret": "d033e22ae348aeb5660fc2140aec35850c4da997",
+        "is_verified": false,
+        "line_number": 18
+      }
+    ],
+    "tests/config/test_omegaconf_config.py": [
+      {
+        "type": "Basic Auth Credentials",
+        "filename": "tests/config/test_omegaconf_config.py",
+        "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684",
+        "is_verified": false,
+        "line_number": 39
+      }
+    ],
+    "tests/framework/context/test_context.py": [
+      {
+        "type": "Basic Auth Credentials",
+        "filename": "tests/framework/context/test_context.py",
+        "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684",
+        "is_verified": false,
+        "line_number": 63
+      }
+    ],
+    "tests/io/conftest.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/io/conftest.py",
+        "hashed_secret": "adb5fabe51f5b45e83fdd91b71c92156fec4a63e",
+        "is_verified": false,
+        "line_number": 71
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/io/conftest.py",
+        "hashed_secret": "3c3b274d119ff5a5ec6c1e215c1cb794d9973ac1",
+        "is_verified": false,
+        "line_number": 117
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/io/conftest.py",
+        "hashed_secret": "15dd2c9ccec914f1470b4dccb45789844e49cf70",
+        "is_verified": false,
+        "line_number": 131
+      }
+    ],
+    "tests/io/test_data_catalog.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/io/test_data_catalog.py",
+        "hashed_secret": "15dd2c9ccec914f1470b4dccb45789844e49cf70",
+        "is_verified": false,
+        "line_number": 529
+      }
+    ],
+    "tests/io/test_kedro_data_catalog.py": [
+      {
+        "type": "Secret Keyword",
+        "filename": "tests/io/test_kedro_data_catalog.py",
+        "hashed_secret": "15dd2c9ccec914f1470b4dccb45789844e49cf70",
+        "is_verified": false,
+        "line_number": 560
+      }
+    ]
+  },
+  "generated_at": "2025-02-06T19:46:16Z"
+}
diff --git a/CITATION.cff b/CITATION.cff
@@ -44,6 +44,6 @@ authors:
 - family-names: Brugman
   given-names: Simon
 title: Kedro
-version: 0.19.10
-date-released: 2024-11-26
+version: 0.19.11
+date-released: 2025-01-29
 url: https://github.com/kedro-org/kedro
diff --git a/Makefile b/Makefile
@@ -27,9 +27,6 @@ e2e-tests-fast:
 pip-compile:
 	pip-compile -q -o -
 
-secret-scan:
-	trufflehog --max_depth 1 --exclude_paths trufflehog-ignore.txt .
-
 build-docs:
 	uv pip install -e ".[docs]"
 	./docs/build-docs.sh "docs"

diff --git a/RELEASE.md b/RELEASE.md
@@ -1,17 +1,40 @@
 # Upcoming Release
 
+## Major features and improvements
+* Added `KedroDataCatalog.filter()` to filter datasets by name and type.
+* Added `Pipeline.grouped_nodes_by_namespace` property which returns a dictionary of nodes grouped by namespace, intended to be used by plugins to facilitate deployment of namespaced nodes together.
+
+## Bug fixes and other changes
+* Updated `_LazyDataset` representation when printing `KedroDataCatalog`.
+* Fixed `MemoryDataset` to infer `assign` copy mode for Ibis Tables, which previously would be inferred as `deepcopy`.
+## Breaking changes to the API
+## Documentation changes
+
+# Release 0.19.11
+
 ## Major features and improvements
 * Implemented `KedroDataCatalog.to_config()` method that converts the catalog instance into a configuration format suitable for serialization.
-* Improve OmegaConfigLoader performance
+* Improve OmegaConfigLoader performance.
+* Replaced `trufflehog` with `detect-secrets` for detecting secrets within a code base.
+* Added support for `%load_ext kedro`.
 
 ## Bug fixes and other changes
 * Added validation to ensure dataset versions consistency across catalog.
 * Fixed a bug in project creation when using a custom starter template offline.
 * Added `node` import to the pipeline template.
+* Update error message when executing kedro run without pipeline.
+* Safeguard hooks when user incorrectly registers a hook class in settings.py.
+* Fixed parsing paths with query and fragment.
+* Remove lowercase transformation in regex validation.
+* Moved `kedro-catalog` JSON schema to `kedro-datasets`.
+* Updated `Partitioned dataset lazy saving` docs page.
+* Fixed `KedroDataCatalog` mutation after pipeline run.
+* Made `KedroDataCatalog._datasets` compatible with `DataCatalog._datasets`.
 
-## Breaking changes to the API
-## Documentation changes
 ## Community contributions
+Many thanks to the following Kedroids for contributing PRs to this release:
+* [Hendrik Scherner](https://github.com/SchernHe)
+* [Chris Schopp](https://github.com/chrisschopp)
 
 # Release 0.19.10
-Original file line number
+Diff line change
@@ Expand Up / @@ -94,6 +94,7 @@ Puneet @@
     Rashida
     Ravi
     Richard
+    Schopp
     Schwarzmann
     Sorokin
     Stichbury
@@ Expand Down @@