Merge branch 'main' into fix/configurable-dynamic-catalog

meltano · Jan 9, 2024 · 1089e4c · 1089e4c
2 parents e7d3340 + 586fc94
commit 1089e4c
Show file tree

Hide file tree

Showing 181 changed files with 6,467 additions and 4,238 deletions.
diff --git a/.flake8 b/.flake8
@@ -1,6 +1,7 @@
 [flake8]
 max-line-length = 88
 exclude = cookiecutter
+ignore = E, W
 per-file-ignores =
     # Don't require docstrings conventions in private modules
     singer_sdk/helpers/_*.py:DAR

diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
@@ -15,7 +15,7 @@ body:
     attributes:
       label: Singer SDK Version
       description: Version of the library you are using
-      placeholder: "0.30.0"
+      placeholder: "0.34.1"
     validations:
       required: true
   - type: checkboxes
@@ -36,6 +36,7 @@ body:
         - "3.8"
         - "3.9"
         - "3.10"
+        - "3.11"
         - "NA"
     validations:
       required: true

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -5,15 +5,13 @@ updates:
     schedule:
       interval: daily
       time: "12:00"
-      timezone: "UTC"
     reviewers: [meltano/engineering]
     labels: [deps]
   - package-ecosystem: pip
     directory: "/.github/workflows"
     schedule:
       interval: daily
       time: "12:00"
-      timezone: "UTC"
     reviewers: [meltano/engineering]
     labels: [deps]
   - package-ecosystem: github-actions
@@ -22,3 +20,7 @@ updates:
       interval: weekly
     reviewers: [meltano/engineering]
     labels: [deps]
+    groups:
+      actions:
+        patterns:
+          - "*"
diff --git a/.github/semantic.yml b/.github/semantic.yml
@@ -30,4 +30,4 @@ scopes:
   - mappers    # mappers only
   - templates  # cookiecutters
   - deps       # production dependencies
-  - deps-dev   # development depencencies (testing, linting, etc.)
+  - deps-dev   # development dependencies (testing, linting, etc.)
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -9,7 +9,13 @@ name: "CodeQL"
 on:
   push:
     branches: [ "main" ]
-  pull_request: {}
+    paths:
+      - .github/workflows/codeql-analysis.yml
+      - '**.py'  # Any Python file
+  pull_request:
+    paths:
+      - .github/workflows/codeql-analysis.yml
+      - '**.py'  # Any Python file
   schedule:
     - cron: '37 10 * * 5'
 
@@ -30,12 +36,10 @@ jobs:
         # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
 
     steps:
-    - name: Checkout repository
-      uses: actions/[email protected]
+    - uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+    - uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -48,8 +52,7 @@ jobs:
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+    - uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -61,5 +64,4 @@ jobs:
     #   echo "Run, Build Application using script"
     #   ./location_of_script_within_repo/buildscript.sh
 
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+    - uses: github/codeql-action/analyze@v3
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
@@ -0,0 +1,54 @@
+name: codspeed
+
+on:
+  push:
+    branches:
+      - "main"
+    paths:
+      - "singer_sdk/**"
+      - "tests/**"
+      - "noxfile.py"
+      - "poetry.lock"
+      - "pyproject.toml"
+      - ".github/workflows/codspeed.yml"
+  pull_request:
+    paths:
+      - "singer_sdk/**"
+      - "tests/**"
+      - "noxfile.py"
+      - "poetry.lock"
+      - "pyproject.toml"
+      - ".github/workflows/codspeed.yml"
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
+        architecture: x64
+
+    - name: Install poetry
+      run: |
+        curl -fsS https://install.python-poetry.org | python - -y
+
+    - name: Configure poetry
+      run: poetry config virtualenvs.create false
+
+    - name: Install project
+      run: >
+        poetry install
+        -vvv
+        --with dev
+        --with benchmark
+        --all-extras
+
+    - uses: CodSpeedHQ/action@v2
+      with:
+        token: ${{ secrets.CODSPEED_TOKEN }}
+        run: pytest tests/ --codspeed
diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt
@@ -1,5 +1,6 @@
-pip==23.2
-poetry==1.5.1
-pre-commit==3.3.3
+pip==23.3.2
+poetry==1.7.1
+poetry-plugin-export==1.6.0
+pre-commit==3.6.0
 nox==2023.4.22
 nox-poetry==1.0.3
diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml
@@ -3,10 +3,16 @@ name: E2E Cookiecutters
 on:
   pull_request:
     types: [opened, synchronize, reopened]
-    paths: ["cookiecutter/**", "e2e-tests/cookiecutters/**"]
+    paths:
+    - "cookiecutter/**"
+    - "e2e-tests/cookiecutters/**"
+    - ".github/workflows/cookiecutter-e2e.yml"
   push:
     branches: [main]
-    paths: ["cookiecutter/**", "e2e-tests/cookiecutters/**"]
+    paths:
+    - "cookiecutter/**"
+    - "e2e-tests/cookiecutters/**"
+    - ".github/workflows/cookiecutter-e2e.yml"
   workflow_dispatch:
 
 concurrency:
@@ -18,18 +24,16 @@ env:
 
 jobs:
   lint:
-    name: Cookiecutter E2E ${{ matrix.python-version }} ${{ matrix.python-version }} / ${{ matrix.os }}
+    name: Cookiecutter E2E Python ${{ matrix.python-version }} / ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: true
       matrix:
         include:
-          - { python-version: "3.10",  os: "ubuntu-latest" }
+          - { python-version: "3.11",  os: "ubuntu-latest" }
 
     steps:
-    - name: Check out the repository
-      uses: actions/[email protected]
-
+    - uses: actions/checkout@v4
     - name: Upgrade pip
       env:
         PIP_CONSTRAINT: .github/workflows/constraints.txt
@@ -38,12 +42,15 @@ jobs:
         pip --version
 
     - name: Install Poetry
+      env:
+        PIP_CONSTRAINT: .github/workflows/constraints.txt
       run: |
         pipx install poetry
+        pipx inject poetry poetry-plugin-export
         poetry --version
+        poetry self show plugins
 
-    - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/[email protected]
+    - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
         architecture: x64
@@ -59,22 +66,25 @@ jobs:
       env:
         PIP_CONSTRAINT: .github/workflows/constraints.txt
       run: |
-        pipx install --pip-args=--constraint=.github/workflows/constraints.txt nox
-        pipx inject --pip-args=--constraint=.github/workflows/constraints.txt nox nox-poetry
+        pipx install nox
+        pipx inject nox nox-poetry
         nox --version
 
     - name: Run Nox
       run: |
         nox --python=${{ matrix.python-version }} --session=test_cookiecutter
 
-    - name: Upload build artifacts
+    - uses: actions/upload-artifact@v4
       if: always()
-      uses: actions/upload-artifact@v3
       with:
+        name: cookiecutter-${{ matrix.os }}-py${{ matrix.python-version }}
         path: |
           /tmp/tap-*
           /tmp/target-*
+          /tmp/mapper-*
           !/tmp/tap-*/.mypy_cache/
           !/tmp/target-*/.mypy_cache/
+          !/tmp/mapper-*/.mypy_cache/
           !/tmp/tap-*/.tox/
           !/tmp/target-*/.tox/
+          !/tmp/mapper-*/.tox/
diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml
@@ -5,37 +5,15 @@ on:
   workflow_dispatch:
     inputs: {}
 
-env:
-  FOSSA_CLI_INSTALLER_VERSION: '3.3.10'
-
 permissions:
   contents: read
 
 jobs:
   dependency-review:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout the repository
-        uses: actions/[email protected]
-
-      - name: GitHub dependency vulnerability check
+      - uses: actions/checkout@v4
+      - uses: actions/dependency-review-action@v3
         if: ${{ github.event_name == 'pull_request_target' }}
-        uses: actions/[email protected]
         with:
           fail-on-severity: high
-
-      - name: FOSSA dependency license check
-        run: |
-          # `$FOSSA_CLI_INSTALLER_VERSION` only controls the version of the installer used - the latest version of `fossa-cli` will always be used.
-          curl --no-progress-meter -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/v${FOSSA_CLI_INSTALLER_VERSION}/install-latest.sh | bash
-
-          echo '## FOSSA dependency license check' >> $GITHUB_STEP_SUMMARY
-          echo '' >> $GITHUB_STEP_SUMMARY
-
-          fossa analyze --fossa-api-key ${{ secrets.MELTYBOT_FOSSA_API_KEY }} --revision ${{ github.sha }} |& tee fossa_analyze.log
-          fossa test --fossa-api-key ${{ secrets.MELTYBOT_FOSSA_API_KEY }} --revision ${{ github.sha }}
-
-          TEST_FAILED=$?
-          FOSSA_REPORT_LINK="$(grep -A 1 '[ INFO] View FOSSA Report:' fossa_analyze.log | tail -n 1 | sed -e 's/^\[ INFO\]\s*//')"
-          echo "[FOSSA detected $([ $TEST_FAILED -ne 0 ] && echo -n '' || echo 'no ')issues](${FOSSA_REPORT_LINK})" >> $GITHUB_STEP_SUMMARY
-          exit $TEST_FAILED
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,8 +1,9 @@
 name: Release
 
 on:
-  release:
-    types: [published]
+  push:
+    tags:
+      - v*
 
 permissions:
   contents: write  # Needed to upload artifacts to the release
@@ -16,12 +17,12 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3.5.3
+      uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4.7.0
+      uses: actions/setup-python@v5
       with:
-        python-version: "3.10"
+        python-version: "3.11"
 
     - name: Upgrade pip
       env:
@@ -56,4 +57,4 @@ jobs:
         file_glob: true
 
     - name: Publish
-      uses: pypa/[email protected].8
+      uses: pypa/[email protected].11