Merge remote-tracking branch 'origin/develop' into m/allow-ci-on-forks

* origin/develop: (102 commits) [DOCS] restore lychee (#10889) [RELEASE] 1.3.3 (#10888) [DOCS] add migration guide to 0.18 docs (#10885) [MAINTENANCE] Migrate from `databricks-sql-connector` to `databricks-sqlalchemy` in tests (#10886) [BUGFIX] Allow adding Expectations with identical attributes to Suites (#10884) [DOCS] Corrections and clarifications for result_format (#10875) [BUGFIX] Make validation results' describe_dict return a serializable dict (#10863) [MAINTENANCE] Convert BatchTestSetup.asset property to BatchTestSetup… (#10864) [MAINTENANCE] Add method to set analytics user_agent_str (#10883) [DOCS] Update documentation around batch_parameters and link to updated API docs. (#10877) [MAINTENANCE] Add user-agent-str to analytics events (#10869) [RELEASE] 1.3.2 (#10874) [DOCS] Add titles to properties and methods sections in API Reference (#10821) [MAINTENANCE] Handle `aws-chunked` encoding type data in `TupleS3StoreBackend` (#10861) [Docs] update screenshot of observed value (#10867) [MAINTENANCE] Enforce a minimum length of 2 for MulticolumnMapExpectation `column_list` (#10850) [BUGFIX] Add input args validators to various Expectations (#10833) [DOCS] fix custom SQL Expectation approach for cloud (#10844) [FEATURE] Enable strict_min/max for ExpectTableRowCountToBeBetween (#10845) [DOCS] Adding keys to duplicated headers (#10846) ...
great-expectations · Jan 27, 2025 · 33de291 · 33de291
2 parents 77fdeb7 + b3d4163
commit 33de291
Show file tree

Hide file tree

Showing 882 changed files with 14,689 additions and 9,107 deletions.
diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
@@ -50,3 +50,6 @@ d170c89167a96b702edc02b16dbf5984619d0e8f
 # enable TRYceratops linting; add noqa comments
 # https://github.com/great-expectations/great_expectations/pull/9684
 2bbfb50a6458f09ef197ee1174666a4c4726a850
+# Add "FIXME CoP" comments to type ignore and noqa ignore lines
+# https://github.com/great-expectations/great_expectations/pull/10817
+078c1611dd9c70fec6d7b6318f21a063ff6aa9b0
diff --git a/.github/teams.yml b/.github/teams.yml
@@ -2,14 +2,14 @@
 # NOTE - this should be kept in sync with the GX org's teams
 
 devrel:
-  - "@kyleaton" # Kyle Eaton
-  - "@rachhouse" # Rachel House
-  - "@r34ctor" # Nevin Tan
-  - "@molliemarie" # Mollie Marie Pettit
-  - "@klavavej" # Kristen Lavavej
   - "@JessSaavedra" # Jessica Saavedra
-  - "@deborahniesz" # Deborah Niesz
   - "@Quantisan" # Paul Lam
+  - "@deborahniesz" # Deborah Niesz
+  - "@klavavej" # Kristen Lavavej
+  - "@kyleaton" # Kyle Eaton
+  - "@molliemarie" # Mollie Marie Pettit
+  - "@r34ctor" # Nevin Tan
+  - "@rachhouse" # Rachel House
 
 core:
   - "@DrewHoo" # Drew Hoover
@@ -19,8 +19,11 @@ core:
   - "@Super-Tanner" # Tanner Beam
   - "@TrangPham" # Thu Pham
   - "@abegong" # Abe Gong
+  - "@alena-hutchinson" # Alena Hutchinson
   - "@allensallinger" # Allen Sallinger
+  - "@annabarr" # Anna Barr
   - "@anthonyburdi" # Anthony Burdi
+  - "@ashmortar" # Aaron Ross
   - "@billdirks" # Bill Dirks
   - "@cdkini" # Chetan Kini
   - "@dctalbot" # David Talbot
@@ -33,12 +36,10 @@ core:
   - "@lockettks" # Kim Mathieu
   - "@nicgrayson" # Nic Grayson
   - "@roblim" # Rob Lim
+  - "@shiplet" # Michael Shiplet
   - "@sujensen" # Susan Jensen
   - "@tyler-hoffman" # Tyler Hoffman
   - "@wookasz" # Łukasz Lempart
-  - "@annabarr" # Anna Barr
-  - "@alena-hutchinson" # Alena Hutchinson
-  - "@ashmortar" # Aaron Ross
 
 bot:
   - "@dependabot"

diff --git a/.github/workflows/README.md b/.github/workflows/README.md
@@ -6,6 +6,8 @@
   - Responsible for keeping PR's up-to-date with `develop` (only works if "auto-merge" is turned on)
 * [CodeSee Architecture Diagrams](codesee-arch-diagram.yml)
   - Generates a visualization of proposed changes to the codebase through the use of https://www.codesee.io/
+* [DataSource Cleanup](data_source_cleanup.yml)
+  - Responsible for cleaning up stray schemas left behind from tests
 * [StaleBot](stale.yml)
   - Responsible for marking PR's and issues as `stale`
 * [PEP-273 Compatability](test-pep273-compatability.yml)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -113,6 +113,8 @@ jobs:
           invoke type-check --ci --pretty  --check-stub-sources
       - name: Marker-coverage-check
         run: invoke marker-coverage
+      - name: Check for linter ignores without comments
+        run: ./scripts/check_linter_ignores.sh
 
   docs-snippets:
     needs: [unit-tests, doc-checks, check-actor-permissions]
@@ -197,6 +199,9 @@ jobs:
           # Authorize access to Google Cloud with a service account
           ./google-cloud-sdk/bin/gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS
 
+      - name: Install mssql odbc driver
+        run: ./scripts/install_mssql_odbc_driver.sh
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -326,8 +331,6 @@ jobs:
         run: python ci/checks/check_name_tag_snippets_referenced.py
       - name: public_api_report
         run: invoke public-api
-      - name: link_checker
-        run: python docs/checks/docs_link_checker.py -p docs/docusaurus/docs -r docs/docusaurus/docs -sr docs/docusaurus/static -s docs -sp static --skip-external
 
   docs-build:
     needs: [doc-checks, check-actor-permissions]
@@ -399,6 +402,13 @@ jobs:
       GX_CLOUD_ORGANIZATION_ID: ${{secrets.MERCURY_ORGANIZATION_ID}}
       GX_CLOUD_ACCESS_TOKEN: ${{secrets.MERCURY_ACCESS_TOKEN}}
       LD_SDK_KEY: ${{secrets.LD_SDK_KEY}}
+      GX_SCHEDULER_LAMBDA_ARN: ${{secrets.GX_SCHEDULER_LAMBDA_ARN}}
+      GX_SCHEDULER_EXECUTION_ROLE_ARN: ${{secrets.GX_SCHEDULER_EXECUTION_ROLE_ARN}}
+      GX_SCHEDULER_EVENTBRIDGE_DLQ_ARN: ${{secrets.GX_SCHEDULER_EVENTBRIDGE_DLQ_ARN}}
+      GX_SCHEDULER_EVENTBRIDGE_REGION: ${{secrets.GX_SCHEDULER_EVENTBRIDGE_REGION}}
+      LOCALSTACK_AWS_ENDPOINT_URL: ${{secrets.LOCALSTACK_AWS_ENDPOINT_URL}}
+      LOCALSTACK_AWS_SECRET_ACCESS_KEY: ${{secrets.LOCALSTACK_AWS_SECRET_ACCESS_KEY}}
+      LOCALSTACK_AWS_ACCESS_KEY_ID: ${{secrets.LOCALSTACK_AWS_ACCESS_KEY_ID}}
       PACT_DO_NOT_TRACK: true
       SNOWFLAKE_CI_ACCOUNT: ${{secrets.SNOWFLAKE_CI_ACCOUNT}}
       SNOWFLAKE_CI_USER_PASSWORD: ${{secrets.SNOWFLAKE_CI_USER_PASSWORD}}
@@ -599,6 +609,9 @@ jobs:
           pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
           invoke deps --gx-install -m '${{ matrix.markers }}' -r test
 
+      - name: Install mssql odbc driver
+        run: ./scripts/install_mssql_odbc_driver.sh
+
       - name: Configure ECR AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:

diff --git a/.github/workflows/data_source_cleanup.yml b/.github/workflows/data_source_cleanup.yml
@@ -0,0 +1,42 @@
+# Script(s) to clean up any loose data left behind from test runs.
+# These can primarily happen if CI is stopped while tests are running
+# for big query, etc.
+
+name: Nightly Data Source Cleanup
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 6 * * *"
+
+jobs:
+  cleanup-big-query:
+    runs-on: ubuntu-latest
+    env:
+      # google
+      GE_TEST_GCP_CREDENTIALS: ${{secrets.GE_TEST_GCP_CREDENTIALS}}
+      GE_TEST_GCP_PROJECT: ${{secrets.GE_TEST_GCP_PROJECT}}
+      GE_TEST_BIGQUERY_DATASET: ${{secrets.GE_TEST_BIGQUERY_DATASET}}
+      GOOGLE_APPLICATION_CREDENTIALS: "gcp-credentials.json"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "pip"
+          cache-dependency-path: |
+            reqs/requirements-dev-test.txt
+            setup.py
+      - name: Create JSON file GCP
+        run: |
+          echo "$GE_TEST_GCP_CREDENTIALS" > gcp-credentials.json
+      - name: Install dependencies
+        run: |
+          pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
+          invoke deps --gx-install -m bigquery
+          pip install -e .
+      - name: Run BigQuery cleanup script
+        run: |
+          python ./scripts/cleanup/cleanup_big_query.py
diff --git a/.gitignore b/.gitignore
@@ -157,3 +157,6 @@ tests/integration/cloud/rest_contracts/pacts
 
 # Local Netlify folder
 .netlify
+
+# mercury container volume
+assets/docker/mercury/volume/
diff --git a/README.md b/README.md
@@ -70,7 +70,7 @@ To ensure the long-term quality of the GX Core codebase, we're not yet ready to
 | -------------------- | ------------------ | ----- |
 | CredentialStore      | 🟢 Ready           |       |
 | BatchDefinition      | 🟡 Partially ready | Formerly known as splitters |
-| Action               | 🔴 Not ready       |       |
+| Action               | 🟢 Ready           |       |
 | DataSource           | 🔴 Not ready       | Includes MetricProvider and ExecutionEngine |
 | DataContext          | 🔴 Not ready       | Also known as Configuration Stores |
 | DataAsset            | 🔴 Not ready       |       |

diff --git a/assets/docker/mercury/docker-compose.yml b/assets/docker/mercury/docker-compose.yml
@@ -79,6 +79,8 @@ services:
     depends_on:
       db-provisioner:
         condition: service_completed_successfully
+      localstack:
+        condition: service_healthy
   mercury-service-api-v1:
     image: 258143015559.dkr.ecr.us-east-1.amazonaws.com/mercury/api:v1api-latest
     platform: linux/amd64
@@ -100,19 +102,26 @@ services:
       AUTH0_MERCURY_API_CLIENT_SECRET: ${AUTH0_MERCURY_API_CLIENT_SECRET}
       SENDGRID_API_KEY: fake_key
       LD_SDK_KEY: ${LD_SDK_KEY}
+      GX_SCHEDULER_LAMBDA_ARN: ${GX_SCHEDULER_LAMBDA_ARN}
+      GX_SCHEDULER_EXECUTION_ROLE_ARN: ${GX_SCHEDULER_EXECUTION_ROLE_ARN}
+      GX_SCHEDULER_EVENTBRIDGE_DLQ_ARN: ${GX_SCHEDULER_EVENTBRIDGE_DLQ_ARN}
+      GX_SCHEDULER_EVENTBRIDGE_REGION: ${GX_SCHEDULER_EVENTBRIDGE_REGION}
+      AWS_ENDPOINT_URL: ${LOCALSTACK_AWS_ENDPOINT_URL}
+      AWS_SECRET_ACCESS_KEY: ${LOCALSTACK_AWS_SECRET_ACCESS_KEY}
+      AWS_ACCESS_KEY_ID: ${LOCALSTACK_AWS_ACCESS_KEY_ID}
     depends_on:
       mercury-service-api:
         condition: service_started
       db-provisioner:
         condition: service_completed_successfully
+      localstack:
+        condition: service_healthy
     entrypoint:
       - /bin/bash
       - -c
       - |
-        sed -i 's/localhost/db/g' /app/services/ge_cloud/mercury/alembic.ini
-        cd /app/services/ge_cloud/mercury
-        alembic upgrade head
         cd /app/services/ge_cloud
+        ./scripts/localstack_init.sh
         gunicorn "mercury.api_v1:get_app()" -c ./fastapi-gunicorn.conf.py
   nginx:
     image: ${ECR_PULL_THROUGH_REPOSITORY_URL}library/nginx:latest
@@ -124,3 +133,16 @@ services:
     depends_on:
       - mercury-service-api
       - mercury-service-api-v1
+  localstack:
+    container_name: localstack
+    image: localstack/localstack
+    ports:
+      - "127.0.0.1:4566:4566" # LocalStack Gateway
+      - "127.0.0.1:4510-4559:4510-4559" # external services port range
+    environment:
+      # LocalStack configuration: https://docs.localstack.cloud/references/configuration/
+      DEBUG: ${DEBUG:-0}
+      SERVICES: scheduler,lambda,sts,iam,ssm,events
+    volumes:
+      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
+      - "/var/run/docker.sock:/var/run/docker.sock"
diff --git a/assets/docker/mssql/README.md b/assets/docker/mssql/README.md
@@ -17,7 +17,7 @@ You should now be able to run the tests via `pytest --mssql`
 
     See: https://github.com/microsoft/mssql-docker/issues/668#issuecomment-1420259510
 
-3. Install the ODSBC 17 driver:
+3. Install the ODBC 18 driver:
 
     https://learn.microsoft.com/en-us/sql/connect/odbc/linux-mac/install-microsoft-odbc-driver-sql-server-macos?view=sql-server-ver15
 
@@ -35,9 +35,3 @@ You should now be able to run the tests via `pytest --mssql`
     python -m pip install --force-reinstall --no-binary :all: pyodbc
     python -c "import pyodbc; print(pyodbc.version)"
     ```
-
-2. If you are getting `Login timeout expired` when using localhost, try setting:
-
-    ```sh
-    export GE_TEST_LOCAL_DB_HOSTNAME=127.0.0.1
-    ```
diff --git a/assets/scripts/gx_cloud/experimental/onboarding_script.py b/assets/scripts/gx_cloud/experimental/onboarding_script.py
diff --git a/ci/azure-pipelines-sqlalchemy-compatibility.yml b/ci/azure-pipelines-sqlalchemy-compatibility.yml
@@ -191,8 +191,12 @@ stages:
           - bash: python -m pip install --upgrade pip
             displayName: "Update pip"
 
+          - script: ./scripts/install_mssql_odbc_driver.sh
+            displayName: "Install mssql odbc driver"
+
           - script: |
               sqlcmd -U sa -P "ReallyStrongPwd1234%^&*" -Q "CREATE DATABASE test_ci;" -o create_db_output.txt
+            displayName: "Create mssql database"
 
           - script: |
               cp constraints-dev.txt constraints-dev-temp.txt

diff --git a/docs/adr/0005-public-api-docstrings.md b/docs/adr/0005-public-api-docstrings.md
@@ -0,0 +1,19 @@
+# 5. Docstrings for public API functions/classes 
+
+Date: 2024-12-19
+
+## Status
+
+Accepted
+
+## Context
+
+By marking an object as part of the public API, it automatically renders in our docs site. In order to provide a clear and informative experience for our end users, we should make sure we always include docstrings for public functions/classes. These docstrings should adhere to a consistent format such that they can be rendered in the same manner by our public API infrastructure. 
+
+## Decision
+
+All objects decorated with `@public_api` will have docstrings. In order to be considered public, they must meet this criteria.
+
+## Consequences
+
+Marginal increase in developer burden but worthwhile due to a higher level of thought and care around what actually gets marked and rendered as part of our public API.