merging with 2.0

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.3.4
+    rev: v0.6.7
     hooks:
       # Run the linter.
       - id: ruff

.prettierignore

@@ -0,0 +1 @@
+docs/references/sources/sql.md

.secrets.baseline

@@ -90,6 +90,10 @@
     {
       "path": "detect_secrets.filters.allowlist.is_line_allowlisted"
     },
+    {
+      "path": "detect_secrets.filters.common.is_baseline_file",
+      "filename": ".secrets.baseline"
+    },
     {
       "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
       "min_level": 2
@@ -223,7 +227,7 @@
         "filename": "tests/integration/orchestration/prefect/flows/test_duckdb_to_sql_server.py",
         "hashed_secret": "74fb0b9813045340022392630bada6fd3d555fe6",
         "is_verified": false,
-        "line_number": 43
+        "line_number": 44
       }
     ],
     "tests/integration/orchestration/prefect/flows/test_genesys_to_adls.py": [
@@ -282,7 +286,7 @@
         "filename": "tests/integration/orchestration/prefect/tasks/test_sql_server.py",
         "hashed_secret": "74fb0b9813045340022392630bada6fd3d555fe6",
         "is_verified": false,
-        "line_number": 28
+        "line_number": 29
       }
     ],
     "tests/integration/test_sharepoint.py": [
@@ -291,7 +295,7 @@
         "filename": "tests/integration/test_sharepoint.py",
         "hashed_secret": "9fb7fe1217aed442b04c0f5e43b5d5a7d3287097",
         "is_verified": false,
-        "line_number": 14
+        "line_number": 15
       }
     ],
     "tests/resources/metadata/model/manifest.json": [
@@ -339,7 +343,7 @@
         "filename": "tests/unit/test_config.py",
         "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
         "is_verified": false,
-        "line_number": 8
+        "line_number": 9
       }
     ],
     "tests/unit/test_genesys.py": [
@@ -348,7 +352,7 @@
         "filename": "tests/unit/test_genesys.py",
         "hashed_secret": "1089adfb1f11b95df31344030507912b5abdf57a",
         "is_verified": false,
-        "line_number": 17
+        "line_number": 18
       }
     ],
     "tests/unit/test_outlook.py": [
@@ -357,14 +361,14 @@
         "filename": "tests/unit/test_outlook.py",
         "hashed_secret": "b235838f76594bf21886c6eec9c06a207e9ec5ce",
         "is_verified": false,
-        "line_number": 17
+        "line_number": 18
       },
       {
         "type": "Secret Keyword",
         "filename": "tests/unit/test_outlook.py",
         "hashed_secret": "1089adfb1f11b95df31344030507912b5abdf57a",
         "is_verified": false,
-        "line_number": 48
+        "line_number": 49
       }
     ],
     "tests/unit/test_redshift_spectrum.py": [
@@ -373,7 +377,7 @@
         "filename": "tests/unit/test_redshift_spectrum.py",
         "hashed_secret": "dc724af18fbdd4e59189f5fe768a5f8311527050",
         "is_verified": false,
-        "line_number": 19
+        "line_number": 21
       }
     ],
     "tests/unit/test_sharepoint.py": [
@@ -382,16 +386,16 @@
         "filename": "tests/unit/test_sharepoint.py",
         "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3",
         "is_verified": false,
-        "line_number": 13
+        "line_number": 14
       },
       {
         "type": "Secret Keyword",
         "filename": "tests/unit/test_sharepoint.py",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 46
+        "line_number": 47
       }
     ]
   },
-  "generated_at": "2024-08-30T12:50:18Z"
+  "generated_at": "2024-09-23T12:45:10Z"
 }

docs/references/orchestration/prefect/flows.md

@@ -4,6 +4,8 @@
 
 ::: viadot.orchestration.prefect.flows.exchange_rates_to_adls
 
+::: viadot.orchestration.prefect.flows.exchange_rates_to_databricks
+
 ::: viadot.orchestration.prefect.flows.sap_to_redshift_spectrum
 
 ::: viadot.orchestration.prefect.flows.sharepoint_to_adls
@@ -17,3 +19,27 @@
 ::: viadot.orchestration.prefect.flows.transform
 
 ::: viadot.orchestration.prefect.flows.transform_and_catalog
+
+::: viadot.orchestration.prefect.flows.duckdb_to_parquet
+
+::: viadot.orchestration.prefect.flows.duckdb_to_sql_server
+
+::: viadot.orchestration.prefect.flows.duckdb_transform
+
+::: viadot.orchestration.prefect.flows.epicor_to_parquet
+
+::: viadot.orchestration.prefect.flows.exchange_rates_api_to_redshift_spectrum
+
+::: viadot.orchestration.prefect.flows.genesys_to_adls
+
+::: viadot.orchestration.prefect.flows.hubspot_to_adls
+
+::: viadot.orchestration.prefect.flows.mindful_to_adls
+
+::: viadot.orchestration.prefect.flows.outlook_to_adls
+
+::: viadot.orchestration.prefect.flows.sap_to_parquet
+
+::: viadot.orchestration.prefect.flows.sql_server_to_minio
+
+::: viadot.orchestration.prefect.flows.sql_server_to_parquet

docs/references/orchestration/prefect/tasks.md

@@ -21,3 +21,25 @@
 ::: viadot.orchestration.prefect.tasks.sharepoint_download_file
 
 ::: viadot.orchestration.prefect.tasks.sharepoint_to_df
+
+::: viadot.orchestration.prefect.tasks.duckdb_query
+
+::: viadot.orchestration.prefect.tasks.epicor_to_df
+
+::: viadot.orchestration.prefect.tasks.genesys_to_df
+
+::: viadot.orchestration.prefect.tasks.hubspot_to_df
+
+::: viadot.orchestration.prefect.tasks.mindful_to_df
+
+::: viadot.orchestration.prefect.tasks.outlook_to_df
+
+::: viadot.orchestration.prefect.tasks.sap_rfc_to_df
+
+::: viadot.orchestration.prefect.tasks.bcp
+
+::: viadot.orchestration.prefect.tasks.create_sql_server_table
+
+::: viadot.orchestration.prefect.tasks.sql_server_query
+
+::: viadot.orchestration.prefect.tasks.sql_server_to_df

docs/references/sources/api_sources.md → docs/references/sources/api.md

@@ -8,10 +8,16 @@
 
 ::: viadot.sources.cloud_for_customers.CloudForCustomers
 
-::: viadot.sources.sap_rfc.SAPRFC
-
 ::: viadot.sources.sharepoint.Sharepoint
 
 ::: viadot.sources.genesys.Genesys
 
+::: viadot.sources.outlook.Outlook
+
+::: viadot.sources.hubspot.Hubspot
+
+::: viadot.sources.epicor.Epicor
+
+::: viadot.sources.mindful.Mindful
+
 ::: viadot.sources.minio.MinIO

docs/references/sources/sql_sources.md → docs/references/sources/sql.md

@@ -16,4 +16,10 @@
 
 ::: viadot.sources.databricks.Databricks
 
-::: viadot.sources.trino.Trino
+::: viadot.sources._trino.Trino
+
+::: viadot.sources._duckdb.DuckDB
+
+::: viadot.sources.sap_rfc.SAPRFC
+
+::: viadot.sources.sap_rfc.SAPRFCV2

mkdocs.yml

@@ -21,8 +21,8 @@ nav:
 
   - References:
       - Sources:
-          - SQL Sources: references/sources/sql_sources.md
-          - API Sources: references/sources/api_sources.md
+          - SQL Sources: references/sources/sql.md
+          - API Sources: references/sources/api.md
       - Orchestration:
           - Prefect:
               - Tasks: references/orchestration/prefect/tasks.md

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "viadot2"
-version = "2.1.18"
+version = "2.1.19"
 description = "A simple data ingestion library to guide data flows from some places to other places."
 authors = [
     { name = "acivitillo", email = "[email protected]" },
@@ -80,7 +80,7 @@ dev-dependencies = [
     "mkdocs-git-revision-date-plugin>=0.3.2",
     "mkdocs-glightbox>=0.4.0",
     "pytest>=8.2.2",
-    "ruff>=0.5.2",
+    "ruff>=0.6.6",
     "pytest-asyncio>=0.23.8",
     "moto>=5.0.13",
 ]

src/viadot/sources/__init__.py

@@ -2,8 +2,9 @@
 
 from importlib.util import find_spec
 
+from ._duckdb import DuckDB
+from ._trino import Trino
 from .cloud_for_customers import CloudForCustomers
-from .duckdb import DuckDB
 from .epicor import Epicor
 from .exchange_rates import ExchangeRates
 from .genesys import Genesys
@@ -13,9 +14,9 @@
 from .sap_bw import Sapbw
 from .sharepoint import Sharepoint
 from .sql_server import SQLServer
-from .trino import Trino
 from .uk_carbon_intensity import UKCarbonIntensity
 
+
 __all__ = [
     "CloudForCustomers",
     "Epicor",
@@ -37,7 +38,7 @@
     __all__.extend(["AzureDataLake"])
 
 if find_spec("duckdb"):
-    from viadot.sources.duckdb import DuckDB  # noqa: F401
+    from viadot.sources._duckdb import DuckDB  # noqa: F401
 
     __all__.extend(["DuckDB"])
 

tests/integration/orchestration/prefect/flows/test_duckdb_to_sql_server.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 
 import pytest
+
 from viadot.orchestration.prefect.flows import duckdb_to_sql_server
 from viadot.orchestration.prefect.tasks import sql_server_query
 from viadot.sources import DuckDB, SQLServer
@@ -12,13 +13,13 @@
 DUCKDB_CREDS = {"database": DATABASE_PATH, "read_only": False}
 
 
-@pytest.fixture()
+@pytest.fixture
 def sql_server():
     # Initialize the SQLServer instance with the test credentials.
     return SQLServer(config_key="sql_server")
 
 
-@pytest.fixture()
+@pytest.fixture
 def duckdb():
     # Initialize the SQLServer instance with the test credentials.
     duckdb = DuckDB(credentials=DUCKDB_CREDS)

tests/integration/orchestration/prefect/flows/test_duckdb_transform.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 
 import pytest
+
 from viadot.orchestration.prefect.flows import duckdb_transform
 from viadot.sources import DuckDB
 

tests/integration/orchestration/prefect/flows/test_sap_to_parquet.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 
 import pandas as pd
+
 from viadot.orchestration.prefect.flows import sap_to_parquet
 
 

tests/integration/orchestration/prefect/tasks/test_adls.py

@@ -1,5 +1,6 @@
 import pandas as pd
 from prefect import flow, task
+
 from viadot.orchestration.prefect.tasks import df_to_adls
 from viadot.utils import skip_test_on_missing_extra
 

tests/integration/orchestration/prefect/tasks/test_databricks.py

@@ -2,6 +2,7 @@
 
 from prefect import flow
 import pytest
+
 from viadot.exceptions import TableDoesNotExistError
 from viadot.utils import skip_test_on_missing_extra
 

tests/integration/orchestration/prefect/tasks/test_dbt.py

@@ -1,4 +1,5 @@
 from prefect import flow
+
 from viadot.orchestration.prefect.tasks import dbt_task
 
 

tests/integration/orchestration/prefect/tasks/test_duckdb.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 
 import pytest
+
 from viadot.orchestration.prefect.tasks import duckdb_query
 from viadot.sources import DuckDB
 

tests/integration/orchestration/prefect/tasks/test_git.py

@@ -2,6 +2,7 @@
 import shutil
 
 from loguru import logger
+
 from viadot.orchestration.prefect.tasks import clone_repo
 
 

tests/integration/orchestration/prefect/tasks/test_luma.py

@@ -1,9 +1,10 @@
 from loguru import logger
 import pytest
+
 from viadot.orchestration.prefect.tasks import luma_ingest_task
 
 
-@pytest.mark.asyncio()
+@pytest.mark.asyncio
 async def test_luma_ingest_task_model_metadata(LUMA_URL):
     logs = await luma_ingest_task.fn(
         metadata_kind="model",
@@ -18,7 +19,7 @@ async def test_luma_ingest_task_model_metadata(LUMA_URL):
     assert success_message in log
 
 
-@pytest.mark.asyncio()
+@pytest.mark.asyncio
 async def test_luma_ingest_task_model_run_metadata(LUMA_URL):
     logs = await luma_ingest_task.fn(
         metadata_kind="model_run",
@@ -33,7 +34,7 @@ async def test_luma_ingest_task_model_run_metadata(LUMA_URL):
     assert success_message in log
 
 
-@pytest.mark.asyncio()
+@pytest.mark.asyncio
 async def test_luma_ingest_task_model_run_metadata_follow(LUMA_URL):
     logs = await luma_ingest_task.fn(
         metadata_kind="model_run",

tests/integration/orchestration/prefect/tasks/test_minio.py

@@ -1,4 +1,5 @@
 import pandas as pd
+
 from viadot.orchestration.prefect.tasks import df_to_minio
 from viadot.orchestration.prefect.utils import get_credentials
 from viadot.sources import MinIO

tests/integration/orchestration/prefect/tasks/test_redshift_spectrum.py

@@ -3,6 +3,7 @@
 import pandas as pd
 from prefect import flow
 import pytest
+
 from viadot.orchestration.prefect.tasks import df_to_redshift_spectrum
 from viadot.utils import skip_test_on_missing_extra