✨ Add Salesforce source and Prefect tasks (#1036)

* ✨ created salesforce source file.

* ✨ created salesforce task file.

* ✨ created salesforce flow file.

* ✅ created integration test file.

* ✅ created salesforce unit test.

* 📝 updated commented lines.

* 🔥 removed credentials from task and flow.

* 📝 updated some comments.

* 📝Update `salesforce.py` doc string

Co-authored-by: Michał Zawadzki <[email protected]>

* 📝Update `src/viadot/sources/salesforce.py` doc

Co-authored-by: Michał Zawadzki <[email protected]>

* 🎨 Rename Salesforce class

* 🔐 adls_path hidden

* 🎨 Salesforce into SimpleSalesforce

* ➕ Adding simple-salesforce dependency

* 🎨 Code Formating changes

* 🎨 Salesforce change name

* ➕ Adding extra dependency

---------

Co-authored-by: MaciejGardziński <[email protected]>
Co-authored-by: Michał Zawadzki <[email protected]>
Co-authored-by: Maciej Gardzinski <[email protected]>

pyproject.toml

@@ -35,6 +35,7 @@ dependencies = [
     "paramiko==2.11.0",
     "defusedxml>=0.7.1",
     "aiohttp>=3.10.5",
+    "simple-salesforce==1.12.6",
     ]
 requires-python = ">=3.10"
 readme = "README.md"

requirements-dev.lock

@@ -7,6 +7,7 @@
 #   all-features: false
 #   with-sources: false
 #   generate-hashes: false
+#   universal: false
 
 -e file:.
 aiohappyeyeballs==2.4.0
@@ -40,6 +41,7 @@ attrs==24.2.0
     # via jsonschema
     # via referencing
     # via visions
+    # via zeep
 babel==2.16.0
     # via mkdocs-material
 bcrypt==4.2.0
@@ -96,12 +98,14 @@ comm==0.2.2
 coolname==2.2.0
     # via prefect
 coverage==7.6.1
+    # via coverage
 croniter==2.0.7
     # via prefect
 cryptography==43.0.0
     # via moto
     # via paramiko
     # via prefect
+    # via pyjwt
 cssselect2==0.7.0
     # via cairosvg
 dateparser==1.2.0
@@ -173,6 +177,7 @@ httpcore==1.0.5
     # via httpx
     # via prefect
 httpx==0.27.0
+    # via httpx
     # via neoteroi-mkdocs
     # via prefect
 humanize==4.10.0
@@ -199,6 +204,8 @@ ipykernel==6.29.5
     # via mkdocs-jupyter
 ipython==8.26.0
     # via ipykernel
+isodate==0.6.1
+    # via zeep
 itsdangerous==2.2.0
     # via prefect
 jedi==0.19.1
@@ -248,6 +255,8 @@ kubernetes==29.0.0
 loguru==0.7.2
 lumacli==0.1.2
     # via viadot2
+lxml==5.3.0
+    # via zeep
 mako==1.3.5
     # via alembic
 markdown==3.7
@@ -306,14 +315,18 @@ mkdocs-include-markdown-plugin==6.2.2
 mkdocs-jupyter==0.24.8
 mkdocs-material==9.5.32
     # via mkdocs-jupyter
+    # via mkdocs-material
 mkdocs-material-extensions==1.3.1
     # via mkdocs-material
 mkdocs-mermaid2-plugin==1.1.1
 mkdocs-table-reader-plugin==3.0.1
 mkdocstrings==0.25.2
+    # via mkdocstrings
     # via mkdocstrings-python
 mkdocstrings-python==1.10.5
     # via mkdocstrings
+more-itertools==10.5.0
+    # via simple-salesforce
 moto==5.0.13
 multidict==6.0.5
     # via aiohttp
@@ -385,6 +398,7 @@ platformdirs==4.2.2
     # via jupyter-core
     # via mkdocs-get-deps
     # via mkdocstrings
+    # via zeep
 pluggy==1.5.0
     # via pytest
 prefect==2.20.2
@@ -428,6 +442,8 @@ pygments==2.18.0
     # via mkdocs-material
     # via nbconvert
     # via rich
+pyjwt==2.9.0
+    # via simple-salesforce
 pymdown-extensions==10.9
     # via mkdocs-material
     # via mkdocs-mermaid2-plugin
@@ -471,6 +487,7 @@ pytz==2024.1
     # via pandas
     # via prefect
     # via trino
+    # via zeep
 pytzdata==2020.1
     # via pendulum
 pywavelets==1.7.0
@@ -512,15 +529,23 @@ requests==2.32.3
     # via mkdocs-mermaid2-plugin
     # via moto
     # via o365
+    # via requests-file
     # via requests-oauthlib
+    # via requests-toolbelt
     # via responses
     # via sharepy
+    # via simple-salesforce
     # via trino
     # via viadot2
+    # via zeep
+requests-file==2.1.0
+    # via zeep
 requests-oauthlib==2.0.0
     # via apprise
     # via kubernetes
     # via o365
+requests-toolbelt==1.0.0
+    # via zeep
 responses==0.25.3
     # via moto
 rfc3339-validator==0.1.4
@@ -546,8 +571,6 @@ scipy==1.14.0
     # via imagehash
 sendgrid==6.11.0
     # via viadot2
-setuptools==73.0.0
-    # via mkdocs-mermaid2-plugin
 sgqlc==16.3
     # via prefect-github
 shapely==2.0.6
@@ -556,9 +579,12 @@ sharepy==2.0.0
     # via viadot2
 shellingham==1.5.4
     # via typer
+simple-salesforce==1.12.6
+    # via viadot2
 six==1.16.0
     # via asttokens
     # via bleach
+    # via isodate
     # via jsbeautifier
     # via kubernetes
     # via paramiko
@@ -618,6 +644,7 @@ trino==0.328.0
 typer==0.12.4
     # via lumacli
     # via prefect
+    # via typer
 typing-extensions==4.12.2
     # via aiosqlite
     # via alembic
@@ -626,6 +653,7 @@ typing-extensions==4.12.2
     # via prefect
     # via pydantic
     # via pydantic-core
+    # via simple-salesforce
     # via sqlalchemy
     # via typer
     # via uvicorn
@@ -670,5 +698,9 @@ xmltodict==0.13.0
     # via moto
 yarl==1.9.4
     # via aiohttp
+zeep==4.2.1
+    # via simple-salesforce
 zipp==3.20.0
     # via importlib-metadata
+setuptools==73.0.0
+    # via mkdocs-mermaid2-plugin

requirements.lock

@@ -7,6 +7,7 @@
 #   all-features: false
 #   with-sources: false
 #   generate-hashes: false
+#   universal: false
 
 -e file:.
 aiohappyeyeballs==2.4.0
@@ -38,6 +39,7 @@ attrs==24.2.0
     # via jsonschema
     # via referencing
     # via visions
+    # via zeep
 bcrypt==4.2.0
     # via paramiko
 beautifulsoup4==4.12.3
@@ -73,6 +75,7 @@ croniter==2.0.7
 cryptography==43.0.0
     # via paramiko
     # via prefect
+    # via pyjwt
 dateparser==1.2.0
     # via prefect
 defusedxml==0.7.1
@@ -117,6 +120,7 @@ httpcore==1.0.5
     # via httpx
     # via prefect
 httpx==0.27.0
+    # via httpx
     # via prefect
 humanize==4.10.0
     # via jinja2-humanize-extension
@@ -135,6 +139,8 @@ importlib-resources==6.1.3
     # via prefect
 iniconfig==2.0.0
     # via pytest
+isodate==0.6.1
+    # via zeep
 itsdangerous==2.2.0
     # via prefect
 jinja2==3.1.4
@@ -154,6 +160,8 @@ kubernetes==29.0.0
     # via prefect
 lumacli==0.1.2
     # via viadot2
+lxml==5.3.0
+    # via zeep
 mako==1.3.5
     # via alembic
 markdown==3.7
@@ -165,6 +173,8 @@ markupsafe==2.1.5
     # via mako
 mdurl==0.1.2
     # via markdown-it-py
+more-itertools==10.5.0
+    # via simple-salesforce
 multidict==6.0.5
     # via aiohttp
     # via yarl
@@ -204,6 +214,8 @@ pendulum==2.1.2
     # via prefect
 pillow==10.4.0
     # via imagehash
+platformdirs==4.3.6
+    # via zeep
 pluggy==1.5.0
     # via pytest
 prefect==2.20.2
@@ -235,6 +247,8 @@ pygit2==1.14.1
     # via viadot2
 pygments==2.18.0
     # via rich
+pyjwt==2.9.0
+    # via simple-salesforce
 pynacl==1.5.0
     # via paramiko
 pyodbc==5.1.0
@@ -264,6 +278,7 @@ pytz==2024.1
     # via pandas
     # via prefect
     # via trino
+    # via zeep
 pytzdata==2020.1
     # via pendulum
 pywavelets==1.7.0
@@ -286,14 +301,22 @@ requests==2.32.3
     # via kubernetes
     # via lumacli
     # via o365
+    # via requests-file
     # via requests-oauthlib
+    # via requests-toolbelt
     # via sharepy
+    # via simple-salesforce
     # via trino
     # via viadot2
+    # via zeep
+requests-file==2.1.0
+    # via zeep
 requests-oauthlib==2.0.0
     # via apprise
     # via kubernetes
     # via o365
+requests-toolbelt==1.0.0
+    # via zeep
 rfc3339-validator==0.1.4
     # via prefect
 rich==13.7.1
@@ -321,7 +344,10 @@ sharepy==2.0.0
     # via viadot2
 shellingham==1.5.4
     # via typer
+simple-salesforce==1.12.6
+    # via viadot2
 six==1.16.0
+    # via isodate
     # via kubernetes
     # via paramiko
     # via python-dateutil
@@ -355,13 +381,15 @@ trino==0.328.0
 typer==0.12.4
     # via lumacli
     # via prefect
+    # via typer
 typing-extensions==4.12.2
     # via aiosqlite
     # via alembic
     # via anyio
     # via prefect
     # via pydantic
     # via pydantic-core
+    # via simple-salesforce
     # via sqlalchemy
     # via typer
     # via uvicorn
@@ -388,3 +416,5 @@ websockets==12.0
     # via prefect
 yarl==1.9.8
     # via aiohttp
+zeep==4.2.1
+    # via simple-salesforce

src/viadot/orchestration/prefect/flows/__init__.py

@@ -15,6 +15,7 @@
 from .mediatool_to_adls import mediatool_to_adls
 from .mindful_to_adls import mindful_to_adls
 from .outlook_to_adls import outlook_to_adls
+from .salesforce_to_adls import salesforce_to_adls
 from .sap_to_parquet import sap_to_parquet
 from .sap_to_redshift_spectrum import sap_to_redshift_spectrum
 from .sftp_to_adls import sftp_to_adls
@@ -45,6 +46,7 @@
     "mediatool_to_adls",
     "mindful_to_adls",
     "outlook_to_adls",
+    "salesforce_to_adls",
     "sap_to_parquet",
     "sap_to_redshift_spectrum",
     "sftp_to_adls",