diff --git a/api/poetry.lock b/api/poetry.lock index 30015a4fb66..91d8dcdfff9 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -35,6 +35,21 @@ files = [ [package.dependencies] requests = ">=2.21,<3.0" +[[package]] +name = "amqp" +version = "5.3.1" +description = "Low-level AMQP client for Python (fork of amqplib)." +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2"}, + {file = "amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432"}, +] + +[package.dependencies] +vine = ">=5.0.0,<6.0.0" + [[package]] name = "annotated-types" version = "0.7.0" @@ -273,6 +288,18 @@ charset-normalizer = ["charset-normalizer"] html5lib = ["html5lib"] lxml = ["lxml"] +[[package]] +name = "billiard" +version = "4.2.1" +description = "Python multiprocessing fork with improvements and bugfixes" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "billiard-4.2.1-py3-none-any.whl", hash = "sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb"}, + {file = "billiard-4.2.1.tar.gz", hash = "sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f"}, +] + [[package]] name = "black" version = "24.10.0" @@ -541,6 +568,64 @@ files = [ {file = "cachetools-5.5.0.tar.gz", hash = "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a"}, ] +[[package]] +name = "celery" +version = "5.4.0" +description = "Distributed Task Queue." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "celery-5.4.0-py3-none-any.whl", hash = "sha256:369631eb580cf8c51a82721ec538684994f8277637edde2dfc0dacd73ed97f64"}, + {file = "celery-5.4.0.tar.gz", hash = "sha256:504a19140e8d3029d5acad88330c541d4c3f64c789d85f94756762d8bca7e706"}, +] + +[package.dependencies] +billiard = ">=4.2.0,<5.0" +click = ">=8.1.2,<9.0" +click-didyoumean = ">=0.3.0" +click-plugins = ">=1.1.1" +click-repl = ">=0.2.0" +kombu = ">=5.3.4,<6.0" +python-dateutil = ">=2.8.2" +redis = {version = ">=4.5.2,<4.5.5 || >4.5.5,<6.0.0", optional = true, markers = "extra == \"redis\""} +tzdata = ">=2022.7" +vine = ">=5.1.0,<6.0" + +[package.extras] +arangodb = ["pyArango (>=2.0.2)"] +auth = ["cryptography (==42.0.5)"] +azureblockblob = ["azure-storage-blob (>=12.15.0)"] +brotli = ["brotli (>=1.0.0)", "brotlipy (>=0.7.0)"] +cassandra = ["cassandra-driver (>=3.25.0,<4)"] +consul = ["python-consul2 (==0.1.5)"] +cosmosdbsql = ["pydocumentdb (==2.3.5)"] +couchbase = ["couchbase (>=3.0.0)"] +couchdb = ["pycouchdb (==1.14.2)"] +django = ["Django (>=2.2.28)"] +dynamodb = ["boto3 (>=1.26.143)"] +elasticsearch = ["elastic-transport (<=8.13.0)", "elasticsearch (<=8.13.0)"] +eventlet = ["eventlet (>=0.32.0)"] +gcs = ["google-cloud-storage (>=2.10.0)"] +gevent = ["gevent (>=1.5.0)"] +librabbitmq = ["librabbitmq (>=2.0.0)"] +memcache = ["pylibmc (==1.6.3)"] +mongodb = ["pymongo[srv] (>=4.0.2)"] +msgpack = ["msgpack (==1.0.8)"] +pymemcache = ["python-memcached (>=1.61)"] +pyro = ["pyro4 (==4.82)"] +pytest = ["pytest-celery[all] (>=1.0.0)"] +redis = ["redis (>=4.5.2,!=4.5.5,<6.0.0)"] +s3 = ["boto3 (>=1.26.143)"] +slmq = ["softlayer-messaging (>=1.0.3)"] +solar = ["ephem (==4.1.5)"] +sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"] +sqs = ["boto3 (>=1.26.143)", "kombu[sqs] (>=5.3.4)", "pycurl (>=7.43.0.5)", "urllib3 (>=1.26.16)"] +tblib = ["tblib (>=1.3.0)", "tblib (>=1.5.0)"] +yaml = ["PyYAML (>=3.10)"] +zookeeper = ["kazoo (>=1.3.1)"] +zstd = ["zstandard (==0.22.0)"] + [[package]] name = "certifi" version = "2024.8.30" @@ -848,6 +933,21 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "click-didyoumean" +version = "0.3.1" +description = "Enables git-like *did-you-mean* feature in click" +optional = false +python-versions = ">=3.6.2" +groups = ["main"] +files = [ + {file = "click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c"}, + {file = "click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463"}, +] + +[package.dependencies] +click = ">=7" + [[package]] name = "click-option-group" version = "0.5.6" @@ -886,6 +986,25 @@ click = ">=4.0" [package.extras] dev = ["coveralls", "pytest (>=3.6)", "pytest-cov", "wheel"] +[[package]] +name = "click-repl" +version = "0.3.0" +description = "REPL plugin for Click" +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9"}, + {file = "click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812"}, +] + +[package.dependencies] +click = ">=7.0" +prompt-toolkit = ">=3.0.36" + +[package.extras] +testing = ["pytest (>=7.2.1)", "pytest-cov (>=4.0.0)", "tox (>=4.4.3)"] + [[package]] name = "clickhouse-driver" version = "0.2.9" @@ -2604,6 +2723,40 @@ files = [ {file = "json5-0.9.25.tar.gz", hash = "sha256:548e41b9be043f9426776f05df8635a00fe06104ea51ed24b67f908856e151ae"}, ] +[[package]] +name = "kombu" +version = "5.4.2" +description = "Messaging library for Python." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "kombu-5.4.2-py3-none-any.whl", hash = "sha256:14212f5ccf022fc0a70453bb025a1dcc32782a588c49ea866884047d66e14763"}, + {file = "kombu-5.4.2.tar.gz", hash = "sha256:eef572dd2fd9fc614b37580e3caeafdd5af46c1eff31e7fba89138cdb406f2cf"}, +] + +[package.dependencies] +amqp = ">=5.1.1,<6.0.0" +tzdata = {version = "*", markers = "python_version >= \"3.9\""} +vine = "5.1.0" + +[package.extras] +azureservicebus = ["azure-servicebus (>=7.10.0)"] +azurestoragequeues = ["azure-identity (>=1.12.0)", "azure-storage-queue (>=12.6.0)"] +confluentkafka = ["confluent-kafka (>=2.2.0)"] +consul = ["python-consul2 (==0.1.5)"] +librabbitmq = ["librabbitmq (>=2.0.0)"] +mongodb = ["pymongo (>=4.1.1)"] +msgpack = ["msgpack (==1.1.0)"] +pyro = ["pyro4 (==4.82)"] +qpid = ["qpid-python (>=0.26)", "qpid-tools (>=0.26)"] +redis = ["redis (>=4.5.2,!=4.5.5,!=5.0.2)"] +slmq = ["softlayer-messaging (>=1.0.3)"] +sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"] +sqs = ["boto3 (>=1.26.143)", "pycurl (>=7.43.0.5)", "urllib3 (>=1.26.16)"] +yaml = ["PyYAML (>=3.10)"] +zookeeper = ["kazoo (>=2.8.0)"] + [[package]] name = "leb128" version = "1.0.8" @@ -5854,7 +6007,6 @@ description = "Provider of IANA time zone data" optional = false python-versions = ">=2" groups = ["main"] -markers = "platform_system == \"Windows\"" files = [ {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, @@ -5908,6 +6060,18 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "vine" +version = "5.1.0" +description = "Python promises." +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc"}, + {file = "vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0"}, +] + [[package]] name = "watchdog" version = "6.0.0" @@ -6391,4 +6555,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.11,<4.0" -content-hash = "1828c52b5922d32c1432227ebdad2fc6701cd970d579f3cf28a4c246d4d959e2" +content-hash = "ccd8282c679ad8b1624b38fac962f7e7fb4d6d5efc382ecd7fb7b728afa51aff" diff --git a/api/pyproject.toml b/api/pyproject.toml index 41d039c4c89..29128c8e2f3 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -80,7 +80,8 @@ dependencies = [ "wtforms-sqlalchemy==0.4.2", "wtforms==3.2.1", "xlsxwriter>=3.2.0", -"zeep==4.3.1" +"zeep==4.3.1", +"celery[redis] (>=5.4.0,<6.0.0)" ] [tool.poetry.group.dev] diff --git a/api/src/pcapi/asynchronous_tasks/__init__.py b/api/src/pcapi/asynchronous_tasks/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/api/src/pcapi/asynchronous_tasks/celery.py b/api/src/pcapi/asynchronous_tasks/celery.py new file mode 100644 index 00000000000..6f2bd02f473 --- /dev/null +++ b/api/src/pcapi/asynchronous_tasks/celery.py @@ -0,0 +1,15 @@ +from celery import Celery, Task +from flask import Flask + +def celery_init_app(app: Flask) -> Celery: + class FlaskTask(Task): + def __call__(self, *args: object, **kwargs: object) -> object: + with app.app_context(): + return self.run(*args, **kwargs) + + celery_app = Celery(app.name, task_cls=FlaskTask) + celery_app.config_from_object(app.config["CELERY"]) + celery_app.set_default() + app.extensions["celery"] = celery_app + return celery_app + diff --git a/api/src/pcapi/asynchronous_tasks/celery_worker.py b/api/src/pcapi/asynchronous_tasks/celery_worker.py new file mode 100644 index 00000000000..199ea6d7939 --- /dev/null +++ b/api/src/pcapi/asynchronous_tasks/celery_worker.py @@ -0,0 +1,4 @@ +from pcapi.flask_app import app as flask_app + + +celery_app = flask_app.extensions["celery"] diff --git a/api/src/pcapi/asynchronous_tasks/sendinblue.py b/api/src/pcapi/asynchronous_tasks/sendinblue.py new file mode 100644 index 00000000000..11d3fce1ce6 --- /dev/null +++ b/api/src/pcapi/asynchronous_tasks/sendinblue.py @@ -0,0 +1,21 @@ +from celery import shared_task + +from pcapi.core.external import sendinblue +from pcapi.core.mails.transactional.send_transactional_email import send_transactional_email +from pcapi.tasks.serialization.sendinblue_tasks import SendTransactionalEmailRequest +from pcapi.tasks.serialization.sendinblue_tasks import UpdateSendinblueContactRequest + + +@shared_task(name="mails.tasks.update_contact_attributes", acks_late=True) +def update_contact_attributes_task(payload: UpdateSendinblueContactRequest) -> None: + sendinblue.make_update_request(payload) + + +@shared_task(name="mails.tasks.send_transactional_email_primary", acks_late=True) +def send_transactional_email_primary_task(payload: SendTransactionalEmailRequest) -> None: + send_transactional_email(payload) + + +@shared_task(name="mails.tasks.send_transactional_email_secondary", acks_late=True) +def send_transactional_email_secondary_task(payload: SendTransactionalEmailRequest) -> None: + send_transactional_email(payload) diff --git a/api/src/pcapi/core/external/sendinblue.py b/api/src/pcapi/core/external/sendinblue.py index 014adae6526..e762ec29c1b 100644 --- a/api/src/pcapi/core/external/sendinblue.py +++ b/api/src/pcapi/core/external/sendinblue.py @@ -14,12 +14,12 @@ from sib_api_v3_sdk.rest import ApiException as SendinblueApiException from pcapi import settings +from pcapi.asynchronous_tasks.sendinblue import update_contact_attributes_task from pcapi.core import mails as mails_api from pcapi.core.cultural_survey import models as cultural_survey_models from pcapi.core.external.attributes import models as attributes_models import pcapi.core.users.models as users_models from pcapi.models.feature import FeatureToggle -from pcapi.tasks.sendinblue_tasks import update_contact_attributes_task from pcapi.tasks.serialization.sendinblue_tasks import UpdateSendinblueContactRequest diff --git a/api/src/pcapi/core/mails/backends/sendinblue.py b/api/src/pcapi/core/mails/backends/sendinblue.py index 9dc2614f2ef..e3d6c2efecb 100644 --- a/api/src/pcapi/core/mails/backends/sendinblue.py +++ b/api/src/pcapi/core/mails/backends/sendinblue.py @@ -7,10 +7,10 @@ from sib_api_v3_sdk.rest import ApiException as SendinblueApiException from pcapi import settings +from pcapi.asynchronous_tasks.sendinblue import send_transactional_email_primary_task +from pcapi.asynchronous_tasks.sendinblue import send_transactional_email_secondary_task from pcapi.core.users.repository import find_user_by_email from pcapi.models.feature import FeatureToggle -from pcapi.tasks.sendinblue_tasks import send_transactional_email_primary_task -from pcapi.tasks.sendinblue_tasks import send_transactional_email_secondary_task import pcapi.tasks.serialization.sendinblue_tasks as serializers from pcapi.utils import email as email_utils from pcapi.utils.email import is_email_whitelisted diff --git a/api/src/pcapi/core/mails/transactional/send_transactional_email.py b/api/src/pcapi/core/mails/transactional/send_transactional_email.py index b2b933a6257..2c31eeade6a 100644 --- a/api/src/pcapi/core/mails/transactional/send_transactional_email.py +++ b/api/src/pcapi/core/mails/transactional/send_transactional_email.py @@ -58,6 +58,8 @@ def send_transactional_email(payload: SendTransactionalEmailRequest) -> None: try: configuration = sib_api_v3_sdk.Configuration() + if settings.PROXY_CERT_BUNDLE is not None: + configuration.ssl_ca_cert = settings.PROXY_CERT_BUNDLE if payload.use_pro_subaccount: configuration.api_key["api-key"] = settings.SENDINBLUE_PRO_API_KEY else: diff --git a/api/src/pcapi/flask_app.py b/api/src/pcapi/flask_app.py index 2b165023f40..9bff4b1a928 100644 --- a/api/src/pcapi/flask_app.py +++ b/api/src/pcapi/flask_app.py @@ -23,6 +23,7 @@ from werkzeug.middleware.proxy_fix import ProxyFix from pcapi import settings +from pcapi.asynchronous_tasks.celery import celery_init_app from pcapi.core import monkeypatches from pcapi.core.finance import utils as finance_utils from pcapi.core.logging import get_or_set_correlation_id @@ -167,6 +168,25 @@ def add_security_headers(response: flask.wrappers.Response) -> flask.wrappers.Re install_commands(app) finance_utils.install_template_filters(app) +app.config.from_mapping( + CELERY=dict( + broker_url=settings.REDIS_URL, + task_acks_late=True, + task_reject_on_worker_lost=True, + # Pickle seems the best pick since we don't support + # anything other than python https://docs.celeryq.dev/en/latest/userguide/calling.html#serializers + task_serializer="pickle", + result_serializer="pickle", + accept_content=["pickle"], + task_routes={ + "mails.tasks.*": {"queue": "mails"}, + }, + task_ignore_result=True, + ), +) + +celery_init_app(app) + backoffice_oauth = OAuth(app) backoffice_oauth.register( diff --git a/api/src/pcapi/settings.py b/api/src/pcapi/settings.py index 62b97859eda..09ddc2568db 100644 --- a/api/src/pcapi/settings.py +++ b/api/src/pcapi/settings.py @@ -652,3 +652,6 @@ # HARVESTR HARVESTR_API_KEY = secrets_utils.get("HARVESTR_API_KEY", "") + +# PROXY configuration +PROXY_CERT_BUNDLE = os.environ.get("PROXY_CERT_BUNDLE", None)