feat: use event producer config

Replace openedx_event signal handlers with new producer config to push to event bus
openedx · Jul 28, 2023 · b46d149 · b46d149
1 parent b94b0a8
commit b46d149
Show file tree

Hide file tree

Showing 11 changed files with 58 additions and 178 deletions.
diff --git a/cms/djangoapps/contentstore/signals/handlers.py b/cms/djangoapps/contentstore/signals/handlers.py
@@ -13,13 +13,7 @@
 from edx_toggles.toggles import SettingToggle
 from opaque_keys.edx.keys import CourseKey
 from openedx_events.content_authoring.data import CourseCatalogData, CourseScheduleData
-from openedx_events.content_authoring.signals import (
-    COURSE_CATALOG_INFO_CHANGED,
-    XBLOCK_DELETED,
-    XBLOCK_DUPLICATED,
-    XBLOCK_PUBLISHED,
-)
-from openedx_events.event_bus import get_producer
+from openedx_events.content_authoring.signals import COURSE_CATALOG_INFO_CHANGED
 from pytz import UTC
 
 from cms.djangoapps.contentstore.courseware_index import (
@@ -159,60 +153,6 @@ def listen_for_course_publish(sender, course_key, **kwargs):  # pylint: disable=
     transaction.on_commit(lambda: emit_catalog_info_changed_signal(course_key))
 
 
-@receiver(COURSE_CATALOG_INFO_CHANGED)
-def listen_for_course_catalog_info_changed(sender, signal, **kwargs):
-    """
-    Publish COURSE_CATALOG_INFO_CHANGED signals onto the event bus.
-    """
-    get_producer().send(
-        signal=COURSE_CATALOG_INFO_CHANGED, topic='course-catalog-info-changed',
-        event_key_field='catalog_info.course_key', event_data={'catalog_info': kwargs['catalog_info']},
-        event_metadata=kwargs['metadata'],
-    )
-
-
-@receiver(XBLOCK_PUBLISHED)
-def listen_for_xblock_published(sender, signal, **kwargs):
-    """
-    Publish XBLOCK_PUBLISHED signals onto the event bus.
-    """
-    if settings.FEATURES.get("ENABLE_SEND_XBLOCK_EVENTS_OVER_BUS"):
-        topic = getattr(settings, "EVENT_BUS_XBLOCK_LIFECYCLE_TOPIC", "course-authoring-xblock-lifecycle")
-        get_producer().send(
-            signal=XBLOCK_PUBLISHED, topic=topic,
-            event_key_field='xblock_info.usage_key', event_data={'xblock_info': kwargs['xblock_info']},
-            event_metadata=kwargs['metadata'],
-        )
-
-
-@receiver(XBLOCK_DELETED)
-def listen_for_xblock_deleted(sender, signal, **kwargs):
-    """
-    Publish XBLOCK_DELETED signals onto the event bus.
-    """
-    if settings.FEATURES.get("ENABLE_SEND_XBLOCK_EVENTS_OVER_BUS"):
-        topic = getattr(settings, "EVENT_BUS_XBLOCK_LIFECYCLE_TOPIC", "course-authoring-xblock-lifecycle")
-        get_producer().send(
-            signal=XBLOCK_DELETED, topic=topic,
-            event_key_field='xblock_info.usage_key', event_data={'xblock_info': kwargs['xblock_info']},
-            event_metadata=kwargs['metadata'],
-        )
-
-
-@receiver(XBLOCK_DUPLICATED)
-def listen_for_xblock_duplicated(sender, signal, **kwargs):
-    """
-    Publish XBLOCK_DUPLICATED signals onto the event bus.
-    """
-    if settings.FEATURES.get("ENABLE_SEND_XBLOCK_EVENTS_OVER_BUS"):
-        topic = getattr(settings, "EVENT_BUS_XBLOCK_LIFECYCLE_TOPIC", "course-authoring-xblock-lifecycle")
-        get_producer().send(
-            signal=XBLOCK_DUPLICATED, topic=topic,
-            event_key_field='xblock_info.usage_key', event_data={'xblock_info': kwargs['xblock_info']},
-            event_metadata=kwargs['metadata'],
-        )
-
-
 @receiver(SignalHandler.course_deleted)
 def listen_for_course_delete(sender, course_key, **kwargs):  # pylint: disable=unused-argument
     """

diff --git a/cms/envs/common.py b/cms/envs/common.py
@@ -1789,6 +1789,9 @@
 
     # Blockstore
     'blockstore.apps.bundles',
+
+    # Openedx events
+    'openedx_events',
 ]
 
 
@@ -2725,3 +2728,29 @@
 
 DISCUSSIONS_INCONTEXT_FEEDBACK_URL = ''
 DISCUSSIONS_INCONTEXT_LEARNMORE_URL = ''
+
+######################## Event bus producer config ########################
+
+# .. setting_name: EVENT_BUS_PRODUCER_CONFIG
+# .. setting_default: {}
+# .. setting_description: Dictionary of event_types mapped to lists of dictionaries containing topic related configuration.
+#    Each topic configuration dictionary contains
+#    * a topic/stream name called `topic` where the event will be pushed to.
+#    * a flag called `enabled` denoting whether the event will be published to the topic.
+#    * `event_key_field` which is a period-delimited string path to event data field to use as event key.
+#    Note: The topic names should not include environment prefix as it will be dynamically added based on
+#    EVENT_BUS_TOPIC_PREFIX setting.
+EVENT_BUS_PRODUCER_CONFIG = {
+    'org.openedx.content_authoring.course.catalog_info.changed.v1': [
+        {'topic': 'course-catalog-info-changed', 'event_key_field': 'catalog_info.course_key', 'enabled': True},
+    ],
+    'org.openedx.content_authoring.xblock.published.v1': [
+        {'topic': 'content-authoring-xblock-lifecycle', 'event_key_field': 'xblock_info.usage_key', 'enabled': True},
+    ],
+    'org.openedx.content_authoring.xblock.deleted.v1': [
+        {'topic': 'content-authoring-xblock-lifecycle', 'event_key_field': 'xblock_info.usage_key', 'enabled': True},
+    ],
+    'org.openedx.content_authoring.xblock.duplicated.v1': [
+        {'topic': 'content-authoring-xblock-lifecycle', 'event_key_field': 'xblock_info.usage_key', 'enabled': True},
+    ],
+}
diff --git a/cms/envs/devstack.py b/cms/envs/devstack.py
@@ -293,21 +293,10 @@ def should_show_debug_toolbar(request):  # lint-amnesty, pylint: disable=missing
 CREDENTIALS_PUBLIC_SERVICE_URL = 'http://localhost:18150'
 
 #################### Event bus backend ########################
-# .. toggle_name: FEATURES['ENABLE_SEND_XBLOCK_EVENTS_OVER_BUS']
-# .. toggle_implementation: DjangoSetting
-# .. toggle_default: False
-# .. toggle_description: Temporary configuration which enables sending xblock events over the event bus.
-# .. toggle_use_cases: open_edx
-# .. toggle_creation_date: 2023-02-21
-# .. toggle_warning: For consistency in user experience, keep the value in sync with the setting of the same name
-#   in the LMS and CMS.
-# .. toggle_tickets: 'https://github.com/openedx/edx-platform/pull/31813'
-FEATURES['ENABLE_SEND_XBLOCK_EVENTS_OVER_BUS'] = True
 EVENT_BUS_PRODUCER = 'edx_event_bus_redis.create_producer'
 EVENT_BUS_REDIS_CONNECTION_URL = 'redis://:[email protected]:6379/'
 EVENT_BUS_TOPIC_PREFIX = 'dev'
 EVENT_BUS_CONSUMER = 'edx_event_bus_redis.RedisEventConsumer'
-EVENT_BUS_XBLOCK_LIFECYCLE_TOPIC = 'course-authoring-xblock-lifecycle'
 
 ################# New settings must go ABOVE this line #################
 ########################################################################

diff --git a/lms/djangoapps/certificates/signals.py b/lms/djangoapps/certificates/signals.py
@@ -6,12 +6,10 @@
 
 from django.db.models.signals import post_save
 from django.dispatch import receiver
-from openedx_events.event_bus import get_producer
 
 from common.djangoapps.course_modes import api as modes_api
 from common.djangoapps.student.models import CourseEnrollment
 from common.djangoapps.student.signals import ENROLLMENT_TRACK_UPDATED
-from lms.djangoapps.certificates.config import SEND_CERTIFICATE_CREATED_SIGNAL
 from lms.djangoapps.certificates.generation_handler import (
     CertificateGenerationNotAllowed,
     generate_allowlist_certificate_task,
@@ -32,7 +30,6 @@
     COURSE_GRADE_NOW_PASSED,
     LEARNER_NOW_VERIFIED
 )
-from openedx_events.learning.signals import CERTIFICATE_CREATED
 
 log = logging.getLogger(__name__)
 
@@ -159,18 +156,3 @@ def _listen_for_enrollment_mode_change(sender, user, course_key, mode, **kwargs)
                 course_key,
             )
             return False
-
-
-@receiver(CERTIFICATE_CREATED)
-def listen_for_certificate_created_event(sender, signal, **kwargs):
-    """
-    Publish `CERTIFICATE_CREATED` events to the event bus.
-    """
-    if SEND_CERTIFICATE_CREATED_SIGNAL.is_enabled():
-        get_producer().send(
-            signal=CERTIFICATE_CREATED,
-            topic='learning-certificate-lifecycle',
-            event_key_field='certificate.course.course_key',
-            event_data={'certificate': kwargs['certificate']},
-            event_metadata=kwargs['metadata']
-        )
diff --git a/lms/djangoapps/certificates/tests/test_signals.py b/lms/djangoapps/certificates/tests/test_signals.py
@@ -3,12 +3,9 @@
 and disabling for instructor-paced courses.
 """
 
-from datetime import datetime, timezone
 from unittest import mock
-from uuid import uuid4
 
 import ddt
-from django.test.utils import override_settings
 from edx_toggles.toggles.testutils import override_waffle_flag, override_waffle_switch
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.tests.factories import CourseFactory
@@ -21,14 +18,10 @@
     CertificateGenerationConfiguration,
     GeneratedCertificate
 )
-from lms.djangoapps.certificates.signals import listen_for_certificate_created_event
 from lms.djangoapps.certificates.tests.factories import CertificateAllowlistFactory, GeneratedCertificateFactory
 from lms.djangoapps.grades.course_grade_factory import CourseGradeFactory
 from lms.djangoapps.grades.tests.utils import mock_passing_grade
 from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification
-from openedx_events.data import EventsMetadata
-from openedx_events.learning.signals import CERTIFICATE_CREATED
-from openedx_events.learning.data import CourseData, UserData, UserPersonalData, CertificateData
 
 
 class SelfGeneratedCertsSignalTest(ModuleStoreTestCase):
@@ -440,78 +433,3 @@ def test_verified_to_audit(self):
         ) as mock_allowlist_task:
             self.verified_enrollment.change_mode('audit')
             mock_allowlist_task.assert_not_called()
-
-
-class CertificateEventBusTests(ModuleStoreTestCase):
-    """
-    Tests for Certificate events that interact with the event bus.
-    """
-    def setUp(self):
-        super().setUp()
-        self.user = UserFactory.create()
-        self.name = f'{self.user.first_name} {self.user.last_name}'
-        self.course = CourseFactory.create(self_paced=True)
-        self.enrollment = CourseEnrollmentFactory(
-            user=self.user,
-            course_id=self.course.id,
-            is_active=True,
-            mode='verified',
-        )
-
-    @override_settings(SEND_CERTIFICATE_CREATED_SIGNAL=False)
-    @mock.patch('lms.djangoapps.certificates.signals.get_producer', autospec=True)
-    def test_event_disabled(self, mock_producer):
-        """
-        Test to verify that we do not push `CERTIFICATE_CREATED` events to the event bus if the
-        `SEND_CERTIFICATE_CREATED_SIGNAL` setting is disabled.
-        """
-        listen_for_certificate_created_event(None, CERTIFICATE_CREATED)
-        mock_producer.assert_not_called()
-
-    @override_settings(SEND_CERTIFICATE_CREATED_SIGNAL=True)
-    @mock.patch('lms.djangoapps.certificates.signals.get_producer', autospec=True)
-    def test_event_enabled(self, mock_producer):
-        """
-        Test to verify that we push `CERTIFICATE_CREATED` events to the event bus if the
-        `SEND_CERTIFICATE_CREATED_SIGNAL` setting is enabled.
-        """
-        expected_course_data = CourseData(course_key=self.course.id)
-        expected_user_data = UserData(
-            pii=UserPersonalData(
-                username=self.user.username,
-                email=self.user.email,
-                name=self.name,
-            ),
-            id=self.user.id,
-            is_active=self.user.is_active
-        )
-        expected_certificate_data = CertificateData(
-            user=expected_user_data,
-            course=expected_course_data,
-            mode='verified',
-            grade='',
-            current_status='downloadable',
-            download_url='',
-            name='',
-        )
-        event_metadata = EventsMetadata(
-            event_type=CERTIFICATE_CREATED.event_type,
-            id=uuid4(),
-            minorversion=0,
-            source='openedx/lms/web',
-            sourcehost='lms.test',
-            time=datetime.now(timezone.utc)
-        )
-
-        event_kwargs = {
-            'certificate': expected_certificate_data,
-            'metadata': event_metadata
-        }
-
-        listen_for_certificate_created_event(None, CERTIFICATE_CREATED, **event_kwargs)
-        # verify that the data sent to the event bus matches what we expect
-        data = mock_producer.return_value.send.call_args.kwargs
-        assert data['signal'].event_type == CERTIFICATE_CREATED.event_type
-        assert data['event_data']['certificate'] == expected_certificate_data
-        assert data['topic'] == 'learning-certificate-lifecycle'
-        assert data['event_key_field'] == 'certificate.course.course_key'
diff --git a/lms/envs/common.py b/lms/envs/common.py
@@ -3274,6 +3274,9 @@ def _make_locale_paths(settings):  # pylint: disable=missing-function-docstring
 
     # Notifications
     'openedx.core.djangoapps.notifications',
+
+    # Openedx events
+    'openedx_events',
 ]
 
 ######################### CSRF #########################################
@@ -5334,3 +5337,20 @@ def _make_locale_paths(settings):  # pylint: disable=missing-function-docstring
 ############## NOTIFICATIONS EXPIRY ##############
 NOTIFICATIONS_EXPIRY = 60
 EXPIRED_NOTIFICATIONS_DELETE_BATCH_SIZE = 10000
+
+######################## Event bus producer config ########################
+
+# .. setting_name: EVENT_BUS_PRODUCER_CONFIG
+# .. setting_default: {}
+# .. setting_description: Dictionary of event_types mapped to lists of dictionaries containing topic related configuration.
+#    Each topic configuration dictionary contains
+#    * a topic/stream name called `topic` where the event will be pushed to.
+#    * a flag called `enabled` denoting whether the event will be published to the topic.
+#    * `event_key_field` which is a period-delimited string path to event data field to use as event key.
+#    Note: The topic names should not include environment prefix as it will be dynamically added based on
+#    EVENT_BUS_TOPIC_PREFIX setting.
+EVENT_BUS_PRODUCER_CONFIG = {
+    'org.openedx.learning.certificate.created.v1': [
+        {'topic': 'learning-certificate-lifecycle', 'event_key_field': 'certificate.course.course_key', 'enabled': True},
+    ],
+}
diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt
@@ -762,7 +762,7 @@ openedx-django-require==2.1.0
     # via -r requirements/edx/kernel.in
 openedx-django-wiki==2.0.0
     # via -r requirements/edx/kernel.in
-openedx-events==8.3.0
+openedx-events @ git+https://github.com/open-craft/openedx-events@navin/configurable-handler
     # via
     #   -r requirements/edx/kernel.in
     #   edx-event-bus-kafka

diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt
@@ -1294,7 +1294,7 @@ openedx-django-wiki==2.0.0
     # via
     #   -r requirements/edx/doc.txt
     #   -r requirements/edx/testing.txt
-openedx-events==8.3.0
+openedx-events @ git+https://github.com/open-craft/openedx-events@navin/configurable-handler
     # via
     #   -r requirements/edx/doc.txt
     #   -r requirements/edx/testing.txt
@@ -2138,6 +2138,7 @@ walrus==0.9.3
     #   edx-event-bus-redis
 watchdog==3.0.0
     # via
+    #   -r requirements/edx/development.in
     #   -r requirements/edx/doc.txt
     #   -r requirements/edx/testing.txt
 wcwidth==0.2.6

diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt
@@ -556,7 +556,7 @@ edx-drf-extensions==8.8.0
     #   edx-rbac
     #   edx-when
     #   edxval
-edx-enterprise==4.0.6
+edx-enterprise==4.0.7
     # via
     #   -c requirements/edx/../constraints.txt
     #   -r requirements/edx/base.txt
@@ -905,7 +905,7 @@ openedx-django-require==2.1.0
     # via -r requirements/edx/base.txt
 openedx-django-wiki==2.0.0
     # via -r requirements/edx/base.txt
-openedx-events==8.3.0
+openedx-events @ git+https://github.com/open-craft/openedx-events@navin/configurable-handler
     # via
     #   -r requirements/edx/base.txt
     #   edx-event-bus-kafka

diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in
@@ -113,7 +113,8 @@ oauthlib                            # OAuth specification support for authentica
 olxcleaner
 openedx-calc                        # Library supporting mathematical calculations for Open edX
 openedx-django-require
-openedx-events>=8.3.0               # Open edX Events from Hooks Extension Framework (OEP-50)
+# openedx-events>=8.3.0               # Open edX Events from Hooks Extension Framework (OEP-50)
+git+https://github.com/open-craft/openedx-events@navin/configurable-handler
 openedx-filters                     # Open edX Filters from Hooks Extension Framework (OEP-50)
 openedx-learning<=0.1
 openedx-mongodbproxy

diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt
@@ -974,7 +974,7 @@ openedx-django-require==2.1.0
     # via -r requirements/edx/base.txt
 openedx-django-wiki==2.0.0
     # via -r requirements/edx/base.txt
-openedx-events==8.3.0
+openedx-events @ git+https://github.com/open-craft/openedx-events@navin/configurable-handler
     # via
     #   -r requirements/edx/base.txt
     #   edx-event-bus-kafka