Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: use event producer config #32874

Closed
wants to merge 1 commit into from
Closed

feat: use event producer config #32874

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 1 addition & 61 deletions cms/djangoapps/contentstore/signals/handlers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,7 @@
from edx_toggles.toggles import SettingToggle
from opaque_keys.edx.keys import CourseKey
from openedx_events.content_authoring.data import CourseCatalogData, CourseScheduleData
from openedx_events.content_authoring.signals import (
COURSE_CATALOG_INFO_CHANGED,
XBLOCK_DELETED,
XBLOCK_DUPLICATED,
XBLOCK_PUBLISHED,
)
from openedx_events.event_bus import get_producer
from openedx_events.content_authoring.signals import COURSE_CATALOG_INFO_CHANGED
from pytz import UTC

from cms.djangoapps.contentstore.courseware_index import (
Expand Down Expand Up @@ -159,60 +153,6 @@ def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=
transaction.on_commit(lambda: emit_catalog_info_changed_signal(course_key))


@receiver(COURSE_CATALOG_INFO_CHANGED)
def listen_for_course_catalog_info_changed(sender, signal, **kwargs):
"""
Publish COURSE_CATALOG_INFO_CHANGED signals onto the event bus.
"""
get_producer().send(
signal=COURSE_CATALOG_INFO_CHANGED, topic='course-catalog-info-changed',
event_key_field='catalog_info.course_key', event_data={'catalog_info': kwargs['catalog_info']},
event_metadata=kwargs['metadata'],
)


@receiver(XBLOCK_PUBLISHED)
def listen_for_xblock_published(sender, signal, **kwargs):
"""
Publish XBLOCK_PUBLISHED signals onto the event bus.
"""
if settings.FEATURES.get("ENABLE_SEND_XBLOCK_EVENTS_OVER_BUS"):
topic = getattr(settings, "EVENT_BUS_XBLOCK_LIFECYCLE_TOPIC", "course-authoring-xblock-lifecycle")
get_producer().send(
signal=XBLOCK_PUBLISHED, topic=topic,
event_key_field='xblock_info.usage_key', event_data={'xblock_info': kwargs['xblock_info']},
event_metadata=kwargs['metadata'],
)


@receiver(XBLOCK_DELETED)
def listen_for_xblock_deleted(sender, signal, **kwargs):
"""
Publish XBLOCK_DELETED signals onto the event bus.
"""
if settings.FEATURES.get("ENABLE_SEND_XBLOCK_EVENTS_OVER_BUS"):
topic = getattr(settings, "EVENT_BUS_XBLOCK_LIFECYCLE_TOPIC", "course-authoring-xblock-lifecycle")
get_producer().send(
signal=XBLOCK_DELETED, topic=topic,
event_key_field='xblock_info.usage_key', event_data={'xblock_info': kwargs['xblock_info']},
event_metadata=kwargs['metadata'],
)


@receiver(XBLOCK_DUPLICATED)
def listen_for_xblock_duplicated(sender, signal, **kwargs):
"""
Publish XBLOCK_DUPLICATED signals onto the event bus.
"""
if settings.FEATURES.get("ENABLE_SEND_XBLOCK_EVENTS_OVER_BUS"):
topic = getattr(settings, "EVENT_BUS_XBLOCK_LIFECYCLE_TOPIC", "course-authoring-xblock-lifecycle")
get_producer().send(
signal=XBLOCK_DUPLICATED, topic=topic,
event_key_field='xblock_info.usage_key', event_data={'xblock_info': kwargs['xblock_info']},
event_metadata=kwargs['metadata'],
)


@receiver(SignalHandler.course_deleted)
def listen_for_course_delete(sender, course_key, **kwargs): # pylint: disable=unused-argument
"""
Expand Down
29 changes: 29 additions & 0 deletions cms/envs/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1789,6 +1789,9 @@

# Blockstore
'blockstore.apps.bundles',

# Openedx events
'openedx_events',
]


Expand Down Expand Up @@ -2725,3 +2728,29 @@

DISCUSSIONS_INCONTEXT_FEEDBACK_URL = ''
DISCUSSIONS_INCONTEXT_LEARNMORE_URL = ''

######################## Event bus producer config ########################

# .. setting_name: EVENT_BUS_PRODUCER_CONFIG
# .. setting_default: {}
# .. setting_description: Dictionary of event_types mapped to lists of dictionaries containing topic related configuration.
# Each topic configuration dictionary contains
# * a topic/stream name called `topic` where the event will be pushed to.
# * a flag called `enabled` denoting whether the event will be published to the topic.
# * `event_key_field` which is a period-delimited string path to event data field to use as event key.
# Note: The topic names should not include environment prefix as it will be dynamically added based on
# EVENT_BUS_TOPIC_PREFIX setting.
EVENT_BUS_PRODUCER_CONFIG = {
'org.openedx.content_authoring.course.catalog_info.changed.v1': [
{'topic': 'course-catalog-info-changed', 'event_key_field': 'catalog_info.course_key', 'enabled': True},
],
'org.openedx.content_authoring.xblock.published.v1': [
{'topic': 'content-authoring-xblock-lifecycle', 'event_key_field': 'xblock_info.usage_key', 'enabled': True},
],
'org.openedx.content_authoring.xblock.deleted.v1': [
{'topic': 'content-authoring-xblock-lifecycle', 'event_key_field': 'xblock_info.usage_key', 'enabled': True},
],
'org.openedx.content_authoring.xblock.duplicated.v1': [
{'topic': 'content-authoring-xblock-lifecycle', 'event_key_field': 'xblock_info.usage_key', 'enabled': True},
],
}
11 changes: 0 additions & 11 deletions cms/envs/devstack.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,21 +293,10 @@ def should_show_debug_toolbar(request): # lint-amnesty, pylint: disable=missing
CREDENTIALS_PUBLIC_SERVICE_URL = 'http://localhost:18150'

#################### Event bus backend ########################
# .. toggle_name: FEATURES['ENABLE_SEND_XBLOCK_EVENTS_OVER_BUS']
# .. toggle_implementation: DjangoSetting
# .. toggle_default: False
# .. toggle_description: Temporary configuration which enables sending xblock events over the event bus.
# .. toggle_use_cases: open_edx
# .. toggle_creation_date: 2023-02-21
# .. toggle_warning: For consistency in user experience, keep the value in sync with the setting of the same name
# in the LMS and CMS.
# .. toggle_tickets: 'https://github.com/openedx/edx-platform/pull/31813'
FEATURES['ENABLE_SEND_XBLOCK_EVENTS_OVER_BUS'] = True
EVENT_BUS_PRODUCER = 'edx_event_bus_redis.create_producer'
EVENT_BUS_REDIS_CONNECTION_URL = 'redis://:[email protected]:6379/'
EVENT_BUS_TOPIC_PREFIX = 'dev'
EVENT_BUS_CONSUMER = 'edx_event_bus_redis.RedisEventConsumer'
EVENT_BUS_XBLOCK_LIFECYCLE_TOPIC = 'course-authoring-xblock-lifecycle'

################# New settings must go ABOVE this line #################
########################################################################
Expand Down
18 changes: 0 additions & 18 deletions lms/djangoapps/certificates/signals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,10 @@

from django.db.models.signals import post_save
from django.dispatch import receiver
from openedx_events.event_bus import get_producer

from common.djangoapps.course_modes import api as modes_api
from common.djangoapps.student.models import CourseEnrollment
from common.djangoapps.student.signals import ENROLLMENT_TRACK_UPDATED
from lms.djangoapps.certificates.config import SEND_CERTIFICATE_CREATED_SIGNAL
from lms.djangoapps.certificates.generation_handler import (
CertificateGenerationNotAllowed,
generate_allowlist_certificate_task,
Expand All @@ -32,7 +30,6 @@
COURSE_GRADE_NOW_PASSED,
LEARNER_NOW_VERIFIED
)
from openedx_events.learning.signals import CERTIFICATE_CREATED

log = logging.getLogger(__name__)

Expand Down Expand Up @@ -159,18 +156,3 @@ def _listen_for_enrollment_mode_change(sender, user, course_key, mode, **kwargs)
course_key,
)
return False


@receiver(CERTIFICATE_CREATED)
def listen_for_certificate_created_event(sender, signal, **kwargs):
"""
Publish `CERTIFICATE_CREATED` events to the event bus.
"""
if SEND_CERTIFICATE_CREATED_SIGNAL.is_enabled():
get_producer().send(
signal=CERTIFICATE_CREATED,
topic='learning-certificate-lifecycle',
event_key_field='certificate.course.course_key',
event_data={'certificate': kwargs['certificate']},
event_metadata=kwargs['metadata']
)
82 changes: 0 additions & 82 deletions lms/djangoapps/certificates/tests/test_signals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,9 @@
and disabling for instructor-paced courses.
"""

from datetime import datetime, timezone
from unittest import mock
from uuid import uuid4

import ddt
from django.test.utils import override_settings
from edx_toggles.toggles.testutils import override_waffle_flag, override_waffle_switch
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
Expand All @@ -21,14 +18,10 @@
CertificateGenerationConfiguration,
GeneratedCertificate
)
from lms.djangoapps.certificates.signals import listen_for_certificate_created_event
from lms.djangoapps.certificates.tests.factories import CertificateAllowlistFactory, GeneratedCertificateFactory
from lms.djangoapps.grades.course_grade_factory import CourseGradeFactory
from lms.djangoapps.grades.tests.utils import mock_passing_grade
from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification
from openedx_events.data import EventsMetadata
from openedx_events.learning.signals import CERTIFICATE_CREATED
from openedx_events.learning.data import CourseData, UserData, UserPersonalData, CertificateData


class SelfGeneratedCertsSignalTest(ModuleStoreTestCase):
Expand Down Expand Up @@ -440,78 +433,3 @@ def test_verified_to_audit(self):
) as mock_allowlist_task:
self.verified_enrollment.change_mode('audit')
mock_allowlist_task.assert_not_called()


class CertificateEventBusTests(ModuleStoreTestCase):
"""
Tests for Certificate events that interact with the event bus.
"""
def setUp(self):
super().setUp()
self.user = UserFactory.create()
self.name = f'{self.user.first_name} {self.user.last_name}'
self.course = CourseFactory.create(self_paced=True)
self.enrollment = CourseEnrollmentFactory(
user=self.user,
course_id=self.course.id,
is_active=True,
mode='verified',
)

@override_settings(SEND_CERTIFICATE_CREATED_SIGNAL=False)
@mock.patch('lms.djangoapps.certificates.signals.get_producer', autospec=True)
def test_event_disabled(self, mock_producer):
"""
Test to verify that we do not push `CERTIFICATE_CREATED` events to the event bus if the
`SEND_CERTIFICATE_CREATED_SIGNAL` setting is disabled.
"""
listen_for_certificate_created_event(None, CERTIFICATE_CREATED)
mock_producer.assert_not_called()

@override_settings(SEND_CERTIFICATE_CREATED_SIGNAL=True)
@mock.patch('lms.djangoapps.certificates.signals.get_producer', autospec=True)
def test_event_enabled(self, mock_producer):
"""
Test to verify that we push `CERTIFICATE_CREATED` events to the event bus if the
`SEND_CERTIFICATE_CREATED_SIGNAL` setting is enabled.
"""
expected_course_data = CourseData(course_key=self.course.id)
expected_user_data = UserData(
pii=UserPersonalData(
username=self.user.username,
email=self.user.email,
name=self.name,
),
id=self.user.id,
is_active=self.user.is_active
)
expected_certificate_data = CertificateData(
user=expected_user_data,
course=expected_course_data,
mode='verified',
grade='',
current_status='downloadable',
download_url='',
name='',
)
event_metadata = EventsMetadata(
event_type=CERTIFICATE_CREATED.event_type,
id=uuid4(),
minorversion=0,
source='openedx/lms/web',
sourcehost='lms.test',
time=datetime.now(timezone.utc)
)

event_kwargs = {
'certificate': expected_certificate_data,
'metadata': event_metadata
}

listen_for_certificate_created_event(None, CERTIFICATE_CREATED, **event_kwargs)
# verify that the data sent to the event bus matches what we expect
data = mock_producer.return_value.send.call_args.kwargs
assert data['signal'].event_type == CERTIFICATE_CREATED.event_type
assert data['event_data']['certificate'] == expected_certificate_data
assert data['topic'] == 'learning-certificate-lifecycle'
assert data['event_key_field'] == 'certificate.course.course_key'
20 changes: 20 additions & 0 deletions lms/envs/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3274,6 +3274,9 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring

# Notifications
'openedx.core.djangoapps.notifications',

# Openedx events
'openedx_events',
]

######################### CSRF #########################################
Expand Down Expand Up @@ -5334,3 +5337,20 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring
############## NOTIFICATIONS EXPIRY ##############
NOTIFICATIONS_EXPIRY = 60
EXPIRED_NOTIFICATIONS_DELETE_BATCH_SIZE = 10000

######################## Event bus producer config ########################

# .. setting_name: EVENT_BUS_PRODUCER_CONFIG
# .. setting_default: {}
# .. setting_description: Dictionary of event_types mapped to lists of dictionaries containing topic related configuration.
# Each topic configuration dictionary contains
# * a topic/stream name called `topic` where the event will be pushed to.
# * a flag called `enabled` denoting whether the event will be published to the topic.
# * `event_key_field` which is a period-delimited string path to event data field to use as event key.
# Note: The topic names should not include environment prefix as it will be dynamically added based on
# EVENT_BUS_TOPIC_PREFIX setting.
EVENT_BUS_PRODUCER_CONFIG = {
'org.openedx.learning.certificate.created.v1': [
{'topic': 'learning-certificate-lifecycle', 'event_key_field': 'certificate.course.course_key', 'enabled': True},
],
}
2 changes: 1 addition & 1 deletion requirements/edx/base.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -762,7 +762,7 @@ openedx-django-require==2.1.0
# via -r requirements/edx/kernel.in
openedx-django-wiki==2.0.0
# via -r requirements/edx/kernel.in
openedx-events==8.3.0
openedx-events @ git+https://github.com/open-craft/openedx-events@navin/configurable-handler
# via
# -r requirements/edx/kernel.in
# edx-event-bus-kafka
Expand Down
3 changes: 2 additions & 1 deletion requirements/edx/development.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1294,7 +1294,7 @@ openedx-django-wiki==2.0.0
# via
# -r requirements/edx/doc.txt
# -r requirements/edx/testing.txt
openedx-events==8.3.0
openedx-events @ git+https://github.com/open-craft/openedx-events@navin/configurable-handler
# via
# -r requirements/edx/doc.txt
# -r requirements/edx/testing.txt
Expand Down Expand Up @@ -2138,6 +2138,7 @@ walrus==0.9.3
# edx-event-bus-redis
watchdog==3.0.0
# via
# -r requirements/edx/development.in
# -r requirements/edx/doc.txt
# -r requirements/edx/testing.txt
wcwidth==0.2.6
Expand Down
4 changes: 2 additions & 2 deletions requirements/edx/doc.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -556,7 +556,7 @@ edx-drf-extensions==8.8.0
# edx-rbac
# edx-when
# edxval
edx-enterprise==4.0.6
edx-enterprise==4.0.7
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/base.txt
Expand Down Expand Up @@ -905,7 +905,7 @@ openedx-django-require==2.1.0
# via -r requirements/edx/base.txt
openedx-django-wiki==2.0.0
# via -r requirements/edx/base.txt
openedx-events==8.3.0
openedx-events @ git+https://github.com/open-craft/openedx-events@navin/configurable-handler
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TODO: update this requirement once openedx/openedx-events#249 is merged

# via
# -r requirements/edx/base.txt
# edx-event-bus-kafka
Expand Down
3 changes: 2 additions & 1 deletion requirements/edx/kernel.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,8 @@ oauthlib # OAuth specification support for authentica
olxcleaner
openedx-calc # Library supporting mathematical calculations for Open edX
openedx-django-require
openedx-events>=8.3.0 # Open edX Events from Hooks Extension Framework (OEP-50)
# openedx-events>=8.3.0 # Open edX Events from Hooks Extension Framework (OEP-50)
git+https://github.com/open-craft/openedx-events@navin/configurable-handler
openedx-filters # Open edX Filters from Hooks Extension Framework (OEP-50)
openedx-learning<=0.1
openedx-mongodbproxy
Expand Down
2 changes: 1 addition & 1 deletion requirements/edx/testing.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -974,7 +974,7 @@ openedx-django-require==2.1.0
# via -r requirements/edx/base.txt
openedx-django-wiki==2.0.0
# via -r requirements/edx/base.txt
openedx-events==8.3.0
openedx-events @ git+https://github.com/open-craft/openedx-events@navin/configurable-handler
# via
# -r requirements/edx/base.txt
# edx-event-bus-kafka
Expand Down