% for config in all_group_configurations: -
+

${config['name']}

${_("Loading")}

diff --git a/cms/templates/js/course-manage-tags.underscore b/cms/templates/js/course-manage-tags.underscore new file mode 100644 index 000000000000..86674ed11bb6 --- /dev/null +++ b/cms/templates/js/course-manage-tags.underscore @@ -0,0 +1,8 @@ +
+

+ <%- gettext('Course tags') %> +

+
+ +<%- gettext('Manage tags') %> +
diff --git a/cms/templates/js/course-outline.underscore b/cms/templates/js/course-outline.underscore index 6525c4dc21e2..eec4be4cb5cf 100644 --- a/cms/templates/js/course-outline.underscore +++ b/cms/templates/js/course-outline.underscore @@ -8,7 +8,7 @@ var userPartitionInfo = xblockInfo.get('user_partition_info'); var selectedGroupsLabel = userPartitionInfo['selected_groups_label']; var selectedPartitionIndex = userPartitionInfo['selected_partition_index']; var xblockId = xblockInfo.get('id') -var tagsCount = (xblockInfo.get('tag_counts_by_unit') || {})[xblockId] || 0; +var tagsCount = (xblockInfo.get('tag_counts_by_block') || {})[xblockId] || 0; var statusMessages = []; var messageType; @@ -187,7 +187,7 @@ if (is_proctored_exam) { <% } %> - <% if (xblockInfo.isVertical() && typeof useTaggingTaxonomyListPage !== "undefined" && useTaggingTaxonomyListPage) { %> + <% if (!isTaggingFeatureDisabled) { %>
  • <% } %> @@ -210,12 +210,12 @@ if (is_proctored_exam) { <%- gettext('Configure') %> <% } %> + <% if (!isTaggingFeatureDisabled) { %> +
  • + <%- gettext('Manage Tags') %> +
    • + <% } %> <% if (xblockInfo.isVertical()) { %> - <% if (typeof useTaggingTaxonomyListPage !== "undefined" && useTaggingTaxonomyListPage) { %> -
  • - <%- gettext('Manage Tags') %> -
    • - <% } %>
  • <%- gettext('Copy to Clipboard') %>
    • diff --git a/cms/templates/settings_graders.html b/cms/templates/settings_graders.html index c3b6f8f73a2a..eb0a057b046e 100644 --- a/cms/templates/settings_graders.html +++ b/cms/templates/settings_graders.html @@ -37,6 +37,7 @@ ), "${grading_url | n, js_escaped_string}", ${course_assignment_lists | n, dump_js_escaped_json}, + ${default_grade_designations | n, dump_js_escaped_json}, ); }); diff --git a/cms/templates/studio_xblock_wrapper.html b/cms/templates/studio_xblock_wrapper.html index b2fc928f4274..282629456633 100644 --- a/cms/templates/studio_xblock_wrapper.html +++ b/cms/templates/studio_xblock_wrapper.html @@ -7,14 +7,15 @@ from openedx.core.djangolib.js_utils import ( dump_js_escaped_json, js_escaped_string ) -from cms.djangoapps.contentstore.toggles import use_new_text_editor, use_new_problem_editor, use_new_video_editor, use_video_gallery_flow, use_tagging_taxonomy_list_page +from cms.djangoapps.contentstore.toggles import use_new_text_editor, use_new_problem_editor, use_new_video_editor, use_video_gallery_flow +from openedx.core.djangoapps.content_tagging.toggles import is_tagging_feature_disabled %> <% use_new_editor_text = use_new_text_editor() use_new_editor_video = use_new_video_editor() use_new_editor_problem = use_new_problem_editor() use_new_video_gallery_flow = use_video_gallery_flow() -use_tagging = use_tagging_taxonomy_list_page() +use_tagging = not is_tagging_feature_disabled() xblock_url = xblock_studio_url(xblock) show_inline = xblock.has_children and not xblock_url section_class = "level-nesting" if show_inline else "level-element" diff --git a/common/djangoapps/student/auth.py b/common/djangoapps/student/auth.py index fb2999eca98d..08b4255feeae 100644 --- a/common/djangoapps/student/auth.py +++ b/common/djangoapps/student/auth.py @@ -23,7 +23,7 @@ OrgContentCreatorRole, OrgInstructorRole, OrgLibraryUserRole, - OrgStaffRole + OrgStaffRole, ) # Studio permissions: @@ -106,6 +106,7 @@ def get_user_permissions(user, course_key, org=None): # Staff have all permissions except EDIT_ROLES: if OrgStaffRole(org=org).has_user(user) or (course_key and user_has_role(user, CourseStaffRole(course_key))): return STUDIO_VIEW_USERS | STUDIO_EDIT_CONTENT | STUDIO_VIEW_CONTENT + # Otherwise, for libraries, users can view only: if course_key and isinstance(course_key, LibraryLocator): if OrgLibraryUserRole(org=org).has_user(user) or user_has_role(user, LibraryUserRole(course_key)): diff --git a/common/djangoapps/student/role_helpers.py b/common/djangoapps/student/role_helpers.py index 8a12bfa0ac90..64ed5cc17efb 100644 --- a/common/djangoapps/student/role_helpers.py +++ b/common/djangoapps/student/role_helpers.py @@ -75,4 +75,4 @@ def get_course_roles(user: User) -> list[CourseAccessRole]: """ # pylint: disable=protected-access role_cache = get_role_cache(user) - return list(role_cache._roles) + return list(role_cache.all_roles_set) diff --git a/common/djangoapps/student/roles.py b/common/djangoapps/student/roles.py index 7bbd0cf92454..971433c9c523 100644 --- a/common/djangoapps/student/roles.py +++ b/common/djangoapps/student/roles.py @@ -4,9 +4,9 @@ """ +from collections import defaultdict import logging from abc import ABCMeta, abstractmethod -from collections import defaultdict from contextlib import contextmanager from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user @@ -23,6 +23,9 @@ # A mapping of roles to the roles that they inherit permissions from. ACCESS_ROLES_INHERITANCE = {} +# The key used to store roles for a user in the cache that do not belong to a course or do not have a course id. +ROLE_CACHE_UNGROUPED_ROLES__KEY = 'ungrouped' + def register_access_role(cls): """ @@ -60,21 +63,52 @@ def strict_role_checking(): ACCESS_ROLES_INHERITANCE.update(OLD_ACCESS_ROLES_INHERITANCE) +def get_role_cache_key_for_course(course_key=None): + """ + Get the cache key for the course key. + """ + return str(course_key) if course_key else ROLE_CACHE_UNGROUPED_ROLES__KEY + + class BulkRoleCache: # lint-amnesty, pylint: disable=missing-class-docstring + """ + This class provides a caching mechanism for roles grouped by users and courses, + using a nested dictionary structure to optimize lookup performance. The cache structure is designed as follows: + + { + user_id_1: { + course_id_1: {role1, role2, role3}, # Set of roles associated with course_id_1 + course_id_2: {role4, role5, role6}, # Set of roles associated with course_id_2 + [ROLE_CACHE_UNGROUPED_ROLES_KEY]: {role7, role8} # Set of roles not tied to any specific course or library + }, + user_id_2: { ... } # Similar structure for another user + } + + - Each top-level dictionary entry keys by `user_id` to access role data for a specific user. + - Nested within each user's dictionary, entries are keyed by `course_id` grouping roles by course. + - The special key `ROLE_CACHE_UNGROUPED_ROLES_KEY` (a constant defined above) + stores roles that are not associated with any specific course or library. + """ + CACHE_NAMESPACE = "student.roles.BulkRoleCache" CACHE_KEY = 'roles_by_user' @classmethod def prefetch(cls, users): # lint-amnesty, pylint: disable=missing-function-docstring - roles_by_user = defaultdict(set) + roles_by_user = defaultdict(lambda: defaultdict(set)) get_cache(cls.CACHE_NAMESPACE)[cls.CACHE_KEY] = roles_by_user for role in CourseAccessRole.objects.filter(user__in=users).select_related('user'): - roles_by_user[role.user.id].add(role) + user_id = role.user.id + course_id = get_role_cache_key_for_course(role.course_id) + + # Add role to the set in roles_by_user[user_id][course_id] + user_roles_set_for_course = roles_by_user[user_id][course_id] + user_roles_set_for_course.add(role) users_without_roles = [u for u in users if u.id not in roles_by_user] for user in users_without_roles: - roles_by_user[user.id] = set() + roles_by_user[user.id] = {} @classmethod def get_user_roles(cls, user): @@ -83,15 +117,32 @@ def get_user_roles(cls, user): class RoleCache: """ - A cache of the CourseAccessRoles held by a particular user + A cache of the CourseAccessRoles held by a particular user. + Internal data structures should be accessed by getter and setter methods; + don't use `_roles_by_course_id` or `_roles` directly. + _roles_by_course_id: This is the data structure as saved in the RequestCache. + It contains all roles for a user as a dict that's keyed by course_id. + The key ROLE_CACHE_UNGROUPED_ROLES__KEY is used for all roles + that are not associated with a course. + _roles: This is a set of all roles for a user, ungrouped. It's used for some types of + lookups and collected from _roles_by_course_id on initialization + so that it doesn't need to be recalculated. + """ def __init__(self, user): try: - self._roles = BulkRoleCache.get_user_roles(user) + self._roles_by_course_id = BulkRoleCache.get_user_roles(user) except KeyError: - self._roles = set( - CourseAccessRole.objects.filter(user=user).all() - ) + self._roles_by_course_id = {} + roles = CourseAccessRole.objects.filter(user=user).all() + for role in roles: + course_id = get_role_cache_key_for_course(role.course_id) + if not self._roles_by_course_id.get(course_id): + self._roles_by_course_id[course_id] = set() + self._roles_by_course_id[course_id].add(role) + self._roles = set() + for roles_for_course in self._roles_by_course_id.values(): + self._roles.update(roles_for_course) @staticmethod def get_roles(role): @@ -100,16 +151,24 @@ def get_roles(role): """ return ACCESS_ROLES_INHERITANCE.get(role, set()) | {role} + @property + def all_roles_set(self): + return self._roles + + @property + def roles_by_course_id(self): + return self._roles_by_course_id + def has_role(self, role, course_id, org): """ Return whether this RoleCache contains a role with the specified role or a role that inherits from the specified role, course_id and org. """ + course_id_string = get_role_cache_key_for_course(course_id) + course_roles = self._roles_by_course_id.get(course_id_string, []) return any( - access_role.role in self.get_roles(role) and - access_role.course_id == course_id and - access_role.org == org - for access_role in self._roles + access_role.role in self.get_roles(role) and access_role.org == org + for access_role in course_roles ) diff --git a/common/djangoapps/student/tests/test_roles.py b/common/djangoapps/student/tests/test_roles.py index 9037eb902f61..da1aad19a803 100644 --- a/common/djangoapps/student/tests/test_roles.py +++ b/common/djangoapps/student/tests/test_roles.py @@ -6,6 +6,7 @@ import ddt from django.test import TestCase from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.locator import LibraryLocator from common.djangoapps.student.roles import ( CourseAccessRole, @@ -22,7 +23,9 @@ OrgContentCreatorRole, OrgInstructorRole, OrgStaffRole, - RoleCache + RoleCache, + get_role_cache_key_for_course, + ROLE_CACHE_UNGROUPED_ROLES__KEY ) from common.djangoapps.student.role_helpers import get_course_roles, has_staff_roles from common.djangoapps.student.tests.factories import AnonymousUserFactory, InstructorFactory, StaffFactory, UserFactory @@ -35,7 +38,7 @@ class RolesTestCase(TestCase): def setUp(self): super().setUp() - self.course_key = CourseKey.from_string('edX/toy/2012_Fall') + self.course_key = CourseKey.from_string('course-v1:course-v1:edX+toy+2012_Fall') self.course_loc = self.course_key.make_usage_key('course', '2012_Fall') self.anonymous_user = AnonymousUserFactory() self.student = UserFactory() @@ -189,8 +192,9 @@ def test_get_orgs_for_user(self): @ddt.ddt class RoleCacheTestCase(TestCase): # lint-amnesty, pylint: disable=missing-class-docstring - IN_KEY = CourseKey.from_string('edX/toy/2012_Fall') - NOT_IN_KEY = CourseKey.from_string('edX/toy/2013_Fall') + IN_KEY_STRING = 'course-v1:edX+toy+2012_Fall' + IN_KEY = CourseKey.from_string(IN_KEY_STRING) + NOT_IN_KEY = CourseKey.from_string('course-v1:edX+toy+2013_Fall') ROLES = ( (CourseStaffRole(IN_KEY), ('staff', IN_KEY, 'edX')), @@ -233,3 +237,75 @@ def test_only_in_role(self, role, target): def test_empty_cache(self, role, target): # lint-amnesty, pylint: disable=unused-argument cache = RoleCache(self.user) assert not cache.has_role(*target) + + def test_get_role_cache_key_for_course_for_course_object_gets_string(self): + """ + Given a valid course key object, get_role_cache_key_for_course + should return the string representation of the key. + """ + course_string = 'course-v1:edX+toy+2012_Fall' + key = CourseKey.from_string(course_string) + key = get_role_cache_key_for_course(key) + assert key == course_string + + def test_get_role_cache_key_for_course_for_undefined_object_returns_default(self): + """ + Given a value None, get_role_cache_key_for_course + should return the default key for ungrouped courses. + """ + key = get_role_cache_key_for_course(None) + assert key == ROLE_CACHE_UNGROUPED_ROLES__KEY + + def test_role_cache_get_roles_set(self): + """ + Test that the RoleCache.all_roles_set getter method returns a flat set of all roles for a user + and that the ._roles attribute is the same as the set to avoid legacy behavior being broken. + """ + lib0 = LibraryLocator.from_string('library-v1:edX+quizzes') + course0 = CourseKey.from_string('course-v1:edX+toy+2012_Summer') + course1 = CourseKey.from_string('course-v1:edX+toy2+2013_Fall') + role_library_v1 = LibraryUserRole(lib0) + role_course_0 = CourseInstructorRole(course0) + role_course_1 = CourseInstructorRole(course1) + + role_library_v1.add_users(self.user) + role_course_0.add_users(self.user) + role_course_1.add_users(self.user) + + cache = RoleCache(self.user) + assert cache.has_role('library_user', lib0, 'edX') + assert cache.has_role('instructor', course0, 'edX') + assert cache.has_role('instructor', course1, 'edX') + + assert len(cache.all_roles_set) == 3 + roles_set = cache.all_roles_set + for role in roles_set: + assert role.course_id.course in ('quizzes', 'toy2', 'toy') + + assert roles_set == cache._roles # pylint: disable=protected-access + + def test_role_cache_roles_by_course_id(self): + """ + Test that the RoleCache.roles_by_course_id getter method returns a dictionary of roles for a user + that are grouped by course_id or if ungrouped by the ROLE_CACHE_UNGROUPED_ROLES__KEY. + """ + lib0 = LibraryLocator.from_string('library-v1:edX+quizzes') + course0 = CourseKey.from_string('course-v1:edX+toy+2012_Summer') + course1 = CourseKey.from_string('course-v1:edX+toy2+2013_Fall') + role_library_v1 = LibraryUserRole(lib0) + role_course_0 = CourseInstructorRole(course0) + role_course_1 = CourseInstructorRole(course1) + role_org_staff = OrgStaffRole('edX') + + role_library_v1.add_users(self.user) + role_course_0.add_users(self.user) + role_course_1.add_users(self.user) + role_org_staff.add_users(self.user) + + cache = RoleCache(self.user) + roles_dict = cache.roles_by_course_id + assert len(roles_dict) == 4 + assert roles_dict.get(ROLE_CACHE_UNGROUPED_ROLES__KEY).pop().role == 'staff' + assert roles_dict.get('library-v1:edX+quizzes').pop().course_id.course == 'quizzes' + assert roles_dict.get('course-v1:edX+toy+2012_Summer').pop().course_id.course == 'toy' + assert roles_dict.get('course-v1:edX+toy2+2013_Fall').pop().course_id.course == 'toy2' diff --git a/common/djangoapps/student/views/management.py b/common/djangoapps/student/views/management.py index eac799c5e1f0..8701919d3725 100644 --- a/common/djangoapps/student/views/management.py +++ b/common/djangoapps/student/views/management.py @@ -434,6 +434,7 @@ def change_enrollment(request, check_access=True): except UnenrollmentNotAllowed as exc: return HttpResponseBadRequest(str(exc)) + log.info("User %s unenrolled from %s; sending REFUND_ORDER", user.username, course_id) REFUND_ORDER.send(sender=None, course_enrollment=enrollment) return HttpResponse() else: diff --git a/common/djangoapps/track/contexts.py b/common/djangoapps/track/contexts.py index 217a514e4c0f..0ac8292e258d 100644 --- a/common/djangoapps/track/contexts.py +++ b/common/djangoapps/track/contexts.py @@ -48,7 +48,7 @@ def course_context_from_course_id(course_id): """ Creates a course context from a `course_id`. - For newer parts of the system (i.e. Blockstore-based libraries/courses/etc.) + For newer parts of the system (i.e. Learning-Core-based libraries/courses/etc.) use context_dict_for_learning_context instead of this method. Example Returned Context:: diff --git a/common/static/data/geoip/GeoLite2-Country.mmdb b/common/static/data/geoip/GeoLite2-Country.mmdb index ffe588ba5afc..27fe30a6ce20 100644 Binary files a/common/static/data/geoip/GeoLite2-Country.mmdb and b/common/static/data/geoip/GeoLite2-Country.mmdb differ diff --git a/docs/concepts/extension_points.rst b/docs/concepts/extension_points.rst index c5ba0d42c050..d4e802baec0e 100644 --- a/docs/concepts/extension_points.rst +++ b/docs/concepts/extension_points.rst @@ -124,7 +124,7 @@ Here are the different integration points that python plugins can use: - By default, the registration page for each instance of Open edX has fields that ask for information such as a user’s name, country, and highest level of education completed. You can add custom fields to the registration page for your own Open edX instance. These fields can be different types, including text entry fields and drop-down lists. See `Adding Custom Fields to the Registration Page`_. * - Learning Context (``openedx.learning_context``) - Trial, Limited - - A "Learning Context" is a course, a library, a program, a blog, an external site, or some other collection of content where learning happens. If you are trying to build a totally new learning experience that's not a type of course, you may need to implement a new learning context. Learning contexts are a new abstraction and are only supported in the nascent Blockstore-based XBlock runtime. Since existing courses use modulestore instead of Blockstore, they are not yet implemented as learning contexts. However, Blockstore-based content libraries are. See |learning_context.py|_ to learn more. + - A "Learning Context" is a course, a library, a program, a blog, an external site, or some other collection of content where learning happens. If you are trying to build a totally new learning experience that's not a type of course, you may need to implement a new learning context. Learning contexts are a new abstraction and are only supported in the nascent Learning-Core-based XBlock runtime. Since existing courses use modulestore instead of Learning Core, they are not yet implemented as learning contexts. However, Learning-Core-based content libraries are. See |learning_context.py|_ to learn more. * - User partition scheme (``openedx.user_partition_scheme`` and ``openedx.dynamic_partition_generator``) - Unknown, Stable - A user partition scheme is a named way for dividing users in a course into groups, usually to show different content to different users or to run experiments. Partitions may be added to a course manually, or automatically added by a "dynamic partition generator." The core platform includes partition scheme plugins like ``random``, ``cohort``, and ``enrollment_track``. See the |UserPartition docstring|_ to learn more. diff --git a/docs/conf.py b/docs/conf.py index 027e56808b6a..f37fc32f6160 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,6 +58,7 @@ 'sphinx.ext.autodoc', 'sphinx.ext.coverage', 'sphinx.ext.doctest', + 'sphinx.ext.graphviz', 'sphinx.ext.ifconfig', 'sphinx.ext.intersphinx', 'sphinx.ext.mathjax', diff --git a/docs/decisions/0017-reimplement-asset-processing.rst b/docs/decisions/0017-reimplement-asset-processing.rst index 1048edf4c075..e689c34eb9cf 100644 --- a/docs/decisions/0017-reimplement-asset-processing.rst +++ b/docs/decisions/0017-reimplement-asset-processing.rst @@ -11,16 +11,14 @@ Overview Status ****** -**Provisional** +**Accepted** -This was `originally authored `_ in March 2023. We `modified it in July 2023 `_ based on learnings from the implementation process. +This was `originally authored `_ in March 2023. We `modified it in July 2023 `_ based on learnings from the implementation process, and then `modified and it again in May 2024 `_ to make the migration easier for operators to understand. -The status will be moved to *Accepted* upon completion of reimplementation. Related work: +Related deprecation tickets: * `[DEPR]: Asset processing in Paver `_ -* `Process edx-platform assets without Paver `_ -* `Process edx-platform assets without Python `_ - +* `[DEPR]: Paver `_ Context ******* @@ -92,7 +90,6 @@ Three particular issues have surfaced in Developer Experience Working Group disc All of these potential solutions would involve refactoring or entirely replacing parts of the current asset processing system. - Decision ******** @@ -114,6 +111,9 @@ Reimplementation Specification Commands and stages ------------------- +**May 2024 update:** See the `static assets reference <../references/static-assets.rst>`_ for +the latest commands. + The three top-level edx-platform asset processing actions are *build*, *collect*, and *watch*. The build action can be further broken down into five stages. Here is how those actions and stages will be reimplemented: @@ -226,6 +226,9 @@ The three top-level edx-platform asset processing actions are *build*, *collect* Build Configuration ------------------- +**May 2024 update:** See the `static assets reference <../references/static-assets.rst>`_ for +the latest configuration settings. + To facilitate a generally Python-free build reimplementation, we will require that certain Django settings now be specified as environment variables, which can be passed to the build like so:: MY_ENV_VAR="my value" npm run build # Set for the whole build. @@ -266,7 +269,7 @@ Some of these options will remain as Django settings because they are used in ed * - ``COMPREHENSIVE_THEME_DIRS`` - Directories that will be searched when compiling themes. - ``COMPREHENSIVE_THEME_DIRS`` - - ``EDX_PLATFORM_THEME_DIRS`` + - ``COMPREHENSIVE_THEME_DIRS`` Migration ========= @@ -285,37 +288,16 @@ As a consequence of this ADR, Tutor will either need to: * reimplement the script as a thin wrapper around the new asset processing commands, or * deprecate and remove the script. -Either way, the migration path is straightforward: - -.. list-table:: - :header-rows: 1 - - * - Existing Tutor-provided command - - New upstream command - * - ``openedx-assets build`` - - ``npm run build`` - * - ``openedx-assets npm`` - - ``scripts/copy-node-modules.sh # (automatically invoked by 'npm install'!)`` - * - ``openedx-assets xmodule`` - - (no longer needed) - * - ``openedx-assets common`` - - ``npm run compile-sass -- --skip-themes`` - * - ``openedx-assets themes`` - - ``npm run compile-sass -- --skip-default`` - * - ``openedx-assets webpack`` - - ``npm run webpack`` - * - ``openedx-assets collect`` - - ``./manage.py [lms|cms] collectstatic --noinput`` - * - ``openedx-assets watch-themes`` - - ``npm run watch`` - -The options accepted by ``openedx-assets`` will all be valid inputs to ``scripts/build-assets.sh``. +**May 2024 update:** The ``openedx-assets`` script will be removed from Tutor, +with migration instructions documented in +`Tutor's changelog `_. non-Tutor migration guide ------------------------- -Operators using distributions other than Tutor should refer to the upstream edx-platform changes described above in **Reimplementation Specification**, and adapt them accordingly to their distribution. - +A migration guide for site operators who are directly referencing Paver will be +included in the +`Paver deprecation ticket `_. See also ******** diff --git a/docs/references/static-assets.rst b/docs/references/static-assets.rst new file mode 100644 index 000000000000..59fae1d65a8f --- /dev/null +++ b/docs/references/static-assets.rst @@ -0,0 +1,160 @@ +Preparing static assets for edx-platform +######################################## + +To run a production or development edx-platform site, you will need to `build +assets`_ assets using ``npm run ...`` commands. Furthermore, for a production +site, you will also need to `collect assets`_. + +*Please note that developing new frontend pages for edx-platform is highly +discouraged. New frontend pages should be built as micro-frontends (MFEs), +which communicate with edx-platform over AJAX, but are built and deployed +independently. Eventually, we expect that MFEs will replace all edx-platform +frontend pages, except perhaps XBlock views.* + +Configuraiton +************* + +To customize the static assets build, set some or all of these variable in your +shell environment before building or collecting static assets. As noted below, +some of these values will automatically become available as Django settings in +LMS or CMS (unless you separately override them in a private Django settings +file or ``LMS_CFG``/``CMS_CFG`` yaml file). + +.. list-table:: + :header-rows: 1 + + * - Environment Variable + - Default + - Description + - LMS Django Setting + - CMS Django Setting + + * - ``COMPREHENSIVE_THEME_DIRS`` + - (empty) + - Directories that will be searched when compiling themes. + Separate multiple paths with colons (``:``). + - ``COMPREHENSIVE_THEME_DIRS`` + - ``COMPREHENSIVE_THEME_DIRS`` + + * - ``WEBPACK_CONFIG_PATH`` + - ``webpack.prod.config.js`` + - Path to Webpack config file + - N/A + - N/A + + * - ``STATIC_ROOT_LMS`` + - ``test_root/staticfiles`` + - Path to which LMS's static assets will be collected + - ``STATIC_ROOT`` + - N/A + + * - ``STATIC_ROOT_CMS`` + - ``$STATIC_ROOT_LMS/studio``. + - Path to which CMS's static assets will be collected + - N/A + - ``STATIC_ROOT`` + + * - ``JS_ENV_EXTRA_CONFIG`` + - ``{}`` + - Global configuration object available to edx-platform JS modules. Specified as a JSON string. + Known keys: + + * ``TINYMCE_ADDITIONAL_PLUGINS`` + * ``TINYMCE_CONFIG_OVERRIDES`` + + - N/A + - N/A + +Build assets +************ + +Building frontend assets requires an active Node and Python environment with +dependencies installed:: + + npm clean-install + pip install -r requirements/edx/assets.txt + + +Once your environment variables are set and build dependencies are installed, +the one-sized-fits-all command to build assets is ``npm run build``. If +your needs are more advanced, though, you can use some combination of the +commands below: + +.. list-table:: + :header-rows: 1 + + * - Command + - Meaning + - Options + * - ``npm run build`` + - Combines ``npm run webpack`` and ``npm run compile-sass`` + - None + * - ``npm run build-dev`` + - Combines ``npm run webpack-dev`` and ``npm run compile-sass-dev`` + - None + * - ``npm run webpack`` + - Build JS bundles with Webpack + - Options are passed through to the `webpack CLI`_ + * - ``npm run webpack-dev`` + - Build JS bundles with Webpack for a development environment + - Options are passed through to the `webpack CLI`_ + * - ``npm run compile-sass`` + - Compile default and/or themed Sass + - Use ``--help`` to see available options + * - ``npm run compile-sass-dev`` + - Compile default and/or themed Sass, uncompressed with source comments + - Use ``--help`` to see available options + * - ``npm run watch`` + - Dev-only. Combine ``npm run watch-webpack`` and ``npm run watch-sass`` + - None. + * - ``npm run watch-webpack`` + - Dev-only. Wait for JS changes and re-run Webpack + - Options are passed through to the `webpack CLI`_ + * - ``npm run watch-sass`` + - Dev-only. Wait for Sass changes and re-compile + - None. + +When supplying options to these commands, separate the command from the options +with a double-hyphen (``--``), like this:: + + npm run compile-sass -- --themes-dir /my/custom/themes/dir + +Omitting the double-hyphen will pass the option to ``npm run`` itself, which +probably isn't what you want to do. + +If you would like to understand these more deeply, they are defined in +`package.json`_. Please note: the ``npm run`` command interfaces are stable and +supported, but their underlying implementations may change without notice. + +.. _webpack CLI: https://webpack.js.org/api/cli/ +.. _package.json: ../package.json + +Collect assets +************** + +Once assets are built, they can be *collected* into another directory for +efficient serving. This is only necessary on production sites; developers can +skip this section. + +First, ensure you have a Python enironment with all edx-platform dependencies +installed:: + + pip install -r requirements/edx/base.txt + +Next, download localized versions of edx-platform assets. Under the hood, this +command uses the `Open edX Atlas`_ tool, which manages aggregated translations +from edx-platform and its various plugins:: + + make pull_translations + +Finally, invoke `Django's collectstatic command`_, once for the Learning +Management System, and once for the Content Management Studio:: + + ./manage.py lms collectstatic --noinput + ./manage.py cms collectstatic --noinput + +The ``--noinput`` option lets you avoid having to type "yes" when overwriting +existing collected assets. + +.. _Open edX Atlas: https://github.com/openedx/openedx-atlas +.. _Django's collectstatic command: https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#collectstatic diff --git a/lms/djangoapps/certificates/api.py b/lms/djangoapps/certificates/api.py index b72113b94788..ec790a4315c1 100644 --- a/lms/djangoapps/certificates/api.py +++ b/lms/djangoapps/certificates/api.py @@ -953,3 +953,21 @@ def invalidate_certificate(user_id, course_key_or_id, source): return False return True + + +def clear_pii_from_certificate_records_for_user(user): + """ + Utility function to remove PII from certificate records when a learner's account is being retired. Used by the + `AccountRetirementView` in the `user_api` Django app (invoked by the /api/user/v1/accounts/retire endpoint). + + The update is performed using a bulk SQL update via the Django ORM. This will not trigger the GeneratedCertificate + model's custom `save()` function, nor fire any Django signals (which is desired at the time of writing). There is + nothing to update in our external systems by this update. + + Args: + user (User): The User instance of the learner actively being retired. + + Returns: + None + """ + GeneratedCertificate.objects.filter(user=user).update(name="") diff --git a/lms/djangoapps/certificates/management/commands/purge_pii_from_generatedcertificates.py b/lms/djangoapps/certificates/management/commands/purge_pii_from_generatedcertificates.py new file mode 100644 index 000000000000..c478fd4fe0c5 --- /dev/null +++ b/lms/djangoapps/certificates/management/commands/purge_pii_from_generatedcertificates.py @@ -0,0 +1,66 @@ +""" +A management command, designed to be run once by Open edX Operators, to obfuscate learner PII from the +`Certificates_GeneratedCertificate` table that should have been purged during learner retirement. + +A fix has been included in the retirement pipeline to properly purge this data during learner retirement. This can be +used to purge PII from accounts that have already been retired. +""" + +import logging + +from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand + +from lms.djangoapps.certificates.models import GeneratedCertificate +from openedx.core.djangoapps.user_api.api import get_retired_user_ids + +User = get_user_model() +log = logging.getLogger(__name__) + + +class Command(BaseCommand): + """ + This management command performs a bulk update on `GeneratedCertificate` instances. This means that it will not + invoke the custom save() function defined as part of the `GeneratedCertificate` model, and thus will not emit any + Django signals throughout the system after the update occurs. This is desired behavior. We are using this + management command to purge remnant PII, retired elsewhere in the system, that should have already been removed + from the Certificates tables. We don't need updates to propogate to external systems (like the Credentials IDA). + + This management command functions by requesting a list of learners' user_ids whom have completed their journey + through the retirement pipeline. The `get_retired_user_ids` utility function is responsible for filtering out any + learners in the PENDING state, as they could still submit a request to cancel their account deletion request (and + we don't want to remove any data that may still be good). + + Example usage: + + # Dry Run (preview changes): + $ ./manage.py lms purge_pii_from_generatedcertificates --dry-run + + # Purge data: + $ ./manage.py lms purge_pii_from_generatedcertificates + """ + + help = """ + Purges learners' full names from the `Certificates_GeneratedCertificate` table if their account has been + successfully retired. + """ + + def add_arguments(self, parser): + parser.add_argument( + "--dry-run", + action="store_true", + help="Shows a preview of what users would be affected by running this management command.", + ) + + def handle(self, *args, **options): + retired_user_ids = get_retired_user_ids() + if not options["dry_run"]: + log.warning( + f"Purging `name` from the certificate records of the following users: {retired_user_ids}" + ) + GeneratedCertificate.objects.filter(user_id__in=retired_user_ids).update(name="") + else: + log.info( + "DRY RUN: running this management command would purge `name` data from the following users: " + f"{retired_user_ids}" + ) diff --git a/lms/djangoapps/certificates/management/commands/tests/test_purge_pii_from_generatedcertificates.py b/lms/djangoapps/certificates/management/commands/tests/test_purge_pii_from_generatedcertificates.py new file mode 100644 index 000000000000..50855ccaa804 --- /dev/null +++ b/lms/djangoapps/certificates/management/commands/tests/test_purge_pii_from_generatedcertificates.py @@ -0,0 +1,114 @@ +""" +Tests for the `purge_pii_from_generatedcertificates` management command. +""" + + +from django.core.management import call_command +from testfixtures import LogCapture + +from common.djangoapps.student.tests.factories import UserFactory +from lms.djangoapps.certificates.data import CertificateStatuses +from lms.djangoapps.certificates.models import GeneratedCertificate +from lms.djangoapps.certificates.tests.factories import GeneratedCertificateFactory +from openedx.core.djangoapps.user_api.models import RetirementState +from openedx.core.djangoapps.user_api.tests.factories import ( + RetirementStateFactory, + UserRetirementRequestFactory, + UserRetirementStatusFactory, +) +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory + + +class PurgePiiFromCertificatesTests(ModuleStoreTestCase): + """ + Tests for the `purge_pii_from_generatedcertificates` management command. + """ + @classmethod + def setUpClass(cls): + """ + The retirement pipeline is not fully enabled by default. In order to properly test the management command, we + must ensure that at least one of the required RetirementState states (`COMPLETE`) exists. + """ + super().setUpClass() + cls.complete = RetirementStateFactory(state_name="COMPLETE") + + @classmethod + def tearDownClass(cls): + # Remove any retirement state objects that we created during this test suite run. We don't want to poison other + # test suites. + RetirementState.objects.all().delete() + super().tearDownClass() + + def setUp(self): + super().setUp() + self.course_run = CourseFactory() + # create an "active" learner that is not associated with any retirement requests, used to verify that the + # management command doesn't purge any info for active users. + self.user_active = UserFactory() + self.user_active_name = "Teysa Karlov" + GeneratedCertificateFactory( + status=CertificateStatuses.downloadable, + course_id=self.course_run.id, + user=self.user_active, + name=self.user_active_name, + grade=1.00, + ) + # create a second learner that is associated with a retirement request, used to verify that the management + # command purges info successfully from a GeneratedCertificate instance associated with a retired learner + self.user_retired = UserFactory() + self.user_retired_name = "Nicol Bolas" + GeneratedCertificateFactory( + status=CertificateStatuses.downloadable, + course_id=self.course_run.id, + user=self.user_retired, + name=self.user_retired_name, + grade=0.99, + ) + UserRetirementStatusFactory( + user=self.user_retired, + current_state=self.complete, + last_state=self.complete, + ) + UserRetirementRequestFactory(user=self.user_retired) + + def test_management_command(self): + """ + Verify the management command purges expected data from a GeneratedCertificate instance if a learner has + successfully had their account retired. + """ + cert_for_active_user = GeneratedCertificate.objects.get(user_id=self.user_active) + assert cert_for_active_user.name == self.user_active_name + cert_for_retired_user = GeneratedCertificate.objects.get(user_id=self.user_retired) + assert cert_for_retired_user.name == self.user_retired_name + + call_command("purge_pii_from_generatedcertificates") + + cert_for_active_user = GeneratedCertificate.objects.get(user_id=self.user_active) + assert cert_for_active_user.name == self.user_active_name + cert_for_retired_user = GeneratedCertificate.objects.get(user_id=self.user_retired) + assert cert_for_retired_user.name == "" + + def test_management_command_dry_run(self): + """ + Verify that the management command does not purge any data when invoked with the `--dry-run` flag + """ + expected_log_msg = ( + "DRY RUN: running this management command would purge `name` data from the following users: " + f"[{self.user_retired.id}]" + ) + + cert_for_active_user = GeneratedCertificate.objects.get(user_id=self.user_active) + assert cert_for_active_user.name == self.user_active_name + cert_for_retired_user = GeneratedCertificate.objects.get(user_id=self.user_retired) + assert cert_for_retired_user.name == self.user_retired_name + + with LogCapture() as logger: + call_command("purge_pii_from_generatedcertificates", "--dry-run") + + cert_for_active_user = GeneratedCertificate.objects.get(user_id=self.user_active) + assert cert_for_active_user.name == self.user_active_name + cert_for_retired_user = GeneratedCertificate.objects.get(user_id=self.user_retired) + assert cert_for_retired_user.name == self.user_retired_name + + assert logger.records[0].msg == expected_log_msg diff --git a/lms/djangoapps/certificates/tests/test_api.py b/lms/djangoapps/certificates/tests/test_api.py index b718fcd2bac9..c766d4250bd9 100644 --- a/lms/djangoapps/certificates/tests/test_api.py +++ b/lms/djangoapps/certificates/tests/test_api.py @@ -40,6 +40,7 @@ can_show_certificate_message, certificate_status_for_student, certificate_downloadable_status, + clear_pii_from_certificate_records_for_user, create_certificate_invalidation_entry, create_or_update_certificate_allowlist_entry, display_date_for_certificate, @@ -76,6 +77,9 @@ from openedx.core.djangoapps.site_configuration.tests.test_util import with_site_configuration CAN_GENERATE_METHOD = 'lms.djangoapps.certificates.generation_handler._can_generate_regular_certificate' +BETA_TESTER_METHOD = 'lms.djangoapps.certificates.api.access.is_beta_tester' +CERTS_VIEWABLE_METHOD = 'lms.djangoapps.certificates.api.certificates_viewable_for_course' +PASSED_OR_ALLOWLISTED_METHOD = 'lms.djangoapps.certificates.api._has_passed_or_is_allowlisted' FEATURES_WITH_CERTS_ENABLED = settings.FEATURES.copy() FEATURES_WITH_CERTS_ENABLED['CERTIFICATES_HTML_VIEW'] = True @@ -1120,10 +1124,6 @@ def test_get_certificate_invalidation_entry_dne(self): for index, message in enumerate(expected_messages): assert message in log.records[index].getMessage() -BETA_TESTER_METHOD = 'lms.djangoapps.certificates.api.access.is_beta_tester' -CERTS_VIEWABLE_METHOD = 'lms.djangoapps.certificates.api.certificates_viewable_for_course' -PASSED_OR_ALLOWLISTED_METHOD = 'lms.djangoapps.certificates.api._has_passed_or_is_allowlisted' - class MockGeneratedCertificate: """ @@ -1268,3 +1268,42 @@ def test_beta_tester(self): with patch(BETA_TESTER_METHOD, return_value=True): assert not can_show_certificate_message(self.course, self.user, grade, certs_enabled) + + +class CertificatesLearnerRetirementFunctionality(ModuleStoreTestCase): + """ + API tests for utility functions used as part of the learner retirement pipeline to remove PII from certificate + records. + """ + def setUp(self): + super().setUp() + self.user = UserFactory() + self.user_full_name = "Maeby Funke" + self.course1 = CourseOverviewFactory() + self.course2 = CourseOverviewFactory() + GeneratedCertificateFactory( + course_id=self.course1.id, + name=self.user_full_name, + user=self.user, + ) + GeneratedCertificateFactory( + course_id=self.course2.id, + name=self.user_full_name, + user=self.user, + ) + + def test_clear_pii_from_certificate_records(self): + """ + Unit test for the `clear_pii_from_certificate_records` utility function, used to wipe PII from certificate + records when a learner's account is being retired. + """ + cert_course1 = GeneratedCertificate.objects.get(user=self.user, course_id=self.course1.id) + cert_course2 = GeneratedCertificate.objects.get(user=self.user, course_id=self.course2.id) + assert cert_course1.name == self.user_full_name + assert cert_course2.name == self.user_full_name + + clear_pii_from_certificate_records_for_user(self.user) + cert_course1 = GeneratedCertificate.objects.get(user=self.user, course_id=self.course1.id) + cert_course2 = GeneratedCertificate.objects.get(user=self.user, course_id=self.course2.id) + assert cert_course1.name == "" + assert cert_course2.name == "" diff --git a/lms/djangoapps/commerce/signals.py b/lms/djangoapps/commerce/signals.py index eaf85bdfba37..51405a020dab 100644 --- a/lms/djangoapps/commerce/signals.py +++ b/lms/djangoapps/commerce/signals.py @@ -2,7 +2,6 @@ Signal handling functions for use with external commerce service. """ - import logging from crum import get_current_request @@ -21,13 +20,15 @@ @receiver(REFUND_ORDER) def handle_refund_order(sender, course_enrollment=None, **kwargs): """ - Signal receiver for unenrollments, used to automatically initiate refunds + Signal receiver for un-enrollments, used to automatically initiate refunds when applicable. """ if not is_commerce_service_configured(): + log.info("Commerce service not configured, skipping refund") return if course_enrollment and course_enrollment.refundable(): + log.info("Handling refund for course enrollment %s", course_enrollment.course_id) try: request_user = get_request_user() or course_enrollment.user if isinstance(request_user, AnonymousUser): @@ -36,7 +37,13 @@ def handle_refund_order(sender, course_enrollment=None, **kwargs): # construct a client to call Otto back anyway, because # the client does not work anonymously, and furthermore, # there's certainly no need to inform Otto about this request. + log.info( + "Anonymous user attempting to initiate refund for course [%s]", + course_enrollment.course_id, + ) return + log.info("Initiating refund_seat for user [%s] for course enrollment %s", + course_enrollment.user.id, course_enrollment.course_id) refund_seat(course_enrollment, change_mode=True) except Exception: # pylint: disable=broad-except # don't assume the signal was fired with `send_robust`. @@ -47,6 +54,13 @@ def handle_refund_order(sender, course_enrollment=None, **kwargs): course_enrollment.user.id, course_enrollment.course_id, ) + elif course_enrollment: + log.info( + "Not refunding seat for course enrollment %s, as its not refundable", + course_enrollment.course_id + ) + else: + log.info("Not refunding seat for course due to missing course enrollment") def get_request_user(): diff --git a/lms/djangoapps/course_api/blocks/tests/test_views.py b/lms/djangoapps/course_api/blocks/tests/test_views.py index e2426708d6bc..c1f673652096 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_views.py +++ b/lms/djangoapps/course_api/blocks/tests/test_views.py @@ -1,27 +1,29 @@ """ Tests for Blocks Views """ -import ddt - from datetime import datetime from unittest import mock from unittest.mock import Mock from urllib.parse import urlencode, urlunparse +import ddt from completion.test_utils import CompletionWaffleTestMixin, submit_completions_for_testing from django.conf import settings from django.urls import reverse from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator +from rest_framework.utils.serializer_helpers import ReturnList from common.djangoapps.student.models import CourseEnrollment from common.djangoapps.student.roles import CourseDataResearcherRole from common.djangoapps.student.tests.factories import AdminFactory, CourseEnrollmentFactory, UserFactory -from openedx.core.djangoapps.discussions.models import ( - DiscussionsConfiguration, - Provider, +from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration, Provider +from xmodule.modulestore.tests.django_utils import ( # lint-amnesty, pylint: disable=wrong-import-order + SharedModuleStoreTestCase +) +from xmodule.modulestore.tests.factories import ( # lint-amnesty, pylint: disable=wrong-import-order + BlockFactory, + ToyCourseFactory ) -from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order -from xmodule.modulestore.tests.factories import BlockFactory, ToyCourseFactory # lint-amnesty, pylint: disable=wrong-import-order from .helpers import deserialize_usage_key @@ -498,17 +500,26 @@ def test_completion_all_course_with_nav_depth(self): assert response.data['blocks'][block_id].get('completion') @ddt.data( - False, - True, + (False, 'list'), + (True, 'list'), + (False, 'dict'), + (True, 'dict'), ) - def test_filter_discussion_xblocks(self, is_openedx_provider): + @ddt.unpack + def test_filter_discussion_xblocks(self, is_openedx_provider, return_type): """ Tests if discussion xblocks are hidden for openedx provider """ + def blocks_has_discussion_xblock(blocks): - for key, value in blocks.items(): - if value.get('type') == 'discussion': - return True + if isinstance(blocks, ReturnList): + for value in blocks: + if value.get('type') == 'discussion': + return True + else: + for key, value in blocks.items(): + if value.get('type') == 'discussion': + return True return False BlockFactory.create( @@ -520,9 +531,14 @@ def blocks_has_discussion_xblock(blocks): ) if is_openedx_provider: DiscussionsConfiguration.objects.create(context_key=self.course_key, provider_type=Provider.OPEN_EDX) - response = self.client.get(self.url, self.query_params) + params = self.query_params.copy() + if return_type == 'list': + params['return_type'] = 'list' + response = self.client.get(self.url, params) - has_discussion_xblock = blocks_has_discussion_xblock(response.data.get('blocks', {})) + has_discussion_xblock = blocks_has_discussion_xblock( + response.data if isinstance(response.data, ReturnList) else response.data.get('blocks', {}) + ) if is_openedx_provider: assert not has_discussion_xblock else: diff --git a/lms/djangoapps/course_api/blocks/utils.py b/lms/djangoapps/course_api/blocks/utils.py index 3a9fea117c23..0af24b951ade 100644 --- a/lms/djangoapps/course_api/blocks/utils.py +++ b/lms/djangoapps/course_api/blocks/utils.py @@ -1,6 +1,8 @@ """ Utils for Blocks """ +from rest_framework.utils.serializer_helpers import ReturnList + from openedx.core.djangoapps.discussions.models import ( DiscussionsConfiguration, Provider, @@ -15,16 +17,28 @@ def filter_discussion_xblocks_from_response(response, course_key): provider = configuration.provider_type if provider == Provider.OPEN_EDX: # Finding ids of discussion xblocks - discussion_xblocks = [ - key for key, value in response.data.get('blocks', {}).items() - if value.get('type') == 'discussion' - ] + if isinstance(response.data, ReturnList): + discussion_xblocks = [ + value.get('id') for value in response.data if value.get('type') == 'discussion' + ] + else: + discussion_xblocks = [ + key for key, value in response.data.get('blocks', {}).items() + if value.get('type') == 'discussion' + ] # Filtering discussion xblocks keys from blocks - filtered_blocks = { - key: value - for key, value in response.data.get('blocks', {}).items() - if value.get('type') != 'discussion' - } + if isinstance(response.data, ReturnList): + filtered_blocks = { + value.get('id'): value + for value in response.data + if value.get('type') != 'discussion' + } + else: + filtered_blocks = { + key: value + for key, value in response.data.get('blocks', {}).items() + if value.get('type') != 'discussion' + } # Removing reference of discussion xblocks from unit # These references needs to be removed because they no longer exist for _, block_data in filtered_blocks.items(): @@ -36,5 +50,8 @@ def filter_discussion_xblocks_from_response(response, course_key): if descendant not in discussion_xblocks ] block_data[key] = descendants - response.data['blocks'] = filtered_blocks + if isinstance(response.data, ReturnList): + response.data = filtered_blocks + else: + response.data['blocks'] = filtered_blocks return response diff --git a/lms/djangoapps/courseware/access_utils.py b/lms/djangoapps/courseware/access_utils.py index dd2f8d8f6dfe..d53699e5e1e4 100644 --- a/lms/djangoapps/courseware/access_utils.py +++ b/lms/djangoapps/courseware/access_utils.py @@ -9,7 +9,6 @@ from crum import get_current_request from django.conf import settings -from edx_toggles.toggles import SettingToggle from enterprise.models import EnterpriseCourseEnrollment, EnterpriseCustomerUser from pytz import UTC @@ -105,7 +104,19 @@ def enterprise_learner_enrolled(request, user, course_key): enterprise_customer_user__user_id=user.id, enterprise_customer_user__enterprise_customer__uuid=enterprise_customer_data['uuid'], ) - return enterprise_enrollments.exists() + enterprise_enrollment_exists = enterprise_enrollments.exists() + log.info( + ( + '[enterprise_learner_enrolled] Checking for an enterprise enrollment for ' + 'lms_user_id=%s in course_key=%s via enterprise_customer_uuid=%s. ' + 'Exists: %s' + ), + user.id, + course_key, + enterprise_customer_data['uuid'], + enterprise_enrollment_exists, + ) + return enterprise_enrollment_exists def check_start_date(user, days_early_for_beta, start, course_key, display_error_to_user=True, now=None): @@ -138,10 +149,9 @@ def check_start_date(user, days_early_for_beta, start, course_key, display_error # Before returning a StartDateError, determine if the learner should be redirected to the enterprise learner # portal by returning StartDateEnterpriseLearnerError instead. - if SettingToggle('COURSEWARE_COURSE_NOT_STARTED_ENTERPRISE_LEARNER_ERROR', default=False).is_enabled(): - request = get_current_request() - if request and enterprise_learner_enrolled(request, user, course_key): - return StartDateEnterpriseLearnerError(start, display_error_to_user=display_error_to_user) + request = get_current_request() + if request and enterprise_learner_enrolled(request, user, course_key): + return StartDateEnterpriseLearnerError(start, display_error_to_user=display_error_to_user) return StartDateError(start, display_error_to_user=display_error_to_user) diff --git a/lms/djangoapps/courseware/tests/test_access.py b/lms/djangoapps/courseware/tests/test_access.py index e89b4f6c4ba0..e0c5f59a83fb 100644 --- a/lms/djangoapps/courseware/tests/test_access.py +++ b/lms/djangoapps/courseware/tests/test_access.py @@ -56,7 +56,7 @@ SharedModuleStoreTestCase ) from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory # lint-amnesty, pylint: disable=wrong-import-order -from xmodule.partitions.partitions import MINIMUM_STATIC_PARTITION_ID, Group, UserPartition # lint-amnesty, pylint: disable=wrong-import-order +from xmodule.partitions.partitions import MINIMUM_UNUSED_PARTITION_ID, Group, UserPartition # lint-amnesty, pylint: disable=wrong-import-order from openedx.features.enterprise_support.api import add_enterprise_customer_to_session from enterprise.api.v1.serializers import EnterpriseCustomerSerializer from openedx.features.enterprise_support.tests.factories import ( @@ -288,9 +288,9 @@ def test_has_access_in_preview_mode_with_group(self): """ # Note about UserPartition and UserPartition Group IDs: these must not conflict with IDs used # by dynamic user partitions. - partition_id = MINIMUM_STATIC_PARTITION_ID - group_0_id = MINIMUM_STATIC_PARTITION_ID + 1 - group_1_id = MINIMUM_STATIC_PARTITION_ID + 2 + partition_id = MINIMUM_UNUSED_PARTITION_ID + group_0_id = MINIMUM_UNUSED_PARTITION_ID + 1 + group_1_id = MINIMUM_UNUSED_PARTITION_ID + 2 user_partition = UserPartition( partition_id, 'Test User Partition', '', [Group(group_0_id, 'Group 1'), Group(group_1_id, 'Group 2')], @@ -903,7 +903,6 @@ def test_course_catalog_access_num_queries_no_enterprise(self, user_attr_name, a ) @ddt.unpack @patch.dict('django.conf.settings.FEATURES', {'DISABLE_START_DATES': False, 'ENABLE_ENTERPRISE_INTEGRATION': True}) - @override_settings(COURSEWARE_COURSE_NOT_STARTED_ENTERPRISE_LEARNER_ERROR=True) def test_course_catalog_access_num_queries_enterprise(self, user_attr_name, course_attr_name): """ Similar to test_course_catalog_access_num_queries_no_enterprise, except enable enterprise features and make the diff --git a/lms/djangoapps/courseware/tests/test_views.py b/lms/djangoapps/courseware/tests/test_views.py index 1d88cff042a8..cde0e8a34ec2 100644 --- a/lms/djangoapps/courseware/tests/test_views.py +++ b/lms/djangoapps/courseware/tests/test_views.py @@ -78,6 +78,8 @@ from lms.djangoapps.courseware.tests.helpers import MasqueradeMixin, get_expiration_banner_text, set_preview_mode from lms.djangoapps.courseware.testutils import RenderXBlockTestMixin from lms.djangoapps.courseware.toggles import ( + COURSEWARE_MICROFRONTEND_ALWAYS_OPEN_AUXILIARY_SIDEBAR, + COURSEWARE_MICROFRONTEND_ENABLE_NAVIGATION_SIDEBAR, COURSEWARE_MICROFRONTEND_SEARCH_ENABLED, COURSEWARE_OPTIMIZED_RENDER_XBLOCK, ) @@ -3262,7 +3264,6 @@ class AccessUtilsTestCase(ModuleStoreTestCase): ) @ddt.unpack @patch.dict('django.conf.settings.FEATURES', {'DISABLE_START_DATES': False, 'ENABLE_ENTERPRISE_INTEGRATION': True}) - @override_settings(COURSEWARE_COURSE_NOT_STARTED_ENTERPRISE_LEARNER_ERROR=True) def test_is_course_open_for_learner( self, start_date_modifier, @@ -3812,3 +3813,68 @@ def test_is_mfe_search_waffle_disabled(self): self.assertEqual(response.status_code, 200) self.assertEqual(body, {'enabled': False}) + + +class TestCoursewareMFENavigationSidebarTogglesAPI(SharedModuleStoreTestCase): + """ + Tests the endpoint to fetch the Courseware Navigation Sidebar waffle flags status. + """ + + def setUp(self): + super().setUp() + + self.course = CourseFactory.create() + + self.client = APIClient() + self.apiUrl = reverse('courseware_navigation_sidebar_toggles_view', kwargs={'course_id': str(self.course.id)}) + + @override_waffle_flag(COURSEWARE_MICROFRONTEND_ENABLE_NAVIGATION_SIDEBAR, active=True) + @override_waffle_flag(COURSEWARE_MICROFRONTEND_ALWAYS_OPEN_AUXILIARY_SIDEBAR, active=False) + def test_courseware_mfe_navigation_sidebar_enabled_aux_disabled(self): + """ + Getter to check if it is allowed to show the Courseware navigation sidebar to a user + and auxiliary sidebar doesn't open. + """ + response = self.client.get(self.apiUrl, content_type='application/json') + body = json.loads(response.content.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(body, {'enable_navigation_sidebar': True, 'always_open_auxiliary_sidebar': False}) + + @override_waffle_flag(COURSEWARE_MICROFRONTEND_ENABLE_NAVIGATION_SIDEBAR, active=True) + @override_waffle_flag(COURSEWARE_MICROFRONTEND_ALWAYS_OPEN_AUXILIARY_SIDEBAR, active=True) + def test_courseware_mfe_navigation_sidebar_enabled_aux_enabled(self): + """ + Getter to check if it is allowed to show the Courseware navigation sidebar to a user + and auxiliary sidebar should always open. + """ + response = self.client.get(self.apiUrl, content_type='application/json') + body = json.loads(response.content.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(body, {'enable_navigation_sidebar': True, 'always_open_auxiliary_sidebar': True}) + + @override_waffle_flag(COURSEWARE_MICROFRONTEND_ENABLE_NAVIGATION_SIDEBAR, active=False) + @override_waffle_flag(COURSEWARE_MICROFRONTEND_ALWAYS_OPEN_AUXILIARY_SIDEBAR, active=True) + def test_courseware_mfe_navigation_sidebar_disabled_aux_enabled(self): + """ + Getter to check if the Courseware navigation sidebar shouldn't be shown to a user + and auxiliary sidebar should always open. + """ + response = self.client.get(self.apiUrl, content_type='application/json') + body = json.loads(response.content.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(body, {'enable_navigation_sidebar': False, 'always_open_auxiliary_sidebar': True}) + + @override_waffle_flag(COURSEWARE_MICROFRONTEND_ENABLE_NAVIGATION_SIDEBAR, active=False) + @override_waffle_flag(COURSEWARE_MICROFRONTEND_ALWAYS_OPEN_AUXILIARY_SIDEBAR, active=False) + def test_courseware_mfe_navigation_sidebar_toggles_disabled(self): + """ + Getter to check if neither navigation sidebar nor auxiliary sidebar is shown. + """ + response = self.client.get(self.apiUrl, content_type='application/json') + body = json.loads(response.content.decode('utf-8')) + + self.assertEqual(response.status_code, 200) + self.assertEqual(body, {'enable_navigation_sidebar': False, 'always_open_auxiliary_sidebar': False}) diff --git a/lms/djangoapps/courseware/toggles.py b/lms/djangoapps/courseware/toggles.py index ca4584b19ba6..43fb40436a5e 100644 --- a/lms/djangoapps/courseware/toggles.py +++ b/lms/djangoapps/courseware/toggles.py @@ -72,9 +72,9 @@ # .. toggle_implementation: WaffleFlag # .. toggle_default: False # .. toggle_description: Disable caching of navigation sidebar blocks on Learning MFE. -# It can be used when caching the structure of large courses for a large number of users -# at the same time can overload the cache storage (memcache or redis). -# .. toggle_use_cases: temporary +# It can be used when caching the structure of large courses for a large number of users +# at the same time can overload the cache storage (memcache or redis). +# .. toggle_use_cases: opt_out, open_edx # .. toggle_creation_date: 2024-03-21 # .. toggle_target_removal_date: None # .. toggle_tickets: FC-0056 @@ -83,6 +83,33 @@ f'{WAFFLE_FLAG_NAMESPACE}.disable_navigation_sidebar_blocks_caching', __name__ ) +# .. toggle_name: courseware.enable_navigation_sidebar +# .. toggle_implementation: WaffleFlag +# .. toggle_default: False +# .. toggle_description: Enable navigation sidebar on Learning MFE +# .. toggle_use_cases: opt_out, open_edx +# .. toggle_creation_date: 2024-03-07 +# .. toggle_target_removal_date: None +# .. toggle_tickets: FC-0056 +COURSEWARE_MICROFRONTEND_ENABLE_NAVIGATION_SIDEBAR = CourseWaffleFlag( + f'{WAFFLE_FLAG_NAMESPACE}.enable_navigation_sidebar', __name__ +) + +# .. toggle_name: courseware.always_open_auxiliary_sidebar +# .. toggle_implementation: WaffleFlag +# .. toggle_default: True +# .. toggle_description: Waffle flag that determines whether the auxiliary sidebar, +# such as discussion or notification, should automatically expand +# on each course unit page within the Learning MFE, without preserving +# the previous state of the sidebar. +# .. toggle_use_cases: temporary +# .. toggle_creation_date: 2024-04-28 +# .. toggle_target_removal_date: 2024-07-28 +# .. toggle_tickets: FC-0056 +COURSEWARE_MICROFRONTEND_ALWAYS_OPEN_AUXILIARY_SIDEBAR = CourseWaffleFlag( + f'{WAFFLE_FLAG_NAMESPACE}.always_open_auxiliary_sidebar', __name__ +) + # .. toggle_name: courseware.mfe_progress_milestones_streak_discount_enabled # .. toggle_implementation: CourseWaffleFlag # .. toggle_default: False diff --git a/lms/djangoapps/courseware/views/views.py b/lms/djangoapps/courseware/views/views.py index 4c3fa982ca69..3bf8ae10e8c2 100644 --- a/lms/djangoapps/courseware/views/views.py +++ b/lms/djangoapps/courseware/views/views.py @@ -88,7 +88,12 @@ from lms.djangoapps.courseware.model_data import FieldDataCache from lms.djangoapps.courseware.models import BaseStudentModuleHistory, StudentModule from lms.djangoapps.courseware.permissions import MASQUERADE_AS_STUDENT, VIEW_COURSE_HOME, VIEW_COURSEWARE -from lms.djangoapps.courseware.toggles import course_is_invitation_only, courseware_mfe_search_is_enabled +from lms.djangoapps.courseware.toggles import ( + course_is_invitation_only, + courseware_mfe_search_is_enabled, + COURSEWARE_MICROFRONTEND_ENABLE_NAVIGATION_SIDEBAR, + COURSEWARE_MICROFRONTEND_ALWAYS_OPEN_AUXILIARY_SIDEBAR, +) from lms.djangoapps.courseware.user_state_client import DjangoXBlockUserStateClient from lms.djangoapps.courseware.utils import ( _use_new_financial_assistance_flow, @@ -2282,3 +2287,19 @@ def courseware_mfe_search_enabled(request, course_id=None): payload = {"enabled": courseware_mfe_search_is_enabled(course_key) if enabled else False} return JsonResponse(payload) + + +@api_view(['GET']) +def courseware_mfe_navigation_sidebar_toggles(request, course_id=None): + """ + GET endpoint to return navigation sidebar toggles. + """ + try: + course_key = CourseKey.from_string(course_id) if course_id else None + except InvalidKeyError: + return JsonResponse({"error": "Invalid course_id"}) + + return JsonResponse({ + "enable_navigation_sidebar": COURSEWARE_MICROFRONTEND_ENABLE_NAVIGATION_SIDEBAR.is_enabled(course_key), + "always_open_auxiliary_sidebar": COURSEWARE_MICROFRONTEND_ALWAYS_OPEN_AUXILIARY_SIDEBAR.is_enabled(course_key), + }) diff --git a/lms/djangoapps/discussion/rest_api/api.py b/lms/djangoapps/discussion/rest_api/api.py index 27a86c3a0192..7244127dc2a3 100644 --- a/lms/djangoapps/discussion/rest_api/api.py +++ b/lms/djangoapps/discussion/rest_api/api.py @@ -13,7 +13,6 @@ from urllib.parse import urlencode, urlunparse from pytz import UTC - from django.conf import settings from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError @@ -34,7 +33,6 @@ ) from lms.djangoapps.course_api.blocks.api import get_blocks -from lms.djangoapps.course_blocks.api import get_course_blocks from lms.djangoapps.courseware.courses import get_course_with_access from lms.djangoapps.courseware.exceptions import CourseAccessRedirect from lms.djangoapps.discussion.toggles import ENABLE_DISCUSSIONS_MFE @@ -82,6 +80,7 @@ from openedx.core.djangoapps.user_api.accounts.api import get_account_settings from openedx.core.lib.exceptions import CourseNotFoundError, DiscussionNotFoundError, PageNotFoundError from xmodule.course_block import CourseBlock +from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore from xmodule.tabs import CourseTabList @@ -131,7 +130,6 @@ is_posting_allowed ) - User = get_user_model() ThreadType = Literal["discussion", "question"] @@ -418,6 +416,7 @@ def sort_categories(category_list): Required arguments: category_list -- list of categories. """ + def convert(text): if text.isdigit(): return int(text) @@ -697,11 +696,19 @@ def get_course_topics_v2( FORUM_ROLE_ADMINISTRATOR, ] ).exists() - course_blocks = get_course_blocks(user, store.make_course_usage_key(course_key)) - accessible_vertical_keys = [ - block for block in course_blocks.get_block_keys() - if block.block_type == 'vertical' - ] + [None] + + with store.branch_setting(ModuleStoreEnum.Branch.draft_preferred, course_key): + blocks = store.get_items( + course_key, + qualifiers={'category': 'vertical'}, + fields=['usage_key', 'discussion_enabled', 'display_name'], + ) + accessible_vertical_keys = [] + for block in blocks: + if block.discussion_enabled and (not block.visible_to_staff_only or user_is_privileged): + accessible_vertical_keys.append(block.usage_key) + accessible_vertical_keys.append(None) + topics_query = DiscussionTopicLink.objects.filter( context_key=course_key, provider_id=provider_type, diff --git a/lms/djangoapps/edxnotes/tests.py b/lms/djangoapps/edxnotes/tests.py index 3cd7bafa5b21..688cd543865f 100644 --- a/lms/djangoapps/edxnotes/tests.py +++ b/lms/djangoapps/edxnotes/tests.py @@ -167,9 +167,9 @@ def test_edxnotes_studio(self): self.problem.runtime.is_author_mode = True assert 'original_get_html' == self.problem.get_html() - def test_edxnotes_blockstore_runtime(self): + def test_edxnotes_learning_core_runtime(self): """ - Tests that get_html is not wrapped when problem is rendered by Blockstore runtime. + Tests that get_html is not wrapped when problem is rendered by the learning core runtime. """ del self.problem.block.runtime.modulestore assert 'original_get_html' == self.problem.get_html() diff --git a/lms/djangoapps/grades/events.py b/lms/djangoapps/grades/events.py index 90279a3e69fe..51d1b13702f0 100644 --- a/lms/djangoapps/grades/events.py +++ b/lms/djangoapps/grades/events.py @@ -6,6 +6,15 @@ from crum import get_current_user from django.conf import settings from eventtracking import tracker +from openedx_events.learning.data import ( + CcxCourseData, + CcxCoursePassingStatusData, + CourseData, + CoursePassingStatusData, + UserData, + UserPersonalData +) +from openedx_events.learning.signals import CCX_COURSE_PASSING_STATUS_UPDATED, COURSE_PASSING_STATUS_UPDATED from common.djangoapps.course_modes.models import CourseMode from common.djangoapps.student.models import CourseEnrollment @@ -174,8 +183,8 @@ def course_grade_passed_first_time(user_id, course_id): def course_grade_now_passed(user, course_id): """ - Emits an edx.course.grade.now_passed event - with data from the course and user passed now . + Emits an edx.course.grade.now_passed and passing status updated events + with data from the course and user passed now. """ event_name = COURSE_GRADE_NOW_PASSED_EVENT_TYPE context = contexts.course_context_from_course_id(course_id) @@ -190,11 +199,13 @@ def course_grade_now_passed(user, course_id): } ) + _emit_course_passing_status_update(user, course_id, is_passing=True) + def course_grade_now_failed(user, course_id): """ - Emits an edx.course.grade.now_failed event - with data from the course and user failed now . + Emits an edx.course.grade.now_failed and passing status updated events + with data from the course and user failed now. """ event_name = COURSE_GRADE_NOW_FAILED_EVENT_TYPE context = contexts.course_context_from_course_id(course_id) @@ -209,6 +220,8 @@ def course_grade_now_failed(user, course_id): } ) + _emit_course_passing_status_update(user, course_id, is_passing=False) + def fire_segment_event_on_course_grade_passed_first_time(user_id, course_locator): """ @@ -258,3 +271,47 @@ def fire_segment_event_on_course_grade_passed_first_time(user_id, course_locator ) log.info("Segment event fired for passed learners. Event: [{}], Data: [{}]".format(event_name, event_properties)) + + +def _emit_course_passing_status_update(user, course_id, is_passing): + """ + Emit course passing status event according to the course type. + The status of event is determined by is_passing parameter. + """ + if hasattr(course_id, 'ccx'): + CCX_COURSE_PASSING_STATUS_UPDATED.send_event( + course_passing_status=CcxCoursePassingStatusData( + is_passing=is_passing, + user=UserData( + pii=UserPersonalData( + username=user.username, + email=user.email, + name=user.get_full_name(), + ), + id=user.id, + is_active=user.is_active, + ), + course=CcxCourseData( + ccx_course_key=course_id, + master_course_key=course_id.to_course_locator(), + ), + ) + ) + else: + COURSE_PASSING_STATUS_UPDATED.send_event( + course_passing_status=CoursePassingStatusData( + is_passing=is_passing, + user=UserData( + pii=UserPersonalData( + username=user.username, + email=user.email, + name=user.get_full_name(), + ), + id=user.id, + is_active=user.is_active, + ), + course=CourseData( + course_key=course_id, + ), + ) + ) diff --git a/lms/djangoapps/grades/tests/test_events.py b/lms/djangoapps/grades/tests/test_events.py index b7843dfc1c1c..eac8cc9a4a70 100644 --- a/lms/djangoapps/grades/tests/test_events.py +++ b/lms/djangoapps/grades/tests/test_events.py @@ -4,16 +4,29 @@ from unittest import mock +from ccx_keys.locator import CCXLocator from django.utils.timezone import now from openedx_events.learning.data import ( + CcxCourseData, + CcxCoursePassingStatusData, CourseData, - PersistentCourseGradeData + CoursePassingStatusData, + PersistentCourseGradeData, + UserData, + UserPersonalData +) +from openedx_events.learning.signals import ( + CCX_COURSE_PASSING_STATUS_UPDATED, + COURSE_PASSING_STATUS_UPDATED, + PERSISTENT_GRADE_SUMMARY_CHANGED ) -from openedx_events.learning.signals import PERSISTENT_GRADE_SUMMARY_CHANGED from openedx_events.tests.utils import OpenEdxEventsTestMixin -from common.djangoapps.student.tests.factories import UserFactory +from common.djangoapps.student.tests.factories import AdminFactory, UserFactory +from lms.djangoapps.ccx.models import CustomCourseForEdX +from lms.djangoapps.grades.course_grade_factory import CourseGradeFactory from lms.djangoapps.grades.models import PersistentCourseGrade +from lms.djangoapps.grades.tests.utils import mock_passing_grade from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory @@ -94,5 +107,148 @@ def test_persistent_grade_event_emitted(self): passed_timestamp=grade.passed_timestamp ) }, - event_receiver.call_args.kwargs + event_receiver.call_args.kwargs, + ) + + +class CoursePassingStatusEventsTest(SharedModuleStoreTestCase, OpenEdxEventsTestMixin): + """ + Tests for Open edX passing status update event. + """ + ENABLED_OPENEDX_EVENTS = [ + "org.openedx.learning.course.passing.status.updated.v1", + ] + + @classmethod + def setUpClass(cls): + """ + Set up class method for the Test class. + """ + super().setUpClass() + cls.start_events_isolation() + + def setUp(self): + super().setUp() + self.course = CourseFactory.create() + self.user = UserFactory.create() + self.receiver_called = False + + def _event_receiver_side_effect(self, **kwargs): + """ + Used show that the Open edX Event was called by the Django signal handler. + """ + self.receiver_called = True + + def test_course_passing_status_updated_emitted(self): + """ + Test whether passing status updated event is sent after the grade is being updated for a user. + """ + event_receiver = mock.Mock(side_effect=self._event_receiver_side_effect) + COURSE_PASSING_STATUS_UPDATED.connect(event_receiver) + grade_factory = CourseGradeFactory() + + with mock_passing_grade(): + grade_factory.update(self.user, self.course) + + self.assertTrue(self.receiver_called) + self.assertDictContainsSubset( + { + "signal": COURSE_PASSING_STATUS_UPDATED, + "sender": None, + "course_passing_status": CoursePassingStatusData( + is_passing=True, + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + name=self.user.get_full_name(), + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course=CourseData( + course_key=self.course.id, + ), + ), + }, + event_receiver.call_args.kwargs, + ) + + +class CCXCoursePassingStatusEventsTest( + SharedModuleStoreTestCase, OpenEdxEventsTestMixin +): + """ + Tests for Open edX passing status update event in a CCX course. + """ + ENABLED_OPENEDX_EVENTS = [ + "org.openedx.learning.ccx.course.passing.status.updated.v1", + ] + + @classmethod + def setUpClass(cls): + """ + Set up class method for the Test class. + """ + super().setUpClass() + cls.start_events_isolation() + + def setUp(self): + super().setUp() + self.course = CourseFactory.create() + self.user = UserFactory.create() + self.coach = AdminFactory.create() + self.ccx = ccx = CustomCourseForEdX( + course_id=self.course.id, display_name="Test CCX", coach=self.coach + ) + ccx.save() + self.ccx_locator = CCXLocator.from_course_locator(self.course.id, ccx.id) + + self.receiver_called = False + + def _event_receiver_side_effect(self, **kwargs): + """ + Used show that the Open edX Event was called by the Django signal handler. + """ + self.receiver_called = True + + def test_ccx_course_passing_status_updated_emitted(self): + """ + Test whether passing status updated event is sent after the grade is being updated in CCX course. + """ + event_receiver = mock.Mock(side_effect=self._event_receiver_side_effect) + CCX_COURSE_PASSING_STATUS_UPDATED.connect(event_receiver) + grade_factory = CourseGradeFactory() + + with mock_passing_grade(): + grade_factory.update(self.user, self.store.get_course(self.ccx_locator)) + + self.assertTrue(self.receiver_called) + self.assertDictContainsSubset( + { + "signal": CCX_COURSE_PASSING_STATUS_UPDATED, + "sender": None, + "course_passing_status": CcxCoursePassingStatusData( + is_passing=True, + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + name=self.user.get_full_name(), + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course=CcxCourseData( + ccx_course_key=self.ccx_locator, + master_course_key=self.course.id, + display_name="", + coach_email="", + start=None, + end=None, + max_students_allowed=self.ccx.max_student_enrollments_allowed, + ), + ), + }, + event_receiver.call_args.kwargs, ) diff --git a/lms/djangoapps/instructor/tests/test_api.py b/lms/djangoapps/instructor/tests/test_api.py index bb2be51e9640..f5d8b0408950 100644 --- a/lms/djangoapps/instructor/tests/test_api.py +++ b/lms/djangoapps/instructor/tests/test_api.py @@ -2252,17 +2252,26 @@ def test_modify_access_bad_role(self): }) assert response.status_code == 400 - def test_modify_access_allow(self): - assert CourseEnrollment.is_enrolled(self.other_user, self.course.id) is False - url = reverse('modify_access', kwargs={'course_id': str(self.course.id)}) - response = self.client.post(url, { - 'unique_student_identifier': self.other_user.email, - 'rolename': 'staff', - 'action': 'allow', - }) - assert response.status_code == 200 - # User should be auto enrolled in the course - assert CourseEnrollment.is_enrolled(self.other_user, self.course.id) + def test_modify_access_api(self): + for rolename in ["staff", "limited_staff", "instructor", "data_researcher"]: + assert CourseEnrollment.is_enrolled(self.other_user, self.course.id) is False + url = reverse('modify_access', kwargs={'course_id': str(self.course.id)}) + response = self.client.post(url, { + 'unique_student_identifier': self.other_user.email, + 'rolename': rolename, + 'action': 'allow', + }) + assert response.status_code == 200 + # User should be auto enrolled in the course + assert CourseEnrollment.is_enrolled(self.other_user, self.course.id) + # Test role revoke action + response = self.client.post(url, { + 'unique_student_identifier': self.other_user.email, + 'rolename': rolename, + 'action': 'revoke', + }) + assert response.status_code == 200 + CourseEnrollment.unenroll(self.other_user, self.course.id) def test_modify_access_allow_with_uname(self): url = reverse('modify_access', kwargs={'course_id': str(self.course.id)}) @@ -2273,15 +2282,6 @@ def test_modify_access_allow_with_uname(self): }) assert response.status_code == 200 - def test_modify_access_revoke(self): - url = reverse('modify_access', kwargs={'course_id': str(self.course.id)}) - response = self.client.post(url, { - 'unique_student_identifier': self.other_staff.email, - 'rolename': 'staff', - 'action': 'revoke', - }) - assert response.status_code == 200 - def test_modify_access_revoke_with_username(self): url = reverse('modify_access', kwargs={'course_id': str(self.course.id)}) response = self.client.post(url, { @@ -2443,6 +2443,19 @@ def assert_update_forum_role_membership(self, current_user, identifier, rolename elif action == 'revoke': assert rolename not in user_roles + def test_autoenroll_on_forum_role_add(self): + """ + Test forum role modification auto enrolls user. + """ + seed_permissions_roles(self.course.id) + user = UserFactory() + for rolename in ["Administrator", "Moderator", "Community TA"]: + assert CourseEnrollment.is_enrolled(user, self.course.id) is False + self.assert_update_forum_role_membership(user, user.email, rolename, "allow") + assert CourseEnrollment.is_enrolled(user, self.course.id) + self.assert_update_forum_role_membership(user, user.email, rolename, "revoke") + CourseEnrollment.unenroll(user, self.course.id) + @ddt.ddt class TestInstructorAPILevelsDataDump(SharedModuleStoreTestCase, LoginEnrollmentTestCase): diff --git a/lms/djangoapps/instructor/views/api.py b/lms/djangoapps/instructor/views/api.py index ddcfa7b7aba3..6d7dfe17a9d7 100644 --- a/lms/djangoapps/instructor/views/api.py +++ b/lms/djangoapps/instructor/views/api.py @@ -2812,7 +2812,8 @@ def update_forum_role_membership(request, course_id): )) user = get_student_from_identifier(unique_student_identifier) - + if action == 'allow' and not is_user_enrolled_in_course(user, course_id): + CourseEnrollment.enroll(user, course_id) try: update_forum_role(course_id, user, rolename, action) except Role.DoesNotExist: diff --git a/lms/djangoapps/lti_provider/tests/test_users.py b/lms/djangoapps/lti_provider/tests/test_users.py index 1d19d8995be9..fa8274eef30e 100644 --- a/lms/djangoapps/lti_provider/tests/test_users.py +++ b/lms/djangoapps/lti_provider/tests/test_users.py @@ -158,6 +158,16 @@ def test_auto_linking_of_users_using_lis_person_contact_email_primary(self, crea users.authenticate_lti_user(request, self.lti_user_id, self.auto_linking_consumer) create_user.assert_called_with(self.lti_user_id, self.auto_linking_consumer, self.old_user.email) + def test_auto_linking_of_users_using_lis_person_contact_email_primary_case_insensitive(self, create_user, switch_user): # pylint: disable=line-too-long + request = RequestFactory().post("/", {"lis_person_contact_email_primary": self.old_user.email.upper()}) + request.user = self.old_user + + users.authenticate_lti_user(request, self.lti_user_id, self.lti_consumer) + create_user.assert_called_with(self.lti_user_id, self.lti_consumer) + + users.authenticate_lti_user(request, self.lti_user_id, self.auto_linking_consumer) + create_user.assert_called_with(self.lti_user_id, self.auto_linking_consumer, request.user.email) + def test_raise_exception_trying_to_auto_link_unauthenticate_user(self, create_user, switch_user): request = RequestFactory().post("/") request.user = AnonymousUser() diff --git a/lms/djangoapps/lti_provider/users.py b/lms/djangoapps/lti_provider/users.py index c2373522b960..d1ade6ead325 100644 --- a/lms/djangoapps/lti_provider/users.py +++ b/lms/djangoapps/lti_provider/users.py @@ -40,8 +40,8 @@ def authenticate_lti_user(request, lti_user_id, lti_consumer): if lti_consumer.require_user_account: # Verify that the email from the LTI Launch and the logged-in user are the same # before linking the LtiUser with the edx_user. - if request.user.is_authenticated and request.user.email == lis_email: - lti_user = create_lti_user(lti_user_id, lti_consumer, lis_email) + if request.user.is_authenticated and request.user.email.lower() == lis_email.lower(): + lti_user = create_lti_user(lti_user_id, lti_consumer, request.user.email) else: # Ask the user to login before linking. raise PermissionDenied() from exc diff --git a/lms/djangoapps/mobile_api/course_info/serializers.py b/lms/djangoapps/mobile_api/course_info/serializers.py index b2bb0ce24701..d7a9471088aa 100644 --- a/lms/djangoapps/mobile_api/course_info/serializers.py +++ b/lms/djangoapps/mobile_api/course_info/serializers.py @@ -82,7 +82,7 @@ class MobileCourseEnrollmentSerializer(serializers.ModelSerializer): """ class Meta: - fields = ('created', 'mode', 'is_active') + fields = ('created', 'mode', 'is_active', 'upgrade_deadline') model = CourseEnrollment lookup_field = 'username' diff --git a/lms/djangoapps/teams/team_partition_scheme.py b/lms/djangoapps/teams/team_partition_scheme.py index d573bf51a2d2..7cbdaa37fbea 100644 --- a/lms/djangoapps/teams/team_partition_scheme.py +++ b/lms/djangoapps/teams/team_partition_scheme.py @@ -58,12 +58,14 @@ class TeamPartitionScheme: This is how it works: - A user partition is created for each team-set in the course with a unused partition ID generated in runtime - by using generate_int_id() with min=MINIMUM_STATIC_PARTITION_ID and max=MYSQL_MAX_INT. + by using generate_int_id() with min=MINIMUM_UNUSED_PARTITION_ID and max=MYSQL_MAX_INT. - A (Content) group is created for each team in the team-set with the database team ID as the group ID, and the team name as the group name. - A user is assigned to a group if they are a member of the team. """ + read_only = True + @classmethod def get_group_for_user(cls, course_key, user, user_partition): """Get the (Content) Group from the specified user partition for the user. diff --git a/lms/envs/common.py b/lms/envs/common.py index d97cd31046b2..da2bfed626e0 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1076,6 +1076,15 @@ # .. toggle_warning: For consistency, keep the value in sync with the setting of the same name in the LMS and CMS. # .. toggle_tickets: https://github.com/openedx/edx-platform/pull/34442 'ENABLE_BLAKE2B_HASHING': False, + + # .. toggle_name: FEATURES['BADGES_ENABLED'] + # .. toggle_implementation: DjangoSetting + # .. toggle_default: False + # .. toggle_description: Set to True to enable badges functionality. + # .. toggle_use_cases: open_edx + # .. toggle_creation_date: 2024-04-02 + # .. toggle_target_removal_date: None + 'BADGES_ENABLED': False, } # Specifies extra XBlock fields that should available when requested via the Course Blocks API @@ -1148,26 +1157,12 @@ DATABASE_ROUTERS = [ 'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter', - 'openedx.core.lib.blockstore_api.db_routers.BlockstoreRouter', 'edx_django_utils.db.read_replica.ReadReplicaRouter', ] ############################ Cache Configuration ############################### CACHES = { - 'blockstore': { - 'KEY_PREFIX': 'blockstore', - 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', - 'LOCATION': ['localhost:11211'], - 'TIMEOUT': '86400', # This data should be long-lived for performance, BundleCache handles invalidation - 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', - 'OPTIONS': { - 'no_delay': True, - 'ignore_exc': True, - 'use_pooling': True, - 'connect_timeout': 0.5 - } - }, 'course_structure_cache': { 'KEY_PREFIX': 'course_structure', 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', @@ -1695,6 +1690,11 @@ def _make_mako_template_dirs(settings): # for more reference. XBLOCK_SETTINGS = {} +# .. setting_name: XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE +# .. setting_default: default +# .. setting_description: The django cache key of the cache to use for storing anonymous user state for XBlocks. +XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'default' + ############# ModuleStore Configuration ########## MODULESTORE_BRANCH = 'published-only' @@ -1928,7 +1928,7 @@ def _make_mako_template_dirs(settings): # Static content STATIC_URL = '/static/' -STATIC_ROOT = ENV_ROOT / "staticfiles" +STATIC_ROOT = os.environ.get('STATIC_ROOT_LMS', ENV_ROOT / "staticfiles") STATIC_URL_BASE = '/static/' STATICFILES_DIRS = [ @@ -2822,7 +2822,14 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring 'STATS_FILE': os.path.join(STATIC_ROOT, 'webpack-worker-stats.json') } } -WEBPACK_CONFIG_PATH = 'webpack.prod.config.js' + +# .. setting_name: WEBPACK_CONFIG_PATH +# .. setting_default: "webpack.prod.config.js" +# .. setting_description: Path to the Webpack configuration file. Used by Paver scripts. +# .. setting_warning: This Django setting is DEPRECATED! Starting in Sumac, Webpack will no longer +# use Django settings. Please set the WEBPACK_CONFIG_PATH environment variable instead. For details, +# see: https://github.com/openedx/edx-platform/issues/31895 +WEBPACK_CONFIG_PATH = os.environ.get('WEBPACK_CONFIG_PATH', 'webpack.prod.config.js') ########################## DJANGO DEBUG TOOLBAR ############################### @@ -3133,7 +3140,7 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # User tours 'lms.djangoapps.user_tours', - # New (Blockstore-based) XBlock runtime + # New (Learning-Core-based) XBlock runtime 'openedx.core.djangoapps.xblock.apps.LmsXBlockAppConfig', # Student support tools @@ -3369,9 +3376,6 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # For edx ace template tags 'edx_ace', - # Blockstore - 'blockstore.apps.bundles', - # MFE API 'lms.djangoapps.mfe_config_api', @@ -3668,9 +3672,13 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring CORS_ORIGIN_WHITELIST = () CORS_ORIGIN_ALLOW_ALL = False CORS_ALLOW_INSECURE = False - CORS_ALLOW_HEADERS = corsheaders_default_headers + ( - 'use-jwt-cookie', - ) + +# Set CORS_ALLOW_HEADERS regardless of whether we've enabled ENABLE_CORS_HEADERS +# because that decision might happen in a later config file. (The headers to +# allow is an application logic, and not site policy.) +CORS_ALLOW_HEADERS = corsheaders_default_headers + ( + 'use-jwt-cookie', +) # Default cache expiration for the cross-domain proxy HTML page. # This is a static page that can be iframed into an external page @@ -3737,6 +3745,9 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # that match a regex in this list. Set to None to allow any email (default). REGISTRATION_EMAIL_PATTERNS_ALLOWED = None +# String length for the configurable part of the auto-generated username +AUTO_GENERATED_USERNAME_RANDOM_STRING_LENGTH = 4 + ########################## CERTIFICATE NAME ######################## CERT_NAME_SHORT = "Certificate" CERT_NAME_LONG = "Certificate of Achievement" @@ -4546,9 +4557,11 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # .. setting_name: COMPREHENSIVE_THEME_DIRS # .. setting_default: [] -# .. setting_description: A list of directories containing themes folders, -# each entry should be a full path to the directory containing the theme folder. -COMPREHENSIVE_THEME_DIRS = [] +# .. setting_description: A list of paths to directories, each of which will +# be searched for comprehensive themes. Do not override this Django setting directly. +# Instead, set the COMPREHENSIVE_THEME_DIRS environment variable, using colons (:) to +# separate paths. +COMPREHENSIVE_THEME_DIRS = os.environ.get("COMPREHENSIVE_THEME_DIRS", "").split(":") # .. setting_name: COMPREHENSIVE_THEME_LOCALE_PATHS # .. setting_default: [] @@ -4728,19 +4741,6 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring ENTERPRISE_MARKETING_FOOTER_QUERY_PARAMS = {} ENTERPRISE_TAGLINE = '' -# .. toggle_name: COURSEWARE_COURSE_NOT_STARTED_ENTERPRISE_LEARNER_ERROR -# .. toggle_implementation: SettingToggle -# .. toggle_default: False -# .. toggle_description: If disabled (False), this switch causes the CourseTabView API (or whatever else calls -# check_course_open_for_learner()) to always return the legacy `course_not_started` error code in ALL cases where the -# course has not started. If enabled (True), the API will respond with `course_not_started_enterprise_learner` in a -# subset of cases where the learner is enrolled via subsidy, and `course_not_started` in all other cases. -# .. toggle_use_cases: temporary -# .. toggle_creation_date: 2023-12-18 -# .. toggle_target_removal_date: 2023-12-19 -# .. toggle_tickets: ENT-8078 -COURSEWARE_COURSE_NOT_STARTED_ENTERPRISE_LEARNER_ERROR = False - ############## Settings for Course Enrollment Modes ###################### # The min_price key refers to the minimum price allowed for an instance # of a particular type of course enrollment mode. This is not to be confused @@ -5184,57 +5184,6 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring ########################## MAILCHIMP SETTINGS ################################# MAILCHIMP_NEW_USER_LIST_ID = "" -########################## BLOCKSTORE ##################################### - -# .. setting_name: XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE -# .. setting_default: default -# .. setting_description: The django cache key of the cache to use for storing anonymous user state for XBlocks. -XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'default' - -# .. setting_name: BLOCKSTORE_BUNDLE_CACHE_TIMEOUT -# .. setting_default: 3000 -# .. setting_description: Maximum time-to-live of cached Bundles fetched from -# Blockstore, in seconds. When the values returned from Blockstore have -# TTLs of their own (such as signed S3 URLs), the maximum TTL of this cache -# must be lower than the minimum TTL of those values. -# We use a default of 3000s (50mins) because temporary URLs are often -# configured to expire after one hour. -BLOCKSTORE_BUNDLE_CACHE_TIMEOUT = 3000 - -# .. setting_name: BUNDLE_ASSET_URL_STORAGE_KEY -# .. setting_default: None -# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_SECRET` is -# set, and `boto3` is installed, this is used as an AWS IAM access key for -# generating signed, read-only URLs for blockstore assets stored in S3. -# Otherwise, URLs are generated based on the default storage configuration. -# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. -BUNDLE_ASSET_URL_STORAGE_KEY = None - -# .. setting_name: BUNDLE_ASSET_URL_STORAGE_SECRET -# .. setting_default: None -# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_KEY` is -# set, and `boto3` is installed, this is used as an AWS IAM secret key for -# generating signed, read-only URLs for blockstore assets stored in S3. -# Otherwise, URLs are generated based on the default storage configuration. -# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. -BUNDLE_ASSET_URL_STORAGE_SECRET = None - -# .. setting_name: BUNDLE_ASSET_STORAGE_SETTINGS -# .. setting_default: dict, appropriate for file system storage. -# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_KEY` is -# set, and `boto3` is installed, this provides the bucket name and location for blockstore assets stored in S3. -# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. -BUNDLE_ASSET_STORAGE_SETTINGS = dict( - # Backend storage - # STORAGE_CLASS='storages.backends.s3boto3.S3Boto3Storage', - # STORAGE_KWARGS=dict(bucket='bundle-asset-bucket', location='/path-to-bundles/'), - STORAGE_CLASS='django.core.files.storage.FileSystemStorage', - STORAGE_KWARGS=dict( - location=MEDIA_ROOT, - base_url=MEDIA_URL, - ), -) - SYSLOG_SERVER = '' FEEDBACK_SUBMISSION_EMAIL = '' GITHUB_REPO_ROOT = '/edx/var/edxapp/data' @@ -5444,7 +5393,12 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring def _should_send_certificate_events(settings): return settings.FEATURES['SEND_LEARNING_CERTIFICATE_LIFECYCLE_EVENTS_TO_BUS'] + #### Event bus producing #### + +def _should_send_learning_badge_events(settings): + return settings.FEATURES['BADGES_ENABLED'] + # .. setting_name: EVENT_BUS_PRODUCER_CONFIG # .. setting_default: all events disabled # .. setting_description: Dictionary of event_types mapped to dictionaries of topic to topic-related configuration. @@ -5517,11 +5471,37 @@ def _should_send_certificate_events(settings): 'course-authoring-xblock-lifecycle': {'event_key_field': 'xblock_info.usage_key', 'enabled': False}, }, + "org.openedx.learning.course.passing.status.updated.v1": { + "learning-badges-lifecycle": { + "event_key_field": "course_passing_status.course.course_key", + "enabled": _should_send_learning_badge_events, + }, + }, + "org.openedx.learning.ccx.course.passing.status.updated.v1": { + "learning-badges-lifecycle": { + "event_key_field": "course_passing_status.course.ccx_course_key", + "enabled": _should_send_learning_badge_events, + }, + }, } derived_collection_entry('EVENT_BUS_PRODUCER_CONFIG', 'org.openedx.learning.certificate.created.v1', 'learning-certificate-lifecycle', 'enabled') derived_collection_entry('EVENT_BUS_PRODUCER_CONFIG', 'org.openedx.learning.certificate.revoked.v1', 'learning-certificate-lifecycle', 'enabled') + +derived_collection_entry( + "EVENT_BUS_PRODUCER_CONFIG", + "org.openedx.learning.course.passing.status.updated.v1", + "learning-badges-lifecycle", + "enabled", +) +derived_collection_entry( + "EVENT_BUS_PRODUCER_CONFIG", + "org.openedx.learning.ccx.course.passing.status.updated.v1", + "learning-badges-lifecycle", + "enabled", +) + BEAMER_PRODUCT_ID = "" #### Survey Report #### diff --git a/lms/envs/devstack.py b/lms/envs/devstack.py index 7f69641055af..82e9134f456f 100644 --- a/lms/envs/devstack.py +++ b/lms/envs/devstack.py @@ -7,8 +7,6 @@ import logging from os.path import abspath, dirname, join -from corsheaders.defaults import default_headers as corsheaders_default_headers - # pylint: enable=unicode-format-string # lint-amnesty, pylint: disable=bad-option-value ##################################################################### from edx_django_utils.plugins import add_plugins @@ -264,9 +262,6 @@ def should_show_debug_toolbar(request): # lint-amnesty, pylint: disable=missing ) }) -############################### BLOCKSTORE ##################################### -BLOCKSTORE_API_URL = "http://edx.devstack.blockstore:18250/api/v1/" - ########################## PROGRAMS LEARNER PORTAL ############################## LEARNER_PORTAL_URL_ROOT = 'http://localhost:8734' @@ -295,9 +290,6 @@ def should_show_debug_toolbar(request): # lint-amnesty, pylint: disable=missing CORS_ALLOW_CREDENTIALS = True CORS_ORIGIN_WHITELIST = () CORS_ORIGIN_ALLOW_ALL = True -CORS_ALLOW_HEADERS = corsheaders_default_headers + ( - 'use-jwt-cookie', -) LOGIN_REDIRECT_WHITELIST.extend([ CMS_BASE, diff --git a/lms/envs/docs/README.rst b/lms/envs/docs/README.rst index bf48cdedf684..9c6b1d8992b7 100644 --- a/lms/envs/docs/README.rst +++ b/lms/envs/docs/README.rst @@ -2,15 +2,10 @@ LMS Configuration Settings ========================== The lms.envs module contains project-wide settings, defined in python modules -using the standard `Django Settings`_ mechanism. +using the standard `Django Settings`_ mechanism, plus some Open edX +particularities, which we describe below. -.. _Django Settings: https://docs.djangoproject.com/en/2.2/topics/settings/ - -Different python modules are used for different setting configuration options. -To prevent duplication of settings, modules import values from other modules, -as shown in the diagram below. - -.. image:: images/lms_settings.png +.. _Django Settings: https://docs.djangoproject.com/en/dev/topics/settings/ YAML Configuration Files diff --git a/lms/envs/docs/images/lms_settings.png b/lms/envs/docs/images/lms_settings.png deleted file mode 100644 index cae36b5d70c8..000000000000 Binary files a/lms/envs/docs/images/lms_settings.png and /dev/null differ diff --git a/lms/envs/production.py b/lms/envs/production.py index d56a5631bb10..014cf59aa36a 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -22,7 +22,6 @@ import os import yaml -from corsheaders.defaults import default_headers as corsheaders_default_headers import django from django.core.exceptions import ImproperlyConfigured from edx_django_utils.plugins import add_plugins @@ -382,9 +381,6 @@ def get_env_setting(setting): CORS_ORIGIN_ALLOW_ALL = ENV_TOKENS.get('CORS_ORIGIN_ALLOW_ALL', False) CORS_ALLOW_INSECURE = ENV_TOKENS.get('CORS_ALLOW_INSECURE', False) - CORS_ALLOW_HEADERS = corsheaders_default_headers + ( - 'use-jwt-cookie', - ) # If setting a cross-domain cookie, it's really important to choose # a name for the cookie that is DIFFERENT than the cookies used @@ -520,11 +516,6 @@ def get_env_setting(setting): EMAIL_HOST_USER = AUTH_TOKENS.get('EMAIL_HOST_USER', '') # django default is '' EMAIL_HOST_PASSWORD = AUTH_TOKENS.get('EMAIL_HOST_PASSWORD', '') # django default is '' -############################### BLOCKSTORE ##################################### -BLOCKSTORE_API_URL = ENV_TOKENS.get('BLOCKSTORE_API_URL', None) # e.g. "https://blockstore.example.com/api/v1/" -# Configure an API auth token at (blockstore URL)/admin/authtoken/token/ -BLOCKSTORE_API_AUTH_TOKEN = AUTH_TOKENS.get('BLOCKSTORE_API_AUTH_TOKEN', None) - # Analytics API ANALYTICS_API_KEY = AUTH_TOKENS.get("ANALYTICS_API_KEY", ANALYTICS_API_KEY) ANALYTICS_API_URL = ENV_TOKENS.get("ANALYTICS_API_URL", ANALYTICS_API_URL) diff --git a/lms/envs/test.py b/lms/envs/test.py index 14c10e52d36d..3c4bb9564927 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -215,13 +215,6 @@ 'course_structure_cache': { 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', }, - # Blockstore caching tests require a cache that actually works: - 'blockstore': { - 'KEY_PREFIX': 'blockstore', - 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', - 'LOCATION': 'edx_loc_mem_cache', - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - }, } ############################# SECURITY SETTINGS ################################ @@ -546,16 +539,6 @@ derive_settings(__name__) -############################### BLOCKSTORE ##################################### -XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'blockstore' # This must be set to a working cache for the tests to pass -BUNDLE_ASSET_STORAGE_SETTINGS = dict( - STORAGE_CLASS='django.core.files.storage.FileSystemStorage', - STORAGE_KWARGS=dict( - location=MEDIA_ROOT, - base_url=MEDIA_URL, - ), -) - # Dummy secret key for dev SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' diff --git a/lms/templates/instructor/instructor_dashboard_2/membership.html b/lms/templates/instructor/instructor_dashboard_2/membership.html index 5a477665868a..9b1f64dfb9d1 100644 --- a/lms/templates/instructor/instructor_dashboard_2/membership.html +++ b/lms/templates/instructor/instructor_dashboard_2/membership.html @@ -229,8 +229,9 @@

    ${_("Course Team Management")}

    ${_("Discussion Admins can edit or delete any post, clear misuse flags, close " "and re-open threads, endorse responses, and see posts from all groups. " "Their posts are marked as 'staff'. They can also add and remove the " - "discussion moderation roles to manage course team membership. Only " - "enrolled users can be added as Discussion Admins.")}" + "discussion moderation roles to manage course team membership. Any users " + "not yet enrolled in the course will be automatically enrolled when added as " + "Discussion Admin")}" data-list-endpoint="${ section_data['list_forum_members_url'] }" data-modify-endpoint="${ section_data['update_forum_role_membership_url'] }" data-add-button-label="${_("Add Discussion Admin")}" @@ -257,8 +258,8 @@

    ${_("Course Team Management")}

    ${_("Discussion Moderators can edit or delete any post, clear misuse flags, close " "and re-open threads, endorse responses, and see posts from all groups. " "Their posts are marked as 'staff'. They cannot manage course team membership by " - "adding or removing discussion moderation roles. Only enrolled users can be " - "added as Discussion Moderators.")}" + "adding or removing discussion moderation roles. Any users not yet enrolled " + "in the course will be automatically enrolled when added as Discussion Moderator")}" data-list-endpoint="${ section_data['list_forum_members_url'] }" data-modify-endpoint="${ section_data['update_forum_role_membership_url'] }" data-add-button-label="${_("Add Moderator")}" @@ -271,8 +272,8 @@

    ${_("Course Team Management")}

    ${_("Group Community TAs are members of the community who help course teams moderate discussions. Group " "Community TAs see only posts by learners in their assigned group. They can edit or delete posts, " "clear flags, close and re-open threads, and endorse responses, but only for posts by learners in " - "their group. Their posts are marked as 'Community TA'. Only enrolled learners can be added as Group " - "Community TAs.")}" + "their group. Their posts are marked as 'Community TA'. Any users not yet enrolled " + "in the course will be automatically enrolled when added as Group Community TA")}" data-list-endpoint="${ section_data['list_forum_members_url'] }" data-modify-endpoint="${ section_data['update_forum_role_membership_url'] }" data-add-button-label="${_("Add Group Community TA")}" @@ -285,7 +286,7 @@

    ${_("Course Team Management")}

    ${_("Community TAs are members of the community who help course teams moderate discussions. " "They can see posts by learners in their assigned cohort or enrollment track, and can edit or delete posts, " "clear flags, close or re-open threads, and endorse responses. Their posts are marked as 'Community TA'. " - "Only enrolled learners can be added as Community TAs.")}" + "Any users not yet enrolled in the course will be automatically enrolled when added as Community TA")}" data-list-endpoint="${ section_data['list_forum_members_url'] }" data-modify-endpoint="${ section_data['update_forum_role_membership_url'] }" data-add-button-label="${_("Add Community TA")}" diff --git a/lms/urls.py b/lms/urls.py index c7a0d49da4d7..5ac6283fddc7 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -336,7 +336,7 @@ name='xblock_resource_url', ), - # New (Blockstore-based) XBlock REST API + # New (Learning-Core-based) XBlock REST API path('', include(('openedx.core.djangoapps.xblock.rest_api.urls', 'openedx.core.djangoapps.xblock'), namespace='xblock_api')), @@ -749,6 +749,11 @@ courseware_views.courseware_mfe_search_enabled, name='courseware_search_enabled_view', ), + re_path( + fr'^courses/{settings.COURSE_ID_PATTERN}/courseware-navigation-sidebar/toggles/$', + courseware_views.courseware_mfe_navigation_sidebar_toggles, + name='courseware_navigation_sidebar_toggles_view', + ), ] urlpatterns += [ diff --git a/lms/wsgi.py b/lms/wsgi.py index bb0d0f7ede8c..169b6929fbfa 100644 --- a/lms/wsgi.py +++ b/lms/wsgi.py @@ -18,13 +18,6 @@ import lms.startup as startup # lint-amnesty, pylint: disable=wrong-import-position startup.run() -from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=wrong-import-position - -# Trigger a forced initialization of our modulestores since this can take a -# while to complete and we want this done before HTTP requests are accepted. -modulestore() - - # This application object is used by the development server # as well as any WSGI server configured to use this file. from django.core.wsgi import get_wsgi_application # lint-amnesty, pylint: disable=wrong-import-order, wrong-import-position diff --git a/openedx/core/djangoapps/content/search/api.py b/openedx/core/djangoapps/content/search/api.py index 24d69dfd9fcf..bbde4fc98230 100644 --- a/openedx/core/djangoapps/content/search/api.py +++ b/openedx/core/djangoapps/content/search/api.py @@ -55,6 +55,8 @@ MAX_ACCESS_IDS_IN_FILTER = 1_000 MAX_ORGS_IN_FILTER = 1_000 +EXCLUDED_XBLOCK_TYPES = ['course', 'course_info'] + @contextmanager def _index_rebuild_lock() -> Generator[str, None, None]: @@ -372,6 +374,7 @@ def add_with_children(block): docs.append(doc) # pylint: disable=cell-var-from-loop _recurse_children(block, add_with_children) # pylint: disable=cell-var-from-loop + # Index course children _recurse_children(course, add_with_children) if docs: @@ -393,6 +396,10 @@ def upsert_xblock_index_doc(usage_key: UsageKey, recursive: bool = True) -> None recursive (bool): If True, also index all children of the XBlock """ xblock = modulestore().get_item(usage_key) + xblock_type = xblock.scope_ids.block_type + + if xblock_type in EXCLUDED_XBLOCK_TYPES: + return docs = [] diff --git a/openedx/core/djangoapps/content/search/tests/test_api.py b/openedx/core/djangoapps/content/search/tests/test_api.py index f156f9bbb22e..cd1acc31b7d9 100644 --- a/openedx/core/djangoapps/content/search/tests/test_api.py +++ b/openedx/core/djangoapps/content/search/tests/test_api.py @@ -6,6 +6,7 @@ import copy from unittest.mock import MagicMock, call, patch +from opaque_keys.edx.keys import UsageKey import ddt from django.test import override_settings @@ -62,6 +63,7 @@ def setUp(self): fields={"display_name": "Test Course"}, ) course_access, _ = SearchAccess.objects.get_or_create(context_key=self.course.id) + self.course_block_key = "block-v1:org1+test_course+test_run+type@course+block@course" # Create XBlocks self.sequential = self.store.create_child(self.user_id, self.course.location, "sequential", "test_sequential") @@ -184,6 +186,12 @@ def test_index_xblock_metadata(self, recursive, mock_meilisearch): mock_meilisearch.return_value.index.return_value.update_documents.assert_called_once_with(expected_docs) + @override_settings(MEILISEARCH_ENABLED=True) + def test_no_index_excluded_xblocks(self, mock_meilisearch): + api.upsert_xblock_index_doc(UsageKey.from_string(self.course_block_key)) + + mock_meilisearch.return_value.index.return_value.update_document.assert_not_called() + @override_settings(MEILISEARCH_ENABLED=True) def test_index_xblock_tags(self, mock_meilisearch): """ diff --git a/openedx/core/djangoapps/content/search/tests/test_handlers.py b/openedx/core/djangoapps/content/search/tests/test_handlers.py index 7f209b4ca601..1ce9c57a1ab9 100644 --- a/openedx/core/djangoapps/content/search/tests/test_handlers.py +++ b/openedx/core/djangoapps/content/search/tests/test_handlers.py @@ -9,7 +9,6 @@ from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangoapps.content_libraries import api as library_api from openedx.core.djangolib.testing.utils import skip_unless_cms -from openedx.core.lib.blockstore_api.tests.base import BlockstoreAppTestMixin from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE, ModuleStoreTestCase @@ -25,11 +24,7 @@ @patch("openedx.core.djangoapps.content.search.api.MeilisearchClient") @override_settings(MEILISEARCH_ENABLED=True) @skip_unless_cms -class TestUpdateIndexHandlers( - ModuleStoreTestCase, - BlockstoreAppTestMixin, - LiveServerTestCase, -): +class TestUpdateIndexHandlers(ModuleStoreTestCase, LiveServerTestCase): """ Test that the search index is updated when XBlocks and Library Blocks are modified """ diff --git a/openedx/core/djangoapps/content_libraries/api.py b/openedx/core/djangoapps/content_libraries/api.py index 3e276a37abf0..3c47e0d2605f 100644 --- a/openedx/core/djangoapps/content_libraries/api.py +++ b/openedx/core/djangoapps/content_libraries/api.py @@ -94,7 +94,7 @@ from xblock.exceptions import XBlockNotFoundError from openedx.core.djangoapps.xblock.api import get_component_from_usage_key, xblock_type_display_name -from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_blockstore +from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_learning_core from xmodule.library_root_xblock import LibraryRoot as LibraryRootV1 from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore @@ -336,6 +336,7 @@ def get_library(library_key): # something that we should remove. It exists to accomodate some complexities # with how Blockstore staged changes, but Learning Core works differently, # and has_unpublished_changes should be sufficient. + # Ref: https://github.com/openedx/edx-platform/issues/34283 has_unpublished_deletes = publishing_api.get_entities_with_unpublished_deletes(learning_package.id) \ .exists() @@ -1012,10 +1013,10 @@ def get_v1_or_v2_library( library = get_library(library_key) if v2_version is not None and library.version != v2_version: raise NotImplementedError( - f"Tried to load version {v2_version} of blockstore-based library {library_key}. " + f"Tried to load version {v2_version} of learning_core-based library {library_key}. " f"Currently, only the latest version ({library.version}) may be loaded. " "This is a known issue. " - "It will be fixed before the production release of blockstore-based (V2) content libraries. " + "It will be fixed before the production release of learning_core-based (V2) content libraries. " ) return library except ContentLibrary.DoesNotExist: @@ -1121,35 +1122,34 @@ def import_block(self, modulestore_key): modulestore_key.block_type, block_id, ) - blockstore_key = library_block.usage_key + dest_key = library_block.usage_key except LibraryBlockAlreadyExists: - blockstore_key = LibraryUsageLocatorV2( + dest_key = LibraryUsageLocatorV2( lib_key=self.library.library_key, block_type=modulestore_key.block_type, usage_id=block_id, ) - get_library_block(blockstore_key) + get_library_block(dest_key) log.warning('Library block already exists: Appending static files ' - 'and overwriting OLX: %s', str(blockstore_key)) + 'and overwriting OLX: %s', str(dest_key)) # Handle static files. files = [ f.path for f in - get_library_block_static_asset_files(blockstore_key) + get_library_block_static_asset_files(dest_key) ] for filename, static_file in block_data.get('static_files', {}).items(): if filename in files: # Files already added, move on. continue file_content = self.get_block_static_data(static_file) - add_library_block_static_asset_file( - blockstore_key, filename, file_content) + add_library_block_static_asset_file(dest_key, filename, file_content) files.append(filename) # Import OLX. - set_library_block_olx(blockstore_key, block_data['olx']) + set_library_block_olx(dest_key, block_data['olx']) def import_blocks_from_course(self, course_key, progress_callback): """ @@ -1200,7 +1200,7 @@ def get_block_data(self, block_key): Get block OLX by serializing it from modulestore directly. """ block = self.modulestore.get_item(block_key) - data = serialize_modulestore_block_for_blockstore(block) + data = serialize_modulestore_block_for_learning_core(block) return {'olx': data.olx_str, 'static_files': {s.name: s for s in data.static_files}} diff --git a/openedx/core/djangoapps/content_libraries/apps.py b/openedx/core/djangoapps/content_libraries/apps.py index 685e9259b6e2..52c3e5179721 100644 --- a/openedx/core/djangoapps/content_libraries/apps.py +++ b/openedx/core/djangoapps/content_libraries/apps.py @@ -16,7 +16,7 @@ class ContentLibrariesConfig(AppConfig): """ name = 'openedx.core.djangoapps.content_libraries' - verbose_name = 'Content Libraries (Blockstore-based)' + verbose_name = 'Content Libraries (Learning-Core-based)' # This is designed as a plugin for now so that # the whole thing is self-contained and can easily be enabled/disabled plugin_app = { diff --git a/openedx/core/djangoapps/content_libraries/constants.py b/openedx/core/djangoapps/content_libraries/constants.py index 0a0614e514d9..9505d52d1cca 100644 --- a/openedx/core/djangoapps/content_libraries/constants.py +++ b/openedx/core/djangoapps/content_libraries/constants.py @@ -1,9 +1,6 @@ """ Constants used for the content libraries. """ from django.utils.translation import gettext_lazy as _ -# ./api.py and ./views.py are only used in Studio, so we always work with this draft of any -# content library bundle: -DRAFT_NAME = 'studio_draft' VIDEO = 'video' COMPLEX = 'complex' diff --git a/openedx/core/djangoapps/content_libraries/library_context.py b/openedx/core/djangoapps/content_libraries/library_context.py index 9408f51e511a..93de022474b0 100644 --- a/openedx/core/djangoapps/content_libraries/library_context.py +++ b/openedx/core/djangoapps/content_libraries/library_context.py @@ -19,7 +19,7 @@ class LibraryContextImpl(LearningContext): """ Implements content libraries as a learning context. - This is the *new* content libraries based on Blockstore, not the old content + This is the *new* content libraries based on Learning Core, not the old content libraries based on modulestore. """ diff --git a/openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py b/openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py index 9b57b42e6aec..cd68112d2b67 100644 --- a/openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py +++ b/openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py @@ -89,7 +89,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): """ Collect all blocks from a course that are "importable" and write them to the - a blockstore library. + a learning core library. """ # Search for the library. diff --git a/openedx/core/djangoapps/content_libraries/migrations/0008_auto_20210818_2148.py b/openedx/core/djangoapps/content_libraries/migrations/0008_auto_20210818_2148.py index fb15e4a0faf2..6c368dec24c4 100644 --- a/openedx/core/djangoapps/content_libraries/migrations/0008_auto_20210818_2148.py +++ b/openedx/core/djangoapps/content_libraries/migrations/0008_auto_20210818_2148.py @@ -40,7 +40,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='ltigradedresource', name='usage_key', - field=opaque_keys.edx.django.models.UsageKeyField(help_text='The usage key string of the blockstore resource serving the content of this launch.', max_length=255), + field=opaque_keys.edx.django.models.UsageKeyField(help_text='The usage key string of the resource serving the content of this launch.', max_length=255), ), migrations.AlterField( model_name='ltiprofile', diff --git a/openedx/core/djangoapps/content_libraries/models.py b/openedx/core/djangoapps/content_libraries/models.py index a4c128c9bcc1..58a9f6b6863f 100644 --- a/openedx/core/djangoapps/content_libraries/models.py +++ b/openedx/core/djangoapps/content_libraries/models.py @@ -8,7 +8,7 @@ LTI 1.3 Models ============== -Content Libraries serves blockstore-based content through LTI 1.3 launches. +Content Libraries serves learning-core-based content through LTI 1.3 launches. The interface supports resource link launches and grading services. Two use cases justify the current data model to support LTI launches. They are: @@ -27,7 +27,7 @@ The data model above is similar to the one provided by the current LTI 1.1 implementation for modulestore and courseware content. But, Content Libraries is orthogonal. Its use-case is to offer standalone, embedded content from a -specific backend (blockstore). As such, it decouples from LTI 1.1. and the +specific backend (learning core). As such, it decouples from LTI 1.1. and the logic assume no relationship or impact across the two applications. The same reasoning applies to steps beyond the data model, such as at the XBlock runtime, authentication, and score handling, etc. @@ -85,9 +85,9 @@ class ContentLibrary(models.Model): """ A Content Library is a collection of content (XBlocks and/or static assets) - All actual content is stored in Blockstore, and any data that we'd want to + All actual content is stored in Learning Core, and any data that we'd want to transfer to another instance if this library were exported and then - re-imported on another Open edX instance should be kept in Blockstore. This + re-imported on another Open edX instance should be kept in Learning Core. This model in Studio should only be used to track settings specific to this Open edX instance, like who has permission to edit this content library. """ @@ -479,7 +479,7 @@ class LtiGradedResource(models.Model): usage_key = UsageKeyField( max_length=255, - help_text=_('The usage key string of the blockstore resource serving the ' + help_text=_('The usage key string of the resource serving the ' 'content of this launch.'), ) diff --git a/openedx/core/djangoapps/content_libraries/permissions.py b/openedx/core/djangoapps/content_libraries/permissions.py index 3c41e0574c36..c7da012c9fb4 100644 --- a/openedx/core/djangoapps/content_libraries/permissions.py +++ b/openedx/core/djangoapps/content_libraries/permissions.py @@ -1,5 +1,5 @@ """ -Permissions for Content Libraries (v2, Blockstore-based) +Permissions for Content Libraries (v2, Learning-Core-based) """ from bridgekeeper import perms, rules from bridgekeeper.rules import Attribute, ManyRelation, Relation, in_current_groups diff --git a/openedx/core/djangoapps/content_libraries/serializers.py b/openedx/core/djangoapps/content_libraries/serializers.py index ee0e48b59c87..13c6a756fd31 100644 --- a/openedx/core/djangoapps/content_libraries/serializers.py +++ b/openedx/core/djangoapps/content_libraries/serializers.py @@ -14,7 +14,6 @@ from openedx.core.djangoapps.content_libraries.models import ( ContentLibraryPermission, ContentLibraryBlockImportTask ) -from openedx.core.lib import blockstore_api from openedx.core.lib.api.serializers import CourseKeyField @@ -175,16 +174,6 @@ class LibraryXBlockStaticFileSerializer(serializers.Serializer): url = serializers.URLField() size = serializers.IntegerField(min_value=0) - def to_representation(self, instance): - """ - Generate the serialized representation of this static asset file. - """ - result = super().to_representation(instance) - # Make sure the URL is one that will work from the user's browser, - # not one that only works from within a docker container: - result['url'] = blockstore_api.force_browser_url(result['url']) - return result - class LibraryXBlockStaticFilesSerializer(serializers.Serializer): """ diff --git a/openedx/core/djangoapps/content_libraries/tasks.py b/openedx/core/djangoapps/content_libraries/tasks.py index 3714dc55f8d6..9f4f7aaaf7dc 100644 --- a/openedx/core/djangoapps/content_libraries/tasks.py +++ b/openedx/core/djangoapps/content_libraries/tasks.py @@ -4,11 +4,11 @@ Architecture note: Several functions in this file manage the copying/updating of blocks in modulestore - and blockstore. These operations should only be performed within the context of CMS. + and learning core. These operations should only be performed within the context of CMS. However, due to existing edx-platform code structure, we've had to define the functions in shared source tree (openedx/) and the tasks are registered in both LMS and CMS. - To ensure that we're not accidentally importing things from blockstore in the LMS context, + To ensure that we're not accidentally importing things from learning core in the LMS context, we use ensure_cms throughout this module. A longer-term solution to this issue would be to move the content_libraries app to cms: @@ -39,7 +39,7 @@ from opaque_keys.edx.keys import CourseKey from openedx.core.djangoapps.content_libraries import api as library_api from openedx.core.djangoapps.xblock.api import load_block -from openedx.core.lib import ensure_cms, blockstore_api +from openedx.core.lib import ensure_cms from xmodule.capa_block import ProblemBlock from xmodule.library_content_block import ANY_CAPA_TYPE_VALUE, LibraryContentBlock from xmodule.library_root_xblock import LibraryRoot as LibraryRootV1 @@ -86,7 +86,7 @@ def on_progress(block_key, block_num, block_count, exception=None): def _import_block(store, user_id, source_block, dest_parent_key): """ - Recursively import a blockstore block and its children.` + Recursively import a learning core block and its children.` """ def generate_block_key(source_key, dest_parent_key): """ @@ -127,7 +127,7 @@ def generate_block_key(source_key, dest_parent_key): # Prepare a list of this block's static assets; any assets that are referenced as /static/{path} (the # recommended way for referencing them) will stop working, and so we rewrite the url when importing. - # Copying assets not advised because modulestore doesn't namespace assets to each block like blockstore, which + # Copying assets not advised because modulestore doesn't namespace assets to each block like learning core, which # might cause conflicts when the same filename is used across imported blocks. if isinstance(source_key, LibraryUsageLocatorV2): all_assets = library_api.get_library_block_static_asset_files(source_key) @@ -139,12 +139,6 @@ def generate_block_key(source_key, dest_parent_key): continue # Only copy authored field data if field.is_set_on(source_block) or field.is_set_on(new_block): field_value = getattr(source_block, field_name) - if isinstance(field_value, str): - # If string field (which may also be JSON/XML data), rewrite /static/... URLs to point to blockstore - for asset in all_assets: - field_value = field_value.replace(f'/static/{asset.path}', asset.url) - # Make sure the URL is one that will work from the user's browser when using the docker devstack - field_value = blockstore_api.force_browser_url(field_value) setattr(new_block, field_name, field_value) new_block.save() store.update_item(new_block, user_id) @@ -178,9 +172,9 @@ def _problem_type_filter(store, library, capa_type): return [key for key in library.children if _filter_child(store, key, capa_type)] -def _import_from_blockstore(user_id, store, dest_block, blockstore_block_ids): +def _import_from_learning_core(user_id, store, dest_block, source_block_ids): """ - Imports a block from a blockstore-based learning context (usually a + Imports a block from a learning-core-based learning context (usually a content library) into modulestore, as a new child of dest_block. Any existing children of dest_block are replaced. """ @@ -190,7 +184,7 @@ def _import_from_blockstore(user_id, store, dest_block, blockstore_block_ids): if user_id is None: raise ValueError("Cannot check user permissions - LibraryTools user_id is None") - if len(set(blockstore_block_ids)) != len(blockstore_block_ids): + if len(set(source_block_ids)) != len(source_block_ids): # We don't support importing the exact same block twice because it would break the way we generate new IDs # for each block and then overwrite existing copies of blocks when re-importing the same blocks. raise ValueError("One or more library component IDs is a duplicate.") @@ -204,7 +198,7 @@ def _import_from_blockstore(user_id, store, dest_block, blockstore_block_ids): # (This could be slow and use lots of memory, except for the fact that LibraryContentBlock which calls this # should be limiting the number of blocks to a reasonable limit. We load them all now instead of one at a # time in order to raise any errors before we start actually copying blocks over.) - orig_blocks = [load_block(UsageKey.from_string(key), user) for key in blockstore_block_ids] + orig_blocks = [load_block(UsageKey.from_string(key), user) for key in source_block_ids] with store.bulk_operations(dest_course_key): child_ids_updated = set() @@ -347,7 +341,7 @@ def _sync_children( str(library_api.LibraryXBlockMetadata.from_component(library_key, component).usage_key) for component in library_api.get_library_components(library_key) ] - _import_from_blockstore(user_id, store, dest_block, source_block_ids) + _import_from_learning_core(user_id, store, dest_block, source_block_ids) dest_block.source_library_version = str(library.version) store.update_item(dest_block, user_id) except Exception as exception: # pylint: disable=broad-except diff --git a/openedx/core/djangoapps/content_libraries/tests/base.py b/openedx/core/djangoapps/content_libraries/tests/base.py index 5f837628c935..2bb94e3d87c7 100644 --- a/openedx/core/djangoapps/content_libraries/tests/base.py +++ b/openedx/core/djangoapps/content_libraries/tests/base.py @@ -1,5 +1,5 @@ """ -Tests for Blockstore-based Content Libraries +Tests for Learning-Core-based Content Libraries """ import uuid from contextlib import contextmanager @@ -12,9 +12,6 @@ from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangoapps.content_libraries.constants import COMPLEX, ALL_RIGHTS_RESERVED from openedx.core.djangolib.testing.utils import skip_unless_cms -from openedx.core.lib.blockstore_api.tests.base import ( - BlockstoreAppTestMixin, -) # Define the URLs here - don't use reverse() because we want to detect # backwards-incompatible changes like changed URLs. @@ -46,9 +43,9 @@ @skip_unless_cms # Content Libraries REST API is only available in Studio -class ContentLibrariesRestApiTest(BlockstoreAppTestMixin, APITransactionTestCase): +class ContentLibrariesRestApiTest(APITransactionTestCase): """ - Base class for Blockstore-based Content Libraries test that use the REST API + Base class for Learning-Core-based Content Libraries test that use the REST API These tests use the REST API, which in turn relies on the Python API. Some tests may use the python API directly if necessary to provide diff --git a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py index c23e728c4b4d..67933f296413 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py @@ -1,5 +1,5 @@ """ -Tests for Blockstore-based Content Libraries +Tests for Learning-Core-based Content Libraries """ from unittest.mock import Mock, patch from unittest import skip @@ -37,7 +37,7 @@ @ddt.ddt class ContentLibrariesTestCase(ContentLibrariesRestApiTest, OpenEdxEventsTestMixin): """ - General tests for Blockstore-based Content Libraries + General tests for Learning-Core-based Content Libraries These tests use the REST API, which in turn relies on the Python API. Some tests may use the python API directly if necessary to provide @@ -278,7 +278,7 @@ def test_library_blocks(self):

    This is a normal capa problem with unicode 🔥. It has "maximum attempts" set to **5**.

    - + XBlock metadata only XBlock data/metadata and associated static asset files @@ -300,7 +300,7 @@ def test_library_blocks(self): # Now view the XBlock's student_view (including draft changes): fragment = self._render_block_view(block_id, "student_view") assert 'resources' in fragment - assert 'Blockstore is designed to store.' in fragment['content'] + assert 'Learning Core is designed to store.' in fragment['content'] # Also call a handler to make sure that's working: handler_url = self._get_block_handler_url(block_id, "xmodule_handler") + "problem_get" @@ -806,7 +806,7 @@ def test_library_block_olx_update_event(self):

    This is a normal capa problem with unicode 🔥. It has "maximum attempts" set to **5**.

    - + XBlock metadata only XBlock data/metadata and associated static asset files @@ -956,7 +956,7 @@ def test_library_block_delete_event(self): @ddt.ddt class ContentLibraryXBlockValidationTest(APITestCase): - """Tests only focused on service validation, no Blockstore needed.""" + """Tests only focused on service validation, no Learning Core interactions here.""" @ddt.data( (URL_BLOCK_METADATA_URL, dict(block_key='totally_invalid_key')), diff --git a/openedx/core/djangoapps/content_libraries/tests/test_runtime.py b/openedx/core/djangoapps/content_libraries/tests/test_runtime.py index f35ba7ea7b90..89b8cdefd86b 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_runtime.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_runtime.py @@ -1,5 +1,5 @@ """ -Test the Blockstore-based XBlock runtime and content libraries together. +Test the Learning-Core-based XBlock runtime and content libraries together. """ import json from gettext import GNUTranslations @@ -14,7 +14,6 @@ from lms.djangoapps.courseware.model_data import get_score from openedx.core.djangoapps.content_libraries import api as library_api from openedx.core.djangoapps.content_libraries.tests.base import ( - BlockstoreAppTestMixin, URL_BLOCK_RENDER_VIEW, URL_BLOCK_GET_HANDLER_URL, URL_BLOCK_METADATA_URL, @@ -60,9 +59,9 @@ def setUp(self): ) -class ContentLibraryRuntimeTestMixin(ContentLibraryContentTestMixin): +class ContentLibraryRuntimeTests(ContentLibraryContentTestMixin): """ - Basic tests of the Blockstore-based XBlock runtime using XBlocks in a + Basic tests of the Learning-Core-based XBlock runtime using XBlocks in a content library. """ @@ -95,7 +94,7 @@ def test_xblock_metadata(self):

    This is a normal capa problem. It has "maximum attempts" set to **5**.

    - + XBlock metadata only XBlock data/metadata and associated static asset files @@ -174,18 +173,10 @@ def test_xblock_fields(self): assert block_saved.display_name == 'New Display Name' -class ContentLibraryRuntimeTest(ContentLibraryRuntimeTestMixin, BlockstoreAppTestMixin): - """ - Tests XBlock runtime using XBlocks in a content library using the installed Blockstore app. - - We run this test with a live server, so that the blockstore asset files can be served. - """ - - # We can remove the line below to enable this in Studio once we implement a session-backed # field data store which we can use for both studio users and anonymous users @skip_unless_lms -class ContentLibraryXBlockUserStateTestMixin(ContentLibraryContentTestMixin): +class ContentLibraryXBlockUserStateTest(ContentLibraryContentTestMixin): """ Test that the Blockstore-based XBlock runtime can store and retrieve student state for XBlocks when learners access blocks directly in a library context, @@ -389,7 +380,7 @@ def test_scores_persisted(self):

    This is a normal capa problem. It has "maximum attempts" set to **5**.

    - + XBlock metadata only XBlock data/metadata and associated static asset files @@ -453,7 +444,7 @@ def test_i18n(self):

    This is a normal capa problem. It has "maximum attempts" set to **5**.

    - + XBlock metadata only XBlock data/metadata and associated static asset files @@ -487,19 +478,8 @@ def test_i18n(self): assert 'Submit' not in dummy_public_view.data['content'] -class ContentLibraryXBlockUserStateTest( # type: ignore[misc] - ContentLibraryXBlockUserStateTestMixin, - BlockstoreAppTestMixin, -): - """ - Tests XBlock user state for XBlocks in a content library using the installed Blockstore app. - - We run this test with a live server, so that the blockstore asset files can be served. - """ - - @skip_unless_lms # No completion tracking in Studio -class ContentLibraryXBlockCompletionTestMixin(ContentLibraryContentTestMixin, CompletionWaffleTestMixin): +class ContentLibraryXBlockCompletionTest(ContentLibraryContentTestMixin, CompletionWaffleTestMixin): """ Test that the Blockstore-based XBlocks can track their completion status using the completion library. @@ -550,16 +530,3 @@ def get_block_completion_status(): # Now the block is completed assert get_block_completion_status() == 1 - - -class ContentLibraryXBlockCompletionTest( - ContentLibraryXBlockCompletionTestMixin, - CompletionWaffleTestMixin, - BlockstoreAppTestMixin, -): - """ - Test that the Blockstore-based XBlocks can track their completion status - using the installed Blockstore app. - - We run this test with a live server, so that the blockstore asset files can be served. - """ diff --git a/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py b/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py index 6a75d63110b8..92ff4c1767d0 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py @@ -1,5 +1,5 @@ """ -Tests for static asset files in Blockstore-based Content Libraries +Tests for static asset files in Learning-Core-based Content Libraries """ from unittest import skip @@ -26,7 +26,7 @@ @skip("Assets are being reimplemented in Learning Core. Disable until that's ready.") class ContentLibrariesStaticAssetsTest(ContentLibrariesRestApiTest): """ - Tests for static asset files in Blockstore-based Content Libraries + Tests for static asset files in Learning-Core-based Content Libraries WARNING: every test should have a unique library slug, because even though the django/mysql database gets reset for each test case, the lookup between @@ -65,7 +65,7 @@ def test_asset_filenames(self): def test_video_transcripts(self): """ - Test that video blocks can read transcript files out of blockstore. + Test that video blocks can read transcript files out of learning core. """ library = self._create_library(slug="transcript-test-lib", title="Transcripts Test Library") block = self._add_block_to_library(library["id"], "video", "video1") @@ -104,7 +104,7 @@ def check_download(): check_sjson() check_download() # Publish the OLX and the transcript file, since published data gets - # served differently by Blockstore and we should test that too. + # served differently by Learning Core and we should test that too. self._commit_library_changes(library["id"]) check_sjson() check_download() diff --git a/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py b/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py index cb306ebbfe48..a25d02761dde 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py @@ -87,5 +87,5 @@ class LibraryBlockLtiUrlViewTest( ContentLibrariesRestApiTest, ): """ - Test generating LTI URL for a block in a library, using the installed Blockstore app. + Test generating LTI URL for a block in a library, using the installed Learning Core app. """ diff --git a/openedx/core/djangoapps/content_libraries/views.py b/openedx/core/djangoapps/content_libraries/views.py index 0df670b1aea6..38f3e7efd6a1 100644 --- a/openedx/core/djangoapps/content_libraries/views.py +++ b/openedx/core/djangoapps/content_libraries/views.py @@ -58,7 +58,8 @@ block. Historical note: These views used to be wrapped with @atomic because we - wanted to make all views that operated on Blockstore data atomic: + wanted to make all views that operated on Blockstore (the predecessor + to Learning Core) atomic: https://github.com/openedx/edx-platform/pull/30456 """ @@ -258,6 +259,7 @@ def post(self, request): # Learning Core. TODO: This can be removed once the frontend stops # sending it to us. This whole bit of deserialization is kind of weird # though, with the renames and such. Look into this later for clennup. + # Ref: https://github.com/openedx/edx-platform/issues/34283 data.pop("collection_uuid", None) try: @@ -708,9 +710,12 @@ def put(self, request, usage_key_str, file_path): ) file_wrapper = request.data['content'] if file_wrapper.size > 20 * 1024 * 1024: # > 20 MiB - # In the future, we need a way to use file_wrapper.chunks() to read - # the file in chunks and stream that to Blockstore, but Blockstore - # currently lacks an API for streaming file uploads. + # TODO: This check was written when V2 Libraries were backed by the Blockstore micro-service. + # Now that we're on Learning Core, do we still need it? Here's the original comment: + # In the future, we need a way to use file_wrapper.chunks() to read + # the file in chunks and stream that to Blockstore, but Blockstore + # currently lacks an API for streaming file uploads. + # Ref: https://github.com/openedx/edx-platform/issues/34737 raise ValidationError("File too big") file_content = file_wrapper.read() try: diff --git a/openedx/core/djangoapps/content_tagging/handlers.py b/openedx/core/djangoapps/content_tagging/handlers.py index 74b4dd5b3331..cc86f7e0dcd6 100644 --- a/openedx/core/djangoapps/content_tagging/handlers.py +++ b/openedx/core/djangoapps/content_tagging/handlers.py @@ -67,9 +67,14 @@ def auto_tag_xblock(**kwargs): if not CONTENT_TAGGING_AUTO.is_enabled(xblock_info.usage_key.course_key): return + if xblock_info.block_type == 'course_info': + # We want to add tags only to the course id, not with its XBlock + return + if xblock_info.block_type == "course": # Course update is handled by XBlock of course type update_course_tags.delay(str(xblock_info.usage_key.course_key)) + return update_xblock_tags.delay(str(xblock_info.usage_key)) diff --git a/openedx/core/djangoapps/content_tagging/rules.py b/openedx/core/djangoapps/content_tagging/rules.py index 265194860159..72b744f5bf8e 100644 --- a/openedx/core/djangoapps/content_tagging/rules.py +++ b/openedx/core/djangoapps/content_tagging/rules.py @@ -273,6 +273,30 @@ def can_view_object_tag_objectid(user: UserType, object_id: str) -> bool: return bool(object_org) and (is_org_admin(user, object_org) or is_org_user(user, object_org)) +@rules.predicate +def can_remove_object_tag_objectid(user: UserType, object_id: str) -> bool: + """ + Everyone that has permission to edit the object should be able remove tags from it. + """ + if not object_id: + raise ValueError("object_id must be provided") + + if not user.is_authenticated: + return False + + try: + context_key = get_context_key_from_key_string(object_id) + assert context_key.org + except (ValueError, AssertionError): + return False + + if has_studio_write_access(user, context_key): + return True + + object_org = rules_cache.get_orgs([context_key.org]) + return bool(object_org) and is_org_admin(user, object_org) + + @rules.predicate def can_change_object_tag( user: UserType, perm_obj: oel_tagging.ObjectTagPermissionItem | None = None @@ -336,3 +360,4 @@ def can_change_taxonomy_tag(user: UserType, tag: oel_tagging.Tag | None = None) rules.set_perm("oel_tagging.view_objecttag_objectid", can_view_object_tag_objectid) rules.set_perm("oel_tagging.change_objecttag_taxonomy", can_view_object_tag_taxonomy) rules.set_perm("oel_tagging.change_objecttag_objectid", can_change_object_tag_objectid) +rules.set_perm("oel_tagging.remove_objecttag_objectid", can_remove_object_tag_objectid) diff --git a/openedx/core/djangoapps/content_tagging/tests/test_tasks.py b/openedx/core/djangoapps/content_tagging/tests/test_tasks.py index ed0cf2c06025..c14adfcce13a 100644 --- a/openedx/core/djangoapps/content_tagging/tests/test_tasks.py +++ b/openedx/core/djangoapps/content_tagging/tests/test_tasks.py @@ -8,14 +8,13 @@ from django.test import override_settings, LiveServerTestCase from django.http import HttpRequest from edx_toggles.toggles.testutils import override_waffle_flag -from openedx_tagging.core.tagging.models import Tag, Taxonomy +from openedx_tagging.core.tagging.models import Tag, Taxonomy, ObjectTag from organizations.models import Organization from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangolib.testing.utils import skip_unless_cms from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE, ModuleStoreTestCase from openedx.core.djangoapps.content_libraries.api import create_library, create_library_block, delete_library_block -from openedx.core.lib.blockstore_api.tests.base import BlockstoreAppTestMixin from .. import api from ..models.base import TaxonomyOrg @@ -59,7 +58,6 @@ def setUp(self): class TestAutoTagging( # type: ignore[misc] LanguageTaxonomyTestMixin, ModuleStoreTestCase, - BlockstoreAppTestMixin, LiveServerTestCase ): """ @@ -111,6 +109,23 @@ def test_create_course(self): # Check if the tags are created in the Course assert self._check_tag(course.id, LANGUAGE_TAXONOMY_ID, "Polski") + def test_only_tag_course_id(self): + # Create course + course = self.store.create_course( + self.orgA.short_name, + "test_course", + "test_run", + self.user_id, + fields={"language": "pl"}, + ) + object_id = str(course.id).replace('course-v1:', '') + + # Check that only one object tag is created for the course + tags = ObjectTag.objects.filter(object_id__contains=object_id) + assert len(tags) == 1 + assert tags[0].value == "Polski" + assert tags[0].object_id == str(course.id) + @override_settings(LANGUAGE_CODE='pt-br') def test_create_course_invalid_language(self): # Create course diff --git a/openedx/core/djangoapps/content_tagging/toggles.py b/openedx/core/djangoapps/content_tagging/toggles.py index 30a21cf77e51..3162a529833c 100644 --- a/openedx/core/djangoapps/content_tagging/toggles.py +++ b/openedx/core/djangoapps/content_tagging/toggles.py @@ -1,12 +1,14 @@ - """ Toggles for content tagging """ +from edx_toggles.toggles import WaffleFlag + from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag + # .. toggle_name: content_tagging.auto -# .. toggle_implementation: WaffleSwitch +# .. toggle_implementation: CourseWaffleFlag # .. toggle_default: False # .. toggle_description: Setting this enables automatic tagging of content # .. toggle_type: feature_flag @@ -15,3 +17,21 @@ # .. toggle_creation_date: 2023-08-30 # .. toggle_tickets: https://github.com/openedx/modular-learning/issues/79 CONTENT_TAGGING_AUTO = CourseWaffleFlag('content_tagging.auto', __name__) + + +# .. toggle_name: content_tagging.disabled +# .. toggle_implementation: WaffleFlag +# .. toggle_default: False +# .. toggle_description: Setting this disables the tagging feature +# .. toggle_type: feature_flag +# .. toggle_category: admin +# .. toggle_use_cases: open_edx +# .. toggle_creation_date: 2024-04-30 +DISABLE_TAGGING_FEATURE = WaffleFlag('content_tagging.disabled', __name__) + + +def is_tagging_feature_disabled(): + """ + Returns a boolean if tagging feature list page is disabled + """ + return DISABLE_TAGGING_FEATURE.is_enabled() diff --git a/openedx/core/djangoapps/contentserver/middleware.py b/openedx/core/djangoapps/contentserver/middleware.py index 12c63365a475..f159a00a56ba 100644 --- a/openedx/core/djangoapps/contentserver/middleware.py +++ b/openedx/core/djangoapps/contentserver/middleware.py @@ -15,6 +15,7 @@ HttpResponsePermanentRedirect ) from django.utils.deprecation import MiddlewareMixin +from edx_django_utils.monitoring import set_custom_attribute from opaque_keys import InvalidKeyError from opaque_keys.edx.locator import AssetLocator @@ -30,10 +31,6 @@ from .models import CdnUserAgentsConfig, CourseAssetCacheTtlConfig log = logging.getLogger(__name__) -try: - import newrelic.agent -except ImportError: - newrelic = None # pylint: disable=invalid-name # TODO: Soon as we have a reasonable way to serialize/deserialize AssetKeys, we need @@ -99,18 +96,17 @@ def process_request(self, request): if safe_course_key.run is None: safe_course_key = safe_course_key.replace(run='only') - if newrelic: - newrelic.agent.add_custom_attribute('course_id', safe_course_key) - newrelic.agent.add_custom_attribute('org', loc.org) - newrelic.agent.add_custom_attribute('contentserver.path', loc.path) + set_custom_attribute('course_id', safe_course_key) + set_custom_attribute('org', loc.org) + set_custom_attribute('contentserver.path', loc.path) - # Figure out if this is a CDN using us as the origin. - is_from_cdn = StaticContentServer.is_cdn_request(request) - newrelic.agent.add_custom_attribute('contentserver.from_cdn', is_from_cdn) + # Figure out if this is a CDN using us as the origin. + is_from_cdn = StaticContentServer.is_cdn_request(request) + set_custom_attribute('contentserver.from_cdn', is_from_cdn) - # Check if this content is locked or not. - locked = self.is_content_locked(content) - newrelic.agent.add_custom_attribute('contentserver.locked', locked) + # Check if this content is locked or not. + locked = self.is_content_locked(content) + set_custom_attribute('contentserver.locked', locked) # Check that user has access to the content. if not self.is_user_authorized(request, content, loc): @@ -168,8 +164,7 @@ def process_request(self, request): response['Content-Length'] = str(last - first + 1) response.status_code = 206 # Partial Content - if newrelic: - newrelic.agent.add_custom_attribute('contentserver.ranged', True) + set_custom_attribute('contentserver.ranged', True) else: log.warning( "Cannot satisfy ranges in Range header: %s for content: %s", @@ -182,9 +177,8 @@ def process_request(self, request): response = HttpResponse(content.stream_data()) response['Content-Length'] = content.length - if newrelic: - newrelic.agent.add_custom_attribute('contentserver.content_len', content.length) - newrelic.agent.add_custom_attribute('contentserver.content_type', content.content_type) + set_custom_attribute('contentserver.content_len', content.length) + set_custom_attribute('contentserver.content_type', content.content_type) # "Accept-Ranges: bytes" tells the user that only "bytes" ranges are allowed response['Accept-Ranges'] = 'bytes' @@ -213,14 +207,12 @@ def set_caching_headers(self, content, response): # indicate there should be no caching whatsoever. cache_ttl = CourseAssetCacheTtlConfig.get_cache_ttl() if cache_ttl > 0 and not is_locked: - if newrelic: - newrelic.agent.add_custom_attribute('contentserver.cacheable', True) + set_custom_attribute('contentserver.cacheable', True) response['Expires'] = StaticContentServer.get_expiration_value(datetime.datetime.utcnow(), cache_ttl) response['Cache-Control'] = "public, max-age={ttl}, s-maxage={ttl}".format(ttl=cache_ttl) elif is_locked: - if newrelic: - newrelic.agent.add_custom_attribute('contentserver.cacheable', False) + set_custom_attribute('contentserver.cacheable', False) response['Cache-Control'] = "private, no-cache, no-store" diff --git a/openedx/core/djangoapps/notifications/apps.py b/openedx/core/djangoapps/notifications/apps.py index 63d110191189..ffad68eccde6 100644 --- a/openedx/core/djangoapps/notifications/apps.py +++ b/openedx/core/djangoapps/notifications/apps.py @@ -18,3 +18,4 @@ def ready(self): """ # pylint: disable=unused-import from . import handlers + from .email import tasks diff --git a/openedx/core/djangoapps/notifications/base_notification.py b/openedx/core/djangoapps/notifications/base_notification.py index 3f080aab8136..2845e9cbe41d 100644 --- a/openedx/core/djangoapps/notifications/base_notification.py +++ b/openedx/core/djangoapps/notifications/base_notification.py @@ -187,7 +187,7 @@ 'filters': [FILTER_AUDIT_EXPIRED_USERS_WITH_NO_ROLE] }, 'ora_staff_notification': { - 'notification_app': 'ora', + 'notification_app': 'grading', 'name': 'ora_staff_notification', 'is_core': False, 'info': '', @@ -227,9 +227,9 @@ 'core_email_cadence': EmailCadence.DAILY, 'non_editable': [] }, - 'ora': { + 'grading': { 'enabled': True, - 'core_info': _('Notifications for Open response submissions.'), + 'core_info': _('Notifications for submission grading.'), 'core_web': True, 'core_email': True, 'core_push': True, diff --git a/openedx/core/djangoapps/notifications/config/waffle.py b/openedx/core/djangoapps/notifications/config/waffle.py index 9d54a0abb8fd..fa1d02adf1d3 100644 --- a/openedx/core/djangoapps/notifications/config/waffle.py +++ b/openedx/core/djangoapps/notifications/config/waffle.py @@ -3,7 +3,7 @@ waffle switches for the notifications app. """ -from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag +from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag, WaffleFlag WAFFLE_NAMESPACE = 'notifications' @@ -48,7 +48,6 @@ # .. toggle_tickets: INF-1145 ENABLE_COURSEWIDE_NOTIFICATIONS = CourseWaffleFlag(f"{WAFFLE_NAMESPACE}.enable_coursewide_notifications", __name__) - # .. toggle_name: notifications.enable_ora_staff_notifications # .. toggle_implementation: CourseWaffleFlag # .. toggle_default: False @@ -58,3 +57,14 @@ # .. toggle_target_removal_date: 2024-06-04 # .. toggle_tickets: INF-1304 ENABLE_ORA_STAFF_NOTIFICATION = CourseWaffleFlag(f"{WAFFLE_NAMESPACE}.enable_ora_staff_notifications", __name__) + +# .. toggle_name: notifications.enable_email_notifications +# .. toggle_implementation: WaffleFlag +# .. toggle_default: False +# .. toggle_description: Waffle flag to enable the Email Notifications feature +# .. toggle_use_cases: temporary, open_edx +# .. toggle_creation_date: 2024-03-25 +# .. toggle_target_removal_date: 2025-06-01 +# .. toggle_warning: When the flag is ON, Email Notifications feature is enabled. +# .. toggle_tickets: INF-1259 +ENABLE_EMAIL_NOTIFICATIONS = WaffleFlag(f'{WAFFLE_NAMESPACE}.enable_email_notifications', __name__) diff --git a/openedx/core/djangoapps/notifications/email/tasks.py b/openedx/core/djangoapps/notifications/email/tasks.py new file mode 100644 index 000000000000..3b1783f42b3a --- /dev/null +++ b/openedx/core/djangoapps/notifications/email/tasks.py @@ -0,0 +1,116 @@ +""" +Celery tasks for sending email notifications +""" +from celery import shared_task +from celery.utils.log import get_task_logger +from django.contrib.auth import get_user_model +from edx_ace import ace +from edx_ace.recipient import Recipient +from edx_django_utils.monitoring import set_code_owner_attribute + +from openedx.core.djangoapps.notifications.email_notifications import EmailCadence +from openedx.core.djangoapps.notifications.models import ( + CourseNotificationPreference, + Notification, + get_course_notification_preference_config_version +) +from .message_type import EmailNotificationMessageType +from .utils import ( + create_app_notifications_dict, + create_email_digest_context, + filter_notification_with_email_enabled_preferences, + get_start_end_date, + get_unique_course_ids, + is_email_notification_flag_enabled +) + + +User = get_user_model() +logger = get_task_logger(__name__) + + +def get_audience_for_cadence_email(cadence_type): + """ + Returns users that are eligible to receive cadence email + """ + if cadence_type not in [EmailCadence.DAILY, EmailCadence.WEEKLY]: + raise ValueError("Invalid value for parameter cadence_type") + start_date, end_date = get_start_end_date(cadence_type) + user_ids = Notification.objects.filter( + email=True, + created__gte=start_date, + created__lte=end_date + ).values_list('user__id', flat=True).distinct() + users = User.objects.filter(id__in=user_ids) + return users + + +def get_user_preferences_for_courses(course_ids, user): + """ + Returns updated user preference for course_ids + """ + # Create new preferences + new_preferences = [] + preferences = CourseNotificationPreference.objects.filter(user=user, course_id__in=course_ids) + preferences = list(preferences) + for course_id in course_ids: + if not any(preference.course_id == course_id for preference in preferences): + pref = CourseNotificationPreference(user=user, course_id=course_id) + new_preferences.append(pref) + if new_preferences: + CourseNotificationPreference.objects.bulk_create(new_preferences, ignore_conflicts=True) + # Update preferences to latest config version + current_version = get_course_notification_preference_config_version() + for preference in preferences: + if preference.config_version != current_version: + preference = preference.get_user_course_preference(user.id, preference.course_id) + new_preferences.append(preference) + return new_preferences + + +def send_digest_email_to_user(user, cadence_type, course_language='en', courses_data=None): + """ + Send [cadence_type] email to user. + Cadence Type can be EmailCadence.DAILY or EmailCadence.WEEKLY + """ + if cadence_type not in [EmailCadence.DAILY, EmailCadence.WEEKLY]: + raise ValueError('Invalid cadence_type') + logger.info(f' Sending email to user {user.username} ==Temp Log==') + if not is_email_notification_flag_enabled(user): + logger.info(f' Flag disabled for {user.username} ==Temp Log==') + return + start_date, end_date = get_start_end_date(cadence_type) + notifications = Notification.objects.filter(user=user, email=True, + created__gte=start_date, created__lte=end_date) + if not notifications: + logger.info(f' No notification for {user.username} ==Temp Log==') + return + course_ids = get_unique_course_ids(notifications) + preferences = get_user_preferences_for_courses(course_ids, user) + notifications = filter_notification_with_email_enabled_preferences(notifications, preferences, cadence_type) + if not notifications: + logger.info(f' No filtered notification for {user.username} ==Temp Log==') + return + apps_dict = create_app_notifications_dict(notifications) + message_context = create_email_digest_context(apps_dict, start_date, end_date, cadence_type, + courses_data=courses_data) + recipient = Recipient(user.id, user.email) + message = EmailNotificationMessageType( + app_label="notifications", name="email_digest" + ).personalize(recipient, course_language, message_context) + ace.send(message) + logger.info(f' Email sent to {user.username} ==Temp Log==') + + +@shared_task(ignore_result=True) +@set_code_owner_attribute +def send_digest_email_to_all_users(cadence_type): + """ + Send email digest to all eligible users + """ + logger.info(f' Sending cadence email of type {cadence_type}') + users = get_audience_for_cadence_email(cadence_type) + courses_data = {} + logger.info(f' Email Cadence Audience {len(users)}') + for user in users: + send_digest_email_to_user(user, cadence_type, courses_data=courses_data) diff --git a/openedx/core/lib/blockstore_api/tests/__init__.py b/openedx/core/djangoapps/notifications/email/tests/__init__.py similarity index 100% rename from openedx/core/lib/blockstore_api/tests/__init__.py rename to openedx/core/djangoapps/notifications/email/tests/__init__.py diff --git a/openedx/core/djangoapps/notifications/email/tests/test_tasks.py b/openedx/core/djangoapps/notifications/email/tests/test_tasks.py new file mode 100644 index 000000000000..5c88ef3edb1a --- /dev/null +++ b/openedx/core/djangoapps/notifications/email/tests/test_tasks.py @@ -0,0 +1,212 @@ +""" +Test cases for notifications/email/tasks.py +""" +import datetime +import ddt + +from unittest.mock import patch + +from edx_toggles.toggles.testutils import override_waffle_flag + +from common.djangoapps.student.tests.factories import UserFactory +from openedx.core.djangoapps.notifications.config.waffle import ENABLE_EMAIL_NOTIFICATIONS +from openedx.core.djangoapps.notifications.email_notifications import EmailCadence +from openedx.core.djangoapps.notifications.email.tasks import ( + get_audience_for_cadence_email, + send_digest_email_to_all_users, + send_digest_email_to_user +) +from openedx.core.djangoapps.notifications.models import CourseNotificationPreference +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory + +from .utils import create_notification + + +@ddt.ddt +class TestEmailDigestForUser(ModuleStoreTestCase): + """ + Tests email notification for a specific user + """ + + def setUp(self): + """ + Setup + """ + super().setUp() + self.user = UserFactory() + self.course = CourseFactory.create(display_name='test course', run="Testing_course") + + @patch('edx_ace.ace.send') + def test_email_is_not_sent_if_no_notifications(self, mock_func): + """ + Tests email is sent iff waffle flag is enabled + """ + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_user(self.user, EmailCadence.DAILY) + assert not mock_func.called + + @ddt.data(True, False) + @patch('edx_ace.ace.send') + def test_email_is_sent_iff_flag_enabled(self, flag_value, mock_func): + """ + Tests email is sent iff waffle flag is enabled + """ + created_date = datetime.datetime.now() - datetime.timedelta(days=1) + create_notification(self.user, self.course.id, created=created_date) + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, flag_value): + send_digest_email_to_user(self.user, EmailCadence.DAILY) + assert mock_func.called is flag_value + + @patch('edx_ace.ace.send') + def test_notification_not_send_if_created_on_next_day(self, mock_func): + """ + Tests email is not sent if notification is created on next day + """ + create_notification(self.user, self.course.id) + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_user(self.user, EmailCadence.DAILY) + assert not mock_func.called + + @patch('edx_ace.ace.send') + def test_notification_not_send_if_created_day_before_yesterday(self, mock_func): + """ + Tests email is not sent if notification is created day before yesterday + """ + created_date = datetime.datetime.now() - datetime.timedelta(days=2) + create_notification(self.user, self.course.id, created=created_date) + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_user(self.user, EmailCadence.DAILY) + assert not mock_func.called + + +@ddt.ddt +class TestEmailDigestAudience(ModuleStoreTestCase): + """ + Tests audience for notification digest email + """ + + def setUp(self): + """ + Setup + """ + super().setUp() + self.user = UserFactory() + self.course = CourseFactory.create(display_name='test course', run="Testing_course") + + @patch('openedx.core.djangoapps.notifications.email.tasks.send_digest_email_to_user') + def test_email_func_not_called_if_no_notification(self, mock_func): + """ + Tests email sending function is not called if user has no notifications + """ + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_all_users(EmailCadence.DAILY) + assert not mock_func.called + + @patch('openedx.core.djangoapps.notifications.email.tasks.send_digest_email_to_user') + def test_email_func_called_if_user_has_notification(self, mock_func): + """ + Tests email sending function is called if user has notification + """ + created_date = datetime.datetime.now() - datetime.timedelta(days=1) + create_notification(self.user, self.course.id, created=created_date) + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_all_users(EmailCadence.DAILY) + assert mock_func.called + + @patch('openedx.core.djangoapps.notifications.email.tasks.send_digest_email_to_user') + def test_email_func_not_called_if_user_notification_is_not_duration(self, mock_func): + """ + Tests email sending function is not called if user has notification + which is not in duration + """ + created_date = datetime.datetime.now() - datetime.timedelta(days=10) + create_notification(self.user, self.course.id, created=created_date) + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_all_users(EmailCadence.DAILY) + assert not mock_func.called + + @patch('edx_ace.ace.send') + def test_email_is_sent_to_user_when_task_is_called(self, mock_func): + created_date = datetime.datetime.now() - datetime.timedelta(days=1) + create_notification(self.user, self.course.id, created=created_date) + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_all_users(EmailCadence.DAILY) + assert mock_func.called + assert mock_func.call_count == 1 + + def test_audience_query_count(self): + with self.assertNumQueries(1): + audience = get_audience_for_cadence_email(EmailCadence.DAILY) + list(audience) # evaluating queryset + + @ddt.data(True, False) + @patch('edx_ace.ace.send') + def test_digest_should_contain_email_enabled_notifications(self, email_value, mock_func): + """ + Tests email is sent only when notifications with email=True exists + """ + created_date = datetime.datetime.now() - datetime.timedelta(days=1) + create_notification(self.user, self.course.id, created=created_date, email=email_value) + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_user(self.user, EmailCadence.DAILY) + assert mock_func.called is email_value + + +@ddt.ddt +class TestPreferences(ModuleStoreTestCase): + """ + Tests preferences + """ + def setUp(self): + """ + Setup + """ + super().setUp() + self.user = UserFactory() + self.course = CourseFactory.create(display_name='test course', run="Testing_course") + self.preference = CourseNotificationPreference.objects.create(user=self.user, course_id=self.course.id) + created_date = datetime.datetime.now() - datetime.timedelta(days=1) + create_notification(self.user, self.course.id, notification_type='new_discussion_post', created=created_date) + + @patch('edx_ace.ace.send') + def test_email_send_for_digest_preference(self, mock_func): + """ + Tests email is send for digest notification preference + """ + config = self.preference.notification_preference_config + types = config['discussion']['notification_types'] + types['new_discussion_post']['email_cadence'] = EmailCadence.DAILY + types['new_discussion_post']['email'] = True + self.preference.save() + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_user(self.user, EmailCadence.DAILY) + assert mock_func.called + + @ddt.data(True, False) + @patch('edx_ace.ace.send') + def test_email_send_for_email_preference_value(self, pref_value, mock_func): + """ + Tests email is sent iff preference value is True + """ + config = self.preference.notification_preference_config + types = config['discussion']['notification_types'] + types['new_discussion_post']['email_cadence'] = EmailCadence.DAILY + types['new_discussion_post']['email'] = pref_value + self.preference.save() + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_user(self.user, EmailCadence.DAILY) + assert mock_func.called is pref_value + + @patch('edx_ace.ace.send') + def test_email_not_send_if_different_digest_preference(self, mock_func): + """ + Tests email is not send if digest notification preference doesnot match + """ + config = self.preference.notification_preference_config + types = config['discussion']['notification_types'] + types['new_discussion_post']['email_cadence'] = EmailCadence.WEEKLY + self.preference.save() + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_user(self.user, EmailCadence.DAILY) + assert not mock_func.called diff --git a/openedx/core/djangoapps/notifications/email/tests/test_utils.py b/openedx/core/djangoapps/notifications/email/tests/test_utils.py new file mode 100644 index 000000000000..d7c9f6c98133 --- /dev/null +++ b/openedx/core/djangoapps/notifications/email/tests/test_utils.py @@ -0,0 +1,196 @@ +""" +Test utils.py +""" +import datetime +import ddt + +from pytz import utc +from waffle import get_waffle_flag_model # pylint: disable=invalid-django-waffle-import + +from common.djangoapps.student.tests.factories import UserFactory +from openedx.core.djangoapps.notifications.config.waffle import ENABLE_EMAIL_NOTIFICATIONS +from openedx.core.djangoapps.notifications.models import Notification +from openedx.core.djangoapps.notifications.email.utils import ( + add_additional_attributes_to_notifications, + create_app_notifications_dict, + create_datetime_string, + create_email_digest_context, + create_email_template_context, + get_course_info, + get_time_ago, + is_email_notification_flag_enabled, +) +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory + +from .utils import assert_list_equal, create_notification + + +class TestUtilFunctions(ModuleStoreTestCase): + """ + Test utils functions + """ + def setUp(self): + """ + Setup + """ + super().setUp() + self.user = UserFactory() + self.course = CourseFactory.create(display_name='test course', run="Testing_course") + + def test_additional_attributes(self): + """ + Tests additional attributes are added when notifications list is passed to + add_additional_attributes_to_notifications function + """ + notification = create_notification(self.user, self.course.id) + additional_params = ['course_name', 'icon', 'time_ago'] + for param in additional_params: + assert not hasattr(notification, param) + add_additional_attributes_to_notifications([notification]) + for param in additional_params: + assert hasattr(notification, param) + + def test_create_app_notifications_dict(self): + """ + Tests notifications are divided based on their app_name + """ + Notification.objects.all().delete() + create_notification(self.user, self.course.id, app_name='discussion', notification_type='new_comment') + create_notification(self.user, self.course.id, app_name='updates', notification_type='course_update') + app_dict = create_app_notifications_dict(Notification.objects.all()) + assert len(app_dict.keys()) == 2 + for key in ['discussion', 'updates']: + assert key in app_dict.keys() + assert app_dict[key]['count'] == 1 + assert len(app_dict[key]['notifications']) == 1 + + def test_get_course_info(self): + """ + Tests get_course_info function + """ + assert get_course_info(self.course.id) == {'name': 'test course'} + + def test_get_time_ago(self): + """ + Tests time_ago string + """ + current_datetime = utc.localize(datetime.datetime.now()) + assert "Today" == get_time_ago(current_datetime) + assert "1d" == get_time_ago(current_datetime - datetime.timedelta(days=1)) + assert "1w" == get_time_ago(current_datetime - datetime.timedelta(days=7)) + + def test_datetime_string(self): + dt = datetime.datetime(2024, 3, 25) + assert create_datetime_string(dt) == "Monday, Mar 25" + + +@ddt.ddt +class TestContextFunctions(ModuleStoreTestCase): + """ + Test template context functions in utils.py + """ + def setUp(self): + """ + Setup + """ + super().setUp() + self.user = UserFactory() + self.course = CourseFactory.create(display_name='test course', run="Testing_course") + + def test_email_template_context(self): + """ + Tests common header and footer context + """ + context = create_email_template_context() + keys = ['platform_name', 'mailing_address', 'logo_url', 'social_media', 'notification_settings_url'] + for key in keys: + assert key in context + + @ddt.data('Daily', 'Weekly') + def test_email_digest_context(self, digest_frequency): + """ + Tests context for email digest + """ + Notification.objects.all().delete() + discussion_notification = create_notification(self.user, self.course.id, app_name='discussion', + notification_type='new_comment') + update_notification = create_notification(self.user, self.course.id, app_name='updates', + notification_type='course_update') + app_dict = create_app_notifications_dict(Notification.objects.all()) + end_date = datetime.datetime(2024, 3, 24, 12, 0) + params = { + "app_notifications_dict": app_dict, + "start_date": end_date - datetime.timedelta(days=0 if digest_frequency == "Daily" else 6), + "end_date": end_date, + "digest_frequency": digest_frequency, + "courses_data": None + } + context = create_email_digest_context(**params) + expected_start_date = 'Sunday, Mar 24' if digest_frequency == 'Daily' else 'Monday, Mar 18' + expected_digest_updates = [ + {'title': 'Total Notifications', 'count': 2}, + {'title': 'Discussion', 'count': 1}, + {'title': 'Updates', 'count': 1}, + ] + expected_email_content = [ + {'title': 'Discussion', 'help_text': '', 'help_text_url': '', 'notifications': [discussion_notification]}, + {'title': 'Updates', 'help_text': '', 'help_text_url': '', 'notifications': [update_notification]} + ] + assert context['start_date'] == expected_start_date + assert context['end_date'] == 'Sunday, Mar 24' + assert context['digest_frequency'] == digest_frequency + assert_list_equal(context['email_digest_updates'], expected_digest_updates) + assert_list_equal(context['email_content'], expected_email_content) + + +class TestWaffleFlag(ModuleStoreTestCase): + """ + Test user level email notifications waffle flag + """ + def setUp(self): + """ + Setup + """ + super().setUp() + self.user_1 = UserFactory() + self.user_2 = UserFactory() + self.course_1 = CourseFactory.create(display_name='test course 1', run="Testing_course_1") + self.course_1 = CourseFactory.create(display_name='test course 2', run="Testing_course_2") + + def test_waffle_flag_for_everyone(self): + """ + Tests if waffle flag is enabled for everyone + """ + assert is_email_notification_flag_enabled() is False + waffle_model = get_waffle_flag_model() + flag, _ = waffle_model.objects.get_or_create(name=ENABLE_EMAIL_NOTIFICATIONS.name) + flag.everyone = True + flag.save() + assert is_email_notification_flag_enabled() is True + + def test_waffle_flag_for_user(self): + """ + Tests user level waffle flag + """ + assert is_email_notification_flag_enabled() is False + waffle_model = get_waffle_flag_model() + flag, _ = waffle_model.objects.get_or_create(name=ENABLE_EMAIL_NOTIFICATIONS.name) + flag.users.add(self.user_1) + flag.save() + assert is_email_notification_flag_enabled(self.user_1) is True + assert is_email_notification_flag_enabled(self.user_2) is False + + def test_waffle_flag_everyone_priority(self): + """ + Tests if everyone field has more priority over user field + """ + assert is_email_notification_flag_enabled() is False + waffle_model = get_waffle_flag_model() + flag, _ = waffle_model.objects.get_or_create(name=ENABLE_EMAIL_NOTIFICATIONS.name) + flag.everyone = False + flag.users.add(self.user_1) + flag.save() + assert is_email_notification_flag_enabled() is False + assert is_email_notification_flag_enabled(self.user_1) is False + assert is_email_notification_flag_enabled(self.user_2) is False diff --git a/openedx/core/djangoapps/notifications/email/tests/utils.py b/openedx/core/djangoapps/notifications/email/tests/utils.py new file mode 100644 index 000000000000..c27f60443b31 --- /dev/null +++ b/openedx/core/djangoapps/notifications/email/tests/utils.py @@ -0,0 +1,40 @@ +""" +Utils for tests +""" +from openedx.core.djangoapps.notifications.models import Notification + + +def create_notification(user, course_key, **kwargs): + """ + Create a test notification + """ + notification_params = { + 'user': user, + 'course_id': course_key, + 'app_name': "discussion", + 'notification_type': "new_comment", + 'content_url': '', + 'content_context': { + "replier_name": "replier", + "username": "username", + "author_name": "author_name", + "post_title": "post_title", + "course_update_content": "Course update content", + "content_type": 'post', + "content": "post_title" + }, + 'email': True, + 'web': True + } + notification_params.update(kwargs) + notification = Notification.objects.create(**notification_params) + return notification + + +def assert_list_equal(list_1, list_2): + """ + Asserts if list is equal + """ + assert len(list_1) == len(list_2) + for element in list_1: + assert element in list_2 diff --git a/openedx/core/djangoapps/notifications/email/utils.py b/openedx/core/djangoapps/notifications/email/utils.py index 2993b68fc2dd..07b1bf1330a5 100644 --- a/openedx/core/djangoapps/notifications/email/utils.py +++ b/openedx/core/djangoapps/notifications/email/utils.py @@ -1,12 +1,46 @@ """ Email Notifications Utils """ +import datetime + from django.conf import settings +from pytz import utc +from waffle import get_waffle_flag_model # pylint: disable=invalid-django-waffle-import + from lms.djangoapps.branding.api import get_logo_url_for_email +from openedx.core.djangoapps.notifications.config.waffle import ENABLE_EMAIL_NOTIFICATIONS +from openedx.core.djangoapps.notifications.email_notifications import EmailCadence +from xmodule.modulestore.django import modulestore + from .notification_icons import NotificationTypeIcons +def is_email_notification_flag_enabled(user=None): + """ + Returns if waffle flag is enabled for user or not + """ + flag_model = get_waffle_flag_model() + try: + flag = flag_model.objects.get(name=ENABLE_EMAIL_NOTIFICATIONS.name) + except flag_model.DoesNotExist: + return False + if flag.everyone is not None: + return flag.everyone + if user: + role_value = flag.is_active_for_user(user) + if role_value is not None: + return role_value + try: + return flag.users.contains(user) + except ValueError: + pass + return False + + def create_datetime_string(datetime_instance): + """ + Returns string for datetime object + """ return datetime_instance.strftime('%A, %b %d') @@ -40,25 +74,172 @@ def create_email_template_context(): } -def create_email_digest_content(start_date, end_date=None, digest_frequency="Daily", - notifications_count=0, updates_count=0, email_content=None): +def create_email_digest_context(app_notifications_dict, start_date, end_date=None, digest_frequency="Daily", + courses_data=None): """ Creates email context based on content + app_notifications_dict: Mapping of notification app and its count, title and notifications start_date: datetime instance end_date: datetime instance + digest_frequency: EmailCadence.DAILY or EmailCadence.WEEKLY + courses_data: Dictionary to cache course info (avoid additional db calls) """ context = create_email_template_context() start_date_str = create_datetime_string(start_date) end_date_str = create_datetime_string(end_date if end_date else start_date) + email_digest_updates = [{ + 'title': 'Total Notifications', + 'count': sum(value['count'] for value in app_notifications_dict.values()) + }] + email_digest_updates.extend([ + { + 'title': value['title'], + 'count': value['count'], + } + for key, value in app_notifications_dict.items() + ]) + email_content = [ + { + 'title': value['title'], + 'help_text': value.get('help_text', ''), + 'help_text_url': value.get('help_text_url', ''), + 'notifications': add_additional_attributes_to_notifications( + value.get('notifications', []), courses_data=courses_data + ) + } + for key, value in app_notifications_dict.items() + ] context.update({ "start_date": start_date_str, "end_date": end_date_str, "digest_frequency": digest_frequency, - "updates": [ - {"count": updates_count, "type": "Updates"}, - {"count": notifications_count, "type": "Notifications"} - ], - "email_content": email_content if email_content else [], - "get_icon_url_for_notification_type": get_icon_url_for_notification_type, + "email_digest_updates": email_digest_updates, + "email_content": email_content, }) return context + + +def get_start_end_date(cadence_type): + """ + Returns start_date and end_date for email digest + """ + if cadence_type not in [EmailCadence.DAILY, EmailCadence.WEEKLY]: + raise ValueError('Invalid cadence_type') + date_today = datetime.datetime.now() + yesterday = date_today - datetime.timedelta(days=1) + end_date = datetime.datetime.combine(yesterday, datetime.time.max) + start_date = end_date + if cadence_type == EmailCadence.WEEKLY: + start_date = end_date - datetime.timedelta(days=6) + start_date = datetime.datetime.combine(start_date, datetime.time.min) + return utc.localize(start_date), utc.localize(end_date) + + +def get_course_info(course_key): + """ + Returns course info for course_key + """ + store = modulestore() + course = store.get_course(course_key) + return {'name': course.display_name} + + +def get_time_ago(datetime_obj): + """ + Returns time_ago for datetime instance + """ + current_date = utc.localize(datetime.datetime.today()) + days_diff = (current_date - datetime_obj).days + if days_diff == 0: + return "Today" + if days_diff >= 7: + return f"{int(days_diff / 7)}w" + return f"{days_diff}d" + + +def add_additional_attributes_to_notifications(notifications, courses_data=None): + """ + Add attributes required for email content to notification instance + notifications: list[Notification] + course_data: Cache course info + """ + if courses_data is None: + courses_data = {} + + for notification in notifications: + notification_type = notification.notification_type + course_key = notification.course_id + course_key_str = str(course_key) + if course_key_str not in courses_data.keys(): + courses_data[course_key_str] = get_course_info(course_key) + course_info = courses_data[course_key_str] + notification.course_name = course_info.get('name', '') + notification.icon = get_icon_url_for_notification_type(notification_type) + notification.time_ago = get_time_ago(notification.created) + return notifications + + +def create_app_notifications_dict(notifications): + """ + Return a dictionary with notification app as key and + title, count and notifications as its value + """ + app_names = list({notification.app_name for notification in notifications}) + app_notifications = { + name: { + 'count': 0, + 'title': name.title(), + 'notifications': [] + } + for name in app_names + } + for notification in notifications: + app_data = app_notifications[notification.app_name] + app_data['count'] += 1 + app_data['notifications'].append(notification) + return app_notifications + + +def get_unique_course_ids(notifications): + """ + Returns unique course_ids from notifications + """ + course_ids = [] + for notification in notifications: + if notification.course_id not in course_ids: + course_ids.append(notification.course_id) + return course_ids + + +def get_enabled_notification_types_for_cadence(preferences, cadence_type=EmailCadence.DAILY): + """ + Returns a dictionary that returns notification_types with cadence_types for course_ids + """ + if cadence_type not in [EmailCadence.DAILY, EmailCadence.WEEKLY]: + raise ValueError('Invalid cadence_type') + course_types = {} + for preference in preferences: + key = preference.course_id + value = [] + config = preference.notification_preference_config + for app_data in config.values(): + for notification_type, type_dict in app_data['notification_types'].items(): + if (type_dict['email_cadence'] == cadence_type) and type_dict['email']: + value.append(notification_type) + if 'core' in value: + value.remove('core') + value.extend(app_data['core_notification_types']) + course_types[key] = value + return course_types + + +def filter_notification_with_email_enabled_preferences(notifications, preferences, cadence_type=EmailCadence.DAILY): + """ + Filter notifications for types with email cadence preference enabled + """ + enabled_course_prefs = get_enabled_notification_types_for_cadence(preferences, cadence_type) + filtered_notifications = [] + for notification in notifications: + if notification.notification_type in enabled_course_prefs[notification.course_id]: + filtered_notifications.append(notification) + return filtered_notifications diff --git a/openedx/core/djangoapps/notifications/management/commands/send_email_digest.py b/openedx/core/djangoapps/notifications/management/commands/send_email_digest.py new file mode 100644 index 000000000000..79857c18cf8a --- /dev/null +++ b/openedx/core/djangoapps/notifications/management/commands/send_email_digest.py @@ -0,0 +1,32 @@ +""" +Management command for sending email digest +""" +from django.core.management.base import BaseCommand + +from openedx.core.djangoapps.notifications.email_notifications import EmailCadence +from openedx.core.djangoapps.notifications.email.tasks import send_digest_email_to_all_users + + +class Command(BaseCommand): + """ + Invoke with: + + python manage.py lms send_email_digest [cadence_type] + cadence_type: Daily or Weekly + """ + help = ( + "Send email digest to user." + ) + + def add_arguments(self, parser): + """ + Adds management commands parser arguments + """ + cadence_type_choices = [EmailCadence.DAILY, EmailCadence.WEEKLY] + parser.add_argument('cadence_type', choices=cadence_type_choices) + + def handle(self, *args, **kwargs): + """ + Start task to send email digest to users + """ + send_digest_email_to_all_users.delay(kwargs['cadence_type']) diff --git a/openedx/core/djangoapps/notifications/models.py b/openedx/core/djangoapps/notifications/models.py index dea3169ef497..2f7da1803bf6 100644 --- a/openedx/core/djangoapps/notifications/models.py +++ b/openedx/core/djangoapps/notifications/models.py @@ -23,7 +23,7 @@ ADDITIONAL_NOTIFICATION_CHANNEL_SETTINGS = ['email_cadence'] # Update this version when there is a change to any course specific notification type or app. -COURSE_NOTIFICATION_CONFIG_VERSION = 9 +COURSE_NOTIFICATION_CONFIG_VERSION = 10 def get_course_notification_preference_config(): diff --git a/openedx/core/djangoapps/notifications/templates/notifications/digest_content.html b/openedx/core/djangoapps/notifications/templates/notifications/digest_content.html index aa1ee903ad7e..1da86f48f0b1 100644 --- a/openedx/core/djangoapps/notifications/templates/notifications/digest_content.html +++ b/openedx/core/djangoapps/notifications/templates/notifications/digest_content.html @@ -1,15 +1,18 @@ -{% for update in email_content %} + +{% for notification_app in email_content %}

    - {{ update.title }} + {{ notification_app.title }}

    - {% if update.help_text %} + {% if notification_app.help_text %}

    - {{ update.help_text }} + {{ notification_app.help_text }} - {% if update.help_text_url %} + {% if notification_app.help_text_url %} - + View all @@ -20,27 +23,27 @@

    - {% for content in update.content %} + {% for notification in notification_app.notifications %} -
    +

    - {{ content.title }} + {{ notification.content | safe }}

    - {{ content.course_name }} - · - {{ content.time_ago }} + {{ notification.course_name }} + {{ "·"|safe }} + {{ notification.time_ago }} - + View diff --git a/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html b/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html index fc91d77fa0bf..7957524e8ae2 100644 --- a/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html +++ b/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html @@ -37,35 +37,27 @@ - {% for update in updates %} - - {% endfor %} + {% for update in email_digest_updates %} + + {% if forloop.counter|divisibleby:3 %} + + {% endif %} + {% endfor %}
    -

    - - - - - - - - - -
    - {{update.count}} -
    - {{update.type}} -
    -

    -     		+ + + + + + + + + +
    + {{update.count}} +
    + {{update.title}} +
    +
    diff --git a/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.html b/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.html index c847f75e4f7b..76658a043665 100644 --- a/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.html +++ b/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.html @@ -1 +1,21 @@ -<%page expression_filter="h"/> +

    + + + + + + + + + + + + +
    + {% include 'notifications/digest_header.html' %} +
    + {% include 'notifications/digest_content.html' %} +
    + {% include 'notifications/digest_footer.html' %} +
    +
    diff --git a/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.txt b/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.txt index e45fd8029e2d..3bbe26faf772 100644 --- a/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.txt +++ b/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.txt @@ -1,22 +1 @@ -
    - - - - - - - - - - - - -
    - {% include 'notifications/digest_header.html' %} -
    - {% include 'notifications/digest_content.html' %} -
    - {% include 'notifications/digest_footer.html' %} -
    - -
    +{{ digest_frequency }} Notifications Digest for {% if digest_frequency == "Weekly" %}the Week of {% endif %}{{ start_date }} diff --git a/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/head.html b/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/head.html index c847f75e4f7b..8d63916b7a49 100644 --- a/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/head.html +++ b/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/head.html @@ -1 +1,3 @@ -<%page expression_filter="h"/> + +{{ platform_name }} + diff --git a/openedx/core/djangoapps/notifications/tests/test_tasks.py b/openedx/core/djangoapps/notifications/tests/test_tasks.py index 83ac9d17633e..036d4d326198 100644 --- a/openedx/core/djangoapps/notifications/tests/test_tasks.py +++ b/openedx/core/djangoapps/notifications/tests/test_tasks.py @@ -433,3 +433,67 @@ def test_course_id_param(self): delete_notifications({'course_id': self.course_1.id}) assert not Notification.objects.filter(course_id=self.course_1.id) assert Notification.objects.filter(course_id=self.course_2.id) + + +@ddt.ddt +class NotificationCreationOnChannelsTests(ModuleStoreTestCase): + """ + Tests for notification creation and channels value. + """ + + def setUp(self): + """ + Create a course and users for tests. + """ + + super().setUp() + self.user = UserFactory() + self.course = CourseFactory.create( + org='testorg', + number='testcourse', + run='testrun' + ) + + self.preference = CourseNotificationPreference.objects.create( + user_id=self.user.id, + course_id=self.course.id, + config_version=0, + ) + + @override_waffle_flag(ENABLE_NOTIFICATIONS, active=True) + @ddt.data( + (False, False, 0), + (False, True, 1), + (True, False, 1), + (True, True, 1), + ) + @ddt.unpack + def test_notification_is_created_when_any_channel_is_enabled(self, web_value, email_value, generated_count): + """ + Tests if notification is created if any preference is enabled + """ + app_name = 'discussion' + notification_type = 'new_discussion_post' + app_prefs = self.preference.notification_preference_config[app_name] + app_prefs['notification_types'][notification_type]['web'] = web_value + app_prefs['notification_types'][notification_type]['email'] = email_value + kwargs = { + 'user_ids': [self.user.id], + 'course_key': str(self.course.id), + 'app_name': app_name, + 'notification_type': notification_type, + 'content_url': 'https://example.com/', + 'context': { + 'post_title': 'Post title', + 'username': 'user name', + }, + } + self.preference.save() + with patch('openedx.core.djangoapps.notifications.tasks.notification_generated_event') as event_mock: + send_notifications(**kwargs) + notifications = Notification.objects.all() + assert len(notifications) == generated_count + if notifications: + notification = Notification.objects.all()[0] + assert notification.web == web_value + assert notification.email == email_value diff --git a/openedx/core/djangoapps/notifications/tests/test_views.py b/openedx/core/djangoapps/notifications/tests/test_views.py index 8dcb335f5fe5..ca5572ed9f55 100644 --- a/openedx/core/djangoapps/notifications/tests/test_views.py +++ b/openedx/core/djangoapps/notifications/tests/test_views.py @@ -298,7 +298,7 @@ def _expected_api_response(self, course=None): }, 'non_editable': {} }, - 'ora': { + 'grading': { 'enabled': True, 'core_notification_types': [], 'notification_types': { @@ -314,7 +314,7 @@ def _expected_api_response(self, course=None): 'email': True, 'push': True, 'email_cadence': 'Daily', - 'info': 'Notifications for Open response submissions.' + 'info': 'Notifications for submission grading.' } }, 'non_editable': {} @@ -601,7 +601,7 @@ def _expected_api_response(self, course=None): }, 'non_editable': {} }, - 'ora': { + 'grading': { 'enabled': True, 'core_notification_types': [], 'notification_types': { @@ -617,7 +617,7 @@ def _expected_api_response(self, course=None): 'email': True, 'push': True, 'email_cadence': 'Daily', - 'info': 'Notifications for Open response submissions.' + 'info': 'Notifications for submission grading.' } }, 'non_editable': {} @@ -926,7 +926,7 @@ def test_get_unseen_notifications_count_with_show_notifications_tray(self, show_ self.assertEqual(response.status_code, 200) self.assertEqual(response.data['count'], 4) self.assertEqual(response.data['count_by_app_name'], { - 'App Name 1': 2, 'App Name 2': 1, 'App Name 3': 1, 'discussion': 0, 'updates': 0, 'ora': 0}) + 'App Name 1': 2, 'App Name 2': 1, 'App Name 3': 1, 'discussion': 0, 'updates': 0, 'grading': 0}) self.assertEqual(response.data['show_notifications_tray'], show_notifications_tray_enabled) def test_get_unseen_notifications_count_for_unauthenticated_user(self): @@ -947,7 +947,7 @@ def test_get_unseen_notifications_count_for_user_with_no_notifications(self): self.assertEqual(response.status_code, 200) self.assertEqual(response.data['count'], 0) - self.assertEqual(response.data['count_by_app_name'], {'discussion': 0, 'updates': 0, 'ora': 0}) + self.assertEqual(response.data['count_by_app_name'], {'discussion': 0, 'updates': 0, 'grading': 0}) def test_get_expiry_days_in_count_view(self): """ diff --git a/openedx/core/djangoapps/olx_rest_api/views.py b/openedx/core/djangoapps/olx_rest_api/views.py index 8977b131530b..1083ee06f68d 100644 --- a/openedx/core/djangoapps/olx_rest_api/views.py +++ b/openedx/core/djangoapps/olx_rest_api/views.py @@ -13,7 +13,7 @@ from openedx.core.lib.api.view_utils import view_auth_classes from xmodule.modulestore.django import modulestore -from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_blockstore +from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_learning_core @api_view(['GET']) @@ -22,7 +22,7 @@ def get_block_olx(request, usage_key_str): """ Given a modulestore XBlock usage ID (block-v1:...), get its OLX and a list of any static asset files it uses. - (There are other APIs for getting the OLX of Blockstore XBlocks.) + (There are other APIs for getting the OLX of Learning Core XBlocks.) """ # Parse the usage key: try: @@ -48,7 +48,7 @@ def serialize_block(block_key): return block = modulestore().get_item(block_key) - serialized_blocks[block_key] = serialize_modulestore_block_for_blockstore(block) + serialized_blocks[block_key] = serialize_modulestore_block_for_learning_core(block) if block.has_children: for child_id in block.children: @@ -103,7 +103,7 @@ def get_block_exportfs_file(request, usage_key_str, path): raise PermissionDenied("You must be a member of the course team in Studio to export OLX using this API.") block = modulestore().get_item(usage_key) - serialized = serialize_modulestore_block_for_blockstore(block) + serialized = serialize_modulestore_block_for_learning_core(block) static_file = None for f in serialized.static_files: if f.name == path: diff --git a/openedx/core/djangoapps/theming/management/commands/compile_sass.py b/openedx/core/djangoapps/theming/management/commands/compile_sass.py index 765ef98aeac3..fbfdd2f222a4 100644 --- a/openedx/core/djangoapps/theming/management/commands/compile_sass.py +++ b/openedx/core/djangoapps/theming/management/commands/compile_sass.py @@ -1,13 +1,14 @@ """ Management command for compiling sass. -""" +DEPRECATED in favor of `npm run compile-sass`. +""" +import shlex -from django.core.management import BaseCommand, CommandError -from paver.easy import call_task +from django.core.management import BaseCommand +from django.conf import settings -from openedx.core.djangoapps.theming.helpers import get_theme_base_dirs, get_themes, is_comprehensive_theming_enabled -from pavelib.assets import ALL_SYSTEMS +from pavelib.assets import run_deprecated_command_wrapper class Command(BaseCommand): @@ -15,7 +16,7 @@ class Command(BaseCommand): Compile theme sass and collect theme assets. """ - help = 'Compile and collect themed assets...' + help = "DEPRECATED. Use 'npm run compile-sass' instead." # NOTE (CCB): This allows us to compile static assets in Docker containers without database access. requires_system_checks = [] @@ -28,7 +29,7 @@ def add_arguments(self, parser): parser (django.core.management.base.CommandParser): parsed for parsing command line arguments. """ parser.add_argument( - 'system', type=str, nargs='*', default=ALL_SYSTEMS, + 'system', type=str, nargs='*', default=["lms", "cms"], help="lms or studio", ) @@ -55,7 +56,7 @@ def add_arguments(self, parser): '--force', action='store_true', default=False, - help="Force full compilation", + help="DEPRECATED. Full recompilation is now always forced.", ) parser.add_argument( '--debug', @@ -64,77 +65,48 @@ def add_arguments(self, parser): help="Disable Sass compression", ) - @staticmethod - def parse_arguments(*args, **options): # pylint: disable=unused-argument - """ - Parse and validate arguments for compile_sass command. - - Args: - *args: Positional arguments passed to the update_assets command - **options: optional arguments passed to the update_assets command - Returns: - A tuple containing parsed values for themes, system, source comments and output style. - 1. system (list): list of system names for whom to compile theme sass e.g. 'lms', 'cms' - 2. theme_dirs (list): list of Theme objects - 3. themes (list): list of Theme objects - 4. force (bool): Force full compilation - 5. debug (bool): Disable Sass compression - """ - system = options.get("system", ALL_SYSTEMS) - given_themes = options.get("themes", ["all"]) - theme_dirs = options.get("theme_dirs", None) - - force = options.get("force", True) - debug = options.get("debug", True) - - if theme_dirs: - available_themes = {} - for theme_dir in theme_dirs: - available_themes.update({t.theme_dir_name: t for t in get_themes(theme_dir)}) - else: - theme_dirs = get_theme_base_dirs() - available_themes = {t.theme_dir_name: t for t in get_themes()} - - if 'no' in given_themes or 'all' in given_themes: - # Raise error if 'all' or 'no' is present and theme names are also given. - if len(given_themes) > 1: - raise CommandError("Invalid themes value, It must either be 'all' or 'no' or list of themes.") - # Raise error if any of the given theme name is invalid - # (theme name would be invalid if it does not exist in themes directory) - elif (not set(given_themes).issubset(list(available_themes.keys()))) and is_comprehensive_theming_enabled(): - raise CommandError( - "Given themes '{themes}' do not exist inside any of the theme directories '{theme_dirs}'".format( - themes=", ".join(set(given_themes) - set(available_themes.keys())), - theme_dirs=theme_dirs, - ), - ) - - if "all" in given_themes: - themes = list(available_themes.values()) - elif "no" in given_themes: - themes = [] - else: - # convert theme names to Theme objects, this will remove all themes if theming is disabled - themes = [available_themes.get(theme) for theme in given_themes if theme in available_themes] - - return system, theme_dirs, themes, force, debug - def handle(self, *args, **options): """ Handle compile_sass command. """ - system, theme_dirs, themes, force, debug = self.parse_arguments(*args, **options) - themes = [theme.theme_dir_name for theme in themes] - - if options.get("themes", None) and not is_comprehensive_theming_enabled(): - # log a warning message to let the user know that asset compilation for themes is skipped - self.stdout.write( - self.style.WARNING( - "Skipping theme asset compilation: enable theming to process themed assets" + systems = set( + {"lms": "lms", "cms": "cms", "studio": "cms"}[sys] + for sys in options.get("system", ["lms", "cms"]) + ) + theme_dirs = options.get("theme_dirs") or settings.COMPREHENSIVE_THEME_DIRS or [] + themes_option = options.get("themes") or [] # '[]' means 'all' + if not settings.ENABLE_COMPREHENSIVE_THEMING: + compile_themes = False + themes = [] + elif "no" in themes_option: + compile_themes = False + themes = [] + elif "all" in themes_option: + compile_themes = True + themes = [] + else: + compile_themes = True + themes = themes_option + run_deprecated_command_wrapper( + old_command="./manage.py [lms|cms] compile_sass", + ignored_old_flags=list(set(["force"]) & set(options)), + new_command=shlex.join([ + "npm", + "run", + ("compile-sass-dev" if options.get("debug") else "compile-sass"), + "--", + *(["--skip-lms"] if "lms" not in systems else []), + *(["--skip-cms"] if "cms" not in systems else []), + *(["--skip-themes"] if not compile_themes else []), + *( + arg + for theme_dir in theme_dirs + for arg in ["--theme-dir", str(theme_dir)] ), - ) - - call_task( - 'pavelib.assets.compile_sass', - options={'system': system, 'theme_dirs': theme_dirs, 'themes': themes, 'force': force, 'debug': debug}, + *( + arg + for theme in themes + for arg in ["--theme", theme] + ), + ]), ) diff --git a/openedx/core/djangoapps/theming/tests/test_commands.py b/openedx/core/djangoapps/theming/tests/test_commands.py index b4abee9bed0e..7ca56f604a41 100644 --- a/openedx/core/djangoapps/theming/tests/test_commands.py +++ b/openedx/core/djangoapps/theming/tests/test_commands.py @@ -2,57 +2,23 @@ Tests for Management commands of comprehensive theming. """ -import pytest -from django.core.management import CommandError, call_command -from django.test import TestCase +from django.core.management import call_command +from django.test import TestCase, override_settings +from unittest.mock import patch -from openedx.core.djangoapps.theming.helpers import get_themes -from openedx.core.djangoapps.theming.management.commands.compile_sass import Command +import pavelib.assets class TestUpdateAssets(TestCase): """ Test comprehensive theming helper functions. """ - def setUp(self): - super().setUp() - self.themes = get_themes() - def test_errors_for_invalid_arguments(self): - """ - Test update_asset command. - """ - # make sure error is raised for invalid theme list - with pytest.raises(CommandError): - call_command("compile_sass", themes=["all", "test-theme"]) - - # make sure error is raised for invalid theme list - with pytest.raises(CommandError): - call_command("compile_sass", themes=["no", "test-theme"]) - - # make sure error is raised for invalid theme list - with pytest.raises(CommandError): - call_command("compile_sass", themes=["all", "no"]) - - # make sure error is raised for invalid theme list - with pytest.raises(CommandError): - call_command("compile_sass", themes=["test-theme", "non-existing-theme"]) - - def test_parse_arguments(self): - """ - Test parse arguments method for update_asset command. - """ - # make sure compile_sass picks all themes when called with 'themes=all' option - parsed_args = Command.parse_arguments(themes=["all"]) - self.assertCountEqual(parsed_args[2], get_themes()) - - # make sure compile_sass picks no themes when called with 'themes=no' option - parsed_args = Command.parse_arguments(themes=["no"]) - self.assertCountEqual(parsed_args[2], []) - - # make sure compile_sass picks only specified themes - parsed_args = Command.parse_arguments(themes=["test-theme"]) - self.assertCountEqual( - parsed_args[2], - [theme for theme in get_themes() if theme.theme_dir_name == "test-theme"] + @patch.object(pavelib.assets, 'sh') + @override_settings(COMPREHENSIVE_THEME_DIRS='common/test') + def test_deprecated_wrapper(self, mock_sh): + call_command('compile_sass', '--themes', 'fake-theme1', 'fake-theme2') + assert mock_sh.called_once_with( + "npm run compile-sass -- " + + "--theme-dir common/test --theme fake-theme-1 --theme fake-theme-2" ) diff --git a/openedx/core/djangoapps/user_api/accounts/tests/test_retirement_views.py b/openedx/core/djangoapps/user_api/accounts/tests/test_retirement_views.py index 0cc2754e4760..9d4efb2fa77c 100644 --- a/openedx/core/djangoapps/user_api/accounts/tests/test_retirement_views.py +++ b/openedx/core/djangoapps/user_api/accounts/tests/test_retirement_views.py @@ -30,25 +30,6 @@ from common.djangoapps.entitlements.models import CourseEntitlementSupportDetail from common.djangoapps.entitlements.tests.factories import CourseEntitlementFactory -from openedx.core.djangoapps.api_admin.models import ApiAccessRequest -from openedx.core.djangoapps.course_groups.models import CourseUserGroup, UnregisteredLearnerCohortAssignments -from openedx.core.djangoapps.credit.models import ( - CreditCourse, - CreditProvider, - CreditRequest, - CreditRequirement, - CreditRequirementStatus -) -from openedx.core.djangoapps.external_user_ids.models import ExternalIdType -from openedx.core.djangoapps.oauth_dispatch.jwt import create_jwt_for_user -from openedx.core.djangoapps.site_configuration.tests.factories import SiteFactory -from openedx.core.djangoapps.user_api.accounts.views import AccountRetirementPartnerReportView -from openedx.core.djangoapps.user_api.models import ( - RetirementState, - UserOrgTag, - UserRetirementPartnerReportingStatus, - UserRetirementStatus -) from common.djangoapps.student.models import ( AccountRecovery, CourseEnrollment, @@ -71,10 +52,31 @@ SuperuserFactory, UserFactory ) +from lms.djangoapps.certificates.api import get_certificate_for_user_id +from lms.djangoapps.certificates.tests.factories import GeneratedCertificateFactory +from openedx.core.djangoapps.api_admin.models import ApiAccessRequest +from openedx.core.djangoapps.course_groups.models import CourseUserGroup, UnregisteredLearnerCohortAssignments +from openedx.core.djangoapps.credit.models import ( + CreditCourse, + CreditProvider, + CreditRequest, + CreditRequirement, + CreditRequirementStatus +) +from openedx.core.djangoapps.external_user_ids.models import ExternalIdType +from openedx.core.djangoapps.oauth_dispatch.jwt import create_jwt_for_user +from openedx.core.djangoapps.site_configuration.tests.factories import SiteFactory +from openedx.core.djangoapps.user_api.accounts.views import AccountRetirementPartnerReportView +from openedx.core.djangoapps.user_api.models import ( + RetirementState, + UserOrgTag, + UserRetirementPartnerReportingStatus, + UserRetirementStatus +) from openedx.core.djangolib.testing.utils import skip_unless_lms -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order -from xmodule.modulestore.tests.factories import CourseFactory # lint-amnesty, pylint: disable=wrong-import-order from openedx.core.djangoapps.oauth_dispatch.tests.factories import ApplicationFactory, AccessTokenFactory +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory from ...tests.factories import UserOrgTagFactory from ..views import USER_PROFILE_PII, AccountRetirementView @@ -1346,6 +1348,46 @@ def setUp(self): self.headers['content_type'] = "application/json" self.url = reverse('accounts_retire') + def _data_sharing_consent_assertions(self): + """ + Helper method for asserting that ``DataSharingConsent`` objects are retired. + """ + self.consent.refresh_from_db() + assert self.retired_username == self.consent.username + test_users_data_sharing_consent = DataSharingConsent.objects.filter( + username=self.original_username + ) + assert not test_users_data_sharing_consent.exists() + + def _entitlement_support_detail_assertions(self): + """ + Helper method for asserting that ``CourseEntitleSupportDetail`` objects are retired. + """ + self.entitlement_support_detail.refresh_from_db() + assert '' == self.entitlement_support_detail.comments + + def _pending_enterprise_customer_user_assertions(self): + """ + Helper method for asserting that ``PendingEnterpriseCustomerUser`` objects are retired. + """ + self.pending_enterprise_user.refresh_from_db() + assert self.retired_email == self.pending_enterprise_user.user_email + pending_enterprise_users = PendingEnterpriseCustomerUser.objects.filter( + user_email=self.original_email + ) + assert not pending_enterprise_users.exists() + + def _sapsf_audit_assertions(self): + """ + Helper method for asserting that ``SapSuccessFactorsLearnerDataTransmissionAudit`` objects are retired. + """ + self.sapsf_audit.refresh_from_db() + assert '' == self.sapsf_audit.sapsf_user_id + audits_for_original_user_id = SapSuccessFactorsLearnerDataTransmissionAudit.objects.filter( + sapsf_user_id=self.test_user.id, + ) + assert not audits_for_original_user_id.exists() + def post_and_assert_status(self, data, expected_status=status.HTTP_204_NO_CONTENT): """ Helper function for making a request to the retire subscriptions endpoint, and asserting the status. @@ -1482,57 +1524,30 @@ def test_can_retire_users_datasharingconsent(self): AccountRetirementView.retire_users_data_sharing_consent(self.test_user.username, self.retired_username) self._data_sharing_consent_assertions() - def _data_sharing_consent_assertions(self): - """ - Helper method for asserting that ``DataSharingConsent`` objects are retired. - """ - self.consent.refresh_from_db() - assert self.retired_username == self.consent.username - test_users_data_sharing_consent = DataSharingConsent.objects.filter( - username=self.original_username - ) - assert not test_users_data_sharing_consent.exists() - def test_can_retire_users_sap_success_factors_audits(self): AccountRetirementView.retire_sapsf_data_transmission(self.test_user) self._sapsf_audit_assertions() - def _sapsf_audit_assertions(self): - """ - Helper method for asserting that ``SapSuccessFactorsLearnerDataTransmissionAudit`` objects are retired. - """ - self.sapsf_audit.refresh_from_db() - assert '' == self.sapsf_audit.sapsf_user_id - audits_for_original_user_id = SapSuccessFactorsLearnerDataTransmissionAudit.objects.filter( - sapsf_user_id=self.test_user.id, - ) - assert not audits_for_original_user_id.exists() - def test_can_retire_user_from_pendingenterprisecustomeruser(self): AccountRetirementView.retire_user_from_pending_enterprise_customer_user(self.test_user, self.retired_email) self._pending_enterprise_customer_user_assertions() - def _pending_enterprise_customer_user_assertions(self): - """ - Helper method for asserting that ``PendingEnterpriseCustomerUser`` objects are retired. - """ - self.pending_enterprise_user.refresh_from_db() - assert self.retired_email == self.pending_enterprise_user.user_email - pending_enterprise_users = PendingEnterpriseCustomerUser.objects.filter( - user_email=self.original_email - ) - assert not pending_enterprise_users.exists() - def test_course_entitlement_support_detail_comments_are_retired(self): AccountRetirementView.retire_entitlement_support_detail(self.test_user) self._entitlement_support_detail_assertions() - def _entitlement_support_detail_assertions(self): + def test_clear_pii_from_certificate_records(self): """ - Helper method for asserting that ``CourseEntitleSupportDetail`` objects are retired. + Test to verify a learner's name is scrubbed from associated certificate records when the AccountRetirementView's + `clear_pii_from_certificate_records` static function is called. """ - self.entitlement_support_detail.refresh_from_db() - assert '' == self.entitlement_support_detail.comments + GeneratedCertificateFactory(course_id=self.course_key, name="Bob Loblaw", user=self.test_user) + cert = get_certificate_for_user_id(self.test_user.id, self.course_key) + assert cert.name == "Bob Loblaw" + + AccountRetirementView.clear_pii_from_certificate_records(self.test_user) + cert = get_certificate_for_user_id(self.test_user.id, self.course_key) + assert cert.name == "" @skip_unless_lms diff --git a/openedx/core/djangoapps/user_api/accounts/views.py b/openedx/core/djangoapps/user_api/accounts/views.py index cfe9872a95f8..720b3ba96af7 100644 --- a/openedx/core/djangoapps/user_api/accounts/views.py +++ b/openedx/core/djangoapps/user_api/accounts/views.py @@ -66,6 +66,7 @@ from openedx.core.djangoapps.user_authn.exceptions import AuthFailedError from openedx.core.lib.api.authentication import BearerAuthenticationAllowInactiveUser from openedx.core.lib.api.parsers import MergePatchParser +from lms.djangoapps.certificates.api import clear_pii_from_certificate_records_for_user from ..errors import AccountUpdateError, AccountValidationError, UserNotAuthorized, UserNotFound from ..message_types import DeletionNotificationMessage @@ -1144,9 +1145,8 @@ def post(self, request): } ``` - Retires the user with the given username. This includes - retiring this username, the associated email address, and - any other PII associated with this user. + Retires the user with the given username. This includes retiring this username, the associated email address, + and any other PII associated with this user. """ username = request.data['username'] @@ -1162,6 +1162,9 @@ def post(self, request): self.delete_users_profile_images(user) self.delete_users_country_cache(user) + # Retire user information from any certificate records associated with the learner + self.clear_pii_from_certificate_records(user) + # Retire data from Enterprise models self.retire_users_data_sharing_consent(username, retired_username) self.retire_sapsf_data_transmission(user) @@ -1197,8 +1200,8 @@ def post(self, request): @staticmethod def clear_pii_from_userprofile(user): """ - For the given user, sets all of the user's profile fields to some retired value. - This also deletes all ``SocialLink`` objects associated with this user's profile. + For the given user, sets all of the user's profile fields to some retired value. This also deletes all + ``SocialLink`` objects associated with this user's profile. """ for model_field, value_to_assign in USER_PROFILE_PII.items(): setattr(user.profile, model_field, value_to_assign) @@ -1250,12 +1253,19 @@ def retire_user_from_pending_enterprise_customer_user(user, retired_email): @staticmethod def retire_entitlement_support_detail(user): """ - Updates all CourseEntitleSupportDetail records for the given - user to have an empty ``comments`` field. + Updates all CourseEntitleSupportDetail records for the given user to have an empty ``comments`` field. """ for entitlement in CourseEntitlement.objects.filter(user_id=user.id): entitlement.courseentitlementsupportdetail_set.all().update(comments='') + @staticmethod + def clear_pii_from_certificate_records(user): + """ + Calls a utility function in the `certificates` Django app responsible for removing PII (name) from any + certificate records associated with the learner being retired. + """ + clear_pii_from_certificate_records_for_user(user) + class UsernameReplacementView(APIView): """ diff --git a/openedx/core/djangoapps/user_api/api.py b/openedx/core/djangoapps/user_api/api.py new file mode 100644 index 000000000000..813845e52582 --- /dev/null +++ b/openedx/core/djangoapps/user_api/api.py @@ -0,0 +1,29 @@ +""" +Python APIs exposed by the user_api app to other in-process apps. +""" + + +from openedx.core.djangoapps.user_api.models import UserRetirementRequest, UserRetirementStatus + + +def get_retired_user_ids(): + """ + Returns a list of learners' user_ids who have retired their account. This utility method removes any learners who + are in the "PENDING" retirement state, they have _requested_ retirement but have yet to have all their data purged. + These learners are still within their cooloff period where they can submit a request to restore their account. + + Args: + None + + Returns: + list[int] - A list of user ids of learners who have retired their account, minus any accounts currently in the + "PENDING" state. + """ + retired_user_ids = set(UserRetirementRequest.objects.values_list("user_id", flat=True)) + pending_retired_user_ids = set( + UserRetirementStatus.objects + .filter(current_state__state_name="PENDING") + .values_list("user_id", flat=True) + ) + + return list(retired_user_ids - pending_retired_user_ids) diff --git a/openedx/core/djangoapps/user_api/tests/factories.py b/openedx/core/djangoapps/user_api/tests/factories.py index c3b11367bd2d..9b6302ae3a1e 100644 --- a/openedx/core/djangoapps/user_api/tests/factories.py +++ b/openedx/core/djangoapps/user_api/tests/factories.py @@ -1,13 +1,20 @@ """Provides factories for User API models.""" -from factory import SubFactory +from factory import Sequence, SubFactory from factory.django import DjangoModelFactory from opaque_keys.edx.locator import CourseLocator from common.djangoapps.student.tests.factories import UserFactory -from ..models import UserCourseTag, UserOrgTag, UserPreference +from openedx.core.djangoapps.user_api.models import ( + RetirementState, + UserCourseTag, + UserOrgTag, + UserPreference, + UserRetirementRequest, + UserRetirementStatus, +) # Factories are self documenting @@ -40,3 +47,44 @@ class Meta: org = 'org' key = None value = None + + +class RetirementStateFactory(DjangoModelFactory): + """ + Factory class for generating RetirementState instances. + """ + class Meta: + model = RetirementState + + state_name = Sequence("STEP_{}".format) + state_execution_order = Sequence(lambda n: n * 10) + is_dead_end_state = False + required = False + + +class UserRetirementStatusFactory(DjangoModelFactory): + """ + Factory class for generating UserRetirementStatus instances. + """ + class Meta: + model = UserRetirementStatus + + user = SubFactory(UserFactory) + original_username = Sequence('learner_{}'.format) + original_email = Sequence("learner{}@email.org".format) + original_name = Sequence("Learner{} Shmearner".format) + retired_username = Sequence("retired__learner_{}".format) + retired_email = Sequence("returned__learner{}@retired.invalid".format) + current_state = None + last_state = None + responses = "" + + +class UserRetirementRequestFactory(DjangoModelFactory): + """ + Factory class for generating UserRetirementRequest instances. + """ + class Meta: + model = UserRetirementRequest + + user = SubFactory(UserFactory) diff --git a/openedx/core/djangoapps/user_api/tests/test_api.py b/openedx/core/djangoapps/user_api/tests/test_api.py new file mode 100644 index 000000000000..a929d1b8c643 --- /dev/null +++ b/openedx/core/djangoapps/user_api/tests/test_api.py @@ -0,0 +1,91 @@ +""" +Unit tests for the `user_api` app's public Python interface. +""" + + +from django.test import TestCase + +from common.djangoapps.student.tests.factories import UserFactory +from openedx.core.djangoapps.user_api.api import get_retired_user_ids +from openedx.core.djangoapps.user_api.models import ( + RetirementState, + UserRetirementRequest, + UserRetirementStatus, +) +from openedx.core.djangoapps.user_api.tests.factories import ( + RetirementStateFactory, + UserRetirementRequestFactory, + UserRetirementStatusFactory, +) + + +class UserApiRetirementTests(TestCase): + """ + Tests for utility functions exposed by the `user_api` app's public Python interface that are related to the user + retirement pipeline. + """ + + @classmethod + def setUpClass(cls): + """ + The retirement pipeline is not fully enabled by default. We must ensure that the required RetirementState's + exist before executing any of our unit tests. + """ + super().setUpClass() + cls.pending = RetirementStateFactory(state_name="PENDING") + cls.complete = RetirementStateFactory(state_name="COMPLETE") + + @classmethod + def tearDownClass(cls): + # Remove any retirement state objects that we created during this test suite run. + RetirementState.objects.all().delete() + super().tearDownClass() + + def tearDown(self): + # clear retirement requests and related data between each test + UserRetirementRequest.objects.all().delete() + UserRetirementStatus.objects.all().delete() + super().tearDown() + + def test_get_retired_user_ids(self): + """ + A unit test to verify that the only user id's returned from the `get_retired_user_ids` function are learners who + aren't in the "PENDING" state. + """ + user_pending = UserFactory() + # create a retirement request and status entry for a learner in the PENDING state + UserRetirementRequestFactory(user=user_pending) + UserRetirementStatusFactory(user=user_pending, current_state=self.pending, last_state=self.pending) + user_complete = UserFactory() + # create a retirement request and status entry for a learner in the COMPLETE state + UserRetirementRequestFactory(user=user_complete) + UserRetirementStatusFactory(user=user_complete, current_state=self.complete, last_state=self.complete) + + results = get_retired_user_ids() + assert len(results) == 1 + assert results == [user_complete.id] + + def test_get_retired_user_ids_no_results(self): + """ + A unit test to verify that if the only retirement requests pending are in the "PENDING" state, we don't return + any learners' user_ids when calling the `get_retired_user_ids` function. + """ + user_pending_1 = UserFactory() + # create a retirement request and status entry for a learner in the PENDING state + UserRetirementRequestFactory(user=user_pending_1) + UserRetirementStatusFactory( + user=user_pending_1, + current_state=self.pending, + last_state=self.pending, + ) + user_pending_2 = UserFactory() + # create a retirement request and status entry for a learner in the PENDING state + UserRetirementRequestFactory(user=user_pending_2) + UserRetirementStatusFactory( + user=user_pending_2, + current_state=self.pending, + last_state=self.pending, + ) + results = get_retired_user_ids() + assert len(results) == 0 + assert not results diff --git a/openedx/core/djangoapps/user_authn/toggles.py b/openedx/core/djangoapps/user_authn/toggles.py index 38d9dbae4016..3ca3b75e9703 100644 --- a/openedx/core/djangoapps/user_authn/toggles.py +++ b/openedx/core/djangoapps/user_authn/toggles.py @@ -33,3 +33,21 @@ def should_redirect_to_authn_microfrontend(): return configuration_helpers.get_value( 'ENABLE_AUTHN_MICROFRONTEND', settings.FEATURES.get('ENABLE_AUTHN_MICROFRONTEND') ) + + +# .. toggle_name: ENABLE_AUTO_GENERATED_USERNAME +# .. toggle_implementation: DjangoSetting +# .. toggle_default: False +# .. toggle_description: Set to True to enable auto-generation of usernames. +# .. toggle_use_cases: open_edx +# .. toggle_creation_date: 2024-02-20 +# .. toggle_warning: Changing this setting may affect user authentication, account management and discussions experience. + + +def is_auto_generated_username_enabled(): + """ + Checks if auto-generated username should be enabled. + """ + return configuration_helpers.get_value( + 'ENABLE_AUTO_GENERATED_USERNAME', settings.FEATURES.get('ENABLE_AUTO_GENERATED_USERNAME') + ) diff --git a/openedx/core/djangoapps/user_authn/views/register.py b/openedx/core/djangoapps/user_authn/views/register.py index cbb0cb49f453..659c90b3d2c4 100644 --- a/openedx/core/djangoapps/user_authn/views/register.py +++ b/openedx/core/djangoapps/user_authn/views/register.py @@ -63,8 +63,12 @@ RegistrationFormFactory, get_registration_extension_form ) +from openedx.core.djangoapps.user_authn.views.utils import get_auto_generated_username from openedx.core.djangoapps.user_authn.tasks import check_pwned_password_and_send_track_event -from openedx.core.djangoapps.user_authn.toggles import is_require_third_party_auth_enabled +from openedx.core.djangoapps.user_authn.toggles import ( + is_require_third_party_auth_enabled, + is_auto_generated_username_enabled +) from common.djangoapps.student.helpers import ( AccountValidationError, authenticate_new_user, @@ -574,6 +578,9 @@ def post(self, request): data = request.POST.copy() self._handle_terms_of_service(data) + if is_auto_generated_username_enabled() and 'username' not in data: + data['username'] = get_auto_generated_username(data) + try: data = StudentRegistrationRequested.run_filter(form_data=data) except StudentRegistrationRequested.PreventRegistration as exc: diff --git a/openedx/core/djangoapps/user_authn/views/tests/test_register.py b/openedx/core/djangoapps/user_authn/views/tests/test_register.py index 37aef7643a88..02d5a72074f5 100644 --- a/openedx/core/djangoapps/user_authn/views/tests/test_register.py +++ b/openedx/core/djangoapps/user_authn/views/tests/test_register.py @@ -65,6 +65,8 @@ password_validators_instruction_texts, password_validators_restrictions ) +ENABLE_AUTO_GENERATED_USERNAME = settings.FEATURES.copy() +ENABLE_AUTO_GENERATED_USERNAME['ENABLE_AUTO_GENERATED_USERNAME'] = True @ddt.ddt @@ -1861,6 +1863,117 @@ def test_rate_limiting_registration_view(self): assert response.status_code == 403 cache.clear() + @override_settings(FEATURES=ENABLE_AUTO_GENERATED_USERNAME) + def test_register_with_auto_generated_username(self): + """ + Test registration functionality with auto-generated username. + + This method tests the registration process when auto-generated username + feature is enabled. It creates a new user account, verifies that the user + account settings are correctly set, and checks if the user is successfully + logged in after registration. + """ + response = self.client.post(self.url, { + "email": self.EMAIL, + "name": self.NAME, + "password": self.PASSWORD, + "honor_code": "true", + }) + self.assertHttpOK(response) + + user = User.objects.get(email=self.EMAIL) + request = RequestFactory().get('/url') + request.user = user + account_settings = get_account_settings(request)[0] + + assert self.EMAIL == account_settings["email"] + assert not account_settings["is_active"] + assert self.NAME == account_settings["name"] + + # Verify that we've been logged in + # by trying to access a page that requires authentication + response = self.client.get(reverse("dashboard")) + self.assertHttpOK(response) + + @override_settings(FEATURES=ENABLE_AUTO_GENERATED_USERNAME) + def test_register_with_empty_name(self): + """ + Test registration field validations when ENABLE_AUTO_GENERATED_USERNAME is enabled. + + Sends a POST request to the registration endpoint with empty name field. + Expects a 400 Bad Request response with the corresponding validation error message for the name field. + """ + response = self.client.post(self.url, { + "email": "bob@example.com", + "name": "", + "password": "password", + "honor_code": "true", + }) + assert response.status_code == 400 + response_json = json.loads(response.content.decode('utf-8')) + self.assertDictEqual( + response_json, + { + "name": [{"user_message": 'Your legal name must be a minimum of one character long'}], + "error_code": "validation-error" + } + ) + + @override_settings(FEATURES=ENABLE_AUTO_GENERATED_USERNAME) + @mock.patch('openedx.core.djangoapps.user_authn.views.utils._get_username_prefix') + @mock.patch('openedx.core.djangoapps.user_authn.views.utils.random.choices') + @mock.patch('openedx.core.djangoapps.user_authn.views.utils.datetime') + @mock.patch('openedx.core.djangoapps.user_authn.views.utils.get_auto_generated_username') + def test_register_autogenerated_duplicate_username(self, + mock_get_auto_generated_username, + mock_datetime, + mock_choices, + mock_get_username_prefix): + """ + Test registering a user with auto-generated username where a duplicate username conflict occurs. + + Mocks various utilities to control the auto-generated username process and verifies the response content + when a duplicate username conflict happens during user registration. + """ + mock_datetime.now.return_value.strftime.return_value = '24 03' + mock_choices.return_value = ['X', 'Y', 'Z', 'A'] # Fixed random string for testing + + mock_get_username_prefix.return_value = None + + current_year_month = f"{datetime.now().year % 100}{datetime.now().month:02d}_" + random_string = 'XYZA' + expected_username = current_year_month + random_string + mock_get_auto_generated_username.return_value = expected_username + + # Register the first user + response = self.client.post(self.url, { + "email": self.EMAIL, + "name": self.NAME, + "password": self.PASSWORD, + "honor_code": "true", + }) + self.assertHttpOK(response) + # Try to create a second user with the same username + response = self.client.post(self.url, { + "email": "someone+else@example.com", + "name": "Someone Else", + "password": self.PASSWORD, + "honor_code": "true", + }) + + assert response.status_code == 409 + response_json = json.loads(response.content.decode('utf-8')) + response_json.pop('username_suggestions') + self.assertDictEqual( + response_json, + { + "username": [{ + "user_message": AUTHN_USERNAME_CONFLICT_MSG, + }], + "error_code": "duplicate-username" + } + ) + def _assert_fields_match(self, actual_field, expected_field): """ Assert that the actual field and the expected field values match. diff --git a/openedx/core/djangoapps/user_authn/views/tests/test_utils.py b/openedx/core/djangoapps/user_authn/views/tests/test_utils.py new file mode 100644 index 000000000000..c931fe339901 --- /dev/null +++ b/openedx/core/djangoapps/user_authn/views/tests/test_utils.py @@ -0,0 +1,77 @@ +""" +Tests for user utils functionality. +""" +from django.test import TestCase +from datetime import datetime +from openedx.core.djangoapps.user_authn.views.utils import get_auto_generated_username, _get_username_prefix +import ddt +from unittest.mock import patch + + +@ddt.ddt +class TestGenerateUsername(TestCase): + """ + Test case for the get_auto_generated_username function. + """ + + @ddt.data( + ({'first_name': 'John', 'last_name': 'Doe'}, "JD"), + ({'name': 'Jane Smith'}, "JS"), + ({'name': 'Jane'}, "J"), + ({'name': 'John Doe Smith'}, "JD") + ) + @ddt.unpack + def test_generate_username_from_data(self, data, expected_initials): + """ + Test get_auto_generated_username function. + """ + random_string = 'XYZA' + current_year_month = f"_{datetime.now().year % 100}{datetime.now().month:02d}_" + + with patch('openedx.core.djangoapps.user_authn.views.utils.random.choices') as mock_choices: + mock_choices.return_value = ['X', 'Y', 'Z', 'A'] + + username = get_auto_generated_username(data) + + expected_username = expected_initials + current_year_month + random_string + self.assertEqual(username, expected_username) + + @ddt.data( + ({'first_name': 'John', 'last_name': 'Doe'}, "JD"), + ({'name': 'Jane Smith'}, "JS"), + ({'name': 'Jane'}, "J"), + ({'name': 'John Doe Smith'}, "JD"), + ({'first_name': 'John Doe', 'last_name': 'Smith'}, "JD"), + ({}, None), + ({'first_name': '', 'last_name': ''}, None), + ({'name': ''}, None), + ({'first_name': '阿提亚', 'last_name': '阿提亚'}, "AT"), + ({'first_name': 'أحمد', 'last_name': 'محمد'}, "HM"), + ({'name': 'أحمد محمد'}, "HM"), + ) + @ddt.unpack + def test_get_username_prefix(self, data, expected_initials): + """ + Test _get_username_prefix function. + """ + username_prefix = _get_username_prefix(data) + self.assertEqual(username_prefix, expected_initials) + + @patch('openedx.core.djangoapps.user_authn.views.utils._get_username_prefix') + @patch('openedx.core.djangoapps.user_authn.views.utils.random.choices') + @patch('openedx.core.djangoapps.user_authn.views.utils.datetime') + def test_get_auto_generated_username_no_prefix(self, mock_datetime, mock_choices, mock_get_username_prefix): + """ + Test get_auto_generated_username function when no name data is provided. + """ + mock_datetime.now.return_value.strftime.return_value = f"{datetime.now().year % 100} {datetime.now().month:02d}" + mock_choices.return_value = ['X', 'Y', 'Z', 'A'] # Fixed random string for testing + + mock_get_username_prefix.return_value = None + + current_year_month = f"{datetime.now().year % 100}{datetime.now().month:02d}_" + random_string = 'XYZA' + expected_username = current_year_month + random_string + + username = get_auto_generated_username({}) + self.assertEqual(username, expected_username) diff --git a/openedx/core/djangoapps/user_authn/views/utils.py b/openedx/core/djangoapps/user_authn/views/utils.py index ac8e2c3950e4..c6107923a3f1 100644 --- a/openedx/core/djangoapps/user_authn/views/utils.py +++ b/openedx/core/djangoapps/user_authn/views/utils.py @@ -1,17 +1,24 @@ """ User Auth Views Utils """ +import logging +import re from django.conf import settings from django.contrib import messages from django.utils.translation import gettext as _ from ipware.ip import get_client_ip +from text_unidecode import unidecode from common.djangoapps import third_party_auth from common.djangoapps.third_party_auth import pipeline from common.djangoapps.third_party_auth.models import clean_username from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers from openedx.core.djangoapps.geoinfo.api import country_code_from_ip +import random +import string +from datetime import datetime +log = logging.getLogger(__name__) API_V1 = 'v1' UUID4_REGEX = '[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}' ENTERPRISE_ENROLLMENT_URL_REGEX = fr'/enterprise/{UUID4_REGEX}/course/{settings.COURSE_KEY_REGEX}/enroll' @@ -108,3 +115,56 @@ def get_mfe_context(request, redirect_to, tpa_hint=None): 'countryCode': country_code, }) return context + + +def _get_username_prefix(data): + """ + Get the username prefix (name initials) based on the provided data. + + Args: + - data (dict): Registration payload. + + Returns: + - str: Name initials or None. + """ + username_regex_partial = settings.USERNAME_REGEX_PARTIAL + full_name = '' + if data.get('first_name', '').strip() and data.get('last_name', '').strip(): + full_name = f"{unidecode(data.get('first_name', ''))} {unidecode(data.get('last_name', ''))}" + elif data.get('name', '').strip(): + full_name = unidecode(data['name']) + + if full_name.strip(): + full_name = re.findall(username_regex_partial, full_name)[0] + name_initials = "".join([name_part[0] for name_part in full_name.split()[:2]]) + return name_initials.upper() if name_initials else None + + return None + + +def get_auto_generated_username(data): + """ + Generate username based on learner's name initials, current date and configurable random string. + + This function creates a username in the format __ + + The length of random string is determined by AUTO_GENERATED_USERNAME_RANDOM_STRING_LENGTH setting. + + Args: + - data (dict): Registration payload. + + Returns: + - str: username. + """ + current_year, current_month = datetime.now().strftime('%y %m').split() + + random_string = ''.join(random.choices( + string.ascii_uppercase + string.digits, + k=settings.AUTO_GENERATED_USERNAME_RANDOM_STRING_LENGTH)) + + username_prefix = _get_username_prefix(data) + username_suffix = f"{current_year}{current_month}_{random_string}" + + # We generate the username regardless of whether the name is empty or invalid. We do this + # because the name validations occur later, ensuring that users cannot create an account without a valid name. + return f"{username_prefix}_{username_suffix}" if username_prefix else username_suffix diff --git a/openedx/core/djangoapps/util/apps.py b/openedx/core/djangoapps/util/apps.py index 843082ea1a16..e194fd7b5110 100644 --- a/openedx/core/djangoapps/util/apps.py +++ b/openedx/core/djangoapps/util/apps.py @@ -18,4 +18,5 @@ def ready(self): """ Registers signal handlers at startup. """ - import openedx.core.djangoapps.util.signals # lint-amnesty, pylint: disable=unused-import, unused-variable + import openedx.core.djangoapps.util.signals # pylint: disable=unused-import, unused-variable + import openedx.core.djangoapps.util.checks # pylint: disable=unused-import, unused-variable diff --git a/openedx/core/djangoapps/util/checks.py b/openedx/core/djangoapps/util/checks.py new file mode 100644 index 000000000000..bcde2fe620d1 --- /dev/null +++ b/openedx/core/djangoapps/util/checks.py @@ -0,0 +1,36 @@ +""" +Miscellaneous system checks +""" +from django.conf import settings +from django.core import checks + + +_DEVSTACK_SETTINGS_MODULES = [ + "lms.envs.devstack", + "lms.envs.devstack_docker", + "lms.envs.devstack_optimized", + "lms.envs.devstack_with_worker", + "cms.envs.devstack", + "cms.envs.devstack_docker", + "cms.envs.devstack_optimized", + "cms.envs.devstack_with_worker", +] + + +@checks.register(checks.Tags.compatibility) +def warn_if_devstack_settings(**kwargs): + """ + Raises a warning if we're using any Devstack settings file. + """ + if settings.SETTINGS_MODULE in _DEVSTACK_SETTINGS_MODULES: + return [ + checks.Warning( + "Open edX Devstack is deprecated, so the Django settings module you are using " + f"({settings.SETTINGS_MODULE}) will be removed from openedx/edx-platform by October 2024. " + "Please either migrate off of Devstack, or modify your Devstack fork to work with an externally-" + "managed Django settings file. " + "For details and discussion, see: https://github.com/openedx/public-engineering/issues/247.", + id="openedx.core.djangoapps.util.W247", + ), + ] + return [] diff --git a/openedx/core/djangoapps/verified_track_content/tests/test_partition_scheme.py b/openedx/core/djangoapps/verified_track_content/tests/test_partition_scheme.py index 87d2235e7194..a2667bfa1c0b 100644 --- a/openedx/core/djangoapps/verified_track_content/tests/test_partition_scheme.py +++ b/openedx/core/djangoapps/verified_track_content/tests/test_partition_scheme.py @@ -12,7 +12,7 @@ from common.djangoapps.student.tests.factories import UserFactory from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase # lint-amnesty, pylint: disable=wrong-import-order from xmodule.modulestore.tests.factories import CourseFactory # lint-amnesty, pylint: disable=wrong-import-order -from xmodule.partitions.partitions import MINIMUM_STATIC_PARTITION_ID, UserPartition, ReadOnlyUserPartitionError # lint-amnesty, pylint: disable=wrong-import-order +from xmodule.partitions.partitions import MINIMUM_UNUSED_PARTITION_ID, UserPartition, ReadOnlyUserPartitionError # lint-amnesty, pylint: disable=wrong-import-order from ..partition_scheme import ENROLLMENT_GROUP_IDS, EnrollmentTrackPartitionScheme, EnrollmentTrackUserPartition @@ -63,11 +63,11 @@ def test_from_json_not_supported(self): def test_group_ids(self): """ - Test that group IDs are all less than MINIMUM_STATIC_PARTITION_ID (to avoid overlapping + Test that group IDs are all less than MINIMUM_UNUSED_PARTITION_ID (to avoid overlapping with group IDs associated with cohort and random user partitions). """ for mode in ENROLLMENT_GROUP_IDS: - assert ENROLLMENT_GROUP_IDS[mode]['id'] < MINIMUM_STATIC_PARTITION_ID + assert ENROLLMENT_GROUP_IDS[mode]['id'] < MINIMUM_UNUSED_PARTITION_ID @staticmethod def get_group_by_name(partition, name): diff --git a/openedx/core/djangoapps/xblock/README.rst b/openedx/core/djangoapps/xblock/README.rst index 0afe70b9d136..245af2259932 100644 --- a/openedx/core/djangoapps/xblock/README.rst +++ b/openedx/core/djangoapps/xblock/README.rst @@ -1,3 +1,8 @@ +This README was written back when the new runtime was backed by Blockstore. +Now that the runtime is backed by Learning Core, this README is out of date. +We need to audit and update it as part of +`this task `_. + XBlock App Suite (New) ====================== diff --git a/openedx/core/djangoapps/xblock/apps.py b/openedx/core/djangoapps/xblock/apps.py index afebd2ec71c1..5ba2361322ec 100644 --- a/openedx/core/djangoapps/xblock/apps.py +++ b/openedx/core/djangoapps/xblock/apps.py @@ -33,10 +33,7 @@ def get_site_root_url(self): def get_learning_context_params(self): """ Get additional kwargs that are passed to learning context implementations - (LearningContext subclass constructors). For example, this can be used to - specify that the course learning context should load the course's list of - blocks from the _draft_ version of the course in studio, but from the - published version of the course in the LMS. + (LearningContext subclass constructors). """ return {} @@ -68,8 +65,6 @@ class StudioXBlockAppConfig(XBlockAppConfig): Studio-specific configuration of the XBlock Runtime django app. """ - BLOCKSTORE_DRAFT_NAME = "studio_draft" - def get_runtime_system_params(self): """ Get the XBlockRuntimeSystem parameters appropriate for viewing and/or @@ -91,14 +86,9 @@ def get_site_root_url(self): def get_learning_context_params(self): """ Get additional kwargs that are passed to learning context implementations - (LearningContext subclass constructors). For example, this can be used to - specify that the course learning context should load the course's list of - blocks from the _draft_ version of the course in studio, but from the - published version of the course in the LMS. - """ - return { - "use_draft": self.BLOCKSTORE_DRAFT_NAME, - } + (LearningContext subclass constructors). + """ + return {} def get_xblock_app_config(): diff --git a/openedx/core/djangoapps/xblock/learning_context/learning_context.py b/openedx/core/djangoapps/xblock/learning_context/learning_context.py index 72a6a7645e9f..1ac621ef244f 100644 --- a/openedx/core/djangoapps/xblock/learning_context/learning_context.py +++ b/openedx/core/djangoapps/xblock/learning_context/learning_context.py @@ -53,15 +53,8 @@ def can_view_block(self, user, usage_key): # pylint: disable=unused-argument def definition_for_usage(self, usage_key, **kwargs): """ - Given a usage key for an XBlock in this context, return the - BundleDefinitionLocator which specifies the actual XBlock definition - (as a path to an OLX in a specific blockstore bundle). + Given a usage key in this context, return the key indicating the actual XBlock definition. - usage_key: the UsageKeyV2 subclass used for this learning context - - kwargs: optional additional parameters unique to the learning context - - Must return a BundleDefinitionLocator if the XBlock exists in this - context, or None otherwise. + Retuns None if the usage key doesn't exist in this context. """ raise NotImplementedError diff --git a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py index 2875c4b468d5..dc5a85de3a02 100644 --- a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py @@ -21,7 +21,7 @@ from xblock.fields import Field, Scope, ScopeIds from xblock.field_data import FieldData -from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_blockstore +from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_learning_core from ..learning_context.manager import get_learning_context_impl from .runtime import XBlockRuntime @@ -234,11 +234,7 @@ def save_block(self, block): log.warning("User %s does not have permission to edit %s", self.user.username, block.scope_ids.usage_id) raise RuntimeError("You do not have permission to edit this XBlock") - # We need Blockstore's serialization so we don't have `url_name` showing - # up in all the OLX. TODO: Rename this later, after we figure out what - # other changes we need to make in the serialization as part of the - # Blockstore -> Learning Core conversion. - serialized = serialize_modulestore_block_for_blockstore(block) + serialized = serialize_modulestore_block_for_learning_core(block) now = datetime.now(tz=timezone.utc) usage_key = block.scope_ids.usage_id with atomic(): diff --git a/openedx/core/djangoapps/xblock/runtime/runtime.py b/openedx/core/djangoapps/xblock/runtime/runtime.py index 15e0f3f0617e..5746af491d10 100644 --- a/openedx/core/djangoapps/xblock/runtime/runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/runtime.py @@ -219,14 +219,14 @@ def applicable_aside_types(self, block: XBlock): def parse_xml_file(self, fileobj): # Deny access to the inherited method - raise NotImplementedError("XML Serialization is only supported with BlockstoreXBlockRuntime") + raise NotImplementedError("XML Serialization is only supported with LearningCoreXBlockRuntime") def add_node_as_child(self, block, node): """ Called by XBlock.parse_xml to treat a child node as a child block. """ # Deny access to the inherited method - raise NotImplementedError("XML Serialization is only supported with BlockstoreXBlockRuntime") + raise NotImplementedError("XML Serialization is only supported with LearningCoreXBlockRuntime") def service(self, block: XBlock, service_name: str): """ @@ -261,8 +261,8 @@ def service(self, block: XBlock, service_name: str): return DjangoXBlockUserService( self.user, - # The value should be updated to whether the user is staff in the context when Blockstore runtime adds - # support for courses. + # The value should be updated to whether the user is staff in the context when Learning Core runtime + # adds support for courses. user_is_staff=self.user.is_staff, # type: ignore anonymous_user_id=self.anonymous_student_id, # See the docstring of `DjangoXBlockUserService`. @@ -437,7 +437,7 @@ def __init__( student_data_mode: Specifies whether student data should be kept in a temporary in-memory store (e.g. Studio) or persisted forever in the database. - runtime_class: What runtime to use, e.g. BlockstoreXBlockRuntime + runtime_class: What runtime to use, e.g. LearningCoreXBlockRuntime """ self.handler_url = handler_url self.id_reader = id_reader or OpaqueKeyReader() diff --git a/openedx/core/djangoapps/xblock/runtime/shims.py b/openedx/core/djangoapps/xblock/runtime/shims.py index 18aa41eb91f3..c306b82bdc8b 100644 --- a/openedx/core/djangoapps/xblock/runtime/shims.py +++ b/openedx/core/djangoapps/xblock/runtime/shims.py @@ -102,7 +102,7 @@ def get_python_lib_zip(self): Only used for capa problems. """ - # TODO: load the python code from Blockstore. Ensure it's not publicly accessible. + # TODO: load the python code from Learning Core. Ensure it's not publicly accessible. return None @property @@ -166,9 +166,8 @@ def resources_fs(self): """ A filesystem that XBlocks can use to read large binary assets. """ - # TODO: implement this to serve any static assets that - # self._active_block has in its blockstore "folder". But this API should - # be deprecated and we should instead get compatible XBlocks to use a + # TODO: implement this to serve any static assets that self._active_block has. + # But this API should be deprecated and we should instead get compatible XBlocks to use a # runtime filesystem service. Some initial exploration of that (as well # as of the 'FileField' concept) has been done and is included in the # XBlock repo at xblock.reference.plugins.FSService and is available in diff --git a/openedx/core/lib/blockstore_api/__init__.py b/openedx/core/lib/blockstore_api/__init__.py deleted file mode 100644 index 855d8a1f96a0..000000000000 --- a/openedx/core/lib/blockstore_api/__init__.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -API Client for Blockstore - -TODO: This should all get ripped out. - -TODO: This wrapper is extraneous now that Blockstore-as-a-service isn't supported. - This whole directory tree should be removed by https://github.com/openedx/blockstore/issues/296. -""" -from blockstore.apps.api.data import ( - BundleFileData, -) -from blockstore.apps.api.exceptions import ( - CollectionNotFound, - BundleNotFound, - DraftNotFound, - BundleVersionNotFound, - BundleFileNotFound, - BundleStorageError, -) -from blockstore.apps.api.methods import ( - # Collections: - get_collection, - create_collection, - update_collection, - delete_collection, - # Bundles: - get_bundles, - get_bundle, - create_bundle, - update_bundle, - delete_bundle, - # Drafts: - get_draft, - get_or_create_bundle_draft, - write_draft_file, - set_draft_link, - commit_draft, - delete_draft, - # Bundles or drafts: - get_bundle_files, - get_bundle_files_dict, - get_bundle_file_metadata, - get_bundle_file_data, - get_bundle_version, - get_bundle_version_files, - # Links: - get_bundle_links, - get_bundle_version_links, - # Misc: - force_browser_url, -) diff --git a/openedx/core/lib/blockstore_api/db_routers.py b/openedx/core/lib/blockstore_api/db_routers.py deleted file mode 100644 index fd0ff50c9510..000000000000 --- a/openedx/core/lib/blockstore_api/db_routers.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Blockstore database router. - -Blockstore started life as an IDA, but is now a Django app plugin within edx-platform. -This router exists to smooth blockstore's transition into edxapp. -""" -from django.conf import settings - - -class BlockstoreRouter: - """ - A Database Router that uses the ``blockstore`` database, if it's configured in settings. - """ - ROUTE_APP_LABELS = {'bundles'} - DATABASE_NAME = 'blockstore' - - def _use_blockstore(self, model): - """ - Return True if the given model should use the blockstore database. - - Ensures that a ``blockstore`` database is configured, and checks the ``model``'s app label. - """ - return (self.DATABASE_NAME in settings.DATABASES) and (model._meta.app_label in self.ROUTE_APP_LABELS) - - def db_for_read(self, model, **hints): # pylint: disable=unused-argument - """ - Use the BlockstoreRouter.DATABASE_NAME when reading blockstore app tables. - """ - if self._use_blockstore(model): - return self.DATABASE_NAME - return None - - def db_for_write(self, model, **hints): # pylint: disable=unused-argument - """ - Use the BlockstoreRouter.DATABASE_NAME when writing to blockstore app tables. - """ - if self._use_blockstore(model): - return self.DATABASE_NAME - return None - - def allow_relation(self, obj1, obj2, **hints): # pylint: disable=unused-argument - """ - Allow relations if both objects are blockstore app models. - """ - if self._use_blockstore(obj1) and self._use_blockstore(obj2): - return True - return None - - def allow_migrate(self, db, app_label, model_name=None, **hints): # pylint: disable=unused-argument - """ - Ensure the blockstore tables only appear in the blockstore database. - """ - if model_name is not None: - model = hints.get('model') - if model is not None and self._use_blockstore(model): - return db == self.DATABASE_NAME - if db == self.DATABASE_NAME: - return False - - return None diff --git a/openedx/core/lib/blockstore_api/tests/base.py b/openedx/core/lib/blockstore_api/tests/base.py deleted file mode 100644 index 1d202d7671b5..000000000000 --- a/openedx/core/lib/blockstore_api/tests/base.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Common code for tests that work with Blockstore -""" -from unittest import mock -from urllib.parse import urlparse - -from django.test.client import RequestFactory - - -class BlockstoreAppTestMixin: - """ - Sets up the environment for tests to be run using the installed Blockstore app. - """ - def setUp(self): - """ - Ensure there's an active request, so that bundle file URLs can be made absolute. - """ - super().setUp() - - # Patch the blockstore get_current_request to use our live_server_url - mock.patch('blockstore.apps.api.methods.get_current_request', - mock.Mock(return_value=self._get_current_request())).start() - self.addCleanup(mock.patch.stopall) - - def _get_current_request(self): - """ - Returns a request object using the live_server_url, if available. - """ - request_args = {} - if hasattr(self, 'live_server_url'): - live_server_url = urlparse(self.live_server_url) - name, port = live_server_url.netloc.split(':') - request_args['SERVER_NAME'] = name - request_args['SERVER_PORT'] = port or '80' - request_args['wsgi.url_scheme'] = live_server_url.scheme - return RequestFactory().request(**request_args) diff --git a/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py b/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py deleted file mode 100644 index 859b24e01d03..000000000000 --- a/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py +++ /dev/null @@ -1,201 +0,0 @@ -""" -Tests for xblock_utils.py -""" - -from uuid import UUID - -import pytest -from django.test import TestCase - -from openedx.core.lib import blockstore_api as api -from openedx.core.lib.blockstore_api.tests.base import ( - BlockstoreAppTestMixin, -) - -# A fake UUID that won't represent any real bundle/draft/collection: -BAD_UUID = UUID('12345678-0000-0000-0000-000000000000') - - -class BlockstoreApiClientTestMixin: - """ - Tests for the Blockstore API Client. - - The goal of these tests is not to test that Blockstore works correctly, but - that the API client can interact with it and all the API client methods - work. - """ - - # Collections - - def test_nonexistent_collection(self): - """ Request a collection that doesn't exist -> CollectionNotFound """ - with pytest.raises(api.CollectionNotFound): - api.get_collection(BAD_UUID) - - def test_collection_crud(self): - """ Create, Fetch, Update, and Delete a Collection """ - title = "Fire 🔥 Collection" - # Create: - coll = api.create_collection(title) - assert coll.title == title - assert isinstance(coll.uuid, UUID) - # Fetch: - coll2 = api.get_collection(coll.uuid) - assert coll == coll2 - # Update: - new_title = "Air 🌀 Collection" - coll3 = api.update_collection(coll.uuid, title=new_title) - assert coll3.title == new_title - coll4 = api.get_collection(coll.uuid) - assert coll4.title == new_title - # Delete: - api.delete_collection(coll.uuid) - with pytest.raises(api.CollectionNotFound): - api.get_collection(coll.uuid) - - # Bundles - - def test_nonexistent_bundle(self): - """ Request a bundle that doesn't exist -> BundleNotFound """ - with pytest.raises(api.BundleNotFound): - api.get_bundle(BAD_UUID) - - def test_bundle_crud(self): - """ Create, Fetch, Update, and Delete a Bundle """ - coll = api.create_collection("Test Collection") - args = { - "title": "Water 💧 Bundle", - "slug": "h2o", - "description": "Sploosh", - } - # Create: - bundle = api.create_bundle(coll.uuid, **args) - for attr, value in args.items(): - assert getattr(bundle, attr) == value - assert isinstance(bundle.uuid, UUID) - # Fetch: - bundle2 = api.get_bundle(bundle.uuid) - assert bundle == bundle2 - # Update: - new_description = "Water Nation Bending Lessons" - bundle3 = api.update_bundle(bundle.uuid, description=new_description) - assert bundle3.description == new_description - bundle4 = api.get_bundle(bundle.uuid) - assert bundle4.description == new_description - # Delete: - api.delete_bundle(bundle.uuid) - with pytest.raises(api.BundleNotFound): - api.get_bundle(bundle.uuid) - - # Drafts, files, and reading/writing file contents: - - def test_nonexistent_draft(self): - """ Request a draft that doesn't exist -> DraftNotFound """ - with pytest.raises(api.DraftNotFound): - api.get_draft(BAD_UUID) - - def test_drafts_and_files(self): - """ - Test creating, reading, writing, committing, and reverting drafts and - files. - """ - coll = api.create_collection("Test Collection") - bundle = api.create_bundle(coll.uuid, title="Earth 🗿 Bundle", slug="earth", description="another test bundle") - # Create a draft - draft = api.get_or_create_bundle_draft(bundle.uuid, draft_name="test-draft") - assert draft.bundle_uuid == bundle.uuid - assert draft.name == 'test-draft' - assert draft.updated_at.year >= 2019 - # And retrieve it again: - draft2 = api.get_or_create_bundle_draft(bundle.uuid, draft_name="test-draft") - assert draft == draft2 - # Also test retrieving using get_draft - draft3 = api.get_draft(draft.uuid) - assert draft == draft3 - - # Write a file into the bundle: - api.write_draft_file(draft.uuid, "test.txt", b"initial version") - # Now the file should be visible in the draft: - draft_contents = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name) - assert draft_contents == b'initial version' - api.commit_draft(draft.uuid) - - # Write a new version into the draft: - api.write_draft_file(draft.uuid, "test.txt", b"modified version") - published_contents = api.get_bundle_file_data(bundle.uuid, "test.txt") - assert published_contents == b'initial version' - draft_contents2 = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name) - assert draft_contents2 == b'modified version' - # Now delete the draft: - api.delete_draft(draft.uuid) - draft_contents3 = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name) - # Confirm the file is now reset: - assert draft_contents3 == b'initial version' - - # Finaly, test the get_bundle_file* methods: - file_info1 = api.get_bundle_file_metadata(bundle.uuid, "test.txt") - assert file_info1.path == 'test.txt' - assert file_info1.size == len(b'initial version') - assert file_info1.hash_digest == 'a45a5c6716276a66c4005534a51453ab16ea63c4' - - assert list(api.get_bundle_files(bundle.uuid)) == [file_info1] - assert api.get_bundle_files_dict(bundle.uuid) == {'test.txt': file_info1} - - # Links - - def test_links(self): - """ - Test operations involving bundle links. - """ - coll = api.create_collection("Test Collection") - # Create two library bundles and a course bundle: - lib1_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib1") - lib1_draft = api.get_or_create_bundle_draft(lib1_bundle.uuid, draft_name="test-draft") - lib2_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib2") - lib2_draft = api.get_or_create_bundle_draft(lib2_bundle.uuid, draft_name="other-draft") - course_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="course") - course_draft = api.get_or_create_bundle_draft(course_bundle.uuid, draft_name="test-draft") - - # To create links, we need valid BundleVersions, which requires having committed at least one change: - api.write_draft_file(lib1_draft.uuid, "lib1-data.txt", "hello world") - api.commit_draft(lib1_draft.uuid) # Creates version 1 - api.write_draft_file(lib2_draft.uuid, "lib2-data.txt", "hello world") - api.commit_draft(lib2_draft.uuid) # Creates version 1 - - # Lib2 has no links: - assert not api.get_bundle_links(lib2_bundle.uuid) - - # Create a link from lib2 to lib1 - link1_name = "lib2_to_lib1" - api.set_draft_link(lib2_draft.uuid, link1_name, lib1_bundle.uuid, version=1) - # Now confirm the link exists in the draft: - lib2_draft_links = api.get_bundle_links(lib2_bundle.uuid, use_draft=lib2_draft.name) - assert link1_name in lib2_draft_links - assert lib2_draft_links[link1_name].direct.bundle_uuid == lib1_bundle.uuid - assert lib2_draft_links[link1_name].direct.version == 1 - # Now commit the change to lib2: - api.commit_draft(lib2_draft.uuid) # Creates version 2 - - # Now create a link from course to lib2 - link2_name = "course_to_lib2" - api.set_draft_link(course_draft.uuid, link2_name, lib2_bundle.uuid, version=2) - api.commit_draft(course_draft.uuid) - - # And confirm the link exists in the resulting bundle version: - course_links = api.get_bundle_links(course_bundle.uuid) - assert link2_name in course_links - assert course_links[link2_name].direct.bundle_uuid == lib2_bundle.uuid - assert course_links[link2_name].direct.version == 2 - # And since the links go course->lib2->lib1, course has an indirect link to lib1: - assert course_links[link2_name].indirect[0].bundle_uuid == lib1_bundle.uuid - assert course_links[link2_name].indirect[0].version == 1 - - # Finally, test deleting a link from course's draft: - api.set_draft_link(course_draft.uuid, link2_name, None, None) - assert not api.get_bundle_links(course_bundle.uuid, use_draft=course_draft.name) - - -class BlockstoreAppApiClientTest(BlockstoreApiClientTestMixin, BlockstoreAppTestMixin, TestCase): - """ - Test the Blockstore API Client, using the installed Blockstore app. - """ diff --git a/openedx/core/lib/teams_config.py b/openedx/core/lib/teams_config.py index c4952b6cdc3a..7210eed0f02f 100644 --- a/openedx/core/lib/teams_config.py +++ b/openedx/core/lib/teams_config.py @@ -19,7 +19,7 @@ TEAM_SCHEME = "team" TEAMS_NAMESPACE = "teams" -# .. toggle_name: course_teams.content_groups_for_teams +# .. toggle_name: teams.content_groups_for_teams # .. toggle_implementation: CourseWaffleFlag # .. toggle_default: False # .. toggle_description: This flag enables content groups for teams. Content groups are virtual groupings of learners diff --git a/openedx/core/lib/xblock_serializer/api.py b/openedx/core/lib/xblock_serializer/api.py index 30dbc8321b3c..8ac1cd5717c3 100644 --- a/openedx/core/lib/xblock_serializer/api.py +++ b/openedx/core/lib/xblock_serializer/api.py @@ -2,7 +2,7 @@ Public python API for serializing XBlocks to OLX """ # pylint: disable=unused-import -from .block_serializer import StaticFile, XBlockSerializer, XBlockSerializerForBlockstore +from .block_serializer import StaticFile, XBlockSerializer, XBlockSerializerForLearningCore def serialize_xblock_to_olx(block): @@ -14,10 +14,10 @@ def serialize_xblock_to_olx(block): return XBlockSerializer(block) -def serialize_modulestore_block_for_blockstore(block): +def serialize_modulestore_block_for_learning_core(block): """ This class will serialize an XBlock, producing: - (1) A new definition ID for use in Blockstore + (1) A new definition ID for use in Learning Core (2) an XML string defining the XBlock and referencing the IDs of its children using syntax (which doesn't actually contain the OLX of its children, just refers to them, so you have to @@ -29,4 +29,4 @@ def serialize_modulestore_block_for_blockstore(block): we have around how we should rewrite this (e.g. are we going to remove ?). """ - return XBlockSerializerForBlockstore(block) + return XBlockSerializerForLearningCore(block) diff --git a/openedx/core/lib/xblock_serializer/block_serializer.py b/openedx/core/lib/xblock_serializer/block_serializer.py index af155a3900b4..966380f25061 100644 --- a/openedx/core/lib/xblock_serializer/block_serializer.py +++ b/openedx/core/lib/xblock_serializer/block_serializer.py @@ -137,10 +137,10 @@ def _serialize_html_block(self, block) -> etree.Element: return olx_node -class XBlockSerializerForBlockstore(XBlockSerializer): +class XBlockSerializerForLearningCore(XBlockSerializer): """ This class will serialize an XBlock, producing: - (1) A new definition ID for use in Blockstore + (1) A new definition ID for use in Learning Core (2) an XML string defining the XBlock and referencing the IDs of its children using syntax (which doesn't actually contain the OLX of its children, just refers to them, so you have to @@ -154,7 +154,7 @@ def __init__(self, block): resulting data in this object. """ super().__init__(block) - self.def_id = utils.blockstore_def_key_from_modulestore_usage_key(self.orig_block_key) + self.def_id = utils.learning_core_def_key_from_modulestore_usage_key(self.orig_block_key) def _serialize_block(self, block) -> etree.Element: """ Serialize an XBlock to OLX/XML. """ @@ -174,12 +174,12 @@ def _serialize_children(self, block, parent_olx_node): # the same block to be used in many places (each with a unique # usage key). However, that functionality is not exposed in # Studio (other than via content libraries). So when we import - # into Blockstore, we assume that each usage is unique, don't + # into Learning Core, we assume that each usage is unique, don't # generate a usage key, and create a new "definition key" from # the original usage key. # So modulestore usage key # block-v1:A+B+C+type@html+block@introduction - # will become Blockstore definition key + # will become Learning Core definition key # html+introduction # # If we needed the real definition key, we could get it via @@ -187,7 +187,7 @@ def _serialize_children(self, block, parent_olx_node): # child_def_id = str(child.scope_ids.def_id) # and then use # - def_id = utils.blockstore_def_key_from_modulestore_usage_key(child_id) + def_id = utils.learning_core_def_key_from_modulestore_usage_key(child_id) parent_olx_node.append(parent_olx_node.makeelement("xblock-include", {"definition": def_id})) def _transform_olx(self, olx_node, usage_id): diff --git a/openedx/core/lib/xblock_serializer/test_api.py b/openedx/core/lib/xblock_serializer/test_api.py index 39ca6bd675e4..8078595b0ea9 100644 --- a/openedx/core/lib/xblock_serializer/test_api.py +++ b/openedx/core/lib/xblock_serializer/test_api.py @@ -68,112 +68,6 @@ """ -EXPECTED_OPENASSESSMENT_OLX = """ - - Open Response Assessment - - - - - Replace this text with your own sample response for this assignment. Then, under Response Score to the right, select an option for each criterion. Learners practice performing peer assessments by assessing this response and comparing the options that they select in the rubric with the options that you specified. - - - - - - Replace this text with another sample response, and then specify the options that you would select for this response. - - - - - - - - - - - - Censorship in the Libraries - - 'All of us can think of a book that we hope none of our children or any other children have taken off the shelf. But if I have the right to remove that book from the shelf -- that work I abhor -- then you also have exactly the same right and so does everyone else. And then we have no books left on the shelf for any of us.' --Katherine Paterson, Author - - Write a persuasive essay to a newspaper reflecting your views on censorship in libraries. Do you believe that certain materials, such as books, music, movies, magazines, etc., should be removed from the shelves if they are found offensive? Support your position with convincing arguments from your own experience, observations, and/or reading. - - Read for conciseness, clarity of thought, and form. - - - - - - Ideas - - Determine if there is a unifying theme or main idea. - - - - - - Content - - Assess the content of the submission - - - - - - -(Optional) What aspects of this response stood out to you? What did it do well? How could it be improved? - - -I think that this response... - - - -""" - - @skip_unless_cms class XBlockSerializationTestCase(SharedModuleStoreTestCase): """ @@ -240,17 +134,17 @@ def test_html_with_static_asset(self): ), ]) - def test_html_with_static_asset_blockstore(self): + def test_html_with_static_asset_learning_core(self): """ - Test the blockstore-specific serialization of an HTML block + Test the learning-core-specific serialization of an HTML block """ block_id = self.course.id.make_usage_key('html', 'just_img') # see sample_courses.py html_block = modulestore().get_item(block_id) serialized = api.serialize_xblock_to_olx(html_block) - serialized_blockstore = api.serialize_modulestore_block_for_blockstore(html_block) + serialized_learning_core = api.serialize_modulestore_block_for_learning_core(html_block) self.assertXmlEqual( - serialized_blockstore.olx_str, - # For blockstore, OLX should never contain "url_name" as that ID is specified by the filename: + serialized_learning_core.olx_str, + # For learning core, OLX should never contain "url_name" as that ID is specified by the filename: """ @@ -259,9 +153,9 @@ def test_html_with_static_asset_blockstore(self): ) self.assertIn("CDATA", serialized.olx_str) # Static files should be identical: - self.assertEqual(serialized.static_files, serialized_blockstore.static_files) - # This is the only other difference - an extra field with the blockstore-specific definition ID: - self.assertEqual(serialized_blockstore.def_id, "html/just_img") + self.assertEqual(serialized.static_files, serialized_learning_core.static_files) + # This is the only other difference - an extra field with the learning-core-specific definition ID: + self.assertEqual(serialized_learning_core.def_id, "html/just_img") def test_html_with_fields(self): """ Test an HTML Block with non-default fields like editor='raw' """ @@ -299,13 +193,13 @@ def test_export_sequential(self): self.assertXmlEqual(serialized.olx_str, EXPECTED_SEQUENTIAL_OLX) - def test_export_sequential_blockstore(self): + def test_export_sequential_learning_core(self): """ - Export a sequential from the toy course, formatted for blockstore. + Export a sequential from the toy course, formatted for learning core. """ sequential_id = self.course.id.make_usage_key('sequential', 'Toy_Videos') # see sample_courses.py sequential = modulestore().get_item(sequential_id) - serialized = api.serialize_modulestore_block_for_blockstore(sequential) + serialized = api.serialize_modulestore_block_for_learning_core(sequential) self.assertXmlEqual(serialized.olx_str, """ @@ -701,10 +595,11 @@ def test_tagged_openassessment_block(self): # Check that the tags data is serialized and omitted from the OLX serialized = api.serialize_xblock_to_olx(openassessment_block) - self.assertXmlEqual( - serialized.olx_str, - EXPECTED_OPENASSESSMENT_OLX - ) + + self.assertNotIn("normal tag", serialized.olx_str) + self.assertNotIn(" tag", serialized.olx_str) + self.assertNotIn("anotherTag", serialized.olx_str) + self.assertEqual(serialized.tags, { str(openassessment_block.location): { self.taxonomy1.id: ["normal tag", " tag", "anotherTag"], diff --git a/openedx/core/lib/xblock_serializer/utils.py b/openedx/core/lib/xblock_serializer/utils.py index 2c736ae2998f..e78c900b1887 100644 --- a/openedx/core/lib/xblock_serializer/utils.py +++ b/openedx/core/lib/xblock_serializer/utils.py @@ -225,17 +225,17 @@ def override_export_fs(block): XmlMixin.export_to_file = old_global_export_to_file -def blockstore_def_key_from_modulestore_usage_key(usage_key): +def learning_core_def_key_from_modulestore_usage_key(usage_key): """ In modulestore, the "definition key" is a MongoDB ObjectID kept in split's definitions table, which theoretically allows the same block to be used in many places (each with a unique usage key). However, that functionality is not exposed in Studio (other than via content libraries). So when we import - into Blockstore, we assume that each usage is unique, don't generate a usage + into learning core, we assume that each usage is unique, don't generate a usage key, and create a new "definition key" from the original usage key. So modulestore usage key block-v1:A+B+C+type@html+block@introduction - will become Blockstore definition key + will become learning core definition key html/introduction """ block_type = usage_key.block_type diff --git a/pavelib/assets.py b/pavelib/assets.py index 466ffc9ed919..f437b6427f93 100644 --- a/pavelib/assets.py +++ b/pavelib/assets.py @@ -1,5 +1,9 @@ """ Asset compilation and collection. + +This entire module is DEPRECATED. In Redwood, it exists just as a collection of temporary compatibility wrappers. +In Sumac, this module will be deleted. To migrate, follow the advice in the printed warnings and/or read the +instructions on the DEPR ticket: https://github.com/openedx/edx-platform/issues/31895 """ import argparse @@ -13,31 +17,48 @@ from paver import tasks from paver.easy import call_task, cmdopts, consume_args, needs, no_help, sh, task from watchdog.events import PatternMatchingEventHandler -from watchdog.observers import Observer # pylint disable=unused-import # Used by Tutor. Remove after Sumac cut. +from watchdog.observers import Observer # pylint: disable=unused-import # Used by Tutor. Remove after Sumac cut. -from .utils.cmd import cmd, django_cmd +from .utils.cmd import django_cmd from .utils.envs import Env -from .utils.process import run_background_process from .utils.timer import timed -# setup baseline paths - -ALL_SYSTEMS = ['lms', 'studio'] - -LMS = 'lms' -CMS = 'cms' SYSTEMS = { - 'lms': LMS, - 'cms': CMS, - 'studio': CMS + 'lms': 'lms', + 'cms': 'cms', + 'studio': 'cms', } -# Collectstatic log directory setting -COLLECTSTATIC_LOG_DIR_ARG = 'collect_log_dir' +WARNING_SYMBOLS = "⚠️ " * 50 # A row of 'warning' emoji to catch CLI users' attention + -# Webpack command -WEBPACK_COMMAND = 'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} webpack {options}' +def run_deprecated_command_wrapper(*, old_command, ignored_old_flags, new_command): + """ + Run the new version of shell command, plus a warning that the old version is deprecated. + """ + depr_warning = ( + "\n" + + f"{WARNING_SYMBOLS}\n" + + "\n" + + f"WARNING: '{old_command}' is DEPRECATED! It will be removed before Sumac.\n" + + "The command you ran is now just a temporary wrapper around a new,\n" + + "supported command, which you should use instead:\n" + + "\n" + + f"\t{new_command}\n" + + "\n" + + "Details: https://github.com/openedx/edx-platform/issues/31895\n" + + "".join( + f" WARNING: ignored deprecated paver flag '{flag}'\n" + for flag in ignored_old_flags + ) + + f"{WARNING_SYMBOLS}\n" + + "\n" + ) + # Print deprecation warning twice so that it's more likely to be seen in the logs. + print(depr_warning) + sh(new_command) + print(depr_warning) def debounce(seconds=1): @@ -45,6 +66,8 @@ def debounce(seconds=1): Prevents the decorated function from being called more than every `seconds` seconds. Waits until calls stop coming in before calling the decorated function. + + This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor. """ def decorator(func): func.timer = None @@ -66,6 +89,8 @@ def call(): class SassWatcher(PatternMatchingEventHandler): """ Watches for sass file changes + + This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor. """ ignore_directories = True patterns = ['*.scss'] @@ -102,7 +127,7 @@ def on_any_event(self, event): ('system=', 's', 'The system to compile sass for (defaults to all)'), ('theme-dirs=', '-td', 'Theme dirs containing all themes (defaults to None)'), ('themes=', '-t', 'The theme to compile sass for (defaults to None)'), - ('debug', 'd', 'DEPRECATED. Debug mode is now determined by NODE_ENV.'), + ('debug', 'd', 'Whether to use development settings'), ('force', '', 'DEPRECATED. Full recompilation is now always forced.'), ]) @timed @@ -143,16 +168,18 @@ def compile_sass(options): This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead. """ - systems = set(get_parsed_option(options, 'system', ALL_SYSTEMS)) - command = shlex.join( - [ + systems = [SYSTEMS[sys] for sys in get_parsed_option(options, 'system', ['lms', 'cms'])] # normalize studio->cms + run_deprecated_command_wrapper( + old_command="paver compile_sass", + ignored_old_flags=(set(["--force"]) & set(options)), + new_command=shlex.join([ "npm", "run", - "compile-sass", + ("compile-sass-dev" if options.get("debug") else "compile-sass"), "--", *(["--dry"] if tasks.environment.dry_run else []), - *(["--skip-lms"] if not systems & {"lms"} else []), - *(["--skip-cms"] if not systems & {"cms", "studio"} else []), + *(["--skip-lms"] if "lms" not in systems else []), + *(["--skip-cms"] if "cms" not in systems else []), *( arg for theme_dir in get_parsed_option(options, 'theme_dirs', []) @@ -160,77 +187,50 @@ def compile_sass(options): ), *( arg - for theme in get_parsed_option(options, "theme", []) + for theme in get_parsed_option(options, "themes", []) for arg in ["--theme", theme] ), - ] - ) - depr_warning = ( - "\n" + - "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + - "\n" + - "WARNING: 'paver compile_sass' is DEPRECATED! It will be removed before Sumac.\n" + - "The command you ran is now just a temporary wrapper around a new,\n" + - "supported command, which you should use instead:\n" + - "\n" + - f"\t{command}\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - "\n" + - ("WARNING: ignoring deprecated flag '--debug'\n" if options.get("debug") else "") + - ("WARNING: ignoring deprecated flag '--force'\n" if options.get("force") else "") + - "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + - "\n" + ]), ) - # Print deprecation warning twice so that it's more likely to be seen in the logs. - print(depr_warning) - sh(command) - print(depr_warning) -def _compile_sass(system, theme, _debug, _force, _timing_info): +def _compile_sass(system, theme, debug, force, _timing_info): """ This is a DEPRECATED COMPATIBILITY WRAPPER It exists to ease the transition for Tutor in Redwood, which directly imported and used this function. """ - command = shlex.join( - [ + run_deprecated_command_wrapper( + old_command="pavelib.assets:_compile_sass", + ignored_old_flags=(set(["--force"]) if force else set()), + new_command=[ "npm", "run", - "compile-sass", + ("compile-sass-dev" if debug else "compile-sass"), "--", *(["--dry"] if tasks.environment.dry_run else []), - *(["--skip-default", "--theme-dir", str(theme.parent), "--theme", str(theme.name)] if theme else []), + *( + ["--skip-default", "--theme-dir", str(theme.parent), "--theme", str(theme.name)] + if theme + else [] + ), ("--skip-cms" if system == "lms" else "--skip-lms"), ] ) - depr_warning = ( - "\n" + - "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + - "\n" + - "WARNING: 'pavelib/assets.py' is DEPRECATED! It will be removed before Sumac.\n" + - "The function you called is just a temporary wrapper around a new, supported command,\n" + - "which you should use instead:\n" + - "\n" + - f"\t{command}\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - "\n" + - "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + - "\n" - ) - # Print deprecation warning twice so that it's more likely to be seen in the logs. - print(depr_warning) - sh(command) - print(depr_warning) def process_npm_assets(): """ Process vendor libraries installed via NPM. + + This is a DEPRECATED COMPATIBILITY WRAPPER. It is now handled as part of `npm clean-install`. + If you need to invoke it explicitly, you can run `npm run postinstall`. """ - sh('scripts/copy-node-modules.sh') + run_deprecated_command_wrapper( + old_command="pavelib.assets:process_npm_assets", + ignored_old_flags=[], + new_command=shlex.join(["npm", "run", "postinstall"]), + ) @task @@ -238,9 +238,21 @@ def process_npm_assets(): def process_xmodule_assets(): """ Process XModule static assets. + + This is a DEPRECATED COMPATIBILITY STUB. Refrences to it should be deleted. """ - print("\t\tProcessing xmodule assets is no longer needed. This task is now a no-op.") - print("\t\tWhen paver is removed from edx-platform, this step will not replaced.") + print( + "\n" + + f"{WARNING_SYMBOLS}", + "\n" + + "WARNING: 'paver process_xmodule_assets' is DEPRECATED! It will be removed before Sumac.\n" + + "\n" + + "Starting with Quince, it is no longer necessary to post-process XModule assets, so \n" + + "'paver process_xmodule_assets' is a no-op. Please simply remove it from your build scripts.\n" + + "\n" + + "Details: https://github.com/openedx/edx-platform/issues/31895\n" + + f"{WARNING_SYMBOLS}", + ) def collect_assets(systems, settings, **kwargs): @@ -249,33 +261,29 @@ def collect_assets(systems, settings, **kwargs): `systems` is a list of systems (e.g. 'lms' or 'studio' or both) `settings` is the Django settings module to use. `**kwargs` include arguments for using a log directory for collectstatic output. Defaults to /dev/null. - """ - for sys in systems: - collectstatic_stdout_str = _collect_assets_cmd(sys, **kwargs) - sh(django_cmd(sys, settings, "collectstatic --noinput {logfile_str}".format( - logfile_str=collectstatic_stdout_str - ))) - print(f"\t\tFinished collecting {sys} assets.") - -def _collect_assets_cmd(system, **kwargs): - """ - Returns the collecstatic command to be used for the given system + This is a DEPRECATED COMPATIBILITY WRAPPER - Unless specified, collectstatic (which can be verbose) pipes to /dev/null + It exists to ease the transition for Tutor in Redwood, which directly imported and used this function. """ - try: - if kwargs[COLLECTSTATIC_LOG_DIR_ARG] is None: - collectstatic_stdout_str = "" - else: - collectstatic_stdout_str = "> {output_dir}/{sys}-collectstatic.log".format( - output_dir=kwargs[COLLECTSTATIC_LOG_DIR_ARG], - sys=system - ) - except KeyError: - collectstatic_stdout_str = "> /dev/null" - - return collectstatic_stdout_str + run_deprecated_command_wrapper( + old_command="pavelib.asset:collect_assets", + ignored_old_flags=[], + new_command=" && ".join( + "( " + + shlex.join( + ["./manage.py", SYSTEMS[sys], f"--settings={settings}", "collectstatic", "--noinput"] + ) + ( + "" + if "collect_log_dir" not in kwargs else + " > /dev/null" + if kwargs["collect_log_dir"] is None else + f"> {kwargs['collect_log_dir']}/{SYSTEMS[sys]}-collectstatic.out" + ) + + " )" + for sys in systems + ), + ) def execute_compile_sass(args): @@ -283,6 +291,8 @@ def execute_compile_sass(args): Construct django management command compile_sass (defined in theming app) and execute it. Args: args: command line argument passed via update_assets command + + This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead. """ for sys in args.system: options = "" @@ -305,12 +315,14 @@ def execute_compile_sass(args): @task @cmdopts([ ('settings=', 's', "Django settings (defaults to devstack)"), - ('watch', 'w', "Watch file system and rebuild on change (defaults to off)"), + ('watch', 'w', "DEPRECATED. This flag never did anything anyway."), ]) @timed def webpack(options): """ Run a Webpack build. + + This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run webpack` instead. """ settings = getattr(options, 'settings', Env.DEVSTACK_SETTINGS) result = Env.get_django_settings(['STATIC_ROOT', 'WEBPACK_CONFIG_PATH'], "lms", settings=settings) @@ -318,44 +330,20 @@ def webpack(options): static_root_cms, = Env.get_django_settings(["STATIC_ROOT"], "cms", settings=settings) js_env_extra_config_setting, = Env.get_django_json_settings(["JS_ENV_EXTRA_CONFIG"], "cms", settings=settings) js_env_extra_config = json.dumps(js_env_extra_config_setting or "{}") - environment = ( - "NODE_ENV={node_env} STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} " - "JS_ENV_EXTRA_CONFIG={js_env_extra_config}" - ).format( - node_env="development" if config_path == 'webpack.dev.config.js' else "production", - static_root_lms=static_root_lms, - static_root_cms=static_root_cms, - js_env_extra_config=js_env_extra_config, - ) - sh( - cmd( - '{environment} webpack --config={config_path}'.format( - environment=environment, - config_path=config_path - ) - ) - ) - - -def execute_webpack_watch(settings=None): - """ - Run the Webpack file system watcher. - """ - # We only want Webpack to re-run on changes to its own entry points, - # not all JS files, so we use its own watcher instead of subclassing - # from Watchdog like the other watchers do. - - result = Env.get_django_settings(["STATIC_ROOT", "WEBPACK_CONFIG_PATH"], "lms", settings=settings) - static_root_lms, config_path = result - static_root_cms, = Env.get_django_settings(["STATIC_ROOT"], "cms", settings=settings) - run_background_process( - 'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} webpack {options}'.format( - options='--watch --config={config_path}'.format( - config_path=config_path - ), - static_root_lms=static_root_lms, - static_root_cms=static_root_cms, - ) + node_env = "development" if config_path == 'webpack.dev.config.js' else "production" + run_deprecated_command_wrapper( + old_command="paver webpack", + ignored_old_flags=(set(["watch"]) & set(options)), + new_command=' '.join([ + f"WEBPACK_CONFIG_PATH={config_path}", + f"NODE_ENV={node_env}", + f"STATIC_ROOT_LMS={static_root_lms}", + f"STATIC_ROOT_CMS={static_root_cms}", + f"JS_ENV_EXTRA_CONFIG={js_env_extra_config}", + "npm", + "run", + "webpack", + ]), ) @@ -411,39 +399,19 @@ def watch_assets(options): return theme_dirs = ':'.join(get_parsed_option(options, 'theme_dirs', [])) - command = shlex.join( - [ + run_deprecated_command_wrapper( + old_command="paver watch_assets", + ignored_old_flags=(set(["debug", "themes", "settings", "background"]) & set(options)), + new_command=shlex.join([ *( - ["env", f"EDX_PLATFORM_THEME_DIRS={theme_dirs}"] if theme_dirs else [] + ["env", f"COMPREHENSIVE_THEME_DIRS={theme_dirs}"] + if theme_dirs else [] ), "npm", "run", "watch", - ] + ]), ) - depr_warning = ( - "\n" + - "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + - "\n" + - "WARNING: 'paver watch_assets' is DEPRECATED! It will be removed before Sumac.\n" + - "The command you ran is now just a temporary wrapper around a new,\n" + - "supported command, which you should use instead:\n" + - "\n" + - f"\t{command}\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - "\n" + - ("WARNING: ignoring deprecated flag '--debug'\n" if options.get("debug") else "") + - ("WARNING: ignoring deprecated flag '--themes'\n" if options.get("themes") else "") + - ("WARNING: ignoring deprecated flag '--settings'\n" if options.get("settings") else "") + - ("WARNING: ignoring deprecated flag '--background'\n" if options.get("background") else "") + - "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + - "\n" - ) - # Print deprecation warning twice so that it's more likely to be seen in the logs. - print(depr_warning) - sh(command) - print(depr_warning) @task @@ -456,10 +424,19 @@ def watch_assets(options): def update_assets(args): """ Compile Sass, then collect static assets. + + This is a DEPRECATED COMPATIBILITY WRAPPER around other DEPRECATED COMPATIBILITY WRAPPERS. + The aggregate affect of this command can be achieved with this sequence of commands instead: + + * pip install -r requirements/edx/assets.txt # replaces install_python_prereqs + * npm clean-install # replaces install_node_prereqs + * npm run build # replaces execute_compile_sass and webpack + * ./manage.py lms collectstatic --noinput # replaces collect_assets (for LMS) + * ./manage.py cms collectstatic --noinput # replaces collect_assets (for CMS) """ parser = argparse.ArgumentParser(prog='paver update_assets') parser.add_argument( - 'system', type=str, nargs='*', default=ALL_SYSTEMS, + 'system', type=str, nargs='*', default=["lms", "studio"], help="lms or studio", ) parser.add_argument( @@ -488,18 +465,17 @@ def update_assets(args): ) parser.add_argument( '--themes', type=str, nargs='+', default=None, - help="list of themes to compile sass for", + help="list of themes to compile sass for. ignored when --watch is used; all themes are watched.", ) parser.add_argument( - '--collect-log', dest=COLLECTSTATIC_LOG_DIR_ARG, default=None, + '--collect-log', dest="collect_log_dir", default=None, help="When running collectstatic, direct output to specified log directory", ) parser.add_argument( '--wait', type=float, default=0.0, - help="How long to pause between filesystem scans" + help="DEPRECATED. Watchdog's default wait time is now used.", ) args = parser.parse_args(args) - collect_log_args = {} # Build Webpack call_task('pavelib.assets.webpack', options={'settings': args.settings}) @@ -508,11 +484,12 @@ def update_assets(args): execute_compile_sass(args) if args.collect: - if args.debug or args.debug_collect: - collect_log_args.update({COLLECTSTATIC_LOG_DIR_ARG: None}) - if args.collect_log_dir: - collect_log_args.update({COLLECTSTATIC_LOG_DIR_ARG: args.collect_log_dir}) + collect_log_args = {"collect_log_dir": args.collect_log_dir} + elif args.debug or args.debug_collect: + collect_log_args = {"collect_log_dir": None} + else: + collect_log_args = {} collect_assets(args.system, args.settings, **collect_log_args) diff --git a/pavelib/paver_tests/test_assets.py b/pavelib/paver_tests/test_assets.py index 029a8db67c4a..3578a5043f7e 100644 --- a/pavelib/paver_tests/test_assets.py +++ b/pavelib/paver_tests/test_assets.py @@ -1,305 +1,130 @@ """Unit tests for the Paver asset tasks.""" - +import json import os +from pathlib import Path from unittest import TestCase from unittest.mock import patch import ddt -import paver.tasks -from paver.easy import call_task, path +import paver.easy +from paver import tasks -from pavelib.assets import COLLECTSTATIC_LOG_DIR_ARG, collect_assets +import pavelib.assets +from pavelib.assets import Env -from ..utils.envs import Env -from .utils import PaverTestCase -ROOT_PATH = path(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) -TEST_THEME_DIR = ROOT_PATH / "common/test/test-theme" +REPO_ROOT = Path(__file__).parent.parent.parent +LMS_SETTINGS = { + "WEBPACK_CONFIG_PATH": "webpack.fake.config.js", + "STATIC_ROOT": "/fake/lms/staticfiles", -@ddt.ddt -class TestPaverAssetTasks(PaverTestCase): - """ - Test the Paver asset tasks. - """ - @ddt.data( - [""], - ["--force"], - ["--debug"], - ["--system=lms"], - ["--system=lms --force"], - ["--system=studio"], - ["--system=studio --force"], - ["--system=lms,studio"], - ["--system=lms,studio --force"], - ) - @ddt.unpack - def test_compile_sass(self, options): - """ - Test the "compile_sass" task. - """ - parameters = options.split(" ") - system = [] - if '--system=studio' not in parameters: - system += ['lms'] - if '--system=lms' not in parameters: - system += ['studio'] - debug = '--debug' in parameters - force = '--force' in parameters - self.reset_task_messages() - call_task('pavelib.assets.compile_sass', options={'system': system, 'debug': debug, 'force': force}) - expected_messages = [] - if force: - expected_messages.append('rm -rf common/static/css/*.css') - expected_messages.append('libsass common/static/sass') +} +CMS_SETTINGS = { + "WEBPACK_CONFIG_PATH": "webpack.fake.config", + "STATIC_ROOT": "/fake/cms/staticfiles", + "JS_ENV_EXTRA_CONFIG": json.dumps({"key1": [True, False], "key2": {"key2.1": 1369, "key2.2": "1369"}}), +} - if "lms" in system: - if force: - expected_messages.append('rm -rf lms/static/css/*.css') - expected_messages.append('libsass lms/static/sass') - expected_messages.append( - 'rtlcss lms/static/css/bootstrap/lms-main.css lms/static/css/bootstrap/lms-main-rtl.css' - ) - expected_messages.append( - 'rtlcss lms/static/css/discussion/lms-discussion-bootstrap.css' - ' lms/static/css/discussion/lms-discussion-bootstrap-rtl.css' - ) - if force: - expected_messages.append('rm -rf lms/static/certificates/css/*.css') - expected_messages.append('libsass lms/static/certificates/sass') - if "studio" in system: - if force: - expected_messages.append('rm -rf cms/static/css/*.css') - expected_messages.append('libsass cms/static/sass') - expected_messages.append( - 'rtlcss cms/static/css/bootstrap/studio-main.css cms/static/css/bootstrap/studio-main-rtl.css' - ) - assert len(self.task_messages) == len(expected_messages) +def _mock_get_django_settings(django_settings, system, settings=None): # pylint: disable=unused-argument + return [(LMS_SETTINGS if system == "lms" else CMS_SETTINGS)[s] for s in django_settings] @ddt.ddt -class TestPaverThemeAssetTasks(PaverTestCase): +@patch.object(Env, 'get_django_settings', _mock_get_django_settings) +@patch.object(Env, 'get_django_json_settings', _mock_get_django_settings) +class TestDeprecatedPaverAssets(TestCase): """ - Test the Paver asset tasks. + Simple test to ensure that the soon-to-be-removed Paver commands are correctly translated into the new npm-run + commands. """ - @ddt.data( - [""], - ["--force"], - ["--debug"], - ["--system=lms"], - ["--system=lms --force"], - ["--system=studio"], - ["--system=studio --force"], - ["--system=lms,studio"], - ["--system=lms,studio --force"], - ) - @ddt.unpack - def test_compile_theme_sass(self, options): - """ - Test the "compile_sass" task. - """ - parameters = options.split(" ") - system = [] - - if '--system=studio' not in parameters: - system += ['lms'] - if "--system=lms" not in parameters: - system += ['studio'] - debug = '--debug' in parameters - force = '--force' in parameters - - self.reset_task_messages() - call_task( - 'pavelib.assets.compile_sass', - options=dict( - system=system, - debug=debug, - force=force, - theme_dirs=[TEST_THEME_DIR.dirname()], - themes=[TEST_THEME_DIR.basename()] - ), - ) - expected_messages = [] - if force: - expected_messages.append('rm -rf common/static/css/*.css') - expected_messages.append('libsass common/static/sass') - - if 'lms' in system: - expected_messages.append('mkdir_p ' + repr(TEST_THEME_DIR / 'lms/static/css')) - if force: - expected_messages.append( - f'rm -rf {str(TEST_THEME_DIR)}/lms/static/css/*.css' - ) - expected_messages.append("libsass lms/static/sass") - expected_messages.append( - 'rtlcss {test_theme_dir}/lms/static/css/bootstrap/lms-main.css' - ' {test_theme_dir}/lms/static/css/bootstrap/lms-main-rtl.css'.format( - test_theme_dir=str(TEST_THEME_DIR), - ) - ) - expected_messages.append( - 'rtlcss {test_theme_dir}/lms/static/css/discussion/lms-discussion-bootstrap.css' - ' {test_theme_dir}/lms/static/css/discussion/lms-discussion-bootstrap-rtl.css'.format( - test_theme_dir=str(TEST_THEME_DIR), - ) - ) - if force: - expected_messages.append( - f'rm -rf {str(TEST_THEME_DIR)}/lms/static/css/*.css' - ) - expected_messages.append( - f'libsass {str(TEST_THEME_DIR)}/lms/static/sass' - ) - if force: - expected_messages.append('rm -rf lms/static/css/*.css') - expected_messages.append('libsass lms/static/sass') - expected_messages.append( - 'rtlcss lms/static/css/bootstrap/lms-main.css lms/static/css/bootstrap/lms-main-rtl.css' - ) - expected_messages.append( - 'rtlcss lms/static/css/discussion/lms-discussion-bootstrap.css' - ' lms/static/css/discussion/lms-discussion-bootstrap-rtl.css' - ) - if force: - expected_messages.append('rm -rf lms/static/certificates/css/*.css') - expected_messages.append('libsass lms/static/certificates/sass') + def setUp(self): + super().setUp() + self.maxDiff = None + os.environ['NO_PREREQ_INSTALL'] = 'true' + tasks.environment = tasks.Environment() - if "studio" in system: - expected_messages.append('mkdir_p ' + repr(TEST_THEME_DIR / 'cms/static/css')) - if force: - expected_messages.append( - f'rm -rf {str(TEST_THEME_DIR)}/cms/static/css/*.css' - ) - expected_messages.append('libsass cms/static/sass') - expected_messages.append( - 'rtlcss {test_theme_dir}/cms/static/css/bootstrap/studio-main.css' - ' {test_theme_dir}/cms/static/css/bootstrap/studio-main-rtl.css'.format( - test_theme_dir=str(TEST_THEME_DIR), - ) - ) - if force: - expected_messages.append( - f'rm -rf {str(TEST_THEME_DIR)}/cms/static/css/*.css' - ) - expected_messages.append( - f'libsass {str(TEST_THEME_DIR)}/cms/static/sass' - ) - if force: - expected_messages.append('rm -rf cms/static/css/*.css') - expected_messages.append('libsass cms/static/sass') - expected_messages.append( - 'rtlcss cms/static/css/bootstrap/studio-main.css cms/static/css/bootstrap/studio-main-rtl.css' - ) - - assert len(self.task_messages) == len(expected_messages) - - -@ddt.ddt -class TestCollectAssets(PaverTestCase): - """ - Test the collectstatic process call. - - ddt data is organized thusly: - * debug: whether or not collect_assets is called with the debug flag - * specified_log_location: used when collect_assets is called with a specific - log location for collectstatic output - * expected_log_location: the expected string to be used for piping collectstatic logs - """ - - @ddt.data( - [{ - "collect_log_args": {}, # Test for default behavior - "expected_log_location": "> /dev/null" - }], - [{ - "collect_log_args": {COLLECTSTATIC_LOG_DIR_ARG: "/foo/bar"}, - "expected_log_location": "> /foo/bar/lms-collectstatic.log" - }], # can use specified log location - [{ - "systems": ["lms", "cms"], - "collect_log_args": {}, - "expected_log_location": "> /dev/null" - }], # multiple systems can be called - ) - @ddt.unpack - def test_collect_assets(self, options): - """ - Ensure commands sent to the environment for collect_assets are as expected - """ - specified_log_loc = options.get("collect_log_args", {}) - specified_log_dict = specified_log_loc - log_loc = options.get("expected_log_location", "> /dev/null") - systems = options.get("systems", ["lms"]) - if specified_log_loc is None: - collect_assets( - systems, - Env.DEVSTACK_SETTINGS - ) - else: - collect_assets( - systems, - Env.DEVSTACK_SETTINGS, - **specified_log_dict - ) - self._assert_correct_messages(log_location=log_loc, systems=systems) - - def test_collect_assets_debug(self): - """ - When the method is called specifically with None for the collectstatic log dir, then - it should run in debug mode and pipe to console. - """ - expected_log_loc = "" - systems = ["lms"] - kwargs = {COLLECTSTATIC_LOG_DIR_ARG: None} - collect_assets(systems, Env.DEVSTACK_SETTINGS, **kwargs) - self._assert_correct_messages(log_location=expected_log_loc, systems=systems) - - def _assert_correct_messages(self, log_location, systems): - """ - Asserts that the expected commands were run. - - We just extract the pieces we care about here instead of specifying an - exact command, so that small arg changes don't break this test. - """ - for i, sys in enumerate(systems): - msg = self.task_messages[i] - assert msg.startswith(f'python manage.py {sys}') - assert ' collectstatic ' in msg - assert f'--settings={Env.DEVSTACK_SETTINGS}' in msg - assert msg.endswith(f' {log_location}') - - -@ddt.ddt -class TestUpdateAssetsTask(PaverTestCase): - """ - These are nearly end-to-end tests, because they observe output from the commandline request, - but do not actually execute the commandline on the terminal/process - """ + def tearDown(self): + super().tearDown() + del os.environ['NO_PREREQ_INSTALL'] @ddt.data( - [{"expected_substring": "> /dev/null"}], # go to /dev/null by default - [{"cmd_args": ["--debug"], "expected_substring": "collectstatic"}] # TODO: make this regex + dict( + task_name='pavelib.assets.compile_sass', + args=[], + kwargs={}, + expected=["npm run compile-sass --"], + ), + dict( + task_name='pavelib.assets.compile_sass', + args=[], + kwargs={"system": "lms,studio"}, + expected=["npm run compile-sass --"], + ), + dict( + task_name='pavelib.assets.compile_sass', + args=[], + kwargs={"debug": True}, + expected=["npm run compile-sass-dev --"], + ), + dict( + task_name='pavelib.assets.compile_sass', + args=[], + kwargs={"system": "lms"}, + expected=["npm run compile-sass -- --skip-cms"], + ), + dict( + task_name='pavelib.assets.compile_sass', + args=[], + kwargs={"system": "studio"}, + expected=["npm run compile-sass -- --skip-lms"], + ), + dict( + task_name='pavelib.assets.compile_sass', + args=[], + kwargs={"system": "cms", "theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes"}, + expected=[ + "npm run compile-sass -- --skip-lms " + + f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes" + ], + ), + dict( + task_name='pavelib.assets.compile_sass', + args=[], + kwargs={"theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes", "themes": "red-theme,test-theme"}, + expected=[ + "npm run compile-sass -- " + + f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes " + + "--theme red-theme --theme test-theme" + ], + ), + dict( + task_name='pavelib.assets.update_assets', + args=["lms", "studio", "--settings=fake.settings"], + kwargs={}, + expected=[ + ( + "WEBPACK_CONFIG_PATH=webpack.fake.config.js " + + "NODE_ENV=production " + + "STATIC_ROOT_LMS=/fake/lms/staticfiles " + + "STATIC_ROOT_CMS=/fake/cms/staticfiles " + + 'JS_ENV_EXTRA_CONFIG=' + + + '"{\\"key1\\": [true, false], \\"key2\\": {\\"key2.1\\": 1369, \\"key2.2\\": \\"1369\\"}}" ' + + "npm run webpack" + ), + "python manage.py lms --settings=fake.settings compile_sass lms ", + "python manage.py cms --settings=fake.settings compile_sass cms ", + ( + "( ./manage.py lms --settings=fake.settings collectstatic --noinput ) && " + + "( ./manage.py cms --settings=fake.settings collectstatic --noinput )" + ), + ], + ), ) @ddt.unpack - def test_update_assets_task_collectstatic_log_arg(self, options): - """ - Scoped test that only looks at what is passed to the collecstatic options - """ - cmd_args = options.get("cmd_args", [""]) - expected_substring = options.get("expected_substring", None) - call_task('pavelib.assets.update_assets', args=cmd_args) - self.assertTrue( - self._is_substring_in_list(self.task_messages, expected_substring), - msg=f"{expected_substring} not found in messages" - ) - - def _is_substring_in_list(self, messages_list, expected_substring): - """ - Return true a given string is somewhere in a list of strings - """ - for message in messages_list: - if expected_substring in message: - return True - return False + @patch.object(pavelib.assets, 'sh') + def test_paver_assets_wrapper_invokes_new_commands(self, mock_sh, task_name, args, kwargs, expected): + paver.easy.call_task(task_name, args=args, options=kwargs) + assert [call_args[0] for (call_args, call_kwargs) in mock_sh.call_args_list] == expected diff --git a/requirements/constraints.txt b/requirements/constraints.txt index e2845fccc734..2300492a488e 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -23,7 +23,7 @@ click>=8.0,<9.0 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==4.16.5 +edx-enterprise==4.19.1 # Stay on LTS version, remove once this is added to common constraint Django<5.0 @@ -47,9 +47,6 @@ django-webpack-loader==0.7.0 # version of py2neo will work with Neo4j 3.5. py2neo<2022 -# scipy version 1.8 requires numpy>=1.17.3, we've pinned numpy to <1.17.0 in requirements/edx-sandbox/py38.in -scipy<1.8.0 - # edx-enterprise, snowflake-connector-python require charset-normalizer==2.0.0 # Can be removed once snowflake-connector-python>2.7.9 is released with the fix. charset-normalizer<2.1.0 @@ -62,14 +59,6 @@ markdown<3.4.0 # Constraint can be removed once the issue https://github.com/PyCQA/pycodestyle/issues/1090 is fixed. pycodestyle<2.9.0 -# pyopenssl>22.0.0 requires cryptography>=38.0 && conflicts with snowflak-connector-python requires cryptography<37 -# which causes the requirements upgrade job to fail due to constraint conflict -# This constraint can be removed once https://github.com/snowflakedb/snowflake-connector-python/issues/1259 is resolved -# and snowflake-connector-python>2.8.0 is released. -pyopenssl==22.0.0 - -cryptography==38.0.4 # greater version has some issues with openssl. - pylint<2.16.0 # greater version failing quality test. Fix them in seperate ticket. # adding these constraints to minimize boto3 and botocore changeset @@ -107,7 +96,7 @@ libsass==0.10.0 click==8.1.6 # pinning this version to avoid updates while the library is being developed -openedx-learning==0.9.2 +openedx-learning==0.9.3 # Open AI version 1.0.0 dropped support for openai.ChatCompletion which is currently in use in enterprise. openai<=0.28.1 @@ -119,9 +108,13 @@ openai<=0.28.1 optimizely-sdk<5.0 # lxml>=5.0 introduced breaking changes related to system dependencies -# lxml==5.2.1 introduced new extra so we'll nee to rename lxml --> lxml[html-clean] +# lxml==5.2.1 introduced new extra so we'll nee to rename lxml --> lxml[html-clean] # This constraint can be removed once we upgrade to Python 3.11 lxml<5.0 +# This has to be constrained as well because newer versions of edx-i18n-tools need the +# newer version of lxml but that requirement was not made expilict in the 1.6.0 version +# of the package. This can be un-pinned when we're upgrading lxml. +edx-i18n-tools<1.6.0 # xmlsec==1.3.14 breaking tests for all builds, can be removed once a fix is available xmlsec<1.3.14 @@ -133,3 +126,5 @@ moto<5.0 # needs to be investigated and fixed separately path<16.12.0 +# Temporary to Support the python 3.11 Upgrade +backports.zoneinfo;python_version<"3.9" # Newer versions have zoneinfo available in the standard library diff --git a/requirements/edx-sandbox/README.rst b/requirements/edx-sandbox/README.rst index 6129aa865b31..51dd7488444e 100644 --- a/requirements/edx-sandbox/README.rst +++ b/requirements/edx-sandbox/README.rst @@ -13,47 +13,64 @@ Files in this directory base.in ======= -This is the current set of requirements or the edx-sandbox -environment, and it is used to generate the ``.txt`` files described below. -These requirements share some constraints with the general edx-platform -requirements (via ``../constraints.txt``), but otherwise, they are completely -separate. +This is the current set of requirements or the edx-sandbox environment, and it +is used to generate the ``.txt`` files described below. These requirements +share some constraints with the general edx-platform requirements (via +``../constraints.txt``), but otherwise, they are completely separate. -We do not recommend installing from this file directly, because -the packages are not pinned. +Installing the edx-sandbox environment from this file is **unsupported** and +**unstable**, because the packages are not pinned. base.txt ======== -These are the latest requirement pins for edx-sandbox. -They are regularly updated with the latest compatible versions of each package. +These are the latest requirement pins for edx-sandbox. They are regularly +updated with the latest compatible versions of each package. -Install from this file if you wish to always run the latest edx-sandbox -environment. Take note that there will periodically be breaking changes to -``base.txt``. For example, we may update the Python version used to generate -the pins, which may break edx-sandbox environments running older Python -versions. +Installing the edx-sandbox environment from this file is **supported** yet +**unstable**. Breaking package upgrades and Python langugae upgrades will +regularly land directly in base.txt. -releases/(RELEASE_NAME).txt -=========================== - -*e.g. releases/redwood.txt, releases/sumac.txt, etc.* +releases/ +========= Starting with Quince, every named Open edX release adds one of these files. -They contain the requirement pins corresponding to ``base.txt`` at the time -of each release. +They contain the requirement pins corresponding to ``base.txt`` at the time of +each release. + +Installing the edx-sandbox environment from the *latest* release file is +**supported** and **stable**. Installing the edx-sandbox environment from +*older* release files is **unsupported** yet **stable**. + +When migrating from one release file to a newer one, be aware of which Python +versions are supported as well as breaking changes in newer packages versions. +You may need to edit the instructor-authored Python code in your platform in +order for it to remain compatible. The edx-platform maintenance team will do their +best to make note of these changes below and in the Open edX release notes. + +releases/quince.txt +------------------- + +* Frozen between the Quince and Redwood releases +* Supports only Python 3.8 + +releases/redwood.txt +---------------------------------- -Install from one of these files if you want to run a stable edx-sandbox -environment without breaking changes. +* Frozen at the time of the Redwood release +* Supports Python 3.8 and Python 3.11 +* BREAKING CHANGE: SciPy is upgraded from 1.7.3 to 1.10.1 (`SciPy changelog`_) +* BREAKING CHANGE: NumPy is upgraded from 1.22.4 to 1.24.4 + (`NumPy changelog`_) +* These upgrades prepare edx-sandbox for the Python 3.12 update in Sumac. -Support windows -*************** +releases/sumac.txt (FUTURE PLAN) +-------------------------------- -Only ``base.txt`` and the latest ``release/*.txt`` from the latest named -release are supported by the Open edX community. However, we will leave -old ``release/*.txt`` files in the repository to assist: +* Frozen at the time of the Sumac release +* BREAKING CHANGE: Drops support for Python 3.8 (`Python changelog`_) +* Supports Python 3.11 and Python 3.12 -* operators who want to stagger their edx-sandbox upgrade from their general - edx-platform upgrade -* operators who need to temporarily roll back their edx-sandbox environments - so that instructors can fix their loncapa Python code. +.. _Python changelog: https://docs.python.org/3.11/whatsnew/changelog.html +.. _SciPy changelog: https://docs.scipy.org/doc/scipy/release.html +.. _NumPy changelog: https://numpy.org/doc/stable/release.html diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index 605de7c4cf6e..db59a1ecbbd1 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -16,17 +16,15 @@ codejail-includes==1.0.0 # via -r requirements/edx-sandbox/base.in contourpy==1.1.1 # via matplotlib -cryptography==38.0.4 - # via - # -c requirements/edx-sandbox/../constraints.txt - # -r requirements/edx-sandbox/base.in +cryptography==42.0.7 + # via -r requirements/edx-sandbox/base.in cycler==0.12.1 # via matplotlib fonttools==4.51.0 # via matplotlib importlib-resources==6.4.0 # via matplotlib -joblib==1.4.0 +joblib==1.4.2 # via nltk kiwisolver==1.4.5 # via matplotlib @@ -46,7 +44,7 @@ nltk==3.8.1 # via # -r requirements/edx-sandbox/base.in # chem -numpy==1.22.4 +numpy==1.24.4 # via # chem # contourpy @@ -71,11 +69,10 @@ python-dateutil==2.9.0.post0 # via matplotlib random2==1.0.2 # via -r requirements/edx-sandbox/base.in -regex==2024.4.28 +regex==2024.5.10 # via nltk -scipy==1.7.3 +scipy==1.10.1 # via - # -c requirements/edx-sandbox/../constraints.txt # -r requirements/edx-sandbox/base.in # chem # openedx-calc @@ -87,7 +84,7 @@ sympy==1.12 # via # -r requirements/edx-sandbox/base.in # openedx-calc -tqdm==4.66.2 +tqdm==4.66.4 # via nltk zipp==3.18.1 # via importlib-resources diff --git a/requirements/edx-sandbox/releases/redwood.txt b/requirements/edx-sandbox/releases/redwood.txt new file mode 100644 index 000000000000..d12c994fd206 --- /dev/null +++ b/requirements/edx-sandbox/releases/redwood.txt @@ -0,0 +1,93 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# make upgrade +# +cffi==1.16.0 + # via cryptography +chem==1.3.0 + # via -r requirements/edx-sandbox/base.in +click==8.1.6 + # via + # -c requirements/edx-sandbox/../constraints.txt + # nltk +codejail-includes==1.0.0 + # via -r requirements/edx-sandbox/base.in +contourpy==1.1.1 + # via matplotlib +cryptography==42.0.7 + # via -r requirements/edx-sandbox/base.in +cycler==0.12.1 + # via matplotlib +fonttools==4.51.0 + # via matplotlib +importlib-resources==6.4.0 + # via matplotlib +joblib==1.4.2 + # via nltk +kiwisolver==1.4.5 + # via matplotlib +lxml==4.9.4 + # via + # -c requirements/edx-sandbox/../constraints.txt + # -r requirements/edx-sandbox/base.in + # openedx-calc +markupsafe==2.1.5 + # via + # chem + # openedx-calc +matplotlib==3.7.5 + # via -r requirements/edx-sandbox/base.in +mpmath==1.3.0 + # via sympy +networkx==3.1 + # via -r requirements/edx-sandbox/base.in +nltk==3.8.1 + # via + # -r requirements/edx-sandbox/base.in + # chem +numpy==1.24.4 + # via + # chem + # contourpy + # matplotlib + # openedx-calc + # scipy +openedx-calc==3.1.0 + # via -r requirements/edx-sandbox/base.in +packaging==24.0 + # via matplotlib +pillow==10.3.0 + # via matplotlib +pycparser==2.22 + # via cffi +pyparsing==3.1.2 + # via + # -r requirements/edx-sandbox/base.in + # chem + # matplotlib + # openedx-calc +python-dateutil==2.9.0.post0 + # via matplotlib +random2==1.0.2 + # via -r requirements/edx-sandbox/base.in +regex==2024.4.28 + # via nltk +scipy==1.10.1 + # via + # -r requirements/edx-sandbox/base.in + # chem + # openedx-calc +six==1.16.0 + # via + # codejail-includes + # python-dateutil +sympy==1.12 + # via + # -r requirements/edx-sandbox/base.in + # openedx-calc +tqdm==4.66.4 + # via nltk +zipp==3.18.1 + # via importlib-resources diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index d407e0ca2a31..6152a1114dca 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -55,19 +55,19 @@ attrs==23.2.0 # edx-ace # jsonschema # lti-consumer-xblock - # openedx-blockstore # openedx-events # openedx-learning # referencing -babel==2.14.0 +babel==2.15.0 # via # -r requirements/edx/kernel.in # enmerkar # enmerkar-underscore backoff==1.10.0 # via analytics-python -backports-zoneinfo[tzdata]==0.2.1 +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" # via + # -c requirements/edx/../constraints.txt # celery # django # edx-milestones @@ -88,13 +88,13 @@ bleach[css]==6.1.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/kernel.in -boto3==1.34.94 +boto3==1.34.104 # via # -r requirements/edx/kernel.in # django-ses # fs-s3fs # ora2 -botocore==1.34.94 +botocore==1.34.104 # via # -r requirements/edx/kernel.in # boto3 @@ -103,7 +103,7 @@ bridgekeeper==0.9 # via -r requirements/edx/kernel.in camel-converter[pydantic]==3.1.2 # via meilisearch -celery==5.3.6 +celery==5.4.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in @@ -166,9 +166,8 @@ coreschema==0.0.4 # drf-yasg crowdsourcehinter-xblock==0.7 # via -r requirements/edx/bundled.in -cryptography==38.0.4 +cryptography==42.0.7 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in # django-fernet-fields-v2 # edx-enterprise @@ -179,7 +178,7 @@ cryptography==38.0.4 # pyopenssl # snowflake-connector-python # social-auth-core -cssutils==2.10.2 +cssutils==2.10.3 # via pynliner defusedxml==0.7.1 # via @@ -188,7 +187,7 @@ defusedxml==0.7.1 # ora2 # python3-openid # social-auth-core -django==4.2.11 +django==4.2.13 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt @@ -216,7 +215,6 @@ django==4.2.11 # djangorestframework # done-xblock # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-ace @@ -250,7 +248,6 @@ django==4.2.11 # help-tokens # jsonfield # lti-consumer-xblock - # openedx-blockstore # openedx-django-pyfs # openedx-django-wiki # openedx-events @@ -289,8 +286,6 @@ django-crum==0.7.9 # edx-rbac # edx-toggles # super-csv -django-environ==0.11.2 - # via openedx-blockstore django-fernet-fields-v2==0.9 # via edx-enterprise django-filter==24.2 @@ -298,7 +293,6 @@ django-filter==24.2 # -r requirements/edx/kernel.in # edx-enterprise # lti-consumer-xblock - # openedx-blockstore django-ipware==7.0.1 # via # -r requirements/edx/kernel.in @@ -308,7 +302,7 @@ django-js-asset==2.2.0 # via django-mptt django-method-override==1.0.4 # via -r requirements/edx/kernel.in -django-model-utils==4.5.0 +django-model-utils==4.5.1 # via # -r requirements/edx/kernel.in # django-user-tasks @@ -341,7 +335,7 @@ django-oauth-toolkit==1.7.1 # edx-enterprise django-object-actions==4.2.0 # via edx-enterprise -django-pipeline==3.0.0 +django-pipeline==3.1.0 # via -r requirements/edx/kernel.in django-ratelimit==4.1.0 # via -r requirements/edx/kernel.in @@ -365,7 +359,7 @@ django-statici18n==2.5.0 # -r requirements/edx/kernel.in # lti-consumer-xblock # xblock-drag-and-drop-v2 -django-storages==1.14.2 +django-storages==1.14.3 # via # -r requirements/edx/kernel.in # edxval @@ -379,7 +373,6 @@ django-waffle==4.1.0 # edx-enterprise # edx-proctoring # edx-toggles - # openedx-blockstore django-webpack-loader==0.7.0 # via # -c requirements/edx/../constraints.txt @@ -392,7 +385,6 @@ djangorestframework==3.14.0 # django-config-models # django-user-tasks # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-api-doc-tools @@ -403,7 +395,6 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions - # openedx-blockstore # openedx-learning # ora2 # super-csv @@ -432,12 +423,9 @@ edx-api-doc-tools==1.8.0 # via # -r requirements/edx/kernel.in # edx-name-affirmation - # openedx-blockstore edx-auth-backends==4.3.0 - # via - # -r requirements/edx/kernel.in - # openedx-blockstore -edx-braze-client==0.2.3 + # via -r requirements/edx/kernel.in +edx-braze-client==0.2.5 # via # -r requirements/edx/bundled.in # edx-enterprise @@ -449,6 +437,7 @@ edx-ccx-keys==1.3.0 # via # -r requirements/edx/kernel.in # lti-consumer-xblock + # openedx-events edx-celeryutils==1.3.0 # via # -r requirements/edx/kernel.in @@ -462,7 +451,6 @@ edx-django-release-util==1.4.0 # via # -r requirements/edx/kernel.in # edxval - # openedx-blockstore edx-django-sites-extensions==4.2.0 # via -r requirements/edx/kernel.in edx-django-utils==5.13.0 @@ -478,7 +466,6 @@ edx-django-utils==5.13.0 # edx-toggles # edx-when # event-tracking - # openedx-blockstore # openedx-events # ora2 # super-csv @@ -494,7 +481,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.16.5 +edx-enterprise==4.19.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in @@ -504,6 +491,7 @@ edx-event-bus-redis==0.5.0 # via -r requirements/edx/kernel.in edx-i18n-tools==1.6.0 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/bundled.in # ora2 edx-milestones==0.6.0 @@ -621,6 +609,7 @@ idna==3.7 importlib-metadata==6.11.0 # via # -c requirements/edx/../common_constraints.txt + # -r requirements/edx/kernel.in # markdown importlib-resources==5.13.0 # via @@ -639,7 +628,7 @@ isodate==0.6.1 # via python3-saml itypes==1.2.0 # via coreapi -jinja2==3.1.3 +jinja2==3.1.4 # via # code-annotations # coreschema @@ -647,7 +636,7 @@ jmespath==1.0.1 # via # boto3 # botocore -joblib==1.4.0 +joblib==1.4.2 # via nltk jsondiff==2.0.0 # via edx-enterprise @@ -660,7 +649,7 @@ jsonfield==3.1.0 # edx-submissions # lti-consumer-xblock # ora2 -jsonschema==4.21.1 +jsonschema==4.22.0 # via # drf-spectacular # optimizely-sdk @@ -707,7 +696,7 @@ lxml-html-clean==0.1.1 # via lxml mailsnake==1.6.4 # via -r requirements/edx/bundled.in -mako==1.3.3 +mako==1.3.5 # via # -r requirements/edx/kernel.in # acid-xblock @@ -731,7 +720,7 @@ markupsafe==2.1.5 # xblock maxminddb==2.6.1 # via geoip2 -meilisearch==0.31.0 +meilisearch==0.31.1 # via -r requirements/edx/kernel.in mock==5.1.0 # via -r requirements/edx/paver.txt @@ -748,10 +737,8 @@ multidict==6.0.5 # aiohttp # yarl mysqlclient==2.2.4 - # via - # -r requirements/edx/kernel.in - # openedx-blockstore -newrelic==9.9.0 + # via -r requirements/edx/kernel.in +newrelic==9.9.1 # via # -r requirements/edx/bundled.in # edx-django-utils @@ -759,7 +746,7 @@ nltk==3.8.1 # via chem nodeenv==1.8.0 # via -r requirements/edx/kernel.in -numpy==1.22.4 +numpy==1.24.4 # via # chem # openedx-calc @@ -780,8 +767,6 @@ openai==0.28.1 # edx-enterprise openedx-atlas==0.6.0 # via -r requirements/edx/kernel.in -openedx-blockstore==1.4.0 - # via -r requirements/edx/kernel.in openedx-calc==3.1.0 # via -r requirements/edx/kernel.in openedx-django-pyfs==3.6.0 @@ -792,7 +777,7 @@ openedx-django-require==2.1.0 # via -r requirements/edx/kernel.in openedx-django-wiki==2.1.0 # via -r requirements/edx/kernel.in -openedx-events==9.9.2 +openedx-events==9.10.0 # via # -r requirements/edx/kernel.in # edx-event-bus-kafka @@ -804,7 +789,7 @@ openedx-filters==1.8.1 # -r requirements/edx/kernel.in # lti-consumer-xblock # ora2 -openedx-learning==0.9.2 +openedx-learning==0.9.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in @@ -812,7 +797,7 @@ optimizely-sdk==4.1.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/bundled.in -ora2==6.7.0 +ora2==6.11.0 # via -r requirements/edx/bundled.in packaging==24.0 # via @@ -882,7 +867,7 @@ pydantic==2.7.1 # via camel-converter pydantic-core==2.18.2 # via pydantic -pygments==2.17.2 +pygments==2.18.0 # via # -r requirements/edx/bundled.in # py2neo @@ -921,9 +906,8 @@ pynacl==1.5.0 # via edx-django-utils pynliner==0.8.0 # via -r requirements/edx/kernel.in -pyopenssl==22.0.0 +pyopenssl==24.1.0 # via - # -c requirements/edx/../constraints.txt # optimizely-sdk # snowflake-connector-python pyparsing==3.1.2 @@ -980,7 +964,6 @@ pytz==2024.1 # icalendar # interchange # olxcleaner - # openedx-blockstore # ora2 # snowflake-connector-python # xblock @@ -1002,11 +985,11 @@ redis==5.0.4 # via # -r requirements/edx/kernel.in # walrus -referencing==0.35.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications -regex==2024.4.28 +regex==2024.5.10 # via nltk requests==2.31.0 # via @@ -1037,7 +1020,7 @@ requests-oauthlib==2.0.0 # via # -r requirements/edx/kernel.in # social-auth-core -rpds-py==0.18.0 +rpds-py==0.18.1 # via # jsonschema # referencing @@ -1055,9 +1038,8 @@ s3transfer==0.10.1 # via boto3 sailthru-client==2.2.3 # via edx-ace -scipy==1.7.3 +scipy==1.10.1 # via - # -c requirements/edx/../constraints.txt # chem # openedx-calc semantic-version==2.10.0 @@ -1129,10 +1111,6 @@ sortedcontainers==2.4.0 soupsieve==2.5 # via beautifulsoup4 sqlparse==0.5.0 - # via - # -r requirements/edx/kernel.in - # django - # openedx-blockstore staff-graded-xblock==2.3.0 # via -r requirements/edx/bundled.in stevedore==5.2.0 @@ -1147,16 +1125,16 @@ stevedore==5.2.0 super-csv==3.2.0 # via edx-bulk-grades sympy==1.12 - # via openedx-calc -testfixtures==8.1.0 + # via openedx-cal +testfixtures==8.2.0 # via edx-enterprise text-unidecode==1.3 # via python-slugify tinycss2==1.2.1 # via bleach -tomlkit==0.12.4 +tomlkit==0.12.5 # via snowflake-connector-python -tqdm==4.66.2 +tqdm==4.66.4 # via # nltk # openai diff --git a/requirements/edx/coverage.txt b/requirements/edx/coverage.txt index 132aed6325ff..e150dc3fe238 100644 --- a/requirements/edx/coverage.txt +++ b/requirements/edx/coverage.txt @@ -6,15 +6,15 @@ # chardet==5.2.0 # via diff-cover -coverage==7.5.0 +coverage==7.5.1 # via -r requirements/edx/coverage.in diff-cover==9.0.0 # via -r requirements/edx/coverage.in -jinja2==3.1.3 +jinja2==3.1.4 # via diff-cover markupsafe==2.1.5 # via jinja2 pluggy==1.5.0 # via diff-cover -pygments==2.17.2 +pygments==2.18.0 # via diff-cover diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 003290829629..1cebd0beaea6 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -68,7 +68,9 @@ annotated-types==0.6.0 anyio==4.3.0 # via # -r requirements/edx/testing.txt + # httpx # starlette + # watchfiles appdirs==1.4.4 # via # -r requirements/edx/doc.txt @@ -105,11 +107,10 @@ attrs==23.2.0 # edx-ace # jsonschema # lti-consumer-xblock - # openedx-blockstore # openedx-events # openedx-learning # referencing -babel==2.14.0 +babel==2.15.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -122,8 +123,9 @@ backoff==1.10.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # analytics-python -backports-zoneinfo[tzdata]==0.2.1 +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # celery @@ -156,14 +158,14 @@ boto==2.49.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -boto3==1.34.94 +boto3==1.34.104 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-ses # fs-s3fs # ora2 -botocore==1.34.94 +botocore==1.34.104 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -186,7 +188,7 @@ camel-converter[pydantic]==3.1.2 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # meilisearch -celery==5.3.6 +celery==5.4.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt @@ -202,6 +204,8 @@ certifi==2024.2.2 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # elasticsearch + # httpcore + # httpx # py2neo # requests # snowflake-connector-python @@ -251,6 +255,7 @@ click==8.1.6 # nltk # pact-python # pip-tools + # typer # user-util # uvicorn click-didyoumean==0.3.1 @@ -298,7 +303,7 @@ coreschema==0.0.4 # -r requirements/edx/testing.txt # coreapi # drf-yasg -coverage[toml]==7.5.0 +coverage[toml]==7.5.1 # via # -r requirements/edx/testing.txt # pytest-cov @@ -306,9 +311,8 @@ crowdsourcehinter-xblock==0.7 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -cryptography==38.0.4 +cryptography==42.0.7 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-fernet-fields-v2 @@ -324,7 +328,7 @@ cssselect==1.2.0 # via # -r requirements/edx/testing.txt # pyquery -cssutils==2.10.2 +cssutils==2.10.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -353,7 +357,7 @@ distlib==0.3.8 # via # -r requirements/edx/testing.txt # virtualenv -django==4.2.11 +django==4.2.13 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt @@ -385,7 +389,6 @@ django==4.2.11 # djangorestframework # done-xblock # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-ace @@ -419,7 +422,6 @@ django==4.2.11 # help-tokens # jsonfield # lti-consumer-xblock - # openedx-blockstore # openedx-django-pyfs # openedx-django-wiki # openedx-events @@ -476,11 +478,6 @@ django-crum==0.7.9 # super-csv django-debug-toolbar==4.3.0 # via -r requirements/edx/development.in -django-environ==0.11.2 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt - # openedx-blockstore django-fernet-fields-v2==0.9 # via # -r requirements/edx/doc.txt @@ -492,7 +489,6 @@ django-filter==24.2 # -r requirements/edx/testing.txt # edx-enterprise # lti-consumer-xblock - # openedx-blockstore django-ipware==7.0.1 # via # -r requirements/edx/doc.txt @@ -508,7 +504,7 @@ django-method-override==1.0.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -django-model-utils==4.5.0 +django-model-utils==4.5.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -552,7 +548,7 @@ django-object-actions==4.2.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-enterprise -django-pipeline==3.0.0 +django-pipeline==3.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -585,7 +581,7 @@ django-statici18n==2.5.0 # -r requirements/edx/testing.txt # lti-consumer-xblock # xblock-drag-and-drop-v2 -django-storages==1.14.2 +django-storages==1.14.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -595,7 +591,7 @@ django-stubs==1.16.0 # -c requirements/edx/../constraints.txt # -r requirements/edx/development.in # djangorestframework-stubs -django-stubs-ext==4.2.7 +django-stubs-ext==5.0.0 # via django-stubs django-user-tasks==3.2.0 # via @@ -610,7 +606,6 @@ django-waffle==4.1.0 # edx-enterprise # edx-proctoring # edx-toggles - # openedx-blockstore django-webpack-loader==0.7.0 # via # -c requirements/edx/../constraints.txt @@ -625,7 +620,6 @@ djangorestframework==3.14.0 # django-config-models # django-user-tasks # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-api-doc-tools @@ -636,7 +630,6 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions - # openedx-blockstore # openedx-learning # ora2 # super-csv @@ -651,9 +644,6 @@ djangorestframework-xml==2.0.0 # edx-enterprise dnspython==2.6.1 # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt - # pymongo docutils==0.19 # via # -r requirements/edx/doc.txt @@ -694,7 +684,6 @@ edx-api-doc-tools==1.8.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-name-affirmation - # openedx-blockstore edx-auth-backends==4.3.0 # via # -r requirements/edx/doc.txt @@ -715,6 +704,7 @@ edx-ccx-keys==1.3.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # lti-consumer-xblock + # openedx-events edx-celeryutils==1.3.0 # via # -r requirements/edx/doc.txt @@ -734,7 +724,6 @@ edx-django-release-util==1.4.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edxval - # openedx-blockstore edx-django-sites-extensions==4.2.0 # via # -r requirements/edx/doc.txt @@ -753,7 +742,6 @@ edx-django-utils==5.13.0 # edx-toggles # edx-when # event-tracking - # openedx-blockstore # openedx-events # ora2 # super-csv @@ -770,7 +758,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.16.5 +edx-enterprise==4.19.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt @@ -785,6 +773,7 @@ edx-event-bus-redis==0.5.0 # -r requirements/edx/testing.txt edx-i18n-tools==1.6.0 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # ora2 @@ -884,6 +873,10 @@ elasticsearch==7.13.4 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-search +email-validator==2.1.1 + # via + # -r requirements/edx/testing.txt + # fastapi enmerkar==0.7.1 # via # -r requirements/edx/doc.txt @@ -893,6 +886,13 @@ enmerkar-underscore==2.3.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt +event-tracking==2.4.0 + # via + # -r requirements/edx/doc.txt + # -r requirements/edx/testing.txt + # edx-completion + # edx-proctoring + # edx-search exceptiongroup==1.2.1 # via # -r requirements/edx/testing.txt @@ -904,14 +904,19 @@ execnet==2.1.1 # pytest-xdist factory-boy==3.3.0 # via -r requirements/edx/testing.txt -faker==25.0.0 +faker==25.2.0 # via # -r requirements/edx/testing.txt # factory-boy -fastapi==0.110.3 +fastapi==0.111.0 # via # -r requirements/edx/testing.txt + # fastapi-cli # pact-python +fastapi-cli==0.0.3 + # via + # -r requirements/edx/testing.txt + # fastapi fastavro==1.9.4 # via # -r requirements/edx/doc.txt @@ -924,7 +929,7 @@ filelock==3.14.0 # snowflake-connector-python # tox # virtualenv -freezegun==1.5.0 +freezegun==1.5.1 # via -r requirements/edx/testing.txt frozenlist==1.4.1 # via @@ -974,6 +979,7 @@ gunicorn==22.0.0 h11==0.14.0 # via # -r requirements/edx/testing.txt + # httpcore # uvicorn help-tokens==2.4.0 # via @@ -984,8 +990,20 @@ html5lib==1.1 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # ora2 +httpcore==1.0.5 + # via + # -r requirements/edx/testing.txt + # httpx httpretty==1.1.4 # via -r requirements/edx/testing.txt +httptools==0.6.1 + # via + # -r requirements/edx/testing.txt + # uvicorn +httpx==0.27.0 + # via + # -r requirements/edx/testing.txt + # fastapi icalendar==5.0.12 # via # -r requirements/edx/doc.txt @@ -995,6 +1013,8 @@ idna==3.7 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # anyio + # email-validator + # httpx # optimizely-sdk # requests # snowflake-connector-python @@ -1055,13 +1075,14 @@ itypes==1.2.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # coreapi -jinja2==3.1.3 +jinja2==3.1.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # code-annotations # coreschema # diff-cover + # fastapi # sphinx jmespath==1.0.1 # via @@ -1069,7 +1090,7 @@ jmespath==1.0.1 # -r requirements/edx/testing.txt # boto3 # botocore -joblib==1.4.0 +joblib==1.4.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1089,7 +1110,7 @@ jsonfield==3.1.0 # edx-submissions # lti-consumer-xblock # ora2 -jsonschema==4.21.1 +jsonschema==4.22.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1167,7 +1188,7 @@ mailsnake==1.6.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -mako==1.3.3 +mako==1.3.5 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1183,6 +1204,10 @@ markdown==3.3.7 # openedx-django-wiki # staff-graded-xblock # xblock-poll +markdown-it-py==3.0.0 + # via + # -r requirements/edx/testing.txt + # rich markupsafe==2.1.5 # via # -r requirements/edx/doc.txt @@ -1201,7 +1226,11 @@ mccabe==0.7.0 # via # -r requirements/edx/testing.txt # pylint -meilisearch==0.31.0 +mdurl==0.1.2 + # via + # -r requirements/edx/testing.txt + # markdown-it-py +meilisearch==0.31.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1245,8 +1274,7 @@ mysqlclient==2.2.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt - # openedx-blockstore -newrelic==9.9.0 +newrelic==9.9.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1261,7 +1289,7 @@ nodeenv==1.8.0 # -r requirements/edx/assets.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -numpy==1.22.4 +numpy==1.24.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1291,10 +1319,6 @@ openedx-atlas==0.6.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -openedx-blockstore==1.4.0 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt openedx-calc==3.1.0 # via # -r requirements/edx/doc.txt @@ -1313,7 +1337,7 @@ openedx-django-wiki==2.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -openedx-events==9.9.2 +openedx-events==9.10.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1327,7 +1351,7 @@ openedx-filters==1.8.1 # -r requirements/edx/testing.txt # lti-consumer-xblock # ora2 -openedx-learning==0.9.2 +openedx-learning==0.9.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt @@ -1337,10 +1361,14 @@ optimizely-sdk==4.1.1 # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -ora2==6.7.0 +ora2==6.11.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt +orjson==3.10.3 + # via + # -r requirements/edx/testing.txt + # fastapi packaging==24.0 # via # -r requirements/edx/../pip-tools.txt @@ -1491,7 +1519,7 @@ pydata-sphinx-theme==0.14.4 # via # -r requirements/edx/doc.txt # sphinx-book-theme -pygments==2.17.2 +pygments==2.18.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1499,6 +1527,7 @@ pygments==2.17.2 # diff-cover # py2neo # pydata-sphinx-theme + # rich # sphinx # sphinx-mdinclude pyjwkest==1.4.2 @@ -1574,9 +1603,8 @@ pynliner==0.8.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -pyopenssl==22.0.0 +pyopenssl==24.1.0 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # optimizely-sdk @@ -1651,6 +1679,10 @@ python-dateutil==2.9.0.post0 # olxcleaner # ora2 # xblock +python-dotenv==1.0.1 + # via + # -r requirements/edx/testing.txt + # uvicorn python-ipware==3.0.0 # via # -r requirements/edx/doc.txt @@ -1660,6 +1692,10 @@ python-memcached==1.62 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt +python-multipart==0.0.9 + # via + # -r requirements/edx/testing.txt + # fastapi python-slugify==8.0.4 # via # -r requirements/edx/doc.txt @@ -1697,7 +1733,6 @@ pytz==2024.1 # icalendar # interchange # olxcleaner - # openedx-blockstore # ora2 # snowflake-connector-python # xblock @@ -1716,6 +1751,7 @@ pyyaml==6.0.1 # edx-django-release-util # edx-i18n-tools # sphinxcontrib-openapi + # uvicorn # xblock random2==1.0.2 # via @@ -1730,13 +1766,13 @@ redis==5.0.4 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # walrus -referencing==0.35.0 +referencing==0.35.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # jsonschema # jsonschema-specifications -regex==2024.4.28 +regex==2024.5.10 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1775,7 +1811,11 @@ requests-oauthlib==2.0.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # social-auth-core -rpds-py==0.18.0 +rich==13.7.1 + # via + # -r requirements/edx/testing.txt + # typer +rpds-py==0.18.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1808,9 +1848,8 @@ sailthru-client==2.2.3 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-ace -scipy==1.7.3 +scipy==1.10.1 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # chem @@ -1824,6 +1863,10 @@ shapely==2.0.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt +shellingham==1.5.4 + # via + # -r requirements/edx/testing.txt + # typer simplejson==3.19.2 # via # -r requirements/edx/doc.txt @@ -1881,6 +1924,7 @@ sniffio==1.3.1 # via # -r requirements/edx/testing.txt # anyio + # httpx snowballstemmer==2.2.0 # via # -r requirements/edx/doc.txt @@ -1976,7 +2020,6 @@ sqlparse==0.5.0 # -r requirements/edx/testing.txt # django # django-debug-toolbar - # openedx-blockstore staff-graded-xblock==2.3.0 # via # -r requirements/edx/doc.txt @@ -2004,7 +2047,7 @@ sympy==1.12 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # openedx-calc -testfixtures==8.1.0 +testfixtures==8.2.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2034,7 +2077,7 @@ tomli==2.0.1 # pytest # tox # vulture -tomlkit==0.12.4 +tomlkit==0.12.5 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2042,12 +2085,16 @@ tomlkit==0.12.4 # snowflake-connector-python tox==4.15.0 # via -r requirements/edx/testing.txt -tqdm==4.66.2 +tqdm==4.66.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # nltk # openai +typer==0.12.3 + # via + # -r requirements/edx/testing.txt + # fastapi-cli types-pytz==2024.1.0.20240417 # via django-stubs types-pyyaml==6.0.12.20240311 @@ -2084,8 +2131,10 @@ typing-extensions==4.11.0 # pydata-sphinx-theme # pylint # pylti1p3 + # rich # snowflake-connector-python # starlette + # typer # uvicorn tzdata==2024.1 # via @@ -2093,6 +2142,10 @@ tzdata==2024.1 # -r requirements/edx/testing.txt # backports-zoneinfo # celery +ujson==5.10.0 + # via + # -r requirements/edx/testing.txt + # fastapi unicodecsv==0.14.1 # via # -r requirements/edx/doc.txt @@ -2121,10 +2174,16 @@ user-util==1.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -uvicorn==0.29.0 +uvicorn[standard]==0.29.0 # via # -r requirements/edx/testing.txt + # fastapi + # fastapi-cli # pact-python +uvloop==0.19.0 + # via + # -r requirements/edx/testing.txt + # uvicorn vine==5.1.0 # via # -r requirements/edx/doc.txt @@ -2132,7 +2191,7 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==20.26.1 +virtualenv==20.26.2 # via # -r requirements/edx/testing.txt # tox @@ -2153,6 +2212,10 @@ watchdog==4.0.0 # -r requirements/edx/development.in # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt +watchfiles==0.21.0 + # via + # -r requirements/edx/testing.txt + # uvicorn wcwidth==0.2.13 # via # -r requirements/edx/doc.txt @@ -2179,6 +2242,10 @@ webob==1.8.7 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # xblock +websockets==12.0 + # via + # -r requirements/edx/testing.txt + # uvicorn wheel==0.43.0 # via # -r requirements/edx/../pip-tools.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 7c9e177cf0d3..b74e81dafc68 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -71,11 +71,10 @@ attrs==23.2.0 # edx-ace # jsonschema # lti-consumer-xblock - # openedx-blockstore # openedx-events # openedx-learning # referencing -babel==2.14.0 +babel==2.15.0 # via # -r requirements/edx/base.txt # enmerkar @@ -86,8 +85,9 @@ backoff==1.10.0 # via # -r requirements/edx/base.txt # analytics-python -backports-zoneinfo[tzdata]==0.2.1 +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # celery # django @@ -114,13 +114,13 @@ bleach[css]==6.1.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.34.94 +boto3==1.34.104 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 -botocore==1.34.94 +botocore==1.34.104 # via # -r requirements/edx/base.txt # boto3 @@ -131,7 +131,7 @@ camel-converter[pydantic]==3.1.2 # via # -r requirements/edx/base.txt # meilisearch -celery==5.3.6 +celery==5.4.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt @@ -209,9 +209,8 @@ coreschema==0.0.4 # drf-yasg crowdsourcehinter-xblock==0.7 # via -r requirements/edx/base.txt -cryptography==38.0.4 +cryptography==42.0.7 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # django-fernet-fields-v2 # edx-enterprise @@ -222,7 +221,7 @@ cryptography==38.0.4 # pyopenssl # snowflake-connector-python # social-auth-core -cssutils==2.10.2 +cssutils==2.10.3 # via # -r requirements/edx/base.txt # pynliner @@ -235,7 +234,7 @@ defusedxml==0.7.1 # ora2 # python3-openid # social-auth-core -django==4.2.11 +django==4.2.13 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt @@ -263,7 +262,6 @@ django==4.2.11 # djangorestframework # done-xblock # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-ace @@ -297,7 +295,6 @@ django==4.2.11 # help-tokens # jsonfield # lti-consumer-xblock - # openedx-blockstore # openedx-django-pyfs # openedx-django-wiki # openedx-events @@ -342,10 +339,6 @@ django-crum==0.7.9 # edx-rbac # edx-toggles # super-csv -django-environ==0.11.2 - # via - # -r requirements/edx/base.txt - # openedx-blockstore django-fernet-fields-v2==0.9 # via # -r requirements/edx/base.txt @@ -355,7 +348,6 @@ django-filter==24.2 # -r requirements/edx/base.txt # edx-enterprise # lti-consumer-xblock - # openedx-blockstore django-ipware==7.0.1 # via # -r requirements/edx/base.txt @@ -367,7 +359,7 @@ django-js-asset==2.2.0 # django-mptt django-method-override==1.0.4 # via -r requirements/edx/base.txt -django-model-utils==4.5.0 +django-model-utils==4.5.1 # via # -r requirements/edx/base.txt # django-user-tasks @@ -404,7 +396,7 @@ django-object-actions==4.2.0 # via # -r requirements/edx/base.txt # edx-enterprise -django-pipeline==3.0.0 +django-pipeline==3.1.0 # via -r requirements/edx/base.txt django-ratelimit==4.1.0 # via -r requirements/edx/base.txt @@ -428,7 +420,7 @@ django-statici18n==2.5.0 # -r requirements/edx/base.txt # lti-consumer-xblock # xblock-drag-and-drop-v2 -django-storages==1.14.2 +django-storages==1.14.3 # via # -r requirements/edx/base.txt # edxval @@ -442,7 +434,6 @@ django-waffle==4.1.0 # edx-enterprise # edx-proctoring # edx-toggles - # openedx-blockstore django-webpack-loader==0.7.0 # via # -c requirements/edx/../constraints.txt @@ -455,7 +446,6 @@ djangorestframework==3.14.0 # django-config-models # django-user-tasks # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-api-doc-tools @@ -466,7 +456,6 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions - # openedx-blockstore # openedx-learning # ora2 # super-csv @@ -507,12 +496,9 @@ edx-api-doc-tools==1.8.0 # via # -r requirements/edx/base.txt # edx-name-affirmation - # openedx-blockstore edx-auth-backends==4.3.0 - # via - # -r requirements/edx/base.txt - # openedx-blockstore -edx-braze-client==0.2.3 + # via -r requirements/edx/base.txt +edx-braze-client==0.2.5 # via # -r requirements/edx/base.txt # edx-enterprise @@ -524,6 +510,7 @@ edx-ccx-keys==1.3.0 # via # -r requirements/edx/base.txt # lti-consumer-xblock + # openedx-events edx-celeryutils==1.3.0 # via # -r requirements/edx/base.txt @@ -537,7 +524,6 @@ edx-django-release-util==1.4.0 # via # -r requirements/edx/base.txt # edxval - # openedx-blockstore edx-django-sites-extensions==4.2.0 # via -r requirements/edx/base.txt edx-django-utils==5.13.0 @@ -553,7 +539,6 @@ edx-django-utils==5.13.0 # edx-toggles # edx-when # event-tracking - # openedx-blockstore # openedx-events # ora2 # super-csv @@ -569,7 +554,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.16.5 +edx-enterprise==4.19.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt @@ -579,6 +564,7 @@ edx-event-bus-redis==0.5.0 # via -r requirements/edx/base.txt edx-i18n-tools==1.6.0 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # ora2 edx-milestones==0.6.0 @@ -743,7 +729,7 @@ itypes==1.2.0 # via # -r requirements/edx/base.txt # coreapi -jinja2==3.1.3 +jinja2==3.1.4 # via # -r requirements/edx/base.txt # code-annotations @@ -754,7 +740,7 @@ jmespath==1.0.1 # -r requirements/edx/base.txt # boto3 # botocore -joblib==1.4.0 +joblib==1.4.2 # via # -r requirements/edx/base.txt # nltk @@ -771,7 +757,7 @@ jsonfield==3.1.0 # edx-submissions # lti-consumer-xblock # ora2 -jsonschema==4.21.1 +jsonschema==4.22.0 # via # -r requirements/edx/base.txt # drf-spectacular @@ -829,7 +815,7 @@ lxml-html-clean==0.1.1 # lxml mailsnake==1.6.4 # via -r requirements/edx/base.txt -mako==1.3.3 +mako==1.3.5 # via # -r requirements/edx/base.txt # acid-xblock @@ -855,7 +841,7 @@ maxminddb==2.6.1 # via # -r requirements/edx/base.txt # geoip2 -meilisearch==0.31.0 +meilisearch==0.31.1 # via -r requirements/edx/base.txt mistune==3.0.2 # via sphinx-mdinclude @@ -878,10 +864,8 @@ multidict==6.0.5 # aiohttp # yarl mysqlclient==2.2.4 - # via - # -r requirements/edx/base.txt - # openedx-blockstore -newrelic==9.9.0 + # via -r requirements/edx/base.txt +newrelic==9.9.1 # via # -r requirements/edx/base.txt # edx-django-utils @@ -891,7 +875,7 @@ nltk==3.8.1 # chem nodeenv==1.8.0 # via -r requirements/edx/base.txt -numpy==1.22.4 +numpy==1.24.4 # via # -r requirements/edx/base.txt # chem @@ -914,8 +898,6 @@ openai==0.28.1 # edx-enterprise openedx-atlas==0.6.0 # via -r requirements/edx/base.txt -openedx-blockstore==1.4.0 - # via -r requirements/edx/base.txt openedx-calc==3.1.0 # via -r requirements/edx/base.txt openedx-django-pyfs==3.6.0 @@ -927,7 +909,7 @@ openedx-django-require==2.1.0 # via -r requirements/edx/base.txt openedx-django-wiki==2.1.0 # via -r requirements/edx/base.txt -openedx-events==9.9.2 +openedx-events==9.10.0 # via # -r requirements/edx/base.txt # edx-event-bus-kafka @@ -939,7 +921,7 @@ openedx-filters==1.8.1 # -r requirements/edx/base.txt # lti-consumer-xblock # ora2 -openedx-learning==0.9.2 +openedx-learning==0.9.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt @@ -947,7 +929,7 @@ optimizely-sdk==4.1.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt -ora2==6.7.0 +ora2==6.11.0 # via -r requirements/edx/base.txt packaging==24.0 # via @@ -1044,7 +1026,7 @@ pydantic-core==2.18.2 # pydantic pydata-sphinx-theme==0.14.4 # via sphinx-book-theme -pygments==2.17.2 +pygments==2.18.0 # via # -r requirements/edx/base.txt # accessible-pygments @@ -1090,9 +1072,8 @@ pynacl==1.5.0 # edx-django-utils pynliner==0.8.0 # via -r requirements/edx/base.txt -pyopenssl==22.0.0 +pyopenssl==24.1.0 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # optimizely-sdk # snowflake-connector-python @@ -1159,7 +1140,6 @@ pytz==2024.1 # icalendar # interchange # olxcleaner - # openedx-blockstore # ora2 # snowflake-connector-python # xblock @@ -1182,12 +1162,12 @@ redis==5.0.4 # via # -r requirements/edx/base.txt # walrus -referencing==0.35.0 +referencing==0.35.1 # via # -r requirements/edx/base.txt # jsonschema # jsonschema-specifications -regex==2024.4.28 +regex==2024.5.10 # via # -r requirements/edx/base.txt # nltk @@ -1221,7 +1201,7 @@ requests-oauthlib==2.0.0 # via # -r requirements/edx/base.txt # social-auth-core -rpds-py==0.18.0 +rpds-py==0.18.1 # via # -r requirements/edx/base.txt # jsonschema @@ -1248,9 +1228,8 @@ sailthru-client==2.2.3 # via # -r requirements/edx/base.txt # edx-ace -scipy==1.7.3 +scipy==1.10.1 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # chem # openedx-calc @@ -1372,7 +1351,6 @@ sqlparse==0.5.0 # via # -r requirements/edx/base.txt # django - # openedx-blockstore staff-graded-xblock==2.3.0 # via -r requirements/edx/base.txt stevedore==5.2.0 @@ -1391,7 +1369,7 @@ sympy==1.12 # via # -r requirements/edx/base.txt # openedx-calc -testfixtures==8.1.0 +testfixtures==8.2.0 # via # -r requirements/edx/base.txt # edx-enterprise @@ -1403,11 +1381,11 @@ tinycss2==1.2.1 # via # -r requirements/edx/base.txt # bleach -tomlkit==0.12.4 +tomlkit==0.12.5 # via # -r requirements/edx/base.txt # snowflake-connector-python -tqdm==4.66.2 +tqdm==4.66.4 # via # -r requirements/edx/base.txt # nltk diff --git a/requirements/edx/github.in b/requirements/edx/github.in index 0e11e0922c8c..720285a4470a 100644 --- a/requirements/edx/github.in +++ b/requirements/edx/github.in @@ -29,16 +29,16 @@ # # For example: # -# # https://github.com/openedx/blockstore/issues/212 -# git+https://github.com/openedx/blockstore.git@v1.3.0#egg=openedx-blockstore==1.3.0 +# # https://github.com/openedx/foobar/issues/212 +# git+https://github.com/openedx/foobar.git@v1.3.0#egg=openedx-foobar==1.3.0 # # where: # -# ISSUE-LINK = https://github.com/openedx/blockstore/issues/212 +# ISSUE-LINK = https://github.com/openedx/foobar/issues/212 # OWNER = openedx -# REPO-NAME = blockstore +# REPO-NAME = foobar # TAG-OR-SHA = v1.3.0 -# DIST-NAME = openedx-blockstore +# DIST-NAME = openedx-foobar # VERSION = 1.3.0 # # Rules to follow: diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index c4a1c8541ae8..8173abf7b3f8 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -120,7 +120,6 @@ openedx-filters # Open edX Filters from Hooks Extension Fram openedx-learning # Open edX Learning core (experimental) openedx-mongodbproxy openedx-django-wiki -openedx-blockstore path piexif # Exif image metadata manipulation, used in the profile_images app Pillow # Image manipulation library; used for course assets, profile images, invoice PDFs, etc. @@ -152,7 +151,6 @@ slumber # The following dependency is unsupported an social-auth-app-django sorl-thumbnail sortedcontainers # Provides SortedKeyList, used for lists of XBlock assets -sqlparse # Required by Django to run migrations.RunSQL stevedore # Support for runtime plugins, used for XBlocks and edx-platform Django app plugins unicodecsv # Easier support for CSV files with unicode text user-util # Functionality for retiring users (GDPR compliance) diff --git a/requirements/edx/semgrep.txt b/requirements/edx/semgrep.txt index 70b90e6f175a..e3c9ca0ce3c0 100644 --- a/requirements/edx/semgrep.txt +++ b/requirements/edx/semgrep.txt @@ -44,7 +44,7 @@ importlib-resources==6.4.0 # via # jsonschema # jsonschema-specifications -jsonschema==4.21.1 +jsonschema==4.22.0 # via semgrep jsonschema-specifications==2023.12.1 # via jsonschema @@ -54,13 +54,13 @@ mdurl==0.1.2 # via markdown-it-py packaging==24.0 # via semgrep -peewee==3.17.3 +peewee==3.17.5 # via semgrep pkgutil-resolve-name==1.3.10 # via jsonschema -pygments==2.17.2 +pygments==2.18.0 # via rich -referencing==0.35.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications @@ -68,7 +68,7 @@ requests==2.31.0 # via semgrep rich==13.7.1 # via semgrep -rpds-py==0.18.0 +rpds-py==0.18.1 # via # jsonschema # referencing diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index b2ef8affc9f7..16aae5fc486b 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -42,7 +42,10 @@ annotated-types==0.6.0 # -r requirements/edx/base.txt # pydantic anyio==4.3.0 - # via starlette + # via + # httpx + # starlette + # watchfiles appdirs==1.4.4 # via # -r requirements/edx/base.txt @@ -73,11 +76,10 @@ attrs==23.2.0 # edx-ace # jsonschema # lti-consumer-xblock - # openedx-blockstore # openedx-events # openedx-learning # referencing -babel==2.14.0 +babel==2.15.0 # via # -r requirements/edx/base.txt # enmerkar @@ -86,8 +88,9 @@ backoff==1.10.0 # via # -r requirements/edx/base.txt # analytics-python -backports-zoneinfo[tzdata]==0.2.1 +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # celery # django @@ -114,13 +117,13 @@ bleach[css]==6.1.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.34.94 +boto3==1.34.104 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 -botocore==1.34.94 +botocore==1.34.104 # via # -r requirements/edx/base.txt # boto3 @@ -133,7 +136,7 @@ camel-converter[pydantic]==3.1.2 # via # -r requirements/edx/base.txt # meilisearch -celery==5.3.6 +celery==5.4.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt @@ -147,6 +150,8 @@ certifi==2024.2.2 # via # -r requirements/edx/base.txt # elasticsearch + # httpcore + # httpx # py2neo # requests # snowflake-connector-python @@ -187,6 +192,7 @@ click==8.1.6 # import-linter # nltk # pact-python + # typer # user-util # uvicorn click-didyoumean==0.3.1 @@ -223,15 +229,14 @@ coreschema==0.0.4 # -r requirements/edx/base.txt # coreapi # drf-yasg -coverage[toml]==7.5.0 +coverage[toml]==7.5.1 # via # -r requirements/edx/coverage.txt # pytest-cov crowdsourcehinter-xblock==0.7 # via -r requirements/edx/base.txt -cryptography==38.0.4 +cryptography==42.0.7 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # django-fernet-fields-v2 # edx-enterprise @@ -246,7 +251,7 @@ cssselect==1.2.0 # via # -r requirements/edx/testing.in # pyquery -cssutils==2.10.2 +cssutils==2.10.3 # via # -r requirements/edx/base.txt # pynliner @@ -265,7 +270,7 @@ dill==0.3.8 # via pylint distlib==0.3.8 # via virtualenv -django==4.2.11 +django==4.2.13 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt @@ -293,7 +298,6 @@ django==4.2.11 # djangorestframework # done-xblock # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-ace @@ -327,7 +331,6 @@ django==4.2.11 # help-tokens # jsonfield # lti-consumer-xblock - # openedx-blockstore # openedx-django-pyfs # openedx-django-wiki # openedx-events @@ -372,10 +375,6 @@ django-crum==0.7.9 # edx-rbac # edx-toggles # super-csv -django-environ==0.11.2 - # via - # -r requirements/edx/base.txt - # openedx-blockstore django-fernet-fields-v2==0.9 # via # -r requirements/edx/base.txt @@ -385,7 +384,6 @@ django-filter==24.2 # -r requirements/edx/base.txt # edx-enterprise # lti-consumer-xblock - # openedx-blockstore django-ipware==7.0.1 # via # -r requirements/edx/base.txt @@ -397,7 +395,7 @@ django-js-asset==2.2.0 # django-mptt django-method-override==1.0.4 # via -r requirements/edx/base.txt -django-model-utils==4.5.0 +django-model-utils==4.5.1 # via # -r requirements/edx/base.txt # django-user-tasks @@ -434,7 +432,7 @@ django-object-actions==4.2.0 # via # -r requirements/edx/base.txt # edx-enterprise -django-pipeline==3.0.0 +django-pipeline==3.1.0 # via -r requirements/edx/base.txt django-ratelimit==4.1.0 # via -r requirements/edx/base.txt @@ -458,7 +456,7 @@ django-statici18n==2.5.0 # -r requirements/edx/base.txt # lti-consumer-xblock # xblock-drag-and-drop-v2 -django-storages==1.14.2 +django-storages==1.14.3 # via # -r requirements/edx/base.txt # edxval @@ -472,7 +470,6 @@ django-waffle==4.1.0 # edx-enterprise # edx-proctoring # edx-toggles - # openedx-blockstore django-webpack-loader==0.7.0 # via # -c requirements/edx/../constraints.txt @@ -485,7 +482,6 @@ djangorestframework==3.14.0 # django-config-models # django-user-tasks # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-api-doc-tools @@ -496,7 +492,6 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions - # openedx-blockstore # openedx-learning # ora2 # super-csv @@ -505,9 +500,7 @@ djangorestframework-xml==2.0.0 # -r requirements/edx/base.txt # edx-enterprise dnspython==2.6.1 - # via - # -r requirements/edx/base.txt - # pymongo + # via email-validator done-xblock==2.3.0 # via -r requirements/edx/base.txt drf-jwt==1.19.2 @@ -532,12 +525,9 @@ edx-api-doc-tools==1.8.0 # via # -r requirements/edx/base.txt # edx-name-affirmation - # openedx-blockstore edx-auth-backends==4.3.0 - # via - # -r requirements/edx/base.txt - # openedx-blockstore -edx-braze-client==0.2.3 + # via -r requirements/edx/base.txt +edx-braze-client==0.2.5 # via # -r requirements/edx/base.txt # edx-enterprise @@ -549,6 +539,7 @@ edx-ccx-keys==1.3.0 # via # -r requirements/edx/base.txt # lti-consumer-xblock + # openedx-events edx-celeryutils==1.3.0 # via # -r requirements/edx/base.txt @@ -562,7 +553,6 @@ edx-django-release-util==1.4.0 # via # -r requirements/edx/base.txt # edxval - # openedx-blockstore edx-django-sites-extensions==4.2.0 # via -r requirements/edx/base.txt edx-django-utils==5.13.0 @@ -578,7 +568,6 @@ edx-django-utils==5.13.0 # edx-toggles # edx-when # event-tracking - # openedx-blockstore # openedx-events # ora2 # super-csv @@ -594,7 +583,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.16.5 +edx-enterprise==4.19.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt @@ -604,6 +593,7 @@ edx-event-bus-redis==0.5.0 # via -r requirements/edx/base.txt edx-i18n-tools==1.6.0 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # ora2 edx-lint==5.3.6 @@ -679,12 +669,20 @@ elasticsearch==7.13.4 # -c requirements/edx/../common_constraints.txt # -r requirements/edx/base.txt # edx-search +email-validator==2.1.1 + # via fastapi enmerkar==0.7.1 # via # -r requirements/edx/base.txt # enmerkar-underscore enmerkar-underscore==2.3.0 # via -r requirements/edx/base.txt +event-tracking==2.4.0 + # via + # -r requirements/edx/base.txt + # edx-completion + # edx-proctoring + # edx-search exceptiongroup==1.2.1 # via # anyio @@ -693,10 +691,14 @@ execnet==2.1.1 # via pytest-xdist factory-boy==3.3.0 # via -r requirements/edx/testing.in -faker==25.0.0 +faker==25.2.0 # via factory-boy -fastapi==0.110.3 - # via pact-python +fastapi==0.111.0 + # via + # fastapi-cli + # pact-python +fastapi-cli==0.0.3 + # via fastapi fastavro==1.9.4 # via # -r requirements/edx/base.txt @@ -707,7 +709,7 @@ filelock==3.14.0 # snowflake-connector-python # tox # virtualenv -freezegun==1.5.0 +freezegun==1.5.1 # via -r requirements/edx/testing.in frozenlist==1.4.1 # via @@ -737,21 +739,31 @@ grimp==3.2 gunicorn==22.0.0 # via -r requirements/edx/base.txt h11==0.14.0 - # via uvicorn + # via + # httpcore + # uvicorn help-tokens==2.4.0 # via -r requirements/edx/base.txt html5lib==1.1 # via # -r requirements/edx/base.txt # ora2 +httpcore==1.0.5 + # via httpx httpretty==1.1.4 # via -r requirements/edx/testing.in +httptools==0.6.1 + # via uvicorn +httpx==0.27.0 + # via fastapi icalendar==5.0.12 # via -r requirements/edx/base.txt idna==3.7 # via # -r requirements/edx/base.txt # anyio + # email-validator + # httpx # optimizely-sdk # requests # snowflake-connector-python @@ -795,19 +807,20 @@ itypes==1.2.0 # via # -r requirements/edx/base.txt # coreapi -jinja2==3.1.3 +jinja2==3.1.4 # via # -r requirements/edx/base.txt # -r requirements/edx/coverage.txt # code-annotations # coreschema # diff-cover + # fastapi jmespath==1.0.1 # via # -r requirements/edx/base.txt # boto3 # botocore -joblib==1.4.0 +joblib==1.4.2 # via # -r requirements/edx/base.txt # nltk @@ -824,7 +837,7 @@ jsonfield==3.1.0 # edx-submissions # lti-consumer-xblock # ora2 -jsonschema==4.21.1 +jsonschema==4.22.0 # via # -r requirements/edx/base.txt # drf-spectacular @@ -884,7 +897,7 @@ lxml-html-clean==0.1.1 # lxml mailsnake==1.6.4 # via -r requirements/edx/base.txt -mako==1.3.3 +mako==1.3.5 # via # -r requirements/edx/base.txt # acid-xblock @@ -898,6 +911,8 @@ markdown==3.3.7 # openedx-django-wiki # staff-graded-xblock # xblock-poll +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.5 # via # -r requirements/edx/base.txt @@ -913,7 +928,9 @@ maxminddb==2.6.1 # geoip2 mccabe==0.7.0 # via pylint -meilisearch==0.31.0 +mdurl==0.1.2 + # via markdown-it-py +meilisearch==0.31.1 # via -r requirements/edx/base.txt mock==5.1.0 # via -r requirements/edx/base.txt @@ -934,10 +951,8 @@ multidict==6.0.5 # aiohttp # yarl mysqlclient==2.2.4 - # via - # -r requirements/edx/base.txt - # openedx-blockstore -newrelic==9.9.0 + # via -r requirements/edx/base.txt +newrelic==9.9.1 # via # -r requirements/edx/base.txt # edx-django-utils @@ -947,7 +962,7 @@ nltk==3.8.1 # chem nodeenv==1.8.0 # via -r requirements/edx/base.txt -numpy==1.22.4 +numpy==1.24.4 # via # -r requirements/edx/base.txt # chem @@ -970,8 +985,6 @@ openai==0.28.1 # edx-enterprise openedx-atlas==0.6.0 # via -r requirements/edx/base.txt -openedx-blockstore==1.4.0 - # via -r requirements/edx/base.txt openedx-calc==3.1.0 # via -r requirements/edx/base.txt openedx-django-pyfs==3.6.0 @@ -983,7 +996,7 @@ openedx-django-require==2.1.0 # via -r requirements/edx/base.txt openedx-django-wiki==2.1.0 # via -r requirements/edx/base.txt -openedx-events==9.9.2 +openedx-events==9.10.0 # via # -r requirements/edx/base.txt # edx-event-bus-kafka @@ -995,7 +1008,7 @@ openedx-filters==1.8.1 # -r requirements/edx/base.txt # lti-consumer-xblock # ora2 -openedx-learning==0.9.2 +openedx-learning==0.9.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt @@ -1003,8 +1016,10 @@ optimizely-sdk==4.1.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt -ora2==6.7.0 +ora2==6.11.0 # via -r requirements/edx/base.txt +orjson==3.10.3 + # via fastapi packaging==24.0 # via # -r requirements/edx/base.txt @@ -1118,12 +1133,13 @@ pydantic-core==2.18.2 # via # -r requirements/edx/base.txt # pydantic -pygments==2.17.2 +pygments==2.18.0 # via # -r requirements/edx/base.txt # -r requirements/edx/coverage.txt # diff-cover # py2neo + # rich pyjwkest==1.4.2 # via # -r requirements/edx/base.txt @@ -1180,9 +1196,8 @@ pynacl==1.5.0 # edx-django-utils pynliner==0.8.0 # via -r requirements/edx/base.txt -pyopenssl==22.0.0 +pyopenssl==24.1.0 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # optimizely-sdk # snowflake-connector-python @@ -1245,12 +1260,16 @@ python-dateutil==2.9.0.post0 # olxcleaner # ora2 # xblock +python-dotenv==1.0.1 + # via uvicorn python-ipware==3.0.0 # via # -r requirements/edx/base.txt # django-ipware python-memcached==1.62 # via -r requirements/edx/base.txt +python-multipart==0.0.9 + # via fastapi python-slugify==8.0.4 # via # -r requirements/edx/base.txt @@ -1282,7 +1301,6 @@ pytz==2024.1 # icalendar # interchange # olxcleaner - # openedx-blockstore # ora2 # snowflake-connector-python # xblock @@ -1295,6 +1313,7 @@ pyyaml==6.0.1 # drf-spectacular # edx-django-release-util # edx-i18n-tools + # uvicorn # xblock random2==1.0.2 # via -r requirements/edx/base.txt @@ -1304,12 +1323,12 @@ redis==5.0.4 # via # -r requirements/edx/base.txt # walrus -referencing==0.35.0 +referencing==0.35.1 # via # -r requirements/edx/base.txt # jsonschema # jsonschema-specifications -regex==2024.4.28 +regex==2024.5.10 # via # -r requirements/edx/base.txt # nltk @@ -1343,7 +1362,9 @@ requests-oauthlib==2.0.0 # via # -r requirements/edx/base.txt # social-auth-core -rpds-py==0.18.0 +rich==13.7.1 + # via typer +rpds-py==0.18.1 # via # -r requirements/edx/base.txt # jsonschema @@ -1370,9 +1391,8 @@ sailthru-client==2.2.3 # via # -r requirements/edx/base.txt # edx-ace -scipy==1.7.3 +scipy==1.10.1 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # chem # openedx-calc @@ -1382,6 +1402,8 @@ semantic-version==2.10.0 # edx-drf-extensions shapely==2.0.4 # via -r requirements/edx/base.txt +shellingham==1.5.4 + # via typer simplejson==3.19.2 # via # -r requirements/edx/base.txt @@ -1427,7 +1449,9 @@ slumber==0.7.1 # edx-enterprise # edx-rest-api-client sniffio==1.3.1 - # via anyio + # via + # anyio + # httpx snowflake-connector-python==3.10.0 # via # -r requirements/edx/base.txt @@ -1459,7 +1483,6 @@ sqlparse==0.5.0 # via # -r requirements/edx/base.txt # django - # openedx-blockstore staff-graded-xblock==2.3.0 # via -r requirements/edx/base.txt starlette==0.37.2 @@ -1480,7 +1503,7 @@ sympy==1.12 # via # -r requirements/edx/base.txt # openedx-calc -testfixtures==8.1.0 +testfixtures==8.2.0 # via # -r requirements/edx/base.txt # -r requirements/edx/testing.in @@ -1501,18 +1524,20 @@ tomli==2.0.1 # pyproject-api # pytest # tox -tomlkit==0.12.4 +tomlkit==0.12.5 # via # -r requirements/edx/base.txt # pylint # snowflake-connector-python tox==4.15.0 # via -r requirements/edx/testing.in -tqdm==4.66.2 +tqdm==4.66.4 # via # -r requirements/edx/base.txt # nltk # openai +typer==0.12.3 + # via fastapi-cli typing-extensions==4.11.0 # via # -r requirements/edx/base.txt @@ -1533,14 +1558,18 @@ typing-extensions==4.11.0 # pydantic-core # pylint # pylti1p3 + # rich # snowflake-connector-python # starlette + # typer # uvicorn tzdata==2024.1 # via # -r requirements/edx/base.txt # backports-zoneinfo # celery +ujson==5.10.0 + # via fastapi unicodecsv==0.14.1 # via # -r requirements/edx/base.txt @@ -1564,15 +1593,20 @@ urllib3==1.26.18 # snowflake-connector-python user-util==1.1.0 # via -r requirements/edx/base.txt -uvicorn==0.29.0 - # via pact-python +uvicorn[standard]==0.29.0 + # via + # fastapi + # fastapi-cli + # pact-python +uvloop==0.19.0 + # via uvicorn vine==5.1.0 # via # -r requirements/edx/base.txt # amqp # celery # kombu -virtualenv==20.26.1 +virtualenv==20.26.2 # via tox voluptuous==0.14.2 # via @@ -1584,6 +1618,8 @@ walrus==0.9.3 # edx-event-bus-redis watchdog==4.0.0 # via -r requirements/edx/base.txt +watchfiles==0.21.0 + # via uvicorn wcwidth==0.2.13 # via # -r requirements/edx/base.txt @@ -1606,6 +1642,8 @@ webob==1.8.7 # via # -r requirements/edx/base.txt # xblock +websockets==12.0 + # via uvicorn wrapt==1.16.0 # via # -r requirements/edx/base.txt diff --git a/scripts/ci-runner.Dockerfile b/scripts/ci-runner.Dockerfile index 291f1d33d32f..bbe7ef16dbab 100644 --- a/scripts/ci-runner.Dockerfile +++ b/scripts/ci-runner.Dockerfile @@ -1,4 +1,4 @@ -FROM summerwind/actions-runner:v2.288.1-ubuntu-20.04-c221b6e as base +FROM summerwind/actions-runner:v2.316.0-ubuntu-20.04-49490c4 as base USER root @@ -45,6 +45,8 @@ COPY setup.py setup.py COPY openedx/core/lib openedx/core/lib COPY lms lms COPY cms cms +COPY common common +COPY xmodule xmodule COPY requirements/pip.txt requirements/pip.txt COPY requirements/pip-tools.txt requirements/pip-tools.txt COPY requirements/edx/testing.txt requirements/edx/testing.txt diff --git a/scripts/compile_sass.py b/scripts/compile_sass.py index 41a2d56b3bda..ec1efee24d2b 100755 --- a/scripts/compile_sass.py +++ b/scripts/compile_sass.py @@ -67,14 +67,12 @@ "theme_dirs", metavar="PATH", multiple=True, - envvar="EDX_PLATFORM_THEME_DIRS", - type=click.Path( - exists=True, file_okay=False, readable=True, writable=True, path_type=Path - ), + envvar="COMPREHENSIVE_THEME_DIRS", + type=click.Path(path_type=Path), help=( "Consider sub-dirs of PATH as themes. " "Multiple theme dirs are accepted. " - "If none are provided, we look at colon-separated paths on the EDX_PLATFORM_THEME_DIRS env var." + "If none are provided, we look at colon-separated paths on the COMPREHENSIVE_THEME_DIRS env var." ), ) @click.option( diff --git a/scripts/user_retirement/README.rst b/scripts/user_retirement/README.rst index 20c99197ed6d..d5417f7f7e75 100644 --- a/scripts/user_retirement/README.rst +++ b/scripts/user_retirement/README.rst @@ -1,7 +1,7 @@ User Retirement Scripts ======================= -`This `_ directory contains python scripts which are migrated from the `tubular `_ respository. +`This `_ directory contains python scripts which are migrated from the `tubular `_ respository. These scripts are intended to drive the user retirement workflow which involves handling the deactivation or removal of user accounts as part of the platform's management process. These scripts could be called from any automation/CD framework. @@ -49,9 +49,9 @@ In-depth Documentation and Configuration Steps For in-depth documentation and essential configurations follow these docs -`Documentation `_ +`Documentation `_ -`Configuration Docs `_ +`Configuration Docs `_ Execute Script diff --git a/scripts/user_retirement/docs/driver_setup.rst b/scripts/user_retirement/docs/driver_setup.rst new file mode 100644 index 000000000000..c6d7b8e6f292 --- /dev/null +++ b/scripts/user_retirement/docs/driver_setup.rst @@ -0,0 +1,134 @@ +.. _driver-setup: + +############################################# +Setting Up the User Retirement Driver Scripts +############################################# + +`scripts/user_retirement `_ +is a directory of Python scripts designed to plug into various automation +tooling. It also contains readme file having details of how to run the scripts. +Included in this directory are two scripts intended to drive the user +retirement workflow. + +``get_learners_to_retire.py`` + Generates a list of users that are ready for immediate retirement. Users + are "ready" after a certain number of days spent in the ``PENDING`` state, + specified by the ``--cool_off_days`` argument. Produces an output intended + for consumption by Jenkins in order to spawn separate downstream builds for + each user. +``retire_one_learner.py`` + Retires the user specified by the ``--username`` argument. + +These two scripts share a required ``--config_file`` argument, which specifies +the driver configuration file for your environment (for example, production). +This configuration file is a YAML file that contains LMS auth secrets, API URLs, +and retirement pipeline stages specific to that environment. Here is an example +of a driver configuration file. + +.. code-block:: yaml + + client_id: + client_secret: + + base_urls: + lms: https://courses.example.com/ + ecommerce: https://ecommerce.example.com/ + credentials: https://credentials.example.com/ + + retirement_pipeline: + - ['RETIRING_EMAIL_LISTS', 'EMAIL_LISTS_COMPLETE', 'LMS', 'retirement_retire_mailings'] + - ['RETIRING_ENROLLMENTS', 'ENROLLMENTS_COMPLETE', 'LMS', 'retirement_unenroll'] + - ['RETIRING_LMS_MISC', 'LMS_MISC_COMPLETE', 'LMS', 'retirement_lms_retire_misc'] + - ['RETIRING_LMS', 'LMS_COMPLETE', 'LMS', 'retirement_lms_retire'] + +The ``client_id`` and ``client_secret`` keys contain the oauth credentials. +These credentials are simply copied from the output of the +``create_dot_application`` management command described in +:ref:`retirement-service-user`. + +The ``base_urls`` section in the configuration file defines the mappings of +IDA to base URLs used by the scripts to construct API URLs. Only the LMS is +mandatory here, but if any of your pipeline states contain API calls to other +services, those services must also be present in the ``base_urls`` section. + +The ``retirement_pipeline`` section defines the steps, state names, and order +of execution for each environment. Each item is a list in the form of: + +#. Start state name +#. End state name +#. IDA to call against (LMS, ECOMMERCE, or CREDENTIALS currently) +#. Method name to call in + `edx_api.py `_ + +For example: ``['RETIRING_CREDENTIALS', 'CREDENTIALS_COMPLETE', 'CREDENTIALS', +'retire_learner']`` will set the user's state to ``RETIRING_CREDENTIALS``, call +a pre-instantiated ``retire_learner`` method in the ``CredentialsApi``, then set +the user's state to ``CREDENTIALS_COMPLETE``. + +******** +Examples +******** + +The following are some examples of how to use the driver scripts. + +================== +Set Up Environment +================== + +Follow this `readme `_ to set up your execution environment. + +========================= +List of Targeted Learners +========================= + +Generate a list of learners that are ready for retirement (those learners who +have selected and confirmed account deletion and have been in the ``PENDING`` +state for the time specified ``cool_off_days``). + +.. code-block:: bash + + mkdir learners_to_retire + get_learners_to_retire.py \ + --config_file=path/to/config.yml \ + --output_dir=learners_to_retire \ + --cool_off_days=5 + +===================== +Run Retirement Script +===================== + +After running these commands, the ``learners_to_retire`` directory contains +several INI files, each containing a single line in the form of ``USERNAME +=``. Iterate over these files while executing the +``retire_one_learner.py`` script on each learner with a command like the following. + +.. code-block:: bash + + retire_one_learner.py \ + --config_file=path/to/config.yml \ + --username= + + +************************************************** +Using the Driver Scripts in an Automated Framework +************************************************** + +At edX, we call the user retirement scripts from +`Jenkins `_ jobs on one of our internal Jenkins +services. The user retirement driver scripts are intended to be agnostic +about which automation framework you use, but they were only fully tested +from Jenkins. + +For more information about how we execute these scripts at edX, see the +following wiki articles: + +* `User Retirement Jenkins Implementation `_ +* `How to: retirement Jenkins jobs development and testing `_ + +And check out the Groovy DSL files we use to seed these jobs: + +* `platform/jobs/RetirementJobs.groovy in edx/jenkins-job-dsl `_ +* `platform/jobs/RetirementJobEdxTriggers.groovy in edx/jenkins-job-dsl `_ + +.. include:: ../../../../links/links.rst + diff --git a/scripts/user_retirement/docs/implementation_overview.rst b/scripts/user_retirement/docs/implementation_overview.rst new file mode 100644 index 000000000000..37a814c1d583 --- /dev/null +++ b/scripts/user_retirement/docs/implementation_overview.rst @@ -0,0 +1,117 @@ +.. _Implmentation: + +####################### +Implementation Overview +####################### + +In the Open edX platform, the user experience is enabled by several +services, such as LMS, Studio, ecommerce, credentials, discovery, and more. +Personally Identifiable Identification (PII) about a user can exist in many of +these services. As a consequence, to remove a user's PII, you must be able +to request each service containing PII to remove, delete, or unlink the +data for that user in that service. + +In the user retirement feature, a centralized process (the *driver* scripts) +orchestrates all of these requests. For information about how to configure the +driver scripts, see :ref:`driver-setup`. + +**************************** +The User Retirement Workflow +**************************** + +The user retirement workflow is a configurable pipeline of building-block +APIs. These APIs are used to: + + * "Forget" a retired user's PII + * Prevent a retired user from logging back in + * Prevent re-use of the username or email address of a retired user + +Depending on which third parties a given Open edX instance integrates with, +the user retirement process may need to call out to external services or to +generate reports for later processing. Any such reports must subsequently be +destroyed. + +Configurability and adaptability were design goals from the beginning, so this +user retirement tooling should be able to accommodate a wide range of Open edX +sites and custom use cases. + +The workflow is designed to be linear and rerunnable, allowing recovery and +continuation in cases where a particular stage fails. Each user who has +requested retirement will be individually processed through this workflow, so +multiple users could be in the same state simultaneously. The LMS is the +authoritative source of information about the state of each user in the +retirement process, and the arbiter of state progressions, using the +``UserRetirementStatus`` model and associated APIs. The LMS also holds a +table of the states themselves (the ``RetirementState`` model), rather than +hard-coding the states. This was done because we cannot predict all the +possible states required by all members of the Open edX community. + +This example state diagram outlines the pathways users follow throughout the +workflow: + +.. digraph:: retirement_states_example + :align: center + + ranksep = "0.3"; + + node[fontname=Courier,fontsize=12,shape=box,group=main] + { rank = same INIT[style=invis] PENDING } + INIT -> PENDING; + "..."[shape=none] + PENDING -> RETIRING_ENROLLMENTS -> ENROLLMENTS_COMPLETE -> RETIRING_FORUMS -> FORUMS_COMPLETE -> "..." -> COMPLETE; + + node[group=""]; + RETIRING_ENROLLMENTS -> ERRORED; + RETIRING_FORUMS -> ERRORED; + PENDING -> ABORTED; + + subgraph cluster_terminal_states { + label = "Terminal States"; + labelloc = b // put label at bottom + {rank = same ERRORED COMPLETE ABORTED} + } + +Unless an error occurs internal to the user retirement tooling, a user's +retirement state should always land in one of the terminal states. At that +point, either their entry should be cleaned up from the +``UserRetirementStatus`` table or, if the state is ``ERRORED``, the +administrator needs to examine the error and resolve it. For more information, +see :ref:`recovering-from-errored`. + +******************* +The User Experience +******************* + +From the learner's perspective, the vast majority of this process is obscured. +The Account page contains a new section titled **Delete My Account**. In this +section, a learner may click the **Delete My Account** button and enter +their password to confirm their request. Subsequently, all of the learner's +browser sessions are logged off, and they become locked out of their account. + +An informational email is immediately sent to the learner to confirm the +deletion of their account. After this email is sent, the learner has a limited +amount of time (defined by the ``--cool_off_days`` argument described in +:ref:`driver-setup`) to contact the site administrators and rescind their +request. + +At this point, the learner's account has been deactivated, but *not* retired. +An entry in the ``UserRetirementStatus`` table is added, and their state set to +``PENDING``. + +By default, the **Delete My Account** section is visible and the button is +enabled, allowing account deletions to queue up. The +``ENABLE_ACCOUNT_DELETION`` feature in django settings toggles the visibility +of this section. See :ref:`django-settings`. + +================ +Third Party Auth +================ + +Learners who registered using social authentication must first unlink their +LMS account from their third-party account. For those learners, the **Delete +My Account** button will be disabled until they do so; meanwhile, they will be +instructed to follow the procedure in this help center article: `How do I link +or unlink my edX account to a social media +account? `_. + +.. include:: ../../../../links/links.rst diff --git a/scripts/user_retirement/docs/index.rst b/scripts/user_retirement/docs/index.rst new file mode 100644 index 000000000000..383c7a6aa8e1 --- /dev/null +++ b/scripts/user_retirement/docs/index.rst @@ -0,0 +1,38 @@ +.. _Enabling User Retirement: + +#################################### +Enabling the User Retirement Feature +#################################### + +There have been many changes to privacy laws (for example, GDPR or the +European Union General Data Protection Regulation) intended to change the way +that businesses think about and handle Personally Identifiable Information +(PII). + +As a step toward enabling Open edX to support some of the key updates in privacy +laws, edX has implemented APIs and tooling that enable Open edX instances to +retire registered users. When you implement this user retirement feature, your +Open edX instance can automatically erase PII for a given user from systems that +are internal to Open edX (for example, the LMS, forums, credentials, and other +independently deployable applications (IDAs)), as well as external systems, such +as third-party marketing services. + +This section is intended not only for instructing Open edX admins to perform +the basic setup, but also to offer some insight into the implementation of the +user retirement feature in order to help the Open edX community build +additional APIs and states that meet their special needs. Custom code, +plugins, packages, or XBlocks in your Open edX instance might store PII, but +this feature will not magically find and clean up that PII. You may need to +create your own custom code to include PII that is not covered by the user +retirement feature. + +.. toctree:: + :maxdepth: 1 + + implementation_overview + service_setup + driver_setup + special_cases + +.. include:: ../../../../links/links.rst + diff --git a/scripts/user_retirement/docs/service_setup.rst b/scripts/user_retirement/docs/service_setup.rst new file mode 100644 index 000000000000..4fd59fcfd3ad --- /dev/null +++ b/scripts/user_retirement/docs/service_setup.rst @@ -0,0 +1,179 @@ +.. _Service Setup: + +##################################### +Setting Up User Retirement in the LMS +##################################### + +This section describes how to set up and configure the user retirement feature +in the Open edX LMS. + +.. _django-settings: + +*************** +Django Settings +*************** + +The following Django settings control the behavior of the user retirement +feature. Note that some of these settings values are lambda functions rather +than standard string literals. This is intentional; it is a pattern for +defining *derived* settings specific to Open edX. Read more about it in +`openedx/core/lib/derived.py +`_. + +.. list-table:: + :header-rows: 1 + + * - Setting Name + - Default + - Description + * - RETIRED_USERNAME_PREFIX + - ``'retired__user_'`` + - The prefix part of hashed usernames. Used in ``RETIRED_USERNAME_FMT``. + * - RETIRED_EMAIL_PREFIX + - ``'retired__user_'`` + - The prefix part of hashed emails. Used in ``RETIRED_EMAIL_FMT``. + * - RETIRED_EMAIL_DOMAIN + - ``'retired.invalid'`` + - The domain part of hashed emails. Used in ``RETIRED_EMAIL_FMT``. + * - RETIRED_USERNAME_FMT + - ``lambda settings: + settings.RETIRED_USERNAME_PREFIX + '{}'`` + - The username field for a retired user gets transformed into this format, + where ``{}`` is replaced with the hash of their username. + * - RETIRED_EMAIL_FMT + - ``lambda settings: + settings.RETIRED_EMAIL_PREFIX + '{}@' + + settings.RETIRED_EMAIL_DOMAIN`` + - The email field for a retired user gets transformed into this format, where + ``{}`` is replaced with the hash of their email. + * - RETIRED_USER_SALTS + - None + - A list of salts used for hashing usernames and emails. Only the last item in this list is used as a salt for all new retirements, but historical salts are preserved in order to guarantee that all hashed usernames and emails can still be checked. The default value **MUST** be overridden! + * - RETIREMENT_SERVICE_WORKER_USERNAME + - ``'RETIREMENT_SERVICE_USER'`` + - The username of the retirement service worker. + * - RETIREMENT_STATES + - See `lms/envs/common.py `_ + in the ``RETIREMENT_STATES`` setting + - A list that defines the name and order of states for the retirement + workflow. See `Retirement States`_ for details. + * - FEATURES['ENABLE_ACCOUNT_DELETION'] + - True + - Whether to display the "Delete My Account" section the account settings page. + + +================= +Retirement States +================= + +The state of each user's retirement is stored in the LMS database, and the +state list itself is also separately stored in the database. We expect the +list of states will be variable over time and across different Open edX +installations, so it is the responsibility of the administrator to populate +the state list. + +The default states are defined in `lms/envs/common.py +`_ +in the ``RETIREMENT_STATES`` setting. There must be, at minimum, a ``PENDING`` +state at the beginning, and ``COMPLETED``, ``ERRORED``, and ``ABORTED`` states +at the end of the list. Also, for every ``RETIRING_foo`` state, there must be +a corresponding ``foo_COMPLETE`` state. + +Override these states if you need to add any states. Typically, these +settings are set in ``lms.yml``. + +After you have defined any custom states, populate the states table with the +following management command: + +.. code-block:: bash + + $ ./manage.py lms --settings= populate_retirement_states + + All states removed and new states added. Differences: + Added: set([u'RETIRING_ENROLLMENTS', u'RETIRING_LMS', u'LMS_MISC_COMPLETE', u'RETIRING_LMS_MISC', u'ENROLLMENTS_COMPLETE', u'LMS_COMPLETE']) + Removed: set([]) + Remaining: set([u'ERRORED', u'PENDING', u'ABORTED', u'COMPLETE']) + States updated successfully. Current states: + PENDING (step 1) + RETIRING_ENROLLMENTS (step 11) + ENROLLMENTS_COMPLETE (step 21) + RETIRING_LMS_MISC (step 31) + LMS_MISC_COMPLETE (step 41) + RETIRING_LMS (step 51) + LMS_COMPLETE (step 61) + ERRORED (step 71) + ABORTED (step 81) + COMPLETE (step 91) + +In this example, some states specified in settings were already present, so +they were listed under ``Remaining`` and were not re-added. The command output +also prints the ``Current states``; this represents all the states in the +states table. The ``populate_retirement_states`` command is idempotent, and +always attempts to make the states table reflect the ``RETIREMENT_STATES`` +list in settings. + +.. _retirement-service-user: + +*********************** +Retirement Service User +*********************** + +The user retirement driver scripts authenticate with the LMS and IDAs as the +retirement service user with oauth client credentials. Therefore, to use the +driver scripts, you must create a retirement service user, and generate a DOT +application and client credentials, as in the following command. + +.. code-block:: bash + + app_name=retirement + user_name=retirement_service_worker + ./manage.py lms --settings= manage_user $user_name $user_name@example.com --staff --superuser + ./manage.py lms --settings= create_dot_application $app_name $user_name + +.. note:: + The client credentials (client ID and client secret) will be printed to the + terminal, so take this opportunity to copy them for future reference. You + will use these credentials to configure the driver scripts. For more + information, see :ref:`driver-setup`. + +The retirement service user needs permission to perform retirement tasks, and +that is done by specifying the ``RETIREMENT_SERVICE_WORKER_USERNAME`` variable +in Django settings: + +.. code-block:: python + + RETIREMENT_SERVICE_WORKER_USERNAME = 'retirement_service_worker' + +************ +Django Admin +************ + +The Django admin interface contains the following models under ``USER_API`` +that relate to user retirement. + +.. list-table:: + :widths: 15 30 55 + :header-rows: 1 + + * - Name + - URI + - Description + * - Retirement States + - ``/admin/user_api/retirementstate/`` + - Represents the table of states defined in ``RETIREMENT_STATES`` and + populated with ``populate_retirement_states``. + * - User Retirement Requests + - ``/admin/user_api/userretirementrequest/`` + - Represents the table that tracks the user IDs of every learner who + has ever requested account deletion. This table is primarily used for + internal bookkeeping, and normally isn't useful for administrators. + * - User Retirement Statuses + - ``/admin/user_api/userretirementstatus/`` + - Model for managing the retirement state for each individual learner. + +In special cases where you may need to manually intervene with the pipeline, +you can use the User Retirement Statuses management page to change the +state for an individual user. For more information about how to handle these +cases, see :ref:`handling-special-cases`. + +.. include:: ../../../../links/links.rst diff --git a/scripts/user_retirement/docs/special_cases.rst b/scripts/user_retirement/docs/special_cases.rst new file mode 100644 index 000000000000..ae544c3208c6 --- /dev/null +++ b/scripts/user_retirement/docs/special_cases.rst @@ -0,0 +1,86 @@ +.. _handling-special-cases: + +###################### +Handling Special Cases +###################### + +.. _recovering-from-errored: + +Recovering from ERRORED +*********************** + +If a retirement API indicates failure (4xx or 5xx status code), the driver +immediately sets the user's state to ``ERRORED``. To debug this error state, +check the ``responses`` field in the user's row in +``user_api_userretirementstatus`` (User Retirement Status) for any relevant +logging. Once the issue is resolved, you need to manually set the user's +``current_state`` to the state immediately prior to the state which should be +re-tried. You can do this using the Django admin. In this example, a user +retirement errored during forums retirement, so we manually reset their state +from ``ERRORED`` to ``ENROLLMENTS_COMPLETE``. + +.. digraph:: retirement_states_example + :align: center + + //rankdir=LR; // Rank Direction Left to Right + ranksep = "0.3"; + + edge[color=grey] + + node[fontname=Courier,fontsize=12,shape=box,group=main] + { rank = same INIT[style=invis] PENDING } + { + edge[style=bold,color=black] + INIT -> PENDING; + "..."[shape=none] + PENDING -> RETIRING_ENROLLMENTS -> ENROLLMENTS_COMPLETE -> RETIRING_FORUMS; + } + RETIRING_FORUMS -> FORUMS_COMPLETE -> "..." -> COMPLETE + + node[group=""]; + RETIRING_ENROLLMENTS -> ERRORED; + RETIRING_FORUMS -> ERRORED[style=bold,color=black]; + PENDING -> ABORTED; + + subgraph cluster_terminal_states { + label = "Terminal States"; + labelloc = b // put label at bottom + {rank = same ERRORED COMPLETE ABORTED} + } + + ERRORED -> ENROLLMENTS_COMPLETE[style="bold,dashed",color=black,label=" via django\nadmin"] + +Now, the user retirement driver scripts will automatically resume this user's +retirement the next time they are executed. + +Rerunning some or all states +***************************** + +If you decide you want to rerun all retirements from the beginning, set +``current_state`` to ``PENDING`` for all retirements with ``current_state`` == +``COMPLETE``. This would be useful in the case where a new stage in the user +retirement workflow is added after running all retirements (but before the +retirement queue is cleaned up), and you want to run all the retirements +through the new stage. Or, perhaps you were developing a stage/API that +didn't work correctly but still indicated success, so the pipeline progressed +all users into ``COMPLETED``. Retirement APIs are designed to be idempotent, +so this should be a no-op for stages already run for a given user. + +Cancelling a retirement +*********************** + +Users who have recently requested account deletion but are still in the +``PENDING`` retirement state may request to rescind their account deletion by +emailing or otherwise contacting the administrators directly. edx-platform +offers a Django management command that administrators can invoke manually to +cancel a retirement, given the user's email address. It restores a given +user's login capabilities and removes them from all retirement queues. The +syntax is as follows: + +.. code-block:: bash + + $ ./manage.py lms --settings= cancel_user_retirement_request + +Keep in mind, this will only work for users which have not had their retirement +states advance beyond ``PENDING``. Additionally, the user will need to reset +their password in order to restore access to their account. diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index da181a5d2a5b..059745ad5ade 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -10,11 +10,13 @@ attrs==23.2.0 # via zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.in -backports-zoneinfo==0.2.1 - # via django -boto3==1.34.94 +backports-zoneinfo==0.2.1 ; python_version < "3.9" + # via + # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt + # django +boto3==1.34.104 # via -r scripts/user_retirement/requirements/base.in -botocore==1.34.94 +botocore==1.34.104 # via # boto3 # s3transfer @@ -35,11 +37,9 @@ click==8.1.6 # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt # -r scripts/user_retirement/requirements/base.in # edx-django-utils -cryptography==38.0.4 - # via - # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt - # pyjwt -django==4.2.11 +cryptography==42.0.7 + # via pyjwt +django==4.2.13 # via # -c scripts/user_retirement/requirements/../../../requirements/common_constraints.txt # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt @@ -50,13 +50,13 @@ django-crum==0.7.9 # via edx-django-utils django-waffle==4.1.0 # via edx-django-utils -edx-django-utils==5.12.0 +edx-django-utils==5.13.0 # via edx-rest-api-client edx-rest-api-client==5.7.0 # via -r scripts/user_retirement/requirements/base.in -google-api-core==2.18.0 +google-api-core==2.19.0 # via google-api-python-client -google-api-python-client==2.127.0 +google-api-python-client==2.129.0 # via -r scripts/user_retirement/requirements/base.in google-auth==2.29.0 # via diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index e850756c8a37..f28846579981 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -14,15 +14,15 @@ attrs==23.2.0 # zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.txt -backports-zoneinfo==0.2.1 +backports-zoneinfo==0.2.1 ; python_version < "3.9" # via # -r scripts/user_retirement/requirements/base.txt # django -boto3==1.34.94 +boto3==1.34.104 # via # -r scripts/user_retirement/requirements/base.txt # moto -botocore==1.34.94 +botocore==1.34.104 # via # -r scripts/user_retirement/requirements/base.txt # boto3 @@ -49,14 +49,14 @@ click==8.1.6 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -cryptography==38.0.4 +cryptography==42.0.7 # via # -r scripts/user_retirement/requirements/base.txt # moto # pyjwt ddt==1.7.2 # via -r scripts/user_retirement/requirements/testing.in -django==4.2.11 +django==4.2.13 # via # -r scripts/user_retirement/requirements/base.txt # django-crum @@ -70,7 +70,7 @@ django-waffle==4.1.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -edx-django-utils==5.12.0 +edx-django-utils==5.13.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-rest-api-client @@ -78,11 +78,11 @@ edx-rest-api-client==5.7.0 # via -r scripts/user_retirement/requirements/base.txt exceptiongroup==1.2.1 # via pytest -google-api-core==2.18.0 +google-api-core==2.19.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-python-client -google-api-python-client==2.127.0 +google-api-python-client==2.129.0 # via -r scripts/user_retirement/requirements/base.txt google-auth==2.29.0 # via @@ -115,7 +115,7 @@ isodate==0.6.1 # zeep jenkinsapi==0.3.13 # via -r scripts/user_retirement/requirements/base.txt -jinja2==3.1.3 +jinja2==3.1.4 # via moto jmespath==1.0.1 # via @@ -138,7 +138,7 @@ more-itertools==10.2.0 # simple-salesforce moto==4.2.14 # via -r scripts/user_retirement/requirements/testing.in -newrelic==9.9.0 +newrelic==9.9.1 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils @@ -287,7 +287,7 @@ urllib3==1.26.18 # botocore # requests # responses -werkzeug==3.0.2 +werkzeug==3.0.3 # via moto xmltodict==0.13.0 # via moto diff --git a/scripts/watch_sass.sh b/scripts/watch_sass.sh index 68d4b1f471a5..58bc3259767d 100755 --- a/scripts/watch_sass.sh +++ b/scripts/watch_sass.sh @@ -4,11 +4,11 @@ # Invoke from repo root as `npm run watch-sass`. # By default, only watches default Sass. -# To watch themes too, provide colon-separated paths in the EDX_PLATFORM_THEME_DIRS environment variable. +# To watch themes too, provide colon-separated paths in the COMPREHENSIVE_THEME_DIRS environment variable. # Each path will be treated as a "theme dir", which means that every immediate child directory is watchable as a theme. # For example: # -# EDX_PLATFORM_THEME_DIRS=/openedx/themes:./themes npm run watch-sass +# COMPREHENSIVE_THEME_DIRS=/openedx/themes:./themes npm run watch-sass # # would watch default Sass as well as /openedx/themes/indigo, /openedx/themes/mytheme, ./themes/red-theme, etc. @@ -93,7 +93,7 @@ start_sass_watch "default theme" \ # Watch each theme's Sass. # If it changes, only recompile that theme. export IFS=":" -for theme_dir in ${EDX_PLATFORM_THEME_DIRS:-} ; do +for theme_dir in ${COMPREHENSIVE_THEME_DIRS:-} ; do for theme_path in "$theme_dir"/* ; do theme_name="${theme_path#"$theme_dir/"}" lms_sass="$theme_path/lms/static/sass" diff --git a/tox.ini b/tox.ini index 9b1937f3640a..1b4252fd1905 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38, quality +envlist = py{38,311} quality # This is needed to prevent the lms, cms, and openedx packages inside the "Open # edX" package (defined in setup.py) from getting installed into site-packages diff --git a/xmodule/README.rst b/xmodule/README.rst index 5096c78c2abe..8bbae713b7a4 100644 --- a/xmodule/README.rst +++ b/xmodule/README.rst @@ -41,14 +41,14 @@ Direction Currently, this directory contains a lot of mission-critical functionality, so continued maintenance and simplification of it is important. Still, we aim to eventually dissolve the directory in favor of more focused & decoupled subsystems: -* ModuleStore is superseded by the `Blockstore`_ storage backend. -* Blockstore-backend content is rendered by a new, simplified `edx-platform XBlock runtime`_. +* ModuleStore is superseded by the `Learning Core`_ storage backend. +* Learning Core-backend content is rendered by a new, simplified `edx-platform XBlock runtime`_. * Navigation, partitioning, and composition of learning content is being re-architected in the `openedx-learning`_ library. * All new XBlocks are implemented in separate repositories, such as `xblock-drag-and-drop-v2`_. To help with this direction, please **do not add new functionality to this directory**. If you feel that you need to add code to this directory, reach out on `the forums`_; it's likely that someone can help you find a different way to implement your change that will be more robust and architecturally sound! -.. _Blockstore: https://github.com/openedx/blockstore/ +.. _Learning Core: https://github.com/openedx/openedx-learning/ .. _edx-platform XBlock runtime: https://github.com/openedx/edx-platform/tree/master/openedx/core/djangoapps/xblock .. _openedx-learning: https://github.com/openedx/openedx-learning .. _xblock-drag-and-drop-v2: https://github.com/openedx/xblock-drag-and-drop-v2 diff --git a/xmodule/html_block.py b/xmodule/html_block.py index 6c883e1322d2..2db198360107 100644 --- a/xmodule/html_block.py +++ b/xmodule/html_block.py @@ -279,7 +279,7 @@ def load_definition(cls, xml_object, system, location, id_generator): # lint-am @classmethod def parse_xml_new_runtime(cls, node, runtime, keys): """ - Parse XML in the new blockstore-based runtime. Since it doesn't yet + Parse XML in the new learning-core-based runtime. Since it doesn't yet support loading separate .html files, the HTML data is assumed to be in a CDATA child or otherwise just inline in the OLX. """ diff --git a/xmodule/library_tools.py b/xmodule/library_tools.py index 7aa0362ea47c..2c077a888482 100644 --- a/xmodule/library_tools.py +++ b/xmodule/library_tools.py @@ -25,7 +25,7 @@ class LibraryToolsService: """ Service for LibraryContentBlock. - Allows to interact with libraries in the modulestore and blockstore. + Allows to interact with libraries in the modulestore and learning core. Should only be used in the CMS. """ diff --git a/xmodule/partitions/partitions.py b/xmodule/partitions/partitions.py index 107a797054a6..9a9d637d318c 100644 --- a/xmodule/partitions/partitions.py +++ b/xmodule/partitions/partitions.py @@ -10,14 +10,21 @@ # pylint: disable=redefined-builtin -# UserPartition IDs must be unique. The Cohort and Random UserPartitions (when they are -# created via Studio) choose an unused ID in the range of 100 (historical) to MAX_INT. Therefore the -# dynamic UserPartitionIDs must be under 100, and they have to be hard-coded to ensure -# they are always the same whenever the dynamic partition is added (since the UserPartition -# ID is stored in the xblock group_access dict). +# Each user partition has an ID that is unique within its learning context. +# The IDs must be valid MySQL primary keys, ie positive integers 1 -> 2^31-1. +# We must carefully manage these IDs, because once they are saved to OLX and the db, they cannot change. +# Here is how we delegate the ID range: +# * 1 -> 49: Unused/Reserved +# * 50: The enrollment track partition +# * 51: The content type gating partition (defined elsewhere) +# * 52-99: Available for other single user partitions, plugged in via setup.py. +# Operators, beware of conflicting IDs between plugins! +# * 100 -> 2^31-1: General namespace for generating IDs at runtime. +# This includes, at least: content partitions, the cohort partition, and teamset partitions. +# When using this range, user partition implementations must check to see that they +# are not conflicting with an existing partition for the course. ENROLLMENT_TRACK_PARTITION_ID = 50 - -MINIMUM_STATIC_PARTITION_ID = 100 +MINIMUM_UNUSED_PARTITION_ID = 100 class UserPartitionError(Exception): diff --git a/xmodule/raw_block.py b/xmodule/raw_block.py index 58b21795f282..f24dd7712d85 100644 --- a/xmodule/raw_block.py +++ b/xmodule/raw_block.py @@ -66,7 +66,7 @@ def parse_xml_new_runtime(cls, node, runtime, keys): Interpret the parsed XML in `node`, creating a new instance of this module. """ - # In the new/blockstore-based runtime, XModule parsing (from + # In the new/learning-core-based runtime, XModule parsing (from # XmlMixin) is disabled, so definition_from_xml will not be # called, and instead the "normal" XBlock parse_xml will be used. # However, it's not compatible with RawMixin, so we implement diff --git a/xmodule/seq_block.py b/xmodule/seq_block.py index 944c73400ab5..6d1a8c59adeb 100644 --- a/xmodule/seq_block.py +++ b/xmodule/seq_block.py @@ -12,6 +12,7 @@ from functools import reduce from django.conf import settings +from edx_django_utils.monitoring import set_custom_attribute from lxml import etree from opaque_keys.edx.keys import UsageKey from pytz import UTC @@ -43,11 +44,6 @@ log = logging.getLogger(__name__) -try: - import newrelic.agent -except ImportError: - newrelic = None # pylint: disable=invalid-name - # HACK: This shouldn't be hard-coded to two types # OBSOLETE: This obsoletes 'type' class_priority = ['video', 'problem'] @@ -839,58 +835,52 @@ def _locations_in_subtree(self, node): def _capture_basic_metrics(self): """ - Capture basic information about this sequence in New Relic. + Capture basic information about this sequence in telemetry. """ - if not newrelic: - return - newrelic.agent.add_custom_attribute('seq.block_id', str(self.location)) - newrelic.agent.add_custom_attribute('seq.display_name', self.display_name or '') - newrelic.agent.add_custom_attribute('seq.position', self.position) - newrelic.agent.add_custom_attribute('seq.is_time_limited', self.is_time_limited) + set_custom_attribute('seq.block_id', str(self.location)) + set_custom_attribute('seq.display_name', self.display_name or '') + set_custom_attribute('seq.position', self.position) + set_custom_attribute('seq.is_time_limited', self.is_time_limited) def _capture_full_seq_item_metrics(self, children): """ Capture information about the number and types of XBlock content in - the sequence as a whole. We send this information to New Relic so that + the sequence as a whole. We record this information in telemetry so that we can do better performance analysis of courseware. """ - if not newrelic: - return # Basic count of the number of Units (a.k.a. VerticalBlocks) we have in # this learning sequence - newrelic.agent.add_custom_attribute('seq.num_units', len(children)) + set_custom_attribute('seq.num_units', len(children)) # Count of all modules (leaf nodes) in this sequence (e.g. videos, # problems, etc.) The units (verticals) themselves are not counted. all_item_keys = self._locations_in_subtree(self) - newrelic.agent.add_custom_attribute('seq.num_items', len(all_item_keys)) + set_custom_attribute('seq.num_items', len(all_item_keys)) # Count of all modules by block_type (e.g. "video": 2, "discussion": 4) block_counts = collections.Counter(usage_key.block_type for usage_key in all_item_keys) for block_type, count in block_counts.items(): - newrelic.agent.add_custom_attribute(f'seq.block_counts.{block_type}', count) + set_custom_attribute(f'seq.block_counts.{block_type}', count) def _capture_current_unit_metrics(self, children): """ Capture information about the current selected Unit within the Sequence. """ - if not newrelic: - return # Positions are stored with indexing starting at 1. If we get into a # weird state where the saved position is out of bounds (e.g. the # content was changed), avoid going into any details about this unit. if 1 <= self.position <= len(children): # Basic info about the Unit... current = children[self.position - 1] - newrelic.agent.add_custom_attribute('seq.current.block_id', str(current.location)) - newrelic.agent.add_custom_attribute('seq.current.display_name', current.display_name or '') + set_custom_attribute('seq.current.block_id', str(current.location)) + set_custom_attribute('seq.current.display_name', current.display_name or '') # Examining all blocks inside the Unit (or split_test, conditional, etc.) child_locs = self._locations_in_subtree(current) - newrelic.agent.add_custom_attribute('seq.current.num_items', len(child_locs)) + set_custom_attribute('seq.current.num_items', len(child_locs)) curr_block_counts = collections.Counter(usage_key.block_type for usage_key in child_locs) for block_type, count in curr_block_counts.items(): - newrelic.agent.add_custom_attribute(f'seq.current.block_counts.{block_type}', count) + set_custom_attribute(f'seq.current.block_counts.{block_type}', count) def _time_limited_student_view(self): """ diff --git a/xmodule/tests/test_library_tools.py b/xmodule/tests/test_library_tools.py index c6185f8754b1..f93066cd5c63 100644 --- a/xmodule/tests/test_library_tools.py +++ b/xmodule/tests/test_library_tools.py @@ -28,7 +28,7 @@ class ContentLibraryToolsTest(MixedSplitTestCase, ContentLibrariesRestApiTest): """ Tests for LibraryToolsService. - Tests interaction with blockstore-based (V2) and mongo-based (V1) content libraries. + Tests interaction with learning-core-based (V2) and mongo-based (V1) content libraries. """ def setUp(self): super().setUp() diff --git a/xmodule/tests/test_split_test_block.py b/xmodule/tests/test_split_test_block.py index 1324d3a806ea..c51c157a7760 100644 --- a/xmodule/tests/test_split_test_block.py +++ b/xmodule/tests/test_split_test_block.py @@ -10,7 +10,7 @@ from xmodule.modulestore.tests.factories import CourseFactory from xmodule.modulestore.tests.utils import MixedSplitTestCase -from xmodule.partitions.partitions import MINIMUM_STATIC_PARTITION_ID, Group, UserPartition +from xmodule.partitions.partitions import MINIMUM_UNUSED_PARTITION_ID, Group, UserPartition from xmodule.partitions.tests.test_partitions import MockPartitionService, MockUserPartitionScheme, PartitionTestCase from xmodule.split_test_block import ( SplitTestBlock, @@ -94,10 +94,10 @@ def setUp(self): self.course.user_partitions = [ self.user_partition, UserPartition( - MINIMUM_STATIC_PARTITION_ID, 'second_partition', 'Second Partition', + MINIMUM_UNUSED_PARTITION_ID, 'second_partition', 'Second Partition', [ - Group(str(MINIMUM_STATIC_PARTITION_ID + 1), 'abel'), - Group(str(MINIMUM_STATIC_PARTITION_ID + 2), 'baker'), Group("103", 'charlie') + Group(str(MINIMUM_UNUSED_PARTITION_ID + 1), 'abel'), + Group(str(MINIMUM_UNUSED_PARTITION_ID + 2), 'baker'), Group("103", 'charlie') ], MockUserPartitionScheme() ) diff --git a/xmodule/video_block/transcripts_utils.py b/xmodule/video_block/transcripts_utils.py index 3dd88c2f313e..d82a5d3f4789 100644 --- a/xmodule/video_block/transcripts_utils.py +++ b/xmodule/video_block/transcripts_utils.py @@ -15,7 +15,7 @@ import simplejson as json from django.conf import settings from lxml import etree -from opaque_keys.edx.locator import BundleDefinitionLocator +from opaque_keys.edx.keys import UsageKeyV2 from pysrt import SubRipFile, SubRipItem, SubRipTime from pysrt.srtexc import Error @@ -856,15 +856,19 @@ def available_translations(self, transcripts, verify_assets=None, is_bumper=Fals # to clean redundant language codes. return list(set(translations)) - def get_default_transcript_language(self, transcripts): + def get_default_transcript_language(self, transcripts, dest_lang=None): """ Returns the default transcript language for this video block. Args: transcripts (dict): A dict with all transcripts and a sub. + dest_lang (unicode): language coming from unit translation language selector. """ sub, other_lang = transcripts["sub"], transcripts["transcripts"] - if self.transcript_language in other_lang: + + if dest_lang and dest_lang in other_lang.keys(): + transcript_language = dest_lang + elif self.transcript_language in other_lang: transcript_language = self.transcript_language elif sub: transcript_language = 'en' @@ -941,7 +945,7 @@ def get_transcript_for_video(video_location, subs_id, file_name, language): """ Get video transcript from content store. This is a lower level function and is used by `get_transcript_from_contentstore`. Prefer that function instead where possible. If you - need to support getting transcripts from VAL or Blockstore as well, use the `get_transcript` + need to support getting transcripts from VAL or Learning Core as well, use the `get_transcript` function instead. NOTE: Transcripts can be searched from content store by two ways: @@ -1029,29 +1033,31 @@ def get_transcript_from_contentstore(video, language, output_format, transcripts return transcript_content, transcript_name, Transcript.mime_types[output_format] -def get_transcript_from_blockstore(video_block, language, output_format, transcripts_info): +def get_transcript_from_learning_core(video_block, language, output_format, transcripts_info): """ - Get video transcript from Blockstore. - - Blockstore expects video transcripts to be placed into the 'static/' - subfolder of the XBlock's folder in a Blockstore bundle. For example, if the - video XBlock's definition is in the standard location of - video/video1/definition.xml - Then the .srt files should be placed at e.g. - video/video1/static/video1-en.srt - This is the same place where other public static files are placed for other - XBlocks, such as image files used by HTML blocks. - - Video XBlocks in Blockstore must set the 'transcripts' XBlock field to a - JSON dictionary listing the filename of the transcript for each language: -