Merge branch 'master' into iamsobanjaved/django-42

cms/djangoapps/contentstore/management/commands/reindex_course.py

@@ -4,6 +4,7 @@
 import logging
 from textwrap import dedent
 from time import time
+from datetime import date
 
 from django.core.management import BaseCommand, CommandError
 from elasticsearch import exceptions
@@ -38,6 +39,9 @@ def add_arguments(self, parser):
         parser.add_argument('--all',
                             action='store_true',
                             help='Reindex all courses')
+        parser.add_argument('--active',
+                            action='store_true',
+                            help='Reindex active courses only')
         parser.add_argument('--setup',
                             action='store_true',
                             help='Reindex all courses on developers stack setup')
@@ -65,12 +69,17 @@ def handle(self, *args, **options):
         """
         course_ids = options['course_ids']
         all_option = options['all']
+        active_option = options['active']
         setup_option = options['setup']
         readable_option = options['warning']
         index_all_courses_option = all_option or setup_option
 
-        if (not len(course_ids) and not index_all_courses_option) or (len(course_ids) and index_all_courses_option):  # lint-amnesty, pylint: disable=len-as-condition
-            raise CommandError("reindex_course requires one or more <course_id>s OR the --all or --setup flags.")
+        if ((not course_ids and not (index_all_courses_option or active_option)) or
+                (course_ids and (index_all_courses_option or active_option))):
+            raise CommandError((
+                "reindex_course requires one or more <course_id>s"
+                " OR the --all, --active or --setup flags."
+            ))
 
         store = modulestore()
 
@@ -104,12 +113,28 @@ def handle(self, *args, **options):
                 course_keys = [course.id for course in modulestore().get_courses()]
             else:
                 return
+        elif active_option:
+            # in case of --active, we get the list of course keys from all courses
+            # that are stored in the modulestore and filter out the non-active
+            course_keys = []
+
+            today = date.today()
+            all_courses = modulestore().get_courses()
+            for course in all_courses:
+                # Omitting courses without a start date as well as
+                # couses that already ended (end date is in the past)
+                if not course.start or (course.end and course.end.date() < today):
+                    continue
+                course_keys.append(course.id)
+
+            logging.warning(f'Selected {len(course_keys)} active courses over a total of {len(all_courses)}.')
+
         else:
             # in case course keys are provided as arguments
             course_keys = list(map(self._parse_course_key, course_ids))
 
         total = len(course_keys)
-        logging.warning(f'Reindexing {total} courses')
+        logging.warning(f'Reindexing {total} courses...')
         reindexed = 0
         start = time()
 
@@ -120,6 +145,6 @@ def handle(self, *args, **options):
                 if reindexed % 10 == 0 or reindexed == total:
                     now = time()
                     t = now - start
-                    logging.warning(f'{reindexed}/{total} reindexed in {t:.1f} seconds')
+                    logging.warning(f'{reindexed}/{total} reindexed in {t:.1f} seconds.')
             except Exception as exc:  # lint-amnesty, pylint: disable=broad-except
-                logging.exception('Error indexing course %s due to the error: %s', course_key, exc)
+                logging.exception('Error indexing course %s due to the error: %s.', course_key, exc)

cms/djangoapps/contentstore/management/commands/tests/test_reindex_courses.py

@@ -10,6 +10,7 @@
 from xmodule.modulestore.django import modulestore  # lint-amnesty, pylint: disable=wrong-import-order
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase  # lint-amnesty, pylint: disable=wrong-import-order
 from xmodule.modulestore.tests.factories import CourseFactory, LibraryFactory  # lint-amnesty, pylint: disable=wrong-import-order
+from datetime import datetime, timedelta
 
 
 @ddt.ddt
@@ -26,11 +27,18 @@ def setUp(self):
             org="test", library="lib2", display_name="run2", default_store=ModuleStoreEnum.Type.split
         )
 
+        yesterday = datetime.min.today() - timedelta(days=1)
+
         self.first_course = CourseFactory.create(
-            org="test", course="course1", display_name="run1"
+            org="test", course="course1", display_name="run1", start=yesterday, end=None
         )
+
         self.second_course = CourseFactory.create(
-            org="test", course="course2", display_name="run1"
+            org="test", course="course2", display_name="run1", start=yesterday, end=yesterday
+        )
+
+        self.third_course = CourseFactory.create(
+            org="test", course="course3", display_name="run1", start=None, end=None
         )
 
     REINDEX_PATH_LOCATION = (
@@ -103,7 +111,7 @@ def test_given_all_key_prompts_and_reindexes_all_courses(self):
                 call_command('reindex_course', all=True)
 
                 patched_yes_no.assert_called_once_with(ReindexCommand.CONFIRMATION_PROMPT, default='no')
-                expected_calls = self._build_calls(self.first_course, self.second_course)
+                expected_calls = self._build_calls(self.first_course, self.second_course, self.third_course)
                 self.assertCountEqual(patched_index.mock_calls, expected_calls)
 
     def test_given_all_key_prompts_and_reindexes_all_courses_cancelled(self):
@@ -116,3 +124,15 @@ def test_given_all_key_prompts_and_reindexes_all_courses_cancelled(self):
 
                 patched_yes_no.assert_called_once_with(ReindexCommand.CONFIRMATION_PROMPT, default='no')
                 patched_index.assert_not_called()
+
+    def test_given_active_key_prompt(self):
+        """
+            Test that reindexes all active courses when --active key is given
+            Active courses have a start date but no end date, or the end date is in the future.
+        """
+        with mock.patch(self.REINDEX_PATH_LOCATION) as patched_index, \
+                mock.patch(self.MODULESTORE_PATCH_LOCATION, mock.Mock(return_value=self.store)):
+            call_command('reindex_course', active=True)
+
+            expected_calls = self._build_calls(self.first_course)
+            self.assertCountEqual(patched_index.mock_calls, expected_calls)

cms/djangoapps/contentstore/signals/handlers.py

@@ -127,6 +127,17 @@ def listen_for_course_publish(sender, course_key, **kwargs):  # pylint: disable=
         dump_course_to_neo4j
     )
 
+    # DEVELOPER README: probably all tasks here should use transaction.on_commit
+    # to avoid stale data, but the tasks are owned by many teams and are often
+    # working well enough. Several instead use a waiting strategy.
+    # If you are in here trying to figure out why your task is not working correctly,
+    # consider whether it is getting stale data and if so choose to wait for the transaction
+    # like exams or put your task to sleep for a while like discussions.
+    # You will not be able to replicate these errors in an environment where celery runs
+    # in process because it will be inside the transaction. Use the settings from
+    # devstack_with_worker.py, and consider adding a time.sleep into send_bulk_published_signal
+    # if you really want to make sure that the task happens before the data is ready.
+
     # register special exams asynchronously after the data is ready
     course_key_str = str(course_key)
     transaction.on_commit(lambda: update_special_exams_and_publish.delay(course_key_str))
@@ -139,10 +150,9 @@ def listen_for_course_publish(sender, course_key, **kwargs):  # pylint: disable=
         # Push the course out to CourseGraph asynchronously.
         dump_course_to_neo4j.delay(course_key_str)
 
-    # Finally, call into the course search subsystem
-    # to kick off an indexing action
+    # Kick off a courseware indexing action after the data is ready
     if CoursewareSearchIndexer.indexing_is_enabled() and CourseAboutSearchIndexer.indexing_is_enabled():
-        update_search_index.delay(course_key_str, datetime.now(UTC).isoformat())
+        transaction.on_commit(lambda: update_search_index.delay(course_key_str, datetime.now(UTC).isoformat()))
 
     update_discussions_settings_from_course_task.apply_async(
         args=[course_key_str],

cms/djangoapps/contentstore/tests/test_courseware_index.py

@@ -5,7 +5,7 @@
 import time
 from datetime import datetime
 from unittest import skip
-from unittest.mock import patch
+from unittest.mock import patch, Mock
 
 import ddt
 import pytest
@@ -585,6 +585,8 @@ def test_large_course_deletion(self):
         self._test_large_course_deletion(self.store)
 
 
+@patch('cms.djangoapps.contentstore.signals.handlers.transaction.on_commit',
+       new=Mock(side_effect=lambda func: func()),)  # run right away
 class TestTaskExecution(SharedModuleStoreTestCase):
     """
     Set of tests to ensure that the task code will do the right thing when

cms/djangoapps/contentstore/views/tests/test_course_index.py

@@ -6,7 +6,6 @@
 import datetime
 import json
 from unittest import mock, skip
-from unittest.mock import patch
 
 import ddt
 import lxml
@@ -644,7 +643,7 @@ def test_verify_warn_only_on_enabled_blocks(self, enabled_block_types, deprecate
         )
 
     @override_settings(FEATURES={'ENABLE_EXAM_SETTINGS_HTML_VIEW': True})
-    @patch('cms.djangoapps.models.settings.course_metadata.CourseMetadata.validate_proctoring_settings')
+    @mock.patch('cms.djangoapps.models.settings.course_metadata.CourseMetadata.validate_proctoring_settings')
     def test_proctoring_link_is_visible(self, mock_validate_proctoring_settings):
         """
         Test to check proctored exam settings mfe url is rendering properly
@@ -685,9 +684,11 @@ class TestCourseReIndex(CourseTestCase):
 
     ENABLED_SIGNALS = ['course_published']
 
+    @mock.patch('cms.djangoapps.contentstore.signals.handlers.transaction.on_commit',
+                new=mock.Mock(side_effect=lambda func: func()), )  # run index right away
     def setUp(self):
         """
-        Set up the for the course outline tests.
+        Set up the for the course reindex tests.
         """
 
         super().setUp()

cms/templates/studio_xblock_wrapper.html

@@ -137,7 +137,7 @@
                                         % endif
                                         % if use_tagging:
                                             <li class="nav-item">
-                                                <a class="manage-tags-button" href="#" role="button">${_("Manage tags")}</a>
+                                                <a class="manage-tags-button" href="#" role="button">${_("Manage Tags")}</a>
                                             </li>
                                         % endif
                                         % if is_course:

common/djangoapps/student/tests/test_email.py

@@ -406,10 +406,9 @@ def test_email_success(self):
             subject='Request to change édX account e-mail',
             body_fragments=[
                 'We received a request to change the e-mail associated with',
-                'your édX account from {old_email} to {new_email}.'.format(
-                    old_email=old_email,
-                    new_email=new_email,
-                ),
+                'your édX account from',
+                old_email,
+                new_email,
                 'If this is correct, please confirm your new e-mail address by visiting:',
                 f'http://edx.org/email_confirm/{registration_key}',
                 'Please do not reply to this e-mail; if you require assistance,',
@@ -467,13 +466,11 @@ def setUp(self):
 
         # Text fragments we expect in the body of the confirmation email
         self.email_fragments = [
-            "This is to confirm that you changed the e-mail associated with {platform_name}"
-            " from {old_email} to {new_email}. If you did not make this request, please contact us immediately."
-            " Contact information is listed at:".format(
-                platform_name=settings.PLATFORM_NAME,
-                old_email=self.user.email,
-                new_email=PendingEmailChange.objects.get(activation_key=self.key).new_email
-            ),
+            "This is to confirm that you changed the e-mail associated with ",
+            str(settings.PLATFORM_NAME),
+            "If you did not make this request, please contact us immediately. Contact information is listed at:",
+            self.user.email,
+            PendingEmailChange.objects.get(activation_key=self.key).new_email,
             "We keep a log of old e-mails, so if this request was unintentional, we can investigate."
         ]
 

common/templates/student/edx_ace/accountactivation/email/body.html

@@ -50,7 +50,7 @@ <h1>
                 {% blocktrans trimmed asvar assist_msg %}
                 If you need help, please use our web form at {start_anchor_web}{{ support_url }}{end_anchor} or email {start_anchor_email}{{ support_email }}{end_anchor}.
                 {% endblocktrans %}
-                {% interpolate_html assist_msg start_anchor_web='<a href="'|add:support_url|add:'">'|safe start_anchor_email='<a href="mailto:'|add:support_email|add:'">'|safe end_anchor='</a>'|safe %}
+                {% interpolate_html assist_msg start_anchor_web='<a style="color: '|add:brand_color|add:';" href="'|add:support_url|add:'">'|safe start_anchor_email='<a style="color: '|add:brand_color|add:';" href="mailto:'|add:support_email|add:'">'|safe end_anchor='</a>'|safe %}
                 <br />
             </p>
         </td>

common/templates/student/edx_ace/emailchange/email/body.html

@@ -3,6 +3,7 @@
 {% load i18n %}
 {% load static %}
 {% block content %}
+{% with old_email='<a href="mailto:'|add:old_email|add:'" style="color: '|add:brand_color|add:';">'|add:old_email|add:'</a>'|safe new_email='<a href="mailto:'|add:new_email|add:'" style="color: '|add:brand_color|add:';">'|add:new_email|add:'</a>'|safe %}
 <table width="100%" align="left" border="0" cellpadding="0" cellspacing="0" role="presentation">
     <tr>
         <td>
@@ -31,4 +32,5 @@ <h1>
         </td>
     </tr>
 </table>
+{% endwith %}
 {% endblock %}

common/templates/student/edx_ace/emailchangeconfirmation/email/body.html

@@ -3,6 +3,7 @@
 {% load i18n %}
 {% load static %}
 {% block content %}
+{% with old_email='<a href="mailto:'|add:old_email|add:'" style="color: '|add:brand_color|add:';">'|add:old_email|add:'</a>'|safe new_email='<a href="mailto:'|add:new_email|add:'" style="color: '|add:brand_color|add:';">'|add:new_email|add:'</a>'|safe %}
 <table width="100%" align="left" border="0" cellpadding="0" cellspacing="0" role="presentation">
     <tr>
         <td>
@@ -30,4 +31,5 @@ <h1>
         </td>
     </tr>
 </table>
+{% endwith %}
 {% endblock %}

conf/locale/ar/LC_MESSAGES/django.po

@@ -70,7 +70,7 @@
 # طاهر <[email protected]>, 2014
 # #-#-#-#-#  django-studio.po (edx-platform)  #-#-#-#-#
 # edX community translations have been downloaded from Arabic (http://app.transifex.com/open-edx/edx-platform/language/ar/).
-# Copyright (C) 2023 EdX
+# Copyright (C) 2024 EdX
 # This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.
 #
 # Translators:
@@ -110,7 +110,7 @@
 # wd3bbas <[email protected]>, 2014
 # #-#-#-#-#  mako.po (edx-platform)  #-#-#-#-#
 # edX community translations have been downloaded from Arabic (http://app.transifex.com/open-edx/edx-platform/language/ar/)
-# Copyright (C) 2023 edX
+# Copyright (C) 2024 edX
 # This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.
 #
 # Translators:
@@ -224,7 +224,7 @@
 # طاهر <[email protected]>, 2014
 # #-#-#-#-#  wiki.po (edx-platform)  #-#-#-#-#
 # edX community translations have been downloaded from Arabic (http://app.transifex.com/open-edx/edx-platform/language/ar/)
-# Copyright (C) 2023 edX
+# Copyright (C) 2024 edX
 # This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.
 #
 # Translators:
@@ -248,9 +248,9 @@
 # SalmaGhazal <[email protected]>, 2014
 # #-#-#-#-#  edx_proctoring_proctortrack.po (0.1a)  #-#-#-#-#
 # edX community translations have been downloaded from Arabic (https://app.transifex.com/open-edx/teams/6205/ar/)
-# Copyright (C) 2023 edX
+# Copyright (C) 2024 edX
 # This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.
-# EdX Team <[email protected]>, 2023.
+# EdX Team <[email protected]>, 2024.
 #
 # Translators:
 # NELC Open edX Translation <[email protected]>, 2020
@@ -259,7 +259,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: 0.1a\n"
 "Report-Msgid-Bugs-To: [email protected]\n"
-"POT-Creation-Date: 2023-12-24 20:43+0000\n"
+"POT-Creation-Date: 2024-01-14 20:43+0000\n"
 "PO-Revision-Date: 2019-01-20 20:43+0000\n"
 "Last-Translator: NELC Open edX Translation <[email protected]>, 2020\n"
 "Language-Team: Arabic (https://app.transifex.com/open-edx/teams/6205/ar/)\n"
@@ -8349,6 +8349,13 @@ msgstr "المهنة"
 msgid "Specialty"
 msgstr "التخصص"
 
+#. Translators: This label appears above a dropdown menu to select
+#. the user's work experience
+#: openedx/core/djangoapps/user_api/accounts/settings_views.py
+#: openedx/core/djangoapps/user_authn/api/form_fields.py
+msgid "Work experience"
+msgstr ""
+
 #: openedx/core/djangoapps/user_api/accounts/utils.py
 #, python-brace-format
 msgid ""
@@ -13053,6 +13060,10 @@ msgstr "نسخة مطابقة لـ ’{0}‘"
 msgid "Invalid prerequisite course key"
 msgstr "رقم المساق الأساسي غير صالح"
 
+#: cms/djangoapps/contentstore/utils.py
+msgid "Set Date"
+msgstr "تعيين التاريخ"
+
 #. #-#-#-#-#  django-studio.po (edx-platform)  #-#-#-#-#
 #. Translators: This is the status of an active video upload
 #: cms/djangoapps/contentstore/video_storage_handlers.py
@@ -13268,10 +13279,6 @@ msgstr ""
 msgid "Course has been successfully reindexed."
 msgstr "جرى إعادة فهرسة المساق بنجاح."
 
-#: cms/djangoapps/contentstore/views/course.py
-msgid "Set Date"
-msgstr "تعيين التاريخ"
-
 #: cms/djangoapps/contentstore/views/course.py
 msgid ""
 "Special characters not allowed in organization, course number, and course "
@@ -23977,6 +23984,10 @@ msgstr "تكبير أو تصغير"
 msgid "Select this problem"
 msgstr ""
 
+#: cms/templates/studio_xblock_wrapper.html
+msgid "Manage Tags"
+msgstr ""
+
 #: cms/templates/studio_xblock_wrapper.html
 msgid "Manage Access"
 msgstr ""
@@ -24343,6 +24354,10 @@ msgstr "تصفّح المساق"
 msgid "Outline"
 msgstr "المخطّط الكلّي"
 
+#: cms/templates/widgets/header.html
+msgid "Videos"
+msgstr ""
+
 #: cms/templates/widgets/header.html
 msgid "Publisher"
 msgstr "ناشر"