Merge branch 'master' into iamsobanjaved/django-42

.gitignore

@@ -61,6 +61,8 @@ conf/locale/fake*/LC_MESSAGES/*.po
 conf/locale/fake*/LC_MESSAGES/*.mo
 # this was a mistake in i18n_tools, now fixed.
 conf/locale/messages.mo
+conf/plugins-locale/
+/*/static/js/xblock.v1-i18n/
 
 ### Testing artifacts
 .testids/

Dockerfile

@@ -147,6 +147,9 @@ COPY . .
 # Install Python requirements again in order to capture local projects
 RUN pip install -e .
 
+# Setting edx-platform directory as safe for git commands
+RUN git config --global --add safe.directory /edx/app/edxapp/edx-platform
+
 # Production target
 FROM base as production
 

Makefile

@@ -4,7 +4,8 @@
   docker_auth docker_build docker_tag_build_push_lms docker_tag_build_push_lms_dev \
   docker_tag_build_push_cms docker_tag_build_push_cms_dev docs extract_translations \
   guides help lint-imports local-requirements migrate migrate-lms migrate-cms \
-  pre-requirements pull pull_translations push_translations requirements shell swagger \
+  pre-requirements pull pull_xblock_translations pull_translations push_translations \
+  requirements shell swagger \
   technical-docs test-requirements ubuntu-requirements upgrade-package upgrade
 
 # Careful with mktemp syntax: it has to work on Mac and Ubuntu, which have differences.
@@ -55,15 +56,30 @@ endif
 push_translations: ## push source strings to Transifex for translation
 	i18n_tool transifex push
 
-pull_translations:  ## pull translations from Transifex
+pull_xblock_translations:  ## pull xblock translations via atlas
+	rm -rf conf/plugins-locale  # Clean up existing atlas translations
+	rm -rf lms/static/i18n/xblock.v1 cms/static/i18n/xblock.v1  # Clean up existing xblock compiled translations
+	mkdir -p conf/plugins-locale/xblock.v1/ lms/static/js/xblock.v1-i18n cms/static/js
+	python manage.py lms pull_xblock_translations --verbose $(ATLAS_OPTIONS)
+	python manage.py lms compile_xblock_translations
+	cp -r lms/static/js/xblock.v1-i18n cms/static/js
+
+pull_translations: ## pull translations from Transifex
 	git clean -fdX conf/locale
+ifeq ($(OPENEDX_ATLAS_PULL),)
 	i18n_tool transifex pull
 	i18n_tool extract
 	i18n_tool dummy
 	i18n_tool generate --verbose 1
 	git clean -fdX conf/locale/rtl
 	git clean -fdX conf/locale/eo
 	i18n_tool validate --verbose
+else
+	make pull_xblock_translations
+	find conf/locale -mindepth 1 -maxdepth 1 -type d -exec rm -r {} \;
+	atlas pull $(ATLAS_OPTIONS) translations/edx-platform/conf/locale:conf/locale
+	i18n_tool generate
+endif
 	paver i18n_compilejs
 
 

cms/djangoapps/contentstore/course_info_model.py

@@ -19,6 +19,7 @@
 from django.http import HttpResponseBadRequest
 from django.utils.translation import gettext as _
 
+from cms.djangoapps.contentstore.utils import track_course_update_event
 from openedx.core.lib.xblock_utils import get_course_update_items
 from xmodule.html_block import CourseInfoBlock  # lint-amnesty, pylint: disable=wrong-import-order
 from xmodule.modulestore.django import modulestore  # lint-amnesty, pylint: disable=wrong-import-order
@@ -85,6 +86,8 @@ def update_course_updates(location, update, passed_id=None, user=None):
 
     # update db record
     save_course_update_items(location, course_updates, course_update_items, user)
+    # track course update event
+    track_course_update_event(location.course_key, user, course_update_dict)
     # remove status key
     if "status" in course_update_dict:
         del course_update_dict["status"]

cms/djangoapps/contentstore/management/commands/reindex_course.py

@@ -3,6 +3,7 @@
 
 import logging
 from textwrap import dedent
+from time import time
 
 from django.core.management import BaseCommand, CommandError
 from elasticsearch import exceptions
@@ -24,7 +25,7 @@ class Command(BaseCommand):
     Examples:
 
         ./manage.py reindex_course <course_id_1> <course_id_2> ... - reindexes courses with provided keys
-        ./manage.py reindex_course --all - reindexes all available courses
+        ./manage.py reindex_course --all --warning - reindexes all available courses with quieter logging
         ./manage.py reindex_course --setup - reindexes all courses for devstack setup
     """
     help = dedent(__doc__)
@@ -40,6 +41,10 @@ def add_arguments(self, parser):
         parser.add_argument('--setup',
                             action='store_true',
                             help='Reindex all courses on developers stack setup')
+        parser.add_argument('--warning',
+                            action='store_true',
+                            help='Reduce logging to a WARNING level of output for progress tracking'
+                            )
 
     def _parse_course_key(self, raw_value):
         """ Parses course key from string """
@@ -61,13 +66,18 @@ def handle(self, *args, **options):
         course_ids = options['course_ids']
         all_option = options['all']
         setup_option = options['setup']
+        readable_option = options['warning']
         index_all_courses_option = all_option or setup_option
 
         if (not len(course_ids) and not index_all_courses_option) or (len(course_ids) and index_all_courses_option):  # lint-amnesty, pylint: disable=len-as-condition
             raise CommandError("reindex_course requires one or more <course_id>s OR the --all or --setup flags.")
 
         store = modulestore()
 
+        if readable_option:
+            logging.disable(level=logging.INFO)
+            logging.warning('Reducing logging to WARNING level for easier progress tracking')
+
         if index_all_courses_option:
             index_names = (CoursewareSearchIndexer.INDEX_NAME, CourseAboutSearchIndexer.INDEX_NAME)
             if setup_option:
@@ -98,8 +108,18 @@ def handle(self, *args, **options):
             # in case course keys are provided as arguments
             course_keys = list(map(self._parse_course_key, course_ids))
 
+        total = len(course_keys)
+        logging.warning(f'Reindexing {total} courses')
+        reindexed = 0
+        start = time()
+
         for course_key in course_keys:
             try:
                 CoursewareSearchIndexer.do_course_reindex(store, course_key)
+                reindexed += 1
+                if reindexed % 10 == 0 or reindexed == total:
+                    now = time()
+                    t = now - start
+                    logging.warning(f'{reindexed}/{total} reindexed in {t:.1f} seconds')
             except Exception as exc:  # lint-amnesty, pylint: disable=broad-except
                 logging.exception('Error indexing course %s due to the error: %s', course_key, exc)

cms/djangoapps/contentstore/rest_api/serializers/common.py

@@ -50,3 +50,23 @@ def to_internal_value(self, data):
                 )
 
         return ret
+
+
+class ProctoringErrorModelSerializer(serializers.Serializer):
+    """
+    Serializer for proctoring error model item.
+    """
+    deprecated = serializers.BooleanField()
+    display_name = serializers.CharField()
+    help = serializers.CharField()
+    hide_on_enabled_publisher = serializers.BooleanField()
+    value = serializers.CharField()
+
+
+class ProctoringErrorListSerializer(serializers.Serializer):
+    """
+    Serializer for proctoring error list.
+    """
+    key = serializers.CharField()
+    message = serializers.CharField()
+    model = ProctoringErrorModelSerializer()

cms/djangoapps/contentstore/rest_api/v1/serializers/__init__.py

@@ -4,6 +4,7 @@
 from .course_details import CourseDetailsSerializer
 from .course_rerun import CourseRerunSerializer
 from .course_team import CourseTeamSerializer
+from .course_index import CourseIndexSerializer
 from .grading import CourseGradingModelSerializer, CourseGradingSerializer
 from .home import CourseHomeSerializer, CourseTabSerializer, LibraryTabSerializer
 from .proctoring import (

cms/djangoapps/contentstore/rest_api/v1/serializers/course_index.py

@@ -0,0 +1,31 @@
+"""
+API Serializers for course index
+"""
+
+from rest_framework import serializers
+
+from cms.djangoapps.contentstore.rest_api.serializers.common import ProctoringErrorListSerializer
+
+
+class InitialIndexStateSerializer(serializers.Serializer):
+    """Serializer for initial course index state"""
+    expanded_locators = serializers.ListSerializer(child=serializers.CharField())
+    locator_to_show = serializers.CharField()
+
+
+class CourseIndexSerializer(serializers.Serializer):
+    """Serializer for course index"""
+    course_release_date = serializers.CharField()
+    course_structure = serializers.DictField()
+    deprecated_blocks_info = serializers.DictField()
+    discussions_incontext_feedback_url = serializers.CharField()
+    discussions_incontext_learnmore_url = serializers.CharField()
+    initial_state = InitialIndexStateSerializer()
+    initial_user_clipboard = serializers.DictField()
+    language_code = serializers.CharField()
+    lms_link = serializers.CharField()
+    mfe_proctored_exam_settings_url = serializers.CharField()
+    notification_dismiss_url = serializers.CharField()
+    proctoring_errors = ProctoringErrorListSerializer(many=True)
+    reindex_link = serializers.CharField()
+    rerun_notification_id = serializers.IntegerField()

cms/djangoapps/contentstore/rest_api/v1/serializers/proctoring.py

@@ -4,6 +4,7 @@
 
 from rest_framework import serializers
 
+from cms.djangoapps.contentstore.rest_api.serializers.common import ProctoringErrorListSerializer
 from xmodule.course_block import get_available_providers
 
 
@@ -31,26 +32,6 @@ class ProctoredExamConfigurationSerializer(serializers.Serializer):
     course_start_date = serializers.DateTimeField()
 
 
-class ProctoringErrorModelSerializer(serializers.Serializer):
-    """
-    Serializer for proctoring error model item.
-    """
-    deprecated = serializers.BooleanField()
-    display_name = serializers.CharField()
-    help = serializers.CharField()
-    hide_on_enabled_publisher = serializers.BooleanField()
-    value = serializers.CharField()
-
-
-class ProctoringErrorListSerializer(serializers.Serializer):
-    """
-    Serializer for proctoring error list.
-    """
-    key = serializers.CharField()
-    message = serializers.CharField()
-    model = ProctoringErrorModelSerializer()
-
-
 class ProctoringErrorsSerializer(serializers.Serializer):
     """
     Serializer for proctoring errors with url to proctored exam settings.

cms/djangoapps/contentstore/rest_api/v1/urls.py

@@ -7,6 +7,7 @@
 from .views import (
     CourseDetailsView,
     CourseTeamView,
+    CourseIndexView,
     CourseGradingView,
     CourseRerunView,
     CourseSettingsView,
@@ -69,6 +70,11 @@
         CourseSettingsView.as_view(),
         name="course_settings"
     ),
+    re_path(
+        fr'^course_index/{COURSE_ID_PATTERN}$',
+        CourseIndexView.as_view(),
+        name="course_index"
+    ),
     re_path(
         fr'^course_details/{COURSE_ID_PATTERN}$',
         CourseDetailsView.as_view(),

cms/djangoapps/contentstore/rest_api/v1/views/__init__.py

@@ -2,6 +2,7 @@
 Views for v1 contentstore API.
 """
 from .course_details import CourseDetailsView
+from .course_index import CourseIndexView
 from .course_team import CourseTeamView
 from .course_rerun import CourseRerunView
 from .grading import CourseGradingView

cms/djangoapps/contentstore/rest_api/v1/views/course_index.py

@@ -0,0 +1,98 @@
+"""API Views for course index"""
+
+import edx_api_doc_tools as apidocs
+from django.conf import settings
+from opaque_keys.edx.keys import CourseKey
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
+from cms.djangoapps.contentstore.rest_api.v1.serializers import CourseIndexSerializer
+from cms.djangoapps.contentstore.utils import get_course_index_context
+from common.djangoapps.student.auth import has_studio_read_access
+from openedx.core.lib.api.view_utils import DeveloperErrorViewMixin, verify_course_exists, view_auth_classes
+
+
+@view_auth_classes(is_authenticated=True)
+class CourseIndexView(DeveloperErrorViewMixin, APIView):
+    """View for Course Index"""
+
+    @apidocs.schema(
+        parameters=[
+            apidocs.string_parameter("course_id", apidocs.ParameterLocation.PATH, description="Course ID"),
+            apidocs.string_parameter(
+                "show",
+                apidocs.ParameterLocation.QUERY,
+                description="Query param to set initial state which fully expanded to see the item",
+            )],
+        responses={
+            200: CourseIndexSerializer,
+            401: "The requester is not authenticated.",
+            403: "The requester cannot access the specified course.",
+            404: "The requested course does not exist.",
+        },
+    )
+    @verify_course_exists()
+    def get(self, request: Request, course_id: str):
+        """
+        Get an object containing course index for outline.
+
+        **Example Request**
+
+            GET /api/contentstore/v1/course_index/{course_id}?show=block-v1:edx+101+y+type@course+block@course
+
+        **Response Values**
+
+        If the request is successful, an HTTP 200 "OK" response is returned.
+
+        The HTTP 200 response contains a single dict that contains keys that
+        are the course's outline.
+
+        **Example Response**
+
+        ```json
+        {
+            "course_release_date": "Set Date",
+            "course_structure": {},
+            "deprecated_blocks_info": {
+                "deprecated_enabled_block_types": [],
+                "blocks": [],
+                "advance_settings_url": "/settings/advanced/course-v1:edx+101+y76"
+            },
+            "discussions_incontext_feedback_url": "",
+            "discussions_incontext_learnmore_url": "",
+            "initial_state": {
+                "expanded_locators": [
+                "block-v1:edx+101+y76+type@chapter+block@03de0adc9d1c4cc097062d80eb04abf6",
+                "block-v1:edx+101+y76+type@sequential+block@8a85e287e30a47e98d8c1f37f74a6a9d"
+                ],
+                "locator_to_show": "block-v1:edx+101+y76+type@chapter+block@03de0adc9d1c4cc097062d80eb04abf6"
+            },
+            "initial_user_clipboard": {
+                "content": null,
+                "source_usage_key": "",
+                "source_context_title": "",
+                "source_edit_url": ""
+            },
+            "language_code": "en",
+            "lms_link": "//localhost:18000/courses/course-v1:edx+101+y76/jump_to/block-v1:edx+101+y76",
+            "mfe_proctored_exam_settings_url": "",
+            "notification_dismiss_url": "/course_notifications/course-v1:edx+101+y76/2",
+            "proctoring_errors": [],
+            "reindex_link": "/course/course-v1:edx+101+y76/search_reindex",
+            "rerun_notification_id": 2
+        }
+        ```
+        """
+
+        course_key = CourseKey.from_string(course_id)
+        if not has_studio_read_access(request.user, course_key):
+            self.permission_denied(request)
+        course_index_context = get_course_index_context(request, course_key)
+        course_index_context.update({
+            "discussions_incontext_learnmore_url": settings.DISCUSSIONS_INCONTEXT_LEARNMORE_URL,
+            "discussions_incontext_feedback_url": settings.DISCUSSIONS_INCONTEXT_FEEDBACK_URL,
+        })
+
+        serializer = CourseIndexSerializer(course_index_context)
+        return Response(serializer.data)