From 531e6b84b1f0164ae88fff1afcfdefbdc673e1f5 Mon Sep 17 00:00:00 2001 From: monteri Date: Thu, 16 May 2024 18:49:25 +0200 Subject: [PATCH] feat: move logic into mobile_api --- lms/djangoapps/mobile_api/apps.py | 7 + .../mobile_api/offline_mode/signals.py | 12 ++ .../mobile_api/offline_mode/tasks.py | 12 ++ .../mobile_api/offline_mode/tests/__init__.py | 0 .../tests/test_assets_management.py | 0 .../tests/test_html_manipulation.py | 0 .../offline_mode/tests/test_signals.py | 0 .../offline_mode/tests/test_tasks.py | 0 .../offline_mode/tests/test_xblock_helpers.py | 0 .../offline_mode/tests/test_zip_management.py | 0 .../mobile_api/offline_mode/utils/__init__.py | 0 .../offline_mode/utils/assets_management.py | 47 ++++++ .../offline_mode/utils/file_management.py | 12 ++ .../offline_mode/utils/html_manipulator.py | 91 ++++++++++ .../offline_mode/utils/xblock_helpers.py | 159 ++++++++++++++++++ .../offline_mode/utils/zip_management.py | 25 +++ .../html_block/mobile_api_module.py | 10 +- 17 files changed, 373 insertions(+), 2 deletions(-) create mode 100644 lms/djangoapps/mobile_api/offline_mode/signals.py create mode 100644 lms/djangoapps/mobile_api/offline_mode/tasks.py create mode 100644 lms/djangoapps/mobile_api/offline_mode/tests/__init__.py create mode 100644 lms/djangoapps/mobile_api/offline_mode/tests/test_assets_management.py create mode 100644 lms/djangoapps/mobile_api/offline_mode/tests/test_html_manipulation.py create mode 100644 lms/djangoapps/mobile_api/offline_mode/tests/test_signals.py create mode 100644 lms/djangoapps/mobile_api/offline_mode/tests/test_tasks.py create mode 100644 lms/djangoapps/mobile_api/offline_mode/tests/test_xblock_helpers.py create mode 100644 lms/djangoapps/mobile_api/offline_mode/tests/test_zip_management.py create mode 100644 lms/djangoapps/mobile_api/offline_mode/utils/__init__.py create mode 100644 lms/djangoapps/mobile_api/offline_mode/utils/assets_management.py create mode 100644 lms/djangoapps/mobile_api/offline_mode/utils/file_management.py create mode 100644 lms/djangoapps/mobile_api/offline_mode/utils/html_manipulator.py create mode 100644 lms/djangoapps/mobile_api/offline_mode/utils/xblock_helpers.py create mode 100644 lms/djangoapps/mobile_api/offline_mode/utils/zip_management.py diff --git a/lms/djangoapps/mobile_api/apps.py b/lms/djangoapps/mobile_api/apps.py index 2e7cb30990d..c7416966c08 100644 --- a/lms/djangoapps/mobile_api/apps.py +++ b/lms/djangoapps/mobile_api/apps.py @@ -12,3 +12,10 @@ class MobileApiConfig(AppConfig): """ name = 'lms.djangoapps.mobile_api' verbose_name = "Mobile API" + + def ready(self): + """ + Connect signal handlers. + """ + from lms.djangoapps.mobile_api.offline_mode import signals # pylint: disable=unused-import + from lms.djangoapps.mobile_api.offline_mode import tasks # pylint: disable=unused-import diff --git a/lms/djangoapps/mobile_api/offline_mode/signals.py b/lms/djangoapps/mobile_api/offline_mode/signals.py new file mode 100644 index 00000000000..573e5648132 --- /dev/null +++ b/lms/djangoapps/mobile_api/offline_mode/signals.py @@ -0,0 +1,12 @@ +import six +from django.dispatch import receiver + +from xmodule.modulestore.django import SignalHandler + +from .tasks import generate_course_media + + +@receiver(SignalHandler.course_published) +def hello_world(sender, course_key, **kwargs): + import pdb; pdb.set_trace() + generate_course_media.delay(six.text_type(course_key)) diff --git a/lms/djangoapps/mobile_api/offline_mode/tasks.py b/lms/djangoapps/mobile_api/offline_mode/tasks.py new file mode 100644 index 00000000000..f4b08a5d2df --- /dev/null +++ b/lms/djangoapps/mobile_api/offline_mode/tasks.py @@ -0,0 +1,12 @@ +from celery import shared_task +from opaque_keys.edx.keys import CourseKey + +from xmodule.modulestore.django import modulestore + + +@shared_task +def generate_course_media(course_id): + course_key = CourseKey.from_string(course_id) + + for xblock_html in modulestore().get_items(course_key, qualifiers={'category': ['html', 'problem']}): + xblock_html.update_info_api() diff --git a/lms/djangoapps/mobile_api/offline_mode/tests/__init__.py b/lms/djangoapps/mobile_api/offline_mode/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lms/djangoapps/mobile_api/offline_mode/tests/test_assets_management.py b/lms/djangoapps/mobile_api/offline_mode/tests/test_assets_management.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lms/djangoapps/mobile_api/offline_mode/tests/test_html_manipulation.py b/lms/djangoapps/mobile_api/offline_mode/tests/test_html_manipulation.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lms/djangoapps/mobile_api/offline_mode/tests/test_signals.py b/lms/djangoapps/mobile_api/offline_mode/tests/test_signals.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lms/djangoapps/mobile_api/offline_mode/tests/test_tasks.py b/lms/djangoapps/mobile_api/offline_mode/tests/test_tasks.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lms/djangoapps/mobile_api/offline_mode/tests/test_xblock_helpers.py b/lms/djangoapps/mobile_api/offline_mode/tests/test_xblock_helpers.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lms/djangoapps/mobile_api/offline_mode/tests/test_zip_management.py b/lms/djangoapps/mobile_api/offline_mode/tests/test_zip_management.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lms/djangoapps/mobile_api/offline_mode/utils/__init__.py b/lms/djangoapps/mobile_api/offline_mode/utils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lms/djangoapps/mobile_api/offline_mode/utils/assets_management.py b/lms/djangoapps/mobile_api/offline_mode/utils/assets_management.py new file mode 100644 index 00000000000..f8e28128a72 --- /dev/null +++ b/lms/djangoapps/mobile_api/offline_mode/utils/assets_management.py @@ -0,0 +1,47 @@ +import os + +from django.core.files.base import ContentFile +from django.core.files.storage import default_storage +from xmodule.assetstore.assetmgr import AssetManager +from xmodule.contentstore.content import StaticContent +from xmodule.exceptions import NotFoundError +from xmodule.modulestore.exceptions import ItemNotFoundError + +from .file_management import get_static_file_path, read_static_file + + +def save_asset_file(xblock, path, filename): + try: + if '/' in filename: + static_path = get_static_file_path(filename) + content = read_static_file(static_path) + else: + asset_key = StaticContent.get_asset_key_from_path(xblock.location.course_key, path) + content = AssetManager.find(asset_key).data + except (ItemNotFoundError, NotFoundError): + pass + else: + base_path = base_storage_path(xblock) + default_storage.save(f'{base_path}assets/{filename}', ContentFile(content)) + + +def remove_old_files(base_path): + try: + directories, files = default_storage.listdir(base_path) + except OSError: + pass + else: + for file_name in files: + default_storage.delete(base_path + file_name) + + try: + directories, files = default_storage.listdir(base_path + 'assets/') + except OSError: + pass + else: + for file_name in files: + default_storage.delete(base_path + 'assets/' + file_name) + + +def base_storage_path(xblock): + return '{loc.org}/{loc.course}/{loc.block_type}/{loc.block_id}/'.format(loc=xblock.location) diff --git a/lms/djangoapps/mobile_api/offline_mode/utils/file_management.py b/lms/djangoapps/mobile_api/offline_mode/utils/file_management.py new file mode 100644 index 00000000000..170889d96b9 --- /dev/null +++ b/lms/djangoapps/mobile_api/offline_mode/utils/file_management.py @@ -0,0 +1,12 @@ +import os +from django.conf import settings + + +def get_static_file_path(relative_path): + base_path = settings.STATIC_ROOT + return os.path.join(base_path, relative_path) + + +def read_static_file(path): + with open(path, 'rb') as file: + return file.read() diff --git a/lms/djangoapps/mobile_api/offline_mode/utils/html_manipulator.py b/lms/djangoapps/mobile_api/offline_mode/utils/html_manipulator.py new file mode 100644 index 00000000000..18d3abfb2c0 --- /dev/null +++ b/lms/djangoapps/mobile_api/offline_mode/utils/html_manipulator.py @@ -0,0 +1,91 @@ +import re +from bs4 import BeautifulSoup + +from .assets_management import save_asset_file + + +class HtmlManipulator: + def __init__(self, xblock, html_data): + self.html_data = html_data + self.xblock = xblock + + def _replace_mathjax_link(self): + mathjax_pattern = re.compile(r'src="https://cdn.jsdelivr.net/npm/mathjax@2.7.5/MathJax.js[^"]*"') + return mathjax_pattern.sub('src="/static/mathjax/MathJax.js"', self.html_data) + + def _replace_static_links(self): + pattern = re.compile(r'/static/[\w./-]+') + return pattern.sub(self._replace_link, self.html_data) + + def _replace_link(self, match): + link = match.group() + filename = link.split('/static/')[-1] + save_asset_file(self.xblock, link, filename) + return f'assets/{filename}' + + def _replace_iframe(self, soup): + for node in soup.find_all('iframe'): + replacement = soup.new_tag('p') + tag_a = soup.new_tag('a') + tag_a['href'] = node.get('src') + tag_a.string = node.get('title', node.get('src')) + replacement.append(tag_a) + node.replace_with(replacement) + + def _add_js_bridge(self, soup): + script_tag = soup.new_tag('script') + script_tag.string = """ + // JS bridge script + function sendMessageToiOS(message) { + window?.webkit?.messageHandlers?.iOSBridge?.postMessage(message); + } + + function sendMessageToAndroid(message) { + window?.AndroidBridge?.postMessage(message); + } + + function receiveMessageFromiOS(message) { + console.log("Message received from iOS:", message); + } + + function receiveMessageFromAndroid(message) { + console.log("Message received from Android:", message); + } + + if (window.webkit && window.webkit.messageHandlers && window.webkit.messageHandlers.iOSBridge) { + window.addEventListener("messageFromiOS", function(event) { + receiveMessageFromiOS(event.data); + }); + } + + if (window.AndroidBridge) { + window.addEventListener("messageFromAndroid", function(event) { + receiveMessageFromAndroid(event.data); + }); + } + + const originalAjax = $.ajax; + + $.ajax = function(options) { + sendMessageToiOS(options); + sendMessageToiOS(JSON.stringify(options)); + sendMessageToAndroid(options); + sendMessageToAndroid(JSON.stringify(options)); + console.log(options, JSON.stringify(options)); + + return originalAjax.call(this, options); + }; + """ + if soup.body: + soup.body.append(script_tag) + else: + soup.append(script_tag) + return soup + + def process_html(self): + self._replace_mathjax_link() + self._replace_static_links() + soup = BeautifulSoup(self.html_data, 'html.parser') + self._replace_iframe(soup) + self._add_js_bridge(soup) + return str(soup) diff --git a/lms/djangoapps/mobile_api/offline_mode/utils/xblock_helpers.py b/lms/djangoapps/mobile_api/offline_mode/utils/xblock_helpers.py new file mode 100644 index 00000000000..d850944e584 --- /dev/null +++ b/lms/djangoapps/mobile_api/offline_mode/utils/xblock_helpers.py @@ -0,0 +1,159 @@ +from django.conf import settings +from django.core.files.base import ContentFile +from django.core.files.storage import default_storage + +from xmodule.modulestore.django import modulestore + +from .utils.html_manipulation import manipulate_html +from .utils.assets_management import save_asset_file, remove_old_files, base_storage_path +from .utils.zip_management import create_zip_file + + +def is_modified(xblock): + file_path = f'{base_storage_path(xblock)}content_html.zip' + + try: + last_modified = default_storage.get_created_time(file_path) + except OSError: + return True + + return xblock.published_on > last_modified + + +def enclosing_sequence_for_gating_checks(block): + seq_tags = ['sequential'] + if block.location.block_type in seq_tags: + return None + + ancestor = block + while ancestor and ancestor.location.block_type not in seq_tags: + ancestor = ancestor.get_parent() # Note: CourseBlock's parent is None + + if ancestor: + return block.runtime.get_block(ancestor.location) + return None + + +def xblock_view_handler(request, xblock, check_if_enrolled=True, disable_staff_debug_info=False): + """ + Helper function to render an XBlock and return the rendered HTML content. + """ + from edx_django_utils.monitoring import set_custom_attribute, set_custom_attributes_for_course_key + from lms.djangoapps.courseware.courses import get_course_with_access + from lms.djangoapps.courseware.block_render import get_block, get_block_by_usage_id, get_block_for_descriptor + from xmodule.modulestore.exceptions import ItemNotFoundError, NoPathToItem + from openedx.features.course_experience.url_helpers import ( + get_courseware_url, + get_learning_mfe_home_url, + is_request_from_learning_mfe + ) + from openedx.core.lib.mobile_utils import is_request_from_mobile_app + from openedx.features.course_experience.utils import dates_banner_should_display + from lms.djangoapps.courseware.masquerade import is_masquerading_as_specific_student, setup_masquerade + from lms.djangoapps.courseware.views.views import get_optimization_flags_for_content + from lms.djangoapps.edxnotes.helpers import is_feature_enabled + from lms.djangoapps.courseware.date_summary import verified_upgrade_deadline_link + from common.djangoapps.edxmako.shortcuts import marketing_link, render_to_response, render_to_string + usage_key = xblock.usage_key + + usage_key = usage_key.replace(course_key=modulestore().fill_in_run(usage_key.course_key)) + course_key = usage_key.course_key + + # Gathering metrics to make performance measurements easier. + set_custom_attributes_for_course_key(course_key) + set_custom_attribute('usage_key', str(usage_key)) + set_custom_attribute('block_type', usage_key.block_type) + + staff_access = has_access(request.user, 'staff', course_key) + + with modulestore().bulk_operations(course_key): + # verify the user has access to the course, including enrollment check + try: + course = get_course_with_access(request.user, 'load', course_key, check_if_enrolled=check_if_enrolled) + except: + return None + + _course_masquerade, request.user = setup_masquerade( + request, + course_key, + staff_access, + ) + + UserActivity.record_user_activity( + request.user, usage_key.course_key, request=request, only_if_mobile_app=True + ) + + recheck_access = request.GET.get('recheck_access') == '1' + try: + block, _ = get_block_by_usage_id( + request, + str(course_key), + str(usage_key), + disable_staff_debug_info=disable_staff_debug_info, + course=course, + will_recheck_access=recheck_access, + ) + except: + return None + + student_view_context = request.GET.dict() + student_view_context['show_bookmark_button'] = request.GET.get('show_bookmark_button', '0') == '1' + student_view_context['show_title'] = request.GET.get('show_title', '1') == '1' + + is_learning_mfe = is_request_from_learning_mfe(request) + student_view_context['hide_access_error_blocks'] = is_learning_mfe and recheck_access + is_mobile_app = is_request_from_mobile_app(request) + student_view_context['is_mobile_app'] = is_mobile_app + + enable_completion_on_view_service = False + completion_service = block.runtime.service(block, 'completion') + if completion_service and completion_service.completion_tracking_enabled(): + if completion_service.blocks_to_mark_complete_on_view({block}): + enable_completion_on_view_service = True + student_view_context['wrap_xblock_data'] = { + 'mark-completed-on-view-after-delay': completion_service.get_complete_on_view_delay_ms() + } + + missed_deadlines, missed_gated_content = dates_banner_should_display(course_key, request.user) + fragment = block.render('student_view', context=student_view_context) + optimization_flags = get_optimization_flags_for_content(block, fragment) + + context = { + 'fragment': fragment, + 'course': course, + 'block': block, + 'disable_accordion': True, + 'allow_iframing': True, + 'disable_header': True, + 'disable_footer': True, + 'disable_window_wrap': True, + 'enable_completion_on_view_service': enable_completion_on_view_service, + 'edx_notes_enabled': is_feature_enabled(course, request.user), + 'staff_access': staff_access, + 'xqa_server': settings.FEATURES.get('XQA_SERVER', 'http://your_xqa_server.com'), + 'missed_deadlines': missed_deadlines, + 'missed_gated_content': missed_gated_content, + 'has_ended': course.has_ended(), + 'web_app_course_url': get_learning_mfe_home_url(course_key=course.id, url_fragment='home'), + 'on_courseware_page': True, + 'verified_upgrade_link': verified_upgrade_deadline_link(request.user, course=course), + 'is_learning_mfe': is_learning_mfe, + 'is_mobile_app': is_mobile_app, + 'render_course_wide_assets': True, + + **optimization_flags, + } + return render_to_string('courseware/courseware-chromeless.html', context) + + +def generate_offline_content(xblock, html_data): + if not is_modified(xblock): + return + + base_path = base_storage_path(xblock) + remove_old_files(base_path) + + manipulated_html = manipulate_html(html_data, lambda path, filename: save_asset_file(xblock, path, filename)) + + default_storage.save(f'{base_path}index.html', ContentFile(manipulated_html)) + create_zip_file(base_path, 'content_html.zip') diff --git a/lms/djangoapps/mobile_api/offline_mode/utils/zip_management.py b/lms/djangoapps/mobile_api/offline_mode/utils/zip_management.py new file mode 100644 index 00000000000..14f9865b7fb --- /dev/null +++ b/lms/djangoapps/mobile_api/offline_mode/utils/zip_management.py @@ -0,0 +1,25 @@ +import os +import zipfile +from django.core.files.storage import default_storage + + +def create_zip_file(base_path, file_name): + zf = zipfile.ZipFile(default_storage.path(base_path + file_name), "w") + zf.write(default_storage.path(base_path + "index.html"), "index.html") + + def add_files_to_zip(zip_file, current_base_path, current_path_in_zip): + try: + directories, files = default_storage.listdir(current_base_path) + except OSError: + return + + for file_name in files: + full_path = os.path.join(current_base_path, file_name) + zip_file.write(full_path, os.path.join(current_path_in_zip, file_name)) + + for directory in directories: + add_files_to_zip(zip_file, os.path.join(current_base_path, directory), + os.path.join(current_path_in_zip, directory)) + + add_files_to_zip(zf, default_storage.path(base_path + "assets/"), 'assets') + zf.close() diff --git a/openedx/features/_mobile_extensions/html_block/mobile_api_module.py b/openedx/features/_mobile_extensions/html_block/mobile_api_module.py index 189268508fe..4a73d7d5bca 100644 --- a/openedx/features/_mobile_extensions/html_block/mobile_api_module.py +++ b/openedx/features/_mobile_extensions/html_block/mobile_api_module.py @@ -33,8 +33,12 @@ def update_info_api(self, html_data=None): base_path = self._base_storage_path() self.remove_old_files(base_path) + # Replace MathJax URL + mathjax_pattern = re.compile(r'src="https://cdn.jsdelivr.net/npm/mathjax@2.7.5/MathJax.js[^"]*"') + data = mathjax_pattern.sub(self._replace_mathjax_link, html_data) + pattern = re.compile(r'/static/[\w./-]+') - data = pattern.sub(self._replace_static_links, html_data) + data = pattern.sub(self._replace_static_links, data) # Parse the HTML with BeautifulSoup soup = BeautifulSoup(data, 'html.parser') @@ -54,6 +58,9 @@ def _replace_static_links(self, match): self.save_asset_file(link, filename) return f'assets/{filename}' + def _replace_mathjax_link(self, match): + return 'src="/static/mathjax/MathJax.js"' + def _replace_iframe(self, soup): for node in soup.find_all('iframe'): replacement = soup.new_tag('p') @@ -64,7 +71,6 @@ def _replace_iframe(self, soup): node.replace_with(replacement) def _add_js_bridge(self, soup): - import pdb; pdb.set_trace() script_tag = soup.new_tag('script') script_tag.string = """ // Function to send messages to iOS