From abcef5ef251bfdff55fca33c62c83ab30f2be8f6 Mon Sep 17 00:00:00 2001 From: Ivan Niedielnitsev <81557788+NiedielnitsevIvan@users.noreply.github.com> Date: Tue, 11 Jun 2024 11:50:36 +0300 Subject: [PATCH] feat: [AXM-349] Implement media generation for problem xblock (#2568) * feat: [AXM-349] Implement media generation for problem xblock * feat: [AXM-349] refactor offline content generation * style: [AXM-349] fix style issues * refactor: [AXM-349] move MathJax to assets * fix: [AXM-349] fix wrong filename * refactor: [AXM-349] refactor generated file pathes and generating task launching * style: [AXM-349] remove unused import --- openedx/features/offline_mode/__init__.py | 0 .../offline_mode/assets_management.py | 162 ++++++++++++++++++ openedx/features/offline_mode/constants.py | 13 ++ .../features/offline_mode/html_manipulator.py | 90 ++++++++++ openedx/features/offline_mode/renderer.py | 2 +- .../static/offline_mode/js/bridge.js | 35 ++++ openedx/features/offline_mode/tasks.py | 36 ++++ openedx/features/offline_mode/utils.py | 72 ++++++++ 8 files changed, 409 insertions(+), 1 deletion(-) create mode 100644 openedx/features/offline_mode/__init__.py create mode 100644 openedx/features/offline_mode/assets_management.py create mode 100644 openedx/features/offline_mode/constants.py create mode 100644 openedx/features/offline_mode/html_manipulator.py create mode 100644 openedx/features/offline_mode/static/offline_mode/js/bridge.js create mode 100644 openedx/features/offline_mode/tasks.py create mode 100644 openedx/features/offline_mode/utils.py diff --git a/openedx/features/offline_mode/__init__.py b/openedx/features/offline_mode/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/offline_mode/assets_management.py b/openedx/features/offline_mode/assets_management.py new file mode 100644 index 000000000000..46c6af06c518 --- /dev/null +++ b/openedx/features/offline_mode/assets_management.py @@ -0,0 +1,162 @@ +""" +This module contains utility functions for managing assets and files. +""" +import shutil +import logging +import os +import requests + +from django.conf import settings +from django.core.files.base import ContentFile +from django.core.files.storage import default_storage + +from xmodule.assetstore.assetmgr import AssetManager +from xmodule.contentstore.content import StaticContent +from xmodule.exceptions import NotFoundError +from xmodule.modulestore.exceptions import ItemNotFoundError + +from .constants import MATHJAX_CDN_URL, MATHJAX_STATIC_PATH + + +log = logging.getLogger(__name__) + + +def get_static_file_path(relative_path): + """ + Constructs the absolute path for a static file based on its relative path. + """ + base_path = settings.STATIC_ROOT + return os.path.join(base_path, relative_path) + + +def read_static_file(path): + """ + Reads the contents of a static file in binary mode. + """ + with open(path, 'rb') as file: + return file.read() + + +def save_asset_file(xblock, path, filename): + """ + Saves an asset file to the default storage. + + If the filename contains a '/', it reads the static file directly from the file system. + Otherwise, it fetches the asset from the AssetManager. + Args: + xblock (XBlock): The XBlock instance + path (str): The path where the asset is located. + filename (str): The name of the file to be saved. + """ + if filename.endswith('djangojs.js'): + return + + try: + if '/' in filename: + static_path = get_static_file_path(filename) + content = read_static_file(static_path) + else: + asset_key = StaticContent.get_asset_key_from_path(xblock.location.course_key, path) + content = AssetManager.find(asset_key).data + except (ItemNotFoundError, NotFoundError): + log.info(f"Asset not found: {filename}") + + else: + base_path = block_storage_path(xblock) + file_path = os.path.join(base_path, 'assets', filename) + default_storage.save(file_path, ContentFile(content)) + + +def remove_old_files(xblock): + """ + Removes the 'assets' directory, 'index.html', and 'offline_content.zip' files + in the specified base path directory. + Args: + (XBlock): The XBlock instance + """ + try: + base_path = block_storage_path(xblock) + assets_path = os.path.join(base_path, 'assets') + index_file_path = os.path.join(base_path, 'index.html') + offline_zip_path = os.path.join(base_path, f'{xblock.location.block_id}.zip') + + # Delete the 'assets' directory if it exists + if os.path.isdir(assets_path): + shutil.rmtree(assets_path) + log.info(f"Successfully deleted the directory: {assets_path}") + + # Delete the 'index.html' file if it exists + if default_storage.exists(index_file_path): + os.remove(index_file_path) + log.info(f"Successfully deleted the file: {index_file_path}") + + # Delete the 'offline_content.zip' file if it exists + if default_storage.exists(offline_zip_path): + os.remove(offline_zip_path) + log.info(f"Successfully deleted the file: {offline_zip_path}") + + except OSError as e: + log.error(f"Error occurred while deleting the files or directory: {e}") + + +def get_offline_block_content_path(xblock=None, usage_key=None): + """ + Checks whether 'offline_content.zip' file is present in the specified base path directory. + + Args: + xblock (XBlock): The XBlock instance + usage_key (UsageKey): The UsageKey of the XBlock + Returns: + bool: True if the file is present, False otherwise + """ + usage_key = usage_key or getattr(xblock, 'location', None) + base_path = block_storage_path(usage_key=usage_key) + offline_zip_path = os.path.join(base_path, f'{usage_key.block_id}.zip') + return offline_zip_path if default_storage.exists(offline_zip_path) else None + + +def block_storage_path(xblock=None, usage_key=None): + """ + Generates the base storage path for the given XBlock. + + The path is constructed based on the XBlock's location, which includes the organization, + course, block type, and block ID. + Args: + xblock (XBlock): The XBlock instance for which to generate the storage path. + usage_key (UsageKey): The UsageKey of the XBlock. + Returns: + str: The constructed base storage path. + """ + loc = usage_key or getattr(xblock, 'location', None) + return f'{str(loc.course_key)}/{loc.block_id}/' if loc else '' + + +def is_modified(xblock): + """ + Check if the xblock has been modified since the last time the offline content was generated. + + Args: + xblock (XBlock): The XBlock instance to check. + """ + file_path = os.path.join(block_storage_path(xblock), f'{xblock.location.block_id}.zip') + + try: + last_modified = default_storage.get_created_time(file_path) + except OSError: + return True + + return xblock.published_on > last_modified + + +def save_mathjax_to_xblock_assets(xblock): + """ + Saves MathJax to the local static directory. + + If MathJax is not already saved, it fetches MathJax from + the CDN and saves it to the local static directory. + """ + file_path = os.path.join(block_storage_path(xblock), MATHJAX_STATIC_PATH) + if not default_storage.exists(file_path): + response = requests.get(MATHJAX_CDN_URL) + default_storage.save(file_path, ContentFile(response.content)) + log.info(f"Successfully saved MathJax to {file_path}") diff --git a/openedx/features/offline_mode/constants.py b/openedx/features/offline_mode/constants.py new file mode 100644 index 000000000000..b6609a4ce520 --- /dev/null +++ b/openedx/features/offline_mode/constants.py @@ -0,0 +1,13 @@ +""" +Constants for offline mode app. +""" +import os + +from django.conf import settings + +MATHJAX_VERSION = '2.7.5' +MATHJAX_CDN_URL = f'https://cdn.jsdelivr.net/npm/mathjax@{MATHJAX_VERSION}/MathJax.js' +MATHJAX_STATIC_PATH = os.path.join('assets', 'js', f'MathJax-{MATHJAX_VERSION}.js') + +DEFAULT_OFFLINE_SUPPORTED_XBLOCKS = ['problem'] +OFFLINE_SUPPORTED_XBLOCKS = getattr(settings, 'OFFLINE_SUPPORTED_XBLOCKS', DEFAULT_OFFLINE_SUPPORTED_XBLOCKS) diff --git a/openedx/features/offline_mode/html_manipulator.py b/openedx/features/offline_mode/html_manipulator.py new file mode 100644 index 000000000000..f8fce464ae2e --- /dev/null +++ b/openedx/features/offline_mode/html_manipulator.py @@ -0,0 +1,90 @@ +""" +Module to prepare HTML content for offline use. +""" +import os +import re + +from bs4 import BeautifulSoup + +from django.conf import settings + +from .assets_management import save_asset_file, save_mathjax_to_xblock_assets +from .constants import MATHJAX_CDN_URL, MATHJAX_STATIC_PATH + + +class HtmlManipulator: + """ + Class to prepare HTML content for offline use. + + Changes links to static files to paths to pre-generated static files for offline use. + """ + + def __init__(self, xblock, html_data): + self.html_data = html_data + self.xblock = xblock + + def _replace_mathjax_link(self): + """ + Replace MathJax CDN link with local path to MathJax.js file. + """ + save_mathjax_to_xblock_assets(self.xblock) + mathjax_pattern = re.compile(fr'src="{MATHJAX_CDN_URL}[^"]*"') + self.html_data = mathjax_pattern.sub(f'src="{MATHJAX_STATIC_PATH}"', self.html_data) + + def _replace_static_links(self): + """ + Replace static links with local links. + """ + static_links_pattern = os.path.join(settings.STATIC_URL, r'[\w./-]+') + pattern = re.compile(fr'{static_links_pattern}') + self.html_data = pattern.sub(self._replace_link, self.html_data) + + def _replace_link(self, match): + """ + Returns the local path of the asset file. + """ + link = match.group() + filename = link.split(settings.STATIC_URL)[-1] + save_asset_file(self.xblock, link, filename) + return f'assets/{filename}' + + @staticmethod + def _replace_iframe(soup): + """ + Replace iframe tags with anchor tags. + """ + for node in soup.find_all('iframe'): + replacement = soup.new_tag('p') + tag_a = soup.new_tag('a') + tag_a['href'] = node.get('src') + tag_a.string = node.get('title', node.get('src')) + replacement.append(tag_a) + node.replace_with(replacement) + + @staticmethod + def _add_js_bridge(soup): + """ + Add JS bridge script to the HTML content. + """ + script_tag = soup.new_tag('script') + with open('openedx/features/offline_mode/static/offline_mode/js/bridge.js', 'r') as file: + script_tag.string = file.read() + if soup.body: + soup.body.append(script_tag) + else: + soup.append(script_tag) + return soup + + def process_html(self): + """ + Prepares HTML content for local use. + + Changes links to static files to paths to pre-generated static files for offline use. + """ + self._replace_static_links() + self._replace_mathjax_link() + + soup = BeautifulSoup(self.html_data, 'html.parser') + self._replace_iframe(soup) + self._add_js_bridge(soup) + return str(soup) diff --git a/openedx/features/offline_mode/renderer.py b/openedx/features/offline_mode/renderer.py index 9de75e76f9a9..2262492f6295 100644 --- a/openedx/features/offline_mode/renderer.py +++ b/openedx/features/offline_mode/renderer.py @@ -9,7 +9,7 @@ from django.contrib.sessions.backends.db import SessionStore from django.http import HttpRequest -from opaque_keys.edx.keys import CourseKey, UsageKey +from opaque_keys.edx.keys import UsageKey from xmodule.modulestore.django import modulestore from common.djangoapps.edxmako.shortcuts import render_to_string diff --git a/openedx/features/offline_mode/static/offline_mode/js/bridge.js b/openedx/features/offline_mode/static/offline_mode/js/bridge.js new file mode 100644 index 000000000000..03e6755f28cd --- /dev/null +++ b/openedx/features/offline_mode/static/offline_mode/js/bridge.js @@ -0,0 +1,35 @@ +function sendMessageToiOS(message) { + window?.webkit?.messageHandlers?.iOSBridge?.postMessage(message); +} + +function sendMessageToAndroid(message) { + window?.AndroidBridge?.postMessage(message); +} + +function receiveMessageFromiOS(message) { + console.log("Message received from iOS:", message); +} + +function receiveMessageFromAndroid(message) { + console.log("Message received from Android:", message); +} + +if (window.webkit && window.webkit.messageHandlers && window.webkit.messageHandlers.iOSBridge) { + window.addEventListener("messageFromiOS", function (event) { + receiveMessageFromiOS(event.data); + }); +} +if (window.AndroidBridge) { + window.addEventListener("messageFromAndroid", function (event) { + receiveMessageFromAndroid(event.data); + }); +} +const originalAjax = $.ajax; +$.ajax = function (options) { + sendMessageToiOS(options); + sendMessageToiOS(JSON.stringify(options)); + sendMessageToAndroid(options); + sendMessageToAndroid(JSON.stringify(options)); + console.log(options, JSON.stringify(options)); + return originalAjax.call(this, options); +}; diff --git a/openedx/features/offline_mode/tasks.py b/openedx/features/offline_mode/tasks.py new file mode 100644 index 000000000000..75e786f43ca4 --- /dev/null +++ b/openedx/features/offline_mode/tasks.py @@ -0,0 +1,36 @@ +""" +Tasks for offline mode feature. +""" +from celery import shared_task +from edx_django_utils.monitoring import set_code_owner_attribute +from opaque_keys.edx.keys import CourseKey, UsageKey + +from xmodule.modulestore.django import modulestore + +from .constants import OFFLINE_SUPPORTED_XBLOCKS +from .renderer import XBlockRenderer +from .utils import generate_offline_content + + +@shared_task +@set_code_owner_attribute +def generate_offline_content_for_course(course_id): + """ + Generates offline content for all supported XBlocks in the course. + """ + course_key = CourseKey.from_string(course_id) + for offline_supported_block_type in OFFLINE_SUPPORTED_XBLOCKS: + for xblock in modulestore().get_items(course_key, qualifiers={'category': offline_supported_block_type}): + html_data = XBlockRenderer(str(xblock.location)).render_xblock_from_lms() + generate_offline_content_for_block.apply_async([str(xblock.location), html_data]) + + +@shared_task +@set_code_owner_attribute +def generate_offline_content_for_block(block_id, html_data): + """ + Generates offline content for the specified block. + """ + block_usage_key = UsageKey.from_string(block_id) + xblock = modulestore().get_item(block_usage_key) + generate_offline_content(xblock, html_data) diff --git a/openedx/features/offline_mode/utils.py b/openedx/features/offline_mode/utils.py new file mode 100644 index 000000000000..772a2fdb9b6b --- /dev/null +++ b/openedx/features/offline_mode/utils.py @@ -0,0 +1,72 @@ +""" +Utility functions and classes for offline mode. +""" +import os +import logging + +from django.contrib.auth import get_user_model +from django.core.files.base import ContentFile +from django.core.files.storage import default_storage + +from zipfile import ZipFile + +from .assets_management import block_storage_path, remove_old_files, is_modified +from .html_manipulator import HtmlManipulator + +User = get_user_model() +log = logging.getLogger(__name__) + + +def create_zip_file(base_path, file_name): + """ + Creates a zip file with the content of the base_path directory. + """ + zf = ZipFile(default_storage.path(base_path + file_name), 'w') + zf.write(default_storage.path(base_path + 'index.html'), 'index.html') + + def add_files_to_zip(zip_file, current_base_path, current_path_in_zip): + """ + Recursively adds files to the zip file. + """ + try: + directories, filenames = default_storage.listdir(current_base_path) + except OSError: + return + + for filename in filenames: + full_path = os.path.join(current_base_path, filename) + zip_file.write(full_path, os.path.join(current_path_in_zip, filename)) + + for directory in directories: + add_files_to_zip( + zip_file, + os.path.join(current_base_path, directory), + os.path.join(current_path_in_zip, directory) + ) + + add_files_to_zip(zf, default_storage.path(base_path + 'assets/'), 'assets') + zf.close() + log.info(f'Offline content for {file_name} has been generated.') + + +def generate_offline_content(xblock, html_data): + """ + Generates archive with XBlock content for offline mode. + + Args: + xblock (XBlock): The XBlock instance + html_data (str): The HTML data of the XBlock + """ + if not is_modified(xblock): + return + + base_path = block_storage_path(xblock) + remove_old_files(xblock) + html_manipulator = HtmlManipulator(xblock, html_data) + updated_html = html_manipulator.process_html() + + default_storage.save( + os.path.join(base_path, 'index.html'), + ContentFile(updated_html), + ) + create_zip_file(base_path, f'{xblock.location.block_id}.zip')