From ec4b0b8ca95a3609e03f5168d3864917814356b3 Mon Sep 17 00:00:00 2001 From: Ahmed Khalid <106074266+ahmed-arb@users.noreply.github.com> Date: Thu, 6 Jun 2024 17:31:45 +0500 Subject: [PATCH 1/2] feat: translate answers for text input type problems --- .../transformers/wiki_transformer.py | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/openedx/features/wikimedia_features/meta_translations/transformers/wiki_transformer.py b/openedx/features/wikimedia_features/meta_translations/transformers/wiki_transformer.py index 9b18af50a897..cf79f1d616ab 100644 --- a/openedx/features/wikimedia_features/meta_translations/transformers/wiki_transformer.py +++ b/openedx/features/wikimedia_features/meta_translations/transformers/wiki_transformer.py @@ -86,7 +86,7 @@ def validate_meta_data(self, data): """ required_fields = ['xml_data', 'encodings'] if not self.validate_keys(required_fields, data): - raise Exception('{} are required in problem meta_data'.format()) + raise Exception('{} are required in problem meta_data'.format(required_fields)) return True def _convert_xpath_to_meta_key_format(self, path): @@ -119,6 +119,15 @@ def _convert_meta_key_format_to_xpath(self, key): converted_path = converted_path[:start] + "[" + converted_path[end-1] + "]" + converted_path[end:] return "/{}".format(converted_path.replace('.','/')) + def _get_element_by_xpath(self, root, xpath): + """ + Return element by xpath + """ + element = root.xpath(xpath) + if not element: + raise Exception("{} not found in xml_data".format(xpath)) + return element[0] + def raw_data_to_meta_data(self, raw_data): """ Convert raw_data of problem (xml) to the meta_data of problem component (dict) @@ -151,8 +160,14 @@ def raw_data_to_meta_data(self, raw_data): problem = etree.XML(raw_data, parser=parser) tree = etree.ElementTree(problem) data_dict = {} + is_text_input = problem.xpath("/problem/stringresponse") for e in problem.iter("*"): - if e.text: + if is_text_input and e.get("answer"): + converted_xpath = self._convert_xpath_to_meta_key_format( + tree.getpath(e) + ) + data_dict.update({converted_xpath: e.get("answer").strip()}) + elif e.text: # have to convert xpath as Meta server only allows '_', '.' and '-' for data keys. converted_xpath = self._convert_xpath_to_meta_key_format(tree.getpath(e)) data_dict.update({converted_xpath: e.text.strip()}) @@ -206,11 +221,13 @@ def meta_data_to_raw_data(self, meta_data): problem = etree.XML(xml_data, parser=parser) for key, value in encodings.items(): xpath = self._convert_meta_key_format_to_xpath(key) - element = problem.xpath(xpath) - if element: - element[0].text = value + element = self._get_element_by_xpath(problem, xpath) + + if element.get("answer"): + element.set("answer", value) else: - raise Exception('{} not found in xml_data'.format(key)) + element.text = value + return etree.tostring(problem) class VideoTranscriptTransformer(WikiTransformer): From 8ba4b6a74ac1c746eaf3d17fda6f73f8c7d5223a Mon Sep 17 00:00:00 2001 From: Ahmed Khalid <106074266+ahmed-arb@users.noreply.github.com> Date: Fri, 7 Jun 2024 17:13:35 +0500 Subject: [PATCH 2/2] feat: adds a management command to backfill CourseBlockData --- .../commands/apply_wiki_parsing_changes.py | 116 ++++++++++++++++++ .../transformers/wiki_transformer.py | 2 + 2 files changed, 118 insertions(+) create mode 100644 openedx/features/wikimedia_features/meta_translations/management/commands/apply_wiki_parsing_changes.py diff --git a/openedx/features/wikimedia_features/meta_translations/management/commands/apply_wiki_parsing_changes.py b/openedx/features/wikimedia_features/meta_translations/management/commands/apply_wiki_parsing_changes.py new file mode 100644 index 000000000000..d2c2c10bf8e9 --- /dev/null +++ b/openedx/features/wikimedia_features/meta_translations/management/commands/apply_wiki_parsing_changes.py @@ -0,0 +1,116 @@ +from logging import getLogger +from lxml import etree + +from django.core.management.base import BaseCommand +from django.core import management + +from common.lib.xmodule.xmodule.modulestore.django import modulestore + +from openedx.features.wikimedia_features.meta_translations.models import ( + CourseBlockData, + CourseBlock, + TranslationVersion, + WikiTranslation, +) +from openedx.features.wikimedia_features.meta_translations.transformers.wiki_transformer import ( + ProblemTransformer, +) + +from opaque_keys.edx.keys import UsageKey + + +log = getLogger(__name__) + + +class Command(BaseCommand): + """ + This command is supposed to run if the parsing rules have changed in ProblemTransformer. It will update parsed_keys in CourseBlockData and reset the translation for every corresponding target block. + + ./manage.py cms apply_wiki_parsing_changes + """ + + help = "Update CourseBlockData entries text input problem blocks." + + def _are_parsed_keys_changed(self, old_parsed_keys, new_parsed_keys): + return old_parsed_keys.keys() != new_parsed_keys.keys() + + def _update_base_course_blocks_data(self, base_course_blocks_data): + """Updates parsed_keys and data in CourseBlockData if the parsing rules have changed in ProblemTransformer. + + Args: + base_course_blocks_data (queryset): queryset of CourseBlockData + + Returns: + list: base course block ids for updated CourseBlockData. + """ + updated_course_blocks_data = [] + + for course_block_data in base_course_blocks_data: + block_id = UsageKey.from_string(str(course_block_data.course_block.block_id)) + + try: + block = modulestore().get_item(block_id) + except Exception: + # Incase there is a disconnected block. Shouldn't happen though in normal case. + log.info("Missing block: {}".format(course_block_data.course_block.block_id)) + continue + + parsed_keys = ProblemTransformer().raw_data_to_meta_data(block.data) + + if self._are_parsed_keys_changed(course_block_data.parsed_keys, parsed_keys): + # Update CourseBlockData data and parsed_keys. + course_block_data.data = block.data + course_block_data.parsed_keys = parsed_keys + course_block_data.content_updated = True + course_block_data.save() + + updated_course_blocks_data.append(course_block_data.id) + log.info("Updated CourseBlockData for block: {}".format(course_block_data.course_block.block_id)) + + return updated_course_blocks_data + + def _unset_old_translations(self, updated_course_blocks_data): + """Resets the translations for all translated rerun courses. + + Args: + updated_course_blocks_data (list): base course block ids for updated CourseBlockData. + """ + # get the block ids of all translated rerun courses. + target_block_ids = ( + WikiTranslation.objects.filter(source_block_data__in=updated_course_blocks_data) + .values_list("target_block__block_id", flat=True) + .distinct() + ) + target_block_ids = [str(block_id) for block_id in target_block_ids] + + updated_wiki_trans = WikiTranslation.objects.filter(target_block__block_id__in=target_block_ids).update( + approved=False, approved_by=None + ) + + updated_course_blocks = CourseBlock.objects.filter(block_id__in=target_block_ids).update( + translated=False, applied_version=None, applied_translation=False + ) + + deleted_trans_ver = TranslationVersion.objects.filter(block_id__in=target_block_ids).delete() + + log.info("Updated {} CourseBlocks and {} WikiTranslations.".format(updated_course_blocks, updated_wiki_trans)) + log.info("Deleted {} translation versions.".format(deleted_trans_ver)) + + def _run_send_and_fetch_jobs(self): + management.call_command("sync_untranslated_strings_to_meta_from_edx", commit=True) + management.call_command("sync_translated_strings_to_edx_from_meta", commit=True) + + def handle(self, *args, **options): + # get CourseBlockData entries to update + course_blocks_data = CourseBlockData.objects.select_related("course_block").filter( + course_block__block_type="problem", + data_type="content", + ) + + updated_course_blocks_data = self._update_base_course_blocks_data(course_blocks_data) + + if updated_course_blocks_data: + self._unset_old_translations(updated_course_blocks_data) + self._run_send_and_fetch_jobs() + + log.info("No. of blocks updated: {}".format(len(updated_course_blocks_data))) diff --git a/openedx/features/wikimedia_features/meta_translations/transformers/wiki_transformer.py b/openedx/features/wikimedia_features/meta_translations/transformers/wiki_transformer.py index cf79f1d616ab..18b0623f55d2 100644 --- a/openedx/features/wikimedia_features/meta_translations/transformers/wiki_transformer.py +++ b/openedx/features/wikimedia_features/meta_translations/transformers/wiki_transformer.py @@ -160,6 +160,8 @@ def raw_data_to_meta_data(self, raw_data): problem = etree.XML(raw_data, parser=parser) tree = etree.ElementTree(problem) data_dict = {} + # TODO move component type attribute list to settings so + # in future we can add more components and attributes for translation is_text_input = problem.xpath("/problem/stringresponse") for e in problem.iter("*"): if is_text_input and e.get("answer"):