Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FC-0049] Handle tags when importing/exporting courses #34356

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
de3decb
refactor: Extract generate tags on csv to a function api
ChrisChV Mar 12, 2024
8056ea4
feat: Add tags.csv to export tarball
ChrisChV Mar 13, 2024
0c74ae2
feat: Verify tag count to generate tags.csv
ChrisChV Mar 14, 2024
82dab62
feat: Add new fields to csv exporter
ChrisChV Mar 19, 2024
2f931b3
test: Fix tests and types on csv exporting
ChrisChV Mar 19, 2024
07db758
feat: Import tags on course and blocks
ChrisChV Mar 26, 2024
2ee29e5
Merge branch 'master' into chris/FAL-3624-import-export-courses
ChrisChV Mar 26, 2024
f7e3260
test: Fixing tests and adding test for import
ChrisChV Mar 26, 2024
abf7b06
fix: Tests and lint
ChrisChV Mar 26, 2024
97571af
style: on comments
ChrisChV Mar 26, 2024
9bba25c
style: Nit on comment
ChrisChV Mar 26, 2024
d143b4f
Merge branch 'master' into chris/FAL-3624-import-export-courses
ChrisChV Mar 27, 2024
476cf95
refactor: Change export/import tags separator to ';'
ChrisChV Mar 27, 2024
1a3fbb6
Merge branch 'master' into chris/FAL-3624-import-export-courses
ChrisChV Mar 27, 2024
70d1d64
chore: bump version
ChrisChV Apr 1, 2024
31eab30
fix: Bug in `get_all_object_tags`
ChrisChV Apr 1, 2024
297e212
Merge branch 'master' into chris/FAL-3624-import-export-courses
ChrisChV Apr 1, 2024
4534de0
style: Nits
ChrisChV Apr 1, 2024
1a8b8b6
chore: Fix merge conflicts
ChrisChV Apr 2, 2024
5435985
style: Fix types checks
ChrisChV Apr 2, 2024
fdf7bc7
Merge branch 'master' into chris/FAL-3624-import-export-courses
ChrisChV Apr 4, 2024
8e8a595
refactor: Move test_objecttag_export_helpers.py to test directory
ChrisChV Apr 4, 2024
bda8da2
fix: Fix tests
ChrisChV Apr 4, 2024
a271fde
style: Nits on tests
ChrisChV Apr 8, 2024
90d6572
Merge branch 'master' into chris/FAL-3624-import-export-courses
ChrisChV Apr 8, 2024
c80f6f7
style: Nit on pylint
ChrisChV Apr 8, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
test: Fixing tests and adding test for import
  • Loading branch information
ChrisChV committed Mar 26, 2024
commit f7e3260ce5f60c298dfec5e3bbd4ce498ed29b8b
25 changes: 12 additions & 13 deletions openedx/core/djangoapps/content_tagging/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -276,12 +276,11 @@ def set_exported_object_tags(
# Clear all tags related with the content.
oel_tagging.delete_object_tags(content_key_str)

for taxonomy_export_id, tags_values in exported_tags.items():
for taxonomy_export_id, tags_values in exported_tags.items():
if not tags_values:
continue

taxonomy = oel_tagging.get_taxonomy_by_export_id(taxonomy_export_id)
tags_values = tags_values.split(',')
oel_tagging.tag_object(
object_id=content_key_str,
taxonomy=taxonomy,
Expand All @@ -296,30 +295,30 @@ def import_course_tags_from_csv(csv_path, course_id) -> None:
Import tags from a csv file generated on export.
"""
# Open csv file and extract the tags
try:
with open(csv_path, 'r') as csv_file:
csv_reader = csv.DictReader(csv_file)
tags_in_blocks = list(csv_reader)
except FileNotFoundError:
return
with open(csv_path, 'r') as csv_file:
csv_reader = csv.DictReader(csv_file)
tags_in_blocks = list(csv_reader)

def get_exported_tags(block) -> TagValuesByTaxonomyExportIdDict:
"""
Returns a map with taxonomy export_id and tags for this block.
"""
result = {}
for key, value in block.items():
if key in ['Type', 'Name', 'ID']:
if key in ['Type', 'Name', 'ID'] or not value:
continue
result[key] = value
result[key] = value.split(',')
return result

course_key = get_content_key_from_string(str(course_id))
course_key = CourseKey.from_string(str(course_id))

for block in tags_in_blocks:
exported_tags = get_exported_tags(block)
block_type = block.get('Type')
block_id = block.get('ID')
block_type = block.get('Type', '')
block_id = block.get('ID', '')

if not block_type or not block_id:
raise ValueError(f"Invalid format of csv in: '{block}'.")

if block_type == 'course':
set_exported_object_tags(course_key, exported_tags)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,23 +25,23 @@ def setUp(self):
self.orgA = Organization.objects.create(name="Organization A", short_name="orgA")
self.taxonomy_1 = api.create_taxonomy(name="Taxonomy 1")
api.set_taxonomy_orgs(self.taxonomy_1, all_orgs=True)
Tag.objects.create(
self.tag_1_1 = Tag.objects.create(
taxonomy=self.taxonomy_1,
value="Tag 1.1",
)
Tag.objects.create(
self.tag_1_2 = Tag.objects.create(
taxonomy=self.taxonomy_1,
value="Tag 1.2",
)

self.taxonomy_2 = api.create_taxonomy(name="Taxonomy 2")
api.set_taxonomy_orgs(self.taxonomy_2, all_orgs=True)

Tag.objects.create(
self.tag_2_1 = Tag.objects.create(
taxonomy=self.taxonomy_2,
value="Tag 2.1",
)
Tag.objects.create(
self.tag_2_2 = Tag.objects.create(
taxonomy=self.taxonomy_2,
value="Tag 2.2",
)
Expand Down Expand Up @@ -100,7 +100,7 @@ def setUp(self):
taxonomy=None,
tag=None,
_value="deleted tag",
_name="deleted taxonomy",
_export_id="deleted_taxonomy",
)

self.expected_course_objecttags = {
Expand Down Expand Up @@ -169,7 +169,7 @@ def setUp(self):
taxonomy=None,
tag=None,
_value="deleted tag",
_name="deleted taxonomy",
_export_id="deleted_taxonomy",
)

self.expected_library_objecttags = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1755,6 +1755,7 @@ def test_get_tags(self):
# Fetch this object's tags for a single taxonomy
expected_tags = [{
'name': 'Multiple Taxonomy',
'export_id': '13-multiple-taxonomy',
'taxonomy_id': taxonomy.pk,
'can_tag_object': True,
'tags': [
Expand Down
73 changes: 71 additions & 2 deletions openedx/core/djangoapps/content_tagging/tests/test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -277,10 +277,19 @@ def test_get_library_object_tags(self):
}


class TestExportTags(TaggedCourseMixin):
class TestExportImportTags(TaggedCourseMixin):
"""
Tests for export functions
Tests for export/import functions
"""
def _create_csv_file(self, content):
"""
Create a csv file and returns the path and name
"""
file_dir_name = tempfile.mkdtemp()
file_name = f'{file_dir_name}/tags.csv'
with open(file_name, 'w') as csv_file:
csv_file.write(content)
return file_name

def test_generate_csv_rows(self) -> None:
buffer = io.StringIO()
Expand All @@ -307,3 +316,63 @@ def test_export_tags_in_csv_file(self) -> None:
cleaned_content = content.replace('\r\n', '\n')
cleaned_expected_csv = self.expected_csv.replace('\r\n', '\n')
self.assertEqual(cleaned_content, cleaned_expected_csv)

def test_import_tags_invalid_format(self) -> None:
csv_path = self._create_csv_file('invalid format, Invalid\r\ntest1, test2')
with self.assertRaises(ValueError) as exc:
api.import_course_tags_from_csv(csv_path, self.course.id)
assert "Invalid format of csv in" in str(exc.exception)

def test_import_tags_valid_taxonomy_and_tags(self) -> None:
csv_path = self._create_csv_file(
'"Name","Type","ID","1-taxonomy-1","2-taxonomy-2"\r\n'
'"Test Course","course","course-v1:orgA+test_course+test_run","Tag 1.1",""\r\n'
)
api.import_course_tags_from_csv(csv_path, self.course.id)
object_tags = list(api.get_object_tags(self.course.id))
assert len(object_tags) == 1

object_tag = object_tags[0]
assert object_tag.tag == self.tag_1_1
assert object_tag.taxonomy == self.taxonomy_1

def test_import_tags_invalid_tag(self) -> None:
csv_path = self._create_csv_file(
'"Name","Type","ID","1-taxonomy-1","2-taxonomy-2"\r\n'
'"Test Course","course","course-v1:orgA+test_course+test_run","Tag 1.11",""\r\n'
)
api.import_course_tags_from_csv(csv_path, self.course.id)
object_tags = list(api.get_object_tags(self.course.id))
assert len(object_tags) == 0

object_tags = list(api.get_object_tags(
self.course.id,
include_deleted=True,
))
assert len(object_tags) == 1

object_tag = object_tags[0]
assert object_tag.tag is None
assert object_tag.value == 'Tag 1.11'
assert object_tag.taxonomy == self.taxonomy_1

def test_import_tags_invalid_taxonomy(self) -> None:
csv_path = self._create_csv_file(
'"Name","Type","ID","1-taxonomy-1-1"\r\n'
'"Test Course","course","course-v1:orgA+test_course+test_run","Tag 1.11"\r\n'
)
api.import_course_tags_from_csv(csv_path, self.course.id)
object_tags = list(api.get_object_tags(self.course.id))
assert len(object_tags) == 0

object_tags = list(api.get_object_tags(
self.course.id,
include_deleted=True,
))
assert len(object_tags) == 1

object_tag = object_tags[0]
assert object_tag.tag is None
assert object_tag.value == 'Tag 1.11'
assert object_tag.taxonomy is None
assert object_tag.export_id == '1-taxonomy-1-1'
2 changes: 2 additions & 0 deletions openedx/core/djangoapps/content_tagging/tests/test_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ def setUp(self):
create the taxonomy, simulating the effect of the following migrations:
1. openedx_tagging.core.tagging.migrations.0012_language_taxonomy
2. content_tagging.migrations.0007_system_defined_org_2
3. openedx_tagging.core.tagging.migrations.0015_taxonomy_export_id
"""
super().setUp()
Taxonomy.objects.get_or_create(id=-1, defaults={
Expand All @@ -47,6 +48,7 @@ def setUp(self):
"allow_multiple": False,
"allow_free_text": False,
"visible_to_authors": True,
"export_id": "-1_languages",
"_taxonomy_class": "openedx_tagging.core.tagging.models.system_defined.LanguageTaxonomy",
})
TaxonomyOrg.objects.get_or_create(taxonomy_id=-1, defaults={"org": None})
Expand Down
8 changes: 6 additions & 2 deletions xmodule/modulestore/xml_importer.py
Original file line number Diff line number Diff line change
Expand Up @@ -583,8 +583,12 @@ def run_imports(self):
courselike = self.import_drafts(courselike, courselike_key, data_path, dest_id)

with self.store.bulk_operations(dest_id):
self.import_tags(data_path, dest_id)

try:
self.import_tags(data_path, dest_id)
except FileNotFoundError:
logging.info(f'Course import {dest_id}: No tags.csv file present.')
except ValueError as e:
logging.info(f'Course import {dest_id}: {str(e)}')
yield courselike


Expand Down
Loading