diff --git a/contentcuration/contentcuration/tests/test_restore_channel.py b/contentcuration/contentcuration/tests/test_restore_channel.py index 6c5e1500ff..faa1130689 100644 --- a/contentcuration/contentcuration/tests/test_restore_channel.py +++ b/contentcuration/contentcuration/tests/test_restore_channel.py @@ -1,24 +1,45 @@ # -*- coding: utf-8 -*- import datetime import json +import os +import sqlite3 import uuid +import zipfile from io import BytesIO +import requests from django.core.files.storage import default_storage from django.template.loader import render_to_string from django.utils.translation import activate from django.utils.translation import deactivate +from le_utils.constants import completion_criteria +from le_utils.constants import content_kinds from le_utils.constants import exercises +from le_utils.constants import format_presets from mixer.backend.django import mixer from mock import MagicMock from mock import patch from .base import StudioTestCase from contentcuration.models import AssessmentItem +from contentcuration.models import ContentNode from contentcuration.models import generate_object_storage_name +from contentcuration.tests.testdata import fileobj_exercise_graphie +from contentcuration.tests.testdata import fileobj_exercise_image +from contentcuration.tests.utils.qti.test_validation import _item_xml +from contentcuration.utils.assessment.perseus import PerseusExerciseGenerator +from contentcuration.utils.assessment.qti.archive import QTIExerciseGenerator +from contentcuration.utils.assessment.qti.convert import hex_to_qti_id +from contentcuration.utils.assessment.qti.imsmanifest import ( + get_assessment_ids_from_manifest, +) +from contentcuration.utils.import_tools import create_assessment_items from contentcuration.utils.import_tools import create_channel +from contentcuration.utils.import_tools import create_files from contentcuration.utils.import_tools import generate_assessment_item from contentcuration.utils.import_tools import process_content +from contentcuration.utils.publish import create_content_database +from contentcuration.utils.publish import set_channel_icon_encoding thumbnail_path = "/content/thumbnail.png" @@ -170,6 +191,8 @@ def test_generate_assessment_item(self): assessment_item = generate_assessment_item( assessment_id, data["order"], data["type"], assessment_data ) + self.assertEqual(assessment_item.assessment_id, assessment_id) + self.assertEqual(assessment_item.order, data["order"]) self.assertEqual(assessment_item.type, data["type"]) self.assertEqual(assessment_item.question, data.get("question", "")) self.assertEqual(assessment_item.randomize, bool(data.get("randomize"))) @@ -188,3 +211,433 @@ def test_generate_assessment_item(self): ) ) deactivate() + + +# Their QTI ids hold both "-" and "_", the urlsafe base64 characters. +LEGACY_ID = "28b7bd766a0506a87ffb4bab40cb3e6b" # KKLe9dmoFBqh_-0urQMs-aw +ENCODED_ID = "850d3ff45bd6b85f883a5332835ee28f" # KhQ0_9FvWuF-IOlMyg17ijw +PERSEUS_ID = "fedcba0987654321fedcba0987654321" + + +class QTIRestoreTestCase(StudioTestCase): + EDITOR_ITEM_XML = _item_xml( + "{identifier}", + "Editor Item", + '' + "choice_0" + "", + 'Pick one. diagram' + 'A' + 'B' + "", + ) + + def setUp(self): + self.setUpBase() + self.source = self._create_exercise_node("1" * 32) + self.restored = self._create_exercise_node("2" * 32) + self.image = fileobj_exercise_image() + + def _create_exercise_node(self, node_id): + return ContentNode.objects.create( + title="Exercise", + node_id=node_id, + content_id=node_id, + kind_id=content_kinds.EXERCISE, + parent=self.channel.main_tree, + ) + + def _add_item(self, item_type, order, **fields): + fields.setdefault("question", "") + fields.setdefault("answers", "[]") + fields.setdefault("hints", "[]") + return AssessmentItem.objects.create( + contentnode=self.source, type=item_type, order=order, **fields + ) + + def _add_editor_item(self, order, identifier="item_1", image=None, **fields): + image = image or self.image + item = self._add_item( + exercises.QTI, + order, + raw_data=self.EDITOR_ITEM_XML.format( + identifier=identifier, + image=image.filename(), + ), + **fields, + ) + image.assessment_item = item + image.save() + return item + + def _add_editor_item_with_encoded_id(self, order): + return self._add_editor_item( + order, + identifier=hex_to_qti_id(ENCODED_ID), + image=fileobj_exercise_image(color="blue"), + assessment_id=ENCODED_ID, + ) + + def _add_legacy_item(self, order, question="Legacy question"): + return self._add_item( + exercises.INPUT_QUESTION, + order, + assessment_id=LEGACY_ID, + question=question, + answers=json.dumps([{"answer": "1", "correct": True, "order": 1}]), + ) + + def _add_perseus_item(self, order, image): + item = self._add_item( + exercises.PERSEUS_QUESTION, + order, + assessment_id=PERSEUS_ID, + raw_data=json.dumps( + { + "question": { + "content": "Perseus ![]({}/{})".format( + exercises.CONTENT_STORAGE_PLACEHOLDER, + image.filename(), + ) + } + }, + ensure_ascii=False, + ), + ) + image.assessment_item = item + image.save() + return item + + def _publish(self, generator_class): + items = self.source.assessment_items.order_by("order") + generator_class( + self.source, + { + "mastery_model": exercises.M_OF_N, + "randomize": True, + "n": 5, + "m": 3, + "all_assessment_items": [i.assessment_id for i in items], + "assessment_mapping": {i.assessment_id: i.type for i in items}, + }, + self.channel.id, + "en-US", + user_id=self.user.id, + ).create_exercise_archive() + + def _restore(self, *presets, package=None, download_url="http://studio"): + """package: QTI package content (or exception) served as-is, listed first.""" + rows, zips = [], {} + if package is not None: + rows.append(("e" * 32, "zip", format_presets.QTI_ZIP)) + zips["e" * 32] = package + for preset in presets: + file_obj = self.source.files.get(preset_id=preset) + zips[file_obj.checksum] = self._stored_package(preset).getvalue() + rows.append((file_obj.checksum, file_obj.file_format_id, preset)) + db = sqlite3.connect(":memory:") + db.execute( + "CREATE TABLE content_file (checksum, extension, preset, contentnode_id)" + ) + db.executemany( + "INSERT INTO content_file VALUES (?, ?, ?, ?)", + [(*row, self.restored.node_id) for row in rows], + ) + + def fetch(url): + content = zips[url.rsplit("/", 1)[1].split(".")[0]] + if isinstance(content, Exception): + raise content + return [content] + + with patch( + "contentcuration.utils.import_tools.requests.get", + side_effect=fetch, + ) as get: + create_assessment_items( + db.cursor(), self.restored, download_url=download_url + ) + if download_url is None: + get.assert_not_called() + return list(self.restored.assessment_items.order_by("order")) + + def _stored_package(self, preset=format_presets.QTI_ZIP): + package = self.source.files.get(preset_id=preset) + with default_storage.open(package.file_on_disk.name, "rb") as f: + return BytesIO(f.read()) + + def _package_contents(self, package): + with zipfile.ZipFile(package) as zf: + return { + name: zf.read(name) + for name in zf.namelist() + # Its manifest identifier is the node's content_id. + if name != "imsmanifest.xml" + } + + def _assert_republishes_source_package(self, *generator_classes): + presets = [g.preset for g in generator_classes] + source_packages = {p: self._stored_package(p) for p in presets} + self.source = self.restored + for generator_class in generator_classes: + self._publish(generator_class) + + self.assertEqual( + get_assessment_ids_from_manifest(self._stored_package()), + get_assessment_ids_from_manifest(source_packages[format_presets.QTI_ZIP]), + ) + for preset, source_package in source_packages.items(): + self.assertEqual( + self._package_contents(self._stored_package(preset)), + self._package_contents(source_package), + ) + + def _file_rows(self, item): + return sorted(item.files.values_list("checksum", "file_format_id", "preset_id")) + + def _assert_restores_source_rows(self, restored, assessment_ids): + """Restored rows match their source items, bar what the package cannot hold. + assessment_ids: each item's expected id, None for an editor slug's fresh id. + """ + source_items = list(self.source.assessment_items.order_by("order")) + for source_item, item, assessment_id in zip( + source_items, restored, assessment_ids, strict=True + ): + with self.subTest(order=source_item.order): + self.assertEqual(item.order, source_item.order) + self.assertEqual(self._file_rows(item), self._file_rows(source_item)) + if assessment_id: + self.assertEqual(item.assessment_id, assessment_id) + else: + # The package names this item only by its editor slug. + self.assertRegex(item.assessment_id, "^[0-9a-f]{32}$") + if source_item.type in (exercises.QTI, exercises.PERSEUS_QUESTION): + self.assertEqual(item.type, source_item.type) + self.assertEqual(item.raw_data, source_item.raw_data) + else: + # Legacy questions are packaged converted to QTI. + self.assertEqual(item.type, exercises.QTI) + + def test_qti_only_node_restores_editor_and_legacy_items_as_qti(self): + self._add_editor_item(0) + self._add_legacy_item(1) + self._add_editor_item_with_encoded_id(2) + self._publish(QTIExerciseGenerator) + + items = self._restore(format_presets.QTI_ZIP) + + self._assert_restores_source_rows(items, [None, LEGACY_ID, ENCODED_ID]) + self.assertIn("Legacy question", items[1].raw_data) + self._assert_republishes_source_package(QTIExerciseGenerator) + + def test_editor_item_with_non_canonical_id_gets_a_new_assessment_id(self): + # Matches hex_to_qti_id's shape but sets its padding bits. + identifier = "K" + "A" * 21 + "B" + self._add_editor_item(0, identifier=identifier) + self._publish(QTIExerciseGenerator) + + (restored,) = self._restore(format_presets.QTI_ZIP) + + self.assertIn(identifier, restored.raw_data) + self.assertNotEqual(restored.assessment_id, "0" * 32) + self.assertRegex(restored.assessment_id, "^[0-9a-f]{32}$") + + def test_dual_published_node_restores_each_question_once(self): + self._add_editor_item(0) + self._add_legacy_item(1) + self._add_editor_item_with_encoded_id(2) + self._publish(QTIExerciseGenerator) + self._publish(PerseusExerciseGenerator) + + items = self._restore(format_presets.QTI_ZIP, format_presets.EXERCISE) + + self._assert_restores_source_rows(items, [None, LEGACY_ID, ENCODED_ID]) + self._assert_republishes_source_package( + QTIExerciseGenerator, PerseusExerciseGenerator + ) + + def test_sized_legacy_image_restores_from_package_and_republishes(self): + item = self._add_legacy_item( + 0, + question="Sized ![](${}/{} =50x50)".format( + exercises.CONTENT_STORAGE_PLACEHOLDER, self.image.filename() + ), + ) + self.image.assessment_item = item + self.image.save() + self._publish(QTIExerciseGenerator) + + (restored,) = self._restore(format_presets.QTI_ZIP) + + self.assertEqual(restored.assessment_id, LEGACY_ID) + image = restored.files.get() + self.assertIn(image.filename(), restored.raw_data) + + # The package holds the resized image only, so republishing is the + # equivalence the restore can keep. + self._assert_republishes_source_package(QTIExerciseGenerator) + + def test_node_without_package_restores_from_exercise_archive(self): + self._add_legacy_item(0) + self._publish(PerseusExerciseGenerator) + + (item,) = self._restore(format_presets.EXERCISE) + + source_item = self.source.assessment_items.get() + self.assertEqual(item.type, exercises.INPUT_QUESTION) + self.assertEqual(item.assessment_id, source_item.assessment_id) + self.assertEqual(item.order, source_item.order) + self.assertEqual(item.question, source_item.question) + # The archive stores a numeric answer as a number, not its source text. + self.assertEqual( + [(float(a["answer"]), a["correct"]) for a in json.loads(item.answers)], + [ + (float(a["answer"]), a["correct"]) + for a in json.loads(source_item.answers) + ], + ) + + def test_local_restore_reads_archives_from_storage(self): + self._add_legacy_item(0) + self._publish(QTIExerciseGenerator) + self._publish(PerseusExerciseGenerator) + + for preset in (format_presets.QTI_ZIP, format_presets.EXERCISE): + with self.subTest(preset=preset): + self.restored.assessment_items.all().delete() + (item,) = self._restore(preset, download_url=None) + self.assertEqual( + item.assessment_id, + self.source.assessment_items.get().assessment_id, + ) + + def test_package_missing_a_listed_file_leaves_no_items(self): + self._add_legacy_item(0) + self._add_editor_item(1) + self._publish(QTIExerciseGenerator) + buf = BytesIO() + with zipfile.ZipFile(self._stored_package()) as src, zipfile.ZipFile( + buf, "w" + ) as dst: + for name in src.namelist(): + if "images/" not in name: + dst.writestr(name, src.read(name)) + + items = self._restore(package=buf.getvalue()) + + self.assertEqual(items, []) + + def test_unusable_package_restores_no_items(self): + # The dual node's exercise archive holds QTI ids, which cannot republish. + self._add_editor_item(0) + self._publish(QTIExerciseGenerator) + self._publish(PerseusExerciseGenerator) + + empty_package = BytesIO() + with zipfile.ZipFile(empty_package, "w") as zf: + zf.writestr( + "imsmanifest.xml", + '' + "", + ) + + for package in ( + empty_package.getvalue(), + b"404", + requests.ConnectionError("gone"), + ): + with self.subTest(package=package), patch( + "contentcuration.utils.import_tools.sys.stderr" + ) as stderr: + self.assertEqual( + self._restore(format_presets.EXERCISE, package=package), [] + ) + self.assertIn("Restoration Process Error", stderr.write.call_args[0][0]) + + def test_unreachable_file_is_logged_not_raised(self): + db = sqlite3.connect(":memory:") + db.execute( + "CREATE TABLE content_file " + "(checksum, extension, file_size, contentnode_id, lang_id, preset)" + ) + db.execute( + "INSERT INTO content_file VALUES (?, ?, ?, ?, ?, ?)", + ("e" * 32, "zip", 1, self.restored.node_id, None, format_presets.QTI_ZIP), + ) + + with patch( + "contentcuration.utils.import_tools.requests.get", + side_effect=requests.ConnectionError("gone"), + ), patch("contentcuration.utils.import_tools.sys.stderr") as stderr: + create_files(db.cursor(), self.restored, download_url="http://studio") + + self.assertIn("gone", stderr.write.call_args[0][0]) + self.assertFalse(self.restored.files.exists()) + + def test_perseus_question_in_package_restores_as_perseus_question(self): + self._add_perseus_item(0, fileobj_exercise_image(color="blue")) + self._add_editor_item(1) + self._publish(QTIExerciseGenerator) + + items = self._restore(format_presets.QTI_ZIP) + + self._assert_restores_source_rows(items, [PERSEUS_ID, None]) + self._assert_republishes_source_package(QTIExerciseGenerator) + + def test_graphie_in_perseus_question_restores_and_republishes(self): + graphie = fileobj_exercise_graphie(original_filename="graph1") + item = self._add_perseus_item(0, fileobj_exercise_image(color="blue")) + graphie_ref = f"web+graphie:{exercises.CONTENT_STORAGE_PLACEHOLDER}/graph1" + item.raw_data = item.raw_data.replace("Perseus", f"![]({graphie_ref})") + item.save() + graphie.assessment_item = item + graphie.save() + self._publish(QTIExerciseGenerator) + + (restored,) = self._restore(format_presets.QTI_ZIP) + + self._assert_restores_source_rows([restored], [PERSEUS_ID]) + restored_graphie = restored.files.get(preset_id=format_presets.EXERCISE_GRAPHIE) + self.assertEqual(restored_graphie.original_filename, "graph1") + self._assert_republishes_source_package(QTIExerciseGenerator) + + def _published_presets(self, node): + """The node's archive presets after publishing the channel database.""" + node.complete = True + node.extra_fields = { + "options": { + "completion_criteria": { + "model": completion_criteria.MASTERY, + "threshold": {"mastery_model": exercises.M_OF_N, "m": 3, "n": 5}, + } + } + } + node.save() + self.channel.language_id = "en" + set_channel_icon_encoding(self.channel) + os.remove(create_content_database(self.channel, True, self.user.id, False)) + return set(node.files.values_list("preset_id", flat=True)) + + def test_restored_legacy_only_node_also_publishes_derived_exercise_archive(self): + # Legacy items restore as QTI, which publish can also derive Perseus from. + self._add_legacy_item(0) + self._add_item( + exercises.MULTIPLE_SELECTION, + 1, + question="Pick both", + answers=json.dumps( + [ + {"answer": "A", "correct": True, "order": 1}, + {"answer": "B", "correct": True, "order": 2}, + ] + ), + ) + self.assertEqual(self._published_presets(self.source), {format_presets.QTI_ZIP}) + + self._restore(format_presets.QTI_ZIP) + + self.assertEqual( + self._published_presets(self.restored), + {format_presets.QTI_ZIP, format_presets.EXERCISE}, + ) diff --git a/contentcuration/contentcuration/tests/utils/qti/test_imsmanifest.py b/contentcuration/contentcuration/tests/utils/qti/test_imsmanifest.py index 949b88ffdd..134f2c908d 100644 --- a/contentcuration/contentcuration/tests/utils/qti/test_imsmanifest.py +++ b/contentcuration/contentcuration/tests/utils/qti/test_imsmanifest.py @@ -1,7 +1,13 @@ import unittest +import zipfile +from io import BytesIO +from contentcuration.utils.assessment.qti.constants import ResourceType from contentcuration.utils.assessment.qti.imsmanifest import Dependency from contentcuration.utils.assessment.qti.imsmanifest import File +from contentcuration.utils.assessment.qti.imsmanifest import ( + get_assessment_item_resources_from_manifest, +) from contentcuration.utils.assessment.qti.imsmanifest import Item from contentcuration.utils.assessment.qti.imsmanifest import Manifest from contentcuration.utils.assessment.qti.imsmanifest import Metadata @@ -135,7 +141,7 @@ def test_imsmanifest_full_integration(self): resources=[ Resource( identifier="t1-test-entry-item1", - type_="imsqti_item_xmlv3p0", + type_=ResourceType.ASSESSMENT_ITEM.value, href="items/choice-single-cardinality.xml", files=[File(href="items/choice-single-cardinality.xml")], dependencies=[Dependency(identifierref="image_resource_1")], @@ -148,25 +154,25 @@ def test_imsmanifest_full_integration(self): ), Resource( identifier="t1-test-entry-item2", - type_="imsqti_item_xmlv3p0", + type_=ResourceType.ASSESSMENT_ITEM.value, href="items/choice-multiple-cardinality.xml", files=[File(href="items/choice-multiple-cardinality.xml")], ), Resource( identifier="t1-test-entry-item3", - type_="imsqti_item_xmlv3p0", + type_=ResourceType.ASSESSMENT_ITEM.value, href="items/text-entry.xml", files=[File(href="items/text-entry.xml")], ), Resource( identifier="t1-test-entry-item4", - type_="imsqti_item_xmlv3p0", + type_=ResourceType.ASSESSMENT_ITEM.value, href="items/extended-text.xml", files=[File(href="items/extended-text.xml")], ), Resource( identifier="t1-test-entry", - type_="imsqti_test_xmlv3p0", + type_=ResourceType.ASSESSMENT_TEST.value, href="assessment.xml", files=[File(href="assessment.xml")], ), @@ -202,3 +208,50 @@ def test_imsmanifest_full_integration(self): "" ) self.assertEqual(manifest.to_xml_string(), expected_xml) + + +class TestGetAssessmentItemResourcesFromManifest(unittest.TestCase): + def _zip_with(self, resources): + manifest = Manifest( + identifier="pkg", resources=Resources(resources=resources) + ).to_xml_string() + buffer = BytesIO() + with zipfile.ZipFile(buffer, "w") as zf: + zf.writestr("imsmanifest.xml", manifest) + return zipfile.ZipFile(buffer) + + def test_items_in_manifest_order_with_dependencies(self): + zf = self._zip_with( + [ + Resource( + identifier="zzz", + type_=ResourceType.ASSESSMENT_ITEM.value, + href="items/zzz.xml", + files=[ + File(href="items/zzz.xml"), + File(href="images/a.png"), + ], + ), + Resource( + identifier="aaa", + type_=ResourceType.ASSESSMENT_ITEM.value, + href="items/aaa.xml", + files=[File(href="items/aaa.xml")], + ), + Resource( + identifier="t", + type_=ResourceType.ASSESSMENT_TEST.value, + href="tests/t.xml", + ), + ] + ) + + resources = get_assessment_item_resources_from_manifest(zf) + + self.assertEqual( + [(r.href, r.dependency_hrefs) for r in resources], + [ + ("items/zzz.xml", ["images/a.png"]), + ("items/aaa.xml", []), + ], + ) diff --git a/contentcuration/contentcuration/utils/assessment/qti/convert.py b/contentcuration/contentcuration/utils/assessment/qti/convert.py index 311334b02f..50046bb814 100644 --- a/contentcuration/contentcuration/utils/assessment/qti/convert.py +++ b/contentcuration/contentcuration/utils/assessment/qti/convert.py @@ -67,6 +67,8 @@ } text_entry_interactions = {exercises.INPUT_QUESTION, exercises.FREE_RESPONSE} +_QTI_ID_PATTERN = re.compile(r"K[A-Za-z0-9_-]{22}") + def hex_to_qti_id(hex_string): """ @@ -76,6 +78,16 @@ def hex_to_qti_id(hex_string): return f"K{base64.urlsafe_b64encode(bytes_data).decode('ascii').rstrip('=')}" +def qti_id_to_hex(qti_id): + """ + Reverse of `hex_to_qti_id`, or None for an id it cannot have produced. + """ + if not qti_id or not _QTI_ID_PATTERN.fullmatch(qti_id): + return None + hex_string = base64.urlsafe_b64decode(f"{qti_id[1:]}==").hex() + return hex_string if hex_to_qti_id(hex_string) == qti_id else None + + @dataclass(frozen=True) class LegacyAssessmentItem: type: str diff --git a/contentcuration/contentcuration/utils/assessment/qti/imsmanifest.py b/contentcuration/contentcuration/utils/assessment/qti/imsmanifest.py index cdb19ba772..3bc49c8377 100644 --- a/contentcuration/contentcuration/utils/assessment/qti/imsmanifest.py +++ b/contentcuration/contentcuration/utils/assessment/qti/imsmanifest.py @@ -2,6 +2,7 @@ import zipfile from typing import Annotated from typing import List +from typing import NamedTuple from typing import Optional from xml.etree import ElementTree as ET @@ -130,24 +131,26 @@ def _get_item_ids_from_assessment_test(zip_file, test_href): namespace_re = re.compile("\\{([^}]+)\\}") -def get_assessment_ids_from_manifest(zip_file_handle): +def _read_manifest_resources(zip_file): + """Raises ValueError for an invalid or missing manifest.""" try: - with zipfile.ZipFile(zip_file_handle, "r") as zip_file: - - # Read and parse the manifest - with zip_file.open("imsmanifest.xml") as manifest_file: - manifest_content = manifest_file.read() - - # Parse the XML - root = ET.fromstring(manifest_content) - - namespace = namespace_re.search(root.tag).group(1) + with zip_file.open("imsmanifest.xml") as manifest_file: + root = ET.fromstring(manifest_file.read()) + except ET.ParseError: + raise ValueError("Invalid XML in manifest") + except KeyError: + raise ValueError("No IMS Manifest found in zip file") + namespace = namespace_re.search(root.tag) + if namespace is None: + raise ValueError("Manifest has no namespace") + namespaces = {"imscp": namespace.group(1)} + return root.findall(".//imscp:resource", namespaces), namespaces - # Define namespace map for IMS Content Packaging - namespaces = {"imscp": namespace} - # Find all resources - resources = root.findall(".//imscp:resource", namespaces) +def get_assessment_ids_from_manifest(zip_file_handle): + try: + with zipfile.ZipFile(zip_file_handle, "r") as zip_file: + resources, _ = _read_manifest_resources(zip_file) assessment_ids = [] @@ -169,9 +172,30 @@ def get_assessment_ids_from_manifest(zip_file_handle): ) return assessment_ids - except ET.ParseError: - raise ValueError("Invalid XML in manifest") except zipfile.BadZipFile: raise ValueError("File is not a valid zip archive") - except KeyError: - raise ValueError("No IMS Manifest found in zip file") + + +class AssessmentItemResource(NamedTuple): + href: str + dependency_hrefs: List[str] + + +def get_assessment_item_resources_from_manifest(zip_file): + """ + Return the package's assessment item resources in manifest (item) order. + `dependency_hrefs` are the resource's files other than the item XML itself. + """ + resources, namespaces = _read_manifest_resources(zip_file) + return [ + AssessmentItemResource( + href=resource.get("href"), + dependency_hrefs=[ + file.get("href") + for file in resource.findall("imscp:file", namespaces) + if file.get("href") != resource.get("href") + ], + ) + for resource in resources + if resource.get("type") == ResourceType.ASSESSMENT_ITEM.value + ] diff --git a/contentcuration/contentcuration/utils/files.py b/contentcuration/contentcuration/utils/files.py index aa7c3a9db4..3643b1be1d 100644 --- a/contentcuration/contentcuration/utils/files.py +++ b/contentcuration/contentcuration/utils/files.py @@ -28,7 +28,7 @@ def create_file_from_contents( - contents, ext=None, node=None, preset_id=None, uploaded_by=None + contents, ext=None, node=None, preset_id=None, uploaded_by=None, **fields ): # Imported here rather than at module level to avoid a circular import: # the GCS storage backend imports this module, and importing models at load @@ -44,6 +44,7 @@ def create_file_from_contents( preset_id=preset_id, contentnode=node, uploaded_by=uploaded_by, + **fields, ) result.file_on_disk.name = path result.save() diff --git a/contentcuration/contentcuration/utils/import_tools.py b/contentcuration/contentcuration/utils/import_tools.py index 0a187ce4c9..697e6a75c4 100644 --- a/contentcuration/contentcuration/utils/import_tools.py +++ b/contentcuration/contentcuration/utils/import_tools.py @@ -8,7 +8,9 @@ import sqlite3 import sys import tempfile +import uuid import zipfile +from contextlib import contextmanager from io import BytesIO import requests @@ -17,11 +19,23 @@ from django.db import transaction from le_utils.constants import content_kinds from le_utils.constants import exercises +from le_utils.constants import file_formats from le_utils.constants import format_presets from le_utils.constants import roles +from lxml import etree from contentcuration import models from contentcuration.api import write_raw_content_to_storage +from contentcuration.utils.assessment.qti.archive import QTIExerciseGenerator +from contentcuration.utils.assessment.qti.convert import qti_id_to_hex +from contentcuration.utils.assessment.qti.imsmanifest import ( + get_assessment_item_resources_from_manifest, +) +from contentcuration.utils.assessment.qti.ingest import ( + find_perseus_custom_interaction_path, +) +from contentcuration.utils.assessment.qti.media import rewrite_qti_media_paths +from contentcuration.utils.assessment.qti.validation import parse_qti_xml from contentcuration.utils.files import create_file_from_contents from contentcuration.utils.files import write_base64_to_file from contentcuration.utils.garbage_collect import get_deleted_chefs_root @@ -396,7 +410,7 @@ def create_files(cursor, contentnode, indent=0, download_url=None): log.warning("\b FAILED (check logs for more details)") sys.stderr.write( "Restoration Process Error: Failed to save file object {}: {}".format( - filename, os.strerror(e.errno) + filename, e ) ) continue @@ -442,8 +456,34 @@ def create_tags(cursor, contentnode, target_id, indent=0): contentnode.save() +@contextmanager +def _download_to_tempfile(filename, download_url): + tempf = tempfile.NamedTemporaryFile( + suffix=os.path.splitext(filename)[1], delete=False + ) + try: + with tempf: + if download_url: + response = requests.get( + "{}/content/storage/{}/{}/{}".format( + download_url, filename[0], filename[1], filename + ) + ) + for chunk in response: + tempf.write(chunk) + else: + checksum = os.path.splitext(filename)[0] + path = models.generate_object_storage_name(checksum, filename) + with default_storage.open(path, "rb") as f: + shutil.copyfileobj(f, tempf) + yield tempf.name + finally: + os.unlink(tempf.name) + + def create_assessment_items(cursor, contentnode, indent=0, download_url=None): - """create_assessment_items: Generate assessment items based on perseus zip + """create_assessment_items: Generate assessment items based on the QTI zip, + or the perseus zip if the node has no QTI zip Args: cursor (sqlite3.Connection): connection to export database contentnode (models.ContentNode): node assessment items reference @@ -454,45 +494,162 @@ def create_assessment_items(cursor, contentnode, indent=0, download_url=None): # Parse database for files referencing content node and make file models sql_command = ( - "SELECT checksum, extension " - "preset FROM {table} WHERE contentnode_id='{id}' AND preset='exercise';".format( - table=FILE_TABLE, id=contentnode.node_id + "SELECT checksum, extension, preset FROM {table} " + "WHERE contentnode_id='{id}' AND preset IN ('{perseus}', '{qti}');".format( + table=FILE_TABLE, + id=contentnode.node_id, + perseus=format_presets.EXERCISE, + qti=format_presets.QTI_ZIP, ) ) - query = cursor.execute(sql_command).fetchall() - for checksum, extension in query: - filename = "{}.{}".format(checksum, extension) - log.info( - "{indent} * EXERCISE {filename}...".format( - indent=" |" * indent, filename=filename - ) - ) - + archives = [ + ("{}.{}".format(checksum, extension), preset) + for checksum, extension, preset in cursor.execute(sql_command).fetchall() + ] + qti_filename = next( + (name for name, preset in archives if preset == format_presets.QTI_ZIP), None + ) + if qti_filename: + # No fallback: a dual node's exercise archive holds QTI ids, which + # cannot republish. try: - # Store the downloaded zip into temporary storage - tempf = tempfile.NamedTemporaryFile( - suffix=".{}".format(extension), delete=False + _restore_archive( + qti_filename, _extract_qti_items, contentnode, indent, download_url ) - response = requests.get( - "{}/content/storage/{}/{}/{}".format( - download_url, filename[0], filename[1], filename + except ( + IOError, + zipfile.BadZipFile, + KeyError, + ValueError, + etree.XMLSyntaxError, + ) as e: + log.warning(f"QTI package unusable: {e}") + sys.stderr.write( + "Restoration Process Error: Failed to restore QTI package {}: {}".format( + qti_filename, e ) ) - for chunk in response: - tempf.write(chunk) - tempf.close() - extract_assessment_items(tempf.name, contentnode, download_url=download_url) + return + for filename, _ in archives: + try: + _restore_archive( + filename, extract_assessment_items, contentnode, indent, download_url + ) except IOError as e: log.warning("\b FAILED (check logs for more details)") sys.stderr.write( "Restoration Process Error: Failed to save file object {}: {}".format( - filename, os.strerror(e.errno) + filename, e ) ) - continue - finally: - os.unlink(tempf.name) + + +def _restore_archive(filename, extract, contentnode, indent, download_url): + log.info( + "{indent} * EXERCISE {filename}...".format( + indent=" |" * indent, filename=filename + ) + ) + with _download_to_tempfile(filename, download_url) as filepath: + extract(filepath, contentnode, download_url=download_url) + + +@transaction.atomic +def _extract_qti_items(filepath, contentnode, download_url=None): + with zipfile.ZipFile(filepath, "r") as zipf: + resources = get_assessment_item_resources_from_manifest(zipf) + if not resources: + raise ValueError("QTI package has no item resources") + for order, resource in enumerate(resources): + item_xml = zipf.read(resource.href) + perseus_path = find_perseus_custom_interaction_path(item_xml) + if perseus_path: + _create_perseus_item(zipf, perseus_path, resource, order, contentnode) + else: + _create_qti_item(zipf, item_xml, resource, order, contentnode) + + +def _create_qti_item(zipf, item_xml, resource, order, contentnode): + # Packaged media sits under images/; the item XML stores bare filenames. + image_hrefs = [ + href for href in resource.dependency_hrefs if href.startswith("images/") + ] + identifier = parse_qti_xml(item_xml).getroot().get("identifier") + assessment_item = models.AssessmentItem.objects.create( + contentnode=contentnode, + # Only a converted legacy item's identifier encodes its assessment_id. + assessment_id=qti_id_to_hex(identifier) or uuid.uuid4().hex, + type=exercises.QTI, + order=order, + raw_data=rewrite_qti_media_paths( + item_xml.decode("utf-8"), + {href: os.path.basename(href) for href in image_hrefs}, + ), + ) + for href in image_hrefs: + # Item-relative hrefs resolve against the item's own directory. + _create_image_file( + zipf, os.path.join(os.path.dirname(resource.href), href), assessment_item + ) + + +def _create_perseus_item(zipf, perseus_path, resource, order, contentnode): + """Restore a packaged Perseus question with its images and graphies (an .svg + with a sibling -data.json). + """ + image_dir = QTIExerciseGenerator.PERSEUS_IMAGE_DIR + hrefs = set(resource.dependency_hrefs) + graphie_hrefs = { + href + for href in hrefs + if href.endswith(".svg") and f"{os.path.splitext(href)[0]}-data.json" in hrefs + } + # Perseus hrefs are relative to the package root. + image_hrefs = [ + href + for href in resource.dependency_hrefs + if href.startswith(f"{image_dir}/") + and not href.endswith("-data.json") + and href not in graphie_hrefs + ] + raw_data = zipf.read(perseus_path).decode("utf-8") + assessment_item = models.AssessmentItem.objects.create( + contentnode=contentnode, + assessment_id=os.path.splitext(os.path.basename(perseus_path))[0], + type=exercises.PERSEUS_QUESTION, + order=order, + raw_data=raw_data.replace( + f"{exercises.IMG_PLACEHOLDER}/{image_dir}", + exercises.CONTENT_STORAGE_PLACEHOLDER, + ), + ) + for href in image_hrefs: + _create_image_file(zipf, href, assessment_item) + for href in sorted(graphie_hrefs): + _create_graphie_file(zipf, os.path.splitext(href)[0], assessment_item) + + +def _create_image_file(zipf, path, assessment_item): + create_file_from_contents( + zipf.read(path), + ext=os.path.splitext(path)[1].lstrip("."), + preset_id=format_presets.EXERCISE_IMAGE, + assessment_item=assessment_item, + ) + + +def _create_graphie_file(zipf, stem, assessment_item): + """Reverse of `_write_raw_perseus_assets`: svg and data json back into one file.""" + create_file_from_contents( + zipf.read(f"{stem}.svg") + + exercises.GRAPHIE_DELIMITER.encode("ascii") + + zipf.read(f"{stem}-data.json"), + ext=file_formats.GRAPHIE, + preset_id=format_presets.EXERCISE_GRAPHIE, + assessment_item=assessment_item, + original_filename=os.path.basename(stem), + ) def extract_assessment_items(filepath, contentnode, download_url=None): @@ -508,13 +665,14 @@ def extract_assessment_items(filepath, contentnode, download_url=None): tempdir = tempfile.mkdtemp() with zipfile.ZipFile(filepath, "r") as zipf: zipf.extractall(tempdir) - os.chdir(tempdir) - with open("exercise.json", "rb") as fobj: + with open(os.path.join(tempdir, "exercise.json"), "rb") as fobj: data = json.load(fobj) for index, assessment_id in enumerate(data["all_assessment_items"]): - with open("{}.json".format(assessment_id), "rb") as fobj: + with open( + os.path.join(tempdir, "{}.json".format(assessment_id)), "rb" + ) as fobj: assessment_item = generate_assessment_item( assessment_id, index,