diff --git a/xblocks_contrib/common/xml_utils.py b/xblocks_contrib/common/xml_utils.py index e7a69e46..1b838e9b 100644 --- a/xblocks_contrib/common/xml_utils.py +++ b/xblocks_contrib/common/xml_utils.py @@ -1,29 +1,27 @@ -"""This module centralizes detection, path computation, and attribute application -for OLX "pointer" tags used by XBlocks during import/export. -Also provides helpers to load definition XML files referenced by pointer tags. - -Note: The functionality has been taken from the edx-platform's XmlMixin class. -https://github.com/openedx/edx-platform/blob/18d5abb2f641db7f364f7566187e57bebdae9fe9/xmodule/xml_block.py +""" +XML utility functions and classes for XBlocks. +Note: Most of the functionality is taken from the edx-platform's XmlMixin. +https://github.com/openedx/edx-platform/blob/master/xmodule/xml_block.py """ +import copy import datetime import json -from typing import Any, TextIO +import logging +import os +from typing import Any from django.core.serializers.json import DjangoJSONEncoder -from fs.osfs import OSFS from lxml import etree -from lxml.etree import _Element as Element +from lxml.etree import ElementTree, XMLParser, _Element from opaque_keys.edx.keys import CourseKey, UsageKey -from opaque_keys.edx.locator import BlockUsageLocator -from xblock.core import XBlock -from xblock.fields import Field, Scope -from xblock.runtime import Runtime +from xblock.core import XML_NAMESPACES, XBlock +from xblock.fields import Dict, Field, Scope, ScopeIds -# Assume all XML files are persisted as utf-8. -EDX_XML_PARSER = etree.XMLParser(dtd_validation=False, load_dtd=False, remove_blank_text=True, encoding="utf-8") +log = logging.getLogger(__name__) -filename_extension = "xml" +# assume all XML files are persisted as utf-8. +EDX_XML_PARSER = XMLParser(dtd_validation=False, load_dtd=False, remove_blank_text=True, encoding='utf-8') class EdxJSONEncoder(DjangoJSONEncoder): @@ -52,10 +50,10 @@ def name_to_pathname(name: str) -> str: Convert a location name for use in a path: replace ':' with '/'. This allows users of the xml format to organize content into directories """ - return name.replace(":", "/") + return name.replace(':', '/') -def is_pointer_tag(xml_obj: Element) -> bool: +def is_pointer_tag(xml_obj: _Element) -> bool: """ Check if xml_obj is a pointer tag: . No children, one attribute named url_name, no text. @@ -68,9 +66,9 @@ def is_pointer_tag(xml_obj: Element) -> bool: Returns a bool. """ if xml_obj.tag != "course": - expected_attr = {"url_name"} + expected_attr = {'url_name'} else: - expected_attr = {"url_name", "course", "org"} + expected_attr = {'url_name', 'course', 'org'} actual_attr = set(xml_obj.attrib.keys()) @@ -90,7 +88,7 @@ def serialize_field(value: Any) -> str: return value elif isinstance(value, datetime.datetime): if value.tzinfo is not None and value.utcoffset() is None: - return value.isoformat() + "Z" + return value.isoformat() + 'Z' return value.isoformat() return json.dumps(value, cls=EdxJSONEncoder) @@ -133,59 +131,429 @@ def own_metadata(block: XBlock) -> dict[str, Any]: Return a JSON-friendly dictionary that contains only non-inherited field keys, mapped to their serialized values """ - return block.get_explicitly_set_fields_by_scope(Scope.settings) - - -def apply_pointer_attributes(node: Element, block: XBlock) -> None: - """Apply required pointer attributes to the relevant node for a block. - - Sets "url_name" for all blocks. For course blocks, additionally assigns - "org" and "course" attributes. + result = {} + for field in block.fields.values(): # lint-amnesty, pylint: disable=no-member + if field.scope == Scope.settings and field.is_set_on(block): + try: + result[field.name] = field.read_json(block) + except TypeError as exception: + exception_message = "{message}, Block-location:{location}, Field-name:{field_name}".format( + message=str(exception), + location=str(block.location), + field_name=field.name + ) + raise TypeError(exception_message) # lint-amnesty, pylint: disable=raise-missing-from + return result + + +class LegacyXmlMixin: """ - if not node.get("url_name"): - node.set("url_name", block.url_name) - - if getattr(block, "category", None) == "course": - # These attributes are required on course pointers - node.set("org", block.location.org) - node.set("course", block.location.course) - - -def format_filepath(category: str, name: str) -> str: - """Formats a path to an XML definition file.""" - return f"{category}/{name}.{filename_extension}" - - -def file_to_xml(file_object: TextIO) -> Element: - """ - Used when this module wants to parse a file object to xml - that will be converted to the definition. - - Returns an lxml Element - """ - return etree.parse(file_object, parser=EDX_XML_PARSER).getroot() - - -def load_file(filepath: str, fs: OSFS, def_id: BlockUsageLocator) -> Element: + Class containing XML parsing functionality for the extracted XBlocks. + NOTE: This is a temporary solution, we don't want new XBlocks to use this mixin. + It will be removed in the future improvements. """ - Open the specified file in fs, and call `file_to_xml` on it, - returning the lxml object. - - Add details and reraise on error. - """ - try: - with fs.open(filepath) as xml_file: - return file_to_xml(xml_file) - except Exception as err: - # Add info about where we are, but keep the traceback - raise Exception(f"Unable to load file contents at path {filepath} for item {def_id}: {err}") from err - - -def load_definition_xml(node: Element, runtime: Runtime, def_id: BlockUsageLocator) -> tuple[Element, str]: - """ - Loads definition_xml stored in a dedicated file - """ - url_name = node.get("url_name") - filepath = format_filepath(node.tag, name_to_pathname(url_name)) - definition_xml = load_file(filepath, runtime.resources_fs, def_id) - return definition_xml, filepath + # Extension to append to filename paths + filename_extension = 'xml' + + xml_attributes = Dict(help="Map of unhandled xml attributes, used only for storage between import and export", + default={}, scope=Scope.settings) + + metadata_to_strip = ('data_dir', + 'tabs', 'grading_policy', + 'discussion_blackouts', + # VS[compat] + # These attributes should have been removed from here once all 2012-fall courses imported into + # the CMS and "inline" OLX format deprecated. But, it never got deprecated. Moreover, it's + # widely used to this date. So, we still have to strip them. Also, removing of "filename" + # changes OLX returned by `/api/olx-export/v1/xblock/{block_id}/`, which indicates that some + # places in the platform rely on it. + 'course', 'org', 'url_name', 'filename', + # Used for storing xml attributes between import and export, for roundtrips + 'xml_attributes', + # Used by _import_xml_node_to_parent in cms/djangoapps/contentstore/helpers.py to prevent + # XmlMixin from treating some XML nodes as "pointer nodes". + "x-is-pointer-node", + ) + + # This is a categories to fields map that contains the block category specific fields which should not be + # cleaned and/or override while adding xml to node. + metadata_to_not_to_clean = { + # A category `video` having `sub` and `transcripts` fields + # which should not be cleaned/override in an xml object. + 'video': ('sub', 'transcripts') + } + + metadata_to_export_to_policy = ('discussion_topics',) + + @staticmethod + def _get_metadata_from_xml(xml_object, remove=True): + """ + Extract the metadata from the XML. + """ + meta = xml_object.find('meta') + if meta is None: + return '' + dmdata = meta.text + if remove: + xml_object.remove(meta) + return dmdata + + @classmethod + def definition_from_xml(cls, xml_object, system): + """ + Return the definition to be passed to the newly created block + during from_xml + + xml_object: An etree Element + """ + raise NotImplementedError("%s does not implement definition_from_xml" % cls.__name__) + + @classmethod + def clean_metadata_from_xml(cls, xml_object, excluded_fields=()): + """ + Remove any attribute named for a field with scope Scope.settings from the supplied + xml_object + """ + for field_name, field in cls.fields.items(): + if (field.scope == Scope.settings + and field_name not in excluded_fields + and xml_object.get(field_name) is not None): + del xml_object.attrib[field_name] + + @classmethod + def file_to_xml(cls, file_object): + """ + Used when this module wants to parse a file object to xml + that will be converted to the definition. + + Returns an lxml Element + """ + return etree.parse(file_object, parser=EDX_XML_PARSER).getroot() + + @classmethod + def load_file(cls, filepath, fs, def_id): # pylint: disable=invalid-name + """ + Open the specified file in fs, and call cls.file_to_xml on it, + returning the lxml object. + + Add details and reraise on error. + """ + try: + with fs.open(filepath) as xml_file: + return cls.file_to_xml(xml_file) + except Exception as err: + # Add info about where we are, but keep the traceback + raise Exception(f'Unable to load file contents at path {filepath} for item {def_id}: {err}') from err + + @classmethod + def load_definition(cls, xml_object, system, def_id, id_generator): + """ + Load a block from the specified xml_object. + Subclasses should not need to override this except in special + cases (e.g. html block) + + Args: + xml_object: an lxml.etree._Element containing the definition to load + system: the modulestore system (aka, runtime) which accesses data and provides access to services + def_id: the definition id for the block--used to compute the usage id and asides ids + id_generator: used to generate the usage_id + """ + + # VS[compat] + # The filename attr should have been removed once all 2012-fall courses imported into the CMS and "inline" OLX + # format deprecated. This never happened, and `filename` is still used, so we have too keep both formats. + filename = xml_object.get('filename') + if filename is None: + definition_xml = copy.deepcopy(xml_object) + filepath = '' + aside_children = [] + else: + filepath = cls._format_filepath(xml_object.tag, filename) + + # VS[compat] + # If the file doesn't exist at the right path, give the class a chance to fix it up. The file will be + # written out again in the correct format. This should have gone away once the CMS became online and had + # imported all 2012-fall courses from XML. + if not system.resources_fs.exists(filepath) and hasattr(cls, 'backcompat_paths'): + candidates = cls.backcompat_paths(filepath) + for candidate in candidates: + if system.resources_fs.exists(candidate): + filepath = candidate + break + + definition_xml = cls.load_file(filepath, system.resources_fs, def_id) + usage_id = id_generator.create_usage(def_id) + aside_children = system.parse_asides(definition_xml, def_id, usage_id, id_generator) + + # Add the attributes from the pointer node + definition_xml.attrib.update(xml_object.attrib) + + definition_metadata = cls._get_metadata_from_xml(definition_xml) + cls.clean_metadata_from_xml(definition_xml) + definition, children = cls.definition_from_xml(definition_xml, system) + if definition_metadata: + definition['definition_metadata'] = definition_metadata + definition['filename'] = [filepath, filename] + + if aside_children: + definition['aside_children'] = aside_children + + return definition, children + + @classmethod + def load_metadata(cls, xml_object): + """ + Read the metadata attributes from this xml_object. + + Returns a dictionary {key: value}. + """ + metadata = {'xml_attributes': {}} + for attr, val in xml_object.attrib.items(): + + if attr in cls.metadata_to_strip: + # don't load these + continue + + if attr not in cls.fields: + metadata['xml_attributes'][attr] = val + else: + metadata[attr] = deserialize_field(cls.fields[attr], val) + return metadata + + @classmethod + def apply_policy(cls, metadata, policy): + """ + Add the keys in policy to metadata, after processing them + through the attrmap. Updates the metadata dict in place. + """ + for attr, value in policy.items(): + if attr not in cls.fields: + # Store unknown attributes coming from policy.json + # in such a way that they will export to xml unchanged + metadata['xml_attributes'][attr] = value + else: + metadata[attr] = value + + @classmethod + def parse_xml(cls, node, runtime, keys): + """ + Use `node` to construct a new block. + + Arguments: + node (etree.Element): The xml node to parse into an xblock. + + runtime (:class:`.Runtime`): The runtime to use while parsing. + + keys (:class:`.ScopeIds`): The keys identifying where this block + will store its data. + + Returns (XBlock): The newly parsed XBlock + + """ + if keys is None: + # Passing keys=None is against the XBlock API but some platform tests do it. + def_id = runtime.id_generator.create_definition(node.tag, node.get('url_name')) + keys = ScopeIds(None, node.tag, def_id, runtime.id_generator.create_usage(def_id)) + aside_children = [] + + # Let the runtime construct the block. It will have a proper, inheritance-aware field data store. + xblock = runtime.construct_xblock_from_class(cls, keys) + + # VS[compat] + # In 2012, when the platform didn't have CMS, and all courses were handwritten XML files, problem tags + # contained XML problem descriptions withing themselves. Later, when Studio has been created, and "pointer" tags + # became the preferred problem format, edX has to add this compatibility code to 1) support both pre- and + # post-Studio course formats simulteneously, and 2) be able to migrate 2012-fall courses to Studio. Old style + # support supposed to be removed, but the deprecation process have never been initiated, so this + # compatibility must stay, probably forever. + if is_pointer_tag(node): + # new style: + # read the actual definition file--named using url_name.replace(':','/') + definition_xml, filepath = cls.load_definition_xml(node, runtime, keys.def_id) + aside_children = runtime.parse_asides(definition_xml, keys.def_id, keys.usage_id, runtime.id_generator) + else: + filepath = None + definition_xml = node + + # Note: removes metadata. + definition, children = cls.load_definition(definition_xml, runtime, keys.def_id, runtime.id_generator) + + # VS[compat] + # Make Ike's github preview links work in both old and new file layouts. + if is_pointer_tag(node): + # new style -- contents actually at filepath + definition['filename'] = [filepath, filepath] + + metadata = cls.load_metadata(definition_xml) + + # move definition metadata into dict + dmdata = definition.get('definition_metadata', '') + if dmdata: + metadata['definition_metadata_raw'] = dmdata + try: + metadata.update(json.loads(dmdata)) + except Exception as err: # lint-amnesty, pylint: disable=broad-except + log.debug('Error in loading metadata %r', dmdata, exc_info=True) + metadata['definition_metadata_err'] = str(err) + + definition_aside_children = definition.pop('aside_children', None) + if definition_aside_children: + aside_children.extend(definition_aside_children) + + # Set/override any metadata specified by policy + cls.apply_policy(metadata, runtime.get_policy(keys.usage_id)) + + # We don't want to use InheritanceKeyValueStore here (doing so will necessitate a lot of code to be moved + # from the `xmodule.modulestore.inheritance` into this class), so we manually set the fields on the block. + field_data = {**metadata, **definition} + + for field_name, value in field_data.items(): + # The 'xml_attributes' field has a special setter logic in its Field class, + # so we must handle it carefully to avoid duplicating data. + if field_name == "xml_attributes": + # The 'filename' attribute is specially handled for git links. + value["filename"] = definition.get("filename", ["", None]) + xblock.xml_attributes.update(value) + elif field_name in xblock.fields: + setattr(xblock, field_name, value) + + xblock.children = children + + if aside_children: + cls.add_applicable_asides_to_block(xblock, runtime, aside_children) + + return xblock + + @classmethod + def add_applicable_asides_to_block(cls, block, runtime, aside_children): + """ + Add asides to the block. Moved this out of the parse_xml method to use it in the VideoBlock.parse_xml + """ + asides_tags = [aside_child.tag for aside_child in aside_children] + asides = runtime.get_asides(block) + for aside in asides: + if aside.scope_ids.block_type in asides_tags: + block.add_aside(aside) + + @classmethod + def parse_xml_new_runtime(cls, node, runtime, keys): + """ + This XML lives within Learning Core and the new runtime doesn't need this + legacy XModule code. Use the "normal" XBlock parsing code. + """ + try: + return super().parse_xml_new_runtime(node, runtime, keys) + except AttributeError: + return super().parse_xml(node, runtime, keys) + + @classmethod + def load_definition_xml(cls, node, runtime, def_id): + """ + Loads definition_xml stored in a dedicated file + """ + url_name = node.get('url_name') + filepath = cls._format_filepath(node.tag, name_to_pathname(url_name)) + definition_xml = cls.load_file(filepath, runtime.resources_fs, def_id) + return definition_xml, filepath + + @classmethod + def _format_filepath(cls, category, name): + return f'{category}/{name}.{cls.filename_extension}' + + def export_to_file(self): + """If this returns True, write the definition of this block to a separate + file. + + NOTE: Do not override this without a good reason. It is here + specifically for customtag... + """ + return True + + def add_xml_to_node(self, node): + """ + For exporting, set data on `node` from ourselves. + """ + xml_object = self.definition_to_xml(self.runtime.export_fs) + + # If xml_object is None, we don't know how to serialize this node, but + # we shouldn't crash out the whole export for it. + if xml_object is None: + return + + for aside in self.runtime.get_asides(self): + if aside.needs_serialization(): + aside_node = etree.Element("unknown_root", nsmap=XML_NAMESPACES) + aside.add_xml_to_node(aside_node) + xml_object.append(aside_node) + + not_to_clean_fields = self.metadata_to_not_to_clean.get(self.category, ()) + self.clean_metadata_from_xml(xml_object, excluded_fields=not_to_clean_fields) + + # Set the tag on both nodes so we get the file path right. + xml_object.tag = self.category + node.tag = self.category + + # Add the non-inherited metadata + for attr in sorted(own_metadata(self)): + # don't want e.g. data_dir + if (attr not in self.metadata_to_strip + and attr not in self.metadata_to_export_to_policy + and attr not in not_to_clean_fields): + val = serialize_field(self.fields[attr].to_json(getattr(self, attr))) + try: + xml_object.set(attr, val) + except Exception: # lint-amnesty, pylint: disable=broad-except + logging.exception( + 'Failed to serialize metadata attribute %s with value %s in module %s. ' + 'This could mean data loss!!!', + attr, val, self.url_name + ) + + for key, value in self.xml_attributes.items(): + if key not in self.metadata_to_strip: + xml_object.set(key, serialize_field(value)) + + if self.export_to_file(): + # Write the definition to a file + url_path = name_to_pathname(self.url_name) + filepath = self._format_filepath(self.category, url_path) + self.runtime.export_fs.makedirs(os.path.dirname(filepath), recreate=True) + with self.runtime.export_fs.open(filepath, 'wb') as fileobj: + ElementTree(xml_object).write(fileobj, pretty_print=True, encoding='utf-8') + else: + # Write all attributes from xml_object onto node + node.clear() + node.tag = xml_object.tag + node.text = xml_object.text + node.tail = xml_object.tail + node.attrib.update(xml_object.attrib) + node.extend(xml_object) + + # Do not override an existing value for the course. + if not node.get('url_name'): + node.set('url_name', self.url_name) + + # We do not need to cater the `course` category here in xblocks_contrib, + # because course export is handled in the edx-platform code. + + # Special case for course pointers: + # if self.category == 'course': + # # add org and course attributes on the pointer tag + # node.set('org', self.location.org) + # node.set('course', self.location.course) + + def definition_to_xml(self, resource_fs): + """ + Return a new etree Element object created from this modules definition. + """ + raise NotImplementedError( + "%s does not implement definition_to_xml" % self.__class__.__name__) + + @property + def non_editable_metadata_fields(self): + """ + Return a list of all metadata fields that cannot be edited. + """ + non_editable_fields = super().non_editable_metadata_fields + non_editable_fields.append(LegacyXmlMixin.xml_attributes) + return non_editable_fields diff --git a/xblocks_contrib/html/html.py b/xblocks_contrib/html/html.py index 90599d6f..2896f421 100644 --- a/xblocks_contrib/html/html.py +++ b/xblocks_contrib/html/html.py @@ -4,7 +4,6 @@ """ import copy -import json import logging import os import re @@ -16,25 +15,15 @@ from django.utils.translation import gettext_noop as _ from fs.errors import ResourceNotFound from lxml import etree -from lxml.etree import ElementTree, XMLParser from opaque_keys.edx.keys import CourseKey, UsageKey from opaque_keys.edx.locator import LibraryLocatorV2 from path import Path as path from web_fragments.fragment import Fragment -from xblock.core import XML_NAMESPACES, XBlock -from xblock.fields import Boolean, Dict, Scope, ScopeIds, String, UserScope +from xblock.core import XBlock +from xblock.fields import Boolean, Scope, String, UserScope from xblock.utils.resources import ResourceLoader -from xblocks_contrib.common.xml_utils import ( - apply_pointer_attributes, - deserialize_field, - format_filepath, - is_pointer_tag, - load_definition_xml, - name_to_pathname, - own_metadata, - serialize_field, -) +from xblocks_contrib.common.xml_utils import LegacyXmlMixin, name_to_pathname log = logging.getLogger(__name__) resource_loader = ResourceLoader(__name__) @@ -42,9 +31,6 @@ # The global (course-agnostic) anonymous user ID for the user. ATTR_KEY_DEPRECATED_ANONYMOUS_USER_ID = "edx-platform.deprecated_anonymous_user_id" -# assume all XML files are persisted as utf-8. -EDX_XML_PARSER = XMLParser(dtd_validation=False, load_dtd=False, remove_blank_text=True, encoding="utf-8") - class MLStripper(HTMLParser): "helper function for html_to_text below" @@ -157,7 +143,7 @@ def stringify_children(node): # This makes our block more resilient. It won't crash in test environments # where the user service might not be available. @XBlock.wants("user") -class HtmlBlock(XBlock): +class HtmlBlock(LegacyXmlMixin, XBlock): # pylint: disable=abstract-method """ The HTML XBlock. """ @@ -173,6 +159,13 @@ class HtmlBlock(XBlock): default=_("Text"), ) data = String(help=_("Html contents to display for this block"), default="", scope=Scope.content) + upstream_data = String( + help=_("Upstream html contents to store upstream data field"), + default=None, + hidden=True, + enforce_type=True, + scope=Scope.content, + ) source_code = String( help=_("Source code for LaTeX documents. This feature is not well-supported."), scope=Scope.settings ) @@ -191,46 +184,9 @@ class HtmlBlock(XBlock): ENABLE_HTML_XBLOCK_STUDENT_VIEW_DATA = "ENABLE_HTML_XBLOCK_STUDENT_VIEW_DATA" uses_xmodule_styles_setup = True - filename_extension = "xml" template_dir_name = "html" show_in_read_only_mode = True - - xml_attributes = Dict( - help="Map of unhandled xml attributes, used only for storage between import and export", - default={}, - scope=Scope.settings, - ) - metadata_to_strip = ( - "data_dir", - "tabs", - "grading_policy", - "discussion_blackouts", - # VS[compat] - # These attributes should have been removed from here once all 2012-fall courses imported into - # the CMS and "inline" OLX format deprecated. But, it never got deprecated. Moreover, it's - # widely used to this date. So, we still have to strip them. Also, removing of "filename" - # changes OLX returned by `/api/olx-export/v1/xblock/{block_id}/`, which indicates that some - # places in the platform rely on it. - "course", - "org", - "url_name", - "filename", - # Used for storing xml attributes between import and export, for roundtrips - "xml_attributes", - # Used by _import_xml_node_to_parent in cms/djangoapps/contentstore/helpers.py to prevent - # XmlMixin from treating some XML nodes as "pointer nodes". - "x-is-pointer-node", - ) - - # This is a categories to fields map that contains the block category specific fields which should not be - # cleaned and/or override while adding xml to node. - metadata_to_not_to_clean = { - # A category `video` having `sub` and `transcripts` fields - # which should not be cleaned/override in an xml object. - "video": ("sub", "transcripts") - } - - metadata_to_export_to_policy = ("discussion_topics",) + icon_class = "other" @property def category(self): @@ -304,6 +260,9 @@ def get_html(self): user_id = current_user.opt_attrs.get(ATTR_KEY_DEPRECATED_ANONYMOUS_USER_ID) if user_id: data = data.replace("%%USER_ID%%", user_id) + if current_user.emails: + email = current_user.emails[0] + data = data.replace("%%USER_EMAIL%%", email) # The course ID replacement is always safe to run. data = data.replace("%%COURSE_ID%%", str(self.scope_ids.usage_id.context_key)) @@ -311,10 +270,16 @@ def get_html(self): def studio_view(self, context=None): # pylint: disable=unused-argument """Return a fragment that contains the html for the studio view.""" - frag = Fragment(self.get_html()) - frag.add_javascript("""function HtmlBlock(runtime, element){}""") - frag.initialize_js("HtmlBlock") - return frag + # Only the ReactJS editor is supported for this block. + # See https://github.com/openedx/frontend-app-authoring/tree/master/src/editors/containers/TextEditor + raise NotImplementedError + + @classmethod + def get_customizable_fields(cls) -> dict[str, str | None]: + return { + "display_name": "upstream_display_name", + "data": "upstream_data", + } # VS[compat] TODO (cpennington): Delete this method once all fall 2012 course # are being edited in the cms @@ -485,27 +450,13 @@ def workbench_scenarios(): ), ] - @classmethod - def clean_metadata_from_xml(cls, xml_object, excluded_fields=()): - """ - Remove any attribute named for a field with scope Scope.settings from the supplied - xml_object - """ - for field_name, field in cls.fields.items(): # pylint: disable=no-member - if ( - field.scope == Scope.settings - and field_name not in excluded_fields - and xml_object.get(field_name) is not None - ): - del xml_object.attrib[field_name] - # NOTE: html descriptors are special. We do not want to parse and # export them ourselves, because that can break things (e.g. lxml # adds body tags when it exports, but they should just be html # snippets that will be included in the middle of pages. @classmethod - def load_definition(cls, xml_object, system, location, id_generator): # pylint: disable=unused-argument + def load_definition(cls, xml_object, system, def_id, id_generator): """Load a descriptor from the specified xml_object: If there is a filename attribute, load it as a string, and @@ -532,7 +483,7 @@ def load_definition(cls, xml_object, system, location, id_generator): # pylint: # from .html # 'filename' in html pointers is a relative path # (not same as 'html/blah.html' when the pointer is in a directory itself) - pointer_path = "{category}/{url_path}".format(category="html", url_path=name_to_pathname(location.block_id)) + pointer_path = "{category}/{url_path}".format(category="html", url_path=name_to_pathname(def_id.block_id)) base = path(pointer_path).dirname() # log.debug("base = {0}, base.dirname={1}, filename={2}".format(base, base.dirname(), filename)) filepath = f"{base}/{filename}.html" @@ -573,140 +524,6 @@ def load_definition(cls, xml_object, system, location, id_generator): # pylint: # add more info and re-raise raise Exception(msg).with_traceback(sys.exc_info()[2]) - @classmethod - def load_metadata(cls, xml_object): - """ - Read the metadata attributes from this xml_object. - - Returns a dictionary {key: value}. - """ - metadata = {"xml_attributes": {}} - for attr, val in xml_object.attrib.items(): - - if attr in cls.metadata_to_strip: - # don't load these - continue - - if attr not in cls.fields: # pylint: disable=unsupported-membership-test - metadata["xml_attributes"][attr] = val - else: - metadata[attr] = deserialize_field(cls.fields[attr], val) # pylint: disable=unsubscriptable-object - return metadata - - @classmethod - def apply_policy(cls, metadata, policy): - """ - Add the keys in policy to metadata, after processing them - through the attrmap. Updates the metadata dict in place. - """ - for attr, value in policy.items(): - if attr not in cls.fields: # pylint: disable=unsupported-membership-test - # Store unknown attributes coming from policy.json - # in such a way that they will export to xml unchanged - metadata["xml_attributes"][attr] = value - else: - metadata[attr] = value - - @classmethod - def parse_xml(cls, node, runtime, keys): - """ - Use `node` to construct a new block. - - Arguments: - node (etree.Element): The xml node to parse into an xblock. - - runtime (:class:`.Runtime`): The runtime to use while parsing. - - keys (:class:`.ScopeIds`): The keys identifying where this block - will store its data. - - Returns (XBlock): The newly parsed XBlock - - """ - - if keys is None: - # Passing keys=None is against the XBlock API but some platform tests do it. - def_id = runtime.id_generator.create_definition(node.tag, node.get("url_name")) - keys = ScopeIds(None, node.tag, def_id, runtime.id_generator.create_usage(def_id)) - aside_children = [] - - # Let the runtime construct the block. It will have a proper, inheritance-aware field data store. - block = runtime.construct_xblock_from_class(cls, keys) - - # VS[compat] - # In 2012, when the platform didn't have CMS, and all courses were handwritten XML files, problem tags - # contained XML problem descriptions withing themselves. Later, when Studio has been created, and "pointer" tags - # became the preferred problem format, edX has to add this compatibility code to 1) support both pre- and - # post-Studio course formats simulteneously, and 2) be able to migrate 2012-fall courses to Studio. Old style - # support supposed to be removed, but the deprecation process have never been initiated, so this - # compatibility must stay, probably forever. - if is_pointer_tag(node): - # new style: - # read the actual definition file--named using url_name.replace(':','/') - definition_xml, filepath = load_definition_xml(node, runtime, keys.def_id) - aside_children = runtime.parse_asides(definition_xml, keys.def_id, keys.usage_id, runtime.id_generator) - else: - filepath = None - definition_xml = node - - # Note: removes metadata. - definition, children = cls.load_definition(definition_xml, runtime, keys.def_id, runtime.id_generator) - - # VS[compat] - # Make Ike's github preview links work in both old and new file layouts. - if is_pointer_tag(node): - # new style -- contents actually at filepath - definition["filename"] = [filepath, filepath] - - metadata = cls.load_metadata(definition_xml) - - # move definition metadata into dict - dmdata = definition.get("definition_metadata", "") - if dmdata: - metadata["definition_metadata_raw"] = dmdata - try: - metadata.update(json.loads(dmdata)) - except Exception as err: # lint-amnesty, pylint: disable=broad-except - log.debug("Error in loading metadata %r", dmdata, exc_info=True) - metadata["definition_metadata_err"] = str(err) - - definition_aside_children = definition.pop("aside_children", None) - if definition_aside_children: - aside_children.extend(definition_aside_children) - - # Set/override any metadata specified by policy - cls.apply_policy(metadata, runtime.get_policy(keys.usage_id)) - - field_data = {**metadata, **definition} - - for field_name, value in field_data.items(): - # The 'xml_attributes' field has a special setter logic in its Field class, - # so we must handle it carefully to avoid duplicating data. - if field_name == "xml_attributes": - # The 'filename' attribute is specially handled for git links. - value["filename"] = definition.get("filename", ["", None]) - block.xml_attributes.update(value) - elif field_name in block.fields: - setattr(block, field_name, value) - - block.children = children - - if aside_children: - cls.add_applicable_asides_to_block(block, runtime, aside_children) - - return block - - @classmethod - def add_applicable_asides_to_block(cls, block, runtime, aside_children): - """ - Add asides to the block. Moved this out of the parse_xml method to use it in the VideoBlock.parse_xml - """ - asides_tags = [aside_child.tag for aside_child in aside_children] - asides = runtime.get_asides(block) - for aside in asides: - if aside.scope_ids.block_type in asides_tags: - block.add_aside(aside) - @classmethod def parse_xml_new_runtime(cls, node, runtime, keys): """ @@ -721,67 +538,6 @@ def parse_xml_new_runtime(cls, node, runtime, keys): cls._set_field_if_present(block, name, value, {}) return block - def export_to_file(self): - """If this returns True, write the definition of this block to a separate - file. - - NOTE: Do not override this without a good reason. It is here - specifically for customtag... - """ - return True - - def add_xml_to_node(self, node): - """For exporting, set data on `node` from ourselves.""" - xml_object = self.definition_to_xml(self.runtime.export_fs) - if xml_object is None: - return - - for aside in self.runtime.get_asides(self): - if aside.needs_serialization(): - aside_node = etree.Element("unknown_root", nsmap=XML_NAMESPACES) - aside.add_xml_to_node(aside_node) - xml_object.append(aside_node) - - not_to_clean_fields = self.metadata_to_not_to_clean.get(self.category, ()) - self.clean_metadata_from_xml(xml_object, excluded_fields=not_to_clean_fields) - xml_object.tag = self.category - node.tag = self.category - - for attr in sorted(own_metadata(self)): - if ( - attr not in self.metadata_to_strip - and attr not in self.metadata_to_export_to_policy - and attr not in not_to_clean_fields - ): - # pylint: disable=unsubscriptable-object - val = serialize_field(self.fields[attr].to_json(getattr(self, attr))) - try: - xml_object.set(attr, val) - except Exception: # pylint: disable=broad-exception-caught - logging.exception("Failed to serialize metadata attribute %s in module %s.", attr, self.url_name) - - for key, value in self.xml_attributes.items(): - if key not in self.metadata_to_strip: - xml_object.set(key, serialize_field(value)) - - if self.export_to_file(): - url_path = name_to_pathname(self.url_name) - filepath = format_filepath( - self.category, self.location.run if self.category == "course" else url_path - ) - self.runtime.export_fs.makedirs(os.path.dirname(filepath), recreate=True) - with self.runtime.export_fs.open(filepath, "wb") as fileobj: - ElementTree(xml_object).write(fileobj, pretty_print=True, encoding="utf-8") - else: - node.clear() - node.tag = xml_object.tag - node.text = xml_object.text - node.tail = xml_object.tail - node.attrib.update(xml_object.attrib) - node.extend(xml_object) - - apply_pointer_attributes(node, self) - def definition_to_xml(self, resource_fs): """ Returns an lxml Element representing the definition of this block. @@ -808,5 +564,4 @@ def definition_to_xml(self, resource_fs): @property def non_editable_metadata_fields(self): """`use_latex_compiler` should not be editable in the Studio settings editor.""" - # pylint: disable=no-member return super().non_editable_metadata_fields + [HtmlBlock.xml_attributes, HtmlBlock.use_latex_compiler]