775: Support Entities from repeats

pyxform/aliases.py

@@ -123,7 +123,6 @@
     "audio": survey_header["audio"],
     "video": survey_header["video"],
 }
-# Note that most of the type aliasing happens in all.xls
 _type_alias_map = {
     "imei": "deviceid",
     "image": "photo",

pyxform/constants.py

@@ -86,6 +86,7 @@
 OSM_TYPE = "binary"
 
 NAMESPACES = "namespaces"
+META = "meta"
 
 # The following are the possible sheet names:
 SUPPORTED_SHEET_NAMES = {
@@ -125,6 +126,7 @@ class EntityColumns(StrEnum):
     ENTITY_ID = "entity_id"
     CREATE_IF = "create_if"
     UPDATE_IF = "update_if"
+    REPEAT = "repeat"
     LABEL = "label"
 
 
@@ -169,3 +171,4 @@ class EntityColumns(StrEnum):
 }
 SUPPORTED_MEDIA_TYPES = {"image", "big-image", "audio", "video"}
 OR_OTHER_CHOICE = {NAME: "other", LABEL: "Other"}
+RESERVED_NAMES_SURVEY_SHEET = {META}

pyxform/entities/entities_parsing.py

@@ -2,10 +2,56 @@
 from typing import Any
 
 from pyxform import constants as const
-from pyxform.errors import PyXFormError
+from pyxform.errors import Detail, PyXFormError
 from pyxform.parsing.expression import is_xml_tag
+from pyxform.utils import PYXFORM_REFERENCE_REGEX
 
 EC = const.EntityColumns
+ENTITY001 = Detail(
+    name="Invalid entity repeat reference",
+    msg=(
+        "[row : 2] On the 'entities' sheet, the 'repeat' value '{value}' is invalid. "
+        "The 'repeat' column, if specified, must contain only a single reference variable "
+        "(like '${{q1}}'), and the reference variable must contain a valid name."
+    ),
+)
+ENTITY002 = Detail(
+    name="Invalid entity repeat: target not found",
+    msg=(
+        "[row : 2] On the 'entities' sheet, the 'repeat' value '{value}' is invalid. "
+        "The entity repeat target was not found in the 'survey' sheet."
+    ),
+)
+ENTITY003 = Detail(
+    name="Invalid entity repeat: target is not a repeat",
+    msg=(
+        "[row : 2] On the 'entities' sheet, the 'repeat' value '{value}' is invalid. "
+        "The entity repeat target is not a repeat."
+    ),
+)
+ENTITY004 = Detail(
+    name="Invalid entity repeat: target is in a repeat",
+    msg=(
+        "[row : 2] On the 'entities' sheet, the 'repeat' value '{value}' is invalid. "
+        "The entity repeat target is inside a repeat."
+    ),
+)
+ENTITY005 = Detail(
+    name="Invalid entity repeat save_to: question in nested repeat",
+    msg=(
+        "[row : {row}] On the 'survey' sheet, the 'save_to' value '{value}' is invalid. "
+        "The entity properties populated with 'save_to' must not be inside of a nested "
+        "repeat within the entity repeat."
+    ),
+)
+ENTITY006 = Detail(
+    name="Invalid entity repeat save_to: question not in entity repeat",
+    msg=(
+        "[row : {row}] On the 'survey' sheet, the 'save_to' value '{value}' is invalid. "
+        "The entity properties populated with 'save_to' must be inside of the entity "
+        "repeat."
+    ),
+)
 
 
 def get_entity_declaration(
@@ -24,6 +70,7 @@ def get_entity_declaration(
     create_condition = entity_row.get(EC.CREATE_IF, None)
     update_condition = entity_row.get(EC.UPDATE_IF, None)
     entity_label = entity_row.get(EC.LABEL, None)
+    entity_repeat = get_validated_repeat_name(entity_row)
 
     if update_condition and not entity_id:
         raise PyXFormError(
@@ -49,6 +96,7 @@ def get_entity_declaration(
             EC.CREATE_IF.value: create_condition,
             EC.UPDATE_IF.value: update_condition,
             EC.LABEL.value: entity_label,
+            EC.REPEAT.value: entity_repeat,
         },
     }
 
@@ -77,11 +125,29 @@ def get_validated_dataset_name(entity):
     return dataset
 
 
+def get_validated_repeat_name(entity) -> str | None:
+    if EC.REPEAT.value not in entity:
+        return None
+
+    value = entity[EC.REPEAT]
+    if not (value and len(value) > 3):
+        raise PyXFormError(ENTITY001.format(value=value))
+    matches = PYXFORM_REFERENCE_REGEX.fullmatch(value)
+    if matches is None:
+        raise PyXFormError(ENTITY001.format(value=value))
+    else:
+        match = matches.group(1)
+        if not is_xml_tag(match):
+            raise PyXFormError(ENTITY001.format(value=value))
+        else:
+            return match
+
+
 def validate_entity_saveto(
     row: dict,
     row_number: int,
-    in_repeat: bool,
     entity_declaration: dict[str, Any] | None = None,
+    stack: Sequence[dict[str, Any]] | None = None,
 ):
     save_to = row.get(const.BIND, {}).get("entities:saveto", "")
     if not save_to:
@@ -97,10 +163,24 @@ def validate_entity_saveto(
             f"{const.ROW_FORMAT_STRING % row_number} Groups and repeats can't be saved as entity properties."
         )
 
-    if in_repeat:
-        raise PyXFormError(
-            f"{const.ROW_FORMAT_STRING % row_number} Currently, you can't create entities from repeats. You may only specify save_to values for form fields outside of repeats."
-        )
+    entity_repeat = entity_declaration[const.PARAMETERS].get(EC.REPEAT, None)
+    if entity_repeat and stack:
+        # Error: saveto in nested repeat inside entity repeat.
+        in_repeat = False
+        located = False
+        for i in reversed(stack):
+            if not i["control_name"] or not i["control_type"]:
+                break
+            elif i["control_type"] == const.REPEAT:
+                if in_repeat:
+                    raise PyXFormError(ENTITY005.format(row=row_number, value=save_to))
+                elif i["control_name"] == entity_repeat:
+                    located = True
+                in_repeat = True
+
+        # Error: saveto not in entity repeat
+        if not located:
+            raise PyXFormError(ENTITY006.format(row=row_number, value=save_to))
 
     error_start = f"{const.ROW_FORMAT_STRING % row_number} Invalid save_to name:"
 
@@ -134,3 +214,56 @@ def validate_entities_columns(row: dict):
             f"pyxform."
         )
         raise PyXFormError(msg)
+
+
+def validate_entity_repeat_target(
+    entity_declaration: dict[str, Any] | None,
+    stack: Sequence[dict[str, Any]] | None = None,
+) -> bool:
+    """
+    Check if the entity repeat target exists, is a repeat, and is a name match.
+
+    Raises an error if the control type or name is None (such as for the Survey), or if
+    the control type is not a repeat.
+
+    :param entity_declaration:
+    :param stack: The control stack from workbook_to_json.
+    :return:
+    """
+    # Ignore: entity already processed.
+    if not entity_declaration:
+        return False
+
+    entity_repeat = entity_declaration[const.PARAMETERS].get(EC.REPEAT, None)
+
+    # Ignore: no repeat declared for the entity.
+    if not entity_repeat:
+        return False
+
+    # Error: repeat not found while processing survey sheet.
+    if not stack:
+        raise PyXFormError(ENTITY002.format(value=entity_repeat))
+
+    control_name = stack[-1]["control_name"]
+    control_type = stack[-1]["control_type"]
+
+    # Ignore: current control is not the target.
+    if control_name and control_name != entity_repeat:
+        return False
+
+    # Error: target is not a repeat.
+    if control_type and control_type != const.REPEAT:
+        raise PyXFormError(ENTITY003.format(value=entity_repeat))
+
+    # Error: repeat is in nested repeat.
+    located = False
+    for i in reversed(stack):
+        if not i["control_name"] or not i["control_type"]:
+            break
+        elif i["control_type"] == const.REPEAT:
+            if located:
+                raise PyXFormError(ENTITY004.format(value=entity_repeat))
+            elif i["control_name"] == entity_repeat:
+                located = True
+
+    return entity_repeat == control_name

pyxform/errors.py

@@ -2,6 +2,41 @@
 Common base classes for pyxform exceptions.
 """
 
+from string import Formatter
+
+
+class _ErrorFormatter(Formatter):
+    """Allows specifying a default for missing format keys."""
+
+    def __init__(self, default_value: str = "unknown"):
+        self.default_value: str = default_value
+
+    def get_value(self, key, args, kwargs):
+        if isinstance(key, str):
+            value = kwargs.get(key, None)
+            if value is None:
+                return self.default_value
+            else:
+                return value
+        else:
+            return super().get_value(key, args, kwargs)
+
+
+_ERROR_FORMATTER = _ErrorFormatter()
+
+
+class Detail:
+    """ErrorCode details."""
+
+    __slots__ = ("msg", "name")
+
+    def __init__(self, name: str, msg: str) -> None:
+        self.name: str = name
+        self.msg: str = msg
+
+    def format(self, **kwargs):
+        return _ERROR_FORMATTER.format(self.msg, **kwargs)
+
 
 class PyXFormError(Exception):
     """Common base class for pyxform exceptions."""

pyxform/question_type_dictionary.py

@@ -2,21 +2,11 @@
 XForm survey question type mapping dictionary module.
 """
 
+from collections.abc import Sequence
 from types import MappingProxyType
+from typing import Any
 
-from pyxform.xls2json import QuestionTypesReader, print_pyobj_to_json
-
-
-def generate_new_dict():
-    """
-    This is just here incase there is ever any need to generate the question
-    type dictionary from all.xls again.
-    It shouldn't be called as part of any application.
-    """
-    path_to_question_types = "/pyxform/question_types/all.xls"
-    json_dict = QuestionTypesReader(path_to_question_types).to_json_dict()
-    print_pyobj_to_json(json_dict, "new_question_type_dict.json")
-
+from pyxform import constants as const
 
 _QUESTION_TYPE_DICT = {
     "q picture": {
@@ -392,3 +382,14 @@ def generate_new_dict():
 
 # Read-only view of the types.
 QUESTION_TYPE_DICT = MappingProxyType(_QUESTION_TYPE_DICT)
+
+
+def get_meta_group(children: Sequence[dict[str, Any]]) -> dict[str, Any]:
+    if children is None:
+        children = []
+    return {
+        const.NAME: "meta",
+        const.TYPE: const.GROUP,
+        const.CONTROL: {"bodyless": True},
+        const.CHILDREN: children,
+    }

pyxform/section.py

@@ -7,10 +7,10 @@
 from typing import TYPE_CHECKING
 
 from pyxform import constants
-from pyxform.errors import PyXFormError
 from pyxform.external_instance import ExternalInstance
 from pyxform.survey_element import SURVEY_ELEMENT_FIELDS, SurveyElement
 from pyxform.utils import DetachableElement, node
+from pyxform.validators.pyxform import unique_names
 
 if TYPE_CHECKING:
     from pyxform.question import Question
@@ -94,18 +94,18 @@ def iter_descendants(
                     iter_into_section_items=iter_into_section_items,
                 )
 
-    # there's a stronger test of this when creating the xpath
-    # dictionary for a survey.
     def _validate_uniqueness_of_element_names(self):
-        element_slugs = set()
+        child_names = set()
+        child_names_lower = set()
+        warnings = []
         for element in self.children:
-            elem_lower = element.name.lower()
-            if elem_lower in element_slugs:
-                raise PyXFormError(
-                    f"There are more than one survey elements named '{elem_lower}' "
-                    f"(case-insensitive) in the section named '{self.name}'."
-                )
-            element_slugs.add(elem_lower)
+            unique_names.validate_question_group_repeat_name(
+                name=element.name,
+                seen_names=child_names,
+                seen_names_lower=child_names_lower,
+                warnings=warnings,
+                check_reserved=False,
+            )
 
     def xml_instance(self, survey: "Survey", **kwargs):
         """

pyxform/survey.py

@@ -21,7 +21,7 @@
 from pyxform.parsing.expression import has_last_saved
 from pyxform.parsing.instance_expression import replace_with_output
 from pyxform.question import Itemset, MultipleChoiceQuestion, Option, Question, Tag
-from pyxform.section import SECTION_EXTRA_FIELDS, Section
+from pyxform.section import SECTION_EXTRA_FIELDS, RepeatingSection, Section
 from pyxform.survey_element import SURVEY_ELEMENT_FIELDS, SurveyElement
 from pyxform.utils import (
     BRACKETED_TAG_REGEX,
@@ -31,6 +31,7 @@
     node,
 )
 from pyxform.validators import enketo_validate, odk_validate
+from pyxform.validators.pyxform import unique_names
 from pyxform.validators.pyxform.iana_subtags.validation import get_languages_with_bad_tags
 
 RE_BRACKET = re.compile(r"\[([^]]+)\]")
@@ -292,21 +293,16 @@ def validate(self):
 
     def _validate_uniqueness_of_section_names(self):
         root_node_name = self.name
-        section_names = set()
-        for element in self.iter_descendants(condition=lambda i: isinstance(i, Section)):
-            if element.name in section_names:
-                if element.name == root_node_name:
-                    # The root node name is rarely explictly set; explain
-                    # the problem in a more helpful way (#510)
-                    msg = (
-                        f"The name '{element.name}' is the same as the form name. "
-                        "Use a different section name (or change the form name in "
-                        "the 'name' column of the settings sheet)."
-                    )
-                    raise PyXFormError(msg)
-                msg = f"There are two sections with the name {element.name}."
-                raise PyXFormError(msg)
-            section_names.add(element.name)
+        repeat_names = set()
+        for element in self.iter_descendants(
+            condition=lambda i: isinstance(i, RepeatingSection)
+        ):
+            unique_names.validate_repeat_name(
+                name=element.name,
+                control_type=constants.REPEAT,
+                instance_element_name=root_node_name,
+                seen_names=repeat_names,
+            )
 
     def get_nsmap(self):
         """Add additional namespaces"""