add: allow entities to be declared for a repeat

- as much processing as possible in entities_parsing.py
- most tests about entity repeat position are in test_create_repeat.py
  - tests specific to update-related fields in test_update_repeat.py
- add more entities xpath helpers

Author: lindsay-stevens

pyxform/entities/entities_parsing.py

@@ -2,10 +2,56 @@
 from typing import Any
 
 from pyxform import constants as const
-from pyxform.errors import PyXFormError
+from pyxform.errors import Detail, PyXFormError
 from pyxform.parsing.expression import is_xml_tag
+from pyxform.utils import PYXFORM_REFERENCE_REGEX
 
 EC = const.EntityColumns
+ENTITY001 = Detail(
+    name="Invalid entity repeat reference",
+    msg=(
+        "[row : 2] On the 'entities' sheet, the 'repeat' value '{value}' is invalid. "
+        "The 'repeat' column, if specified, must contain only a single reference variable "
+        "(like '${{q1}}'), and the reference variable must contain a valid name."
+    ),
+)
+ENTITY002 = Detail(
+    name="Invalid entity repeat: target not found",
+    msg=(
+        "[row : 2] On the 'entities' sheet, the 'repeat' value '{value}' is invalid. "
+        "The entity repeat target was not found in the 'survey' sheet."
+    ),
+)
+ENTITY003 = Detail(
+    name="Invalid entity repeat: target is not a repeat",
+    msg=(
+        "[row : 2] On the 'entities' sheet, the 'repeat' value '{value}' is invalid. "
+        "The entity repeat target is not a repeat."
+    ),
+)
+ENTITY004 = Detail(
+    name="Invalid entity repeat: target is in a repeat",
+    msg=(
+        "[row : 2] On the 'entities' sheet, the 'repeat' value '{value}' is invalid. "
+        "The entity repeat target is inside a repeat."
+    ),
+)
+ENTITY005 = Detail(
+    name="Invalid entity repeat save_to: question in nested repeat",
+    msg=(
+        "[row : {row}] On the 'survey' sheet, the 'save_to' value '{value}' is invalid. "
+        "The entity properties populated with 'save_to' must not be inside of a nested "
+        "repeat within the entity repeat."
+    ),
+)
+ENTITY006 = Detail(
+    name="Invalid entity repeat save_to: question not in entity repeat",
+    msg=(
+        "[row : {row}] On the 'survey' sheet, the 'save_to' value '{value}' is invalid. "
+        "The entity properties populated with 'save_to' must be inside of the entity "
+        "repeat."
+    ),
+)
 
 
 def get_entity_declaration(
@@ -24,6 +70,7 @@ def get_entity_declaration(
     create_condition = entity_row.get(EC.CREATE_IF, None)
     update_condition = entity_row.get(EC.UPDATE_IF, None)
     entity_label = entity_row.get(EC.LABEL, None)
+    entity_repeat = get_validated_repeat_name(entity_row)
 
     if update_condition and not entity_id:
         raise PyXFormError(
@@ -49,6 +96,7 @@ def get_entity_declaration(
             EC.CREATE_IF.value: create_condition,
             EC.UPDATE_IF.value: update_condition,
             EC.LABEL.value: entity_label,
+            EC.REPEAT.value: entity_repeat,
         },
     }
 
@@ -77,11 +125,29 @@ def get_validated_dataset_name(entity):
     return dataset
 
 
+def get_validated_repeat_name(entity) -> str | None:
+    if EC.REPEAT.value not in entity:
+        return None
+
+    value = entity[EC.REPEAT]
+    if not (value and len(value) > 3):
+        raise PyXFormError(ENTITY001.format(value=value))
+    matches = PYXFORM_REFERENCE_REGEX.fullmatch(value)
+    if matches is None:
+        raise PyXFormError(ENTITY001.format(value=value))
+    else:
+        match = matches.group(1)
+        if not is_xml_tag(match):
+            raise PyXFormError(ENTITY001.format(value=value))
+        else:
+            return match
+
+
 def validate_entity_saveto(
     row: dict,
     row_number: int,
-    in_repeat: bool,
     entity_declaration: dict[str, Any] | None = None,
+    stack: Sequence[dict[str, Any]] | None = None,
 ):
     save_to = row.get(const.BIND, {}).get("entities:saveto", "")
     if not save_to:
@@ -97,10 +163,24 @@ def validate_entity_saveto(
             f"{const.ROW_FORMAT_STRING % row_number} Groups and repeats can't be saved as entity properties."
         )
 
-    if in_repeat:
-        raise PyXFormError(
-            f"{const.ROW_FORMAT_STRING % row_number} Currently, you can't create entities from repeats. You may only specify save_to values for form fields outside of repeats."
-        )
+    entity_repeat = entity_declaration[const.PARAMETERS].get(EC.REPEAT, None)
+    if entity_repeat and stack:
+        # Error: saveto in nested repeat inside entity repeat.
+        in_repeat = False
+        located = False
+        for i in reversed(stack):
+            if not i["control_name"] or not i["control_type"]:
+                break
+            elif i["control_type"] == const.REPEAT:
+                if in_repeat:
+                    raise PyXFormError(ENTITY005.format(row=row_number, value=save_to))
+                elif i["control_name"] == entity_repeat:
+                    located = True
+                in_repeat = True
+
+        # Error: saveto not in entity repeat
+        if not located:
+            raise PyXFormError(ENTITY006.format(row=row_number, value=save_to))
 
     error_start = f"{const.ROW_FORMAT_STRING % row_number} Invalid save_to name:"
 
@@ -134,3 +214,56 @@ def validate_entities_columns(row: dict):
             f"pyxform."
         )
         raise PyXFormError(msg)
+
+
+def validate_entity_repeat_target(
+    entity_declaration: dict[str, Any] | None,
+    stack: Sequence[dict[str, Any]] | None = None,
+) -> bool:
+    """
+    Check if the entity repeat target exists, is a repeat, and is a name match.
+
+    Raises an error if the control type or name is None (such as for the Survey), or if
+    the control type is not a repeat.
+
+    :param entity_declaration:
+    :param stack: The control stack from workbook_to_json.
+    :return:
+    """
+    # Ignore: entity already processed.
+    if not entity_declaration:
+        return False
+
+    entity_repeat = entity_declaration[const.PARAMETERS].get(EC.REPEAT, None)
+
+    # Ignore: no repeat declared for the entity.
+    if not entity_repeat:
+        return False
+
+    # Error: repeat not found while processing survey sheet.
+    if not stack:
+        raise PyXFormError(ENTITY002.format(value=entity_repeat))
+
+    control_name = stack[-1]["control_name"]
+    control_type = stack[-1]["control_type"]
+
+    # Ignore: current control is not the target.
+    if control_name and control_name != entity_repeat:
+        return False
+
+    # Error: target is not a repeat.
+    if control_type and control_type != const.REPEAT:
+        raise PyXFormError(ENTITY003.format(value=entity_repeat))
+
+    # Error: repeat is in nested repeat.
+    located = False
+    for i in reversed(stack):
+        if not i["control_name"] or not i["control_type"]:
+            break
+        elif i["control_type"] == const.REPEAT:
+            if located:
+                raise PyXFormError(ENTITY004.format(value=entity_repeat))
+            elif i["control_name"] == entity_repeat:
+                located = True
+
+    return entity_repeat == control_name

pyxform/question_type_dictionary.py

@@ -2,7 +2,11 @@
 XForm survey question type mapping dictionary module.
 """
 
+from collections.abc import Sequence
 from types import MappingProxyType
+from typing import Any
+
+from pyxform import constants as const
 
 _QUESTION_TYPE_DICT = {
     "q picture": {
@@ -378,3 +382,14 @@
 
 # Read-only view of the types.
 QUESTION_TYPE_DICT = MappingProxyType(_QUESTION_TYPE_DICT)
+
+
+def get_meta_group(children: Sequence[dict[str, Any]]) -> dict[str, Any]:
+    if children is None:
+        children = []
+    return {
+        const.NAME: "meta",
+        const.TYPE: const.GROUP,
+        const.CONTROL: {"bodyless": True},
+        const.CHILDREN: children,
+    }

pyxform/xls2json.py

@@ -17,11 +17,13 @@
 )
 from pyxform.entities.entities_parsing import (
     get_entity_declaration,
+    validate_entity_repeat_target,
     validate_entity_saveto,
 )
 from pyxform.errors import PyXFormError
 from pyxform.parsing.expression import is_pyxform_reference, is_xml_tag
 from pyxform.parsing.sheet_headers import dealias_and_group_headers
+from pyxform.question_type_dictionary import get_meta_group
 from pyxform.utils import (
     PYXFORM_REFERENCE_REGEX,
     coalesce,
@@ -781,6 +783,15 @@ def workbook_to_json(
         if end_control_parse:
             parse_dict = end_control_parse.groupdict()
             if parse_dict.get("end") and "type" in parse_dict:
+                if validate_entity_repeat_target(
+                    entity_declaration=entity_declaration,
+                    stack=stack,
+                ):
+                    parent_children_array.append(
+                        get_meta_group(children=[entity_declaration])
+                    )
+                    json_dict[constants.ENTITY_FEATURES] = ["create", "update", "offline"]
+                    entity_declaration = None
                 control_type = aliases.control[parse_dict["type"]]
                 if prev_control_type != control_type or len(stack) == 1:
                     raise PyXFormError(
@@ -817,11 +828,12 @@ def workbook_to_json(
             seen_names_lower=child_names_lower,
             warnings=warnings,
         )
-
-        in_repeat = any(
-            ancestor["control_type"] == constants.REPEAT for ancestor in stack
+        validate_entity_saveto(
+            row=row,
+            row_number=row_number,
+            entity_declaration=entity_declaration,
+            stack=stack,
         )
-        validate_entity_saveto(row, row_number, in_repeat, entity_declaration)
 
         # Try to parse question as begin control statement
         # (i.e. begin loop/repeat/group):
@@ -961,6 +973,10 @@ def workbook_to_json(
                         "row_number": row_number,
                     }
                 )
+                validate_entity_repeat_target(
+                    stack=stack,
+                    entity_declaration=entity_declaration,
+                )
                 continue
 
         # Assuming a question is anything not processed above as a loop/repeat/group.
@@ -1428,16 +1444,12 @@ def workbook_to_json(
         )
 
     if entity_declaration:
+        validate_entity_repeat_target(entity_declaration=entity_declaration)
         json_dict[constants.ENTITY_FEATURES] = ["create", "update", "offline"]
         meta_children.append(entity_declaration)
 
     if len(meta_children) > 0:
-        meta_element = {
-            "name": "meta",
-            "type": "group",
-            "control": {"bodyless": True},
-            "children": meta_children,
-        }
+        meta_element = get_meta_group(children=meta_children)
         survey_children_array = stack[0]["parent_children"]
         survey_children_array.append(meta_element)