From 61f18041fbdbb671205df781f4f35f6a2b715a9f Mon Sep 17 00:00:00 2001
From: olabusayoT <50379531+olabusayoT@users.noreply.github.com>
Date: Thu, 21 Mar 2024 16:22:03 -0400
Subject: [PATCH] !fixup Enable dfdl:extraEscapedCharcaters to take an
 expression

- implement Ev val and class for extraEscapedCharacters similar to  escapeCharacter and escapeEscapeCharacter
- add optimization to DynamicEscapeSchemeCombinatorElement adding isConstant to checks
- fix bug in hasDynamicEscapeScheme by adding consideration for escapeSchemeUnparseEv
- add optionExtraEscapedCharsEv to localElementPropertyReferencedElements
- add tests for literal and expression examples
- refactor extraEscaoedCgarCooked to evalAndConvertExtraEscapedCharacters to more closely match other Evs
- add test for escapeBlocks and literal and expressioned extraEscapeChars

DAFFODIL-2876
---
 .../core/dsom/RuntimePropertyMixins.scala     | 11 ++-
 .../grammar/ElementBaseGrammarMixin.scala     |  2 +-
 .../DelimiterAndEscapeRelated.scala           | 10 +--
 .../runtime1/processors/EvEscapeSchemes.scala | 18 +++--
 .../section07/escapeScheme/escapeScheme.tdml  | 67 +++++++++++++++++++
 .../escapeScheme/TestEscapeScheme.scala       |  2 +
 6 files changed, 97 insertions(+), 13 deletions(-)

diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/RuntimePropertyMixins.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/RuntimePropertyMixins.scala
index 82ad4a720e..e04bca9c51 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/RuntimePropertyMixins.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/dsom/RuntimePropertyMixins.scala
@@ -637,7 +637,16 @@ trait ElementRuntimeValuedPropertiesMixin
             f(es.optionEscapeEscapeCharacterEv.get)
           else
             ReferencedElementInfos.None
-        ee ++ propExprElts(es.optionEscapeCharacterRaw, es.escapeCharacterEv, f)
+        val extraEscapeChars =
+          if (es.optionExtraEscapedCharactersEv.isDefined)
+            f(es.optionExtraEscapedCharactersEv.get)
+          else
+            ReferencedElementInfos.None
+        ee ++ extraEscapeChars ++ propExprElts(
+          es.optionEscapeCharacterRaw,
+          es.escapeCharacterEv,
+          f,
+        )
       } else {
         ReferencedElementInfos.None
       }
diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala
index f34f1e1bd9..f94daf1536 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala
@@ -1409,7 +1409,7 @@ trait ElementBaseGrammarMixin
   }
 
   private lazy val hasDynamicEscapeScheme =
-    this.optionEscapeScheme.isDefined && !this.optionEscapeScheme.get.escapeSchemeParseEv.isConstant
+    this.optionEscapeScheme.isDefined && (!this.optionEscapeScheme.get.escapeSchemeParseEv.isConstant || !this.optionEscapeScheme.get.escapeSchemeUnparseEv.isConstant)
 
   private def withEscapeScheme(body: Gram) = {
     if (hasDynamicEscapeScheme) DynamicEscapeSchemeCombinatorElement(this, body)
diff --git a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/DelimiterAndEscapeRelated.scala b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/DelimiterAndEscapeRelated.scala
index 88036ff308..6c0fe81edf 100644
--- a/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/DelimiterAndEscapeRelated.scala
+++ b/daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/primitives/DelimiterAndEscapeRelated.scala
@@ -120,18 +120,20 @@ case class DynamicEscapeSchemeCombinatorElement(e: ElementBase, body: Gram)
   val schemeParseOpt = e.optionEscapeScheme.map { _.escapeSchemeParseEv }
   val schemeUnparseOpt = e.optionEscapeScheme.map { _.escapeSchemeUnparseEv }
 
-  Assert.invariant(schemeParseOpt.isDefined && !schemeParseOpt.get.isConstant)
-  Assert.invariant(schemeUnparseOpt.isDefined && !schemeUnparseOpt.get.isConstant)
+  val schemeParseIsConstant = schemeParseOpt.map(_.isConstant).getOrElse(true)
+  val schemeUnparseIsConstant = schemeUnparseOpt.map(_.isConstant).getOrElse(true)
+
+  Assert.invariant(!schemeParseIsConstant || !schemeUnparseIsConstant)
 
   lazy val parser: DaffodilParser = {
     val p = body.parser
-    if (p.isEmpty) p
+    if (p.isEmpty || schemeParseIsConstant) p
     else new DynamicEscapeSchemeParser(schemeParseOpt.get, e.termRuntimeData, p)
   }
 
   override lazy val unparser: DaffodilUnparser = {
     val u = body.unparser
-    if (u.isEmpty) u
+    if (u.isEmpty || schemeUnparseIsConstant) u
     else new DynamicEscapeSchemeUnparser(schemeUnparseOpt.get, e.termRuntimeData, u)
   }
 }
diff --git a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/EvEscapeSchemes.scala b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/EvEscapeSchemes.scala
index 2d8ded33b8..a4bdb885b3 100644
--- a/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/EvEscapeSchemes.scala
+++ b/daffodil-runtime1/src/main/scala/org/apache/daffodil/runtime1/processors/EvEscapeSchemes.scala
@@ -85,11 +85,12 @@ abstract class EscapeSchemeUnparseEv(ci: DPathCompileInfo)
 
   def extraEscapedChars: Maybe[ExtraEscapedCharsEv]
 
-  def extraEscapedCharsCooked(state: State) = {
+  def evalAndConvertExtraEscapedCharacters(state: ParseOrUnparseState): Seq[Char] = {
     if (extraEscapedChars.isDefined) {
-      extraEscapedChars.get
+      val extEscChar = extraEscapedChars.get
         .evaluate(state)
-        .map { _.charAt(0) }
+        .map(_.charAt(0))
+      extEscChar
     } else {
       Seq()
     }
@@ -118,15 +119,17 @@ class EscapeSchemeCharUnparseEv(
   ci: DPathCompileInfo,
 ) extends EscapeSchemeUnparseEv(ci) {
 
-  override val runtimeDependencies = Vector(escapeChar) ++ optEscapeEscapeChar.toList
+  override val runtimeDependencies =
+    Vector(escapeChar) ++ optEscapeEscapeChar.toList ++ extraEscapedChars.toList
 
   def compute(state: ParseOrUnparseState) = {
     val escChar = escapeChar.evaluate(state).charAt(0)
     val optEscEscChar = evalAndConvertEEC(state)
+    val extEscChars = evalAndConvertExtraEscapedCharacters(state)
     new EscapeSchemeCharUnparserHelper(
       escChar,
       optEscEscChar,
-      extraEscapedCharsCooked(state),
+      extEscChars,
       ci,
     )
   }
@@ -159,18 +162,19 @@ class EscapeSchemeBlockUnparseEv(
   ci: DPathCompileInfo,
 ) extends EscapeSchemeUnparseEv(ci) {
 
-  override val runtimeDependencies = optEscapeEscapeChar.toList
+  override val runtimeDependencies = optEscapeEscapeChar.toList ++ extraEscapedChars.toList
 
   val bs = EscapeBlockStartCooker.convertConstant(blockStart, ci, forUnparse = true)
   val be = EscapeBlockEndCooker.convertConstant(blockEnd, ci, forUnparse = true)
 
   def compute(state: ParseOrUnparseState) = {
     val optEscEscChar = evalAndConvertEEC(state)
+    val extEscChars = evalAndConvertExtraEscapedCharacters(state)
     new EscapeSchemeBlockUnparserHelper(
       optEscEscChar,
       bs,
       be,
-      extraEscapedCharsCooked(state),
+      extEscChars,
       generateEscapeBlock,
       ci,
     )
diff --git a/daffodil-test/src/test/resources/org/apache/daffodil/section07/escapeScheme/escapeScheme.tdml b/daffodil-test/src/test/resources/org/apache/daffodil/section07/escapeScheme/escapeScheme.tdml
index 9257577960..54c7bf6b9e 100644
--- a/daffodil-test/src/test/resources/org/apache/daffodil/section07/escapeScheme/escapeScheme.tdml
+++ b/daffodil-test/src/test/resources/org/apache/daffodil/section07/escapeScheme/escapeScheme.tdml
@@ -680,6 +680,18 @@
         escapeEscapeCharacter=")"  extraEscapedCharacters="" generateEscapeBlock="whenNeeded"/>
     </dfdl:defineEscapeScheme>
 
+    <dfdl:defineEscapeScheme name="eBlkEndParen_eecExp">
+      <dfdl:escapeScheme escapeBlockStart="("
+        escapeBlockEnd=")" escapeKind="escapeBlock"
+        escapeEscapeCharacter=")" extraEscapedCharacters="{../tns:extraEscapedCharacters }" generateEscapeBlock="whenNeeded"/>
+    </dfdl:defineEscapeScheme>
+
+    <dfdl:defineEscapeScheme name="eBlkEndParen_eecLit">
+      <dfdl:escapeScheme escapeBlockStart="("
+        escapeBlockEnd=")" escapeKind="escapeBlock"
+        escapeEscapeCharacter=")" extraEscapedCharacters="? *" generateEscapeBlock="whenNeeded"/>
+    </dfdl:defineEscapeScheme>
+
     <xs:element name="record">
       <xs:complexType>
         <xs:sequence dfdl:separator="," >
@@ -688,6 +700,27 @@
         </xs:sequence>
       </xs:complexType>
     </xs:element>
+
+    <xs:element name="record2">
+      <xs:complexType>
+        <xs:sequence dfdl:separator="," >
+          <xs:element name="extraEscapedCharacters" type="xs:string"/>
+          <xs:element name="item" type="xs:string" dfdl:lengthKind="delimited" maxOccurs="5"
+	    dfdl:escapeSchemeRef="tns:eBlkEndParen_eecExp" />
+        </xs:sequence>
+      </xs:complexType>
+    </xs:element>
+
+    <xs:element name="record3">
+      <xs:complexType>
+        <xs:sequence dfdl:separator="," >
+          <xs:element name="extraEscapedCharacters" type="xs:string"/>
+          <xs:element name="item" type="xs:string" dfdl:lengthKind="delimited" maxOccurs="5"
+	    dfdl:escapeSchemeRef="tns:eBlkEndParen_eecLit" />
+        </xs:sequence>
+      </xs:complexType>
+    </xs:element>
+
   </defineSchema>
  
   <parserTestCase name="escBlkEndSame" model="eBlkEndSame"
@@ -706,6 +739,40 @@
     </infoset>
   </parserTestCase>
 
+  <unparserTestCase name="escBlkEndSame2" model="eBlkEndSame"
+    description="Section 7 defineEscapeScheme - escapeBlock with expression extraEscapedCharacters" root="record2" roundTrip="true">
+    <document>? *,1,(Column Number? Two*),3,4,5</document>
+    <infoset>
+      <dfdlInfoset>
+    	<tns:record2>
+    	  <tns:extraEscapedCharacters>? *</tns:extraEscapedCharacters>
+    	  <tns:item>1</tns:item>
+    	  <tns:item>Column Number? Two*</tns:item>
+    	  <tns:item>3</tns:item>
+    	  <tns:item>4</tns:item>
+    	  <tns:item>5</tns:item>
+  	    </tns:record2>
+      </dfdlInfoset>
+    </infoset>
+  </unparserTestCase>
+
+  <unparserTestCase name="escBlkEndSame3" model="eBlkEndSame"
+    description="Section 7 defineEscapeScheme - escapeBlock with literal extraEscapedCharacters" root="record3" roundTrip="true">
+    <document>? *,1,(Column Number? Two*),3,4,5</document>
+    <infoset>
+      <dfdlInfoset>
+    	<tns:record3>
+    	  <tns:extraEscapedCharacters>? *</tns:extraEscapedCharacters>
+    	  <tns:item>1</tns:item>
+    	  <tns:item>Column Number? Two*</tns:item>
+    	  <tns:item>3</tns:item>
+    	  <tns:item>4</tns:item>
+    	  <tns:item>5</tns:item>
+  	    </tns:record3>
+      </dfdlInfoset>
+    </infoset>
+  </unparserTestCase>
+
   <!--
        Test Name: escBlkMultipleEEC
           Schema: eBlkMultipleEEC
diff --git a/daffodil-test/src/test/scala/org/apache/daffodil/section07/escapeScheme/TestEscapeScheme.scala b/daffodil-test/src/test/scala/org/apache/daffodil/section07/escapeScheme/TestEscapeScheme.scala
index f2d52cec64..28ee31deaa 100644
--- a/daffodil-test/src/test/scala/org/apache/daffodil/section07/escapeScheme/TestEscapeScheme.scala
+++ b/daffodil-test/src/test/scala/org/apache/daffodil/section07/escapeScheme/TestEscapeScheme.scala
@@ -100,6 +100,8 @@ class TestEscapeScheme {
 
   @Test def test_escBlkAllQuotes(): Unit = { runner.runOneTest("escBlkAllQuotes") }
   @Test def test_escBlkEndSame(): Unit = { runner.runOneTest("escBlkEndSame") }
+  @Test def test_escBlkEndSame2(): Unit = { runner.runOneTest("escBlkEndSame2") }
+  @Test def test_escBlkEndSame3(): Unit = { runner.runOneTest("escBlkEndSame3") }
   // @Test def test_escBlkMultipleEEC() { runner.runOneTest("escBlkMultipleEEC") } // DAFFODIL-1972
 
   @Test def test_escapeScheme_with_comment(): Unit = {