Skip to content

Commit

Permalink
Add support for lengthUnit=bits for any type
Browse files Browse the repository at this point in the history 
- we currently report a warning when we use lengthUnit=bits with certain types ex date/time types. Per the discussions on dev (https://lists.apache.org/thread/01sn3fk2wk7jtos2svw4995f7dxxt7g1), we want to open up use of lengthUnit=bits to all types.
- remove no-op checkLengthUnits function
- allow lengthUnits bits for binarySeconds/milliSeconds for xs:datetime
- remove SDE for requirement that lengthUnit=bytes for binarySeconds/milliSeconds
- verify test that uses lengthUnit=bits with calendar type no longer produces warning
- verify test that uses lengthUnit=bits with integer type no longer produces warning
- verify test that uses lengthUnit=bits with decimal type no longer produces warning
- add tests for xs:double/xs:float verifying LU=bits
- add tests for double/float showing runtime lengths not allowed
- add tests for xs:dateTime with binaryCalendarRep set to binarySeconds/milliSeconds
- add test for xs:date with binaryCalendarRep set to binarySeconds/milliSeconds (SDEs)
- deprecate allowBigIntegerBits tunable and the DeprecatedBigIntegerBits WarnID

Deprecation/Compatibility
- allowBigIntegerBits tunable no longer has any effect. All types are allowed to use lengthUnit=bits.
- WarnID DeprecatedBigIntegerBits has been deprecated. It is no longer in use in the codebase.

DAFFODIL-2931
  • Loading branch information
@olabusayoT
olabusayoT committed Oct 28, 2024
1 parent 3096dea commit 36f2d87
Show file tree
Hide file tree
Showing 8 changed files with 280 additions and 101 deletions.
44 changes: 2 additions & 42 deletions daffodil-core/src/main/scala/org/apache/daffodil/core/grammar/ElementBaseGrammarMixin.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,6 @@ trait ElementBaseGrammarMixin
lazy val prefixedLengthElementDecl: PrefixLengthQuasiElementDecl =
LV('prefixedLengthElementDecl) {
Assert.invariant(lengthKind == LengthKind.Prefixed)
checkLengthUnits()
val detachedNode =
<element name={name + " (prefixLength)"} type={prefixLengthType.toQNameString}/>
.copy(scope = prefixLengthTypeGSTD.xml.scope)
Expand Down Expand Up @@ -591,7 +590,6 @@ trait ElementBaseGrammarMixin
val lengthFromProp: JLong = repElement.lengthEv.optConstant.get
val nbits = repElement.lengthUnits match {
case LengthUnits.Bits =>
checkLengthUnits(repElement)
lengthFromProp.longValue()
case LengthUnits.Bytes => lengthFromProp.longValue() * 8
case LengthUnits.Characters =>
Expand Down Expand Up @@ -1069,10 +1067,8 @@ trait ElementBaseGrammarMixin
(primType, binaryCalendarRep) match {
case (PrimType.DateTime, BinaryCalendarRep.BinarySeconds) =>
(lengthUnits, binaryNumberKnownLengthInBits) match {
case (LengthUnits.Bytes, 32) =>
case (LengthUnits.Bytes | LengthUnits.Bits, 32) =>
new ConvertBinaryDateTimeSecMilliPrim(this, binaryNumberKnownLengthInBits)
case (_, 32) =>
SDE("lengthUnits must be 'bytes' when binaryCalendarRep='binarySeconds'")
case (_, n) =>
SDE(
"binary xs:dateTime must be 32 bits when binaryCalendarRep='binarySeconds'. Length in bits was %s.",
Expand All @@ -1083,10 +1079,8 @@ trait ElementBaseGrammarMixin
SDE("binaryCalendarRep='binarySeconds' is not allowed with type %s", primType.name)
case (PrimType.DateTime, BinaryCalendarRep.BinaryMilliseconds) =>
(lengthUnits, binaryNumberKnownLengthInBits) match {
case (LengthUnits.Bytes, 64) =>
case (LengthUnits.Bytes | LengthUnits.Bits, 64) =>
new ConvertBinaryDateTimeSecMilliPrim(this, binaryNumberKnownLengthInBits)
case (_, 64) =>
SDE("lengthUnits must be 'bytes' when binaryCalendarRep='binaryMilliseconds'")
case (_, n) =>
SDE(
"binary xs:dateTime must be 64 bits when binaryCalendarRep='binaryMilliseconds'. Length in bits was %s.",
Expand Down Expand Up @@ -1713,38 +1707,4 @@ trait ElementBaseGrammarMixin
prod("mandatoryTextAlignment", impliedRepresentation eq Representation.Text) {
mtaBase
}

val allowedBitTypes = Set[PrimType](
PrimType.Boolean,
PrimType.Byte,
PrimType.Short,
PrimType.Int,
PrimType.Long,
PrimType.UnsignedByte,
PrimType.UnsignedShort,
PrimType.UnsignedInt,
PrimType.UnsignedLong
)
val allowedBitTypesText = allowedBitTypes.map("xs:" + _.toString).toList.sorted.mkString(", ")

private def checkLengthUnits(elem: ElementBase = context): Unit = {
elem.lengthUnits match {
case LengthUnits.Bits if elem.representation == Representation.Binary =>
elem.optPrimType match {
case Some(primType) =>
if (!allowedBitTypes.contains(primType))
if (tunable.allowBigIntegerBits)
elem.SDW(
WarnID.DeprecatedBigIntegerBits,
s"In a future release, lengthUnits='bits' will only be supported for the following types: $allowedBitTypesText"
)
else
elem.SDE(
"lengthUnits='bits' is only supported for the following types: $allowedBitTypesText"
)
case None =>
}
case _ =>
}
}
}
15 changes: 8 additions & 7 deletions daffodil-propgen/src/main/resources/org/apache/daffodil/xsd/dafext.xsd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,12 +104,8 @@
<xs:element name="allowBigIntegerBits" type="xs:boolean" default="true" minOccurs="0">
<xs:annotation>
<xs:documentation>
Previous Daffodil releases let schemas define every type's length using "bits" as the length unit
even though the specification allows bit length units only for a specific set of types' binary
representations and does not allow bit length units for any other type's binary representation
or any type's text representation. When this tunable is true, a deprecation warning is issued
when bit length units are incorrectly used. When this tunable is false, a schema definition
error will be issued instead.
Deprecated. This tunable no longer has any affect and is only kept for
backwards compatability.
</xs:documentation>
</xs:annotation>
</xs:element>
Expand Down Expand Up @@ -703,7 +699,12 @@
<xs:enumeration value="appinfoNoSource" />
<xs:enumeration value="choiceInsideHiddenGroup" />
<xs:enumeration value="codeGenerator" />
<xs:enumeration value="deprecatedBigIntegerBits" />
<!-- deprecated -->
<xs:enumeration value="deprecatedBigIntegerBits">
<xs:annotation>
<xs:documentation>Deprecated.</xs:documentation>
</xs:annotation>
</xs:enumeration>
<xs:enumeration value="deprecatedBuiltInFormats" />
<xs:enumeration value="deprecatedEncodingNameUSASCII7BitPacked" />
<xs:enumeration value="deprecatedExpressionResultCoercion" />
Expand Down
73 changes: 70 additions & 3 deletions daffodil-test/src/test/resources/org/apache/daffodil/section05/simple_types/SimpleTypes.tdml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,12 @@
dfdl:binaryCalendarRep="binarySeconds" dfdl:binaryCalendarEpoch="01-01-2000T00:00:00"/>
<xs:element name="dateTimeBin12" type="xs:dateTime" dfdl:lengthKind="delimited" dfdl:terminator=";"
dfdl:binaryCalendarRep="binarySeconds" dfdl:binaryCalendarEpoch="1977-01-01T00:00:07"/>
<xs:element name="dateTimeBin13" type="xs:dateTime" dfdl:lengthKind="explicit" dfdl:length="{ 32 }"
dfdl:lengthUnits="bits" dfdl:binaryCalendarRep="binarySeconds" dfdl:binaryCalendarEpoch="2018-01-01T09:13:42+09:00"/>
<xs:element name="dateTimeBin14" type="xs:dateTime" dfdl:lengthKind="explicit" dfdl:length="{ 64 }"
dfdl:lengthUnits="bits" dfdl:binaryCalendarRep="binaryMilliseconds" dfdl:binaryCalendarEpoch="2000-06-15T03:25:19"/>
<xs:element name="dateTimeBin15" type="xs:date" dfdl:lengthKind="explicit" dfdl:length="{ 64 }"
dfdl:lengthUnits="bits" dfdl:binaryCalendarRep="binaryMilliseconds" dfdl:binaryCalendarEpoch="2000-06-15T03:25:19"/>

<xs:element name="dateBinBCD" type="xs:date" dfdl:calendarPattern="MMddyy" dfdl:calendarPatternKind="explicit"
dfdl:lengthKind="explicit" dfdl:length="{ 3 }" dfdl:lengthUnits="bytes" dfdl:binaryCalendarRep="bcd" />
Expand Down Expand Up @@ -338,6 +344,9 @@
dfdl:lengthKind="explicit" dfdl:length="{ 3 }" dfdl:lengthUnits="bytes" dfdl:binaryCalendarRep="ibm4690Packed" />
<xs:element name="dateBinIBM4690Packed2" type="xs:date" dfdl:calendarPattern="MMddyy" dfdl:calendarPatternKind="explicit"
dfdl:lengthKind="explicit" dfdl:length="{ 18 }" dfdl:lengthUnits="bits" dfdl:binaryCalendarRep="ibm4690Packed"/>
<xs:element name="dateBinIBM4690Packed3" type="xs:date" dfdl:calendarPattern="MMddyy" dfdl:calendarPatternKind="explicit"
dfdl:lengthKind="explicit" dfdl:length="{ 24 }" dfdl:lengthUnits="bits" dfdl:binaryCalendarRep="ibm4690Packed" />


<xs:element name="dateTimeBinIBM4690Packed">
<xs:complexType>
Expand Down Expand Up @@ -2410,9 +2419,11 @@
<tdml:document>
<tdml:documentPart type="bits">00000000 00000000 00000000 00111101</tdml:documentPart>
</tdml:document>
<tdml:errors>
<tdml:error>Schema Definition Error: lengthUnits must be 'bytes' when binaryCalendarRep='binarySeconds'</tdml:error>
</tdml:errors>
<tdml:infoset>
<tdml:dfdlInfoset>
<ex:dateTimeBin2 xmlns:ex="http://example.com">1970-01-01T00:01:01</ex:dateTimeBin2>
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:parserTestCase>

<tdml:parserTestCase name="dateTimeBin3" root="dateTimeBin3"
Expand Down Expand Up @@ -2697,6 +2708,48 @@
</tdml:errors>
</tdml:parserTestCase>

<tdml:parserTestCase name="dateTimeBin23" root="dateTimeBin13"
model="SimpleTypes-binary" description="binarySeconds with LengthUnits=bits"
roundTrip="true">

<tdml:document>
<tdml:documentPart type="bits">00000000 00000000 00000000 00000001</tdml:documentPart>
</tdml:document>
<tdml:infoset>
<tdml:dfdlInfoset>
<dateTimeBin13>2018-01-01T09:13:43+09:00</dateTimeBin13>
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:parserTestCase>

<tdml:parserTestCase name="dateTimeBin24" root="dateTimeBin14"
model="SimpleTypes-binary" description="binaryMilliSeconds with lengthUnits=bits"
roundTrip="true">

<tdml:document>
<tdml:documentPart type="bits">00000000 00000000 00000000 00000000 11111111 11111111 11111111 11111111</tdml:documentPart>
</tdml:document>
<tdml:infoset>
<tdml:dfdlInfoset>
<dateTimeBin14>2000-08-03T20:28:06.295000</dateTimeBin14>
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:parserTestCase>

<tdml:parserTestCase name="dateTimeBin25" root="dateTimeBin15"
model="SimpleTypes-binary" description="binaryMilliSeconds with lengthUnits=bits"
roundTrip="true">

<tdml:document>
<tdml:documentPart type="bits">00000000 00000000 00000000 00000000 11111111 11111111 11111111 11111111</tdml:documentPart>
</tdml:document>
<tdml:errors>
<tdml:error>binaryCalendarRep='binaryMilliseconds'</tdml:error>
<tdml:error>not allowed</tdml:error>
<tdml:error>type date</tdml:error>
</tdml:errors>
</tdml:parserTestCase>

<tdml:parserTestCase name="dateBinBCD" root="dateBinBCD"
model="SimpleTypes-binary" description="Section 5 Schema types-dateTime - DFDL-5-016R"
roundTrip="true">
Expand Down Expand Up @@ -2992,6 +3045,20 @@
</tdml:errors>
</tdml:parserTestCase>

<tdml:parserTestCase name="dateBinIBM4690Packed4" root="dateBinIBM4690Packed3"
model="SimpleTypes-binary" description="binary packed calendar representation with lengthUnit=bits"
roundTrip="true" ignoreUnexpectedWarnings="false">

<tdml:document>
<tdml:documentPart type="byte">12 14 23</tdml:documentPart>
</tdml:document>
<tdml:infoset>
<tdml:dfdlInfoset>
<dateBinIBM4690Packed3>2023-12-14</dateBinIBM4690Packed3>
</tdml:dfdlInfoset>
</tdml:infoset>
</tdml:parserTestCase>

<tdml:parserTestCase name="dateTimeBinIBM4690Packed" root="dateTimeBinIBM4690Packed"
model="SimpleTypes-binary" description="Section 5 Schema types-dateTime - DFDL-5-016R"
roundTrip="true">
Expand Down
Loading

0 comments on commit 36f2d87

Please sign in to comment.