Add App2E HTJ2K constraints validation (#375)

Validate the CPL essence descriptors against the APP2.HT.REV and APP2.HT.IRV constraint sets specified in ST 2067-21, Annex I.

Co-authored-by: Pierre-Anthony Lemieux <[email protected]>

Author: fschleich

README.md

@@ -1,7 +1,14 @@
 
 # Photon
 
-Photon is a Java implementation of the [Interoperable Master Format (IMF)](https://www.imfug.com/explainer/imf-explainer-en/#what-is-imf) standard. IMF core constraints are defined by [SMPTE](https://www.smpte.org/who-we-are) specification [st2067-2:2013](https://ieeexplore.ieee.org/document/7291584) (paywall). Photon offers tools for parsing, interpreting and validating constituent files that make an Interoperable Master Package (IMP). These include AssetMap (st429-9:2014), PackingList (st429-8:2007), Composition Playlist (st2067-3:2013), and the essence containing IMF track file (st2067-5:2013) which follows the Material eXchange Format (MXF) format (st377-1:2011). Specifically, Photon parses and completely reads an MXF file containing a single audio or video essence as defined by the IMF Essence Component specification (st2067-5:2013) and serializes the metadata into the IMF Composition Playlist structure.
+Photon is a Java implementation of the [Interoperable Master Format (IMF)](https://www.smpte.org/standards/st2067) standard. Photon offers tools for parsing, interpreting and validating constituent files that make an Interoperable Master Package (IMP). These include:
+
+- AssetMap (ST 429-9)
+- PackingList (ST 429-8) 
+- Composition Playlist (ST 2067-3) 
+- IMF track files (ST 2067-5)
+
+Photon parses and reads IMF track files and serializes the metadata into the IMF Composition Playlist structure. Currently, Photon provides support for IMF Application #2E (ST 2067-21) and Application #5 ACES (ST 2067-50), and the Immersive Audio Bitstream (IAB) Plug-in (ST 2067-201).
 
 The goal of the Photon is to provide a simple standardized interface to completely validate an IMP.
 
@@ -14,12 +21,16 @@ Photon can be built using JDK-8. Support for earlier jdk versions has not been t
 ### Gradle
 Photon can be built very easily by using the included Gradle wrapper. Having downloaded the sources, simply invoke the following commands inside the folder containing the sources:
 
+```
 $ ./gradlew clean
 $ ./gradlew build
+```
 
 For Windows
+```
 $ gradlew.bat clean
 $ gradlew.bat build
+```
 
 ## Full Documentation
 

build.gradle

@@ -104,7 +104,9 @@ dependencies {
     /*compile "com.sandflow:regxmllib:${revRegXMLSNAPSHOT}"*/
     testImplementation "org.mockito:mockito-core:3.3+"
     testImplementation "org.testng:testng:7.5+"
-    testImplementation "org.slf4j:slf4j-simple:latest.release"
+    implementation "org.slf4j:slf4j-simple:1.7.2"
+    implementation "org.slf4j:slf4j-api:1.7.2"
+
 
     // JAX-B dependencies for JDK 9+
     implementation "jakarta.xml.bind:jakarta.xml.bind-api:2.3.2"

codequality/findbugs-excludeFilter-GeneratedCode.xml

@@ -38,4 +38,9 @@
         <Bug pattern="RCN_REDUNDANT_NULLCHECK_OF_NULL_VALUE" />
     </Match>
 
+    <!-- It should be possible to define a class that is tested but not fully used at this time -->
+    <Match>
+        <Bug pattern="URF_UNREAD_FIELD" />
+    </Match>
+
 </FindBugsFilter>

src/main/java/com/netflix/imflibrary/st2067_2/Application2E2021.java

@@ -4,9 +4,12 @@
 import com.netflix.imflibrary.Colorimetry.ColorModel;
 import com.netflix.imflibrary.Colorimetry.Quantization;
 import com.netflix.imflibrary.Colorimetry.Sampling;
+import com.netflix.imflibrary.st0377.header.GenericPictureEssenceDescriptor.FrameLayoutType;
 import com.netflix.imflibrary.st0377.header.UL;
 import com.netflix.imflibrary.IMFErrorLogger;
 import com.netflix.imflibrary.st2067_2.ApplicationCompositionFactory.ApplicationCompositionType;
+import com.netflix.imflibrary.st2067_2.CompositionImageEssenceDescriptorModel.J2KHeaderParameters;
+import com.netflix.imflibrary.st2067_2.CompositionImageEssenceDescriptorModel.ProgressionOrder;
 import com.netflix.imflibrary.utils.Fraction;
 import com.netflix.imflibrary.JPEG2000;
 
@@ -269,13 +272,291 @@ public Application2E2021(@Nonnull IMFCompositionPlaylistType imfCompositionPlayl
         }
     }
 
+    /* Validate codestream parameters against constraints listed in SMPTE ST 2067-21:2023 Annex I */
+
+    private static boolean validateHT(CompositionImageEssenceDescriptorModel imageDescriptor,
+                                     IMFErrorLogger logger) {
+        boolean isValid = true;
+
+        J2KHeaderParameters p = imageDescriptor.getJ2KHeaderParameters();
+
+        if (p == null) {
+            logger.addError(
+                    IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                    IMFErrorLogger.IMFErrors.ErrorLevels.FATAL,
+                    "APP2.HT: Missing or incomplete JPEG 2000 Sub-descriptor");
+            return false;
+        }
+
+        if (p.xosiz != 0 || p.yosiz != 0 || p.xtosiz != 0 || p.ytosiz != 0) {
+            logger.addError(
+                    IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                    IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                    "APP2.HT: Invalid XOsiz, YOsiz, XTOsiz or YTOsiz");
+            isValid = false;
+        }
+
+        if (p.xtsiz < p.xsiz || p.ytsiz < p.ysiz) {
+            logger.addError(
+                    IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                    IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                    "APP2.HT: Invalid XTsiz or XYsiz");
+            isValid = false;
+        }
+
+        /* components constraints */
+
+        if (p.csiz.length <= 0 || p.csiz.length > 4) {
+            logger.addError(
+                    IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                    IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                    String.format("APP2.HT: Invalid number (%d) of components", p.csiz.length));
+            isValid = false;
+        }
+
+        /* x sub-sampling */
+        if (p.csiz[0].xrsiz != 1) {
+            logger.addError(
+                    IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                    IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                    "APP2.HT: invalid horizontal sub-sampling for component 1");
+            isValid = false;
+        }
+        if (p.csiz.length > 1 && p.csiz[1].xrsiz != 1 &&
+            (p.csiz.length <= 2 || p.csiz[1].xrsiz != 2 || p.csiz[2].xrsiz != 2)) {
+            logger.addError(
+                    IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                    IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                    "APP2.HT: invalid horizontal sub-sampling for component 2");
+            isValid = false;
+        }
+        if (p.csiz.length > 2 && p.csiz[2].xrsiz != 1 && (p.csiz[1].xrsiz != 2 || p.csiz[2].xrsiz != 2)) {
+            logger.addError(
+                    IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                    IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                    "APP2.HT: invalid horizontal sub-sampling for component 3");
+            isValid = false;
+        }
+        if (p.csiz.length > 3 && p.csiz[3].xrsiz != 1) {
+            logger.addError(
+                    IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                    IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                    "APP2.HT: invalid horizontal sub-sampling for component 4");
+            isValid = false;
+        }
+
+        /* y sub-sampling and sample width */
+        if (p.csiz[0].ssiz > 15 || p.csiz[0].ssiz < 7) {
+            logger.addError(
+                    IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                    IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                    String.format("APP2.HT: Invalid bit depth (%d)", p.csiz[0].ssiz + 1));
+            isValid = false;
+        }
+        for (int i = 0; i < p.csiz.length; i++) {
+            if (p.csiz[i].yrsiz != 1) {
+                logger.addError(
+                        IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                        IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                        String.format("APP2.HT: invalid vertical sub-sampling for component %d", i));
+                isValid = false;
+            }
+            if (p.csiz[i].ssiz != p.csiz[0].ssiz) {
+                logger.addError(
+                        IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                        IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                        "APP2.HT: all components must have the same bit depth");
+                isValid = false;
+            }
+        }
+        /* CAP constraints */
+
+        /* Pcapi is 1 for i = 15, and 0 otherwise, per ST 2067-21 Annex I; therefore, pcap = 2^(32-15) = 131072 */
+        if (p.cap == null || p.cap.pcap != 131072 || p.cap.ccap == null || p.cap.ccap.length != 1) {
+            /* codestream shall require only Part 15 capabilities */
+            logger.addError(
+                    IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                    IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                    "APP2.HT: missing or invalid CAP marker");
+            return false;
+        }
+
+        if ((p.cap.ccap[0] & 0b1111000000000000) != 0) {
+            /* Bits 12-15 of Ccap15 shall be 0 */
+            logger.addError(
+                    IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                    IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                    "APP2.HT: Bits 12-15 of Ccap15 shall be 0");
+            isValid = false;
+        }
+
+        boolean isHTREV = (p.cap.ccap[0] & 0b100000) == 0;
+
+        /* COD */
+
+        if (p.cod == null) {
+            logger.addError(
+                    IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                    IMFErrorLogger.IMFErrors.ErrorLevels.FATAL,
+                    "APP2.HT: Missing COD marker");
+            return false;
+        }
+
+        /* no scod constraints */
+
+        /* code-block style */
+        if (p.cod.cbStyle != 0b01000000) {
+            /* bad code-block style */
+            logger.addError(
+                    IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                    IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                    "APP2.HT: Invalid default code-block style");
+            isValid = false;
+        }
+
+        /* progression order - RPCL is not required, but ST 2067-21:2023 Annex I Note 3 implies a preference */
+        if (p.cod.progressionOrder != ProgressionOrder.RPCL.value())
+            logger.addError(
+                IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                IMFErrorLogger.IMFErrors.ErrorLevels.WARNING,
+                "APP2.HT: JPEG 2000 progression order is not RPCL");
+
+        /* resolution layers */
+        if (p.cod.numDecompLevels == 0) {
+            logger.addError(
+                IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                "APP2.HT: Number of decomposition levels must be greater than 0");
+            isValid = false;
+        }
+
+
+        long maxSz = Math.max(p.xsiz, p.ysiz);
+        if ((maxSz <= 2048 && p.cod.numDecompLevels > 5) ||
+            (maxSz <= 4096 && p.cod.numDecompLevels > 6) ||
+            (maxSz <= 8192 && p.cod.numDecompLevels > 7)) {
+            logger.addError(
+                IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                "APP2.HT: Invalid number of decomposition levels");
+            isValid = false;
+        }
+
+        /* number of layers */
+
+        if (p.cod.numLayers != 1) {
+            logger.addError(
+                IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                String.format("APP2.HT: Number of layers (%d) is not 1", p.cod.numLayers));
+            isValid = false;
+        }
+
+        /* code-block sizes */
+
+        if (p.cod.ycb < 5 || p.cod.ycb > 6) {
+            logger.addError(
+                IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                String.format("APP2.HT: Invalid vertical code-block size (ycb = %d)", p.cod.ycb));
+            isValid = false;
+        }
+
+        if (p.cod.xcb < 5 || p.cod.xcb > 7) {
+            logger.addError(
+                IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                String.format("APP2.HT: Invalid horizontal code-block size (xcb = %d)", p.cod.xcb));
+            isValid = false;
+        }
+
+
+        /* transformation */
+
+        boolean isReversibleFilter = (p.cod.transformation == 1);
+
+        if (isHTREV && !isReversibleFilter) {
+            logger.addError(
+                IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                "APP2.HT: 9-7 irreversible filter is used but HTREV is signaled in CAP");
+            isValid = false;
+        }
+
+        /* precinct size */
+
+        if (p.cod.precinctSizes.length == 0 || p.cod.precinctSizes[0] != 0x77) {
+            logger.addError(
+                IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                "APP2.HT: Invalid precinct sizes");
+            isValid = false;
+        }
+
+        for (int i = 1; i < p.cod.precinctSizes.length; i++)
+            if (p.cod.precinctSizes[i] != 0x88) {
+                logger.addError(
+                    IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                    IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                    "APP2.HT: Invalid precinct sizes");
+                isValid = false;
+                break;
+            }
+
+        /* magbp - calculation according to ITU-T T.814 */
+
+
+        int maxB = p.csiz[0].ssiz + 2;
+        if (isReversibleFilter) {
+            maxB += 2 + p.cod.multiComponentTransform;
+            if (p.cod.numDecompLevels > 5)
+                maxB += 1;
+        } else if (p.cod.multiComponentTransform == 1 && p.csiz[0].ssiz > 9) {
+            maxB += 1;
+        }
+
+        int codestreamB = (p.cap.ccap[0] & 0b11111) + 8;
+
+        /*
+        *   NOTE: The Parameter B constraints in ST 2067-21:2023 are arguably too narrow, and existing implementations do violate them under certain circumstances.
+        *   Since practical issues are not expected from software decoders otherwise, an ERROR is currently returned only for values that exceed the max value (21)
+        *   allowed for any configuration by ST 2067-21:2023. A WARNING is provided for values that exceed the limit stated in ST 2067-21:2023, but not 21.
+        *
+        *   TODO: This should be revisited as more implementations become available. Discussion for reference: https://github.com/SMPTE/st2067-21/issues/7
+        */
+
+        if (codestreamB > 21) {
+            logger.addError(
+                    IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                    IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,
+                    "APP2.HT: Parameter B has exceeded its limit to an extent that decoder issues are to be expected");
+            isValid = false;
+        } else if (codestreamB > maxB) {
+            logger.addError(
+                IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
+                IMFErrorLogger.IMFErrors.ErrorLevels.WARNING,
+                "APP2.HT: Parameter B has exceeded its limits");
+        }
+
+        return isValid;
+    }
+
+    /**
+     * @deprecated Instead use {@link #isValidJ2KProfile(CompositionImageEssenceDescriptorModel imageDescriptor, IMFErrorLogger logger)}
+     */
+    @Deprecated
     public static boolean isValidJ2KProfile(CompositionImageEssenceDescriptorModel imageDescriptor) {
+        return isValidJ2KProfile(imageDescriptor, new com.netflix.imflibrary.IMFErrorLoggerImpl());
+    }
+
+    public static boolean isValidJ2KProfile(CompositionImageEssenceDescriptorModel imageDescriptor,
+                                            IMFErrorLogger logger) {
         UL essenceCoding = imageDescriptor.getPictureEssenceCodingUL();
         Integer width = imageDescriptor.getStoredWidth();
         Integer height = imageDescriptor.getStoredHeight();
 
         if (JPEG2000.isAPP2HT(essenceCoding))
-            return true;
+            return validateHT(imageDescriptor, logger);
 
         if (JPEG2000.isIMF4KProfile(essenceCoding))
             return width > 2048 && width <= 4096 && height > 0 && height <= 3112;
@@ -293,7 +574,7 @@ public static void validateImageCharacteristics(CompositionImageEssenceDescripto
             IMFErrorLogger logger) {
 
         // J2K profiles
-        if (!isValidJ2KProfile(imageDescriptor)) {
+        if (!isValidJ2KProfile(imageDescriptor, logger)) {
             logger.addError(
                     IMFErrorLogger.IMFErrors.ErrorCodes.APPLICATION_COMPOSITION_ERROR,
                     IMFErrorLogger.IMFErrors.ErrorLevels.NON_FATAL,