Reorg base2 (#88)

* Reorganize tests, to make the main tests faster. Signed-off-by: [email protected] <[email protected]> * Lots of minor tweaks to tests, mostly fixing documentation comments. Signed-off-by: [email protected] <[email protected]> * Stripped down run-tests. Signed-off-by: [email protected] <[email protected]> --------- Signed-off-by: [email protected] <[email protected]> Co-authored-by: [email protected] <[email protected]>
AcademySoftwareFoundation · May 3, 2024 · 37def3c · 37def3c
1 parent 9ab083d
commit 37def3c
Show file tree

Hide file tree

Showing 38 changed files with 2,572 additions and 85 deletions.
diff --git a/enctests/test_configs/base_tests.yml → enctests/base_tests.yml b/enctests/test_configs/base_tests.yml → enctests/base_tests.yml
diff --git a/enctests/runtest.sh b/enctests/runtest.sh
@@ -1,54 +1,24 @@
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/prores_tests.yml --output prores-results.otio --encoded-folder prores-encode
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/prores_qscale_tests.yml --output prores-qscale-results.otio --encoded-folder prores-qscale-encode
-
-python3 -m testframework.otio2html --test-config test_configs/prores_tests.yml --results prores-results.otio
-
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/codec_tests.yml --output codec-results.otio --encoded-folder codec-encode
-
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/h264_tests.yml --output h264-results.otio --encoded-folder h264-encode
-
-python3 -m testframework.otio2html --test-config test_configs/h264_tests.yml --results h264-results.otio
-
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/h264_crf_tests.yml --output h264-crf-results.otio --encoded-folder h264-crf-encode
-
-python3 -m testframework.otio2html --test-config test_configs/h264_crf_tests.yml --results h264-crf-results.otio
-
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/color_tests.yml --output color-results.otio --encoded-folder color-encode
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/hevc_color_tests.yml --output hevc-color-results.otio --encoded-folder hevc-color-encode
-
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/rgb_color_tests.yml --output rgb-color-results.otio --encoded-folder rgb-color-encode
-
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/rgb_tests.yml --output rgb-results.otio --encoded-folder rgb-encode
-
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/mjpeg_color_tests.yml --output mjpeg-color-results.otio --encoded-folder mjpeg-color-encode
-
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/av1_color_tests.yml --output av1-color-results.otio --encoded-folder av1-color-encode
-
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/vp9_color_tests.yml --output vp9-color-results.otio --encoded-folder vp9-color-encode
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/vp9_crf_tests.yml --output vp9-crf-results.otio --encoded-folder vp9-crf-encode
-
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/dnxhd_color_tests.yml --output dnxhd-color-results.otio --encoded-folder dnxhd-color-encode
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/dnxhd_tests.yml --output dnxhd-results.otio --encoded-folder dnxhd-encode
-
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/hevc_tests.yml --output hevc-results.otio --encoded-folder hevc-encode
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/av1_crf_tests.yml --output av1-crf-results.otio --encoded-folder av1-crf-encode
+python3 -m testframework.generatetests
+python3 -m testframework.main 
+python3 -m testframework.main --test-config test_wedge_configs --results-folder wedge_results
 
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/hevc_crf_tests.yml --output hevc-crf-results.otio --encoded-folder hevc-crf-encode
-python3 -m testframework.otio2html --test-config test_configs/hevc_crf_tests.yml --results hevc-crf-results.otio
+# python3 -m testframework.otio2html --test-config test_configs/prores_tests.yml --results prores-results.otio
 
-python3 -m testframework.main --source-folder sources/enc_sources --test-config test_configs/color_tests.yml --output color-results.otio --encoded-folder color-encode
 
 # Windows only with nvenc
-python -m testframework.main --source-folder sources/enc_sources --test-config test_configs/hevc_nvenc_color_tests.yml --output hevc-nvenc-color-results.otio --encoded-folder hevc-color-encode
-python -m testframework.main --source-folder sources/enc_sources --test-config test_configs/hevc_nvenc_tests.yml --output hevc-nvenc-results.otio --encoded-folder hevc-nvenc-encode
-python -m testframework.otio2html --test-config test_configs/hevc_nvenc_tests.yml --results hevc-nvenc-results.otio
+#python -m testframework.main --source-folder sources/enc_sources --test-config test_configs/hevc_nvenc_color_tests.yml --output hevc-nvenc-color-results.otio --encoded-folder hevc-color-encode
+#python -m testframework.main --source-folder sources/enc_sources --test-config test_configs/hevc_nvenc_tests.yml --output hevc-nvenc-results.otio --encoded-folder hevc-nvenc-encode
+#python -m testframework.otio2html --test-config test_configs/hevc_nvenc_tests.yml --results hevc-nvenc-results.otio
 
 # OSX Only tests
-python -m testframework.main --source-folder sources/enc_sources --test-config test_configs/osx_prores_tests.yml --output osx-prores-results.otio --encoded-folder osx-prores-encode
+#python -m testframework.main --source-folder sources/enc_sources --test-config test_configs/osx_prores_tests.yml --output osx-prores-results.otio --encoded-folder osx-prores-encode
 
 
 #WIP
-python3 -m testframework.generatetests
 
-python3 -m testframework.main --test-config test_configs/documentation_tests.yml --output doc-results.otio --encoded-folder docs-encode
-python3 -m testframework.otio2html --test-config test_configs/documentation_tests.yml --results doc-results.otio
+#python3 -m testframework.main --test-config test_configs/documentation_tests.yml --output doc-results.otio --encoded-folder docs-encode
+#python3 -m testframework.otio2html --test-config test_configs/documentation_tests.yml --results doc-results.otio
+
+# Below will re-do all the tests in "test_wedge_configs" where the output data is in --results-folder
+# python3 -m testframework.otio2html --test-config-dir test_wedge_configs --results-folder wedge_results
+
diff --git a/enctests/test_configs/aomav1_color_tests.yml b/enctests/test_configs/aomav1_color_tests.yml
@@ -0,0 +1,66 @@
+
+---
+test_av1_color:
+  app: ffmpeg
+  comparisontest:
+  - testtype: idiff
+    compare_image: ../sourceimages/chip-chart-1080-16bit-noicc-yuv420p10le.png
+  - testtype: assertresults
+    tests:
+    - assert: less
+      value: max_error
+      less: 0.00195
+  description: av1 color tests of different pix-fmts
+  encoding_template: ffmpeg -y {input_args} -i "{source}" {encoding_args} -y "{outfile}"
+  name: test_av1_color
+  sources:
+  - ../sourceimages/chip-chart-1080-16bit-noicc.png.yml
+  suffix: .mp4
+  wedges:
+    av1-yuv422p10le: &baseav1
+      -c:v: libaom-av1
+      -crf: 25
+      -row-mt: 1
+      -cpu-used: 6
+      -vf: '"scale=in_color_matrix=bt709:out_color_matrix=bt709"'
+      -color_primaries: bt709
+      -color_range: tv
+      -color_trc: bt709
+      -colorspace: bt709
+      -pix_fmt: yuv422p10le
+      -strict: experimental
+
+    av1-yuv422p10le-cpu5:
+      << : *baseav1
+      -cpu-used: 5
+
+    av1-yuv422p10le-cpu6-pass1:
+      << : *baseav1
+      -cpu-used: 5
+      -aom-params: passes=1
+      -pass: 1
+
+    av1-yuv422p10le-cpu5-g1:
+      << : *baseav1
+      -g: 1
+
+
+    svtav1-yuv422p10le:
+      -c:v: libsvtav1 
+      -crf: 18
+      -row-mt: 1 
+      -preset: 9
+      -vf: '"scale=in_color_matrix=bt709:out_color_matrix=bt709"'
+      -color_primaries: bt709
+      -color_range: tv
+      -color_trc: bt709
+      -colorspace: bt709
+      -pix_fmt: yuv420p10le
+
+---
+reports:
+  description: This is testing av1 color encoding (both libaom-av1 and libsvtav1)
+  directory: aom-av1-color-encode
+  name: aom-av1-color-tests
+  templatefile: colorweb.html.jinja
+  title: Ffmpeg AV1 encoding color tests
diff --git a/enctests/test_configs/av1_color_tests.yml b/enctests/test_configs/av1_color_tests.yml
@@ -9,7 +9,7 @@ test_av1_color:
     tests:
     - assert: less
       value: max_error
-      less: 0.00195
+      less: 0.004
   description: av1 color tests of different pix-fmts
   encoding_template: ffmpeg -y {input_args} -i "{source}" {encoding_args} -y "{outfile}"
   name: test_av1_color

diff --git a/enctests/test_configs/bitdepth_color_tests.yml b/enctests/test_configs/bitdepth_color_tests.yml
@@ -0,0 +1,80 @@
+
+
+test_bitdepth:
+  app: ffmpeg
+  comparisontest:
+  - extracttemplate: ffmpeg -y -i {newfile} -compression_level 10 -pred mixed -pix_fmt rgb48be -frames:v 1 {newpngfile}
+    testtype: idiff
+  - testtype: assertresults
+    tests:
+    - assert: less
+      value: max_error
+      less: 0.00195
+  description: Bitdepth tests.
+  encoding_template: ffmpeg -y {input_args} -i "{source}"              {encoding_args}
+    -y "{outfile}"
+  name: test_rgb
+  sources:
+  - ../sourceimages/chip-chart-1080-16bit-noicc.png.yml
+  suffix: .mp4
+  wedges:
+    h264:
+      -c:v: libx264
+      -preset: placebo
+      -qp: 0
+      -pix_fmt: yuv420p10le
+      -vf: '"scale=in_range=full:in_color_matrix=bt709:out_range=tv:out_color_matrix=bt709"'
+      -color_primaries: bt709
+      -color_range: tv
+      -color_trc: bt709
+      -colorspace: bt709
+    10bit_h264:
+      -c:v: libx264
+      -preset: placebo
+      -qp: 0
+      -pix_fmt: yuv420p10le
+      -vf: '"scale=in_range=full:in_color_matrix=bt709:out_range=tv:out_color_matrix=bt709"'
+      -color_primaries: bt709
+      -color_range: tv
+      -color_trc: bt709
+      -colorspace: bt709
+    vp9_8bit:
+      -c:v: libvpx-vp9
+      -quality: good
+      -crf: 0
+      -pix_fmt: yuv420p
+      -colorspace: bt709
+      -vf: '"scale=in_range=full:in_color_matrix=bt709:out_range=tv:out_color_matrix=bt709"'
+      -color_range: tv
+      -color_primaries: bt709
+      -color_range: tv
+      -color_trc: bt709
+    vp9_10bit:
+      -c:v: libvpx-vp9
+      -quality: good
+      -crf: 0
+      -pix_fmt: yuv420p10le
+      -colorspace: bt709
+      -vf: '"scale=in_range=full:in_color_matrix=bt709:out_range=tv:out_color_matrix=bt709"'
+      -color_range: tv
+      -color_primaries: bt709
+      -color_range: tv
+      -color_trc: bt709
+    vp9_12bit:
+      -c:v: libvpx-vp9
+      -quality: good
+      -crf: 0
+      -pix_fmt: yuv420p12le
+      -colorspace: bt709
+      -vf: '"scale=in_range=full:in_color_matrix=bt709:out_range=tv:out_color_matrix=bt709"'
+      -color_primaries: bt709
+      -color_range: tv
+      -color_trc: bt709
+
+---
+reports:
+  description: Testing colorbars at different bit-depths for h264 and vp9, showing the file size.
+  directory: bitdepth-color-encode
+  name: bitdepth-color-tests
+  templatefile: colorweb.html.jinja
+  title:  Bit-depth color comparisons. Browser support.