Add scripts/check_binaries.py to detect unknown binaries and run it i…

…n CI

.github/workflows/code_checks.yml

@@ -101,6 +101,20 @@ jobs:
         # SC2129: (style): Consider using { cmd1; cmd2; } >> file instead of individual redirects
         run: shellcheck -e SC2086,SC2046,SC2164,SC2054,SC2129 $(find . -name '*.sh' -a -not -name ltmain.sh -a -not -wholename "./autotest/*" -a -not -wholename "./.github/*")
 
+  binary_files:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Install Requirements
+        run: |
+            sudo apt-get install -y python3 coreutils
+
+      - name: Checkout
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Detect binary files
+        run: python3 ./scripts/check_binaries.py
+
   linting:
     runs-on: ubuntu-latest
     steps:

scripts/binaries_allow_list.csv

@@ -0,0 +1,19 @@
+filename,sha256sum
+"swig/java/test_data/poly.shp",0785601f2dfa49ab2c50c6c13806fb95ce9c4c58c0182273850e066799b1c344
+"swig/java/test_data/byte.tif",59ed6e9dd19291bbdc230092ab7f6bc46fb0537ee4c9eec15dd58021cd9c12be
+"swig/java/test_data/poly.shx",ac735bd4487f979081d723ec6611861dc90aab3560d170963ac3c5a886594ea2
+"swig/java/test_data/poly.dbf",19a2f2d8ff31e21df0cb5eb7afb0512760f1a01070f67b0f78dfc748ab264dfb
+"swig/csharp/gdal.snk",93bc052177485a64850ebe7d126ba741a8b5d0b57e6c6ea08f8ae86d4827e5a7
+"ogr/ogrsf_frmts/miramon/data/MM_m_idofic.csv",69c57dbb95e172744e3c236bbe732e97105a029678e7011df67ce31a85b29604
+"ogr/ogrsf_frmts/sxf/data/default.rsc",9e1c9d7081d52775e750d4155d49255252b04ca81fef27cb2997675d3cd9626b
+"ogr/ogrsf_frmts/dgn/data/seed_3d.dgn",97c2f00ee6ea96873b7d16e5e898b4850e3d35448299d7d9d37e7d1792b56896
+"ogr/ogrsf_frmts/dgn/data/seed_2d.dgn",dd8465f18569d9289809e9e0962115d365d0a56de021393952a5e7a0a20b527c
+"resources/gdal2tiles/opacity-slider.png",f747e7f6a0e0ccb5d5209deb67511035a3fab98aa7394fa703e19243ee97d679
+"resources/gdal2tiles/none.png",f1bc566c46d73b7cc6f25957c308abbad7af113c69c40af7001115f1a40d96b5
+"frmts/jpipkak/components.PNG",41da97dfa2b78e8e3e2b8cf4379e5794a6a4a8ec1c4bab884a3981a94c7ffc7c
+"frmts/jpipkak/jpipsequence.PNG",bd3e07fc7d450b01fa5af23905d74cfc427c0faf86f109d2a2cbbc145eebc9ed
+"frmts/jpipkak/gdalsequence.PNG",b811e2529f8d4076e6e854d4a900b9b65e6b744e9dc9d95e02107932a47e646a
+"data/GDALLogoGS.svg",1ae6e2d92e6b0439f3baee0cb55dad8b97abfb4fcc58c9189b23b63c818aeeca
+"data/GDALLogoBW.svg",aac9dab33d479e0b832b8bd7c9e5510dcbef34a7c6cc9aea96bbe2aab5161c53
+"data/gdalicon.png",d90f50a8f74c325086c867bf006a92266c259a270d9166b2f41f586b67755729
+"data/GDALLogoColor.svg",2c102a31c6e9116aa9b98787d8bf5cf648dbcc2b930aee653cd35df65ed118d0

scripts/check_binaries.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+"""This file checks that the root directory does not contain an unexpected binary file"""
+
+import csv
+import os
+import subprocess
+import sys
+
+this_dir = os.path.dirname(__file__)
+if not os.path.isabs(this_dir):
+    this_dir = os.path.join(os.getcwd(), this_dir)
+this_dir = os.path.abspath(this_dir)
+root_dir = os.path.dirname(this_dir)
+
+binaries_allow_list = {}
+with open(
+    os.path.join(this_dir, "binaries_allow_list.csv"), newline="", encoding="utf-8"
+) as f:
+    reader = csv.DictReader(f)
+    assert reader.fieldnames == ["filename", "sha256sum"]
+    for row in reader:
+        binaries_allow_list[row["filename"]] = row["sha256sum"]
+
+error_code = 0
+
+for dirname in os.listdir(root_dir):
+    # We skip doc and autotest as they are not included in the GDAL source tarball
+    if dirname not in ("doc", "autotest", ".git"):
+        p = subprocess.Popen(
+            [
+                "find",
+                os.path.join(root_dir, dirname),
+                "-type",
+                "f",
+                "-exec",
+                "file",
+                "{}",
+                ";",
+            ],
+            stdout=subprocess.PIPE,
+        )
+        out, _ = p.communicate()
+        for line in out.decode("utf-8").split("\n"):
+            if not line:
+                continue
+            tokens = line.split(":")
+            filename = tokens[0]
+            kind = ":".join(tokens[1:]).strip()
+            if (
+                "text" not in kind
+                and "AutoCAD" not in kind
+                and kind not in ("empty", "JSON data")
+            ):
+                p = subprocess.Popen(["sha256sum", filename], stdout=subprocess.PIPE)
+                sha256sum, _ = p.communicate()
+                sha256sum = sha256sum.decode("utf-8").split("  ")[0]
+
+                rel_filename = filename[len(root_dir) + 1 :]
+                if rel_filename not in binaries_allow_list:
+                    error_code = 1
+                    print(
+                        f'Found unknown binary file {rel_filename} of kind "{kind}". If it is legit, add the following line in scripts/binaries_allow_list.csv:\n"{rel_filename}",{sha256sum}'
+                    )
+                elif binaries_allow_list[rel_filename] != sha256sum:
+                    error_code = 1
+                    print(
+                        f'Binary file {rel_filename} has a different sha256sum than expected. If it is legit, update the following line in scripts/binaries_allow_list.csv:\n"{rel_filename}",{sha256sum}'
+                    )
+
+sys.exit(error_code)