Update ci process

.github/workflows/sanity-check.yml

@@ -34,19 +34,18 @@ jobs:
         id: find_files
         run: |
           FILES=$(git diff --diff-filter=A --name-only origin/main...HEAD | grep "_i8_" || true)
-          {
-            echo "files<<EOF"
-            echo "$FILES"
-            echo "EOF"
-          } >> $GITHUB_OUTPUT
+          if [ -z "$FILES" ]; then
+            COUNT=0
+          else
+            COUNT=$(echo "$FILES" | wc -l | tr -d ' ')
+          fi
+          echo "count=$COUNT" >> $GITHUB_OUTPUT
+          echo "files<<EOF" >> $GITHUB_OUTPUT
+          echo "$FILES" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
 
       - name: Run model checks
-        if: steps.find_files.outputs.files != ''
+        if: ${{ steps.find_files.outputs.count != '0' }}
         run: |
-          while read -r f; do
-            [ -z "$f" ] && continue
-            echo "Checking model: $f"
-            git lfs pull --include="$f" --exclude=""
-            python CI/check_model.py --model "$f"
-          done <<< "${{ steps.find_files.outputs.files }}"
-
+          FILES="${{ steps.find_files.outputs.files }}"
+          python CI/post_filter.py -- CI/check_model.py --models $FILES

CI/check_model.py

@@ -1,14 +1,10 @@
 import argparse
-
-import akida
-from cnn2snn import convert
-from quantizeml import load_model
-
-from compute_device import compute_min_device
+import os
+import subprocess
 
 
 def process_model(file_path):
-    try:
+    try:       
         print(f"Loading model: {file_path}")
         base_model = load_model(file_path)
     except Exception as e:
@@ -29,8 +25,21 @@ def process_model(file_path):
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("--model", required=True, help="Path to model file (.h5 or .onnx)")
+    parser.add_argument("--models", nargs="+", help="Path to model files (.h5 or .onnx)")    
     args = parser.parse_args()
-
-    process_model(args.model)
-
+
+    from cnn2snn import convert
+    from quantizeml import load_model
+    from compute_device import compute_min_device    
+
+    # Process each model
+    for model_file in args.models:
+        if os.path.exists(model_file):
+            # Move this step of the YAML workflow into the Python script
+            # Download the file via Git LFS if necessary
+            # git lfs pull --include="$f" --exclude=""
+            subprocess.run(["git", "lfs", "pull", 
+                "--include", model_file,"--exclude", ""], 
+                capture_output=True)                
+
+            process_model(model_file)    

CI/post_filter.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""Simple post-capture filter for noisy commands.
+Runs a subprocess, captures stdout+stderr, and prints allowed lines to the console. 
+Usage: 
+    python CI/post_filter.py -- CI/check_model.py --models $FILES
+    The `--` separates options to this script from the command to run.
+"""
+import os
+import sys
+import argparse
+import subprocess
+import re
+
+def run_and_filter(cmd:str):
+    """Runs a subprocess, captures stdout+stderr, and prints filtred lines to the console. 
+    Args:
+        cmd (str): Python command sent from GitHub action 
+    """
+    allow = re.compile(r'(Checking model:|Loading model:|✅.*needs \d+ Akida nodes)', re.IGNORECASE)
+    deny = re.compile(
+        r'(cuda|cufft|cudnn|cublas|tensorflow|xla|absl::InitializeLog|Unable to register|WARNING:|'
+        r'failed call to|loop_optimizer\.cc|computation_placer|computation placer|gpu_device|Created device|'
+        r'TF_FORCE_GPU_ALLOW_GROWTH|Overriding orig_value)'
+        , re.IGNORECASE)        
+
+    # If the command is a Python script path and not executable, prefix
+    # with the current Python interpreter to avoid PermissionError on CI.
+    if isinstance(cmd, (list, tuple)) and cmd:
+        first = cmd[0]
+        if isinstance(first, str) and first.endswith('.py') and not os.access(first, os.X_OK):
+            cmd = [sys.executable] + list(cmd)
+
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+
+    try:
+        # Read lines as they arrive and print
+        for raw in iter(proc.stdout.readline, ''):
+            if raw is None:
+                break
+            line = raw.rstrip('\n')
+            keep = True
+            if allow.search(line):
+                keep = True
+            else:
+                if deny.search(line):
+                    keep = False
+                else:
+                    keep = True
+
+            if keep:
+                print(line, flush=True)
+
+        proc.wait()
+    finally:
+        try:
+            proc.stdout.close()
+        except Exception:
+            pass
+
+def main(argv=None):
+    p = argparse.ArgumentParser(prog='post_filter', description='Run command and filter its output')
+    p.add_argument('cmd', nargs=argparse.REMAINDER, help='Command to run (use -- before the command)')
+    args = p.parse_args(argv)
+
+    if not args.cmd:
+        print('No command provided.' 
+              'Use -- to separate command, e.g.: python CI/post_filter.py -- CI/check_model.py --models $FILES')
+        return 2
+
+    # Remove leading '--' if present
+    cmd = args.cmd
+    if cmd and cmd[0] == '--':
+        cmd = cmd[1:]
+
+    run_and_filter(cmd)
+
+if __name__ == '__main__':
+    sys.exit(main())