ci(mlx): GMM probe v3 + fork-check v7 #24
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: MLX PR check (fork) | |
| # CAMPAGNE v7 — sonde v3 lazy-safe + génération des artefacts fiabilisée. | |
| # | |
| # Post-mortem v6 : | |
| # 1. `python -m ...test_utils <cfg> generate` est un NO-OP : le module | |
| # n'a pas de __main__. Rien d'écrit, `ls` mort, runner jamais lancé. | |
| # 2. Le crash du runner sous PROBE=1 (harnais : FAILED + stdout coupé à | |
| # "Executing forward...") était la sonde v2 elle-même : exec_* tourne | |
| # à la CONSTRUCTION du graphe lazy ; A/B/rhs_idx n'y sont pas encore | |
| # évalués (B sort d'un TransposeNode) et dump() appelait | |
| # .data<char>() en premier -> mort avant le moindre fprintf, hors de | |
| # portée du try/catch (dumps placés avant) et incatchable (segfault). | |
| # Ce crash prouve au passage que l'instrumentation est atteinte, | |
| # PROBE lu, sorted=1. | |
| # | |
| # Changements v7 : | |
| # 1. Sonde v3 (probe-gmm-v3.patch, sur v2) : marqueur "enter v3" | |
| # immédiat, verdict via ops lazy + item() (qui évalue), dumps de | |
| # pointeurs APRÈS matérialisation seulement, catch(std::exception) | |
| # + catch(...). | |
| # 2. Ordre inversé : harnais d'abord (il génère model.pte/input.bin | |
| # dans le get_test_dir() de site-packages), step direct ensuite qui | |
| # RÉUTILISE ces artefacts. Fallback : generate_test_files() en | |
| # python -c si le dossier manque. | |
| # 3. A/B perturbation conservé : PROBE=1 vs PROBE=0, cmp des sorties. | |
| # | |
| # Verdict ([GMM-PROBE] harnais ou direct) : | |
| # wrapped >> 0 && natcopies ~ 0 -> PROVENANCE -> fix ET dans la PR. | |
| # wrapped >> 0 && natcopies >> 0 -> état de processus pré-appel | |
| # (l'image statique est innocentée par L3-exact depuis v5). | |
| # EXCEPTION: ... -> la sonde explique. | |
| # "enter v3" seul -> crash plus profond, mais localisé après l'entrée. | |
| # | |
| # Prérequis : probe v1+v2+v3 appliqués ; repro_gather_mm.cpp à la racine. | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - ci-mlx-check | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| provenance-vs-link: | |
| runs-on: macos-14 | |
| timeout-minutes: 300 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Preuve — commit, sonde v3 présente | |
| run: | | |
| set -eux | |
| git log --oneline -3 | |
| grep -c "GMM-PROBE" backends/mlx/runtime/MLXInterpreter.h | |
| grep -n "enter v3" backends/mlx/runtime/MLXInterpreter.h | |
| - name: Install ExecuTorch | |
| run: | | |
| set -eux | |
| python -m pip install --upgrade pip | |
| python install_executorch.py | |
| - name: Configure + build op_test_runner | |
| id: build | |
| run: | | |
| set -eux | |
| cmake --preset mlx-release \ | |
| -DEXECUTORCH_BUILD_TESTS=ON \ | |
| -DEXECUTORCH_MLX_ENABLE_SANITIZERS=OFF | |
| cmake --build cmake-out --target op_test_runner -j2 | |
| df -h / | |
| # ------------------------------------------------------------------ | |
| # 1) Harnais — génère les artefacts ET porte le verdict (sonde v3 | |
| # sur stdout, réimprimée via "C++ binary output:" même à rc=0). | |
| # ------------------------------------------------------------------ | |
| - name: SONDE P harnais — famille gather_mm (génère + verdict) | |
| if: steps.build.outcome == 'success' | |
| continue-on-error: true | |
| run: | | |
| set -eux | |
| export ET_MLX_GMM_PROBE=1 | |
| python -m executorch.backends.mlx.test.run_all_tests gather_mm -v \ | |
| 2>&1 | tee /tmp/harness.log | |
| echo "=== GMM-PROBE dans le harnais ===" | |
| grep -n 'GMM-PROBE' /tmp/harness.log || echo "AUCUNE" | |
| echo "=== contexte des configs sorted ===" | |
| grep -n -B3 -A16 'e4_i64_o128_sorted' /tmp/harness.log | head -140 | |
| # ------------------------------------------------------------------ | |
| # 2) Direct — réutilise les artefacts du harnais, flux bruts, | |
| # A/B perturbation. | |
| # ------------------------------------------------------------------ | |
| - name: SONDE P directe — flux bruts + A/B perturbation | |
| if: steps.build.outcome == 'success' | |
| continue-on-error: true | |
| run: | | |
| set -eux | |
| CFG=gather_mm_e4_i64_o128_sorted | |
| R=$(python -c "import executorch.backends.mlx.test.test_utils as t; print(t.find_op_test_runner())") | |
| D=$(python -c "import executorch.backends.mlx.test.test_utils as t, pathlib; print(pathlib.Path(t.__file__).parent/'op_tests'/'$CFG')") | |
| echo "runner=$R"; echo "testdir=$D" | |
| echo "probe-in-binary: $(strings "$R" | grep -c 'GMM-PROBE')" | |
| if [ ! -f "$D/model.pte" ]; then | |
| python -c "import executorch.backends.mlx.test.test_ops as _reg; from executorch.backends.mlx.test.test_utils import get_all_test_configs; c=dict(get_all_test_configs())['$CFG']; print(c.generate_test_files(verbose=False))" | |
| fi | |
| ls -la "$D" | |
| set +e | |
| ET_MLX_GMM_PROBE=1 "$R" \ | |
| --pte "$D/model.pte" --input "$D/input.bin" \ | |
| --output /tmp/out_probe.bin --verbose \ | |
| > /tmp/p_out.txt 2> /tmp/p_err.txt | |
| rc1=$? | |
| "$R" \ | |
| --pte "$D/model.pte" --input "$D/input.bin" \ | |
| --output /tmp/out_noprobe.bin --verbose \ | |
| > /tmp/n_out.txt 2> /tmp/n_err.txt | |
| rc2=$? | |
| set -e | |
| echo "=== rc probe=$rc1 / noprobe=$rc2 ===" | |
| echo "=== STDOUT (probe) ==="; cat /tmp/p_out.txt | |
| echo "=== STDERR (probe) ==="; cat /tmp/p_err.txt | |
| echo "=== lignes GMM-PROBE ===" | |
| grep -h 'GMM-PROBE' /tmp/p_out.txt /tmp/p_err.txt || echo "AUCUNE" | |
| echo "=== A/B perturbation ===" | |
| if [ "$rc1" -eq 0 ] && [ "$rc2" -eq 0 ]; then | |
| cmp /tmp/out_probe.bin /tmp/out_noprobe.bin \ | |
| && echo "SORTIES IDENTIQUES (sonde non perturbante)" \ | |
| || echo "SORTIES DIFFERENTES (bug sensible a l'ordre d'execution !)" | |
| else | |
| echo "cmp saute (rc non nuls)" | |
| fi | |
| - name: CENSUS — symboles mlx/Metal en double dans la closure | |
| if: steps.build.outcome == 'success' | |
| continue-on-error: true | |
| run: | | |
| set -eux | |
| RUNNER=$(find cmake-out -name op_test_runner -type f | head -1) | |
| otool -L "$RUNNER" | |
| LINKTXT=$(find cmake-out -path "*op_test_runner.dir/link.txt" | head -1) | |
| rm -f /tmp/defs.txt | |
| for a in $(tr ' ' '\n' < "$LINKTXT" | grep '\.a$' | sort -u); do | |
| [ -f "$a" ] || continue | |
| nm -gU "$a" 2>/dev/null | awk -v A="$a" \ | |
| '$2 ~ /^[TSDW]$/ {print $3, A}' >> /tmp/defs.txt || true | |
| done | |
| grep -iE 'mlx|metal|MTL' /tmp/defs.txt | sort | \ | |
| awk '{print $1}' | uniq -d | head -80 || true | |
| - name: ÉCHELLE — L0 mlx seul (contrôle) | |
| if: steps.build.outcome == 'success' | |
| continue-on-error: true | |
| run: | | |
| set -eux | |
| MLXA=$(find cmake-out -name libmlx.a | head -1) | |
| METALLIB=$(find cmake-out -name mlx.metallib | head -1) | |
| cp "$METALLIB" . | |
| clang++ -std=c++17 -O2 repro_gather_mm.cpp \ | |
| -I backends/mlx/third-party/mlx \ | |
| -I "$(dirname "$MLXA")" \ | |
| "$MLXA" \ | |
| -framework Metal -framework Foundation -framework QuartzCore \ | |
| -o rung_L0 | |
| ./rung_L0 && echo "L0: OK" || echo "L0: DIVERGE/CRASH rc=$?" | |
| - name: ÉCHELLE — L3 closure exacte (contrôle de régression) | |
| if: steps.build.outcome == 'success' | |
| continue-on-error: true | |
| run: | | |
| set -eux | |
| MLXA=$(find cmake-out -name libmlx.a | head -1) | |
| LINKTXT=$(find cmake-out -path "*op_test_runner.dir/link.txt" | head -1) | |
| BLD=$(dirname "$(dirname "$(dirname "$LINKTXT")")") | |
| clang++ -std=c++17 -O2 -c repro_gather_mm.cpp \ | |
| -I backends/mlx/third-party/mlx \ | |
| -I "$(dirname "$MLXA")" \ | |
| -o /tmp/repro.o | |
| sed -E \ | |
| -e "s#[^ ]*op_test_runner\.cpp\.o#/tmp/repro.o#" \ | |
| -e "s# -o [^ ]*op_test_runner# -o $PWD/rung_L3#" \ | |
| "$LINKTXT" > /tmp/l3cmd.sh | |
| (cd "$BLD" && bash /tmp/l3cmd.sh) 2>&1 | tail -40 | |
| ./rung_L3 && echo "L3: OK" || echo "L3: DIVERGE/CRASH rc=$?" | |
| - name: Diagnostics finaux | |
| if: always() | |
| run: df -h / |