ci(mlx): GMM probe v2 on stdout + fork-check v5 #22
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: MLX PR check (fork) | |
| # CAMPAGNE v5 — même objectif que v4 (provenance vs image de link), mais | |
| # la sonde ne peut plus être avalée. | |
| # | |
| # Post-mortem v4 : la sonde écrivait sur stderr ; or run_cpp_test_runner | |
| # fait capture_output=True et ne réimprime stderr QUE si le runner sort | |
| # != 0. Nos configs rouges sortent 0 (l'échec est le compare Python | |
| # d'après). Silence garanti par construction — pas un verdict. | |
| # | |
| # Changements v5 : | |
| # 1. Sonde v2 (probe-gmm-v2.patch, sur v1) : sortie sur STDOUT + fflush | |
| # + try/catch qui imprime toute exception MLX au lieu d'un silence. | |
| # 2. Invocation DIRECTE du runner (sans harnais) : generate via le CLI | |
| # test_utils, puis exécution binaire avec l'env, stdout/stderr bruts | |
| # streamés dans le log + rc affiché. Aucun avaleur possible. | |
| # 3. `strings` sur le binaire : preuve que la sonde est DANS l'image. | |
| # 4. L3 = closure EXACTE du runner : on réutilise le link.txt de CMake | |
| # en substituant l'objet et la cible. Le L3 de v4 (force_load sur | |
| # toutes les archives) collisionne par construction (symboles en | |
| # double) — c'était mon artefact, pas une donnée. | |
| # | |
| # Verdict (lignes [GMM-PROBE] de l'étape "SONDE P directe") : | |
| # wrapped >> 0 && natcopies ~ 0 -> PROVENANCE (wrap zero-copy x kernel | |
| # sorted) -> fix côté ET dans la PR, xfail retiré. | |
| # wrapped >> 0 && natcopies >> 0 -> IMAGE DE PROCESSUS -> l'échelle | |
| # L0→L3 (closure exacte) nomme la bibliothèque. | |
| # EXCEPTION: ... -> la sonde dit pourquoi elle ne peut pas mesurer. | |
| # | |
| # Prérequis branche ci-mlx-check : | |
| # - probe-gmm-provenance.patch (v1) PUIS probe-gmm-v2.patch appliqués | |
| # - repro_gather_mm.cpp à la racine (inchangé) | |
| # - ce fichier en .github/workflows/mlx-fork-check.yml | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - ci-mlx-check | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| provenance-vs-link: | |
| runs-on: macos-14 | |
| timeout-minutes: 300 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Preuve — commit, sonde v2 présente (stdout + try/catch) | |
| run: | | |
| set -eux | |
| git log --oneline -3 | |
| grep -n "GMM-PROBE" backends/mlx/runtime/MLXInterpreter.h | head -5 | |
| grep -n "fprintf(\s*stdout" backends/mlx/runtime/MLXInterpreter.h | head -3 | |
| grep -cn "EXCEPTION" backends/mlx/runtime/MLXInterpreter.h | |
| - name: Install ExecuTorch | |
| run: | | |
| set -eux | |
| python -m pip install --upgrade pip | |
| python install_executorch.py | |
| - name: Configure + build op_test_runner | |
| id: build | |
| run: | | |
| set -eux | |
| cmake --preset mlx-release \ | |
| -DEXECUTORCH_BUILD_TESTS=ON \ | |
| -DEXECUTORCH_MLX_ENABLE_SANITIZERS=OFF | |
| cmake --build cmake-out --target op_test_runner -j2 | |
| df -h / | |
| # ------------------------------------------------------------------ | |
| # SONDE P directe — le verdict. Zéro harnais entre la sonde et le | |
| # log : generate via CLI, puis binaire à la main, flux bruts. | |
| # ------------------------------------------------------------------ | |
| - name: SONDE P directe — binaire à la main, flux bruts | |
| if: steps.build.outcome == 'success' | |
| continue-on-error: true | |
| run: | | |
| set -eux | |
| RUNNER=$(find cmake-out -name op_test_runner -type f | head -1) | |
| echo "probe-in-binary: $(strings "$RUNNER" | grep -c 'GMM-PROBE')" | |
| CFG=gather_mm_e4_i64_o128_sorted | |
| python -m executorch.backends.mlx.test.test_utils "$CFG" generate | |
| D=backends/mlx/test/op_tests/$CFG | |
| ls -la "$D" | |
| set +e | |
| ET_MLX_GMM_PROBE=1 "$RUNNER" \ | |
| --pte "$D/model.pte" \ | |
| --input "$D/input.bin" \ | |
| --output /tmp/probe_out.bin \ | |
| --verbose > /tmp/p_out.txt 2> /tmp/p_err.txt | |
| rc=$? | |
| set -e | |
| echo "=== rc=$rc ===" | |
| echo "=== STDOUT brut ==="; cat /tmp/p_out.txt | |
| echo "=== STDERR brut ==="; cat /tmp/p_err.txt | |
| echo "=== lignes GMM-PROBE ===" | |
| grep -h 'GMM-PROBE' /tmp/p_out.txt /tmp/p_err.txt || \ | |
| echo "AUCUNE — sonde non atteinte malgré binaire ($?)" | |
| # ------------------------------------------------------------------ | |
| # Corroboration harnais — la famille gather_mm complète (les 2 | |
| # sorted rouges + les verts de contrôle). La sonde v2 étant sur | |
| # stdout, elle est maintenant visible via "C++ binary output:". | |
| # ------------------------------------------------------------------ | |
| - name: SONDE P harnais — famille gather_mm (corroboration) | |
| if: steps.build.outcome == 'success' | |
| continue-on-error: true | |
| run: | | |
| set -eux | |
| export ET_MLX_GMM_PROBE=1 | |
| python -m executorch.backends.mlx.test.run_all_tests gather_mm -v | |
| # ------------------------------------------------------------------ | |
| # CENSUS — closure de link réelle + symboles mlx/Metal définis dans | |
| # plusieurs archives liées (candidats ODR). | |
| # ------------------------------------------------------------------ | |
| - name: CENSUS — otool/nm du runner et des archives liées | |
| if: steps.build.outcome == 'success' | |
| continue-on-error: true | |
| run: | | |
| set -eux | |
| RUNNER=$(find cmake-out -name op_test_runner -type f | head -1) | |
| otool -L "$RUNNER" | |
| LINKTXT=$(find cmake-out -path "*op_test_runner.dir/link.txt" | head -1) | |
| echo "==== link.txt ===="; cat "$LINKTXT" | |
| rm -f /tmp/defs.txt | |
| for a in $(tr ' ' '\n' < "$LINKTXT" | grep '\.a$' | sort -u); do | |
| [ -f "$a" ] || continue | |
| nm -gU "$a" 2>/dev/null | awk -v A="$a" \ | |
| '$2 ~ /^[TSDW]$/ {print $3, A}' >> /tmp/defs.txt || true | |
| done | |
| echo "==== symboles mlx/Metal définis dans >=2 archives ====" | |
| grep -iE 'mlx|metal|MTL' /tmp/defs.txt | sort | \ | |
| awk '{print $1}' | uniq -d | head -80 || true | |
| # ------------------------------------------------------------------ | |
| # ÉCHELLE — ne sert que si la sonde dit "image de processus". | |
| # ------------------------------------------------------------------ | |
| - name: ÉCHELLE — L0 mlx seul (contrôle) | |
| if: steps.build.outcome == 'success' | |
| continue-on-error: true | |
| run: | | |
| set -eux | |
| MLXA=$(find cmake-out -name libmlx.a | head -1) | |
| METALLIB=$(find cmake-out -name mlx.metallib | head -1) | |
| cp "$METALLIB" . | |
| clang++ -std=c++17 -O2 repro_gather_mm.cpp \ | |
| -I backends/mlx/third-party/mlx \ | |
| -I "$(dirname "$MLXA")" \ | |
| "$MLXA" \ | |
| -framework Metal -framework Foundation -framework QuartzCore \ | |
| -o rung_L0 | |
| ./rung_L0 && echo "L0: OK" || echo "L0: DIVERGE/CRASH rc=$?" | |
| - name: ÉCHELLE — L1 + mlxdelegate + mlx_schema + executorch_core | |
| if: steps.build.outcome == 'success' | |
| continue-on-error: true | |
| run: | | |
| set -eux | |
| MLXA=$(find cmake-out -name libmlx.a | head -1) | |
| EXTRA="" | |
| for n in libmlxdelegate.a libmlx_schema.a libexecutorch_core.a; do | |
| f=$(find cmake-out -name "$n" | head -1) || true | |
| if [ -n "$f" ]; then EXTRA="$EXTRA -Wl,-force_load,$f"; \ | |
| else echo "ABSENT: $n"; fi | |
| done | |
| echo "EXTRA=$EXTRA" | |
| clang++ -std=c++17 -O2 repro_gather_mm.cpp \ | |
| -I backends/mlx/third-party/mlx \ | |
| -I "$(dirname "$MLXA")" \ | |
| "$MLXA" $EXTRA \ | |
| -framework Metal -framework Foundation -framework QuartzCore \ | |
| -o rung_L1 2>&1 | tail -40 | |
| ./rung_L1 && echo "L1: OK" || echo "L1: DIVERGE/CRASH rc=$?" | |
| - name: ÉCHELLE — L2 + executorch complet + kernels + threadpool | |
| if: steps.build.outcome == 'success' | |
| continue-on-error: true | |
| run: | | |
| set -eux | |
| MLXA=$(find cmake-out -name libmlx.a | head -1) | |
| EXTRA="" | |
| for n in libmlxdelegate.a libmlx_schema.a libexecutorch_core.a \ | |
| libexecutorch.a libcpuinfo.a libpthreadpool.a \ | |
| libextension_threadpool.a; do | |
| f=$(find cmake-out -name "$n" | head -1) || true | |
| if [ -n "$f" ]; then EXTRA="$EXTRA -Wl,-force_load,$f"; \ | |
| else echo "ABSENT: $n"; fi | |
| done | |
| for f in $(find cmake-out -name 'lib*kernels*.a' -o -name 'lib*ops_lib*.a'); do | |
| EXTRA="$EXTRA -Wl,-force_load,$f" | |
| done | |
| echo "EXTRA=$EXTRA" | |
| clang++ -std=c++17 -O2 repro_gather_mm.cpp \ | |
| -I backends/mlx/third-party/mlx \ | |
| -I "$(dirname "$MLXA")" \ | |
| "$MLXA" $EXTRA \ | |
| -framework Metal -framework Foundation -framework QuartzCore \ | |
| -framework Accelerate \ | |
| -o rung_L2 2>&1 | tail -40 | |
| ./rung_L2 && echo "L2: OK" || echo "L2: DIVERGE/CRASH rc=$?" | |
| - name: ÉCHELLE — L3 closure EXACTE (link.txt substitué) | |
| if: steps.build.outcome == 'success' | |
| continue-on-error: true | |
| run: | | |
| set -eux | |
| MLXA=$(find cmake-out -name libmlx.a | head -1) | |
| LINKTXT=$(find cmake-out -path "*op_test_runner.dir/link.txt" | head -1) | |
| BLD=$(dirname "$(dirname "$(dirname "$LINKTXT")")") | |
| echo "link dir: $BLD" | |
| clang++ -std=c++17 -O2 -c repro_gather_mm.cpp \ | |
| -I backends/mlx/third-party/mlx \ | |
| -I "$(dirname "$MLXA")" \ | |
| -o /tmp/repro.o | |
| sed -E \ | |
| -e "s#[^ ]*op_test_runner\.cpp\.o#/tmp/repro.o#" \ | |
| -e "s# -o [^ ]*op_test_runner# -o $PWD/rung_L3#" \ | |
| "$LINKTXT" > /tmp/l3cmd.sh | |
| echo "==== commande L3 ===="; cat /tmp/l3cmd.sh | |
| (cd "$BLD" && bash /tmp/l3cmd.sh) 2>&1 | tail -60 | |
| ./rung_L3 && echo "L3: OK" || echo "L3: DIVERGE/CRASH rc=$?" | |
| - name: Diagnostics finaux | |
| if: always() | |
| run: df -h / |