fix: leo test now returns non-zero exit status when one or more tests fail

compiler/compiler/src/run.rs

@@ -318,10 +318,14 @@ pub fn run_with_ledger(config: &Config, case_sets: &[Vec<Case>]) -> Result<Vec<V
         let mut deploy = || -> Result<()> {
             // Add the program to the ledger.
             // Note that this function performs an additional validity check on the bytecode.
-            let deployment = ledger
-                .vm()
-                .deploy(&genesis_private_key, &aleo_program, None, 0, None, &mut rng)
-                .map_err(|e| anyhow!("Failed to deploy program {name}: {e}"))?;
+            // Catch panics during deployment (e.g., constraint failures during key synthesis).
+            let deployment = match catch_unwind(AssertUnwindSafe(|| {
+                ledger.vm().deploy(&genesis_private_key, &aleo_program, None, 0, None, &mut rng)
+            })) {
+                Ok(Ok(deployment)) => deployment,
+                Ok(Err(e)) => return Err(anyhow!("Failed to deploy program {name}: {e}").into()),
+                Err(e) => return Err(anyhow!("Panic during deployment of program {name}: {e:?}").into()),
+            };
             let block = ledger
                 .prepare_advance_to_next_beacon_block(&genesis_private_key, vec![], vec![], vec![deployment], &mut rng)
                 .map_err(|e| anyhow!("Failed to prepare to advance block for program {name}: {e}"))?;

errors/src/errors/cli/cli_errors.rs

@@ -334,4 +334,11 @@ create_messages!(
         msg: format!("{msg}"),
         help: None,
     }
+
+    @backtraced
+    tests_failed {
+        args: (failed: impl Display, total: impl Display),
+        msg: format!("{failed} out of {total} tests failed"),
+        help: None,
+    }
 );

leo/cli/commands/test.rs

@@ -178,6 +178,6 @@ fn handle_test(command: LeoTest, package: Package) -> Result<()> {
             }
         }
 
-        Ok(())
+        if total_passed < total { Err(CliError::tests_failed(total - total_passed, total).into()) } else { Ok(()) }
     }
 }

leo/tests/integration.rs

@@ -115,6 +115,12 @@ fn run_test(test: &Test, force_rewrite: bool) -> Option<String> {
     let stderr_utf8 = std::str::from_utf8(&output.stderr).expect("stderr should be utf8");
     fs::write(&stderr_path, filter_stderr(stderr_utf8).as_bytes()).expect("Failed to write STDERR");
 
+    // Capture exit code if the command exited normally
+    let exitcode_path = test_context_directory.path().join("EXITCODE");
+    if let Some(code) = output.status.code() {
+        fs::write(&exitcode_path, code.to_string().as_bytes()).expect("Failed to write EXITCODE");
+    }
+
     if force_rewrite {
         copy_recursively(test_context_directory.path(), &test.expectation_directory)
             .expect("Failed to copy directory.");

tests/expectations/cli/network_dependency/EXITCODE

@@ -0,0 +1 @@
+0

tests/expectations/cli/program_name_mismatch/EXITCODE

@@ -0,0 +1 @@
+196

tests/expectations/cli/test_add/EXITCODE

@@ -0,0 +1 @@
+0

tests/expectations/cli/test_command_shortcuts/EXITCODE

@@ -0,0 +1 @@
+0

tests/expectations/cli/test_deploy/EXITCODE

@@ -0,0 +1 @@
+0

tests/expectations/cli/test_failure_exit_code/COMMANDS

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+LEO_BIN=${1}
+
+${LEO_BIN} test

tests/expectations/cli/test_failure_exit_code/EXITCODE

@@ -0,0 +1 @@
+248

tests/expectations/cli/test_failure_exit_code/STDOUT

@@ -0,0 +1,11 @@
+⚠️ No network specified, defaulting to 'testnet'.
+⚠️ No endpoint specified, defaulting to 'https://api.explorer.provable.com/v1'.
+       Leo     2 statements before dead code elimination.
+       Leo     2 statements after dead code elimination.
+       Leo     The program checksum is: '[169u8, 44u8, 143u8, 18u8, 47u8, 196u8, 2u8, 77u8, 84u8, 100u8, 131u8, 15u8, 54u8, 173u8, 74u8, 56u8, 55u8, 110u8, 248u8, 62u8, 168u8, 75u8, 162u8, 68u8, 220u8, 201u8, 90u8, 144u8, 1u8, 150u8, 69u8, 68u8]'.
+       Leo ✅ Compiled 'test_failure_exit_code.aleo' into Aleo instructions.
+       Leo     3 statements before dead code elimination.
+       Leo     3 statements after dead code elimination.
+       Leo     The program checksum is: '[102u8, 47u8, 165u8, 186u8, 89u8, 127u8, 235u8, 8u8, 218u8, 104u8, 153u8, 232u8, 55u8, 84u8, 184u8, 59u8, 59u8, 81u8, 222u8, 235u8, 199u8, 43u8, 69u8, 129u8, 112u8, 197u8, 74u8, 39u8, 13u8, 220u8, 22u8, 196u8]'.
+       Leo ✅ Compiled 'test_failure.aleo' into Aleo instructions.
+       Leo Loading the ledger from storage...

tests/expectations/cli/test_failure_exit_code/contents/.gitignore

@@ -0,0 +1,5 @@
+outputs/
+build/
+*.avm
+*.prover
+*.verifier

tests/expectations/cli/test_failure_exit_code/contents/program.json

@@ -0,0 +1,6 @@
+{
+    "program": "test_failure_exit_code.aleo",
+    "version": "1.0.0",
+    "description": "Test program for exit code verification",
+    "license": "MIT"
+}

tests/expectations/cli/test_failure_exit_code/contents/src/main.leo

@@ -0,0 +1,8 @@
+program test_failure_exit_code.aleo {
+    transition main() -> u8 {
+        return 42u8;
+    }
+
+    @noupgrade
+    async constructor() {}
+}

tests/expectations/cli/test_failure_exit_code/contents/tests/test_failure.leo

@@ -0,0 +1,9 @@
+// Test to verify that test failures exit with non-zero exit code
+import test_failure_exit_code.aleo;
+
+program test_failure.aleo {
+    @test
+    transition test_should_fail() {
+        assert_eq(1u8, 2u8);
+    }
+}

tests/expectations/cli/test_simple_build/EXITCODE

@@ -0,0 +1 @@
+0

tests/tests/cli/test_failure_exit_code/COMMANDS

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+LEO_BIN=${1}
+
+${LEO_BIN} test

tests/tests/cli/test_failure_exit_code/contents/.gitignore

@@ -0,0 +1,5 @@
+outputs/
+build/
+*.avm
+*.prover
+*.verifier

tests/tests/cli/test_failure_exit_code/contents/program.json

@@ -0,0 +1,6 @@
+{
+    "program": "test_failure_exit_code.aleo",
+    "version": "1.0.0",
+    "description": "Test program for exit code verification",
+    "license": "MIT"
+}

tests/tests/cli/test_failure_exit_code/contents/src/main.leo

@@ -0,0 +1,8 @@
+program test_failure_exit_code.aleo {
+    transition main() -> u8 {
+        return 42u8;
+    }
+
+    @noupgrade
+    async constructor() {}
+}

tests/tests/cli/test_failure_exit_code/contents/tests/test_failure.leo

@@ -0,0 +1,9 @@
+// Test to verify that test failures exit with non-zero exit code
+import test_failure_exit_code.aleo;
+
+program test_failure.aleo {
+    @test
+    transition test_should_fail() {
+        assert_eq(1u8, 2u8);
+    }
+}