Better bindings tests

This comes from my experiences with mozilla#2333 and uniffi-bindgen-gecko-js.
For those, I needed to run a lot of tests while (re)implementing a
bindings generator.  I love how the current tests cover basically all
UniFFI features, but I think there a few things that can be improved.
This commit adds a suite of tests specifically aimed at the bindings.

The new tests target specific features rather than trying to mimic
real-world examples.  Before, I spent a lot of time trying to figure out
what a test failure meant.  I'm hoping that when one of these tests
fail, it's obvious what's not working.  It also means that the new test
suites don't require writing so much foreign code.

They also have a better external bindings story.  I'm thinking we can
direct external buindings authors to copy everything inside the
`bindings-tests` directory and use that to test their bindings.  They
can use the test macro now, since it's doesn't hard code the mapping
from file extension to test runner.  Copying the code feels a bit weird,
but I like it better than having to publish all these test fixture
crates and I think it will work okay in practice.

I still want to keep the current examples/fixtures. They're nice because
they showcase real-world use-cases and test that those use-cases work
like we expect them to. If this gets merged, we can rework those crates
to focus on that.  Maybe we rework them to only test one bindings
language per example/fixture.

Cargo.toml

@@ -24,6 +24,9 @@ members = [
   "examples/todolist",
   "examples/traits",
 
+  "bindings-tests",
+  "bindings-tests/fixtures/*",
+
   "fixtures/benchmarks",
   "fixtures/coverall",
   "fixtures/callbacks",

bindings-tests/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "bindings-tests"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+uniffi = { path = "../uniffi", features = ["bindgen-tests"] }
+
+[features]
+ffi-trace = ["uniffi/ffi-trace"]

bindings-tests/fixtures/fn-calls/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "uniffi-fixture-fn-calls"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+uniffi = { workspace = true }
+
+[features]
+ffi-trace = ["uniffi/ffi-trace"]

bindings-tests/fixtures/fn-calls/src/lib.rs

@@ -0,0 +1,9 @@
+//! Extremely simple fixture to test making scaffolding calls without any arguments or return types
+//!
+//! The test is if the bindings can make a call to `test_func`.  If in doubt, run the tests with
+//! `--features=ffi-trace` to check that the function is actually called.
+
+#[uniffi::export]
+pub fn test_func() {}
+
+uniffi::setup_scaffolding!("fn_calls");

bindings-tests/fixtures/primitive-types/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "uniffi-fixture-primitive-types"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+uniffi = { workspace = true }
+
+[features]
+ffi-trace = ["uniffi/ffi-trace"]

bindings-tests/fixtures/primitive-types/src/lib.rs

@@ -0,0 +1,46 @@
+//! Test lifting/lowering primitive types
+
+// Simple tests
+
+#[uniffi::export]
+pub fn roundtrip(a: u32) -> u32 {
+    a
+}
+
+#[uniffi::export]
+pub fn roundtrip_bool(a: bool) -> bool {
+    a
+}
+
+#[uniffi::export]
+pub fn roundtrip_string(a: String) -> String {
+    a
+}
+
+/// Complex test: input a bunch of different values and add them together
+#[uniffi::export]
+pub fn sum(
+    a: u8,
+    b: i8,
+    c: u16,
+    d: i16,
+    e: u32,
+    f: i32,
+    g: u64,
+    h: i64,
+    i: f32,
+    j: f64,
+    negate: bool,
+) -> f64 {
+    let all_values = [
+        a as f64, b as f64, c as f64, d as f64, e as f64, f as f64, g as f64, h as f64, i as f64, j,
+    ];
+    let sum: f64 = all_values.into_iter().sum();
+    if negate {
+        -sum
+    } else {
+        sum
+    }
+}
+
+uniffi::setup_scaffolding!("primitive_types");

bindings-tests/python/uniffi-fixture-fn-calls.py

@@ -0,0 +1,12 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import fn_calls
+import unittest
+
+class FnCallsTest(unittest.TestCase):
+    def test_fn_call(self):
+        fn_calls.test_func()
+
+unittest.main()

bindings-tests/python/uniffi-fixture-primitive-types.py

@@ -0,0 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import primitive_types
+import unittest
+
+class FnCallsTest(unittest.TestCase):
+    def test_roundtrip(self):
+        self.assertEqual(primitive_types.roundtrip(0), 0)
+        self.assertEqual(primitive_types.roundtrip(42), 42)
+
+    def test_roundtrip_bool(self):
+        self.assertEqual(primitive_types.roundtrip_bool(True), True)
+        self.assertEqual(primitive_types.roundtrip_bool(False), False)
+
+    def test_roundtrip_string(self):
+        self.assertEqual(primitive_types.roundtrip_string("Hello"), "Hello")
+
+    def test_sum(self):
+        self.assertEqual(primitive_types.sum(1, -1, 2, -2, 3, -3, 4, -4, 0.5, 1.5, True), -2.0)
+
+unittest.main()

bindings-tests/src/lib.rs

@@ -0,0 +1 @@
+

bindings-tests/tests/tests.rs

@@ -0,0 +1,11 @@
+use uniffi::deps::anyhow::Result;
+
+uniffi::bindings_tests!(
+    py: run_python_test,
+    // TODO Kotlin/Swift/Ruby
+);
+
+pub fn run_python_test(tmp_dir: &str, fixture_name: &str) -> Result<()> {
+    let script_name = format!("python/{fixture_name}.py");
+    uniffi::python_test::run_test(tmp_dir, fixture_name, &script_name)
+}

uniffi/src/lib.rs

@@ -25,6 +25,8 @@ pub use uniffi_bindgen::{
 #[cfg(feature = "build")]
 pub use uniffi_build::{generate_scaffolding, generate_scaffolding_for_crate};
 #[cfg(feature = "bindgen-tests")]
+pub use uniffi_macros::bindings_tests;
+#[cfg(feature = "bindgen-tests")]
 pub use uniffi_macros::build_foreign_language_testcases;
 
 #[cfg(feature = "cli")]

uniffi_bindgen/src/bindings/python/test.rs

@@ -5,7 +5,7 @@ License, v. 2.0. If a copy of the MPL was not distributed with this
 use crate::bindings::RunScriptOptions;
 use crate::cargo_metadata::CrateConfigSupplier;
 use crate::library_mode::generate_bindings;
-use anyhow::{Context, Result};
+use anyhow::{bail, Context, Result};
 use camino::Utf8Path;
 use std::env;
 use std::ffi::OsString;
@@ -33,7 +33,11 @@ pub fn run_script(
     args: Vec<String>,
     _options: &RunScriptOptions,
 ) -> Result<()> {
-    let script_path = Utf8Path::new(script_file).canonicalize_utf8()?;
+    let script_path = Utf8Path::new(script_file);
+    if !script_path.exists() {
+        bail!("{script_path} not found");
+    }
+    let script_path = script_path.canonicalize_utf8()?;
     let test_helper = UniFFITestHelper::new(crate_name)?;
     let out_dir = test_helper.create_out_dir(tmp_dir, &script_path)?;
     let cdylib_path = test_helper.copy_cdylib_to_out_dir(&out_dir)?;

uniffi_macros/src/lib.rs

@@ -32,6 +32,14 @@ use self::{
     object::expand_object, record::expand_record,
 };
 
+/// A macro to build test cases for a bindings generator.
+///
+/// See `bindings-tests/tests/tests.rs` for an example of how this is used.
+#[proc_macro]
+pub fn bindings_tests(tokens: TokenStream) -> TokenStream {
+    test::bindings_tests(tokens)
+}
+
 /// A macro to build testcases for a component's generated bindings.
 ///
 /// This macro provides some plumbing to write automated tests for the generated

uniffi_macros/src/test.rs

@@ -6,7 +6,88 @@ use camino::{Utf8Path, Utf8PathBuf};
 use proc_macro::TokenStream;
 use quote::{format_ident, quote};
 use std::env;
-use syn::{parse_macro_input, punctuated::Punctuated, LitStr, Token};
+use syn::{parse_macro_input, punctuated::Punctuated, Ident, LitStr, Token};
+
+const TEST_FIXTURES: &[&'static str] = &[
+    "uniffi-fixture-fn-calls",
+    "uniffi-fixture-primitive-types",
+    // TODO
+    // "uniffi-fixture-compound-types",
+    // "uniffi-fixture-extra-types",
+    // "uniffi-fixture-errors",
+    // "uniffi-fixture-records",
+    // "uniffi-fixture-enums",
+    // "uniffi-fixture-interfaces",
+    // "uniffi-fixture-interface-errors",
+    // "uniffi-fixture-callbacks",
+    // "uniffi-fixture-trait-interfaces",
+    // "uniffi-fixture-async",
+    // "uniffi-fixture-async-callbacks",
+    // "uniffi-fixture-custom-types",
+    // "uniffi-fixture-external-types",
+    // "uniffi-fixture-keywords",
+    // "uniffi-fixture-defaults",
+    // "uniffi-fixture-rust-trait-methods",
+];
+
+pub(crate) fn bindings_tests(tokens: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(tokens as BindingsTestsInput);
+
+    // For each test file found, generate a matching testcase.
+    let test_functions = input
+        .runners
+        .iter()
+        .flat_map(|runner| TEST_FIXTURES.iter().map(move |fixture| (fixture, runner)))
+        .map(|(fixture, runner)| {
+            let runner_name = &runner.name;
+            let runner_ident = &runner.runner;
+            let test_name = format_ident!(
+                "{}_{runner_name}",
+                fixture.replace(|c: char| !c.is_alphanumeric(), "_")
+            );
+            quote! {
+                #[test]
+                fn #test_name () -> ::uniffi::deps::anyhow::Result<()> {
+                    #runner_ident(std::env!("CARGO_TARGET_TMPDIR"), #fixture)
+                }
+            }
+        })
+        .collect::<Vec<proc_macro2::TokenStream>>();
+    let test_module = quote! {
+        #(#test_functions)*
+    };
+    TokenStream::from(test_module)
+}
+
+struct BindingsTestsInput {
+    runners: Vec<BindingsTestRunner>,
+}
+
+struct BindingsTestRunner {
+    name: Ident,
+    _sep: Token![:],
+    runner: Ident,
+}
+
+impl syn::parse::Parse for BindingsTestsInput {
+    fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
+        let runners = Punctuated::<BindingsTestRunner, Token![,]>::parse_terminated(input)?
+            .into_iter()
+            .collect();
+
+        Ok(Self { runners })
+    }
+}
+
+impl syn::parse::Parse for BindingsTestRunner {
+    fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
+        Ok(Self {
+            name: input.parse()?,
+            _sep: input.parse()?,
+            runner: input.parse()?,
+        })
+    }
+}
 
 pub(crate) fn build_foreign_language_testcases(tokens: TokenStream) -> TokenStream {
     let input = parse_macro_input!(tokens as BuildForeignLanguageTestCaseInput);