diff --git a/pytest_pyodide/decorator.py b/pytest_pyodide/decorator.py
index ac51f93c..0f29f81f 100644
--- a/pytest_pyodide/decorator.py
+++ b/pytest_pyodide/decorator.py
@@ -1,5 +1,6 @@
 import ast
 import functools
+import os
 import pickle
 import sys
 from base64 import b64decode, b64encode
@@ -383,6 +384,7 @@ def __init__(
         packages: Collection[str] = (),
         pytest_assert_rewrites: bool = True,
         *,
+        skip_if_not_all_built: bool = "CI" in os.environ,
         _force_assert_rewrites: bool = False,
     ):
         """
@@ -402,8 +404,8 @@ def __init__(
             If True, use pytest assertion rewrites. This gives better error messages
             when an assertion fails, but requires us to load pytest.
         """
-
         self._pkgs = list(packages)
+        self._skip_if_not_all_built = skip_if_not_all_built
         pytest_assert_rewrites = _force_assert_rewrites or (
             pytest_assert_rewrites and package_is_built("pytest")
         )
@@ -447,10 +449,18 @@ def __call__(self, f: Callable) -> Callable:
     def _run(self, selenium: SeleniumType, args: tuple):
         """The main runner, called from the AST generated in _create_outer_func."""
         __tracebackhide__ = True
-        code = self._code_template(args)
-        if self._pkgs:
-            selenium.load_package(self._pkgs)
 
+        unbuilt = sorted(pkg for pkg in self._pkgs if not package_is_built(pkg))
+        if unbuilt:
+            msg = "Requires unbuilt packages: " + ", ".join(unbuilt)
+            if "PYTEST_CURRENT_TEST" not in os.environ:
+                raise RuntimeError(msg)
+            if self._skip_if_not_all_built:
+                pytest.skip(msg)
+            pytest.fail(msg)
+        selenium.load_package(self._pkgs)
+
+        code = self._code_template(args)
         r = selenium.run_async(code)
         [status, result] = r
 
diff --git a/pytest_pyodide/runner.py b/pytest_pyodide/runner.py
index b2f71c77..32341c05 100644
--- a/pytest_pyodide/runner.py
+++ b/pytest_pyodide/runner.py
@@ -327,6 +327,8 @@ def run_webworker(self, code):
         )
 
     def load_package(self, packages):
+        if not packages:
+            return
         self.run_js(f"await pyodide.loadPackage({packages!r})")