diff --git a/CHANGELOG.md b/CHANGELOG.md index 508397ec..d06e7986 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.34.0] - 2026/03/31 + ### Added - pyodide xbuildenv install can skip eager installation, and cross-build packages are installed on first build-time use. diff --git a/pyodide_build/build_env.py b/pyodide_build/build_env.py index ebbc699d..6737dd83 100644 --- a/pyodide_build/build_env.py +++ b/pyodide_build/build_env.py @@ -359,8 +359,6 @@ def ensure_emscripten(skip_install: bool = False) -> None: RuntimeError If emcc is not found and auto-install is skipped, or if version mismatch. """ - from pyodide_build.logger import logger - from pyodide_build.xbuildenv import CrossBuildEnvManager # avoid circular import # Check if auto-install should be skipped env_skip = os.environ.get("PYODIDE_SKIP_EMSCRIPTEN_INSTALL", "") @@ -380,20 +378,8 @@ def ensure_emscripten(skip_install: bool = False) -> None: f"No Emscripten compiler found. Need Emscripten version {needed_version}" ) from None - # Get the xbuildenv path from the already-initialized pyodide root - # pyodide_root is at {xbuild_root}/xbuildenv/pyodide-root - # so xbuild_root is pyodide_root.parent.parent - pyodide_root = get_pyodide_root() - xbuild_root = pyodide_root.parent.parent - emsdk_dir = xbuild_root / "emsdk" - emsdk_env_script = emsdk_dir / "emsdk_env.sh" - - if emsdk_env_script.exists(): - logger.info("Emscripten found but not activated, activating...") - else: - logger.info("Emscripten not found, installing...") - manager = CrossBuildEnvManager(xbuild_root.parent) - emsdk_dir = manager.install_emscripten(needed_version) + manager = get_current_xbuildenv_manager() + emsdk_dir = manager.install_emscripten(needed_version) env_vars = activate_emscripten_env(emsdk_dir) os.environ.update(env_vars) diff --git a/pyodide_build/cli/xbuildenv.py b/pyodide_build/cli/xbuildenv.py index 0d6e7919..3b267727 100644 --- a/pyodide_build/cli/xbuildenv.py +++ b/pyodide_build/cli/xbuildenv.py @@ -263,14 +263,25 @@ def _search( default=DEFAULT_PATH, help="Pyodide cross-env path", ) +@click.option( + "--force", + "-f", + is_flag=True, + default=False, + help="force reinstallation even if the same version is already installed.", +) def _install_emscripten( version: str | None, path: Path, + force: bool, ) -> None: """Install Emscripten SDK into the cross-build environment. This command clones the emsdk repository, installs and activates the specified Emscripten version, and applies Pyodide-specific patches. + + If the requested version is already installed, the command is a no-op unless + --force is passed. """ check_xbuildenv_root(path) manager = CrossBuildEnvManager(path) @@ -280,7 +291,7 @@ def _install_emscripten( print("Installing emsdk...") - emsdk_dir = manager.install_emscripten(version) + emsdk_dir = manager.install_emscripten(version, force=force) print("Installing emsdk complete.") print(f"Use `source {emsdk_dir}/emsdk_env.sh` to set up the environment.") diff --git a/pyodide_build/tests/test_cli_install_emscripten.py b/pyodide_build/tests/test_cli_install_emscripten.py index ff04a9a0..6f268e7d 100644 --- a/pyodide_build/tests/test_cli_install_emscripten.py +++ b/pyodide_build/tests/test_cli_install_emscripten.py @@ -39,7 +39,7 @@ def test_install_emscripten_default_version(tmp_path, monkeypatch): called = {} - def fake_install(self, version): + def fake_install(self, version, *, force=False): called["version"] = version return self.env_dir / "emsdk" @@ -72,7 +72,7 @@ def test_install_emscripten_specific_version(tmp_path, monkeypatch): called = {} - def fake_install(self, version): + def fake_install(self, version, *, force=False): called["version"] = version return self.env_dir / "emsdk" @@ -112,7 +112,7 @@ def test_install_emscripten_with_existing_emsdk(tmp_path, monkeypatch): lambda name: "latest", ) - def fake_install(self, version): + def fake_install(self, version, *, force=False): assert version == "latest" assert existing_emsdk.exists() return existing_emsdk @@ -144,7 +144,7 @@ def test_install_emscripten_git_failure(tmp_path, monkeypatch): monkeypatch.setattr( "pyodide_build.xbuildenv.CrossBuildEnvManager.install_emscripten", - lambda self, version: (_ for _ in ()).throw( + lambda self, version, *, force=False: (_ for _ in ()).throw( subprocess.CalledProcessError(1, "git clone") ), ) @@ -170,7 +170,7 @@ def test_install_emscripten_emsdk_install_failure(tmp_path, monkeypatch): monkeypatch.setattr( "pyodide_build.xbuildenv.CrossBuildEnvManager.install_emscripten", - lambda self, version: (_ for _ in ()).throw( + lambda self, version, *, force=False: (_ for _ in ()).throw( subprocess.CalledProcessError(1, "./emsdk install") ), ) @@ -189,6 +189,42 @@ def test_install_emscripten_emsdk_install_failure(tmp_path, monkeypatch): assert isinstance(result.exception, subprocess.CalledProcessError) +def test_install_emscripten_force_flag(tmp_path, monkeypatch): + """Test that --force flag triggers reinstallation""" + envpath = Path(tmp_path) / ".xbuildenv" + envpath.mkdir() + + called = {} + + def fake_install(self, version, *, force=False): + called["version"] = version + called["force"] = force + return self.env_dir / "emsdk" + + monkeypatch.setattr( + "pyodide_build.cli.xbuildenv.get_build_flag", + lambda name: "3.1.46", + ) + monkeypatch.setattr( + "pyodide_build.xbuildenv.CrossBuildEnvManager.install_emscripten", + fake_install, + ) + + result = runner.invoke( + xbuildenv.app, + [ + "install-emscripten", + "--force", + "--path", + str(envpath), + ], + ) + + assert result.exit_code == 0, result.output + assert called["force"] is True + assert called["version"] == "3.1.46" + + def test_install_emscripten_output_format(tmp_path, monkeypatch): """Test that the output message format is correct""" envpath = Path(tmp_path) / ".xbuildenv" @@ -203,7 +239,7 @@ def test_install_emscripten_output_format(tmp_path, monkeypatch): monkeypatch.setattr( "pyodide_build.xbuildenv.CrossBuildEnvManager.install_emscripten", - lambda self, version: expected_path, + lambda self, version, *, force=False: expected_path, ) result = runner.invoke( diff --git a/pyodide_build/tests/test_install_emscripten.py b/pyodide_build/tests/test_install_emscripten.py index d0054476..6c4e399e 100644 --- a/pyodide_build/tests/test_install_emscripten.py +++ b/pyodide_build/tests/test_install_emscripten.py @@ -53,26 +53,33 @@ def test_clone_emscripten_fresh_clone(tmp_path, monkeypatch): def test_clone_emscripten_already_exists(tmp_path, monkeypatch): - """Test that _clone_emscripten pulls updates when emsdk already exists""" + """Test that _clone_emscripten removes existing dir and clones fresh""" manager = CrossBuildEnvManager(tmp_path) - # Setup: create active xbuildenv with existing emsdk version_dir = tmp_path / "0.28.0" version_dir.mkdir() emsdk_dir = version_dir / "emsdk" - emsdk_dir.mkdir() # Simulate existing emsdk + emsdk_dir.mkdir() manager.use_version("0.28.0") - # Mock subprocess.run mock_run = MagicMock(return_value=subprocess.CompletedProcess([], 0)) monkeypatch.setattr(subprocess, "run", mock_run) - # Execute result = manager._clone_emscripten() - # Verify - should pull instead of clone assert result == emsdk_dir - mock_run.assert_called_once_with(["git", "-C", str(emsdk_dir), "pull"], check=True) + assert not emsdk_dir.exists() + mock_run.assert_called_once_with( + [ + "git", + "clone", + "--depth", + "1", + "https://github.com/emscripten-core/emsdk.git", + str(emsdk_dir), + ], + check=True, + ) def test_install_emscripten_no_active_xbuildenv(tmp_path): @@ -210,51 +217,54 @@ def mock_run_side_effect(cmd, **kwargs): def test_install_emscripten_with_existing_emsdk(tmp_path, monkeypatch): - """Test installing Emscripten when emsdk directory already exists""" + """Test installing Emscripten removes existing emsdk and clones fresh""" manager = CrossBuildEnvManager(tmp_path) - # Setup: create active xbuildenv with existing emsdk version_dir = tmp_path / "0.28.0" version_dir.mkdir() emsdk_dir = version_dir / "emsdk" - emsdk_dir.mkdir() # Simulate existing emsdk - patches_dir = emsdk_dir / "patches" - patches_dir.mkdir() - (patches_dir / "test.patch").write_text("--- a/test\n+++ b/test\n") + emsdk_dir.mkdir() upstream_emscripten = emsdk_dir / "upstream" / "emscripten" - upstream_emscripten.mkdir(parents=True) - manager.use_version("0.28.0") - # Mock subprocess.run - mock_run = MagicMock(return_value=subprocess.CompletedProcess([], 0)) + def mock_run_side_effect(cmd, **kwargs): + if isinstance(cmd, list) and "clone" in cmd: + upstream_emscripten.mkdir(parents=True, exist_ok=True) + return subprocess.CompletedProcess([], 0) + + mock_run = MagicMock(side_effect=mock_run_side_effect) monkeypatch.setattr(subprocess, "run", mock_run) + manager.use_version("0.28.0") - # Execute result = manager.install_emscripten() - # Verify - should pull, then install, patch, and activate assert result == emsdk_dir assert mock_run.call_count == 4 calls = mock_run.call_args_list - # 1. Pull latest changes (not clone) - assert calls[0] == call(["git", "-C", str(emsdk_dir), "pull"], check=True) + assert calls[0] == call( + [ + "git", + "clone", + "--depth", + "1", + "https://github.com/emscripten-core/emsdk.git", + str(emsdk_dir), + ], + check=True, + ) - # 2. Install emsdk assert calls[1] == call( ["./emsdk", "install", "--build=Release", "latest"], cwd=emsdk_dir, check=True, ) - # 3. Apply patches patch_cmd = calls[2][0][0] assert "patch" in patch_cmd assert calls[2][1]["shell"] is True assert calls[2][1]["cwd"] == upstream_emscripten - # 4. Activate emsdk assert calls[3] == call( ["./emsdk", "activate", "--embedded", "--build=Release", "latest"], cwd=emsdk_dir, @@ -262,6 +272,103 @@ def test_install_emscripten_with_existing_emsdk(tmp_path, monkeypatch): ) +def test_install_emscripten_skips_when_already_installed(tmp_path, monkeypatch): + manager = CrossBuildEnvManager(tmp_path) + + version_dir = tmp_path / "0.28.0" + version_dir.mkdir() + manager.use_version("0.28.0") + + emsdk_dir = version_dir / "emsdk" + emsdk_dir.mkdir() + + marker = version_dir / ".emscripten-version" + marker.write_text("3.1.46") + + mock_run = MagicMock(return_value=subprocess.CompletedProcess([], 0)) + monkeypatch.setattr(subprocess, "run", mock_run) + + result = manager.install_emscripten("3.1.46") + + assert result == emsdk_dir + mock_run.assert_not_called() + + +def test_install_emscripten_reinstalls_different_version(tmp_path, monkeypatch): + manager = CrossBuildEnvManager(tmp_path) + + version_dir = tmp_path / "0.28.0" + version_dir.mkdir() + manager.use_version("0.28.0") + + emsdk_dir = version_dir / "emsdk" + emsdk_dir.mkdir() + upstream_emscripten = emsdk_dir / "upstream" / "emscripten" + upstream_emscripten.mkdir(parents=True) + + marker = version_dir / ".emscripten-version" + marker.write_text("3.1.45") + + mock_run = MagicMock(return_value=subprocess.CompletedProcess([], 0)) + monkeypatch.setattr(subprocess, "run", mock_run) + + result = manager.install_emscripten("3.1.46") + + assert result == emsdk_dir + assert mock_run.call_count == 4 + assert marker.read_text() == "3.1.46" + + +def test_install_emscripten_force_reinstalls_same_version(tmp_path, monkeypatch): + manager = CrossBuildEnvManager(tmp_path) + + version_dir = tmp_path / "0.28.0" + version_dir.mkdir() + manager.use_version("0.28.0") + + emsdk_dir = version_dir / "emsdk" + emsdk_dir.mkdir() + upstream_emscripten = emsdk_dir / "upstream" / "emscripten" + upstream_emscripten.mkdir(parents=True) + + marker = version_dir / ".emscripten-version" + marker.write_text("3.1.46") + + mock_run = MagicMock(return_value=subprocess.CompletedProcess([], 0)) + monkeypatch.setattr(subprocess, "run", mock_run) + + result = manager.install_emscripten("3.1.46", force=True) + + assert result == emsdk_dir + assert mock_run.call_count == 4 + assert marker.read_text() == "3.1.46" + + +def test_install_emscripten_writes_marker_on_success(tmp_path, monkeypatch): + manager = CrossBuildEnvManager(tmp_path) + + version_dir = tmp_path / "0.28.0" + version_dir.mkdir() + manager.use_version("0.28.0") + + emsdk_dir = version_dir / "emsdk" + upstream_emscripten = emsdk_dir / "upstream" / "emscripten" + + def mock_run_side_effect(cmd, **kwargs): + if isinstance(cmd, list) and "clone" in cmd: + upstream_emscripten.mkdir(parents=True, exist_ok=True) + return subprocess.CompletedProcess([], 0) + + mock_run = MagicMock(side_effect=mock_run_side_effect) + monkeypatch.setattr(subprocess, "run", mock_run) + + manager.install_emscripten("3.1.46") + + marker = version_dir / ".emscripten-version" + assert marker.exists() + assert marker.read_text() == "3.1.46" + + def test_install_emscripten_patch_fails(tmp_path, monkeypatch): """Test handling of patch application failure due to version mismatch""" manager = CrossBuildEnvManager(tmp_path) diff --git a/pyodide_build/xbuildenv.py b/pyodide_build/xbuildenv.py index 6cff42c0..78a87880 100644 --- a/pyodide_build/xbuildenv.py +++ b/pyodide_build/xbuildenv.py @@ -19,6 +19,7 @@ CDN_BASE = "https://cdn.jsdelivr.net/pyodide/v{version}/full/" PYTHON_VERSION_MARKER_FILE = ".build-python-version" CROSS_BUILD_PACKAGES_MARKER_FILE = ".cross-build-packages-installed" +EMSCRIPTEN_VERSION_MARKER_FILE = ".emscripten-version" class CrossBuildEnvManager: @@ -473,24 +474,26 @@ def _clone_emscripten(self, emsdk_dir: Path | str | None = None) -> Path: logger.info("Cloning Emscripten SDK into %s", emsdk_dir) if emsdk_dir.exists(): - logger.info("Emsdk directory already exists, pulling latest changes...") - subprocess.run(["git", "-C", str(emsdk_dir), "pull"], check=True) - else: - subprocess.run( - [ - "git", - "clone", - "--depth", - "1", - "https://github.com/emscripten-core/emsdk.git", - str(emsdk_dir), - ], - check=True, - ) + logger.info("Removing existing emsdk directory %s", emsdk_dir) + shutil.rmtree(emsdk_dir) + + subprocess.run( + [ + "git", + "clone", + "--depth", + "1", + "https://github.com/emscripten-core/emsdk.git", + str(emsdk_dir), + ], + check=True, + ) return emsdk_dir - def install_emscripten(self, emscripten_version: str = "latest") -> Path: + def install_emscripten( + self, emscripten_version: str = "latest", *, force: bool = False + ) -> Path: """ Install and activate Emscripten SDK inside the currently selected xbuildenv. @@ -498,6 +501,8 @@ def install_emscripten(self, emscripten_version: str = "latest") -> Path: ---------- emscripten_version The Emscripten SDK version to install (default: 'latest'). + force + If True, force reinstallation even if the same version is already installed. Returns ------- @@ -514,6 +519,17 @@ def install_emscripten(self, emscripten_version: str = "latest") -> Path: patches_dir = self.pyodide_root / "emsdk" / "patches" emscripten_root = emsdk_dir / "upstream" / "emscripten" + marker = xbuild_root / EMSCRIPTEN_VERSION_MARKER_FILE + if not force and marker.exists(): + installed_version = marker.read_text().strip() + if installed_version == emscripten_version: + logger.debug( + "Emscripten SDK (version: %s) is already installed at %s, skipping", + emscripten_version, + emsdk_dir, + ) + return emsdk_dir + logger.info( "Installing Emscripten SDK (version: %s) into %s", emscripten_version, @@ -559,6 +575,8 @@ def install_emscripten(self, emscripten_version: str = "latest") -> Path: check=True, ) + marker.write_text(emscripten_version) + logger.info("Emscripten SDK installed successfully at %s", emsdk_dir) return emsdk_dir