diff --git a/micropip/__init__.py b/micropip/__init__.py index 3a890d3..be7729e 100644 --- a/micropip/__init__.py +++ b/micropip/__init__.py @@ -1,19 +1,24 @@ -from ._commands.freeze import freeze -from ._commands.index_urls import set_index_urls -from ._commands.install import install -from ._commands.list import _list as list -from ._commands.mock_package import ( - add_mock_package, - list_mock_packages, - remove_mock_package, -) from ._commands.uninstall import uninstall +from .package_manager import PackageManager try: from ._version import __version__ except ImportError: pass +singleton_package_manager = PackageManager() + +install = singleton_package_manager.install +set_index_urls = singleton_package_manager.set_index_urls +list = singleton_package_manager.list +freeze = singleton_package_manager.freeze +add_mock_package = singleton_package_manager.add_mock_package +list_mock_packages = singleton_package_manager.list_mock_packages +remove_mock_package = singleton_package_manager.remove_mock_package + +# TODO: port uninstall +# uninstall = singleton_package_manager.uninstall + __all__ = [ "install", "list", diff --git a/micropip/_commands/freeze.py b/micropip/_commands/freeze.py deleted file mode 100644 index 3cbfe57..0000000 --- a/micropip/_commands/freeze.py +++ /dev/null @@ -1,7 +0,0 @@ -from micropip.freeze import freeze_lockfile - -from .._compat import REPODATA_INFO, REPODATA_PACKAGES - - -def freeze() -> str: - return freeze_lockfile(REPODATA_PACKAGES, REPODATA_INFO) diff --git a/micropip/_commands/index_urls.py b/micropip/_commands/index_urls.py deleted file mode 100644 index aae9cce..0000000 --- a/micropip/_commands/index_urls.py +++ /dev/null @@ -1,27 +0,0 @@ -from .. import package_index - - -def set_index_urls(urls: list[str] | str) -> None: - """ - Set the index URLs to use when looking up packages. - - - The index URL should support the - `JSON API `__ . - - - The index URL may contain the placeholder {package_name} which will be - replaced with the package name when looking up a package. If it does not - contain the placeholder, the package name will be appended to the URL. - - - If a list of URLs is provided, micropip will try each URL in order until - it finds a package. If no package is found, an error will be raised. - - Parameters - ---------- - urls - A list of URLs or a single URL to use as the package index. - """ - - if isinstance(urls, str): - urls = [urls] - - package_index.INDEX_URLS = urls[:] diff --git a/micropip/_commands/install.py b/micropip/_commands/install.py deleted file mode 100644 index 74538b5..0000000 --- a/micropip/_commands/install.py +++ /dev/null @@ -1,21 +0,0 @@ -from micropip import package_index - -from ..install import install as _install - - -async def install( - requirements: str | list[str], - keep_going: bool = False, - deps: bool = True, - credentials: str | None = None, - pre: bool = False, - index_urls: list[str] | str | None = None, - *, - verbose: bool | int | None = None, -) -> None: - if index_urls is None: - index_urls = package_index.INDEX_URLS[:] - - return await _install( - requirements, keep_going, deps, credentials, pre, index_urls, verbose=verbose - ) diff --git a/micropip/_commands/list.py b/micropip/_commands/list.py deleted file mode 100644 index 3ce5d4b..0000000 --- a/micropip/_commands/list.py +++ /dev/null @@ -1,7 +0,0 @@ -from .._compat import REPODATA_PACKAGES -from ..list import list_installed_packages -from ..package import PackageDict - - -def _list() -> PackageDict: - return list_installed_packages(REPODATA_PACKAGES) diff --git a/micropip/_commands/mock_package.py b/micropip/_commands/mock_package.py deleted file mode 100644 index 1ddd9ab..0000000 --- a/micropip/_commands/mock_package.py +++ /dev/null @@ -1,191 +0,0 @@ -import importlib -import importlib.metadata -import shutil -import site -from pathlib import Path -from textwrap import dedent - -from .._mock_package import _add_in_memory_distribution, _remove_in_memory_distribution - -MOCK_INSTALL_NAME_MEMORY = "micropip in-memory mock package" -MOCK_INSTALL_NAME_PERSISTENT = "micropip mock package" - - -def add_mock_package( - name: str, - version: str, - *, - modules: dict[str, str | None] | None = None, - persistent: bool = False, -) -> None: - """ - Add a mock version of a package to the package dictionary. - - This means that if it is a dependency, it is skipped on install. - - By default a single empty module is installed with the same - name as the package. You can alternatively give one or more modules to make a - set of named modules. - - The modules parameter is usually a dictionary mapping module name to module text. - - .. code-block:: python - - { - "mylovely_module":''' - def module_method(an_argument): - print("This becomes a module level argument") - - module_value = "this value becomes a module level variable" - print("This is run on import of module") - ''' - } - - If you are adding the module in non-persistent mode, you can also pass functions - which are used to initialize the module on loading (as in `importlib.abc.loader.exec_module` ). - This allows you to do things like use `unittest.mock.MagicMock` classes for modules. - - .. code-block:: python - - def init_fn(module): - module.dict["WOO"]="hello" - print("Initing the module now!") - - ... - - { - "mylovely_module": init_fn - } - - Parameters - ---------- - name : - - Package name to add - - version : - - Version of the package. This should be a semantic version string, - e.g. 1.2.3 - - modules : - - Dictionary of module_name:string pairs. - The string contains the source of the mock module or is blank for - an empty module. - - persistent : - - If this is True, modules will be written to the file system, so they - persist between runs of python (assuming the file system persists). - If it is False, modules will be stored inside micropip in memory only. - """ - - if modules is None: - # make a single mock module with this name - modules = {name: ""} - - # make the metadata - METADATA = f"""Metadata-Version: 1.1 -Name: {name} -Version: {version} -Summary: {name} mock package generated by micropip -Author-email: {name}@micro.pip.non-working-fake-host -""" - - for module_name in modules.keys(): - METADATA += f"Provides: {module_name}\n" - - if persistent: - # make empty mock modules with the requested names in user site packages - site_packages = Path(site.getsitepackages()[0]) - - # should exist already, but just in case - site_packages.mkdir(parents=True, exist_ok=True) - - dist_dir = site_packages / f"{name}-{version}.dist-info" - dist_dir.mkdir(parents=True, exist_ok=False) - - metadata_file = dist_dir / "METADATA" - record_file = dist_dir / "RECORD" - installer_file = dist_dir / "INSTALLER" - - file_list = [metadata_file, installer_file] - - metadata_file.write_text(METADATA) - installer_file.write_text(MOCK_INSTALL_NAME_PERSISTENT) - - for module_name, content in modules.items(): - if not content: - content = "" - - content = dedent(content) - path_parts = module_name.split(".") - - dir_path = Path(site_packages, *path_parts) - dir_path.mkdir(exist_ok=True, parents=True) - init_file = dir_path / "__init__.py" - file_list.append(init_file) - - init_file.write_text(content) - - with open(record_file, "w") as f: - for file in file_list: - f.write(f"{file},,{file.stat().st_size}\n") - f.write(f"{record_file},,\n") - else: - # make memory mocks of files - INSTALLER = MOCK_INSTALL_NAME_MEMORY - metafiles = {"METADATA": METADATA, "INSTALLER": INSTALLER} - _add_in_memory_distribution(name, metafiles, modules) - - importlib.invalidate_caches() - - -def list_mock_packages() -> list[str]: - """ - List all mock packages currently installed. - """ - mock_packages = [ - dist.name - for dist in importlib.metadata.distributions() - if dist.read_text("INSTALLER") - in (MOCK_INSTALL_NAME_PERSISTENT, MOCK_INSTALL_NAME_MEMORY) - ] - return mock_packages - - -def remove_mock_package(name: str) -> None: - """ - Remove a mock package. - """ - - d = importlib.metadata.distribution(name) - installer = d.read_text("INSTALLER") - - if installer == MOCK_INSTALL_NAME_MEMORY: - _remove_in_memory_distribution(name) - return - - elif installer is None or installer != MOCK_INSTALL_NAME_PERSISTENT: - raise ValueError( - f"Package {name} doesn't seem to be a micropip mock. \n" - "Are you sure it was installed with micropip?" - ) - - # a real mock package - kill it - # remove all files - folders: set[Path] = set() - if d.files is not None: - for file in d.files: - p = Path(file.locate()) - p.unlink() - folders.add(p.parent) - - # delete all folders except site_packages - # (that check is just to avoid killing - # undesirable things in case of weird micropip errors) - site_packages = Path(site.getsitepackages()[0]) - for f in folders: - if f != site_packages: - shutil.rmtree(f) diff --git a/micropip/_mock_package.py b/micropip/_mock_package.py index 555eb5e..7578584 100644 --- a/micropip/_mock_package.py +++ b/micropip/_mock_package.py @@ -1,10 +1,17 @@ +import importlib import importlib.abc import importlib.metadata import importlib.util +import shutil +import site import sys from collections.abc import Callable +from pathlib import Path from textwrap import dedent +MOCK_INSTALL_NAME_MEMORY = "micropip in-memory mock package" +MOCK_INSTALL_NAME_PERSISTENT = "micropip mock package" + class MockDistribution(importlib.metadata.Distribution): def __init__(self, file_dict, modules): @@ -97,3 +104,183 @@ def _remove_in_memory_distribution(name): del sys.modules[module] del _mock_modules[module] del _mock_distributions[name] + + +def add_mock_package( + name: str, + version: str, + *, + modules: dict[str, str | None] | None = None, + persistent: bool = False, +) -> None: + """ + Add a mock version of a package to the package dictionary. + + This means that if it is a dependency, it is skipped on install. + + By default a single empty module is installed with the same + name as the package. You can alternatively give one or more modules to make a + set of named modules. + + The modules parameter is usually a dictionary mapping module name to module text. + + .. code-block:: python + + { + "mylovely_module":''' + def module_method(an_argument): + print("This becomes a module level argument") + + module_value = "this value becomes a module level variable" + print("This is run on import of module") + ''' + } + + If you are adding the module in non-persistent mode, you can also pass functions + which are used to initialize the module on loading (as in `importlib.abc.loader.exec_module` ). + This allows you to do things like use `unittest.mock.MagicMock` classes for modules. + + .. code-block:: python + + def init_fn(module): + module.dict["WOO"]="hello" + print("Initing the module now!") + + ... + + { + "mylovely_module": init_fn + } + + Parameters + ---------- + name : + + Package name to add + + version : + + Version of the package. This should be a semantic version string, + e.g. 1.2.3 + + modules : + + Dictionary of module_name:string pairs. + The string contains the source of the mock module or is blank for + an empty module. + + persistent : + + If this is True, modules will be written to the file system, so they + persist between runs of python (assuming the file system persists). + If it is False, modules will be stored inside micropip in memory only. + """ + + if modules is None: + # make a single mock module with this name + modules = {name: ""} + + # make the metadata + METADATA = f"""Metadata-Version: 1.1 +Name: {name} +Version: {version} +Summary: {name} mock package generated by micropip +Author-email: {name}@micro.pip.non-working-fake-host +""" + + for module_name in modules.keys(): + METADATA += f"Provides: {module_name}\n" + + if persistent: + # make empty mock modules with the requested names in user site packages + site_packages = Path(site.getsitepackages()[0]) + + # should exist already, but just in case + site_packages.mkdir(parents=True, exist_ok=True) + + dist_dir = site_packages / f"{name}-{version}.dist-info" + dist_dir.mkdir(parents=True, exist_ok=False) + + metadata_file = dist_dir / "METADATA" + record_file = dist_dir / "RECORD" + installer_file = dist_dir / "INSTALLER" + + file_list = [metadata_file, installer_file] + + metadata_file.write_text(METADATA) + installer_file.write_text(MOCK_INSTALL_NAME_PERSISTENT) + + for module_name, content in modules.items(): + if not content: + content = "" + + content = dedent(content) + path_parts = module_name.split(".") + + dir_path = Path(site_packages, *path_parts) + dir_path.mkdir(exist_ok=True, parents=True) + init_file = dir_path / "__init__.py" + file_list.append(init_file) + + init_file.write_text(content) + + with open(record_file, "w") as f: + for file in file_list: + f.write(f"{file},,{file.stat().st_size}\n") + f.write(f"{record_file},,\n") + else: + # make memory mocks of files + INSTALLER = MOCK_INSTALL_NAME_MEMORY + metafiles = {"METADATA": METADATA, "INSTALLER": INSTALLER} + _add_in_memory_distribution(name, metafiles, modules) + + importlib.invalidate_caches() + + +def list_mock_packages() -> list[str]: + """ + List all mock packages currently installed. + """ + mock_packages = [ + dist.name + for dist in importlib.metadata.distributions() + if dist.read_text("INSTALLER") + in (MOCK_INSTALL_NAME_PERSISTENT, MOCK_INSTALL_NAME_MEMORY) + ] + return mock_packages + + +def remove_mock_package(name: str) -> None: + """ + Remove a mock package. + """ + + d = importlib.metadata.distribution(name) + installer = d.read_text("INSTALLER") + + if installer == MOCK_INSTALL_NAME_MEMORY: + _remove_in_memory_distribution(name) + return + + elif installer is None or installer != MOCK_INSTALL_NAME_PERSISTENT: + raise ValueError( + f"Package {name} doesn't seem to be a micropip mock. \n" + "Are you sure it was installed with micropip?" + ) + + # a real mock package - kill it + # remove all files + folders: set[Path] = set() + if d.files is not None: + for file in d.files: + p = Path(file.locate()) + p.unlink() + folders.add(p.parent) + + # delete all folders except site_packages + # (that check is just to avoid killing + # undesirable things in case of weird micropip errors) + site_packages = Path(site.getsitepackages()[0]) + for f in folders: + if f != site_packages: + shutil.rmtree(f) diff --git a/micropip/install.py b/micropip/install.py index bd05388..a18947a 100644 --- a/micropip/install.py +++ b/micropip/install.py @@ -6,7 +6,6 @@ from packaging.markers import default_environment -from . import package_index from ._compat import loadPackage, to_js from .constants import FAQ_URLS from .logging import setup_logging @@ -15,11 +14,11 @@ async def install( requirements: str | list[str], + index_urls: list[str] | str, keep_going: bool = False, deps: bool = True, credentials: str | None = None, pre: bool = False, - index_urls: list[str] | str | None = None, *, verbose: bool | int | None = None, ) -> None: @@ -128,9 +127,6 @@ async def install( wheel_base = Path(getsitepackages()[0]) - if index_urls is None: - index_urls = package_index.INDEX_URLS[:] - transaction = Transaction( ctx=ctx, # type: ignore[arg-type] ctx_extras=[], diff --git a/micropip/package_index.py b/micropip/package_index.py index dd3fff8..88bac43 100644 --- a/micropip/package_index.py +++ b/micropip/package_index.py @@ -16,7 +16,6 @@ from .wheelinfo import WheelInfo DEFAULT_INDEX_URLS = ["https://pypi.org/simple"] -INDEX_URLS = DEFAULT_INDEX_URLS _formatter = string.Formatter() @@ -232,8 +231,8 @@ def _select_parser(content_type: str, pkgname: str) -> Callable[[str], ProjectIn async def query_package( name: str, + index_urls: list[str] | str, fetch_kwargs: dict[str, Any] | None = None, - index_urls: list[str] | str | None = None, ) -> ProjectInfo: """ Query for a package from package indexes. @@ -242,16 +241,13 @@ async def query_package( ---------- name Name of the package to search for. - fetch_kwargs - Keyword arguments to pass to the fetch function. index_urls A list of URLs or a single URL to use as the package index. - If None, the default index URL is used. - If a list of URLs is provided, it will be tried in order until it finds a package. If no package is found, an error will be raised. + fetch_kwargs + Keyword arguments to pass to the fetch function. """ - global INDEX_URLS _fetch_kwargs = fetch_kwargs.copy() if fetch_kwargs else {} @@ -263,9 +259,7 @@ async def query_package( "accept", "application/vnd.pypi.simple.v1+json, */*;q=0.01" ) - if index_urls is None: - index_urls = INDEX_URLS - elif isinstance(index_urls, str): + if isinstance(index_urls, str): index_urls = [index_urls] for url in index_urls: diff --git a/micropip/package_manager.py b/micropip/package_manager.py index c3b874c..b09f1ce 100644 --- a/micropip/package_manager.py +++ b/micropip/package_manager.py @@ -3,8 +3,7 @@ List, ) -from . import package_index -from ._commands import mock_package +from . import _mock_package, package_index from ._compat import REPODATA_INFO, REPODATA_PACKAGES from .freeze import freeze_lockfile from .install import install @@ -37,18 +36,18 @@ async def install( pre: bool = False, index_urls: list[str] | str | None = None, *, - verbose: bool | int = False, + verbose: bool | int | None = None, ): if index_urls is None: index_urls = self.index_urls return await install( requirements, + index_urls, keep_going, deps, credentials, pre, - index_urls, verbose=verbose, ) @@ -66,15 +65,15 @@ def add_mock_package( modules: dict[str, str | None] | None = None, persistent: bool = False, ): - return mock_package.add_mock_package( + return _mock_package.add_mock_package( name, version, modules=modules, persistent=persistent ) def list_mock_packages(self): - return mock_package.list_mock_packages() + return _mock_package.list_mock_packages() def remove_mock_package(self, name: str): - return mock_package.remove_mock_package(name) + return _mock_package.remove_mock_package(name) def uninstall(self): raise NotImplementedError() diff --git a/micropip/transaction.py b/micropip/transaction.py index 8c1f2ca..c3b11d5 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -28,7 +28,7 @@ class Transaction: deps: bool pre: bool fetch_kwargs: dict[str, str] - index_urls: list[str] | str | None + index_urls: list[str] | str locked: dict[str, PackageMetadata] = field(default_factory=dict) wheels: list[WheelInfo] = field(default_factory=list) @@ -184,7 +184,9 @@ async def _add_requirement_from_package_index(self, req: Requirement): add it to the package list and return True. Otherwise, return False. """ metadata = await package_index.query_package( - req.name, self.fetch_kwargs, index_urls=self.index_urls + req.name, + self.index_urls, + self.fetch_kwargs, ) wheel = find_wheel(metadata, req) diff --git a/tests/conftest.py b/tests/conftest.py index af55b88..33785ab 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -286,7 +286,7 @@ def add_pkg_version( self.metadata_map[filename] = metadata self.top_level_map[filename] = top_level - async def query_package(self, pkgname, kwargs, index_urls=None): + async def query_package(self, pkgname, index_urls, kwargs): from micropip.package_index import ProjectInfo try: diff --git a/tests/test_package_index.py b/tests/test_package_index.py index 5552f60..5f1169a 100644 --- a/tests/test_package_index.py +++ b/tests/test_package_index.py @@ -1,7 +1,6 @@ import pytest from conftest import TEST_PYPI_RESPONSE_DIR, _read_gzipped_testfile -import micropip._commands.index_urls as index_urls import micropip.package_index as package_index @@ -89,24 +88,6 @@ def test_project_info_equal(name): assert f_json.sha256 == f_simple_json.sha256 -def test_set_index_urls(): - default_index_urls = package_index.DEFAULT_INDEX_URLS - assert package_index.INDEX_URLS == default_index_urls - - valid_url1 = "https://pkg-index.com/{package_name}/json/" - valid_url2 = "https://another-pkg-index.com/{package_name}" - valid_url3 = "https://another-pkg-index.com/simple/" - try: - index_urls.set_index_urls(valid_url1) - assert package_index.INDEX_URLS == [valid_url1] - - index_urls.set_index_urls([valid_url1, valid_url2, valid_url3]) - assert package_index.INDEX_URLS == [valid_url1, valid_url2, valid_url3] - finally: - index_urls.set_index_urls(default_index_urls) - assert package_index.INDEX_URLS == default_index_urls - - def test_contain_placeholder(): assert package_index._contain_placeholder("https://pkg-index.com/{package_name}/") assert package_index._contain_placeholder( diff --git a/tests/test_transaction.py b/tests/test_transaction.py index 326490d..3acb1c6 100644 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -48,6 +48,8 @@ def test_parse_wheel_url3(): def create_transaction(Transaction): + from micropip.package_index import DEFAULT_INDEX_URLS + return Transaction( wheels=[], locked={}, @@ -59,7 +61,7 @@ def create_transaction(Transaction): ctx={}, ctx_extras=[], fetch_kwargs={}, - index_urls=None, + index_urls=DEFAULT_INDEX_URLS, )