diff --git a/.github/workflows/node.yaml b/.github/workflows/node.yaml index 90683f96..53424a93 100644 --- a/.github/workflows/node.yaml +++ b/.github/workflows/node.yaml @@ -26,6 +26,11 @@ jobs: poetry-version: - '1.1.13' runs-on: ubuntu-22.04 + services: + verdaccio: + image: verdaccio/verdaccio + ports: + - 4873:4873 steps: - uses: actions/checkout@v3 - name: Configure git @@ -45,6 +50,8 @@ jobs: flatpak --user remote-add flathub https://flathub.org/repo/flathub.flatpakrepo flatpak --user install -y flathub \ org.freedesktop.{Platform,Sdk{,.Extension.node{14,16,18}}}//22.08 + - name: Setup the local npm registry + run: tools/setup-local-registry.sh - name: Install dependencies run: poetry install - name: Run checks diff --git a/node/README.md b/node/README.md index 06c2cca4..8ad3ae72 100644 --- a/node/README.md +++ b/node/README.md @@ -321,6 +321,19 @@ $ poetry run pytest -n auto Note that these tests can take up quite a bit of space in /tmp, so if you hit `No space left on device` errors, try expanding `/tmp` or changing `$TMPDIR`. +### Local Registry + +Some of the tests require a local npm registry to work. For this purpose, you can use +[Verdaccio](https://verdaccio.org/), preferably via Docker / podman: + +```bash +$ docker run --rm -it -p 4873:4873 verdaccio/verdaccio +``` + +Then run `tools/setup-local-registry.sh` to set up this registry with a pre-published +package. (Note that running it twice will result in an error, since it tries to publish +the same package twice.) + ### Utility Scripts A few utility scripts are included in the `tools` directory: @@ -333,6 +346,8 @@ A few utility scripts are included in the `tools` directory: - `lockfile-utils.sh peek-cache PACKAGE-MANAGER PACKAGE` will install the dependencies from the corresponding lockfile and then extract the resulting package cache (npm) or mirror directory (yarn), for closer examination. +- `setup-local-registry.sh` will set up a local npm registry as [described + above](#local-registry). - `b64-to-hex.sh` will convert a base64 hash value from npm into hex, e.g.: ``` $ echo x+sXyT4RLLEIb6bY5R+wZnt5pfk= | tools/b64-to-hex.sh diff --git a/node/flatpak_node_generator/main.py b/node/flatpak_node_generator/main.py index f5e831c1..604259bf 100644 --- a/node/flatpak_node_generator/main.py +++ b/node/flatpak_node_generator/main.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Iterator, List, Set +from typing import Dict, Iterator, List, Set import argparse import asyncio @@ -12,7 +12,7 @@ from .node_headers import NodeHeaders from .package import Package from .progress import GeneratorProgress -from .providers import ProviderFactory +from .providers import Config, ProviderFactory from .providers.npm import NpmLockfileProvider, NpmModuleProvider, NpmProviderFactory from .providers.special import SpecialSourceProvider from .providers.yarn import YarnProviderFactory @@ -189,20 +189,20 @@ async def _async_main() -> None: print('Reading packages from lockfiles...') packages: Set[Package] = set() - rcfile_node_headers: Set[NodeHeaders] = set() + config_node_headers: Set[NodeHeaders] = set() - for lockfile in lockfiles: - lockfile_provider = provider_factory.create_lockfile_provider() - rcfile_providers = provider_factory.create_rcfile_providers() + lockfile_provider = provider_factory.create_lockfile_provider() + config_provider = provider_factory.create_config_provider() + + lockfile_configs: Dict[Path, Config] = {} + for lockfile in lockfiles: packages.update(lockfile_provider.process_lockfile(lockfile)) + lockfile_configs[lockfile] = config = config_provider.load_config(lockfile) - for rcfile_provider in rcfile_providers: - rcfile = lockfile.parent / rcfile_provider.RCFILE_NAME - if rcfile.is_file(): - nh = rcfile_provider.get_node_headers(rcfile) - if nh is not None: - rcfile_node_headers.add(nh) + nh = config.get_node_headers() + if nh is not None: + config_node_headers.add(nh) print(f'{len(packages)} packages read.') @@ -220,14 +220,18 @@ async def _async_main() -> None: ) special = SpecialSourceProvider(gen, options) - with provider_factory.create_module_provider(gen, special) as module_provider: + with provider_factory.create_module_provider( + gen, + special, + lockfile_configs, + ) as module_provider: with GeneratorProgress( packages, module_provider, args.max_parallel, ) as progress: await progress.run() - for headers in rcfile_node_headers: + for headers in config_node_headers: print(f'Generating headers {headers.runtime} @ {headers.target}') await special.generate_node_headers(headers) diff --git a/node/flatpak_node_generator/providers/__init__.py b/node/flatpak_node_generator/providers/__init__.py index ca6f5685..ad340895 100644 --- a/node/flatpak_node_generator/providers/__init__.py +++ b/node/flatpak_node_generator/providers/__init__.py @@ -1,7 +1,8 @@ +from dataclasses import dataclass from pathlib import Path -from typing import ContextManager, Dict, Iterator, List, Optional +from typing import Any, ContextManager, Dict, Iterator, List, Optional, Tuple -import re +import dataclasses import urllib.parse from ..manifest import ManifestGenerator @@ -45,32 +46,48 @@ def process_lockfile(self, lockfile: Path) -> Iterator[Package]: raise NotImplementedError() -class RCFileProvider: - RCFILE_NAME: str +@dataclass +class Config: + data: Dict[str, Any] = dataclasses.field(default_factory=lambda: {}) - def parse_rcfile(self, rcfile: Path) -> Dict[str, str]: - with open(rcfile, 'r') as r: - rcfile_text = r.read() - parser_re = re.compile( - r'^(?!#|;)(\S+)(?:\s+|\s*=\s*)(?:"(.+)"|(\S+))$', re.MULTILINE - ) - result: Dict[str, str] = {} - for key, quoted_val, val in parser_re.findall(rcfile_text): - result[key] = quoted_val or val - return result - - def get_node_headers(self, rcfile: Path) -> Optional[NodeHeaders]: - rc_data = self.parse_rcfile(rcfile) - if 'target' not in rc_data: + def merge_new_keys_only(self, other: Dict[str, Any]) -> None: + for key, value in other.items(): + if key not in self.data: + self.data[key] = value + + def get_node_headers(self) -> Optional[NodeHeaders]: + if 'target' not in self.data: return None - target = rc_data['target'] - runtime = rc_data.get('runtime') - disturl = rc_data.get('disturl') + target = self.data['target'] + runtime = self.data.get('runtime') + disturl = self.data.get('disturl') assert isinstance(runtime, str) and isinstance(disturl, str) return NodeHeaders.with_defaults(target, runtime, disturl) + def get_registry_for_scope(self, scope: str) -> Optional[str]: + return self.data.get(f'{scope}:registry') + + +class ConfigProvider: + @property + def _filename(self) -> str: + raise NotImplementedError() + + def parse_config(self, path: Path) -> Dict[str, Any]: + raise NotImplementedError() + + def load_config(self, lockfile: Path) -> Config: + config = Config() + + for parent in lockfile.parents: + path = parent / self._filename + if path.exists(): + config.merge_new_keys_only(self.parse_config(path)) + + return config + class ModuleProvider(ContextManager['ModuleProvider']): async def generate_package(self, package: Package) -> None: @@ -81,10 +98,13 @@ class ProviderFactory: def create_lockfile_provider(self) -> LockfileProvider: raise NotImplementedError() - def create_rcfile_providers(self) -> List[RCFileProvider]: + def create_config_provider(self) -> ConfigProvider: raise NotImplementedError() def create_module_provider( - self, gen: ManifestGenerator, special: SpecialSourceProvider + self, + gen: ManifestGenerator, + special: SpecialSourceProvider, + lockfile_configs: Dict[Path, Config], ) -> ModuleProvider: raise NotImplementedError() diff --git a/node/flatpak_node_generator/providers/npm.py b/node/flatpak_node_generator/providers/npm.py index a60cb08e..b0aa895b 100644 --- a/node/flatpak_node_generator/providers/npm.py +++ b/node/flatpak_node_generator/providers/npm.py @@ -34,7 +34,7 @@ ) from ..requests import Requests from ..url_metadata import RemoteUrlMetadata -from . import LockfileProvider, ModuleProvider, ProviderFactory, RCFileProvider +from . import Config, ConfigProvider, LockfileProvider, ModuleProvider, ProviderFactory from .special import SpecialSourceProvider _NPM_CORGIDOC = ( @@ -170,8 +170,103 @@ def process_lockfile(self, lockfile: Path) -> Iterator[Package]: ) -class NpmRCFileProvider(RCFileProvider): - RCFILE_NAME = '.npmrc' +class NpmConfigProvider(ConfigProvider): + _COMMENT = ('#', ';') + + @property + def _filename(self) -> str: + return '.npmrc' + + def _parse_value_as_json(self, string: str) -> Any: + try: + return json.loads(string) + except json.JSONDecodeError: + return string + + def _parse_value_literal(self, string: str) -> str: + result = '' + escaped = False + for c in string: + if escaped: + if c not in self._COMMENT and c != '\\': + result += '\\' + result += c + escaped = False + elif c == '\\': + escaped = True + elif c in self._COMMENT: + break + else: + result += c + + if escaped: + result += '\\' + return result.strip() + + def _parse_value(self, string: str) -> Any: + SINGLE_QUOTE = "'" + DOUBLE_QUOTE = '"' + + string = string.strip() + + if string.startswith(SINGLE_QUOTE) and string.endswith(SINGLE_QUOTE): + return self._parse_value_as_json(string[1:-1]) + elif string.startswith(DOUBLE_QUOTE) and string.endswith(DOUBLE_QUOTE): + return self._parse_value_as_json(string) + else: + return self._parse_value_literal(string) + + def _coalesce_to_string(self, value: Any) -> Any: + if isinstance(value, list): + return ','.join(map(self._coalesce_to_string, value)) + elif isinstance(value, dict): + return '[object Object]' + else: + return str(value) + + def parse_config(self, path: Path) -> Dict[str, Any]: + LITERALS = { + 'true': True, + 'false': False, + 'null': None, + } + + result: Dict[str, Any] = {} + + with path.open() as fp: + for line in fp: + line = line.strip() + if not line or line.startswith(self._COMMENT): + continue + + try: + key_s, value_s = line.split('=', 1) + except ValueError: + key_s = line + value_s = 'true' + + key = self._coalesce_to_string(self._parse_value(key_s)) + is_array = False + if key.endswith('[]'): + is_array = True + key = key[:-2] + + value = self._parse_value(value_s) + if isinstance(value, str): + value = LITERALS.get(value, value) + + if is_array and key not in result: + result[key] = [] + elif is_array and not isinstance(result[key], list): + result[key] = [result[key]] + + previous_value = result.get(key) + if isinstance(previous_value, list): + previous_value.append(value) + else: + result[key] = value + + return result class NpmModuleProvider(ModuleProvider): @@ -191,6 +286,7 @@ def __init__( special: SpecialSourceProvider, lockfile_root: Path, options: Options, + lockfile_configs: Dict[Path, Config], ) -> None: self.gen = gen self.special_source_provider = special @@ -198,6 +294,7 @@ def __init__( self.registry = options.registry self.no_autopatch = options.no_autopatch self.no_trim_index = options.no_trim_index + self.lockfile_configs = lockfile_configs self.npm_cache_dir = self.gen.data_root / 'npm-cache' self.cacache_dir = self.npm_cache_dir / '_cacache' # Awaitable so multiple tasks can be waiting on the same package info. @@ -210,8 +307,6 @@ def __init__( self.git_sources: DefaultDict[ Path, Dict[Path, GitSource] ] = collections.defaultdict(lambda: {}) - # FIXME better pass the same provider object we created in main - self.rcfile_provider = NpmRCFileProvider() def __exit__( self, @@ -383,21 +478,16 @@ async def generate_package(self, package: Package) -> None: def relative_lockfile_dir(self, lockfile: Path) -> Path: return lockfile.parent.relative_to(self.lockfile_root) - @functools.lru_cache(typed=True) - def get_lockfile_rc(self, lockfile: Path) -> Dict[str, str]: - rc = {} - rcfile_path = lockfile.parent / self.rcfile_provider.RCFILE_NAME - if rcfile_path.is_file(): - rc.update(self.rcfile_provider.parse_rcfile(rcfile_path)) - return rc - def get_package_registry(self, package: Package) -> str: assert isinstance(package.source, RegistrySource) - rc = self.get_lockfile_rc(package.lockfile) - if rc and '/' in package.name: - scope, _ = package.name.split('/', maxsplit=1) - if f'{scope}:registry' in rc: - return rc[f'{scope}:registry'] + if '/' in package.name: + config = self.lockfile_configs.get(package.lockfile) + if config is not None: + scope, _ = package.name.split('/', maxsplit=1) + registry = config.get_registry_for_scope(scope) + if registry is not None: + return registry + return self.registry def _finalize(self) -> None: @@ -527,10 +617,19 @@ def __init__(self, lockfile_root: Path, options: Options) -> None: def create_lockfile_provider(self) -> NpmLockfileProvider: return NpmLockfileProvider(self.options.lockfile) - def create_rcfile_providers(self) -> List[RCFileProvider]: - return [NpmRCFileProvider()] + def create_config_provider(self) -> NpmConfigProvider: + return NpmConfigProvider() def create_module_provider( - self, gen: ManifestGenerator, special: SpecialSourceProvider + self, + gen: ManifestGenerator, + special: SpecialSourceProvider, + lockfile_configs: Dict[Path, Config], ) -> NpmModuleProvider: - return NpmModuleProvider(gen, special, self.lockfile_root, self.options.module) + return NpmModuleProvider( + gen, + special, + self.lockfile_root, + self.options.module, + lockfile_configs, + ) diff --git a/node/flatpak_node_generator/providers/yarn.py b/node/flatpak_node_generator/providers/yarn.py index e18c26e8..39bbe951 100644 --- a/node/flatpak_node_generator/providers/yarn.py +++ b/node/flatpak_node_generator/providers/yarn.py @@ -10,8 +10,8 @@ from ..integrity import Integrity from ..manifest import ManifestGenerator from ..package import GitSource, LocalSource, Package, PackageSource, ResolvedSource -from . import LockfileProvider, ModuleProvider, ProviderFactory, RCFileProvider -from .npm import NpmRCFileProvider +from . import Config, ConfigProvider, LockfileProvider, ModuleProvider, ProviderFactory +from .npm import NpmConfigProvider from .special import SpecialSourceProvider GIT_URL_PATTERNS = [ @@ -25,6 +25,47 @@ GIT_URL_HOSTS = ['github.com', 'gitlab.com', 'bitbucket.com', 'bitbucket.org'] +def _unquote(string: str) -> str: + if string.startswith('"'): + assert string.endswith('"') + return string[1:-1] + else: + return string + + +def _parse_lockfile(lockfile: Path) -> Dict[str, Any]: + def _iter_lines() -> Iterator[Tuple[int, str]]: + indent = ' ' + for line in lockfile.open(): + level = 0 + while line.startswith(indent): + level += 1 + line = line[len(indent) :] + yield level, line.strip() + + root_entry: Dict[str, Any] = {} + parent_entries = [root_entry] + + for level, line in _iter_lines(): + if line.startswith('#') or not line: + continue + assert level <= len(parent_entries) - 1 + parent_entries = parent_entries[: level + 1] + if line.endswith(':'): + key = line[:-1] + child_entry = parent_entries[-1][key] = {} + parent_entries.append(child_entry) + else: + # NOTE shlex.split is handy, but slow; + # to speed up parsing we can use something less robust, e.g. + # _key, _value = line.split(' ', 1) + # parent_entries[-1][self.unquote(_key)] = self.unquote(_value) + key, value = shlex.split(line) + parent_entries[-1][key] = value + + return root_entry + + class YarnLockfileProvider(LockfileProvider): _LOCAL_PKG_RE = re.compile(r'^(?:file|link):') @@ -38,51 +79,12 @@ def is_git_version(version: str) -> bool: return len([p for p in url.path.split('/') if p]) == 2 return False - def parse_lockfile(self, lockfile: Path) -> Dict[str, Any]: - def _iter_lines() -> Iterator[Tuple[int, str]]: - indent = ' ' - for line in lockfile.open(): - level = 0 - while line.startswith(indent): - level += 1 - line = line[len(indent) :] - yield level, line.strip() - - root_entry: Dict[str, Any] = {} - parent_entries = [root_entry] - - for level, line in _iter_lines(): - if line.startswith('#') or not line: - continue - assert level <= len(parent_entries) - 1 - parent_entries = parent_entries[: level + 1] - if line.endswith(':'): - key = line[:-1] - child_entry = parent_entries[-1][key] = {} - parent_entries.append(child_entry) - else: - # NOTE shlex.split is handy, but slow; - # to speed up parsing we can use something less robust, e.g. - # _key, _value = line.split(' ', 1) - # parent_entries[-1][self.unquote(_key)] = self.unquote(_value) - key, value = shlex.split(line) - parent_entries[-1][key] = value - - return root_entry - - def unquote(self, string: str) -> str: - if string.startswith('"'): - assert string.endswith('"') - return string[1:-1] - else: - return string - def process_package( self, lockfile: Path, name_line: str, entry: Dict[str, Any] ) -> Package: assert name_line and entry - name = self.unquote(name_line.split(',', 1)[0]) + name = _unquote(name_line.split(',', 1)[0]) name, version_constraint = name.rsplit('@', 1) source: PackageSource @@ -103,12 +105,29 @@ def process_package( ) def process_lockfile(self, lockfile: Path) -> Iterator[Package]: - for name_line, package in self.parse_lockfile(lockfile).items(): + for name_line, package in _parse_lockfile(lockfile).items(): yield self.process_package(lockfile, name_line, package) -class YarnRCFileProvider(RCFileProvider): - RCFILE_NAME = '.yarnrc' +class YarnConfigProvider(ConfigProvider): + def __init__(self) -> None: + self._npm_config_provider = NpmConfigProvider() + + @property + def _filename(self) -> str: + return '.yarnrc' + + def parse_config(self, path: Path) -> Dict[str, Any]: + # yarn's config format is identical to its lockfile format. + return _parse_lockfile(path) + + def load_config(self, lockfile: Path) -> Config: + config = super().load_config(lockfile) + + npm_config = self._npm_config_provider.load_config(lockfile) + config.merge_new_keys_only(npm_config.data) + + return config class YarnModuleProvider(ModuleProvider): @@ -178,10 +197,13 @@ def __init__(self) -> None: def create_lockfile_provider(self) -> YarnLockfileProvider: return YarnLockfileProvider() - def create_rcfile_providers(self) -> List[RCFileProvider]: - return [YarnRCFileProvider(), NpmRCFileProvider()] + def create_config_provider(self) -> YarnConfigProvider: + return YarnConfigProvider() def create_module_provider( - self, gen: ManifestGenerator, special: SpecialSourceProvider + self, + gen: ManifestGenerator, + special: SpecialSourceProvider, + lockfile_configs: Dict[Path, Config], ) -> YarnModuleProvider: return YarnModuleProvider(gen, special) diff --git a/node/tests/conftest.py b/node/tests/conftest.py index adafed37..54625623 100644 --- a/node/tests/conftest.py +++ b/node/tests/conftest.py @@ -171,6 +171,14 @@ class ProviderPaths: def package_json(self) -> Path: return self.root / 'package.json' + @property + def npmrc(self) -> Path: + return self.root / '.npmrc' + + @property + def yarnrc(self) -> Path: + return self.root / '.yarnrc' + @property def lockfile_source(self) -> Path: if self.type == ProviderFactoryType.NPM: @@ -199,6 +207,11 @@ def lockfile_dest(self) -> str: def add_to_manifest(self, gen: ManifestGenerator) -> None: gen.add_local_file_source(self.package_json) gen.add_local_file_source(self.lockfile_source, Path(self.lockfile_dest)) + + for rc in self.npmrc, self.yarnrc: + if rc.exists(): + gen.add_local_file_source(rc) + if self.type == ProviderFactoryType.YARN: gen.add_data_source( f'yarn-offline-mirror "./flatpak-node/yarn-mirror"', Path('.yarnrc') @@ -273,7 +286,12 @@ async def generate_modules( ) special = SpecialSourceProvider(gen, self.special) - with factory.create_module_provider(gen, special) as module: + config_provider = factory.create_config_provider() + lockfile_configs = { + paths.lockfile_source: config_provider.load_config(paths.lockfile_source), + } + + with factory.create_module_provider(gen, special, lockfile_configs) as module: for package in factory.create_lockfile_provider().process_lockfile( paths.lockfile_source ): diff --git a/node/tests/data/packages/custom-registry/.npmrc b/node/tests/data/packages/custom-registry/.npmrc new file mode 100644 index 00000000..adb0501e --- /dev/null +++ b/node/tests/data/packages/custom-registry/.npmrc @@ -0,0 +1 @@ +"@flatpak-node-generator-tests:registry" = "http://localhost:4873" diff --git a/node/tests/data/packages/custom-registry/package-lock.v1.json b/node/tests/data/packages/custom-registry/package-lock.v1.json new file mode 100644 index 00000000..615b0656 --- /dev/null +++ b/node/tests/data/packages/custom-registry/package-lock.v1.json @@ -0,0 +1,13 @@ +{ + "name": "@flatpak-node-generator-tests/custom-registry", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@flatpak-node-generator-tests/registry-package": { + "version": "1.0.0", + "resolved": "http://localhost:4873/@flatpak-node-generator-tests%2fregistry-package/-/registry-package-1.0.0.tgz", + "integrity": "sha512-dyHOPsVMVjCUtyL7jeqs+ISkpRnXV88lkObwNJvYl33f/nWiKj/luzfoE4wgYOcmRuYwLNgAV/Y2SD19aHWDng==" + } + } +} diff --git a/node/tests/data/packages/custom-registry/package-lock.v2.json b/node/tests/data/packages/custom-registry/package-lock.v2.json new file mode 100644 index 00000000..2cdd7e89 --- /dev/null +++ b/node/tests/data/packages/custom-registry/package-lock.v2.json @@ -0,0 +1,27 @@ +{ + "name": "@flatpak-node-generator-tests/custom-registry", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "@flatpak-node-generator-tests/custom-registry", + "version": "1.0.0", + "dependencies": { + "@flatpak-node-generator-tests/registry-package": "*" + } + }, + "node_modules/@flatpak-node-generator-tests/registry-package": { + "version": "1.0.0", + "resolved": "http://localhost:4873/@flatpak-node-generator-tests%2fregistry-package/-/registry-package-1.0.0.tgz", + "integrity": "sha512-dyHOPsVMVjCUtyL7jeqs+ISkpRnXV88lkObwNJvYl33f/nWiKj/luzfoE4wgYOcmRuYwLNgAV/Y2SD19aHWDng==" + } + }, + "dependencies": { + "@flatpak-node-generator-tests/registry-package": { + "version": "1.0.0", + "resolved": "http://localhost:4873/@flatpak-node-generator-tests%2fregistry-package/-/registry-package-1.0.0.tgz", + "integrity": "sha512-dyHOPsVMVjCUtyL7jeqs+ISkpRnXV88lkObwNJvYl33f/nWiKj/luzfoE4wgYOcmRuYwLNgAV/Y2SD19aHWDng==" + } + } +} diff --git a/node/tests/data/packages/custom-registry/package-lock.v3.json b/node/tests/data/packages/custom-registry/package-lock.v3.json new file mode 100644 index 00000000..67f1a921 --- /dev/null +++ b/node/tests/data/packages/custom-registry/package-lock.v3.json @@ -0,0 +1,20 @@ +{ + "name": "@flatpak-node-generator-tests/custom-registry", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@flatpak-node-generator-tests/custom-registry", + "version": "1.0.0", + "dependencies": { + "@flatpak-node-generator-tests/registry-package": "*" + } + }, + "node_modules/@flatpak-node-generator-tests/registry-package": { + "version": "1.0.0", + "resolved": "http://localhost:4873/@flatpak-node-generator-tests%2fregistry-package/-/registry-package-1.0.0.tgz", + "integrity": "sha512-dyHOPsVMVjCUtyL7jeqs+ISkpRnXV88lkObwNJvYl33f/nWiKj/luzfoE4wgYOcmRuYwLNgAV/Y2SD19aHWDng==" + } + } +} diff --git a/node/tests/data/packages/custom-registry/package.json b/node/tests/data/packages/custom-registry/package.json new file mode 100644 index 00000000..5628efb8 --- /dev/null +++ b/node/tests/data/packages/custom-registry/package.json @@ -0,0 +1,7 @@ +{ + "name": "@flatpak-node-generator-tests/custom-registry", + "version": "1.0.0", + "dependencies": { + "@flatpak-node-generator-tests/registry-package": "*" + } +} diff --git a/node/tests/data/packages/custom-registry/registry-package/index.js b/node/tests/data/packages/custom-registry/registry-package/index.js new file mode 100644 index 00000000..12d6ec24 --- /dev/null +++ b/node/tests/data/packages/custom-registry/registry-package/index.js @@ -0,0 +1,7 @@ +const fs = require('fs') + +module.exports = { + sayHello: () => { + fs.writeFileSync('hello.txt', 'Hello!') + }, +} diff --git a/node/tests/data/packages/custom-registry/registry-package/package.json b/node/tests/data/packages/custom-registry/registry-package/package.json new file mode 100644 index 00000000..2f21b269 --- /dev/null +++ b/node/tests/data/packages/custom-registry/registry-package/package.json @@ -0,0 +1,4 @@ +{ + "name": "@flatpak-node-generator-tests/registry-package", + "version": "1.0.0" +} diff --git a/node/tests/data/packages/custom-registry/yarn.lock b/node/tests/data/packages/custom-registry/yarn.lock new file mode 100644 index 00000000..491c7945 --- /dev/null +++ b/node/tests/data/packages/custom-registry/yarn.lock @@ -0,0 +1,8 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@flatpak-node-generator-tests/registry-package@*": + version "1.0.0" + resolved "http://localhost:4873/@flatpak-node-generator-tests%2fregistry-package/-/registry-package-1.0.0.tgz#1c677ccdfe9442572d73b64b4f01268fa3f302a9" + integrity sha512-dyHOPsVMVjCUtyL7jeqs+ISkpRnXV88lkObwNJvYl33f/nWiKj/luzfoE4wgYOcmRuYwLNgAV/Y2SD19aHWDng== diff --git a/node/tests/test_npm.py b/node/tests/test_npm.py new file mode 100644 index 00000000..11160d06 --- /dev/null +++ b/node/tests/test_npm.py @@ -0,0 +1,52 @@ +from pathlib import Path + +from flatpak_node_generator.providers.npm import NpmConfigProvider + +TEST_CONFIG = r""" +# comment +1 = 1 ; inline comment +; comment +'2' = '2' +"3" = "3" +a = x # inline comment +b = "x" +c = 'x' +d = "true" +e = false +f = 'null' +g = \;\#\3\4\ +"h=1" = 1=2 +i +'[1,[2,3],{}]' = '{"4": 5}' + +r1[] = a +r1 = b +r2 = c +r2[] = d +""" + + +def test_config_loading(tmp_path: Path) -> None: + config_provider = NpmConfigProvider() + + npmrc = tmp_path / '.npmrc' + npmrc.write_text(TEST_CONFIG) + + config = config_provider.load_config(npmrc / 'lockfile') + assert config.data == { + '1': '1', + '2': 2, + '3': '3', + 'a': 'x', + 'b': 'x', + 'c': 'x', + 'd': True, + 'e': False, + 'f': None, + 'g': r';#\3\4' + '\\', + '"h': '1" = 1=2', + 'i': True, + '1,2,3,[object Object]': {'4': 5}, + 'r1': ['a', 'b'], + 'r2': ['c', 'd'], + } diff --git a/node/tests/test_providers.py b/node/tests/test_providers.py index 3d29c815..2bf5fd80 100644 --- a/node/tests/test_providers.py +++ b/node/tests/test_providers.py @@ -1,4 +1,5 @@ from pathlib import Path +from typing import Any, Dict import itertools @@ -6,6 +7,59 @@ from conftest import FlatpakBuilder, ProviderFactorySpec from flatpak_node_generator.manifest import ManifestGenerator +from flatpak_node_generator.providers import ConfigProvider + +TEST_CONFIG_FILENAME = 'node-test.rc' + + +async def test_config_loading(tmp_path: Path) -> None: + class TestConfigProvider(ConfigProvider): + @property + def _filename(self) -> str: + return TEST_CONFIG_FILENAME + + def parse_config(self, path: Path) -> Dict[str, Any]: + return dict(x.split('=') for x in path.read_text().split(',')) + + inner_dir = tmp_path / 'inner' + inner_dir.mkdir(parents=True) + + inner_rc = inner_dir / TEST_CONFIG_FILENAME + inner_rc.write_text('inner=a,override=b') + + outer_rc = tmp_path / TEST_CONFIG_FILENAME + outer_rc.write_text('outer=c,override=d') + + config_provider = TestConfigProvider() + config = config_provider.load_config(inner_dir / 'lockfile') + assert config.data == { + 'inner': 'a', + 'override': 'b', + 'outer': 'c', + } + + +async def test_custom_registry( + flatpak_builder: FlatpakBuilder, + provider_factory_spec: ProviderFactorySpec, + node_version: int, +) -> None: + with ManifestGenerator() as gen: + await provider_factory_spec.generate_modules( + 'custom-registry', gen, node_version + ) + + flatpak_builder.build( + sources=gen.ordered_sources(), + commands=[ + provider_factory_spec.install_command, + """node -e 'require("@flatpak-node-generator-tests/registry-package").sayHello()'""", + ], + use_node=node_version, + ) + + hello_txt = flatpak_builder.module_dir / 'hello.txt' + assert hello_txt.read_text() == 'Hello!' async def test_minimal_git( diff --git a/node/tests/test_yarn.py b/node/tests/test_yarn.py index 1bba1eb8..00becf20 100644 --- a/node/tests/test_yarn.py +++ b/node/tests/test_yarn.py @@ -1,9 +1,11 @@ from pathlib import Path -from conftest import ProviderFactorySpec from flatpak_node_generator.integrity import Integrity from flatpak_node_generator.package import GitSource, Package, ResolvedSource -from flatpak_node_generator.providers.yarn import YarnLockfileProvider +from flatpak_node_generator.providers.yarn import ( + YarnConfigProvider, + YarnLockfileProvider, +) TEST_LOCKFILE = """ # random comment @@ -34,6 +36,17 @@ """ +TEST_YARNRC = """ +yarnrc "a b c" +override "from yarnrc" +""" + +TEST_NPMRC = """ +npmrc = "d e f" +override = "from npmrc" +""" + + def test_lockfile_parsing(tmp_path: Path) -> None: lockfile_provider = YarnLockfileProvider() @@ -88,3 +101,20 @@ def test_lockfile_parsing(tmp_path: Path) -> None: ), ), ] + + +def test_config_loading(tmp_path: Path) -> None: + config_provider = YarnConfigProvider() + + yarnrc = tmp_path / '.yarnrc' + yarnrc.write_text(TEST_YARNRC) + + npmrc = tmp_path / '.npmrc' + npmrc.write_text(TEST_NPMRC) + + config = config_provider.load_config(tmp_path / 'lockfile') + assert config.data == { + 'yarnrc': 'a b c', + 'npmrc': 'd e f', + 'override': 'from yarnrc', + } diff --git a/node/tools/lockfile-utils.sh b/node/tools/lockfile-utils.sh index 56f71c31..faa54a58 100755 --- a/node/tools/lockfile-utils.sh +++ b/node/tools/lockfile-utils.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -e +shopt -s nullglob die() { echo "$@" >&2 @@ -62,7 +63,7 @@ package_path="$(dirname "$0")/../tests/data/packages/$package_arg" tmpdir=$(mktemp -d) trap 'rm -rf -- "$tmpdir"' EXIT -cp "$package_path/package.json" "$tmpdir" +cp "$package_path/package.json" "$package_path/".*rc "$tmpdir" # Special-case handling for our test of a local package. [[ -d "$package_path/subdir" ]] && cp -r "$package_path/subdir" "$tmpdir" diff --git a/node/tools/setup-local-registry.sh b/node/tools/setup-local-registry.sh new file mode 100755 index 00000000..1b850145 --- /dev/null +++ b/node/tools/setup-local-registry.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -e + +REGISTRY=localhost:4873 + +curl -X PUT http://$REGISTRY/-/user/org.couchdb.user:test \ + -H 'Content-Type: application/json' \ + -d '{"name": "test", "password": "test"}' +echo + +pkg_path="$(dirname "$0")/../tests/data/packages/custom-registry/registry-package" + +tmpdir=$(mktemp -d) +trap 'rm -rf -- "$tmpdir"' EXIT + +cp -r "$pkg_path/"* "$tmpdir" + +cat > "$tmpdir/.npmrc" <