diff --git a/packages/komodo_defi_framework/app_build/BUILD_CONFIG_README.md b/packages/komodo_defi_framework/app_build/BUILD_CONFIG_README.md index 2334b9660..6742f30bc 100644 --- a/packages/komodo_defi_framework/app_build/BUILD_CONFIG_README.md +++ b/packages/komodo_defi_framework/app_build/BUILD_CONFIG_README.md @@ -40,19 +40,26 @@ Paths in the config are relative to that package directory. - Use `--concurrent` for faster downloads in development - Override behavior per build via env `OVERRIDE_DEFI_API_DOWNLOAD=true|false` -### Using Nebula mirror and short commit hashes +### Using mirror sites and short commit hashes -- You can add Nebula as an additional source in `api.source_urls`: +- You can add mirror sites as additional sources in `api.source_urls`: ``` "source_urls": [ "https://api.github.com/repos/GLEECBTC/komodo-defi-framework", - "https://sdk.devbuilds.komodo.earth/", - "https://nebula.decker.im/kdf/" + "https://devbuilds.gleec.com/", + "https://nebula.decker.im/kdf/", + "https://sdk.devbuilds.komodo.earth/" ] ``` -- The downloader expects branch-scoped directory listings (e.g., `.../dev/`) on both devbuilds and Nebula mirrors and will fallback to the base listing when available. It searches for artifacts that match the platform patterns and contain either the full commit hash or a 7-char short hash. +#### Supported mirror types + +- **GLEEC DevBuilds** (`devbuilds.gleec.com`): Caddy-based file server with JSON API support. Uses `Accept: application/json` header for structured directory listings. +- **Nebula** (`nebula.decker.im`): HTML-based directory listing. +- **Komodo DevBuilds** (`sdk.devbuilds.komodo.earth`): HTML-based directory listing. + +- The downloader expects branch-scoped directory listings (e.g., `.../dev/`) on mirror sites and will fallback to the base listing when available. It searches for artifacts that match the platform patterns and contain either the full commit hash or a 7-char short hash. - To pin a specific commit (e.g., `4025b8c`) without changing branches, update `api.api_commit_hash` or use the CLI with `--commit`: ```bash @@ -70,6 +77,7 @@ dart run packages/komodo_wallet_cli/bin/update_api_config.dart \ - Re-run the CLI with a different `--commit ` value. Notes: + - Nebula index includes additional files like `komodo-wallet-*`; these are automatically ignored by the downloader. - macOS on Nebula uses `kdf-macos-universal2-.zip` (special case handled in `matching_pattern`). Other platforms use `kdf_-.zip`. diff --git a/packages/komodo_wallet_build_transformer/lib/src/steps/defi_api_build_step/artefact_downloader_factory.dart b/packages/komodo_wallet_build_transformer/lib/src/steps/defi_api_build_step/artefact_downloader_factory.dart index 2e0643eaa..46fab36b8 100644 --- a/packages/komodo_wallet_build_transformer/lib/src/steps/defi_api_build_step/artefact_downloader_factory.dart +++ b/packages/komodo_wallet_build_transformer/lib/src/steps/defi_api_build_step/artefact_downloader_factory.dart @@ -1,10 +1,21 @@ import 'package:komodo_wallet_build_transformer/src/steps/defi_api_build_step/artefact_downloader.dart'; +import 'package:komodo_wallet_build_transformer/src/steps/defi_api_build_step/caddy_artefact_downloader.dart'; import 'package:komodo_wallet_build_transformer/src/steps/defi_api_build_step/dev_builds_artefact_downloader.dart'; import 'package:komodo_wallet_build_transformer/src/steps/defi_api_build_step/github_artefact_downloader.dart'; import 'package:komodo_wallet_build_transformer/src/steps/github/github_api_provider.dart'; import 'package:komodo_wallet_build_transformer/src/steps/models/api/api_build_config.dart'; class ArtefactDownloaderFactory { + /// Known Caddy-based mirror hosts that support JSON directory listings. + static const _caddyHosts = ['devbuilds.gleec.com']; + + /// Checks if the given URL is hosted on a known Caddy server. + static bool _isCaddyServer(String url) { + final uri = Uri.tryParse(url); + if (uri == null) return false; + return _caddyHosts.any((host) => uri.host == host); + } + static Map fromBuildConfig( ApiBuildConfig buildConfig, { String? githubToken, @@ -18,6 +29,12 @@ class ArtefactDownloaderFactory { sourceUrl, githubToken: githubToken, ); + } else if (_isCaddyServer(sourceUrl)) { + downloaders[sourceUrl] = CaddyArtefactDownloader( + apiBranch: buildConfig.branch, + apiCommitHash: buildConfig.apiCommitHash, + sourceUrl: sourceUrl, + ); } else { downloaders[sourceUrl] = DevBuildsArtefactDownloader( apiBranch: buildConfig.branch, diff --git a/packages/komodo_wallet_build_transformer/lib/src/steps/defi_api_build_step/caddy_artefact_downloader.dart b/packages/komodo_wallet_build_transformer/lib/src/steps/defi_api_build_step/caddy_artefact_downloader.dart new file mode 100644 index 000000000..23ba98298 --- /dev/null +++ b/packages/komodo_wallet_build_transformer/lib/src/steps/defi_api_build_step/caddy_artefact_downloader.dart @@ -0,0 +1,247 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:http/http.dart' as http; +import 'package:komodo_wallet_build_transformer/src/steps/defi_api_build_step/artefact_downloader.dart'; +import 'package:komodo_wallet_build_transformer/src/steps/models/api/api_file_matching_config.dart'; +import 'package:logging/logging.dart'; +import 'package:path/path.dart' as path; + +/// A file entry returned by Caddy's JSON directory listing API. +class CaddyFileEntry { + CaddyFileEntry({ + required this.name, + required this.size, + required this.url, + required this.modTime, + required this.isDir, + required this.isSymlink, + }); + + factory CaddyFileEntry.fromJson(Map json) { + return CaddyFileEntry( + name: json['name'] as String, + size: json['size'] as int, + url: json['url'] as String, + modTime: DateTime.parse(json['mod_time'] as String), + isDir: json['is_dir'] as bool, + isSymlink: json['is_symlink'] as bool, + ); + } + + final String name; + final int size; + final String url; + final DateTime modTime; + final bool isDir; + final bool isSymlink; +} + +/// Artefact downloader for Caddy file servers using the JSON directory API. +/// +/// Caddy provides a JSON directory listing when the `Accept: application/json` +/// header is included. This is more reliable than HTML scraping. +class CaddyArtefactDownloader implements ArtefactDownloader { + CaddyArtefactDownloader({ + required this.apiBranch, + required this.apiCommitHash, + required this.sourceUrl, + }); + + final _log = Logger('CaddyArtefactDownloader'); + + @override + final String apiBranch; + + @override + final String apiCommitHash; + + @override + final String sourceUrl; + + /// Fetches directory listing from Caddy using JSON API. + Future> _fetchDirectoryListing(Uri uri) async { + final response = await http.get( + uri, + headers: {'Accept': 'application/json'}, + ); + response.throwIfNotSuccessResponse(); + + final List jsonList = jsonDecode(response.body) as List; + return jsonList + .map((e) => CaddyFileEntry.fromJson(e as Map)) + .toList(); + } + + /// Recursively searches for matching files in the directory tree. + Future> _searchForFiles({ + required Uri baseUri, + required ApiFileMatchingConfig matchingConfig, + required String fullHash, + required String shortHash, + int maxDepth = 3, + int currentDepth = 0, + }) async { + if (currentDepth >= maxDepth) { + return {}; + } + + final candidates = {}; + + try { + final entries = await _fetchDirectoryListing(baseUri); + + for (final entry in entries) { + if (entry.isDir) { + // Recursively search subdirectories + final subUri = baseUri.resolve(entry.url); + final subCandidates = await _searchForFiles( + baseUri: subUri, + matchingConfig: matchingConfig, + fullHash: fullHash, + shortHash: shortHash, + maxDepth: maxDepth, + currentDepth: currentDepth + 1, + ); + candidates.addAll(subCandidates); + } else { + // Check if file matches criteria + final fileName = entry.name; + + // Skip non-zip files + if (!fileName.endsWith('.zip')) continue; + + // Skip wallet archives + if (fileName.contains('wallet')) continue; + + // Check pattern match + if (!matchingConfig.matches(fileName)) continue; + + // Check hash match + final containsHash = + fileName.contains(fullHash) || fileName.contains(shortHash); + if (!containsHash) continue; + + // Build absolute URL + final resolvedUrl = baseUri.resolve(entry.url).toString(); + candidates[fileName] = resolvedUrl; + _log.fine('Found candidate: $fileName at $resolvedUrl'); + } + } + } catch (e) { + _log.fine('Failed to fetch directory listing from $baseUri: $e'); + } + + return candidates; + } + + @override + Future fetchDownloadUrl( + ApiFileMatchingConfig matchingConfig, + String platform, + ) async { + final normalizedSource = sourceUrl.endsWith('/') + ? sourceUrl + : '$sourceUrl/'; + final baseUri = Uri.parse(normalizedSource); + + final fullHash = apiCommitHash; + final shortHash = apiCommitHash.substring(0, 7); + _log.info('Looking for files with hash $fullHash or $shortHash'); + + // Try branch-scoped directory first, then fall back to base + final candidateListingUrls = { + if (apiBranch.isNotEmpty) baseUri.resolve('$apiBranch/'), + baseUri, + }; + + for (final listingUrl in candidateListingUrls) { + _log.info('Searching in $listingUrl'); + + final candidates = await _searchForFiles( + baseUri: listingUrl, + matchingConfig: matchingConfig, + fullHash: fullHash, + shortHash: shortHash, + ); + + if (candidates.isNotEmpty) { + final preferred = matchingConfig.choosePreferred(candidates.keys); + final url = candidates[preferred] ?? candidates.values.first; + _log.info('Selected file: $preferred from $listingUrl'); + return url; + } + + _log.fine('No matching files found in $listingUrl'); + } + + throw Exception( + 'Zip file not found for platform $platform from $sourceUrl', + ); + } + + @override + Future downloadArtefact({ + required String url, + required String destinationPath, + }) async { + _log.info('Downloading $url...'); + final response = await http.get(Uri.parse(url)); + response.throwIfNotSuccessResponse(); + + final zipFileName = path.basename(url); + final zipFilePath = path.join(destinationPath, zipFileName); + + final directory = Directory(destinationPath); + if (!directory.existsSync()) { + await directory.create(recursive: true); + } + + final zipFile = File(zipFilePath); + try { + await zipFile.writeAsBytes(response.bodyBytes); + } catch (e) { + _log.info('Error writing file', e); + rethrow; + } + + _log.info('Downloaded $zipFileName'); + return zipFilePath; + } + + @override + Future extractArtefact({ + required String filePath, + required String destinationFolder, + }) async { + try { + if (Platform.isMacOS || Platform.isLinux) { + final result = await Process.run('unzip', [ + '-o', + filePath, + '-d', + destinationFolder, + ]); + if (result.exitCode != 0) { + throw Exception('Error extracting zip file: ${result.stderr}'); + } + } else if (Platform.isWindows) { + final result = await Process.run('powershell', [ + '-Command', + 'Expand-Archive -Path "$filePath" -DestinationPath "$destinationFolder" -Force', + ]); + if (result.exitCode != 0) { + throw Exception('Error extracting zip file: ${result.stderr}'); + } + } else { + _log.severe('Unsupported platform: ${Platform.operatingSystem}'); + throw UnsupportedError('Unsupported platform'); + } + _log.info('Extraction completed.'); + } catch (e) { + _log.shout('Failed to extract zip file: $e'); + rethrow; + } + } +} + diff --git a/packages/komodo_wallet_build_transformer/lib/src/steps/defi_api_build_step/dev_builds_artefact_downloader.dart b/packages/komodo_wallet_build_transformer/lib/src/steps/defi_api_build_step/dev_builds_artefact_downloader.dart index eea044333..ce728490f 100644 --- a/packages/komodo_wallet_build_transformer/lib/src/steps/defi_api_build_step/dev_builds_artefact_downloader.dart +++ b/packages/komodo_wallet_build_transformer/lib/src/steps/defi_api_build_step/dev_builds_artefact_downloader.dart @@ -160,11 +160,8 @@ class DevBuildsArtefactDownloader implements ArtefactDownloader { } else if (Platform.isWindows) { // For Windows, use PowerShell's Expand-Archive command final result = await Process.run('powershell', [ - 'Expand-Archive', - '-Path', - filePath, - '-DestinationPath', - destinationFolder, + '-Command', + 'Expand-Archive -Path "$filePath" -DestinationPath "$destinationFolder" -Force', ]); if (result.exitCode != 0) { throw Exception('Error extracting zip file: ${result.stderr}'); diff --git a/packages/komodo_wallet_build_transformer/lib/src/steps/defi_api_build_step/github_artefact_downloader.dart b/packages/komodo_wallet_build_transformer/lib/src/steps/defi_api_build_step/github_artefact_downloader.dart index dd7e40633..4de88f978 100644 --- a/packages/komodo_wallet_build_transformer/lib/src/steps/defi_api_build_step/github_artefact_downloader.dart +++ b/packages/komodo_wallet_build_transformer/lib/src/steps/defi_api_build_step/github_artefact_downloader.dart @@ -158,11 +158,8 @@ class GithubArtefactDownloader implements ArtefactDownloader { } else if (Platform.isWindows) { // For Windows, use PowerShell's Expand-Archive command final result = await Process.run('powershell', [ - 'Expand-Archive', - '-Path', - filePath, - '-DestinationPath', - destinationFolder, + '-Command', + 'Expand-Archive -Path "$filePath" -DestinationPath "$destinationFolder" -Force', ]); if (result.exitCode != 0) { throw Exception('Error extracting zip file: ${result.stderr}'); diff --git a/packages/komodo_wallet_cli/bin/update_api_config.dart b/packages/komodo_wallet_cli/bin/update_api_config.dart index 3441b1027..8a9b90bb9 100644 --- a/packages/komodo_wallet_cli/bin/update_api_config.dart +++ b/packages/komodo_wallet_cli/bin/update_api_config.dart @@ -68,12 +68,12 @@ void main(List arguments) async { ..addOption( 'source', abbr: 's', - help: 'Source to fetch from (github or mirror)', + help: 'Source to fetch from (github, mirror, or caddy)', defaultsTo: 'github', ) ..addOption( 'mirror-url', - help: 'Mirror URL if using mirror source', + help: 'Mirror URL if using mirror or caddy source', defaultsTo: 'https://sdk.devbuilds.komodo.earth', ) ..addFlag( @@ -205,8 +205,10 @@ calculates their checksums, and updates the build config with this information i the branch name and commit hash. It does not extract or set up the files - that is the responsibility of the build step. -It supports both GitHub releases and the internal mirror site at: -https://sdk.devbuilds.komodo.earth/ +Supported sources: +- github: GitHub releases API +- mirror: HTML-based mirror sites (e.g., sdk.devbuilds.komodo.earth, nebula.decker.im) +- caddy: Caddy file servers with JSON API (e.g., devbuilds.gleec.com) Usage: dart run komodo_wallet_cli:update_api_config [options] @@ -243,7 +245,15 @@ Examples: --config packages/komodo_defi_framework/app_build/build_config.json \ --output-dir packages/komodo_defi_framework/app_build/temp_downloads - # Using a custom mirror URL + # Using GLEEC Caddy mirror (JSON API) + dart run komodo_wallet_cli:update_api_config \ + --branch dev \ + --source caddy \ + --mirror-url https://devbuilds.gleec.com \ + --config packages/komodo_defi_framework/app_build/build_config.json \ + --output-dir packages/komodo_defi_framework/app_build/temp_downloads + + # Using a custom HTML mirror URL dart run komodo_wallet_cli:update_api_config \ --branch dev \ --source mirror \ @@ -279,8 +289,8 @@ class KdfFetcher { outputDirObj.createSync(recursive: true); } - if (source != 'github' && source != 'mirror') { - throw ArgumentError('Source must be either "github" or "mirror"'); + if (source != 'github' && source != 'mirror' && source != 'caddy') { + throw ArgumentError('Source must be "github", "mirror", or "caddy"'); } } @@ -488,6 +498,14 @@ class KdfFetcher { matchingKeyword, matchingPreference, ); + } else if (source == 'caddy') { + return _fetchCaddyDownloadUrl( + platform, + commitHash, + matchingPattern, + matchingKeyword, + matchingPreference, + ); } else { return _fetchMirrorDownloadUrl( platform, @@ -738,6 +756,173 @@ class KdfFetcher { ); } + /// Fetches download URL from a Caddy file server using JSON API. + /// + /// Caddy provides JSON directory listings when the `Accept: application/json` + /// header is included. This is more reliable than HTML scraping. + Future _fetchCaddyDownloadUrl( + String platform, + String commitHash, + String? matchingPattern, + String? matchingKeyword, + List matchingPreference, + ) async { + final normalizedMirror = mirrorUrl.endsWith('/') + ? mirrorUrl + : '$mirrorUrl/'; + final mirrorUri = Uri.parse(normalizedMirror); + final listingUrls = { + if (branch.isNotEmpty) mirrorUri.resolve('$branch/'), + mirrorUri, + }; + + final fullHash = commitHash; + final shortHash = commitHash.substring(0, 7); + log.info( + 'Looking for files with hash $fullHash or $shortHash (Caddy JSON)', + ); + + for (final baseUrl in listingUrls) { + log.fine('Fetching files from Caddy mirror: $baseUrl'); + try { + final candidates = await _searchCaddyDirectory( + baseUrl: baseUrl, + matchingPattern: matchingPattern, + matchingKeyword: matchingKeyword, + fullHash: fullHash, + shortHash: shortHash, + ); + + if (candidates.isNotEmpty) { + final preferred = _choosePreferred( + candidates.keys, + matchingPreference, + ); + final resolved = candidates[preferred] ?? candidates.values.first; + log.info('Found matching files for commit; selected: $resolved'); + return resolved; + } + + // Second pass without commit constraint (only when not strict) + if (!strict) { + final looseCandidates = await _searchCaddyDirectory( + baseUrl: baseUrl, + matchingPattern: matchingPattern, + matchingKeyword: matchingKeyword, + fullHash: null, + shortHash: null, + ); + if (looseCandidates.isNotEmpty) { + final preferred = _choosePreferred( + looseCandidates.keys, + matchingPreference, + ); + final resolved = + looseCandidates[preferred] ?? looseCandidates.values.first; + log.warning( + 'Could not find exact commit match. Using latest matching asset: $resolved', + ); + return resolved; + } + } + + log.fine('No matching files found in $baseUrl'); + } catch (e) { + log.fine('Error querying Caddy mirror $baseUrl: $e'); + } + } + + throw Exception( + 'No matching asset found for platform $platform and commit $commitHash', + ); + } + + /// Recursively searches a Caddy directory for matching files using JSON API. + Future> _searchCaddyDirectory({ + required Uri baseUrl, + required String? matchingPattern, + required String? matchingKeyword, + required String? fullHash, + required String? shortHash, + int maxDepth = 3, + int currentDepth = 0, + }) async { + if (currentDepth >= maxDepth) { + return {}; + } + + final candidates = {}; + + try { + final response = await http.get( + baseUrl, + headers: {'Accept': 'application/json'}, + ); + + if (response.statusCode != 200) { + log.fine('Caddy listing failed at $baseUrl: ${response.statusCode}'); + return {}; + } + + final List entries = jsonDecode(response.body) as List; + + for (final entry in entries) { + final entryMap = entry as Map; + final name = entryMap['name'] as String; + final url = entryMap['url'] as String; + final isDir = entryMap['is_dir'] as bool; + + if (isDir) { + // Recursively search subdirectories + final subUrl = baseUrl.resolve(url); + final subCandidates = await _searchCaddyDirectory( + baseUrl: subUrl, + matchingPattern: matchingPattern, + matchingKeyword: matchingKeyword, + fullHash: fullHash, + shortHash: shortHash, + maxDepth: maxDepth, + currentDepth: currentDepth + 1, + ); + candidates.addAll(subCandidates); + } else { + // Check if file matches criteria + if (!name.endsWith('.zip')) continue; + if (name.contains('wallet')) continue; + + var matches = false; + if (matchingPattern != null) { + try { + final regex = RegExp(matchingPattern); + matches = regex.hasMatch(name); + } catch (e) { + log.warning('Invalid regex pattern: $matchingPattern'); + } + } else if (matchingKeyword != null) { + matches = name.contains(matchingKeyword); + } + + if (!matches) continue; + + // Check hash match if hashes are provided + if (fullHash != null && shortHash != null) { + final containsHash = + name.contains(fullHash) || name.contains(shortHash); + if (!containsHash) continue; + } + + final resolvedUrl = baseUrl.resolve(url).toString(); + candidates[name] = resolvedUrl; + log.fine('Found candidate: $name at $resolvedUrl'); + } + } + } catch (e) { + log.fine('Failed to fetch Caddy directory listing from $baseUrl: $e'); + } + + return candidates; + } + /// Downloads a binary from the given URL Future downloadBinary(String url, String platform) async { log.info('Downloading from: $url');