diff --git a/build.sbt b/build.sbt index ff80b9a5b8ce..68527b1e80e8 100644 --- a/build.sbt +++ b/build.sbt @@ -4688,16 +4688,17 @@ lazy val editions = project .dependsOn( Def.task { Editions.writeEditionConfig( - editionsRoot = file("distribution") / "editions", - ensoVersion = ensoVersion, - editionName = currentEdition, - libraryVersion = stdLibVersion, - log = streams.value.log + editionsRoot = file("distribution") / "editions", + editionTemplate = file("distribution") / "edition.template.yaml", + ensoVersion = ensoVersion, + editionName = currentEdition, + libraryVersion = stdLibVersion, + log = streams.value.log ) } ) .value, - cleanFiles += baseDirectory.value / ".." / ".." / "distribution" / "editions" + cleanFiles += (ThisBuild / baseDirectory).value / "distribution" / "editions" ) .dependsOn(semver) .dependsOn(testkit % Test) @@ -4728,22 +4729,6 @@ lazy val semver = project (`scala-yaml` / Compile / exportedModule).value ) ) - .settings( - (Compile / compile) := (Compile / compile) - .dependsOn( - Def.task { - Editions.writeEditionConfig( - editionsRoot = file("distribution") / "editions", - ensoVersion = ensoVersion, - editionName = currentEdition, - libraryVersion = stdLibVersion, - log = streams.value.log - ) - } - ) - .value, - cleanFiles += baseDirectory.value / ".." / ".." / "distribution" / "editions" - ) .dependsOn(`scala-yaml`) .dependsOn(testkit % Test) @@ -5008,6 +4993,7 @@ lazy val `locking-test-helper` = project ) val `std-lib-root` = file("distribution/lib/Standard/") + def stdLibComponentRoot(name: String): File = `std-lib-root` / name / stdLibVersion val `base-polyglot-root` = stdLibComponentRoot("Base") / "polyglot" / "java" @@ -5309,12 +5295,21 @@ lazy val `opencv-wrapper` = project ), inputJar := "org.openpnp" % "opencv" % opencvVersion, jarExtractor := JarExtractor( - "nu/pattern/opencv/linux/x86_64/*.so" -> PolyglotLib(LinuxAMD64), - "nu/pattern/opencv/osx/ARMv8/*.dylib" -> PolyglotLib(MacOSArm64), - "nu/pattern/opencv/windows/x86_64/*.dll" -> PolyglotLib(WindowsAMD64), - "nu/pattern/*.class" -> CopyToOutputJar, - "META-INF/**" -> CopyToOutputJar, - "org/**" -> CopyToOutputJar + "nu/pattern/opencv/linux/x86_64/*.so" -> PolyglotLib( + LinuxAMD64, + matchArch = false + ), + "nu/pattern/opencv/osx/ARMv8/*.dylib" -> PolyglotLib( + MacOSArm64, + matchArch = false + ), + "nu/pattern/opencv/windows/x86_64/*.dll" -> PolyglotLib( + WindowsAMD64, + matchArch = false + ), + "nu/pattern/*.class" -> CopyToOutputJar, + "META-INF/**" -> CopyToOutputJar, + "org/**" -> CopyToOutputJar ) ) diff --git a/build_tools/build/paths.yaml b/build_tools/build/paths.yaml index 98f514930c6d..f222419ee2f0 100644 --- a/build_tools/build/paths.yaml +++ b/build_tools/build/paths.yaml @@ -94,6 +94,7 @@ rust/: test-results/: small-jdk/: + extension-libs.zip: test/: Benchmarks/: tools/: diff --git a/build_tools/build/src/engine.rs b/build_tools/build/src/engine.rs index 529b726c1970..6a4f7188fca8 100644 --- a/build_tools/build/src/engine.rs +++ b/build_tools/build/src/engine.rs @@ -198,6 +198,9 @@ pub struct BuildConfigurationFlags { pub verify_packages: bool, pub stdlib_api_check: bool, pub run_enso_lint: bool, + /// URL that should be used for a `main` repository in the + /// edition. + pub library_repo_url: Option, } #[derive(Clone, Debug)] @@ -336,6 +339,7 @@ impl Default for BuildConfigurationFlags { extra_java_tool_opts: None, build_small_jdk: false, small_jdk_dir: None, + library_repo_url: None, build_benchmarks: false, check_enso_benchmarks: false, execute_benchmarks: default(), diff --git a/build_tools/build/src/engine/context.rs b/build_tools/build/src/engine/context.rs index bebd13338c4d..0069ea80b2f6 100644 --- a/build_tools/build/src/engine/context.rs +++ b/build_tools/build/src/engine/context.rs @@ -21,10 +21,11 @@ use crate::project::ProcessWrapper; use ide_ci::actions::workflow::is_in_env; use ide_ci::cache; -use ide_ci::github::release::IsReleaseExt; +use ide_ci::github::release::{Handle, IsReleaseExt}; use ide_ci::platform::DEFAULT_SHELL; use ide_ci::programs::sbt; use ide_ci::programs::Sbt; +use octocrab::models::repos::Release; use std::env::consts::DLL_EXTENSION; use std::env::consts::EXE_EXTENSION; @@ -265,6 +266,10 @@ impl RunContext { // we don't want to call this in environments like GH-hosted runners. // === Build distributions and native images === + if let Some(repo_url) = self.config.library_repo_url.clone() { + debug!("Using library repository in main edition: {}", repo_url); + env::ENSO_MAIN_LIBRARY_REPOSITORY_URL.set(&repo_url)?; + } let mut tasks = vec![]; let mut run_sbt_clean = false; if self.config.build_engine_package { @@ -443,7 +448,7 @@ impl RunContext { package.verify_package_sbt(&sbt).await?; } if self.config.build_engine_package { - for libname in ["Base", "Table", "Image", "Database"] { + for libname in ["Base", "Table", "Database"] { let lib_path = self .repo_root .built_distribution @@ -488,6 +493,10 @@ impl RunContext { if TARGET_OS == OS::Linux { release.upload_asset_file(self.paths.manifest_file()).await?; release.upload_asset_file(self.paths.launcher_manifest_file()).await?; + if self.config.library_repo_url.clone().is_some() { + let zip = self.create_extension_libs_zip().await?; + release.upload_asset_file(zip).await?; + } } } }, @@ -632,6 +641,27 @@ impl RunContext { } } + /// Creates a zip archive that contains all the extension libraries. + /// This ZIP should be later uploaded as an artifact to the GH + /// release. + async fn create_extension_libs_zip(&self) -> Result { + debug!("Creating ZIP of all extension libraries"); + let zip_path = self.repo_root.target.extension_libs_zip.path.clone(); + let libs_root_dir = self.extension_libs_root_dir()?; + let paths = vec![libs_root_dir]; + ide_ci::archive::create(&zip_path, paths).await?; + Ok(zip_path) + } + + /// Returns root directory for all the extension libraries. + fn extension_libs_root_dir(&self) -> Result { + let lib_root_dir = + self.repo_root.built_distribution.enso_engine_triple.engine_package.lib.clone(); + let lib_root_dir = lib_root_dir.canonicalize()?; + let extension_libs_root_dir = lib_root_dir.join("Enso"); + Ok(extension_libs_root_dir) + } + fn short_path(&self, full: &Path) -> PathBuf { let strip = full.strip_prefix(self.repo_root.path.clone()); if let Ok(relative) = strip { diff --git a/build_tools/build/src/engine/env.rs b/build_tools/build/src/engine/env.rs index f7ed2bce35b2..d7cdce2c4638 100644 --- a/build_tools/build/src/engine/env.rs +++ b/build_tools/build/src/engine/env.rs @@ -22,4 +22,8 @@ define_env_var! { //// path to JUnit output directory ENSO_TEST_JUNIT_DIR, String; + + /// URL of the main editions library repository which will be written + /// to the `editions.yaml` by sbt. + ENSO_MAIN_LIBRARY_REPOSITORY_URL, String; } diff --git a/build_tools/build/src/release/manifest.rs b/build_tools/build/src/release/manifest.rs index 050ed36da4f4..e9ca316ca298 100644 --- a/build_tools/build/src/release/manifest.rs +++ b/build_tools/build/src/release/manifest.rs @@ -35,7 +35,7 @@ impl Asset { Self { os: triple.os, arch: triple.arch, url, target_pretty } } - /// Description od the asset with IDE image. + /// Description of the asset with IDE image. pub fn new_ide(repo: &impl IsRepo, triple: &TargetTriple) -> Self { let filename = project::ide::electron_image_filename(triple.os, triple.arch, &triple.versions.version); @@ -43,7 +43,7 @@ impl Asset { Self::new(url, triple) } - /// Description od the asset with Engine bundle. + /// Description of the asset with Engine bundle. pub fn new_engine(repo: &impl IsRepo, triple: &TargetTriple) -> Self { use crate::paths::generated::RepoRootBuiltDistributionEnsoBundleTriple; let stem = RepoRootBuiltDistributionEnsoBundleTriple::segment_name(triple.to_string()); diff --git a/build_tools/cli/src/lib.rs b/build_tools/cli/src/lib.rs index 7e5cc57b48e7..a9889dcc8d8e 100644 --- a/build_tools/cli/src/lib.rs +++ b/build_tools/cli/src/lib.rs @@ -26,7 +26,6 @@ use crate::arg::WatchJob; use anyhow::Context; use arg::BuildDescription; use clap::Parser; -use enso_build::cloud_tests; use enso_build::config::Config; use enso_build::context::BuildContext; use enso_build::engine::context::EnginePackageProvider; @@ -57,6 +56,7 @@ use enso_build::source::Source; use enso_build::source::WatchTargetJob; use enso_build::source::WithDestination; use enso_build::version; +use enso_build::{cloud_tests, env}; use ide_ci::actions::workflow::is_in_env; use ide_ci::cache::goodie::graalvm; use ide_ci::cache::Cache; @@ -303,8 +303,10 @@ impl Processor { let repo = self.remote_repo.clone(); let context = self.context(); let small_jdk_dir = context.repo_root.target.small_jdk.path.clone(); + let cloned_self = self.clone(); async move { let input = input.await?; + let extension_lib_url = cloned_self.extensions_lib_url().await?; let operation = enso_build::engine::Operation::Release( enso_build::engine::ReleaseOperation { repo, @@ -318,6 +320,7 @@ impl Processor { build_small_jdk: true, small_jdk_dir: Some(small_jdk_dir), verify_packages: true, + library_repo_url: extension_lib_url, ..default() }; let context = input.prepare_context(context, config)?; @@ -524,6 +527,34 @@ impl Processor { .boxed() } + /// Returns URL to the extension library zip archive. This URL points to an artifact from the release. + /// If [env::RELEASE_ID] is not set, it returns None. + /// + /// The returned URL has a format similar to: + /// `jar:https://github.com/enso-org/enso/releases/download/2025.4.1-nightly.2025.12.28/extension-libs.zip` + async fn extensions_lib_url(&self) -> Result> { + match env::ENSO_RELEASE_ID.get() { + Ok(_) => { + let extension_libs_fname = self + .repo_root + .target + .extension_libs_zip + .as_path() + .file_name() + .unwrap() + .to_str() + .unwrap(); + let tag = version::ENSO_VERSION.get()?; + let final_url = format!( + "jar:https://github.com/enso-org/enso/releases/download/{}/{}", + tag, extension_libs_fname + ); + Ok(Some(final_url)) + } + Err(_) => Ok(None), + } + } + /// Add options to produce heap dumps on OOM errors. /// It is essential to pass the `-XX:+HeapDumpOnOutOfMemoryError` option both via /// `JAVA_TOOL_OPTIONS` env var and as a command line argument to the runner. diff --git a/distribution/edition.template.yaml b/distribution/edition.template.yaml new file mode 100644 index 000000000000..41c0fe410235 --- /dev/null +++ b/distribution/edition.template.yaml @@ -0,0 +1,73 @@ +# This file is a template and should not be used directly. +# Replace placeholders in double brackets with actual values. +# +# See [editions.md](https://github.com/enso-org/enso/blob/87ce78615afecb8bd8d586c798c97ab5e28083cd/docs/libraries/editions.md) +# for the description of its contents. +# +# Parameters for the template: +# - ENGINE_VERSION: the version of the engine to use in this edition. +# - LIBS_VERSION: version of all the standard libraries. +# - MAIN_REPO_URL: URL of the main repository hosting the standard libraries. +# - STD_IMAGE_REPO_URL: URL of the repository hosting the Standard.Image library. + +engine-version: {{ENGINE_VERSION}} +repositories: + - name: main + url: {{MAIN_REPO_URL}} + - name: std_image_repo + url: {{STD_IMAGE_REPO_URL}} +libraries: + - name: Standard.Base + repository: main + version: {{LIBS_VERSION}} + - name: Standard.Test + repository: main + version: {{LIBS_VERSION}} + - name: Standard.Table + repository: main + version: {{LIBS_VERSION}} + - name: Standard.Database + repository: main + version: {{LIBS_VERSION}} + - name: Standard.AWS + repository: main + version: {{LIBS_VERSION}} + - name: Standard.Geo + repository: main + version: {{LIBS_VERSION}} + - name: Standard.Visualization + repository: main + version: {{LIBS_VERSION}} + - name: Standard.Examples + repository: main + version: {{LIBS_VERSION}} + - name: Standard.Searcher + repository: main + version: {{LIBS_VERSION}} + - name: Standard.Generic_JDBC + repository: main + version: {{LIBS_VERSION}} + - name: Standard.Google + repository: main + version: {{LIBS_VERSION}} + - name: Standard.Google_Api + repository: main + version: {{LIBS_VERSION}} + - name: Standard.Snowflake + repository: main + version: {{LIBS_VERSION}} + - name: Standard.Microsoft + repository: main + version: {{LIBS_VERSION}} + - name: Standard.Tableau + repository: main + version: {{LIBS_VERSION}} + - name: Standard.Saas + repository: main + version: {{LIBS_VERSION}} + - name: Standard.DuckDB + repository: main + version: {{LIBS_VERSION}} + - name: Standard.Image + repository: std_image_repo + version: {{LIBS_VERSION}} diff --git a/project/DistributionPackage.scala b/project/DistributionPackage.scala index 0f3d45751de4..15b0453e6094 100644 --- a/project/DistributionPackage.scala +++ b/project/DistributionPackage.scala @@ -168,11 +168,12 @@ object DistributionPackage { (distributionRoot / "editions").mkdirs() Editions.writeEditionConfig( - editionsRoot = distributionRoot / "editions", - ensoVersion = ensoVersion, - editionName = editionName, - libraryVersion = targetStdlibVersion, - log = log + editionsRoot = distributionRoot / "editions", + editionTemplate = file("distribution/edition.template.yaml"), + ensoVersion = ensoVersion, + editionName = editionName, + libraryVersion = targetStdlibVersion, + log = log ) copyLibraryCacheIncremental( @@ -641,20 +642,22 @@ object DistributionPackage { val targetPackageRoot = destinationRoot / prefix / libName / targetVersion val libSourceDir = sourceRoot / prefix / libName / sourceVersion - val copied = copyDirectoryIncremental( - source = libSourceDir, - destination = targetPackageRoot, - cache = cacheFactory.make(s"$prefix.$libName") - ) - val bindingsDir = targetPackageRoot / ".enso" / "cache" / "bindings" - if (copied && bindingsDir.exists()) { - log.info( - s"Clearing cached bindings for $prefix.$libName, because library sources were changed." + if (libSourceDir.exists() && libSourceDir.isDirectory) { + val copied = copyDirectoryIncremental( + source = libSourceDir, + destination = targetPackageRoot, + cache = cacheFactory.make(s"$prefix.$libName") ) - IO.delete(bindingsDir) + val bindingsDir = targetPackageRoot / ".enso" / "cache" / "bindings" + if (copied && bindingsDir.exists()) { + log.info( + s"Clearing cached bindings for $prefix.$libName, because library sources were changed." + ) + IO.delete(bindingsDir) + } + fixLibraryManifest(targetPackageRoot, targetVersion, log) + existingLibraries.append((prefix, libName)) } - fixLibraryManifest(targetPackageRoot, targetVersion, log) - existingLibraries.append((prefix, libName)) } } diff --git a/project/Editions.scala b/project/Editions.scala index 654c6c69b0bb..8da9e3a919e9 100644 --- a/project/Editions.scala +++ b/project/Editions.scala @@ -1,42 +1,25 @@ import sbt._ object Editions { + val MAIN_REPO_ENV = "ENSO_MAIN_LIBRARY_REPOSITORY_URL" + val STD_IMAGE_REPO_ENV = "ENSO_STD_IMAGE_REPOSITORY_URL" - /** List of libraries that are shipped with the engine and reside in the - * engine repository. - * - * They currently all share the version number. - */ - val standardLibraries: Seq[String] = Seq( - "Standard.Base", - "Standard.Test", - "Standard.Table", - "Standard.Database", - "Standard.AWS", - "Standard.Image", - "Standard.Geo", - "Standard.Visualization", - "Standard.Examples", - "Standard.Searcher", - "Standard.Generic_JDBC", - "Standard.Google", - "Standard.Google_Api", - "Standard.Snowflake", - "Standard.Microsoft", - "Standard.Tableau", - "Standard.Saas", - "Standard.DuckDB" - ) - - case class ContribLibrary(name: String, version: String) - - /** A list of additional libraries from external sources that are published in - * the main repository and should be available in the default edition. - */ - val contribLibraries: Seq[ContribLibrary] = Seq() + private val fallbackUrl = "https://libraries.release.enso.org/libraries" /** The URL to the main library repository. */ - val mainLibraryRepositoryUrl = "https://libraries.release.enso.org/libraries" + private val mainLibraryRepositoryUrl: String = { + System.getenv(MAIN_REPO_ENV) match { + case null | "" => fallbackUrl + case url => url + } + } + + private val stdImageRepositoryUrl: String = { + System.getenv(STD_IMAGE_REPO_ENV) match { + case null | "" => fallbackUrl + case url => url + } + } private val extension = ".yaml" @@ -45,11 +28,18 @@ object Editions { */ def writeEditionConfig( editionsRoot: File, + editionTemplate: File, ensoVersion: String, editionName: String, libraryVersion: String, log: Logger ): Unit = { + if (!editionTemplate.exists()) { + log.error( + s"Edition template file [${editionTemplate.getAbsolutePath}] does " + + s"not exist. Skipping edition generation." + ) + } IO.createDirectory(editionsRoot) val edition = editionsRoot / (editionName + extension) @@ -60,31 +50,26 @@ object Editions { } } + val templateContent = IO.read(editionTemplate) + val comment = + """ + |# This file was generated automatically by `project/Editions.scala`. + |# Do not edit it directly. + |""".stripMargin val editionConfigContent = { - val standardLibrariesConfigs = standardLibraries.map { libName => - s""" - name: $libName - | repository: main - | version: $libraryVersion""".stripMargin + val replaced = templateContent + .replaceAll("\\{\\{ENGINE_VERSION}}", ensoVersion) + .replaceAll("\\{\\{LIBS_VERSION}}", libraryVersion) + .replaceAll("\\{\\{MAIN_REPO_URL}}", mainLibraryRepositoryUrl) + .replaceAll("\\{\\{STD_IMAGE_REPO_URL}}", stdImageRepositoryUrl) + val allVarsReplaced = !replaced.contains("{{") + if (!allVarsReplaced) { + log.error( + s"Not all template variables were replaced in edition template " + + s"[${editionTemplate.getAbsolutePath}]." + ) } - - val contribLibrariesConfigs = contribLibraries.map { - case ContribLibrary(name, version) => - s""" - name: $name - | repository: main - | version: $version""".stripMargin - } - - val librariesConfigs = standardLibrariesConfigs ++ contribLibrariesConfigs - - val editionConfig = - s"""engine-version: $ensoVersion - |repositories: - | - name: main - | url: $mainLibraryRepositoryUrl - |libraries: - |${librariesConfigs.mkString("\n")} - |""".stripMargin - editionConfig + comment + System.lineSeparator() + replaced } val currentContent = if (edition.exists()) Some(IO.read(edition)) else None diff --git a/project/JarExtractor.scala b/project/JarExtractor.scala index a16782614d1d..377c52261f48 100644 --- a/project/JarExtractor.scala +++ b/project/JarExtractor.scala @@ -62,14 +62,15 @@ object JarExtractor { * For example, if the entry is `foo.so` and the `arch` parameter is * [[LinuxAMD64]], the entry will be copied to `amd64/linux/foo.so`. * - * The entry will be copied only if the architecture matches the current - * platform's architecture. + * If `matchArch` is true, the entry will be copied only if the architecture matches the current + * platform's architecture, otherwise it will be copied unconditionally. * * @param arch If specified, will be copied only iff the architecture is * the same as the current platform. */ case class PolyglotLib( - arch: NativeLibArch + arch: NativeLibArch, + matchArch: Boolean = true ) extends Command /** Traverses all the entries in the input JAR file and extracts files @@ -111,7 +112,7 @@ object JarExtractor { command match { case CopyToOutputJar => copyEntry(outputJar, inputJar, entry, logger) - case PolyglotLib(arch) => + case PolyglotLib(arch, matchArch) => // Silently rename the old `*.jnilib` files to `*.dylib`. val fullPath = entryPath.getFileName.toString val idx = fullPath.lastIndexOf('.') @@ -137,7 +138,9 @@ object JarExtractor { val destPath = polyglotLibDir .resolve(arch.path) .resolve(fullPath2) - if (archMatchesCurPlatform(arch)) { + val shouldCopy = + if (matchArch) archMatchesCurPlatform(arch) else true + if (shouldCopy) { copyEntry(destPath, inputJar, entry, logger) } }