Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 30 additions & 27 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -4715,7 +4715,7 @@ lazy val editions = project
}
)
.value,
cleanFiles += baseDirectory.value / ".." / ".." / "distribution" / "editions"
cleanFiles += (ThisBuild / baseDirectory).value / "distribution" / "editions"
)
.dependsOn(semver)
.dependsOn(testkit % Test)
Expand Down Expand Up @@ -4746,22 +4746,6 @@ lazy val semver = project
(`scala-yaml` / Compile / exportedModule).value
)
)
.settings(
(Compile / compile) := (Compile / compile)
.dependsOn(
Def.task {
Editions.writeEditionConfig(
editionsRoot = file("distribution") / "editions",
ensoVersion = ensoVersion,
editionName = currentEdition,
libraryVersion = stdLibVersion,
log = streams.value.log
)
}
)
.value,
cleanFiles += baseDirectory.value / ".." / ".." / "distribution" / "editions"
)
.dependsOn(`scala-yaml`)
.dependsOn(testkit % Test)

Expand Down Expand Up @@ -5025,13 +5009,20 @@ lazy val `locking-test-helper` = project
assembly / assemblyOutputPath := file("locking-test-helper.jar")
)

val `std-lib-root` = file("distribution/lib/Standard/")
val `std-lib-root` = file("distribution/lib/Standard/")
val `extension-lib-root` = file("distribution/lib/Enso/")

def stdLibComponentRoot(name: String): File =
`std-lib-root` / name / stdLibVersion
def extensionLibComponentRoot(name: String): File =
`extension-lib-root` / name / stdLibVersion

val `base-polyglot-root` = stdLibComponentRoot("Base") / "polyglot" / "java"
val `table-polyglot-root` = stdLibComponentRoot("Table") / "polyglot" / "java"
val `image-polyglot-root` = stdLibComponentRoot("Image") / "polyglot" / "java"
val `image-native-libs` = stdLibComponentRoot("Image") / "polyglot" / "lib"
val `image-polyglot-root` =
extensionLibComponentRoot("Image") / "polyglot" / "java"
val `image-native-libs` =
extensionLibComponentRoot("Image") / "polyglot" / "lib"
val `generic-jdbc-polyglot-root` =
stdLibComponentRoot("Generic_JDBC") / "polyglot" / "java"
val `generic-jdbc-native-libs` =
Expand Down Expand Up @@ -5327,13 +5318,25 @@ lazy val `opencv-wrapper` = project
),
inputJar := "org.openpnp" % "opencv" % opencvVersion,
jarExtractor := JarExtractor(
"nu/pattern/opencv/linux/x86_64/*.so" -> PolyglotLib(LinuxAMD64),
"nu/pattern/opencv/osx/ARMv8/*.dylib" -> PolyglotLib(MacOSArm64),
"nu/pattern/opencv/osx/x86_64/*.dylib" -> PolyglotLib(MacOSAMD64),
"nu/pattern/opencv/windows/x86_64/*.dll" -> PolyglotLib(WindowsAMD64),
"nu/pattern/*.class" -> CopyToOutputJar,
"META-INF/**" -> CopyToOutputJar,
"org/**" -> CopyToOutputJar
"nu/pattern/opencv/linux/x86_64/*.so" -> PolyglotLib(
LinuxAMD64,
matchArch = false
),
"nu/pattern/opencv/osx/ARMv8/*.dylib" -> PolyglotLib(
MacOSArm64,
matchArch = false
),
"nu/pattern/opencv/osx/x86_64/*.dylib" -> PolyglotLib(
MacOSAMD64,
matchArch = false
),
"nu/pattern/opencv/windows/x86_64/*.dll" -> PolyglotLib(
WindowsAMD64,
matchArch = false
),
"nu/pattern/*.class" -> CopyToOutputJar,
"META-INF/**" -> CopyToOutputJar,
"org/**" -> CopyToOutputJar
)
)

Expand Down
1 change: 1 addition & 0 deletions build_tools/build/paths.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@
rust/:
test-results/:
small-jdk/:
extension-libs.zip:
test/:
Benchmarks/:
tools/:
Expand Down
4 changes: 4 additions & 0 deletions build_tools/build/src/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,9 @@ pub struct BuildConfigurationFlags {
pub verify_packages: bool,
pub stdlib_api_check: bool,
pub run_enso_lint: bool,
/// URL that should be used for a `main` repository in the
/// edition.
pub library_repo_url: Option<String>,
}

#[derive(Clone, Debug)]
Expand Down Expand Up @@ -336,6 +339,7 @@ impl Default for BuildConfigurationFlags {
extra_java_tool_opts: None,
build_small_jdk: false,
small_jdk_dir: None,
library_repo_url: None,
build_benchmarks: false,
check_enso_benchmarks: false,
execute_benchmarks: default(),
Expand Down
34 changes: 32 additions & 2 deletions build_tools/build/src/engine/context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,11 @@ use crate::project::ProcessWrapper;

use ide_ci::actions::workflow::is_in_env;
use ide_ci::cache;
use ide_ci::github::release::IsReleaseExt;
use ide_ci::github::release::{Handle, IsReleaseExt};
use ide_ci::platform::DEFAULT_SHELL;
use ide_ci::programs::sbt;
use ide_ci::programs::Sbt;
use octocrab::models::repos::Release;
use std::env::consts::DLL_EXTENSION;
use std::env::consts::EXE_EXTENSION;

Expand Down Expand Up @@ -265,6 +266,10 @@ impl RunContext {
// we don't want to call this in environments like GH-hosted runners.

// === Build distributions and native images ===
if let Some(repo_url) = self.config.library_repo_url.clone() {
debug!("Using library repository in main edition: {}", repo_url);
env::ENSO_MAIN_LIBRARY_REPOSITORY_URL.set(&repo_url)?;
}
let mut tasks = vec![];
let mut run_sbt_clean = false;
if self.config.build_engine_package {
Expand Down Expand Up @@ -443,7 +448,7 @@ impl RunContext {
package.verify_package_sbt(&sbt).await?;
}
if self.config.build_engine_package {
for libname in ["Base", "Table", "Image", "Database"] {
for libname in ["Base", "Table", "Database"] {
let lib_path = self
.repo_root
.built_distribution
Expand Down Expand Up @@ -488,6 +493,10 @@ impl RunContext {
if TARGET_OS == OS::Linux {
release.upload_asset_file(self.paths.manifest_file()).await?;
release.upload_asset_file(self.paths.launcher_manifest_file()).await?;
if self.config.library_repo_url.clone().is_some() {
let zip = self.create_extension_libs_zip().await?;
release.upload_asset_file(zip).await?;
}
}
}
},
Expand Down Expand Up @@ -632,6 +641,27 @@ impl RunContext {
}
}

/// Creates a zip archive that contains all the extension libraries.
/// This ZIP should be later uploaded as an artifact to the GH
/// release.
async fn create_extension_libs_zip(&self) -> Result<PathBuf> {
debug!("Creating ZIP of all extension libraries");
let zip_path = self.repo_root.target.extension_libs_zip.path.clone();
let libs_root_dir = self.extension_libs_root_dir()?;
let paths = vec![libs_root_dir];
ide_ci::archive::create(&zip_path, paths).await?;
Ok(zip_path)
}

/// Returns root directory for all the extension libraries.
fn extension_libs_root_dir(&self) -> Result<PathBuf> {
let lib_root_dir =
self.repo_root.built_distribution.enso_engine_triple.engine_package.lib.clone();
let lib_root_dir = lib_root_dir.canonicalize()?;
let extension_libs_root_dir = lib_root_dir.join("Enso");
Ok(extension_libs_root_dir)
}

fn short_path(&self, full: &Path) -> PathBuf {
let strip = full.strip_prefix(self.repo_root.path.clone());
if let Ok(relative) = strip {
Expand Down
4 changes: 4 additions & 0 deletions build_tools/build/src/engine/env.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,8 @@ define_env_var! {

//// path to JUnit output directory
ENSO_TEST_JUNIT_DIR, String;

/// URL of the main editions library repository which will be written
/// to the `editions.yaml` by sbt.
ENSO_MAIN_LIBRARY_REPOSITORY_URL, String;
}
4 changes: 2 additions & 2 deletions build_tools/build/src/release/manifest.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,15 @@ impl Asset {
Self { os: triple.os, arch: triple.arch, url, target_pretty }
}

/// Description od the asset with IDE image.
/// Description of the asset with IDE image.
pub fn new_ide(repo: &impl IsRepo, triple: &TargetTriple) -> Self {
let filename =
project::ide::electron_image_filename(triple.os, triple.arch, &triple.versions.version);
let url = ide_ci::github::release::download_asset(repo, &triple.versions.version, filename);
Self::new(url, triple)
}

/// Description od the asset with Engine bundle.
/// Description of the asset with Engine bundle.
pub fn new_engine(repo: &impl IsRepo, triple: &TargetTriple) -> Self {
use crate::paths::generated::RepoRootBuiltDistributionEnsoBundleTriple;
let stem = RepoRootBuiltDistributionEnsoBundleTriple::segment_name(triple.to_string());
Expand Down
33 changes: 32 additions & 1 deletion build_tools/cli/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ use crate::arg::WatchJob;
use anyhow::Context;
use arg::BuildDescription;
use clap::Parser;
use enso_build::cloud_tests;
use enso_build::config::Config;
use enso_build::context::BuildContext;
use enso_build::engine::context::EnginePackageProvider;
Expand Down Expand Up @@ -57,6 +56,7 @@ use enso_build::source::Source;
use enso_build::source::WatchTargetJob;
use enso_build::source::WithDestination;
use enso_build::version;
use enso_build::{cloud_tests, env};
use ide_ci::actions::workflow::is_in_env;
use ide_ci::cache::goodie::graalvm;
use ide_ci::cache::Cache;
Expand Down Expand Up @@ -303,8 +303,10 @@ impl Processor {
let repo = self.remote_repo.clone();
let context = self.context();
let small_jdk_dir = context.repo_root.target.small_jdk.path.clone();
let cloned_self = self.clone();
async move {
let input = input.await?;
let extension_lib_url = cloned_self.extensions_lib_url().await?;
let operation = enso_build::engine::Operation::Release(
enso_build::engine::ReleaseOperation {
repo,
Expand All @@ -318,6 +320,7 @@ impl Processor {
build_small_jdk: true,
small_jdk_dir: Some(small_jdk_dir),
verify_packages: true,
library_repo_url: extension_lib_url,
..default()
};
let context = input.prepare_context(context, config)?;
Expand Down Expand Up @@ -524,6 +527,34 @@ impl Processor {
.boxed()
}

/// Returns URL to the extension library zip archive. This URL points to an artifact from the release.
/// If [env::RELEASE_ID] is not set, it returns None.
///
/// The returned URL has a format similar to:
/// `jar:https://github.com/enso-org/enso/releases/download/2025.4.1-nightly.2025.12.28/extension-libs.zip`
async fn extensions_lib_url(&self) -> Result<Option<String>> {
match env::ENSO_RELEASE_ID.get() {
Ok(_) => {
let extension_libs_fname = self
.repo_root
.target
.extension_libs_zip
.as_path()
.file_name()
.unwrap()
.to_str()
.unwrap();
let tag = version::ENSO_VERSION.get()?;
let final_url = format!(
"jar:https://github.com/enso-org/enso/releases/download/{}/{}",
tag, extension_libs_fname
);
Ok(Some(final_url))
}
Err(_) => Ok(None),
}
}

/// Add options to produce heap dumps on OOM errors.
/// It is essential to pass the `-XX:+HeapDumpOnOutOfMemoryError` option both via
/// `JAVA_TOOL_OPTIONS` env var and as a command line argument to the runner.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: Image
namespace: Standard
namespace: Enso
version: 0.0.0-dev
license: APLv2
authors:
Expand All @@ -11,4 +11,4 @@ maintainers:

services:
- provides: Standard.Base.System.File_Format.File_Format_SPI
with: Standard.Image.Image_Services.Impl
with: Enso.Image.Image_Services.Impl
6 changes: 6 additions & 0 deletions distribution/lib/Enso/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
## Enso library namespace

As opposed to the `Standard` library namespace, the libraries in this namespace
(under this directory) are meant as an _extension_ libraries that will be downloaded
automatically on demand. These libraries are not distributed with the core Enso
distribution.
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import Standard.Base.Runtime.Context

from Standard.Table import Column, Table

from Standard.Image import Image, Matrix, Read_Flag
from Enso.Image import Image, Matrix, Read_Flag

## An example error type used in a number of examples.
type Example_Error_Type
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,12 @@ public void standardImageShouldHaveNativeLib() {
// Evaluate dummy sources to force loading Standard.Image
ctxRule.evalModule(
"""
from Standard.Image import all
from Enso.Image import all
main = 42
""");
var ensoCtx = ctxRule.ensoContext();
var stdImg =
ensoCtx
.getPackageRepository()
.getPackageForLibraryJava(LibraryName.apply("Standard", "Image"));
ensoCtx.getPackageRepository().getPackageForLibraryJava(LibraryName.apply("Enso", "Image"));
assertThat(stdImg.isPresent(), is(true));
this.stdImgPkg = stdImg.get();
var nativeLibs =
Expand Down
26 changes: 14 additions & 12 deletions project/DistributionPackage.scala
Original file line number Diff line number Diff line change
Expand Up @@ -614,20 +614,22 @@ object DistributionPackage {
val targetPackageRoot =
destinationRoot / prefix / libName / targetVersion
val libSourceDir = sourceRoot / prefix / libName / sourceVersion
val copied = copyDirectoryIncremental(
source = libSourceDir,
destination = targetPackageRoot,
cache = cacheFactory.make(s"$prefix.$libName")
)
val bindingsDir = targetPackageRoot / ".enso" / "cache" / "bindings"
if (copied && bindingsDir.exists()) {
log.info(
s"Clearing cached bindings for $prefix.$libName, because library sources were changed."
if (libSourceDir.exists() && libSourceDir.isDirectory) {
val copied = copyDirectoryIncremental(
source = libSourceDir,
destination = targetPackageRoot,
cache = cacheFactory.make(s"$prefix.$libName")
)
IO.delete(bindingsDir)
val bindingsDir = targetPackageRoot / ".enso" / "cache" / "bindings"
if (copied && bindingsDir.exists()) {
log.info(
s"Clearing cached bindings for $prefix.$libName, because library sources were changed."
)
IO.delete(bindingsDir)
}
fixLibraryManifest(targetPackageRoot, targetVersion, log)
existingLibraries.append((prefix, libName))
}
fixLibraryManifest(targetPackageRoot, targetVersion, log)
existingLibraries.append((prefix, libName))
}
}

Expand Down
Loading
Loading