diff --git a/.github/workflows/bazel-build.yml b/.github/workflows/bazel-build.yml index d42cf4c986b9..13bdbc13e898 100644 --- a/.github/workflows/bazel-build.yml +++ b/.github/workflows/bazel-build.yml @@ -54,12 +54,16 @@ jobs: runs-on: - ubuntu-latest steps: + - name: Install native image dependencies + run: | + sudo apt-get update + sudo apt-get install build-essential zlib1g-dev - uses: bazel-contrib/setup-bazel@0.13.0 with: bazelisk-cache: true bazelrc: build --remote_cache=grpcs://${{ vars.ENSO_BAZEL_CACHE_URI }} --remote_cache_header="authorization=Basic ${{ secrets.ENSO_BAZEL_CACHE_TOKEN }}" - uses: actions/checkout@v4 - - run: bazel build //:sbt_build_engine_distribution + - run: bazel build --verbose_failures //:sbt_build_native_engine_distribution - name: Get Engine build output location id: get_bazel_output run: | diff --git a/BUILD.bazel b/BUILD.bazel index 8a6383a95a44..2f90f5728889 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -2,7 +2,7 @@ load("@aspect_bazel_lib//lib:write_source_files.bzl", "write_source_files") load("@aspect_rules_ts//ts:defs.bzl", "ts_config") load("@buildifier_prebuilt//:rules.bzl", "buildifier") load("@npm//:defs.bzl", "npm_link_all_packages") -load("//:bazel_scripts/run_engine_distribution.bzl", "run_enso") +load("//:bazel_scripts/run_engine_distribution.bzl", "run_enso", "ensure_native_enso") load("//:bazel_scripts/stdlibs.bzl", "stdlib_source_patterns") load("//toolchains/sbt:run_sbt.bzl", "run_sbt") @@ -213,6 +213,14 @@ ENV = { "SBT_SERVER_FORCESTART": "1", } +NATIVE_IMAGE_ENV = { + "JAVA_HOME": "$(JAVABASE)", + "PATH": "$(FLATC_PATH):$(JAVABASE)/bin:$$PATH", + "LC_ALL": "C.UTF-8", + "SBT_SERVER_FORCESTART": "1", + "ENSO_LAUNCHER": "native,fast,-ls" +} + run_sbt( name = "sbt_build_engine_distribution", srcs = SRCS, @@ -225,12 +233,31 @@ run_sbt( visibility = ["//visibility:public"], ) +run_sbt( + name = "sbt_build_native_engine_distribution", + srcs = SRCS, + args = [ + "buildEngineDistribution" + ], + env = NATIVE_IMAGE_ENV, + out_dir = "built-distribution-native", + system_props = SBT_SYSTEM_PROPS + EXTRA_SYSTEM_PROPS, + visibility = ["//visibility:public"], +) + # This is just a sanity check that runs equivalent of `enso --run test/Base_Tests/**/Maybe_Spec.enso`. run_enso( name = "run_enso_test", src = "//test/Base_Tests:maybe_spec", - distribution = ":sbt_build_engine_distribution", + distribution = ":sbt_build_native_engine_distribution", run_args = [ "--no-ir-caches", ], ) + +# Sanity check that ensure that the distribution was built with native image. +ensure_native_enso( + name = "ensure_native_enso", + distribution = ":sbt_build_native_engine_distribution", + visibility = ["//visibility:public"], +) diff --git a/bazel_scripts/run_engine_distribution.bzl b/bazel_scripts/run_engine_distribution.bzl index 60546553e786..a0dd3c6ec9e0 100644 --- a/bazel_scripts/run_engine_distribution.bzl +++ b/bazel_scripts/run_engine_distribution.bzl @@ -5,7 +5,7 @@ Simple rule that runs Enso distribution via a shell script. def _run_enso_impl(ctx): distribution = ctx.attr.distribution[DefaultInfo].files binary = ctx.actions.declare_file(ctx.label.name + ".sh") - dist_dir = distribution.to_list()[0].path + dist_dir = distribution.to_list()[0].basename src_file = ctx.file.src.path java_toolchain_type = ctx.toolchains["@bazel_tools//tools/jdk:runtime_toolchain_type"] java_home = java_toolchain_type.java_runtime.java_home_runfiles_path @@ -15,7 +15,7 @@ def _run_enso_impl(ctx): content = """#!/bin/bash export JAVA_HOME="{java_home}" export PATH="$JAVA_HOME/bin:$PATH" - binary_path=built-distribution/enso-engine-*/enso-*/bin/enso + binary_path={dist}/enso-engine-*/enso-*/bin/enso if [ ! -f $binary_path ]; then echo "Error: Could not find enso binary in {dist}" exit 1 @@ -42,6 +42,53 @@ def _run_enso_impl(ctx): runfiles = all_runfiles, )] + +def _ensure_native_enso_impl(ctx): + """ Runs enso executable with `--version` argument to ensure it was built with native image """ + distribution = ctx.attr.distribution[DefaultInfo].files + binary = ctx.actions.declare_file(ctx.label.name + ".sh") + dist_dir = distribution.to_list()[0].basename + ctx.actions.write( + output = binary, + content = """#!/bin/bash + binary_path={dist}/enso-engine-*/enso-*/bin/enso + if [ ! -f $binary_path ]; then + echo "Error: Could not find enso binary in {dist}" + exit 1 + fi + $PWD/$binary_path --version + $PWD/$binary_path --version | grep Substrate > /dev/null + if [ $? -ne 0 ]; then + echo "Error: enso binary is not a native image build" + exit 1 + fi + """.format( + dist = dist_dir + ), + is_executable = True, + ) + + all_runfiles = ctx.runfiles( + files = distribution.to_list() + ) + + return [DefaultInfo( + executable = binary, + runfiles = all_runfiles, + )] + + +ensure_native_enso = rule( + implementation = _ensure_native_enso_impl, + attrs = { + "distribution": attr.label( + mandatory = True, + allow_files = True + ) + }, + executable = True +) + run_enso = rule( implementation = _run_enso_impl, toolchains = [ diff --git a/build.sbt b/build.sbt index f649aa48b41b..f0bec7490e29 100644 --- a/build.sbt +++ b/build.sbt @@ -4047,6 +4047,13 @@ lazy val `engine-runner` = project "-Dnic=nic" ) else Seq() + val cCompilerOpts = if ((Bazel / wasStartedFromBazel).value) { + Seq( + "-H:CCompilerPath=" + (Bazel / cCompilerPath).value.getAbsolutePath + ) + } else { + Seq() + } val mp = (Runtime / modulePath).value.map(_.getAbsolutePath) NativeImage .buildNativeImage( @@ -4077,7 +4084,7 @@ lazy val `engine-runner` = project "--add-opens=org.graalvm.nativeimage.builder/com.oracle.svm.core.jdk=ALL-UNNAMED", // Snowflake uses Apache Arrow (equivalent of #9664 in native-image setup) "--add-opens=java.base/java.nio=ALL-UNNAMED" - ) ++ enableHeapDumpOpts ++ debugOpts, + ) ++ enableHeapDumpOpts ++ debugOpts ++ cCompilerOpts, mainModule = Some("org.enso.runner"), mainClass = Some("org.enso.runner.Main"), initializeAtRuntime = Seq( diff --git a/project/BazelSupport.scala b/project/BazelSupport.scala index 9991df44a38e..e2a4d5a371ff 100644 --- a/project/BazelSupport.scala +++ b/project/BazelSupport.scala @@ -23,6 +23,7 @@ object BazelSupport extends AutoPlugin { val EXTRACTED_PYTHON_RESOURCES_PROP = "enso.BazelSupport.python.resourceDir" val YDOC_SERVER_POLYGLOT_MAIN_JS = "enso.BazelSupport.ydocServer.polyglotMainJs" + val C_COMPILER_PATH = "enso.BazelSupport.CCompilerPath" object autoImport { lazy val wasStartedFromBazel = settingKey[Boolean]( @@ -52,6 +53,9 @@ object BazelSupport extends AutoPlugin { lazy val extractedPythonResourceDir = taskKey[File]( "Directory containing extracted Python resources" ) + lazy val cCompilerPath = taskKey[File]( + "Path to the C Compiler. Will be passed to native-image via `-H:CCompilerPath`." + ) lazy val ydocServerPolyglotMainJs = taskKey[File]( "Path to the ydoc-server polyglot main JS file." ) @@ -170,6 +174,23 @@ object BazelSupport extends AutoPlugin { ) } jsFile + }, + Bazel / cCompilerPath := { + val logger = streams.value.log + val prop = System.getProperty(C_COMPILER_PATH) + if (prop == null) { + logger.error( + s"C Compiler path not set in ${C_COMPILER_PATH} property." + ) + } + val compiler = new File(prop) + if (!compiler.exists()) { + logger.warn( + s"C Compiler not found at $compiler. " + + "Make sure to provide a valid C Compiler." + ) + } + compiler } ) } diff --git a/toolchains/sbt/run_sbt.bzl b/toolchains/sbt/run_sbt.bzl index cc97f0128211..b815019e81ea 100644 --- a/toolchains/sbt/run_sbt.bzl +++ b/toolchains/sbt/run_sbt.bzl @@ -11,6 +11,9 @@ def _run_sbt_impl(ctx): sbt_bin = ctx.toolchains["@//toolchains/sbt:toolchain_type"].sbt_info.sbt_bin java_runtime = ctx.attr._java_runtime java_executable_path = java_runtime[java_common.JavaRuntimeInfo].java_executable_exec_path + cc_toolchain = ctx.toolchains["@bazel_tools//tools/cpp:toolchain_type"] + cc_path = cc_toolchain.cc.compiler_executable + cc_deps = cc_toolchain.cc.all_files out_dir = ctx.actions.declare_directory(ctx.attr.out_dir) outputs = [out_dir] @@ -22,9 +25,10 @@ def _run_sbt_impl(ctx): for k, v in ctx.attr.env.items(): envs[k] = expand_variables(ctx, ctx.expand_location(v, targets = ctx.attr.srcs), outs = outputs, attribute_name = "env") - inputs = depset(ctx.files.srcs, transitive = [java_runtime.files]) + inputs = depset(ctx.files.srcs, transitive = [java_runtime.files, cc_deps]) system_props = [ "-Denso.BazelSupport.outDir=" + out_dir.path, + "-Denso.BazelSupport.CCompilerPath=" + cc_path, ] for p in ctx.attr.system_props: system_props = system_props + split_args(expand_variables(ctx, ctx.expand_location(p, targets = ctx.attr.srcs), outs = outputs)) @@ -48,6 +52,7 @@ run_sbt = rule( "@//toolchains/sbt:toolchain_type", "@//toolchains/flatc:toolchain_type", "@bazel_tools//tools/jdk:runtime_toolchain_type", + "@bazel_tools//tools/cpp:toolchain_type" ], attrs = { "args": attr.string_list(