Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -3843,9 +3843,6 @@ lazy val `engine-runner` = project
.listFiles("*.jar")
.map(_.getAbsolutePath()) ++
`std-aws-polyglot-root`.listFiles("*.jar").map(_.getAbsolutePath()) ++
`std-snowflake-polyglot-root`
.listFiles("*.jar")
.map(_.getAbsolutePath()) ++
`std-tableau-polyglot-root`
.listFiles("*.jar")
.map(_.getAbsolutePath()) ++
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,11 @@
- dec x:(Standard.Base.Data.Text.Text|Standard.Base.Data.Numbers.Integer|Standard.Base.Data.Numbers.Float) mc:(Standard.Base.Data.Numeric.Math_Context.Math_Context|Standard.Base.Nothing.Nothing)= -> Standard.Base.Data.Decimal.Decimal!(Standard.Base.Errors.Common.Arithmetic_Error|Standard.Base.Data.Numbers.Number_Parse_Error)
- error_if_from_float left:Standard.Base.Any.Any right:Standard.Base.Any.Any ~action:Standard.Base.Any.Any -> Standard.Base.Any.Any
- from_big_decimal that:Standard.Base.Any.Any -> Standard.Base.Any.Any
- get_big_decimal that:Standard.Base.Data.Decimal.Decimal -> Standard.Base.Any.Any
- handle_java_exception ~action:Standard.Base.Any.Any extra_message:Standard.Base.Data.Text.Text= -> Standard.Base.Any.Any
- handle_number_format_exception ~action:Standard.Base.Any.Any -> Standard.Base.Any.Any
- handle_precision_loss original_value:Standard.Base.Any.Any conversion_result:Standard.Base.Data.Decimal.ConversionResult -> Standard.Base.Any.Any
- handle_unsupported_argument_types ~action:Standard.Base.Any.Any -> Standard.Base.Any.Any
- value_with_scale that:Standard.Base.Data.Decimal.Decimal -> (Standard.Base.Data.Pair.Pair Standard.Base.Data.Numbers.Integer Standard.Base.Data.Numbers.Integer)
- Standard.Base.Data.Ordering.Comparable.from that:Standard.Base.Data.Decimal.Decimal -> Standard.Base.Data.Ordering.Comparable
- Standard.Base.Data.Ordering.Comparable.from that:Standard.Base.Data.Numbers.Number -> Standard.Base.Data.Ordering.Comparable
- Standard.Base.Data.Decimal.Decimal.from that:Standard.Base.Data.Text.Text -> Standard.Base.Data.Decimal.Decimal
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import project.Data.Numeric.Math_Context.Math_Context
import project.Data.Numeric.Rounding_Mode.Rounding_Mode
import project.Data.Text.Text
import project.Data.Vector.Vector
import project.Data.Pair.Pair
import project.Error.Error
import project.Errors.Deprecated.Deprecated
import project.Errors.Illegal_Argument.Illegal_Argument
Expand Down Expand Up @@ -1467,8 +1468,10 @@ Float.from (that : Decimal) =
## ---
private: true
---
Helper method allowing access to the backing field.
get_big_decimal (that : Decimal) = that.big_decimal
Returns two numbers: the actual value and the scale associated to it

value_with_scale (that : Decimal) -> Pair Integer Integer =
Pair.new that.big_decimal.unscaledValue that.big_decimal.scale

## ---
private: true
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from Standard.Base import all
import Standard.Base.Errors.Illegal_Argument.Illegal_Argument
import Standard.Base.Errors.Illegal_State.Illegal_State
from Standard.Base.Data.Decimal import get_big_decimal
from Standard.Base.Data.Decimal import value_with_scale

from Standard.Table import Value_Type

Expand Down Expand Up @@ -35,7 +35,9 @@ type Statement_Setter
Nothing -> stmt.setNull i Types.NULL
_ : Boolean -> stmt.setBoolean i value
_ : Integer -> jdbc_value_setter.setInteger stmt i value
_ : Decimal -> stmt.setBigDecimal i (get_big_decimal value)
d : Decimal ->
pair = value_with_scale d
jdbc_value_setter.setBigDecimal stmt i pair.first pair.second
_ : Float -> stmt.setDouble i value
_ : Text -> stmt.setString i value
_ : Date_Time ->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ from Standard.Base import all
import Standard.Base.Errors.Common.Index_Out_Of_Bounds
import Standard.Base.Errors.Illegal_Argument.Illegal_Argument
import Standard.Base.Errors.Illegal_State.Illegal_State
from Standard.Base.Data.Decimal import from_big_decimal, get_big_decimal
from Standard.Base.Data.Decimal import from_big_decimal, value_with_scale

import project.Internal.Value_Type_Helpers
import project.Value_Type.Bits
Expand Down Expand Up @@ -89,7 +89,9 @@ closest_storage_type value_type = case value_type of
This step is unnecessary for primitive and builtin values, but necessary for
values such as `Decimal`/`BigDecimal`.
enso_to_java x = case x of
_ : Decimal -> get_big_decimal x
_ : Decimal ->
pair = value_with_scale x
BigDecimal.new pair.first pair.second
_ -> x

## ---
Expand All @@ -98,10 +100,20 @@ enso_to_java x = case x of
Convert a Java value to an Enso value before returning it to Enso from a Java
`Column`. This step is unnecessary for primitive and builtin values, but
necessary for values such as `Decimal`/`BigDecimal`.
java_to_enso x = case x of
_ : Nothing -> Nothing
_ : BigDecimal -> from_big_decimal x
_ -> x
java_to_enso x =
case x of
_ : Nothing -> Nothing
_ : Integer -> x
_ : Float -> x
_ : Text -> x
_ : BigDecimal -> from_big_decimal x
_ ->
type_name = (Meta.type_of x).to_text
case type_name of
"java.math.BigDecimal" ->
big = BigDecimal.new x.unscaledValue x.scale
from_big_decimal big
_ -> x

## ---
private: true
Expand Down
10 changes: 5 additions & 5 deletions project/GraalVM.scala
Original file line number Diff line number Diff line change
Expand Up @@ -109,11 +109,11 @@ object GraalVM {
// Expected production NI sizes deduced from sizes on latest
// nightly builds: https://github.com/enso-org/enso/pull/12843#issuecomment-2869897463
// With maximal size relaxed by 30 MB.
private val windowsX64Release = NativeImageSize(200, 470)
private val linuxX64Release = NativeImageSize(200, 490)
private val macX64Release = NativeImageSize(200, 457)
private val macARM64Release = NativeImageSize(200, 473)
private val testNISize = NativeImageSize(100, 592)
private val windowsX64Release = NativeImageSize(200, 390)
private val linuxX64Release = NativeImageSize(200, 410)
private val macX64Release = NativeImageSize(200, 380)
private val macARM64Release = NativeImageSize(200, 390)
private val testNISize = NativeImageSize(100, 510)
}

/** Has the user requested to use Espresso for Java interop? */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,18 @@ public static long coerceToLong(Object o) {
}

public static BigInteger coerceToBigInteger(Object o) {
if (o instanceof BigInteger bigInteger) {
return bigInteger;
} else {
long longValue = coerceToLong(o);
return BigInteger.valueOf(longValue);
try {
return switch (o) {
case BigInteger big -> big;
case Double d -> new BigDecimal(d).toBigIntegerExact();
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

case Float f -> new BigDecimal(f).toBigIntegerExact();
default -> {
var longValue = coerceToLong(o);
yield BigInteger.valueOf(longValue);
}
};
} catch (ArithmeticException ex) {
throw new UnsupportedOperationException("Cannot coerce " + o + " to a big integer.");
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package org.enso.database;

import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Types;
Expand Down Expand Up @@ -65,4 +67,11 @@ public void setLocalDate(PreparedStatement stmt, int columnIndex, LocalDate loca
throws SQLException {
stmt.setObject(columnIndex, localDate, Types.DATE);
}

public void setBigDecimal(
PreparedStatement stmt, int columnIndex, BigInteger unscaledValue, int scale)
throws SQLException {
var big = new BigDecimal(unscaledValue, scale);
stmt.setBigDecimal(columnIndex, big);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -107,14 +107,14 @@ static <T> ColumnStorage<T> makeLocal(ColumnStorage<T> storage) {
return (ColumnStorage<T>) new TypedStorage(localType, new Object[0]);
}
var data = storage.addressOfData();
var size = Math.toIntExact(storage.getSize());
var proxyType = storage.getType();
var localType = StorageType.fromTypeCharAndSize(proxyType.typeChar(), proxyType.size());
if (data != 0) {
var size = Math.toIntExact(storage.getSize());
var validity = storage.addressOfValidity();
var proxyType = storage.getType();
var localType = StorageType.fromTypeCharAndSize(proxyType.typeChar(), proxyType.size());
var localStorage =
switch (localType) {
case BooleanType type -> BoolBuilder.fromAddress(size, data, validity).seal(storage);
case BooleanType _ -> BoolBuilder.fromAddress(size, data, validity).seal(storage);
case IntegerType type ->
LongBuilder.fromAddress(size, data, validity, type).seal(storage, type);
case FloatType type ->
Expand All @@ -126,10 +126,23 @@ static <T> ColumnStorage<T> makeLocal(ColumnStorage<T> storage) {
assert assertSameStorages(storage, localStorage);
return (ColumnStorage<T>) localStorage;
} else {
if (BuilderUtil.LOG.isTraceEnabled()) {
var t = storage.getType();
BuilderUtil.LOG.trace(
"makeLocal unsuccessful for {}:{} size {}", t.typeChar(), t.size(), storage.getSize());
switch (localType) {
case BigIntegerType _ -> {
Copy link
Member Author

@JaroslavTulach JaroslavTulach Dec 22, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sbt:enso> runEngineDistribution 
  --vm.D=polyglot.enso.classLoading=Standard.Snowflake:guest,hosted 
  --env ENSO_SNOWFLAKE_USER=aaa 
  --env ENSO_SNOWFLAKE_PASSWORD=bbb
  --env ENSO_SNOWFLAKE_ACCOUNT=ccc
  --env ENSO_SNOWFLAKE_DATABASE=CI_TEST_DB
  --run test/Snowflake_Tests should.be.able.to.round-trip.a.BigInteger.column

[FAILED] [Snowflake] Edge Cases: [0/1, 2127ms]                        

    - [FAILED] should be able to round-trip a BigInteger column [2127ms]
        Reason: (1.1805916207174113E21): Expected a value of type Standard.Base.Data.Numbers.Integer 
        but got a value [1.1805916207174113E21] of type Standard.Base.Data.Numbers.Float instead
       (at enso/enso/test/Snowflake_Tests/src/Snowflake_Spec.enso:375:17-37).
  • the problem is that with Proxy Storage there is Truffle value conversion
  • when obtaining values from the storage - e.g. when calling from Enso to Java
    • and BigInteger may be returned as double or float
    • when it can be exactly represented by double or float
  • by eagerly copying the content of the storage to local JVM we convert everything back to BigInteger
  • conversions to big integer had to be improved

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  • it would be faster to use off-heap memory transfer
  • but I don't know what's appropriate Arrow format for java.math.BigInteger!?
  • there is Int+bit width, but the width can be 64 bits at max...

var b = Builder.getForBigInteger(size, null);
b.appendBulkStorage(storage);
var localStorage = b.seal();
return (ColumnStorage<T>) localStorage;
}
default -> {
if (BuilderUtil.LOG.isTraceEnabled()) {
var t = storage.getType();
BuilderUtil.LOG.trace(
"makeLocal unsuccessful for {}:{} size {}",
t.typeChar(),
t.size(),
storage.getSize());
}
}
}
}
return storage;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,12 @@
import org.enso.table.problems.ProblemAggregator;
import org.graalvm.polyglot.Context;
import org.graalvm.polyglot.Value;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/** A representation of a column. Consists of a column name and the underlying storage. */
public final class Column {
private static final Logger LOG = LoggerFactory.getLogger(Column.class);
private final String name;
private final ColumnStorage<?> storage;

Expand All @@ -29,7 +32,15 @@ public final class Column {
public Column(String name, ColumnStorage<?> storage) {
ensureNameIsValid(name);
this.name = name;
this.storage = Proxy.isProxyClass(storage.getClass()) ? Builder.makeLocal(storage) : storage;
var isProxy = Proxy.isProxyClass(storage.getClass());
this.storage = isProxy ? Builder.makeLocal(storage) : storage;
var type = this.storage.getType();
LOG.debug(
Copy link
Member Author

@JaroslavTulach JaroslavTulach Dec 22, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  • Run any tests with org.enso.table.Logger.level=trace to see Column instances being created
  • for example running:
sbt:enso> runEngineDistribution 
  --vm.D=org.enso.table.Logger.level=trace
  --env ENSO_SNOWFLAKE_USER=aaa 
  --env ENSO_SNOWFLAKE_PASSWORD=bbb 
  --env ENSO_SNOWFLAKE_ACCOUNT=ccc 
  --env ENSO_SNOWFLAKE_DATABASE=CI_TEST_DB 
  --run test/Snowflake_Tests nicnicnic
  

8 tests succeeded.
0 tests failed.
0 tests skipped.
0 groups skipped.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  • a25e91c delays the initialization a lot, but not fully

"Column[{}] of {}:{} type with size: {}",
name,
type.typeChar(),
type.size(),
storage.getSize());
}

public static boolean isColumnNameValid(String name) {
Expand Down
3 changes: 2 additions & 1 deletion test/Snowflake_Tests/src/Snowflake_Spec.enso
Original file line number Diff line number Diff line change
Expand Up @@ -535,7 +535,8 @@ add_snowflake_specs suite_builder create_connection_fn db_name =

common_selection = Common_Table_Operations.Main.Test_Selection.Config supported_replace_params=Snowflake_Connection.get_dialect.get_supported_replace_params run_advanced_edge_case_tests_by_default=False
aggregate_selection = Common_Table_Operations.Aggregate_Spec.Test_Selection.Config first_last_row_order=False aggregation_problems=False text_concat=False
agg_in_memory_table = ((Project_Description.new enso_dev.Table_Tests).data / "data.csv") . read
agg_in_memory_table =
((Project_Description.new enso_dev.Table_Tests).data / "data.csv") . read

agg_table_fn = _->
agg_in_memory_table.select_into_database_table default_connection.get (Name_Generator.random_name "Agg1") primary_key=Nothing temporary=True
Expand Down
Loading
Loading