Add support for reading YXDB files

CHANGELOG.md

@@ -13,12 +13,14 @@
 - [Standard.Test pending field is lazy][14536].
 - [Using dual JVM mode for Standard.AWS][14568].
 - [Polishing Standard.Test API][14599].
+- [Support for reading Alteryx YXDB files][14602].
 
 [14522]: https://github.com/enso-org/enso/pull/14522
 [14476]: https://github.com/enso-org/enso/pull/14476
 [14536]: https://github.com/enso-org/enso/pull/14536
 [14568]: https://github.com/enso-org/enso/pull/14568
 [14599]: https://github.com/enso-org/enso/pull/14599
+[14602]: https://github.com/enso-org/enso/pull/14602
 
 #### Enso Language & Runtime
 

build.sbt

@@ -5235,6 +5235,7 @@ lazy val `std-table` = project
       "org.antlr"                % "antlr4-runtime"        % antlrVersion,
       "org.apache.logging.log4j" % "log4j"                 % "2.24.3",
       "org.apache.logging.log4j" % "log4j-to-slf4j"        % "2.24.3", // org.apache.poi uses log4j
+      "uk.co.jdunkerley"         % "yxdb-java"             % "0.1.4",
       "org.graalvm.truffle"      % "truffle-api"           % graalMavenPackagesVersion % Test,
       "junit"                    % "junit"                 % junitVersion              % Test,
       "com.github.sbt"           % "junit-interface"       % junitIfVersion            % Test,

distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/NOTICE

@@ -85,3 +85,8 @@ Copyright notices related to this dependency can be found in the directory `org.
 The license file can be found at `licenses/MIT`.
 Copyright notices related to this dependency can be found in the directory `org.slf4j.slf4j-api-2.0.16`.
 
+
+'yxdb-java', licensed under the MIT License, is distributed with the Table.
+The license information can be found along with the copyright notices.
+Copyright notices related to this dependency can be found in the directory `uk.co.jdunkerley.yxdb-java-0.1.4`.
+

distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/uk.co.jdunkerley.yxdb-java-0.1.4/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2022 tlarsendataguy
+Copyright (c) 2025 jdunkerley
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

distribution/lib/Standard/Table/0.0.0-dev/docs/api/Alteryx_Format.md

@@ -0,0 +1,13 @@
+## Enso Signatures 1.0
+## module Standard.Table.Alteryx_Format
+- type Alteryx_Format
+    - Alteryx_Format
+    - for_file_write file:Standard.Base.Any.Any -> Standard.Base.Any.Any
+    - for_read file:Standard.Base.System.File_Format_Metadata.File_Format_Metadata -> Standard.Base.Any.Any
+    - get_dropdown_options -> Standard.Base.Any.Any
+    - get_name_patterns -> (Standard.Base.Data.Vector.Vector Standard.Base.System.File_Format.File_Name_Pattern)
+    - read self file:Standard.Base.Any.Any on_problems:Standard.Base.Errors.Problem_Behavior.Problem_Behavior -> Standard.Base.Any.Any
+    - read_stream self stream:Standard.Base.System.Input_Stream.Input_Stream metadata:Standard.Base.System.File_Format_Metadata.File_Format_Metadata= -> Standard.Base.Any.Any
+    - resolve constructor:Standard.Base.Any.Any -> Standard.Base.Any.Any
+    - spatial_column_to_geojson table:(Standard.Table.Table.Table&Standard.Table.In_Memory_Table.In_Memory_Table)= column:(Standard.Base.Data.Text.Text|Standard.Base.Data.Numbers.Integer)= -> Standard.Base.Any.Any
+    - spatial_to_geojson spatial_object:(Standard.Base.Data.Array.Array|Standard.Base.Data.Vector.Vector)= -> Standard.Base.Data.Text.Text

distribution/lib/Standard/Table/0.0.0-dev/src/Alteryx_Format.enso

@@ -0,0 +1,159 @@
+from Standard.Base import all
+import Standard.Base.Enso_Cloud.Data_Link_Helpers
+import Standard.Base.Errors.Common.Type_Error
+import Standard.Base.Errors.Illegal_Argument.Illegal_Argument
+import Standard.Base.Errors.Illegal_State.Illegal_State
+import Standard.Base.Errors.File_Error.File_Error
+import Standard.Base.Runtime.Context
+import Standard.Base.System.File.Generic.Writable_File.Writable_File
+import Standard.Base.System.File_Format.File_Name_Pattern
+import Standard.Base.System.File_Format_Metadata.File_Format_Metadata
+import Standard.Base.System.Input_Stream.Input_Stream
+from Standard.Base.Metadata.Choice import Option
+from Standard.Base.Metadata.Widget import Text_Input
+
+import project.Internal.Java_Problems
+import project.Internal.Telemetry
+import project.Internal.Widget_Helpers
+import project.Table.Table
+import project.In_Memory_Table.In_Memory_Table
+from project.In_Memory_Table import from_java_table
+
+polyglot java import java.io.FileNotFoundException
+polyglot java import java.lang.IllegalArgumentException
+polyglot java import java.lang.IllegalStateException
+polyglot java import org.enso.table.read.AlteryxYXDBReader
+
+## A file format for reading Alteryx YXDB files.
+type Alteryx_Format
+    Alteryx_Format
+
+    ## ---
+       icon: Spatial
+       ---
+       Converts an Alteryx Spatial Object to a GeoJSON representation.
+
+       ## Arguments
+       - spatial_object: The spatial object to convert, represented as a byte array.
+
+       ## Returns
+       A GeoJSON representation of the spatial object.
+    spatial_to_geojson (spatial_object : Array | Vector = Missing_Argument.throw "spatial_object") -> Text =
+        AlteryxYXDBReader.spatialObjectToGeoJSON spatial_object
+
+    ## ---
+       icon: Spatial
+       ---
+       Converts a column of Alteryx Spatial Objects to a GeoJSON representation.
+
+       ## Arguments
+       - table: The table containing the spatial objects.
+       - column: The name of the column containing the spatial objects.
+
+       ## Returns
+       An updated table with the specified column converted to GeoJSON.
+    @column _column_widget
+    spatial_column_to_geojson (table : Table & In_Memory_Table = Missing_Argument.throw "table") (column : Text | Integer = Missing_Argument.throw "column") =
+        input_column = table.at column
+        converted_column = input_column.map v-> Alteryx_Format.spatial_to_geojson v
+        table.set converted_column input_column.name
+
+    ## ---
+       private: true
+       ---
+       Resolve an unresolved constructor to the actual type.
+    resolve : Function -> Alteryx_Format | Nothing
+    resolve constructor =
+        Panic.catch Type_Error (constructor:Alteryx_Format) _->Nothing
+
+    ## ---
+       private: true
+       ---
+       If the File_Format supports reading from the file, return a configured
+       instance.
+    for_read : File_Format_Metadata -> Alteryx_Format | Nothing
+    for_read file:File_Format_Metadata =
+        case file.guess_extension of
+            ".yxdb" -> Alteryx_Format.Alteryx_Format
+            _ -> Nothing
+
+    ## ---
+       private: true
+       ---
+       If this File_Format should be used for writing to that file, return a
+       configured instance.
+    for_file_write : Writable_File -> Alteryx_Format | Nothing
+    for_file_write file =
+        _ = file
+        Nothing
+
+    ## ---
+       private: true
+       ---
+    get_dropdown_options : Vector Option
+    get_dropdown_options = [Option "Alteryx Database" "..Alteryx_Format"]
+
+    ## ---
+       private: true
+       ---
+    get_name_patterns -> Vector File_Name_Pattern =
+        [File_Name_Pattern.Value "Alteryx Database" ["*.yxdb"]]
+
+    ## ---
+       private: true
+       ---
+       Implements the `File.read` for this `File_Format`
+    read : File -> Problem_Behavior -> Any
+    read self file on_problems:Problem_Behavior =
+        result = _read_file file on_problems
+        result.if_not_error <|
+            Telemetry.log "File_Format.read" "Read file: format={}, output={}, row_count={}, column_count={}" ["Alteryx_Format", "Table", result.row_count, result.column_count]
+            result
+
+    ## ---
+       private: true
+       ---
+       Implements decoding the format from a stream.
+    read_stream : Input_Stream -> File_Format_Metadata -> Any
+    read_stream self stream:Input_Stream (metadata : File_Format_Metadata = File_Format_Metadata.no_information) =
+        _ = metadata
+
+        ## Currently stream must be materialised (but no actual reason...)
+        tmp_file = File.create_temporary_file "alteryx_database_read" ".yxdb"
+
+        ## Write stream to temporary file
+        write_result = Context.Output.with_enabled <|
+            inner = Panic.catch Any (stream.write_to_file tmp_file) caught_panic-> Error.throw caught_panic.payload
+            if inner.is_error then tmp_file.delete
+            inner
+        Error.return_if_error write_result
+
+        result = Panic.recover Any <|
+            self.read tmp_file ..Report_Warning
+
+        ## Clean up temporary file
+        Context.Output.with_enabled <|
+            Panic.catch Any (tmp_file.delete) _->Nothing
+
+        result
+
+private _read_file path on_problems:Problem_Behavior =
+    Data_Link_Helpers.as_file path file->
+        Java_Problems.with_problem_aggregator on_problems java_problem_aggregator->
+            java_table = Panic.catch Any handler=handle_java_exceptions <|
+                AlteryxYXDBReader.read file.path java_problem_aggregator
+            from_java_table java_table
+
+private handle_java_exceptions caught_panic =
+    error = case caught_panic.payload of
+        ex : IllegalArgumentException -> Illegal_Argument.Error ex.getMessage ex
+        ex : IllegalStateException -> Illegal_State.Error ex.getMessage ex
+        ex : FileNotFoundException -> File_Error.Not_Found ex.getMessage
+        _ -> Illegal_State.Error "Unexpected error reading Alteryx YXDB file." caught_panic.payload
+    Error.throw error
+
+private _column_widget arg cache=Nothing =
+    _ = arg
+    table = cache.if_not_nothing <| cache "table"
+    if table.is_nothing then Text_Input display=..Always else
+        Widget_Helpers.make_column_name_selector table display=..Always

distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Table_Services.enso

@@ -6,6 +6,7 @@ import Standard.Base.System.File_Format.File_Format_SPI
 import project.Delimited.Delimited_Format.Delimited_Format
 import project.Excel.Excel_Format.Excel_Format
 import project.Fixed_Width.Fixed_Width_Format.Fixed_Width_Format
+import project.Alteryx_Format.Alteryx_Format
 import project.IO.EDI_Format.EDI_Format
 import project.Return_As_Table.Return_As_Table
 
@@ -16,8 +17,9 @@ File_Format_SPI.from (that:Impl) =
     excel = File_Format_SPI.new Excel_Format "excel"
     delim = File_Format_SPI.new Delimited_Format "delimited"
     fixed = File_Format_SPI.new Fixed_Width_Format "fixed_width"
+    alteryx = File_Format_SPI.new Alteryx_Format "alteryx"
     edi = File_Format_SPI.new EDI_Format "edi"
-    excel+delim+fixed+edi
+    excel+delim+fixed+alteryx+edi
 
 Return_As.SPI.from (that:Impl) =
     _ = that

distribution/lib/Standard/Table/0.0.0-dev/src/Main.enso

@@ -1,5 +1,7 @@
 from Standard.Base import all
 
+export project.Alteryx_Format.Alteryx_Format
+
 export project.Aggregate_Column.Aggregate_Column
 
 export project.Blank_Selector.Blank_Selector
@@ -69,4 +71,3 @@ export project.Table.Table
 export project.Value_Type.Auto
 export project.Value_Type.Bits
 export project.Value_Type.Value_Type
-

project/Dependencies.scala

@@ -61,8 +61,13 @@ object Dependencies {
   // Keep in sync with GraalVM.version. Do not change the name of this variable,
   // it is used by the Rust build script via regex matching.
   val graalMavenPackagesVersion = "25.0.1"
-  val targetJavaVersion         = "17"
-  val defaultDevEnsoVersion     = "0.0.0-dev"
+
+  def runningInAnIde: Boolean = {
+    val idea = System.getProperty("idea.managed")
+    idea != null && idea.nonEmpty
+  }
+  val targetJavaVersion     = if (runningInAnIde) "21" else "17"
+  val defaultDevEnsoVersion = "0.0.0-dev"
   val ensoVersion = sys.env.getOrElse(
     "ENSO_VERSION",
     defaultDevEnsoVersion

std-bits/table/src/main/java/org/enso/table/read/AlteryxYXDBReader.java

@@ -0,0 +1,116 @@
+package org.enso.table.read;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.format.DateTimeParseException;
+import java.util.Arrays;
+import java.util.stream.IntStream;
+import org.enso.table.data.column.builder.Builder;
+import org.enso.table.data.column.storage.type.AnyObjectType;
+import org.enso.table.data.column.storage.type.BigDecimalType;
+import org.enso.table.data.column.storage.type.BooleanType;
+import org.enso.table.data.column.storage.type.DateTimeType;
+import org.enso.table.data.column.storage.type.DateType;
+import org.enso.table.data.column.storage.type.FloatType;
+import org.enso.table.data.column.storage.type.IntegerType;
+import org.enso.table.data.column.storage.type.StorageType;
+import org.enso.table.data.column.storage.type.TextType;
+import org.enso.table.data.column.storage.type.TimeOfDayType;
+import org.enso.table.data.table.Column;
+import org.enso.table.data.table.Table;
+import org.enso.table.problems.ProblemAggregator;
+import uk.co.jdunkerley.yxdb.Spatial;
+import uk.co.jdunkerley.yxdb.YxdbField;
+import uk.co.jdunkerley.yxdb.YxdbReader;
+import uk.co.jdunkerley.yxdb.YxdbType;
+
+public final class AlteryxYXDBReader {
+  /**
+   * Reads an Alteryx YXDB file and returns its contents as a Table.
+   *
+   * @param path the path to the YXDB file.
+   * @return a Table containing the data from the YXDB file.
+   */
+  public static Table read(String path, ProblemAggregator problemAggregator)
+      throws FileNotFoundException, IllegalArgumentException, IllegalStateException {
+    // Test that the path exists
+    if (!Files.exists(Path.of(path))) {
+      throw new FileNotFoundException(path);
+    }
+
+    try (var yxdbReader = new YxdbReader(path)) {
+      var recordCount = yxdbReader.numRecords();
+      var fields = yxdbReader.fields();
+      var storageTypes =
+          Arrays.stream(fields).map(AlteryxYXDBReader::mapYXDBField).toArray(StorageType[]::new);
+      var storages =
+          Arrays.stream(storageTypes)
+              .map(st -> st.makeBuilder(recordCount, problemAggregator))
+              .toArray(Builder[]::new);
+
+      while (yxdbReader.next()) {
+        try {
+          for (int i = 0; i < storages.length; i++) {
+            var yxdbValue =
+                storageTypes[i] instanceof AnyObjectType
+                    ? yxdbReader.readBlob(i)
+                    : yxdbReader.read(i);
+            storages[i].append(yxdbValue);
+          }
+        } catch (IndexOutOfBoundsException _) {
+          throw new IllegalArgumentException(
+              "The YXDB file appears to be corrupted on row " + storages[0].getCurrentSize());
+        } catch (DateTimeParseException _) {
+          throw new IllegalArgumentException(
+              "The YXDB file contains invalid date/time data on row "
+                  + storages[0].getCurrentSize());
+        }
+      }
+
+      var columns =
+          IntStream.range(0, storages.length)
+              .mapToObj(i -> new Column(fields[i].name(), storages[i].seal()))
+              .toArray(Column[]::new);
+      return new Table(columns);
+    } catch (IllegalArgumentException exc) {
+      throw exc;
+    } catch (IOException exc) {
+      var message = exc.getMessage();
+      throw new IllegalArgumentException(exc.getMessage(), exc);
+    } catch (Exception exc) {
+      throw new IllegalStateException("An unexpected error occurred: " + exc.getMessage(), exc);
+    }
+  }
+
+  /**
+   * Converts a spatial object in byte array format to its GeoJSON representation.
+   *
+   * @param spatialObj the spatial object as a byte array.
+   * @return the GeoJSON representation of the spatial object.
+   */
+  public static String spatialObjectToGeoJSON(byte[] spatialObj) {
+    return spatialObj == null ? null : Spatial.toGeoJson(spatialObj);
+  }
+
+  private static StorageType<?> mapYXDBField(YxdbField field) {
+    return switch (field.yxdbType()) {
+      case YxdbType.BOOLEAN -> BooleanType.INSTANCE;
+      case YxdbType.BYTE -> IntegerType.INT_8;
+      case YxdbType.INT16 -> IntegerType.INT_16;
+      case YxdbType.INT32 -> IntegerType.INT_32;
+      case YxdbType.INT64 -> IntegerType.INT_64;
+      case YxdbType.FLOAT, YxdbType.DOUBLE -> FloatType.FLOAT_64;
+      case YxdbType.DECIMAL -> BigDecimalType.INSTANCE;
+      case YxdbType.STRING, YxdbType.WSTRING -> TextType.variableLengthWithLimit(field.size());
+      case YxdbType.V_STRING, YxdbType.V_WSTRING -> TextType.VARIABLE_LENGTH;
+      case YxdbType.DATE -> DateType.INSTANCE;
+      case YxdbType.TIME -> TimeOfDayType.INSTANCE;
+      case YxdbType.DATETIME -> DateTimeType.INSTANCE;
+      case YxdbType.BLOB, YxdbType.SPATIAL_OBJ -> AnyObjectType.INSTANCE;
+      default ->
+          throw new IllegalStateException("Unsupported YXDB field type: " + field.yxdbType());
+    };
+  }
+}