diff --git a/CHANGELOG.md b/CHANGELOG.md index d124b99813ed..698e4fb26cbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,12 +13,14 @@ - [Standard.Test pending field is lazy][14536]. - [Using dual JVM mode for Standard.AWS][14568]. - [Polishing Standard.Test API][14599]. +- [Support for reading Alteryx YXDB files][14602]. [14522]: https://github.com/enso-org/enso/pull/14522 [14476]: https://github.com/enso-org/enso/pull/14476 [14536]: https://github.com/enso-org/enso/pull/14536 [14568]: https://github.com/enso-org/enso/pull/14568 [14599]: https://github.com/enso-org/enso/pull/14599 +[14602]: https://github.com/enso-org/enso/pull/14602 #### Enso Language & Runtime diff --git a/build.sbt b/build.sbt index ff80b9a5b8ce..806dc78f31a9 100644 --- a/build.sbt +++ b/build.sbt @@ -5235,6 +5235,7 @@ lazy val `std-table` = project "org.antlr" % "antlr4-runtime" % antlrVersion, "org.apache.logging.log4j" % "log4j" % "2.24.3", "org.apache.logging.log4j" % "log4j-to-slf4j" % "2.24.3", // org.apache.poi uses log4j + "uk.co.jdunkerley" % "yxdb-java" % "0.1.4", "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % Test, "junit" % "junit" % junitVersion % Test, "com.github.sbt" % "junit-interface" % junitIfVersion % Test, diff --git a/distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/NOTICE b/distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/NOTICE index 6aaf10f9de03..efebf8c138fa 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/NOTICE +++ b/distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/NOTICE @@ -85,3 +85,8 @@ Copyright notices related to this dependency can be found in the directory `org. The license file can be found at `licenses/MIT`. Copyright notices related to this dependency can be found in the directory `org.slf4j.slf4j-api-2.0.16`. + +'yxdb-java', licensed under the MIT License, is distributed with the Table. +The license information can be found along with the copyright notices. +Copyright notices related to this dependency can be found in the directory `uk.co.jdunkerley.yxdb-java-0.1.4`. + diff --git a/distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/uk.co.jdunkerley.yxdb-java-0.1.4/LICENSE b/distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/uk.co.jdunkerley.yxdb-java-0.1.4/LICENSE new file mode 100644 index 000000000000..3412dc19aa0d --- /dev/null +++ b/distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/uk.co.jdunkerley.yxdb-java-0.1.4/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2022 tlarsendataguy +Copyright (c) 2025 jdunkerley + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/distribution/lib/Standard/Table/0.0.0-dev/docs/api/Alteryx_Format.md b/distribution/lib/Standard/Table/0.0.0-dev/docs/api/Alteryx_Format.md new file mode 100644 index 000000000000..115c444db8fe --- /dev/null +++ b/distribution/lib/Standard/Table/0.0.0-dev/docs/api/Alteryx_Format.md @@ -0,0 +1,13 @@ +## Enso Signatures 1.0 +## module Standard.Table.Alteryx_Format +- type Alteryx_Format + - Alteryx_Format + - for_file_write file:Standard.Base.Any.Any -> Standard.Base.Any.Any + - for_read file:Standard.Base.System.File_Format_Metadata.File_Format_Metadata -> Standard.Base.Any.Any + - get_dropdown_options -> Standard.Base.Any.Any + - get_name_patterns -> (Standard.Base.Data.Vector.Vector Standard.Base.System.File_Format.File_Name_Pattern) + - read self file:Standard.Base.Any.Any on_problems:Standard.Base.Errors.Problem_Behavior.Problem_Behavior -> Standard.Base.Any.Any + - read_stream self stream:Standard.Base.System.Input_Stream.Input_Stream metadata:Standard.Base.System.File_Format_Metadata.File_Format_Metadata= -> Standard.Base.Any.Any + - resolve constructor:Standard.Base.Any.Any -> Standard.Base.Any.Any + - spatial_column_to_geojson table:(Standard.Table.Table.Table&Standard.Table.In_Memory_Table.In_Memory_Table)= column:(Standard.Base.Data.Text.Text|Standard.Base.Data.Numbers.Integer)= -> Standard.Base.Any.Any + - spatial_to_geojson spatial_object:(Standard.Base.Data.Array.Array|Standard.Base.Data.Vector.Vector)= -> Standard.Base.Data.Text.Text diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Alteryx_Format.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Alteryx_Format.enso new file mode 100644 index 000000000000..44f942186201 --- /dev/null +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Alteryx_Format.enso @@ -0,0 +1,159 @@ +from Standard.Base import all +import Standard.Base.Enso_Cloud.Data_Link_Helpers +import Standard.Base.Errors.Common.Type_Error +import Standard.Base.Errors.Illegal_Argument.Illegal_Argument +import Standard.Base.Errors.Illegal_State.Illegal_State +import Standard.Base.Errors.File_Error.File_Error +import Standard.Base.Runtime.Context +import Standard.Base.System.File.Generic.Writable_File.Writable_File +import Standard.Base.System.File_Format.File_Name_Pattern +import Standard.Base.System.File_Format_Metadata.File_Format_Metadata +import Standard.Base.System.Input_Stream.Input_Stream +from Standard.Base.Metadata.Choice import Option +from Standard.Base.Metadata.Widget import Text_Input + +import project.Internal.Java_Problems +import project.Internal.Telemetry +import project.Internal.Widget_Helpers +import project.Table.Table +import project.In_Memory_Table.In_Memory_Table +from project.In_Memory_Table import from_java_table + +polyglot java import java.io.FileNotFoundException +polyglot java import java.lang.IllegalArgumentException +polyglot java import java.lang.IllegalStateException +polyglot java import org.enso.table.read.AlteryxYXDBReader + +## A file format for reading Alteryx YXDB files. +type Alteryx_Format + Alteryx_Format + + ## --- + icon: Spatial + --- + Converts an Alteryx Spatial Object to a GeoJSON representation. + + ## Arguments + - spatial_object: The spatial object to convert, represented as a byte array. + + ## Returns + A GeoJSON representation of the spatial object. + spatial_to_geojson (spatial_object : Array | Vector = Missing_Argument.throw "spatial_object") -> Text = + AlteryxYXDBReader.spatialObjectToGeoJSON spatial_object + + ## --- + icon: Spatial + --- + Converts a column of Alteryx Spatial Objects to a GeoJSON representation. + + ## Arguments + - table: The table containing the spatial objects. + - column: The name of the column containing the spatial objects. + + ## Returns + An updated table with the specified column converted to GeoJSON. + @column _column_widget + spatial_column_to_geojson (table : Table & In_Memory_Table = Missing_Argument.throw "table") (column : Text | Integer = Missing_Argument.throw "column") = + input_column = table.at column + converted_column = input_column.map v-> Alteryx_Format.spatial_to_geojson v + table.set converted_column input_column.name + + ## --- + private: true + --- + Resolve an unresolved constructor to the actual type. + resolve : Function -> Alteryx_Format | Nothing + resolve constructor = + Panic.catch Type_Error (constructor:Alteryx_Format) _->Nothing + + ## --- + private: true + --- + If the File_Format supports reading from the file, return a configured + instance. + for_read : File_Format_Metadata -> Alteryx_Format | Nothing + for_read file:File_Format_Metadata = + case file.guess_extension of + ".yxdb" -> Alteryx_Format.Alteryx_Format + _ -> Nothing + + ## --- + private: true + --- + If this File_Format should be used for writing to that file, return a + configured instance. + for_file_write : Writable_File -> Alteryx_Format | Nothing + for_file_write file = + _ = file + Nothing + + ## --- + private: true + --- + get_dropdown_options : Vector Option + get_dropdown_options = [Option "Alteryx Database" "..Alteryx_Format"] + + ## --- + private: true + --- + get_name_patterns -> Vector File_Name_Pattern = + [File_Name_Pattern.Value "Alteryx Database" ["*.yxdb"]] + + ## --- + private: true + --- + Implements the `File.read` for this `File_Format` + read : File -> Problem_Behavior -> Any + read self file on_problems:Problem_Behavior = + result = _read_file file on_problems + result.if_not_error <| + Telemetry.log "File_Format.read" "Read file: format={}, output={}, row_count={}, column_count={}" ["Alteryx_Format", "Table", result.row_count, result.column_count] + result + + ## --- + private: true + --- + Implements decoding the format from a stream. + read_stream : Input_Stream -> File_Format_Metadata -> Any + read_stream self stream:Input_Stream (metadata : File_Format_Metadata = File_Format_Metadata.no_information) = + _ = metadata + + ## Currently stream must be materialised (but no actual reason...) + tmp_file = File.create_temporary_file "alteryx_database_read" ".yxdb" + + ## Write stream to temporary file + write_result = Context.Output.with_enabled <| + inner = Panic.catch Any (stream.write_to_file tmp_file) caught_panic-> Error.throw caught_panic.payload + if inner.is_error then tmp_file.delete + inner + Error.return_if_error write_result + + result = Panic.recover Any <| + self.read tmp_file ..Report_Warning + + ## Clean up temporary file + Context.Output.with_enabled <| + Panic.catch Any (tmp_file.delete) _->Nothing + + result + +private _read_file path on_problems:Problem_Behavior = + Data_Link_Helpers.as_file path file-> + Java_Problems.with_problem_aggregator on_problems java_problem_aggregator-> + java_table = Panic.catch Any handler=handle_java_exceptions <| + AlteryxYXDBReader.read file.path java_problem_aggregator + from_java_table java_table + +private handle_java_exceptions caught_panic = + error = case caught_panic.payload of + ex : IllegalArgumentException -> Illegal_Argument.Error ex.getMessage ex + ex : IllegalStateException -> Illegal_State.Error ex.getMessage ex + ex : FileNotFoundException -> File_Error.Not_Found ex.getMessage + _ -> Illegal_State.Error "Unexpected error reading Alteryx YXDB file." caught_panic.payload + Error.throw error + +private _column_widget arg cache=Nothing = + _ = arg + table = cache.if_not_nothing <| cache "table" + if table.is_nothing then Text_Input display=..Always else + Widget_Helpers.make_column_name_selector table display=..Always diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Table_Services.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Table_Services.enso index 888efb26cb22..9d62e825e9ae 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Table_Services.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Internal/Table_Services.enso @@ -6,6 +6,7 @@ import Standard.Base.System.File_Format.File_Format_SPI import project.Delimited.Delimited_Format.Delimited_Format import project.Excel.Excel_Format.Excel_Format import project.Fixed_Width.Fixed_Width_Format.Fixed_Width_Format +import project.Alteryx_Format.Alteryx_Format import project.IO.EDI_Format.EDI_Format import project.Return_As_Table.Return_As_Table @@ -16,8 +17,9 @@ File_Format_SPI.from (that:Impl) = excel = File_Format_SPI.new Excel_Format "excel" delim = File_Format_SPI.new Delimited_Format "delimited" fixed = File_Format_SPI.new Fixed_Width_Format "fixed_width" + alteryx = File_Format_SPI.new Alteryx_Format "alteryx" edi = File_Format_SPI.new EDI_Format "edi" - excel+delim+fixed+edi + excel+delim+fixed+alteryx+edi Return_As.SPI.from (that:Impl) = _ = that diff --git a/distribution/lib/Standard/Table/0.0.0-dev/src/Main.enso b/distribution/lib/Standard/Table/0.0.0-dev/src/Main.enso index 7368c4a036f4..e2d60c3d6a89 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/src/Main.enso +++ b/distribution/lib/Standard/Table/0.0.0-dev/src/Main.enso @@ -1,5 +1,7 @@ from Standard.Base import all +export project.Alteryx_Format.Alteryx_Format + export project.Aggregate_Column.Aggregate_Column export project.Blank_Selector.Blank_Selector @@ -69,4 +71,3 @@ export project.Table.Table export project.Value_Type.Auto export project.Value_Type.Bits export project.Value_Type.Value_Type - diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 4bb4e4847adc..aebce13f20b3 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -61,8 +61,13 @@ object Dependencies { // Keep in sync with GraalVM.version. Do not change the name of this variable, // it is used by the Rust build script via regex matching. val graalMavenPackagesVersion = "25.0.1" - val targetJavaVersion = "17" - val defaultDevEnsoVersion = "0.0.0-dev" + + def runningInAnIde: Boolean = { + val idea = System.getProperty("idea.managed") + idea != null && idea.nonEmpty + } + val targetJavaVersion = if (runningInAnIde) "21" else "17" + val defaultDevEnsoVersion = "0.0.0-dev" val ensoVersion = sys.env.getOrElse( "ENSO_VERSION", defaultDevEnsoVersion diff --git a/std-bits/table/src/main/java/org/enso/table/read/AlteryxYXDBReader.java b/std-bits/table/src/main/java/org/enso/table/read/AlteryxYXDBReader.java new file mode 100644 index 000000000000..1cc097ca5562 --- /dev/null +++ b/std-bits/table/src/main/java/org/enso/table/read/AlteryxYXDBReader.java @@ -0,0 +1,116 @@ +package org.enso.table.read; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.format.DateTimeParseException; +import java.util.Arrays; +import java.util.stream.IntStream; +import org.enso.table.data.column.builder.Builder; +import org.enso.table.data.column.storage.type.AnyObjectType; +import org.enso.table.data.column.storage.type.BigDecimalType; +import org.enso.table.data.column.storage.type.BooleanType; +import org.enso.table.data.column.storage.type.DateTimeType; +import org.enso.table.data.column.storage.type.DateType; +import org.enso.table.data.column.storage.type.FloatType; +import org.enso.table.data.column.storage.type.IntegerType; +import org.enso.table.data.column.storage.type.StorageType; +import org.enso.table.data.column.storage.type.TextType; +import org.enso.table.data.column.storage.type.TimeOfDayType; +import org.enso.table.data.table.Column; +import org.enso.table.data.table.Table; +import org.enso.table.problems.ProblemAggregator; +import uk.co.jdunkerley.yxdb.Spatial; +import uk.co.jdunkerley.yxdb.YxdbField; +import uk.co.jdunkerley.yxdb.YxdbReader; +import uk.co.jdunkerley.yxdb.YxdbType; + +public final class AlteryxYXDBReader { + /** + * Reads an Alteryx YXDB file and returns its contents as a Table. + * + * @param path the path to the YXDB file. + * @return a Table containing the data from the YXDB file. + */ + public static Table read(String path, ProblemAggregator problemAggregator) + throws FileNotFoundException, IllegalArgumentException, IllegalStateException { + // Test that the path exists + if (!Files.exists(Path.of(path))) { + throw new FileNotFoundException(path); + } + + try (var yxdbReader = new YxdbReader(path)) { + var recordCount = yxdbReader.numRecords(); + var fields = yxdbReader.fields(); + var storageTypes = + Arrays.stream(fields).map(AlteryxYXDBReader::mapYXDBField).toArray(StorageType[]::new); + var storages = + Arrays.stream(storageTypes) + .map(st -> st.makeBuilder(recordCount, problemAggregator)) + .toArray(Builder[]::new); + + while (yxdbReader.next()) { + try { + for (int i = 0; i < storages.length; i++) { + var yxdbValue = + storageTypes[i] instanceof AnyObjectType + ? yxdbReader.readBlob(i) + : yxdbReader.read(i); + storages[i].append(yxdbValue); + } + } catch (IndexOutOfBoundsException _) { + throw new IllegalArgumentException( + "The YXDB file appears to be corrupted on row " + storages[0].getCurrentSize()); + } catch (DateTimeParseException _) { + throw new IllegalArgumentException( + "The YXDB file contains invalid date/time data on row " + + storages[0].getCurrentSize()); + } + } + + var columns = + IntStream.range(0, storages.length) + .mapToObj(i -> new Column(fields[i].name(), storages[i].seal())) + .toArray(Column[]::new); + return new Table(columns); + } catch (IllegalArgumentException exc) { + throw exc; + } catch (IOException exc) { + var message = exc.getMessage(); + throw new IllegalArgumentException(exc.getMessage(), exc); + } catch (Exception exc) { + throw new IllegalStateException("An unexpected error occurred: " + exc.getMessage(), exc); + } + } + + /** + * Converts a spatial object in byte array format to its GeoJSON representation. + * + * @param spatialObj the spatial object as a byte array. + * @return the GeoJSON representation of the spatial object. + */ + public static String spatialObjectToGeoJSON(byte[] spatialObj) { + return spatialObj == null ? null : Spatial.toGeoJson(spatialObj); + } + + private static StorageType mapYXDBField(YxdbField field) { + return switch (field.yxdbType()) { + case YxdbType.BOOLEAN -> BooleanType.INSTANCE; + case YxdbType.BYTE -> IntegerType.INT_8; + case YxdbType.INT16 -> IntegerType.INT_16; + case YxdbType.INT32 -> IntegerType.INT_32; + case YxdbType.INT64 -> IntegerType.INT_64; + case YxdbType.FLOAT, YxdbType.DOUBLE -> FloatType.FLOAT_64; + case YxdbType.DECIMAL -> BigDecimalType.INSTANCE; + case YxdbType.STRING, YxdbType.WSTRING -> TextType.variableLengthWithLimit(field.size()); + case YxdbType.V_STRING, YxdbType.V_WSTRING -> TextType.VARIABLE_LENGTH; + case YxdbType.DATE -> DateType.INSTANCE; + case YxdbType.TIME -> TimeOfDayType.INSTANCE; + case YxdbType.DATETIME -> DateTimeType.INSTANCE; + case YxdbType.BLOB, YxdbType.SPATIAL_OBJ -> AnyObjectType.INSTANCE; + default -> + throw new IllegalStateException("Unsupported YXDB field type: " + field.yxdbType()); + }; + } +} diff --git a/test/Table_Tests/data/alteryx/AllNormalFields.yxdb b/test/Table_Tests/data/alteryx/AllNormalFields.yxdb new file mode 100644 index 000000000000..0f8d29abb782 Binary files /dev/null and b/test/Table_Tests/data/alteryx/AllNormalFields.yxdb differ diff --git a/test/Table_Tests/data/alteryx/ampdata.yxdb b/test/Table_Tests/data/alteryx/ampdata.yxdb new file mode 100644 index 000000000000..cdd23e012684 Binary files /dev/null and b/test/Table_Tests/data/alteryx/ampdata.yxdb differ diff --git a/test/Table_Tests/data/alteryx/line.yxdb b/test/Table_Tests/data/alteryx/line.yxdb new file mode 100644 index 000000000000..b6c118d955db Binary files /dev/null and b/test/Table_Tests/data/alteryx/line.yxdb differ diff --git a/test/Table_Tests/src/IO/Alteryx_YXDB_Spec.enso b/test/Table_Tests/src/IO/Alteryx_YXDB_Spec.enso new file mode 100644 index 000000000000..a181467b7c81 --- /dev/null +++ b/test/Table_Tests/src/IO/Alteryx_YXDB_Spec.enso @@ -0,0 +1,89 @@ +from Standard.Base import all +import Standard.Base.Errors.Illegal_Argument.Illegal_Argument +from Standard.Table import all + +from Standard.Test import all + +add_specs suite_builder = + suite_builder.group "Alteryx_Format" group_builder-> + group_builder.specify "Should read a YXDB without error (with explicit format)" <| + data_file = enso_project.data / 'alteryx' / 'AllNormalFields.yxdb' + yxdb_read = Data.read data_file ..Alteryx_Format + yxdb_read.should_succeed + yxdb_read.should_be_a Table + + group_builder.specify "Should read a YXDB without error (with auto format)" <| + data_file = enso_project.data / 'alteryx' / 'AllNormalFields.yxdb' + yxdb_read = Data.read data_file + yxdb_read.should_succeed + yxdb_read.should_be_a Table + + group_builder.specify "Should read normal data from YXDB" <| + data_file = enso_project.data / 'alteryx' / 'AllNormalFields.yxdb' + table = Data.read data_file ..Alteryx_Format + + table.row_count.should_equal 1 + + table.column_count.should_equal 16 + table.column_names.should_equal ["ByteField", "BoolField", "Int16Field", "Int32Field", "Int64Field", "FixedDecimalField", "FloatField", "DoubleField", "StringField", "WStringField", "V_StringShortField", "V_StringLongField", "V_WStringShortField", "V_WStringLongField", "DateField", "DateTimeField"] + + expected = [1, True, 16, 32, 64, (dec "123.450000"), 678.9000244140625, 0.12345, 'A', 'AB', 'ABC', 'BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB', 'XZY', 'WWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWWW', Date.new 2020 1 1, Date_Time.new 2020 2 3 4 5 6] + actual = table.rows.first.to_vector + actual.should_equal expected + + group_builder.specify "Should read spatial data from YXDB" <| + data_file = enso_project.data / 'alteryx' / 'line.yxdb' + table = Data.read data_file ..Alteryx_Format + + table.row_count.should_equal 1 + table.column_count.should_equal 2 + table.column_names.should_equal ["RecordID", "Spatial"] + + table.get_value "RecordID" 0 . should_equal 1 + + spatial_data = table.get_value "Spatial" 0 + + ## Spatial data is a byte[] + spatial_data.should_be_a Array + spatial_data.length.should_equal 144 + + group_builder.specify "Should be able to convert spatial data to GeoJSON" <| + byte_vec = [3, 0, 0, 0, 0, 0, 0, 0, 0, -103, 91, -64, -90, 97, -8, -120, -104, 58, 61, 64, 0, 0, 0, 0, 0, 24, 85, -64, 56, -92, 81, -127, -109, 37, 69, 64, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -72, 90, -64, 56, -92, 81, -127, -109, 37, 69, 64, 0, 0, 0, 0, 0, 24, 85, -64, -2, 124, 91, -80, 84, -97, 68, 64, -122, -112, -13, -2, 63, -106, 90, -64, -46, -60, 59, -64, -109, 94, 66, 64, -122, -112, -13, -2, 63, 80, 85, -64, 63, -28, 45, 87, 63, -106, 65, 64, 0, 0, 0, 0, 0, -103, 91, -64, -118, 3, -24, -9, -3, 69, 64, 64, -122, -112, -13, -2, 63, 94, 86, -64, -90, 97, -8, -120, -104, 58, 61, 64] + expected = '{"yxdbType":"LineString","coordinates":[[-106.875,42.293564],[-84.375,41.244772],[-106.347656,36.738884],[-85.253906,35.173808],[-110.390625,32.546813],[-89.472656,29.22889]]}' + actual = Alteryx_Format.spatial_to_geojson byte_vec + actual.should_equal expected + + group_builder.specify "Should be able to convert spatial column to GeoJSON" <| + data_file = enso_project.data / 'alteryx' / 'line.yxdb' + table = Data.read data_file ..Alteryx_Format + + expected = '{"yxdbType":"LineString","coordinates":[[-106.875,42.293564],[-84.375,41.244772],[-106.347656,36.738884],[-85.253906,35.173808],[-110.390625,32.546813],[-89.472656,29.22889]]}' + + converted_table = Alteryx_Format.spatial_column_to_geojson table "Spatial" + converted_table.row_count.should_equal 1 + converted_table.column_count.should_equal 2 + converted_table.column_names.should_equal ["RecordID", "Spatial"] + converted_table.get_value "RecordID" 0 . should_equal 1 + converted_table.get_value "Spatial" 0 . should_equal expected + + group_builder.specify "Should error gracefully when reading an AMP file" <| + data_file = enso_project.data / 'alteryx' / 'ampdata.yxdb' + + yxdb_read = Data.read data_file ..Alteryx_Format + yxdb_read.should_fail_with Illegal_Argument + yxdb_read.catch.message . should_equal "Reading AMP YXDB files is not supported." + + data_read = Data.read data_file + data_read . should_fail_with Illegal_Argument + data_read.catch.message . should_equal "Reading AMP YXDB files is not supported." + + group_builder.specify "Should error gracefully when reading a non-YXDB file" <| + data_file = enso_project.data / 'data.csv' + + yxdb_read = Data.read data_file ..Alteryx_Format + yxdb_read.should_fail_with Illegal_Argument + +main filter=Nothing = + suite = Test.build suite_builder-> + add_specs suite_builder + suite.run_with_filter filter diff --git a/test/Table_Tests/src/IO/Main.enso b/test/Table_Tests/src/IO/Main.enso index baec30379b5c..61785bc1bd4f 100644 --- a/test/Table_Tests/src/IO/Main.enso +++ b/test/Table_Tests/src/IO/Main.enso @@ -2,6 +2,7 @@ from Standard.Base import all from Standard.Test import all +import project.IO.Alteryx_YXDB_Spec import project.IO.Cloud_Spec import project.IO.Csv_Spec import project.IO.Data_Link_Formats_Spec @@ -13,6 +14,7 @@ import project.IO.Json_Spec import project.IO.Read_Many_Spec add_specs suite_builder = + Alteryx_YXDB_Spec.add_specs suite_builder Cloud_Spec.add_specs suite_builder Csv_Spec.add_specs suite_builder Delimited_Read_Spec.add_specs suite_builder diff --git a/tools/legal-review/Table/report-state b/tools/legal-review/Table/report-state index aa829a643cd7..adbee422ceda 100644 --- a/tools/legal-review/Table/report-state +++ b/tools/legal-review/Table/report-state @@ -1,3 +1,3 @@ -F5B107EDB522F398BDA55C6EA2EF4A1DBC32C2470DFD2AC73DBDB7882B98A804 -F8A107FF4612F102D1F3311C94A9C1B203C2AC8B246B2DD7B62BDA1BF36343D6 +6E3CE1063B51808B8203CF9601E5D8D46641204D02CEDD331B412BA9BF362875 +2883761A2F257ECD4D0F5D652B07B0F684C9DD5DD32134C1B30E4CB9CB0CE696 0 diff --git a/tools/legal-review/Table/uk.co.jdunkerley.yxdb-java-0.1.4/custom-license b/tools/legal-review/Table/uk.co.jdunkerley.yxdb-java-0.1.4/custom-license new file mode 100644 index 000000000000..6b1d0bfabc3c --- /dev/null +++ b/tools/legal-review/Table/uk.co.jdunkerley.yxdb-java-0.1.4/custom-license @@ -0,0 +1 @@ +LICENSE diff --git a/tools/legal-review/Table/uk.co.jdunkerley.yxdb-java-0.1.4/files-keep b/tools/legal-review/Table/uk.co.jdunkerley.yxdb-java-0.1.4/files-keep new file mode 100644 index 000000000000..80d7ea8ec71a --- /dev/null +++ b/tools/legal-review/Table/uk.co.jdunkerley.yxdb-java-0.1.4/files-keep @@ -0,0 +1 @@ +/jdunkerley/yxdb-java/blob/main/LICENSE