diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/builder/BoolBuilder.java b/std-bits/table/src/main/java/org/enso/table/data/column/builder/BoolBuilder.java index db4e5041abcd..93929e582505 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/builder/BoolBuilder.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/builder/BoolBuilder.java @@ -1,5 +1,6 @@ package org.enso.table.data.column.builder; +import java.lang.foreign.MemorySegment; import java.util.BitSet; import org.enso.table.data.column.storage.BoolStorage; import org.enso.table.data.column.storage.ColumnBooleanStorage; @@ -8,18 +9,31 @@ import org.enso.table.data.column.storage.type.NullType; import org.enso.table.data.column.storage.type.StorageType; import org.enso.table.error.ValueTypeMismatchException; -import org.enso.table.util.BitSets; +import org.enso.table.util.ImmutableBitSet; /** A builder for boolean columns. */ final class BoolBuilder implements BuilderForBoolean, BuilderWithRetyping { private final BitSet vals; private final BitSet validityMap; - int size = 0; + private int size; // ** Creates a new builder for boolean columns. Should be built via Builder.getForBoolean. */ BoolBuilder(int capacity) { - vals = new BitSet(capacity); - validityMap = new BitSet(capacity); + this(new BitSet(capacity), new BitSet(capacity), 0); + } + + private BoolBuilder(BitSet vals, BitSet validityMap, int size) { + this.vals = vals; + this.validityMap = validityMap; + this.size = size; + } + + static BoolBuilder fromAddress(int sizeInBits, long data, long validity) { + var bytesSize = (sizeInBits + 7) / 8; + var vals = BitSet.valueOf(MemorySegment.ofAddress(data).reinterpret(bytesSize).asByteBuffer()); + var validityMap = + BitSet.valueOf(MemorySegment.ofAddress(validity).reinterpret(bytesSize).asByteBuffer()); + return new BoolBuilder(vals, validityMap, sizeInBits); } @Override @@ -72,7 +86,7 @@ public void appendBulkStorage(ColumnStorage storage) { if (storage instanceof BoolStorage boolStorage) { // We know this is valid for a BoolStorage. int toCopy = (int) boolStorage.getSize(); - BitSets.copy(boolStorage.getValues(), vals, size, toCopy); + boolStorage.getValues().copyTo(vals, size, toCopy); boolStorage.getValidityMap().copyTo(validityMap, size, toCopy); size += toCopy; } else if (storage instanceof ColumnBooleanStorage columnBooleanStorage) { @@ -92,7 +106,13 @@ public void appendBulkStorage(ColumnStorage storage) { @Override public ColumnStorage seal() { - return new BoolStorage(vals, validityMap, size, false); + return seal(null); + } + + final ColumnStorage seal(ColumnStorage other) { + var vals = new ImmutableBitSet(this.vals, size); + var validityMap = new ImmutableBitSet(this.validityMap, size); + return new BoolStorage(vals, validityMap, size, false, other); } @Override diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/builder/Builder.java b/std-bits/table/src/main/java/org/enso/table/data/column/builder/Builder.java index c858a8f895f4..9ef4c284bc3e 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/builder/Builder.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/builder/Builder.java @@ -114,6 +114,7 @@ static ColumnStorage makeLocal(ColumnStorage storage) { var localType = StorageType.fromTypeCharAndSize(proxyType.typeChar(), proxyType.size()); var localStorage = switch (localType) { + case BooleanType type -> BoolBuilder.fromAddress(size, data, validity).seal(storage); case IntegerType type -> LongBuilder.fromAddress(size, data, validity, type).seal(storage, type); default -> storage; diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/IsInOperation.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/IsInOperation.java index 94c567592506..c40326c33864 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/IsInOperation.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/IsInOperation.java @@ -267,7 +267,8 @@ private static ColumnStorage applyBooleanIsIn( // If had both true and false, then return all true when not nothing if (flags.hadTrue && flags.hadFalse) { var validityMap = makeValidityMap(boolStorage, checkedSize); - return new BoolStorage(new BitSet(), validityMap, checkedSize, true); + return new BoolStorage( + ImmutableBitSet.allFalse(checkedSize), validityMap, checkedSize, true, null); } // Only have one of true or false @@ -296,7 +297,7 @@ private static ColumnStorage applyBooleanIsIn( private static ColumnStorage applyBoolStorage( boolean keepValue, BoolStorage boolStorage, int checkedSize) { - BitSet values = boolStorage.getValues(); + BitSet values = boolStorage.getValues().cloneBitSet(); BitSet isNothing = boolStorage.getValidityMap().cloneBitSet(); isNothing.flip(0, Math.toIntExact(boolStorage.getSize())); diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/binary/FillMissingOperation.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/binary/FillMissingOperation.java index 81b8680e8810..0092c9bf645e 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/binary/FillMissingOperation.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/binary/FillMissingOperation.java @@ -2,7 +2,6 @@ import java.math.BigDecimal; import java.math.BigInteger; -import java.util.BitSet; import org.enso.base.polyglot.NumericConverter; import org.enso.table.data.column.builder.Builder; import org.enso.table.data.column.operation.BinaryOperation; @@ -100,7 +99,7 @@ public ColumnStorage applyZip( public static class BooleanFillMissingOperation extends FillMissingOperation { public static BoolStorage fillMissingBoolStorage(BoolStorage storage, boolean fillValue) { var size = (int) storage.getSize(); - var newValues = (BitSet) storage.getValues().clone(); + var newValues = storage.getValues().cloneBitSet(); var isNothingMap = storage.getValidityMap().cloneBitSet(); isNothingMap.flip(0, size); if (fillValue != storage.isNegated()) { @@ -109,7 +108,8 @@ public static BoolStorage fillMissingBoolStorage(BoolStorage storage, boolean fi newValues.andNot(isNothingMap); } var validity = ImmutableBitSet.allTrue(size); - return new BoolStorage(newValues, validity, size, storage.isNegated()); + return new BoolStorage( + new ImmutableBitSet(newValues, size), validity, size, storage.isNegated(), null); } public BooleanFillMissingOperation(StorageType resultType) { diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/binary/LogicalOperations.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/binary/LogicalOperations.java index aa8648b62f30..316290827df1 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/binary/LogicalOperations.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/binary/LogicalOperations.java @@ -80,7 +80,7 @@ protected ColumnBooleanStorage applySpecializedMapOverBoolStorage( } int size = (int) left.getSize(); - BitSet values = left.getValues(); + BitSet values = left.getValues().cloneBitSet(); if (left.isNegated()) { var newMissing = new BitSet(size); newMissing.flip(0, size); @@ -110,10 +110,10 @@ protected ColumnBooleanStorage applySpecializedZipOverBoolStorage( BoolStorage left, BoolStorage right, MapOperationProblemAggregator problemAggregator) { int size = (int) left.getSize(); int rightSize = (int) right.getSize(); - BitSet values = left.getValues(); + BitSet values = left.getValues().cloneBitSet(); // Compute the output set - BitSet out = right.getValues().get(0, size); + BitSet out = right.getValues().cloneBitSet().get(0, size); boolean negated; if (left.isNegated()) { if (right.isNegated()) { @@ -209,7 +209,7 @@ protected ColumnBooleanStorage applySpecializedMapOverBoolStorage( } int size = (int) left.getSize(); - BitSet values = left.getValues(); + BitSet values = left.getValues().cloneBitSet(); if (left.isNegated()) { var newValidity = left.getValidityMap().cloneBitSet(); newValidity.andNot(values); @@ -232,10 +232,10 @@ protected ColumnBooleanStorage applySpecializedZipOverBoolStorage( BoolStorage left, BoolStorage right, MapOperationProblemAggregator problemAggregator) { int size = (int) left.getSize(); int rightSize = (int) right.getSize(); - BitSet values = left.getValues(); + BitSet values = left.getValues().cloneBitSet(); // Compute the output set - BitSet out = right.getValues().get(0, size); + BitSet out = right.getValues().cloneBitSet().get(0, size); boolean negated; if (left.isNegated()) { if (right.isNegated()) { diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/comparators/BooleanComparators.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/comparators/BooleanComparators.java index 1fea39919530..cd534725d1fa 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/comparators/BooleanComparators.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/comparators/BooleanComparators.java @@ -1,6 +1,5 @@ package org.enso.table.data.column.operation.comparators; -import java.util.BitSet; import org.enso.table.data.column.builder.Builder; import org.enso.table.data.column.operation.BinaryOperationBoolean; import org.enso.table.data.column.operation.BinaryOperationTyped; @@ -8,6 +7,7 @@ import org.enso.table.data.column.storage.BoolStorage; import org.enso.table.data.column.storage.ColumnBooleanStorage; import org.enso.table.data.table.problems.MapOperationProblemAggregator; +import org.enso.table.util.ImmutableBitSet; final class BooleanComparators { public static final BinaryOperationTyped EQ = @@ -60,10 +60,15 @@ protected ColumnBooleanStorage applySpecializedMapOverBoolStorage( boolean rightBoolean, boolean rightIsNothing, MapOperationProblemAggregator problemAggregator) { + var size = (int) left.getSize(); return rightBoolean ? NotOperation.applySpecializedBoolStorage(left) : new BoolStorage( - new BitSet(), left.getValidityMap(), Builder.checkSize(left.getSize()), false); + ImmutableBitSet.allFalse(size), + left.getValidityMap(), + Builder.checkSize(size), + false, + null); } }; @@ -81,9 +86,14 @@ protected ColumnBooleanStorage applySpecializedMapOverBoolStorage( boolean rightBoolean, boolean rightIsNothing, MapOperationProblemAggregator problemAggregator) { + var size = (int) left.getSize(); return rightBoolean ? new BoolStorage( - new BitSet(), left.getValidityMap(), Builder.checkSize(left.getSize()), true) + ImmutableBitSet.allFalse(size), + left.getValidityMap(), + Builder.checkSize(size), + true, + null) : NotOperation.applySpecializedBoolStorage(left); } }; @@ -102,9 +112,10 @@ protected ColumnBooleanStorage applySpecializedMapOverBoolStorage( boolean rightBoolean, boolean rightIsNothing, MapOperationProblemAggregator problemAggregator) { + var size = Builder.checkSize(left.getSize()); return rightBoolean ? new BoolStorage( - new BitSet(), left.getValidityMap(), Builder.checkSize(left.getSize()), false) + ImmutableBitSet.allFalse(size), left.getValidityMap(), size, false, null) : left; } }; @@ -123,10 +134,15 @@ protected ColumnBooleanStorage applySpecializedMapOverBoolStorage( boolean rightBoolean, boolean rightIsNothing, MapOperationProblemAggregator problemAggregator) { + var size = (int) left.getSize(); return rightBoolean ? left : new BoolStorage( - new BitSet(), left.getValidityMap(), Builder.checkSize(left.getSize()), true); + ImmutableBitSet.allFalse(size), + left.getValidityMap(), + Builder.checkSize(left.getSize()), + true, + null); } }; } diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/unary/DoubleIsOperation.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/unary/DoubleIsOperation.java index 7cbe91f1cb72..ec5bb9c37fc4 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/unary/DoubleIsOperation.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/unary/DoubleIsOperation.java @@ -1,6 +1,5 @@ package org.enso.table.data.column.operation.unary; -import java.util.BitSet; import java.util.function.DoublePredicate; import org.enso.table.data.column.builder.Builder; import org.enso.table.data.column.operation.StorageIterators; @@ -14,6 +13,7 @@ import org.enso.table.data.column.storage.type.IntegerType; import org.enso.table.data.column.storage.type.StorageType; import org.enso.table.data.table.problems.MapOperationProblemAggregator; +import org.enso.table.util.ImmutableBitSet; public class DoubleIsOperation implements UnaryOperation { public static final String FINITE_NAME = "is_finite"; @@ -62,8 +62,13 @@ public ColumnStorage apply( // For Finite if (isAllFinite(storage.getType())) { if (storage instanceof ColumnStorageWithValidityMap withNothingMap) { + var size = (int) storage.getSize(); return new BoolStorage( - new BitSet(), withNothingMap.getValidityMap(), (int) storage.getSize(), finiteValue); + ImmutableBitSet.allFalse(size), + withNothingMap.getValidityMap(), + size, + finiteValue, + null); } return StorageIterators.mapOverStorage( diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/unary/IsNothingOperation.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/unary/IsNothingOperation.java index 2df516680ef4..17543b9861f7 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/unary/IsNothingOperation.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/unary/IsNothingOperation.java @@ -31,7 +31,7 @@ public ColumnStorage apply( if (storage instanceof ColumnStorageWithValidityMap validityMap) { var size = (int) storage.getSize(); var allValidity = ImmutableBitSet.allTrue(size); - return new BoolStorage(validityMap.getValidityMap().cloneBitSet(), allValidity, size, true); + return new BoolStorage(validityMap.getValidityMap(), allValidity, size, true, null); } return StorageIterators.buildOverStorage( diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/operation/unary/NotOperation.java b/std-bits/table/src/main/java/org/enso/table/data/column/operation/unary/NotOperation.java index 96023361ea81..a7aa420d8cf9 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/operation/unary/NotOperation.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/operation/unary/NotOperation.java @@ -64,7 +64,8 @@ public static ColumnBooleanStorage applySpecializedBoolStorage(BoolStorage boolS boolStorage.getValues(), boolStorage.getValidityMap(), (int) boolStorage.getSize(), - !boolStorage.isNegated()); + !boolStorage.isNegated(), + null); } public static ColumnBooleanStorage applySpecializedNullStorage(ColumnStorage storage) { diff --git a/std-bits/table/src/main/java/org/enso/table/data/column/storage/BoolStorage.java b/std-bits/table/src/main/java/org/enso/table/data/column/storage/BoolStorage.java index 756add3c607a..523fbfa80943 100644 --- a/std-bits/table/src/main/java/org/enso/table/data/column/storage/BoolStorage.java +++ b/std-bits/table/src/main/java/org/enso/table/data/column/storage/BoolStorage.java @@ -1,5 +1,6 @@ package org.enso.table.data.column.storage; +import java.lang.foreign.MemorySegment; import java.util.BitSet; import java.util.NoSuchElementException; import org.enso.table.data.column.storage.iterators.ColumnBooleanStorageIterator; @@ -9,21 +10,43 @@ /** A boolean column storage. */ public final class BoolStorage extends Storage implements ColumnBooleanStorage, ColumnStorageWithValidityMap { - private final BitSet values; + private final ImmutableBitSet values; private final ImmutableBitSet validityMap; private final int size; private final boolean negated; + private final ColumnStorage proxy; public BoolStorage(BitSet values, BitSet validityMap, int size, boolean negated) { - this(values, new ImmutableBitSet(validityMap, size), size, negated); + this( + new ImmutableBitSet(values, size), + new ImmutableBitSet(validityMap, size), + size, + negated, + null); } - public BoolStorage(BitSet values, ImmutableBitSet validityMap, int size, boolean negated) { + public BoolStorage( + ImmutableBitSet values, + ImmutableBitSet validityMap, + int size, + boolean negated, + ColumnStorage other) { super(BooleanType.INSTANCE); this.values = values; this.validityMap = validityMap; this.size = size; this.negated = negated; + this.proxy = other; + } + + @Override + public long addressOfData() { + return MemorySegment.ofBuffer(values.rawData()).address(); + } + + @Override + public long addressOfValidity() { + return MemorySegment.ofBuffer(validityMap.rawData()).address(); } @Override @@ -57,7 +80,7 @@ public boolean isNegated() { return negated; } - public BitSet getValues() { + public ImmutableBitSet getValues() { return values; } diff --git a/std-bits/tests/src/test/java/org/enso/base/polyglot/tests/BoolStorageTest.java b/std-bits/tests/src/test/java/org/enso/base/polyglot/tests/BoolStorageTest.java new file mode 100644 index 000000000000..f69f991ba4e9 --- /dev/null +++ b/std-bits/tests/src/test/java/org/enso/base/polyglot/tests/BoolStorageTest.java @@ -0,0 +1,36 @@ +package org.enso.base.polyglot.tests; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; + +import org.enso.table.data.column.builder.Builder; +import org.enso.test.utils.ContextUtils; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +public class BoolStorageTest { + @ClassRule + public static final ContextUtils ctx = ContextUtils.newBuilder("enso").assertGC(false).build(); + + @BeforeClass + public static void importAll() { + ctx.eval("enso", "from Standard.Base import all"); + } + + @Test + public void makeLocalFromLongStorage() { + var b = Builder.getForBoolean(3); + b.append(false).appendNulls(1).append(true); + var storage = b.seal(); + var localStorage = Builder.makeLocal(storage); + assertNotSame("local storage is a copy of storage", storage, localStorage); + assertEquals("They have the same size", storage.getSize(), localStorage.getSize()); + assertEquals("They have the same type", storage.getType(), localStorage.getType()); + for (var i = 0L; i < storage.getSize(); i++) { + var elem = storage.getItemBoxed(i); + var localElem = localStorage.getItemBoxed(i); + assertEquals("At " + i, elem, localElem); + } + } +} diff --git a/std-bits/tests/src/test/java/org/enso/table/data/column/operation/unary/IsNothingOperationTest.java b/std-bits/tests/src/test/java/org/enso/table/data/column/operation/unary/IsNothingOperationTest.java index 6e863bfba880..057c6ecc4546 100644 --- a/std-bits/tests/src/test/java/org/enso/table/data/column/operation/unary/IsNothingOperationTest.java +++ b/std-bits/tests/src/test/java/org/enso/table/data/column/operation/unary/IsNothingOperationTest.java @@ -36,7 +36,8 @@ private void checkBoolStorage(boolean negate) { var storage = Builder.getForBoolean(3).append(true).append(false).appendNulls(1).seal(); if (negate) { if (storage instanceof BoolStorage bs) { - storage = new BoolStorage(bs.getValues(), bs.getValidityMap(), (int) bs.getSize(), true); + storage = + new BoolStorage(bs.getValues(), bs.getValidityMap(), (int) bs.getSize(), true, null); } else { fail("Expecting bool storage: " + storage); }