Skip to content

Commit 3ec78e6

Browse files
kenwenzelkenwenzel
committed
GH-4950 LMDB: support full 64-bit doubles and fix float encoding
1 parent 1d600f6 commit 3ec78e6

File tree

4 files changed

+143
-115
lines changed

4 files changed

+143
-115
lines changed

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/ValueStore.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -467,7 +467,7 @@ protected byte[] getData(long id) throws IOException {
467467
* @return the value object or <code>null</code> if not found
468468
*/
469469
LmdbValue cachedValue(long id) {
470-
LmdbValue value = valueCache[(int) (id % valueCache.length)];
470+
LmdbValue value = valueCache[(int) ((id < 0 ? -id : id) % valueCache.length)];
471471
if (value != null && value.getInternalID() == id) {
472472
return value;
473473
}
@@ -484,7 +484,7 @@ LmdbValue cachedValue(long id) {
484484
* @return the value object or <code>null</code> if not found
485485
*/
486486
void cacheValue(long id, LmdbValue value) {
487-
valueCache[(int) (id % valueCache.length)] = value;
487+
valueCache[(int) ((id < 0 ? -id : id) % valueCache.length)] = value;
488488
}
489489

490490
/**

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/Varint.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,11 @@ private Varint() {
8989
* @param value value to encode
9090
*/
9191
public static void writeUnsigned(final ByteBuffer bb, final long value) {
92-
if (value <= 240) {
92+
if (value < 0) {
93+
int bytes = descriptor(value) + 1;
94+
bb.put((byte) (250 + (bytes - 3)));
95+
writeSignificantBits(bb, value, bytes);
96+
} else if (value <= 240) {
9397
bb.put((byte) value);
9498
} else if (value <= 2287) {
9599
bb.put((byte) ((value - 240) / 256 + 241));
@@ -112,7 +116,10 @@ public static void writeUnsigned(final ByteBuffer bb, final long value) {
112116
* @return length in bytes
113117
*/
114118
public static int calcLengthUnsigned(long value) {
115-
if (value <= 240) {
119+
if (value < 0) {
120+
int bytes = descriptor(value) + 1;
121+
return 1 + bytes;
122+
} else if (value <= 240) {
116123
return 1;
117124
} else if (value <= 2287) {
118125
return 2;

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/inlined/Decimals.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ static long packDouble(double value) {
113113
}
114114

115115
static long packFloat(float value) {
116-
return ValueIds.createId(ValueIds.T_FLOAT, Float.floatToIntBits(value));
116+
return ValueIds.createId(ValueIds.T_FLOAT, Integers.encodeZigZag(Float.floatToRawIntBits(value)));
117117
}
118118

119119
static Literal unpackDecimal(long value, ValueFactory valueFactory) {
@@ -147,7 +147,7 @@ static Literal unpackDouble(long value, ValueFactory valueFactory) {
147147
}
148148

149149
static Literal unpackFloat(long value, ValueFactory valueFactory) {
150-
float floatValue = Float.intBitsToFloat((int) ValueIds.getValue(value));
150+
float floatValue = Float.intBitsToFloat((int) Integers.decodeZigZag(ValueIds.getValue(value)));
151151
return valueFactory.createLiteral(floatValue);
152152
}
153153
}

‎core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/inlined/ValuesTest.java‎

Lines changed: 130 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
import java.math.BigDecimal;
1616
import java.math.BigInteger;
17+
import java.nio.ByteBuffer;
1718
import java.time.LocalDate;
1819
import java.time.LocalDateTime;
1920
import java.util.Arrays;
@@ -23,123 +24,125 @@
2324
import org.eclipse.rdf4j.model.ValueFactory;
2425
import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
2526
import org.eclipse.rdf4j.model.vocabulary.XSD;
27+
import org.eclipse.rdf4j.sail.lmdb.Varint;
2628
import org.junit.jupiter.api.Test;
2729

2830
class ValuesTest {
2931

3032
private final ValueFactory vf = SimpleValueFactory.getInstance();
3133

34+
private List<Literal> literals = Arrays.asList(
35+
// DECIMAL
36+
vf.createLiteral(BigDecimal.ZERO),
37+
vf.createLiteral(BigDecimal.ONE.negate()),
38+
vf.createLiteral(new BigDecimal("123456789.987654321")),
39+
vf.createLiteral(new BigDecimal("0.00000000000000000001")),
40+
vf.createLiteral(BigDecimal.valueOf(42.42)),
41+
vf.createLiteral(BigDecimal.TEN),
42+
// DOUBLE
43+
vf.createLiteral(Double.NaN),
44+
vf.createLiteral(Double.POSITIVE_INFINITY),
45+
vf.createLiteral(Double.NEGATIVE_INFINITY),
46+
vf.createLiteral(Double.MIN_VALUE),
47+
vf.createLiteral(Double.MAX_VALUE),
48+
// vf.createLiteral(-0.0d),
49+
vf.createLiteral(3.14159d),
50+
vf.createLiteral(2.0d),
51+
vf.createLiteral(7.11d),
52+
// FLOAT
53+
vf.createLiteral(Float.NaN),
54+
vf.createLiteral(Float.POSITIVE_INFINITY),
55+
vf.createLiteral(Float.NEGATIVE_INFINITY),
56+
vf.createLiteral(Float.MIN_VALUE),
57+
vf.createLiteral(Float.MAX_VALUE),
58+
vf.createLiteral(-0.0f),
59+
vf.createLiteral(1.5f),
60+
vf.createLiteral(0.25f),
61+
// INTEGER
62+
vf.createLiteral(BigInteger.ZERO),
63+
vf.createLiteral(BigInteger.ONE.negate()),
64+
vf.createLiteral(BigInteger.valueOf(Long.MAX_VALUE)),
65+
vf.createLiteral(BigInteger.valueOf(Long.MIN_VALUE)),
66+
vf.createLiteral(BigInteger.valueOf(100)),
67+
vf.createLiteral(BigInteger.valueOf(-12345)),
68+
// LONG
69+
vf.createLiteral(Long.MAX_VALUE),
70+
vf.createLiteral(Long.MIN_VALUE),
71+
vf.createLiteral(0L),
72+
vf.createLiteral(123456789L),
73+
// INT
74+
vf.createLiteral(Integer.MAX_VALUE),
75+
vf.createLiteral(Integer.MIN_VALUE),
76+
vf.createLiteral(0),
77+
vf.createLiteral(42),
78+
// SHORT
79+
vf.createLiteral(Short.MAX_VALUE),
80+
vf.createLiteral(Short.MIN_VALUE),
81+
vf.createLiteral((short) 0),
82+
vf.createLiteral((short) 999),
83+
// BYTE
84+
vf.createLiteral(Byte.MAX_VALUE),
85+
vf.createLiteral(Byte.MIN_VALUE),
86+
vf.createLiteral((byte) 0),
87+
vf.createLiteral((byte) 42),
88+
// UNSIGNED_LONG
89+
vf.createLiteral("0", XSD.UNSIGNED_LONG),
90+
vf.createLiteral("18446744073709551615", XSD.UNSIGNED_LONG), // 2^64-1
91+
vf.createLiteral("123456789", XSD.UNSIGNED_LONG),
92+
// UNSIGNED_INT
93+
vf.createLiteral("0", XSD.UNSIGNED_INT),
94+
vf.createLiteral("4294967295", XSD.UNSIGNED_INT), // 2^32-1
95+
vf.createLiteral("123456", XSD.UNSIGNED_INT),
96+
// UNSIGNED_SHORT
97+
vf.createLiteral("0", XSD.UNSIGNED_SHORT),
98+
vf.createLiteral("65535", XSD.UNSIGNED_SHORT), // 2^16-1
99+
vf.createLiteral("12345", XSD.UNSIGNED_SHORT),
100+
// UNSIGNED_BYTE
101+
vf.createLiteral("0", XSD.UNSIGNED_BYTE),
102+
vf.createLiteral("255", XSD.UNSIGNED_BYTE), // 2^8-1
103+
vf.createLiteral("42", XSD.UNSIGNED_BYTE),
104+
// POSITIVE_INTEGER
105+
vf.createLiteral("1", XSD.POSITIVE_INTEGER),
106+
vf.createLiteral("999999999999999999999999", XSD.POSITIVE_INTEGER),
107+
vf.createLiteral("42", XSD.POSITIVE_INTEGER),
108+
// NEGATIVE_INTEGER
109+
vf.createLiteral("-1", XSD.NEGATIVE_INTEGER),
110+
vf.createLiteral("-999999999999999999999999", XSD.NEGATIVE_INTEGER),
111+
vf.createLiteral("-42", XSD.NEGATIVE_INTEGER),
112+
// NON_NEGATIVE_INTEGER
113+
vf.createLiteral("0", XSD.NON_NEGATIVE_INTEGER),
114+
vf.createLiteral("123456789012345678", XSD.NON_NEGATIVE_INTEGER),
115+
vf.createLiteral("123", XSD.NON_NEGATIVE_INTEGER),
116+
// NON_POSITIVE_INTEGER
117+
vf.createLiteral("0", XSD.NON_POSITIVE_INTEGER),
118+
vf.createLiteral("-123456789012345678", XSD.NON_POSITIVE_INTEGER),
119+
vf.createLiteral("-99", XSD.NON_POSITIVE_INTEGER),
120+
// STRING (short string; edge + standard)
121+
vf.createLiteral("", XSD.STRING),
122+
vf.createLiteral("a", XSD.STRING),
123+
vf.createLiteral("abcdefg", XSD.STRING), // max inlined length
124+
vf.createLiteral("RDF4J", XSD.STRING),
125+
vf.createLiteral("test", XSD.STRING),
126+
// DATETIME
127+
vf.createLiteral(LocalDateTime.of(1970, 1, 1, 0, 0, 0)),
128+
vf.createLiteral(LocalDateTime.of(9999, 12, 31, 23, 59, 59)),
129+
vf.createLiteral(LocalDateTime.of(2020, 2, 29, 12, 0, 0)),
130+
vf.createLiteral(LocalDateTime.of(1999, 12, 31, 23, 59, 59)),
131+
// DATETIMESTAMP
132+
// vf.createLiteral(OffsetDateTime.of(1970, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)),
133+
// vf.createLiteral(OffsetDateTime.of(9999, 12, 31, 23, 59, 59, 0, ZoneOffset.ofHours(14))),
134+
// vf.createLiteral(OffsetDateTime.of(2000, 1, 1, 0, 0, 0, 0, ZoneOffset.ofHours(-5))),
135+
// DATE
136+
vf.createLiteral(LocalDate.of(1970, 1, 1)),
137+
vf.createLiteral(LocalDate.of(9999, 12, 31)),
138+
vf.createLiteral(LocalDate.of(2024, 6, 13)),
139+
// BOOLEAN
140+
vf.createLiteral(true),
141+
vf.createLiteral(false)
142+
);
143+
32144
@Test
33145
void testPackAndUnpack_AllLiteralTypesWithEdgeAndStandardCases() {
34-
List<Literal> literals = Arrays.asList(
35-
// DECIMAL
36-
vf.createLiteral(BigDecimal.ZERO),
37-
vf.createLiteral(BigDecimal.ONE.negate()),
38-
vf.createLiteral(new BigDecimal("123456789.987654321")),
39-
vf.createLiteral(new BigDecimal("0.00000000000000000001")),
40-
vf.createLiteral(BigDecimal.valueOf(42.42)),
41-
vf.createLiteral(BigDecimal.TEN),
42-
// DOUBLE
43-
vf.createLiteral(Double.NaN),
44-
vf.createLiteral(Double.POSITIVE_INFINITY),
45-
vf.createLiteral(Double.NEGATIVE_INFINITY),
46-
vf.createLiteral(Double.MIN_VALUE),
47-
vf.createLiteral(Double.MAX_VALUE),
48-
// vf.createLiteral(-0.0d),
49-
vf.createLiteral(3.14159d),
50-
vf.createLiteral(2.0d),
51-
// FLOAT
52-
vf.createLiteral(Float.NaN),
53-
vf.createLiteral(Float.POSITIVE_INFINITY),
54-
vf.createLiteral(Float.NEGATIVE_INFINITY),
55-
vf.createLiteral(Float.MIN_VALUE),
56-
vf.createLiteral(Float.MAX_VALUE),
57-
vf.createLiteral(-0.0f),
58-
vf.createLiteral(1.5f),
59-
vf.createLiteral(0.25f),
60-
// INTEGER
61-
vf.createLiteral(BigInteger.ZERO),
62-
vf.createLiteral(BigInteger.ONE.negate()),
63-
vf.createLiteral(BigInteger.valueOf(Long.MAX_VALUE)),
64-
vf.createLiteral(BigInteger.valueOf(Long.MIN_VALUE)),
65-
vf.createLiteral(BigInteger.valueOf(100)),
66-
vf.createLiteral(BigInteger.valueOf(-12345)),
67-
// LONG
68-
vf.createLiteral(Long.MAX_VALUE),
69-
vf.createLiteral(Long.MIN_VALUE),
70-
vf.createLiteral(0L),
71-
vf.createLiteral(123456789L),
72-
// INT
73-
vf.createLiteral(Integer.MAX_VALUE),
74-
vf.createLiteral(Integer.MIN_VALUE),
75-
vf.createLiteral(0),
76-
vf.createLiteral(42),
77-
// SHORT
78-
vf.createLiteral(Short.MAX_VALUE),
79-
vf.createLiteral(Short.MIN_VALUE),
80-
vf.createLiteral((short) 0),
81-
vf.createLiteral((short) 999),
82-
// BYTE
83-
vf.createLiteral(Byte.MAX_VALUE),
84-
vf.createLiteral(Byte.MIN_VALUE),
85-
vf.createLiteral((byte) 0),
86-
vf.createLiteral((byte) 42),
87-
// UNSIGNED_LONG
88-
vf.createLiteral("0", XSD.UNSIGNED_LONG),
89-
vf.createLiteral("18446744073709551615", XSD.UNSIGNED_LONG), // 2^64-1
90-
vf.createLiteral("123456789", XSD.UNSIGNED_LONG),
91-
// UNSIGNED_INT
92-
vf.createLiteral("0", XSD.UNSIGNED_INT),
93-
vf.createLiteral("4294967295", XSD.UNSIGNED_INT), // 2^32-1
94-
vf.createLiteral("123456", XSD.UNSIGNED_INT),
95-
// UNSIGNED_SHORT
96-
vf.createLiteral("0", XSD.UNSIGNED_SHORT),
97-
vf.createLiteral("65535", XSD.UNSIGNED_SHORT), // 2^16-1
98-
vf.createLiteral("12345", XSD.UNSIGNED_SHORT),
99-
// UNSIGNED_BYTE
100-
vf.createLiteral("0", XSD.UNSIGNED_BYTE),
101-
vf.createLiteral("255", XSD.UNSIGNED_BYTE), // 2^8-1
102-
vf.createLiteral("42", XSD.UNSIGNED_BYTE),
103-
// POSITIVE_INTEGER
104-
vf.createLiteral("1", XSD.POSITIVE_INTEGER),
105-
vf.createLiteral("999999999999999999999999", XSD.POSITIVE_INTEGER),
106-
vf.createLiteral("42", XSD.POSITIVE_INTEGER),
107-
// NEGATIVE_INTEGER
108-
vf.createLiteral("-1", XSD.NEGATIVE_INTEGER),
109-
vf.createLiteral("-999999999999999999999999", XSD.NEGATIVE_INTEGER),
110-
vf.createLiteral("-42", XSD.NEGATIVE_INTEGER),
111-
// NON_NEGATIVE_INTEGER
112-
vf.createLiteral("0", XSD.NON_NEGATIVE_INTEGER),
113-
vf.createLiteral("123456789012345678", XSD.NON_NEGATIVE_INTEGER),
114-
vf.createLiteral("123", XSD.NON_NEGATIVE_INTEGER),
115-
// NON_POSITIVE_INTEGER
116-
vf.createLiteral("0", XSD.NON_POSITIVE_INTEGER),
117-
vf.createLiteral("-123456789012345678", XSD.NON_POSITIVE_INTEGER),
118-
vf.createLiteral("-99", XSD.NON_POSITIVE_INTEGER),
119-
// STRING (short string; edge + standard)
120-
vf.createLiteral("", XSD.STRING),
121-
vf.createLiteral("a", XSD.STRING),
122-
vf.createLiteral("abcdefg", XSD.STRING), // max inlined length
123-
vf.createLiteral("RDF4J", XSD.STRING),
124-
vf.createLiteral("test", XSD.STRING),
125-
// DATETIME
126-
vf.createLiteral(LocalDateTime.of(1970, 1, 1, 0, 0, 0)),
127-
vf.createLiteral(LocalDateTime.of(9999, 12, 31, 23, 59, 59)),
128-
vf.createLiteral(LocalDateTime.of(2020, 2, 29, 12, 0, 0)),
129-
vf.createLiteral(LocalDateTime.of(1999, 12, 31, 23, 59, 59)),
130-
// DATETIMESTAMP
131-
// vf.createLiteral(OffsetDateTime.of(1970, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)),
132-
// vf.createLiteral(OffsetDateTime.of(9999, 12, 31, 23, 59, 59, 0, ZoneOffset.ofHours(14))),
133-
// vf.createLiteral(OffsetDateTime.of(2000, 1, 1, 0, 0, 0, 0, ZoneOffset.ofHours(-5))),
134-
// DATE
135-
vf.createLiteral(LocalDate.of(1970, 1, 1)),
136-
vf.createLiteral(LocalDate.of(9999, 12, 31)),
137-
vf.createLiteral(LocalDate.of(2024, 6, 13)),
138-
// BOOLEAN
139-
vf.createLiteral(true),
140-
vf.createLiteral(false)
141-
);
142-
143146
for (Literal literal : literals) {
144147
long packed = Values.packLiteral(literal);
145148
// If the literal is not inlined, packed==0. Only test roundtrip if it is inlined.
@@ -153,6 +156,24 @@ void testPackAndUnpack_AllLiteralTypesWithEdgeAndStandardCases() {
153156
}
154157
}
155158

159+
@Test
160+
void testPackAndUnpack_AllLiteralTypesWithVarintConversion() {
161+
ByteBuffer bb = ByteBuffer.allocate(Long.BYTES + 1);
162+
for (Literal literal : literals) {
163+
long packed = Values.packLiteral(literal);
164+
// If the literal is not inlined, packed==0. Only test roundtrip if it is inlined.
165+
if (packed != 0L) {
166+
bb.clear();
167+
Varint.writeUnsigned(bb, packed);
168+
bb.flip();
169+
assertThat(Varint.readUnsigned(bb)).isEqualTo(packed);
170+
} else {
171+
// (optional) ensure non-inlined values can be detected
172+
assertThat(packed).isZero();
173+
}
174+
}
175+
}
176+
156177
private void assertEqualLiterals(Literal actual, Literal expected) {
157178
assertThat(actual).isEqualTo(expected);
158179
}

0 commit comments

Comments
 (0)