Skip to content

Commit

Permalink
use long as ByteVector64
Browse files Browse the repository at this point in the history
  • Loading branch information
@wenshao
wenshao committed Jan 11, 2025
1 parent fc96048 commit f524539
Show file tree
Hide file tree
Showing 8 changed files with 189 additions and 259 deletions.
10 changes: 0 additions & 10 deletions core/src/main/java/com/alibaba/fastjson2/JSONFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ public static String getProperty(String key) {
static final NameCacheEntry[] NAME_CACHE = new NameCacheEntry[8192];
static final NameCacheEntry2[] NAME_CACHE2 = new NameCacheEntry2[8192];

static final Function<JSONWriter.Context, JSONWriter> INCUBATOR_VECTOR_WRITER_CREATOR_UTF8;
static final Function<JSONWriter.Context, JSONWriter> INCUBATOR_VECTOR_WRITER_CREATOR_UTF16;
static final JSONReaderUTF8Creator INCUBATOR_VECTOR_READER_CREATOR_ASCII;
static final JSONReaderUTF8Creator INCUBATOR_VECTOR_READER_CREATOR_UTF8;
Expand Down Expand Up @@ -231,20 +230,12 @@ public NameCacheEntry2(String name, long value0, long value1) {

boolean readerVector = getPropertyBool(properties, "fastjson2.readerVector", false);

Function<JSONWriter.Context, JSONWriter> incubatorVectorCreatorUTF8 = null;
Function<JSONWriter.Context, JSONWriter> incubatorVectorCreatorUTF16 = null;
JSONReaderUTF8Creator readerCreatorASCII = null;
JSONReaderUTF8Creator readerCreatorUTF8 = null;
JSONReaderUTF16Creator readerCreatorUTF16 = null;
if (JDKUtils.VECTOR_SUPPORT) {
if (VECTOR_BIT_LENGTH >= 64) {
try {
Class<?> factoryClass = Class.forName("com.alibaba.fastjson2.JSONWriterUTF8Vector$Factory");
incubatorVectorCreatorUTF8 = (Function<JSONWriter.Context, JSONWriter>) factoryClass.newInstance();
} catch (Throwable e) {
initErrorLast = e;
}

try {
Class<?> factoryClass = Class.forName("com.alibaba.fastjson2.JSONWriterUTF16Vector$Factory");
incubatorVectorCreatorUTF16 = (Function<JSONWriter.Context, JSONWriter>) factoryClass.newInstance();
Expand Down Expand Up @@ -278,7 +269,6 @@ public NameCacheEntry2(String name, long value0, long value1) {
}
}
}
INCUBATOR_VECTOR_WRITER_CREATOR_UTF8 = incubatorVectorCreatorUTF8;
INCUBATOR_VECTOR_WRITER_CREATOR_UTF16 = incubatorVectorCreatorUTF16;
INCUBATOR_VECTOR_READER_CREATOR_ASCII = readerCreatorASCII;
INCUBATOR_VECTOR_READER_CREATOR_UTF8 = readerCreatorUTF8;
Expand Down
74 changes: 12 additions & 62 deletions core/src/main/java/com/alibaba/fastjson2/JSONWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -547,15 +547,7 @@ public static JSONWriter of() {
jsonWriter = new JSONWriterUTF16JDK8(writeContext);
}
} else if ((defaultWriterFeatures & OptimizedForAscii.mask) != 0) {
if (STRING_VALUE != null) {
if (INCUBATOR_VECTOR_WRITER_CREATOR_UTF8 != null) {
jsonWriter = INCUBATOR_VECTOR_WRITER_CREATOR_UTF8.apply(writeContext);
} else {
jsonWriter = new JSONWriterUTF8JDK9(writeContext);
}
} else {
jsonWriter = new JSONWriterUTF8(writeContext);
}
jsonWriter = ofUTF8(writeContext);
} else {
if (INCUBATOR_VECTOR_WRITER_CREATOR_UTF16 != null) {
jsonWriter = INCUBATOR_VECTOR_WRITER_CREATOR_UTF16.apply(writeContext);
Expand Down Expand Up @@ -588,11 +580,7 @@ public static JSONWriter of(Context context) {
}
} else if ((context.features & OptimizedForAscii.mask) != 0) {
if (STRING_VALUE != null) {
if (INCUBATOR_VECTOR_WRITER_CREATOR_UTF8 != null) {
jsonWriter = INCUBATOR_VECTOR_WRITER_CREATOR_UTF8.apply(context);
} else {
jsonWriter = new JSONWriterUTF8JDK9(context);
}
jsonWriter = new JSONWriterUTF8JDK9(context);
} else {
jsonWriter = new JSONWriterUTF8(context);
}
Expand All @@ -617,15 +605,7 @@ public static JSONWriter of(Feature... features) {
jsonWriter = new JSONWriterUTF16JDK8(writeContext);
}
} else if ((writeContext.features & OptimizedForAscii.mask) != 0) {
if (STRING_VALUE != null) {
if (INCUBATOR_VECTOR_WRITER_CREATOR_UTF8 != null) {
jsonWriter = INCUBATOR_VECTOR_WRITER_CREATOR_UTF8.apply(writeContext);
} else {
jsonWriter = new JSONWriterUTF8JDK9(writeContext);
}
} else {
jsonWriter = new JSONWriterUTF8(writeContext);
}
jsonWriter = ofUTF8(writeContext);
} else {
if (INCUBATOR_VECTOR_WRITER_CREATOR_UTF16 != null) {
jsonWriter = INCUBATOR_VECTOR_WRITER_CREATOR_UTF16.apply(writeContext);
Expand Down Expand Up @@ -699,51 +679,21 @@ public static JSONWriter ofPretty(JSONWriter writer) {
}

public static JSONWriter ofUTF8() {
Context context = createWriteContext();
JSONWriter jsonWriter;
if (STRING_VALUE != null) {
if (INCUBATOR_VECTOR_WRITER_CREATOR_UTF8 != null) {
jsonWriter = INCUBATOR_VECTOR_WRITER_CREATOR_UTF8.apply(context);
} else {
jsonWriter = new JSONWriterUTF8JDK9(context);
}
} else {
jsonWriter = new JSONWriterUTF8(context);
}

return jsonWriter;
return ofUTF8(
createWriteContext()
);
}

public static JSONWriter ofUTF8(JSONWriter.Context context) {
JSONWriter jsonWriter;
if (STRING_VALUE != null) {
if (INCUBATOR_VECTOR_WRITER_CREATOR_UTF8 != null) {
jsonWriter = INCUBATOR_VECTOR_WRITER_CREATOR_UTF8.apply(context);
} else {
jsonWriter = new JSONWriterUTF8JDK9(context);
}
} else {
jsonWriter = new JSONWriterUTF8(context);
}

return jsonWriter;
return STRING_VALUE != null
? new JSONWriterUTF8JDK9(context)
: new JSONWriterUTF8(context);
}

public static JSONWriter ofUTF8(Feature... features) {
Context context = createWriteContext(features);

JSONWriter jsonWriter;
if (STRING_VALUE != null) {
if (INCUBATOR_VECTOR_WRITER_CREATOR_UTF8 != null) {
jsonWriter = INCUBATOR_VECTOR_WRITER_CREATOR_UTF8.apply(context);
} else {
jsonWriter = new JSONWriterUTF8JDK9(context);
}
} else {
jsonWriter = new JSONWriterUTF8(context);
}

return jsonWriter;
return ofUTF8(
createWriteContext(features)
);
}

public void writeBinary(byte[] bytes) {
Expand Down
47 changes: 39 additions & 8 deletions core/src/main/java/com/alibaba/fastjson2/JSONWriterUTF8.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ class JSONWriterUTF8

final CacheItem cacheItem;
protected byte[] bytes;
protected final long byteVectorQuote;

JSONWriterUTF8(Context ctx) {
super(ctx, null, false, StandardCharsets.UTF_8);
Expand All @@ -62,6 +63,7 @@ class JSONWriterUTF8
bytes = new byte[8192];
}
this.bytes = bytes;
this.byteVectorQuote = this.useSingleQuote ? 0x2727_2727_2727_2727L : 0x2222_2222_2222_2222L;
}

public final void writeNull() {
Expand Down Expand Up @@ -479,16 +481,24 @@ public void writeString(String str) {
this.bytes[this.off++] = (byte) quote;
}

public void writeStringLatin1(byte[] values) {
public void writeStringLatin1(byte[] value) {
boolean escape = false;
if ((context.features & BrowserSecure.mask) != 0) {
writeStringLatin1BrowserSecure(values);
writeStringLatin1BrowserSecure(value);
return;
}

final byte quote = (byte) this.quote;
for (int i = 0; i < values.length; i++) {
byte c = values[i];
final long vecQuote = this.byteVectorQuote;
int i = 0;
final int upperBound = (value.length - i) & ~7;
for (; i < upperBound; i += 8) {
if (containsEscaped(IOUtils.getLongLittleEndian(value, i), vecQuote)) {
break;
}
}
for (; i < value.length; i++) {
byte c = value[i];
if (c == quote || c == '\\' || c < ' ') {
escape = true;
break;
Expand All @@ -497,19 +507,40 @@ public void writeStringLatin1(byte[] values) {

int off = this.off;
if (!escape) {
int minCapacity = off + values.length + 2;
int minCapacity = off + value.length + 2;
byte[] bytes = this.bytes;
if (minCapacity > bytes.length) {
bytes = grow(minCapacity);
}
bytes[off] = quote;
System.arraycopy(values, 0, bytes, off + 1, values.length);
off += values.length + 1;
System.arraycopy(value, 0, bytes, off + 1, value.length);
off += value.length + 1;
bytes[off] = quote;
this.off = off + 1;
return;
}
writeStringEscaped(values);
writeStringEscaped(value);
}

private static boolean containsEscaped(long data, long quote) {
// c == quote || c == '\\' || c < ' '
/*
for (int i = 0; i < 8; ++i) {
byte c = (byte) data;
if (c == quote || c == '\\' || c < ' ') {
return true;
}
data >>>= 8;
}
return false;
*/
long xed = data ^ quote;
long xf0 = data ^ 0x5c5c5c5c5c5c5c5cL;

xed = (xed - 0x0101010101010101L) & ~xed;
xf0 = (xf0 - 0x0101010101010101L) & ~xf0;

return ((xed | xf0 | (0x7F7F_7F7F_7F7F_7F7FL - data + 0x1010_1010_1010_1010L) | data) & 0x8080808080808080L) != 0;
}

protected final void writeStringLatin1BrowserSecure(byte[] values) {
Expand Down
8 changes: 8 additions & 0 deletions core/src/main/java/com/alibaba/fastjson2/util/IOUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1619,4 +1619,12 @@ public static long getLongBigEndian(byte[] bytes, int offset) {
}
return v;
}

public static long getLongLittleEndian(byte[] bytes, int offset) {
long v = UNSAFE.getLong(bytes, ARRAY_BYTE_BASE_OFFSET + offset);
if (BIG_ENDIAN) {
v = Long.reverseBytes(v);
}
return v;
}
}
9 changes: 9 additions & 0 deletions core/src/test/java/com/alibaba/fastjson2/JSONWriterUTF8Test.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1103,4 +1103,13 @@ public void grow() {
}
}
}

@Test
public void testSpecial() {
assertEquals("'\\'01234567890123456789'", new String(JSON.toJSONBytes("'01234567890123456789", UseSingleQuotes)));
assertEquals("\"'01234567890123456789\"", new String(JSON.toJSONBytes("'01234567890123456789")));

assertEquals("\"\\\"01234567890123456789\"", new String(JSON.toJSONBytes("\"01234567890123456789")));
assertEquals("'\"01234567890123456789'", new String(JSON.toJSONBytes("\"01234567890123456789", UseSingleQuotes)));
}
}
121 changes: 121 additions & 0 deletions core/src/test/java/com/alibaba/fastjson2/util/IOUtilsTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.alibaba.fastjson2.util;

import com.alibaba.fastjson2.JSONException;
import org.apache.arrow.flatbuf.Int;
import org.junit.jupiter.api.Test;

import java.nio.charset.StandardCharsets;
Expand Down Expand Up @@ -373,4 +374,124 @@ public void indexOf() throws Throwable {
IOUtils.indexOfChar(
bytes, 'a', 1));
}

@Test
public void x0() {
for (int i = 0; i < 256; i++) {
byte b = (byte) i;
if (b < ' ') {
System.out.println(i + "\t" + Integer.toHexString(i));
}
}
}

@Test
public void xor() {
for (int i = 0; i < 256; i++) {
byte b = (byte) i;
// if ((b ^ 0x22) != 0 && (b ^ 0x27) != 0 && (b ^ 0x5c) != 0) {
// continue;
// }
// if (((i & 0x80) != 0)) {
// continue;
// }
////
// System.out.println(i + "\t" + Integer.toHexString(i) + "\t" + Integer.toHexString((0x70 - i + 0x30) & 0x80) + "\t" + (char) i);
//
// if (contains(i, 0x22)) {
// System.out.println(i + "\t" + Integer.toHexString(i) + "\t" + (char) i);
// }
// if ((b ^ 0x22) == 0) {
// System.out.println(i + "\t" + Integer.toHexString(i) + "\t" + (char) i);
// continue;
// }
if (isSpecial(i)) {
continue;
}

System.out.println(i + "\t" + Integer.toHexString(i) + "\t" + (char) i);
}
}

@Test
public void xor_0() {
for (int i = 0; i < 256; i++) {
if (((i & 0x80) != 0)) {
continue;
}

int x = i >> 4;

System.out.println(i + "\t" + Integer.toHexString(x) + "\t" + Integer.toHexString((((0x7 - x + 0x3) & 0x8))) + "\t" + (char) i);
}
}

public static boolean isSpecial(long data) {
long xed = data ^ 0x2222_2222_2222_2222L;
long xf0 = data ^ 0x5c5c5c5c5c5c5c5cL;

xed = (xed - 0x0101010101010101L) & ~xed;
xf0 = (xf0 - 0x0101010101010101L) & ~xf0;

return ((xed | xf0 | (0x7F7F_7F7F_7F7F_7F7FL - data + 0x1010_1010_1010_1010L) | data) & 0x8080808080808080L) != 0;
}

public static boolean isSpecial1(long data) {
// 在Java中，long是有符号的，所以我们需要确保我们正确处理高字节。
long xed = data ^ 0xededededededededL;
long xf0 = (data | 0x1010101010101010L) ^ 0xf0f0f0f0f0f0f0f0L;
long xf4 = data ^ 0xf4f4f4f4f4f4f4f4L;

// Java没有&^运算符，所以用~和&来模拟
xed = (xed - 0x0101010101010101L) & ~xed;
xf0 = (xf0 - 0x0101010101010101L) & ~xf0;
xf4 = (xf4 - 0x0101010101010101L) & ~xf4;

return ((xed | xf0 | xf4) & 0x8080808080808080L) != 0;
}


public static boolean contains(long data, long mask) {
// Perform XOR on data and mask
data ^= mask;

// Subtract the magic number from data
// Use a mask to emulate unsigned behavior for the highest bit
long magic = 0x0101010101010101L;
long highBitMask = 0x8080808080808080L;
long result = (data - magic) & ~data & highBitMask;

// Return true if any byte in the result is non-zero
return result != 0;
}



@Test
public void xor_1() {
long MASK = 0x2222_2222_2222_2222L;
long[] values = {
0x1522_1314_1522_1314L,
0x1534_1534_1534_1534L,
MASK,
};
for (long i : values) {
System.out.println(Long.toHexString(i) + "\t" + isSpecial(i));
}
}

@Test
public void x6() {
for (int i = 0; i < 6; i++) {
int x = i << 4;
System.out.println(i + "\t" + x + "\t" + (x ^ 0x70) + "\t" + (((x ^ 0x70) + 0x20) & 0x80));
}
}

@Test
public void hex() {
for (int i = 0; i < 16; i++) {
System.out.println(i + "\t" + Integer.toHexString(i) + "\t" + Integer.toBinaryString(i));
}
}
}
Loading

0 comments on commit f524539

Please sign in to comment.