More perf work

pbj-core/buildSrc/src/main/kotlin/com.hedera.pbj.spotless-java-conventions.gradle.kts

@@ -25,7 +25,7 @@ spotless {
         toggleOffOn()
         // don't need to set target, it is inferred from java
         // apply a specific flavor of google-java-format
-        googleJavaFormat("1.15.0").aosp().reflowLongStrings()
+        palantirJavaFormat()
         // make sure every file has the following copyright header.
         // optionally, Spotless can set copyright years by digging
         // through git history (see "license" section below).

pbj-core/pbj-runtime/build.gradle.kts

@@ -23,6 +23,8 @@ plugins {
 }
 
 dependencies {
+    implementation("org.eclipse.collections:eclipse-collections-api:11.1.0")
+    implementation("org.eclipse.collections:eclipse-collections:11.1.0")
     compileOnly(libs.spotbugs.annotations)
     testImplementation(testLibs.bundles.protobuf)
     testImplementation(testLibs.bundles.junit)

pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoParserTools.java

@@ -2,6 +2,8 @@
 
 import com.hedera.pbj.runtime.io.Bytes;
 import com.hedera.pbj.runtime.io.DataInput;
+import org.eclipse.collections.impl.list.mutable.FastList;
+import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList;
 
 import java.io.IOException;
 import java.nio.ByteOrder;
@@ -42,7 +44,11 @@
      */
     public static <T> List<T> addToList(List<T> list, T newItem) {
         if (list == Collections.EMPTY_LIST) {
-            list = new ArrayList<>();
+//            if (newItem instanceof Integer) {
+//                list = new FastList();
+//            }
+//            list = new ArrayList<>(20);
+            list = new FastList<>(20);
         }
         list.add(newItem);
         return list;
@@ -215,9 +221,11 @@
      */
     public static String readString(final DataInput input) throws IOException {
         final int length = input.readVarInt(false);
+        if (length == 0) return "";
         byte[] bytes = new byte[length];
         input.readBytes(bytes);
         return new String(bytes,StandardCharsets.UTF_8);
+//        return input.readUtf8String(length);
     }
 
     /**

pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoTestTools.java

@@ -25,11 +25,11 @@ public final class ProtoTestTools {
     private ProtoTestTools() {}
     /** Thread local set of reusable buffers */
     private static final ThreadLocal<DataBuffer> THREAD_LOCAL_BUFFERS =
-            ThreadLocal.withInitial(() -> DataBuffer.allocate(BUFFER_SIZE, false));
+            ThreadLocal.withInitial(() -> DataBuffer.allocate(BUFFER_SIZE, true));
 
     /** Thread local set of reusable buffers, second buffer for each thread */
     private static final ThreadLocal<DataBuffer> THREAD_LOCAL_BUFFERS_2 =
-            ThreadLocal.withInitial(() -> DataBuffer.allocate(BUFFER_SIZE, false));
+            ThreadLocal.withInitial(() -> DataBuffer.allocate(BUFFER_SIZE, true));
 
     /**
      * Get the thread local instance of DataBuffer, reset and ready to use.

pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoWriterTools.java

@@ -13,7 +13,7 @@
 /**
  * Static helper methods for Writers
  */
-@SuppressWarnings({"DuplicatedCode", "OptionalUsedAsFieldOrParameterType"})
+@SuppressWarnings({"DuplicatedCode", "OptionalUsedAsFieldOrParameterType", "ForLoopReplaceableByForEach"})
 public final class ProtoWriterTools {
 
     /** The number of leading bits of the tag that are used to store field type, the rest is field number */
@@ -222,7 +222,7 @@ public static void writeString(DataOutput out, FieldDefinition field, String val
      */
     private static void writeStringNoChecks(DataOutput out, FieldDefinition field, String value) throws IOException {
         // When not a oneOf don't write default value
-        if (!field.oneOf() && (value == null || value.isBlank())) {
+        if (!field.oneOf() && (value == null || value.isEmpty())) {
             return;
         }
         writeTag(out, field, ProtoConstants.WIRE_TYPE_DELIMITED);
@@ -460,47 +460,55 @@ assert switch(field.type()) {
             return;
         }
 
+        final int listSize = list.size();
         switch (field.type()) {
             case INT32 -> {
                 int size = 0;
-                for (final int i : list) {
-                    size += sizeOfVarInt32(i);
+                for (int i = 0; i < listSize; i++) {
+                    final int value = list.get(i);
+                    size += sizeOfVarInt32(value);
                 }
                 writeTag(out, field, ProtoConstants.WIRE_TYPE_DELIMITED);
                 out.writeVarInt(size, false);
-                for (final int i : list) {
-                    out.writeVarInt(i, false);
+                for (int i = 0; i < list.size(); i++) {
+                    final int  value = list.get(i);
+                    out.writeVarInt(value, false);
                 }
             }
             case UINT32 -> {
                 int size = 0;
-                for (final int i : list) {
-                    size += sizeOfUnsignedVarInt64(Integer.toUnsignedLong(i));
+                for (int i = 0; i < listSize; i++) {
+                    final int value = list.get(i);
+                    size += sizeOfUnsignedVarInt64(Integer.toUnsignedLong(value));
                 }
                 writeTag(out, field, ProtoConstants.WIRE_TYPE_DELIMITED);
                 out.writeVarInt(size, false);
-                for (final int i : list) {
-                    out.writeVarLong(Integer.toUnsignedLong(i), false);
+                for (int i = 0; i < listSize; i++) {
+                    final int value = list.get(i);
+                    out.writeVarLong(Integer.toUnsignedLong(value), false);
                 }
             }
             case SINT32 -> {
                 int size = 0;
-                for (final int i : list) {
-                    size += sizeOfUnsignedVarInt64(((long)i << 1) ^ ((long)i >> 63));
+                for (int i = 0; i < listSize; i++) {
+                    final int value = list.get(i);
+                    size += sizeOfUnsignedVarInt64(((long)value << 1) ^ ((long)value >> 63));
                 }
                 writeTag(out, field, ProtoConstants.WIRE_TYPE_DELIMITED);
                 out.writeVarInt(size, false);
-                for (final int i : list) {
-                    out.writeVarInt(i, true);
+                for (int i = 0; i < listSize; i++) {
+                    final int value = list.get(i);
+                    out.writeVarInt(value, true);
                 }
             }
             case SFIXED32, FIXED32 -> {
                 // The bytes in protobuf are in little-endian order -- backwards for Java.
                 // Smallest byte first.
                 writeTag(out, field, ProtoConstants.WIRE_TYPE_DELIMITED);
                 out.writeVarLong((long)list.size() * FIXED32_SIZE, false);
-                for (final int i : list) {
-                    out.writeInt(i, ByteOrder.LITTLE_ENDIAN);
+                for (int i = 0; i < listSize; i++) {
+                    final int value = list.get(i);
+                    out.writeInt(value, ByteOrder.LITTLE_ENDIAN);
                 }
             }
             default -> throw unsupported();
@@ -527,36 +535,42 @@ assert switch(field.type()) {
             return;
         }
 
+        final int listSize = list.size();
         switch (field.type()) {
             case INT64, UINT64 -> {
                 int size = 0;
-                for (final long i : list) {
-                    size += sizeOfUnsignedVarInt64(i);
+                for (int i = 0; i < listSize; i++) {
+                    final long value = list.get(i);
+                    size += sizeOfUnsignedVarInt64(value);
                 }
                 writeTag(out, field, ProtoConstants.WIRE_TYPE_DELIMITED);
                 out.writeVarInt(size, false);
-                for (final long i : list) {
-                    out.writeVarLong(i, false);
+                for (int i = 0; i < listSize; i++) {
+                    final long value = list.get(i);
+                    out.writeVarLong(value, false);
                 }
             }
             case SINT64 -> {
                 int size = 0;
-                for (final long i : list) {
-                    size += sizeOfUnsignedVarInt64((i << 1) ^ (i >> 63));
+                for (int i = 0; i < listSize; i++) {
+                    final long value = list.get(i);
+                    size += sizeOfUnsignedVarInt64((value << 1) ^ (value >> 63));
                 }
                 writeTag(out, field, ProtoConstants.WIRE_TYPE_DELIMITED);
                 out.writeVarInt(size, false);
-                for (final long i : list) {
-                    out.writeVarLong(i, true);
+                for (int i = 0; i < listSize; i++) {
+                    final long value = list.get(i);
+                    out.writeVarLong(value, true);
                 }
             }
             case SFIXED64, FIXED64 -> {
                 // The bytes in protobuf are in little-endian order -- backwards for Java.
                 // Smallest byte first.
                 writeTag(out, field, ProtoConstants.WIRE_TYPE_DELIMITED);
                 out.writeVarLong((long)list.size() * FIXED64_SIZE, false);
-                for (final long i : list) {
-                    out.writeLong(i, ByteOrder.LITTLE_ENDIAN);
+                for (int i = 0; i < listSize; i++) {
+                    final long value = list.get(i);
+                    out.writeLong(value, ByteOrder.LITTLE_ENDIAN);
                 }
             }
             default -> throw unsupported();
@@ -581,8 +595,9 @@ public static void writeFloatList(DataOutput out, FieldDefinition field, List<Fl
         final int size = list.size() * FIXED32_SIZE;
         writeTag(out, field, ProtoConstants.WIRE_TYPE_DELIMITED);
         out.writeVarInt(size, false);
-        for (final Float i : list) {
-            out.writeFloat(i, ByteOrder.LITTLE_ENDIAN);
+        final int listSize = list.size();
+        for (int i = 0; i < listSize; i++) {
+            out.writeFloat(list.get(i), ByteOrder.LITTLE_ENDIAN);
         }
     }
 
@@ -604,8 +619,9 @@ public static void writeDoubleList(DataOutput out, FieldDefinition field, List<D
         final int size = list.size() * FIXED64_SIZE;
         writeTag(out, field, ProtoConstants.WIRE_TYPE_DELIMITED);
         out.writeVarInt(size, false);
-        for (final Double i : list) {
-            out.writeDouble(i, ByteOrder.LITTLE_ENDIAN);
+        final int listSize = list.size();
+        for (int i = 0; i < listSize; i++) {
+            out.writeDouble(list.get(i), ByteOrder.LITTLE_ENDIAN);
         }
     }
 
@@ -627,7 +643,9 @@ public static void writeBooleanList(DataOutput out, FieldDefinition field, List<
         // write
         writeTag(out, field, ProtoConstants.WIRE_TYPE_DELIMITED);
         out.writeVarInt(list.size(), false);
-        for (final boolean b : list) {
+        final int listSize = list.size();
+        for (int i = 0; i < listSize; i++) {
+            final boolean b = list.get(i);
             out.writeVarInt(b ? 1 : 0, false);
         }
     }
@@ -647,14 +665,15 @@ public static void writeEnumList(DataOutput out, FieldDefinition field, List<? e
         if (!field.oneOf() && list.isEmpty()) {
             return;
         }
+        final int listSize = list.size();
         int size = 0;
-        for (final EnumWithProtoOrdinal enumValue : list) {
-            size += sizeOfUnsignedVarInt32(enumValue.protoOrdinal());
+        for (int i = 0; i < listSize; i++) {
+            size += sizeOfUnsignedVarInt32(list.get(i).protoOrdinal());
         }
         writeTag(out, field, ProtoConstants.WIRE_TYPE_DELIMITED);
         out.writeVarInt(size, false);
-        for (final EnumWithProtoOrdinal enumValue : list) {
-            out.writeVarInt(enumValue.protoOrdinal(), false);
+        for (int i = 0; i < listSize; i++) {
+            out.writeVarInt(list.get(i).protoOrdinal(), false);
         }
     }
 
@@ -673,7 +692,9 @@ public static void writeStringList(DataOutput out, FieldDefinition field, List<S
         if (!field.oneOf() && list.isEmpty()) {
             return;
         }
-        for (final String value : list) {
+        final int listSize = list.size();
+        for (int i = 0; i < listSize; i++) {
+            final String value = list.get(i);
             writeTag(out, field, ProtoConstants.WIRE_TYPE_DELIMITED);
             out.writeVarInt(sizeOfStringNoTag(field,value), false);
             Utf8Tools.encodeUtf8(value,out);
@@ -698,8 +719,9 @@ public static <T> void writeMessageList(DataOutput out, FieldDefinition field, L
         if (!field.oneOf() && list.isEmpty()) {
             return;
         }
-        for (final T value : list) {
-            writeMessageNoChecks(out, field, value, writer, sizeOf);
+        final int listSize = list.size();
+        for (int i = 0; i < listSize; i++) {
+            writeMessageNoChecks(out, field, list.get(i), writer, sizeOf);
         }
     }
 
@@ -718,8 +740,9 @@ public static void writeBytesList(DataOutput out, FieldDefinition field, List<By
         if (!field.oneOf() && list.isEmpty()) {
             return;
         }
-        for (final Bytes value : list) {
-            writeBytesNoChecks(out, field, value, false);
+        final int listSize = list.size();
+        for (int i = 0; i < listSize; i++) {
+            writeBytesNoChecks(out, field, list.get(i), false);
         }
     }
 
@@ -1002,7 +1025,7 @@ public static int sizeOfEnum(FieldDefinition field, EnumWithProtoOrdinal enumVal
      */
     public static int sizeOfString(FieldDefinition field, String value) {
         // When not a oneOf don't write default value
-        if (!field.oneOf() && (value == null || value.isBlank())) {
+        if (!field.oneOf() && (value == null || value.isEmpty())) {
             return 0;
         }
         final int size = sizeOfStringNoTag(field, value);
@@ -1018,7 +1041,7 @@ public static int sizeOfString(FieldDefinition field, String value) {
      */
     private static int sizeOfStringNoTag(FieldDefinition field, String value) {
         // When not a oneOf don't write default value
-        if (!field.oneOf() && (value == null || value.isBlank())) {
+        if (!field.oneOf() && (value == null || value.isEmpty())) {
             return 0;
         }
         try {

pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/ByteOverByteBuffer.java

@@ -44,7 +44,23 @@ public ByteOverByteBuffer(byte[] data) {
         this.length = data.length;
     }
 
+    /**
+     * Package protected direct access to buffer
+     *
+     * @return the internal buffer
+     */
+    ByteBuffer getBuffer() {
+        return buffer;
+    }
 
+    /**
+     * Package protected direct access to start offset
+     *
+     * @return the internal start offset
+     */
+    int getStart() {
+        return start;
+    }
 
     // ================================================================================================================
     // Bytes Methods

pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/DataBuffer.java

@@ -4,6 +4,8 @@
 import java.nio.BufferUnderflowException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 
 /**
  * A Buffer backed by a ByteBuffer that implements {@code DataInput} and {@code DataOutput}.
@@ -76,6 +78,13 @@ public static DataBuffer allocate(int size, boolean offHeap) {
     // ================================================================================================================
     // DataOutput Methods
 
+
+    static String bufToString(ByteBuffer buf) {
+        byte[] b = new byte[Math.min(30,buf.remaining())];
+        buf.get(0, b);
+        return "ByteBuffer{direct="+buf.isDirect()+",pos="+buf.position()+",remaining="+buf.remaining()+" ,d="+ Arrays.toString(b)+"}";
+    }
+
     /**
      * Set the limit to current position and position to origin. This is useful when you have just finished writing
      * into a buffer and want to flip it ready to read back from.
@@ -468,6 +477,23 @@ public long readVarLong(boolean zigZag) throws IOException {
         return zigZag ? (x >>> 1) ^ -(x & 1) : x;
     }
 
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public String readUtf8String(int lengthInBytes) throws IOException {
+//        if (buffer.remaining() < lengthInBytes) throw new IOException("Not enough bytes to remaining [" +
+//                buffer.remaining() + "] to read string of [" + lengthInBytes + "] bytes");
+//        int oldLimit = buffer.limit();
+//        buffer.limit(buffer.position()+lengthInBytes);
+//        final String readStr = StandardCharsets.UTF_8.decode(buffer).toString();
+//        buffer.limit(oldLimit);
+//        return readStr;
+        final String readStr = new String(buffer.array(), buffer.position(), lengthInBytes, StandardCharsets.UTF_8);
+        buffer.position(buffer.position() + lengthInBytes);
+        return readStr;
+    }
+
     // ================================================================================================================
     // DataOutput Write Methods
 
@@ -508,8 +534,6 @@ public void writeBytes(byte[] src) {
     @Override
     public void writeBytes(DataBuffer src) throws IOException {
         if ((getLimit() - getPosition()) < src.getRemaining()) {
-            System.err.println("Trying to write [" + src.getRemaining() + "] bytes but only [" +
-                    (getLimit() - getPosition()) + "] remaining of [" + getCapacity() + "]");
             throw new BufferUnderflowException();
         }
         buffer.put(src.buffer);