move compress native into plugin repo (#604)

* move compress native into plugin repo

Signed-off-by: [email protected] <[email protected]>

* spotless and add a missing header

Signed-off-by: [email protected] <[email protected]>

* some cleanup

Signed-off-by: [email protected] <[email protected]>

---------

Signed-off-by: [email protected] <[email protected]>

acceptance-tests/build.gradle

@@ -40,6 +40,7 @@ solidity {
 tasks.register('acceptanceTests', Test) {
   description = 'Runs acceptance tests.'
 
+  dependsOn(project(":native:compress").compileJava)
   dependsOn(project(":arithmetization").copyPluginJar)
 
   inputs.property("integration.date", LocalTime.now()) // so it runs on every invocation
@@ -55,6 +56,7 @@ tasks.register('acceptanceTests', Test) {
 
 dependencies {
   testImplementation project(":arithmetization")
+  testImplementation project(":native:compress")
   testImplementation "${besuArtifactGroup}.internal:dsl:$besuVersion"
   testImplementation "${besuArtifactGroup}:besu-datatypes:$besuVersion"
   testImplementation "${besuArtifactGroup}.internal:eth:$besuVersion"

acceptance-tests/src/test/java/linea/plugin/acc/test/compress/CompressTest.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright Consensys Software Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package linea.plugin.acc.test.compress;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+import net.consensys.linea.compress.LibCompress;
+import org.hyperledger.besu.tests.acceptance.dsl.AcceptanceTestBase;
+import org.junit.jupiter.api.Test;
+
+public class CompressTest extends AcceptanceTestBase {
+
+  @Test
+  public void testCompress() {
+    final int compressedSize = LibCompress.CompressedSize(new byte[128], 128);
+
+    // should not error
+    assertThat(compressedSize).isGreaterThan(0);
+
+    // should have compressed into 1 backref + header, must be less than 10
+    assertThat(compressedSize).isLessThan(10);
+  }
+}

arithmetization/build.gradle

@@ -47,18 +47,18 @@ dependencies {
   implementation "${besuArtifactGroup}.internal:core:${besuVersion}"
   implementation "${besuArtifactGroup}.internal:rlp:${besuVersion}"
   implementation "${besuArtifactGroup}.internal:algorithms:${besuVersion}"
+  implementation project(":native:compress")
 
   implementation 'info.picocli:picocli'
 
-  implementation 'io.tmio:tuweni-toml'
-
   compileOnly 'io.vertx:vertx-core'
 
   implementation 'com.fasterxml.jackson.core:jackson-databind'
   implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
 
   implementation 'io.tmio:tuweni-bytes'
   implementation 'io.tmio:tuweni-units'
+  implementation 'io.tmio:tuweni-toml'
 
   implementation 'org.bouncycastle:bcprov-jdk18on'
   implementation 'org.hibernate.validator:hibernate-validator'

build.gradle

@@ -1,3 +1,4 @@
+
 buildscript {
   ext {
     besuIdentifier = "linea-besu-${besuVersion}"

gradle/dependency-management.gradle

@@ -100,8 +100,8 @@ dependencyManagement {
     }
 
     dependency 'junit:junit:4.13.2'
-    dependency 'org.junit.vintage:junit-vintage-engine:5.10.0'
-    dependencySet(group: 'org.junit.jupiter', version: '5.10.0') {
+    dependency 'org.junit.vintage:junit-vintage-engine:5.10.2'
+    dependencySet(group: 'org.junit.jupiter', version: '5.10.2') {
       entry 'junit-jupiter-api'
       entry 'junit-jupiter-engine'
       entry 'junit-jupiter-params'
@@ -121,5 +121,7 @@ dependencyManagement {
     dependency 'com.slack.api:slack-api-client:1.32.1'
 
     dependency 'org.hibernate.validator:hibernate-validator:8.0.1.Final'
+
+    dependency 'net.java.dev.jna:jna:5.12.1'
   }
 }

gradle/versions.gradle

@@ -0,0 +1 @@
+dependency 'net.java.dev.jna:jna:5.14.0'

native/build.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+# Initialize external vars - need this to get around unbound variable errors
+SKIP_GRADLE="$SKIP_GRADLE"
+
+# Exit script if you try to use an uninitialized variable.
+set -o nounset
+
+# Exit script if a statement returns a non-true return value.
+set -o errexit
+
+# Use the error status of the first failure, rather than that of the last item in a pipeline.
+set -o pipefail
+
+# Resolve the directory that contains this script. We have to jump through a few
+# hoops for this because the usual one-liners for this don't work if the script
+# is a symlink
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )"
+
+# Determine core count for parallel make
+if [[ "$OSTYPE" == "linux-gnu" ]];  then
+  CORE_COUNT=$(nproc)
+  OSARCH=${OSTYPE%%[0-9.]*}-`arch`
+fi
+
+if [[ "$OSTYPE" == "darwin"* ]];  then
+  CORE_COUNT=$(sysctl -n hw.ncpu)
+  if [[ "`machine`" == "arm"* ]]; then
+    arch_name="aarch64"
+    export CFLAGS="-arch arm64"
+  else
+    arch_name="x86-64"
+    export CFLAGS="-arch x86_64"
+  fi
+  OSARCH="darwin-$arch_name"
+fi
+
+# add to path cargo
+[ -f $HOME/.cargo/env ] && . $HOME/.cargo/env
+
+# add to path brew
+[ -f $HOME/.zprofile ] && . $HOME/.zprofile
+
+build_compress() {
+  cat <<EOF
+  ############################
+  ####### build compress #######
+  ############################
+EOF
+
+  cd "$SCRIPTDIR/compress/compress-jni"
+
+  # delete old build dir, if exists
+  rm -rf "$SCRIPTDIR/compress/build" || true
+  mkdir -p "$SCRIPTDIR/compress/build/lib"
+
+  if [[ "$OSTYPE" == "msys" ]]; then
+    	LIBRARY_EXTENSION=dll
+  elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
+    LIBRARY_EXTENSION=so
+  elif [[ "$OSTYPE" == "darwin"* ]]; then
+    LIBRARY_EXTENSION=dylib
+  fi
+
+  go build -buildmode=c-shared -o libcompress_jni.$LIBRARY_EXTENSION compress-jni.go
+
+  mkdir -p "$SCRIPTDIR/compress/build/${OSARCH}/lib"
+  cp libcompress_jni.* "$SCRIPTDIR/compress/build/${OSARCH}/lib"
+}
+
+build_compress
+
+exit

native/compress/build.gradle

@@ -0,0 +1,103 @@
+/*
+ * Copyright Hyperledger Besu Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+plugins {
+    id 'java-library'
+    id 'maven-publish'
+    id 'com.jfrog.artifactory' version '4.20.0'
+}
+
+apply from: rootProject.file("gradle/dependency-management.gradle")
+
+repositories {
+    mavenCentral()
+}
+
+test {
+    useJUnitPlatform()
+}
+
+tasks.register('callBuildSh', Exec) {
+    workingDir buildscript.sourceFile.parentFile
+    commandLine 'bash', '../build.sh'
+}
+
+compileJava{
+    dependsOn callBuildSh
+}
+
+dependencies {
+    implementation 'net.java.dev.jna:jna'
+    implementation 'io.tmio:tuweni-bytes'
+
+    testImplementation 'org.junit.jupiter:junit-jupiter-api'
+    testImplementation 'org.junit.jupiter:junit-jupiter-engine'
+    testImplementation 'org.assertj:assertj-core'
+    testImplementation 'org.mockito:mockito-core'
+}
+
+
+tasks.register('macArmLibCopy', Copy) {
+    from 'build/darwin-aarch64/lib/libcompress_jni.dylib'
+    into 'build/resources/main/darwin-aarch64'
+}
+processResources.dependsOn macArmLibCopy
+
+tasks.register('macLibCopy', Copy) {
+    from 'build/darwin-x86-64/lib/libcompress_jni.dylib'
+    into 'build/resources/main/darwin-x86-64'
+}
+processResources.dependsOn macLibCopy
+
+tasks.register('linuxLibCopy', Copy) {
+    from 'build/linux-gnu-x86_64/lib/libcompress_jni.so'
+    into 'build/resources/main/linux-x86-64'
+}
+processResources.dependsOn linuxLibCopy
+
+tasks.register('linuxArm64LibCopy', Copy) {
+    from 'build/linux-gnu-aarch64/lib/libcompress_jni.so'
+    into 'build/resources/main/linux-aarch64'
+}
+processResources.dependsOn linuxArm64LibCopy
+
+jar {
+    archiveBaseName = 'linea-native-compress'
+    includeEmptyDirs = false
+    manifest {
+        attributes(
+                'Specification-Title': archiveBaseName,
+                'Specification-Version': project.version,
+                'Implementation-Title': archiveBaseName,
+                'Implementation-Version': project.version,
+                'Automatic-Module-Name': 'org.hyperledger.besu.nativelib.compress'
+        )
+    }
+}
+
+jar.dependsOn(callBuildSh)
+
+sourceSets {
+    main {
+        resources {
+            srcDirs '${buildDir}/darwin-aaarch64'
+            srcDirs '${buildDir}/darwin-x86-64'
+            srcDirs '${buildDir}/linux-gnu-x86_64'
+            srcDirs '${buildDir}/linux-gnu-aarch64'
+        }
+    }
+}
+
+

native/compress/compress-jni/compress-jni.go

@@ -0,0 +1,87 @@
+package main
+
+import "C"
+import (
+	"errors"
+	"os"
+	"sync"
+	"unsafe"
+
+	"github.com/consensys/compress/lzss"
+)
+
+var (
+	compressor *lzss.Compressor
+	lastError  error // last error that occurred
+	lock       sync.Mutex
+)
+
+const compressionLevel = lzss.BestCompression
+
+// Init initializes the compressor.
+// Returns true if the compressor was initialized, false otherwise.
+// If false is returned, the Error() method will return a string describing the error.
+//
+//export Init
+func Init(dictPath *C.char) bool {
+	fPath := C.GoString(dictPath)
+	return initGo(fPath)
+}
+
+func initGo(dictPath string) bool {
+	lock.Lock()
+	defer lock.Unlock()
+
+	// read the dictionary
+	dict, err := os.ReadFile(dictPath)
+	if err != nil {
+		lastError = err
+		return false
+	}
+
+	compressor, lastError = lzss.NewCompressor(dict, compressionLevel)
+
+	return lastError == nil
+}
+
+// Compress compresses the input and returns the length of the compressed data.
+// If an error occurred, returns -1.
+// User must call Error() to get the error message.
+//
+// This function is thread-safe.
+//
+//export CompressedSize
+func CompressedSize(input *C.char, inputLength C.int) C.int {
+	inputSlice := C.GoBytes(unsafe.Pointer(input), inputLength)
+
+	n, err := compressor.CompressedSize256k(inputSlice)
+	if err != nil {
+		lock.Lock()
+		lastError = errors.Join(lastError, err)
+		lock.Unlock()
+		return -1
+	}
+	if n > len(inputSlice) {
+		// this simulates the fallback to "no compression"
+		// this case may happen if the input is not compressible
+		// in which case the compressed size is the input size + the header size
+		n = len(inputSlice) + lzss.HeaderSize
+	}
+	return C.int(n)
+}
+
+// Error returns the last encountered error.
+// If no error was encountered, returns nil.
+//
+//export Error
+func Error() *C.char {
+	lock.Lock()
+	defer lock.Unlock()
+	if lastError != nil {
+		// this leaks memory, but since this represents a fatal error, it's probably ok.
+		return C.CString(lastError.Error())
+	}
+	return nil
+}
+
+func main() {}

native/compress/compress-jni/go.mod

@@ -0,0 +1,7 @@
+module github.com/consensys/besu-sequencer-plugins
+
+go 1.20
+
+require github.com/consensys/compress v0.2.4
+
+require github.com/icza/bitio v1.1.0 // indirect

native/compress/compress-jni/go.sum

@@ -0,0 +1,10 @@
+github.com/consensys/compress v0.2.4 h1:d6WGZtlrdBCZjGZ76xO236JvYhU1zbb7zhQ1uHdLRoI=
+github.com/consensys/compress v0.2.4/go.mod h1:Ne8+cGKjqgjF1dlHapZx38pHzWpaBYhsKxQa+JPl0zM=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/icza/bitio v1.1.0 h1:ysX4vtldjdi3Ygai5m1cWy4oLkhWTAi+SyO6HC8L9T0=
+github.com/icza/bitio v1.1.0/go.mod h1:0jGnlLAx8MKMr9VGnn/4YrvZiprkvBelsVIbA9Jjr9A=
+github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lToqnXgA8Mz1DP11X4zSJ159C3k=
+github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=