Add Lance connector

.github/workflows/ci.yml

@@ -351,6 +351,7 @@ jobs:
             !:trino-jdbc,
             !:trino-kafka,
             !:trino-lakehouse,
+            !:trino-lance,
             !:trino-main,
             !:trino-mariadb,
             !:trino-memory,
@@ -472,6 +473,7 @@ jobs:
             - { modules: plugin/trino-ignite }
             - { modules: plugin/trino-kafka }
             - { modules: plugin/trino-lakehouse }
+            - { modules: plugin/trino-lance }
             - { modules: plugin/trino-mariadb }
             - { modules: plugin/trino-mongodb }
             - { modules: plugin/trino-mysql }
@@ -893,6 +895,7 @@ jobs:
             - suite-storage-formats-detailed
             - suite-parquet
             - suite-oauth2
+            - suite-lance
             - suite-ldap
             - suite-compatibility
             - suite-all-connectors-smoke

core/trino-server/src/main/provisio/trino.xml

@@ -157,6 +157,12 @@
         </artifact>
     </artifactSet>
 
+    <artifactSet to="plugin/lance">
+        <artifact id="${project.groupId}:trino-lance:zip:${project.version}">
+            <unpack />
+        </artifact>
+    </artifactSet>
+
     <artifactSet to="plugin/loki">
         <artifact id="${project.groupId}:trino-loki:zip:${project.version}">
             <unpack />

lib/trino-lance-file/pom.xml

@@ -0,0 +1,179 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>io.trino</groupId>
+        <artifactId>trino-root</artifactId>
+        <version>478-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>trino-lance-file</artifactId>
+    <description>Trino - Lance file format</description>
+
+    <properties>
+        <air.compiler.fail-warnings>true</air.compiler.fail-warnings>
+        <air.test.jvm.additional-arguments>${air.test.jvm.additional-arguments.default}
+            --add-opens=java.base/java.nio=ALL-UNNAMED
+            --sun-misc-unsafe-memory-access=allow</air.test.jvm.additional-arguments>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>build.buf.gen</groupId>
+            <artifactId>lancedb_lance_protocolbuffers_java</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.errorprone</groupId>
+            <artifactId>error_prone_annotations</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.github.luohao</groupId>
+            <artifactId>fastlanes-java</artifactId>
+            <version>1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>io.trino</groupId>
+            <artifactId>trino-filesystem</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.trino</groupId>
+            <artifactId>trino-memory-context</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.trino</groupId>
+            <artifactId>trino-plugin-toolkit</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>it.unimi.dsi</groupId>
+            <artifactId>fastutil</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>jakarta.annotation</groupId>
+            <artifactId>jakarta.annotation-api</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.airlift</groupId>
+            <artifactId>slice</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>io.trino</groupId>
+            <artifactId>trino-spi</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.openjdk.jol</groupId>
+            <artifactId>jol-core</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.lancedb</groupId>
+            <artifactId>lance-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>io.airlift</groupId>
+            <artifactId>junit-extensions</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>io.trino</groupId>
+            <artifactId>trino-main</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>io.trino</groupId>
+            <artifactId>trino-testing-services</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-memory-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-vector</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.calcite.avatica</groupId>
+            <artifactId>avatica-core</artifactId>
+            <version>1.25.0</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.openjdk.jmh</groupId>
+            <artifactId>jmh-generator-annprocess</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <repositories>
+        <repository>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+            <id>buf</id>
+            <name>Buf Maven Repository</name>
+            <url>https://buf.build/gen/maven</url>
+        </repository>
+    </repositories>
+</project>

lib/trino-lance-file/src/main/java/io/trino/lance/file/AbstractLanceDataSource.java

@@ -0,0 +1,118 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.lance.file;
+
+import io.airlift.slice.Slice;
+import io.airlift.slice.Slices;
+
+import java.io.IOException;
+
+import static java.lang.Math.min;
+import static java.lang.Math.toIntExact;
+import static java.util.Objects.requireNonNull;
+
+public abstract class AbstractLanceDataSource
+        implements LanceDataSource
+{
+    private final LanceDataSourceId id;
+    private final long estimatedSize;
+    private long readTimeNanos;
+    private long readBytes;
+
+    public AbstractLanceDataSource(LanceDataSourceId id, long estimatedSize)
+    {
+        this.id = requireNonNull(id, "id is null");
+        this.estimatedSize = estimatedSize;
+    }
+
+    @Override
+    public LanceDataSourceId getId()
+    {
+        return id;
+    }
+
+    @Override
+    public long getReadBytes()
+    {
+        return readBytes;
+    }
+
+    @Override
+    public long getReadTimeNanos()
+    {
+        return readTimeNanos;
+    }
+
+    @Override
+    public long getEstimatedSize()
+    {
+        return estimatedSize;
+    }
+
+    @Override
+    public Slice readTail(int length)
+            throws IOException
+    {
+        long start = System.nanoTime();
+
+        Slice tailSlice = readTailInternal(length);
+
+        readTimeNanos += System.nanoTime() - start;
+        readBytes += tailSlice.length();
+
+        return tailSlice;
+    }
+
+    protected Slice readTailInternal(int length)
+            throws IOException
+    {
+        int readSize = toIntExact(min(estimatedSize, length));
+        return readFully(estimatedSize - readSize, readSize);
+    }
+
+    @Override
+    public long getRetainedSize()
+    {
+        return 0;
+    }
+
+    @Override
+    public final Slice readFully(long position, int length)
+            throws IOException
+    {
+        byte[] buffer = new byte[length];
+        readFully(position, buffer, 0, length);
+        return Slices.wrappedBuffer(buffer);
+    }
+
+    protected abstract void readInternal(long position, byte[] buffer, int bufferOffset, int bufferLength)
+            throws IOException;
+
+    private void readFully(long position, byte[] buffer, int bufferOffset, int bufferLength)
+            throws IOException
+    {
+        long start = System.nanoTime();
+
+        readInternal(position, buffer, bufferOffset, bufferLength);
+
+        readTimeNanos += System.nanoTime() - start;
+        readBytes += bufferLength;
+    }
+
+    @Override
+    public final String toString()
+    {
+        return id.toString();
+    }
+}

lib/trino-lance-file/src/main/java/io/trino/lance/file/FileLanceDataSource.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.lance.file;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+public class FileLanceDataSource
+        extends AbstractLanceDataSource
+{
+    private final RandomAccessFile input;
+
+    public FileLanceDataSource(File path)
+            throws FileNotFoundException
+    {
+        super(new LanceDataSourceId(path.getPath()), path.length());
+        this.input = new RandomAccessFile(path, "r");
+    }
+
+    @Override
+    protected void readInternal(long position, byte[] buffer, int bufferOffset, int bufferLength)
+            throws IOException
+    {
+        input.seek(position);
+        input.readFully(buffer, bufferOffset, bufferLength);
+    }
+
+    @Override
+    public void close()
+            throws IOException
+    {
+        input.close();
+    }
+}