[SEDONA-723] Add write format for (Geo)Arrow

spark/spark-3.5/pom.xml

@@ -48,6 +48,12 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-java-root</artifactId>
+            <version>18.2.0</version>
+            <type>pom</type>
+        </dependency>
         <dependency>
             <groupId>org.apache.sedona</groupId>
             <artifactId>sedona-spark-common-${spark.compat.version}_${scala.compat.version}</artifactId>

spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/arrow/ArrowBatchWrite.scala

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sedona.sql.datasources.arrow
+
+import org.apache.spark.sql.connector.write.BatchWrite
+import org.apache.spark.sql.connector.write.{DataWriterFactory, PhysicalWriteInfo}
+import org.apache.spark.sql.connector.write.WriterCommitMessage
+import org.apache.spark.sql.connector.write.LogicalWriteInfo
+import org.apache.spark.sql.catalyst.InternalRow
+
+case class ArrowBatchWrite(logicalInfo: LogicalWriteInfo) extends BatchWrite {
+  def createBatchWriterFactory(info: PhysicalWriteInfo): DataWriterFactory = {
+    return new ArrowDataWriterFactory(logicalInfo, info)
+  }
+
+  def commit(messages: Array[WriterCommitMessage]): Unit = {}
+
+  def abort(messages: Array[WriterCommitMessage]): Unit = {}
+
+}

spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/arrow/ArrowDataSource.scala

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sedona.sql.datasources.arrow
+
+import org.apache.spark.sql.connector.catalog.Table
+import org.apache.spark.sql.execution.datasources.FileFormat
+import org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2
+import org.apache.spark.sql.sources.DataSourceRegister
+import org.apache.spark.sql.util.CaseInsensitiveStringMap
+
+import java.util.Locale
+import scala.jdk.CollectionConverters._
+import scala.util.Try
+import org.apache.sedona.sql.datasources.geopackage.ArrowTable
+
+class ArrowDataSource extends FileDataSourceV2 with DataSourceRegister {
+
+  override def fallbackFileFormat: Class[_ <: FileFormat] = {
+    null
+  }
+
+  override protected def getTable(options: CaseInsensitiveStringMap): Table = {
+    ArrowTable("", sparkSession, options, getPaths(options), None)
+  }
+
+  override def shortName(): String = "arrows"
+}

spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/arrow/ArrowDataWriterFactory.scala

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sedona.sql.datasources.arrow
+
+import org.apache.spark.sql.connector.write.LogicalWriteInfo
+import org.apache.spark.sql.connector.write.PhysicalWriteInfo
+import org.apache.spark.sql.connector.write.DataWriterFactory
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.connector.write.DataWriter
+
+case class ArrowDataWriterFactory(logicalInfo: LogicalWriteInfo, physicalInfo: PhysicalWriteInfo)
+    extends DataWriterFactory {
+  override def createWriter(partitionId: Int, taskId: Long): DataWriter[InternalRow] = {
+    return new ArrowWriter(logicalInfo, physicalInfo, partitionId, taskId)
+  }
+}

spark/spark-3.5/src/main/scala/org/apache/sedona/sql/datasources/arrow/ArrowEncoderUtils.scala

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sedona.sql.datasources.arrow
+
+import scala.collection.JavaConverters._
+import scala.reflect.ClassTag
+
+import org.apache.arrow.vector.{FieldVector, VectorSchemaRoot}
+import org.apache.arrow.vector.complex.StructVector
+
+private[arrow] object ArrowEncoderUtils {
+  object Classes {
+    val WRAPPED_ARRAY: Class[_] = classOf[scala.collection.mutable.WrappedArray[_]]
+    val ITERABLE: Class[_] = classOf[scala.collection.Iterable[_]]
+    val MAP: Class[_] = classOf[scala.collection.Map[_, _]]
+    val JLIST: Class[_] = classOf[java.util.List[_]]
+    val JMAP: Class[_] = classOf[java.util.Map[_, _]]
+  }
+
+  def isSubClass(cls: Class[_], tag: ClassTag[_]): Boolean = {
+    cls.isAssignableFrom(tag.runtimeClass)
+  }
+
+  def unsupportedCollectionType(cls: Class[_]): Nothing = {
+    throw new RuntimeException(s"Unsupported collection type: $cls")
+  }
+}
+
+private[arrow] object StructVectors {
+  def unapply(v: AnyRef): Option[(StructVector, Seq[FieldVector])] = v match {
+    case root: VectorSchemaRoot => Option((null, root.getFieldVectors.asScala.toSeq))
+    case struct: StructVector => Option((struct, struct.getChildrenFromFields.asScala.toSeq))
+    case _ => None
+  }
+}