diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index 64b0d510..c5c068b6 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -1,9 +1,12 @@ +import org.gradle.jvm.toolchain.JavaLanguageVersion + object Versions : Dsl { const val project = "2.0.0-SNAPSHOT" const val kotlinSparkApiGradlePlugin = "2.0.0-SNAPSHOT" const val groupID = "org.jetbrains.kotlinx.spark" const val kotlin = "2.0.0-RC3" - const val jvmTarget = "8" + const val jvmTarget = "1.8" + val jvmLanguageVersion = JavaLanguageVersion.of(8) const val jupyterJvmTarget = "8" inline val spark get() = System.getProperty("spark") as String inline val scala get() = System.getProperty("scala") as String diff --git a/compiler-plugin/build.gradle.kts b/compiler-plugin/build.gradle.kts index 02aac811..f94b15dc 100644 --- a/compiler-plugin/build.gradle.kts +++ b/compiler-plugin/build.gradle.kts @@ -1,3 +1,4 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.dsl.KotlinVersion import org.jetbrains.kotlin.gradle.tasks.KotlinCompile @@ -74,13 +75,17 @@ tasks.withType().configureEach { kotlin { jvmToolchain { - languageVersion = JavaLanguageVersion.of(8) + languageVersion = Versions.jvmLanguageVersion + } + compilerOptions { + jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget) } } java { toolchain { languageVersion = JavaLanguageVersion.of(8) } + targetCompatibility = JavaVersion.toVersion(Versions.jvmTarget) } val generateTests by tasks.creating(JavaExec::class) { diff --git a/examples/build.gradle.kts b/examples/build.gradle.kts index 8683926a..2cc93906 100644 --- a/examples/build.gradle.kts +++ b/examples/build.gradle.kts @@ -1,4 +1,4 @@ -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import org.jetbrains.kotlin.gradle.dsl.JvmTarget plugins { // Needs to be installed in the local maven repository or have the bootstrap jar on the classpath @@ -49,8 +49,10 @@ dependencies { } kotlin { - jvmToolchain(8) jvmToolchain { - languageVersion = JavaLanguageVersion.of(Versions.jvmTarget) + languageVersion = Versions.jvmLanguageVersion + } + compilerOptions { + jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget) } } \ No newline at end of file diff --git a/gradle-plugin/build.gradle.kts b/gradle-plugin/build.gradle.kts index fb77282e..42fb7941 100644 --- a/gradle-plugin/build.gradle.kts +++ b/gradle-plugin/build.gradle.kts @@ -1,5 +1,8 @@ @file:Suppress("UnstableApiUsage") +import org.jetbrains.kotlin.gradle.dsl.JvmTarget + + plugins { `java-gradle-plugin` kotlin @@ -51,7 +54,10 @@ dependencies { kotlin { jvmToolchain { - languageVersion = JavaLanguageVersion.of(Versions.jvmTarget) + languageVersion = Versions.jvmLanguageVersion + } + compilerOptions { + jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget) } } diff --git a/gradle/bootstraps/compiler-plugin.jar b/gradle/bootstraps/compiler-plugin.jar index 78fe0355..29aa66d2 100644 Binary files a/gradle/bootstraps/compiler-plugin.jar and b/gradle/bootstraps/compiler-plugin.jar differ diff --git a/gradle/bootstraps/gradle-plugin.jar b/gradle/bootstraps/gradle-plugin.jar index cbc37327..10a5be5d 100644 Binary files a/gradle/bootstraps/gradle-plugin.jar and b/gradle/bootstraps/gradle-plugin.jar differ diff --git a/kotlin-spark-api/build.gradle.kts b/kotlin-spark-api/build.gradle.kts index ede91174..c4a748e7 100644 --- a/kotlin-spark-api/build.gradle.kts +++ b/kotlin-spark-api/build.gradle.kts @@ -4,6 +4,7 @@ import com.igormaznitsa.jcp.gradle.JcpTask import com.vanniktech.maven.publish.JavadocJar.Dokka import com.vanniktech.maven.publish.KotlinJvm import org.jetbrains.dokka.gradle.AbstractDokkaLeafTask +import org.jetbrains.kotlin.gradle.dsl.JvmTarget plugins { kotlin @@ -66,6 +67,9 @@ dependencies { kafkaStreamsTestUtils, sparkMl, ) + + compileOnly(scalaLibrary) + testCompileOnly(scalaLibrary) } } @@ -154,8 +158,11 @@ tasks.compileTestKotlin { } kotlin { + compilerOptions { + jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget) + } jvmToolchain { - languageVersion = JavaLanguageVersion.of(Versions.jvmTarget) + languageVersion = Versions.jvmLanguageVersion } } diff --git a/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt index 219be4ae..e931890e 100644 --- a/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt +++ b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt @@ -76,6 +76,7 @@ import kotlin.reflect.jvm.javaGetter import kotlin.reflect.jvm.javaMethod import kotlin.reflect.jvm.jvmName import kotlin.reflect.typeOf +import scala.collection.immutable.Seq as ImmutableSeq fun kotlinEncoderFor( kClass: KClass, @@ -683,7 +684,7 @@ object KotlinTypeInference : Serializable { JavaBeanEncoder( ClassTag.apply(jClass), - fields.asScalaSeq(), + fields.asScalaSeq() as ImmutableSeq?, ) } } diff --git a/scala-helpers/build.gradle.kts b/scala-helpers/build.gradle.kts index b636c14d..f74ed7e0 100644 --- a/scala-helpers/build.gradle.kts +++ b/scala-helpers/build.gradle.kts @@ -21,8 +21,7 @@ repositories { dependencies { Dependencies { - api( - scalaLibrary, + implementation( reflect, ) @@ -32,18 +31,20 @@ dependencies { implementation( // sparkSql, not needed atm ) + compileOnly(scalaLibrary) + testCompileOnly(scalaLibrary) } } java { toolchain { if (Versions.scalaCompat.toDouble() > 2.12) { // scala 2.12 will always target java 8 - languageVersion = JavaLanguageVersion.of(Versions.jvmTarget) - + languageVersion = Versions.jvmLanguageVersion } else if (Versions.jvmTarget == "1.8" || Versions.jvmTarget == "8") { languageVersion = JavaLanguageVersion.of(8) } } + targetCompatibility = JavaVersion.toVersion(Versions.jvmTarget) } tasks.withType { @@ -54,9 +55,12 @@ tasks.withType { } } -val scalaMainSources = sourceSets.main.get().scala.sourceDirectories +val scalaMainSources = + sourceSets.main + .get() + .scala.sourceDirectories -val preprocessMain by tasks.creating(JcpTask::class) { +val preprocessMain by tasks.creating(JcpTask::class) { sources = scalaMainSources clearTarget = true fileExtensions = listOf("scala") @@ -99,4 +103,4 @@ mavenPublishing { val skipScalaOnlyDependent = System.getProperty("skipScalaOnlyDependent").toBoolean() tasks .filter { "publish" in it.name } - .forEach { it.onlyIf { !skipScalaOnlyDependent } } \ No newline at end of file + .forEach { it.onlyIf { !skipScalaOnlyDependent } } diff --git a/scala-tuples-in-kotlin/build.gradle.kts b/scala-tuples-in-kotlin/build.gradle.kts index cc99e208..0ba2bdb2 100644 --- a/scala-tuples-in-kotlin/build.gradle.kts +++ b/scala-tuples-in-kotlin/build.gradle.kts @@ -3,9 +3,7 @@ import com.vanniktech.maven.publish.JavadocJar.Dokka import com.vanniktech.maven.publish.KotlinJvm import org.jetbrains.dokka.gradle.AbstractDokkaLeafTask -import org.jetbrains.dokka.gradle.DokkaTask -import org.jetbrains.dokka.gradle.DokkaTaskPartial -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import org.jetbrains.kotlin.gradle.dsl.JvmTarget plugins { scala @@ -30,24 +28,26 @@ dependencies { Dependencies { implementation( kotlinStdLib, - scalaLibrary, ) testImplementation( kotest, atrium, kotlinTest, ) + compileOnly(scalaLibrary) + testCompileOnly(scalaLibrary) } } - kotlin { jvmToolchain { - languageVersion = JavaLanguageVersion.of(Versions.jvmTarget) + languageVersion = Versions.jvmLanguageVersion + } + compilerOptions { + jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget) } } - tasks.withType { dokkaSourceSets { all { diff --git a/spark-connect-examples/build.gradle.kts b/spark-connect-examples/build.gradle.kts index 1f62c6fa..f6885edc 100644 --- a/spark-connect-examples/build.gradle.kts +++ b/spark-connect-examples/build.gradle.kts @@ -5,6 +5,7 @@ plugins { id("org.jetbrains.kotlinx.spark.api") kotlin("jvm") application + shadow } // run with `./gradlew run` @@ -44,21 +45,21 @@ dependencies { implementation(sparkConnectClient) implementation(kotlinDateTime) + + compileOnly(scalaLibrary) } } -// spark-connect seems to work well with java 17 as client and java 1.8 as server -// also set gradle and your project sdk to java 17 kotlin { jvmToolchain { - languageVersion = JavaLanguageVersion.of(17) + languageVersion = JavaLanguageVersion.of(17)//Versions.jvmLanguageVersion } compilerOptions { - jvmTarget = JvmTarget.JVM_17 + jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget) } } tasks.withType { - sourceCompatibility = JavaVersion.VERSION_17.toString() - targetCompatibility = JavaVersion.VERSION_17.toString() + sourceCompatibility = JavaVersion.VERSION_1_8.toString() + targetCompatibility = JavaVersion.VERSION_1_8.toString() } diff --git a/spark-connect-examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/Main.kt b/spark-connect-examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/Main.kt index e81f261d..f5bc5c84 100644 --- a/spark-connect-examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/Main.kt +++ b/spark-connect-examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/Main.kt @@ -5,30 +5,39 @@ import org.jetbrains.kotlinx.spark.api.plugin.annotations.Sparkify import org.jetbrains.kotlinx.spark.api.showDS import org.jetbrains.kotlinx.spark.api.toList import org.jetbrains.kotlinx.spark.api.tuples.X -import org.jetbrains.kotlinx.spark.api.tuples.t import org.jetbrains.kotlinx.spark.api.withSparkConnect import scala.Tuple2 import java.time.LocalDate -// run with `./gradlew run` or set VM options: "--add-opens=java.base/java.nio=ALL-UNNAMED" in the IDE +// run with `./gradlew runShadow` or set VM options: "--add-opens=java.base/java.nio=ALL-UNNAMED" in the IDE fun main() = withSparkConnect("sc://localhost") { val classFinder = REPLClassDirMonitor("/mnt/data/Projects/kotlin-spark-api/spark-connect-examples/build/classes") spark.registerClassFinder(classFinder) - spark.addArtifact("/mnt/data/Projects/kotlin-spark-api/spark-connect-examples/build/libs/spark-connect-examples-2.0.0-SNAPSHOT.jar") + // make jar first, preferably a fat jar with shadow, but be careful it doesn't contain scala depencencies + spark.addArtifact("/mnt/data/Projects/kotlin-spark-api/spark-connect-examples/build/libs/spark-connect-examples-2.0.0-SNAPSHOT-all.jar") + val data = listOf( Person("Alice", 25, LocalDate.of(1996, 1, 1), "Alice" X Address("1 Main St", "Springfield", "IL", 62701)), Person("Bob", 30, LocalDate.of(1991, 1, 1), "Bob" X Address("2 Main St", "Springfield", "IL", 62701)), - Person("Charlie", 35, LocalDate.of(1986, 1, 1), "Charlie" X Address("3 Main St", "Springfield", "IL", 62701)), + Person( + "Charlie", + 35, + LocalDate.of(1986, 1, 1), + "Charlie" X Address("3 Main St", "Springfield", "IL", 62701), + ), ) val ds = data.toDS().showDS() - ds.toList().forEach { - println(it) - } + ds + .filter { it.age > 26 } + .toList() + .forEach { + println(it) + } } @Sparkify