diff --git a/build.sbt b/build.sbt index 07fb3ec..938afb6 100644 --- a/build.sbt +++ b/build.sbt @@ -50,7 +50,7 @@ lazy val core = project name := "ammonite-spark", generatePropertyFile("org/apache/spark/sql/ammonitesparkinternals/ammonite-spark.properties"), libraryDependencies ++= Seq( - Deps.ammoniteRepl % "provided", + Deps.ammoniteReplApi.value % "provided", Deps.sparkSql.value % "provided", Deps.jettyServer ) @@ -65,7 +65,7 @@ lazy val tests = project generateDependenciesFile, testSettings, libraryDependencies ++= Seq( - Deps.ammoniteRepl, + Deps.ammoniteRepl.value, Deps.utest ) ) diff --git a/modules/core/src/main/scala-2.11/ammonite/interp/api/package.scala b/modules/core/src/main/scala-2.11/ammonite/interp/api/package.scala new file mode 100644 index 0000000..ac0a67a --- /dev/null +++ b/modules/core/src/main/scala-2.11/ammonite/interp/api/package.scala @@ -0,0 +1,7 @@ +package ammonite.interp + +package object api { + + type InterpAPI = ammonite.interp.InterpAPI + +} diff --git a/modules/core/src/main/scala-2.11/ammonite/repl/api/package.scala b/modules/core/src/main/scala-2.11/ammonite/repl/api/package.scala new file mode 100644 index 0000000..57a366b --- /dev/null +++ b/modules/core/src/main/scala-2.11/ammonite/repl/api/package.scala @@ -0,0 +1,10 @@ +package ammonite.repl + +package object api { + + type Frame = ammonite.runtime.Frame + val Frame = ammonite.runtime.Frame + + type ReplAPI = ammonite.repl.ReplAPI + +} diff --git a/modules/core/src/main/scala-2.11/org/apache/spark/sql/ammonitesparkinternals/Compatibility.scala b/modules/core/src/main/scala-2.11/org/apache/spark/sql/ammonitesparkinternals/Compatibility.scala new file mode 100644 index 0000000..bfeaf6b --- /dev/null +++ b/modules/core/src/main/scala-2.11/org/apache/spark/sql/ammonitesparkinternals/Compatibility.scala @@ -0,0 +1,12 @@ +package org.apache.spark.sql.ammonitesparkinternals + +import ammonite.runtime.SpecialClassLoader + +object Compatibility { + + implicit class SpecialLoaderOps(private val cl: SpecialClassLoader) { + def inMemoryClasses: Map[String, Array[Byte]] = + cl.newFileDict.toMap + } + +} diff --git a/modules/core/src/main/scala-2.12/org/apache/spark/sql/ammonitesparkinternals/Compatibility.scala b/modules/core/src/main/scala-2.12/org/apache/spark/sql/ammonitesparkinternals/Compatibility.scala new file mode 100644 index 0000000..5194456 --- /dev/null +++ b/modules/core/src/main/scala-2.12/org/apache/spark/sql/ammonitesparkinternals/Compatibility.scala @@ -0,0 +1,3 @@ +package org.apache.spark.sql.ammonitesparkinternals + +object Compatibility diff --git a/modules/core/src/main/scala/org/apache/spark/sql/AmmoniteSparkSession.scala b/modules/core/src/main/scala/org/apache/spark/sql/AmmoniteSparkSession.scala index 9801fd8..88ac687 100644 --- a/modules/core/src/main/scala/org/apache/spark/sql/AmmoniteSparkSession.scala +++ b/modules/core/src/main/scala/org/apache/spark/sql/AmmoniteSparkSession.scala @@ -1,7 +1,7 @@ package org.apache.spark.sql -import ammonite.repl.ReplAPI -import ammonite.interp.InterpAPI +import ammonite.repl.api.ReplAPI +import ammonite.interp.api.InterpAPI import org.apache.spark.sql.ammonitesparkinternals.AmmoniteSparkSessionBuilder object AmmoniteSparkSession { diff --git a/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteClassServer.scala b/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteClassServer.scala index 7e4c8c1..f274013 100644 --- a/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteClassServer.scala +++ b/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteClassServer.scala @@ -5,12 +5,14 @@ import java.net.{InetSocketAddress, ServerSocket, URI} import java.nio.file.Files import javax.servlet.http.{HttpServletRequest, HttpServletResponse} -import ammonite.runtime.Frame +import ammonite.repl.api.Frame import org.eclipse.jetty.server.{Request, Server} import org.eclipse.jetty.server.handler.AbstractHandler final class AmmoniteClassServer(host: String, bindTo: String, port: Int, frames: => List[Frame]) { + import Compatibility._ + private val socketAddress = InetSocketAddress.createUnresolved(bindTo, port) private val handler = new AbstractHandler { @@ -22,7 +24,7 @@ final class AmmoniteClassServer(host: String, bindTo: String, port: Int, frames: val fromClassMaps = frames .toStream - .flatMap(_.classloader.newFileDict.get(item)) + .flatMap(_.classloader.inMemoryClasses.get(item)) .headOption def fromDirs = diff --git a/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteSparkSessionBuilder.scala b/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteSparkSessionBuilder.scala index eeb2b3d..284da21 100644 --- a/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteSparkSessionBuilder.scala +++ b/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteSparkSessionBuilder.scala @@ -5,8 +5,8 @@ import java.net.{InetAddress, URI, URL} import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} -import ammonite.repl.ReplAPI -import ammonite.interp.InterpAPI +import ammonite.repl.api.ReplAPI +import ammonite.interp.api.InterpAPI import org.apache.spark.SparkContext import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd} import org.apache.spark.sql.SparkSession diff --git a/modules/tests/src/main/scala-2.11/ammonite/repl/api/package.scala b/modules/tests/src/main/scala-2.11/ammonite/repl/api/package.scala new file mode 100644 index 0000000..cbaba55 --- /dev/null +++ b/modules/tests/src/main/scala-2.11/ammonite/repl/api/package.scala @@ -0,0 +1,9 @@ +package ammonite.repl + +package object api { + + type FrontEnd = ammonite.repl.FrontEnd + type History = ammonite.runtime.History + type ReplLoad = ammonite.repl.ReplLoad + +} diff --git a/modules/tests/src/main/scala/ammonite/spark/SparkReplTests.scala b/modules/tests/src/main/scala/ammonite/spark/SparkReplTests.scala index 34885c4..29692ec 100644 --- a/modules/tests/src/main/scala/ammonite/spark/SparkReplTests.scala +++ b/modules/tests/src/main/scala/ammonite/spark/SparkReplTests.scala @@ -330,8 +330,8 @@ class SparkReplTests( @ val list = List((1, Foo(1)), (1, Foo(2))) list: List[(Int, Foo)] = List((1, Foo(1)), (1, Foo(2))) - @ val res = sc.parallelize(list).groupByKey().collect() - res: Array[(Int, Iterable[Foo])] = Array((1, List(Foo(1), Foo(2)))) + @ val res = sc.parallelize(list).groupByKey().collect().map { case (k, v) => k -> v.toList } + res: Array[(Int, List[Foo])] = Array((1, List(Foo(1), Foo(2)))) """ ) } diff --git a/modules/tests/src/main/scala/ammonite/spark/fromammonite/TestRepl.scala b/modules/tests/src/main/scala/ammonite/spark/fromammonite/TestRepl.scala index 671a49d..a2670f3 100644 --- a/modules/tests/src/main/scala/ammonite/spark/fromammonite/TestRepl.scala +++ b/modules/tests/src/main/scala/ammonite/spark/fromammonite/TestRepl.scala @@ -4,7 +4,8 @@ import ammonite.interp.{CodeClassWrapper, CodeWrapper, Interpreter} import ammonite.main.Defaults import ammonite.ops.{Path, read} import ammonite.repl._ -import ammonite.runtime.{Frame, History, Storage} +import ammonite.repl.api.{FrontEnd, History, ReplLoad} +import ammonite.runtime.{Frame, ImportHook, Storage} import ammonite.util.Util.normalizeNewlines import ammonite.util._ import utest._ @@ -88,15 +89,15 @@ class TestRepl { } )) - val frames: Ref[List[Frame]] = Ref(List(Frame.createInitial())) + private val initialLoader = Thread.currentThread().getContextClassLoader + val frames: Ref[List[Frame]] = Ref(List(Frame.createInitial(initialLoader))) val sess0 = new SessionApiImpl(frames) var currentLine = 0 lazy val interp: Interpreter = try { new Interpreter( - printer0, + printer = printer0, storage = storage, - wd = ammonite.ops.pwd, basePredefs = Seq( PredefInfo( Name("defaultPredef"), @@ -108,7 +109,9 @@ class TestRepl { ), customPredefs = Seq(), extraBridges = extraBridges, + wd = ammonite.ops.pwd, colors = Ref(Colors.BlackWhite), + verboseOutput = true, getFrame = () => frames().head, createFrame = () => { val f = sess0.childFrame(frames().head); frames() = f :: frames(); f }, replCodeWrapper = codeWrapper, diff --git a/project/Deps.scala b/project/Deps.scala index 8b20b43..1d1eeda 100644 --- a/project/Deps.scala +++ b/project/Deps.scala @@ -5,7 +5,28 @@ import sbt.Keys._ object Deps { - def ammoniteRepl = ("com.lihaoyi" % "ammonite-repl" % "1.6.7").cross(CrossVersion.full) + private val ammoniteVersion = setting { + val sv = scalaVersion.value + if (sv.startsWith("2.11.")) + "1.6.7" + else + "1.6.9-8-2a27ffe" + } + + def ammoniteReplApi = setting { + val sv = scalaVersion.value + val mod = + if (sv.startsWith("2.11.")) + "com.lihaoyi" % "ammonite-repl" + else + "com.lihaoyi" % "ammonite-repl-api" + val ver = ammoniteVersion.value + (mod % ver).cross(CrossVersion.full) + } + def ammoniteRepl = setting { + val ver = ammoniteVersion.value + ("com.lihaoyi" % "ammonite-repl" % ver).cross(CrossVersion.full) + } def jettyServer = "org.eclipse.jetty" % "jetty-server" % "9.4.19.v20190610" def utest = "com.lihaoyi" %% "utest" % "0.6.7" diff --git a/project/Settings.scala b/project/Settings.scala index 020a074..dfb2547 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -26,7 +26,7 @@ object Settings { } lazy val shared = Seq( - scalaVersion := scala211, + scalaVersion := scala212, crossScalaVersions := Seq(scala212, scala211), scalacOptions ++= Seq( "-deprecation",