Merge pull request #329 from alexarchambault/syntax

Add various syntax and Toree compatibility helpers

build.sc

@@ -287,6 +287,29 @@ class AlmondSpark(val crossScalaVersion: String) extends CrossSbtModule with Amm
   }
 }
 
+object `almond-toree-spark` extends Cross[AlmondToreeSpark](Versions.scala: _*)
+class AlmondToreeSpark(val crossScalaVersion: String) extends CrossSbtModule
+    with AmmSparkPublishModule
+    with AmmSparkMima {
+  def moduleDeps = super.moduleDeps ++ Seq(
+    `almond-spark`()
+  )
+  def ivyDeps = super.ivyDeps() ++ Agg(
+    Deps.almondToreeHooks
+  )
+  def compileIvyDeps = super.compileIvyDeps() ++ Agg(
+    Deps.scalaKernelApi
+      .exclude(("com.lihaoyi", s"ammonite-compiler_$crossScalaVersion"))
+      .exclude(("com.lihaoyi", s"ammonite-repl-api_$crossScalaVersion")),
+    Deps.sparkSql(scalaVersion())
+  )
+  def repositoriesTask = T.task {
+    super.repositoriesTask() ++ Seq(
+      coursier.Repositories.jitpack
+    )
+  }
+}
+
 def publishSonatype(tasks: mill.main.Tasks[PublishModule.PublishData]) = T.command {
   publishSonatype0(
     data = define.Target.sequence(tasks.value)(),

modules/almond-spark/src/main/scala/almond/spark/syntax.scala

@@ -0,0 +1,17 @@
+package almond.spark
+
+import almond.display.Html
+import org.apache.spark.sql._
+import org.apache.spark.{SparkConf, SparkContext}
+
+object syntax {
+  implicit class KernelToreeSparkOps(private val kernel: almond.api.JupyterApi) {
+    def sparkContext: SparkContext = sparkSession.sparkContext
+    def sparkConf: SparkConf       = sparkContext.getConf
+    def sparkSession: SparkSession = SparkSession.builder().getOrCreate()
+  }
+  implicit class AlmondNetflixDataFrameOps(private val df: DataFrame) {
+    def render(limit: Int = 10): Html =
+      Html(DataFrameRenderer.render(df, limit = limit))
+  }
+}

modules/almond-toree-spark/src/main/scala/almond/spark/ToreeSql.scala

@@ -0,0 +1,20 @@
+package almond.spark
+
+object ToreeSql {
+
+  var sqlLimit = 10
+
+  private def sqlMagic(content: String): String = {
+    val tq = "\"\"\""
+    s"""_root_.almond.display.Html(_root_.almond.dfrenderer.AlmondDataFrameRenderer.render(spark.sql($tq$content$tq), limit = $sqlLimit))"""
+  }
+
+  def setup(): Unit = {
+    almond.toree.CellMagicHook.addHandler("sql") { (_, content) =>
+      Right(sqlMagic(content))
+    }
+    almond.toree.CellMagicHook.addHandler("sparksql") { (_, content) =>
+      Right(sqlMagic(content))
+    }
+  }
+}

project/deps.sc

@@ -6,11 +6,13 @@ object Versions {
 
   def scala = Seq(scala213, scala212)
 
+  def almond        = "0.14.0-RC6"
   def ammonite      = "3.0.0-M0-40-d95c3b3d"
   def jsoniterScala = "2.13.5"
 }
 
 object Deps {
+  def almondToreeHooks = ivy"sh.almond::toree-hooks:${Versions.almond}"
   def ammoniteCompiler = ivy"sh.almond.tmp.ammonite:::ammonite-compiler:${Versions.ammonite}"
   def ammoniteReplApi  = ivy"sh.almond.tmp.ammonite:::ammonite-repl-api:${Versions.ammonite}"
   def ammoniteRepl     = ivy"sh.almond.tmp.ammonite:::ammonite-repl:${Versions.ammonite}"
@@ -22,7 +24,7 @@ object Deps {
   def jsoniterScalaMacros =
     ivy"com.github.plokhotnyuk.jsoniter-scala::jsoniter-scala-macros:${Versions.jsoniterScala}"
   def log4j2         = ivy"org.apache.logging.log4j:log4j-core:2.17.2"
-  def scalaKernelApi = ivy"sh.almond:::scala-kernel-api:0.14.0-RC6"
+  def scalaKernelApi = ivy"sh.almond:::scala-kernel-api:${Versions.almond}"
   def scalatags      = ivy"com.lihaoyi::scalatags:0.12.0"
   def sparkSql(sv: String) = {
     val ver =