Expand SPARK_DIST_CLASSPATH (#303)

Seems it tends to have globs
alexarchambault · May 15, 2023 · 3f1a8c2 · 3f1a8c2
1 parent cfa7bf5
commit 3f1a8c2
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 3 deletions.
diff --git a/build.sc b/build.sc
@@ -146,6 +146,7 @@ class Core(val crossScalaVersion: String) extends CrossSbtModule with WithProper
     Deps.sparkSql(scalaVersion())
   )
   def ivyDeps = super.ivyDeps() ++ Agg(
+    Deps.classPathUtil,
     Deps.jettyServer
   )
   def propertyFilePath =

diff --git a/.../main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteSparkSessionBuilder.scala b/.../main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteSparkSessionBuilder.scala
@@ -7,6 +7,7 @@ import java.nio.file.{Files, Paths}
 
 import ammonite.interp.api.InterpAPI
 import ammonite.repl.api.ReplAPI
+import coursier.cputil.ClassPathUtil
 import coursierapi.Dependency
 import org.apache.spark.SparkContext
 import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
@@ -334,8 +335,7 @@ class AmmoniteSparkSessionBuilder(implicit
 
         val sparkDistClassPath = sys.env.get("SPARK_DIST_CLASSPATH")
           .toList
-          .flatMap(_.split(File.pathSeparator).toList)
-          .map(Paths.get(_))
+          .flatMap(ClassPathUtil.classPath(_))
 
         (fromBaseCp ++ fromSparkDistrib, sparkDistClassPath)
     }

diff --git a/project/deps.sc b/project/deps.sc
@@ -15,7 +15,8 @@ object Deps {
   def ammoniteReplApi  = ivy"com.lihaoyi:::ammonite-repl-api:${Versions.ammonite}"
   def ammoniteRepl     = ivy"com.lihaoyi:::ammonite-repl:${Versions.ammonite}"
 
-  def jettyServer = ivy"org.eclipse.jetty:jetty-server:9.4.51.v20230217"
+  def classPathUtil = ivy"io.get-coursier::class-path-util:0.1.4"
+  def jettyServer   = ivy"org.eclipse.jetty:jetty-server:9.4.51.v20230217"
   def jsoniterScalaCore =
     ivy"com.github.plokhotnyuk.jsoniter-scala::jsoniter-scala-core:${Versions.jsoniterScala}"
   def jsoniterScalaMacros =