Skip to content

Commit

Permalink
Add keepJar method on AmmoniteSparkSessionBuilder
Browse files Browse the repository at this point in the history
Handy to filter out specific JARs if needed
  • Loading branch information
alexarchambault committed May 18, 2023
1 parent 858eaeb commit 1a6fec6
Showing 1 changed file with 8 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ class AmmoniteSparkSessionBuilder(implicit
private var sendSparkYarnJars0 = true
private var sendSparkJars0 = true
private var ignoreJars0 = Set.empty[URI]
private var keepJars0 = Seq.empty[URI => Boolean]

def sendSparkYarnJars(force: Boolean = true): this.type = {
sendSparkYarnJars0 = force
Expand All @@ -219,6 +220,11 @@ class AmmoniteSparkSessionBuilder(implicit
this
}

def keepJars(keep: URI => Boolean): this.type = {
keepJars0 = keepJars0 :+ keep
this
}

var classServerOpt = Option.empty[AmmoniteClassServer]

private def isYarn(): Boolean =
Expand Down Expand Up @@ -349,7 +355,8 @@ class AmmoniteSparkSessionBuilder(implicit
.map(normalize)
.toSet
val nonSparkJars = jars.filter(uri => !sparkJarFileSet.contains(normalize(uri)))
config("spark.jars", nonSparkJars.map(_.toASCIIString).mkString(","))
val finalNonSparkJars = keepJars0.foldLeft(nonSparkJars)(_.filter(_))
config("spark.jars", finalNonSparkJars.map(_.toASCIIString).mkString(","))
}

if (interpApi != null)
Expand Down

0 comments on commit 1a6fec6

Please sign in to comment.