Skip to content

Commit

Permalink
Don't send source JARs by default
Browse files Browse the repository at this point in the history
These are not really needed on executors…
  • Loading branch information
alexarchambault committed May 18, 2023
1 parent 1a6fec6 commit 0749b49
Showing 1 changed file with 10 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ class AmmoniteSparkSessionBuilder(implicit
private var sendSparkYarnJars0 = true
private var sendSparkJars0 = true
private var ignoreJars0 = Set.empty[URI]
private var sendSourceJars0 = false
private var keepJars0 = Seq.empty[URI => Boolean]

def sendSparkYarnJars(force: Boolean = true): this.type = {
Expand All @@ -220,6 +221,11 @@ class AmmoniteSparkSessionBuilder(implicit
this
}

def sendSourceJars(send: Boolean): this.type = {
sendSourceJars0 = send
this
}

def keepJars(keep: URI => Boolean): this.type = {
keepJars0 = keepJars0 :+ keep
this
Expand Down Expand Up @@ -355,7 +361,10 @@ class AmmoniteSparkSessionBuilder(implicit
.map(normalize)
.toSet
val nonSparkJars = jars.filter(uri => !sparkJarFileSet.contains(normalize(uri)))
val finalNonSparkJars = keepJars0.foldLeft(nonSparkJars)(_.filter(_))
val nonSparkJars0 =
if (sendSourceJars0) nonSparkJars
else nonSparkJars.filter(uri => !uri.toASCIIString.endsWith("-sources.jar"))
val finalNonSparkJars = keepJars0.foldLeft(nonSparkJars0)(_.filter(_))
config("spark.jars", finalNonSparkJars.map(_.toASCIIString).mkString(","))
}

Expand Down

0 comments on commit 0749b49

Please sign in to comment.