From 858eaeb61aeb88b8f3cb0e7d2ad6d8097257fcae Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Thu, 18 May 2023 11:53:43 +0200 Subject: [PATCH 1/3] Fix formatting on main branch --- build.sc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build.sc b/build.sc index 8504384..e8709f3 100644 --- a/build.sc +++ b/build.sc @@ -19,10 +19,12 @@ implicit def millModuleBasePath: define.BasePath = trait AmmSparkPublishModule extends PublishModule { import mill.scalalib.publish._ def publishVersion = T { - val v = VcsVersion.vcsState().format() + val v = VcsVersion.vcsState().format() val dirtyIdx = v.indexOf("-DIRTY") + def endsWithCommitHash = + v.length > 6 && v.substring(v.length - 6).forall(c => c.isDigit || (c >= 'a' && c <= 'f')) if (dirtyIdx >= 0) v.take(dirtyIdx) + "-SNAPSHOT" - else if (v.length > 6 && v.substring(v.length - 6).forall(c => c.isDigit || (c >= 'a' && c <= 'f'))) v + "-SNAPSHOT" + else if (endsWithCommitHash) v + "-SNAPSHOT" else v } def pomSettings = PomSettings( From 1a6fec6a8122717d97eeb75f3f3bfcb5ff67b0e3 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Wed, 17 May 2023 17:40:47 +0200 Subject: [PATCH 2/3] Add keepJar method on AmmoniteSparkSessionBuilder Handy to filter out specific JARs if needed --- .../AmmoniteSparkSessionBuilder.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteSparkSessionBuilder.scala b/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteSparkSessionBuilder.scala index d78cf00..e043361 100644 --- a/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteSparkSessionBuilder.scala +++ b/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteSparkSessionBuilder.scala @@ -203,6 +203,7 @@ class AmmoniteSparkSessionBuilder(implicit private var sendSparkYarnJars0 = true private var sendSparkJars0 = true private var ignoreJars0 = Set.empty[URI] + private var keepJars0 = Seq.empty[URI => Boolean] def sendSparkYarnJars(force: Boolean = true): this.type = { sendSparkYarnJars0 = force @@ -219,6 +220,11 @@ class AmmoniteSparkSessionBuilder(implicit this } + def keepJars(keep: URI => Boolean): this.type = { + keepJars0 = keepJars0 :+ keep + this + } + var classServerOpt = Option.empty[AmmoniteClassServer] private def isYarn(): Boolean = @@ -349,7 +355,8 @@ class AmmoniteSparkSessionBuilder(implicit .map(normalize) .toSet val nonSparkJars = jars.filter(uri => !sparkJarFileSet.contains(normalize(uri))) - config("spark.jars", nonSparkJars.map(_.toASCIIString).mkString(",")) + val finalNonSparkJars = keepJars0.foldLeft(nonSparkJars)(_.filter(_)) + config("spark.jars", finalNonSparkJars.map(_.toASCIIString).mkString(",")) } if (interpApi != null) From 0749b494fe47e8996c02ba47db988afbbf3baf77 Mon Sep 17 00:00:00 2001 From: Alex Archambault Date: Thu, 18 May 2023 11:49:54 +0200 Subject: [PATCH 3/3] Don't send source JARs by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These are not really needed on executors… --- .../AmmoniteSparkSessionBuilder.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteSparkSessionBuilder.scala b/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteSparkSessionBuilder.scala index e043361..fa8b74f 100644 --- a/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteSparkSessionBuilder.scala +++ b/modules/core/src/main/scala/org/apache/spark/sql/ammonitesparkinternals/AmmoniteSparkSessionBuilder.scala @@ -203,6 +203,7 @@ class AmmoniteSparkSessionBuilder(implicit private var sendSparkYarnJars0 = true private var sendSparkJars0 = true private var ignoreJars0 = Set.empty[URI] + private var sendSourceJars0 = false private var keepJars0 = Seq.empty[URI => Boolean] def sendSparkYarnJars(force: Boolean = true): this.type = { @@ -220,6 +221,11 @@ class AmmoniteSparkSessionBuilder(implicit this } + def sendSourceJars(send: Boolean): this.type = { + sendSourceJars0 = send + this + } + def keepJars(keep: URI => Boolean): this.type = { keepJars0 = keepJars0 :+ keep this @@ -355,7 +361,10 @@ class AmmoniteSparkSessionBuilder(implicit .map(normalize) .toSet val nonSparkJars = jars.filter(uri => !sparkJarFileSet.contains(normalize(uri))) - val finalNonSparkJars = keepJars0.foldLeft(nonSparkJars)(_.filter(_)) + val nonSparkJars0 = + if (sendSourceJars0) nonSparkJars + else nonSparkJars.filter(uri => !uri.toASCIIString.endsWith("-sources.jar")) + val finalNonSparkJars = keepJars0.foldLeft(nonSparkJars0)(_.filter(_)) config("spark.jars", finalNonSparkJars.map(_.toASCIIString).mkString(",")) }