[CORE] Fix the java.nio.file.NoSuchFileException: default in spark 3.5 (

apache#6175)
gaoyangxiaozhu · Jun 24, 2024 · 9cceba6 · 9cceba6
1 parent 4ed161b
commit 9cceba6
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala b/gluten-core/src/main/scala/org/apache/spark/HdfsConfGenerator.scala
@@ -41,8 +41,8 @@ object HdfsConfGenerator extends Logging {
     addFileMethod.invoke(sc, path, Boolean.box(false), Boolean.box(true), Boolean.box(false))
     // Overwrite the spark internal config `spark.app.initial.file.urls`,
     // so that the file can be available before initializing executor plugin.
-    assert(sc.addedFiles.nonEmpty)
-    sc.conf.set("spark.app.initial.file.urls", sc.addedFiles.keys.toSeq.mkString(","))
+    assert(sc.listFiles.nonEmpty)
+    sc.conf.set("spark.app.initial.file.urls", sc.listFiles().mkString(","))
   }
 
   private def ignoreKey(key: String): Boolean = {