fic spark job submit

harbby · Jul 26, 2020 · faaa049 · faaa049
1 parent 442042e
commit faaa049
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 1 deletion.
diff --git a/sylph-dist/src/data/data.db b/sylph-dist/src/data/data.db
diff --git a/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/yarn/SylphSparkYarnClient.java b/sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/yarn/SylphSparkYarnClient.java
@@ -40,7 +40,8 @@ public SylphSparkYarnClient(ClientArguments clientArgs, SparkConf sparkConf, Yar
         this.yarnQueue = yarnQueue;
 
         //String key = DRIVER_MEMORY; //test
-        Field field = Client.class.getDeclaredField("org$apache$spark$deploy$yarn$Client$$hadoopConf");
+        //Field field = Client.class.getDeclaredField("org$apache$spark$deploy$yarn$Client$$hadoopConf"); //scala 2.11
+        Field field = Client.class.getDeclaredField("hadoopConf");    //scala 2.12
         field.setAccessible(true);
         YarnConfiguration yarnConfiguration = new YarnConfiguration(yarnClient.getConfig());
         field.set(this, yarnConfiguration);

diff --git a/sylph-yarn/build.gradle b/sylph-yarn/build.gradle
@@ -1,3 +1,7 @@
+configurations.all {
+  resolutionStrategy { preferProjectModules() }
+}
+
 dependencies {
   // conflict was found between the following modules:
   compileOnly group: 'org.apache.hadoop', name: 'hadoop-client', version: deps.hadoop