From 0600861f7e77062be3caab22899547d916bb3e06 Mon Sep 17 00:00:00 2001 From: xutingjun Date: Thu, 4 Jun 2015 20:38:26 +0800 Subject: [PATCH 1/4] set executor cores into system --- .../org/apache/spark/deploy/yarn/ApplicationMaster.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index 002d7b6eaf498..fc6234fda1a69 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -598,6 +598,10 @@ object ApplicationMaster extends Logging { def main(args: Array[String]): Unit = { SignalLogger.register(log) val amArgs = new ApplicationMasterArguments(args) + if (amArgs.userClass != null) { + System.setProperty("spark.executor.cores", amArgs.executorCores.toString) + } + SparkHadoopUtil.get.runAsSparkUser { () => master = new ApplicationMaster(amArgs, new YarnRMClient(amArgs)) System.exit(master.run()) From e51cc9e7461df733d6112eb72283929c31efcaa5 Mon Sep 17 00:00:00 2001 From: Xutingjun Date: Fri, 5 Jun 2015 10:14:01 +0800 Subject: [PATCH 2/4] set executor cores into system --- core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 8cf4d58847d8e..7521f517e94b7 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -446,6 +446,7 @@ object SparkSubmit { OptionAssigner(args.keytab, YARN, CLUSTER, clOption = "--keytab"), // Other options + OptionAssigner(args.executorCores, YARN, CLUSTER, sysProp = "spark.executor.cores"), OptionAssigner(args.executorCores, STANDALONE, ALL_DEPLOY_MODES, sysProp = "spark.executor.cores"), OptionAssigner(args.executorMemory, STANDALONE | MESOS | YARN, ALL_DEPLOY_MODES, From ce33148f5b9d5df6bc2094edafba9e17a73ddf28 Mon Sep 17 00:00:00 2001 From: Xutingjun Date: Fri, 5 Jun 2015 10:26:41 +0800 Subject: [PATCH 3/4] set executor cores into system --- .../org/apache/spark/deploy/yarn/ApplicationMaster.scala | 4 ---- 1 file changed, 4 deletions(-) diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala index fc6234fda1a69..002d7b6eaf498 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala @@ -598,10 +598,6 @@ object ApplicationMaster extends Logging { def main(args: Array[String]): Unit = { SignalLogger.register(log) val amArgs = new ApplicationMasterArguments(args) - if (amArgs.userClass != null) { - System.setProperty("spark.executor.cores", amArgs.executorCores.toString) - } - SparkHadoopUtil.get.runAsSparkUser { () => master = new ApplicationMaster(amArgs, new YarnRMClient(amArgs)) System.exit(master.run()) From 80b18cd0c7899ba13b87c1766664cc4563b94c43 Mon Sep 17 00:00:00 2001 From: Xutingjun Date: Fri, 5 Jun 2015 10:46:13 +0800 Subject: [PATCH 4/4] change to STANDALONE | YARN --- core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 7521f517e94b7..0e840460ea589 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -425,7 +425,6 @@ object SparkSubmit { // Yarn client only OptionAssigner(args.queue, YARN, CLIENT, sysProp = "spark.yarn.queue"), OptionAssigner(args.numExecutors, YARN, CLIENT, sysProp = "spark.executor.instances"), - OptionAssigner(args.executorCores, YARN, CLIENT, sysProp = "spark.executor.cores"), OptionAssigner(args.files, YARN, CLIENT, sysProp = "spark.yarn.dist.files"), OptionAssigner(args.archives, YARN, CLIENT, sysProp = "spark.yarn.dist.archives"), OptionAssigner(args.principal, YARN, CLIENT, sysProp = "spark.yarn.principal"), @@ -446,8 +445,7 @@ object SparkSubmit { OptionAssigner(args.keytab, YARN, CLUSTER, clOption = "--keytab"), // Other options - OptionAssigner(args.executorCores, YARN, CLUSTER, sysProp = "spark.executor.cores"), - OptionAssigner(args.executorCores, STANDALONE, ALL_DEPLOY_MODES, + OptionAssigner(args.executorCores, STANDALONE | YARN, ALL_DEPLOY_MODES, sysProp = "spark.executor.cores"), OptionAssigner(args.executorMemory, STANDALONE | MESOS | YARN, ALL_DEPLOY_MODES, sysProp = "spark.executor.memory"),