오류가 발생하면 오류의 아래 부분에 제시하는데로 yarn.scheduler.maximum-allocation-mb' and/or 'yarn.nodemanager.resource.memory-mb 혹은(또는) yarn.scheduler.maximum-allocation-mb' and/or 'yarn.nodemanager.resource.memory-mb의 값을 증가 시켜 지정해준다.

------------오류내용

18/06/08 14:56:48 WARN cluster.YarnSchedulerBackend$YarnSchedulerEndpoint: Attempted to request executors before the AM has registered!

18/06/08 14:56:48 ERROR util.Utils: Uncaught exception in thread main

java.lang.NullPointerException

        at org.apache.spark.network.shuffle.ExternalShuffleClient.close(ExternalShuffleClient.java:152)

        at org.apache.spark.storage.BlockManager.stop(BlockManager.scala:1338)

        at org.apache.spark.SparkEnv.stop(SparkEnv.scala:97)

        at org.apache.spark.SparkContext$$anonfun$stop$12.apply$mcV$sp(SparkContext.scala:1786)

        at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1221)

        at org.apache.spark.SparkContext.stop(SparkContext.scala:1785)

        at org.apache.spark.SparkContext.<init>(SparkContext.scala:610)

        at org.apache.spark.repl.SparkILoop.createSparkContext(SparkILoop.scala:1022)

        at $line3.$read$$iwC$$iwC.<init>(<console>:15)

        at $line3.$read$$iwC.<init>(<console>:25)

        at $line3.$read.<init>(<console>:27)

        at $line3.$read$.<init>(<console>:31)

        at $line3.$read$.<clinit>(<console>)

        at $line3.$eval$.<init>(<console>:7)

        at $line3.$eval$.<clinit>(<console>)

        at $line3.$eval.$print(<console>)

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorI

.....

java.lang.IllegalArgumentException: Required executor memory (1024+384 MB) is above the max threshold (1024 MB) of this cluster! Please check the values of 'yarn.scheduler.maximum-allocation-mb' and/or 'yarn.nodemanager.resource.memory-mb'.

        at org.apache.spark.deploy.yarn.Client.verifyClusterResources(Client.scala:281)

        at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:140)

        at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:57)

        at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:151)

        at org.apache.spark.SparkContext.<init>(SparkContext.scala:538)

        at org.apache.spark.repl.SparkILoop.createSparkContext(SparkILoop.scala:1022)

        at $iwC$$iwC.<init>(<console>:15)

        at $iwC.<init>(<console>:25)

        at <init>(<console>:27)

        at .<init>(<console>:31)

        at .<clinit>(<console>)

        at .<init>(<console>:7)

        at .<clinit>(<console>)

        at $print(<console>)

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)

        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

        at java.lang.reflect.Method.invoke(Method.java:606)

        at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1045)

        at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1326)

        at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:821)

        at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:852)

        at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:800)

        at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:857)

        at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902)

        at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814)

        at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:125)

        at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:124)

        at org.apache.spark.repl.SparkIMain.beQuietDuring(SparkIMain.scala:305)

        at org.apache.spark.repl.SparkILoopInit$class.initializeSpark(SparkILoopInit.scala:124)

        at org.apache.spark.repl.SparkILoop.initializeSpark(SparkILoop.scala:64)