报错日志：

2022-05-23 14:33:05,581 INFO yarn.ApplicationMaster: Unregistering ApplicationMaster with FAILED (diag message: User class threw exception: java.lang.IllegalStateException: Promise already completed.

at scala.concurrent.Promise.complete(Promise.scala:53)

at scala.concurrent.Promise.complete$(Promise.scala:52)

at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:187)

at scala.concurrent.Promise.success(Promise.scala:86)

at scala.concurrent.Promise.success$(Promise.scala:86)

at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:187)

at org.apache.spark.deploy.yarn.ApplicationMaster.org$apache$spark$deploy$yarn$ApplicationMaster$$sparkContextInitialized(ApplicationMaster.scala:404)

at org.apache.spark.deploy.yarn.ApplicationMaster$.sparkContextInitialized(ApplicationMaster.scala:895)

at org.apache.spark.scheduler.cluster.YarnClusterScheduler.postStartHook(YarnClusterScheduler.scala:32)

at org.apache.spark.SparkContext.(SparkContext.scala:608)

at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2555)

at org.apache.spark.sql.SparkSession$Builder.$anonfun$getOrCreate$1(SparkSession.scala:930)

at scala.Option.getOrElse(Option.scala:189)

at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:921)

at MovieDataAnalysisBySparkRDD$.demand_1_by_sparkSql(Run.scala:153)

at Run$.main(Run.scala:187)

at Run.main(Run.scala)

at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)

at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

at java.lang.reflect.Method.invoke(Method.java:498)

at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:728)

)

报错原因：一个application 应该只能有一个SparkSession对象或者SparkConext对象。可以将spark或sc在主函数中创建，然后再以参数的形式传递。

object Run {

def main(args: Array[String]): Unit = {

// 1 创建上下文环境配置对象

val conf: SparkConf = new SparkConf()

.setAppName("movie_data_analysis")

.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") //替换默认序列化机制

.registerKryoClasses(Array(classOf[MovieQuota])) //注册使用kryo序列化的自定义类

.setMaster("yarn")

// 2 创建SparkSession对象

val spark: SparkSession = SparkSession.builder().config(conf).getOrCreate()

println("------------字符编码------------")

println(System.getProperty("file.encoding"))

MovieDataAnalysisBySparkRDD.demand_1_by_sparkRdd(spark.sparkContext)

MovieDataAnalysisBySparkRDD.demand_1_by_sparkSql(spark)

spark.stop()

}

}

参考链接

评论可见，请评论后查看内容，谢谢！！！
 您阅读本篇文章共花了：