SparkSql插入ES时报错解决（针对增加用户权限之后的报错解决）

org.elasticsearch.hadoop.EsHadoopIllegalArgumentException: Cannot detect ES version - typically this happens if the network/Elasticsearch cluster is not accessible or when targeting a WAN/Cloud instance without the proper setting ‘es.nodes.wan.only’

以下是报错信息

Exception in thread "main" org.elasticsearch.hadoop.EsHadoopIllegalArgumentException: Cannot detect ES version - typically this happens if the network/Elasticsearch cluster is not accessible or when targeting a WAN/Cloud instance without the proper setting 'es.nodes.wan.only'

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.elasticsearch.hadoop.rest.InitializationUtils.discoverClusterInfo(InitializationUtils.java:340)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.elasticsearch.spark.sql.EsSparkSQL$.saveToEs(EsSparkSQL.scala:97)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.elasticsearch.spark.sql.ElasticsearchRelation.insert(DefaultSource.scala:620)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.elasticsearch.spark.sql.DefaultSource.createRelation(DefaultSource.scala:107)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:45)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:86)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:80)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:80)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:676)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:676)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:78)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:125)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:73)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:676)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:285)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:271)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:229)

15-11-2022 14:43:56 CST playrecordtoes INFO - at com.pukka.bigdata.dealwith.toes.SparkToEs$.main(SparkToEs.scala:112)

15-11-2022 14:43:56 CST playrecordtoes INFO - at com.pukka.bigdata.dealwith.toes.SparkToEs.main(SparkToEs.scala)

15-11-2022 14:43:56 CST playrecordtoes INFO - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

15-11-2022 14:43:56 CST playrecordtoes INFO - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)

15-11-2022 14:43:56 CST playrecordtoes INFO - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

15-11-2022 14:43:56 CST playrecordtoes INFO - at java.lang.reflect.Method.invoke(Method.java:498)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:845)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:161)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:184)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:920)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:929)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

15-11-2022 14:43:56 CST playrecordtoes INFO - Caused by: org.elasticsearch.hadoop.rest.EsHadoopInvalidRequest: org.elasticsearch.hadoop.rest.EsHadoopRemoteException: security_exception: missing authentication credentials for REST request [/]

15-11-2022 14:43:56 CST playrecordtoes INFO - null

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.elasticsearch.hadoop.rest.RestClient.checkResponse(RestClient.java:469)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.elasticsearch.hadoop.rest.RestClient.execute(RestClient.java:426)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.elasticsearch.hadoop.rest.RestClient.execute(RestClient.java:388)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.elasticsearch.hadoop.rest.RestClient.execute(RestClient.java:392)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.elasticsearch.hadoop.rest.RestClient.get(RestClient.java:168)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.elasticsearch.hadoop.rest.RestClient.mainInfo(RestClient.java:735)

15-11-2022 14:43:56 CST playrecordtoes INFO - at org.elasticsearch.hadoop.rest.InitializationUtils.discoverClusterInfo(InitializationUtils.java:330)

报错解决 在配置参数中加入 “es.net.http.auth.user”、 “es.net.http.auth.pass” 即可

val options = Map(

//如果没有index,则新建

"es.index.auto.create" -> "true",

"es.nodes.wan.only" -> "true",

"es.nodes" -> "192.168.20.131,192.168.20.130",

//"es.nodes" -> "10.0.6.56",

//"es.nodes" -> "60.167.225.130",

//es的端口

"es.port" -> "9200",

"es.net.http.auth.user" ->"elastic",

"es.net.http.auth.pass" ->"test@2022!"

)

val esPath=s"play_record_$year$month/_doc"

dataFrame

.write

.format("org.elasticsearch.spark.sql")

.options(options)

.mode(SaveMode.Append)

//index为epg_itv_play_record_$year$month

.save(esPath)

spark.stop()

精彩内容

评论可见，请评论后查看内容，谢谢！！！
 您阅读本篇文章共花了：