使用spark 向Doris插入数据,报:node and exceeded the max retry times
3个datefarm 做union 并做repartition(200)
求big-old讲解
23/01/17 21:49:50 INFO scheduler.TaskSetBlacklist: Blacklisting worker50.center.lon for stage 123
23/01/17 21:49:50 WARN scheduler.TaskSetManager: Lost task 28.0 in stage 123.0 (TID 10275, worker50.center.lon, executor 3): java.io.IOException: Failed to load data on BE: http://xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx? node and exceeded the max retry times.
at org.apache.doris.spark.sql.DorisSourceProvider$$anonfun$createRelation$1$$anonfun$org$apache$doris$spark$sql$DorisSourceProvider$$anonfun$$flush$1$1.apply$mcV$sp(DorisSourceProvider.scala:118)
at scala.util.control.Breaks.breakable(Breaks.scala:38)
at org.apache.doris.spark.sql.DorisSourceProvider$$anonfun$createRelation$1.org$apache$doris$spark$sql$DorisSourceProvider$$anonfun$$flush$1(DorisSourceProvider.scala:92)
at org.apache.doris.spark.sql.DorisSourceProvider$$anonfun$createRelation$1.apply(DorisSourceProvider.scala:83)
at org.apache.doris.spark.sql.DorisSourceProvider$$anonfun$createRelation$1.apply(DorisSourceProvider.scala:68)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$28.apply(RDD.scala:935)
at org.apache.spark.rdd.RDD$$anonfun$foreachPartition$1$$anonfun$apply$28.apply(RDD.scala:935)
at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2121)
at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:2121)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:121)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$11.apply(Executor.scala:407)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1408)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:413)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)