我刚刚学者用python连接spark，操作很简单。关联查询集群上数据，因为好像spark.SQL不能写子查询，我就分开写的，很奇怪，第一个 建表语句没有问题，但是第二个建表语句总是执行不下去，我还试过注释掉第一个语句，但是第二个语句还是出错,why?。
import sys
import os
from pyspark import SparkContext
from pyspark import SparkConf
from pyspark.sql import HiveContext
from pyspark import SQLContext
import pandas as pd
os.environ["SPARK_HOME"]='/opt/hadoopclient/Spark/spark/'
sys.path.append('/opt/hadoopclient/Spark/spark/python')
string_test='pyspark_test'
conf=SparkConf().setAppName(string_test).setMaster('local[*]')
sc=SparkContext(conf=conf)
sparksql=HiveContext(sc)
sparksql.sql("drop table if exists sfz616_result1")
sparksql.sql("drop table if exists sfz616_result2")
第一个建表语句
sparksql.sql("create table sfz616_result1 as \
select a.zjhm,a.fcrq,a.hcpcc,a.hcpcfd,a.hcpmdd from bigdata.gd_tl_gtsmzxx a,sfz_616 b where a.zjhm=b.sfz")
第二个建表语句
sparksql.sql("create table sfz616_result2 as select zjhm,max(fcrq) from sfz616_result1 group by zjhm")
sparksql.sql("create table sfz616_result3 as select distinct t1.* from sfz616_result1 t1,sfz616_result2 t2 where t1.zjhm=t2.zjhm and t1.fcrq=t2.fcrq2")
df=sparksql.sql("select * from sfz616_result2")
df1=df.toPandas()
df1.to_excel("./2.xlsx")
主要报错
/ 3003]2019-10-19 18:04:26,677 | ERROR | Executor task launch worker-27 | Exception in task 9.0 in stage 3.0 (TID 3048) | org.apache.spark.Logging$class.logError(Logging.scala:96)
java.io.FileNotFoundException: /tmp/blockmgr-9b005652-860d-4552-9dbd-8011b0be57a7/1d/temp_shuffle_36fb36cc-d273-4a4d-b6de-eae2a72299d7 (Too many open files)
| ERROR | Executor task launch worker-18 | Uncaught exception while reverting partial writes to file /tmp/blockmgr-9b005652-860d-4552-9dbd-8011b0be57a7/0d/temp_shuffle_37a3e05a-c49b-4d2b-a68a-07e8902da71e | org.apache.spark.Logging$class.logError(Logging.scala:96)
java.io.FileNotFoundException: /tmp/blockmgr-9b005652-860d-4552-9dbd-8011b0be57a7/0d/temp_shuffle_37a3e05a-c49b-4d2b-a68a-07e8902da71e (No such file or directory)
