Apache spark Py4JJavaError键PySpark笔记本中的错误
我对使用PySpark还不熟悉，我正在尝试构建一些东西，列出特定文件夹中的所有文档，将其划分为bucket，最后对所有这些文档运行PandasUDF函数 我已经将代码剥离到了错误仍然发生的最低限度Apache spark Py4JJavaError键PySpark笔记本中的错误,apache-spark,pyspark,Apache Spark,Pyspark,我对使用PySpark还不熟悉，我正在尝试构建一些东西，列出特定文件夹中的所有文档，将其划分为bucket，最后对所有这些文档运行PandasUDF函数 我已经将代码剥离到了错误仍然发生的最低限度 import pandas as pd import os from pyspark.sql.functions import pandas_udf, PandasUDFType from pyspark.sql import functions as F from pyspark.sql.wind
import pandas as pd
import os
from pyspark.sql.functions import pandas_udf, PandasUDFType
from pyspark.sql import functions as F
from pyspark.sql.window import Window
from pyspark.sql.types import *
spark.conf.set('spark.sql.execution.arrow.enabled', 'true')
spark.conf.set('spark.sql.execution.arrow.pyspark.enabled', 'true')
spark.conf.set('spark.sql.execution.arrow.pyspark.fallback.enabled', 'true')
def bucketizer(spark_df, nr_of_buckets, sort_col):
spark_count = spark_df.count()
spark_df = (spark_df
.withColumn('count', F.lit(spark_count))
.withColumn('rank_in_group', F.row_number().over(Window.orderBy(sort_col)))
.withColumn('bucket', F.floor(nr_of_buckets * (F.col('rank_in_group')-1) / F.col('count')+1))
.drop('count')
.drop('rank_in_group')
)
return spark_df
schema = StructType([
StructField('value', StringType(), True)
])
@pandas_udf(schema, PandasUDFType.GROUPED_MAP)
def function(df_docs):
documents = df_docs[0].tolist() # pandas to list
results = pd.DataFrame(documents) # list back to pandas
return results
# Get list of paths to all files in specified directory
documents = ['path_doc1', 'path_doc2', 'path_docn'] # Normally these are document paths
# Read documents into dataframe
dfs = spark.createDataFrame(documents, StringType())
dfs.show()
# Add buckets
dfs_bucketized = bucketizer(dfs, 10, 'value')
# Start processing
spark_results = dfs_bucketized.groupBy('bucket').apply(function)
spark_results.show()
+---------+
| value|
+---------+
|path_doc1|
|path_doc2|
|path_docn|
+---------+
---------------------------------------------------------------------------
Py4JJavaError Traceback (most recent call last)
<ipython-input-4-58b3ff1f2625> in <module>
9 # Start processing
10 spark_results = dfs_bucketized.groupBy('bucket').apply(function)
---> 11 spark_results.show()
/usr/hdp/current/spark2-client/python/pyspark/sql/dataframe.py in show(self, n, truncate, vertical)
348 """
349 if isinstance(truncate, bool) and truncate:
--> 350 print(self._jdf.showString(n, 20, vertical))
351 else:
352 print(self._jdf.showString(n, int(truncate), vertical))
/opt/pyvenv/py36-venv/lib64/python3.6/site-packages/py4j/java_gateway.py in __call__(self, *args)
1255 answer = self.gateway_client.send_command(command)
1256 return_value = get_return_value(
-> 1257 answer, self.gateway_client, self.target_id, self.name)
1258
1259 for temp_arg in temp_args:
/usr/hdp/current/spark2-client/python/pyspark/sql/utils.py in deco(*a, **kw)
61 def deco(*a, **kw):
62 try:
---> 63 return f(*a, **kw)
64 except py4j.protocol.Py4JJavaError as e:
65 s = e.java_exception.toString()
/opt/pyvenv/py36-venv/lib64/python3.6/site-packages/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
326 raise Py4JJavaError(
327 "An error occurred while calling {0}{1}{2}.\n".
--> 328 format(target_id, ".", name), value)
329 else:
330 raise Py4JError(
Py4JJavaError: An error occurred while calling o100.showString.
: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 5.0 failed 4 times, most recent failure: Lost task 0.3 in stage 5.0 (TID 10, usl16226.unix.corp, executor 1): org.apache.spark.api.python.PythonException: Traceback (most recent call last):
File "/opt/pyvenv/py36-venv/lib64/python3.6/site-packages/pandas/core/indexes/base.py", line 2646, in get_loc
return self._engine.get_loc(key)
File "pandas/_libs/index.pyx", line 111, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/index.pyx", line 138, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/hashtable_class_helper.pxi", line 1618, in pandas._libs.hashtable.PyObjectHashTable.get_item
File "pandas/_libs/hashtable_class_helper.pxi", line 1626, in pandas._libs.hashtable.PyObjectHashTable.get_item
KeyError: 0
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/hdp/current/spark2-client/python/pyspark/worker.py", line 229, in main
process()
File "/usr/hdp/current/spark2-client/python/pyspark/worker.py", line 224, in process
serializer.dump_stream(func(split_index, iterator), outfile)
File "/usr/hdp/current/spark2-client/python/pyspark/serializers.py", line 260, in dump_stream
for series in iterator:
File "<string>", line 1, in <lambda>
File "/usr/hdp/current/spark2-client/python/pyspark/worker.py", line 97, in wrapped
result = f(pd.concat(series, axis=1))
File "<ipython-input-3-6f41326fa838>", line 6, in function
File "/opt/pyvenv/py36-venv/lib64/python3.6/site-packages/pandas/core/frame.py", line 2800, in __getitem__
indexer = self.columns.get_loc(key)
File "/opt/pyvenv/py36-venv/lib64/python3.6/site-packages/pandas/core/indexes/base.py", line 2648, in get_loc
return self._engine.get_loc(self._maybe_cast_indexer(key))
File "pandas/_libs/index.pyx", line 111, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/index.pyx", line 138, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/hashtable_class_helper.pxi", line 1618, in pandas._libs.hashtable.PyObjectHashTable.get_item
File "pandas/_libs/hashtable_class_helper.pxi", line 1626, in pandas._libs.hashtable.PyObjectHashTable.get_item
KeyError: 0
at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:298)
at org.apache.spark.sql.execution.python.ArrowPythonRunner$$anon$1.read(ArrowPythonRunner.scala:164)
at org.apache.spark.sql.execution.python.ArrowPythonRunner$$anon$1.read(ArrowPythonRunner.scala:114)
at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:252)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:439)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage3.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$10$$anon$1.hasNext(WholeStageCodegenExec.scala:614)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:253)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:247)
at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:830)
at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:830)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
at org.apache.spark.scheduler.Task.run(Task.scala:109)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1599)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1587)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1586)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1586)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:831)
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:831)
at scala.Option.foreach(Option.scala:257)
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:831)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1820)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1769)
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1758)
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:642)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2034)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2055)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2074)
at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:363)
at org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:38)
at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:3272)
at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2484)
at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2484)
at org.apache.spark.sql.Dataset$$anonfun$52.apply(Dataset.scala:3253)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:77)
at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3252)
at org.apache.spark.sql.Dataset.head(Dataset.scala:2484)
at org.apache.spark.sql.Dataset.take(Dataset.scala:2698)
at org.apache.spark.sql.Dataset.showString(Dataset.scala:254)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:214)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.spark.api.python.PythonException: Traceback (most recent call last):
File "/opt/pyvenv/py36-venv/lib64/python3.6/site-packages/pandas/core/indexes/base.py", line 2646, in get_loc
return self._engine.get_loc(key)
File "pandas/_libs/index.pyx", line 111, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/index.pyx", line 138, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/hashtable_class_helper.pxi", line 1618, in pandas._libs.hashtable.PyObjectHashTable.get_item
File "pandas/_libs/hashtable_class_helper.pxi", line 1626, in pandas._libs.hashtable.PyObjectHashTable.get_item
KeyError: 0
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/hdp/current/spark2-client/python/pyspark/worker.py", line 229, in main
process()
File "/usr/hdp/current/spark2-client/python/pyspark/worker.py", line 224, in process
serializer.dump_stream(func(split_index, iterator), outfile)
File "/usr/hdp/current/spark2-client/python/pyspark/serializers.py", line 260, in dump_stream
for series in iterator:
File "<string>", line 1, in <lambda>
File "/usr/hdp/current/spark2-client/python/pyspark/worker.py", line 97, in wrapped
result = f(pd.concat(series, axis=1))
File "<ipython-input-3-6f41326fa838>", line 6, in function
File "/opt/pyvenv/py36-venv/lib64/python3.6/site-packages/pandas/core/frame.py", line 2800, in __getitem__
indexer = self.columns.get_loc(key)
File "/opt/pyvenv/py36-venv/lib64/python3.6/site-packages/pandas/core/indexes/base.py", line 2648, in get_loc
return self._engine.get_loc(self._maybe_cast_indexer(key))
File "pandas/_libs/index.pyx", line 111, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/index.pyx", line 138, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/hashtable_class_helper.pxi", line 1618, in pandas._libs.hashtable.PyObjectHashTable.get_item
File "pandas/_libs/hashtable_class_helper.pxi", line 1626, in pandas._libs.hashtable.PyObjectHashTable.get_item
KeyError: 0
at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.handlePythonException(PythonRunner.scala:298)
at org.apache.spark.sql.execution.python.ArrowPythonRunner$$anon$1.read(ArrowPythonRunner.scala:164)
at org.apache.spark.sql.execution.python.ArrowPythonRunner$$anon$1.read(ArrowPythonRunner.scala:114)
at org.apache.spark.api.python.BasePythonRunner$ReaderIterator.hasNext(PythonRunner.scala:252)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)
at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:439)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage3.processNext(Unknown Source)
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$10$$anon$1.hasNext(WholeStageCodegenExec.scala:614)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:253)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:247)
at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:830)
at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:830)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
at org.apache.spark.scheduler.Task.run(Task.scala:109)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
+---------+
|价值观|
+---------+
|路径1|
|路径2|
|路径文档|
+---------+
---------------------------------------------------------------------------
Py4JJavaError回溯（最近一次调用）
在里面
9#开始处理
10 spark\u results=dfs\u bucketized.groupBy（'bucket'）。应用（函数）
--->11 spark_结果显示（）
/show中的usr/hdp/current/spark2 client/python/pyspark/sql/dataframe.py（self，n，truncate，vertical）
348 """
349如果存在（截断，布尔）和截断：
-->350打印（self._jdf.showString（n，20，垂直））
351其他：
352打印（self._jdf.showString（n，int（截断），垂直））
/opt/pyvenv/py36-venv/lib64/python3.6/site-packages/py4j/java_-gateway.py in____调用（self，*args）
1255 answer=self.gateway\u client.send\u命令（command）
1256返回值=获取返回值(
->1257应答，self.gateway_客户端，self.target_id，self.name）
1258
1259对于临时参数中的临时参数：
/装饰中的usr/hdp/current/spark2-client/python/pyspark/sql/utils.py（*a，**kw）
61 def装饰（*a，**千瓦）：
62尝试：
--->63返回f（*a，**kw）
64除py4j.protocol.Py4JJavaError外的其他错误为e:
65 s=e.java_exception.toString（）
/获取返回值中的opt/pyvenv/py36-venv/lib64/python3.6/site-packages/py4j/protocol.py（应答、网关客户端、目标id、名称）
326 raise Py4JJavaError(
327“调用{0}{1}{2}时出错。\n”。
-->328格式（目标id，“.”，名称），值）
329其他：
330升起Py4JError(
Py4JJavaError:调用o100.showString时出错。
：org.apache.spark.sparkeexception:作业因阶段失败而中止：阶段5.0中的任务0失败4次，最近的失败：阶段5.0中的任务0.3丢失（TID 10，usl16226.unix.corp，executor 1）：org.apache.spark.api.python.python异常：回溯（最近一次调用）：
文件“/opt/pyvenv/py36-venv/lib64/python3.6/site-packages/pandas/core/index/base.py”，第2646行，在get_loc中
返回发动机。获取位置（钥匙）
文件“pandas/_libs/index.pyx”，第111行，在pandas._libs.index.IndexEngine.get_loc中
文件“pandas/_libs/index.pyx”，第138行，在pandas._libs.index.IndexEngine.get_loc中
文件“pandas/_libs/hashtable_class_helper.pxi”，第1618行，在pandas._libs.hashtable.PyObjectHashTable.get_项中
文件“pandas/_libs/hashtable_class_helper.pxi”，第1626行，在pandas._libs.hashtable.PyObjectHashTable.get_项中
关键错误：0
在处理上述异常期间，发生了另一个异常：
回溯（最近一次呼叫最后一次）：
文件“/usr/hdp/current/spark2 client/python/pyspark/worker.py”，第229行，在main中
过程（）
文件“/usr/hdp/current/spark2 client/python/pyspark/worker.py”，第224行，正在处理中
serializer.dump_流（func（拆分索引，迭代器），outfile）
文件“/usr/hdp/current/spark2 client/python/pyspark/serializers.py”，第260行，在dump_流中
对于迭代器中的系列：
文件“”，第1行，在
文件“/usr/hdp/current/spark2 client/python/pyspark/worker.py”，第97行，已包装
结果=f（局部浓度（系列，轴=1））
文件“”，第6行，在函数中
文件“/opt/pyvenv/py36-venv/lib64/python3.6/site-packages/pandas/core/frame.py”，第2800行，在__
indexer=self.columns.get_loc（键）
文件“/opt/pyvenv/py36-venv/lib64/python3.6/site-packages/pandas/core/index/base.py”，第2648行，在get_loc中
返回self.\u引擎。获取\u loc（self.\u可能\u cast\u索引器（键））
文件“pandas/_libs/index.pyx”，第111行，在pandas._libs.index.IndexEngine.get_loc中
文件“pandas/_libs/index.pyx”，第138行，在pandas._libs.index.IndexEngine.get_loc中
文件“pandas/_libs/hashtable_class_helper.pxi”，第1618行，在pandas._libs.hashtable.PyObjectHashTable.get_项中
文件“pandas/_libs/hashtable_class_helper.pxi”，第1626行，在pandas._libs.hashtable.PyObjectHashTable.get_项中
关键错误：0
位于org.apache.spark.api.python.BasePythonRunner$readeriator.handlePythonException（PythonRunner.scala:298）
位于org.apache.spark.sql.execution.python.ArrowPythonRunner$$anon$1.read（ArrowPythonRunner.scala:164）
位于org.apache.spark.sql.execution.python.ArrowPythonRunner$$anon$1.read（ArrowPythonRunner.scala:114）
位于org.apache.spark.api.python.BasePythonRunner$readerierator.hasNext（PythonRunner.scala:252）
在org.apache.spark.interruptblediator.hasNext（interruptblediator.scala:37）
位于scala.collection.Iterator$$anon$12.hasNext（Iterator.scala:439）
位于scala.collection.Iterator$$anon$11.hasNext（Iterator.scala:408）
位于org.apache.spark.sql.catalyst.expressions.GeneratedClass$GenerateEditorForCodeGenStage3.processNext（未知源）
位于org.apache.spark.sql.execution.BufferedRowIterator.hasNext（BufferedRowIterator.java:43）
位于org.apache.spark.sql.execution.whisttagecodegenexec$$anonfun$10$$anon$1.hasNext（whisttagecodegenexec.scala:614）
位于org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply（SparkPlan.scala:253）
位于org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply（SparkPlan.scala:247）
位于org.apache.spark.rdd.rdd$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply（rdd.scala:830）
位于org.apache.spark.rdd.rdd$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply（rdd.scala:830）
在org.apache.spark.rdd.MapPartitionsRDD.compute上（MapPartitionsRDD.scala:38）
在org.apache.spark.rdd.rdd.computeOrReadCheckpoint（rdd.scala:324）
位于org.apache.spark.rdd.rdd.iterator（rdd.scala:288）
位于org.apache.spark.rdd.MapPartitionsRDD.comp