Python 在PySpark中运行collect（）操作时出错

在使用Python练习Apache Spark时，我在运行collect（）命令时出错。 我正在Jupyter笔记本中尝试以下代码

vehicledata = sc.textFile("/home/newuser/Desktop/vehicle_data.csv")
vehicledata.collect()

[u'suzuki,gas,std,two,hatchback,fwd,four,69,4900,31,36,5118',
 u'volvo,gas,std,two,hatchback,fwd,four,75,5400,37,40,3700',
 u'chevrolet,gas,std,two,sedan,fwd,four,65,5600,30,32,4800',
 u'toyota,gas,std,two,sedan,fwd,four,45,7800,45,47,3900',
 u'tata,gas,std,two,hatchback,fwd,four,40,5900,50,52,7800',
 u'tata,gas,std,four,hatchback,fwd,four,20,3100,30,35,2355',
 u'suzuki,gas,std,four,hatchback,fwd,four,20,2500,28,30,6355',
 u'chevrolet,gas,std,two,sedan,fwd,four,47,4900,23,27,2300',
 u'hyundai,cell,std,two,SUV,fwd,four,55,6500,20,25,7000',
 u'benz,gas,std,two,SUV,fwd,four,60,7300,26,30,7900',
 u'']

在这里，我通过拆分车辆品牌和第八个字段值来获取两个值

cyldata = vehicledata.map(lambda x: (x.split(",")[0], x.split(",")[7]))
cyldata.collect()

当我运行上面的collect（）命令时，我得到了以下错误。 因为我是Spark的新手，所以我无法辨别错误

我可以知道是否有任何错误的编码或错误，由于不正确的spark anaconda安装

---------------------------------------------------------------------------
Py4JJavaError                             Traceback (most recent call last)
<ipython-input-9-795ae9e596f7> in <module>()
----> 1 cyldata.collect()

/home/newuser/spark/python/pyspark/rdd.pyc in collect(self)
    771         """
    772         with SCCallSiteSync(self.context) as css:
--> 773             port = self.ctx._jvm.PythonRDD.collectAndServe(self._jrdd.rdd())
    774         return list(_load_from_socket(port, self._jrdd_deserializer))
    775 

/home/newuser/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py in __call__(self, *args)
    536         answer = self.gateway_client.send_command(command)
    537         return_value = get_return_value(answer, self.gateway_client,
--> 538                 self.target_id, self.name)
    539 
    540         for temp_arg in temp_args:

/home/newuser/spark/python/pyspark/sql/utils.pyc in deco(*a, **kw)
     34     def deco(*a, **kw):
     35         try:
---> 36             return f(*a, **kw)
     37         except py4j.protocol.Py4JJavaError as e:
     38             s = e.java_exception.toString()

/home/newuser/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
    298                 raise Py4JJavaError(
    299                     'An error occurred while calling {0}{1}{2}.\n'.
--> 300                     format(target_id, '.', name), value)
    301             else:
    302                 raise Py4JError(

Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.collectAndServe.
: org.apache.spark.SparkException: Job aborted due to stage failure: Task 1 in stage 2.0 failed 1 times, most recent failure: Lost task 1.0 in stage 2.0 (TID 5, localhost): org.apache.spark.api.python.PythonException: Traceback (most recent call last):
  File "/home/newuser/spark/python/lib/pyspark.zip/pyspark/worker.py", line 111, in main
    process()
  File "/home/newuser/spark/python/lib/pyspark.zip/pyspark/worker.py", line 106, in process
    serializer.dump_stream(func(split_index, iterator), outfile)
  File "/home/newuser/spark/python/lib/pyspark.zip/pyspark/serializers.py", line 263, in dump_stream
    vs = list(itertools.islice(iterator, batch))
  File "<ipython-input-8-1adc0c950e09>", line 1, in <lambda>
IndexError: list index out of range

    at org.apache.spark.api.python.PythonRunner$$anon$1.read(PythonRDD.scala:166)
    at org.apache.spark.api.python.PythonRunner$$anon$1.next(PythonRDD.scala:129)
    at org.apache.spark.api.python.PythonRunner$$anon$1.next(PythonRDD.scala:125)
    at org.apache.spark.InterruptibleIterator.next(InterruptibleIterator.scala:43)
    at scala.collection.Iterator$class.foreach(Iterator.scala:727)
    at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)
    at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48)
    at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103)
    at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47)
    at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273)
    at org.apache.spark.InterruptibleIterator.to(InterruptibleIterator.scala:28)
    at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265)
    at org.apache.spark.InterruptibleIterator.toBuffer(InterruptibleIterator.scala:28)
    at scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252)
    at org.apache.spark.InterruptibleIterator.toArray(InterruptibleIterator.scala:28)
    at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:909)
    at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:909)
    at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850)
    at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850)
    at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
    at org.apache.spark.scheduler.Task.run(Task.scala:88)
    at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:745)

Driver stacktrace:
    at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1283)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1271)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1270)
    at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
    at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
    at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1270)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697)
    at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:697)
    at scala.Option.foreach(Option.scala:236)
    at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:697)
    at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1496)
    at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1458)
    at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1447)
    at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
    at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567)
    at org.apache.spark.SparkContext.runJob(SparkContext.scala:1824)
    at org.apache.spark.SparkContext.runJob(SparkContext.scala:1837)
    at org.apache.spark.SparkContext.runJob(SparkContext.scala:1850)
    at org.apache.spark.SparkContext.runJob(SparkContext.scala:1921)
    at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:909)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:147)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:108)
    at org.apache.spark.rdd.RDD.withScope(RDD.scala:310)
    at org.apache.spark.rdd.RDD.collect(RDD.scala:908)
    at org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:405)
    at org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
    at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
    at py4j.Gateway.invoke(Gateway.java:259)
    at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
    at py4j.commands.CallCommand.execute(CallCommand.java:79)
    at py4j.GatewayConnection.run(GatewayConnection.java:207)
    at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.spark.api.python.PythonException: Traceback (most recent call last):
  File "/home/newuser/spark/python/lib/pyspark.zip/pyspark/worker.py", line 111, in main
    process()
  File "/home/newuser/spark/python/lib/pyspark.zip/pyspark/worker.py", line 106, in process
    serializer.dump_stream(func(split_index, iterator), outfile)
  File "/home/newuser/spark/python/lib/pyspark.zip/pyspark/serializers.py", line 263, in dump_stream
    vs = list(itertools.islice(iterator, batch))
  File "<ipython-input-8-1adc0c950e09>", line 1, in <lambda>
IndexError: list index out of range

    at org.apache.spark.api.python.PythonRunner$$anon$1.read(PythonRDD.scala:166)
    at org.apache.spark.api.python.PythonRunner$$anon$1.next(PythonRDD.scala:129)
    at org.apache.spark.api.python.PythonRunner$$anon$1.next(PythonRDD.scala:125)
    at org.apache.spark.InterruptibleIterator.next(InterruptibleIterator.scala:43)
    at scala.collection.Iterator$class.foreach(Iterator.scala:727)
    at org.apache.spark.InterruptibleIterator.foreach(InterruptibleIterator.scala:28)
    at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48)
    at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103)
    at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47)
    at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273)
    at org.apache.spark.InterruptibleIterator.to(InterruptibleIterator.scala:28)
    at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265)
    at org.apache.spark.InterruptibleIterator.toBuffer(InterruptibleIterator.scala:28)
    at scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252)
    at org.apache.spark.InterruptibleIterator.toArray(InterruptibleIterator.scala:28)
    at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:909)
    at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:909)
    at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850)
    at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1850)
    at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
    at org.apache.spark.scheduler.Task.run(Task.scala:88)
    at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    ... 1 more

---------------------------------------------------------------------------
Py4JJavaError回溯（最近一次调用）
在（）
---->1.cyldata.collect（）
/home/newuser/spark/python/pyspark/rdd.pyc in collect（self）
771         """
772使用SCCallSiteSync（self.context）作为css：
-->773 port=self.ctx.\u jvm.PythonRDD.collectAndServe（self.\u jrdd.rdd（））
774返回列表（_从_套接字加载（端口，self._jrdd_反序列化器））
775
/home/newuser/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py in____调用（self，*args）
536 answer=self.gateway\u client.send\u命令（command）
537返回值=获取返回值（应答，self.gateway\u客户端，
-->538 self.target_id，self.name）
539
540对于临时参数中的临时参数：
/装饰中的home/newuser/spark/python/pyspark/sql/utils.pyc（*a，**kw）
34 def装饰（*a，**千瓦）：
35尝试：
--->36返回f（*a，**kw）
37除py4j.protocol.Py4JJavaError为e外：
38 s=e.java_exception.toString（）
/获取返回值（应答、网关客户端、目标id、名称）中的home/newuser/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py
298 raise Py4JJavaError(
299'调用{0}{1}{2}时出错。\n'。
-->300格式（目标id，，，，名称），值）
301其他：
302升起Py4JError(
Py4JJavaError:调用z:org.apache.spark.api.python.PythonRDD.collectAndServe时出错。
：org.apache.spark.sparkeexception:作业因阶段失败而中止：阶段2.0中的任务1失败1次，最近的失败：阶段2.0中的任务1.0丢失（TID 5，本地主机）：org.apache.spark.api.python.python异常：回溯（最近一次调用）：
文件“/home/newuser/spark/python/lib/pyspark.zip/pyspark/worker.py”，主文件第111行
过程（）
文件“/home/newuser/spark/python/lib/pyspark.zip/pyspark/worker.py”，第106行，正在处理中
serializer.dump_流（func（拆分索引，迭代器），outfile）
文件“/home/newuser/spark/python/lib/pyspark.zip/pyspark/serializers.py”，第263行，在dump_流中
vs=列表（itertools.islice（迭代器，批处理））
文件“”，第1行，在
索引器：列表索引超出范围
位于org.apache.spark.api.python.PythonRunner$$anon$1.read（PythonRDD.scala:166）
位于org.apache.spark.api.python.PythonRunner$$anon$1.next（PythonRDD.scala:129）
位于org.apache.spark.api.python.PythonRunner$$anon$1.next（PythonRDD.scala:125）
在org.apache.spark.interruptblediator.next（interruptblediator.scala:43）
位于scala.collection.Iterator$class.foreach（Iterator.scala:727）
在org.apache.spark.interruptblediator.foreach（interruptblediator.scala:28）
在scala.collection.generic.growtable$class.$plus$plus$eq（growtable.scala:48）
在scala.collection.mutable.ArrayBuffer.$plus$plus$eq（ArrayBuffer.scala:103）
在scala.collection.mutable.ArrayBuffer.$plus$plus$eq（ArrayBuffer.scala:47）
在scala.collection.TraversableOnce$class.to处（TraversableOnce.scala:273）
在org.apache.spark.interruptableiterator.to（interruptableiterator.scala:28）
在scala.collection.TraversableOnce$class.toBuffer处（TraversableOnce.scala:265）
在org.apache.spark.interruptableiterator.toBuffer上（interruptableiterator.scala:28）
位于scala.collection.TraversableOnce$class.toArray（TraversableOnce.scala:252）
在org.apache.spark.interruptableiterator.toArray上（interruptableiterator.scala:28）
位于org.apache.spark.rdd.rdd$$anonfun$collect$1$$anonfun$12.apply（rdd.scala:909）
位于org.apache.spark.rdd.rdd$$anonfun$collect$1$$anonfun$12.apply（rdd.scala:909）
位于org.apache.spark.SparkContext$$anonfun$runJob$5.apply（SparkContext.scala:1850）
位于org.apache.spark.SparkContext$$anonfun$runJob$5.apply（SparkContext.scala:1850）
位于org.apache.spark.scheduler.ResultTask.runTask（ResultTask.scala:66）
位于org.apache.spark.scheduler.Task.run（Task.scala:88）
位于org.apache.spark.executor.executor$TaskRunner.run（executor.scala:214）
位于java.util.concurrent.ThreadPoolExecutor.runWorker（ThreadPoolExecutor.java:1142）
位于java.util.concurrent.ThreadPoolExecutor$Worker.run（ThreadPoolExecutor.java:617）
运行（Thread.java:745）
驱动程序堆栈跟踪：
位于org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages（DAGScheduler.scala:1283）
位于org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply（DAGScheduler.scala:1271）
位于org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply（DAGScheduler.scala:1270）
位于scala.collection.mutable.resizeblearray$class.foreach（resizeblearray.scala:59）
位于scala.collection.mutable.ArrayBuffer.foreach（ArrayBuffer.scala:47）
位于org.apache.spark.scheduler.DAGScheduler.abortStage（DAGScheduler.scala:1270）
位于org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply（DAGScheduler.scala:697）
位于org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply（DAGScheduler.scala:697）
位于scala.Option.foreach（Option.scala:236）
位于org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed（DAGScheduler.scala:697）
位于org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive（DAGScheduler.scala:1496）
位于org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive（DAGScheduler.scala:1458）