Warning: file_get_contents(/data/phpspider/zhask/data//catemap/3/apache-spark/6.json): failed to open stream: No such file or directory in /data/phpspider/zhask/libs/function.php on line 167

Warning: Invalid argument supplied for foreach() in /data/phpspider/zhask/libs/tag.function.php on line 1116

Notice: Undefined index: in /data/phpspider/zhask/libs/function.php on line 180

Warning: array_chunk() expects parameter 1 to be array, null given in /data/phpspider/zhask/libs/function.php on line 181
Python 我不知道如果你第一次输入多个参数为什么不起作用,但是当你从一个参数开始,然后继续添加时,它就起作用了_Python_Apache Spark_Pyspark_Apache Spark Mllib - Fatal编程技术网

Python 我不知道如果你第一次输入多个参数为什么不起作用,但是当你从一个参数开始,然后继续添加时,它就起作用了

Python 我不知道如果你第一次输入多个参数为什么不起作用,但是当你从一个参数开始,然后继续添加时,它就起作用了,python,apache-spark,pyspark,apache-spark-mllib,Python,Apache Spark,Pyspark,Apache Spark Mllib,这不是一个永久的、伟大的解决方案,但对我有效我用rf.featureSubsetStrategy、['auto'、'onethird']替换了它,但我得到了同样的错误。然后,我从ParamGridBuilder中删除了rf.featureSubsetStrategy、['auto'、'onethird'],再次出现了相同的错误。@mikeL您的代码存在与ParamGrid无关的其他问题,但我确信,如果您修复了此特定问题,您不会出现相同的错误。是的,这是一个不同的错误,是的,还有其他问题。数据框的

这不是一个永久的、伟大的解决方案,但对我有效

我用rf.featureSubsetStrategy、['auto'、'onethird']替换了它,但我得到了同样的错误。然后,我从ParamGridBuilder中删除了rf.featureSubsetStrategy、['auto'、'onethird'],再次出现了相同的错误。@mikeL您的代码存在与
ParamGrid
无关的其他问题,但我确信,如果您修复了此特定问题,您不会出现相同的错误。是的,这是一个不同的错误,是的,还有其他问题。数据框的功能和标签似乎位于错误的位置。感谢所有反馈。我仍然没有让它运行,但你解决了我问的问题,还有一些其他问题!谢谢
from pyspark.ml import Pipeline
from pyspark.ml.classification import RandomForestClassifier
from pyspark.ml.tuning import CrossValidator, ParamGridBuilder
from pyspark.ml.evaluation import MulticlassClassificationEvaluator

data = nds.sc.parallelize([
 LabeledPoint(0.0, [0,402,6,0]),
 LabeledPoint(0.0, [3,500,3,0]),
 LabeledPoint(1.0, [1,590,1,1]),
 LabeledPoint(1.0, [3,328,5,0]),
 LabeledPoint(1.0, [4,351,4,0]),
 LabeledPoint(0.0, [2,372,2,0]),
 LabeledPoint(0.0, [4,302,5,0]),
 LabeledPoint(1.0, [1,387,2,0]),
 LabeledPoint(1.0, [1,419,3,0]),
 LabeledPoint(0.0, [1,370,5,0]),
 LabeledPoint(0.0, [1,410,4,0]),
 LabeledPoint(0.0, [2,509,7,1]),
 LabeledPoint(0.0, [1,307,5,0]),
 LabeledPoint(0.0, [0,424,4,1]),
 LabeledPoint(0.0, [1,509,2,1]),
 LabeledPoint(1.0, [3,361,4,0]),
 ])


train=data.toDF(['label','features'])

numfolds =2

rf = RandomForestClassifier(labelCol="label", featuresCol="features")
evaluator = MulticlassClassificationEvaluator()  


paramGrid = ParamGridBuilder().addGrid(rf.maxDepth,      
[4,8,10]).addGrid(rf.impurity, ['entropy','gini']).addGrid(rf.featureSubsetStrategy, [6,8,10]).build()

pipeline = Pipeline(stages=[rf])

crossval = CrossValidator(
    estimator=pipeline,
    estimatorParamMaps=paramGrid,
    evaluator=evaluator,
    numFolds= numfolds)

model = crossval.fit(train)
Py4JJavaError                             Traceback (most recent call last)
<ipython-input-87-7ea70f89086a> in <module>()
 66     numFolds=num)
 67 
---> 68 model = crossval.fit(train)

/opt/spark/current/python/pyspark/ml/pipeline.py in fit(self, dataset, params)
 67                 return self.copy(params)._fit(dataset)
 68             else:
---> 69                 return self._fit(dataset)
 70         else:
 71             raise ValueError("Params must be either a param map or a list/tuple of param maps, "

/opt/spark/current/python/pyspark/ml/tuning.py in _fit(self, dataset)
237             train = df.filter(~condition)
238             for j in range(numModels):
--> 239                 model = est.fit(train, epm[j])
240                 # TODO: duplicate evaluator to take extra params from input
241                 metric = eva.evaluate(model.transform(validation, epm[j]))

/opt/spark/current/python/pyspark/ml/pipeline.py in fit(self, dataset, params)
 65         elif isinstance(params, dict):
 66             if params:
---> 67                 return self.copy(params)._fit(dataset)
 68             else:
 69                 return self._fit(dataset)

/opt/spark/current/python/pyspark/ml/pipeline.py in _fit(self, dataset)
211                     dataset = stage.transform(dataset)
212                 else:  # must be an Estimator
--> 213                     model = stage.fit(dataset)
214                     transformers.append(model)
215                     if i < indexOfLastEstimator:

/opt/spark/current/python/pyspark/ml/pipeline.py in fit(self, dataset, params)
 67                 return self.copy(params)._fit(dataset)
 68             else:
---> 69                 return self._fit(dataset)
 70         else:
 71             raise ValueError("Params must be either a param map or a list/tuple of param maps, "

/opt/spark/current/python/pyspark/ml/wrapper.py in _fit(self, dataset)
130 
131     def _fit(self, dataset):
--> 132         java_model = self._fit_java(dataset)
133         return self._create_model(java_model)
134 

/opt/spark/current/python/pyspark/ml/wrapper.py in _fit_java(self, dataset)
126         :return: fitted Java model
127         """
--> 128         self._transfer_params_to_java()
129         return self._java_obj.fit(dataset._jdf)
130 

/opt/spark/current/python/pyspark/ml/wrapper.py in _transfer_params_to_java(self)
 80         for param in self.params:
 81             if param in paramMap:
---> 82                 pair = self._make_java_param_pair(param, paramMap[param])
 83                 self._java_obj.set(pair)
 84 

/opt/spark/current/python/pyspark/ml/wrapper.py in _make_java_param_pair(self, param, value)
 71         java_param = self._java_obj.getParam(param.name)
 72         java_value = _py2java(sc, value)
---> 73         return java_param.w(java_value)
 74 
 75     def _transfer_params_to_java(self):

   /opt/spark/current/python/lib/py4j-0.9-src.zip/py4j/java_gateway.py in __call__(self, *args)
811         answer = self.gateway_client.send_command(command)
812         return_value = get_return_value(
   --> 813             answer, self.gateway_client, self.target_id, self.name)
814 
815         for temp_arg in temp_args:

/opt/spark/current/python/pyspark/sql/utils.py in deco(*a, **kw)
 43     def deco(*a, **kw):
 44         try:
    ---> 45             return f(*a, **kw)
 46         except py4j.protocol.Py4JJavaError as e:
 47             s = e.java_exception.toString()

/opt/spark/current/python/lib/py4j-0.9-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
306                 raise Py4JJavaError(
307                     "An error occurred while calling {0}{1}{2}.\n".
--> 308                     format(target_id, ".", name), value)
309             else:
310                 raise Py4JError(

Py4JJavaError: An error occurred while calling o1434.w.
: java.lang.ClassCastException: java.lang.Integer cannot be cast to      java.lang.String
at org.apache.spark.ml.tree.RandomForestParams$$anonfun$5.apply(treeParams.scala:340)
at org.apache.spark.ml.param.Param.validate(params.scala:71)
at org.apache.spark.ml.param.ParamPair.<init>(params.scala:509)
at org.apache.spark.ml.param.Param.$minus$greater(params.scala:85)
at org.apache.spark.ml.param.Param.w(params.scala:82)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:381)
at py4j.Gateway.invoke(Gateway.java:259)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:209)
at java.lang.Thread.run(Thread.java:745)
data.toDF()
from operator import attrgetter

data.map(attrgetter("label", "features")).toDF(["some_name", "some_other_name"])
lr_parameter_grid_ = ParamGridBuilder().addGrid(lr.maxIter, [50, 200, 500])\
    .addGrid(lr.regParam, [0, 0.3, 1])\
    .addGrid(lr.elasticNetParam, [0, 0.3, 1]).build()