Python 尝试使用Gridsearchcv时出现内存错误
我正在开发一个定制的估计器,它应该在sklearn的gridsearchcv函数中传递。我现在已经创建了估计器,但是遇到了内存错误。在下面的代码中,您将看到一些常量,如“KxRange[0]”或数组,如retain_rate。它们只是用一些随机值预先定义的。 这是我的密码:Python 尝试使用Gridsearchcv时出现内存错误,python,scikit-learn,memory-leaks,gridsearchcv,Python,Scikit Learn,Memory Leaks,Gridsearchcv,我正在开发一个定制的估计器,它应该在sklearn的gridsearchcv函数中传递。我现在已经创建了估计器,但是遇到了内存错误。在下面的代码中,您将看到一些常量,如“KxRange[0]”或数组,如retain_rate。它们只是用一些随机值预先定义的。 这是我的密码: # sklearn grid search from sklearn.model_selection import GridSearchCV # import the base estimator from sklearn.
# sklearn grid search
from sklearn.model_selection import GridSearchCV
# import the base estimator
from sklearn.base import BaseEstimator, RegressorMixin
# define my own estimator
class MyEstimator(BaseEstimator,RegressorMixin):
# define constructor
# possible tau: int/float
# other parameters: array of int/floats, length 9
def __init__(self, tau=0, \
K1=K1Range[0], K2=K2Range[0], K3=K3Range[0], K4=K4Range[0], K5=K5Range[0], K6=K6Range[0], K7=K7Range[0], K8=K8Range[0], K9=K9Range[0], \
S1=0, S2=0, S3=0, S4=0, S5=0, S6=0, S7=0, S8=0, S9=0, \
alpha1=1, alpha2=1, alpha3=1, alpha4=1, alpha5=1, alpha6=1, alpha7=1, alpha8=1, alpha9=1, \
beta1=1, beta2=1, beta3=1, beta4=1, beta5=1, beta6=1, beta7=1, beta8=1, beta9=1):
# initialize parameters
self.tau = tau
self.K1 = K1
self.K2 = K2
self.K3 = K3
self.K4 = K4
self.K5 = K5
self.K6 = K6
self.K7 = K7
self.K8 = K8
self.K9 = K9
self.S1 = S1
self.S2 = S2
self.S3 = S3
self.S4 = S4
self.S5 = S5
self.S6 = S6
self.S7 = S7
self.S8 = S8
self.S9 = S9
self.alpha1 = alpha1
self.alpha2 = alpha2
self.alpha3 = alpha3
self.alpha4 = alpha4
self.alpha5 = alpha5
self.alpha6 = alpha6
self.alpha7 = alpha7
self.alpha8 = alpha8
self.alpha9 = alpha9
self.beta1 = beta1
self.beta2 = beta2
self.beta3 = beta3
self.beta4 = beta4
self.beta5 = beta5
self.beta6 = beta6
self.beta7 = beta7
self.beta8 = beta8
self.beta9 = beta9
# to fit the model
def fit(self, X,y=None):
# define the mu vector
self.mu_ = np.ones((N))
# define lag weights
lag_weights = np.ones((max_lag))
# define retain_rate
retain_rate = np.array([alpha1, alpha2, alpha3, alpha4, alpha5, alpha6, alpha7, alpha8, alpha9])
# define cum_effect, set to a random value
cum_effect = 1
# define cum_effects_hill
cum_effects_hill = np.ones((N, num_media))
# parameter transformation
for nn in range(N):
for m in range(num_media):
for l in range(max_lag):
lag_weights[l] = retain_rate[m]**l
cum_effect = Adstock(X[nn][m], lag_weights)
cum_effects_hill[nn][m] = Hill(cum_effect, ec[m], slope[m])
self.mu_[nn] = tau + np.dot(cum_effects_hill[nn], beta_medias)
return self
# the predict function
def predict(self, X, y=None):
# try to get the mu_ argument. If it does not exist, we throw an error
try:
getattr(self, "mu_")
except AttributeError:
raise RuntimeError("You must train classifer before predicting data!")
return self.mu_
# the score function
def score(self, X, y):
# calculate the MSE
return np.dot(y - self.predict(X), y - self.predict(X))/len(X)
以下类似于“main”函数
这是错误消息:
MemoryError Traceback (most recent call last)
<ipython-input-22-de0388db8453> in <module>
14 #
15 clf = GridSearchCV(t, param_grid)
---> 16 clf.fit(X_media, actual_sales)
17 #clf.predict(X_media)
~\Anaconda3\lib\site-packages\sklearn\utils\validation.py in inner_f(*args, **kwargs)
71 FutureWarning)
72 kwargs.update({k: arg for k, arg in zip(sig.parameters, args)})
---> 73 return f(**kwargs)
74 return inner_f
75
~\Anaconda3\lib\site-packages\sklearn\model_selection\_search.py in fit(self, X, y, groups, **fit_params)
734 return results
735
--> 736 self._run_search(evaluate_candidates)
737
738 # For multi-metric evaluation, store the best_index_, best_params_ and
~\Anaconda3\lib\site-packages\sklearn\model_selection\_search.py in _run_search(self, evaluate_candidates)
1186 def _run_search(self, evaluate_candidates):
1187 """Search all candidates in param_grid"""
-> 1188 evaluate_candidates(ParameterGrid(self.param_grid))
1189
1190
~\Anaconda3\lib\site-packages\sklearn\model_selection\_search.py in evaluate_candidates(candidate_params)
698
699 def evaluate_candidates(candidate_params):
--> 700 candidate_params = list(candidate_params)
701 n_candidates = len(candidate_params)
702
MemoryError:
MemoryError回溯(最近一次调用)
在里面
14 #
15 clf=网格搜索CV(t,参数网格)
--->16 clf.fit(X_媒体,实际销售额)
17#clf.预测(X#U媒体)
内部文件中的~\Anaconda3\lib\site packages\sklearn\utils\validation.py(*args,**kwargs)
71未来警告)
72 kwargs.update({k:arg代表k,arg在zip中(sig.parameters,args)})
--->73返回f(**kwargs)
74返回内部\u f
75
~\Anaconda3\lib\site packages\sklearn\model\u selection\\u search.py in fit(self、X、y、groups、**fit\u参数)
734返回结果
735
-->736自我评估-运行搜索(评估候选人)
737
738#对于多指标评估,存储最佳指数、最佳参数和
~\Anaconda3\lib\site packages\sklearn\model\u selection\u search.py in\u run\u search(self,evaluate\u候选者)
1186定义-运行-搜索(自我评估-候选人):
1187“搜索参数网格中的所有候选项”
->1188评估候选参数(参数网格(self.param网格))
1189
1190
~\Anaconda3\lib\site packages\sklearn\model\u selection\\u search.py in evaluate\u候选者(候选者参数)
698
699 def评估候选人(候选人参数):
-->700候选参数=列表(候选参数)
701 n_候选者=len(候选者参数)
702
记忆错误:
有人能告诉我如何修复这个错误吗?或者我的代码有问题吗?谢谢 回溯显示网格搜索在尝试生成候选列表(即网格)时已耗尽内存。显然,您有37个参数,每个参数都有两个可能的值,因此候选参数的数量是
2^(37)
,超过1370亿。无论如何,你可能真的不想尝试那么多候选人,所以也许随机搜索CV
更合适?谢谢你,本。我现在正在尝试做RandomizedSearchCV
MemoryError Traceback (most recent call last)
<ipython-input-22-de0388db8453> in <module>
14 #
15 clf = GridSearchCV(t, param_grid)
---> 16 clf.fit(X_media, actual_sales)
17 #clf.predict(X_media)
~\Anaconda3\lib\site-packages\sklearn\utils\validation.py in inner_f(*args, **kwargs)
71 FutureWarning)
72 kwargs.update({k: arg for k, arg in zip(sig.parameters, args)})
---> 73 return f(**kwargs)
74 return inner_f
75
~\Anaconda3\lib\site-packages\sklearn\model_selection\_search.py in fit(self, X, y, groups, **fit_params)
734 return results
735
--> 736 self._run_search(evaluate_candidates)
737
738 # For multi-metric evaluation, store the best_index_, best_params_ and
~\Anaconda3\lib\site-packages\sklearn\model_selection\_search.py in _run_search(self, evaluate_candidates)
1186 def _run_search(self, evaluate_candidates):
1187 """Search all candidates in param_grid"""
-> 1188 evaluate_candidates(ParameterGrid(self.param_grid))
1189
1190
~\Anaconda3\lib\site-packages\sklearn\model_selection\_search.py in evaluate_candidates(candidate_params)
698
699 def evaluate_candidates(candidate_params):
--> 700 candidate_params = list(candidate_params)
701 n_candidates = len(candidate_params)
702
MemoryError: