Python TfDif和自定义要素之间的FeatureUnion上的KeyError
我正在尝试创建一个模型，在该模型中，我将在文本列上使用TfidfVectorizer，还将在文本上使用其他两列的额外数据。下面的代码再现了我试图做的事情和我得到的错误Python TfDif和自定义要素之间的FeatureUnion上的KeyError,python,scikit-learn,sklearn-pandas,Python,Scikit Learn,Sklearn Pandas,我正在尝试创建一个模型，在该模型中，我将在文本列上使用TfidfVectorizer，还将在文本上使用其他两列的额外数据。下面的代码再现了我试图做的事情和我得到的错误 import pandas as pd from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.base import BaseEstimator, TransformerMixin from sklearn.pipeline import
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.pipeline import FeatureUnion
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction import DictVectorizer
from sklearn.naive_bayes import BernoulliNB
class ParStats(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X):
print(X[0])
return [{'feat_1': x['feat_1'],
'feat_2': x['feat_2']}
for x in X]
class ItemSelector(BaseEstimator, TransformerMixin):
def __init__(self, key):
self.key = key
def fit(self, x, y=None):
return self
def transform(self, data_dict):
return data_dict[self.key]
def feature_union_test():
# create test data frame
test_data = {
'text': ['And the silken, sad, uncertain rustling of each purple curtain',
'Thrilled me filled me with fantastic terrors never felt before',
'So that now, to still the beating of my heart, I stood repeating',
'Tis some visitor entreating entrance at my chamber door',
'Some late visitor entreating entrance at my chamber door',
'This it is and nothing more'],
'feat_1': [4, 7, 10, 7, 4, 6],
'feat_2': [1, 5, 5, 1, 1, 10],
'ignore': [1, 1, 1, 0, 0, 0]
}
test_df = pd.DataFrame(data=test_data)
y_train = test_df['ignore'].values.astype('int')
# Feature Union Pipeline
pipeline = FeatureUnion([
('text', Pipeline([
('selector', ItemSelector(key='text')),
('tfidf', TfidfVectorizer(max_df=0.5)),
])),
('parstats', Pipeline([
('stats', ParStats()),
('vect', DictVectorizer()),
]))
])
tfidf = pipeline.fit_transform(test_df)
# fits Naive Bayes
clf = BernoulliNB().fit(tfidf, y_train)
feature_union_test()
Traceback (most recent call last):
File "C:\Users\Rogerio\Python VENV\lib\site-packages\pandas\core\indexes\base.py", line 3064, in get_loc
return self._engine.get_loc(key)
File "pandas\_libs\index.pyx", line 140, in pandas._libs.index.IndexEngine.get_loc
File "pandas\_libs\index.pyx", line 162, in pandas._libs.index.IndexEngine.get_loc
File "pandas\_libs\hashtable_class_helper.pxi", line 1492, in pandas._libs.hashtable.PyObjectHashTable.get_item
File "pandas\_libs\hashtable_class_helper.pxi", line 1500, in pandas._libs.hashtable.PyObjectHashTable.get_item
KeyError: 0
我尝试了管道的几个不同的迭代，我总是会遇到一些错误，所以很明显我遗漏了一些东西。我做错了什么？错误发生在
ParStatstransform首先，
pandasprint（X[0]）而且，您不能以这样的方式迭代数据帧
以下是该函数的可能工作版本：
def transform(self, X):
return [{'feat_1': x[0], 'feat_2': x[1]}
for x in X[['feat_1', 'feat_2']].values]
当然，还有很多其他可能的解决方案，但您已经明白了。错误发生在
ParStatstransform首先，
pandasprint（X[0]）而且，您不能以这样的方式迭代数据帧
以下是该函数的可能工作版本：
def transform(self, X):
return [{'feat_1': x[0], 'feat_2': x[1]}
for x in X[['feat_1', 'feat_2']].values]
当然，还有很多其他可能的解决方案，但是你知道了。好的。在评论中讨论之后，这是你的问题陈述 您希望将列
feat_1feat_2text# Feature Union Pipeline
pipeline = FeatureUnion([('text', Pipeline([('selector', ItemSelector(key='text')),
('tfidf', TfidfVectorizer(max_df=0.5)),
])),
('non_text', ItemSelector(key=['feat_1', 'feat_2']))
])
tfidf = pipeline.fit_transform(test_df)
默认的
ItemSelectortextfeat_1feat_2text# Feature Union Pipeline
pipeline = FeatureUnion([('text', Pipeline([('selector', ItemSelector(key='text')),
('tfidf', TfidfVectorizer(max_df=0.5)),
])),
('non_text', ItemSelector(key=['feat_1', 'feat_2']))
])
tfidf = pipeline.fit_transform(test_df)
默认的
ItemSelectortextParStatsX[0]0feat1feat2feat1feat2数据帧ParStatsX[0]0feat1feat2feat1feat2数据帧