Python 在spacy中执行多类分类时出错
我正在尝试使用crowdflower文本分类进行多类分类。下面是我的代码:Python 在spacy中执行多类分类时出错,python,nlp,spacy,multiclass-classification,Python,Nlp,Spacy,Multiclass Classification,我正在尝试使用crowdflower文本分类进行多类分类。下面是我的代码: from __future__ import unicode_literals, print_function from __future__ import unicode_literals from pathlib import Path import pandas as pd import spacy from spacy.util import minibatch, compounding def ma
from __future__ import unicode_literals, print_function
from __future__ import unicode_literals
from pathlib import Path
import pandas as pd
import spacy
from spacy.util import minibatch, compounding
def main(model=None, output_dir=None, n_iter=20):
if model is not None:
nlp = spacy.load(model) # load existing spaCy model
print("Loaded model '%s'" % model)
else:
nlp = spacy.blank('en') # create blank Language class
print("Created blank 'en' model")
# add the text classifier to the pipeline if it doesn't exist
# nlp.create_pipe works for built-ins that are registered with spaCy
if 'textcat' not in nlp.pipe_names:
textcat = nlp.create_pipe('textcat')
nlp.add_pipe(textcat, last=True)
# otherwise, get it, so we can add labels to it
else:
textcat = nlp.get_pipe('textcat')
# add label to text classifier
for i in ['neutral','worry','happiness','sadness','love','surprise','fun','relief','hate','enthusiasm','boredom','anger']:
textcat.add_label(i)
df = pd.read_csv('text_emotion.csv')
df.drop(['tweet_id', 'author'], axis=1, inplace=True)
df = df[df['sentiment'] != 'empty']
train_data = list(zip(unicode(df['content']),
[{u'cats': unicode(cats)} for cats in df['sentiment']]))
# get names of other pipes to disable them during training
other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'textcat']
with nlp.disable_pipes(*other_pipes): # only train textcat
optimizer = nlp.begin_training()
print("Training the model...")
print('{:^5}\t'.format('LOSS'))
for i in range(n_iter):
losses = {}
# batch up the examples using spaCy's minibatch
batches = minibatch(train_data, size=compounding(4., 32., 1.001))
for batch in batches:
texts, annotations = zip(*batch)
# print('texts: '+str(texts))
# print('annotations: '+str(annotations))
nlp.update(texts, annotations, sgd=optimizer, drop=0.2,losses=losses)
# with textcat.model.use_params(optimizer.averages):
# evaluate on the dev data split off in load_data()
print('{0:.3f}' # print a simple table
.format(losses['textcat']))
# test the trained model
test_text = "This movie sucked"
doc = nlp(test_text)
print(test_text, doc.cats)
if output_dir is not None:
output_dir = Path(output_dir)
if not output_dir.exists():
output_dir.mkdir()
nlp.to_disk(output_dir)
print("Saved model to", output_dir)
# test the saved model
print("Loading from", output_dir)
nlp2 = spacy.load(output_dir)
doc2 = nlp2(test_text)
print(test_text, doc2.cats)
if __name__ == '__main__':
main()
我收到以下错误:
Traceback (most recent call last):
batch: [(u'1', {u'cats': u'sadness'}), (u' ', {u'cats': u'sadness'}), (u' ', {u'cats': u'enthusiasm'}), (u' ', {u'cats': u'neutral'})]
File "/Users/loginofdeath/Documents/24Feb/emo_cat.py", line 91, in <module>
main()
File "/Users/loginofdeath/Document/24Feb/emo_cat.py", line 63, in main
nlp.update(texts, annotations, sgd=optimizer, drop=0.2,losses=losses)
File "/usr/local/lib/python2.7/site-packages/spacy/language.py", line 399, in update
gold = GoldParse(doc, **gold)
File "gold.pyx", line 430, in spacy.gold.GoldParse.__init__
ValueError: dictionary update sequence element #0 has length 1; 2 is required
回溯(最近一次呼叫最后一次):
批次:[(u'1',{u'cats:u'said'},(u',{u'cats:u'said'},(u',{u'cats:u'faciency'},(u',{u'cats:u'neutral'})]
文件“/Users/loginofdeath/Documents/24Feb/emo_cat.py”,第91行,在
main()
文件“/Users/loginofdeath/Document/24Feb/emo_cat.py”,主文件第63行
更新(文本、注释、sgd=优化器、drop=0.2、损失=损失)
文件“/usr/local/lib/python2.7/site packages/spacy/language.py”,第399行,在更新中
黄金=黄金解析(文件,**黄金)
文件“gold.pyx”,第430行,在spacy.gold.GoldParse.\uu init中__
ValueError:字典更新序列元素#0的长度为1;2是必需的
我正在使用:
Python版本:2.7.14
平台:Darwin-16.4.0-x86_64-i386-64位
spaCy版本:2.0.9
型号:en
有人能帮我吗?我对spacy中多类分类的方法正确吗?提前感谢。此答案的全部功劳归于。 代码如下:
from __future__ import unicode_literals, print_function
from __future__ import unicode_literals
from pathlib import Path
import pandas as pd
import spacy
import copy
from spacy.util import minibatch, compounding
import re
def clean_string(mystring):
return re.sub('[^A-Za-z\ 0-9 ]+', '', mystring)
def main(model=None, output_dir=None, n_iter=2):
if model is not None:
nlp = spacy.load(model) # load existing spaCy model
print("Loaded model '%s'" % model)
else:
nlp = spacy.blank('en') # create blank Language class
print("Created blank 'en' model")
# add the text classifier to the pipeline if it doesn't exist
# nlp.create_pipe works for built-ins that are registered with spaCy
if 'textcat' not in nlp.pipe_names:
textcat = nlp.create_pipe('textcat')
nlp.add_pipe(textcat, last=True)
# otherwise, get it, so we can add labels to it
else:
textcat = nlp.get_pipe('textcat')
# add label to text classifier
for i in ['neutral','worry','happiness','sadness','love','surprise','fun','relief','hate','enthusiasm','boredom','anger']:
textcat.add_label(i)
df = pd.read_csv('text_emotion.csv')
df.drop(['tweet_id', 'author'], axis=1, inplace=True)
df = df[df['sentiment'] != 'empty']
sentiment_values = df['sentiment'].unique()
labels_default = dict((v, 0) for v in sentiment_values)
train_data = []
for i, row in df.iterrows():
label_values = copy.deepcopy(labels_default)
label_values[row['sentiment']] = 1
train_data.append((unicode(clean_string(row['content'])), {"cats": label_values}))
train_data = train_data[:5000]
# get names of other pipes to disable them during training
other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'textcat']
with nlp.disable_pipes(*other_pipes): # only train textcat
optimizer = nlp.begin_training()
print("Training the model...")
print('{:^5}\t'.format('LOSS'))
for i in range(n_iter):
losses = {}
# batch up the examples using spaCy's minibatch
batches = minibatch(train_data, size=compounding(4., 32., 1.001))
for batch in batches:
texts, annotations = zip(*batch)
# print('texts: '+str(texts))
# print('annotations: '+str(annotations))
nlp.update(texts, annotations, sgd=optimizer, drop=0.2,losses=losses)
# with textcat.model.use_params(optimizer.averages):
# evaluate on the dev data split off in load_data()
print('{0:.3f}' # print a simple table
.format(losses['textcat']))
# test the trained model
test_text = "This movie sucked"
doc = nlp(test_text)
print(test_text, sorted(doc.cats.items(), key=lambda val: val[1], reverse=True))
if output_dir is not None:
output_dir = Path(output_dir)
if not output_dir.exists():
output_dir.mkdir()
nlp.to_disk(output_dir)
print("Saved model to", output_dir)
# test the saved model
print("Loading from", output_dir)
nlp2 = spacy.load(output_dir)
doc2 = nlp2(test_text)
print(test_text, doc2.cats)
if __name__ == '__main__':
main()
请注意,在代码中我们只训练5000个数据点的训练数据。我希望这将澄清spaCy中有关多类分类的大多数查询