Python 读取给定文本文件中的所有单词，并打印每个单词的计数
Test.txt包含以下句子：如果一只土拨鼠可以，一只土拨鼠会吸走多少木头 查克·伍德 这个程序应该读取给定文本文件中的所有单词，直到eof 并打印出每个单词的计数。这个词应该是 已处理，不区分大小写所有大写字母，标点符号应为 已删除，输出应按 频率 然而，我遇到了一个简单的问题，那就是数数行，而不是单词，帮助一个兄弟 制作一个翻译表以去除非单词字符 打印表格 f=打开“Test.txt” cnt=0 对于f.read.split中的单词： 印刷字 cnt+=1 印刷碳纳米管 这可能会帮助你，兄弟……尽管我也是python的新手 此代码：Python 读取给定文本文件中的所有单词，并打印每个单词的计数,python,list,sorting,dictionary,Python,List,Sorting,Dictionary,Test.txt包含以下句子：如果一只土拨鼠可以，一只土拨鼠会吸走多少木头 查克·伍德 这个程序应该读取给定文本文件中的所有单词，直到eof 并打印出每个单词的计数。这个词应该是 已处理，不区分大小写所有大写字母，标点符号应为 已删除，输出应按 频率 然而，我遇到了一个简单的问题，那就是数数行，而不是单词，帮助一个兄弟 制作一个翻译表以去除非单词字符 打印表格 f=打开“Test.txt” cnt=0 对于f.read.split中的单词： 印刷字 cnt+=1 印刷碳纳米管 这可能会帮助你，兄
from collections import Counter
data = open( 'Test1.txt' ).read() # read the file
data = ''.join( [i.upper() if i.isalpha() else ' ' for i in data] ) # remove the punctuation
c = Counter( data.split() ) # count the words
c.most_common()
[('A', 2), ('CHUCK', 2), ('WOODCHUCK', 2), ('WOOD', 2), ('WOULD', 1), ('COULD', 1), ('HOW', 1), ('MUCH', 1), ('IF', 1)]
我想知道代码是否太短了= 首先你必须决定你对一个词的定义是什么 定义1：单词是由空格分隔的字符序列。所以你已经知道了一个单词，而点钟也是一个单词 定义2：一个词是讲话或写作中一个独特的有意义的元素。在这种情况下，你有两个不同的单词，而o点钟是一个单词 因此，如果您运行：
import string
import re
import nltk
import pandas as pd
s = "How much wood would a woodchuck chuck if a woodchuck could chuck wood. \n And also another line you've read from the file with something else. I wake up daily before eight o'clock."
def tokenize(text,semantic=True,sep=" "):
if semantic:
#Definition 2
return nltk.word_tokenize(text)
else:
#Definition 1
return [x for x in text.split(sep) ]
def remove_punctuation(text):
pattern = re.compile('[{}]'.format(re.escape(string.punctuation)))
return list(filter(None, [pattern.sub('',token) for token in text]))
def lowercase(text):
return [token.lower() for token in text]
result = nltk.FreqDist(remove_punctuation(lowercase(tokenize(s)))).most_common()
table = pd.DataFrame(result)
table.to_csv('result.csv')
result = nltk.FreqDist(remove_punctuation(lowercase(tokenize(s,semantic=False)))).most_common()
import string
import re
import nltk
import pandas as pd
s = "How much wood would a woodchuck chuck if a woodchuck could chuck wood. \n And also another line you've read from the file with something else. I wake up daily before eight o'clock."
CONTRACTION_MAP = {"ain't": "is not", "aren't": "are not","can't": "cannot",
"can't've": "cannot have", "'cause": "because", "could've": "could have",
"couldn't": "could not", "couldn't've": "could not have","didn't": "did not",
"doesn't": "does not", "don't": "do not", "hadn't": "had not",
"hadn't've": "had not have", "hasn't": "has not", "haven't": "have not",
"he'd": "he would", "he'd've": "he would have", "he'll": "he will",
"he'll've": "he he will have", "he's": "he is", "how'd": "how did",
"how'd'y": "how do you", "how'll": "how will", "how's": "how is",
"I'd": "I would", "I'd've": "I would have", "I'll": "I will",
"I'll've": "I will have","I'm": "I am", "I've": "I have",
"i'd": "i would", "i'd've": "i would have", "i'll": "i will",
"i'll've": "i will have","i'm": "i am", "i've": "i have",
"isn't": "is not", "it'd": "it would", "it'd've": "it would have",
"it'll": "it will", "it'll've": "it will have","it's": "it is",
"let's": "let us", "ma'am": "madam", "mayn't": "may not",
"might've": "might have","mightn't": "might not","mightn't've": "might not have",
"must've": "must have", "mustn't": "must not", "mustn't've": "must not have",
"needn't": "need not", "needn't've": "need not have","o'clock": "of the clock",
"oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not",
"sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would",
"she'd've": "she would have", "she'll": "she will", "she'll've": "she will have",
"she's": "she is", "should've": "should have", "shouldn't": "should not",
"shouldn't've": "should not have", "so've": "so have","so's": "so as",
"this's": "this is",
"that'd": "that would", "that'd've": "that would have","that's": "that is",
"there'd": "there would", "there'd've": "there would have","there's": "there is",
"they'd": "they would", "they'd've": "they would have", "they'll": "they will",
"they'll've": "they will have", "they're": "they are", "they've": "they have",
"to've": "to have", "wasn't": "was not", "we'd": "we would",
"we'd've": "we would have", "we'll": "we will", "we'll've": "we will have",
"we're": "we are", "we've": "we have", "weren't": "were not",
"what'll": "what will", "what'll've": "what will have", "what're": "what are",
"what's": "what is", "what've": "what have", "when's": "when is",
"when've": "when have", "where'd": "where did", "where's": "where is",
"where've": "where have", "who'll": "who will", "who'll've": "who will have",
"who's": "who is", "who've": "who have", "why's": "why is",
"why've": "why have", "will've": "will have", "won't": "will not",
"won't've": "will not have", "would've": "would have", "wouldn't": "would not",
"wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would",
"y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have",
"you'd": "you would", "you'd've": "you would have", "you'll": "you will",
"you'll've": "you will have", "you're": "you are", "you've": "you have" }
# Credit for this function: https://www.kaggle.com/saxinou/nlp-01-preprocessing-data
def expand_contractions(sentence, contraction_mapping):
contractions_pattern = re.compile('({})'.format('|'.join(contraction_mapping.keys())),
flags=re.IGNORECASE|re.DOTALL)
def expand_match(contraction):
match = contraction.group(0)
first_char = match[0]
expanded_contraction = contraction_mapping.get(match) if contraction_mapping.get(match) else contraction_mapping.get(match.lower())
expanded_contraction = first_char+expanded_contraction[1:]
return expanded_contraction
expanded_sentence = contractions_pattern.sub(expand_match, sentence)
return expanded_sentence
def tokenize(text,semantic=True,sep=" "):
if semantic:
#Definition 2
return nltk.word_tokenize(text)
else:
#Definition 1
return [x for x in text.split(sep) ]
def remove_punctuation(text):
pattern = re.compile('[{}]'.format(re.escape(string.punctuation)))
return list(filter(None, [pattern.sub('',token) for token in text]))
def lowercase(text):
return [token.lower() for token in text]
result = nltk.FreqDist(remove_punctuation(lowercase(tokenize(expand_contractions(s,CONTRACTION_MAP))))).most_common()
table = pd.DataFrame(result)
table.to_csv('result.csv')
这可能是有帮助的核心python解决方案
data = "How much wood would a woodchuck chuck if a woodchuck could chuck wood."
data = "".join(i.strip('\n') for i in data if ord(i) < 127)
data_arr = data.upper().split(' ')
a = {}
for i in data_arr:
if i not in a:
a[i] = 1
else:
a[i] = a[i] + 1
data = sorted(a.items(), key=lambda a: a[0])
print(data)
你为什么不在空间上创造一个词呢？我不知道一个词的频率是如何被使用的。你能示范一下吗？
result = nltk.FreqDist(remove_punctuation(lowercase(tokenize(s,semantic=False)))).most_common()
import string
import re
import nltk
import pandas as pd
s = "How much wood would a woodchuck chuck if a woodchuck could chuck wood. \n And also another line you've read from the file with something else. I wake up daily before eight o'clock."
CONTRACTION_MAP = {"ain't": "is not", "aren't": "are not","can't": "cannot",
"can't've": "cannot have", "'cause": "because", "could've": "could have",
"couldn't": "could not", "couldn't've": "could not have","didn't": "did not",
"doesn't": "does not", "don't": "do not", "hadn't": "had not",
"hadn't've": "had not have", "hasn't": "has not", "haven't": "have not",
"he'd": "he would", "he'd've": "he would have", "he'll": "he will",
"he'll've": "he he will have", "he's": "he is", "how'd": "how did",
"how'd'y": "how do you", "how'll": "how will", "how's": "how is",
"I'd": "I would", "I'd've": "I would have", "I'll": "I will",
"I'll've": "I will have","I'm": "I am", "I've": "I have",
"i'd": "i would", "i'd've": "i would have", "i'll": "i will",
"i'll've": "i will have","i'm": "i am", "i've": "i have",
"isn't": "is not", "it'd": "it would", "it'd've": "it would have",
"it'll": "it will", "it'll've": "it will have","it's": "it is",
"let's": "let us", "ma'am": "madam", "mayn't": "may not",
"might've": "might have","mightn't": "might not","mightn't've": "might not have",
"must've": "must have", "mustn't": "must not", "mustn't've": "must not have",
"needn't": "need not", "needn't've": "need not have","o'clock": "of the clock",
"oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not",
"sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would",
"she'd've": "she would have", "she'll": "she will", "she'll've": "she will have",
"she's": "she is", "should've": "should have", "shouldn't": "should not",
"shouldn't've": "should not have", "so've": "so have","so's": "so as",
"this's": "this is",
"that'd": "that would", "that'd've": "that would have","that's": "that is",
"there'd": "there would", "there'd've": "there would have","there's": "there is",
"they'd": "they would", "they'd've": "they would have", "they'll": "they will",
"they'll've": "they will have", "they're": "they are", "they've": "they have",
"to've": "to have", "wasn't": "was not", "we'd": "we would",
"we'd've": "we would have", "we'll": "we will", "we'll've": "we will have",
"we're": "we are", "we've": "we have", "weren't": "were not",
"what'll": "what will", "what'll've": "what will have", "what're": "what are",
"what's": "what is", "what've": "what have", "when's": "when is",
"when've": "when have", "where'd": "where did", "where's": "where is",
"where've": "where have", "who'll": "who will", "who'll've": "who will have",
"who's": "who is", "who've": "who have", "why's": "why is",
"why've": "why have", "will've": "will have", "won't": "will not",
"won't've": "will not have", "would've": "would have", "wouldn't": "would not",
"wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would",
"y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have",
"you'd": "you would", "you'd've": "you would have", "you'll": "you will",
"you'll've": "you will have", "you're": "you are", "you've": "you have" }
# Credit for this function: https://www.kaggle.com/saxinou/nlp-01-preprocessing-data
def expand_contractions(sentence, contraction_mapping):
contractions_pattern = re.compile('({})'.format('|'.join(contraction_mapping.keys())),
flags=re.IGNORECASE|re.DOTALL)
def expand_match(contraction):
match = contraction.group(0)
first_char = match[0]
expanded_contraction = contraction_mapping.get(match) if contraction_mapping.get(match) else contraction_mapping.get(match.lower())
expanded_contraction = first_char+expanded_contraction[1:]
return expanded_contraction
expanded_sentence = contractions_pattern.sub(expand_match, sentence)
return expanded_sentence
def tokenize(text,semantic=True,sep=" "):
if semantic:
#Definition 2
return nltk.word_tokenize(text)
else:
#Definition 1
return [x for x in text.split(sep) ]
def remove_punctuation(text):
pattern = re.compile('[{}]'.format(re.escape(string.punctuation)))
return list(filter(None, [pattern.sub('',token) for token in text]))
def lowercase(text):
return [token.lower() for token in text]
result = nltk.FreqDist(remove_punctuation(lowercase(tokenize(expand_contractions(s,CONTRACTION_MAP))))).most_common()
table = pd.DataFrame(result)
table.to_csv('result.csv')
data = "How much wood would a woodchuck chuck if a woodchuck could chuck wood."
data = "".join(i.strip('\n') for i in data if ord(i) < 127)
data_arr = data.upper().split(' ')
a = {}
for i in data_arr:
if i not in a:
a[i] = 1
else:
a[i] = a[i] + 1
data = sorted(a.items(), key=lambda a: a[0])
print(data)