1 import os 2 import jieba 3 import random 4 """ 5 函数说明:中文文本处理 6 Parameters: 7 folder_path - 文本存放的路径 8 test_size - 测试集占比，默认占所有数据集的百分之20 9 Returns: 10 all_words_list - 按词频降序排序的训练集列表 11 train_data_list - 训练集列表 12 test_data_list - 测试集列表 13 train_class_list - 训练集标签列表 14 test_class_list - 测试集标签列表 15 """ 16 def TextProcessing(folder_path): 17 folder_list = os.listdir(folder_path) #查看folder_path下的文件 18 data_list = [] #训练集 19 class_list = [] 20 21 #遍历每个子文件夹 22 for folder in folder_list: 23 new_folder_path = os.path.join(folder_path, folder) #根据子文件夹，生成新的路径 24 files = os.listdir(new_folder_path) #存放子文件夹下的txt文件的列表 25 26 j = 1 27 #遍历每个txt文件 28 for file in files: 29 if j > 100: #每类txt样本数最多100个 30 break 31 with open(os.path.join(new_folder_path, file), 'r', encoding = 'utf-8') as f: #打开txt文件 32 raw = f.read() 33 34 word_cut = jieba.cut(raw, cut_all = False) #精简模式，返回一个可迭代的generator 35 word_list = list(word_cut) #generator转换为list 36 37 data_list.append(word_list) 38 class_list.append(folder) 39 j += 1 40 data_class_list = list(zip(data_list, class_list)) # zip压缩合并，将数据与标签对应压缩 41 random.shuffle(data_class_list) # 将data_class_list乱序 42 index = int(len(data_class_list) * test_size) + 1 # 训练集和测试集切分的索引值 43 train_list = data_class_list[index:] # 训练集 44 test_list = data_class_list[:index] # 测试集 45 train_data_list, train_class_list = zip(*train_list) # 训练集解压缩 46 test_data_list, test_class_list = zip(*test_list) # 测试集解压缩 47 48 all_words_dict = {} # 统计训练集词频 49 for word_list in train_data_list: 50 for word in word_list: 51 if word in all_words_dict.keys(): 52 all_words_dict[word] += 1 53 else: 54 all_words_dict[word] = 1 55 56 # 根据键的值倒序排序 57 all_words_tuple_list = sorted(all_words_dict.items(), key=lambda f: f[1], reverse=True) 58 all_words_list, all_words_nums = zip(*all_words_tuple_list) # 解压缩 59 all_words_list = list(all_words_list) # 转换成列表 60 return all_words_list, train_data_list, test_data_list, train_class_list, test_class_list 61 62 if __name__ == '__main__': 63 # 文本预处理 64 folder_path = './SogouC/Sample' # 训练集存放地址 65 all_words_list, train_data_list, test_data_list, train_class_list, test_class_list = TextProcessing( 66 folder_path, test_size=0.2) 67 print(all_words_list) 68 """ 69 函数说明:读取文件里的内容，并去重 70 Parameters: 71 words_file - 文件路径 72 Returns: 73 words_set - 读取的内容的set集合 74 """ 75 def MakeWordsSet(words_file): 76 words_set = set() # 创建set集合 77 with open(words_file, 'r', encoding='utf-8') as f: # 打开文件 78 for line in f.readlines(): # 一行一行读取 79 word = line.strip() # 去回车 80 if len(word) > 0: # 有文本，则添加到words_set中 81 words_set.add(word) 82 return words_set # 返回处理结果 83 """ 84 函数说明:文本特征选取 85 Parameters: 86 all_words_list - 训练集所有文本列表 87 deleteN - 删除词频最高的deleteN个词 88 stopwords_set - 指定的结束语 89 Returns: 90 feature_words - 特征集 91 """ 92 def words_dict(all_words_list, deleteN, stopwords_set=set()): 93 feature_words = [] # 特征列表 94 n = 1 95 for t in range(deleteN, len(all_words_list), 1): 96 if n > 1000: # feature_words的维度为1000 97 break 98 # 如果这个词不是数字，并且不是指定的结束语，并且单词长度大于1小于5，那么这个词就可以作为特征词 99 if not all_words_list[t].isdigit() and all_words_list[t] not in stopwords_set and 1 < len( 100 all_words_list[t]) < 5: 101 feature_words.append(all_words_list[t]) 102 n += 1 103 return feature_words 104 105 if __name__ == '__main__': 106 # 文本预处理 107 folder_path = './SogouC/Sample' # 训练集存放地址 108 all_words_list, train_data_list, test_data_list, train_class_list, test_class_list = TextProcessing( 109 folder_path, test_size=0.2) 110 111 # 生成stopwords_set 112 stopwords_file = './stopwords_cn.txt' 113 stopwords_set = MakeWordsSet(stopwords_file) 114 feature_words = words_dict(all_words_list, 100, stopwords_set) 115 print(feature_words) 116 117 # print(data_list) 118 # print(class_list) 119 if __name__ == '__main__': 120 #文本预处理 121 folder_path = './SogouC/Sample' #训练集存放地址 122 TextProcessing(folder_path) 123 ''' 124 将所有文本分成训练集和测试集，并对训练集中的所有单词进行词频统计，并按降序排序。 125 使用Sklearn构建朴素贝叶斯分类器 126 在scikit-learn中，一共有3个朴素贝叶斯的分类算法类。 127 分别是GaussianNB，MultinomialNB和BernoulliNB。 128 其中GaussianNB就是先验为高斯分布的朴素贝叶斯， 129 MultinomialNB就是先验为多项式分布的朴素贝叶斯， 130 而BernoulliNB就是先验为伯努利分布的朴素贝叶斯。 131 ''' 132 from sklearn.naive_bayes import MultinomialNB 133 import matplotlib.pyplot as plt 134 """ 135 函数说明:根据feature_words将文本向量化 136 Parameters: 137 train_data_list - 训练集 138 test_data_list - 测试集 139 feature_words - 特征集 140 Returns: 141 train_feature_list - 训练集向量化列表 142 test_feature_list - 测试集向量化列表 143 """ 144 def TextFeatures(train_data_list, test_data_list, feature_words): 145 def text_features(text, feature_words): #出现在特征集中，则置1 146 text_words = set(text) 147 features = [1 if word in text_words else 0 for word in feature_words] 148 return features 149 train_feature_list = [text_features(text, feature_words) for text in train_data_list] 150 test_feature_list = [text_features(text, feature_words) for text in test_data_list] 151 return train_feature_list, test_feature_list #返回结果 152 153 """ 154 函数说明:新闻分类器 155 156 Parameters: 157 train_feature_list - 训练集向量化的特征文本 158 test_feature_list - 测试集向量化的特征文本 159 train_class_list - 训练集分类标签 160 test_class_list - 测试集分类标签 161 Returns: 162 test_accuracy - 分类器精度 163 """ 164 def TextClassifier(train_feature_list, test_feature_list, train_class_list, test_class_list): 165 classifier = MultinomialNB().fit(train_feature_list, train_class_list) 166 test_accuracy = classifier.score(test_feature_list, test_class_list) 167 return test_accuracy 168 169 # if __name__ == '__main__': 170 # #文本预处理 171 # folder_path = './SogouC/Sample' #训练集存放地址 172 # all_words_list, train_data_list, test_data_list, train_class_list, test_class_list = TextProcessing(folder_path, test_size=0.2) 173 # 174 # # 生成stopwords_set 175 # stopwords_file = './stopwords_cn.txt' 176 # stopwords_set = MakeWordsSet(stopwords_file) 177 178 test_accuracy_list = [] 179 deleteNs = range(0, 1000, 20) #0 20 40 60 ... 980 180 for deleteN in deleteNs: 181 feature_words = words_dict(all_words_list, deleteN, stopwords_set) 182 train_feature_list, test_feature_list = TextFeatures(train_data_list, test_data_list, feature_words) 183 test_accuracy = TextClassifier(train_feature_list, test_feature_list, train_class_list, test_class_list) 184 test_accuracy_list.append(test_accuracy) 185 186 plt.figure() 187 plt.plot(deleteNs, test_accuracy_list) 188 plt.title('Relationship of deleteNs and test_accuracy') 189 plt.xlabel('deleteNs') 190 plt.ylabel('test_accuracy') 191 plt.show() 192 if __name__ == '__main__': 193 # 文本预处理 194 folder_path = './SogouC/Sample' # 训练集存放地址 195 all_words_list, train_data_list, test_data_list, train_class_list, test_class_list = TextProcessing(folder_path, 196 test_size=0.2) 197 198 # 生成stopwords_set 199 stopwords_file = './stopwords_cn.txt' 200 stopwords_set = MakeWordsSet(stopwords_file) 201 202 test_accuracy_list = [] 203 feature_words = words_dict(all_words_list, 450, stopwords_set) 204 train_feature_list, test_feature_list = TextFeatures(train_data_list, test_data_list, feature_words) 205 test_accuracy = TextClassifier(train_feature_list, test_feature_list, train_class_list, test_class_list) 206 test_accuracy_list.append(test_accuracy) 207 ave = lambda c: sum(c) / len(c)