吴沂钊
177 lines
5.3 KiB
Plaintext
177 lines
5.3 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "9d091a01",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import pandas as pd\n",
|
||
"import numpy as np\n",
|
||
"import re\n",
|
||
"import jieba\n",
|
||
"import pickle\n",
|
||
"from wordcloud import WordCloud\n",
|
||
"from sklearn.naive_bayes import GaussianNB\n",
|
||
"from sklearn.model_selection import train_test_split\n",
|
||
"from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
|
||
"from sklearn.metrics import confusion_matrix\n",
|
||
"import seaborn as sns\n",
|
||
"import matplotlib.pyplot as plt\n",
|
||
"plt.rcParams['font.sans-serif'] = 'SimHei'\n",
|
||
"plt.rcParams['axes.unicode_minus'] = False\n",
|
||
"import warnings\n",
|
||
"warnings.filterwarnings('ignore')"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "eff92c98",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def data_process(file='message80W1.csv'):\n",
|
||
"#读取数据并进行清洗\n",
|
||
" data = pd.read_csv(file, header=None, index_col=0)\n",
|
||
" data.columns = ['label', 'message']\n",
|
||
" n = 5000\n",
|
||
"\n",
|
||
" a = data[data['label'] == 0].sample(n)\n",
|
||
" b = data[data['label'] == 1].sample(n)\n",
|
||
" data_new = pd.concat([a, b], axis=0)\n",
|
||
"\n",
|
||
" data_dup = data_new['message'].drop_duplicates()\n",
|
||
" data_qumin = data_dup.apply(lambda x: re.sub('x', '', x))\n",
|
||
"\n",
|
||
" jieba.load_userdict('newdic1.txt')\n",
|
||
" data_cut = data_qumin.apply(lambda x: jieba.lcut(x))\n",
|
||
"\n",
|
||
" stopWords = pd.read_csv('stopword.txt', encoding='GB18030', sep='hahaha', header=None)\n",
|
||
" stopWords = ['≮', '≯', '≠', '≮', ' ', '会', '月', '日', '–'] + list(stopWords.iloc[:, 0])\n",
|
||
" data_after_stop = data_cut.apply(lambda x: [i for i in x if i not in stopWords])\n",
|
||
" labels = data_new.loc[data_after_stop.index, 'label']\n",
|
||
" adata = data_after_stop.apply(lambda x: ' '.join(x))\n",
|
||
" return adata, data_after_stop, labels"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "c3ae80f9",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues):\n",
|
||
" sns.heatmap(cm, annot=True)\n",
|
||
" plt.ylabel('True label')\n",
|
||
" plt.xlabel('Predicted label')\n",
|
||
" plt.title(title)\n",
|
||
" plt.xticks(np.arange(len(classes)), classes)\n",
|
||
" plt.yticks(np.arange(len(classes)), classes)\n",
|
||
" plt.show()\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "56dbb0d1",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"\n",
|
||
"adata, data_after_stop, lables = data_process()\n",
|
||
"data_tr, data_te, labels_tr, labels_te = train_test_split(adata, lables, test_size=0.2)\n",
|
||
"countVectorizer = CountVectorizer()\n",
|
||
"data_tr = countVectorizer.fit_transform(data_tr)\n",
|
||
"X_tr = TfidfTransformer().fit_transform(data_tr.toarray()).toarray()\n",
|
||
"data_te = CountVectorizer(vocabulary=countVectorizer.vocabulary_).fit_transform(data_te)\n",
|
||
"X_te = TfidfTransformer().fit_transform(data_te.toarray()).toarray()\n",
|
||
"model = GaussianNB()\n",
|
||
"model.fit(X_tr, labels_tr)\n",
|
||
"pred = model.predict(X_te)\n",
|
||
"score = model.score(X_te, labels_te)\n",
|
||
"with open('model.pkl', 'wb') as f:\n",
|
||
" pickle.dump(model, f)\n",
|
||
"with open('countVectorizer.pkl', 'wb') as f:\n",
|
||
" pickle.dump(countVectorizer, f)\n",
|
||
"plot_confusion_matrix(confusion_matrix(labels_te, pred), [1, 0], title=\"模型分类准确率{:.2f}%\".format(score * 100))"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "cc83543e",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"\n",
|
||
"word_fre = {}\n",
|
||
"for i in data_after_stop[lables == 0]:\n",
|
||
" for j in i:\n",
|
||
" if j not in word_fre.keys():\n",
|
||
" word_fre[j] = 1\n",
|
||
" else:\n",
|
||
" word_fre[j] += 1\n",
|
||
"\n",
|
||
"wc = WordCloud( background_color='white', font_path=r'C:/Windows/Fonts/SimHei.ttf')\n",
|
||
"wc.fit_words(word_fre)\n",
|
||
"plt.imshow(wc)\n",
|
||
"plt.show()\n",
|
||
"word_fre = {}\n",
|
||
"for i in data_after_stop[lables == 0]:\n",
|
||
" for j in i:\n",
|
||
" if j not in word_fre.keys():\n",
|
||
" word_fre[j] = 1\n",
|
||
" else:\n",
|
||
" word_fre[j] += 1\n",
|
||
"\n",
|
||
"wc = WordCloud( background_color='white', font_path=r'C:/Windows/Fonts/SimHei.ttf')\n",
|
||
"wc.fit_words(word_fre)\n",
|
||
"plt.imshow(wc)\n",
|
||
"plt.show()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "bde6f19c",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"0"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "5f089b32",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "Python 3 (ipykernel)",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.10.7"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|