diff --git a/购物评论 情感分类/短信.ipynb b/购物评论 情感分类/短信.ipynb deleted file mode 100644 index df39bfd..0000000 --- a/购物评论 情感分类/短信.ipynb +++ /dev/null @@ -1,176 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "9d091a01", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "import re\n", - "import jieba\n", - "import pickle\n", - "from wordcloud import WordCloud\n", - "from sklearn.naive_bayes import GaussianNB\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n", - "from sklearn.metrics import confusion_matrix\n", - "import seaborn as sns\n", - "import matplotlib.pyplot as plt\n", - "plt.rcParams['font.sans-serif'] = 'SimHei'\n", - "plt.rcParams['axes.unicode_minus'] = False\n", - "import warnings\n", - "warnings.filterwarnings('ignore')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "eff92c98", - "metadata": {}, - "outputs": [], - "source": [ - "def data_process(file='message80W1.csv'):\n", - "#读取数据并进行清洗\n", - " data = pd.read_csv(file, header=None, index_col=0)\n", - " data.columns = ['label', 'message']\n", - " n = 5000\n", - "\n", - " a = data[data['label'] == 0].sample(n)\n", - " b = data[data['label'] == 1].sample(n)\n", - " data_new = pd.concat([a, b], axis=0)\n", - "\n", - " data_dup = data_new['message'].drop_duplicates()\n", - " data_qumin = data_dup.apply(lambda x: re.sub('x', '', x))\n", - "\n", - " jieba.load_userdict('newdic1.txt')\n", - " data_cut = data_qumin.apply(lambda x: jieba.lcut(x))\n", - "\n", - " stopWords = pd.read_csv('stopword.txt', encoding='GB18030', sep='hahaha', header=None)\n", - " stopWords = ['≮', '≯', '≠', '≮', ' ', '会', '月', '日', '–'] + list(stopWords.iloc[:, 0])\n", - " data_after_stop = data_cut.apply(lambda x: [i for i in x if i not in stopWords])\n", - " labels = data_new.loc[data_after_stop.index, 'label']\n", - " adata = data_after_stop.apply(lambda x: ' '.join(x))\n", - " return adata, data_after_stop, labels" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c3ae80f9", - "metadata": {}, - "outputs": [], - "source": [ - "def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues):\n", - " sns.heatmap(cm, annot=True)\n", - " plt.ylabel('True label')\n", - " plt.xlabel('Predicted label')\n", - " plt.title(title)\n", - " plt.xticks(np.arange(len(classes)), classes)\n", - " plt.yticks(np.arange(len(classes)), classes)\n", - " plt.show()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "56dbb0d1", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "adata, data_after_stop, lables = data_process()\n", - "data_tr, data_te, labels_tr, labels_te = train_test_split(adata, lables, test_size=0.2)\n", - "countVectorizer = CountVectorizer()\n", - "data_tr = countVectorizer.fit_transform(data_tr)\n", - "X_tr = TfidfTransformer().fit_transform(data_tr.toarray()).toarray()\n", - "data_te = CountVectorizer(vocabulary=countVectorizer.vocabulary_).fit_transform(data_te)\n", - "X_te = TfidfTransformer().fit_transform(data_te.toarray()).toarray()\n", - "model = GaussianNB()\n", - "model.fit(X_tr, labels_tr)\n", - "pred = model.predict(X_te)\n", - "score = model.score(X_te, labels_te)\n", - "with open('model.pkl', 'wb') as f:\n", - " pickle.dump(model, f)\n", - "with open('countVectorizer.pkl', 'wb') as f:\n", - " pickle.dump(countVectorizer, f)\n", - "plot_confusion_matrix(confusion_matrix(labels_te, pred), [1, 0], title=\"模型分类准确率{:.2f}%\".format(score * 100))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cc83543e", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "word_fre = {}\n", - "for i in data_after_stop[lables == 0]:\n", - " for j in i:\n", - " if j not in word_fre.keys():\n", - " word_fre[j] = 1\n", - " else:\n", - " word_fre[j] += 1\n", - "\n", - "wc = WordCloud( background_color='white', font_path=r'C:/Windows/Fonts/SimHei.ttf')\n", - "wc.fit_words(word_fre)\n", - "plt.imshow(wc)\n", - "plt.show()\n", - "word_fre = {}\n", - "for i in data_after_stop[lables == 0]:\n", - " for j in i:\n", - " if j not in word_fre.keys():\n", - " word_fre[j] = 1\n", - " else:\n", - " word_fre[j] += 1\n", - "\n", - "wc = WordCloud( background_color='white', font_path=r'C:/Windows/Fonts/SimHei.ttf')\n", - "wc.fit_words(word_fre)\n", - "plt.imshow(wc)\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bde6f19c", - "metadata": {}, - "outputs": [], - "source": [ - "0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5f089b32", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.7" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}