lzh/machine_learning_projects
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

上传病马数据分析

Browse Source 
Signed-off-by: 邓凯洋 <13202611+deng-kaiyang@user.noreply.gitee.com>
This commit is contained in:
邓凯洋
2023-07-16 04:33:58 +00:00
committed by Gitee
parent ef68674b68
commit aecba00a0c
1 changed files with 539 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
微博谣言分析（第八组）/平时作业/病马数据分析.ipynb
+539
View File
@@ -0,0 +1,539 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 17,
"id": "066bb5b7",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import seaborn as sns\n",
"from sklearn import preprocessing\n",
"from sklearn.preprocessing import StandardScaler\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.metrics import accuracy_score,confusion_matrix,roc_curve,roc_auc_score\n",
"from sklearn.ensemble import RandomForestClassifier,BaggingClassifier,AdaBoostClassifier\n",
"from sklearn.naive_bayes import GaussianNB\n",
"from sklearn.tree import DecisionTreeClassifier\n",
"from sklearn.svm import SVC\n",
"from sklearn.linear_model import LogisticRegression\n",
"from sklearn.neighbors import KNeighborsClassifier\n",
"# 中文正常显示\n",
"plt.rcParams['font.sans-serif'] = ['SimHei']\n",
"plt.rcParams['axes.unicode_minus'] = False"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "4c7866bc",
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>直肠温度</th>\n",
" <th>脉搏</th>\n",
" <th>呼吸频率</th>\n",
" <th>红细胞体积</th>\n",
" <th>总蛋白值</th>\n",
" <th>y</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>38.5</td>\n",
" <td>66</td>\n",
" <td>28</td>\n",
" <td>45.0</td>\n",
" <td>8.4</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>39.2</td>\n",
" <td>88</td>\n",
" <td>20</td>\n",
" <td>50.0</td>\n",
" <td>85.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>38.3</td>\n",
" <td>40</td>\n",
" <td>24</td>\n",
" <td>33.0</td>\n",
" <td>6.7</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>39.1</td>\n",
" <td>164</td>\n",
" <td>84</td>\n",
" <td>48.0</td>\n",
" <td>7.2</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>37.3</td>\n",
" <td>104</td>\n",
" <td>35</td>\n",
" <td>74.0</td>\n",
" <td>7.4</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" 直肠温度 脉搏 呼吸频率 红细胞体积 总蛋白值 y\n",
"0 38.5 66 28 45.0 8.4 0.0\n",
"1 39.2 88 20 50.0 85.0 0.0\n",
"2 38.3 40 24 33.0 6.7 1.0\n",
"3 39.1 164 84 48.0 7.2 0.0\n",
"4 37.3 104 35 74.0 7.4 0.0"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df = pd.read_csv(\"data_horse.csv\",encoding=\"gbk\")\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "6b170345",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "7ea4f6d8c3db45a79c13b723b97dd20e",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Summarize dataset: 0%| | 0/5 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b9adcb8a22924cba954fb39b5613b26b",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Generate report structure: 0%| | 0/1 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "5f8a8815e4e44c67ad341e6b4c83f80b",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Render HTML: 0%| | 0/1 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "92bf5783980c495395b66dc4feedd203",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Export report to file: 0%| | 0/1 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"from pandas_profiling import ProfileReport\n",
"profile = ProfileReport(df, title='Pandas Profiling Report', html={'style':{'full_width':True}})\n",
"profile.to_file(output_file=\"your_report.html\")"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "7a3827d1",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"y 1.000000\n",
"直肠温度 0.172629\n",
"总蛋白值 0.114651\n",
"呼吸频率 -0.036938\n",
"红细胞体积 -0.152119\n",
"脉搏 -0.276255\n",
"Name: y, dtype: float64"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"## 可视化列的缺失值、唯一值、类型\n",
"df.corr()\n",
"# 可视化变量y与其他的相关性曲线\n",
"df.corr()['y'].sort_values(ascending=False)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "44e52860",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>直肠温度</th>\n",
" <th>脉搏</th>\n",
" <th>红细胞体积</th>\n",
" <th>总蛋白值</th>\n",
" <th>y</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>38.5</td>\n",
" <td>66</td>\n",
" <td>45.0</td>\n",
" <td>8.4</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>39.2</td>\n",
" <td>88</td>\n",
" <td>50.0</td>\n",
" <td>85.0</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>38.3</td>\n",
" <td>40</td>\n",
" <td>33.0</td>\n",
" <td>6.7</td>\n",
" <td>1.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>39.1</td>\n",
" <td>164</td>\n",
" <td>48.0</td>\n",
" <td>7.2</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>37.3</td>\n",
" <td>104</td>\n",
" <td>74.0</td>\n",
" <td>7.4</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" 直肠温度 脉搏 红细胞体积 总蛋白值 y\n",
"0 38.5 66 45.0 8.4 0.0\n",
"1 39.2 88 50.0 85.0 0.0\n",
"2 38.3 40 33.0 6.7 1.0\n",
"3 39.1 164 48.0 7.2 0.0\n",
"4 37.3 104 74.0 7.4 0.0"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# 剔除相关性绝对值小于0.1的变量\n",
"df = df[df.columns[df.corr()['y'].abs()>0.1]]\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "4cf93c3e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"直肠温度 0\n",
"脉搏 0\n",
"红细胞体积 0\n",
"总蛋白值 0\n",
"y 9\n",
"dtype: int64"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.isnull().sum()"
]
},
{
"cell_type": "code",
"execution_count": 30,
"id": "730813c8",
"metadata": {},
"outputs": [],
"source": [
"#可视化每一个变量\n",
"import warnings\n",
"warnings.filterwarnings(\"ignore\")\n",
"for i in df.columns:\n",
" plt.figure(figsize=(10,5))\n",
" plt.title(i)\n",
" # 设置字体旋转角度\n",
" plt.xticks(rotation=90)\n",
" sns.countplot(df[i])\n",
" plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "34e32440",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"直肠温度 69\n",
"脉搏 26\n",
"红细胞体积 36\n",
"总蛋白值 42\n",
"y 9\n",
"dtype: int64"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# 我们认为这些变量中的0值都是不合理的，所以我们将除了y(最后一列)以外的变量中的0值都替换为np.nan\n",
"df[df.columns[:-1]] = df[df.columns[:-1]].replace(0,np.nan)\n",
"df.isnull().sum()"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "4a7f1ad7",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"直肠温度 0\n",
"脉搏 0\n",
"红细胞体积 0\n",
"总蛋白值 0\n",
"y 9\n",
"dtype: int64"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# 用众数填充缺失值\n",
"# 观察发现直肠温度列是正太分布的，因此可以用均值填充缺失值，其他使用中位数\n",
"df['直肠温度'] = df['直肠温度'].fillna(df['直肠温度'].mean())\n",
"df[['脉搏','红细胞体积','总蛋白值']] = df[['脉搏','红细胞体积','总蛋白值']].fillna(df[['脉搏','红细胞体积','总蛋白值']].median())\n",
"df.isnull().sum()"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "b671b6a2",
"metadata": {},
"outputs": [],
"source": [
"## 删除缺失值\n",
"df = df.dropna()"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "f1a812b6",
"metadata": {},
"outputs": [],
"source": [
"# 再查看一下每个变量的分布\n",
"for i in df.columns:\n",
" plt.figure(figsize=(10,5))\n",
" plt.title(i)\n",
" # 设置字体旋转角度\n",
" plt.xticks(rotation=90)\n",
" sns.countplot(df[i])\n",
" plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "78edccee",
"metadata": {},
"outputs": [],
"source": [
"# 由于变量的取值范围不一样，因此需要对数据进行标准化\n",
"import time\n",
"X_train,X_test,y_train,y_test = train_test_split(df[df.columns[:-1]],df['y'],test_size=0.3,random_state=0)\n",
"X_train = StandardScaler().fit_transform(X_train)\n",
"X_test = StandardScaler().fit_transform(X_test)\n",
"\n",
"model_result = {}\n",
"\n",
"models = [RandomForestClassifier(),BaggingClassifier(),AdaBoostClassifier(),GaussianNB(),LogisticRegression(),DecisionTreeClassifier(),SVC(),KNeighborsClassifier()]\n",
"for model in models:\n",
" try:\n",
" model_name = str(model).split('(')[0]\n",
" start = time.time()\n",
" model.fit(X_train,y_train)\n",
" y_pred = model.predict(X_test)\n",
" end = time.time()\n",
" # 存储准确率和混淆矩阵\n",
" model_result[model_name] = [accuracy_score(y_test,y_pred),confusion_matrix(y_test,y_pred),end-start]\n",
" except Exception as e:\n",
" print(model_name,e)\n",
"\n",
"## 使用df保存模型的结果\n",
"df_result = pd.DataFrame(model_result).T\n",
"df_result.columns = ['accuracy_score','confusion_matrix','time']\n",
"\n",
"df_result.sort_values(by='accuracy_score',ascending=False,inplace=True)"
]
},
{
"cell_type": "code",
"execution_count": 29,
"id": "bea009fc",
"metadata": {},
"outputs": [],
"source": [
"# 绘制各个模型的准确率柱状图，并在柱状图上标注值\n",
"plt.figure(figsize=(10,5))\n",
"plt.title('各个模型的准确率')\n",
"sns.barplot(x=df_result.index,y=df_result['accuracy_score'])\n",
"for i in range(df_result.shape[0]):\n",
" plt.text(i,df_result['accuracy_score'][i],round(df_result['accuracy_score'][i],3),ha='center')\n",
"# 设置字体选择\n",
"plt.xticks(rotation=90)\n",
"plt.show()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.9"
}
},
"nbformat": 4,
"nbformat_minor": 5
}