lzh/Python-DA-and-DM
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
0c2ba099a8b8140847887aa65a645404f726f39a
Python-DA-and-DM/美赛2020C题/Problem_C_Data/hw1 part2/新建文件夹/.ipynb_checkpoints/Untitled-checkpoint.ipynb
T
lzh 0c2ba099a8 2020美赛c题建模数据+代码+论文H
2020-06-14 13:12:12 +08:00

352 lines
17 KiB
Plaintext
Raw Blame History

{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Q2"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"and daughter (V.W.).\n",
")[Gal(beta 1,3/4)Glc-NAc(beta 1,?)]\n",
"[5] is to jointly learn an em- bedding of words into an n-dimensional vector space and to use these vectors to predict how likely a word is given its context.\n",
".\n",
".\n",
", xm ).\n",
"(1992) J. Biol.\n",
"267, 968-974; Zhou et al.\n",
"(1995) Mol.\n",
"9, 208-218).\n",
"41, 115-123) largely failed to stain tubular basement membranes, suggesting the presence of heparan sulfate chains lacking the specific JM-403 epitope.\n",
"['and daughter (V.W.).', ')[Gal(beta 1,3/4)Glc-NAc(beta 1,?)]', '[5] is to jointly learn an em- bedding of words into an n-dimensional vector space and to use these vectors to predict how likely a word is given its context.', '.', '.', ', xm ).', '(1992) J. Biol.', '267, 968-974; Zhou et al.', '(1995) Mol.', '9, 208-218).', '41, 115-123) largely failed to stain tubular basement membranes, suggesting the presence of heparan sulfate chains lacking the specific JM-403 epitope.']\n"
]
}
],
"source": [
"import nltk\n",
"from nltk.tokenize import sent_tokenize\n",
"from nltk.tokenize import word_tokenize\n",
"f=open('sentence_splitter_input.txt','r',encoding='utf-8')\n",
"ssi=f.read()\n",
"split_result=sent_tokenize(ssi)\n",
"under=[]\n",
"over=[]\n",
"for i in split_result:\n",
" if i[0][0].istitle()==False:\n",
" print(i)\n",
" over.append(i)\n",
"print(over)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Q3"
]
},
{
"cell_type": "code",
"execution_count": 106,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[((' a ', ' aback ', ' abandon '), 1), ((' aback ', ' abandon ', ' abandoned '), 1), ((' abandon ', ' abandoned ', ' abandoning '), 1), ((' abandoned ', ' abandoning ', ' abandonment '), 1), ((' abandoning ', ' abandonment ', ' abaringe '), 1), ((' abandonment ', ' abaringe ', ' abasement '), 1), ((' abaringe ', ' abasement ', ' abated '), 1), ((' abasement ', ' abated ', ' abatuno '), 1), ((' abated ', ' abatuno ', ' abbas '), 1), ((' abatuno ', ' abbas ', \" abbas's \"), 1)]\n"
]
}
],
"source": [
"import pandas as pd\n",
"from nltk import FreqDist\n",
"from nltk import ngrams\n",
"data=pd.read_csv('training set.txt',sep='\\t',header=None)\n",
"bigrams = ngrams(data[0], 3)\n",
"bigramsDist = FreqDist(bigrams)\n",
"print(bigramsDist.most_common(10))"
]
},
{
"cell_type": "code",
"execution_count": 108,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[(('and', 'and', 'and'), 28933), (('a', 'a', 'a'), 23261), (('an', 'an', 'an'), 3739), (('all', 'all', 'all'), 3092), (('about', 'about', 'about'), 1815), (('after', 'after', 'after'), 1075), (('also', 'also', 'also'), 1067), (('af', 'af', 'af'), 1003), (('against', 'against', 'against'), 626), (('american', 'american', 'american'), 599)]\n"
]
}
],
"source": [
"bigrams = ngrams(tri, 3)\n",
"bigramsDist = FreqDist(bigrams)\n",
"print(bigramsDist.most_common(10))"
]
},
{
"cell_type": "code",
"execution_count": 109,
"metadata": {},
"outputs": [],
"source": [
"for key,value in bigramsDist.items():\n",
" for k in key:\n",
" if(k=='hello'):\n",
" print(key,value)\n",
" break"
]
},
{
"cell_type": "code",
"execution_count": 110,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[(('a', 'aback', 'abandon'), 1), (('aback', 'abandon', 'abandoned'), 1), (('abandon', 'abandoned', 'abandoning'), 1), (('abandoned', 'abandoning', 'abandonment'), 1), (('abandoning', 'abandonment', 'abaringe'), 1), (('abandonment', 'abaringe', 'abasement'), 1), (('abaringe', 'abasement', 'abated'), 1), (('abasement', 'abated', 'abatuno'), 1), (('abated', 'abatuno', 'abbas'), 1), (('abatuno', 'abbas', \"abbas's\"), 1)]\n"
]
}
],
"source": [
"from nltk import FreqDist\n",
"from nltk import ngrams\n",
"from nltk.book import text6\n",
"\n",
"bigrams = ngrams(data[0], 3)\n",
"bigramsDist = FreqDist(bigrams)\n",
"print(bigramsDist.most_common(10))\n"
]
},
{
"cell_type": "code",
"execution_count": 111,
"metadata": {},
"outputs": [],
"source": [
"for key,value in bigramsDist.items():\n",
" for k in key:\n",
" if(k==' absence '):\n",
" print(key,value)\n",
" break"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Q4"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['Arizona.docx', 'California.docx', 'Florida.docx', 'Idaho.docx', 'Maryland.docx', 'Nevada.docx', 'New_Jersey.docx', 'North_Carolina.docx', 'Pennsylvania.docx', 'Vermont.docx', '~$rizona.docx']\n",
"=============Arizona.docx==============\n",
"['the copper state']\n",
"=============California.docx==============\n",
"['caltrans']\n",
"=============Florida.docx==============\n",
"['sunshine state']\n",
"=============Idaho.docx==============\n",
"=============Maryland.docx==============\n",
"[]\n",
"['bay state']\n",
"['bay state']\n",
"=============Nevada.docx==============\n",
"=============New_Jersey.docx==============\n",
"[]\n",
"=============North_Carolina.docx==============\n",
"=============Pennsylvania.docx==============\n",
"[]\n",
"[]\n",
"[]\n",
"=============Vermont.docx==============\n"
]
},
{
"ename": "PackageNotFoundError",
"evalue": "Package not found at './state/~$rizona.docx'",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mPackageNotFoundError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m<ipython-input-3-32ea4b732b7b>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m 10\u001b[0m \u001b[0mgsp\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 11\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mstat\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mstats\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m \u001b[0mfile\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdocx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDocument\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdir_path\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mstat\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 13\u001b[0m \u001b[0mstat_name\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'.'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 14\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"=============\"\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mstat\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;34m\"==============\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32mE:\\anaconda3\\lib\\site-packages\\docx\\api.py\u001b[0m in \u001b[0;36mDocument\u001b[1;34m(docx)\u001b[0m\n\u001b[0;32m 23\u001b[0m \"\"\"\n\u001b[0;32m 24\u001b[0m \u001b[0mdocx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_default_docx_path\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mdocx\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mdocx\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m \u001b[0mdocument_part\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mPackage\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdocx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmain_document_part\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 26\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mdocument_part\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontent_type\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mCT\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mWML_DOCUMENT_MAIN\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 27\u001b[0m \u001b[0mtmpl\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"file '%s' is not a Word file, content type is '%s'\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32mE:\\anaconda3\\lib\\site-packages\\docx\\opc\\package.py\u001b[0m in \u001b[0;36mopen\u001b[1;34m(cls, pkg_file)\u001b[0m\n\u001b[0;32m 126\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0mpkg_file\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 127\u001b[0m \"\"\"\n\u001b[1;32m--> 128\u001b[1;33m \u001b[0mpkg_reader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mPackageReader\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_file\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpkg_file\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 129\u001b[0m \u001b[0mpackage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 130\u001b[0m \u001b[0mUnmarshaller\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munmarshal\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpkg_reader\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpackage\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mPartFactory\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32mE:\\anaconda3\\lib\\site-packages\\docx\\opc\\pkgreader.py\u001b[0m in \u001b[0;36mfrom_file\u001b[1;34m(pkg_file)\u001b[0m\n\u001b[0;32m 30\u001b[0m \u001b[0mReturn\u001b[0m \u001b[0ma\u001b[0m \u001b[1;33m|\u001b[0m\u001b[0mPackageReader\u001b[0m\u001b[1;33m|\u001b[0m \u001b[0minstance\u001b[0m \u001b[0mloaded\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0mcontents\u001b[0m \u001b[0mof\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0mpkg_file\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 31\u001b[0m \"\"\"\n\u001b[1;32m---> 32\u001b[1;33m \u001b[0mphys_reader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mPhysPkgReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpkg_file\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 33\u001b[0m \u001b[0mcontent_types\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_ContentTypeMap\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_xml\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mphys_reader\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontent_types_xml\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 34\u001b[0m \u001b[0mpkg_srels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mPackageReader\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_srels_for\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mphys_reader\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mPACKAGE_URI\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;32mE:\\anaconda3\\lib\\site-packages\\docx\\opc\\phys_pkg.py\u001b[0m in \u001b[0;36m__new__\u001b[1;34m(cls, pkg_file)\u001b[0m\n\u001b[0;32m 29\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 30\u001b[0m raise PackageNotFoundError(\n\u001b[1;32m---> 31\u001b[1;33m \u001b[1;34m\"Package not found at '%s'\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mpkg_file\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 32\u001b[0m )\n\u001b[0;32m 33\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;31m# assume it's a stream and pass it to Zip reader to sort out\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
"\u001b[1;31mPackageNotFoundError\u001b[0m: Package not found at './state/~$rizona.docx'"
]
}
],
"source": [
"import docx\n",
"import re\n",
"import os\n",
"#获取文档对象\n",
"dir_path=\"./state/\"\n",
"stats=os.listdir(dir_path)\n",
"print(stats)\n",
"stat_name=[]\n",
"nickname=[]\n",
"gsp=[]\n",
"for stat in stats:\n",
" file=docx.Document(dir_path+stat)\n",
" stat_name.append(stat.split('.')[0])\n",
" print(\"=============\"+stat+\"==============\")\n",
" n_name=[]\n",
" n_money=[]\n",
" for para in file.paragraphs:\n",
" para=para.text.lower()\n",
" # print(re.findall(r'nickname(.*?)', para))\n",
" \n",
" para_t=sent_tokenize(para)\n",
" for p in para_t:\n",
" if 'gross state product' in p:\n",
" money=re.findall('\\$\\d+\\.\\d+',p,re.S)\n",
" n_money.extend(money)\n",
" if 'nickname' in p:\n",
" n_name.extend(re.findall('nickname.*?\\\"(.*?)\"',p,re.S))\n",
" print(n_name)\n",
" nickname.append(n_name)\n",
" gsp.append(n_money)"
]
},
{
"cell_type": "code",
"execution_count": 190,
"metadata": {},
"outputs": [],
"source": [
"# print(gsp)\n",
"# print(nickname)\n",
"# print(stat_name)\n",
"for i,j,k in zip(gsp,nickname,stat_name):\n",
" if i==None:\n",
" print(i)\n",
" print(6)\n",
"res=pd.DataFrame({\n",
" 'stat':stat_name,\n",
" 'nickname':nickname,\n",
" 'gsp':gsp\n",
"})\n",
"res.to_csv('res.tsv',sep='\\t',index=None)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"=============~$rizona.docx==============\n",
"['the copper state']\n"
]
}
],
"source": [
"file=docx.Document(dir_path+'Arizona.docx')\n",
"stat_name.append(stat.split('.')[0])\n",
"print(\"=============\"+stat+\"==============\")\n",
"n_name=[]\n",
"n_money=[]\n",
"for para in file.paragraphs:\n",
" para=para.text.lower()\n",
"# print(re.findall(r'nickname(.*?)', para))\n",
"\n",
" para_t=sent_tokenize(para)\n",
" for p in para_t:\n",
" if 'gross state product' in p:\n",
" money=re.findall('\\$\\d+\\.\\d+',p,re.S)\n",
" n_money.extend(money)\n",
" if 'nickname' in p:\n",
" n_name.extend(re.findall('nickname.*?\\\"(.*?)\"',p,re.S))\n",
" print(n_name)\n",
"# tables=file.tables\n",
"# for i in range(len(tables)):\n",
"# tb=tables[i]\n",
"# #获取表格的行\n",
"# tb_rows=tb.rows\n",
"# #读取每一行内容\n",
"# for i in range(len(tb_rows)):\n",
"# row_data=[]\n",
"# row_cells=tb_rows[i].cells\n",
"# #读取每一行单元格内容\n",
"# for cell in row_cells:\n",
"# #单元格内容\n",
"# row_data.append(cell.text)\n",
"# print(row_data)\n",
"children = file.element.body.iter()\n",
"child_iters = []\n",
"for child in children:\n",
" # 通过类型判断目录\n",
" if child.tag.endswith('textbox'):\n",
" for ci in child.iter():\n",
" if ci.tag.endswith('main}r'):\n",
" child_iters.append(ci)\n",
"textbox = [ci.text for ci in child_iters]\n"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Arizona\n"
]
}
],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Reference in New Issue View Git Blame Copy Permalink