新增抓取中国新闻网chinanews.com的爬虫，以及从该网站抓取的一万条新闻

2020-04-04 22:54:40 +08:00
parent 59e3fb030a
commit 60862d5c9e
1148 changed files with 202 additions and 19075 deletions
@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Sat Apr  4 17:38:38 2020
+
+@author: Zhenlin
+"""
+
+#清理乱码新闻
+
+import os
+import xml.etree.ElementTree as ET
+import configparser
+
+config = configparser.ConfigParser()
+config.read('../config.ini', 'utf-8')
+
+folder=config['DEFAULT']['doc_dir_path']
+
+files=os.listdir(folder)
+
+files = sorted(files, key=lambda x: int(os.path.splitext(x)[0]))
+
+keyword='编辑'
+
+i=1
+for j,file in enumerate(files):
+    print('%d/%d'%(j,len(files)))
+    path='%s\%s'%(folder,file)
+
+    tree =  ET.parse(path)
+    root = tree.getroot()
+    body = root.find('body').text
+    docid = root.find('id')
+    if keyword in body:
+        docid.text = str(i)
+        tree.write('%s\%d.xml'%(folder,i), encoding=config['DEFAULT']['doc_encoding'])
+        i+=1
+    else:
+        os.remove(path)
+        
+print('# valid files = %d'%(i-1))