update code

code/recommendation_module.py

@@ -2,7 +2,7 @@
 """
 Created on Wed Dec 23 14:06:10 2015
 
-@author: czl
+@author: bitjoy.net
 """
 
 from os import listdir
@@ -13,7 +13,6 @@ import sqlite3
 import configparser
 from datetime import *
 import math
-import re
 
 import pandas as pd
 import numpy as np
@@ -147,46 +146,17 @@ class RecommendationModule:
         for word, df in idf.items():
             idf_file.write('%s %.9f\n'%(word, math.log(n / df)))
         idf_file.close()
-    
-    def natural_sort(self, l): 
-        convert = lambda text: int(text) if text.isdigit() else text.lower() 
-        alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ] 
-        return sorted(l, key = alphanum_key)
-    
-    def partition_files(self, n_partitions):
-        files = listdir(self.doc_dir_path)
-        files = self.natural_sort(files)
-        n = float(len(files))
-        partitions = []
-        sz = int(n / n_partitions)
-        x = 0
-        for i in range(n_partitions - 1):
-            partitions.append(files[x: x + sz])
-            x += sz
-        partitions.append(files[x:])
-        return partitions
         
-    def find_k_nearest(self, k, topK, n_partitions):
-        
-        print('-----start gen_idf_file time: %s-----'%(datetime.today()))
+    def find_k_nearest(self, k, topK):
         self.gen_idf_file()
-        
-        partitions = self.partition_files(n_partitions)
-        i = 1
-        for p in partitions:
-            print('-----start %d construct_dt_matrix time: %s-----'%(i, datetime.today()))
-            dt_matrix = self.construct_dt_matrix(p, topK)
-            
-            print('-----start %d construct_k_nearest_matrix time: %s-----'%(i, datetime.today()))
-            self.construct_k_nearest_matrix(dt_matrix, k)
-            i += 1
-      
-        print('-----start write_k_nearest_matrix_to_db time: %s-----'%(datetime.today()))
+        files = listdir(self.doc_dir_path)
+        dt_matrix = self.construct_dt_matrix(files, topK)
+        self.construct_k_nearest_matrix(dt_matrix, k)
         self.write_k_nearest_matrix_to_db()
         
 if __name__ == "__main__":
     print('-----start time: %s-----'%(datetime.today()))
     rm = RecommendationModule('../config.ini', 'utf-8')
-    rm.find_k_nearest(5, 25, 10)
+    rm.find_k_nearest(5, 25)
     print('-----finish time: %s-----'%(datetime.today()))
     

code/setup.py

@@ -2,18 +2,48 @@
 """
 Created on Fri Dec 25 00:04:40 2015
 
-@author: czl
+@author: bitjoy.net
 """
 
+from spider import get_news_pool
+from spider import crawl_news
 from index_module import IndexModule
 from recommendation_module import RecommendationModule
 from datetime import *
+import urllib.request
+import configparser
 
+def get_max_page(root):
+    response = urllib.request.urlopen(root)
+    html = str(response.read())
+    html = html[html.find('var maxPage =') : ]
+    html = html[:html.find(';')]
+    max_page = int(html[html.find('=') + 1 : ])
+    #print(max_page)
+    return(max_page)
 
-print('-----start indexing time: %s-----'%(datetime.today()))
-im = IndexModule('../config.ini', 'utf-8')
-im.construct_postings_lists()
-print('-----start recommending time: %s-----'%(datetime.today()))
-rm = RecommendationModule('../config.ini', 'utf-8')
-rm.find_k_nearest(5, 25, 10)
-print('-----finish time: %s-----'%(datetime.today()))
+def crawling():
+    print('-----start crawling time: %s-----'%(datetime.today()))
+    config = configparser.ConfigParser()
+    config.read('../config.ini', 'utf-8')
+    root = 'http://news.sohu.com/1/0903/61/subject212846158'
+    max_page = get_max_page(root+'.shtml')
+    news_pool = get_news_pool(root, max_page, max_page - 5)
+    crawl_news(news_pool, 140, config['DEFAULT']['doc_dir_path'], config['DEFAULT']['doc_encoding'])
+    
+if __name__ == "__main__":
+    print('-----start time: %s-----'%(datetime.today()))
+    
+    #抓取新闻数据
+    crawling()
+    
+    #构建索引
+    print('-----start indexing time: %s-----'%(datetime.today()))
+    im = IndexModule('../config.ini', 'utf-8')
+    im.construct_postings_lists()
+    
+    #推荐阅读
+    print('-----start recommending time: %s-----'%(datetime.today()))
+    rm = RecommendationModule('../config.ini', 'utf-8')
+    rm.find_k_nearest(5, 25)
+    print('-----finish time: %s-----'%(datetime.today()))