update index module code

code/index_module.py

@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Sat Dec  5 23:31:22 2015
+
+@author: bitjoy.net
+"""
+
+from os import listdir
+import xml.etree.ElementTree as ET
+import jieba
+import sqlite3
+import configparser
+
+class Doc:
+    docid = 0
+    date_time = ''
+    tf = 0
+    ld = 0
+    def __init__(self, docid, date_time, tf, ld):
+        self.docid = docid
+        self.date_time = date_time
+        self.tf = tf
+        self.ld = ld
+    def __repr__(self):
+        return(str(self.docid) + '\t' + self.date_time + '\t' + str(self.tf) + '\t' + str(self.ld))
+    def __str__(self):
+        return(str(self.docid) + '\t' + self.date_time + '\t' + str(self.tf) + '\t' + str(self.ld))
+
+class IndexModule:
+    stop_words = set()
+    postings_lists = {}
+    
+    config_path = ''
+    config_encoding = ''
+    
+    def __init__(self, config_path, config_encoding):
+        self.config_path = config_path
+        self.config_encoding = config_encoding
+        config = configparser.ConfigParser()
+        config.read(config_path, config_encoding)
+        f = open(config['DEFAULT']['stop_words_path'], encoding = config['DEFAULT']['stop_words_encoding'])
+        words = f.read()
+        self.stop_words = set(words.split('\n'))
+
+    def is_number(self, s):
+        try:
+            float(s)
+            return True
+        except ValueError:
+            return False
+        
+    def clean_list(self, seg_list):
+        cleaned_dict = {}
+        n = 0
+        for i in seg_list:
+            i = i.strip().lower()
+            if i != '' and not self.is_number(i) and i not in self.stop_words:
+                n = n + 1
+                if i in cleaned_dict:
+                    cleaned_dict[i] = cleaned_dict[i] + 1
+                else:
+                    cleaned_dict[i] = 1
+        return n, cleaned_dict
+    
+    def write_postings_to_db(self, db_path):
+        conn = sqlite3.connect(db_path)
+        c = conn.cursor()
+        
+        c.execute('''DROP TABLE IF EXISTS postings''')
+        c.execute('''CREATE TABLE postings
+                     (term TEXT PRIMARY KEY, df INTEGER, docs TEXT)''')
+
+        for key, value in self.postings_lists.items():
+            doc_list = '\n'.join(map(str,value[1]))
+            t = (key, value[0], doc_list)
+            c.execute("INSERT INTO postings VALUES (?, ?, ?)", t)
+
+        conn.commit()
+        conn.close()
+    
+    def construct_postings_lists(self):
+        config = configparser.ConfigParser()
+        config.read(self.config_path, self.config_encoding)
+        files = listdir(config['DEFAULT']['doc_dir_path'])
+        AVG_L = 0
+        for i in files:
+            root = ET.parse(config['DEFAULT']['doc_dir_path'] + i).getroot()
+            title = root.find('title').text
+            body = root.find('body').text
+            docid = int(root.find('id').text)
+            date_time = root.find('datetime').text
+            seg_list = jieba.lcut(title + '。' + body, cut_all=False)
+            
+            ld, cleaned_dict = self.clean_list(seg_list)
+            
+            AVG_L = AVG_L + ld
+            
+            for key, value in cleaned_dict.items():
+                d = Doc(docid, date_time, value, ld)
+                if key in self.postings_lists:
+                    self.postings_lists[key][0] = self.postings_lists[key][0] + 1 # df++
+                    self.postings_lists[key][1].append(d)
+                else:
+                    self.postings_lists[key] = [1, [d]] # [df, [Doc]]
+        AVG_L = AVG_L / len(files)
+        config.set('DEFAULT', 'N', str(len(files)))
+        config.set('DEFAULT', 'avg_l', str(AVG_L))
+        with open(self.config_path, 'w', encoding = self.config_encoding) as configfile:
+            config.write(configfile)
+        self.write_postings_to_db(config['DEFAULT']['db_path'])
+
+if __name__ == "__main__":
+    im = IndexModule('../config.ini', 'utf-8')
+    im.construct_postings_lists()

code/recommendation_module.py

@@ -0,0 +1,192 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Dec 23 14:06:10 2015
+
+@author: czl
+"""
+
+from os import listdir
+import xml.etree.ElementTree as ET
+import jieba
+import jieba.analyse
+import sqlite3
+import configparser
+from datetime import *
+import math
+import re
+
+import pandas as pd
+import numpy as np
+
+from sklearn.metrics import pairwise_distances
+
+class RecommendationModule:
+    stop_words = set()
+    k_nearest = []
+    
+    config_path = ''
+    config_encoding = ''
+    
+    doc_dir_path = ''
+    doc_encoding = ''
+    stop_words_path = ''
+    stop_words_encoding = ''
+    idf_path = ''
+    db_path = ''
+    
+    def __init__(self, config_path, config_encoding):
+        self.config_path = config_path
+        self.config_encoding = config_encoding
+        config = configparser.ConfigParser()
+        config.read(config_path, config_encoding)
+        
+        self.doc_dir_path = config['DEFAULT']['doc_dir_path']
+        self.doc_encoding = config['DEFAULT']['doc_encoding']
+        self.stop_words_path = config['DEFAULT']['stop_words_path']
+        self.stop_words_encoding = config['DEFAULT']['stop_words_encoding']
+        self.idf_path = config['DEFAULT']['idf_path']
+        self.db_path = config['DEFAULT']['db_path']
+
+        f = open(self.stop_words_path, encoding = self.stop_words_encoding)
+        words = f.read()
+        self.stop_words = set(words.split('\n'))
+    
+    def write_k_nearest_matrix_to_db(self):
+        conn = sqlite3.connect(self.db_path)
+        c = conn.cursor()
+        
+        c.execute('''DROP TABLE IF EXISTS knearest''')
+        c.execute('''CREATE TABLE knearest
+                     (id INTEGER PRIMARY KEY, first INTEGER, second INTEGER,
+                     third INTEGER, fourth INTEGER, fifth INTEGER)''')
+
+        for docid, doclist in self.k_nearest:
+            c.execute("INSERT INTO knearest VALUES (?, ?, ?, ?, ?, ?)", tuple([docid] + doclist))
+
+        conn.commit()
+        conn.close()
+    
+    def is_number(self, s):
+        try:
+            float(s)
+            return True
+        except ValueError:
+            return False
+            
+    
+    def construct_dt_matrix(self, files, topK = 200):
+        jieba.analyse.set_stop_words(self.stop_words_path)
+        jieba.analyse.set_idf_path(self.idf_path)
+        M = len(files)
+        N = 1
+        terms = {}
+        dt = []
+        for i in files:
+            root = ET.parse(self.doc_dir_path + i).getroot()
+            title = root.find('title').text
+            body = root.find('body').text
+            docid = int(root.find('id').text)
+            tags = jieba.analyse.extract_tags(title + '。' + body, topK=topK, withWeight=True)
+            #tags = jieba.analyse.extract_tags(title, topK=topK, withWeight=True)
+            cleaned_dict = {}
+            for word, tfidf in tags:
+                word = word.strip().lower()
+                if word == '' or self.is_number(word):
+                    continue
+                cleaned_dict[word] = tfidf
+                if word not in terms:
+                    terms[word] = N
+                    N += 1
+            dt.append([docid, cleaned_dict])
+        dt_matrix = [[0 for i in range(N)] for j in range(M)]
+        i =0
+        for docid, t_tfidf in dt:
+            dt_matrix[i][0] = docid
+            for term, tfidf in t_tfidf.items():
+                dt_matrix[i][terms[term]] = tfidf
+            i += 1
+
+        dt_matrix = pd.DataFrame(dt_matrix)
+        dt_matrix.index = dt_matrix[0]
+        print('dt_matrix shape:(%d %d)'%(dt_matrix.shape))
+        return dt_matrix
+        
+    def construct_k_nearest_matrix(self, dt_matrix, k):
+        tmp = np.array(1 - pairwise_distances(dt_matrix[dt_matrix.columns[1:]], metric = "cosine"))
+        similarity_matrix = pd.DataFrame(tmp, index = dt_matrix.index.tolist(), columns = dt_matrix.index.tolist())
+        for i in similarity_matrix.index:
+            tmp = [int(i),[]]
+            j = 0
+            while j < k:
+                max_col = similarity_matrix.loc[i].idxmax(axis = 1)
+                similarity_matrix.loc[i][max_col] =  -1
+                if max_col != i:
+                    tmp[1].append(int(max_col)) #max column name
+                    j += 1
+            self.k_nearest.append(tmp)
+    
+    def gen_idf_file(self):
+        files = listdir(self.doc_dir_path)
+        n = float(len(files))
+        idf = {}
+        for i in files:
+            root = ET.parse(self.doc_dir_path + i).getroot()
+            title = root.find('title').text
+            body = root.find('body').text
+            seg_list = jieba.lcut(title + '。' + body, cut_all=False)
+            seg_list = set(seg_list) - self.stop_words
+            for word in seg_list:
+                word = word.strip().lower()
+                if word == '' or self.is_number(word):
+                    continue
+                if word not in idf:
+                    idf[word] = 1
+                else:
+                    idf[word] = idf[word] + 1
+        idf_file = open(self.idf_path, 'w', encoding = 'utf-8')
+        for word, df in idf.items():
+            idf_file.write('%s %.9f\n'%(word, math.log(n / df)))
+        idf_file.close()
+    
+    def natural_sort(self, l): 
+        convert = lambda text: int(text) if text.isdigit() else text.lower() 
+        alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ] 
+        return sorted(l, key = alphanum_key)
+    
+    def partition_files(self, n_partitions):
+        files = listdir(self.doc_dir_path)
+        files = self.natural_sort(files)
+        n = float(len(files))
+        partitions = []
+        sz = int(n / n_partitions)
+        x = 0
+        for i in range(n_partitions - 1):
+            partitions.append(files[x: x + sz])
+            x += sz
+        partitions.append(files[x:])
+        return partitions
+        
+    def find_k_nearest(self, k, topK, n_partitions):
+        
+        print('-----start gen_idf_file time: %s-----'%(datetime.today()))
+        self.gen_idf_file()
+        
+        partitions = self.partition_files(n_partitions)
+        i = 1
+        for p in partitions:
+            print('-----start %d construct_dt_matrix time: %s-----'%(i, datetime.today()))
+            dt_matrix = self.construct_dt_matrix(p, topK)
+            
+            print('-----start %d construct_k_nearest_matrix time: %s-----'%(i, datetime.today()))
+            self.construct_k_nearest_matrix(dt_matrix, k)
+            i += 1
+      
+        print('-----start write_k_nearest_matrix_to_db time: %s-----'%(datetime.today()))
+        self.write_k_nearest_matrix_to_db()
+        
+if __name__ == "__main__":
+    print('-----start time: %s-----'%(datetime.today()))
+    rm = RecommendationModule('../config.ini', 'utf-8')
+    rm.find_k_nearest(5, 25, 10)
+    print('-----finish time: %s-----'%(datetime.today()))
+    

code/setup.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Dec 25 00:04:40 2015
+
+@author: czl
+"""
+
+from index_module import IndexModule
+from recommendation_module import RecommendationModule
+from datetime import *
+
+
+print('-----start indexing time: %s-----'%(datetime.today()))
+im = IndexModule('../config.ini', 'utf-8')
+im.construct_postings_lists()
+print('-----start recommending time: %s-----'%(datetime.today()))
+rm = RecommendationModule('../config.ini', 'utf-8')
+rm.find_k_nearest(5, 25, 10)
+print('-----finish time: %s-----'%(datetime.today()))

config.ini

@@ -7,6 +7,6 @@ idf_path = ../data/idf.txt
 db_path = ../data/ir.db
 k1 = 1.5
 b = 0.75
-n = 102416
+n = 1000
-avg_l = 348.61414232151225
+avg_l = 462.781