lzh/flask_spider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

修改配置

Browse Source
This commit is contained in:
lzh
2022-03-05 17:10:36 +08:00
parent d78995e280
commit 8a4abfe540
10 changed files with 116716 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
BosonNLP_sentiment_score.txt
+114768
View File
File diff suppressed because it is too large Load Diff
app.py
+6 -1
View File
@@ -6,7 +6,6 @@ from utils import get_data
app = Flask(__name__)
# 连接数据库
# app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://root:LZHlzh.rootOOT123@101.35.53.113:3306/flask_spider'
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://root:123456@localhost:3306/flask_spider'
# 实例化orm框架的操作对象，后续数据库操作，都要基于操作对象来完成
db = SQLAlchemy(app)
@@ -27,6 +26,7 @@ def hello_world():
@app.route("/login", methods=["post"])
def login():
"""登录"""
code, msg = 200, "success"
username = request.json.get("username")
password = request.json.get("password")
@@ -39,6 +39,7 @@ def login():
@app.route("/regist", methods=["post"])
def regist():
"""注册"""
code, msg = 200, "success"
username = request.json.get("username")
password = request.json.get("password")
@@ -54,6 +55,10 @@ def regist():
@app.route("/display", methods=["post"])
def display():
"""
获取绘图数据
:return:
"""
code, msg = 200, "success"
url = request.json.get("url")
username = request.json.get("username")
create_sql.sql
+16 -11
View File
@@ -5,32 +5,37 @@ SET FOREIGN_KEY_CHECKS = 0;
-- ----------------------------
-- Table structure for tb_comments
-- ----------------------------
CREATE TABLE IF NOT EXISTS `tb_comments` (
DROP TABLE IF EXISTS `tb_comments`;
CREATE TABLE `tb_comments` (
`username` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL,
`url` varchar(255) CHARACTER SET latin1 COLLATE latin1_swedish_ci NULL DEFAULT NULL,
`comment` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
`product_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
`update_time` bigint(20) NULL DEFAULT NULL
) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;
-- ----------------------------
-- Table structure for tb_tags
-- ----------------------------
CREATE TABLE IF NOT EXISTS `tb_tags` (
`username` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL,
`url` varchar(255) CHARACTER SET latin1 COLLATE latin1_swedish_ci NULL DEFAULT NULL,
`tag` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL,
DROP TABLE IF EXISTS `tb_tags`;
CREATE TABLE `tb_tags` (
`username` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL,
`url` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL,
`tag` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL,
`count` int(11) NULL DEFAULT NULL,
`update_time` bigint(20) NULL DEFAULT NULL
) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = DYNAMIC;
`update_time` bigint(20) NULL DEFAULT NULL,
`product_id` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL
) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci ROW_FORMAT = DYNAMIC;
-- ----------------------------
-- Table structure for tb_user
CREATE TABLE IF NOT EXISTS `tb_user` (
-- ----------------------------
DROP TABLE IF EXISTS `tb_user`;
CREATE TABLE `tb_user` (
`username` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL,
`password` varchar(255) CHARACTER SET latin1 COLLATE latin1_swedish_ci NOT NULL,
`id` int(11) NOT NULL AUTO_INCREMENT,
PRIMARY KEY (`id`) USING BTREE
) ENGINE = InnoDB AUTO_INCREMENT = 2 CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;
) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;
SET FOREIGN_KEY_CHECKS = 1;
SET FOREIGN_KEY_CHECKS = 1;
run.py
+10
View File
@@ -0,0 +1,10 @@
# -*- coding: utf-8 -*-
# @Time :2022/3/2 15:57
# @Author :lzh
# @File : run.py
# @Software: PyCharm
import os
if __name__ == '__main__':
# 启动脚本，启动vue、flask服务
os.system("python app.py")
os.system("cd static && npm run serve")
static/src/views/about/About.vue
+4 -1
View File
@@ -55,6 +55,10 @@ export default {
};
info.display(params).then((res) => {
if (res.data.code == 200) {
localStorage.setItem("positiveChartData", JSON.stringify( { xData: [], yData: [] }));
localStorage.setItem("negativeChartData", JSON.stringify( { xData: [], yData: [] }));
localStorage.setItem("frequencyChartData", JSON.stringify([]));
localStorage.setItem("pieChartData", JSON.stringify([]));
this.chartsData = res.data.data;
let positiveChartData = res.data.data.positive;
let negativeChartData = res.data.data.negative;
@@ -91,7 +95,6 @@ export default {
localStorage.setItem("negativeChartData", JSON.stringify(this.negativeChartData));
localStorage.setItem("frequencyChartData", JSON.stringify(this.frequencyChartData));
localStorage.setItem("pieChartData", JSON.stringify(this.pieChartData));
localStorage.setItem("url", this.url);
this.$message.success("数据获取成功！请点击相关页签查看更新数据")
} else {
static/src/views/negative/negative.vue
+3
View File
@@ -84,6 +84,9 @@ export default {
);
this.drawChart();
if (negativeChartData){
this.$message.warning("该链接评论数据为空")
}
this.negativeChart.resize();
}
});
stopword.txt
+1902
View File
File diff suppressed because it is too large Load Diff
text_analysis/code.py
+2 -2
View File
@@ -2,6 +2,7 @@ import jieba
import os
def stopwordslist(filepath):
"""读取停用词表"""
stopwords = [line.strip() for line in open(filepath, 'r', encoding='gbk').readlines()]
return stopwords
@@ -37,7 +38,7 @@ def get_top_positive_negative_frequency(data, top=10):
:param top:
:return:
"""
senti_dict = sentiment_dict(os.path.join(os.path.abspath("."),'BosonNLP_sentiment_score.txt'))
senti_dict = sentiment_dict(os.path.join(os.path.abspath("."),'text_analysis\\BosonNLP_sentiment_score.txt'))
print(os.path.join(os.path.abspath("."),'BosonNLP_sentiment_score.txt'))
content_seg = movestopwords(data["comments"])
scores = {}
@@ -62,6 +63,5 @@ def get_top_positive_negative_frequency(data, top=10):
if __name__ == '__main__':
# 读取nlp情感词分数
from utils import get_data
data = get_data("lzh","1512002")
print(get_top_positive_negative_frequency(data,15))
text_analysis/文件说明.txt
-2
View File
@@ -1,5 +1,3 @@
BosonNLP_sentiment_score.txt 情感词对应评分
degree.csv 程度副词表
msyh.ttc 字体文件(用于绘制词云)
not.csv 否定词表
stopword.txt 停用词表
utils.py
+5 -6
View File
@@ -13,9 +13,9 @@ headers = {
class dbutil:
def __init__(self):
self.db = pymysql.connect(host='101.35.53.113',
self.db = pymysql.connect(host='127.0.0.1',
user='root',
password='LZHlzh.rootOOT123',
password='123456',
database='flask_spider')
def insert(self, sql):
@@ -69,8 +69,7 @@ def parse_comments(username, product_id, page_limit=20):
try:
comment = item.get("content", "")
update_time = item.get("createTime", -1)
sql = "insert into tb_comments(username,url,comment,update_time) values ('{}','{}','{}',{});".format(username, comments_url,
comment, update_time)
sql = "insert into tb_comments(username,url,product_id,comment,update_time) values ('{}','{}','{}','{}',{});".format(username, comments_url,product_id,comment, update_time)
dbutil().insert(sql)
except Exception as e:
traceback.print_exc(e)
@@ -88,7 +87,7 @@ def parse_tags(username, product_id):
try:
tag = item.get("name", "")
count = item.get("strCount", 0)
sql = "insert into tb_tags(username,url,tag,count,update_time) values ('{}','{}','{}',{},{});".format(username, tags_url, tag, count,
sql = "insert into tb_tags(username,url,product_id,tag,count,update_time) values ('{}','{}','{}','{}',{},{});".format(username, tags_url, product_id,tag, count,
int(time.time()))
dbutil().insert(sql)
except Exception as e:
@@ -103,7 +102,7 @@ def get_data(username, product_id):
:param product_id:
:return:
"""
sql = "select * from {} where url like '%itemId={}%' and username='{}'"
sql = "select * from {} where product_id='{}' and username='{}'"
res = {
"tags": {},
"comments": None