启动脚本优化，注册邮箱长度限制修改为32

Auto_maching_learning/settings.py

@@ -81,13 +81,17 @@ WSGI_APPLICATION = 'Auto_maching_learning.wsgi.application'
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.mysql',  # 数据库引擎
-        'NAME': 'AML',  # 你要存储数据的库名，事先要创建之
-        'USER': 'root',  # 数据库用户名
-        'PASSWORD': 'LZHlzh.root123',  # 密码
-        'HOST': 'localhost',  # 主机
-        'PORT': '3306',  # 数据库使用的端口
+        "ENGINE": "django.db.backends.sqlite3",
+        'NAME': os.path.join(BASE_DIR, 'db.sqlite3')
     },
+    # 'mysql': {
+    #     'ENGINE': 'django.db.backends.mysql',  # 数据库引擎
+    #     'NAME': 'AML',  # 你要存储数据的库名，事先要创建之
+    #     'USER': 'root',  # 数据库用户名
+    #     'PASSWORD': '',  # 密码
+    #     'HOST': 'localhost',  # 主机
+    #     'PORT': '3306',  # 数据库使用的端口
+    # },
     'mongotest': {
         'ENGINE': None,
     }

ModelSelection/dataset_process_model.py

@@ -13,13 +13,30 @@ import pandas as pd
 import pandas_profiling
 import pymongo
 
+from Auto_maching_learning.settings import LOG_DIR
 from ModelSelection.models import DatasetModel
+from utils.MODEL_DICT import CLEAN_DICT
+from utils.logutil import set_log
+logger = set_log(os.path.join(LOG_DIR, os.path.split(__file__)[1].split(".")[0]))
+
+template_path = os.path.join(os.path.abspath(''), 'code_templates')
+
+
+def joint_code(code_path, encoding='utf-8'):
+    """拼接代码文件"""
+    text = ""
+    try:
+        try:
+            f = open(os.path.join(template_path, code_path), 'r', encoding=encoding)
+            text = f.read()
+        except Exception as e:
+            f = open(os.path.join(template_path, code_path), 'r', encoding='gbk')
+            text = f.read()
+    except Exception as e:
+        logger.exception(e)
+    return text
 
 
-# username='root'
-# password='lzh.mongo.admin'
-# url='47.97.197.244'
-# port=27017
 class DatasetProcess:
     def __init__(self, database="AML", collection="user_model", username="admin"):
         self.client = pymongo.MongoClient(host="localhost", port=27017)
@@ -46,7 +63,7 @@ class DatasetProcess:
         postfix = os.path.split(file_path)[-1].split(".")
         filename = postfix[0] + "_" + postfix[1]
         if not self.isVip:
-            if (len(self.datasets) > 5):
+            if len(self.datasets) > 5:
                 return False, "非会员最多存储五份数据集"
         if filename in self.datasets:
             return False, "该数据集已存在"
@@ -119,7 +136,6 @@ class DatasetProcess:
         except Exception as e:
             raise e
 
-    # todo 后续确保filename为相对路径
     def generate_report(self, dataset_name):
         try:
             current_path = os.getcwd()
@@ -137,6 +153,38 @@ class DatasetProcess:
         except Exception as e:
             raise e
 
+
+    def generate_clean_code(self, user_name, dataset_name, conditions):
+        """
+        传入数据集，清洗条件
+        :param user_name:
+        :param dataset_name:
+        :param conditions:
+        :return: str : 数据清洗代码
+        """
+        generate_code = joint_code("main_clean.py") % dataset_name
+        try:
+            for condition in conditions:
+                try:
+                    cols = condition.get("columns", [])
+                    clean_method = condition.get("clean_method", "")
+                    clean_expression = CLEAN_DICT.get(clean_method, "")
+
+                    # 清洗方法为字典，说明含有子方法，从映射表中取出对应表达式后执行语句
+                    if clean_expression and isinstance(clean_expression, dict):
+                        sub_method = condition.get("sub_method", "")
+                        clean_expression = CLEAN_DICT.get(clean_method, {}).get(sub_method)
+
+                    generate_code += """\ncols = {}\n{}
+                                        """.format(cols, clean_expression)
+                except Exception as e:
+                    raise Exception(e)
+            generate_code += "\ndf.to_csv('clean_'+FILE_PATH+'.csv')"
+            timestamp = time.time()
+            return generate_code
+        except Exception as e:
+            raise e
+
     def delete(self, dataset_name):
         '''
         根据数据集名称删除数据

ModelSelection/model_process_model.py

@@ -16,9 +16,10 @@ import os
 - 主函数预先定义一个代码文件，相关参数通过占位符填充，填充的参数来源于前段输入，包括：主要包括特征列、目标列，文件名
 """
 from utils.MODEL_DICT import MODEL_DICT
+from ModelSelection.dataset_process_model import joint_code
 
 
-class SetModel():
+class SetModel:
     """
 
     用于与前端界面交互，获取特征列，以及数据处理步骤。
@@ -42,7 +43,6 @@ class SetModel():
         :param model_name(str of list):模型
         :param evaluate_methods(str of list,非必填):模型评估方法
         """
-        self.code_files = os.path.join(os.path.abspath(''), 'code_templates')
         self.name = name
         self.dataset_name = dataset_name
         self.target = target
@@ -54,22 +54,10 @@ class SetModel():
         self.generate = ''
         self.clean_code = ''
 
-    def joint_code(self, code_path, encoding='utf-8'):
-        """拼接代码文件"""
-        try:
-            f = open(os.path.join(self.code_files, code_path), 'r', encoding=encoding)
-            self.generate += f.read()
-        except Exception as e:
-            f = open(os.path.join(self.code_files, code_path), 'r', encoding='gbk')
-            self.generate += f.read()
-
-    def get_clean_code(self):
-        pass
-
     def get_code(self):
         """ 生成代码"""
         # 拼接导入的库
-        self.joint_code('ImportPackages.py')
+        self.generate += joint_code('ImportPackages.py')
         # 拼接模型需要的库
         for model in self.model_name:
             self.generate += MODEL_DICT[self.model_type][model] + "\n"
@@ -88,21 +76,21 @@ MODEL = [{}]
 
         # 拼接分类/回归/聚类的主函数与必要评估方法
         if self.model_type == "分类":
-            self.joint_code('main_supervisied.py')
-            self.joint_code("evaluation_classifier.py")
+            self.generate += joint_code('main_supervisied.py')
+            self.generate += joint_code("evaluation_classifier.py")
         elif self.model_type == "回归":
-            self.joint_code('main_supervisied.py')
-            self.joint_code("evaluation_regressor.py")
+            self.generate += joint_code('main_supervisied.py')
+            self.generate += joint_code("evaluation_regressor.py")
         elif self.model_type == "聚类":
-            self.joint_code("main_unsupervised.py")
-            self.joint_code("evaluation_cluster.py")
+            self.generate += joint_code("main_unsupervised.py")
+            self.generate += joint_code("evaluation_cluster.py")
 
         # 拼接用户自选的可视化的评估方法
         if len(self.evaluate_methods) != 0:
             for method in self.evaluate_methods:
                 location = MODEL_DICT.get(method)
                 if location:
-                    self.joint_code(location)
+                    self.generate += joint_code(location)
 
         # 生成代码文件
         filename = "generate_{}_{}.py".format(self.username, self.name)

ModelSelection/model_selection_views.py

@@ -1,20 +1,20 @@
 import json
 import os
-import sys
-import traceback
 
 import pandas as pd
 from django.http import JsonResponse, HttpResponse
 from django.shortcuts import render
 from django.views.decorators.http import require_http_methods
 
+from Auto_maching_learning.settings import LOG_DIR
 from ModelSelection.dataset_process_model import DatasetProcess
 from ModelSelection.model_process_model import SetModel
 from utils.MODEL_DICT import CLEAN_DICT, MODEL_DICT, METRICS_DICT
 from utils.logutil import set_log
-from Auto_maching_learning.settings import LOG_DIR
 
 logger = set_log(os.path.join(LOG_DIR, os.path.split(__file__)[1].split(".")[0]))
+
+
 # Create your views here.
 
 def get_datesets_list(request):
@@ -258,7 +258,7 @@ def get_methods(request):
         "clean": CLEAN_DICT,
         "metrics": METRICS_DICT
     }
-    code_dict = types.get(request.GET.get("type"),{})
+    code_dict = types.get(request.GET.get("type"), {})
     for method, sub_method in code_dict.items():
         if isinstance(sub_method, dict):
             res[method] = list(sub_method.keys())
@@ -267,6 +267,25 @@ def get_methods(request):
     return JsonResponse({"data": res}, status=200)
 
 
+@require_http_methods(['POST'])
+def generate_clean_code(request):
+    data, code, msg = None, 200, None
+    try:
+        post_body = json.loads(request.body)
+        dataset_name = post_body.pop("dataset", "")
+        user_name = post_body.pop("user_name", "")
+        conditions = post_body.pop("conditions", {})
+        dp = DatasetProcess(username=user_name)
+        data = dp.generate_clean_code(user_name, dataset_name, conditions)
+        msg = "生成成功！"
+    except Exception as e:
+        msg = str(e)
+        code = 500
+        logger.exception(e)
+
+    return JsonResponse({"data": data, "msg": msg, "code": code})
+
+
 @require_http_methods(['POST'])
 def plot_graph(request):
-    pass
+    pass

ModelSelection/urls.py

@@ -14,7 +14,8 @@ urlpatterns = [
     path('generate_code', generate_code),  # 生成代码
     path('export_code', export_code),  # 导出代码文件
     path('del_dataset', del_dataset),  # 删除数据集
-    path('plot_graph', plot_graph)  # 绘制图形
+    path('plot_graph', plot_graph),  # 绘制图形
+    path('generate_clean_code', generate_clean_code) # 生成数据清洗代码
 
 ]
 

UserAuthority/models.py

@@ -6,6 +6,6 @@ class UserModel(mongoengine.Document):
     username = mongoengine.StringField(max_length=16)
     password = mongoengine.StringField(max_length=16)
     phone = mongoengine.StringField(max_length=16)
-    email = mongoengine.StringField(max_length=16)
+    email = mongoengine.StringField(max_length=32)
     isVip = mongoengine.BooleanField(default=False)  # 会员权限
     dataset = mongoengine.ListField()

UserAuthority/user_authority_model.py

@@ -37,7 +37,7 @@ class UserProcess():
 def Regist(infos, **params):
     '''
     注册信息
-    :param infos:dict
+    :param infos:dict 注册表单信息
     :return:
     '''
     try:

UserAuthority/user_authority_views.py

@@ -74,6 +74,7 @@ def regist(request):
             data['code'] = 200
             data['msg'] = '注册成功'
         else:
+            logger.error("注册表单:{};注册验证码:{}".format(infos, checkcode))
             raise Exception("验证码错误，注册失败")
     except Exception as e:
         logger.exception(e)

code_templates/main_clean.py

@@ -0,0 +1,9 @@
+import pandas as pd
+
+FILE_PATH = '%s'
+df = None
+# 读取数据
+if FILE_PATH.split('.')[-1] == 'xls' or FILE_PATH.split('.')[-1] == 'xlsx':
+    df = pd.read_excel(FILE_PATH)
+else:
+    df = pd.read_csv(FILE_PATH)

run_start.py

@@ -6,6 +6,7 @@
 # @description :前后端启动脚本
 import os
 import platform
+import sys
 import threading
 
 
@@ -19,13 +20,34 @@ class myThread(threading.Thread):
 
 
 # 创建新线程
-start_backend = "python3 manage.py runserver"
+start_backend = "python manage.py runserver"
 if platform.system().lower() == "linux":
-    start_backend = "python manage.py runserver"
-start_fronend = "cd ../AML_frontend/ && npm run dev"
+    start_backend = "python3 manage.py runserver"
+
+# 前端打包后运行
+# 带参数，则打包静态资源部署到compressedAML/public目录下
+BASE_PATH = os.getcwd()
+STATIC_PATH = os.path.join(BASE_PATH, "..", "compressedAML")
+FRONTEND_PATH = os.path.join(BASE_PATH, "..", "AML_frontend")
+
+if not os.path.exists(STATIC_PATH):
+    os.system("cd .. && npm install express-generator -g && express compressedAML && cd compressedAML && npm install ")
+
+start_fronend = "cd {} && npm run dev".format(FRONTEND_PATH)
+# dist参数执行打包好后的静态文件
+if len(sys.argv) > 1 and sys.argv[1] == "dist":
+    start_fronend = "cd {} && npm start".format(STATIC_PATH)
+# build 执行打包命令
+elif len(sys.argv) > 1 and sys.argv[1] == "build":
+    os.system("cd {} and npm run build".format(FRONTEND_PATH))
+    # 打包后的静态文件移到到
+    os.system("mv {} {}".format(os.path.join(FRONTEND_PATH, "dist", "*"), os.path.join(STATIC_PATH,"public")))
+    start_fronend = "cd {} && npm start".format(STATIC_PATH)
+
 thread_backend = myThread(start_backend)
 thread_frontend = myThread(start_fronend)
 
 # 开启新线程
 thread_backend.start()
 thread_frontend.start()
+# pipreqs. / --encoding = utf - 8

utils/EmailClient.py

@@ -76,8 +76,10 @@ class EmailService():
         print(regist_info)
         if regist_info:
             send_time = regist_info.get('send_time')
+            # 验证码保留五分钟
             if time.time() - send_time > self.valid_time:
                 self.temp_collection.delete_one(query)
+
                 return False
             if text == str(regist_info.get("check_code")):
                 self.temp_collection.delete_one(query)

utils/MODEL_DICT.py

@@ -55,9 +55,9 @@ CLEAN_DICT = {
     "重复项删除": "df[cols] = df[cols].dropna()",
     "排序": "df.sort_values(by=cols, inplace= True)",
     "筛选": {
-        "大于": "df[cols]",
-        "等于": "df[cols]",
-        "小于": "df[cols]",
+        "大于": "df[df[cols]>",
+        "等于": "df[df[cols]=",
+        "小于": "df[df[cols]<",
         "包含": "df[cols].str().contains()"
     },
     "标准化": {