fix(backend): sql导入增加最大文件上传和最大文件预览限制 #3486

dbm-ui/backend/components/mysql_backup/client.py

@@ -39,7 +39,7 @@ def __init__(self):
             method="POST",
             url="backupapi/client/install",
             description=_("backup_client下载，同步任务"),
-            default_timeout=600,
+            default_timeout=300,
             max_retry_times=1,
         )
 

dbm-ui/backend/db_services/dbbase/instances/handlers.py

@@ -13,7 +13,6 @@
 
 from django.db.models import F, Q
 
-from backend import env
 from backend.constants import IP_PORT_DIVIDER
 from backend.db_meta.models import Machine, ProxyInstance, StorageInstance
 from backend.db_services.ipchooser.handlers.host_handler import HostHandler
@@ -119,9 +118,7 @@ def check_instances(self, query_instances: List[Union[str, Dict]], cluster_ids:
             db_instances.append(db_inst)
 
         # 查询补充主机信息
-        host_infos = HostHandler.check(
-            [{"bk_biz_id": env.DBA_APP_BK_BIZ_ID, "scope_type": "biz"}], [], [], bk_host_ids
-        )
+        host_infos = HostHandler.check([{"bk_biz_id": self.bk_biz_id, "scope_type": "biz"}], [], [], bk_host_ids)
         host_id_info_map = {host_info["host_id"]: host_info for host_info in host_infos}
         return [
             {**host_id_instance_map[str(db_inst)], **{"host_info": host_id_info_map.get(db_inst.bk_host_id, {})}}

dbm-ui/backend/db_services/mysql/open_area/handlers.py

@@ -144,6 +144,7 @@ def __get_openarea_rules_set(cls, config, config_data, operator, cluster_id__clu
                 "cluster_type": cluster_id__cluster[data["cluster_id"]].cluster_type,
             }
             for data in config_data
+            if data.get("authorize_ips")
             for user in user__dbs_rules.keys()
         ]
         return authorize_details

dbm-ui/backend/db_services/mysql/sql_import/constants.py

@@ -19,6 +19,11 @@
 CACHE_SEMANTIC_SKIP_PAUSE_FILED = "{bk_biz_id}_{root_id}_semantic_check_skip_pause"
 SQL_SEMANTIC_CHECK_DATA_EXPIRE_TIME = 7 * 24 * 60 * 60
 
+# 最大预览SQL文件大小200MB
+MAX_PREVIEW_SQL_FILE_SIZE = 200 * 1024 * 1024
+# 最大上传SQL文件大小1G
+MAX_UPLOAD_SQL_FILE_SIZE = 1024 * 1024 * 1024
+
 
 class SQLCharset(str, StructuredEnum):
     """sql语句的字符集类型"""

dbm-ui/backend/db_services/mysql/sql_import/handlers.py

@@ -25,6 +25,7 @@
     CACHE_SEMANTIC_AUTO_COMMIT_FIELD,
     CACHE_SEMANTIC_SKIP_PAUSE_FILED,
     CACHE_SEMANTIC_TASK_FIELD,
+    MAX_PREVIEW_SQL_FILE_SIZE,
     SQL_SEMANTIC_CHECK_DATA_EXPIRE_TIME,
     SQLImportMode,
 )
@@ -70,7 +71,9 @@ def _upload_sql_file(
         # 如果上传的是sql内容, 则创建一个sql文件
         if sql_content:
             sql_file = tempfile.NamedTemporaryFile(suffix=".sql")
-            sql_file.write(str.encode(sql_content, encoding="utf-8"))
+            content_byte = str.encode(sql_content, encoding="utf-8")
+            sql_file.write(content_byte)
+            sql_file.size = len(content_byte)
             sql_file.seek(0)
             sql_file_list = [sql_file]
 
@@ -83,9 +86,13 @@ def _upload_sql_file(
 
                 # 恢复文件指针为文件头，否则会无法读取内容
                 file.seek(0)
-                sql_file_info.update(
-                    sql_path=sql_path, sql_content=file.read().decode("utf-8"), raw_file_name=file.name
-                )
+                # 超过最大预览限制，则不支持预览
+                if file.size > MAX_PREVIEW_SQL_FILE_SIZE:
+                    sql_content = _("当前SQL文件过大，暂不提供内容预览...")
+                else:
+                    sql_content = file.read().decode("utf-8")
+
+                sql_file_info.update(sql_path=sql_path, sql_content=sql_content, raw_file_name=file.name)
 
             sql_file_info_list.append(sql_file_info)
 
@@ -113,7 +120,7 @@ def grammar_check(
             params={"path": dir_name, "files": file_name_list, "cluster_type": self.cluster_type}
         )
 
-        # 填充sql内容。TODO：如果sql内容过大需要进行压缩吗？
+        # 填充sql内容。
         for sql_file_info in sql_file_info_list:
             sql_path = sql_file_info["sql_path"]
             file_name = os.path.split(sql_path)[1]

dbm-ui/backend/db_services/mysql/sql_import/serializers.py

@@ -15,6 +15,7 @@
 from backend.db_services.mysql.sql_import import mock_data
 from backend.db_services.mysql.sql_import.constants import (
     BKREPO_SQLFILE_PATH,
+    MAX_UPLOAD_SQL_FILE_SIZE,
     SQLCharset,
     SQLExecuteTicketMode,
     SQLImportMode,
@@ -43,6 +44,8 @@ def validate(self, attrs):
             raise ValidationError(_("不允许语法检查的sql的内容为空！"))
 
         for file in attrs.get("sql_files", []):
+            if file.size > MAX_UPLOAD_SQL_FILE_SIZE:
+                raise ValidationError(_("请保证单个文件{}不超过1G").fromat(file.name))
             if file.name.rsplit(".")[-1] != "sql":
                 raise ValidationError(_("请保证sql文件[{}]的后缀为.sql").format(file.name))
 

helm-charts/bk-dbm/Chart.lock

@@ -4,25 +4,25 @@ dependencies:
   version: 1.13.0
 - name: mysql
   repository: https://charts.bitnami.com/bitnami
-  version: 9.15.0
+  version: 9.23.0
 - name: redis
   repository: https://charts.bitnami.com/bitnami
   version: 16.13.2
 - name: etcd
   repository: https://charts.bitnami.com/bitnami
-  version: 9.7.5
+  version: 9.15.2
 - name: reloader
   repository: https://stakater.github.io/stakater-charts
-  version: 1.0.54
+  version: 1.0.69
 - name: grafana
   repository: file://./charts/grafana
   version: 7.9.8
 - name: dbm
   repository: file://charts/dbm
-  version: 0.1.31
+  version: 0.1.32
 - name: dbconfig
   repository: file://charts/dbconfig
-  version: 0.1.12
+  version: 0.1.13
 - name: dbpriv
   repository: file://charts/dbpriv
   version: 0.1.33
@@ -56,5 +56,5 @@ dependencies:
 - name: backup-consumer
   repository: file://charts/backup-consumer
   version: 0.0.3
-digest: sha256:fb23872583e43da18a82a66979e2c0be8bc737d32fa04fc5e567741392484e25
-generated: "2023-12-16T16:58:39.846364+08:00"
+digest: sha256:cdce7dfdf738437c0905f61ac8d058a90a1740ca22ed2da5c9ab6c2e863f91d1
+generated: "2024-03-07T20:39:45.062582+08:00"

helm-charts/bk-dbm/Chart.yaml

@@ -27,11 +27,11 @@ dependencies:
   - condition: dbm.enabled
     name: dbm
     repository: file://charts/dbm
-    version: 0.1.31
+    version: 0.1.32
   - condition: dbconfig.enabled
     name: dbconfig
     repository: file://charts/dbconfig
-    version: 0.1.12
+    version: 0.1.13
   - condition: dbpriv.enabled
     name: dbpriv
     repository: file://charts/dbpriv
@@ -79,5 +79,5 @@ dependencies:
 description: A Helm chart for bk-dbm
 name: bk-dbm
 type: application
-version: 1.3.0-alpha.39
-appVersion: 1.3.0-alpha.39
+version: 1.3.0-alpha.40
+appVersion: 1.3.0-alpha.40

helm-charts/bk-dbm/charts/dbconfig/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
-appVersion: 0.0.1-alpha.78
+appVersion: 0.0.1-alpha.80
 description: A Helm chart for dbconfig
 name: dbconfig
 type: application
-version: 0.1.12
+version: 0.1.13

helm-charts/bk-dbm/charts/dbm/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
-appVersion: 1.3.0-alpha.431
+appVersion: 1.3.0-alpha.489
 description: A Helm chart for dbm
 name: dbm
 type: application
-version: 0.1.31
+version: 0.1.32