From acdcdc35cb18623b764ab457755e28d5873c3c93 Mon Sep 17 00:00:00 2001 From: gaoxinxing <15031259256@163.com> Date: Tue, 12 Sep 2023 11:27:26 +0800 Subject: [PATCH 1/3] init --- .../dataloader/mapper/DataReadLogMapper.java | 49 +++++++++++-------- .../dataloader/vo/ConsumptionStatistic.java | 34 +++++++++++++ 2 files changed, 62 insertions(+), 21 deletions(-) create mode 100644 server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/vo/ConsumptionStatistic.java diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/mapper/DataReadLogMapper.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/mapper/DataReadLogMapper.java index dd451e6867..bf4b8596a6 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/mapper/DataReadLogMapper.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/mapper/DataReadLogMapper.java @@ -17,6 +17,7 @@ package ai.starwhale.mlops.domain.dataset.dataloader.mapper; import ai.starwhale.mlops.domain.dataset.dataloader.po.DataReadLogEntity; +import ai.starwhale.mlops.domain.dataset.dataloader.vo.ConsumptionStatistic; import java.util.List; import java.util.Objects; import org.apache.ibatis.annotations.Insert; @@ -41,17 +42,17 @@ public interface DataReadLogMapper { int insert(DataReadLogEntity dataBlock); @Insert({ - ""}) + ""}) int batchInsert(@Param("records") List theCollection); @Update("UPDATE dataset_read_log SET " @@ -67,16 +68,16 @@ public static String updateToProcessedSql(Long sessionId, String end, String status) { return new SQL() {{ - UPDATE("dataset_read_log"); - SET("consumer_id=#{consumerId}", "status=#{status}", "finished_time=NOW()"); - WHERE("session_id=#{sessionId}", "start=#{start}"); - if (Objects.isNull(end)) { - WHERE("end is null"); - } else { - WHERE("end=#{end}"); - } - }} - .toString(); + UPDATE("dataset_read_log"); + SET("consumer_id=#{consumerId}", "status=#{status}", "finished_time=NOW()"); + WHERE("session_id=#{sessionId}", "start=#{start}"); + if (Objects.isNull(end)) { + WHERE("end is null"); + } else { + WHERE("end=#{end}"); + } + }} + .toString(); } } @@ -123,4 +124,10 @@ int updateToProcessed(Long sessionId, String consumerId, String start, @Select("SELECT sum(assigned_num) from dataset_read_log " + "WHERE session_id in (SELECT id from dataset_read_session where session_id=#{sessionId})") int totalAssignedNum(String sessionId); + + @Select("SELECT drs.dataset_name, drl.consumer_id as consumer, drl.status, sum(drl.size) as number " + + "FROM dataset_read_session drs, dataset_read_log drl" + + "WHERE drs.id = drl.session_id and drs.session_id = #{sessionId}" + + "GROUP BY dataset_name, status, consumer_id") + List statistic(String sessionId); } diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/vo/ConsumptionStatistic.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/vo/ConsumptionStatistic.java new file mode 100644 index 0000000000..cdc1a26930 --- /dev/null +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/vo/ConsumptionStatistic.java @@ -0,0 +1,34 @@ +/* + * Copyright 2022 Starwhale, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ai.starwhale.mlops.domain.dataset.dataloader.vo; + +import ai.starwhale.mlops.domain.dataset.dataloader.Status; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ConsumptionStatistic { + private String datasetName; + private String consumer; + private Status.DataStatus status; + private Long number; +} From 11c3c15752f01f28226f3d368a4614f2101c5ec1 Mon Sep 17 00:00:00 2001 From: gaoxinxing <15031259256@163.com> Date: Tue, 12 Sep 2023 17:53:13 +0800 Subject: [PATCH 2/3] fix --- .../dataloader/mapper/DataReadLogMapper.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/mapper/DataReadLogMapper.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/mapper/DataReadLogMapper.java index bf4b8596a6..2612d33780 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/mapper/DataReadLogMapper.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/mapper/DataReadLogMapper.java @@ -125,9 +125,17 @@ int updateToProcessed(Long sessionId, String consumerId, String start, + "WHERE session_id in (SELECT id from dataset_read_session where session_id=#{sessionId})") int totalAssignedNum(String sessionId); - @Select("SELECT drs.dataset_name, drl.consumer_id as consumer, drl.status, sum(drl.size) as number " - + "FROM dataset_read_session drs, dataset_read_log drl" + @Select("SELECT" + + " drs.dataset_name," + + " drl.consumer_id as consumer," + + " drs.batch_size," + + " drl.status," + + " sum(drl.size) as total_number," + + " sum(drl.assigned_num) as assigned_num," + + " sum(drl.size * drl.assigned_num) as total_consumption_number " + + "FROM dataset_read_session drs, dataset_read_log drl " + "WHERE drs.id = drl.session_id and drs.session_id = #{sessionId}" - + "GROUP BY dataset_name, status, consumer_id") + + "GROUP BY dataset_name, batch_size, status, consumer_id" + ) List statistic(String sessionId); } From 4ca2d7e6c0ef69ecb68294a7edb140e55b388d7f Mon Sep 17 00:00:00 2001 From: gaoxinxing <15031259256@163.com> Date: Wed, 13 Sep 2023 14:14:22 +0800 Subject: [PATCH 3/3] optimise --- .../dataloader/mapper/DataReadLogMapper.java | 24 +++++++++---------- .../dataloader/vo/ConsumptionStatistic.java | 4 +++- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/mapper/DataReadLogMapper.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/mapper/DataReadLogMapper.java index 2612d33780..db34b0802f 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/mapper/DataReadLogMapper.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/mapper/DataReadLogMapper.java @@ -68,15 +68,15 @@ public static String updateToProcessedSql(Long sessionId, String end, String status) { return new SQL() {{ - UPDATE("dataset_read_log"); - SET("consumer_id=#{consumerId}", "status=#{status}", "finished_time=NOW()"); - WHERE("session_id=#{sessionId}", "start=#{start}"); - if (Objects.isNull(end)) { - WHERE("end is null"); - } else { - WHERE("end=#{end}"); - } - }} + UPDATE("dataset_read_log"); + SET("consumer_id=#{consumerId}", "status=#{status}", "finished_time=NOW()"); + WHERE("session_id=#{sessionId}", "start=#{start}"); + if (Objects.isNull(end)) { + WHERE("end is null"); + } else { + WHERE("end=#{end}"); + } + }} .toString(); } } @@ -130,12 +130,12 @@ int updateToProcessed(Long sessionId, String consumerId, String start, + " drl.consumer_id as consumer," + " drs.batch_size," + " drl.status," - + " sum(drl.size) as total_number," + + " sum(drl.size) as data_size," + " sum(drl.assigned_num) as assigned_num," - + " sum(drl.size * drl.assigned_num) as total_consumption_number " + + " sum(drl.size * drl.assigned_num) as consumption_num " + "FROM dataset_read_session drs, dataset_read_log drl " + "WHERE drs.id = drl.session_id and drs.session_id = #{sessionId}" + "GROUP BY dataset_name, batch_size, status, consumer_id" ) - List statistic(String sessionId); + List consumptionStatistic(String sessionId); } diff --git a/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/vo/ConsumptionStatistic.java b/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/vo/ConsumptionStatistic.java index cdc1a26930..c2906e4f8f 100644 --- a/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/vo/ConsumptionStatistic.java +++ b/server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/vo/ConsumptionStatistic.java @@ -30,5 +30,7 @@ public class ConsumptionStatistic { private String datasetName; private String consumer; private Status.DataStatus status; - private Long number; + private Long dataSize; + private Long assignedNum; + private Long consumedNum; }