WIP chore(controller): dataset consumption statistic

server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/mapper/DataReadLogMapper.java

@@ -17,6 +17,7 @@
 package ai.starwhale.mlops.domain.dataset.dataloader.mapper;
 
 import ai.starwhale.mlops.domain.dataset.dataloader.po.DataReadLogEntity;
+import ai.starwhale.mlops.domain.dataset.dataloader.vo.ConsumptionStatistic;
 import java.util.List;
 import java.util.Objects;
 import org.apache.ibatis.annotations.Insert;
@@ -41,17 +42,17 @@ public interface DataReadLogMapper {
     int insert(DataReadLogEntity dataBlock);
 
     @Insert({
-        "<script>",
-        "INSERT INTO dataset_read_log"
-            + "(id, session_id, start, start_inclusive, end, end_inclusive, size, status)",
-        "VALUES"
-            + "<foreach item='data' collection='records' open='' separator=',' close=''>"
-            + "("
-            + "#{data.id}, #{data.sessionId}, #{data.start}, #{data.startInclusive},"
-            + "#{data.end}, #{data.endInclusive}, #{data.size}, #{data.status}"
-            + ")"
-            + "</foreach>",
-        "</script>"})
+            "<script>",
+            "INSERT INTO dataset_read_log"
+                    + "(id, session_id, start, start_inclusive, end, end_inclusive, size, status)",
+            "VALUES"
+                    + "<foreach item='data' collection='records' open='' separator=',' close=''>"
+                    + "("
+                    + "#{data.id}, #{data.sessionId}, #{data.start}, #{data.startInclusive},"
+                    + "#{data.end}, #{data.endInclusive}, #{data.size}, #{data.status}"
+                    + ")"
+                    + "</foreach>",
+            "</script>"})
     int batchInsert(@Param("records") List<DataReadLogEntity> theCollection);
 
     @Update("UPDATE dataset_read_log SET "
@@ -67,16 +68,16 @@ public static String updateToProcessedSql(Long sessionId,
                                                   String end,
                                                   String status) {
             return new SQL() {{
-                    UPDATE("dataset_read_log");
-                    SET("consumer_id=#{consumerId}", "status=#{status}", "finished_time=NOW()");
-                    WHERE("session_id=#{sessionId}", "start=#{start}");
-                    if (Objects.isNull(end)) {
-                        WHERE("end is null");
-                    } else {
-                        WHERE("end=#{end}");
-                    }
-                }}
-                .toString();
+                        UPDATE("dataset_read_log");
+                        SET("consumer_id=#{consumerId}", "status=#{status}", "finished_time=NOW()");
+                        WHERE("session_id=#{sessionId}", "start=#{start}");
+                        if (Objects.isNull(end)) {
+                            WHERE("end is null");
+                        } else {
+                            WHERE("end=#{end}");
+                        }
+                    }}
+                    .toString();
         }
     }
 
@@ -123,4 +124,18 @@ int updateToProcessed(Long sessionId, String consumerId, String start,
     @Select("SELECT sum(assigned_num) from dataset_read_log "
             + "WHERE session_id in (SELECT id from dataset_read_session where session_id=#{sessionId})")
     int totalAssignedNum(String sessionId);
+
+    @Select("SELECT"
+            + "  drs.dataset_name,"
+            + "  drl.consumer_id as consumer,"
+            + "  drs.batch_size,"
+            + "  drl.status,"
+            + "  sum(drl.size) as data_size,"
+            + "  sum(drl.assigned_num) as assigned_num,"
+            + "  sum(drl.size * drl.assigned_num) as consumption_num "
+            + "FROM dataset_read_session drs, dataset_read_log drl "
+            + "WHERE drs.id = drl.session_id and drs.session_id = #{sessionId}"
+            + "GROUP BY dataset_name, batch_size, status, consumer_id"
+    )
+    List<ConsumptionStatistic> consumptionStatistic(String sessionId);
 }

server/controller/src/main/java/ai/starwhale/mlops/domain/dataset/dataloader/vo/ConsumptionStatistic.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2022 Starwhale, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.starwhale.mlops.domain.dataset.dataloader.vo;
+
+import ai.starwhale.mlops.domain.dataset.dataloader.Status;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ConsumptionStatistic {
+    private String datasetName;
+    private String consumer;
+    private Status.DataStatus status;
+    private Long dataSize;
+    private Long assignedNum;
+    private Long consumedNum;
+}