fix: 列表添加排序

clickhouse 初始化错误修复 trace treeline 中间丢失dapr sidercar trace，导致后续traces不显示
masastack · Apr 26, 2024 · 61e0b97 · 61e0b97
1 parent 65691ca
commit 61e0b97
Show file tree

Hide file tree

Showing 11 changed files with 63 additions and 28 deletions.
diff --git a/src/Infrastructure/Masa.Tsc.Storage.Clickhouse.Apm/ApmClickhouseInit.cs b/src/Infrastructure/Masa.Tsc.Storage.Clickhouse.Apm/ApmClickhouseInit.cs
@@ -13,6 +13,7 @@ public static void Init(MasaStackClickhouseConnection connection)
 
     private static void InitErrorTable(MasaStackClickhouseConnection connection)
     {
+        var viewTable = Constants.ErrorTable.Replace(".", ".v_");
         var sql = new string[]{
         @$"CREATE TABLE {Constants.ErrorTable}
 (
@@ -47,7 +48,7 @@ TTL toDateTime(Timestamp) + toIntervalDay(30)
 SETTINGS index_granularity = 8192,
  ttl_only_drop_parts = 1;
 ",
-$@"CREATE MATERIALIZED VIEW {Constants.ErrorTable.Replace(".",".v_")} TO {Constants.ErrorTable}
+$@"CREATE MATERIALIZED VIEW {viewTable} TO {Constants.ErrorTable}
 AS
 SELECT
     Timestamp,TraceId,SpanId, if(position(Body, '\n') > 0,extract(Body, '[^\n\r]+'),Body) `Attributes.exception.message`,LogAttributes['exception.type'] AS `Attributes.exception.type`,
@@ -58,7 +59,8 @@ TTL toDateTime(Timestamp) + toIntervalDay(30)
 FROM {MasaStackClickhouseConnection.LogSourceTable}
 WHERE mapContains(LogAttributes, 'exception.type')
 "};
-        ClickhouseInit.InitTable(connection, Constants.ErrorTable, sql);
+        ClickhouseInit.InitTable(connection, Constants.ErrorTable, sql[0]);
+        ClickhouseInit.InitTable(connection, viewTable, sql[1]);
     }
 
     private static void InitAggregateTable(MasaStackClickhouseConnection connection)
@@ -71,6 +73,7 @@ private static void InitAggregateTable(MasaStackClickhouseConnection connection)
 
     private static void InitAggregateTable(MasaStackClickhouseConnection connection, string interval, string tableName)
     {
+        var viewTable = tableName.Replace(".", ".v_");
         var sql = new string[] {
         $@"CREATE TABLE {tableName}
 (
@@ -95,7 +98,7 @@ ORDER BY (
  )
  TTL toDateTime(Timestamp) + toIntervalDay(30)
 SETTINGS index_granularity = 8192",
-$@"CREATE MATERIALIZED VIEW {tableName.Replace(".",".v_")} TO {tableName}
+$@"CREATE MATERIALIZED VIEW {viewTable} TO {tableName}
 (
     `ServiceName` String,
     `Resource.service.namespace` String,
@@ -110,16 +113,16 @@ TTL toDateTime(Timestamp) + toIntervalDay(30)
 ) AS
 SELECT
     ServiceName,
-    `Resource.service.namespace`,
-    `Attributes.http.target`,
-    `Attributes.http.method`,
+    ResourceAttributes['service.namespace'] `Resource.service.namespace`,
+    SpanAttributes['http.target'] `Attributes.http.target`,
+    SpanAttributes['http.method'] `Attributes.http.method`,
     toStartOfInterval(Timestamp,INTERVAL {interval}) AS Timestamp,
     avgState(Duration) AS Latency,
     countState(1) AS Throughput,
-    sumState(has(['400','500','501','502','503'],`Attributes.http.status_code`)) as Failed,
+    sumState(has(['400','500','501','502','503'],SpanAttributes['http.status_code'])) as Failed,
     quantileState(0.99)(Duration) as P99,
     quantileState(0.95)(Duration) as P95 
-FROM {MasaStackClickhouseConnection.TraceTable}
+FROM {MasaStackClickhouseConnection.TraceSourceTable}
 WHERE
 SpanKind='SPAN_KIND_SERVER'
 GROUP BY
@@ -129,6 +132,8 @@ GROUP BY
     `Attributes.http.method`,
     Timestamp"
         };
-        ClickhouseInit.InitTable(connection, tableName, sql);
+
+        ClickhouseInit.InitTable(connection, tableName, sql[0]);
+        ClickhouseInit.InitTable(connection, viewTable, sql[1]);
     }
 }
diff --git a/src/Infrastructure/Masa.Tsc.Storage.Clickhouse/Extensions/ClickhouseInit.cs b/src/Infrastructure/Masa.Tsc.Storage.Clickhouse/Extensions/ClickhouseInit.cs
@@ -37,6 +37,7 @@ public static void Init(IServiceCollection services)
 
     private static void InitLog()
     {
+        var viewTable = MasaStackClickhouseConnection.LogTable.Replace(".", ".v_");
         string[] sql = new string[] {
             @$"CREATE TABLE {MasaStackClickhouseConnection.LogTable}
 (
@@ -91,7 +92,7 @@ TTL toDateTime(Timestamp) + toIntervalDay(30)
 SETTINGS index_granularity = 8192,
  ttl_only_drop_parts = 1;
 ",
-            $@"CREATE MATERIALIZED VIEW {MasaStackClickhouseConnection.LogTable.Replace(".",".v_")} TO {MasaStackClickhouseConnection.LogTable}
+            $@"CREATE MATERIALIZED VIEW {viewTable} TO {MasaStackClickhouseConnection.LogTable}
 AS
 SELECT
 Timestamp,TraceId,SpanId,TraceFlags,SeverityText,SeverityNumber,ServiceName,Body,ResourceSchemaUrl,toJSONString(ResourceAttributes) as Resources,
@@ -107,12 +108,13 @@ TTL toDateTime(Timestamp) + toIntervalDay(30)
 FROM {MasaStackClickhouseConnection.LogSourceTable}
 ",
         };
-        InitTable(MasaStackClickhouseConnection.LogTable, sql);
+        InitTable(MasaStackClickhouseConnection.LogTable, sql[0]);
+        InitTable(viewTable, sql[1]);
     }
 
     private static void InitTrace(string table, string? where = null)
     {
-
+        var viewTable = table.Replace(".", ".v_");
         string[] sql = new string[] {
             @$"CREATE TABLE {table}
 (
@@ -178,7 +180,7 @@ TTL toDateTime(Timestamp) + toIntervalDay(30)
 SETTINGS index_granularity = 8192,
  ttl_only_drop_parts = 1;
 ",
-            $@"CREATE MATERIALIZED VIEW {table.Replace(".",".v_")} TO {table}
+            $@"CREATE MATERIALIZED VIEW {viewTable} TO {table}
 AS
 SELECT
     Timestamp,TraceId,SpanId,ParentSpanId,TraceState,SpanName,SpanKind,ServiceName,toJSONString(ResourceAttributes) AS Resources,
@@ -204,7 +206,8 @@ TTL toDateTime(Timestamp) + toIntervalDay(30)
 FROM {MasaStackClickhouseConnection.TraceSourceTable}
 {where}
 " };
-        InitTable(table, sql);
+        InitTable(table, sql[0]);
+        InitTable(viewTable, sql[1]);
     }
 
     private static void InitMappingTable()

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Endpoint.razor b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Endpoint.razor
@@ -18,6 +18,7 @@
             ServerItemsLength="total"
             ItemsPerPage="defaultSize"
             Items="data"
+            MustSort
             TItem="ListChartData"
             Loading="@isTableLoading"
             OnOptionsUpdate="OnTableOptionsChanged">

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Endpoints/TimeLine.razor.cs b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Endpoints/TimeLine.razor.cs
@@ -76,6 +76,7 @@ private void CaculateTimelines(List<TraceResponseDto>? traces)
         traceLinkUrl = default;
         spanLinkUrl = default;
         timeLines.Clear();
+        services.Clear();
         if (traces == null || !traces.Any())
             return;
 
@@ -99,19 +100,36 @@ private void CaculateTimelines(List<TraceResponseDto>? traces)
             limit = 100;
         }
 
+        start = traces.Min(item => item.Timestamp);
+        end = traces.Max(item => item.EndTimestamp);
+        CaculateTotalTime(start, end);
+
+        do
+        {
+            SetRootTimeLine(traces);
+        }
+        while (traces.Count > 0);
+
+        timeLines = timeLines.OrderBy(item => item.Trace.Timestamp).ToList();
+        services = services.Distinct().ToList();
+        traceLinkUrl = GetUrl(timeLines[0]);
+    }
+
+
+    private void SetRootTimeLine(List<TraceResponseDto> traces)
+    {
         var roots = GetEmptyParentNodes(traces!);
         if (!roots.Any())
         {
             roots = GetInteruptRootNodes(traces!);
             if (!roots.Any())
                 roots = traces.OrderBy(t => t.Timestamp).ToList();
         }
-        services = traces.Select(item => item.Resource["service.name"].ToString()).Distinct().ToList()!;
-        start = traces.Min(item => item.Timestamp);
-        end = traces.Max(item => item.EndTimestamp);
-        CaculateTotalTime(start, end);
+        services.AddRange(traces.Select(item => item.Resource["service.name"].ToString())!);
+
         foreach (var item in roots)
         {
+            traces.Remove(item);
             var timeLine = new TreeLineDto
             {
                 ParentSpanId = item.ParentSpanId,
@@ -131,8 +149,6 @@ private void CaculateTimelines(List<TraceResponseDto>? traces)
             }
             timeLines.Add(timeLine);
         }
-
-        traceLinkUrl = GetUrl(timeLines[0]);
     }
 
     private List<TreeLineDto> GetChildren(string spanId, List<TraceResponseDto> traces)
@@ -142,6 +158,7 @@ private List<TreeLineDto> GetChildren(string spanId, List<TraceResponseDto> trac
         var result = new List<TreeLineDto>();
         foreach (var item in children)
         {
+            traces.Remove(item);
             var timeLine = new TreeLineDto
             {
                 ParentSpanId = item.ParentSpanId,
@@ -365,6 +382,10 @@ public void SetValue(TraceResponseDto trace, DateTime start, DateTime end, int t
                     if (matches.Count > 0)
                         table = matches[0].Value;
                 }
+                else
+                {
+                    table = database.System;
+                }
 
                 Type = $"{action} {table}";
             }

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Errors.razor b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Errors.razor
@@ -16,6 +16,7 @@
             ServerItemsLength="total"
             ItemsPerPage="defaultSize"
             Items="data"
+            MustSort
             TItem="ErrorMessageDto"
             Loading="@isTableLoading"
             OnOptionsUpdate="OnTableOptionsChanged">

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Logs.razor b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Logs.razor
@@ -16,6 +16,7 @@
             FixedHeader
             ServerItemsLength="total"
             ItemsPerPage="defaultSize"
+            MustSort
             Items="data"
             TItem="LogResponseDto"
             Loading="@isTableLoading"

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Logs.razor.cs b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Logs.razor.cs
@@ -24,8 +24,8 @@ public partial class Logs
     private int page = 1;
     private List<LogResponseDto> data = new();
     private bool isTableLoading = false;
-    private string? sortFiled;
-    private bool? sortBy;
+    private string? sortFiled = nameof(LogResponseDto.Timestamp);
+    private bool? sortBy = true;
     private bool dialogShow = false;
     private LogResponseDto current = null;
 
@@ -97,7 +97,7 @@ private async Task LoadPageDataAsync()
             IsDesc = sortBy ?? false,
             SortField = sortFiled!,
             Query = Search.Text,
-            IsLimitEnv=false
+            IsLimitEnv = false
         };
         var result = await ApiCaller.LogService.GetPageAsync(query);
         data.Clear();

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceEndpoints.razor b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceEndpoints.razor
@@ -31,6 +31,7 @@
         <SDataTable Headers="headers"
                     Height="590"
                     Stripe
+                    MustSort
                     FixedHeader
                     ServerItemsLength="total"
                     ItemsPerPage="defaultSize"

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceErrors.razor b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceErrors.razor
@@ -17,6 +17,7 @@
     <div class="col-12 rounded-lg mt-4" style="border:solid #ccc 1px">
         <SDataTable Headers="headers"
                     Stripe
+                    MustSort
                     Height="300"
                     FixedHeader
                     ServerItemsLength="total"

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceLogs.razor b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceLogs.razor
@@ -18,6 +18,7 @@
         <SDataTable Headers="headers"
                     Stripe
                     Height="300"
+                    MustSort
                     FixedHeader
                     ServerItemsLength="total"
                     ItemsPerPage="defaultSize"

diff --git a/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceLogs.razor.cs b/src/Web/Masa.Tsc.Web.Admin.Rcl/Pages/Apm/Services/ServiceLogs.razor.cs
@@ -22,12 +22,12 @@ public partial class ServiceLogs
     private int defaultSize = 20;
     private int total = 0;
     private int page = 1;
-    private List<LogResponseDto> data = new();
+    private readonly List<LogResponseDto> data = new();
     private bool isTableLoading = false;
-    private string? sortFiled;
-    private bool? sortBy;
+    private string? sortFiled = nameof(LogResponseDto.Timestamp);
+    private bool? sortBy = true;
     private string lastKey = string.Empty;
-    private ChartData chart = new();
+    private readonly ChartData chart = new();
     private bool dialogShow = false;
     private LogResponseDto current = null;
 
@@ -70,7 +70,7 @@ private async Task LoadAsync(SearchData data = null)
         {
             return;
         }
-        await LoadPageDataAsync();       
+        await LoadPageDataAsync();
     }
 
     private async Task LoadChartDataAsync()