feat: support trace & log query with clickhouse (#676)

* feat: add trace & log query with clickhouse * Update pr_run_test_ci.yml * chore: update test project * chore: update * chore: fix path * chore: test update * chore: update test case * chore: update * chore: update * chore: update * chore: update * chore: update * fix: fix bugs * refactor: update table schema * restore: restore test clickhouse connect * chore: update * chore: update * fix: fix bugs
masastack · Nov 14, 2023 · fb54095 · fb54095
1 parent ee994e9
commit fb54095
Show file tree

Hide file tree

Showing 25 changed files with 1,435 additions and 6 deletions.
diff --git a/.github/workflows/pr_run_test_ci.yml b/.github/workflows/pr_run_test_ci.yml
@@ -87,6 +87,11 @@ jobs:
           analysis-icu
           analysis-smartcn
           analysis-kuromoji
+    - name: Setup ClickHouse     
+      uses: vahid-sohrabloo/clickhouse-action@v1
+      with:
+         # Version of ClickHouse to use
+        version: latest # optional, default is latest
     - name: Build and analyze for internal
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any

diff --git a/Masa.Framework.sln b/Masa.Framework.sln
@@ -709,9 +709,13 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tsc", "tsc", "{6042AE23-A07
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DynamicsCRM", "DynamicsCRM", "{64B54122-44F1-4379-9422-953EF706A3A6}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Masa.Utils.DynamicsCrm.Core", "src\Utils\DynamicsCrm\Masa.Utils.DynamicsCrm.Core\Masa.Utils.DynamicsCrm.Core.csproj", "{83310F46-E1C7-4438-B32A-9F6F7EA13FCF}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.Utils.DynamicsCrm.Core", "src\Utils\DynamicsCrm\Masa.Utils.DynamicsCrm.Core\Masa.Utils.DynamicsCrm.Core.csproj", "{83310F46-E1C7-4438-B32A-9F6F7EA13FCF}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Masa.Utils.DynamicsCrm.EntityFrameworkCore", "src\Utils\DynamicsCrm\Masa.Utils.DynamicsCrm.EntityFrameworkCore\Masa.Utils.DynamicsCrm.EntityFrameworkCore.csproj", "{8A51A2A9-FBF4-40DC-AD89-AD3B9D3A50DC}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.Utils.DynamicsCrm.EntityFrameworkCore", "src\Utils\DynamicsCrm\Masa.Utils.DynamicsCrm.EntityFrameworkCore\Masa.Utils.DynamicsCrm.EntityFrameworkCore.csproj", "{8A51A2A9-FBF4-40DC-AD89-AD3B9D3A50DC}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Masa.Contrib.StackSdks.Tsc.Clickhouse", "src\Contrib\StackSdks\Masa.Contrib.StackSdks.Tsc.Clickhouse\Masa.Contrib.StackSdks.Tsc.Clickhouse.csproj", "{43389D12-17E1-4F07-9A42-5CFCC24D08B2}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests", "src\Contrib\StackSdks\Tests\Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests\Masa.Contrib.StackSdks.Tsc.Clickhouse.Tests.csproj", "{289BF8C8-968F-4105-A65E-C1C6FD8857F2}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -2569,6 +2573,22 @@ Global
 		{8A51A2A9-FBF4-40DC-AD89-AD3B9D3A50DC}.Release|Any CPU.Build.0 = Release|Any CPU
 		{8A51A2A9-FBF4-40DC-AD89-AD3B9D3A50DC}.Release|x64.ActiveCfg = Release|Any CPU
 		{8A51A2A9-FBF4-40DC-AD89-AD3B9D3A50DC}.Release|x64.Build.0 = Release|Any CPU
+		{43389D12-17E1-4F07-9A42-5CFCC24D08B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{43389D12-17E1-4F07-9A42-5CFCC24D08B2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{43389D12-17E1-4F07-9A42-5CFCC24D08B2}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{43389D12-17E1-4F07-9A42-5CFCC24D08B2}.Debug|x64.Build.0 = Debug|Any CPU
+		{43389D12-17E1-4F07-9A42-5CFCC24D08B2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{43389D12-17E1-4F07-9A42-5CFCC24D08B2}.Release|Any CPU.Build.0 = Release|Any CPU
+		{43389D12-17E1-4F07-9A42-5CFCC24D08B2}.Release|x64.ActiveCfg = Release|Any CPU
+		{43389D12-17E1-4F07-9A42-5CFCC24D08B2}.Release|x64.Build.0 = Release|Any CPU
+		{289BF8C8-968F-4105-A65E-C1C6FD8857F2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{289BF8C8-968F-4105-A65E-C1C6FD8857F2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{289BF8C8-968F-4105-A65E-C1C6FD8857F2}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{289BF8C8-968F-4105-A65E-C1C6FD8857F2}.Debug|x64.Build.0 = Debug|Any CPU
+		{289BF8C8-968F-4105-A65E-C1C6FD8857F2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{289BF8C8-968F-4105-A65E-C1C6FD8857F2}.Release|Any CPU.Build.0 = Release|Any CPU
+		{289BF8C8-968F-4105-A65E-C1C6FD8857F2}.Release|x64.ActiveCfg = Release|Any CPU
+		{289BF8C8-968F-4105-A65E-C1C6FD8857F2}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -2921,6 +2941,8 @@ Global
 		{64B54122-44F1-4379-9422-953EF706A3A6} = {5944A182-13B8-4DA6-AEE2-0A01E64A9648}
 		{83310F46-E1C7-4438-B32A-9F6F7EA13FCF} = {64B54122-44F1-4379-9422-953EF706A3A6}
 		{8A51A2A9-FBF4-40DC-AD89-AD3B9D3A50DC} = {64B54122-44F1-4379-9422-953EF706A3A6}
+		{43389D12-17E1-4F07-9A42-5CFCC24D08B2} = {6042AE23-A07E-4F6F-B1C3-F17617AEB722}
+		{289BF8C8-968F-4105-A65E-C1C6FD8857F2} = {E4AD67C8-9255-4013-A3C4-962694399770}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {40383055-CC50-4600-AD9A-53C14F620D03}

diff --git a/.../Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Aggregate/SimpleAggregateRequestDto.cs b/.../Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Model/Aggregate/SimpleAggregateRequestDto.cs
@@ -17,4 +17,9 @@ public class SimpleAggregateRequestDto : BaseRequestDto
     /// currently support elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/7.17/search-aggregations-bucket-datehistogram-aggregation.html
     /// </summary>
     public string Interval { get; set; }
+
+    /// <summary>
+    /// only fro type Group by, true return type is IEnumerable<KeyValuePair<string, int>>,false is IEnumerable<string>
+    /// </summary>
+    public bool AllValue { get; set; }
 }
diff --git a/...ildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Service/ILogService.cs b/...ildingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Service/ILogService.cs
@@ -10,7 +10,7 @@ public interface ILogService
     Task<IEnumerable<MappingResponseDto>> GetMappingAsync();
 
     /// <summary>
-    /// when query type: Count,Sum,Avg and DistinctCount return type is double, DateHistogram return IEnumerable<KeyValuePair<double, long>> ,GroupBy return IEnumerable<string>
+    /// when query type: Count,Sum,Avg and DistinctCount return type is double, DateHistogram return IEnumerable<KeyValuePair<double, long>> ,GroupBy return IEnumerable<string>, AllValue is true return IEnumerable<KeyValueParir<string,long>>
     /// </summary>
     /// <param name="query"></param>
     /// <returns></returns>

diff --git a/...dingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Service/ITraceService.cs b/...dingBlocks/StackSdks/Masa.BuildingBlocks.StackSdks.Tsc.Contracts/Service/ITraceService.cs
@@ -17,4 +17,6 @@ public interface ITraceService
     /// <param name="query"></param>
     /// <returns></returns>
     Task<object> AggregateAsync(SimpleAggregateRequestDto query);
+
+    Task<string> GetMaxDelayTraceIdAsync(BaseRequestDto query);
 }