Incremental refresh for PBI reports (microsoft#976)

Co-authored-by: msbrett <[email protected]> Co-authored-by: Michael Flanakin <[email protected]>
helderpinto · Sep 27, 2024 · 1d4650c · 1d4650c
1 parent cbf21df
commit 1d4650c
Show file tree

Hide file tree

Showing 2 changed files with 80 additions and 9 deletions.
diff --git a/docs/_resources/changelog.md b/docs/_resources/changelog.md
@@ -81,6 +81,8 @@ Legend:
 📊 Power BI reports
 {: .fs-5 .fw-500 .mt-4 mb-0 }
 
+> ➕ Added:
+>
 > - [Governance](../_reporting/power-bi/governance.md)
 >   1. Added Policy compliance.
 >   2. Added Virtual machines and managed disks.
@@ -89,6 +91,11 @@ Legend:
 > - [Workload optimization](../_reporting/power-bi/workload-optimization.md)
 >   1. Added Azure Advisor cost recommendations.
 >   2. Added Unattached disks.
+>
+> 🛠️ Fixed:
+>
+> - General
+>   1. Improved import performance by using parquet metadata to filter files by date (if configured)
 
 🏦 FinOps hubs
 {: .fs-5 .fw-500 .mt-4 mb-0 }

diff --git a/src/power-bi/Shared.Dataset/definition/expressions.tmdl b/src/power-bi/Shared.Dataset/definition/expressions.tmdl
@@ -11,15 +11,24 @@ expression RangeStart = null meta [IsParameterQuery=true, Type="DateTime", IsPar
 	lineageTag: 90085870-f708-4bc2-90c1-3a343e88a26b
 	queryGroup: '🛠️ Setup'
 
-	annotation PBI_ResultType = Exception
+	annotation PBI_ResultType = DateTime
 
 	annotation PBI_NavigationStepName = Navigation
 
 expression RangeEnd = null meta [IsParameterQuery=true, Type="DateTime", IsParameterQueryRequired=false]
 	lineageTag: 695f1d3b-7b1a-49fa-8d83-f2c25dc787de
 	queryGroup: '🛠️ Setup'
 
-	annotation PBI_ResultType = Exception
+	annotation PBI_ResultType = DateTime
+
+	annotation PBI_NavigationStepName = Navigation
+
+/// Optional. Number of months to include in the report before the RangeEnd date. Ignored if RangeStart is specified. Default: (all available data).
+expression 'Number of Months' = null meta [IsParameterQuery=true, Type="Number", IsParameterQueryRequired=false]
+	lineageTag: 1d47668d-4de6-404c-8e53-f89a35081e8b
+	queryGroup: '🛠️ Setup'
+
+	annotation PBI_ResultType = Null
 
 	annotation PBI_NavigationStepName = Navigation
 
@@ -109,7 +118,11 @@ expression ftk_Storage = ```
 		            Table.AddColumn(Source, "EndDate", each if [StartDate] = null then null else Date.EndOfMonth([StartDate]), type datetime), 
 		            each (start = null or [StartDate] = null or [StartDate] >= Date.StartOfMonth(Date.From(start))) and (end = null or [EndDate] = null or [EndDate] <= Date.EndOfMonth(Date.From(end)))
 		        ),
-
+		    AddMetadata = Table.AddColumn(FilterFilesByDate, "Metadata", each 
+		        if Text.Lower([Extension]) = ".parquet" and datasetType = "focuscost" then ftk_Metadata([Content], "ChargePeriodStart")
+		        else null ),
+		    ExpandMetadata = Table.ExpandTableColumn(AddMetadata, "Metadata", {"Min"}, {"ChargePeriodStart"}),
+		    FilterMetadata = Table.SelectRows(ExpandMetadata, each (start = null or [ChargePeriodStart] = null or [ChargePeriodStart] >= ftk_DatetimeToJulianDate(start)) and (end = null or [ChargePeriodStart] = null or [ChargePeriodStart] < ftk_DatetimeToJulianDate(end))),
 		    // Extract
 		    _dataNotFound = () =>
 		        if IsHub then "HubDataNotFound: No " & data & " data found in the storage account (" & Text.SplitAny(hubUrl, "/."){2} & "). Please confirm data was exported and processed by hub ingestion pipelines. Refer to the troubleshooting guide for assistance: https://aka.ms/finops/hubs/troubleshoot."
@@ -126,7 +139,7 @@ expression ftk_Storage = ```
 		    ,
 		    ReadContent = if FilterFilesByDate = null then null else Table.SelectRows(
 		        Table.SelectColumns(
-		            Table.AddColumn(FilterFilesByDate, "Data",
+		            Table.AddColumn(FilterMetadata, "Data",
 		                each if [Extension] = ".parquet" then Parquet.Document([Content]) 
 		                else if [Extension] = ".gz"      then _parseCsv(Binary.Decompress([Content], Compression.GZip)) 
 		                else if [Extension] = ".csv"     then _parseCsv([Content])
@@ -414,12 +427,63 @@ expression 'Export Storage URL' = null meta [IsParameterQuery=true, Type="Text",
 
 	annotation PBI_ResultType = Text
 
-/// Optional. Number of months to include in the report before the RangeEnd date. Ignored if RangeStart is specified. Default: (all available data).
-expression 'Number of Months' = null meta [IsParameterQuery=true, Type="Number", IsParameterQueryRequired=false]
-	lineageTag: 1d47668d-4de6-404c-8e53-f89a35081e8b
-	queryGroup: '🛠️ Setup'
+expression ftk_DatetimeToJulianDate = ```
+		(InputDate) =>
+		    let
+		        StartDate = #date(1899, 12, 30),
+		        NumberOfDays = Duration.Days(Date.From(InputDate) - StartDate),
+		        JulianDay = NumberOfDays + 2415018.5 // 2415019 or 2415018.5 
+		    in
+		        JulianDay
+		```
+	lineageTag: 4b2bc0fe-5c3d-4275-ad19-3b4ff0b720e9
+	queryGroup: Functions
 
 	annotation PBI_NavigationStepName = Navigation
 
-	annotation PBI_ResultType = Exception
+	annotation PBI_ResultType = Function
 
+expression ftk_ImpalaToJulianDate =
+		(binaryData) =>
+		let
+		    // Your 12-byte binary data
+		    // binaryData = Binary.FromText("AAAAAAAAAADBiSUA", BinaryEncoding.Base64),
+		    // Get the last 4 bytes
+		    last4Bytes = Binary.Range(binaryData, 8, 4),
+		    // Convert the binary to a list of numbers
+		    listOfNumbers = Binary.ToList(last4Bytes),
+		    // Reverse the list of numbers
+		    reversedListOfNumbers = List.Reverse(listOfNumbers),
+		    // Convert the list of numbers to an integer
+		    number = List.Accumulate(reversedListOfNumbers, 0, (state, current) => state * 256 + current) - 0.5
+		in
+		    number
+	lineageTag: 89863516-9c1f-4f21-aaf1-b84e73909525
+	queryGroup: Functions
+
+	annotation PBI_NavigationStepName = Navigation
+
+	annotation PBI_ResultType = Function
+
+expression ftk_Metadata =
+		(fileContents, dateColumn as text) =>
+		let
+		  m = Parquet.Metadata(fileContents) as any,
+		  Navigation = m[RowGroups],
+		  #"Expanded Columns" = Table.ExpandTableColumn(Navigation, "Columns", {"MetaData"}, {"MetaData"}),
+		  #"Expanded MetaData" = Table.ExpandRecordColumn(#"Expanded Columns", "MetaData", {"PathInSchema", "Statistics"}, {"PathInSchema", "Statistics"}),
+		  #"Expanded PathInSchema" = Table.ExpandListColumn(#"Expanded MetaData", "PathInSchema"),
+		  #"Filtered rows" = Table.SelectRows(#"Expanded PathInSchema", each ([PathInSchema] = dateColumn)),
+		  #"Expanded Statistics" = Table.ExpandRecordColumn(#"Filtered rows", "Statistics", {"MaxValue", "MinValue"}, {"MaxValue", "MinValue"}),
+		  #"Added min" = Table.AddColumn(#"Expanded Statistics", "Min", each if [MinValue] = null then null else ftk_ImpalaToJulianDate([MinValue])),
+		  #"Added max" = Table.AddColumn(#"Added min", "Max", each if [MaxValue] = null then null else ftk_ImpalaToJulianDate([MaxValue])),
+		  #"Removed other columns" = Table.SelectColumns(#"Added max", {"Min", "Max"}),
+		  #"Removed duplicates" = Table.Distinct(#"Removed other columns")
+		in
+		  #"Removed duplicates"
+	lineageTag: c697294c-e969-4248-8d51-2232bfbd8068
+	queryGroup: Functions
+
+	annotation PBI_NavigationStepName = Navigation
+
+	annotation PBI_ResultType = Function