Release 0.7 tweaks (microsoft#1172)

helderpinto · Dec 1, 2024 · 25a8c27 · 25a8c27
1 parent 2d77b51
commit 25a8c27
Show file tree

Hide file tree

Showing 14 changed files with 616 additions and 322 deletions.
diff --git a/docs/_reporting/hubs/configure-scopes.md b/docs/_reporting/hubs/configure-scopes.md
@@ -43,7 +43,7 @@ If you cannot grant permissions for your scope, you can create Cost Management e
 1. [Create a new FOCUS cost export](https://learn.microsoft.com/azure/cost-management-billing/costs/tutorial-export-acm-data?tabs=azure-portal) using the following settings:
 
    - **Type of data** = `Cost and usage details (FOCUS)`<sup>1</sup>
-   - **Dataset version** = `1.0`<sup>2</sup>
+   - **Dataset version** = `1.0` or `1.0r2`<sup>2</sup>
    - **Frequency** = `Daily export of month-to-date costs`<sup>3</sup>
    - **Storage account** = (Use subscription/resource deployed with your hub)
    - **Container** = `msexports`
@@ -77,7 +77,7 @@ If you cannot grant permissions for your scope, you can create Cost Management e
 5. Repeat steps 1-4 for each scope you want to monitor.
 
 _<sup>1) FinOps hubs 0.2 and beyond requires FOCUS cost data. As of July 2024, the option to export FOCUS cost data is only accessible from the central Cost Management experience in the Azure portal. If you do not see this option, please search for or navigate to [Cost Management Exports](https://portal.azure.com/#blade/Microsoft_Azure_CostManagement/Menu/open/exports).</sup>_
-_<sup>2) FinOps hubs 0.4 supports both FOCUS 1.0 and FOCUS 1.0 preview. Power BI reports in 0.4 are aligned to FOCUS 1.0 regardless of whether data was ingested as FOCUS 1.0 preview. If you need 1.0 preview data and reports, please use FinOps hubs 0.3.</sup>_
+_<sup>2) FinOps hubs 0.4 supports FOCUS 1.0r2, 1.0, 1.0 preview. Power BI reports in 0.4 are aligned to FOCUS 1.0 regardless of whether data was ingested as FOCUS 1.0 preview. If you need 1.0 preview data and reports, please use FinOps hubs 0.3. The only difference in FOCUS 1.0r2 is the inclusion of seconds in date columns.</sup>_
 _<sup>3) Configuring a daily export starts in the current month. If you want to backfill historical data, create a one-time export and set the start/end dates to the desired date range.</sup>_
 _<sup>4) While most settings are required, overwriting is optional. We recommend **not** overwriting files so you can monitor your ingestion pipeline using the [Data ingestion](../power-bi/data-ingestion.md) report. If you do not plan to use that report, please enable overwriting.</sup>_
 _<sup>5) Export paths can be any value but must be unique per scope. We recommended using a path that identifies the source scope (e.g., subscription or billing account). If 2 scopes share the same path, there could be ingestion errors.</sup>_

diff --git a/docs/_resources/changelog.md b/docs/_resources/changelog.md
@@ -129,10 +129,6 @@ Legend:
 🏦 FinOps hubs
 {: .fs-5 .fw-500 .mt-4 mb-0 }
 
-> ➕ Added:
-> 
-> 1. Infrastructure encryption - Added an optional enableInfrastructureEncryption template parameter to support storage account infrastructure encryption.
-
 <small>**Breaking change**</small>
 {: .label .label-red .pt-0 .pl-3 .pr-3 .m-0 }
 
@@ -151,6 +147,7 @@ Legend:
 >    - Added param to disable external access to Azure Data Lake and Azure Data Explorer.
 >    - Added param to specify subnet range of virtual network - minimum size = /26
 > 1. Support for storage account infrastructure encryption.
+> 1. Published a [schema file](https://aka.ms/finops/hubs/settings-schema) for the hub settings.json file.
 >
 > ✏️ Changed:
 >
@@ -176,7 +173,7 @@ Legend:
 > ➕ Added:
 >
 > - [Optimization workbook](../_optimize/workbooks/optimization/README.md)
->   1. On the Storagetab, included the **RSVaultBackup** tag in the list of non-idle disks.
+>   1. On the Storage tab, included the **RSVaultBackup** tag in the list of non-idle disks.
 >
 > 🛠️ Fixed:
 >

diff --git a/src/power-bi/kql/CostSummary.Report/report.json b/src/power-bi/kql/CostSummary.Report/report.json
@@ -154,7 +154,7 @@
           "z": 1000.00
         },
         {
-          "config": "{\"name\":\"907968064288c59539c5\",\"layouts\":[{\"id\":0,\"position\":{\"x\":15.562310030395137,\"y\":16,\"z\":0,\"width\":1264.0000000000002,\"height\":96,\"tabOrder\":5000}}],\"singleVisual\":{\"visualType\":\"textbox\",\"drillFilterOtherVisuals\":true,\"objects\":{\"general\":[{\"properties\":{\"paragraphs\":[{\"textRuns\":[{\"value\":\"Cost summary report\",\"textStyle\":{\"fontWeight\":\"bold\",\"fontSize\":\"42pt\"}},{\"value\":\"   v24.11.28\",\"textStyle\":{\"color\":\"#808080\"}}]}]}}]},\"vcObjects\":{\"visualHeader\":[{\"properties\":{\"show\":{\"expr\":{\"Literal\":{\"Value\":\"false\"}}}}}],\"background\":[{\"properties\":{\"show\":{\"expr\":{\"Literal\":{\"Value\":\"false\"}}}}}]}}}",
+          "config": "{\"name\":\"907968064288c59539c5\",\"layouts\":[{\"id\":0,\"position\":{\"x\":15.562310030395137,\"y\":16,\"z\":0,\"width\":1264.0000000000002,\"height\":96,\"tabOrder\":5000}}],\"singleVisual\":{\"visualType\":\"textbox\",\"drillFilterOtherVisuals\":true,\"objects\":{\"general\":[{\"properties\":{\"paragraphs\":[{\"textRuns\":[{\"value\":\"Cost summary report\",\"textStyle\":{\"fontWeight\":\"bold\",\"fontSize\":\"42pt\"}},{\"value\":\"   v24.11.30\",\"textStyle\":{\"color\":\"#808080\"}}]}]}}]},\"vcObjects\":{\"visualHeader\":[{\"properties\":{\"show\":{\"expr\":{\"Literal\":{\"Value\":\"false\"}}}}}],\"background\":[{\"properties\":{\"show\":{\"expr\":{\"Literal\":{\"Value\":\"false\"}}}}}]}}}",
           "filters": "[]",
           "height": 96.00,
           "width": 1264.00,

diff --git a/src/power-bi/kql/CostSummary.SemanticModel/definition/cultures/en-US.tmdl b/src/power-bi/kql/CostSummary.SemanticModel/definition/cultures/en-US.tmdl
@@ -19844,82 +19844,6 @@ cultureInfo en-US
 			        }
 			      ]
 			    },
-			    "tag_cost.tag": {
-			      "Definition": {
-			        "Binding": {
-			          "ConceptualEntity": "TagCosts",
-			          "ConceptualProperty": "Tags"
-			        }
-			      },
-			      "State": "Generated",
-			      "Terms": [
-			        {
-			          "tag": {
-			            "State": "Generated"
-			          }
-			        },
-			        {
-			          "device": {
-			            "Type": "Noun",
-			            "State": "Suggested",
-			            "Source": {
-			              "Agent": "Thesaurus"
-			            },
-			            "Weight": 0.491
-			          }
-			        },
-			        {
-			          "ticket": {
-			            "Type": "Noun",
-			            "State": "Suggested",
-			            "Source": {
-			              "Agent": "Thesaurus"
-			            },
-			            "Weight": 0.476
-			          }
-			        },
-			        {
-			          "tab": {
-			            "Type": "Noun",
-			            "State": "Suggested",
-			            "Source": {
-			              "Agent": "Thesaurus"
-			            },
-			            "Weight": 0.476
-			          }
-			        },
-			        {
-			          "docket": {
-			            "Type": "Noun",
-			            "State": "Suggested",
-			            "Source": {
-			              "Agent": "Thesaurus"
-			            },
-			            "Weight": 0.476
-			          }
-			        },
-			        {
-			          "chip": {
-			            "Type": "Noun",
-			            "State": "Suggested",
-			            "Source": {
-			              "Agent": "Thesaurus"
-			            },
-			            "Weight": 0.476
-			          }
-			        },
-			        {
-			          "mark": {
-			            "Type": "Noun",
-			            "State": "Suggested",
-			            "Source": {
-			              "Agent": "Thesaurus"
-			            },
-			            "Weight": 0.476
-			          }
-			        }
-			      ]
-			    },
 			    "tag_cost.x_commitment_discount_savings": {
 			      "Definition": {
 			        "Binding": {

diff --git a/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/ChargeBreakdown.tmdl b/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/ChargeBreakdown.tmdl
@@ -299,6 +299,24 @@ table ChargeBreakdown
 
 		annotation PBI_FormatHint = {"isGeneralNumber":true}
 
+	column x_ChargePeriodMonth
+		dataType: dateTime
+		formatString: General Date
+		lineageTag: 487aae09-ebbf-43b1-baa2-5220b477575d
+		summarizeBy: none
+		sourceColumn: x_ChargePeriodMonth
+
+		annotation SummarizationSetBy = Automatic
+
+	column x_ReportingDate
+		dataType: dateTime
+		formatString: General Date
+		lineageTag: 1c5359f9-6c85-4157-9274-8a4bb4f040b9
+		summarizeBy: none
+		sourceColumn: x_ReportingDate
+
+		annotation SummarizationSetBy = Automatic
+
 	partition ChargeBreakdown = m
 		mode: import
 		queryGroup: 'Data Explorer'
@@ -307,18 +325,20 @@ table ChargeBreakdown
 				    lookback = Text.From(if #"Number of Months" = "" or #"Number of Months" = null then 999 else #"Number of Months"),
 				    Source = AzureDataExplorer.Contents(#"Cluster URI", "Hub", "
 
+				    let numberOfMonths = " & lookback & ";
+				    let monthlyGranularity = " & Text.From(#"Daily or Monthly" = "Monthly") & ";
+				    let allowCostManagementAllocationRules = false;
+				    let summarizeAfterRowCount = 400000;
 				    Costs_v1_0
-				    | where ChargePeriodStart >= monthsago(" & lookback & ")
-				    //
-				    // Filter out cost allocation changes
-				    | where isempty(x_CostAllocationRuleName)
-				    //
-				    // Fix missing costs
-				    | extend ContractedCost = iff(isempty(ContractedCost) or ContractedCost == 0, EffectiveCost, ContractedCost)
-				    | extend ListCost = iff(isempty(ListCost) or ListCost == 0, ContractedCost, ListCost)
+				    | where ChargePeriodStart >= monthsago(numberOfMonths)
+				    | where allowCostManagementAllocationRules or isempty(x_CostAllocationRuleName)
+				    | extend x_ChargePeriodMonth = startofmonth(ChargePeriodStart)
+				    | extend x_ReportingDate = iff(monthlyGranularity, x_ChargePeriodMonth, startofday(ChargePeriodStart))
 				    //
 				    | summarize
 				        BilledCost     = sum(BilledCost),
+				        ChargePeriodEnd = max(ChargePeriodEnd),
+				        ChargePeriodStart = min(ChargePeriodStart),
 				        ContractedCost = sum(ContractedCost),
 				        EffectiveCost  = sum(EffectiveCost),
 				        ListCost       = sum(ListCost)
@@ -328,8 +348,6 @@ table ChargeBreakdown
 				            BillingCurrency,
 				            ChargeCategory,
 				            ChargeClass,
-				            ChargePeriodEnd,
-				            ChargePeriodStart,
 				            CommitmentDiscountCategory,
 				            CommitmentDiscountStatus,
 				            CommitmentDiscountType,
@@ -340,9 +358,11 @@ table ChargeBreakdown
 				            ServiceCategory,
 				            ServiceName,
 				            SubAccountName,
+				            x_ChargePeriodMonth,
 				            x_Operation,
 				            x_PricingSubcategory,
 				            x_PublisherCategory,
+				            x_ReportingDate,
 				            x_SkuMeterCategory,
 				            x_SkuMeterName,
 				            x_SkuMeterSubcategory,
@@ -373,11 +393,13 @@ table ChargeBreakdown
 				        ServiceCategory,
 				        ServiceName,
 				        SubAccountName,
+				        x_ChargePeriodMonth,
 				        x_CommitmentDiscountSavings = ContractedCost - EffectiveCost,
 				        x_NegotiatedDiscountSavings = ListCost - ContractedCost,
 				        x_Operation,
 				        x_PricingSubcategory,
 				        x_PublisherCategory,
+				        x_ReportingDate,
 				        x_SkuMeterCategory,
 				        x_SkuMeterName,
 				        x_SkuMeterSubcategory,
@@ -387,8 +409,53 @@ table ChargeBreakdown
 				        x_SkuTier,
 				        x_TotalSavings = ListCost - EffectiveCost,
 				        x_UsageType
+				    | serialize
+				    | extend ROW = row_number()
+				    | as allData
+				    | limit summarizeAfterRowCount
+				    | union (
+				        allData
+				        | where ROW > summarizeAfterRowCount
+				        | summarize
+				            BilledCost        = sum(BilledCost),
+				            ChargePeriodEnd   = max(ChargePeriodEnd),
+				            ChargePeriodStart = min(ChargePeriodStart),
+				            ContractedCost    = sum(ContractedCost),
+				            EffectiveCost     = sum(EffectiveCost),
+				            ListCost          = sum(ListCost)
+				            by
+				                BillingAccountId,
+				                BillingAccountName,
+				                BillingCurrency,
+				                ChargeCategory,
+				                ChargeClass,
+				                CommitmentDiscountCategory,
+				                CommitmentDiscountStatus,
+				                CommitmentDiscountType,
+				                PricingCategory,
+				                ProviderName,
+				                PublisherName,
+				                ResourceType = '(multiple)',
+				                ServiceCategory,
+				                ServiceName,
+				                SubAccountName = '(multiple)',
+				                x_ChargePeriodMonth,
+				                x_Operation = '(multiple)',
+				                x_PricingSubcategory,
+				                x_PublisherCategory,
+				                x_ReportingDate,
+				                x_SkuMeterCategory = '(multiple)',
+				                x_SkuMeterName = '(multiple)',
+				                x_SkuMeterSubcategory = '(multiple)',
+				                x_SkuPartNumber = '(multiple)',
+				                x_SkuServiceFamily = '(multiple)',
+				                x_SkuTerm,
+				                x_SkuTier = '(multiple)',
+				                x_UsageType = '(multiple)'
+				    )
+				    | project-away ROW
 
-				    ", [MaxRows=null, MaxSize=null, NoTruncate=false, AdditionalSetStatements=null, ClientRequestIdPrefix="ftk-Resources-PurchaseCosts"])
+				    ", [MaxRows=null, MaxSize=1073741824, NoTruncate=false, AdditionalSetStatements=null, ClientRequestIdPrefix="ftk-CostSummary-ChargeBreakdown"])
 				in
 				    Source