[data_sources] Relax unparsed object check for datasources (#2043)

* relax unparsed check for datasources

* re-add dashboard list unparsed relax in fwprovider

* Update datadog/fwprovider/data_source_datadog_dashboard_list.go

---------

Co-authored-by: Kevin Zou <[email protected]>

datadog/data_source_datadog_cloud_workload_security_agent_rules.go

@@ -2,6 +2,7 @@ package datadog
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/terraform-providers/terraform-provider-datadog/datadog/internal/utils"
 
@@ -62,17 +63,21 @@ func dataSourceDatadogCloudWorkloadSecurityAgentRulesRead(ctx context.Context, d
 	auth := providerConf.Auth
 
 	agentRules := make([]map[string]interface{}, 0)
-
 	response, httpresp, err := apiInstances.GetCloudWorkloadSecurityApiV2().ListCloudWorkloadSecurityAgentRules(auth)
-
 	if err != nil {
 		return utils.TranslateClientErrorDiag(err, httpresp, "error listing agent rules")
 	}
-	if err := utils.CheckForUnparsed(response); err != nil {
-		return diag.FromErr(err)
-	}
 
+	diags := diag.Diagnostics{}
 	for _, agentRule := range response.GetData() {
+		if err := utils.CheckForUnparsed(agentRule); err != nil {
+			diags = append(diags, diag.Diagnostic{
+				Severity: diag.Warning,
+				Summary:  fmt.Sprintf("skipping agent rule with id: %s", agentRule.GetId()),
+				Detail:   fmt.Sprintf("rule contains unparsed object: %v", err),
+			})
+			continue
+		}
 
 		// extract agent rule
 		agentRuleTF := make(map[string]interface{})
@@ -90,5 +95,5 @@ func dataSourceDatadogCloudWorkloadSecurityAgentRulesRead(ctx context.Context, d
 	d.SetId("cloud-workload-security-agent-rules")
 	d.Set("agent_rules", agentRules)
 
-	return nil
+	return diags
 }

datadog/data_source_datadog_dashboard.go

@@ -55,9 +55,6 @@ func dataSourceDatadogDashboardRead(ctx context.Context, d *schema.ResourceData,
 			}
 			return retry.NonRetryableError(utils.TranslateClientError(err, httpresp, "error querying dashboard"))
 		}
-		if err := utils.CheckForUnparsed(dashResponse); err != nil {
-			return retry.NonRetryableError(err)
-		}
 
 		searchedName := d.Get("name")
 		var foundDashes []datadogV1.DashboardSummaryDefinition
@@ -74,6 +71,10 @@ func dataSourceDatadogDashboardRead(ctx context.Context, d *schema.ResourceData,
 			return retry.NonRetryableError(fmt.Errorf("%s returned more than one dashboard", searchedName))
 		}
 
+		if err := utils.CheckForUnparsed(foundDashes[0]); err != nil {
+			return retry.NonRetryableError(err)
+		}
+
 		d.SetId(foundDashes[0].GetId())
 		d.Set("url", foundDashes[0].GetUrl())
 		d.Set("title", foundDashes[0].GetTitle())

datadog/data_source_datadog_integration_aws_logs_services.go

@@ -2,6 +2,7 @@ package datadog
 
 import (
 	"context"
+	"fmt"
 	"sort"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
@@ -50,13 +51,18 @@ func dataSourceDatadogIntegrationAWSLogsServicesRead(ctx context.Context, d *sch
 	if err != nil {
 		return utils.TranslateClientErrorDiag(err, httpresp, "error querying AWS logs services")
 	}
-	if err := utils.CheckForUnparsed(awsLogsServices); err != nil {
-		return diag.FromErr(err)
-	}
 
+	diags := diag.Diagnostics{}
 	tfLogsServices := make([]map[string]interface{}, 0)
-
 	for _, awsLogsService := range awsLogsServices {
+		if err := utils.CheckForUnparsed(awsLogsService); err != nil {
+			diags = append(diags, diag.Diagnostic{
+				Severity: diag.Warning,
+				Summary:  fmt.Sprintf("skipping integration aws logs services with id: %s", awsLogsService.GetId()),
+				Detail:   fmt.Sprintf("aws logs service contains unparsed object: %v", err),
+			})
+			continue
+		}
 
 		// extract agent rule
 		awsLogsServiceTF := make(map[string]interface{})
@@ -81,5 +87,5 @@ func dataSourceDatadogIntegrationAWSLogsServicesRead(ctx context.Context, d *sch
 
 	d.SetId("aws-logs-services")
 
-	return nil
+	return diags
 }

datadog/data_source_datadog_logs_archives_order.go

@@ -37,9 +37,6 @@ func dataSourceDatadogLogsArchivesOrderRead(ctx context.Context, d *schema.Resou
 	if err != nil {
 		return utils.TranslateClientErrorDiag(err, httpresp, "error querying the order of your logs archives")
 	}
-	if err := utils.CheckForUnparsed(logsArchiveOrder); err != nil {
-		return diag.FromErr(err)
-	}
 
 	if err := d.Set("archive_ids", logsArchiveOrder.Data.Attributes.ArchiveIds); err != nil {
 		return diag.FromErr(err)

datadog/data_source_datadog_logs_indexes.go

@@ -2,6 +2,7 @@ package datadog
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/terraform-providers/terraform-provider-datadog/datadog/internal/utils"
 
@@ -108,12 +109,19 @@ func dataSourceDatadogLogsIndexesRead(ctx context.Context, d *schema.ResourceDat
 	if err != nil {
 		return utils.TranslateClientErrorDiag(err, httpresp, "error querying log indexes")
 	}
-	if err := utils.CheckForUnparsed(logsIndexes); err != nil {
-		return diag.FromErr(err)
-	}
 
+	diags := diag.Diagnostics{}
 	tfLogsIndexes := make([]map[string]interface{}, len(logsIndexes.GetIndexes()))
 	for i, l := range logsIndexes.GetIndexes() {
+		if err := utils.CheckForUnparsed(l); err != nil {
+			diags = append(diags, diag.Diagnostic{
+				Severity: diag.Warning,
+				Summary:  fmt.Sprintf("skipping logs index with name: %s", l.GetName()),
+				Detail:   fmt.Sprintf("logs index contains unparsed object: %v", err),
+			})
+			continue
+		}
+
 		tfLogsIndexes[i] = map[string]interface{}{
 			"name":             l.GetName(),
 			"daily_limit":      l.GetDailyLimit(),
@@ -128,5 +136,5 @@ func dataSourceDatadogLogsIndexesRead(ctx context.Context, d *schema.ResourceDat
 
 	d.SetId("log-indexes")
 
-	return nil
+	return diags
 }

datadog/data_source_datadog_logs_indexes_order.go

@@ -37,9 +37,6 @@ func dataSourceDatadogLogsIndexesOrderRead(ctx context.Context, d *schema.Resour
 	if err != nil {
 		return utils.TranslateClientErrorDiag(err, httpresp, "error querying the order of your log indexes")
 	}
-	if err := utils.CheckForUnparsed(logsIndexesOrder); err != nil {
-		return diag.FromErr(err)
-	}
 
 	if err := d.Set("index_names", logsIndexesOrder.GetIndexNames()); err != nil {
 		return diag.FromErr(err)

datadog/data_source_datadog_logs_pipelines.go

@@ -2,6 +2,7 @@ package datadog
 
 import (
 	"context"
+	"fmt"
 	"strconv"
 	"strings"
 
@@ -95,15 +96,22 @@ func dataSourceDatadogLogsPipelinesRead(ctx context.Context, d *schema.ResourceD
 	if err != nil {
 		return utils.TranslateClientErrorDiag(err, httpresp, "error querying log pipelines")
 	}
-	if err := utils.CheckForUnparsed(logsPipelines); err != nil {
-		return diag.FromErr(err)
-	}
 
+	diags := diag.Diagnostics{}
 	vStr, ok := d.GetOk("is_read_only")
 	v, _ := strconv.ParseBool(vStr.(string))
 	tflogsPipelines := make([]map[string]interface{}, 0)
 	for _, pipeline := range logsPipelines {
 		if !ok || (ok && v == *pipeline.IsReadOnly) {
+			if err := utils.CheckForUnparsed(pipeline); err != nil {
+				diags = append(diags, diag.Diagnostic{
+					Severity: diag.Warning,
+					Summary:  fmt.Sprintf("skipping logs pipeline with id: %s", pipeline.GetId()),
+					Detail:   fmt.Sprintf("logs pipeline contains unparsed object: %v", err),
+				})
+				continue
+			}
+
 			tflogsPipelines = append(tflogsPipelines, map[string]interface{}{
 				"name":         pipeline.GetName(),
 				"id":           pipeline.GetId(),
@@ -119,7 +127,7 @@ func dataSourceDatadogLogsPipelinesRead(ctx context.Context, d *schema.ResourceD
 	}
 
 	d.SetId(computePipelinesDatasourceID(d))
-	return nil
+	return diags
 }
 
 func computePipelinesDatasourceID(d *schema.ResourceData) string {

datadog/data_source_datadog_monitor.go

@@ -294,15 +294,15 @@ func dataSourceDatadogMonitorRead(ctx context.Context, d *schema.ResourceData, m
 	}
 
 	monitors, httpresp, err := apiInstances.GetMonitorsApiV1().ListMonitors(auth, *optionalParams)
-	if err != nil {
-		return utils.TranslateClientErrorDiag(err, httpresp, "error querying monitors")
-	}
 	if len(monitors) > 1 {
 		return diag.Errorf("your query returned more than one result, please try a more specific search criteria")
 	}
 	if len(monitors) == 0 {
 		return diag.Errorf("your query returned no result, please try a less specific search criteria")
 	}
+	if err != nil {
+		return utils.TranslateClientErrorDiag(err, httpresp, "error querying monitors")
+	}
 
 	m := monitors[0]
 	if err := utils.CheckForUnparsed(m); err != nil {

datadog/data_source_datadog_monitor_config_policies.go

@@ -2,6 +2,7 @@ package datadog
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
@@ -77,12 +78,19 @@ func dataSourceDatadogMonitorConfigPoliciesRead(ctx context.Context, d *schema.R
 	if err != nil {
 		return utils.TranslateClientErrorDiag(err, httpresp, "error querying monitor config policies")
 	}
-	if err := utils.CheckForUnparsed(monitorConfigPolicies); err != nil {
-		return diag.FromErr(err)
-	}
 
+	diags := diag.Diagnostics{}
 	tfMonitorConfigPolicies := make([]map[string]interface{}, len(monitorConfigPolicies.Data))
 	for i, mcp := range monitorConfigPolicies.Data {
+		if err := utils.CheckForUnparsed(mcp); err != nil {
+			diags = append(diags, diag.Diagnostic{
+				Severity: diag.Warning,
+				Summary:  fmt.Sprintf("skipping monitor config policy with id: %s", mcp.GetId()),
+				Detail:   fmt.Sprintf("aws logs service contains unparsed object: %v", err),
+			})
+			continue
+		}
+
 		attributes := mcp.GetAttributes()
 		tfMonitorConfigPolicies[i] = map[string]interface{}{
 			"id":          mcp.GetId(),
@@ -102,5 +110,5 @@ func dataSourceDatadogMonitorConfigPoliciesRead(ctx context.Context, d *schema.R
 	d.SetId("monitor-config-policies")
 	d.Set("monitor_config_policies", tfMonitorConfigPolicies)
 
-	return nil
+	return diags
 }

datadog/data_source_datadog_monitors.go

@@ -2,6 +2,7 @@ package datadog
 
 import (
 	"context"
+	"fmt"
 	"strings"
 
 	"github.com/terraform-providers/terraform-provider-datadog/datadog/internal/utils"
@@ -86,17 +87,24 @@ func dataSourceDatadogMonitorsRead(ctx context.Context, d *schema.ResourceData,
 	if err != nil {
 		return utils.TranslateClientErrorDiag(err, httpresp, "error querying monitors")
 	}
-	if err := utils.CheckForUnparsed(monitors); err != nil {
-		return diag.FromErr(err)
-	}
 	if len(monitors) == 0 {
 		return diag.Errorf("your query returned no result, please try a less specific search criteria")
 	}
 
 	d.SetId(computeMonitorsDatasourceID(d))
 
+	diags := diag.Diagnostics{}
 	tfMonitors := make([]map[string]interface{}, len(monitors))
 	for i, m := range monitors {
+		if err := utils.CheckForUnparsed(m); err != nil {
+			diags = append(diags, diag.Diagnostic{
+				Severity: diag.Warning,
+				Summary:  fmt.Sprintf("skipping monitor with id: %v", m.GetId()),
+				Detail:   fmt.Sprintf("monitor contains unparsed object: %v", err),
+			})
+			continue
+		}
+
 		tfMonitors[i] = map[string]interface{}{
 			"id":   m.GetId(),
 			"name": m.GetName(),
@@ -107,7 +115,7 @@ func dataSourceDatadogMonitorsRead(ctx context.Context, d *schema.ResourceData,
 		return diag.FromErr(err)
 	}
 
-	return nil
+	return diags
 }
 
 func computeMonitorsDatasourceID(d *schema.ResourceData) string {

datadog/data_source_datadog_permissions.go

@@ -2,6 +2,7 @@ package datadog
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/terraform-providers/terraform-provider-datadog/datadog/internal/utils"
 
@@ -45,22 +46,30 @@ func dataSourceDatadogPermissionsRead(ctx context.Context, d *schema.ResourceDat
 	if err != nil {
 		return utils.TranslateClientErrorDiag(err, resp, "error listing permissions")
 	}
-	if err := utils.CheckForUnparsed(res); err != nil {
-		return diag.FromErr(err)
-	}
+
+	diags := diag.Diagnostics{}
 	perms := res.GetData()
 	permsNameToID := make(map[string]string, len(perms))
 	includeRestricted := d.Get("include_restricted").(bool)
 	for _, perm := range perms {
 		if !includeRestricted && perm.Attributes.GetRestricted() {
 			continue
 		}
+
+		if err := utils.CheckForUnparsed(perm); err != nil {
+			diags = append(diags, diag.Diagnostic{
+				Severity: diag.Warning,
+				Summary:  fmt.Sprintf("skipping permission with id: %s", perm.GetId()),
+				Detail:   fmt.Sprintf("permission contains unparsed object: %v", err),
+			})
+			continue
+		}
 		permsNameToID[perm.Attributes.GetName()] = perm.GetId()
 	}
 	if err := d.Set("permissions", permsNameToID); err != nil {
 		return diag.FromErr(err)
 	}
 	d.SetId("datadog-permissions")
 
-	return nil
+	return diags
 }

datadog/data_source_datadog_role.go

@@ -53,9 +53,7 @@ func dataSourceDatadogRoleRead(ctx context.Context, d *schema.ResourceData, meta
 	if err != nil {
 		return utils.TranslateClientErrorDiag(err, httpresp, "error querying roles")
 	}
-	if err := utils.CheckForUnparsed(res); err != nil {
-		return diag.FromErr(err)
-	}
+
 	roles := res.GetData()
 	roleIndex := 0
 	if len(roles) > 1 {
@@ -79,6 +77,10 @@ func dataSourceDatadogRoleRead(ctx context.Context, d *schema.ResourceData, meta
 		return diag.Errorf("your query returned no result, please try a less specific search criteria")
 	}
 
+	if err := utils.CheckForUnparsed(roles[roleIndex]); err != nil {
+		return diag.FromErr(err)
+	}
+
 	r := roles[roleIndex]
 	d.SetId(r.GetId())
 	if err := d.Set("name", r.Attributes.GetName()); err != nil {