Skip to content

Commit

Permalink
Add optional property for custom fields
Browse files Browse the repository at this point in the history
  • Loading branch information
@fivetran-jovanmanojlovic
fivetran-jovanmanojlovic committed Sep 4, 2024
1 parent 5180556 commit fbd0360
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 50 deletions.
14 changes: 14 additions & 0 deletions fivetran/common/config_field.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ type ConfigField struct {
Description map[string]string `json:"description"`
SensitiveExclusions map[string]bool `json:"sensitive_exclusions,omitempty"`
ApiField string `json:"api_field"`
OptionalFor map[string]bool `json:"optional_for"`
}

func (c ConfigField) GetIsSensitiveForSchema() bool {
Expand Down Expand Up @@ -67,12 +68,14 @@ var (
destinationFieldsByService = make(map[string]map[string]ConfigField)

destinationSchemaFields = make(map[string]map[string]bool)
optionalForFields = make(map[string]map[string]bool)
)

func GetFieldsForService(service string) (map[string]ConfigField, error) {
if len(configFieldsByService) == 0 {
panic("Fields for config are not loaded")
}

if r, ok := configFieldsByService[service]; ok {
return r, nil
}
Expand Down Expand Up @@ -130,6 +133,10 @@ func GetDestinationFieldsMap() map[string]ConfigField {
return destinationFields
}

func IsOptionalFor(service string, field string) bool {
return optionalForFields[service] != nil && optionalForFields[service][field]
}

func LoadAuthFieldsMap() {
if len(authFields) == 0 {
readAuthFieldsFromJson(&authFields)
Expand Down Expand Up @@ -178,10 +185,17 @@ func readDestinationFieldsFromJson(target *map[string]ConfigField) {
func handleDestinationSchemaField(fieldName string) {
if schema, ok := configFields[fieldName]; ok {
for k := range schema.Description {
optionalForFields[k] = make(map[string]bool)
if _, ok := destinationSchemaFields[k]; !ok {
destinationSchemaFields[k] = make(map[string]bool)
if optional, ok := schema.OptionalFor[k]; ok {
optionalForFields[k][fieldName] = optional
}
}
destinationSchemaFields[k][fieldName] = true
if optional, ok := schema.OptionalFor[k]; ok {
optionalForFields[k][fieldName] = optional
}
}
delete(configFields, fieldName)
}
Expand Down
7 changes: 5 additions & 2 deletions fivetran/common/fields.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -659,8 +659,7 @@
"key_field": "",
"item_type": {},
"description": {
"adobe_analytics": "The table name unique within the schema to which connector will sync the data. Required for connector creation.",
"share_point": "The table name within the schema to which the connector will sync the data."
"adobe_analytics": "The table name unique within the schema to which connector will sync the data. Required for connector creation."
},
"api_field": ""
}
Expand Down Expand Up @@ -10490,6 +10489,9 @@
"fields": null,
"key_field": "",
"item_type": {},
"optional_for" : {
"share_point" : true
},
"description": {
"adroll": "Destination table. Table is permanent and cannot be changed after connection creation",
"aws_cost_report": "Destination table. Table is permanent and cannot be changed after connection creation",
Expand All @@ -10509,6 +10511,7 @@
"kinesis": "Destination table. Table is permanent and cannot be changed after connection creation",
"s3": "Destination table. Table is permanent and cannot be changed after connection creation",
"sftp": "Destination table. Table is permanent and cannot be changed after connection creation",
"share_point": "Destination table. Table is permanent and cannoddt be changed after connection creation",
"wasabi_cloud_storage": "Destination table. Table is permanent and cannot be changed after connection creation",
"webhooks": "Destination table. Table is permanent and cannot be changed after connection creation",
"workday": "Destination table. Table is permanent and cannot be changed after connection creation",
Expand Down
96 changes: 48 additions & 48 deletions fivetran/framework/core/model/connector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ type ConnectorDatasourceModel struct {
PauseAfterTrial types.Bool `tfsdk:"pause_after_trial"`
DailySyncTime types.String `tfsdk:"daily_sync_time"`

ProxyAgentId types.String `tfsdk:"proxy_agent_id"`
NetworkingMethod types.String `tfsdk:"networking_method"`
LocalProcessingAgentId types.String `tfsdk:"local_processing_agent_id"`
ProxyAgentId types.String `tfsdk:"proxy_agent_id"`
NetworkingMethod types.String `tfsdk:"networking_method"`
LocalProcessingAgentId types.String `tfsdk:"local_processing_agent_id"`

Status types.Object `tfsdk:"status"`

Expand Down Expand Up @@ -122,9 +122,9 @@ type ConnectorResourceModel struct {
Service types.String `tfsdk:"service"`
DestinationSchema types.Object `tfsdk:"destination_schema"`

ProxyAgentId types.String `tfsdk:"proxy_agent_id"`
NetworkingMethod types.String `tfsdk:"networking_method"`
LocalProcessingAgentId types.String `tfsdk:"local_processing_agent_id"`
ProxyAgentId types.String `tfsdk:"proxy_agent_id"`
NetworkingMethod types.String `tfsdk:"networking_method"`
LocalProcessingAgentId types.String `tfsdk:"local_processing_agent_id"`

Config types.Object `tfsdk:"config"`
Auth types.Object `tfsdk:"auth"`
Expand Down Expand Up @@ -195,24 +195,24 @@ func (d *ConnectorResourceModel) ReadFromContainer(c ConnectorModelContainer, fo
d.CreatedAt = types.StringValue(c.CreatedAt)
d.GroupId = types.StringValue(c.GroupId)
d.Service = types.StringValue(c.Service)
if c.LocalProcessingAgentId != "" {
d.LocalProcessingAgentId = types.StringValue(c.LocalProcessingAgentId)
} else {
d.LocalProcessingAgentId = types.StringNull()
}

if c.LocalProcessingAgentId != "" {
d.LocalProcessingAgentId = types.StringValue(c.LocalProcessingAgentId)
} else {
d.LocalProcessingAgentId = types.StringNull()
}

d.DestinationSchema = getDestinationSchemaValue(c.Service, c.Schema)

if c.ProxyAgentId != "" {
d.ProxyAgentId = types.StringValue(c.ProxyAgentId)
} else {
d.ProxyAgentId = types.StringNull()
}
if c.ProxyAgentId != "" {
d.ProxyAgentId = types.StringValue(c.ProxyAgentId)
} else {
d.ProxyAgentId = types.StringNull()
}

if c.NetworkingMethod != "" {
d.NetworkingMethod = types.StringValue(c.NetworkingMethod)
}
if c.NetworkingMethod != "" {
d.NetworkingMethod = types.StringValue(c.NetworkingMethod)
}

if forceReadConfig || (!d.Config.IsNull() && !d.Config.IsUnknown()) {
d.Config = getValue(
Expand All @@ -230,24 +230,24 @@ func (d *ConnectorDatasourceModel) ReadFromContainer(c ConnectorModelContainer)
d.CreatedAt = types.StringValue(c.CreatedAt)
d.GroupId = types.StringValue(c.GroupId)
d.Service = types.StringValue(c.Service)
if c.LocalProcessingAgentId != "" {
d.LocalProcessingAgentId = types.StringValue(c.LocalProcessingAgentId)
} else {
d.LocalProcessingAgentId = types.StringNull()
}

if c.LocalProcessingAgentId != "" {
d.LocalProcessingAgentId = types.StringValue(c.LocalProcessingAgentId)
} else {
d.LocalProcessingAgentId = types.StringNull()
}

d.DestinationSchema = getDestinationSchemaValue(c.Service, c.Schema)

if c.ProxyAgentId != "" {
d.ProxyAgentId = types.StringValue(c.ProxyAgentId)
} else {
d.ProxyAgentId = types.StringNull()
}
if c.ProxyAgentId != "" {
d.ProxyAgentId = types.StringValue(c.ProxyAgentId)
} else {
d.ProxyAgentId = types.StringNull()
}

if c.NetworkingMethod != "" {
d.NetworkingMethod = types.StringValue(c.NetworkingMethod)
}
if c.NetworkingMethod != "" {
d.NetworkingMethod = types.StringValue(c.NetworkingMethod)
}

d.Config = getValue(
types.ObjectType{AttrTypes: getAttrTypes(common.GetConfigFieldsMap())},
Expand All @@ -267,9 +267,9 @@ type ConnectorModelContainer struct {
Service string
Schema string

ProxyAgentId string
NetworkingMethod string
LocalProcessingAgentId string
ProxyAgentId string
NetworkingMethod string
LocalProcessingAgentId string

Config map[string]interface{}

Expand All @@ -288,17 +288,17 @@ func (c *ConnectorModelContainer) ReadFromResponseData(data connectors.DetailsRe
c.Schema = data.Schema
c.Config = config

if data.ProxyAgentId != "" {
c.ProxyAgentId = data.ProxyAgentId
}
if data.ProxyAgentId != "" {
c.ProxyAgentId = data.ProxyAgentId
}

if data.NetworkingMethod != "" {
c.NetworkingMethod = data.NetworkingMethod
}
if data.NetworkingMethod != "" {
c.NetworkingMethod = data.NetworkingMethod
}

if data.LocalProcessingAgentId != "" {
c.LocalProcessingAgentId = data.LocalProcessingAgentId
}
if data.LocalProcessingAgentId != "" {
c.LocalProcessingAgentId = data.LocalProcessingAgentId
}
}

func getDestinatonSchemaForConfig(serviceId, nameAttr, tableAttr, prefixAttr attr.Value) (map[string]interface{}, error) {
Expand All @@ -321,14 +321,14 @@ func getDestinatonSchemaForConfig(serviceId, nameAttr, tableAttr, prefixAttr att
"schema_prefix": prefix,
}, nil
} else {
if nameAttr.IsNull() {
if nameAttr.IsNull() && !common.IsOptionalFor(service, "name") {
return nil, fmt.Errorf("`destination_schema.name` field is required to create `%v` connector", service)
}
result := map[string]interface{}{
"schema": nameAttr.(types.String).ValueString(),
}
if common.GetDestinationSchemaFields()[service]["table"] {
if tableAttr.IsNull() {
if tableAttr.IsNull() && !common.IsOptionalFor(service, "table") {
return nil, fmt.Errorf("`destination_schema.table` field is required to create `%v` connector", service)
}
result["table"] = tableAttr.(types.String).ValueString()
Expand Down

0 comments on commit fbd0360

Please sign in to comment.