confgenerator: omit semantic validation on user config (#1081)

* confgenerator: omit semantic validation on user config Signed-off-by: Ridwan Sharif <[email protected]> * confgenerator: add windows unit test * Update confgenerator/confmerger.go Co-authored-by: igorpeshansky <[email protected]> * confmeger: rename original to result --------- Signed-off-by: Ridwan Sharif <[email protected]> Co-authored-by: igorpeshansky <[email protected]>
GoogleCloudPlatform · Feb 3, 2023 · 6705187 · 6705187
1 parent aeaf23e
commit 6705187
Show file tree

Hide file tree

Showing 16 changed files with 1,610 additions and 9 deletions.
diff --git a/confgenerator/confmerger.go b/confgenerator/confmerger.go
@@ -19,8 +19,8 @@ package confgenerator
 func MergeConfFiles(userConfPath, platform string, builtInConfStructs map[string]*UnifiedConfig) (*UnifiedConfig, error) {
 	builtInStruct := builtInConfStructs[platform]
 
-	// Read the built-in config file.
-	original, err := builtInStruct.DeepCopy(platform)
+	// Start with the built-in config.
+	result, err := builtInStruct.DeepCopy(platform)
 	if err != nil {
 		return nil, err
 	}
@@ -32,19 +32,19 @@ func MergeConfFiles(userConfPath, platform string, builtInConfStructs map[string
 
 	// Optionally merge the user config file.
 	if overrides != nil {
-		mergeConfigs(original, overrides)
+		mergeConfigs(result, overrides)
 	}
 
-	if err := original.Validate(); err != nil {
+	if err := result.Validate(); err != nil {
 		return nil, err
 	}
 
 	// Ensure the merged config struct fields are valid.
 	v := newValidator()
-	if err := v.Struct(original); err != nil {
+	if err := v.Struct(result); err != nil {
 		panic(err)
 	}
-	return original, nil
+	return result, nil
 }
 
 func mergeConfigs(original, overrides *UnifiedConfig) {

diff --git a/confgenerator/files.go b/confgenerator/files.go
@@ -43,9 +43,6 @@ func ReadUnifiedConfigFromFile(path, platform string) (*UnifiedConfig, error) {
 	if err != nil {
 		return nil, err
 	}
-	if err := uc.Validate(); err != nil {
-		return nil, err
-	}
 
 	return uc, nil
 }

diff --git a/...valid/linux/all-custom_use_built_in_receivers/golden/f120d4527bd717cab023dbbe5fbdc332.lua b/...valid/linux/all-custom_use_built_in_receivers/golden/f120d4527bd717cab023dbbe5fbdc332.lua
@@ -0,0 +1,42 @@
+
+function process(tag, timestamp, record)
+local __field_0 = (function()
+return record["agent.googleapis.com/log_file_path"]
+end)();
+local __field_1 = (function()
+if record["logging.googleapis.com/labels"] == nil
+then
+return nil
+end
+return record["logging.googleapis.com/labels"]["compute.googleapis.com/resource_name"]
+end)();
+local __field_2 = (function()
+return record["logging.googleapis.com/logName"]
+end)();
+(function(value)
+record["agent.googleapis.com/log_file_path"] = value
+end)(nil);
+local v = __field_0;
+(function(value)
+if record["logging.googleapis.com/labels"] == nil
+then
+record["logging.googleapis.com/labels"] = {}
+end
+record["logging.googleapis.com/labels"]["agent.googleapis.com/log_file_path"] = value
+end)(v)
+local v = __field_1;
+if v == nil then v = "" end;
+(function(value)
+if record["logging.googleapis.com/labels"] == nil
+then
+record["logging.googleapis.com/labels"] = {}
+end
+record["logging.googleapis.com/labels"]["compute.googleapis.com/resource_name"] = value
+end)(v)
+local v = __field_2;
+if v == nil then v = "syslog" end;
+(function(value)
+record["logging.googleapis.com/logName"] = value
+end)(v)
+return 2, timestamp, record
+end
diff --git a/confgenerator/testdata/valid/linux/all-custom_use_built_in_receivers/golden/features.yaml b/confgenerator/testdata/valid/linux/all-custom_use_built_in_receivers/golden/features.yaml
@@ -0,0 +1,8 @@
+- module: logging
+  feature: service:pipelines
+  key: default_pipeline_overridden
+  value: "false"
+- module: metrics
+  feature: service:pipelines
+  key: default_pipeline_overridden
+  value: "false"
diff --git a/...erator/testdata/valid/linux/all-custom_use_built_in_receivers/golden/fluent_bit_main.conf b/...erator/testdata/valid/linux/all-custom_use_built_in_receivers/golden/fluent_bit_main.conf
@@ -0,0 +1,105 @@
+@SET buffers_dir=/var/lib/google-cloud-ops-agent/fluent-bit/buffers
+@SET logs_dir=/var/log/google-cloud-ops-agent/subagents
+
+[SERVICE]
+    Daemon                    off
+    Flush                     1
+    Log_Level                 info
+    dns.resolver              legacy
+    storage.backlog.mem_limit 50M
+    storage.checksum          off
+    storage.max_chunks_up     128
+    storage.metrics           on
+    storage.sync              normal
+
+[INPUT]
+    Name            fluentbit_metrics
+    Scrape_Interval 60
+    Scrape_On_Start True
+
+[INPUT]
+    Buffer_Chunk_Size 512k
+    Buffer_Max_Size   2M
+    DB                ${buffers_dir}/default_pipeline_syslog
+    DB.locking        true
+    Key               message
+    Mem_Buf_Limit     10M
+    Name              tail
+    Path              /var/log/messages,/var/log/syslog
+    Read_from_Head    True
+    Rotate_Wait       30
+    Skip_Long_Lines   On
+    Tag               default_pipeline.syslog
+    storage.type      filesystem
+
+[INPUT]
+    Buffer_Chunk_Size 512k
+    Buffer_Max_Size   2M
+    DB                ${buffers_dir}/host_syslog
+    DB.locking        true
+    Key               message
+    Mem_Buf_Limit     10M
+    Name              tail
+    Path              /var/log/messages,/var/log/syslog
+    Read_from_Head    True
+    Rotate_Wait       30
+    Skip_Long_Lines   On
+    Tag               host.syslog
+    storage.type      filesystem
+
+[INPUT]
+    Buffer_Chunk_Size 512k
+    Buffer_Max_Size   2M
+    DB                ${buffers_dir}/ops-agent-fluent-bit
+    DB.locking        true
+    Key               message
+    Mem_Buf_Limit     10M
+    Name              tail
+    Path              ${logs_dir}/logging-module.log
+    Read_from_Head    True
+    Rotate_Wait       30
+    Skip_Long_Lines   On
+    Tag               ops-agent-fluent-bit
+    storage.type      memory
+
+[FILTER]
+    Match  default_pipeline.syslog
+    Name   lua
+    call   process
+    script f120d4527bd717cab023dbbe5fbdc332.lua
+
+[FILTER]
+    Match  host.syslog
+    Name   lua
+    call   process
+    script f120d4527bd717cab023dbbe5fbdc332.lua
+
+[OUTPUT]
+    Match_Regex                   ^(default_pipeline\.syslog|host\.syslog)$
+    Name                          stackdriver
+    Retry_Limit                   3
+    http_request_key              logging.googleapis.com/httpRequest
+    net.connect_timeout_log_error False
+    resource                      gce_instance
+    stackdriver_agent             Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
+    tls                           On
+    tls.verify                    Off
+    workers                       8
+
+[OUTPUT]
+    Match_Regex                   ^(ops-agent-fluent-bit)$
+    Name                          stackdriver
+    Retry_Limit                   3
+    http_request_key              logging.googleapis.com/httpRequest
+    net.connect_timeout_log_error False
+    resource                      gce_instance
+    stackdriver_agent             Google-Cloud-Ops-Agent-Logging/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version)
+    tls                           On
+    tls.verify                    Off
+    workers                       8
+
+[OUTPUT]
+    Match *
+    Name  prometheus_exporter
+    host  0.0.0.0
+    port  20202
diff --git a/...ator/testdata/valid/linux/all-custom_use_built_in_receivers/golden/fluent_bit_parser.conf b/...ator/testdata/valid/linux/all-custom_use_built_in_receivers/golden/fluent_bit_parser.conf