From a7c4abf8f08afe2cd2a6217fa3072fc751585539 Mon Sep 17 00:00:00 2001 From: avilevy Date: Mon, 30 Oct 2023 18:36:27 +0000 Subject: [PATCH] Bug fixes --- Dockerfile | 72 +++++++------------ cmd/agent_wrapper/main_linux_test.go | 3 +- debian/rules | 4 +- dockerfiles/compile_test.go | 4 +- dockerfiles/template | 6 +- ...transformation_config.yaml => config.yaml} | 0 .../{transformation_input.txt => input.log} | 0 ...tion_output.yaml => output_fluentbit.yaml} | 0 ...transformation_config.yaml => config.yaml} | 0 .../{transformation_input.txt => input.log} | 0 ...tion_output.yaml => output_fluentbit.yaml} | 0 .../testdata/parse_json_upstream/config.yaml | 4 ++ .../testdata/parse_json_upstream/input.log | 2 + .../parse_json_upstream/output_fluentbit.yaml | 6 ++ .../testdata/test_timezone/config.yaml | 6 ++ .../testdata/test_timezone/input.log | 1 + .../test_timezone/output_fluentbit.yaml | 3 + transformation_test/transformation_test.go | 9 +-- 18 files changed, 59 insertions(+), 61 deletions(-) rename transformation_test/testdata/modify_fields/{transformation_config.yaml => config.yaml} (100%) rename transformation_test/testdata/modify_fields/{transformation_input.txt => input.log} (100%) rename transformation_test/testdata/modify_fields/{transformation_output.yaml => output_fluentbit.yaml} (100%) rename transformation_test/testdata/parse_json/{transformation_config.yaml => config.yaml} (100%) rename transformation_test/testdata/parse_json/{transformation_input.txt => input.log} (100%) rename transformation_test/testdata/parse_json/{transformation_output.yaml => output_fluentbit.yaml} (100%) create mode 100644 transformation_test/testdata/parse_json_upstream/config.yaml create mode 100644 transformation_test/testdata/parse_json_upstream/input.log create mode 100644 transformation_test/testdata/parse_json_upstream/output_fluentbit.yaml create mode 100644 transformation_test/testdata/test_timezone/config.yaml create mode 100644 transformation_test/testdata/test_timezone/input.log create mode 100644 transformation_test/testdata/test_timezone/output_fluentbit.yaml diff --git a/Dockerfile b/Dockerfile index 990245e61e..4663d65651 100644 --- a/Dockerfile +++ b/Dockerfile @@ -156,10 +156,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM centos7-build-golang-base AS centos7-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -257,10 +255,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM centos8-build-golang-base AS centos8-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -361,10 +357,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM rockylinux9-build-golang-base AS rockylinux9-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -460,10 +454,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM bookworm-build-golang-base AS bookworm-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -559,10 +551,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM bullseye-build-golang-base AS bullseye-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -660,10 +650,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM buster-build-golang-base AS buster-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -778,10 +766,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM sles12-build-golang-base AS sles12-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -882,10 +868,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM sles15-build-golang-base AS sles15-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -981,10 +965,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM focal-build-golang-base AS focal-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -1080,10 +1062,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM jammy-build-golang-base AS jammy-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -1179,10 +1159,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM lunar-build-golang-base AS lunar-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang @@ -1278,10 +1256,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM mantic-build-golang-base AS mantic-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang diff --git a/cmd/agent_wrapper/main_linux_test.go b/cmd/agent_wrapper/main_linux_test.go index be7840c833..3adcecc473 100644 --- a/cmd/agent_wrapper/main_linux_test.go +++ b/cmd/agent_wrapper/main_linux_test.go @@ -17,11 +17,12 @@ package main import ( + "fmt" "os/exec" "strings" ) // Get a command that will write the given number of bytes func getCommand(writeBytes int) *exec.Cmd { - return exec.Command("echo", strings.Repeat("a", int(writeBytes-1))) + return exec.Command("/bin/sh", "-c", fmt.Sprintf("echo %s", strings.Repeat("a", writeBytes-1))) } diff --git a/debian/rules b/debian/rules index 02e20a4e46..9845a639ed 100755 --- a/debian/rules +++ b/debian/rules @@ -8,9 +8,7 @@ override_dh_auto_configure: override_dh_auto_build: true override_dh_auto_test: - pwd - ls -la /work/cache/opt/google-cloud-ops-agent/subagents/fluent-bit/bin - go test ./... + PATH="/usr/local/go/bin:$$PATH" FLB="../cache/opt/google-cloud-ops-agent/subagents/fluent-bit/bin" go test ./... override_dh_strip_nondeterminism: dh_strip_nondeterminism -X.jar override_dh_auto_install: diff --git a/dockerfiles/compile_test.go b/dockerfiles/compile_test.go index dfd135bfe6..717b14f958 100644 --- a/dockerfiles/compile_test.go +++ b/dockerfiles/compile_test.go @@ -7,7 +7,9 @@ import ( func TestDockerfileMatches(t *testing.T) { gotDockerfile, err := os.ReadFile(getDockerfilePath()) - if err != nil { + if os.IsNotExist(err) { + t.Skip("Dockerfile not found") + } else if err != nil { t.Error(err) } expectedDockerfile, err := getDockerfile() diff --git a/dockerfiles/template b/dockerfiles/template index 79f986c9db..ef5a2c8897 100644 --- a/dockerfiles/template +++ b/dockerfiles/template @@ -71,10 +71,8 @@ RUN ./agent_wrapper.sh /work/cache/ FROM {target_name}-build-golang-base AS {target_name}-build WORKDIR /work -COPY cmd/google_cloud_ops_agent_engine cmd/google_cloud_ops_agent_engine -COPY VERSION build.sh ./ -COPY debian debian -COPY pkg pkg +COPY . /work + # Run the build script once to build the ops agent engine to a cache RUN mkdir -p /tmp/cache_run/golang && cp -r . /tmp/cache_run/golang WORKDIR /tmp/cache_run/golang diff --git a/transformation_test/testdata/modify_fields/transformation_config.yaml b/transformation_test/testdata/modify_fields/config.yaml similarity index 100% rename from transformation_test/testdata/modify_fields/transformation_config.yaml rename to transformation_test/testdata/modify_fields/config.yaml diff --git a/transformation_test/testdata/modify_fields/transformation_input.txt b/transformation_test/testdata/modify_fields/input.log similarity index 100% rename from transformation_test/testdata/modify_fields/transformation_input.txt rename to transformation_test/testdata/modify_fields/input.log diff --git a/transformation_test/testdata/modify_fields/transformation_output.yaml b/transformation_test/testdata/modify_fields/output_fluentbit.yaml similarity index 100% rename from transformation_test/testdata/modify_fields/transformation_output.yaml rename to transformation_test/testdata/modify_fields/output_fluentbit.yaml diff --git a/transformation_test/testdata/parse_json/transformation_config.yaml b/transformation_test/testdata/parse_json/config.yaml similarity index 100% rename from transformation_test/testdata/parse_json/transformation_config.yaml rename to transformation_test/testdata/parse_json/config.yaml diff --git a/transformation_test/testdata/parse_json/transformation_input.txt b/transformation_test/testdata/parse_json/input.log similarity index 100% rename from transformation_test/testdata/parse_json/transformation_input.txt rename to transformation_test/testdata/parse_json/input.log diff --git a/transformation_test/testdata/parse_json/transformation_output.yaml b/transformation_test/testdata/parse_json/output_fluentbit.yaml similarity index 100% rename from transformation_test/testdata/parse_json/transformation_output.yaml rename to transformation_test/testdata/parse_json/output_fluentbit.yaml diff --git a/transformation_test/testdata/parse_json_upstream/config.yaml b/transformation_test/testdata/parse_json_upstream/config.yaml new file mode 100644 index 0000000000..fe50911c9f --- /dev/null +++ b/transformation_test/testdata/parse_json_upstream/config.yaml @@ -0,0 +1,4 @@ +- type: parse_json + field: message + time_key: time + time_format: "%Y-%m-%dT%H:%M:%S %z" diff --git a/transformation_test/testdata/parse_json_upstream/input.log b/transformation_test/testdata/parse_json_upstream/input.log new file mode 100644 index 0000000000..f3c3aa24b9 --- /dev/null +++ b/transformation_test/testdata/parse_json_upstream/input.log @@ -0,0 +1,2 @@ +{"key1": 12345, "key2": "abc", "time": "2006-07-28T13:22:04Z"} +{"key1": 12345, "key2": "abc", "time": "2006-07-28T13:22:04+01:00"} diff --git a/transformation_test/testdata/parse_json_upstream/output_fluentbit.yaml b/transformation_test/testdata/parse_json_upstream/output_fluentbit.yaml new file mode 100644 index 0000000000..359dbb6bd4 --- /dev/null +++ b/transformation_test/testdata/parse_json_upstream/output_fluentbit.yaml @@ -0,0 +1,6 @@ +- date: 2006-07-28T13:22:04Z + key1: 12345 + key2: abc +- date: 2006-07-28T12:22:04Z + key1: 12345 + key2: abc diff --git a/transformation_test/testdata/test_timezone/config.yaml b/transformation_test/testdata/test_timezone/config.yaml new file mode 100644 index 0000000000..662470751c --- /dev/null +++ b/transformation_test/testdata/test_timezone/config.yaml @@ -0,0 +1,6 @@ +# When a timezone is not within a log Fluent-bit assumes UTC instead of system timezone +# https://github.com/fluent/fluent-bit/issues/593#issuecomment-811183471 +- type: parse_json + field: message + time_key: time + time_format: "%Y-%m-%dT%H:%M" diff --git a/transformation_test/testdata/test_timezone/input.log b/transformation_test/testdata/test_timezone/input.log new file mode 100644 index 0000000000..2327f46633 --- /dev/null +++ b/transformation_test/testdata/test_timezone/input.log @@ -0,0 +1 @@ +{"key1": 12345, "key2": "abc", "time": "2006-07-28T13:22"} diff --git a/transformation_test/testdata/test_timezone/output_fluentbit.yaml b/transformation_test/testdata/test_timezone/output_fluentbit.yaml new file mode 100644 index 0000000000..0124b1ec10 --- /dev/null +++ b/transformation_test/testdata/test_timezone/output_fluentbit.yaml @@ -0,0 +1,3 @@ +- date: 2006-07-28T13:22:00Z + key1: 12345 + key2: abc diff --git a/transformation_test/transformation_test.go b/transformation_test/transformation_test.go index d5caed736f..f2b62db2b1 100644 --- a/transformation_test/transformation_test.go +++ b/transformation_test/transformation_test.go @@ -7,6 +7,7 @@ import ( "flag" "fmt" "io" + "os" "os/exec" "path/filepath" "strconv" @@ -24,13 +25,13 @@ import ( const ( flbMainConf = "fluent_bit_main.conf" flbParserConf = "fluent_bit_parser.conf" - transformationInput = "transformation_input.txt" - transformationOutput = "transformation_output.yaml" + transformationInput = "input.log" + transformationOutput = "output_fluentbit.yaml" flbTag = "transformation_test" ) var ( - flbPath = flag.String("flb", "", "Fluent-bit path") + flbPath = flag.String("flb", os.Getenv("FLB"), "Fluent-bit path") ) //go:embed testdata @@ -163,7 +164,7 @@ func readTransformationConfig(dir string) (transformationTest, error) { var transformationTestData []byte var config transformationTest - transformationTestData, err := readFileFromTestDir(filepath.Join(dir, "transformation_config.yaml")) + transformationTestData, err := readFileFromTestDir(filepath.Join(dir, "config.yaml")) if err != nil { return config, err }