From 62da5380655c253a9403481985f2be3dda57b446 Mon Sep 17 00:00:00 2001 From: grego952 Date: Thu, 25 Jan 2024 08:24:51 +0100 Subject: [PATCH 01/27] Add testing strategy --- docs/contributor/testing-strategy.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 docs/contributor/testing-strategy.md diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md new file mode 100644 index 00000000..d0f0fd75 --- /dev/null +++ b/docs/contributor/testing-strategy.md @@ -0,0 +1,16 @@ +# Testing Strategy + +Each pull request to the repository triggers the following CI/CD jobs: + +- `golangci-lint / lint (pull_request)` - Is responsible for linting and static code analysis. +- `PR Markdown Link Check / markdown-link-check (pull_request)` - Checks if there are no broken links in the pull request `.md` files. +- `Run unit tests / validate (pull_request)` - Executes basic create/update/delete functional tests of the reconciliation logic. +- `Run vuln check / test (pull_request)` - Executes the [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) tool on the code to detect known vulnerabilities. +- `pull-infrastructure-manager-build` - builds the Docker image and pushes it to the registry. + +After the pull request is merged, the following CI/CD jobs are executed: + + - `Run unit tests / validate (push)` - Executes basic create/update/delete functional tests of the reconciliation logic. + - `Run vuln check / test (push)` - Executes the [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) tool on the code to detect known vulnerabilities. + - `golangci-lint / lint (push)` - Is responsible for linting and static code analysis. + - `main-infrastructure-manager-build` - Rebuilds the image and pushes it into the `prod` registry. \ No newline at end of file From bf68b90af15d77ee5e054bea842e61685efd7b66 Mon Sep 17 00:00:00 2001 From: grego952 Date: Thu, 25 Jan 2024 08:30:22 +0100 Subject: [PATCH 02/27] Small fixes --- docs/contributor/testing-strategy.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index d0f0fd75..adc72257 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -5,12 +5,12 @@ Each pull request to the repository triggers the following CI/CD jobs: - `golangci-lint / lint (pull_request)` - Is responsible for linting and static code analysis. - `PR Markdown Link Check / markdown-link-check (pull_request)` - Checks if there are no broken links in the pull request `.md` files. - `Run unit tests / validate (pull_request)` - Executes basic create/update/delete functional tests of the reconciliation logic. -- `Run vuln check / test (pull_request)` - Executes the [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) tool on the code to detect known vulnerabilities. +- `Run vuln check / test (pull_request)` - Runs [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) on the code to detect known vulnerabilities. - `pull-infrastructure-manager-build` - builds the Docker image and pushes it to the registry. After the pull request is merged, the following CI/CD jobs are executed: - `Run unit tests / validate (push)` - Executes basic create/update/delete functional tests of the reconciliation logic. - - `Run vuln check / test (push)` - Executes the [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) tool on the code to detect known vulnerabilities. + - `Run vuln check / test (push)` - Runs [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) on the code to detect known vulnerabilities. - `golangci-lint / lint (push)` - Is responsible for linting and static code analysis. - - `main-infrastructure-manager-build` - Rebuilds the image and pushes it into the `prod` registry. \ No newline at end of file + - `main-infrastructure-manager-build` - Rebuilds the image and pushes it to the registry. \ No newline at end of file From e9a109877ef1ac12c6486ff5544be5ac1a89316b Mon Sep 17 00:00:00 2001 From: Grzegorz Karaluch Date: Thu, 25 Jan 2024 11:39:12 +0100 Subject: [PATCH 03/27] Apply suggestions from Disper review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michał 'Disper' Drzewiecki --- docs/contributor/testing-strategy.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index adc72257..45253638 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -2,15 +2,15 @@ Each pull request to the repository triggers the following CI/CD jobs: -- `golangci-lint / lint (pull_request)` - Is responsible for linting and static code analysis. -- `PR Markdown Link Check / markdown-link-check (pull_request)` - Checks if there are no broken links in the pull request `.md` files. -- `Run unit tests / validate (pull_request)` - Executes basic create/update/delete functional tests of the reconciliation logic. -- `Run vuln check / test (pull_request)` - Runs [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) on the code to detect known vulnerabilities. -- `pull-infrastructure-manager-build` - builds the Docker image and pushes it to the registry. +- `golangci-lint / lint (pull_request)` - Is responsible for linting and static code analysis. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.golangci.yaml) and [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.github/workflows/golangci-lint.yaml). +- `PR Markdown Link Check / markdown-link-check (pull_request)` - Checks if there are no broken links in the pull request `.md` files. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/mlc.config.json). +- `Run unit tests / validate (pull_request)` - Executes basic create/update/delete functional tests of the reconciliation logic. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.github/workflows/run-tests.yaml). +- `Run vuln check / test (pull_request)` - Runs [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) on the code to detect known vulnerabilities. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.github/workflows/run-vuln-check.yaml). +- `pull-infrastructure-manager-build` - builds the Docker image and pushes it to the registry. It's configured [here](https://github.com/kyma-project/test-infra/blob/a3c2a07da4ba42e468f69cf42f1960d7bfcc3fff/prow/jobs/kyma-project/infrastructure-manager/infrastructure-manager.yaml). After the pull request is merged, the following CI/CD jobs are executed: - `Run unit tests / validate (push)` - Executes basic create/update/delete functional tests of the reconciliation logic. - `Run vuln check / test (push)` - Runs [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) on the code to detect known vulnerabilities. - `golangci-lint / lint (push)` - Is responsible for linting and static code analysis. - - `main-infrastructure-manager-build` - Rebuilds the image and pushes it to the registry. \ No newline at end of file + - `main-infrastructure-manager-build` - Rebuilds the image and pushes it to the registry. It's configured [here](https://github.com/kyma-project/test-infra/blob/a3c2a07da4ba42e468f69cf42f1960d7bfcc3fff/prow/jobs/kyma-project/infrastructure-manager/infrastructure-manager.yaml). \ No newline at end of file From 5a3f392899d6261d5a395c174d09c4821bd8eb6b Mon Sep 17 00:00:00 2001 From: grego952 Date: Thu, 25 Jan 2024 13:04:56 +0100 Subject: [PATCH 04/27] Apply Disper suggestions --- docs/contributor/testing-strategy.md | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index 45253638..c408f599 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -1,16 +1,12 @@ # Testing Strategy -Each pull request to the repository triggers the following CI/CD jobs: +The following CI/CD jobs are a part of the development cycle: -- `golangci-lint / lint (pull_request)` - Is responsible for linting and static code analysis. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.golangci.yaml) and [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.github/workflows/golangci-lint.yaml). -- `PR Markdown Link Check / markdown-link-check (pull_request)` - Checks if there are no broken links in the pull request `.md` files. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/mlc.config.json). -- `Run unit tests / validate (pull_request)` - Executes basic create/update/delete functional tests of the reconciliation logic. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.github/workflows/run-tests.yaml). -- `Run vuln check / test (pull_request)` - Runs [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) on the code to detect known vulnerabilities. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.github/workflows/run-vuln-check.yaml). -- `pull-infrastructure-manager-build` - builds the Docker image and pushes it to the registry. It's configured [here](https://github.com/kyma-project/test-infra/blob/a3c2a07da4ba42e468f69cf42f1960d7bfcc3fff/prow/jobs/kyma-project/infrastructure-manager/infrastructure-manager.yaml). - -After the pull request is merged, the following CI/CD jobs are executed: +> **NOTE:** Jobs marked with `pull_request` are triggered with each pull request. Jobs marked with `push` are executed after the merge. - - `Run unit tests / validate (push)` - Executes basic create/update/delete functional tests of the reconciliation logic. - - `Run vuln check / test (push)` - Runs [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) on the code to detect known vulnerabilities. - - `golangci-lint / lint (push)` - Is responsible for linting and static code analysis. - - `main-infrastructure-manager-build` - Rebuilds the image and pushes it to the registry. It's configured [here](https://github.com/kyma-project/test-infra/blob/a3c2a07da4ba42e468f69cf42f1960d7bfcc3fff/prow/jobs/kyma-project/infrastructure-manager/infrastructure-manager.yaml). \ No newline at end of file +- `golangci-lint / lint (pull_request/push)` - Is responsible for linting and static code analysis. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.golangci.yaml) and [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.github/workflows/golangci-lint.yaml). +- `PR Markdown Link Check / markdown-link-check (pull_request)` - Checks if there are no broken links in the pull request `.md` files. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/mlc.config.json). +- `Run unit tests / validate (pull_request/push)` - Executes basic create/update/delete functional tests of the reconciliation logic. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.github/workflows/run-tests.yaml). +- `Run vuln check / test (pull_request/push)` - Runs [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) on the code to detect known vulnerabilities. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.github/workflows/run-vuln-check.yaml). +- `pull-infrastructure-manager-build` - Triggered with each pull request. It builds the Docker image and pushes it to the registry. It's configured [here](https://github.com/kyma-project/test-infra/blob/a3c2a07da4ba42e468f69cf42f1960d7bfcc3fff/prow/jobs/kyma-project/infrastructure-manager/infrastructure-manager.yaml). +- `main-infrastructure-manager-build` - Triggered after the merge. Rebuilds the image and pushes it to the registry. It's configured [here](https://github.com/kyma-project/test-infra/blob/a3c2a07da4ba42e468f69cf42f1960d7bfcc3fff/prow/jobs/kyma-project/infrastructure-manager/infrastructure-manager.yaml). \ No newline at end of file From e96382ebf61609c5182def8418acb9bbe5f8d21d Mon Sep 17 00:00:00 2001 From: Tobias Schuhmacher Date: Fri, 26 Jan 2024 14:24:42 +0100 Subject: [PATCH 05/27] adding tobis ideas --- docs/contributor/testing-strategy.md | 32 +++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index c408f599..ef89c213 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -1,4 +1,34 @@ -# Testing Strategy +# Testing Strategy Infrastructure-Manager + +## Introduction +This testing strategy outlines the approach and methodologies to be used for testing a Kubernetes project built using the Kubebuilder framework. It includes various levels of testing such as unit testing, integration testing, and end-to-end testing to ensure the stability, reliability, and correctness of the project. +## Testing Levels +1. **Unit Testing:** Writing and executing tests for individual functions, methods, and components to verify their behavior and correctness in isolation. +2. **Integration Testing:** Validating the integration and interaction between different components, modules, and services in the project. +3. **End-to-End Testing:** Testing the application as a whole in a production-like environment, mimicking real-world scenarios to ensure the entire system functions correctly. +## Testing Tools and Frameworks +Use the following tools and frameworks to implement the above-mentioned testing levels: +- **Ginkgo and Gomega**: For writing and executing unit tests with a BDD-style syntax and assertions. +- **Kubebuilder Test Framework**: For creating and executing integration tests that interact with the Kubernetes cluster. +- **Helm**: For deploying and managing test clusters and environments for end-to-end testing. +- **Kubernetes clients for Go**: Use the official Kubernetes client libraries to interact with the Kubernetes API and validate the behavior of the project. +## Testing Approach +### Unit Testing +1. Identify critical functions, methods, and components that require testing. +2. Write unit tests using Ginkgo and Gomega frameworks. +3. Ensure tests cover various scenarios, edge cases, and possible failure scenarios. +4. Test for both positive and negative inputs to validate the expected behavior. +5. Mock external dependencies and use stubs or fakes to isolate the unit under test. +6. Run unit tests periodically during development and before each commit to prevent regressions. +### Integration Testing +1. Create a separate test suite for integration testing. +2. Use the Kubebuilder Test Framework to create test cases that interact with the Kubernetes cluster. +3. Test the interaction and integration of your custom resources, controllers, and other components with the Kubernetes API. +4. Ensure test cases cover various aspects such as resource creation, updating, deletion, and handling of edge cases. +5. Validate the correctness of event handling, reconciliation, and other control logic. +### End-to-End Testing +1. Use Helm to create, deploy, and manage test clusters and environments that closely resemble the + The following CI/CD jobs are a part of the development cycle: From 806e7973a5b62dc0fbb6c871e45d1bad22abc442 Mon Sep 17 00:00:00 2001 From: Tobias Schuhmacher Date: Fri, 26 Jan 2024 14:28:53 +0100 Subject: [PATCH 06/27] update docs --- docs/contributor/testing-strategy.md | 65 +++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index ef89c213..10fc2bc3 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -1,18 +1,33 @@ # Testing Strategy Infrastructure-Manager ## Introduction -This testing strategy outlines the approach and methodologies to be used for testing a Kubernetes project built using the Kubebuilder framework. It includes various levels of testing such as unit testing, integration testing, and end-to-end testing to ensure the stability, reliability, and correctness of the project. -## Testing Levels -1. **Unit Testing:** Writing and executing tests for individual functions, methods, and components to verify their behavior and correctness in isolation. -2. **Integration Testing:** Validating the integration and interaction between different components, modules, and services in the project. -3. **End-to-End Testing:** Testing the application as a whole in a production-like environment, mimicking real-world scenarios to ensure the entire system functions correctly. -## Testing Tools and Frameworks -Use the following tools and frameworks to implement the above-mentioned testing levels: -- **Ginkgo and Gomega**: For writing and executing unit tests with a BDD-style syntax and assertions. -- **Kubebuilder Test Framework**: For creating and executing integration tests that interact with the Kubernetes cluster. -- **Helm**: For deploying and managing test clusters and environments for end-to-end testing. -- **Kubernetes clients for Go**: Use the official Kubernetes client libraries to interact with the Kubernetes API and validate the behavior of the project. +This testing strategy describes how the Framefrog team is testing the product `Kyma Infrastructure Manager`. It outlines the approach and methodologies to be used for testing all layers of this product to ensure the stability, reliability, and correctness. + + +## Testing Methodology + +We investigate the product by separating it into layers: + +1. Business Logic + + Includes the technical frameworks (e.g. Kubebuilder) and custom Golang code. + +2. Business Features + + Combines the business logic into feature which is consumed by our customers. + +3. Product Integration + + Verifies how our product is integarted into the technical landscape, how it interacts with 3rd party systems and how it is accessible by customers or remote systems. + +For each layer is a dedicated testing approach used: + +1. **Unit Testing for Business Logic:** Writing and executing tests for individual functions, methods, and components to verify their behavior and correctness in isolation. +2. **Integration Testing for Business Features:** Validating the integration and interaction between different components, modules, and services in the project. +3. **End-to-End Testing:** Testing the application as a whole in a production-like environment, mimicking real-world scenarios to ensure the entire system functions correctly, is performing well and secure. + ## Testing Approach + ### Unit Testing 1. Identify critical functions, methods, and components that require testing. 2. Write unit tests using Ginkgo and Gomega frameworks. @@ -20,17 +35,43 @@ Use the following tools and frameworks to implement the above-mentioned testing 4. Test for both positive and negative inputs to validate the expected behavior. 5. Mock external dependencies and use stubs or fakes to isolate the unit under test. 6. Run unit tests periodically during development and before each commit to prevent regressions. + ### Integration Testing 1. Create a separate test suite for integration testing. 2. Use the Kubebuilder Test Framework to create test cases that interact with the Kubernetes cluster. 3. Test the interaction and integration of your custom resources, controllers, and other components with the Kubernetes API. 4. Ensure test cases cover various aspects such as resource creation, updating, deletion, and handling of edge cases. 5. Validate the correctness of event handling, reconciliation, and other control logic. + ### End-to-End Testing 1. Use Helm to create, deploy, and manage test clusters and environments that closely resemble the +### Testing Tools and Frameworks +Use the following tools and frameworks to implement the above-mentioned testing levels: + +- **GoTest**: For unit testing of Golang code. +- **Kubebuilder Test Framework and EnvTest**: For creating and executing integration tests that interact with the Kubernetes API. +- **Ginkgo and Gomega**: For writing and executing unit tests with a BDD-style syntax and assertions. +- **Kubebuilder Test Framework and EnvTest**: For creating and executing integration tests that interact with the Kubernetes API. +- **K3d**: For creating short-living and lightweight Kubernetes clustes running within a Docker context. +- **Helm**: For deploying and managing test clusters and environments for end-to-end testing. +- **K6:**: For performance and stress testing + +|Framework|Unit Testing|Integration Testing|End-to-End Testing| +|--|--|--|--| +|GoTest| X | | | +|Kubebuilder Test Framework| X | X | | +|EnvTest| X | X | | +|Ginkgo| | X | | +|Gomega| | X | | +|K3d| | | X | +|Helm| | | X | +|K6| | | X | + + +## Test Automation -The following CI/CD jobs are a part of the development cycle: +The following CI/CD jobs are a part of the development cycle and executing quality assurance related steps: > **NOTE:** Jobs marked with `pull_request` are triggered with each pull request. Jobs marked with `push` are executed after the merge. From 7d6d1188c625dc3c2f387c180a054505b89686a4 Mon Sep 17 00:00:00 2001 From: Tobias Schuhmacher Date: Fri, 26 Jan 2024 14:39:00 +0100 Subject: [PATCH 07/27] add few more sentences --- docs/contributor/testing-strategy.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index 10fc2bc3..41c0fc78 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -26,25 +26,28 @@ For each layer is a dedicated testing approach used: 2. **Integration Testing for Business Features:** Validating the integration and interaction between different components, modules, and services in the project. 3. **End-to-End Testing:** Testing the application as a whole in a production-like environment, mimicking real-world scenarios to ensure the entire system functions correctly, is performing well and secure. + ## Testing Approach ### Unit Testing 1. Identify critical functions, methods, and components that require testing. -2. Write unit tests using Ginkgo and Gomega frameworks. -3. Ensure tests cover various scenarios, edge cases, and possible failure scenarios. +2. Write unit tests using GoUnit tests, Ginkgo and Gomega frameworks. +3. Ensure tests cover various scenarios, edge cases, and possible failure scenarios. We try to verify business relevant logic with at least 65% code coverage. 4. Test for both positive and negative inputs to validate the expected behavior. 5. Mock external dependencies and use stubs or fakes to isolate the unit under test. 6. Run unit tests periodically during development and before each commit to prevent regressions. ### Integration Testing -1. Create a separate test suite for integration testing. -2. Use the Kubebuilder Test Framework to create test cases that interact with the Kubernetes cluster. -3. Test the interaction and integration of your custom resources, controllers, and other components with the Kubernetes API. -4. Ensure test cases cover various aspects such as resource creation, updating, deletion, and handling of edge cases. -5. Validate the correctness of event handling, reconciliation, and other control logic. +1. The PO together with the team create an registry of implemented business features and define for each feature a suitable test scenario. +2. Create a separate test suite for integration testing. +3. Each test scenario is implemented in a separte test case. Use the Kubebuilder Test Framework and others to create test cases that interact with the Kubernetes cluster. +4. Test the interaction and integration of your custom resources, controllers, and other components with the Kubernetes API. +5. Ensure test cases cover various aspects such as resource creation, updating, deletion, and handling of edge cases. +6. Validate the correctness of event handling, reconciliation, and other control logic. ### End-to-End Testing -1. Use Helm to create, deploy, and manage test clusters and environments that closely resemble the +1. Use Helm to create, deploy, and manage test clusters and environments that closely resemble the productive execution context. +2. Run regularly, but at least once per release, a performance test that measures product KPIs to indicate KPI violations or performance differences between release candidates. ### Testing Tools and Frameworks Use the following tools and frameworks to implement the above-mentioned testing levels: @@ -62,8 +65,8 @@ Use the following tools and frameworks to implement the above-mentioned testing |GoTest| X | | | |Kubebuilder Test Framework| X | X | | |EnvTest| X | X | | -|Ginkgo| | X | | -|Gomega| | X | | +|Ginkgo| X | X | | +|Gomega| X | X | | |K3d| | | X | |Helm| | | X | |K6| | | X | From 59fdae61bd39cfed854232929bf4d4dc8e039e16 Mon Sep 17 00:00:00 2001 From: Tobias Schuhmacher Date: Fri, 26 Jan 2024 14:41:10 +0100 Subject: [PATCH 08/27] extend E2E test description --- docs/contributor/testing-strategy.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index 41c0fc78..c977fe0e 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -47,7 +47,8 @@ For each layer is a dedicated testing approach used: ### End-to-End Testing 1. Use Helm to create, deploy, and manage test clusters and environments that closely resemble the productive execution context. -2. Run regularly, but at least once per release, a performance test that measures product KPIs to indicate KPI violations or performance differences between release candidates. +2. For shot living Kubernetes clusters, use K3d or other lightweight Kubernetes cluster providers. +3. Run regularly, but at least once per release, a performance test that measures product KPIs to indicate KPI violations or performance differences between release candidates. ### Testing Tools and Frameworks Use the following tools and frameworks to implement the above-mentioned testing levels: From d1443b4d402d84ff36a40e648a79ebe3aa33faad Mon Sep 17 00:00:00 2001 From: Tobias Schuhmacher Date: Fri, 26 Jan 2024 14:45:30 +0100 Subject: [PATCH 09/27] add a hint about execution frequency --- docs/contributor/testing-strategy.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index c977fe0e..e0f445f8 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -35,7 +35,8 @@ For each layer is a dedicated testing approach used: 3. Ensure tests cover various scenarios, edge cases, and possible failure scenarios. We try to verify business relevant logic with at least 65% code coverage. 4. Test for both positive and negative inputs to validate the expected behavior. 5. Mock external dependencies and use stubs or fakes to isolate the unit under test. -6. Run unit tests periodically during development and before each commit to prevent regressions. +6. Run unit tests periodically during development and before each PR to prevent regressions. +7. Unit tests have to be executed as fast as possible to minimize roundtrip times. Long running tests should be excluded from requently executed test runs and be triggered periodically (e.g. 4 times a day) ### Integration Testing 1. The PO together with the team create an registry of implemented business features and define for each feature a suitable test scenario. @@ -44,6 +45,7 @@ For each layer is a dedicated testing approach used: 4. Test the interaction and integration of your custom resources, controllers, and other components with the Kubernetes API. 5. Ensure test cases cover various aspects such as resource creation, updating, deletion, and handling of edge cases. 6. Validate the correctness of event handling, reconciliation, and other control logic. +7. Unit tests have to be executed as fast as possible to minimize roundtrip times and be applied for each PR. Long running tests should be excluded from requently executed test runs and be triggered periodically (e.g. 4 times a day) ### End-to-End Testing 1. Use Helm to create, deploy, and manage test clusters and environments that closely resemble the productive execution context. From 3d737693cb8895a6e25fef11c2db023007635ecb Mon Sep 17 00:00:00 2001 From: Tobias Schuhmacher Date: Fri, 26 Jan 2024 14:50:54 +0100 Subject: [PATCH 10/27] add execution table --- docs/contributor/testing-strategy.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index e0f445f8..2be31357 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -52,6 +52,12 @@ For each layer is a dedicated testing approach used: 2. For shot living Kubernetes clusters, use K3d or other lightweight Kubernetes cluster providers. 3. Run regularly, but at least once per release, a performance test that measures product KPIs to indicate KPI violations or performance differences between release candidates. +|Testing Approach|Per Commit|Per PR|Per Release|In intervals| +|--|--|--|--|--| +|Unit Testing|X|X||Only long running tests daily| +|Integration Testing||X||Only long running tests daily| +|End-to-End Testing|||X|Daily| + ### Testing Tools and Frameworks Use the following tools and frameworks to implement the above-mentioned testing levels: From 527b825f812a32b13e79dba27d53dea428bbb516 Mon Sep 17 00:00:00 2001 From: Tobias Schuhmacher Date: Fri, 2 Feb 2024 10:10:05 +0100 Subject: [PATCH 11/27] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Grzegorz Karaluch Co-authored-by: Michał 'Disper' Drzewiecki --- docs/contributor/testing-strategy.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index 2be31357..ca0d66ae 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -1,7 +1,7 @@ -# Testing Strategy Infrastructure-Manager +# Testing Strategy for Infrastructure Manager ## Introduction -This testing strategy describes how the Framefrog team is testing the product `Kyma Infrastructure Manager`. It outlines the approach and methodologies to be used for testing all layers of this product to ensure the stability, reliability, and correctness. +This testing strategy describes how the Framefrog team tests Kyma Infrastructure Manager. It outlines the approach and methodologies used for testing all layers of this product to ensure stability, reliability, and correctness. ## Testing Methodology @@ -20,7 +20,7 @@ We investigate the product by separating it into layers: Verifies how our product is integarted into the technical landscape, how it interacts with 3rd party systems and how it is accessible by customers or remote systems. -For each layer is a dedicated testing approach used: +For each layer, there is a dedicated testing approach used: 1. **Unit Testing for Business Logic:** Writing and executing tests for individual functions, methods, and components to verify their behavior and correctness in isolation. 2. **Integration Testing for Business Features:** Validating the integration and interaction between different components, modules, and services in the project. @@ -39,7 +39,7 @@ For each layer is a dedicated testing approach used: 7. Unit tests have to be executed as fast as possible to minimize roundtrip times. Long running tests should be excluded from requently executed test runs and be triggered periodically (e.g. 4 times a day) ### Integration Testing -1. The PO together with the team create an registry of implemented business features and define for each feature a suitable test scenario. +1. The PO and the team create a registry of implemented business features and define a suitable test scenario for each feature. 2. Create a separate test suite for integration testing. 3. Each test scenario is implemented in a separte test case. Use the Kubebuilder Test Framework and others to create test cases that interact with the Kubernetes cluster. 4. Test the interaction and integration of your custom resources, controllers, and other components with the Kubernetes API. @@ -49,7 +49,7 @@ For each layer is a dedicated testing approach used: ### End-to-End Testing 1. Use Helm to create, deploy, and manage test clusters and environments that closely resemble the productive execution context. -2. For shot living Kubernetes clusters, use K3d or other lightweight Kubernetes cluster providers. +2. For short-living Kubernetes clusters, use k3d or other lightweight Kubernetes cluster providers. 3. Run regularly, but at least once per release, a performance test that measures product KPIs to indicate KPI violations or performance differences between release candidates. |Testing Approach|Per Commit|Per PR|Per Release|In intervals| @@ -64,10 +64,9 @@ Use the following tools and frameworks to implement the above-mentioned testing - **GoTest**: For unit testing of Golang code. - **Kubebuilder Test Framework and EnvTest**: For creating and executing integration tests that interact with the Kubernetes API. - **Ginkgo and Gomega**: For writing and executing unit tests with a BDD-style syntax and assertions. -- **Kubebuilder Test Framework and EnvTest**: For creating and executing integration tests that interact with the Kubernetes API. -- **K3d**: For creating short-living and lightweight Kubernetes clustes running within a Docker context. +- **k3d**: For creating short-living and lightweight Kubernetes clusters running within a Docker context. - **Helm**: For deploying and managing test clusters and environments for end-to-end testing. -- **K6:**: For performance and stress testing +- **k6:**: For performance and stress testing |Framework|Unit Testing|Integration Testing|End-to-End Testing| |--|--|--|--| @@ -76,9 +75,9 @@ Use the following tools and frameworks to implement the above-mentioned testing |EnvTest| X | X | | |Ginkgo| X | X | | |Gomega| X | X | | -|K3d| | | X | +|k3d| | | X | |Helm| | | X | -|K6| | | X | +|k6| | | X | ## Test Automation From 9728b6e7aa8c955a6354da2f86d5b055a58f2a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Disper=27=20Drzewiecki?= Date: Mon, 5 Feb 2024 08:39:16 +0100 Subject: [PATCH 12/27] Update docs/contributor/testing-strategy.md Co-authored-by: Tobias Schuhmacher --- docs/contributor/testing-strategy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index ca0d66ae..7431bcc2 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -8,7 +8,7 @@ This testing strategy describes how the Framefrog team tests Kyma Infrastructure We investigate the product by separating it into layers: -1. Business Logic +1. Code Includes the technical frameworks (e.g. Kubebuilder) and custom Golang code. From 2dfa87b28841b7c5bffe8246009bcc6a7cbdf0bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Disper=27=20Drzewiecki?= Date: Mon, 5 Feb 2024 08:39:30 +0100 Subject: [PATCH 13/27] Update docs/contributor/testing-strategy.md Co-authored-by: Tobias Schuhmacher --- docs/contributor/testing-strategy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index 7431bcc2..6faffb9b 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -45,7 +45,7 @@ For each layer, there is a dedicated testing approach used: 4. Test the interaction and integration of your custom resources, controllers, and other components with the Kubernetes API. 5. Ensure test cases cover various aspects such as resource creation, updating, deletion, and handling of edge cases. 6. Validate the correctness of event handling, reconciliation, and other control logic. -7. Unit tests have to be executed as fast as possible to minimize roundtrip times and be applied for each PR. Long running tests should be excluded from requently executed test runs and be triggered periodically (e.g. 4 times a day) +7. Integration tests have to be executed fast to minimize roundtrip times and be applied for each PR. Long-running tests should be excluded from frequently executed test runs and be triggered periodically (e.g. 4 times a day) ### End-to-End Testing 1. Use Helm to create, deploy, and manage test clusters and environments that closely resemble the productive execution context. From c1dd3325f169aa52ac4f025654fe7dc5aa707e26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Disper=27=20Drzewiecki?= Date: Mon, 5 Feb 2024 08:39:38 +0100 Subject: [PATCH 14/27] Update docs/contributor/testing-strategy.md Co-authored-by: Tobias Schuhmacher --- docs/contributor/testing-strategy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index 6faffb9b..884f576f 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -48,7 +48,7 @@ For each layer, there is a dedicated testing approach used: 7. Integration tests have to be executed fast to minimize roundtrip times and be applied for each PR. Long-running tests should be excluded from frequently executed test runs and be triggered periodically (e.g. 4 times a day) ### End-to-End Testing -1. Use Helm to create, deploy, and manage test clusters and environments that closely resemble the productive execution context. +1. Use a mainstream Kubernetes management tool (e.g. [Helm](https://helm.sh/) or [Kustomize](https://kustomize.io/)) to create, deploy, and manage test clusters and environments that closely resemble the productive execution context. 2. For short-living Kubernetes clusters, use k3d or other lightweight Kubernetes cluster providers. 3. Run regularly, but at least once per release, a performance test that measures product KPIs to indicate KPI violations or performance differences between release candidates. From 90bd26623c2f43002873b532049491303cf5c2e4 Mon Sep 17 00:00:00 2001 From: Grzegorz Karaluch Date: Mon, 5 Feb 2024 08:50:47 +0100 Subject: [PATCH 15/27] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michał 'Disper' Drzewiecki --- docs/contributor/testing-strategy.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index 884f576f..6b6598f7 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -14,7 +14,7 @@ We investigate the product by separating it into layers: 2. Business Features - Combines the business logic into feature which is consumed by our customers. + Combines the code into feature which is consumed by our customers. 3. Product Integration @@ -22,7 +22,7 @@ We investigate the product by separating it into layers: For each layer, there is a dedicated testing approach used: -1. **Unit Testing for Business Logic:** Writing and executing tests for individual functions, methods, and components to verify their behavior and correctness in isolation. +1. **Unit Testing for Code:** Writing and executing tests for individual functions, methods, and components to verify their behavior and correctness in isolation. 2. **Integration Testing for Business Features:** Validating the integration and interaction between different components, modules, and services in the project. 3. **End-to-End Testing:** Testing the application as a whole in a production-like environment, mimicking real-world scenarios to ensure the entire system functions correctly, is performing well and secure. From eb67b448a25a78fc2ce7d588b43e9c7415fbe2f9 Mon Sep 17 00:00:00 2001 From: grego952 Date: Tue, 20 Feb 2024 07:56:53 +0100 Subject: [PATCH 16/27] Apply suggestions from tobiscr review --- docs/contributor/testing-strategy.md | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index 6b6598f7..a8aba208 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -1,7 +1,12 @@ # Testing Strategy for Infrastructure Manager ## Introduction -This testing strategy describes how the Framefrog team tests Kyma Infrastructure Manager. It outlines the approach and methodologies used for testing all layers of this product to ensure stability, reliability, and correctness. +This testing strategy describes how the Framefrog team tests the Kyma Infrastructure Manager product. It outlines the approach and methodologies used for testing all layers of this product to ensure the following: + +* Stability by applying load and performance tests that verify the product resilience under load peaks. +* Reliability by running chaos tests and operational awareness workshops to ensure product reliability in a productive context. +* Functional correctness by documenting and tracing all product features, their acceptance criteria and the used end-2-end tests to verify a correct implementation. +* Maintainability by regularly measuring the code quality metrics like reusability, modularity and, package qualities. ## Testing Methodology @@ -61,15 +66,19 @@ For each layer, there is a dedicated testing approach used: ### Testing Tools and Frameworks Use the following tools and frameworks to implement the above-mentioned testing levels: +- **[golanci-lint](https://github.com/golangci/golangci-lint)**: Golang code linting for better code quality. +- **[go-critic](https://github.com/go-critic/go-critic)**: Another linter for measuring different code quality metrics. - **GoTest**: For unit testing of Golang code. -- **Kubebuilder Test Framework and EnvTest**: For creating and executing integration tests that interact with the Kubernetes API. -- **Ginkgo and Gomega**: For writing and executing unit tests with a BDD-style syntax and assertions. -- **k3d**: For creating short-living and lightweight Kubernetes clusters running within a Docker context. -- **Helm**: For deploying and managing test clusters and environments for end-to-end testing. -- **k6:**: For performance and stress testing +- **Kubebuilder Test Framework and [EnvTest](https://book.kubebuilder.io/reference/envtest.html)**: For creating and executing integration tests that interact with the Kubernetes API. +- **[Ginkgo](https://github.com/onsi/ginkgo) and [Gomega](https://github.com/onsi/gomega)**: For writing and executing unit tests with a BDD-style syntax and assertions. +- **[k3d](https://k3d.io/)**: For creating short-living and lightweight Kubernetes clusters running within a Docker context. +- **[Helm](https://helm.sh/)**: For deploying and managing test clusters and environments for end-to-end testing. +- **[k6](https://k6.io/):**: For performance and stress testing. |Framework|Unit Testing|Integration Testing|End-to-End Testing| |--|--|--|--| +|Golangci-lint| X | | | +|Go-critic| X | | | |GoTest| X | | | |Kubebuilder Test Framework| X | X | | |EnvTest| X | X | | From ae84be222d988ee61cdb99b62a10908e5fe7f2c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Tue, 20 Feb 2024 10:27:36 +0100 Subject: [PATCH 17/27] upgrades golangci-lint to v4 --- .github/workflows/golangci-lint.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index 766fca07..19ca4296 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -21,12 +21,12 @@ jobs: go-version: '1.21' cache: false - name: golangci-lint - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v4 with: # Require: The version of golangci-lint to use. # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version. # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit. - version: v1.54.2 + version: v1.56.2 # Optional: working directory, useful for monorepos # working-directory: somedir From 94bc3d0c503824494ac667fb4651b142702b0e07 Mon Sep 17 00:00:00 2001 From: Tobias Schuhmacher Date: Tue, 20 Feb 2024 15:39:21 +0100 Subject: [PATCH 18/27] Apply suggestions from code review --- docs/contributor/testing-strategy.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index a8aba208..2ede225e 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -77,6 +77,8 @@ Use the following tools and frameworks to implement the above-mentioned testing |Framework|Unit Testing|Integration Testing|End-to-End Testing| |--|--|--|--| +|Golangci-lint| X | | | +|Go-critic| X | | | |Golangci-lint| X | | | |Go-critic| X | | | |GoTest| X | | | From 0ac801c3f5423e30f783281bd78fa795ac64ed49 Mon Sep 17 00:00:00 2001 From: Tobias Schuhmacher Date: Tue, 20 Feb 2024 15:40:37 +0100 Subject: [PATCH 19/27] Update docs/contributor/testing-strategy.md --- docs/contributor/testing-strategy.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index 2ede225e..a8aba208 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -77,8 +77,6 @@ Use the following tools and frameworks to implement the above-mentioned testing |Framework|Unit Testing|Integration Testing|End-to-End Testing| |--|--|--|--| -|Golangci-lint| X | | | -|Go-critic| X | | | |Golangci-lint| X | | | |Go-critic| X | | | |GoTest| X | | | From c0ff1d4181b12f2f4f8fe2540e19f0e6abfe52d4 Mon Sep 17 00:00:00 2001 From: Grzegorz Karaluch Date: Wed, 21 Feb 2024 11:19:30 +0100 Subject: [PATCH 20/27] Apply suggestions from IwonaLanger review Co-authored-by: Iwona Langer --- docs/contributor/testing-strategy.md | 38 ++++++++++++---------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index a8aba208..5cdcce9f 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -6,61 +6,55 @@ This testing strategy describes how the Framefrog team tests the Kyma Infrastruc * Stability by applying load and performance tests that verify the product resilience under load peaks. * Reliability by running chaos tests and operational awareness workshops to ensure product reliability in a productive context. * Functional correctness by documenting and tracing all product features, their acceptance criteria and the used end-2-end tests to verify a correct implementation. -* Maintainability by regularly measuring the code quality metrics like reusability, modularity and, package qualities. +* Maintainability by regularly measuring code quality metrics like reusability, modularity, and package qualities. ## Testing Methodology We investigate the product by separating it into layers: -1. Code +1. Code: Includes the technical frameworks (e.g. Kubebuilder) and custom Golang code. - Includes the technical frameworks (e.g. Kubebuilder) and custom Golang code. +2. Business Features: Combines the code into a feature our customers consume. -2. Business Features - - Combines the code into feature which is consumed by our customers. - -3. Product Integration - - Verifies how our product is integarted into the technical landscape, how it interacts with 3rd party systems and how it is accessible by customers or remote systems. +3. Product Integration: Verifies how our product is integrated into the technical landscape, how it interacts with 3rd party systems, and how it is accessible by customers or remote systems. -For each layer, there is a dedicated testing approach used: +For each layer, a dedicated testing approach is used: 1. **Unit Testing for Code:** Writing and executing tests for individual functions, methods, and components to verify their behavior and correctness in isolation. 2. **Integration Testing for Business Features:** Validating the integration and interaction between different components, modules, and services in the project. -3. **End-to-End Testing:** Testing the application as a whole in a production-like environment, mimicking real-world scenarios to ensure the entire system functions correctly, is performing well and secure. +3. **End-to-End Testing:** Testing the application as a whole in a production-like environment, mimicking real-world scenarios to ensure the entire system is secure, functions correctly, and performs well. ## Testing Approach ### Unit Testing 1. Identify critical functions, methods, and components that require testing. -2. Write unit tests using GoUnit tests, Ginkgo and Gomega frameworks. +2. Write unit tests using GoUnit tests, Ginkgo, and Gomega frameworks. 3. Ensure tests cover various scenarios, edge cases, and possible failure scenarios. We try to verify business relevant logic with at least 65% code coverage. 4. Test for both positive and negative inputs to validate the expected behavior. 5. Mock external dependencies and use stubs or fakes to isolate the unit under test. -6. Run unit tests periodically during development and before each PR to prevent regressions. -7. Unit tests have to be executed as fast as possible to minimize roundtrip times. Long running tests should be excluded from requently executed test runs and be triggered periodically (e.g. 4 times a day) +6. Run unit tests periodically during development and before each PR is merged to prevent regressions. +7. Unit tests must be executed as fast as possible to minimize roundtrip times. Long-running tests should be excluded from frequently executed test runs and be triggered periodically, for example, 4 times a day. ### Integration Testing 1. The PO and the team create a registry of implemented business features and define a suitable test scenario for each feature. 2. Create a separate test suite for integration testing. -3. Each test scenario is implemented in a separte test case. Use the Kubebuilder Test Framework and others to create test cases that interact with the Kubernetes cluster. +3. Each test scenario is implemented in a separate test case. Use the Kubebuilder Test Framework and others to create test cases that interact with the Kubernetes cluster. 4. Test the interaction and integration of your custom resources, controllers, and other components with the Kubernetes API. 5. Ensure test cases cover various aspects such as resource creation, updating, deletion, and handling of edge cases. 6. Validate the correctness of event handling, reconciliation, and other control logic. -7. Integration tests have to be executed fast to minimize roundtrip times and be applied for each PR. Long-running tests should be excluded from frequently executed test runs and be triggered periodically (e.g. 4 times a day) +7. Integration tests must be executed fast to minimize roundtrip times and be applied for each PR. Long-running tests should be excluded from frequently executed test runs and be triggered periodically, for example, 4 times a day. ### End-to-End Testing -1. Use a mainstream Kubernetes management tool (e.g. [Helm](https://helm.sh/) or [Kustomize](https://kustomize.io/)) to create, deploy, and manage test clusters and environments that closely resemble the productive execution context. +1. Use a mainstream Kubernetes management tool (for example, [Helm](https://helm.sh/) or [Kustomize](https://kustomize.io/)) to create, deploy, and manage test clusters and environments that closely resemble the productive execution context. 2. For short-living Kubernetes clusters, use k3d or other lightweight Kubernetes cluster providers. 3. Run regularly, but at least once per release, a performance test that measures product KPIs to indicate KPI violations or performance differences between release candidates. |Testing Approach|Per Commit|Per PR|Per Release|In intervals| |--|--|--|--|--| -|Unit Testing|X|X||Only long running tests daily| -|Integration Testing||X||Only long running tests daily| +|Unit Testing|X|X||Only long-running tests daily| +|Integration Testing||X||Only long-running tests daily| |End-to-End Testing|||X|Daily| ### Testing Tools and Frameworks @@ -73,7 +67,7 @@ Use the following tools and frameworks to implement the above-mentioned testing - **[Ginkgo](https://github.com/onsi/ginkgo) and [Gomega](https://github.com/onsi/gomega)**: For writing and executing unit tests with a BDD-style syntax and assertions. - **[k3d](https://k3d.io/)**: For creating short-living and lightweight Kubernetes clusters running within a Docker context. - **[Helm](https://helm.sh/)**: For deploying and managing test clusters and environments for end-to-end testing. -- **[k6](https://k6.io/):**: For performance and stress testing. +- **[k6](https://k6.io/)**: For performance and stress testing. |Framework|Unit Testing|Integration Testing|End-to-End Testing| |--|--|--|--| @@ -91,7 +85,7 @@ Use the following tools and frameworks to implement the above-mentioned testing ## Test Automation -The following CI/CD jobs are a part of the development cycle and executing quality assurance related steps: +The following CI/CD jobs are a part of the development cycle and execute quality assurance-related steps: > **NOTE:** Jobs marked with `pull_request` are triggered with each pull request. Jobs marked with `push` are executed after the merge. From 807e7f14d48e3f87b43bf51a70501f5f2b42b21d Mon Sep 17 00:00:00 2001 From: grego952 Date: Wed, 21 Feb 2024 11:55:40 +0100 Subject: [PATCH 21/27] Apply remaining suggestions from IwonaLanger review --- docs/contributor/testing-strategy.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index 5cdcce9f..648d3571 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -11,7 +11,7 @@ This testing strategy describes how the Framefrog team tests the Kyma Infrastruc ## Testing Methodology -We investigate the product by separating it into layers: +We investigate the product by dividing it into layers: 1. Code: Includes the technical frameworks (e.g. Kubebuilder) and custom Golang code. @@ -29,6 +29,9 @@ For each layer, a dedicated testing approach is used: ## Testing Approach ### Unit Testing + +This section focuses on ensuring the functionality and reliability of critical functions, methods, and components within our system. + 1. Identify critical functions, methods, and components that require testing. 2. Write unit tests using GoUnit tests, Ginkgo, and Gomega frameworks. 3. Ensure tests cover various scenarios, edge cases, and possible failure scenarios. We try to verify business relevant logic with at least 65% code coverage. @@ -38,6 +41,9 @@ For each layer, a dedicated testing approach is used: 7. Unit tests must be executed as fast as possible to minimize roundtrip times. Long-running tests should be excluded from frequently executed test runs and be triggered periodically, for example, 4 times a day. ### Integration Testing + +This section focuses on the integration testing process, which involves testing the interaction and integration of various components and custom resources with the Kubernetes API. It provides you with a step-by-step guide to conduct integration testing, ensuring the correctness and functionality of the implemented business features. + 1. The PO and the team create a registry of implemented business features and define a suitable test scenario for each feature. 2. Create a separate test suite for integration testing. 3. Each test scenario is implemented in a separate test case. Use the Kubebuilder Test Framework and others to create test cases that interact with the Kubernetes cluster. @@ -47,6 +53,9 @@ For each layer, a dedicated testing approach is used: 7. Integration tests must be executed fast to minimize roundtrip times and be applied for each PR. Long-running tests should be excluded from frequently executed test runs and be triggered periodically, for example, 4 times a day. ### End-to-End Testing + +This section describes how to create and manage test clusters using mainstream Kubernetes management tools like Helm or Kustomize, and how to perform regular performance tests to ensure your application functions correctly and meets the KPIs in a production-like environment. + 1. Use a mainstream Kubernetes management tool (for example, [Helm](https://helm.sh/) or [Kustomize](https://kustomize.io/)) to create, deploy, and manage test clusters and environments that closely resemble the productive execution context. 2. For short-living Kubernetes clusters, use k3d or other lightweight Kubernetes cluster providers. 3. Run regularly, but at least once per release, a performance test that measures product KPIs to indicate KPI violations or performance differences between release candidates. @@ -62,7 +71,7 @@ Use the following tools and frameworks to implement the above-mentioned testing - **[golanci-lint](https://github.com/golangci/golangci-lint)**: Golang code linting for better code quality. - **[go-critic](https://github.com/go-critic/go-critic)**: Another linter for measuring different code quality metrics. -- **GoTest**: For unit testing of Golang code. +- **[go test](https://pkg.go.dev/testing)**: For unit testing of Golang code. - **Kubebuilder Test Framework and [EnvTest](https://book.kubebuilder.io/reference/envtest.html)**: For creating and executing integration tests that interact with the Kubernetes API. - **[Ginkgo](https://github.com/onsi/ginkgo) and [Gomega](https://github.com/onsi/gomega)**: For writing and executing unit tests with a BDD-style syntax and assertions. - **[k3d](https://k3d.io/)**: For creating short-living and lightweight Kubernetes clusters running within a Docker context. @@ -89,9 +98,9 @@ The following CI/CD jobs are a part of the development cycle and execute quality > **NOTE:** Jobs marked with `pull_request` are triggered with each pull request. Jobs marked with `push` are executed after the merge. -- `golangci-lint / lint (pull_request/push)` - Is responsible for linting and static code analysis. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.golangci.yaml) and [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.github/workflows/golangci-lint.yaml). -- `PR Markdown Link Check / markdown-link-check (pull_request)` - Checks if there are no broken links in the pull request `.md` files. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/mlc.config.json). -- `Run unit tests / validate (pull_request/push)` - Executes basic create/update/delete functional tests of the reconciliation logic. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.github/workflows/run-tests.yaml). -- `Run vuln check / test (pull_request/push)` - Runs [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) on the code to detect known vulnerabilities. It's configured [here](https://github.com/kyma-project/infrastructure-manager/blob/main/.github/workflows/run-vuln-check.yaml). -- `pull-infrastructure-manager-build` - Triggered with each pull request. It builds the Docker image and pushes it to the registry. It's configured [here](https://github.com/kyma-project/test-infra/blob/a3c2a07da4ba42e468f69cf42f1960d7bfcc3fff/prow/jobs/kyma-project/infrastructure-manager/infrastructure-manager.yaml). -- `main-infrastructure-manager-build` - Triggered after the merge. Rebuilds the image and pushes it to the registry. It's configured [here](https://github.com/kyma-project/test-infra/blob/a3c2a07da4ba42e468f69cf42f1960d7bfcc3fff/prow/jobs/kyma-project/infrastructure-manager/infrastructure-manager.yaml). \ No newline at end of file +- `golangci-lint / lint (pull_request/push)` - Is responsible for linting and static code analysis. For the configuration details, see [`golangci.yaml`](https://github.com/kyma-project/infrastructure-manager/blob/main/.golangci.yaml) and [`golangci-lint.yaml`](https://github.com/kyma-project/infrastructure-manager/blob/main/.github/workflows/golangci-lint.yaml). +- `PR Markdown Link Check / markdown-link-check (pull_request)` - Checks if there are no broken links in the pull request `.md` files. For the configuration details, see [`mlc.config.json`](https://github.com/kyma-project/infrastructure-manager/blob/main/mlc.config.json). +- `Run unit tests / validate (pull_request/push)` - Executes basic create/update/delete functional tests of the reconciliation logic. For the configuration details, see [`run-tests.yaml`](https://github.com/kyma-project/infrastructure-manager/blob/main/.github/workflows/run-tests.yaml). +- `Run vuln check / test (pull_request/push)` - Runs [govulncheck](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck) on the code to detect known vulnerabilities. For the configuration details, see [`run-vuln-check.yaml`](https://github.com/kyma-project/infrastructure-manager/blob/main/.github/workflows/run-vuln-check.yaml). +- `pull-infrastructure-manager-build` - Triggered with each pull request. It builds the Docker image and pushes it to the registry. For the configuration details, see [`infrastructure-manager.yaml`](https://github.com/kyma-project/test-infra/blob/a3c2a07da4ba42e468f69cf42f1960d7bfcc3fff/prow/jobs/kyma-project/infrastructure-manager/infrastructure-manager.yaml). +- `main-infrastructure-manager-build` - Triggered after the merge. Rebuilds the image and pushes it to the registry. For the configuration details, see [`infrastructure-manager.yaml`](https://github.com/kyma-project/test-infra/blob/a3c2a07da4ba42e468f69cf42f1960d7bfcc3fff/prow/jobs/kyma-project/infrastructure-manager/infrastructure-manager.yaml). \ No newline at end of file From 14ac3d49fae08690b5fe4fad2db78d2c39bc7844 Mon Sep 17 00:00:00 2001 From: grego952 Date: Wed, 21 Feb 2024 11:57:15 +0100 Subject: [PATCH 22/27] Change GoTest to go test --- docs/contributor/testing-strategy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index 648d3571..9ba3fb7c 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -82,7 +82,7 @@ Use the following tools and frameworks to implement the above-mentioned testing |--|--|--|--| |Golangci-lint| X | | | |Go-critic| X | | | -|GoTest| X | | | +|go test| X | | | |Kubebuilder Test Framework| X | X | | |EnvTest| X | X | | |Ginkgo| X | X | | From 659195ff1697da715c314ee40233d6c434dbebd4 Mon Sep 17 00:00:00 2001 From: grego952 Date: Thu, 22 Feb 2024 10:12:20 +0100 Subject: [PATCH 23/27] Change ordered list to unordered one --- docs/contributor/testing-strategy.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/contributor/testing-strategy.md b/docs/contributor/testing-strategy.md index 9ba3fb7c..db5eb0ce 100644 --- a/docs/contributor/testing-strategy.md +++ b/docs/contributor/testing-strategy.md @@ -13,17 +13,15 @@ This testing strategy describes how the Framefrog team tests the Kyma Infrastruc We investigate the product by dividing it into layers: -1. Code: Includes the technical frameworks (e.g. Kubebuilder) and custom Golang code. - -2. Business Features: Combines the code into a feature our customers consume. - -3. Product Integration: Verifies how our product is integrated into the technical landscape, how it interacts with 3rd party systems, and how it is accessible by customers or remote systems. +* Code: Includes the technical frameworks (e.g. Kubebuilder) and custom Golang code. +* Business Features: Combines the code into a feature our customers consume. +* Product Integration: Verifies how our product is integrated into the technical landscape, how it interacts with 3rd party systems, and how it is accessible by customers or remote systems. For each layer, a dedicated testing approach is used: -1. **Unit Testing for Code:** Writing and executing tests for individual functions, methods, and components to verify their behavior and correctness in isolation. -2. **Integration Testing for Business Features:** Validating the integration and interaction between different components, modules, and services in the project. -3. **End-to-End Testing:** Testing the application as a whole in a production-like environment, mimicking real-world scenarios to ensure the entire system is secure, functions correctly, and performs well. +* **Unit Testing for Code:** Writing and executing tests for individual functions, methods, and components to verify their behavior and correctness in isolation. +* **Integration Testing for Business Features:** Validating the integration and interaction between different components, modules, and services in the project. +* **End-to-End Testing:** Testing the application as a whole in a production-like environment, mimicking real-world scenarios to ensure the entire system is secure, functions correctly, and performs well. ## Testing Approach From 960cb9b8d95d41fe6355341b6081a0bd7c3c6573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20=27Disper=27=20Drzewiecki?= Date: Mon, 26 Feb 2024 12:16:33 +0100 Subject: [PATCH 24/27] Update golangci-lint.yaml 4.0.0 --- .github/workflows/golangci-lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index 19ca4296..b481ea27 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -21,7 +21,7 @@ jobs: go-version: '1.21' cache: false - name: golangci-lint - uses: golangci/golangci-lint-action@v4 + uses: golangci/golangci-lint-action@v4.0.0 with: # Require: The version of golangci-lint to use. # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version. From 108228c275901bb9c3f40820aec4b783ee1023d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Mon, 26 Feb 2024 12:24:49 +0100 Subject: [PATCH 25/27] linter --- internal/controller/find_last_sync_time_test.go | 4 ++-- internal/controller/gardener_cluster_controller_test.go | 4 ++-- internal/controller/next_requeue_test.go | 4 ++-- internal/controller/suite_test.go | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/controller/find_last_sync_time_test.go b/internal/controller/find_last_sync_time_test.go index 60c7af33..9d41e0c8 100644 --- a/internal/controller/find_last_sync_time_test.go +++ b/internal/controller/find_last_sync_time_test.go @@ -3,8 +3,8 @@ package controller import ( "time" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" + . "github.com/onsi/ginkgo/v2" //nolint:revive + . "github.com/onsi/gomega" //nolint:revive ) var _ = Describe("findLastSyncTime", func() { diff --git a/internal/controller/gardener_cluster_controller_test.go b/internal/controller/gardener_cluster_controller_test.go index f60ae6fd..854fd91a 100644 --- a/internal/controller/gardener_cluster_controller_test.go +++ b/internal/controller/gardener_cluster_controller_test.go @@ -5,8 +5,8 @@ import ( "time" imv1 "github.com/kyma-project/infrastructure-manager/api/v1" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" + . "github.com/onsi/ginkgo/v2" //nolint:revive + . "github.com/onsi/gomega" //nolint:revive corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/internal/controller/next_requeue_test.go b/internal/controller/next_requeue_test.go index fc496db6..711db27d 100644 --- a/internal/controller/next_requeue_test.go +++ b/internal/controller/next_requeue_test.go @@ -3,8 +3,8 @@ package controller import ( "time" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" + . "github.com/onsi/ginkgo/v2" //nolint:revive + . "github.com/onsi/gomega" //nolint:revive ) var _ = Describe("nextRequeueAfter", func() { diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index d4c08cf4..087152e6 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -24,10 +24,10 @@ import ( infrastructuremanagerv1 "github.com/kyma-project/infrastructure-manager/api/v1" "github.com/kyma-project/infrastructure-manager/internal/controller/mocks" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" + . "github.com/onsi/ginkgo/v2" //nolint:revive + . "github.com/onsi/gomega" //nolint:revive "github.com/pkg/errors" - . "github.com/stretchr/testify/mock" + . "github.com/stretchr/testify/mock" //nolint:revive "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" @@ -46,7 +46,7 @@ var ( testEnv *envtest.Environment //nolint:gochecknoglobals suiteCtx context.Context //nolint:gochecknoglobals cancelSuiteCtx context.CancelFunc //nolint:gochecknoglobals - anyContext = MatchedBy(func(ctx context.Context) bool { return true }) + anyContext = MatchedBy(func(_ context.Context) bool { return true }) ) const TestKubeconfigValidityTime = 24 * time.Hour From 83cec25142d74c716b03300bbc1c1be2077a29ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Mon, 26 Feb 2024 12:27:57 +0100 Subject: [PATCH 26/27] linter --- internal/controller/suite_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 087152e6..81f8a669 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -41,12 +41,12 @@ import ( // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. var ( - cfg *rest.Config //nolint:gochecknoglobals - k8sClient client.Client //nolint:gochecknoglobals - testEnv *envtest.Environment //nolint:gochecknoglobals - suiteCtx context.Context //nolint:gochecknoglobals - cancelSuiteCtx context.CancelFunc //nolint:gochecknoglobals - anyContext = MatchedBy(func(_ context.Context) bool { return true }) + cfg *rest.Config //nolint:gochecknoglobals + k8sClient client.Client //nolint:gochecknoglobals + testEnv *envtest.Environment //nolint:gochecknoglobals + suiteCtx context.Context //nolint:gochecknoglobals + cancelSuiteCtx context.CancelFunc //nolint:gochecknoglobals + anyContext = MatchedBy(func(_ context.Context) bool { return true }) //nolint:gochecknoglobals ) const TestKubeconfigValidityTime = 24 * time.Hour From 783f5996f1dd694f8563fd31f91c4fb31f1bf61f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Drzewiecki?= Date: Tue, 27 Feb 2024 11:43:46 +0100 Subject: [PATCH 27/27] checkout upgraded to v4 and setup-go upgraded to v5 --- .github/workflows/golangci-lint.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index b481ea27..7d29bffa 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -15,8 +15,8 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: '1.21' cache: false