diff --git a/.github/workflows/cli.unit.yml b/.github/workflows/cli.unit.yml new file mode 100644 index 000000000..197da5ae1 --- /dev/null +++ b/.github/workflows/cli.unit.yml @@ -0,0 +1,42 @@ + +name: CLI Unit Tests +on: + push: + paths: + - 'cli/**' + - 'workflows/cli.*.yml' + pull_request: + workflow_dispatch: + +defaults: + run: + shell: bash + working-directory: cli + +jobs: + unit_tests: + name: Run Unit Tests -- Python 3.${{ matrix.pyver }} + runs-on: ubuntu-latest + strategy: + matrix: + pyver: [8, 9] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.${{ matrix.pyver }} + - name: Install tox + run: pip install tox + - name: Install Kind Cluster + run: kind create cluster --name=pytest-kind --kubeconfig ./kubeconfig + - name: Run Tox + run: tox -e py3${{ matrix.pyver }}-unit -- --kind-bin=/usr/local/bin/kind --kubeconfig ./kubeconfig --keep-cluster + - name: Generate Coverage Report + if: matrix.pyver == 9 + run: tox -e coverage + - name: Upload Coverage Report + if: matrix.pyver == 9 + uses: codecov/codecov-action@v1 + with: + files: cli/coverage.xml + flags: unit,gha,python-3.${{ matrix.pyver }} \ No newline at end of file diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 000000000..2e7ed3299 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,88 @@ +name: Run e2e tests +on: + pull_request_target: + types: [labeled, synchronize] + push: + branches: + - master + workflow_dispatch: + inputs: + test_list: + description: Only run tests that match the regular expression + default: "" + img: + description: Benchmark-operator image location + default: quay.io/rht_perf_ci/benchmark-operator:e2e +concurrency: + group: e2e-ci +jobs: + e2e: + name: E2E testing + runs-on: ubuntu-latest + timeout-minutes: 120 + if: "github.event_name == 'push' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ok to test'))" + env: + BATS_TESTS: ${{ github.event.inputs.test_list }} + IMG: ${{ github.event.inputs.img }} + steps: + + - name: Set env vars # Workaround: When the workflow is triggered by events different from workflow_dispatch the IMG env var is not set + run: echo "IMG=quay.io/rht_perf_ci/benchmark-operator:e2e" >> $GITHUB_ENV + if: github.event_name != 'workflow_dispatch' + + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + if: github.event_name == 'pull_request_target' + + - name: Check out code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + if: "github.event_name == 'workflow_dispatch' || github.event_name == 'push'" + + - name: Install bats + run: make install-bats + + - name: Authenticate against OCP cluster + run: oc login -u ${OPENSHIFT_USER} -p ${OPENSHIFT_PASSWORD} ${OPENSHIFT_SERVER} --insecure-skip-tls-verify=true + env: + OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }} + OPENSHIFT_USER: ${{ secrets.OPENSHIFT_USER }} + OPENSHIFT_PASSWORD: ${{ secrets.OPENSHIFT_PASSWORD }} + + - name: Login in quay + run: podman login quay.io -u ${QUAY_USER} -p ${QUAY_TOKEN} + env: + QUAY_USER: ${{ secrets.QUAY_CI_USER }} + QUAY_TOKEN: ${{ secrets.QUAY_CI_TOKEN }} + + - name: Build, push and deploy benchmark-operator + run: make image-build image-push deploy + + - name: Ensure benchmark-operator is running + run: kubectl rollout status -n benchmark-operator --timeout=5m deploy/benchmark-controller-manager + + - name: Execute e2e tests + run: e2e/run_tests_ci.sh + env: + TERM: linux + + - name: Undeploy benchmark-operator + run: make undeploy + if: always() # always run even if the previous step fails + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + if: failure() + with: + name: benchmark-operator-debug-${{ github.run_id }} + path: e2e/artifacts + + - name: Publish Test Report + uses: mikepenz/action-junit-report@v2.4.3 + if: always() + with: + report_paths: e2e/report.xml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..72a1c5753 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,60 @@ +name: Release benchmark-operator container images +on: + push: + branches: + - master + tags: + - "*" # triggers only if push new tag version + +jobs: + containers: + name: Build container images + runs-on: ubuntu-latest + strategy: + matrix: + arch: + - arm64 + - amd64 + - ppc64le + env: + ORG: cloud-bulldozer + IMAGE_ARCH: ${{ matrix.arch }} + steps: + + - name: Install dependencies required for multi-arch builds + run: sudo apt-get update && sudo apt-get install qemu-user-static podman fuse-overlayfs + + - name: Check out code + uses: actions/checkout@v2 + + - name: Login in quay + run: podman login quay.io -u ${QUAY_USER} -p ${QUAY_TOKEN} + env: + QUAY_USER: ${{ secrets.QUAY_USER }} + QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} + + - name: Build benchmark-operator container + run: make image-build + + - name: Push benchmark-operator container + run: make image-push + + manifest: + name: Build container manifest + runs-on: ubuntu-latest + env: + MANIFEST_ARCHS: amd64 arm64 ppc64le + needs: containers + steps: + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Login in quay + run: podman login quay.io -u ${QUAY_USER} -p ${QUAY_TOKEN} + env: + QUAY_USER: ${{ secrets.QUAY_USER }} + QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} + + - name: Build and push container manifest + run: make manifest diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 index afd4228f9..fa0908090 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,160 @@ -# Ignore +# Binaries for programs and plugins +*.dll +*.so +*.dylib +bin + +# editor and IDE paraphernalia +.idea *.swp +*.swo +*~ + +.kube +.pytest-kind +.kind + + + +### Pulled from https://github.com/github/gitignore/blob/master/Python.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +cython_debug/ +.cache +e2e/report.xml +e2e/artifacts diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..8762bb269 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,75 @@ +--- +files: ^cli/ +repos: + - repo: git://github.com/Lucas-C/pre-commit-hooks + rev: v1.1.10 + hooks: + - id: remove-tabs + + - repo: git://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: trailing-whitespace + - id: check-merge-conflict + - id: end-of-file-fixer + - id: check-case-conflict + - id: detect-private-key + - id: check-ast + + - repo: https://github.com/psf/black + rev: 21.7b0 + hooks: + - id: black + args: ['--line-length', '110', 'cli'] + + - repo: https://github.com/pycqa/isort + rev: 5.9.2 + hooks: + - id: isort + args: + - --profile=black + - --color + - --line-length=110 + + - repo: https://gitlab.com/PyCQA/flake8 + rev: '3.9.2' + hooks: + - id: flake8 + args: ['--config', 'cli/.flake8'] + additional_dependencies: ['pep8-naming', 'flake8-docstrings'] + + - repo: https://github.com/PyCQA/pylint + rev: v2.9.5 + hooks: + - id: pylint + args: ['--rcfile', 'cli/.pylintrc'] + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.910 + hooks: + - id: mypy + args: + - --ignore-missing-imports + - --install-types + - --non-interactive + + - repo: https://github.com/asottile/pyupgrade + rev: v2.21.2 + hooks: + - id: pyupgrade + + - repo: https://github.com/PyCQA/bandit + rev: 1.7.0 + hooks: + - id: bandit + args: ["--exclude", "cli/tests/", "--skip", "B404,B603"] + +ci: + autofix_commit_msg: | + [pre-commit.ci] auto fixes from pre-commit.com hooks + for more information, see https://pre-commit.ci + autofix_prs: true + autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' + autoupdate_schedule: weekly + skip: [] + submodules: false \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 594e194b3..000000000 --- a/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -sudo: required -services: docker -language: python -install: - - pip install docker molecule openshift -script: - - molecule test -s test-local diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index faeee6d78..650b4254d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,14 +37,6 @@ In the interactive rebase screen, set the first commit to `pick` and all others Push your rebased commits (you may need to force), then issue your PR. -## Container Images - -Custom container image definitions are maintained in [magazine](https://github.com/cloud-bulldozer/magazine). -We use quay for all storing all our custom container images, and if you're adding a new -workload and not sure of where to add/maintain the container image. We highly recommend, to -add the Dockerfile to magazine, as we've automation setup for updating images in Quay, when -a git push happens to magazine. - ## Add workload Adding new workloads are always welcome, but before you submit PR: @@ -79,7 +71,7 @@ Example `playbook.yml`: ### Workload triggers [CRD](https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/) holds the definition of the resource. -The operator triggers roles based on the conditions defined in a CR ([example](resources/crds/ripsaw_v1alpha1_uperf_cr.yaml)) which will influence which roles the +The operator triggers roles based on the conditions defined in a CR ([example](config/samples/uperf/cr.yaml) which will influence which roles the [playbook](playbook.yml) executes. Other vars may be defined that can modify the workload run conditions. @@ -91,7 +83,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: your_workload_name @@ -103,16 +95,16 @@ spec: my_key_3: my_value_3 ``` -Note: The Benchmark has to be created in the namespace `my-ripsaw` +Note: The Benchmark has to be created in the namespace `benchmark-operator` ### Additional guidance for adding a workload * Please keep the [workload status](README.md#workloads-status) updated * To help users understand how the workload can be run, please add a guide similar to [uperf](docs/uperf.md) * Add the link for your workload guide to [installation guide](docs/installation.md#running-workloads) -* Ensure all resources created are within the `my-ripsaw` namespace, this can be done by setting namespace +* Ensure all resources created are within the `benchmark-operator` namespace, this can be done by setting namespace to use `operator_namespace` var. This is to ensure that the resources aren't defaulted to current active -namespace which is what `meta.namespace` would default to. +namespace which is what `ansible_operator_meta.namespace` would default to. * All resources created as part of your role should use `trunc_uuid` ansible var in their names and labels, so for example [fio-client job template](roles/fio-distributed/templates/client.yaml) has the name `fio-client` and label `app: fiod-client`, instead we'll append the var `trunc_uuid` to both the name and label so it'll be `fio-client-{{ trunc_uuid }}` and label would be `app:fiod-client-{{ trunc_uuid }}`. The reason for this @@ -129,40 +121,15 @@ case of failure or when disabled. This ensures no interference with subsequent w ### The operator container image Any changes to the [roles](roles/) tree or to the [playbook](playbook.yml) file will necessitate a new build of the operator container image. -The container is built using the [Operator SDK](https://github.com/operator-framework/operator-sdk) and pushed to a public repository. -The public repository could be [quay](https://quay.io) in which case you'll need to: - -```bash -$ operator-sdk build quay.io//benchmark-operator:testing --image-builder podman -$ podman push quay.io//benchmark-operator:testing -``` - -Note: you can also use docker, and no, we'll not judge ;) - -`:testing` is simply a tag. You can define different tags to use with your image, like `:latest` or `:master` - -To test with your own operator image, you will need the [operator](resources/operator.yml) file to point the container image to your testing version. -Be sure to do this outside of your git tree to avoid mangling the official file that points to our stable image. +The benchmark-operator container can be built and pushed to a public repository, which could be [quay](https://quay.io), using the provided Makefile. -This can be done as follows: +To test with your own operator image, you will need to delete the current deployment and then build, push, and redeploy using your operator container image as follows: ```bash -$ sed 's/image:.*/image: quay.io\/\/benchmark-operator:testing/' resources/operator.yaml > /my/testing/operator.yaml -``` - -You can then redeploy operator -```bash -# kubectl delete -f resources/operator.yaml -# kubectl apply -f /my/testing/operator.yaml -``` -Redefine CRD -```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -``` -Apply a new CR -```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_uperf_cr.yaml +# kubectl delete deployment -n benchmark-operator benchmark-controller-manager +# make image-build image-push deploy IMG=quay.io//benchmark-operator:testing ``` +`:testing` is simply a tag. You can define different tags to use with your image, like `:latest` or `:master` ## CI Currently we have a CI that runs against PRs. @@ -173,7 +140,7 @@ To ensure that adding new a workload will not break other workloads and its behavior can be predicted, we've mandated writing tests before PR can be merged. If a new workload is added, please follow the instructions to add a testcase to -[test.sh](test,sh): +[test.sh](test.sh): * copy an existing test like [uperf test](tests/test_uperf.sh) * Add commands needed to setup the workload specific requirements if any * Add a valid cr file to [test_crs](tests/test_crs/) directory for your workload diff --git a/build/Dockerfile b/Dockerfile similarity index 63% rename from build/Dockerfile rename to Dockerfile index a32b21982..046fda623 100644 --- a/build/Dockerfile +++ b/Dockerfile @@ -1,18 +1,18 @@ -FROM quay.io/operator-framework/ansible-operator:v0.17.1 +FROM quay.io/operator-framework/ansible-operator:v1.32.0 USER root COPY requirements.yml ${HOME}/requirements.yml +RUN python3 -m pip install --no-cache-dir jmespath +RUN ansible-galaxy collection install community.general +RUN ansible-galaxy collection list RUN ansible-galaxy collection install -r ${HOME}/requirements.yml \ && chmod -R ug+rwx ${HOME}/.ansible COPY image_resources/centos8-appstream.repo /etc/yum.repos.d/centos8-appstream.repo -RUN dnf install -y --nodocs redis openssl --enablerepo=centos8-appstream && dnf clean all +RUN dnf install -y --nodocs redis openssl --enablerepo=centos8-appstream-* && dnf clean all COPY resources/kernel-cache-drop-daemonset.yaml /opt/kernel_cache_dropper/ -COPY group_vars/ ${HOME}/group_vars/ -COPY roles/ ${HOME}/roles/ -COPY templates/ ${HOME}/templates/ -COPY meta/ ${HOME}/meta/ COPY watches.yaml ${HOME}/watches.yaml -COPY playbook.yml ${HOME}/playbook.yml +COPY roles/ ${HOME}/roles/ +COPY playbooks/ ${HOME}/playbooks/ USER 1001 diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..33341a3c0 --- /dev/null +++ b/Makefile @@ -0,0 +1,218 @@ +# VERSION defines the project version for the bundle. +# Update this value when you upgrade the version of your project. +# To re-generate a bundle for another specific version without changing the standard setup, you can: +# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) +# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) + +# CHANNELS define the bundle channels used in the bundle. +# Add a new line here if you would like to change its default config. (E.g CHANNELS = "preview,fast,stable") +# To re-generate a bundle for other specific channels without changing the standard setup, you can: +# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=preview,fast,stable) +# - use environment variables to overwrite this value (e.g export CHANNELS="preview,fast,stable") +ifneq ($(origin CHANNELS), undefined) +BUNDLE_CHANNELS := --channels=$(CHANNELS) +endif + +# DEFAULT_CHANNEL defines the default channel used in the bundle. +# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") +# To re-generate a bundle for any other default channel without changing the default setup, you can: +# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) +# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") +ifneq ($(origin DEFAULT_CHANNEL), undefined) +BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) +endif +BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) + +# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. +# This variable is used to construct full image tags for bundle and catalog images. +# +# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both +# cloudbulldozer.io/ripsaw-bundle:$VERSION and cloudbulldozer.io/ripsaw-catalog:$VERSION. +IMAGE_TAG_BASE ?= cloudbulldozer.io/ripsaw + +# BUNDLE_IMG defines the image:tag used for the bundle. +# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) +BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) + +OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') +ARCH ?= $(shell uname -m | sed 's/x86_64/amd64/') + +# Image URL to use all building/pushing image targets +REGISTRY ?= quay.io +ORG ?= cloud-bulldozer +# Get the current branch name +# In case is the master branch, rename it to latest +VERSION ?= $(shell hack/tag_name.sh) +IMG ?= $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION) +ifdef IMAGE_ARCH +IMG := $(REGISTRY)/$(ORG)/benchmark-operator:$(VERSION)-$(IMAGE_ARCH) +BUILD_FLAGS := --arch=$(IMAGE_ARCH) +endif +MANIFEST_ARCHS ?= amd64 arm64 ppc64le + +# Containers +ifeq (, $(shell which podman)) + ENGINE := docker +else + ENGINE := podman +endif + +# E2E testing +E2E_DIR = e2e +E2E_RESULTS = test-results.xml +BATS_FLAGS = -j 6 -F pretty --report-formatter junit -T +ifdef BATS_TESTS + FILTERED_TESTS := -f "$(BATS_TESTS)" +endif + +all: image-build + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk commands is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Build + +run: ansible-operator ## Run against the configured Kubernetes cluster in ~/.kube/config + $(ANSIBLE_OPERATOR) run + +image-build: ## Build container image with the manager. + ${ENGINE} build $(BUILD_FLAGS) -t $(IMG) . + +image-push: ## Push container image with the manager. + ${ENGINE} push $(IMG) + +manifest: manifest-build ## Builds a container manifest and push it to the registry + $(ENGINE) manifest push $(IMG) $(IMG) + +manifest-build: + $(ENGINE) manifest create $(IMG) + @for arch in $(MANIFEST_ARCHS); do \ + echo "Adding $(IMG)-$${arch} to manifest ${IMG}"; \ + $(ENGINE) manifest add $(IMG) $(IMG)-$${arch}; \ + done + +##@ Deployment + +install: kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl apply -f - + +uninstall: kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl delete -f - + +deploy: kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default | kubectl apply -f - + +undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/default | kubectl delete -f - + + +.PHONY: kustomize +KUSTOMIZE = $(shell pwd)/bin/kustomize +kustomize: ## Download kustomize locally if necessary. +ifeq (,$(wildcard $(KUSTOMIZE))) +ifeq (,$(shell which kustomize 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(KUSTOMIZE)) ;\ + curl -sSLo - https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v3.5.4/kustomize_v3.5.4_$(OS)_$(ARCH).tar.gz | \ + tar xzf - -C bin/ ;\ + } +else +KUSTOMIZE = $(shell which kustomize) +endif +endif + +.PHONY: ansible-operator +ANSIBLE_OPERATOR = $(shell pwd)/bin/ansible-operator +ansible-operator: ## Download ansible-operator locally if necessary, preferring the $(pwd)/bin path over global if both exist. +ifeq (,$(wildcard $(ANSIBLE_OPERATOR))) +ifeq (,$(shell which ansible-operator 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(ANSIBLE_OPERATOR)) ;\ + curl -sSLo $(ANSIBLE_OPERATOR) https://github.com/operator-framework/operator-sdk/releases/download/v1.7.2/ansible-operator_$(OS)_$(ARCH) ;\ + chmod +x $(ANSIBLE_OPERATOR) ;\ + } +else +ANSIBLE_OPERATOR = $(shell which ansible-operator) +endif +endif + +.PHONY: bundle +bundle: kustomize ## Generate bundle manifests and metadata, then validate generated files. + operator-sdk generate kustomize manifests -q + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) + $(KUSTOMIZE) build config/manifests | operator-sdk generate bundle -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) + operator-sdk bundle validate ./bundle + +.PHONY: bundle-build +bundle-build: ## Build the bundle image. + ${ENGINE} build -f bundle.Dockerfile -t $(BUNDLE_IMG) . + +.PHONY: bundle-push +bundle-push: ## Push the bundle image. + $(MAKE) image-push IMG=$(BUNDLE_IMG) + +.PHONY: opm +OPM = ./bin/opm +opm: ## Download opm locally if necessary. +ifeq (,$(wildcard $(OPM))) +ifeq (,$(shell which opm 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(OPM)) ;\ + curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.15.1/$(OS)-$(ARCH)-opm ;\ + chmod +x $(OPM) ;\ + } +else +OPM = $(shell which opm) +endif +endif + +# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0). +# These images MUST exist in a registry and be pull-able. +BUNDLE_IMGS ?= $(BUNDLE_IMG) + +# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). +CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) + +# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. +ifneq ($(origin CATALOG_BASE_IMG), undefined) +FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) +endif + +# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'. +# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see: +# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator +.PHONY: catalog-build +catalog-build: opm ## Build a catalog image. + $(OPM) index add --container-tool docker --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) + +# Push the catalog image. +.PHONY: catalog-push +catalog-push: ## Push a catalog image. + $(MAKE) image-push IMG=$(CATALOG_IMG) + +##@ Test + +.PHONY: e2e-tests +e2e-tests: ## Triggers e2e testing, by default all e2e/*.bats tests are executed. You can execute specific tests by setting a regix in the vabiable BATS_TESTS like: BATS_TESTS="fio|uperf|ycsb" make e2e-tests + cd $(E2E_DIR); bats $(BATS_FLAGS) $(FILTERED_TESTS) . + +.PHONY: install-bats +install-bats: + VERSION=v1.5.0 ./hack/install-bats.sh diff --git a/PROJECT b/PROJECT new file mode 100644 index 000000000..56220cc3c --- /dev/null +++ b/PROJECT @@ -0,0 +1,16 @@ +domain: cloudbulldozer.io +layout: +- ansible.sdk.operatorframework.io/v1 +plugins: + manifests.sdk.operatorframework.io/v2: {} + scorecard.sdk.operatorframework.io/v2: {} +projectName: ripsaw +resources: +- api: + crdVersion: v1 + namespaced: true + domain: cloudbulldozer.io + group: ripsaw + kind: Benchmark + version: v1alpha1 +version: "3" diff --git a/README.md b/README.md index 22435c14f..6c86b18f3 100644 --- a/README.md +++ b/README.md @@ -3,25 +3,51 @@ The intent of this Operator is to deploy common workloads to establish a performance baseline of Kubernetes cluster on your provider. -## Installation -Installing the benchmark-operator is easiest by using the helm chart and can be done with the following commands. This requires +## Installation (Default) +The easiest way to install the operator is through the operator-sdk methods provided in the `Makefile`. + +```bash +git clone https://github.com/cloud-bulldozer/benchmark-operator +cd benchmark-operator +make deploy +``` + +Watch how to do this on Youtube: + +[![Benchmark Operator Quickstart](http://img.youtube.com/vi/eQxKl_uve84/0.jpg)](http://www.youtube.com/watch?v=eQxKl_uve84 "Benchmark Operator Quickstart") + +If you wish to build a version of the operator from your local copy of the repo, you can run + +```bash +git clone https://github.com/cloud-bulldozer/benchmark-operator +cd benchmark-operator +make image-build image-push deploy IMG=$YOUR_IMAGE +``` + +> Note: building the image requires podman + +## Installation (Helm) + +Installing the benchmark-operator via Helm can be done with the following commands. This requires your machine to have Helm installed. [Install Helm](https://helm.sh/docs/intro/install/) -> Note: If running on openshift you'll need to run this command before installing the chart. `oc adm policy -n my-ripsaw add-scc-to-user privileged -z benchmark-operator` +> Note: If running on openshift you'll need to run this command before installing the chart. `oc adm policy -n benchmark-operator add-scc-to-user privileged -z benchmark-operator` ```bash git clone https://github.com/cloud-bulldozer/benchmark-operator cd benchmark-operator/charts/benchmark-operator -helm install benchmark-operator . -n my-ripsaw --create-namespace +kubectl create namespace benchmark-operator +oc adm policy -n benchmark-operator add-scc-to-user privileged -z benchmark-operator # Openshift Only +helm install benchmark-operator . -n benchmark-operator --create-namespace ``` To delete this release, you can do so with the following command: ```bash -helm delete benchmark-operator -n my-ripsaw --purge +helm uninstall benchmark-operator -n benchmark-operator ``` @@ -31,7 +57,7 @@ helm delete benchmark-operator -n my-ripsaw --purge | Workload | Use | ElasticSearch indexing | Reconciliation usage | VM support (kubevirt) | Kata Containers | CI Tested | | ------------------------------ | ---------------------- | ------------------ | -------------------------- | --------------------- | --------------- | ------------ | | [UPerf](docs/uperf.md) | Network Performance | Yes | Used, default : 3second | Working | Working | Yes | -| [Iperf3](docs/iperf.md) | Network Performance | No | Used, default : 3second | Not Supported | Preview | Yes | +| [Iperf3](docs/iperf3.md) | Network Performance | No | Used, default : 3second | Not Supported | Preview | Yes | | [fio](docs/fio_distributed.md) | Storage IO | Yes | Used, default : 3second | Working | Working | Yes | | [Sysbench](docs/sysbench.md) | System Performance | No | Used, default : 3second | Not Supported | Preview | Yes | | [YCSB](docs/ycsb.md) | Database Performance | Yes | Used, default : 3second | Not Supported | Preview | Yes | @@ -39,7 +65,7 @@ helm delete benchmark-operator -n my-ripsaw --purge | [Pgbench](docs/pgbench.md) | Postgres Performance | Yes | Used, default : 3second | Not Supported | Preview | Yes | | [Smallfile](docs/smallfile.md) | Storage IO Performance | Yes | Used, default : 3second | Not Supported | Preview | Yes | | [fs-drift](docs/fs-drift.md) | Storage IO Longevity | Yes | Not used | Not Supported | Preview | Yes | -| [hammerdb](docs/hammerdb.md) | Database Performance | Yes | Used, default : 3second | Not Supported | Preview | Yes | +| [hammerdb](docs/hammerdb.md) | Database Performance | Yes | Used, default : 3second | Working | Preview | Yes | | [Service Mesh](docs/servicemesh.md) | Microservices | No | Used, default : 3second | Not Supported | Preview | No | | [Vegeta](docs/vegeta.md) | HTTP Performance | Yes | Used, default : 3second | Not Supported | Preview | Yes | | [Scale Openshift](docs/scale_openshift.md) | Scale Openshift Cluster | Yes | Used, default : 3second | Not Supported | Preview | Yes | @@ -49,6 +75,8 @@ helm delete benchmark-operator -n my-ripsaw --purge | [oslat](docs/oslat.md) | Real-Time Latency | Yes | Used, default : 3second | Not Supported | Preview | No | | [testpmd](docs/testpmd.md) | TestPMD DPDK App | No | Used | Not Supported | Preview | No | | [Flent](docs/flent.md) | Network Performance | Yes | Used, default : 3second | Not Supported | Not Supported | Yes | +| [Log-Generator](docs/log_generator.md) | Log Throughput to Backend | Yes | Used, default : 3second | Not Supported | Yes | Yes | +| [Image-Pull](docs/image_pull.md) | Time to Pull Image from Container Repo | Yes | Used, default : 3second | Not Supported | Yes | Yes | ### Reconciliation @@ -59,11 +87,14 @@ Why did we decide to switch to this? Our operator would implement long running t However, long running tasks blocks the Operator, causing us to delete the Operator and re-create the operator to un-block it. The benchmarks mentioned above that state `Used` for Reconciliation, no longer have this issue. +# E2E tests +Benchmark-operator includes a series of end 2 end tests that can be triggered in local. More info in the [documentation.](docs/e2e-ci.md#running-in-local) + ## Optional workload images Optional locations for workload images can now be added easily without the need to rebuild the operator. To do so in the workload args section of the CR add image: [location] -NOTE: This is not a required arguement. If omitted it will default to the currently verified workload image. +NOTE: This is not a required argument. If omitted it will default to the currently verified workload image. Additionally, this is *NOT* enabled for YCSB For Example: @@ -73,7 +104,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://my-es.foo.bar:80" @@ -85,6 +116,32 @@ spec: image: my.location/foo:latest ``` +## Optional debug out for benchmark-wrapper workloads +Workloads that are triggered through [benchmark-wrapper](https://github.com/cloud-bulldozer/benchmark-wrapper) +can optionally pass the debug flag through the workload CR. + +NOTE: This is not a required argument. If omitted it will default to the default logging level of +the benchmark-wrapper. + +For Example: + +``` +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: example-benchmark + namespace: benchmark-operator +spec: + elasticsearch: + url: "http://my-es.foo.bar:80" + metadata_collection: true + cleanup: false + workload: + name: snafu_workload + args: + debug: true +``` + ## User Provided UUID All benchmarks in the benchmark-operator utilize a UUID for tracking and indexing purposes. This UUID is, by default, generated when the workload is first started. However, if desired, a user provided UUID can @@ -98,7 +155,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: uuid: 6060004a-7515-424e-93bb-c49844600dde elasticsearch: @@ -111,18 +168,12 @@ spec: image: my.location/foo:latest ``` -## Installation -[Installation](docs/installation.md) - ## Contributing [Contributing](CONTRIBUTING.md) ## Metadata Collection [Metadata Collection](docs/metadata.md) -## Cerberus Integration -[Cerberus Integration](docs/cerberus.md) - ## Indexing to Elasticsearch [Indexing to Elasticsearch](docs/elastic.md) @@ -133,6 +184,6 @@ spec: [Cache dropping](docs/cache_dropping.md) ## Community -Key Members(slack_usernames): ravi, mohit, dry923, rsevilla or rook +Key Members(slack_usernames): sejug, mohit, dry923, rsevilla or rook * [**#sig-scalability on Kubernetes Slack**](https://kubernetes.slack.com) * [**#forum-perfscale on CoreOS Slack**](https://coreos.slack.com) diff --git a/build/test-framework/Dockerfile b/build/test-framework/Dockerfile deleted file mode 100644 index e6c041367..000000000 --- a/build/test-framework/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -ARG BASEIMAGE -FROM ${BASEIMAGE} -USER 0 -RUN yum install -y python-devel gcc libffi-devel && pip install molecule -ARG NAMESPACEDMAN -ADD $NAMESPACEDMAN /namespaced.yaml -ADD build/test-framework/ansible-test.sh /ansible-test.sh -RUN chmod +x /ansible-test.sh -USER 1001 -ADD . /opt/ansible/project diff --git a/build/test-framework/ansible-test.sh b/build/test-framework/ansible-test.sh deleted file mode 100644 index 9719f2609..000000000 --- a/build/test-framework/ansible-test.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -export WATCH_NAMESPACE=${TEST_NAMESPACE} -(/usr/local/bin/entrypoint)& -trap "kill $!" SIGINT SIGTERM EXIT - -cd ${HOME}/project -exec molecule test -s test-cluster diff --git a/charts/benchmark-operator/README.md b/charts/benchmark-operator/README.md index 6ff20dfaf..0f2d4a83a 100644 --- a/charts/benchmark-operator/README.md +++ b/charts/benchmark-operator/README.md @@ -12,13 +12,15 @@ A Helm chart for Kubernetes | nameOverride | string | `""` | | | operator.affinity.nodeAffinity | object | `{"preferredDuringSchedulingIgnoredDuringExecution":[{"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/workload","operator":"In","values":[""]}]},"weight":100}]}` | nodeAffinity is set to make the operator run on workload nodes, but it's not required | | operator.image.pullPolicy | string | `"Always"` | | -| operator.image.repository | string | `"quay.io/benchmark-operator/benchmark-operator"` | | +| operator.image.repository | string | `"quay.io/cloud-bulldozer/benchmark-operator"` | | | operator.image.tag | string | `""` | | +| operator.redisImage.pullPolicy | string | `"Always"` | | +| operator.redisImage.repository | string | `"k8s.gcr.io/redis"` | | +| operator.redisImage.tag | string | `"v1"` | | | operator.nodeSelector | object | `{}` | | | operator.replicaCount | int | `1` | how many replicas for the operator deployment | | operator.resources.limits.cpu | string | `"0.1"` | | | operator.tolerations[0] | object | `{"effect":"NoSchedule","key":"role","value":"workload"}` | schedule on a workload node even if it's tainted | -| rbac | object | `{"clusterRoles":{"kubevirt":false,"selfProvisioner":false}}` | map for custom cluster roles for certain benchmarks like kubevirt | ---------------------------------------------- Autogenerated from chart metadata using [helm-docs v1.4.0](https://github.com/norwoodj/helm-docs/releases/v1.4.0) diff --git a/charts/benchmark-operator/crds/crd.yaml b/charts/benchmark-operator/crds/crd.yaml new file mode 120000 index 000000000..f28d729c9 --- /dev/null +++ b/charts/benchmark-operator/crds/crd.yaml @@ -0,0 +1 @@ +../../../config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml \ No newline at end of file diff --git a/charts/benchmark-operator/crds/ripsaw_v1alpha1_ripsaw_crd.yaml b/charts/benchmark-operator/crds/ripsaw_v1alpha1_ripsaw_crd.yaml deleted file mode 120000 index 54e584ebe..000000000 --- a/charts/benchmark-operator/crds/ripsaw_v1alpha1_ripsaw_crd.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml \ No newline at end of file diff --git a/charts/benchmark-operator/templates/self-provisioner-cluster-role-binding.yaml b/charts/benchmark-operator/templates/clusterrolebinding.yaml similarity index 79% rename from charts/benchmark-operator/templates/self-provisioner-cluster-role-binding.yaml rename to charts/benchmark-operator/templates/clusterrolebinding.yaml index 40b9faaef..8a4b72544 100644 --- a/charts/benchmark-operator/templates/self-provisioner-cluster-role-binding.yaml +++ b/charts/benchmark-operator/templates/clusterrolebinding.yaml @@ -1,14 +1,14 @@ -{{ if .Values.rbac.clusterRoles.selfProvisioner }} -apiVersion: rbac.authorization.k8s.io/v1 +--- kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ include "benchmark-operator.fullname" . }} + namespace: {{ .Release.Namespace }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: self-provisioner + name: cluster-admin subjects: - kind: ServiceAccount name: {{ include "benchmark-operator.fullname" . }} namespace: {{ .Release.Namespace }} -{{ end }} \ No newline at end of file diff --git a/charts/benchmark-operator/templates/kubevirt-cluster-role.yaml b/charts/benchmark-operator/templates/kubevirt-cluster-role.yaml deleted file mode 100644 index 02e33f6ff..000000000 --- a/charts/benchmark-operator/templates/kubevirt-cluster-role.yaml +++ /dev/null @@ -1,76 +0,0 @@ -{{ if .Values.rbac.clusterRoles.kubevirt }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "benchmark-operator.fullname" . }}-kubevirt -rules: -- apiGroups: - - rbac.authorization.k8s.io - resources: - - roles - - rolebindings - - clusterroles - - clusterrolebindings - - daemonsets - verbs: - - '*' -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - get - - list -- apiGroups: - - rbac.authorization.k8s.io - resources: - - nodes - verbs: - - get - - list -- apiGroups: - - subresources.kubevirt.io - resources: - - virtualmachineinstances/console - - virtualmachineinstances/vnc - verbs: - - get -- apiGroups: - - kubevirt.io - resources: - - virtualmachineinstances - - virtualmachines - - virtualmachineinstancepresets - - virtualmachineinstancereplicasets - verbs: - - get - - delete - - create - - update - - patch - - list - - watch - - deletecollection -- apiGroups: [""] - resources: - - configmaps - resourceNames: - - kubevirt-config - verbs: - - update - - get - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "benchmark-operator.fullname" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "benchmark-operator.fullname" . }}-kubevirt -subjects: -- kind: ServiceAccount - name: {{ include "benchmark-operator.fullname" . }} - namespace: {{ .Release.Namespace }} -{{ end }} \ No newline at end of file diff --git a/charts/benchmark-operator/templates/operator.yaml b/charts/benchmark-operator/templates/operator.yaml index a2f5b7cb9..514eeabd4 100644 --- a/charts/benchmark-operator/templates/operator.yaml +++ b/charts/benchmark-operator/templates/operator.yaml @@ -11,21 +11,11 @@ spec: template: metadata: labels: + control-plane: controller-manager name: {{ include "benchmark-operator.fullname" . }} spec: serviceAccountName: {{ include "benchmark-operator.fullname" . }} containers: - - name: ansible - command: - - /usr/local/bin/ao-logs - - /tmp/ansible-operator/runner - - stdout - image: "{{ .Values.operator.image.repository }}:{{ .Values.operator.image.tag }}" - imagePullPolicy: {{ .Values.operator.image.pullPolicy}} - volumeMounts: - - mountPath: /tmp/ansible-operator/runner - name: runner - readOnly: true - name: benchmark-operator image: "{{ .Values.operator.image.repository }}:{{ .Values.operator.image.tag }}" imagePullPolicy: {{ .Values.operator.image.pullPolicy}} @@ -48,10 +38,12 @@ spec: - mountPath: /tmp/ansible-operator/runner name: runner - name: redis-master - image: k8s.gcr.io/redis:v1 + image: "{{ .Values.operator.redisImage.repository }}:{{ .Values.operator.redisImage.tag }}" env: - name: MASTER value: "true" + - name: ALLOW_EMPTY_PASSWORD + value: "yes" ports: - containerPort: 6379 resources: @@ -72,5 +64,3 @@ spec: emptyDir: {} - name: runner emptyDir: {} - - diff --git a/charts/benchmark-operator/templates/psp.yaml b/charts/benchmark-operator/templates/psp.yaml deleted file mode 100644 index a8e05fb49..000000000 --- a/charts/benchmark-operator/templates/psp.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: privileged -spec: - privileged: true - allowPrivilegeEscalation: true - allowedCapabilities: - - '*' - volumes: - - '*' - hostNetwork: true - hostPorts: - - min: 0 - max: 65535 - hostIPC: true - hostPID: true - runAsUser: - rule: 'RunAsAny' - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'RunAsAny' - fsGroup: - rule: 'RunAsAny' diff --git a/charts/benchmark-operator/templates/role-binding.yaml b/charts/benchmark-operator/templates/role-binding.yaml deleted file mode 100644 index 638a77f58..000000000 --- a/charts/benchmark-operator/templates/role-binding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "benchmark-operator.fullname" . }} - namespace: {{ .Release.Namespace }} -subjects: -- kind: ServiceAccount - name: {{ include "benchmark-operator.fullname" . }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: Role - name: {{ include "benchmark-operator.fullname" . }} - apiGroup: rbac.authorization.k8s.io diff --git a/charts/benchmark-operator/templates/role.yaml b/charts/benchmark-operator/templates/role.yaml deleted file mode 100644 index 2e9711eb3..000000000 --- a/charts/benchmark-operator/templates/role.yaml +++ /dev/null @@ -1,103 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "benchmark-operator.fullname" . }} - namespace: {{ .Release.Namespace }} -rules: -- apiGroups: - - "" - resources: - - pods - - daemonsets - - services - - endpoints - - persistentvolumeclaims - - virtualmachineinstances - - events - - configmaps - - secrets - - pods/log - verbs: - - '*' -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - apps - resources: - - deployments - - daemonsets - - replicasets - - statefulsets - - deployments/finalizers - verbs: - - '*' -- apiGroups: - - monitoring.coreos.com - resources: - - servicemonitors - verbs: - - get - - create -- apiGroups: - - kubevirt.io - resources: - - virtualmachineinstances - verbs: - - '*' -- apiGroups: - - ripsaw.cloudbulldozer.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - batch - - extensions - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - policy - resources: - - podsecuritypolicies - verbs: - - use - resourceNames: - - privileged -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use -- apiGroups: - - hyperfoil.io - resources: - - hyperfoils - verbs: - - '*' -- apiGroups: - - networking.k8s.io - resources: - - networkpolicies - verbs: - - get - - list - - watch - - create - - update - - patch - - delete diff --git a/charts/benchmark-operator/templates/service-account.yaml b/charts/benchmark-operator/templates/service-account.yaml index 71a3ffc67..eb24d0509 100644 --- a/charts/benchmark-operator/templates/service-account.yaml +++ b/charts/benchmark-operator/templates/service-account.yaml @@ -1,5 +1,6 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ include "benchmark-operator.fullname" . }} + # do not change - hard coded in ansible roles + name: "benchmark-operator" namespace: {{ .Release.Namespace }} diff --git a/charts/benchmark-operator/values.yaml b/charts/benchmark-operator/values.yaml index 600cdfd18..1db5dddb0 100644 --- a/charts/benchmark-operator/values.yaml +++ b/charts/benchmark-operator/values.yaml @@ -5,32 +5,27 @@ nameOverride: "" # -- (string) overrides naming of all resources fullnameOverride: "benchmark-operator" - -# -- map for custom cluster roles for certain benchmarks like kubevirt -rbac: - clusterRoles: - kubevirt: false - selfProvisioner: false - - -operator: +operator: # -- how many replicas for the operator deployment replicaCount: 1 image: - repository: quay.io/benchmark-operator/benchmark-operator + repository: quay.io/cloud-bulldozer/benchmark-operator + pullPolicy: Always + tag: latest + + redisImage: + repository: quay.io/cloud-bulldozer/redis pullPolicy: Always - tag: master + tag: latest resources: limits: cpu: "0.1" - nodeSelector: {} - tolerations: # -- schedule on a workload node even if it's tainted - key: role diff --git a/cli/.coveragerc b/cli/.coveragerc new file mode 100644 index 000000000..7ae4f689f --- /dev/null +++ b/cli/.coveragerc @@ -0,0 +1,4 @@ +[run] +omit = + **/__init__.py + ripsaw/cli.py diff --git a/cli/.flake8 b/cli/.flake8 new file mode 100644 index 000000000..31cf5219b --- /dev/null +++ b/cli/.flake8 @@ -0,0 +1,11 @@ +[flake8] +ignore= + E123,E125,E731,E722,W503,W605, # Black takes care of these + D202, # no blank lines after docstring + D203, # blank line before class docstring + D212, # multi-line summary starts at first line + D401, # imperitive mood + D413, # blank line after last section +max-line-length=110 +docstring-convention=numpy +select=E,W,F diff --git a/cli/.pylintrc b/cli/.pylintrc new file mode 100644 index 000000000..c8abffeb5 --- /dev/null +++ b/cli/.pylintrc @@ -0,0 +1,20 @@ +[MASTER] +jobs=0 +ignore-paths= + cli/tests +[MESSAGES CONTROL] +disable= + duplicate-code, + import-error, + too-few-public-methods, + too-many-arguments, + too-many-instance-attributes, + fixme, + logging-fstring-interpolation, + no-self-use, + logging-format-interpolation + + + +[FORMAT] +max-line-length=110 diff --git a/cli/MANIFEST.in b/cli/MANIFEST.in new file mode 100644 index 000000000..48715ba55 --- /dev/null +++ b/cli/MANIFEST.in @@ -0,0 +1 @@ +include version.txt diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 000000000..64bcdde9c --- /dev/null +++ b/cli/README.md @@ -0,0 +1,26 @@ +# Ripsaw CLI + +## Overview + +Ripsaw CLI is a portable, lightweight CLI that can be used to install the benchmark-operator into a specified cluster and run user-defined benchmarks. + +## Installation + +> Note: Ripsaw CLI is only tested against Python 3.8+ + +You must first start by installing the python package like so: + +```bash +cd cli +pip install . +``` + +If you want to make changes to the underlying code, you can set it up as editable by running `pip install -e .` instead. + + +After that, you can run `ripsaw --help` to see the command options. + + +## Commands + +There are two top-level command groups: `operator` and `benchmark`. You can run `ripsaw operator --help` to see options for installing/deleting the operator, and `ripsaw benchmark --help` to see options for running benchmarks. diff --git a/cli/pytest.ini b/cli/pytest.ini new file mode 100644 index 000000000..6757ecf58 --- /dev/null +++ b/cli/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +markers = + integration: mark a test as integration tests + unit: mark a test as a unit test diff --git a/cli/ripsaw/__init__.py b/cli/ripsaw/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/cli/ripsaw/cli.py b/cli/ripsaw/cli.py new file mode 100755 index 000000000..1634c47e4 --- /dev/null +++ b/cli/ripsaw/cli.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Main Entrypoint Module for the CLI""" + + +import click +from ripsaw.commands import benchmark, operator +from ripsaw.util import logging + +logger = logging.get_logger(__name__) + + +@click.group() +def cli(): + """Top level Click CLI group that is used to add all the groups defined in the commands module.""" + + +cli.add_command(operator.operator_group) +cli.add_command(benchmark.benchmark_group) + +if __name__ == "__main__": + cli() diff --git a/cli/ripsaw/clients/__init__.py b/cli/ripsaw/clients/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cli/ripsaw/clients/elastic.py b/cli/ripsaw/clients/elastic.py new file mode 100755 index 000000000..6e07c5935 --- /dev/null +++ b/cli/ripsaw/clients/elastic.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Create portable functions around interacting with ES hosting benchmark data""" + + +import ssl + +import elasticsearch +import urllib3 +from ripsaw.util import logging + +logger = logging.get_logger(__name__) + + +def check_index(server, uuid, index, es_ssl=False): + """Checks index on ES Server for benchmark with a specific uuid""" + + if es_ssl: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + ssl_ctx = ssl.create_default_context() + ssl_ctx.check_hostname = False + ssl_ctx.verify_mode = ssl.CERT_NONE + es_client = elasticsearch.Elasticsearch( + [server], send_get_body_as="POST", ssl_context=ssl_ctx, use_ssl=True + ) + else: + es_client = elasticsearch.Elasticsearch([server], send_get_body_as="POST") + es_client.indices.refresh(index=index) + results = es_client.search(index=index, body={"query": {"term": {"uuid.keyword": uuid}}}, size=1) + if results["hits"]["total"]["value"] > 0: + return True + + print("No result found in ES") + return False diff --git a/cli/ripsaw/clients/k8s.py b/cli/ripsaw/clients/k8s.py new file mode 100755 index 000000000..2f3b67746 --- /dev/null +++ b/cli/ripsaw/clients/k8s.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create portable definition of a Kubernetes/Openshift Cluster""" + + +import time +import urllib3 + +from kubernetes import client, config +from ripsaw.util import logging +from ripsaw.util.exceptions import BenchmarkFailedError, BenchmarkTimeoutError, PodsNotFoundError +import os + +logger = logging.get_logger(__name__) + +# how many seconds to sleep before looping back in wait commands +DEFAULT_WAIT_TIME = 1 + + +class Cluster: + """ + Class definition of a Kubernetes/Openshift Cluster + + Attributes: + api_client (ApiClient) + core_client (CoreV1Api) + batch_client (BatchV1Api) + crd_client (CustomObjectsApi) + """ + + def __init__(self, kubeconfig_path=None): + """Initialize object and create clients from specified kubeconfig""" + client_config = client.Configuration() + http_proxy = os.getenv('http_proxy', None) + """Proxy has auth header""" + if http_proxy and "@" in http_proxy: + proxy_auth = http_proxy.split("@")[0].split("//")[1] + proxy_url = http_proxy + if proxy_url: + client_config.proxy = proxy_url + if proxy_auth: + client_config.proxy_headers = urllib3.make_headers(proxy_basic_auth=proxy_auth) + config.load_kube_config(config_file=kubeconfig_path, client_configuration=client_config) + self.api_client = client.ApiClient(client_config) + self.core_client = client.CoreV1Api(self.api_client) + self.batch_client = client.BatchV1Api(self.api_client) + self.crd_client = client.CustomObjectsApi(self.api_client) + + def get_pods(self, label_selector, namespace): + """Get pods matching specified label selector in specified namespace""" + return self.core_client.list_namespaced_pod(namespace, label_selector=label_selector, watch=False) + + def get_pod_logs(self, label_selector, namespace, container): + """Get container logs from pods with labels in namespace""" + pods = self.get_pods(label_selector, namespace).items + return [ + self.core_client.read_namespaced_pod_log( + name=pod.metadata.name, namespace=namespace, container=container + ) + for pod in pods + ] + + def get_jobs(self, label_selector, namespace): + """Get jobs with specified labels in namespace""" + return self.batch_client.list_namespaced_job(namespace, label_selector=label_selector, watch=False) + + def get_nodes(self, label_selector=None): + """Get all nodes with label selector, default is to get all nodes""" + return self.core_client.list_node(label_selector=label_selector) + + def get_node_names(self, label_selector=None): + """Get names of all nodes with label selector""" + return [node.metadata.name for node in self.get_nodes(label_selector).items] + + def get_namespaces(self, label_selector=None): + """Get namespaces with labels""" + return self.core_client.list_namespace(label_selector=label_selector) + + def get_benchmark(self, name, namespace="benchmark-operator"): + """Get benchmark resource""" + return self.crd_client.get_namespaced_custom_object( + group="ripsaw.cloudbulldozer.io", + version="v1alpha1", + namespace=namespace, + plural="benchmarks", + name=name, + ) + + def get_benchmark_metadata(self, name, namespace="benchmark-operator"): + """Get benchmark metadata from the cluster resource""" + benchmark = self.get_benchmark(name, namespace) + return { + "name": benchmark["metadata"]["name"], + "namespace": benchmark["metadata"]["namespace"], + "uuid": benchmark.get("status", {}).get("uuid", "Not Assigned Yet"), + "suuid": benchmark.get("status", {}).get("suuid", "Not Assigned Yet"), + "status": benchmark.get("status", {}).get("state", ""), + } + + # Waiters + def wait_for_pods(self, label_selector, namespace, timeout=300): + """wait for pods with label in namespace for specified timeout""" + waiting_for_pods = True + timeout_interval = 0 + while waiting_for_pods: + if timeout_interval >= timeout: + raise TimeoutError() + pods = self.get_pods(label_selector, namespace).items + if len(pods) == 0 and timeout_interval < 60: + continue + + if len(pods) == 0 and timeout_interval > 60: + raise PodsNotFoundError(f"Found no pods with label selector {label_selector}") + + _ = [ + logger.info(f"{pod.metadata.namespace}\t{pod.metadata.name}\t{pod.status.phase}") + for pod in pods + ] + waiting_for_pods = any(pod.status.phase != "Running" for pod in pods) + + time.sleep(DEFAULT_WAIT_TIME) + timeout_interval += DEFAULT_WAIT_TIME + + def wait_for_benchmark( + self, name, namespace="benchmark-operator", desired_state="Completed", timeout=300 + ): + """Wait for benchmark to hit desired state for specified timeout""" + waiting_for_benchmark = True + logger.info(f"Waiting for state: {desired_state}") + timeout_interval = 0 + logger.info("BENCHMARK\tUUID\t\t\t\t\tSTATE") + while waiting_for_benchmark: + if timeout_interval >= timeout: + raise BenchmarkTimeoutError(name) + benchmark = self.get_benchmark(name, namespace) + bench_status = benchmark.get("status", {}) + uuid = bench_status.get("uuid", "Not Assigned Yet") + current_state = bench_status.get("state", "") + logger.info(f"{benchmark['metadata']['name']}\t{uuid}\t{current_state}") + if current_state == "Failed": + raise BenchmarkFailedError(benchmark["metadata"]["name"], benchmark["status"]["uuid"]) + + waiting_for_benchmark = current_state != desired_state + time.sleep(DEFAULT_WAIT_TIME) + + timeout_interval += DEFAULT_WAIT_TIME + logger.info( + f"{benchmark['metadata']['name']} with uuid {uuid} has reached the desired state {desired_state}" + ) + + # Create Functions + + def create_benchmark(self, benchmark): + """Create benchmark in the cluster""" + self.crd_client.create_namespaced_custom_object( + group="ripsaw.cloudbulldozer.io", + version="v1alpha1", + namespace=benchmark["metadata"]["namespace"], + plural="benchmarks", + body=benchmark, + ) + + # Delete Functions + + def delete_benchmark(self, name, namespace="benchmark-operator"): + """Delete the benchmark from the cluster""" + logger.info(f"Deleting benchmark {name} in namespace {namespace}") + self.crd_client.delete_namespaced_custom_object( + group="ripsaw.cloudbulldozer.io", + version="v1alpha1", + namespace=namespace, + plural="benchmarks", + name=name, + ) + logger.info(f"Deleted benchmark {name} in namespace {namespace}") + + def delete_all_benchmarks(self, namespace="benchmark-operator"): + """Delete all the benchmarks from the cluster""" + all_benchmarks = self.crd_client.list_namespaced_custom_object( + group="ripsaw.cloudbulldozer.io", version="v1alpha1", namespace=namespace, plural="benchmarks" + ) + + _ = [ + self.delete_benchmark(benchmark["metadata"]["name"], namespace) + for benchmark in all_benchmarks.get("items", []) + ] + + def delete_namespace(self, namespace): + """Delete namespace from the cluster""" + return self.core_client.delete_namespace(namespace) + + def delete_namespaces_with_label(self, label_selector): + """Delete all namespaces with label from the cluster.""" + return [ + self.core_client.delete_namespace(namespace.metadata.name) + for namespace in self.get_namespaces(label_selector=label_selector).items + ] diff --git a/cli/ripsaw/commands/__init__.py b/cli/ripsaw/commands/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/cli/ripsaw/commands/benchmark.py b/cli/ripsaw/commands/benchmark.py new file mode 100755 index 000000000..c0535540f --- /dev/null +++ b/cli/ripsaw/commands/benchmark.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CLI Commands for benchmark operations""" + + +import json +from urllib.parse import urlparse + +import click +from ripsaw.clients import elastic +from ripsaw.clients.k8s import Cluster +from ripsaw.models.benchmark import Benchmark + + +@click.group("benchmark") +def benchmark_group(): + """Function to define Click CLI group, used as a decorator to declare functions as group commands + This function is used as a decorator for the benchmark commands (eg. ripsaw benchmark ) + """ + + +@benchmark_group.command("run") +@click.option("-f", "--file", "file", required=True, type=click.File("rb")) +@click.option("-t", "--timeout", "timeout", default=300, type=int) +@click.option("--check-es", is_flag=True) +def run_benchmark(file, timeout, check_es): + """Create benchmark class from file and submit the resource to configured cluster.""" + cluster = Cluster() + benchmark = Benchmark(file, cluster) + click.echo(f"Starting Benchmark {benchmark.name}, timeout set to {timeout} seconds") + benchmark.run(timeout=timeout) + click.secho(f"Benchmark {benchmark.name} Complete", fg="green") + run_metadata = benchmark.get_metadata() + click.echo(f"Run Metadata: \n{json.dumps(run_metadata, indent=4)}\n") + if check_es: + results_found = _check_es(benchmark) + if results_found: + click.secho("Results Found, Benchmark Complete", fg="green") + benchmark.cleanup() + else: + click.secho("No Results Found", fg="red") + + +@benchmark_group.command("delete") +@click.argument("name") +@click.option("-n", "--namespace", "namespace", default="benchmark-operator") +def delete_benchmark(name, namespace): + """delete benchmark resource from cluster""" + cluster = Cluster() + cluster.delete_benchmark(name, namespace) + click.echo(f"Benchmark {name} deleted") + + +def _check_es(benchmark): + """check es for benchmark results""" + run_metadata = benchmark.get_metadata() + es_url = benchmark.yaml["spec"]["elasticsearch"]["url"] + index_name = benchmark.yaml["spec"]["elasticsearch"]["index_name"] + click.echo(f"Checking ES: \n Host: {_get_es_hostname(es_url)} \n Index: {index_name} \n") + return elastic.check_index(es_url, run_metadata["uuid"], f"{index_name}-results") + + +def _get_es_hostname(es_url): + """parse es url and get the hostname""" + parsed_url = urlparse(es_url) + return parsed_url.hostname diff --git a/cli/ripsaw/commands/operator.py b/cli/ripsaw/commands/operator.py new file mode 100755 index 000000000..92cad8a60 --- /dev/null +++ b/cli/ripsaw/commands/operator.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CLI Commands for benchmark operations""" + +import os +import subprocess +import sys +import tempfile + +import click +import git +from ripsaw.clients.k8s import Cluster +from ripsaw.util import logging + +logger = logging.get_logger(__name__) +DEFAULT_REPO = "https://github.com/cloud-bulldozer/benchmark-operator" +DEFAULT_BRANCH = "master" + + +@click.group("operator") +def operator_group(): + """Function to define Click CLI group, used as a decorator to declare functions as group commands + This function is used as a decorator for the operator commands (eg. ripsaw operator ) + """ + + +@operator_group.command("install") +@click.option("--repo", help="Repo URL for benchmark-operator", show_default=True, default=DEFAULT_REPO) +@click.option("--branch", help="Branch to checkout", show_default=True, default=DEFAULT_BRANCH) +@click.option("--force-remote", help="Forcibly use remote repo for install", is_flag=True) +@click.option("--kubeconfig", help=" path to kubeconfig of cluster", default=None) +def install( + repo=DEFAULT_REPO, branch=DEFAULT_BRANCH, force_remote=False, command="make deploy", kubeconfig=None +): + """installs the operator from repo into specified cluster""" + _perform_operator_action(repo, branch, force_remote, command, kubeconfig) + wait_for_operator(kubeconfig=kubeconfig) + click.secho("Operator Running", fg="green") + + +@operator_group.command("delete") +@click.option("--repo", help="Repo URL for benchmark-operator", show_default=True, default=DEFAULT_REPO) +@click.option("--branch", help="Branch to checkout", show_default=True, default=DEFAULT_BRANCH) +@click.option("--force-remote", help="Forcibly use remote repo for install", is_flag=True) +@click.option("--kubeconfig", help="kubeconfig of cluster", default=None) +def delete( + repo=DEFAULT_REPO, + branch=DEFAULT_BRANCH, + force_remote=False, + command="make kustomize undeploy", + kubeconfig=None, +): + """delete the operator from configured cluster""" + click.echo("Deleting Operator") + _perform_operator_action(repo, branch, force_remote, command, kubeconfig) + click.secho("Operator Deleted", fg="green") + + +def wait_for_operator(namespace="benchmark-operator", kubeconfig=None): + """wait for operator pods to be ready""" + cluster = Cluster(kubeconfig_path=kubeconfig) + label_selector = "control-plane=controller-manager" + cluster.wait_for_pods(label_selector, namespace, timeout=120) + + +def _perform_operator_action(repo, branch, force_remote, command, kubeconfig=None): + """run command from operator repo""" + shell_env = os.environ.copy() + if kubeconfig is not None: + shell_env["KUBECONFIG"] = kubeconfig + local_git_repo = _find_git_repo(os.getcwd()) + result = None + if not force_remote and local_git_repo is not None and "benchmark-operator" in local_git_repo: + shell_env["VERSION"] = "latest" + logger.info("Detected CLI is running from local repo, using that instead of remote") + click.echo(f"Installing Operator from local repo at {local_git_repo}") + result = _run_command(command, shell_env, local_git_repo) + else: + with tempfile.TemporaryDirectory(dir=".") as temp_dir: + click.echo(f"Installing Operator from repo {repo} and branch {branch}") + _clone_repo(repo, branch, temp_dir) + result = _run_command(command, shell_env, temp_dir) + + if result.stderr: + logger.warning(result.stderr) + if result.stdout: + logger.debug("Command Result: {}".format(result.stdout)) + + +def _run_command(command, shell_env, cwd): + try: + return subprocess.run( + command.split(" "), + shell=False, + env=shell_env, + cwd=cwd, + capture_output=True, + check=True, + encoding="utf-8", + ) + except subprocess.CalledProcessError as err: + logger.error(f"Something went wrong when running '{command}'") + logger.error(f"Return Code {err.returncode}") + logger.error(f"stdout -- {err.output}") + logger.error(f"stderr -- {err.stderr}") + sys.exit(1) + + +def _clone_repo(repo, branch, directory): + """clone git repo into directory""" + git.Repo.clone_from(url=repo, single_branch=True, depth=1, to_path=directory, branch=branch) + + +def _find_git_repo(path): + """get working dir of first git root on tree, return None otherwise""" + try: + return git.Repo(path, search_parent_directories=True).working_dir + except git.exc.InvalidGitRepositoryError: + return None diff --git a/cli/ripsaw/models/__init__.py b/cli/ripsaw/models/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/cli/ripsaw/models/benchmark.py b/cli/ripsaw/models/benchmark.py new file mode 100755 index 000000000..e2cd8b3e8 --- /dev/null +++ b/cli/ripsaw/models/benchmark.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Defines a Class representing a Benchmark""" + +import yaml +from benedict import benedict +from ripsaw.clients.k8s import Cluster +from ripsaw.util import logging +from ripsaw.util.exceptions import BenchmarkNotStartedError + +logger = logging.get_logger(__name__) + + +class Benchmark: + """ + This class represents a Benchmark to run against a specified cluster + + Attributes: + file (file): Benchmark CR + cluster (ripsaw.clients.k8s.Cluster): Kubernetes/Openshift Cluster + name (str): Name of the benchmark, defined in the file + namespace (str): Namespace for the benchmark, defined in the file + metadata (dict): Metadata from the benchmark + + """ + + def __init__(self, file, cluster=None): + """Initialize Benchmark Class""" + self.yaml = benedict(yaml.full_load(file), keypath_separator=",") + if cluster is None: + self.cluster = Cluster() + else: + self.cluster = cluster + self.name = self.yaml["metadata"]["name"] + self.namespace = self.yaml["metadata"]["namespace"] + self.metadata = {} + + def update_spec(self, key_path, new_value): + """Update YAML Key Path with new value""" + self.yaml[key_path] = new_value + + def run(self, timeout=600): + """Run benchmark with specified timeout in seconds (default: 600)""" + self.metadata = self.cluster.create_benchmark(self.yaml) + logger.info(f"Benchmark {self.name} created") + self.wait(timeout=timeout) + return self.metadata + + def wait(self, desired_state="Complete", timeout=600): + """Wait for benchmark to enter desired state with specified timeout in seconds (default: 600)""" + if self.metadata == {}: + raise BenchmarkNotStartedError(self.name) + try: + self.cluster.wait_for_benchmark( + self.name, self.namespace, desired_state=desired_state, timeout=timeout + ) + except Exception as err: + logger.error(f"Benchmark exception: {err}") + exit(1) + self.metadata = self.cluster.get_benchmark_metadata(self.name, self.namespace) + return self.metadata + + def cleanup(self): + """Delete this benchmark from the cluster""" + self.cluster.delete_benchmark(self.name, self.namespace) + + def get_metadata(self): + """Fetch Benchmark Metadata from the cluster""" + return self.cluster.get_benchmark_metadata(self.name, self.namespace) diff --git a/cli/ripsaw/models/workload.py b/cli/ripsaw/models/workload.py new file mode 100755 index 000000000..694519411 --- /dev/null +++ b/cli/ripsaw/models/workload.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Defines a Class representing a Workload""" + + +from os import listdir +from os.path import isfile, join + +from ripsaw.clients.k8s import Cluster +from ripsaw.models.benchmark import Benchmark +from ripsaw.util import logging + +logger = logging.get_logger(__name__) + + +class Workload: + """Defines a Class representing a Workload""" + + def __init__(self, name, cluster=None, benchmark_dir="."): + self.name = name + if cluster is None: + self.cluster = Cluster() + else: + self.cluster = cluster + self.workload_dir = f"{benchmark_dir}/{name}" + self.benchmarks = [ + Benchmark(join(self.workload_dir, f), self.cluster) + for f in listdir(self.workload_dir) + if isfile(join(self.workload_dir, f)) and f.endswith(".yaml") + ] + + def run_all(self): + """Run all benchmarks in workload""" + runs = [] + for run in self.benchmarks: + run.start() + run.wait() + runs.append(run.metadata) + return runs + + def run(self, run_name): + """Run a benchmark""" + run = next((run for run in self.benchmarks if run.name == run_name), None) + run.start() + run.wait() + return run.metadata + + def inject_overrides(self, overrides): + """Override all benchmark specs with overrides""" + for run in self.benchmarks: + _ = [run.update_spec(key, value) for key, value in overrides.items()] diff --git a/cli/ripsaw/util/__init__.py b/cli/ripsaw/util/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/cli/ripsaw/util/exceptions.py b/cli/ripsaw/util/exceptions.py new file mode 100755 index 000000000..8dac49999 --- /dev/null +++ b/cli/ripsaw/util/exceptions.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Defines Custom Exceptions""" + + +class BenchmarkFailedError(Exception): + """Exception raised when Benchmark hits a Failed State""" + + def __init__(self, name, uuid, msg=None): + if msg is None: + msg = f"The benchmark {name} with uuid {uuid} failed" + super().__init__(msg) + self.name = name + self.uuid = uuid + + +class BenchmarkNotStartedError(Exception): + """Exception raised when Benchmark doesn't hit the Running state before timeout""" + + def __init__(self, name, msg=None): + if msg is None: + msg = f"The benchmark {name} has not started yet" + + super().__init__(msg) + + +class BenchmarkTimeoutError(Exception): + """Exception raised when Benchmark doesn't hit the Completed state before timeout""" + + def __init__(self, name, msg=None): + if msg is None: + msg = f"The benchmark {name} timed out" + + super().__init__(msg) + self.name = name + + +class PodsNotFoundError(Exception): + """Exception raised when query for pods returns none""" + + def __init__(self, msg=None): + if msg is None: + msg = "No Pods found" + super().__init__(msg) diff --git a/cli/ripsaw/util/logging.py b/cli/ripsaw/util/logging.py new file mode 100755 index 000000000..71a92e25d --- /dev/null +++ b/cli/ripsaw/util/logging.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Defines Custom Logging Resources""" + +import logging + + +class DuplicateFilter(logging.Filter): + """Filter class to apply to loggers to prevent duplicate log lines""" + + def filter(self, record): + """Filters our duplicate log lines""" + # add other fields if you need more granular comparison, depends on your app + current_log = (record.module, record.levelno, record.msg) + if current_log != getattr(self, "last_log", None): + self.last_log = current_log # pylint: disable=attribute-defined-outside-init + return True + return False + + +logging.basicConfig(level=logging.INFO, format="ripsaw-cli:%(name)s:%(levelname)s :: %(message)s") +logger = logging.getLogger() # get the root logger +logger.addFilter(DuplicateFilter()) + + +def get_logger(name): + """Get new logger with name""" + new_logger = logging.getLogger(name) + new_logger.addFilter(DuplicateFilter()) + return new_logger diff --git a/cli/setup.cfg b/cli/setup.cfg new file mode 100644 index 000000000..f223274b3 --- /dev/null +++ b/cli/setup.cfg @@ -0,0 +1,54 @@ +# This file is used to configure your project. +# Read more about the various options under: +# http://setuptools.readthedocs.io/en/latest/setuptools.html#configuring-setup-using-setup-cfg-files + +[metadata] +name = ripsaw-cli +description = To run benchmarks against openshift/kubernetes from the CLI +author = red-hat-performance +author_email = perf-dept@redhat.com +license = Apache License 2.0 +long_description = file: README.md +long_description_content-type = text/markdown; charset=UTF-8 +classifiers = + Development Status :: 4 - Beta + Programming Language :: Python +version = file: version.txt +home-page = https://github.com/cloud-bulldozer/benchmark-operator/cli + +[options] +zip_safe = False +packages = find: +include_package_data = True +# Add here dependencies of your project (semicolon/line-separated), e.g. +install_requires = + python-benedict==0.24.0 + click==8.0.1 + kubernetes==17.17.0 + urllib3==1.25.7 + PyYAML==5.4.1 + elasticsearch==7.13.4 + GitPython==3.1.18 + python-dateutil + requests +python_requires = >=3.8 + +[options.extras_require] +tests = + pytest + pytest-cov + tox + pytest-kind + ElasticMock + pytest-env +[options.entry_points] +# Add here console scripts like: +console_scripts = + ripsaw = ripsaw.cli:cli + +[aliases] +dists = bdist_wheel + +[bdist_wheel] +# Use this option if your package is pure-python +universal = 1 diff --git a/cli/setup.py b/cli/setup.py new file mode 100755 index 000000000..fc2d23911 --- /dev/null +++ b/cli/setup.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Setup ripsaw package""" + +import sys + +from pkg_resources import VersionConflict, require +from setuptools import setup + +try: + require("setuptools>=38.3") +except VersionConflict: + print("Error: version of setuptools is too old (<38.3)!") + sys.exit(1) + + +if __name__ == "__main__": + setup() diff --git a/cli/tests/__init__.py b/cli/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cli/tests/clients/test_elastic.py b/cli/tests/clients/test_elastic.py new file mode 100755 index 000000000..9e8c60de8 --- /dev/null +++ b/cli/tests/clients/test_elastic.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import elasticsearch +import pytest +from elasticmock import elasticmock +from ripsaw.clients import elastic + + +@pytest.mark.unit +class TestElastic: + @elasticmock + def test_check_index(self): + server_url = "http://localhost:9200" + index = "test-index" + document = {"uuid": "foo", "data": "bar"} + + elastic_client = elasticsearch.Elasticsearch(hosts=[{"host": "localhost", "port": 9200}]) + elastic_client.index(index, document) + assert elastic.check_index(server_url, document["uuid"], index) + assert not elastic.check_index(server_url, "random-uuid", index) diff --git a/cli/tests/clients/test_k8s.py b/cli/tests/clients/test_k8s.py new file mode 100755 index 000000000..f20faa31b --- /dev/null +++ b/cli/tests/clients/test_k8s.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import kubernetes +import pytest +from ripsaw.util.exceptions import BenchmarkTimeoutError + + +@pytest.mark.unit +class TestCluster: + def test_get_pods(self, cluster): + label_selector = "component=etcd" + pods = cluster.get_pods(label_selector=label_selector, namespace="kube-system") + assert len(pods.items) == 1 + assert "etcd-pytest-kind-control-plane" in pods.items[0].metadata.name + + def test_get_pod_logs(self, cluster): + label_selector = "component=etcd" + logs = cluster.get_pod_logs(label_selector=label_selector, namespace="kube-system", container="etcd") + assert len(logs) > 0 + + def test_get_jobs(self, cluster, test_job): + label_selector = "app=test-job" + jobs = cluster.get_jobs(label_selector=label_selector, namespace="default") + cluster.wait_for_pods("job-name=busybox", "default") + assert len(jobs.items) == 1 + assert jobs.items[0].metadata.name == "busybox" + + def test_get_nodes(self, cluster): + nodes = cluster.get_nodes() + assert len(nodes.items) == 1 + + def test_get_node_names(self, cluster): + names = cluster.get_node_names() + assert len(names) == 1 + assert type(names[0]) is str + assert names[0] == "pytest-kind-control-plane" + + def test_get_namespaces(self, cluster): + namespaces = cluster.get_namespaces() + names = [namespace.metadata.name for namespace in namespaces.items] + assert "kube-system" in names + + def test_get_benchmark(self, cluster, test_benchmark): + benchmark = cluster.get_benchmark(name="byowl") + assert benchmark["metadata"]["name"] == "byowl" + + def test_get_benchmark_metadata(self, cluster, test_benchmark): + benchmark_metadata = cluster.get_benchmark_metadata(name="byowl") + assert benchmark_metadata == { + "name": "byowl", + "namespace": "benchmark-operator", + "uuid": "Not Assigned Yet", + "suuid": "Not Assigned Yet", + "status": "", + } + + def test_wait_for_pods(self, cluster, test_job): + label_selector = "job-name=busybox" + namespace = "default" + cluster.wait_for_pods(label_selector, namespace) + pods = cluster.get_pods(label_selector, namespace) + assert len(pods.items) == 1 + assert pods.items[0].status.phase == "Running" + + def test_create_benchmark_async(self, cluster, test_benchmark_model): + name = test_benchmark_model.name + namespace = test_benchmark_model.namespace + cluster.create_benchmark(test_benchmark_model.yaml) + assert name == cluster.get_benchmark(name, namespace)["metadata"]["name"] + + def test_wait_for_benchmark_timeout(self, cluster, test_benchmark): + name = "byowl" + namespace = "benchmark-operator" + with pytest.raises(BenchmarkTimeoutError): + cluster.wait_for_benchmark(name, namespace, timeout=5) + + def test_delete_benchmark(self, cluster, test_benchmark): + name = "byowl" + namespace = "benchmark-operator" + cluster.delete_benchmark(name, namespace) + with pytest.raises(kubernetes.client.exceptions.ApiException): + cluster.get_benchmark(name, namespace) + + def test_delete_all_benchmarks(self, cluster, test_benchmark): + name = "byowl" + namespace = "benchmark-operator" + cluster.delete_all_benchmarks(namespace) + with pytest.raises(kubernetes.client.exceptions.ApiException): + cluster.get_benchmark(name, namespace) + + def test_delete_namespace(self, cluster, test_namespace): + expected_name = test_namespace[0] + label_selector = test_namespace[1] + namespace = cluster.get_namespaces(label_selector=label_selector).items[0] + assert namespace.metadata.name == expected_name + response = cluster.delete_namespace(namespace.metadata.name) + assert response.status == "{'phase': 'Terminating'}" + + def test_delete_namespaces_with_label(self, cluster, test_multiple_namespaces): + expected_namespaces = test_multiple_namespaces[0] + label_selector = test_multiple_namespaces[1] + namespaces = cluster.get_namespaces(label_selector=label_selector).items + responses = cluster.delete_namespaces_with_label(label_selector=label_selector) + assert expected_namespaces == [namespace.metadata.name for namespace in namespaces] + assert len(responses) == len(namespaces) + assert all([response.status == "{'phase': 'Terminating'}" for response in responses]) diff --git a/cli/tests/commands/test_benchmark_commands.py b/cli/tests/commands/test_benchmark_commands.py new file mode 100755 index 000000000..c2b2ecb1c --- /dev/null +++ b/cli/tests/commands/test_benchmark_commands.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest + + +@pytest.mark.integration +class TestBenchmarkCommands: + def test_run_benchmark(self): + pass + + def test_delete_benchmark(self): + pass diff --git a/cli/tests/commands/test_operator_commands.py b/cli/tests/commands/test_operator_commands.py new file mode 100755 index 000000000..478a5f44f --- /dev/null +++ b/cli/tests/commands/test_operator_commands.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from ripsaw.commands import operator + + +@pytest.mark.integration +class TestOperatorCommands: + def test_operator_commands(self, kind_kubeconfig, cluster, benchmark_namespace): + operator.install(kubeconfig=kind_kubeconfig) + pods = cluster.get_pods( + label_selector="control-plane=controller-manager", namespace="benchmark-operator" + ) + assert len(pods.items) == 1 + assert pods.items[0].status.phase == "Running" + operator.delete(kubeconfig=kind_kubeconfig) + pods = cluster.get_pods( + label_selector="control-plane=controller-manager", namespace="benchmark-operator" + ) + assert len(pods.items) == 0 diff --git a/cli/tests/conftest.py b/cli/tests/conftest.py new file mode 100755 index 000000000..04c37ee32 --- /dev/null +++ b/cli/tests/conftest.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import pytest +import pytest_kind # noqa: F401 +from ripsaw.clients.k8s import Cluster +from ripsaw.models.benchmark import Benchmark + +# Kind Cluster Fixtures + + +@pytest.fixture(scope="session") +def cluster(kind_cluster): + time.sleep(15) + return Cluster(kubeconfig_path=str(kind_cluster.kubeconfig_path.resolve())) + + +@pytest.fixture(scope="session") +def kind_kubeconfig(kind_cluster): + return str(kind_cluster.kubeconfig_path.resolve()) + + +@pytest.fixture(scope="function") +def test_job(kind_cluster): + yield kind_cluster.kubectl("apply", "-f", "tests/resources/job.yaml") + kind_cluster.kubectl("delete", "-f", "tests/resources/job.yaml", "--ignore-not-found") + + +@pytest.fixture(scope="session") +def benchmark_crd(kind_cluster): + yield kind_cluster.kubectl("apply", "-f", "../config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml") + kind_cluster.kubectl( + "delete", "-f", "../config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml", "--ignore-not-found" + ) + + +@pytest.fixture(scope="session") +def benchmark_namespace(kind_cluster): + yield kind_cluster.kubectl("create", "namespace", "benchmark-operator") + kind_cluster.kubectl("delete", "namespace", "benchmark-operator", "--ignore-not-found") + + +@pytest.fixture(scope="function") +def test_benchmark(kind_cluster, benchmark_crd, benchmark_namespace): + yield kind_cluster.kubectl("apply", "-f", "tests/resources/benchmark.yaml") + kind_cluster.kubectl("delete", "-f", "tests/resources/benchmark.yaml", "--ignore-not-found") + + +@pytest.fixture(scope="function") +def test_benchmark_path(kind_cluster, benchmark_crd, benchmark_namespace): + return "tests/resources/benchmark.yaml" + + +@pytest.fixture(scope="function") +def test_namespace(kind_cluster): + name = "test-namespace" + label = "test=true" + kind_cluster.kubectl("create", "namespace", name) + kind_cluster.kubectl("label", "namespaces", name, label) + yield name, label + kind_cluster.kubectl("delete", "namespace", name, "--ignore-not-found") + + +@pytest.fixture(scope="function") +def test_multiple_namespaces(kind_cluster): + name = "test-namespace" + label = "multi-namespace=true" + namespaces = f"{name}-1 {name}-2 {name}-3".split(" ") + [kind_cluster.kubectl("create", "namespace", namespace) for namespace in namespaces] + [kind_cluster.kubectl("label", "namespaces", namespace, label) for namespace in namespaces] + yield namespaces, label + [kind_cluster.kubectl("delete", "namespace", namespace, "--ignore-not-found") for namespace in namespaces] + + +@pytest.fixture(scope="function") +def test_benchmark_model(kind_cluster, cluster, test_benchmark_path): + yield Benchmark(test_benchmark_path, cluster) + kind_cluster.kubectl("delete", "benchmark", "byowl", "-n", "benchmark-operator", "--ignore-not-found") diff --git a/cli/tests/models/test_benchmark.py b/cli/tests/models/test_benchmark.py new file mode 100755 index 000000000..a68e7a178 --- /dev/null +++ b/cli/tests/models/test_benchmark.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest + + +@pytest.mark.integration +class TestBenchmark: + def test_run(self): + pass + + def test_wait(self): + pass + + def test_cleanup(self): + pass + + def test_get_metadata(self): + pass diff --git a/cli/tests/models/test_workload.py b/cli/tests/models/test_workload.py new file mode 100755 index 000000000..5a9a9166d --- /dev/null +++ b/cli/tests/models/test_workload.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest + + +@pytest.mark.integration +class TestWorkload: + def test_run(self): + pass + + def test_run_all(self): + pass + + def test_inject_overrides(self): + pass diff --git a/cli/tests/resources/benchmark.yaml b/cli/tests/resources/benchmark.yaml new file mode 100644 index 000000000..bc226fd1b --- /dev/null +++ b/cli/tests/resources/benchmark.yaml @@ -0,0 +1,16 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: byowl + namespace: benchmark-operator +spec: + system_metrics: + collection: false + metadata: + collection: false + workload: + name: byowl + args: + image: "quay.io/jtaleric/uperf:testing" + clients: 1 + commands: "echo Test" diff --git a/cli/tests/resources/job.yaml b/cli/tests/resources/job.yaml new file mode 100755 index 000000000..6f7f611ef --- /dev/null +++ b/cli/tests/resources/job.yaml @@ -0,0 +1,19 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: busybox + namespace: default + labels: + app: test-job +spec: + template: + spec: + containers: + - image: busybox + command: + - sleep + - "3600" + imagePullPolicy: IfNotPresent + name: busybox + restartPolicy: Never + terminationGracePeriodSeconds: 0 diff --git a/cli/tox.ini b/cli/tox.ini new file mode 100644 index 000000000..c8b771b20 --- /dev/null +++ b/cli/tox.ini @@ -0,0 +1,19 @@ +[tox] +envlist = + py{38,39}{-unit} +skip_missing_interpreters = true + +[testenv:py{38,39}-unit] +extras = + tests +setenv = + py{38,39}-unit: COVERAGE_FILE = .coverage.{envname} +commands = + pytest -m unit --cov-config=.coveragerc --cov=ripsaw --cov-report term-missing {posargs} + +[testenv:coverage] +skip_install = true +deps = coverage +commands = + coverage combine + coverage xml diff --git a/cli/version.txt b/cli/version.txt new file mode 100644 index 000000000..8acdd82b7 --- /dev/null +++ b/cli/version.txt @@ -0,0 +1 @@ +0.0.1 diff --git a/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml similarity index 57% rename from resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml rename to config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml index 417f46c0b..27df9b3c0 100644 --- a/resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml +++ b/config/crd/bases/ripsaw.cloudbulldozer.io_benchmarks.yaml @@ -21,19 +21,36 @@ spec: spec: type: object properties: - uuid: + uuid: type: string elasticsearch: + default: {} type: object properties: url: type: string + default: "" index_name: type: string parallel: type: boolean + default: false verify_cert: type: boolean + default: true + snappy: + default: {} + type: object + properties: + url: + type: string + default: "" + user: + type: string + default: "" + password: + type: string + default: "" prometheus: type: object properties: @@ -45,9 +62,51 @@ spec: type: string prom_token: type: string + system_metrics: + default: {} + type: object + properties: + collection: + type: boolean + default: false + es_url: + type: string + index_name: + type: string + default: system-metrics + step: + type: string + default: 30s + metrics_profile: + type: string + default: node-metrics.yml + prom_url: + type: string + default: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + image: + type: string + default: quay.io/cloud-bulldozer/kube-burner:latest + prom_token: + type: string workload: x-kubernetes-preserve-unknown-fields: true type: object + properties: + name: + type: string + description: Workload name + args: + x-kubernetes-preserve-unknown-fields: true + type: object + properties: + debug: + type: boolean + default: false + description: Set debug log level in the workload + job_timeout: + type: integer + description: Timeout used for jobs deployed by this benchmark + default: 3600 job_params: type: array items: @@ -76,9 +135,6 @@ spec: privileged: default: false type: boolean - serviceaccount: - default: default - type: string stockpileSkipTags: default: [] items: @@ -96,8 +152,6 @@ spec: default: false type: boolean type: object - cerberus_url: - type: string cleanup: type: boolean test_user: @@ -108,21 +162,44 @@ spec: type: string drop_cache_kernel: type: boolean + drop_cache_rook_ceph: + type: boolean status: + default: {} type: object properties: complete: type: boolean + default: false + description: Benchmark is completed metadata: type: string + description: Metadata collection status state: type: string + description: Benchmark status suuid: type: string + description: Benchmark short UUID uuid: type: string - cerberus: + description: Benchmark UUID + pod_hi_idx: + type: string + pod_low_idx: + type: string + node_hi_idx: + type: string + node_low_idx: + type: string + pod_idx: + type: string + node_idx: + type: string + system_metrics: type: string + default: Not collected + description: System metrics collection status additionalPrinterColumns: - name: Type type: string @@ -136,10 +213,10 @@ spec: type: string description: The state of metadata collection jsonPath: .status.metadata - - name: Cerberus + - name: System metrics type: string - description: If Cerberus is connected or not - jsonPath: .status.cerberus + description: System metrics collect status + jsonPath: .status.system_metrics - name: UUID type: string jsonPath: .status.uuid diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 000000000..cdd73b4f8 --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,6 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/ripsaw.cloudbulldozer.io_benchmarks.yaml +#+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml new file mode 100644 index 000000000..0b6d73728 --- /dev/null +++ b/config/default/kustomization.yaml @@ -0,0 +1,30 @@ +# Adds namespace to all resources. +namespace: benchmark-operator + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: benchmark- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +bases: +- ../crd +- ../rbac +- ../manager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus + +patchesStrategicMerge: [] +# Protect the /metrics endpoint by putting it behind auth. +# If you want your controller-manager to expose the /metrics +# endpoint w/o any authn/z, please comment the following line. +# - manager_auth_proxy_patch.yaml + +# Mount the controller config file for loading manager configurations +# through a ComponentConfig type +#- manager_config_patch.yaml diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml new file mode 100644 index 000000000..151724322 --- /dev/null +++ b/config/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,27 @@ +# This patch inject a sidecar container which is a HTTP proxy for the +# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=10" + ports: + - containerPort: 8443 + name: https + - name: manager + args: + - "--health-probe-bind-address=:6789" + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" + - "--leader-election-id=ripsaw" diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml new file mode 100644 index 000000000..6c400155c --- /dev/null +++ b/config/default/manager_config_patch.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--config=controller_manager_config.yaml" + volumeMounts: + - name: manager-config + mountPath: /controller_manager_config.yaml + subPath: controller_manager_config.yaml + volumes: + - name: manager-config + configMap: + name: manager-config diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml new file mode 100644 index 000000000..7d61df648 --- /dev/null +++ b/config/manager/controller_manager_config.yaml @@ -0,0 +1,10 @@ +apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 +kind: ControllerManagerConfig +health: + healthProbeBindAddress: :6789 +metrics: + bindAddress: 127.0.0.1:8080 + +leaderElection: + leaderElect: true + resourceName: 811c9dc5.cloudbulldozer.io diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml new file mode 100644 index 000000000..f6241927a --- /dev/null +++ b/config/manager/kustomization.yaml @@ -0,0 +1,16 @@ +resources: +- manager.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- files: + - controller_manager_config.yaml + name: manager-config +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: quay.io/cloud-bulldozer/benchmark-operator + newTag: latest diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml new file mode 100644 index 000000000..234917434 --- /dev/null +++ b/config/manager/manager.yaml @@ -0,0 +1,92 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: operator +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + labels: + control-plane: controller-manager + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/workload + operator: Exists + containers: + - name: manager + args: + - --leader-elect + - --leader-election-id=ripsaw + image: controller:latest + imagePullPolicy: Always + env: + - name: ANSIBLE_GATHERING + value: explicit + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: "benchmark-operator" + - name: MAX_CONCURRENT_RECONCILES_BENCHMARK_RIPSAW_CLOUDBULLDOZER_IO + value: "4" + - name: ANSIBLE_VERBOSITY_BENCHMARK_RIPSAW_CLOUDBULLDOZER_IO + value: "4" + securityContext: + allowPrivilegeEscalation: false + livenessProbe: + httpGet: + path: /healthz + port: 6789 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 6789 + initialDelaySeconds: 5 + periodSeconds: 10 + - name: redis-master + image: quay.io/cloud-bulldozer/redis:latest + env: + - name: MASTER + value: "true" + - name: ALLOW_EMPTY_PASSWORD + value: "yes" + ports: + - containerPort: 6379 + resources: + requests: + cpu: "100m" + limits: + cpu: "2.0" + volumeMounts: + - mountPath: /data + name: data + serviceAccountName: benchmark-operator + terminationGracePeriodSeconds: 10 + volumes: + - name: data + emptyDir: {} diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml new file mode 100644 index 000000000..2dc3e9a73 --- /dev/null +++ b/config/manifests/kustomization.yaml @@ -0,0 +1,7 @@ +# These resources constitute the fully configured set of manifests +# used to generate the 'manifests/' directory in a bundle. +resources: +- bases/ripsaw.clusterserviceversion.yaml +- ../default +- ../samples +- ../scorecard diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml new file mode 100644 index 000000000..ed137168a --- /dev/null +++ b/config/prometheus/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- monitor.yaml diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml new file mode 100644 index 000000000..d19136ae7 --- /dev/null +++ b/config/prometheus/monitor.yaml @@ -0,0 +1,20 @@ + +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager diff --git a/resources/self_provisioner_binding.yaml b/config/rbac/clusterrolebinding.yaml similarity index 63% rename from resources/self_provisioner_binding.yaml rename to config/rbac/clusterrolebinding.yaml index ae3a7484b..2996ee665 100644 --- a/resources/self_provisioner_binding.yaml +++ b/config/rbac/clusterrolebinding.yaml @@ -1,12 +1,14 @@ -apiVersion: rbac.authorization.k8s.io/v1 +--- kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: benchmark-operator-self-provisioner + name: operator-binding + namespace: benchmark-operator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: self-provisioner + name: cluster-admin subjects: - kind: ServiceAccount name: benchmark-operator - namespace: my-ripsaw + namespace: benchmark-operator diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml new file mode 100644 index 000000000..af5e06965 --- /dev/null +++ b/config/rbac/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- service_account.yaml +- clusterrolebinding.yaml diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml new file mode 100644 index 000000000..36e029d5b --- /dev/null +++ b/config/rbac/service_account.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operator + namespace: benchmark-operator diff --git a/resources/crds/ripsaw_v1alpha1_byowl_cr.yaml b/config/samples/byowl/cr.yaml similarity index 89% rename from resources/crds/ripsaw_v1alpha1_byowl_cr.yaml rename to config/samples/byowl/cr.yaml index 467fa3113..bf5d7a506 100644 --- a/resources/crds/ripsaw_v1alpha1_byowl_cr.yaml +++ b/config/samples/byowl/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: byowl-benchmark-example - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: byowl diff --git a/config/samples/concurrent-builds/cr.yaml b/config/samples/concurrent-builds/cr.yaml new file mode 100644 index 000000000..90ca8a5be --- /dev/null +++ b/config/samples/concurrent-builds/cr.yaml @@ -0,0 +1,77 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-concurrent-builds-example + namespace: benchmark-operator +spec: + # Metadata information + elasticsearch: + # Elastic search instance with full URL format. https://elastic.apps.org:9200 + url: http://my.elasticsearch.server:80 + prometheus: + # Elastic search instance with full URL format. https://elastic.apps.org:9200 + es_url: http://my.elasticsearch.server:80 + # Prometheus bearer token + prom_token: PROMETHEUS_BEARER_TOKEN + # Prometheus URL with full URL format. https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + # Workload name + workload: concurrent-builds + # Kube-burner Job timeout + job_timeout: 7200 + # ES index name + default_index: ripsaw-kube-burner + # Number of job iterations + job_iterations: 100 + # Pin kube-burner to a node using the value of the label kubernetes.io/hostname + pin_server: {"node-role.kubernetes.io/worker": ""} + # Wait for pods to be runnig before finishing kube-burner workload + wait_when_finished: true + # Wait for all pods to be running before moving forward to the next job iteration + pod_wait: false + # Use a custom kube-burner image + image: quay.io/cloud-bulldozer/kube-burner:latest + # Queries per second + qps: 25 + # Max number of burst queries to perform + burst: 25 + # Log level. Allowed, info and debug + log_level: info + # Delete old namespaces for the selected workload before starting benchmark + cleanup: true + # Verify object creation + verify_objects: true + # Exit w/o indexing if a verify error happened + error_on_verify: true + # Prometheus step size + step: 30s + # kube-burner metrics profile + metrics_profile: metrics.yaml + # Remote configuration file + # remote_config: http://yourdomain/kube-burner.yml + # Remote metrics profile + # remote_metrics_profile: http://yourdomain/metrics-profile.yml + # kube-burner pod tolerations + # Application to build + app: django + source_strat_env: PIP_INDEX_URL + source_strat_from: python + source_strat_from_version: latest + # Script to run after build + post_commit_script: "./manage.py test" + # Build Image name + build_image_stream: django-psql-example + # Git url for application + git_url: https://github.com/sclorg/django-ex.git + build_image: image-registry.openshift-image-registry.svc:5000/svt-django/django-psql-example + tolerations: + - key: role + value: workload + effect: NoSchedule + # Pod nodeSelector + node_selector: + key: node-role.kubernetes.io/worker + value: diff --git a/resources/crds/ripsaw_v1alpha1_cyclictest_cr.yaml b/config/samples/cyclictest/cr.yaml similarity index 94% rename from resources/crds/ripsaw_v1alpha1_cyclictest_cr.yaml rename to config/samples/cyclictest/cr.yaml index 60ad70e27..618ae1569 100644 --- a/resources/crds/ripsaw_v1alpha1_cyclictest_cr.yaml +++ b/config/samples/cyclictest/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: cyclictest-example - namespace: my-ripsaw + namespace: benchmark-operator spec: # where elastic search is running elasticsearch: diff --git a/resources/crds/ripsaw_v1alpha1_fio_distributed_cr.yaml b/config/samples/fio/cr.yaml similarity index 89% rename from resources/crds/ripsaw_v1alpha1_fio_distributed_cr.yaml rename to config/samples/fio/cr.yaml index 679018358..c6e1c19c5 100644 --- a/resources/crds/ripsaw_v1alpha1_fio_distributed_cr.yaml +++ b/config/samples/fio/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: fio-benchmark-example - namespace: my-ripsaw + namespace: benchmark-operator spec: # where elastic search is running elasticsearch: @@ -53,8 +53,12 @@ spec: log_sample_rate: 3000 #storageclass: rook-ceph-block #storagesize: 5Gi - #rook_ceph_drop_caches: True - #rook_ceph_drop_cache_pod_ip: 192.168.111.20 + # use drop_cache_kernel to have set of labeled nodes drop kernel buffer cache before each sample + #drop_cache_kernel: False + # use drop_cache_rook_ceph to have Ceph OSDs drop their cache before each sample + #drop_cache_rook_ceph: False + # increase this if you want fio to run for more than 1 hour without being terminated by K8S + #job_timeout: 3600 ####################################### # EXPERT AREA - MODIFY WITH CAUTION # ####################################### diff --git a/resources/crds/ripsaw_v1alpha1_fio_distributed_vm_cr.yaml b/config/samples/fio/vm-cr.yaml similarity index 96% rename from resources/crds/ripsaw_v1alpha1_fio_distributed_vm_cr.yaml rename to config/samples/fio/vm-cr.yaml index 5855a19c6..2a39af1a9 100644 --- a/resources/crds/ripsaw_v1alpha1_fio_distributed_vm_cr.yaml +++ b/config/samples/fio/vm-cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: fio-vm-benchmark-example - namespace: my-ripsaw + namespace: benchmark-operator spec: # where elastic search is running elasticsearch: @@ -33,7 +33,10 @@ spec: # vm_cores: 2 # Memory that will be available inside the VM # Default: 5G - # vm_cores: 10G + # vm_memory: 10G + # VM bus type: virtio, sata, or scsi + # Default: virtio + # vm_bus: virtio # test types, see fio documentation jobs: - write @@ -67,8 +70,6 @@ spec: # Can be one of Filesystem,Block Default: Filesystem pvcvolumemode: Block storagesize: 5Gi - #rook_ceph_drop_caches: True - #rook_ceph_drop_cache_pod_ip: 192.168.111.20 ####################################### # EXPERT AREA - MODIFY WITH CAUTION # ####################################### diff --git a/resources/crds/ripsaw_v1alpha1_flent_cr.yaml b/config/samples/flent/cr.yaml similarity index 93% rename from resources/crds/ripsaw_v1alpha1_flent_cr.yaml rename to config/samples/flent/cr.yaml index 5dc9bb680..96cae8e2d 100644 --- a/resources/crds/ripsaw_v1alpha1_flent_cr.yaml +++ b/config/samples/flent/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: flent-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: myk8scluster elasticsearch: diff --git a/resources/crds/ripsaw_v1alpha1_fs_drift_cr.yaml b/config/samples/fs-drift/cr.yaml similarity index 97% rename from resources/crds/ripsaw_v1alpha1_fs_drift_cr.yaml rename to config/samples/fs-drift/cr.yaml index e1f5170d8..31f3be7b6 100644 --- a/resources/crds/ripsaw_v1alpha1_fs_drift_cr.yaml +++ b/config/samples/fs-drift/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: fs-drift-example - namespace: my-ripsaw + namespace: benchmark-operator spec: # where elastic search is running elasticsearch: diff --git a/config/samples/hammerdb/cr.yaml b/config/samples/hammerdb/cr.yaml new file mode 100644 index 000000000..e72bd1c34 --- /dev/null +++ b/config/samples/hammerdb/cr.yaml @@ -0,0 +1,78 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-benchmark + namespace: benchmark-operator +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + #image: quay.io/user/hammerdb:latest # add custom hammerdb image + pin: false # true for nodeSelector + pin_node: "node1" + resources: false # true for resources requests/limits + requests_cpu: 200m + requests_memory: 100Mi + limits_cpu: 4 + limits_memory: 16Gi + db_type: "mssql" + timed_test: true + test_type: "tpc-c" + db_init: true # true only for first run to build schema + db_benchmark: true + db_server: "mssql-deployment.mssql-db" + db_port: "1433" + db_warehouses: 1 + db_num_workers: 1 + db_user: "SA" + db_pass: "s3curePasswordString" + db_name: "tpcc" + transactions: 10000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # ElasticSearch custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "2019" + es_os_version: "centos8" + es_kind: "pod" \ No newline at end of file diff --git a/config/samples/hammerdb/mariadb/cr.yaml b/config/samples/hammerdb/mariadb/cr.yaml new file mode 100644 index 000000000..48405675d --- /dev/null +++ b/config/samples/hammerdb/mariadb/cr.yaml @@ -0,0 +1,78 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-benchmark + namespace: benchmark-operator +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + #image: quay.io/user/hammerdb:latest # add custom hammerdb image + pin: false # true for nodeSelector + pin_node: "node1" + resources: false # true for resources requests/limits + requests_cpu: 200m + requests_memory: 100Mi + limits_cpu: 4 + limits_memory: 16Gi + db_type: "mariadb" + timed_test: true + test_type: "tpc-c" + db_init: true # true only for first run to build schema + db_benchmark: true + db_server: "mariadb-deployment.mariadb-db" + db_port: "3306" + db_warehouses: 1 + db_num_workers: 1 + db_user: "root" + db_pass: "mysql" + db_name: "tpcc" + transactions: 10000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # ElasticSearch custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "10.3.0" + es_os_version: "centos8" + es_kind: "pod" diff --git a/config/samples/hammerdb/mariadb/hostpath-cr.yaml b/config/samples/hammerdb/mariadb/hostpath-cr.yaml new file mode 100644 index 000000000..71dded4cf --- /dev/null +++ b/config/samples/hammerdb/mariadb/hostpath-cr.yaml @@ -0,0 +1,123 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: mariadb-db +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-custom-config + namespace: mariadb-db +data: + custom.conf: | + # This group is read both both by the client and the server + # use it for options that affect everything + # + [client-server] + # + # include all files from the config directory + # + !includedir /etc/my.cnf.d + port=3306 + ssl=0 + [mysqld] + innodb_file_per_table=1 + innodb_flush_method=O_DIRECT + innodb_adaptive_flushing=1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mariadb-deployment + namespace: mariadb-db +spec: + selector: + matchLabels: + app: mariadb + template: + metadata: + labels: + app: mariadb + type: mariadb-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "worker-0" + terminationGracePeriodSeconds: 10 + containers: + - name: mariadb + image: centos/mariadb-103-centos8:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 3306 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 4 + memory: 16Gi + env: + - name: MYSQL_USER + value: "test" + - name: MYSQL_PASSWORD + value: "test" + - name: MYSQL_ROOT_PASSWORD + value: "mysql" + securityContext: + privileged: true + volumeMounts: + - name: mariadb-custom-config + mountPath: /etc/my.cnf + subPath: custom.conf #should be the name used in the ConfigMap + - name: maria-persistent-storage + mountPath: /var/lib/mysql + readOnly: false + volumes: + - name: mariadb-custom-config + configMap: + name: mariadb-custom-config + - name: maria-persistent-storage + persistentVolumeClaim: + claimName: maria-persistent-storage +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvdb + labels: + type: local +spec: + persistentVolumeReclaimPolicy: Delete + storageClassName: manual + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /tmp +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: maria-persistent-storage + namespace: mariadb-db +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: manual +--- +apiVersion: v1 +kind: Service +metadata: + name: mariadb-deployment + namespace: mariadb-db +spec: + selector: + app: mariadb + ports: + - protocol: TCP + port: 3306 + targetPort: 3306 diff --git a/config/samples/hammerdb/mariadb/local-cr.yaml b/config/samples/hammerdb/mariadb/local-cr.yaml new file mode 100644 index 000000000..794695bd6 --- /dev/null +++ b/config/samples/hammerdb/mariadb/local-cr.yaml @@ -0,0 +1,133 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: mariadb-db +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-custom-config + namespace: mariadb-db +data: + custom.conf: | + # This group is read both both by the client and the server + # use it for options that affect everything + # + [client-server] + # + # include all files from the config directory + # + !includedir /etc/my.cnf.d + port=3306 + ssl=0 + [mysqld] + innodb_file_per_table=1 + innodb_flush_method=O_DIRECT + innodb_adaptive_flushing=1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mariadb-deployment + namespace: mariadb-db +spec: + selector: + matchLabels: + app: mariadb + template: + metadata: + labels: + app: mariadb + type: mariadb-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "master-0" + terminationGracePeriodSeconds: 10 + containers: + - name: mariadb + image: centos/mariadb-103-centos8:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 3306 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 4 + memory: 16Gi + env: + - name: MYSQL_USER + value: "test" + - name: MYSQL_PASSWORD + value: "test" + - name: MYSQL_ROOT_PASSWORD + value: "mysql" + securityContext: + privileged: true + volumeMounts: + - name: mariadb-custom-config + mountPath: /etc/my.cnf + subPath: custom.conf #should be the name used in the ConfigMap + - name: maria-persistent-storage + mountPath: /var/lib/mysql + readOnly: false + volumes: + - name: mariadb-custom-config + configMap: + name: mariadb-custom-config + - name: maria-persistent-storage + persistentVolumeClaim: + claimName: maria-persistent-storage +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvdb +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + storageClassName: localblock + local: + # mkdir inside node + path: /var/lib/mysql + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - master-0 +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: maria-persistent-storage + namespace: mariadb-db +spec: + storageClassName: localblock + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: mariadb-deployment + namespace: mariadb-db +spec: + selector: + app: mariadb + ports: + - protocol: TCP + port: 3306 + targetPort: 3306 +--- \ No newline at end of file diff --git a/config/samples/hammerdb/mariadb/pvc-cr.yaml b/config/samples/hammerdb/mariadb/pvc-cr.yaml new file mode 100644 index 000000000..a5a633ab6 --- /dev/null +++ b/config/samples/hammerdb/mariadb/pvc-cr.yaml @@ -0,0 +1,106 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: mariadb-db +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-custom-config + namespace: mariadb-db +data: + custom.conf: | + # This group is read both both by the client and the server + # use it for options that affect everything + # + [client-server] + # + # include all files from the config directory + # + !includedir /etc/my.cnf.d + port=3306 + ssl=0 + [mysqld] + innodb_file_per_table=1 + innodb_flush_method=O_DIRECT + innodb_adaptive_flushing=1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mariadb-deployment + namespace: mariadb-db +spec: + selector: + matchLabels: + app: mariadb + template: + metadata: + labels: + app: mariadb + type: mariadb-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "worker-0" + terminationGracePeriodSeconds: 10 + containers: + - name: mariadb + image: centos/mariadb-103-centos8:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 3306 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 4 + memory: 16Gi + env: + - name: MYSQL_USER + value: "test" + - name: MYSQL_PASSWORD + value: "test" + - name: MYSQL_ROOT_PASSWORD + value: "mysql" + volumeMounts: + - name: mariadb-custom-config + mountPath: /etc/my.cnf + subPath: custom.conf #should be the name used in the ConfigMap + - name: mariadb-persistent-storage + mountPath: /var/lib/mysql + readOnly: false + volumes: + - name: mariadb-custom-config + configMap: + name: mariadb-custom-config + - name: mariadb-persistent-storage + persistentVolumeClaim: + claimName: mariadb-persistent-storage +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mariadb-persistent-storage + namespace: mariadb-db +spec: + storageClassName: ocs-storagecluster-ceph-rbd + accessModes: [ "ReadWriteOnce" ] + volumeMode: Filesystem + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: mariadb-deployment + namespace: mariadb-db +spec: + selector: + app: mariadb + ports: + - protocol: TCP + port: 3306 + targetPort: 3306 +--- diff --git a/config/samples/hammerdb/mariadb/vm-cr.yaml b/config/samples/hammerdb/mariadb/vm-cr.yaml new file mode 100644 index 000000000..f0841a497 --- /dev/null +++ b/config/samples/hammerdb/mariadb/vm-cr.yaml @@ -0,0 +1,105 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-vm-benchmark + namespace: benchmark-operator +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + # image: "quay.io/test/hammerdb:latest" # add custom hammerdb image + pin: false # enable for nodeSelector + pin_node: "node1" + db_type: "mariadb" + timed_test: true + test_type: "tpc-c" + # true only for first run to build schema + db_init: true + db_benchmark: true + db_server: "127.0.0.1" + db_port: "3306" + db_warehouses: 1 + db_num_workers: 1 + db_user: "root" + db_pass: "mysql" + db_name: "tpcc" + transactions: 10000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # ElasticSearch custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "10.3.0" + es_os_version: "centos8" + es_kind: "vm" + kind: vm + client_vm: + dedicatedcpuplacement: false + sockets: 1 + cores: 2 + threads: 1 + image: quay.io/cloud-bulldozer/centos-stream8-mariadb103-container-disk:latest + requests: + memory: 100Mi + limits: + memory: 16Gi # at least 16Gi for VM + network: + front_end: bridge # or masquerade + multiqueue: + enabled: false # if set to true, highly recommend to set selinux to permissive on the nodes where the vms would be scheduled + queues: 0 # must be given if enabled is set to true and ideally should be set to vcpus ideally so sockets*threads*cores, your image must've ethtool installed + extra_options: + - none + #- hostpassthrough + ## OCS PVC + pvc: false # enable for OCS PVC + pvc_storageclass: ocs-storagecluster-ceph-rbd + # Can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce + pvc_pvcaccessmode: ReadWriteMany + # Can be one of Filesystem,Block Default: Filesystem + pvc_pvcvolumemode: Block + pvc_storagesize: 10Gi + ## HostPath - Configuring SELinux on cluster workers + hostpath: false # enable for hostPath + hostpath_path: /var/tmp/disk.img + hostpath_storagesize: 10Gi diff --git a/config/samples/hammerdb/mssql/cr.yaml b/config/samples/hammerdb/mssql/cr.yaml new file mode 100644 index 000000000..e72bd1c34 --- /dev/null +++ b/config/samples/hammerdb/mssql/cr.yaml @@ -0,0 +1,78 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-benchmark + namespace: benchmark-operator +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + #image: quay.io/user/hammerdb:latest # add custom hammerdb image + pin: false # true for nodeSelector + pin_node: "node1" + resources: false # true for resources requests/limits + requests_cpu: 200m + requests_memory: 100Mi + limits_cpu: 4 + limits_memory: 16Gi + db_type: "mssql" + timed_test: true + test_type: "tpc-c" + db_init: true # true only for first run to build schema + db_benchmark: true + db_server: "mssql-deployment.mssql-db" + db_port: "1433" + db_warehouses: 1 + db_num_workers: 1 + db_user: "SA" + db_pass: "s3curePasswordString" + db_name: "tpcc" + transactions: 10000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # ElasticSearch custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "2019" + es_os_version: "centos8" + es_kind: "pod" \ No newline at end of file diff --git a/config/samples/hammerdb/mssql/hostpath-cr.yaml b/config/samples/hammerdb/mssql/hostpath-cr.yaml new file mode 100644 index 000000000..6f00b05af --- /dev/null +++ b/config/samples/hammerdb/mssql/hostpath-cr.yaml @@ -0,0 +1,96 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: mssql-db +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mssql-deployment + namespace: mssql-db +spec: + replicas: 1 + selector: + matchLabels: + app: mssql + template: + metadata: + labels: + app: mssql + type: mssql-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "worker-0" + terminationGracePeriodSeconds: 10 + containers: + - name: mssql + image: mcr.microsoft.com/mssql/rhel/server:2019-latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 1433 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 4 + memory: 16Gi + env: + - name: MSSQL_PID + value: "Enterprise" + - name: ACCEPT_EULA + value: "Y" + - name: MSSQL_SA_PASSWORD + value: "s3curePasswordString" + securityContext: + privileged: true + volumeMounts: + - name: mssql-persistent-storage + mountPath: /var/opt/mssql + readOnly: false + volumes: + - name: mssql-persistent-storage + persistentVolumeClaim: + claimName: mssql-persistent-storage +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvdb + labels: + type: local +spec: + persistentVolumeReclaimPolicy: Delete + storageClassName: manual + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /tmp +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mssql-persistent-storage + namespace: mssql-db +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: manual +--- +apiVersion: v1 +kind: Service +metadata: + name: mssql-deployment + namespace: mssql-db +spec: + selector: + app: mssql + ports: + - protocol: TCP + port: 1433 + targetPort: 1433 diff --git a/config/samples/hammerdb/mssql/local-cr.yaml b/config/samples/hammerdb/mssql/local-cr.yaml new file mode 100644 index 000000000..e147debbc --- /dev/null +++ b/config/samples/hammerdb/mssql/local-cr.yaml @@ -0,0 +1,105 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: mssql-db +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mssql-deployment + namespace: mssql-db +spec: + replicas: 1 + selector: + matchLabels: + app: mssql + template: + metadata: + labels: + app: mssql + type: mssql-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "master-0" + terminationGracePeriodSeconds: 10 + containers: + - name: mssql + image: mcr.microsoft.com/mssql/rhel/server:2019-latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 1433 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 4 + memory: 16Gi + env: + - name: MSSQL_PID + value: "Enterprise" + - name: ACCEPT_EULA + value: "Y" + - name: MSSQL_SA_PASSWORD + value: "s3curePasswordString" + securityContext: + privileged: true + volumeMounts: + - name: mssql-persistent-storage + mountPath: /var/opt/mssql + readOnly: false + volumes: + - name: mssql-persistent-storage + persistentVolumeClaim: + claimName: mssql-persistent-storage +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvdb +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + storageClassName: localblock + local: + # mkdir inside node + path: /var/opt/mssql + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - master-0 +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mssql-persistent-storage + namespace: mssql-db +spec: + storageClassName: localblock + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: mssql-deployment + namespace: mssql-db +spec: + selector: + app: mssql + ports: + - protocol: TCP + port: 1433 + targetPort: 1433 \ No newline at end of file diff --git a/config/samples/hammerdb/mssql/vm-cr.yaml b/config/samples/hammerdb/mssql/vm-cr.yaml new file mode 100644 index 000000000..65970581a --- /dev/null +++ b/config/samples/hammerdb/mssql/vm-cr.yaml @@ -0,0 +1,105 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-vm-benchmark + namespace: benchmark-operator +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + #image: quay.io/user/hammerdb:latest # add custom hammerdb image + pin: false # true for nodeSelector + pin_node: "node1" + db_type: "mssql" + timed_test: true + test_type: "tpc-c" + db_init: true # true only for first run to build schema + db_benchmark: true + db_server: "127.0.0.1" + db_port: "1433" + db_warehouses: 1 + db_num_workers: 1 + db_user: "SA" + db_pass: "s3curePasswordString" + db_name: "tpcc" + transactions: 10000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # ElasticSearch custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "2019" + es_os_version: "centos8" + es_kind: "vm" + kind: vm + client_vm: + dedicatedcpuplacement: false + sockets: 1 + cores: 2 + threads: 1 + image: quay.io/cloud-bulldozer/centos-stream8-mssql2019-container-disk:latest + requests: + memory: 100Mi + limits: + memory: 16Gi # at least 16Gi for VM + network: + front_end: bridge # or masquerade + multiqueue: + enabled: false # if set to true, highly recommend to set selinux to permissive on the nodes where the vms would be scheduled + queues: 0 # must be given if enabled is set to true and ideally should be set to vcpus ideally so sockets*threads*cores, your image must've ethtool installed + extra_options: + - none + #- hostpassthrough + ## OCS PVC + pvc: false # enable for OCS PVC + pvc_storageclass: ocs-storagecluster-ceph-rbd + # Can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce + pvc_pvcaccessmode: ReadWriteMany + # Can be one of Filesystem,Block Default: Filesystem + pvc_pvcvolumemode: Block + pvc_storagesize: 10Gi + ## HostPath - Configuring SELinux on cluster workers + hostpath: false # enable for hostPath + hostpath_path: /var/tmp/disk.img + hostpath_storagesize: 10Gi + diff --git a/config/samples/hammerdb/postgres/cr.yaml b/config/samples/hammerdb/postgres/cr.yaml new file mode 100644 index 000000000..c64a29240 --- /dev/null +++ b/config/samples/hammerdb/postgres/cr.yaml @@ -0,0 +1,78 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-benchmark + namespace: benchmark-operator +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + #image: quay.io/user/hammerdb:latest # add custom hammerdb image + pin: false # true for nodeSelector + pin_node: "node1" + resources: false # true for resources requests/limits + requests_cpu: 200m + requests_memory: 100Mi + limits_cpu: 4 + limits_memory: 16Gi + db_type: "pg" + timed_test: true + test_type: "tpc-c" + db_init: true # true only for first run to build schema + db_benchmark: true + db_server: "postgres-deployment.postgres-db" + db_port: "5432" + db_warehouses: 1 + db_num_workers: 1 + db_user: "postgres" + db_pass: "postgres" + db_name: "tpcc" + transactions: 10000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "postgres" + db_postgresql_superuser_pass: "postgres" + db_postgresql_defaultdbase: "tpcc" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # ElasticSearch custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "10" + es_os_version: "centos8" + es_kind: "pod" \ No newline at end of file diff --git a/config/samples/hammerdb/postgres/hostpath-cr.yaml b/config/samples/hammerdb/postgres/hostpath-cr.yaml new file mode 100644 index 000000000..15734c035 --- /dev/null +++ b/config/samples/hammerdb/postgres/hostpath-cr.yaml @@ -0,0 +1,117 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: postgres-db +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-custom-config + namespace: postgres-db +data: + custom.pg_hba.conf: | + # TYPE DATABASE USER ADDRESS METHOD + # "local" is for Unix domain socket connections only + local all all trust + # IPv4 local connections: + host all all all trust +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-deployment + namespace: postgres-db +spec: + selector: + matchLabels: + app: postgres + replicas: 1 + template: + metadata: + labels: + app: postgres + type: postgres-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "master-0" + terminationGracePeriodSeconds: 10 + containers: + - name: postgres + image: centos/postgresql-10-centos8:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 5432 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 4 + memory: 16Gi + env: + - name: POSTGRESQL_USER + value: "test" + - name: POSTGRESQL_PASSWORD + value: "test" + - name: POSTGRESQL_DATABASE + value: "test" + - name: POSTGRESQL_ADMIN_PASSWORD + value: "postgres" + securityContext: + privileged: true + volumeMounts: + - name: postgres-custom-config + mountPath: /var/lib/pgsql/data/pg_hba.conf + subPath: custom.pg_hba.conf #should be the name used in the ConfigMap + - name: postgres-persistent-storage + mountPath: /var/lib/pgsql/data + readOnly: false + volumes: + - name: postgres-custom-config + configMap: + name: postgres-custom-config + - name: postgres-persistent-storage + persistentVolumeClaim: + claimName: postgres-persistent-storage +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvdb + labels: + type: local +spec: + persistentVolumeReclaimPolicy: Delete + storageClassName: manual + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /tmp +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: postgres-persistent-storage + namespace: postgres-db +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: manual +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres-deployment + namespace: postgres-db +spec: + selector: + app: postgres + ports: + - protocol: TCP + port: 5432 + targetPort: 5432 \ No newline at end of file diff --git a/config/samples/hammerdb/postgres/local-cr.yaml b/config/samples/hammerdb/postgres/local-cr.yaml new file mode 100644 index 000000000..f0b15004c --- /dev/null +++ b/config/samples/hammerdb/postgres/local-cr.yaml @@ -0,0 +1,126 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: postgres-db +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-custom-config + namespace: postgres-db +data: + custom.pg_hba.conf: | + # TYPE DATABASE USER ADDRESS METHOD + # "local" is for Unix domain socket connections only + local all all trust + # IPv4 local connections: + host all all all trust +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-deployment + namespace: postgres-db +spec: + selector: + matchLabels: + app: postgres + replicas: 1 + template: + metadata: + labels: + app: postgres + type: postgres-database-server + spec: + nodeSelector: + kubernetes.io/hostname: "master-0" + terminationGracePeriodSeconds: 10 + containers: + - name: postgres + image: centos/postgresql-10-centos8:latest + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 5432 + resources: + requests: + cpu: 200m + memory: 100Mi + limits: + cpu: 4 + memory: 16Gi + env: + - name: POSTGRESQL_USER + value: "test" + - name: POSTGRESQL_PASSWORD + value: "test" + - name: POSTGRESQL_DATABASE + value: "test" + - name: POSTGRESQL_ADMIN_PASSWORD + value: "postgres" + securityContext: + privileged: true + volumeMounts: + - name: postgres-custom-config + mountPath: /var/lib/pgsql/data/pg_hba.conf + subPath: custom.pg_hba.conf #should be the name used in the ConfigMap + - name: postgres-persistent-storage + mountPath: /var/lib/pgsql/data + readOnly: false + volumes: + - name: postgres-custom-config + configMap: + name: postgres-custom-config + - name: postgres-persistent-storage + persistentVolumeClaim: + claimName: postgres-persistent-storage +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvdb +spec: + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + storageClassName: localblock + local: + # mkdir inside node + path: /var/lib/pgsql/data + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - master-0 +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: postgres-persistent-storage + namespace: postgres-db +spec: + storageClassName: localblock + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres-deployment + namespace: postgres-db +spec: + selector: + app: postgres + ports: + - protocol: TCP + port: 5432 + targetPort: 5432 \ No newline at end of file diff --git a/config/samples/hammerdb/postgres/vm-cr.yaml b/config/samples/hammerdb/postgres/vm-cr.yaml new file mode 100644 index 000000000..fa136f5c9 --- /dev/null +++ b/config/samples/hammerdb/postgres/vm-cr.yaml @@ -0,0 +1,105 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-vm-benchmark + namespace: benchmark-operator +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + #image: quay.io/user/hammerdb:latest # add custom hammerdb image + pin: false # true for nodeSelector + pin_node: "node1" + db_type: "pg" + timed_test: true + test_type: "tpc-c" + db_init: true # true only for first run to build schema + db_benchmark: true + db_server: "127.0.0.1" + db_port: "5432" + db_warehouses: 1 + db_num_workers: 1 + db_user: "postgres" + db_pass: "s3curePasswordString" + db_name: "tpcc" + transactions: 500000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "postgres" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "tpcc" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # ElasticSearch custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "10" + es_os_version: "centos8" + es_kind: "vm" + kind: vm + client_vm: + dedicatedcpuplacement: false + sockets: 1 + cores: 2 + threads: 1 + image: quay.io/cloud-bulldozer/centos-stream8-postgres10-container-disk:latest + requests: + memory: 100Mi + limits: + memory: 16Gi # at least 16Gi for VM + network: + front_end: bridge # or masquerade + multiqueue: + enabled: false # if set to true, highly recommend to set selinux to permissive on the nodes where the vms would be scheduled + queues: 0 # must be given if enabled is set to true and ideally should be set to vcpus ideally so sockets*threads*cores, your image must've ethtool installed + extra_options: + - none + #- hostpassthrough + ## OCS PVC + pvc: false # enable for OCS PVC + pvc_storageclass: ocs-storagecluster-ceph-rbd + # Can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce + pvc_pvcaccessmode: ReadWriteMany + # Can be one of Filesystem,Block Default: Filesystem + pvc_pvcvolumemode: Block + pvc_storagesize: 10Gi + ## HostPath - Configuring SELinux on cluster workers + hostpath: false # enable for hostPath + hostpath_path: /var/tmp/disk.img + hostpath_storagesize: 10Gi + diff --git a/config/samples/hammerdb/vm-cr.yaml b/config/samples/hammerdb/vm-cr.yaml new file mode 100644 index 000000000..97e2ec7b0 --- /dev/null +++ b/config/samples/hammerdb/vm-cr.yaml @@ -0,0 +1,105 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb-vm-benchmark + namespace: benchmark-operator +spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false + workload: + name: hammerdb + args: + #image: quay.io/user/hammerdb:latest # add custom hammerdb image + pin: false # true for nodeSelector + pin_node: "node1" + db_type: "mssql" + timed_test: true + test_type: "tpc-c" + # true only for first run to build schema + db_init: true # true only for first run to build schema + db_benchmark: true + db_server: "127.0.0.1" + db_port: "1433" + db_warehouses: 1 + db_num_workers: 1 + db_user: "SA" + db_pass: "s3curePasswordString" + db_name: "tpcc" + transactions: 500000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # ElasticSearch custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "2019" + es_os_version: "centos8" + es_kind: "vm" + kind: vm + client_vm: + dedicatedcpuplacement: false + sockets: 1 + cores: 2 + threads: 1 + image: quay.io/cloud-bulldozer/centos-stream8-mssql2019-container-disk:latest + limits: + memory: 16Gi # at least 16Gi for VM + requests: + memory: 100Mi + network: + front_end: bridge # or masquerade + multiqueue: + enabled: false # if set to true, highly recommend to set selinux to permissive on the nodes where the vms would be scheduled + queues: 0 # must be given if enabled is set to true and ideally should be set to vcpus ideally so sockets*threads*cores, your image must've ethtool installed + extra_options: + - none + #- hostpassthrough + ## OCS PVC + pvc: false # enable for OCS PVC + pvc_storageclass: ocs-storagecluster-ceph-rbd + # Can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce + pvc_pvcaccessmode: ReadWriteMany + # Can be one of Filesystem,Block Default: Filesystem + pvc_pvcvolumemode: Block + pvc_storagesize: 10Gi + ## HostPath - Configuring SELinux on cluster workers + hostpath: false # enable for hostPath + hostpath_path: /var/tmp/disk.img + hostpath_storagesize: 10Gi diff --git a/resources/crds/ripsaw_v1alpha1_iperf3_cr.yaml b/config/samples/iperf3/cr.yaml similarity index 87% rename from resources/crds/ripsaw_v1alpha1_iperf3_cr.yaml rename to config/samples/iperf3/cr.yaml index f47f7e6dc..2a95a56ae 100644 --- a/resources/crds/ripsaw_v1alpha1_iperf3_cr.yaml +++ b/config/samples/iperf3/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: iperf3-benchmark-example - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: iperf3 @@ -12,7 +12,7 @@ spec: transmit_type: time transmit_value: 60 omit_start: 0 - length_buffer: 128K +# length_buffer: 128K window_size: 64k ip_tos: 0 mss: 900 diff --git a/resources/crds/ripsaw_v1alpha1_kube-burner_cr.yaml b/config/samples/kube-burner/cr.yaml similarity index 81% rename from resources/crds/ripsaw_v1alpha1_kube-burner_cr.yaml rename to config/samples/kube-burner/cr.yaml index 6e47d1ed1..86e68ae8a 100644 --- a/resources/crds/ripsaw_v1alpha1_kube-burner_cr.yaml +++ b/config/samples/kube-burner/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: kube-burner-cluster-density-example - namespace: my-ripsaw + namespace: benchmark-operator spec: # Metadata information elasticsearch: @@ -55,6 +55,18 @@ spec: # Remote metrics profile # remote_metrics_profile: http://yourdomain/metrics-profile.yml # kube-burner pod tolerations + # Application to build + #app: django + #source_strat_env: PIP_INDEX_URL + #source_strat_from: python + #source_strat_from_version: latest + # Script to run after build + #post_commit_script: "./manage.py test" + # Build Image name + #build_image_stream: django-psql-example + # Git url for application + #git_url: https://github.com/sclorg/django-ex.git + #build_image: image-registry.openshift-image-registry.svc:5000/svt-django/django-psql-example tolerations: - key: role value: workload @@ -62,4 +74,4 @@ spec: # Pod nodeSelector node_selector: key: node-role.kubernetes.io/worker - value: + value: \ No newline at end of file diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml new file mode 100644 index 000000000..13d4215f6 --- /dev/null +++ b/config/samples/kustomization.yaml @@ -0,0 +1,33 @@ +## Append samples you want in your CSV to this file as resources ## +resources: +- ripsaw_v1alpha1_benchmark.yaml +- byowl/cr.yaml +- concurrent-builds/cr.yaml +- cyclictest/cr.yaml +- fio/cr.yaml +- fio/vm-cr.yaml +- fs-drift/cr.yaml +- hammerdb/cr.yaml +- hammerdb/vm-cr.yaml +- hammerdb/mariadb/cr.yaml +- hammerdb/mariadb/vm-cr.yaml +- hammerdb/mssql/cr.yaml +- hammerdb/mssql/vm-cr.yaml +- hammerdb/postgres/cr.yaml +- hammerdb/postgres/vm-cr.yaml +- iperf3/cr.yaml +- kube-burner/cr.yaml +- log-generator/cr.yaml +- oslat/cr.yaml +- pgbench/cr.yaml +- servicemesh/cr.yaml +- smallfile/cr.yaml +- stressng/cr.yaml +- stressng/vm-cr.yaml +- sysbench/cr.yaml +- testpmd/cr.yaml +- uperf/cr.yaml +- uperf/vm-cr.yaml +- vegeta/cr.yaml +- ycsb/cr.yaml +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/log-generator/cr.yaml b/config/samples/log-generator/cr.yaml new file mode 100644 index 000000000..af8e8045b --- /dev/null +++ b/config/samples/log-generator/cr.yaml @@ -0,0 +1,38 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: log-generator + namespace: benchmark-operator +spec: + elasticsearch: + # Elastic search instance with full URL format. https://elastic.apps.org:9200 + url: http://my.elasticsearch.server:80 + index_name: log-generator + prometheus: + # Elastic search instance with full URL format. https://elastic.apps.org:9200 + es_url: http://my.elasticsearch.server:80 + # Prometheus bearer token + prom_token: PROMETHEUS_BEARER_TOKEN + # Prometheus URL with full URL format. https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: log_generator + args: + # Number of log generator pods to launch + pod_count: 1 + # size of message in bytes + size: 512 + # number of messages per second to send + messages_per_second: 10 + # duration of time to send messages in minutes + duration: 1 + # url of the backend elasticsearch to be used for verification + es_url: "https://my_es_server" + # bearer token to use to access the backend elasticsearch + es_token: "sha256~myToken" + # amount of time after sending all the messages to wait for the backend to receive them all + timeout: 600 + # Label of host to target + label: + key: foo + value: "" diff --git a/resources/crds/ripsaw_v1alpha1_oslat_cr.yaml b/config/samples/oslat/cr.yaml similarity index 94% rename from resources/crds/ripsaw_v1alpha1_oslat_cr.yaml rename to config/samples/oslat/cr.yaml index 004f187d2..8d5d309e3 100644 --- a/resources/crds/ripsaw_v1alpha1_oslat_cr.yaml +++ b/config/samples/oslat/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: oslat-example - namespace: my-ripsaw + namespace: benchmark-operator spec: # where elastic search is running elasticsearch: diff --git a/resources/crds/ripsaw_v1alpha1_pgbench_cr.yaml b/config/samples/pgbench/cr.yaml similarity index 98% rename from resources/crds/ripsaw_v1alpha1_pgbench_cr.yaml rename to config/samples/pgbench/cr.yaml index eb40ae377..84c22f6bf 100644 --- a/resources/crds/ripsaw_v1alpha1_pgbench_cr.yaml +++ b/config/samples/pgbench/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: pgbench-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: myk8scluster elasticsearch: diff --git a/config/samples/ripsaw_v1alpha1_benchmark.yaml b/config/samples/ripsaw_v1alpha1_benchmark.yaml new file mode 100644 index 000000000..45d204117 --- /dev/null +++ b/config/samples/ripsaw_v1alpha1_benchmark.yaml @@ -0,0 +1,7 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: benchmark-sample +spec: + # Add fields here + foo: bar diff --git a/resources/crds/ripsaw_v1alpha1_servicemesh_cr.yaml b/config/samples/servicemesh/cr.yaml similarity index 97% rename from resources/crds/ripsaw_v1alpha1_servicemesh_cr.yaml rename to config/samples/servicemesh/cr.yaml index 5f6d83362..158e499af 100644 --- a/resources/crds/ripsaw_v1alpha1_servicemesh_cr.yaml +++ b/config/samples/servicemesh/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: servicemesh-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: servicemesh diff --git a/config/samples/smallfile/cr.yaml b/config/samples/smallfile/cr.yaml new file mode 100644 index 000000000..90ad63f6b --- /dev/null +++ b/config/samples/smallfile/cr.yaml @@ -0,0 +1,27 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: example-benchmark + namespace: benchmark-operator +spec: + test_user: homer_simpson + clustername: nuclear-power-plant + elasticsearch: + url: http://my.elasticsearch.server:9200 + workload: + name: smallfile + args: + #drop_cache_kernel: true + #drop_cache_rook_ceph: true + #storageclass: my-kubernetes-storageclass + storagesize: 2Gi + clients: 1 + samples: 1 + operation: ["create", "read"] + threads: 5 + file_size: 64 + files: 100 + # more parameters available, see smallfile documentation + # at https://github.com/distributed-system-analysis/smallfile/blob/master/README.md + # subset of parameters supported for benchmark-operator are in + # roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 diff --git a/resources/crds/ripsaw_v1alpha1_stressng_cr.yaml b/config/samples/stressng/cr.yaml similarity index 65% rename from resources/crds/ripsaw_v1alpha1_stressng_cr.yaml rename to config/samples/stressng/cr.yaml index b8cd8c549..452da28de 100644 --- a/resources/crds/ripsaw_v1alpha1_stressng_cr.yaml +++ b/config/samples/stressng/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: stressng-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: http://es.server.com:80 @@ -15,12 +15,17 @@ spec: runtype: "parallel" timeout: "30" instances: 1 - # nodeselector: - # key1: value1 - # key2: value2 + pin: false # true for nodeSelector + pin_node: "node1" + resources: false # true for resources requests/limits + requests_cpu: 200m + requests_memory: 100Mi + limits_cpu: 2 + limits_memory: 4Gi # cpu stressor options cpu_stressors: "1" cpu_percentage: "100" + cpu_method: "all" # vm stressor option vm_stressors: "1" vm_bytes: "128M" diff --git a/resources/crds/ripsaw_v1alpha1_stressng_vm.yaml b/config/samples/stressng/vm-cr.yaml similarity index 92% rename from resources/crds/ripsaw_v1alpha1_stressng_vm.yaml rename to config/samples/stressng/vm-cr.yaml index 1969bd7ae..28b8d28f1 100644 --- a/resources/crds/ripsaw_v1alpha1_stressng_vm.yaml +++ b/config/samples/stressng/vm-cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: stressng-vm-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: http://es.server.com:80 @@ -15,9 +15,8 @@ spec: runtype: "parallel" timeout: "30" instances: 1 - # nodeselector: - # key1: value1 - # key2: value2 + pin: false # true for nodeSelector + pin_node: "node1" # cpu stressor options cpu_stressors: "1" cpu_percentage: "100" diff --git a/resources/crds/ripsaw_v1alpha1_sysbench_cr.yaml b/config/samples/sysbench/cr.yaml similarity index 72% rename from resources/crds/ripsaw_v1alpha1_sysbench_cr.yaml rename to config/samples/sysbench/cr.yaml index b48d9a8a3..f52d90dfb 100644 --- a/resources/crds/ripsaw_v1alpha1_sysbench_cr.yaml +++ b/config/samples/sysbench/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: sysbench-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: sysbench @@ -10,6 +10,9 @@ spec: enabled: true #kind: vm # If you want to run this as a VM uncomment the above + #pin_node: "worker-0.mylab.example.com" + #storageclass: ocs-storagecluster-ceph-rbd + #storagesize: 200Gi tests: - name: cpu parameters: diff --git a/resources/crds/ripsaw_v1alpha1_testpmd_cr.yaml b/config/samples/testpmd/cr.yaml similarity index 93% rename from resources/crds/ripsaw_v1alpha1_testpmd_cr.yaml rename to config/samples/testpmd/cr.yaml index 2f90d68d3..5e50987de 100644 --- a/resources/crds/ripsaw_v1alpha1_testpmd_cr.yaml +++ b/config/samples/testpmd/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: testpmd-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: myk8scluster workload: diff --git a/config/samples/uperf-scale/cr.yaml b/config/samples/uperf-scale/cr.yaml new file mode 100644 index 000000000..4fcc0b2fd --- /dev/null +++ b/config/samples/uperf-scale/cr.yaml @@ -0,0 +1,56 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-scale-benchmark + namespace: benchmark-operator +spec: + clustername: myk8scluster + elasticsearch: + url: "http://es-instance.com:9200" + #test_user: username_to_attach_to_metadata + workload: + # cleanup: true + name: uperf-scale + args: + serviceip: false + hostnetwork: false + networkpolicy: false + multus: + enabled: false + samples: 5 + kind: pod + # pin: false + # + # 'pair' sepcifies fixed number of client-server pairs for "Pin" mode, + # If 'pair' is NOT present, it will use 'density_range' which allows + # enumeration in addition to fixed number of pair. + test_types: + - stream + protos: + - tcp + sizes: + - 16384 + nthrs: + - 1 + runtime: 30 + + # The following variables are for 'Scale' mode. + # The 'Scale' mode is activated when 'pin=false' or undefined. + # The Scale mode params are: colocate, denstisy_range, node_range and step_size. + # + colocate: false + density_range: [1, 1] + node_range: [1, 1] + step_size: add1 + # Valid step_size values are: addN or log2 + # N can be any decimal number + # Enumeration examples: + # add1: 1,2,3,4 ,,, + # add2: 1,3,5,7 ... + # add10: 1,11,21,31 ... + # log2: 1,2,4,8,16,32 ,,, + # + # 'exclude_labels' specifies the list of ineligible worker nodes. + # exclude_labels: (OR conditional, every node that matches any of these labels is excluded) + # - "bad=true" + # - "fc640=true" \ No newline at end of file diff --git a/config/samples/uperf/cr.yaml b/config/samples/uperf/cr.yaml new file mode 100644 index 000000000..1570a9eda --- /dev/null +++ b/config/samples/uperf/cr.yaml @@ -0,0 +1,64 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-benchmark + namespace: benchmark-operator +spec: + clustername: myk8scluster + elasticsearch: + url: http://es.server.com:80 + #test_user: username_to_attach_to_metadata + workload: + # cleanup: true + name: uperf + args: + serviceip: false + hostnetwork: false + networkpolicy: false + multus: + enabled: false + pin: false + # + # pin: true/false - default=false + # - true will run 'Pin' mode using 1 server (pin_server:) and 1 client (pin_clien:) nodes. + # - false will run 'Scale' mode. See colocate, density_range, node_range and step_size. + pin_server: "node-0" + pin_client: "node-1" + samples: 1 + kind: pod + pair: 1 + # + # 'pair' sepcifies fixed number of client-server pairs for "Pin" mode, + # If 'pair' is NOT present, it will use 'density_range' which allows + # enumeration in addition to fixed number of pair. + test_types: + - stream + protos: + - tcp + sizes: + - 16384 + nthrs: + - 1 + runtime: 30 + + # The following variables are for 'Scale' mode. + # The 'Scale' mode is activated when 'pin=false' or undefined. + # The Scale mode params are: colocate, denstisy_range, node_range and step_size. + # + # colocate: true/false - default=false + # density_range: [n, m] - default=[1,1] + # node_range: [x, y] - default=[1,1] + # step_size: log2 - default=add1 + # Valid step_size values are: addN or log2 + # N can be any decimal number + # Enumeration examples: + # add1: 1,2,3,4 ,,, + # add2: 1,3,5,7 ... + # add10: 1,11,21,31 ... + # log2: 1,2,4,8,16,32 ,,, + # + # 'exclude_labels' specifies the list of ineligible worker nodes. + # exclude_labels: (OR conditional, every node that matches any of these labels is excluded) + # - "bad=true" + # - "fc640=true" + diff --git a/resources/crds/ripsaw_v1alpha1_uperf_vm.yaml b/config/samples/uperf/vm-cr.yaml similarity index 98% rename from resources/crds/ripsaw_v1alpha1_uperf_vm.yaml rename to config/samples/uperf/vm-cr.yaml index 0b3ad04f3..24ecf4c73 100644 --- a/resources/crds/ripsaw_v1alpha1_uperf_vm.yaml +++ b/config/samples/uperf/vm-cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: test-cluster test_user: user # user is a key that points to user triggering ripsaw, useful to search results in ES diff --git a/resources/crds/ripsaw_v1alpha1_vegeta_cr.yaml b/config/samples/vegeta/cr.yaml similarity index 91% rename from resources/crds/ripsaw_v1alpha1_vegeta_cr.yaml rename to config/samples/vegeta/cr.yaml index fa706ef30..ece98d77c 100644 --- a/resources/crds/ripsaw_v1alpha1_vegeta_cr.yaml +++ b/config/samples/vegeta/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: vegeta-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: # Elastic search instance with full URL format. https://elastic.apps.org:9200 @@ -12,6 +12,7 @@ spec: workload: name: vegeta args: + # node_selector: "vegeta=true" hostnetwork: false clients: 2 targets: diff --git a/resources/crds/ripsaw_v1alpha1_ycsb_cr.yaml b/config/samples/ycsb/cr.yaml similarity index 95% rename from resources/crds/ripsaw_v1alpha1_ycsb_cr.yaml rename to config/samples/ycsb/cr.yaml index f7ede7797..15c33288c 100644 --- a/resources/crds/ripsaw_v1alpha1_ycsb_cr.yaml +++ b/config/samples/ycsb/cr.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: ycsb-couchbase-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: server: http://my.elasticsearch.server:80 diff --git a/config/scorecard/bases/config.yaml b/config/scorecard/bases/config.yaml new file mode 100644 index 000000000..c77047841 --- /dev/null +++ b/config/scorecard/bases/config.yaml @@ -0,0 +1,7 @@ +apiVersion: scorecard.operatorframework.io/v1alpha3 +kind: Configuration +metadata: + name: config +stages: +- parallel: true + tests: [] diff --git a/config/scorecard/kustomization.yaml b/config/scorecard/kustomization.yaml new file mode 100644 index 000000000..50cd2d084 --- /dev/null +++ b/config/scorecard/kustomization.yaml @@ -0,0 +1,16 @@ +resources: +- bases/config.yaml +patchesJson6902: +- path: patches/basic.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +- path: patches/olm.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +#+kubebuilder:scaffold:patchesJson6902 diff --git a/config/scorecard/patches/basic.config.yaml b/config/scorecard/patches/basic.config.yaml new file mode 100644 index 000000000..2d4e0a9e7 --- /dev/null +++ b/config/scorecard/patches/basic.config.yaml @@ -0,0 +1,10 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - basic-check-spec + image: quay.io/operator-framework/scorecard-test:v1.7.2 + labels: + suite: basic + test: basic-check-spec-test diff --git a/config/scorecard/patches/olm.config.yaml b/config/scorecard/patches/olm.config.yaml new file mode 100644 index 000000000..59c557683 --- /dev/null +++ b/config/scorecard/patches/olm.config.yaml @@ -0,0 +1,50 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-bundle-validation + image: quay.io/operator-framework/scorecard-test:v1.7.2 + labels: + suite: olm + test: olm-bundle-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-validation + image: quay.io/operator-framework/scorecard-test:v1.7.2 + labels: + suite: olm + test: olm-crds-have-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-resources + image: quay.io/operator-framework/scorecard-test:v1.7.2 + labels: + suite: olm + test: olm-crds-have-resources-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-spec-descriptors + image: quay.io/operator-framework/scorecard-test:v1.7.2 + labels: + suite: olm + test: olm-spec-descriptors-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-status-descriptors + image: quay.io/operator-framework/scorecard-test:v1.7.2 + labels: + suite: olm + test: olm-status-descriptors-test diff --git a/config/testing/debug_logs_patch.yaml b/config/testing/debug_logs_patch.yaml new file mode 100644 index 000000000..3fb3d559f --- /dev/null +++ b/config/testing/debug_logs_patch.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + env: + - name: ANSIBLE_DEBUG_LOGS + value: "TRUE" diff --git a/config/testing/kustomization.yaml b/config/testing/kustomization.yaml new file mode 100644 index 000000000..410916239 --- /dev/null +++ b/config/testing/kustomization.yaml @@ -0,0 +1,23 @@ +# Adds namespace to all resources. +namespace: osdk-test + +namePrefix: osdk- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +patchesStrategicMerge: +- manager_image.yaml +- debug_logs_patch.yaml +- ../default/manager_auth_proxy_patch.yaml + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../crd +- ../rbac +- ../manager +images: +- name: testing + newName: testing-operator diff --git a/config/testing/manager_image.yaml b/config/testing/manager_image.yaml new file mode 100644 index 000000000..e44f542d9 --- /dev/null +++ b/config/testing/manager_image.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + image: testing diff --git a/config/testing/pull_policy/Always.yaml b/config/testing/pull_policy/Always.yaml new file mode 100644 index 000000000..6b0a8e2a8 --- /dev/null +++ b/config/testing/pull_policy/Always.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: Always diff --git a/config/testing/pull_policy/IfNotPresent.yaml b/config/testing/pull_policy/IfNotPresent.yaml new file mode 100644 index 000000000..2f52f496c --- /dev/null +++ b/config/testing/pull_policy/IfNotPresent.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: IfNotPresent diff --git a/config/testing/pull_policy/Never.yaml b/config/testing/pull_policy/Never.yaml new file mode 100644 index 000000000..86f13d816 --- /dev/null +++ b/config/testing/pull_policy/Never.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: Never diff --git a/deploy/10_service_account.yaml b/deploy/10_service_account.yaml deleted file mode 100644 index 92cc0365b..000000000 --- a/deploy/10_service_account.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: benchmark-operator - namespace: my-ripsaw diff --git a/deploy/20_role.yaml b/deploy/20_role.yaml deleted file mode 100644 index 12b1be6b7..000000000 --- a/deploy/20_role.yaml +++ /dev/null @@ -1,112 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - creationTimestamp: null - name: benchmark-operator - namespace: my-ripsaw -rules: -- apiGroups: - - "" - resources: - - pods - - daemonsets - - services - - endpoints - - persistentvolumeclaims - - virtualmachineinstances - - events - - configmaps - - secrets - - pods/log - verbs: - - '*' -- apiGroups: - - "" - resources: - - namespaces - verbs: - - get -- apiGroups: - - apps - resources: - - deployments - - daemonsets - - replicasets - - statefulsets - - deployments/finalizers - verbs: - - '*' -- apiGroups: - - monitoring.coreos.com - resources: - - servicemonitors - verbs: - - get - - create -- apiGroups: - - kubevirt.io - resources: - - virtualmachineinstances - verbs: - - '*' -- apiGroups: - - ripsaw.cloudbulldozer.io - resources: - - '*' - verbs: - - '*' -- apiGroups: - - batch - - extensions - resources: - - jobs - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - policy - resources: - - podsecuritypolicies - verbs: - - use - resourceNames: - - privileged -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use -- apiGroups: - - hyperfoil.io - resources: - - hyperfoils - verbs: - - '*' -- apiGroups: - - networking.k8s.io - resources: - - networkpolicies - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - k8s.cni.cncf.io - resources: - - network-attachment-definitions - verbs: - - get - - list - - watch diff --git a/deploy/30_role_binding.yaml b/deploy/30_role_binding.yaml deleted file mode 100644 index 9ff9929a1..000000000 --- a/deploy/30_role_binding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: benchmark-operator - namespace: my-ripsaw -subjects: -- kind: ServiceAccount - name: benchmark-operator - namespace: my-ripsaw -roleRef: - kind: Role - name: benchmark-operator - apiGroup: rbac.authorization.k8s.io diff --git a/deploy/40_cluster_role_kubevirt.yaml b/deploy/40_cluster_role_kubevirt.yaml deleted file mode 100644 index 31c2cd6db..000000000 --- a/deploy/40_cluster_role_kubevirt.yaml +++ /dev/null @@ -1,61 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: benchmark-operator-kube -rules: -- apiGroups: - - rbac.authorization.k8s.io - resources: - - roles - - rolebindings - - clusterroles - - clusterrolebindings - - daemonsets - verbs: - - '*' -- apiGroups: - - apps - resources: - - daemonsets - verbs: - - get - - list -- apiGroups: - - rbac.authorization.k8s.io - resources: - - nodes - verbs: - - get - - list -- apiGroups: - - subresources.kubevirt.io - resources: - - virtualmachineinstances/console - - virtualmachineinstances/vnc - verbs: - - get -- apiGroups: - - kubevirt.io - resources: - - virtualmachineinstances - - virtualmachines - - virtualmachineinstancepresets - - virtualmachineinstancereplicasets - verbs: - - get - - delete - - create - - update - - patch - - list - - watch - - deletecollection -- apiGroups: [""] - resources: - - configmaps - resourceNames: - - kubevirt-config - verbs: - - update - - get - - patch diff --git a/deploy/50_pod_security_policy.yml b/deploy/50_pod_security_policy.yml deleted file mode 100644 index a8e05fb49..000000000 --- a/deploy/50_pod_security_policy.yml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: privileged -spec: - privileged: true - allowPrivilegeEscalation: true - allowedCapabilities: - - '*' - volumes: - - '*' - hostNetwork: true - hostPorts: - - min: 0 - max: 65535 - hostIPC: true - hostPID: true - runAsUser: - rule: 'RunAsAny' - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'RunAsAny' - fsGroup: - rule: 'RunAsAny' diff --git a/docs/api_load.md b/docs/api_load.md new file mode 100644 index 000000000..9d6412230 --- /dev/null +++ b/docs/api_load.md @@ -0,0 +1,104 @@ +# API Load + +## What does it do? + +The API Load workload will execute a load test using [OCM API Load](https://github.com/cloud-bulldozer/ocm-api-load) +After running it will perform an ES indexing for all the requests that have been generated and upload all the files to a snappy server. + +## Sections + +### Required sections + +`elasticsearch` section, it is needed to index all the documents that are used for results. + +### Optional sections + +[`snappy`](https://github.com/cloud-bulldozer/snappy-data-server) section, it is needed to upload the raw files in case we need them for further analysis. + +## Variables + +### Required variables + +`test_list` a list of the test that are goping to be run + +`gateway_url` url for openshift API + +`ocm_token` authorization token for API + +`duration` default duration of each the attack + +`rate` default rate of each attack + +`aws_access_key` access key for AWS auhtentication + +`aws_access_secret` access secret for AWS auhtentication + +`aws_account_id` 12 digit account number for AWS auhtentication + +### Optional variables + +`cooldown` time in seconds, wait before the next attack occurs, default 60 seconds + +`output_path` path were to write the results, default `/tmp/results` + +`override` used to run a specific sub-command of `ocm-api-load`, like `version` or `help` + +`sleep` time in seconds, checks redis status each seconds, default 360 seconds + +Each test can be provided with it's own configurations + +`duration` default duration of each the attack + +`rate` default rate of each attack + +### Example CR + +Your resource file with test list and custom configuraiton for some attacks: + +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: api-load + namespace: benchmark-operator +spec: + elasticsearch: + url: http://elastic.search.server + index_name: "api-load" + snappy: + url: http://snappy.files.server + user: user + password: password + workload: + name: api_load + args: + gateway_url: https://api.openshift.com + ocm_token: realTokenHere + duration: 1 + rate: 5/s + output_path: /tmp/results + aws_access_key: realKey + aws_access_secret: realSecret + aws_account_id: 12DigitAccountNumber + cooldown: 10 + sleep: 300 + test_list: + self-access-token: + rate: "1/s" + duration: 1 + list-subscriptions: + duration: 1 + access-review: + rate: "7/s" + register-new-cluster: + register-existing-cluster: + create-cluster: + list-clusters: + get-current-account: + quota-cost: + resource-review: + cluster-authorizations: + self-terms-review: + certificates: + +``` diff --git a/docs/byowl.md b/docs/byowl.md index a9c31d0d8..fbb614813 100644 --- a/docs/byowl.md +++ b/docs/byowl.md @@ -12,7 +12,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: byowl-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: byowl @@ -44,3 +44,11 @@ spec: This will launch the uperf container, and simply print the messages above into the log of the container. + +### A generic support: +You can add any section under `specoptions` and `containeroptions` in `byowl` CRD, which will go under POD's `spec` and `containers` respectively as below: + + +![BYOWL_DOC](https://user-images.githubusercontent.com/4022122/112431010-f31de200-8d64-11eb-9179-e6ae7eb0e2cd.png) + + diff --git a/docs/cache_dropping.md b/docs/cache_dropping.md index 360ecf57e..32fd2b2b0 100644 --- a/docs/cache_dropping.md +++ b/docs/cache_dropping.md @@ -23,17 +23,27 @@ There are different types of caching that occur in the system - (Ceph OCS) OSD caching (not yet supported fully) you can control which type of cache dropping -is done using these CR fields in the workload args section: +is done using one or both of these CR fields in the workload args section: ``` drop_cache_kernel: true +drop_cache_rook_ceph: true ``` +## how to drop kernel cache + For this to work, you must **label** the nodes that you want to drop kernel cache, for example: ``` # kubectl label node minikube kernel-cache-dropper=yes ``` + +or to label all worker nodes: + +``` +# kubectl label node kernel-cache-dropper=yes -l node-role.kubernetes.io/worker='' +``` + If you do not do this, benchmark-operator will reject the benchmark with an error to the effect that none of the cluster nodes have this label. This label controls where kernel cache is dropped. @@ -41,14 +51,40 @@ There will be a short delay after kernel cache is dropped in order to allow the some key cache contents before stressing it with a workload. This is controllable via the CACHE_RELOAD_TIME env. var. and defaults to 10 sec. -You must also execute this command before running a benchmark. +The specific benchmark must support this feature. +Benchmarks supported for kernel cache dropping at present are: + +- fio +- smallfile + +## how to drop Ceph OSD cache + +For this to work with OpenShift Container Storage, you must do these steps once the benchmark-operator is running: +and the cache dropper pod, and enable benchmark-operator to see into the openshift-storage namespace. +You can do this with: ``` -oc create -f ripsaw.l/resources/kernel-cache-drop-clusterrole.yaml + +# enable the benchmark operator to look for pods in the openshift-storage namespace +oc create -f roles/ceph_osd_cache_drop/ocs-cache-drop-clusterrole.yaml + +# start the Ceph toolbox pod in openshift-storage namespace +oc patch OCSInitialization ocsinit -n openshift-storage --type json --patch \ + '[{ "op": "replace", "path": "/spec/enableCephTools", "value": true }]' + +# start the OSD cache dropper pod in openshift-storage namespace +oc create -f roles/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml + +# repeat until you see if the 2 pods are both running +oc -n openshift-storage get pod | awk '/tool/||/drop/' + ``` -Lastly, the specific benchmark must support this feature. -Benchmarks supported for kernel cache dropping at present are: +when you see both of these pods in the running state, then you can use benchmark operator. The reason that +you have to manually start these two pods running is that the benchmark-operator does not have authorization +to run them in the openshift-storage namespace and get access to the secrets needed to do this. + +Benchmarks supported for Ceph OSD cache dropping are: - fio - smallfile @@ -84,3 +120,18 @@ For example, in your workload.yml.j2 where it creates the environment variables - name: KCACHE_DROP_PORT_NUM value: "{{ kernel_cache_drop_svc_port }}" ``` + +similarly, for Ceph OSD cache dropping, you must add this to one of your workload pods' environment variables: +``` + +{% if ceph_osd_cache_drop_pod_ip is defined %} + - name: ceph_osd_cache_drop_pod_ip + value: "{{ ceph_osd_cache_drop_pod_ip }}" + - name: CEPH_CACHE_DROP_PORT_NUM + value: "{{ ceph_cache_drop_svc_port }}" +{% endif %} + +``` +The ansible playbook for benchmark-operator will look up the ceph_osd_cache_drop_pod_ip IP address and fill in this var, +all you have to do is pass it in. See the ceph_osd_cache_drop ansible role for details. + diff --git a/docs/cerberus.md b/docs/cerberus.md deleted file mode 100644 index 5bdc0a05a..000000000 --- a/docs/cerberus.md +++ /dev/null @@ -1,59 +0,0 @@ -# Cerberus Integration - -How To: -* [What is it](#what-is-it) -* [How it Works](#how-it-works) -* [How to enable Cerberus](#how-to-enable-Cerberus) - -# What is it - -What is Cerberus? [Cerberus](https://github.com/openshift-scale/cerberus) is a project that will watch an Openshift/Kubernernetes cluster -for dead nodes, component failures, etc and provide a healthly/unhealthy (True/False) signal. - -# How it Works - -For installation and startup instructions for Cerberus please see [https://github.com/openshift-scale/cerberus](https://github.com/openshift-scale/cerberus) - -Ripsaw has been enabled to check a provided Cerberus url for a True/False signal. True being healthy and False being unhealthy (meaning a component -has failed). If ripsaw encounters a False signal it will set the State of the benchmark to Error and the benchmark will not proceed -further. - -It will NOT stop any running components or kill any pods when a failure signal is found. That means that any pods that are running will -continue to run. Ripsaw will simply not proceed to any next steps of the workload. Everything is left in this state to aid in any -potential debugging that may need to be done. - -Once an Error state is entered it will not go back to its previous state. This means that you will either need to restart the benchmark -entirely or manually change the state of the benchmark. - -# How to enable Cerberus - -Enabling Cerberus is very easy. Simply define the cerberus_url variable in your CR file with the url:port of the Cerberus service you wish -to use. - -For example, if Cerberus is running at 1.2.3.4:8080 - -``` -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: byowl-benchmark - namespace: my-ripsaw -spec: - elasticsearch: - url: "http://foo.bar.com:9200" - cerberus_url: "http://1.2.3.4:8080" - workload: - name: byowl -... -``` - -NOTE: The cerberus url MUST BE in the format http://[address]:[port] - -Once Cerberus is enabled the connection status can be viewed by getting the benchmark status. If Cerberus is not -enabled the connection status will simply be "not connected" - -``` -$ kubectl -n my-ripsaw get benchmarks.ripsaw.cloudbulldozer.io -NAME TYPE STATE METADATA STATE CERBERUS UUID AGE -byowl-benchmark byowl Error Complete Connected 11faec65-4009-58e8-ac36-0233f0fc822d 10m -``` diff --git a/docs/cyclictest.md b/docs/cyclictest.md index f665de0ce..8c58acf34 100644 --- a/docs/cyclictest.md +++ b/docs/cyclictest.md @@ -10,7 +10,7 @@ SCHED_FIFO real-time tasks which are higher priority. ## Running cyclictest -Given that you followed instructions to deploy operator, you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_cyclictest.yaml) to your needs. +Given that you followed instructions to deploy operator, you can modify [cr.yaml](../config/samples/cyclictest/cr.yaml) to your needs. It is recommended to define pod requests and limits when running cyclict test, to give guaranteed CPUs to the pods. It is also expected to have the realtime kernel installed with required isolation for pods using the [Performance Add-On Operator](https://github.com/openshift-kni/performance-addon-operators). @@ -21,13 +21,14 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: cyclictest - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: server: workload: name: "cyclictest" args: + node_selector: "" duration: "1m" disable_cpu_balance: true stressng: false @@ -43,7 +44,8 @@ spec: You can run it by: ```bash -oc apply -f resources/crds/ripsaw_v1alpha1_cyclictest_cr.yaml # if edited the original one +# kubectl apply -f config/samples/cyclictest/cr.yaml # if edited the original one +# kubectl apply -f # if created a new cr file ``` ## Looking at results @@ -51,7 +53,7 @@ You can look at the results from the cyclictest benchmark by doing ```bash NAME=cyclictest-workload-xxxxxxx -oc logs -n my-ripsaw $NAME +oc logs -n benchmark-operator $NAME ``` ## Cleanup diff --git a/docs/e2e-ci.md b/docs/e2e-ci.md new file mode 100644 index 000000000..cff4c41e1 --- /dev/null +++ b/docs/e2e-ci.md @@ -0,0 +1,71 @@ +# E2E CI + +## Introduction + +Benchmark-operator ships an end to end CI mechanism that runs by default in the events described in the [e2e workflow](../.github/workflows/e2e.yml). +The main goal of this workflow is to test breaking/dangerous changes pushed into benchmark-operator, this means that it's not needed to run all tests before merging every pull request. + +## Workflow + +### Bats + +[Bats](https://github.com/bats-core/bats-core) is a TAP-compliant testing framework for Bash. It provides a simple way to verify that the UNIX programs you write behave as expected. + +A Bats test file is a Bash script with special syntax for defining test cases. Under the hood, each test case is just a function with a description. + +```bash +#!/usr/bin/env bats + +@test "addition using bc" { + result="$(echo 2+2 | bc)" + [ "$result" -eq 4 ] +} + +@test "addition using dc" { + result="$(echo 2 2+p | dc)" + [ "$result" -eq 4 ] +} +``` + +If every command in the test case exits with a 0 status code (success), the test passes. This workflow uses bats framework to execute end to end tests, these tests are defined in files with the termination *.bats under the project's e2e directory. + +### Events + +This workflow is at the moment of writing this document triggered by the following events: + +- A commit is pushed to the master branch: This allows to detect issues when we want a commit is merged to this branch + +- Workflow dispatch: This event allows a repository administrator to **manually trigger the worklflow**. The interesting part of this event is that allows to run specific tests using an input text dialog, this input dialog expectes a regular expression which is passed to the `-f` flag of the bats command. (`-f, --filter Only run tests that match the regular expression`). For example if you fill out this this dialog with uperf|fio, the workflow will trigger only the tests for FIO and smallfile, test names are defined with the special @test suffix in each bats file. `grep -h @test e2e/*.bats` + +- pull_request_target: This event is generated by pull requests, **the PRs must be labeled with "ok to test" to trigger this workflow.** + +## Running in local + +It's possible to run e2e tests in local, you can trigger them with executing the command: `make e2e-tests` +> Benchmark-operator pod must be running before triggering the tests. + +In case you want to test your own code you can use the following command to build, push and deploy your benchmark-operator bits, and eventually trigger e2e test suite. + +```shell +$ IMG= make image-build image-push deploy e2e-tests +``` + +It's also possible to trigger specific tests by setting the BATS_TESTS env var with a regex that will be passed to the `-f` flag of bats. + +```shell +$ IMG=container-registry.io/org/benchmark-operator:mybranch BATS_TESTS="uperf|fio" make image-build image-push deploy e2e-tests +``` + +A preconfigured Elasticsearch endpoint is set by default in the file [helpers.bash](../e2e/helpers.bash), however it's possible to set a custom one by exporting the variable `ES_SERVER`. + +### Software requirements + +Running tests in local has several requirements: + +- podman +- parallel +- bats +- make +- kubectl +- oc +- yq diff --git a/docs/elastic.md b/docs/elastic.md index 6f44b5dcf..feaa398f0 100644 --- a/docs/elastic.md +++ b/docs/elastic.md @@ -29,7 +29,7 @@ Current supported ES + Prometheus integrated workloads: | Workload | Status | | ------------------------------ | ---------------------- | | [UPerf](docs/uperf.md) | Not Supported | -| [Iperf3](docs/iperf.md) | Not Supported | +| [Iperf3](docs/iperf3.md) | Not Supported | | [fio](docs/fio_distributed.md) | Supported | | [Sysbench](docs/sysbench.md) | Not Supported | | [YCSB](docs/ycsb.md) | Not Supported | @@ -72,7 +72,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: smallfile-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: test_user: homer_simpson clustername: test_ci @@ -118,7 +118,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: smallfile-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: test_user: homer_simpson clustername: test_ci @@ -158,9 +158,9 @@ cluster-monitoring-view ,or use the Prometheus user account's token. Command to setup a service account and retrieve token: ``` -oc create serviceaccount snafu -n my-ripsaw -oc create clusterrolebinding grafana-cluster-monitoring-view --clusterrole=cluster-monitoring-view --serviceaccount=my-ripsaw:snafu -oc sa get-token snafu -n my-ripsaw +oc create serviceaccount snafu -n benchmark-operator +oc create clusterrolebinding grafana-cluster-monitoring-view --clusterrole=cluster-monitoring-view --serviceaccount=benchmark-operator:snafu +oc sa get-token snafu -n benchmark-operator ``` *NOTE:* User tokens can expire, for long running test it is recommend to either set the necessary permissions diff --git a/docs/fio_distributed.md b/docs/fio_distributed.md index fddf09d10..7f5576e0b 100644 --- a/docs/fio_distributed.md +++ b/docs/fio_distributed.md @@ -12,82 +12,8 @@ point for executing the workload on the server pods in parallel. ## Running Distributed FIO The Custom Resource (CR) file for fio includes a significant number of options to offer the user -flexibility. +flexibility. [Click here for an example of an fio CR](../config/samples/fio/cr.yaml). -```yaml -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: fio-benchmark - namespace: my-ripsaw -spec: - elasticsearch: - url: "http://my.es.server:9200" - clustername: myk8scluster - test_user: my_test_user_name - workload: - name: "fio_distributed" - args: - prefill: true - # for compressed volume uncomment the next 2 lines and make the cmp_bs same as bs - # prefill_bs: 8KiB - # cmp_ratio: 70 - samples: 3 - servers: 3 - # Chose to run servers in 'pod' or 'vm' - # 'vm' needs kubevirt to be available - # Default: pod - kind: pod - runtime_class: class_name - jobs: - - write - - read - bs: - - 4KiB - - 64KiB - numjobs: - - 1 - - 8 - iodepth: 4 - read_runtime: 60 - read_ramp_time: 5 - filesize: 2GiB - log_sample_rate: 1000 - storageclass: rook-ceph-block - storagesize: 5Gi - rook_ceph_drop_caches: True - rook_ceph_drop_cache_pod_ip: 192.168.111.20 -####################################### -# EXPERT AREA - MODIFY WITH CAUTION # -####################################### -# global_overrides: -# - key=value - job_params: - - jobname_match: w - params: - - fsync_on_close=1 - - create_on_open=1 - - jobname_match: read - params: - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: rw - params: - - rwmixread=50 - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} - - jobname_match: readwrite - params: - - rwmixread=50 - - time_based=1 - - runtime={{ workload_args.read_runtime }} - - ramp_time={{ workload_args.read_ramp_time }} -# - jobname_match: -# params: -# - key=value -``` ### NodeSelector and Taint/Tolerations vs. pin server @@ -157,19 +83,6 @@ The workload loops are nested as such from the CR options: > Therefore, be aware in providing units to the CR values that fio options should use the `MiB` format > while the `storagesize` option used for the K8S persistent volume claim should use the `Mi` format -#### metadata - -*Values here will usually be left unchanged* - -- **name**: The name the Ripsaw operator will use for the benchmark resource -- **namespace**: The namespace in which the benchmark will run - -#### spec - -- **elasticsearch**: (optional) Values are used to enable indexing of fio data; [further details are below](#indexing-in-elasticsearch-and-visualization-through-grafana) -- **clustername**: (optional) An arbitrary name for your system under test (SUT) that can aid indexing -- **test_user**: (optional) An arbitrary name for the user performing the tests that can aid indexing - #### spec.workload - **name**: **DO NOT CHANGE** This value is used by the Ripsaw operator to trigger the correct Ansible role @@ -187,14 +100,21 @@ The workload loops are nested as such from the CR options: > to instantiate the files themselves prior to beginning the benchmark workload. - **runtime_class** : If this is set, the benchmark-operator will apply the runtime_class to the podSpec runtimeClassName. > Note: For Kata containers +- **annotations** : If this is set, the benchmark-operator will set + > the specified annotations on the pods' metadata. +- **server_annotations** : If this is set, the benchmark-operator will set + > the specified annotations on the server pods' metadata. +- **client_annotations** : If this is set, the benchmark-operator will set + > the specified annotations on the client pods' metadata. - **kind**: Can either be `pod` or `vm` to determine if the fio workload is run in a Pod or in a VM > Note: For VM workloads, you need to install Openshift Virtualization first -- **vm_image**: Whether to use a pre-defined VM image with pre-installed requirements. Necessary for disconnected installs. - > Note: You can use my fedora image here: quay.io/mulbc/fed-fio \ - > Note: Only applies when kind is set to `vm` +- **vm_image**: VM image to use for the test, benchmark dependencies are provided by a pod container image (Default pod image: quay.io/cloud-bulldozer/fio:latest). Container images should be pre-loaded for disconnected installs. Default: quay.io/kubevirt/fedora-container-disk-images:latest + > Note: Only applies when kind is set to `vm`, current assumption is the firewall is disabled in the VM (default for most cloud images) - **vm_cores**: The number of CPU cores that will be available inside the VM. Default=1 > Note: Only applies when kind is set to `vm` -- **vm_cores**: The amount of Memory that will be available inside the VM in Kubernetes format. Default=5G +- **vm_memory**: The amount of Memory that will be available inside the VM in Kubernetes format. Default=5G + > Note: Only applies when kind is set to `vm` +- **vm_bus**: The VM bus type (virtio, scsi, or sata). Default=virtio > Note: Only applies when kind is set to `vm` - **bs**: (list) blocksize values to use for I/O transactions > Note: We set the `direct=1` fio option in the jobfile configmap. In order to avoid errors, the `bs` values @@ -219,22 +139,26 @@ The workload loops are nested as such from the CR options: > using these parameters, but note this behavious is configured via the EXPERT AREA section of the CR as > [described below](#expert-specjob_params), and therefore this may be adjusted to user preferences. - **filesize**: The size of the file used for each job in the workload (per `numjobs * servers` as described above) -- **log_sample_rate**: Applied to fio options `log_avg_msec` and `log_hist_msec` in the jobfile configmap; see `fio(1)` +- **log_sample_rate**: (optional) Applied to fio options `log_avg_msec` (milliseconds) in the jobfile configmap; see `fio(1)` +- **log_hist_msec** (optional) if set, enables histogram logging at this specified interval in milliseconds - **storageclass**: (optional) The K8S StorageClass to use for persistent volume claims (PVC) per server pod + > Note: Currently when the type is set to `vm` only Block-device storageclasses are supported +- **hostpath**: (optional) The local storage path to be used for hostPath (pods) or hostDisk (VMs) + > Note: When the type is set to `vm` make sure the hostDisk feature-gate is enabled and proper permission settings are applied to the path if applicable (i.e. chcon -Rt container_file_t ) - **pvcaccessmode**: (optional) The AccessMode to request with the persistent volume claim (PVC) for the fio server. Can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce - **pvcvolumemode**: (optional) The volmeMode to request with the persistent volume claim (PVC) for the fio server. Can be one of Filesystem,Block Default: Filesystem > Note: It is recommended to change this to `Block` for VM tests - **storagesize**: (optional) The size of the PVCs to request from the StorageClass ([note units quirk per above](#understanding-the-cr-options)) -- **rook_ceph_drop_caches**: (optional) If set to `True`, the Rook-Ceph OSD caches will be dropped prior to each sample -- **rook_ceph_drop_cache_pod_ip**: (optional) The IP address of the pod hosting the Rook-Ceph cache drop URL -- See [cache drop pod instructions](#dropping-rook-ceph-osd-caches) below +- **drop_cache_kernel**: (optional, default false) If set to `True`, kernel cache will be dropped on the labeled hosts before each sample. See [here for how to set up kernel cache dropping](./cache_dropping.md#how-to-drop-kernel-cache) +- **drop_cache_rook_ceph**: (optional, default false) The IP address of the pod hosting the Rook-Ceph cache drop URL -- See [here for how to set up Ceph OSD cache dropping](./cache_dropping.md#how-to-drop-Ceph-OSD-cache) > Technical Note: If you are running kube/openshift on VMs make sure the diskimage or volume is preallocated. - **prefill**: (Optional) boolean to enable/disable prefill SDS - prefill requirement stems from Ceph RBD thin-provisioning - just creating the RBD volume doesn't mean that there is space allocated to read and write out there. For example, reads to an uninitialized volume don't even talk to the Ceph OSDs, they just return immediately with zeroes in the client. - **prefill_bs** (Optional) The Block size that need to used for the prefill. - When running against compressed volumes, the prefill operation need to be done with the same block size as using in the test, otherwise the compression ratio will not be as expected. - **cmp_ratio** (Optional) When running against compressed volumes, the expected compression ratio (0-100) -- **fio_json_to_log**: (Optional) boolean to enable/disable sending job results in json format to client pod log. - +- **fio_json_to_log**: (Optional, default false) boolean to enable/disable sending job results in json format to client pod log. +- **job_timeout**: (Optional, default 3600) set this parameter if you want the fio to run for more than 1 hour without automatic termination #### EXPERT: spec.global_overrides The `key=value` combinations provided in the list here will be appended to the `[global]` section of the fio @@ -245,29 +169,6 @@ jobfile configmap. These options will therefore override the global values for a Under most circumstances, the options provided in the EXPERT AREA here should not be modified. The `key=value` pairs under `params` here are used to append additional fio job options based on the job type. Each `jobname_match` in the list uses a "search string" to match a job name per `fio(1)`, and if a match is made, the `key=value` list items under `params` are appended to the `[job]` section of the fio jobfile configmap. -## Dropping Rook-Ceph OSD Caches - -Dropping the OSD caches before workloads is a normal and advised part of tests that involve storage I/O. -Doing this with Rook-Ceph requires a privileged pod running the same namespace as the Ceph pods and with -the Ceph command tools available. To facilitate this, we provide the -[resources/rook_ceph_drop_cache_pod.yaml](../resources/rook_ceph_drop_cache_pod.yaml) file, which will -deploy a pod with the correct permissions and tools, as well as running a simple HTTP listener to trigger -the cache drop by URL. -**You must deploy this privileged pod in order for the drop caches requests in the workload to function.** - -```bash -kubectl apply -f resources/rook_ceph_drop_cache_pod.yaml -``` - -*Note: If Ceph is in a namespace other than `rook-ceph` you will need to modify the provided YAML accordingly.* - -Since the cache drop pod is deployed with host networking, the pod will take on the IP address -of the node on which it is running. You will need to use this IP address in the CR file as described above. - -```bash -kubectl get pod -n rook-ceph rook-ceph-osd-cache-drop --template={{.status.podIP}} -``` - ## Indexing in elasticsearch and visualization through Grafana ### Setup of Elasticsearch and Grafana @@ -303,4 +204,4 @@ The field for timestamp will always be `time_ms` . In order to index your fio results to elasticsearch, you will need to define the parameters appropriately in your workload CR file. The `spec.elasticsearch.url` parameter is required. -The `spec.clustername` and `spec.test_user` values are advised to allow for better indexing of your data. +The `spec.clustername` and `spec.test_user` values are advised to make it easier to find your set of test results in elasticsearch. diff --git a/docs/flent.md b/docs/flent.md index 23f21bad5..e4af62416 100644 --- a/docs/flent.md +++ b/docs/flent.md @@ -5,19 +5,18 @@ ## Running Flent Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_flent_cr.yaml) +you can modify [cr.yaml](../config/samples/flent/cr.yaml) ```yaml apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: flent-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: myk8scluster #elasticsearch: - # server: elk.server.com - # port: 9200 + # url: http://my.elasticsearch.server:80 #test_user: username_to_attach_to_metadata workload: # cleanup: true @@ -233,7 +232,7 @@ To enable Multus in Ripsaw, here is the relevant config. Once done creating/editing the resource file, you can run it by: ```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_flent_cr.yaml # if edited the original one +# kubectl apply -f config/samples/flent/cr.yaml # if edited the original one # kubectl apply -f # if created a new cr file ``` @@ -244,7 +243,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: myk8scluster test_user: test_user # user is a key that points to user triggering ripsaw, useful to search results in ES @@ -266,20 +265,15 @@ spec: runtime: 30 ``` -The new fields : - -`elasticsearch.server` this is the elasticsearch cluster ip you want to send the result data to for long term storage. - -`elasticsearch.port` port which elasticsearch is listening, typically `9200`. - -`user` provide a user id to the metadata that will be sent to Elasticsearch, this makes finding the results easier. +- `elasticsearch.url` Elastic search instance with full URL format. https://elastic.apps.org:9200 +- `user` provide a user id to the metadata that will be sent to Elasticsearch, this makes finding the results easier. By default we will utilize the `flent-results` index for Elasticsearch. Deploying the above(assuming pairs is set to 1) would result in ```bash -# oc get pods -n my-ripsaw +# oc get pods -n benchmark-operator NAME READY STATUS RESTARTS AGE benchmark-operator-f84bdbd8f-n6cnc 3/3 Running 0 34m flent-bench-client-10.116.0.54-533b6892-dc5cd 0/1 Completed 0 32m diff --git a/docs/fs-drift.md b/docs/fs-drift.md index 445544c1b..8ff3e796e 100644 --- a/docs/fs-drift.md +++ b/docs/fs-drift.md @@ -9,16 +9,16 @@ tree - see the benchmark documentation for details. fs-drift requires that a st ## Running fs-drift Benchmark -Once the operator has been installed following the instructions, one needs to modify the [cr.yaml](../resources/crds/ripsaw_v1alpha1_fs-drift_cr.yaml) to customize workload parameters - the defaults are selected to demonstrate its operation and are not intended to specify a long-duration test. +Once the operator has been installed following the instructions, one needs to modify the [cr.yaml](../config/samples/fs-drift/cr.yaml) to customize workload parameters - the defaults are selected to demonstrate its operation and are not intended to specify a long-duration test. -The parameters in [cr.yaml](../resources/crds/ripsaw_v1alpha1_fs-drift_cr.yaml) would look similar to this example: +The parameters in [cr.yaml](../config/samples/fs-drift/cr.yaml) would look similar to this example: ```yaml apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: fs-drift - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: fs-drift @@ -37,7 +37,7 @@ specified in a YAML input file also. YAML input style is a little different -- is omitted, and single dashes are converted to underscores. So "--parameter-foo bar" becomes "parameter_foo: bar". Operator CRs apparently also do not allow dashes in key names. So for the above example, use the -syntax "parameter_foo" instead of "--parameter-foo". See resources/crds/ for an example of fs-drift CR. +syntax "parameter_foo" instead of "--parameter-foo". See `config/samples/fs-drift/` for an example of fs-drift CR. The following fs-drift parameters will be overridden when fs-drift is used in ripsaw - do not specify these parameters yourself! @@ -53,10 +53,14 @@ The option **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option **annotations** can be set to apply the specified +annotations to the pod metadata. + Once done creating/editing the CR file below, one can run it by: ```bash -kubectl apply -f resources/crds/ripsaw_v1alpha1_fs-drift_cr.yaml +# kubectl apply -f config/samples/fs-drift/cr.yaml # if edited the original one +# kubectl apply -f # if created a new cr file ``` Deploying the above(assuming worker_pods set to 2) would result in diff --git a/docs/hammerdb.md b/docs/hammerdb.md index 8af4140c5..e19582429 100644 --- a/docs/hammerdb.md +++ b/docs/hammerdb.md @@ -5,9 +5,9 @@ ## Running hammerdb Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_hammerdb_cr.yaml) to your needs. +you can modify [cr.yaml](../config/samples/hammerdb/cr.yaml) to your needs. -The pgbench workload needs to be pointed at an existing MS-SQL databases via the CR file. +The hammerdb workload needs to be pointed at an existing [MSSQL](../config/samples/hammerdb/mssql/cr.yaml), [MARIADB](../config/samples/hammerdb/mariadb/cr.yaml), [POSTGRES](../config/samples/hammerdb/postgres/cr.yaml) database via the CR file. An example CR might look like this @@ -16,44 +16,234 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: hammerdb - namespace: my-ripsaw + namespace: benchmark-operator spec: + elasticsearch: + url: http://my.elasticsearch.server:80 + index_name: ripsaw-hammerdb + metadata: + collection: false workload: - name: "hammerdb" + name: hammerdb args: - db_init: false + #image: quay.io/user/hammerdb:latest # add custom hammerdb image + pin: false # true for nodeSelector + pin_client: "node1" + resources: false # true for resources requests/limits + requests_cpu: 200m + requests_memory: 100Mi + limits_cpu: 4 + limits_memory: 16Gi + db_type: "mssql" + timed_test: true + test_type: "tpc-c" + db_init: true # true only for first run to build schema db_benchmark: true - db_server: "mssql-deployment.sql-server" - db_port: "1443" - db_tcp: "true" - db_user: "SA" - db_pass: "s3curePasswordString" + db_server: "mssql-deployment.mssql-db" + db_port: "1433" db_warehouses: 1 db_num_workers: 1 - transactions: 20000 - runtime: 1 + db_user: "SA" + db_pass: "s3curePasswordString" + db_name: "tpcc" + transactions: 10000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mariadb: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + db_mysql_socket: "/var/lib/mysql/mysql.sock" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + # ElasticSearch custom fields + es_custom_field: false + es_ocp_version: "4.7.0" + es_cnv_version: "2.6.2" + es_db_version: "2019" + es_os_version: "centos8" + es_kind: "pod" ``` +The `pin` feature is `true` for node selector, `pin_node` is the node name + +The `resource` feature is `true` for resources configurations: `requests_cpu`, `requests_memory`, `limits_cpu`, `limits_memory` -The `db_init` feature determines wether the database has already been initialized (false) or needs to be initialized (true). If the DB has been used previously to run benchmarks against it, it needs to be set to `false`. +The `db_type` is the database type: mariadb, pg or mssql -The `db_benchmark` feature is used to run the actual benchmark when set to true. `db_server` either holds the name or the IP address of the DB server, `db_port` the port on which the DB is accessible. If `db_tcp` is set to true the client will use a TCP connection, if it's set to `false` UDP will be used. +The `db_init` feature determines wether the database has already been initialized `false` or needs to be initialized `true`. If the DB has been used previously to run benchmarks against it, it needs to be set to `false`. -`db_user` and `db_pass` need to be set identical to the settings on the DB server side. +The `db_benchmark` feature used to run the actual benchmark when set to `true`. -The tpcc benchmark which we use can set up an arbitrary number of warehouses between which goods will be transferred in order to simulate a real-world scenario. The higher the number of warehouses is, the more complex and load-heavy the benchmark can get. `db_warehouses` is used to define this number. -`db_num_workers` controls the number of virtual users, acting upon the warehouses and the goods in them. This number needs to lesser or equal to the number of warehouses. +The `db_server` either holds the name or the IP address of the DB server, + +The `db_port` the port on which the DB is accessible. If `db_mssql_tcp` is set to `true` the client will use a TCP connection, if it's set to `false` UDP will be used. + +The `db_user` and `db_pass` need to be set identical to the settings on the DB server side. + +The tpcc benchmark which we use can set up an arbitrary number of warehouses between which goods will be transferred in order to simulate a real-world scenario. The higher the number of warehouses is, the more complex and load-heavy the benchmark can get. +The `db_warehouses` is used to define this number. +The `db_num_workers` is used to controls the number of virtual users, acting upon the warehouses and the goods in them. This number needs to lesser or equal to the number of warehouses. +* db_warehouses >= db_num_workers: virtual users must be less than or equal to number of warehouses With `runtime`, `rampup` and `samples` the time for a single run, the rampup time per run and the number of runs can be controlled. +The `es_custom_field` feature is `true` to enable distribute the following fields to Elastic Search: `es_ocp_version`, `es_cnv_version`, `es_db_version`, `es_os_version`, `es_kind` + +There are several options to store database data on Pod, the default one is ephemeral: + +HostPath: The data will be stored on a local disk where the OS is placed. + +Local: The data will be stored on local disk on separate disk, separate disk from OS. + +PVC: The data will be stored on Container Storage, it's required a Pre-installed Container Storage + +MSSQL examples: + +[MSSQL ephemeral](../config/samples/hammerdb/mssql/cr.yaml), +[MSSQL HostPath](../config/samples/hammerdb/mssql/hostpath-cr.yaml), +[MSSQL Local](../config/samples/hammerdb/mssql/local-cr.yaml), +[MSSQL PVC](../config/samples/hammerdb/mssql/vm-cr.yaml) + +Postgres examples: + +[Postgres ephemeral](../config/samples/hammerdb/postgres/cr.yaml), +[Postgres HostPath](../config/samples/hammerdb/postgres/hostpath-cr.yaml), +[Postgres Local](../config/samples/hammerdb/postgres/local-cr.yaml), +[Postgres PVC](../config/samples/hammerdb/postgres/vm-cr.yaml) + +Mariadb examples: + +[Mariadb ephemeral](../config/samples/hammerdb/mariadb/cr.yaml), +[Mariadb HostPath](../config/samples/hammerdb/mariadb/hostpath-cr.yaml), +[Mariadb Local](../config/samples/hammerdb/mariadb/local-cr.yaml), +[Mariadb PVC](../config/samples/hammerdb/mariadb/vm-cr.yaml) + + The option **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option **annotations** can be set to apply the specified +annotations to the pod metadata. To set annotations only on the +workload, use **client_annotations**; to set annotations only on the +database server, use **server_annotations**. + Once done creating/editing the resource file, you can run it by: ```bash # kubectl apply -f ``` +## Running hammerdb in VMs through kubevirt/cnv [Working] + + +### changes to cr file + +```yaml + kind: vm + client_vm: + dedicatedcpuplacement: false + sockets: 1 + cores: 2 + threads: 1 + image: kubevirt/fedora-cloud-container-disk-demo:latest + requests: + memory: 100Mi + limits: + memory: 16Gi # at least 16Gi for VM + network: + front_end: bridge # or masquerade + multiqueue: + enabled: false # if set to true, highly recommend to set selinux to permissive on the nodes where the vms would be scheduled + queues: 0 # must be given if enabled is set to true and ideally should be set to vcpus ideally so sockets*threads*cores, your image must've ethtool installed + extra_options: + - none + #- hostpassthrough +- ## OCS PVC + pvc: false # enable for OCS PVC + pvc_storageclass: ocs-storagecluster-ceph-rbd + pvc_pvcaccessmode: ReadWriteMany + pvc_pvcvolumemode: Block + pvc_storagesize: 10Gi + ## HostPath - Configuring SELinux on cluster workers + hostpath: false # enable for hostPath + hostpath_path: /var/tmp/disk.img + hostpath_storagesize: 10Gi +``` + +The above is the additional changes required to run hammerdb in vms. + +There several options to store database data on VM, the default one is ephemeral: + +PVC: The data will be stored on Container Storage, it's required a Pre-installed Container Storage + +The `pvc` feature is `true` for enabling container storage PVC on VM, +there several parameters that must be configured: + +`pvc_storageclass` for pvc storage class (`kubectl get sc`) + +`pvc_pvcaccessmode` can be one of ReadWriteOnce,ReadOnlyMany,ReadWriteMany Default: ReadWriteOnce + +`pvc_pvcvolumemode` can be one of Filesystem,Block Default: Filesystem + +`pvc_storagesize` the PVC storage size + +HostPath: The data will be stored on the local disk where the OS is placed. + +The `hostpath` feature is `true` for enabling HostPath on VM, there several parameters that must be configured: + +`hostpath_path` The image path to hold the hostPath + +`hostpath_storagesize` the HostPath storage size + +examples: +[MSSQL](../config/samples/hammerdb/mssql/vm-cr.yaml), +[Postgres](../config/samples/hammerdb/postgres/vm-cr.yaml), +[Mariadb](../config/samples/hammerdb/mariadb/vm-cr.yaml), + + +Currently, we only support images that can be used as [containerDisk](https://docs.openshift.com/container-platform/4.6/virt/virtual_machines/virtual_disks/virt-using-container-disks-with-vms.html#virt-preparing-container-disk-for-vms_virt-using-container-disks-with-vms). + +You can easily make your own container-disk-image as follows by downloading your qcow2 image of choice. +You can then make changes to your qcow2 image as needed using virt-customize. + +```bash +cat << END > Dockerfile +FROM scratch +ADD .qcow2 /disk/ +END + +podman build -t . +podman push +``` + +You can either access results by indexing them directly or by accessing the console. +The results are stored in /tmp/ directory +You can console into VM by running `virtctl console vmi_name` diff --git a/docs/image_pull.md b/docs/image_pull.md new file mode 100644 index 000000000..88007a201 --- /dev/null +++ b/docs/image_pull.md @@ -0,0 +1,50 @@ +# Image Pull + +## What does it do? + +The Image Pull workload will take a list of images that exist in a container repository (e.x. Quay) +and copy them to the containers local working directory via [skopeo](https://github.com/containers/skopeo). +It will then display and/or index relevant data regarding the amount of time it took as well as any retries +or failures that occurred. + +Additioanlly, it can be configured to run multiple pods at the same time, each reporting its own relevant data. +This allows the user to test concurrency of image pulls on a container image. + +## Variables + +### Required variables: + +`image_list` a list of images in the format [image_transport]://[image_location] + +### Optional variables: + +`pod_count` total number of concurrent pods/tests to run (default: 1) + +`timeout` how long, in seconds, to wait for the image to copy (default: 600) + +`retries` how many times to retry failed copies (default: 0) + +### Example CR + +Your resource file may look like this when running 10 concurrent pods with 1 retry: + +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: image-pull + namespace: benchmark-operator +spec: + elasticsearch: + url: "http://es-instance.com:9200" + index_name: "image-pull" + workload: + name: image_pull + args: + pod_count: 2 + timeout: 600 + retries: 1 + image_list: + - docker://quay.io/cloud-bulldozer/backpack + - docker://quay.io/cloud-bulldozer/fio +``` diff --git a/docs/installation.md b/docs/installation.md deleted file mode 100644 index 5ad67650a..000000000 --- a/docs/installation.md +++ /dev/null @@ -1,120 +0,0 @@ -## Installation -This guide uses minikube version v1.5.2+ as the local Kubernetes cluster -and quay.io for the public registry. We also test on crc version v1.7.0+. -`kubectl` is used but can be substituted with `oc`. - -### Tested versions -* [OpenShift® Container Platform](https://www.openshift.com/products/container-platform/) - * Tested on: 4.3 and later -* [kubernetes](https://kubernetes.io/) - * Tested on: 1.16.2 and later - -Note: -* Experimental tag refers to some workloads that might be functioning -* To use versions of Openshift and kubernetes prior to 4.3 and 1.16.2 respectively, please use version 0.0.2 of ripsaw - -### Requirements - - -The main requirements are as follows: -* Running Kubernetes cluster - [supported versions](#Supported-Versions) -* [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) -* [git](https://git-scm.com/downloads) - -Note: Please login as admin user - -The following requirements are needed to make/test changes to operator: -* [operator-sdk](https://github.com/operator-framework/operator-sdk) -* [docker](https://docs.docker.com/install/) - -Note: You also require a [quay](https://quay.io/) account to push images - -The following optional requirements are needed if using OKD/OCP < 4.0: -* [dep](https://golang.github.io/dep/docs/installation.html) -* [operator-sdk](https://github.com/operator-framework/operator-sdk) -* [go](https://golang.org/dl/) -* [OLM](https://github.com/operator-framework/operator-lifecycle-manager) - -The workloads could also have their own requirements which would be specified -in the installation guide. - -### Deploying operator -Note: The benchmark-operator's code-name is ripsaw, so the names have been -used interchangeably. - -Note: If you're on a vanilla k8s distribution, then you can also deploy Ripsaw through - operatorhub.io, please check [ripsaw in operatorhub](https://operatorhub.io/operator/ripsaw) for more details. - -First we'll need to clone the operator: - -```bash -# git clone https://github.com/cloud-bulldozer/ripsaw -# cd ripsaw -# export KUBECONFIG= # if not already done -``` - -We try to maintain all resources created/required by ripsaw in the namespace `my-ripsaw`, -as this would be the namespace, ripsaw would be installed into if deployed through operatorhub.io. - -Note: But in case you've a specific usecase where you want the resources to be in a different namespace, you'll just need to edit the namespace in deploy/ -as well as the operator definition. - -But for sake of the documentation, let's proceed with the namespace `my-ripsaw` - -so we'll create the namespace as follows - -```bash -# kubectl apply -f resources/namespace.yaml -``` - -or if you're used to `oc` it'd be `oc new-project my-ripsaw` and `oc project my-ripsaw` - -We'll now apply the permissions and operator definitions. - -```bash -# kubectl apply -f deploy -# kubectl apply -f resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -# kubectl apply -f resources/operator.yaml -``` - -### Running workload -Now that we've deployed our operator, follow workload specific instructions to -run workloads: -* [uperf](uperf.md) -* [fio](fio_distributed.md) -* [sysbench](sysbench.md) -* [YCSB](ycsb.md) -* [Bring your own workload](byowl.md) -* [pgbench](pgbench.md) -* [fs-drift](fs-drift.md) -* [servicemesh](servicemesh.md) -* [cyclictest](cyclictest.md) -* [oslat](oslat.md) - -If you want to add a new workload please follow these [instructions](../CONTRIBUTE.md#Add-workload) to submit a PR - -### Clean up -Now that we're running workloads we can cleanup by running following commands - -```bash -# kubectl delete -f -# kubectl delete -f resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -# kubectl delete -f resources/operator.yaml -# kubectl delete -f deploy -``` - -## running CI - -If you want to run CI on your laptop as part of developing a PR, you may want to use your own image location and account (i.e. your -own image repository account). To do this, set 2 environment variables: - -* RIPSAW_CI_IMAGE_LOCATION - host where image repository lives (default is quay.io) -* RIPSAW_CI_IMAGE_ACCOUNT - user account (default is rht_perf_ci). - -This allows you to have the CI run on your own private image built with your PR. This assumes that your benchmark's CI -script in tests/ utilizes the common code in tests/common.sh to launch ripsaw. - -You can modify your ripsaw image to use a test version of your benchmark image as well. For examples, see workload.yml.j2 files in the roles/ tree and look for the image: tag. diff --git a/docs/iperf.md b/docs/iperf3.md similarity index 97% rename from docs/iperf.md rename to docs/iperf3.md index 3d50ff0f9..ce9d91d48 100644 --- a/docs/iperf.md +++ b/docs/iperf3.md @@ -5,7 +5,7 @@ ## Running iperf Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_iperf3_cr.yaml) +you can modify [cr.yaml](../config/samples/iperf3/cr.yaml) ```yaml @@ -13,20 +13,20 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: iperf3-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: name: iperf3 args: pairs: 1 hostnetwork: false - pin: true - pin_server: "master-0" - pin_client: "master-1" + #pin: true + #pin_server: "master-0" + #pin_client: "master-1" port: 5201 transmit_type: time transmit_value: 60 omit_start: 0 - length_buffer: 128K + #length_buffer: 128K window_size: 64k ip_tos: 0 mss: 900 @@ -50,6 +50,9 @@ then you'd probably give a `retries` of 60, as 60*15 is 900s. runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +`annotations` can be set to apply the specified +annotations to the pod metadata. + The rest of the args are compulsory arguments that need to be passed and can cause issues if missed, they are: @@ -116,7 +119,7 @@ $ oc adm policy add-scc-to-user privileged -z benchmark-operator Once done creating/editing the resource file, you can run it by: ```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_iperf3_cr.yaml # if edited the original one +# kubectl apply -f config/samples/iperf3/cr.yaml # if edited the original one # kubectl apply -f # if created a new cr file ``` diff --git a/docs/kube-burner.md b/docs/kube-burner.md index 3fd8acc35..2ad49490d 100644 --- a/docs/kube-burner.md +++ b/docs/kube-burner.md @@ -19,7 +19,7 @@ The benchmark-operator integration here is meant to run only some workloads usef ## Running kube-burner Given that you followed instructions to deploy benchmark-operator. Kube-burner needs an additional serviceaccount and clusterrole to run. Available at [kube-burner-role.yml](../resources/kube-burner-role.yml) -You can modify kube-burner's [cr.yaml](../resources/crds/ripsaw_v1alpha1_kube-burner_cr.yaml) to fit your requirements. +You can modify kube-burner's [cr.yaml](../config/samples/kube-burner/cr.yaml) to fit your requirements. ---- @@ -49,6 +49,23 @@ Each iteration of this workload creates the following objects: - 1 deployment holding a client application for the previous database - 1 service pointing to the postgresl database +- **node-density-cni**. Creates a **single namespace with a number of applications equals to job_iterations**. This application consists on two deployments (a node.js webserver and a simple client that curls the webserver) and a service that is used by the client to reach the webserver. +Each iteration of this workload creates the following objects: + - 1 deployment holding a node.js webserver + - 1 deployment holding a client application for curling the webserver + - 1 service pointing to the webserver + + The Readiness Probe of the client pod depends on being able to reach the webserver so that the PodReady latencies collected by kube-burner reflect network connectivity. + +- **node-density-cni-policy**. Creates a **single namespace with a number of applications equals to job_iterations**. This application consists on two deployments (a node.js webserver and a simple client that curls the webserver) and a service that is used by the client to reach the webserver. +Each iteration of this workload creates the following objects: + - 1 deployment holding a node.js webserver + - 1 deployment holding a client application for curling the webserver + - 1 service pointing to the webserver + + A NetworkPolicy to deny all connections is created in the namspace first and then NetworkPolicies specifically applying the connection of each client-webserver pair are applied. The Readiness Probe of the client pod depends on being able to reach the webserver so that the PodReady latencies collected by kube-burner reflect network connectivity. + + - **max-namespaces**: This workload is a cluster limits focused test which creates maximum possible namespaces across the cluster. This is a namespaced workload, meaning that kube-burner **will create as many namespaces with these objects as the configured job_iterations**. - 1 deployment holding a postgresql database - 5 deployments consisting of a client application for the previous database @@ -63,13 +80,20 @@ Each iteration of this workload creates the following objects: Each iteration of this workload creates the following object: - 1 pod. (sleep) +- **concurrent-builds**: Creates a buildconfig, imagestream and corresponding build for a set application. **This will create as many namespaces with these objects as the configured job_iterations**. +See https://github.com/cloud-bulldozer/e2e-benchmarking/tree/master/workloads/kube-burner for example parameters for each application +Each iteration of this workload creates the following object: + - 1 imagestream (dependent on application type set) + - 1 buildconfig (also dependent on application type set) + - 1 build created from buildconfig + The workload type is specified by the parameter `workload` from the `args` object of the configuration. Each workload supports several configuration parameters, detailed in the [configuration section](#configuration) ## Configuration All kube-burner's workloads support the following parameters: -- **`workload`**: Type of kube-burner workload. As mentioned before, allowed values are cluster-density, node-density and node-density-heavy +- **``workload``**: Type of kube-burner workload. As mentioned before, allowed values are cluster-density, node-density and node-density-heavy - **``default_index``**: ElasticSearch index name. Defaults to __ripsaw-kube-burner__ - **``job_iterations``**: How many iterations to execute of the specified kube-burner workload - **``qps``**: Limit object creation queries per second. Defaults to __5__ @@ -96,6 +120,8 @@ Where key defaults to __node-role.kubernetes.io/worker__ and value defaults to e - **``step``**: Prometheus step size, useful for long benchmarks. Defaults to 30s - **``metrics_profile``**: kube-burner metric profile that indicates what prometheus metrics kube-burner will collect. Defaults to `metrics.yaml` in node-density workloads and `metrics-aggregated.yaml` in the remaining. Detailed in the [Metrics section](#Metrics) of this document - **``runtime_class``** : If this is set, the benchmark-operator will apply the runtime_class to the podSpec runtimeClassName. +- **``annotations``** : If this is set, the benchmark-operator will set the specified annotations on the pod's metadata. +- **``extra_env_vars``** : This dictionary defines a set of fields that will be injected to the kube-burner pod as environment variables. e.g. `extra_env_vars: {"foo": "bar", "foo2": "bar2"}` kube-burner is able to collect complex prometheus metrics and index them in a ElasticSearch. This feature can be configured by the prometheus object of kube-burner's CR. @@ -179,14 +205,23 @@ Keep in mind that the object templated declared in this remote configuration fil replicas: 1 ``` -> `kube-burner` is able to use go template based configuration files, in addition to the default behaviour, this template can reference environment variables using the syntax `{{ .MY_ENV_VAR }}`. The kube-burner job created by `benchmark-operator` always injects the environment variable `prom_es` with the value of `prometheus.es_url`. This can be useful to overwrite the ElasticSearch URL in remote configuration files as shown in the code snippet below. +> `kube-burner` is able to use go template based configuration files, in addition to the default behaviour, this template can reference environment variables using the syntax `{{ .MY_ENV_VAR }}`. The kube-burner job created by `benchmark-operator` always injects a list of environment variables which can be defined with the parameter `extra_env_vars` mentioned previously. This can be useful to parametrize remote configuration files as shown in the code snippet below. + +Supossing a CR with `extra_env_vars` configured as: +```yaml +workload: + args: + extra_env_vars: + INDEXING: true + ES_SERVER: https://example-es.instance.com:9200 +``` ```yaml global: writeToFile: false indexerConfig: - enabled: true - esServers: ["{{.prom_es}}"] + enabled: {{.INDEXING}} + esServers: ["{{.ES_SERVER}}"] insecureSkipVerify: true defaultIndex: ripsaw-kube-burner type: elastic @@ -236,3 +271,40 @@ It supports different severities: - critical: Prints a fatal message with the alarm description to stdout and exits execution inmediatly with rc != 0 More information can be found at the [Kube-burner docs site.](https://kube-burner.readthedocs.io/en/latest/alerting/) + +## Reading configuration from a configmap + +Kube-burner is able to fetch it's own configuration from a configmap. To do so you just have to set the argument `configmap` pointing to a configmap in the same namespace where kube-burner is in the CR. This configmap needs to have a config.yml file to hold the main kube-burner's configuration file(apart from the required object templates), and optionally can contain a metrics.yml and alerts.yml files. An example configuration CR would look like: + +```yaml +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-configmap-cfg + namespace: benchmark-operator +spec: + metadata: + collection: false + prometheus: + prom_token: ThisIsNotAValidToken + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + configmap: kube-burner-config + cleanup: true + pin_server: {"node-role.kubernetes.io/worker": ""} + image: quay.io/cloud-bulldozer/kube-burner:latest + log_level: info + step: 30s + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule +``` + +To create a configmap with the kube-burner configurations you can use `kubectl create configmap --from-file= kube-burner-config` + diff --git a/docs/log_generator.md b/docs/log_generator.md new file mode 100644 index 000000000..0f6c45bca --- /dev/null +++ b/docs/log_generator.md @@ -0,0 +1,144 @@ +# Log Generator + +## What does it do? + +The Log Generator writes messages of a set size and at a set rate to stdout. If provided +it will also verify that all messages were recieved by the backend log aggregator. +This data will also be indexed if Elasticsearch information is provided. + +*NOTE* This workload will not deploy the backend log aggregator + +## Variables + +### Required variables: + +`size` the size, in bytes, of the message to send + +`duration` how long, in minutes, messages should be sent for + +`messages_per_second` the number of messages per second (mutually exclusive with messages_per_minute) + +`messages_per_minute` the number of messages per minute (mutually exclusive with messages_per_second) + +### Optional variables: + +`pod_count` total number of log generator pods to launch (default: 1) + +`timeout` how long, in seconds, after have been sent to allow the backend service to receive all the messages (default: 600) + +`snafu_disable_logs` Disable all logging in the pod from the snafu logger, thereby only leaving the generated log messages on stdout (default: False) + +### Verification variables: + +To verify your messages have been received by the backend aggregator you must provide information for ONLY ONE of the supported +backends: Elasticsearch or AWS CloudWatch + +Elasticsearch Backend: + +`es_url` the elasticsearch url to query for results. + +`es_index` the index to search for the sent messages (default: app*) + +`es_token` the bearer token to use to access elasticsearch if required + +AWS CloudWatch: + +`cloudwatch_log_group` the aws cloudwatch log group to query + +`aws_region` the region that cloudwatch is deployed to + +`aws_access_key` the access key to use to query cloudwatch + +`aws_secret_key` the secret key to use to query cloudwatch + +Kafka: + +`kafka_bootstrap_server` the connection details to kafka + +`kafka_topic` the topic where logs are stored + +`kafka_check` if you want to verify that log messages made it to kafka sink (requires a high timeout) + +Your resource file may look like this when using an Elasticsearch Backend: + +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: log-generator + namespace: benchmark-operator +spec: + elasticsearch: + url: "http://es-instance.com:9200" + index_name: log-generator + workload: + name: log_generator + args: + pod_count: 2 + size: 512 + messages_per_second: 10 + duration: 1 + es_url: "https://my-es-backend.com" + es_token: "sha256~myToken" + timeout: 600 + label: + key: foo + value: "" +``` + +Your resource file may look like this when using an AWS CloudWatch Backend: + +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: log-generator + namespace: benchmark-operator +spec: + elasticsearch: + url: "http://es-instance.com:9200" + index_name: log-generator + workload: + name: log_generator + args: + pod_count: 10 + size: 1024 + messages_per_second: 100 + duration: 10 + cloudwatch_log_group: "my_log_group" + aws_region: "us-west-2" + aws_access_key: "myKey" + aws_secret_token: "myToken" + timeout: 800 + label: + key: bar + value: "" +``` + +Your resource file may look like this when using a Kafka Backend: + +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: log-generator + namespace: benchmark-operator +spec: + elasticsearch: + url: "http://es-instance.com:9200" + index_name: log-generator + workload: + name: log_generator + args: + pod_count: 2 + size: 512 + messages_per_second: 10 + duration: 1 + kafka_bootstrap_server: "my-cluster-kafka-bootstrap.amq:9092" + kafka_topic: "topic-logging-app" + timeout: 600 + label: + key: foo + value: "" +``` + diff --git a/docs/metadata.md b/docs/metadata.md index 108c6c0d7..79871c1e2 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -36,9 +36,6 @@ the workload template with an init container section that looks like: ```jinja {% if metadata.collection is sameas true and metadata.targeted is sameas true %} -{% if metadata.serviceaccount != "default" %} - serviceAccountName: {{ metadata.serviceaccount }} -{% endif %} initContainers: - name: backpack-{{ trunc_uuid }} image: {{ metadata.image }} @@ -56,11 +53,11 @@ the workload template with an init container section that looks like: {% if metadata.force is sameas true %} --force {% endif %} -{% if metadata.stockpile_tags|length > 0 %} - --tags={{ metadata.stockpile_tags|join(",") }} +{% if metadata.stockpileTags|length > 0 %} + --tags={{ metadata.stockpileTags|join(",") }} {% endif %} -{% if metadata.stockpile_skip_tags|length > 0 %} - --skip-tags={{ metadata.stockpile_skip_tags|join(",") }} +{% if metadata.stockpileSkipTags|length > 0 %} + --skip-tags={{ metadata.stockpileSkipTags|join(",") }} {% endif %} {% if metadata.ssl is sameas true %} --sslskipverify @@ -112,7 +109,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: byowl-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://my.elastic.server.foo:9200" @@ -165,7 +162,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: byowl-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: byowl @@ -181,7 +178,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: byowl-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: metadata: collection: true @@ -202,7 +199,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: byowl-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://my.elastic.server.foo:9200" @@ -261,71 +258,6 @@ metadata: ssl: true ``` -### Additional k8s Information - -There are multiple kubernetes (k8s) based modules that are run in stockpile. -These modules are not accessable by default as the default service account -that runs the pod does not have the required privileges to view them. - -To allow the pods view permissions on the cluster you can apply -the following yaml to create a service account with the appropriate -privileges. - -You will need to change the namespace to fit with your environment -Note: You can also find this yaml in [backpack_role.yaml](../resources/backpack_role.yaml) - -```yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: backpack_role -rules: -- apiGroups: - - "*" - resources: - - "*" - verbs: - - get - - list - - watch ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: backpack-view - namespace: my-ripsaw ---- -apiVersion: v1 -kind: Secret -metadata: - name: backpack-view - namespace: my-ripsaw - annotations: - kubernetes.io/service-account.name: backpack-view -type: kubernetes.io/service-account-token ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: backpack-view - namespace: my-ripsaw -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: backpack_role -subjects: -- kind: ServiceAccount - name: backpack-view - namespace: my-ripsaw -``` - -Once the `backpack-view` service account is created you can modify the default backpack service account setting: - -```yaml -metadata: - serviceaccount: backpack-view -``` - ### Stockpile tags Backpack leverages [stockpile](https://github.com/cloud-bulldozer/stockpile/) to collect metadata. Stockpile is basically a set of Ansible roles, these roles have different tags. It's possible to pass custom tags to the inner `ansible-playbook` command through the parameters `stockpileSkipTags` and `stockpileTags` from the metadata section. These parameters are translated to the Ansible's flags _--tags_ and _--skip-tags_ respectively. diff --git a/docs/oslat.md b/docs/oslat.md index 60fc1694b..e53d671bf 100644 --- a/docs/oslat.md +++ b/docs/oslat.md @@ -5,7 +5,7 @@ The goal of the oslat workload in the benchmark-operator is to run oslat inside ## Running oslat -Given that you followed instructions to deploy operator, you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_oslat.yaml) to your needs. +Given that you followed instructions to deploy operator, you can modify [cr.yaml](../config/samples/oslat/cr.yaml) to your needs. It is recommended to define pod requests and limits when running oslat test, to give guaranteed CPUs to the pods. It is also expected to have the realtime kernel installed with required isolation for pods using the [Performance Add-On Operator](https://github.com/openshift-kni/performance-addon-operators). @@ -13,6 +13,9 @@ The option **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option **annotations** can be set to apply the specified +annotations to the pod metadata. + An example CR might look like this ```yaml @@ -20,13 +23,14 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: oslat - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: server: workload: name: "oslat" args: + node_selector: "" runtime: "1m" disable_cpu_balance: true use_taskset: true @@ -42,7 +46,8 @@ spec: You can run it by: ```bash -oc apply -f resources/crds/ripsaw_v1alpha1_oslat_cr.yaml # if edited the original one +# kubectl apply -f config/samples/oslat/cr.yaml # if edited the original one +# kubectl apply -f # if created a new cr file ``` ## Looking at results @@ -50,7 +55,7 @@ You can look at the results from the oslat benchmark by doing ```bash NAME=oslat-workload-xxxxxxx -oc logs -n my-ripsaw $NAME +oc logs -n benchmark-operator $NAME ``` ## Cleanup diff --git a/docs/pgbench.md b/docs/pgbench.md index 9621f2c0f..784634d45 100644 --- a/docs/pgbench.md +++ b/docs/pgbench.md @@ -5,7 +5,7 @@ ## Running pgbench Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_pgbench_cr.yaml) to your needs. +you can modify [cr.yaml](../config/samples/pgbench/cr.yaml) to your needs. The pgbench workload needs to be pointed at one or more existing PostgreSQL databases via the CR file. @@ -16,7 +16,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: pgbench-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: "pgbench" @@ -78,6 +78,9 @@ The `runtime_class` option can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option `annotations` can be set to apply the specified +annotations to the pod metadata. + Once done creating/editing the resource file, you can run it by: ```bash diff --git a/docs/prometheus.md b/docs/prometheus.md index ccb65d9f2..b4fa1b502 100644 --- a/docs/prometheus.md +++ b/docs/prometheus.md @@ -43,7 +43,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: smallfile-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: test_user: homer_simpson clustername: test_ci diff --git a/docs/scale_openshift.md b/docs/scale_openshift.md index dd365b7d8..b048162ea 100644 --- a/docs/scale_openshift.md +++ b/docs/scale_openshift.md @@ -12,8 +12,6 @@ The scale workload takes the following required variables: `scale` the target number of workers to scale to. -`serviceaccount` the service account to run as. Note the "default" account generally does not have enough privileges - Optional variables: `poll_interval` how long, in seconds, to wait between polls to see if the scaling is complete. default: 5 @@ -28,6 +26,24 @@ Optional variables: `runtime_class` If this is set, the benchmark-operator will apply the runtime_class to the podSpec runtimeClassName. +`annotations` If this is set, the benchmark-operator will set the specified annotations on the pods' metadata. + +`rosa` For clusters installed using ROSA. Following parameters will be required: + +`rosa.cluster_name` Name of cluster as it is shown on `rosa list clusters` command + +`rosa.env` ROSA environment where cluster is installed. + +`rosa.token` ROSA token required to execute commands + +`aws` AWS credentials required by ROSA cli to execute scalation commands + +`aws.access_key_id` Exported as AWS_ACCESS_KEY_ID + +`aws.secret_access_key` Exported as AWS_SECRET_ACCESS_KEY + +`aws.default_region` Exported as AWS_DEFAULT_REGION + Your resource file may look like this when using all avaible options: ```yaml @@ -35,7 +51,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: scale - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://es-instance.com:9200" @@ -44,7 +60,6 @@ spec: name: scale_openshift args: scale: 25 - serviceaccount: scaler poll_interval: 2 post_sleep: 300 label: @@ -54,51 +69,16 @@ spec: key: role value: workload effect: NoSchedule + rosa: + cluster_name: rosa-test-01 + env: staging + token: "xxxx" + aws: + access_key_id: XXXXX + secret_access_key: XXXXXX + default_region: us-west-2 ``` *NOTE:* If the cluster is already at the desired scale the timings will still be captured and uploaded to Elasticsearch (if provided). The overall time will simply be the time it took the scale process to confirm that the cluster is at the correct scale. - -## Service Account and Permissions - -The default service account generally does not have the required visibility/permissions to allow the Scale job -to modify the cluster machine sets. An example service account setup has been provided in resources/scale_role.yaml -as well as described below: - -``` -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: scale_role -rules: -- apiGroups: - - "*" - resources: - - machines - - machinesets - - nodes - - infrastructures - verbs: - - '*' ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: scaler - namespace: my-ripsaw ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: scaler - namespace: my-ripsaw -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: scale_role -subjects: -- kind: ServiceAccount - name: scaler - namespace: my-ripsaw -``` diff --git a/docs/servicemesh.md b/docs/servicemesh.md index 1c4f6766b..2b916b59d 100644 --- a/docs/servicemesh.md +++ b/docs/servicemesh.md @@ -20,16 +20,19 @@ The option **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option **annotations** can be set to apply the specified +annotations to the pod metadata. + ## Running the benchmark -Here is an example of the [benchmark CR](../resources/crds/ripsaw_v1alpha1_servicemesh_cr.yaml): +Here is an example of the [benchmark CR](../config/samples/servicemesh/cr.yaml): ```yaml apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: example - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: servicemesh @@ -71,7 +74,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: servicemesh-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: servicemesh @@ -108,7 +111,7 @@ spec: You can run it by: ```bash -oc apply -f resources/crds/ripsaw_v1alpha1_servicemesh_cr.yaml # if edited the original one +oc apply -f config/samples/servicemesh/cr.yaml # if edited the original one ``` ## Visualize the report @@ -117,7 +120,7 @@ While the output from benchmark is a JSON, you can easily display ```bash NAME=servicemesh-benchmark-xxxxxxx -oc logs -n my-ripsaw $NAME > /tmp/$NAME.json +oc logs -n benchmark-operator $NAME > /tmp/$NAME.json cat /tmp/$NAME.json | docker run -i --rm quay.io/hyperfoil/hyperfoil-report /opt/report.sh > /tmp/$NAME.html ``` diff --git a/docs/smallfile.md b/docs/smallfile.md index 2c3a21e89..de8cdb795 100644 --- a/docs/smallfile.md +++ b/docs/smallfile.md @@ -3,111 +3,35 @@ [Smallfile](https://github.com/distributed-system-analysis/smallfile) is a python-based distributed POSIX workload generator which can be used to quickly measure performance for a variety of metadata-intensive workloads across an entire cluster. ## Running Smallfile Benchmark using Ripsaw -Once the operator has been installed following the instructions, one needs to modify the clients parameter(which is currently set to 0), to value greater than 0 in [ripsaw_v1alpha1_smallfile_cr.yaml](../resources/crds/ripsaw_v1alpha1_smallfile_cr.yaml) to run default "create" the test. Also, in addtion to that, smallfile operator is completely dependent on storageclass and storagesize. Please make sure to double check the parameters in CRD file. +Once the operator has been installed following the instructions, one needs to modify the clients parameter(which is currently set to 0), to value greater than 0 in [cr.yaml](../config/samples/smallfile/cr.yaml) to run default "create" the test. Also, in addtion to that, smallfile operator is completely dependent on storageclass and storagesize. Please make sure to double check the parameters in CRD file. -```yaml -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: example-benchmark - namespace: my-ripsaw -spec: - workload: - name: smallfile - args: - clients: 1 - operation: ["create"] - threads: 5 - file_size: 64 - files: 100 - storageclass: rook-ceph-block # Provide if PV is needed - storagesize: 5Gi # Provide if PV is needed -``` - -Smallfile operator also gives the leverage to run multiple test operations in a user-defined sequence. Like in the [Custom Resource Definition file](../resources/crds/ripsaw_v1alpha1_smallfile_cr.yaml), the series of operation can be mentioned as: +Smallfile operator also gives the leverage to run multiple test operations in a user-defined sequence. Like in the [Custom Resource Definition file](../config/samples/smallfile/cr.yaml), the series of operations can be specified as a list. -```yaml -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: example-benchmark - namespace: my-ripsaw -spec: - workload: - name: smallfile - args: - clients: 1 - operation: ["create","read","append", "delete"] - threads: 5 - file_size: 64 - files: 100 - storageclass: rook-ceph-block # Provide if PV is needed - storagesize: 5Gi # Provide if PV is needed -``` +NOTE: While running the sequence of tests using smallfile workload, please make sure that the initial operation must be create, and the "cleanup" operation should come in termination, else smallfile might produce error due to meaningless sequence of tests. For example: -NOTE: While running the sequence of tests using smallfile workload, please make sure that the initial operation must be create, and the "delete" operation should come in termination, else smallfile might produce error due to meaningless sequence of tests. For example: ```bash -operation: ["read","append","create", "delete"] + +operation: ["read","append","create", "delete","cleanup"] #This will be meaningless sequence of test as trying to read something, which has not been created yet.The same logic applies for the append test as well. Hence, smallfile will produce error. ``` ## Adding More options in smallfile tests -Smallfile also comes with a variety of configurable options for running tests, following are the parameters that can be added to CR file: - - * **response_times** – if Y then save response time for each file operation in a - rsptimes\*csv file in the shared network directory. Record format is - operation-type, start-time, response-time. The operation type is included so - that you can run different workloads at the same time and easily merge the - data from these runs. The start-time field is the time that the file - operation started, down to microsecond resolution. The response time field is - the file operation duration down to microsecond resolution. - * **file_size_distribution** – only supported value today is exponential. - * **record_size** -- record size in KB, how much data is transferred in a single - read or write system call. If 0 then it is set to the minimum of the file - size and 1-MiB record size limit. - * **files_per_dir** -- maximum number of files contained in any one directory. - * **dirs_per_dir** -- maximum number of subdirectories contained in any one - directory. - * **hash_into_dirs** – if Y then assign next file to a directory using a hash - function, otherwise assign next –files-per-dir files to next directory. - * **same_dir** -- if Y then threads will share a single directory. - * **network_sync_dir** – don't need to specify unless you run a multi-host test - and the –top parameter points to a non-shared directory (see discussion - below). Default: network_shared subdirectory under –top dir. - * **permute_host_dirs** – if Y then have each host process a different - subdirectory tree than it otherwise would (see below for directory tree - structure). - * **xattr_size** -- size of extended attribute value in bytes (names begin with - 'user.smallfile-') - * **xattr_count** -- number of extended attributes per file - * **prefix** -- a string prefix to prepend to files (so they don't collide with -previous runs for example) - * **suffix** -- a string suffix to append to files (so they don't collide with - previous runs for example) - * **incompressible** – if Y then generate a pure-random file that - will not be compressible (useful for tests where intermediate network or file - copy utility attempts to compress data - * **record_ctime_size** -- if Y then label each created file with an - xattr containing a time of creation and a file size. This will be used by - –await-create operation to compute performance of asynchonous file - replication/copy. - * **finish** -- if Y, thread will complete all requested file operations even if - measurement has finished. - * **stonewall** -- if Y then thread will measure throughput as soon as it detects - that another thread has finished. - * **verify_read** – if Y then smallfile will verify read data is correct. - * **remote_pgm_dir** – don't need to specify this unless the smallfile software - lives in a different directory on the target hosts and the test-driver host. - * **pause** -- integer (microseconds) each thread will wait before starting next - file. - * **runtime_class** - If this is set, the benchmark-operator will apply the runtime_class to the podSpec runtimeClassName. +Smallfile also comes with a variety of configurable options that can be added by the user to the CR for running tests, documented [at its github site here](https://github.com/distributed-system-analysis/smallfile#readme) . To obtain the YAML parameter name, remove the 2 preceding dashes and convert remaining dashes to underscores and append a colon character. For example, **--file-size** becomes **file_size:** . Parameters that **are not** usable in the smallfile CR include: +* --yaml-input-file - used by the benchmark +* --operation - you specify this in the CR in the operation list +* --top - you do not specify this, Kubernetes points smallfile to the PV mountpoint +* --response-times - used by the benchmark +* --output-json - used by the benchmark +* --network-sync-dir - not applicable for pods +* --permute-host-dirs - not applicable for pods +* --remote-pgm-dir - not applicable for pods Once done creating/editing the resource file, one can run it by: ```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_smallfile_cr.yaml # if edited the original one +# kubectl apply -f config/samples/smallfile/cr.yaml # if edited the original one # kubectl apply -f # if created a new cr file ``` @@ -117,57 +41,55 @@ $ kubectl get pods NAME READY STATUS RESTARTS AGE benchmark-operator-7c6bc98b8c-2j5x5 2/2 Running 0 47s example-benchmark-smallfile-client-1-benchmark-hwj4b 1/1 Running 0 33s +example-benchmark-smallfile-publisher... ``` -To see the output of the run one has to run `kubectl logs `. This is shown below: +To see the output of the run one has to run `kubectl logs `. This looks *approximately* like: ```bash -$ kubectl logs example-benchmark-smallfile-client-1-benchmark-hwj4b -create -RUnning_OPERATION_create -top: /mnt/pvc/smallfile_test_data -threads: 5 -file-size: 64 -files: 100 - version : 3.1 - hosts in test : None - launch by daemon : False - top test directory(s) : ['/mnt/pvc/smallfile_test_data'] - operation : create - files/thread : 100 - threads : 5 - record size (KB, 0 = maximum) : 0 - file size (KB) : 64 - file size distribution : fixed - files per dir : 100 - dirs per dir : 10 - threads share directories? : N - filename prefix : - filename suffix : - hash file number into dir.? : N - fsync after modify? : N - pause between files (microsec) : 0 - minimum directories per sec : 50 - finish all requests? : Y - stonewall? : Y - measure response times? : N - verify read? : Y - verbose? : False - log to stderr? : False - ext.attr.size : 0 - ext.attr.count : 0 -host = example-benchmark-smallfile-client-1-benchmark-hwj4b,thr = 00,elapsed = 0.015719,files = 100,records = 100,status = ok -host = example-benchmark-smallfile-client-1-benchmark-hwj4b,thr = 01,elapsed = 0.020679,files = 100,records = 100,status = ok -host = example-benchmark-smallfile-client-1-benchmark-hwj4b,thr = 02,elapsed = 0.028791,files = 100,records = 100,status = ok -host = example-benchmark-smallfile-client-1-benchmark-hwj4b,thr = 03,elapsed = 0.029367,files = 100,records = 100,status = ok -host = example-benchmark-smallfile-client-1-benchmark-hwj4b,thr = 04,elapsed = 0.029750,files = 100,records = 100,status = ok -total threads = 5 -total files = 500 -total IOPS = 500 -total data = 0.031 GiB -100.00% of requested files processed, minimum is 70.00 -elapsed time = 0.030 -files/sec = 16806.661271 -IOPS = 16806.661271 -MiB/sec = 1050.41633 +$ kubectl logs example-benchmark-smallfile-client-1-benchmark-hwj4b +Waiting For all Smallfile Pods to get ready ... +Executing Smallfile... +2021-08-24T20:26:32Z - INFO - MainProcess - trigger_smallfile: running:smallfile_cli.py --operation create --top /mnt/pvc/smallfile_test_data --output-json /var/tmp/RESULTS/1/create.json --response-times Y --yaml-input-file /tmp/smallfile/smallfilejob +2021-08-24T20:26:32Z - INFO - MainProcess - trigger_smallfile: from current directory /opt/snafu + version : 3.2 + hosts in test : None + launch by daemon : False + top test directory(s) : ['/mnt/pvc/smallfile_test_data'] + operation : create + files/thread : 10000 + threads : 1 + record size (KB, 0 = maximum) : 0 + file size (KB) : 4 + file size distribution : fixed + files per dir : 100 + dirs per dir : 10 + threads share directories? : N + filename prefix : + filename suffix : + hash file number into dir.? : N + fsync after modify? : N + pause between files (microsec) : 0 + auto-pause? : N + delay after cleanup per file (microsec) : 0 + minimum directories per sec : 50 + total hosts : 30 + finish all requests? : Y + stonewall? : Y + measure response times? : Y + verify read? : Y + verbose? : False + log to stderr? : False +host = smallfile-client-1-benchmark-84ad212e-9h454,thr = 00,elapsed = 0.592771,files = 10000,records = 10000,status = ok +total threads = 1 +total files = 10000 +total IOPS = 16869 +total data = 0.038 GiB +100.00% of requested files processed, warning threshold is 70.00 +elapsed time = 0.593 +files/sec = 16869.919582 +IOPS = 16869.919582 +MiB/sec = 65.898123 +2021-08-24T20:26:37Z - INFO - MainProcess - trigger_smallfile: completed sample 1 for operation create , results in /var/tmp/RESULTS/1/create.json +... ``` diff --git a/docs/stressng.md b/docs/stressng.md index 421218ec5..7fbb9e8c4 100644 --- a/docs/stressng.md +++ b/docs/stressng.md @@ -5,12 +5,15 @@ ## Running stressng Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_stressng.yaml) to your needs. +you can modify [cr.yaml](../config/samples/stressng/cr.yaml) to your needs. The optional argument **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option **annotations** can be set to apply the specified +annotations to the pod metadata. + An example CR might look like this ```yaml @@ -18,7 +21,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: stressng - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://es-instance.com:9200" @@ -36,6 +39,7 @@ spec: # cpu stressor options cpu_stressors: "1" cpu_percentage: "100" + cpu_method: "all" # vm stressor option vm_stressors: "1" vm_bytes: "128M" @@ -94,7 +98,7 @@ Note: this is currently in preview mode. ``` The above is the additional changes required to run stressng in vms. -Currently, we only support images that can be used as [containerDisk](https://kubevirt.io/user-guide/docs/latest/creating-virtual-machines/disks-and-volumes.html#containerdisk). +Currently, we only support images that can be used as [containerDisk](https://docs.openshift.com/container-platform/4.6/virt/virtual_machines/virtual_disks/virt-using-container-disks-with-vms.html#virt-preparing-container-disk-for-vms_virt-using-container-disks-with-vms). You can easily make your own container-disk-image as follows by downloading your qcow2 image of choice. You can then make changes to your qcow2 image as needed using virt-customize. diff --git a/docs/sysbench.md b/docs/sysbench.md index 0983ee7e7..91848a7b9 100644 --- a/docs/sysbench.md +++ b/docs/sysbench.md @@ -5,14 +5,29 @@ ## Running Sysbench Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_sysbench_cr.yaml) +you can modify [cr.yaml](../config/samples/sysbench/cr.yaml) The optional argument **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option **annotations** can be set to apply the specified +annotations to the pod metadata. + +The `pin_node` parameter allows to place the sysbench pod +on a specific node, using the `hostname` label. + +The `storageclass` and `storagesize` options can be set to have sysbench run on a particular StorageClass volume. +If `storageclass` is not provided, sysbench runs on an `emptyDir` volume by default. + +With the `fileio` test, you can define parameters for the `prepare`, `run` and `cleanup` steps +independently, using `prepare_parameters`, `run_parameters`, and `cleanup_parameters` respectively. +The `parameters` option is a global set of parameters each step inherits, but can be overriden. To +disable a value from `parameters` within `_parameters`, simply set the value to +`null`. + Note: please ensure you set 0 for other workloads if editing the -[cr.yaml](../resources/crds/ripsaw_v1alpha1_sysbench_cr.yaml) file otherwise +[cr.yaml](../config/samples/sysbench/cr.yaml) file otherwise your resource file may look like this: @@ -21,7 +36,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: sysbench-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: workload: name: sysbench @@ -29,6 +44,9 @@ spec: enabled: true #kind: vm # If you want to run this as a VM uncomment the above + #pin_node: "worker-0.mylab.example.com" + #storageclass: ocs-storagecluster-ceph-rbd + #storagesize: 200Gi tests: - name: cpu parameters: @@ -36,6 +54,10 @@ spec: - name: fileio parameters: file-test-mode: rndrw + # This removes the file-test-mode parameter during the cleanup step + # Since run_parameters and prepare_parameters are not defined, they use file-test-mode: rndrw + #cleanup_parameters: + #file-test-mode: null ``` Name here refers to testname and can be cpu or fileio or memory etc and the parameters are the parametes for the particular test. @@ -44,7 +66,7 @@ You can find more information at [sysbench documentation](https://github.com/ako Once done creating/editing the resource file, you can run it by: ```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_sysbench_cr.yaml # if edited the original one +# kubectl apply -f config/samples/sysbench/cr.yaml # if edited the original one # kubectl apply -f # if created a new cr file ``` diff --git a/docs/system-metrics.md b/docs/system-metrics.md new file mode 100644 index 000000000..096297604 --- /dev/null +++ b/docs/system-metrics.md @@ -0,0 +1,57 @@ +# System-metrics collection + +Benchmark-operator is able to collect prometheus metrics from the cluster at the end of a benchmark. To do so, it creates a k8s job that uses [kube-burner](https://github.com/cloud-bulldozer/kube-burner) to collect the Prometheus metrics given by a configuration file. This system metrics collection mechanism in available for all workloads except `kube-burner` and `backpack`. + +This feature is disabled by default, it can be enabled by adding a `system_metrics` section to a benchmark CR. + +```yaml + system_metrics: + collection: true (Defaults to false) + prom_url: + index_name: + es_url: + prom_token: + metrics_profile: + step: + image: +``` + +As stated in the example above, `metrics_profile` points to node-metrics.yml, (this file is available within the system-metrics role of this repo), however it can be configured pointing to an external URL like in the example below: + +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: example-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_token: eyJhbGciOiJSUzI1NiIsImtpZCI6IlljTUxlUHBTY2hvUVJQYUlZWmV5MTE4d3VnRFpjUUh5MWZtdE9hdnlvNFUifQ.eyJpc3MiOiJrdWJlcnopeVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJvcGVuc2hpZnQtbW9uaXRvcmluZyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJwcm9tZXRoZXVzLWs4cy10b2tlbi12NGo3YyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJwcm9tZXRoZXVzLWs4cyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjFkYTdkMTRkLWE2MTktNDZjYS1iZGRlLTMzOTYxOWYxMmM4MiIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDpvcGVuc2hpZnQtbW9uaXRvcmluZzpwcm9tZXRoZXVzLWs4cyJ9.PJp5pD_CjMG05vVLFdxUDRWGA8C71TNyRsUcHmpMlnZLQWBwxZSDZ-Uh3y6g1O-Yz3nopeCoZLB6lugxxalcT1DhiEC9yNK53Lr6HLqaz8nWUbRPbex0913KcuSsnpeRj7tzlwQ2K3WbtIeyyHpG5vAeff07LDvHUcPsc3B_dyetGnInClBHFVEJRES6f5DbIUidtXZEfYKANJNcssly0qZMZicwvM4a_pRp6ctGB-zzR6Ac4lh3b1JLfl_5TLGuuoYEOAeJPVUF4TjemsNNJ5BlycEkVI377LKNdHf83wua5pn3ItJtKE5gdrG833203p-y0pj-UDJj2bAv0cjUQ + metrics_profile: https://raw.githubusercontent.com/cloud-bulldozer/benchmark-operator/master/roles/kube-burner/files/metrics-aggregated.yaml + elasticsearch: + url: "http://es-instance.com:9200" + index_name: ripsaw-uperf + metadata: + collection: false + cleanup: false + workload: + name: uperf + args: + hostnetwork: false + serviceip: false + pin: false + multus: + enabled: false + samples: 1 + pair: 1 + test_types: + - stream + protos: + - tcp + sizes: + - 1024 + nthrs: + - 1 + runtime: 120 +``` diff --git a/docs/testpmd.md b/docs/testpmd.md index bc2e9874e..eb6aae8b7 100644 --- a/docs/testpmd.md +++ b/docs/testpmd.md @@ -13,6 +13,8 @@ This workload assumes a healthy cluster is runing with SR-IOV and Performance Ad These are used to create VFs and SRIOV networks, interface might vary. Creating `SriovNetworkNodePolicy` will reboot your worker nodes one by one. +Below node policy is for Intel XXV710 NICs, for a Mellanox NIC to work in Data Plane Development Kit (DPDK) mode, use the `netdevice` driver type and include additional spec `isRdma: true`. + ```yaml apiVersion: sriovnetwork.openshift.io/v1 kind: SriovNetworkNodePolicy @@ -51,7 +53,7 @@ spec: spoofChk: "on" trust: "on" resourceName: intelnics - networkNamespace: my-ripsaw + networkNamespace: benchmark-operator ``` @@ -101,7 +103,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: testpmd-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: myk8scluster workload: @@ -180,8 +182,8 @@ This CR will create TestPMD pod and TRex pod using the SRIOV network provided an ```sh # oc get benchmark -NAME TYPE STATE METADATA STATE CERBERUS UUID AGE -testpmd-benchmark testpmd Complete not collected not connected 35daf5ac-2edf-5e34-a6cc-17fcda055937 4m36s +NAME TYPE STATE METADATA STATE UUID AGE +testpmd-benchmark testpmd Complete not collected 35daf5ac-2edf-5e34-a6cc-17fcda055937 4m36s # oc get pods NAME READY STATUS RESTARTS AGE diff --git a/docs/uperf-scale.md b/docs/uperf-scale.md new file mode 100644 index 000000000..c596d153c --- /dev/null +++ b/docs/uperf-scale.md @@ -0,0 +1,303 @@ +# Uperf-Scale + +[Uperf](http://uperf.org/) is a network performance tool + +## Running UPerf-Scale + +Given that you followed instructions to deploy operator, +you can modify [cr.yaml](../config/samples/uperf-scale/cr.yaml) + +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-scale-benchmark + namespace: benchmark-operator +spec: + elasticsearch: + url: "http://es-instance.com:9200" + workload: + name: uperf-scale + args: + client_resources: + requests: + cpu: 500m + memory: 500Mi + limits: + cpu: 500m + memory: 500Mi + server_resources: + requests: + cpu: 500m + memory: 500Mi + limits: + cpu: 500m + memory: 500Mi + serviceip: false + runtime_class: class_name + hostnetwork: false + networkpolicy: false + kind: pod + multus: + enabled: false + samples: 1 + test_types: + - stream + protos: + - tcp + sizes: + - 16384 + nthrs: + - 1 + runtime: 30 + colocate: false + density_range: [low, high] + node_range: [low, high] + step_size: addN, log2 +``` + +`client_resources` and `server_resources` will create uperf client's and server's containers with the given k8s compute resources respectively [k8s resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) + +`serviceip` will place the uperf server behind a K8s [Service](https://kubernetes.io/docs/concepts/services-networking/service/) + +`runtime_class` If this is set, the benchmark-operator will apply the runtime_class to the podSpec runtimeClassName. + +*Note:* `runtime_class` has only been tested with Kata containers. Only include `runtime_class` if using Kata containers. + +`annotations` If this is set, the benchmark-operator will set the specified annotations on the pods' metadata. + +`server_annotations` If this is set, the benchmark-operator will set the specified annotations on the server pods' metadata. + +`client_annotations` If this is set, the benchmark-operator will set the specified annotations on the client pods' metadata. + +`hostnetwork` will test the performance of the node the pod will run on. + +`networkpolicy` will create a simple networkpolicy for ingress + +`density_range` to determine the number of pairs and `node_range` to determine the number of nodes. + +`multus[1]` Configure our pods to use multus. + +`samples` how many times to run the tests. For example + +[1] https://github.com/intel/multus-cni/tree/master/examples + +```yaml + samples: 3 + density_range: [1,1] + test_types: + - stream + protos: + - tcp + sizes: + - 1024 + - 16384 + nthrs: + - 1 + runtime: 30 +``` + +Will run `stream` w/ `tcp` and message size `1024` three times and +`stream` w/ `tcp` and message size `16384` three times. This will help us +gain confidence in our results. + +### Asymmetric Request-Response + +For the request-response (rr) `test_type`, it is possible to provide the `sizes` values as a +list of two values where the first value is the write size and the second value is the read +size. + +For example: +```yaml + samples: 3 + density_range: [1,1] + test_types: + - rr + protos: + - tcp + sizes: + - 1024 + - [8192, 4096] + nthrs: + - 1 + runtime: 30 +``` +Will run the `rr` test with `tcp`, first with a symmectic size of `1024` and then with an +asymmetric size of `8192` write and `4096` read. + +### Multus + +If the user desires to test with Multus, use the below Multus `NetworkAtachmentDefinition` +as an example: + +``` +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: macvlan-range-0 +spec: + config: '{ + "cniVersion": "0.3.1", + "type": "macvlan", + "master": "eno1", + "mode": "bridge", + "ipam": { + "type": "host-local", + "ranges": [ + [ { + "subnet": "11.10.0.0/16", + "rangeStart": "11.10.1.20", + "rangeEnd": "11.10.3.50" + } ] ] + } + }' +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: macvlan-range-1 +spec: + config: '{ + "cniVersion": "0.3.1", + "type": "macvlan", + "master": "eno1", + "mode": "bridge", + "ipam": { + "type": "host-local", + "ranges": [ + [ { + "subnet": "11.10.0.0/16", + "rangeStart": "11.10.1.60", + "rangeEnd": "11.10.3.90" + } ] ] + } + }' +``` + +This will use the same IP subnet across nodes, but not overlap +IP addresses. + +To enable Multus in Ripsaw, here is the relevant config. + +``` + ... + multus: + enabled: true + client: "macvlan-range-0" + server: "macvlan-range-1" + ... + +``` +### Scale mode params +Scale in this context refers to the ability to enumerate UPERF +client-server pairs during test in a control fashion using the following knobs. + +`colocate: true` will place each client and server pod pair on the same node. + +`density_range` to specify the range of client-server pairs that the test will iterate. + +`node_range` to specify the range of nodes that the test will iterate. + +`step_size` to specify the incrementing method. + +Here is one scale example: + +``` + ... + colocate: false + density_range: [1,10] + node_range: [1,128] + step_size: log2 + ... +``` +In the above sample, the `scale` mode will be activated. The `colocate` option is to keep both server and client pods on same node. In the first phase, the +pod instantion phase, the system gathers node inventory and may reduce the `node_range.high` value +to match the number of worker node available in the cluster. + +According to `node_range: [1,128]`, and `density_range:[1,10]`, the system will instantiate 10 pairs on +each of 128 nodes. Each pair has a node_idx and a pod_idx that are used later to control +which one and when they should run the UPERF workload, After all pairs are up and ready, +next comes the test execution phase. + +The scale mode iterates the test as a double nested loop as follows: +``` + for node with node_idx less-or-equal node_range(low, high. step_size): + for pod with pod_idx less-or-equal density_range(low, high, step_size): + run uperf +``` +Hence, with the above params, the first iteration runs the pair with node_idx/pod_idx of {1,1}. After the first +run has completed, the second interation runs 2 pairs of {1,1} and {1,2} and so on. + +The valid `step_size` methods are: addN and log2. `N` can be any integer and `log2` will double the value at each iteration i.e. 1,2,4,8,16 ... +By choosing the appropriate values for `density_range` and `node_range`, the user can generate most if not all +combinations of UPERF data points to exercise datapath performance from many angles. + +Once done creating/editing the resource file, you can run it by: + +```bash +# kubectl apply -f config/samples/uperf/cr.yaml # if edited the original one +# kubectl apply -f # if created a new cr file +``` +### Advanced Service types + +Benchmark operator now also supports different service types, it can create `NodePort` and `LoadBalancer` (only metallb) +type services along with the current default `ClusterIP` type. + +No pre-requisites needed for `NodePort` service, as long as the ports used by uperf(30000 to 30012) are allowed at the node level, +which is the cluster default. + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) +uperf-service-np NodePort 172.30.177.81 30000:31955/TCP,30001:31935/TCP,30002:31942/TCP,30001:31935/UDP,30002:31942/UDP +``` + +For `metallb` type, there are certain pre-requisites, +1. Installation of [MetalLB](https://metallb.universe.tf/) operator and CRD +2. Configuration of [BGP](https://github.com/metallb/metallb-operator#create-a-bgp-peer-object) +3. Configuration of [AddressPool](https://github.com/metallb/metallb-operator#create-an-address-pool-object) for lb service +4. Configuration of extenal router for BGP + +`metallb` type creates 2 services per benchmark CR (for each protocol, `tcp` and `udp`) and they will share the external IP like below + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) +uperf-service-lb LoadBalancer 172.30.177.99 192.168.216.102 30000:30976/TCP,30001:30652/TCP,30002:30099/TCP +uperf-service-lb2 LoadBalancer 172.30.126.71 192.168.216.102 30001:31312/UDP,30002:30776/UDP +``` + +#### CR file inputs + +##### For NodePort +```yaml + ... + name: uperf + serviceip: true + servicetype: "nodeport" + ... +``` + +##### For MetalLB +`metallb` +```yaml + ... + name: uperf + serviceip: true + servicetype: "metallb" + metallb: + addresspool: "addresspool-l3" + service_etp: "Cluster" # Either `Cluster` or `Local` + ... +``` +You can either access results by indexing them directly or by accessing the console. +The results are stored in /tmp/ directory + + +### Dashboard example + +Using the Elasticsearch storage describe above, we can build dashboards like the below. + +![UPerf Dashboard](https://i.imgur.com/gSVZ9MX.png) + +To reuse the dashboard above, use the json [here](https://github.com/cloud-bulldozer/arsenal/tree/master/uperf/grafana) + +Additionally, by default we will utilize the `uperf-results` index for Elasticsearch. \ No newline at end of file diff --git a/docs/uperf.md b/docs/uperf.md index 3d05dbc55..3a969c501 100644 --- a/docs/uperf.md +++ b/docs/uperf.md @@ -5,14 +5,14 @@ ## Running UPerf Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_uperf_cr.yaml) +you can modify [cr.yaml](../config/samples/uperf/cr.yaml) ```yaml apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: uperf-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://es-instance.com:9200" @@ -41,10 +41,10 @@ spec: kind: pod pin_server: "node-0" pin_client: "node-1" + pair: 1 multus: enabled: false samples: 1 - pair: 1 test_types: - stream protos: @@ -56,6 +56,8 @@ spec: runtime: 30 ``` + By default, the uperf server and client pods will be preferably scheduled on different nodes, thanks to a `podAntiAffinity` rule. In scenarios with more than 1 pair, all clients and servers will be scheduled to the same node. + `client_resources` and `server_resources` will create uperf client's and server's containers with the given k8s compute resources respectively [k8s resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) `serviceip` will place the uperf server behind a K8s [Service](https://kubernetes.io/docs/concepts/services-networking/service/) @@ -64,6 +66,12 @@ spec: *Note:* `runtime_class` has only been tested with Kata containers. Only include `runtime_class` if using Kata containers. +`annotations` If this is set, the benchmark-operator will set the specified annotations on the pods' metadata. + +`server_annotations` If this is set, the benchmark-operator will set the specified annotations on the server pods' metadata. + +`client_annotations` If this is set, the benchmark-operator will set the specified annotations on the client pods' metadata. + `hostnetwork` will test the performance of the node the pod will run on. `networkpolicy` will create a simple networkpolicy for ingress @@ -74,6 +82,10 @@ spec: `pin_client` what node to pin the client pod to. +`pair` how many instances of uperf client-server pairs. `pair` is applicable for `pin: true` only. + +`protos`: choose between `tcp`, `udp` or/and `sctp`. In case `sctp` is choosen, SCTP module has to be previously loaded in the environment. + `multus[1]` Configure our pods to use multus. `samples` how many times to run the tests. For example @@ -82,7 +94,7 @@ spec: ```yaml samples: 3 - pair: 1 + density_range: [1,1] test_types: - stream protos: @@ -108,7 +120,7 @@ size. For example: ```yaml samples: 3 - pair: 1 + density_range: [1,1] test_types: - rr protos: @@ -189,12 +201,55 @@ To enable Multus in Ripsaw, here is the relevant config. ... ``` +### Advanced Service types -Once done creating/editing the resource file, you can run it by: +Benchmark operator now also supports different service types, it can create `NodePort` and `LoadBalancer` (only metallb) +type services along with the current default `ClusterIP` type. -```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_uperf_cr.yaml # if edited the original one -# kubectl apply -f # if created a new cr file +No pre-requisites needed for `NodePort` service, as long as the ports used by uperf(30000 to 30012) are allowed at the node level, +which is the cluster default. + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) +uperf-service-np NodePort 172.30.177.81 30000:31955/TCP,30001:31935/TCP,30002:31942/TCP,30001:31935/UDP,30002:31942/UDP +``` + +For `metallb` type, there are certain pre-requisites, +1. Installation of [MetalLB](https://metallb.universe.tf/) operator and CRD +2. Configuration of [BGP](https://github.com/metallb/metallb-operator#create-a-bgp-peer-object) +3. Configuration of [AddressPool](https://github.com/metallb/metallb-operator#create-an-address-pool-object) for lb service +4. Configuration of extenal router for BGP + +`metallb` type creates 2 services per benchmark CR (for each protocol, `tcp` and `udp`) and they will share the external IP like below + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) +uperf-service-lb LoadBalancer 172.30.177.99 192.168.216.102 30000:30976/TCP,30001:30652/TCP,30002:30099/TCP +uperf-service-lb2 LoadBalancer 172.30.126.71 192.168.216.102 30001:31312/UDP,30002:30776/UDP +``` + +#### CR file inputs + +##### For NodePort +```yaml + ... + name: uperf + serviceip: true + servicetype: "nodeport" + ... +``` + +##### For MetalLB +`metallb` +```yaml + ... + name: uperf + serviceip: true + servicetype: "metallb" + metallb: + addresspool: "addresspool-l3" + service_etp: "Cluster" # Either `Cluster` or `Local` + ... ``` ## Running Uperf in VMs through kubevirt/cnv [Preview] diff --git a/docs/vegeta.md b/docs/vegeta.md index e14205e29..743ed9f03 100644 --- a/docs/vegeta.md +++ b/docs/vegeta.md @@ -5,18 +5,24 @@ ## Running Vegeta Given that you followed instructions to deploy operator, -you can modify Vegeta's [cr.yaml](../resources/crds/ripsaw_v1alpha1_vegeta_cr.yaml) to make it fit with your requirements. +you can modify Vegeta's [cr.yaml](../config/samples/vegeta/cr.yaml) to make it fit with your requirements. The option **runtime_class** can be set to specify an optional runtime_class to the podSpec runtimeClassName. This is primarily intended for Kata containers. +The option **annotations** can be set to apply the specified +annotations to the pod metadata. + +The **node_selector** option can be used to limit the nodes where +the vegeta pods are deployed. + ```yaml apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: vegeta-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: "http://esinstance.com:9200" @@ -24,6 +30,7 @@ spec: workload: name: vegeta args: + # node_selector: "vegeta=true" clients: 2 image: quay.io/cloud-bulldozer/vegeta:latest hostnetwork: false diff --git a/docs/ycsb.md b/docs/ycsb.md index 346b4396c..268109a19 100644 --- a/docs/ycsb.md +++ b/docs/ycsb.md @@ -5,7 +5,7 @@ ## Running YCSB Given that you followed instructions to deploy operator, -you can modify [cr.yaml](../resources/crds/ripsaw_v1alpha1_ycsb_cr.yaml) to your needs. +you can modify [cr.yaml](../config/samples/ycsb/cr.yaml) to your needs. YCSB is a workload that requires a database/key-value store to run workloads against and benchmark. @@ -15,7 +15,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: ycsb-mongo-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: clustername: myk8scluster workload: @@ -68,9 +68,11 @@ so that the ycsb pod can access the API of database. `runtime_class`: If this is set, the benchmark-operator will apply the runtime_class to the podSpec runtimeClassName. +`annotations` If this is set, the benchmark-operator will set the specified annotations on the pods' metadata. + Once done creating/editing the resource file, you can run it by: ```bash -# kubectl apply -f resources/crds/ripsaw_v1alpha1_ycsb_cr.yaml # if edited the original one +# kubectl apply -f config/samples/ycsb/cr.yaml # if edited the original one # kubectl apply -f # if created a new cr file ``` diff --git a/e2e/001-scale_openshift.bats b/e2e/001-scale_openshift.bats new file mode 100755 index 000000000..1256230f8 --- /dev/null +++ b/e2e/001-scale_openshift.bats @@ -0,0 +1,42 @@ +#!/usr/bin/env bats + +# vi: ft=bash + + +load helpers.bash + +ES_INDEX=openshift-cluster-timings + + +@test "scale_openshift-up" { + CR=scale_openshift/scale_up.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + # Make node no schedulable as soon as benchmark finishes + kubectl cordon $(kubectl get node --sort-by='{.metadata.creationTimestamp}' -o name | tail -1) + # Annotate machine to ensure it's the one deleted in the scale_openshift-down test + # Reference: https://github.com/openshift/machine-api-operator/blob/master/FAQ.md#what-decides-which-machines-to-destroy-when-a-machineset-is-scaled-down + kubectl -n openshift-machine-api annotate $(kubectl get machine -n openshift-machine-api --sort-by='{.metadata.creationTimestamp}' -o name | tail -1) machine.openshift.io/cluster-api-delete-machine=true --overwrite + check_es +} + +@test "scale_openshift-down" { + CR=scale_openshift/scale_down.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 || die "Timeout waiting for benchmark/${CR_NAME} to complete" + check_es +} + +setup_file() { + # Prevent running scale down/up simultaneously + export BATS_NO_PARALLELIZE_WITHIN_FILE=true + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/002-sysbench.bats b/e2e/002-sysbench.bats new file mode 100755 index 000000000..3c2fef6f9 --- /dev/null +++ b/e2e/002-sysbench.bats @@ -0,0 +1,24 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=ripsaw-sysbench-results + + +@test "sysbench-standard" { + CR=sysbench/sysbench.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/003-byowl.bats b/e2e/003-byowl.bats new file mode 100755 index 000000000..748283cf0 --- /dev/null +++ b/e2e/003-byowl.bats @@ -0,0 +1,32 @@ +#!/usr/bin/env bats + +# vi: ft=bash + + +load helpers.bash + + + +@test "byowl-targeted" { + CR=byowl/byowl-targeted.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 +} + +@test "byowl-not-targeted" { + CR=byowl/byowl-not-targeted.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/004-backpack.bats b/e2e/004-backpack.bats new file mode 100755 index 000000000..e67c8f11c --- /dev/null +++ b/e2e/004-backpack.bats @@ -0,0 +1,34 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +indexes=(cpu_vulnerabilities-metadata cpuinfo-metadata dmidecode-metadata k8s_nodes-metadata lspci-metadata meminfo-metadata sysctl-metadata ocp_network_operator-metadata ocp_install_config-metadata ocp_kube_apiserver-metadata ocp_dns-metadata ocp_kube_controllermanager-metadata) + + +@test "backpack-init" { + CR=backpack/backpack-init.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +@test "backpack-daemonset" { + CR=backpack/backpack-daemonset.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/005-stressng.bats b/e2e/005-stressng.bats new file mode 100755 index 000000000..a4ca9893b --- /dev/null +++ b/e2e/005-stressng.bats @@ -0,0 +1,25 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=ripsaw-stressng-results + + +@test "stressng-standard" { + CR=stressng/stressng.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/006-vegeta.bats b/e2e/006-vegeta.bats new file mode 100755 index 000000000..92188dc72 --- /dev/null +++ b/e2e/006-vegeta.bats @@ -0,0 +1,34 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=ripsaw-vegeta-results + + +@test "vegeta-standard" { + CR=vegeta/vegeta.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +@test "vegeta-hostpath" { + CR=vegeta/vegeta_hostnetwork.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/007-smallfile.bats b/e2e/007-smallfile.bats new file mode 100755 index 000000000..652e04284 --- /dev/null +++ b/e2e/007-smallfile.bats @@ -0,0 +1,34 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +indexes=(ripsaw-smallfile-results ripsaw-smallfile-rsptimes) + + +@test "smallfile-standard" { + CR=smallfile/smallfile.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +@test "smallfile-hostpath" { + CR=smallfile/smallfile_hostpath.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/008-kube-burner.bats b/e2e/008-kube-burner.bats new file mode 100755 index 000000000..ec8cb6d5e --- /dev/null +++ b/e2e/008-kube-burner.bats @@ -0,0 +1,96 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=ripsaw-kube-burner + + +@test "kube-burner-cluster-density" { + CR=kube-burner/cluster-density.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +@test "kube-burner-node-density" { + CR=kube-burner/node-density.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 +} + +@test "kube-burner-node-density-heavy" { + CR=kube-burner/node-density-heavy.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 +} + +@test "kube-burner-node-density-cni" { + CR=kube-burner/node-density-cni.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 +} + +@test "kube-burner-node-density-cni-networkpolicy" { + CR=kube-burner/node-density-cni-networkpolicy.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 +} + +@test "kube-burner-max-services" { + CR=kube-burner/max-services.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 +} + +@test "kube-burner-max-namespaces" { + CR=kube-burner/max-namespaces.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 +} + +@test "kube-burner-concurrent-builds" { + CR=kube-burner/concurrent-builds.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 +} + +@test "kube-burner-configmap" { + CR=kube-burner/configmap.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +setup_file() { + basic_setup + kubectl apply -f kube-burner/configmap-cfg.yaml +} + +teardown_file() { + kubectl delete -f kube-burner/configmap-cfg.yaml +} + +teardown() { + basic_teardown + kubectl delete ns -l kube-burner-uuid=${uuid} --ignore-not-found +} diff --git a/e2e/009-iperf3.bats b/e2e/009-iperf3.bats new file mode 100755 index 000000000..ed0786ce5 --- /dev/null +++ b/e2e/009-iperf3.bats @@ -0,0 +1,23 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + + + +@test "iperf3-standard" { + CR=iperf3/iperf3.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/010-fio.bats b/e2e/010-fio.bats new file mode 100755 index 000000000..c676094e0 --- /dev/null +++ b/e2e/010-fio.bats @@ -0,0 +1,64 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +indexes=(ripsaw-fio-results ripsaw-fio-log ripsaw-fio-analyzed-result) + + +@test "fio-standard" { + CR=fio/fio.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +@test "fio-bsrange" { + CR=fio/fio_bsrange.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +@test "fio-hostpath" { + CR=fio/fio_hostpath.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +@test "fio-ocs-cachedrop" { + CR=fio/fio_ocs_cache_drop.yaml + CR_NAME=$(get_benchmark_name ${CR}) + openshift_storage_present=$(kubectl get ns | awk '/openshift-storage/' | wc -l) + if [ $openshift_storage_present -gt 0 ]; then + kubectl label nodes -l node-role.kubernetes.io/worker= kernel-cache-dropper=yes --overwrite + kubectl patch OCSInitialization ocsinit -n openshift-storage --type json --patch \ + '[{ "op": "replace", "path": "/spec/enableCephTools", "value": true }]' + drop_cache_pods=$(kubectl -n openshift-storage get pod | awk '/drop/' | awk '/unning/' | wc -l) + if [ $drop_cache_pods -eq 0 ] ; then + kubectl create -f ../resources/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml + kubectl wait --for=condition=Initialized pods/rook-ceph-osd-cache-drop -n openshift-storage --timeout=100s + fi + sleep 5 + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es + fi +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/011-ycsb.bats b/e2e/011-ycsb.bats new file mode 100755 index 000000000..2baf8d271 --- /dev/null +++ b/e2e/011-ycsb.bats @@ -0,0 +1,33 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +indexes=(ripsaw-ycsb-summary ripsaw-ycsb-results) + + +@test "ycsb-mongo" { + CR=ycsb/ycsb-mongo.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +setup() { + kubectl_exec apply -f ycsb/mongo.yaml +} + +setup_file() { + basic_setup +} + +teardown_file() { + kubectl_exec delete -f ycsb/mongo.yaml --ignore-not-found +} + +teardown() { + basic_teardown +} diff --git a/e2e/012-flent.bats b/e2e/012-flent.bats new file mode 100755 index 000000000..f25da6daf --- /dev/null +++ b/e2e/012-flent.bats @@ -0,0 +1,35 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=ripsaw-flent-results + + +@test "flent-standard" { + CR=flent/flent.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + + +@test "flent-resources" { + CR=flent/flent_resources.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/013-uperf.bats b/e2e/013-uperf.bats new file mode 100755 index 000000000..e733b23ff --- /dev/null +++ b/e2e/013-uperf.bats @@ -0,0 +1,64 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=ripsaw-uperf-results + + +@test "uperf-hostnetwork-nodeport" { + # Merging nodeport and hostnetwork test to avoid port binding conflicts + for CR in uperf/uperf_hostnetwork.yaml uperf/uperf_serviceip_nodeport.yaml; do + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es + kubectl_exec delete benchmark ${CR_NAME} --ignore-not-found + done +} + +@test "uperf-standard" { + CR=uperf/uperf.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +@test "uperf-resources" { + CR=uperf/uperf_resources.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +@test "uperf-network policy" { + CR=uperf/uperf_networkpolicy.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +@test "uperf-serviceip" { + CR=uperf/uperf_serviceip.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/014-pgbench.bats b/e2e/014-pgbench.bats new file mode 100755 index 000000000..d57c9b035 --- /dev/null +++ b/e2e/014-pgbench.bats @@ -0,0 +1,35 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +indexes=(ripsaw-pgbench-summary ripsaw-pgbench-raw) + + +@test "pgbench-standard" { + CR=pgbench/pgbench.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +setup() { + kubectl_exec apply -f pgbench/postgres.yaml + kubectl_exec rollout status deploy/postgres --timeout=60s + export POSTGRES_IP=$(kubectl_exec get pod -l app=postgres -o jsonpath="{.items[*].status.podIP}") +} + +setup_file() { + basic_setup +} + +teardown_file() { + kubectl_exec delete -f pgbench/postgres.yaml --ignore-not-found +} + +teardown() { + basic_teardown +} diff --git a/e2e/015-image_pull.bats b/e2e/015-image_pull.bats new file mode 100755 index 000000000..8be95e022 --- /dev/null +++ b/e2e/015-image_pull.bats @@ -0,0 +1,25 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=image-pull-results + + +@test "image_pull-standard" { + CR=image_pull/image_pull.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/016-hammerdb.bats b/e2e/016-hammerdb.bats new file mode 100755 index 000000000..44cfb7ace --- /dev/null +++ b/e2e/016-hammerdb.bats @@ -0,0 +1,34 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=ripsaw-hammerdb-results + + +@test "hammerdb-standard" { + CR=hammerdb/hammerdb.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +setup() { + kubectl apply -f hammerdb/sql-server.yaml + kubectl rollout status -n sql-server deploy/mssql-deployment --timeout=60s +} + +setup_file() { + basic_setup +} + +teardown_file() { + kubectl delete ns sql-server --wait=false --ignore-not-found +} + +teardown() { + basic_teardown +} diff --git a/e2e/017-fs-drift.bats b/e2e/017-fs-drift.bats new file mode 100755 index 000000000..21a3302de --- /dev/null +++ b/e2e/017-fs-drift.bats @@ -0,0 +1,35 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +indexes=(ripsaw-fs-drift-results ripsaw-fs-drift-rsptimes ripsaw-fs-drift-rates-over-time) + + +@test "fs-drift-standard" { + CR=fs-drift/fs-drift.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + + +@test "fs-drift-hostpath" { + CR=fs-drift/fs-drift_hostpath.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/018-log_generator.bats b/e2e/018-log_generator.bats new file mode 100755 index 000000000..425564d37 --- /dev/null +++ b/e2e/018-log_generator.bats @@ -0,0 +1,26 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + +ES_INDEX=log-generator-results + + +@test "log_generator-standard" { + CR=log_generator/log_generator.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 1200 + check_es +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} + diff --git a/e2e/019-api_load.bats b/e2e/019-api_load.bats new file mode 100755 index 000000000..8595c1d5d --- /dev/null +++ b/e2e/019-api_load.bats @@ -0,0 +1,22 @@ +#!/usr/bin/env bats + +# vi: ft=bash + +load helpers.bash + + +@test "api_load-standard" { + CR=api_load/api_load.yaml + CR_NAME=$(get_benchmark_name ${CR}) + envsubst < ${CR} | kubectl apply -f - + get_uuid "${CR_NAME}" + check_benchmark 120 +} + +setup_file() { + basic_setup +} + +teardown() { + basic_teardown +} diff --git a/e2e/api_load/api_load.yaml b/e2e/api_load/api_load.yaml new file mode 100644 index 000000000..eba253938 --- /dev/null +++ b/e2e/api_load/api_load.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: api-load + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + index_name: "api-load" + snappy: + url: SNAPPY_SERVER + user: realuser + password: realpassword + workload: + name: api_load + args: + override: version + gateway_url: http://localhost:8080 + ocm_token: notARealToken + duration: 1 + rate: 1/s + output_path: /tmp/results + aws_access_key: empty + aws_access_secret: empty + aws_account_id: empty + aws_region: empty + cooldown: 10 + sleep: 5 + test_list: + self-access-token: diff --git a/e2e/backpack/backpack-daemonset.yaml b/e2e/backpack/backpack-daemonset.yaml new file mode 100644 index 000000000..48a529d9d --- /dev/null +++ b/e2e/backpack/backpack-daemonset.yaml @@ -0,0 +1,14 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: backpack-daemonset + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + targeted: false + privileged: true + workload: + name: backpack diff --git a/e2e/backpack/backpack-init.yaml b/e2e/backpack/backpack-init.yaml new file mode 100644 index 000000000..4711e6138 --- /dev/null +++ b/e2e/backpack/backpack-init.yaml @@ -0,0 +1,17 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: backpack-init + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + privileged: true + workload: + name: byowl + args: + image: "quay.io/jtaleric/uperf:testing" + clients: 1 + commands: "echo Test" diff --git a/e2e/byowl/byowl-not-targeted.yaml b/e2e/byowl/byowl-not-targeted.yaml new file mode 100644 index 000000000..f2f32fea9 --- /dev/null +++ b/e2e/byowl/byowl-not-targeted.yaml @@ -0,0 +1,23 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: byowl-standard + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + targeted: false + workload: + name: byowl + args: + image: "quay.io/cloud-bulldozer/uperf:latest" + clients: 1 + commands: "echo Test" diff --git a/e2e/byowl/byowl-targeted.yaml b/e2e/byowl/byowl-targeted.yaml new file mode 100644 index 000000000..655eb0116 --- /dev/null +++ b/e2e/byowl/byowl-targeted.yaml @@ -0,0 +1,23 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: byowl-targeted-metadta + namespace: benchmark-operator +spec: + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + targeted: true + workload: + name: byowl + args: + image: "quay.io/cloud-bulldozer/uperf:latest" + clients: 1 + commands: "echo Test" diff --git a/e2e/fio/fio.yaml b/e2e/fio/fio.yaml new file mode 100644 index 000000000..6c9321ae8 --- /dev/null +++ b/e2e/fio/fio.yaml @@ -0,0 +1,85 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fio-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-fio + metadata: + collection: true + cleanup: false + workload: + name: "fio_distributed" + args: + samples: 2 + servers: 2 + jobs: + - write + - read + - randwrite + - randread + - randrw + bs: + - 4KiB + numjobs: + - 2 + iodepth: 1 + read_runtime: 5 + write_runtime: 5 + read_ramp_time: 1 + write_ramp_time: 1 + filesize: 10MiB + log_sample_rate: 2000 + storagesize: 16Mi + debug: true +####################################### +# EXPERT AREA - MODIFY WITH CAUTION # +####################################### + job_params: + - jobname_match: write + params: + - time_based=1 + - fsync_on_close=1 + - create_on_open=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} + - jobname_match: read + params: + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: rw + params: + - rwmixread=50 + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: readwrite + params: + - rwmixread=50 + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: randread + params: + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: randwrite + params: + - time_based=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} + - jobname_match: randrw + params: + - time_based=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} diff --git a/e2e/fio/fio_bsrange.yaml b/e2e/fio/fio_bsrange.yaml new file mode 100644 index 000000000..77c5fbf4a --- /dev/null +++ b/e2e/fio/fio_bsrange.yaml @@ -0,0 +1,50 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fio-bsrange + namespace: benchmark-operator +spec: + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-fio + metadata: + collection: false + cleanup: false + workload: + name: "fio_distributed" + args: + samples: 1 + servers: 2 + jobs: + - write + bsrange: + - 4KiB-16KiB + numjobs: + - 2 + iodepth: 1 + read_runtime: 5 + write_runtime: 5 + read_ramp_time: 1 + write_ramp_time: 1 + filesize: 10MiB + log_sample_rate: 2000 + log_hist_msec: 3000 + storagesize: 16Mi + debug: true +####################################### +# EXPERT AREA - MODIFY WITH CAUTION # +####################################### + job_params: + - jobname_match: write + params: + - time_based=1 + - fsync_on_close=1 + - create_on_open=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} diff --git a/e2e/fio/fio_hostpath.yaml b/e2e/fio/fio_hostpath.yaml new file mode 100644 index 000000000..71ec9b2da --- /dev/null +++ b/e2e/fio/fio_hostpath.yaml @@ -0,0 +1,49 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fio-hostpath + namespace: benchmark-operator +spec: + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-fio + metadata: + collection: false + cleanup: false + hostpath: /mnt/vda1/workload_args/ + workload: + name: "fio_distributed" + args: + samples: 1 + servers: 2 + prefill: true + jobs: + - write + bs: + - 4KiB + numjobs: + - 2 + iodepth: 2 + read_runtime: 5 + write_runtime: 5 + read_ramp_time: 1 + write_ramp_time: 1 + filesize: 10MiB + debug: false +####################################### +# EXPERT AREA - MODIFY WITH CAUTION # +####################################### + job_params: + - jobname_match: write + params: + - time_based=1 + - fsync_on_close=1 + - create_on_open=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} diff --git a/e2e/fio/fio_ocs_cache_drop.yaml b/e2e/fio/fio_ocs_cache_drop.yaml new file mode 100644 index 000000000..6b07f116b --- /dev/null +++ b/e2e/fio/fio_ocs_cache_drop.yaml @@ -0,0 +1,51 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fio-ocs-cachedrop + namespace: benchmark-operator +spec: + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-fio + metadata: + collection: false + cleanup: false + workload: + name: "fio_distributed" + args: + drop_cache_rook_ceph: true + drop_cache_kernel: true + samples: 1 + servers: 2 + jobs: + - write + bs: + - 4KiB + numjobs: + - 2 + iodepth: 1 + read_runtime: 5 + write_runtime: 5 + read_ramp_time: 1 + write_ramp_time: 1 + filesize: 10MiB + log_sample_rate: 2000 + storagesize: 16Mi + debug: false +####################################### +# EXPERT AREA - MODIFY WITH CAUTION # +####################################### + job_params: + - jobname_match: write + params: + - time_based=1 + - fsync_on_close=1 + - create_on_open=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} diff --git a/e2e/flent/flent.yaml b/e2e/flent/flent.yaml new file mode 100644 index 000000000..a733ab7f1 --- /dev/null +++ b/e2e/flent/flent.yaml @@ -0,0 +1,30 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: flent + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-flent + metadata: + collection: true + clustername: myk8scluster + workload: + # cleanup: true + name: flent + args: + hostnetwork: false + multus: + enabled: false + pair: 1 + test_types: + - tcp_download + runtime: 2 + debug: true diff --git a/e2e/flent/flent_resources.yaml b/e2e/flent/flent_resources.yaml new file mode 100644 index 000000000..725b50e38 --- /dev/null +++ b/e2e/flent/flent_resources.yaml @@ -0,0 +1,38 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: flent-resources + namespace: benchmark-operator +spec: + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-flent + metadata: + collection: false + workload: + # cleanup: true + name: flent + args: + client_resources: + requests: + cpu: 100m + memory: 100Mi + server_resources: + requests: + cpu: 100m + memory: 100Mi + pin: false + hostnetwork: false + multus: + enabled: false + pair: 1 + test_types: + - tcp_download + runtime: 2 + debug: true diff --git a/e2e/fs-drift/fs-drift.yaml b/e2e/fs-drift/fs-drift.yaml new file mode 100644 index 000000000..fd168c2f5 --- /dev/null +++ b/e2e/fs-drift/fs-drift.yaml @@ -0,0 +1,31 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fs-drift-benchmark + namespace: benchmark-operator +spec: + test_user: homer_simpson + # to separate this test run from everyone else's + clustername: test_ci + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + # where elastic search is running + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-fs-drift + metadata: + collection: true + workload: + name: fs-drift + args: + worker_pods: 1 + threads: 5 + max_file_size_kb: 4 + max_files: 1000 + duration: 240 + debug: true + response_times: true diff --git a/e2e/fs-drift/fs-drift_hostpath.yaml b/e2e/fs-drift/fs-drift_hostpath.yaml new file mode 100644 index 000000000..ea5bb6d43 --- /dev/null +++ b/e2e/fs-drift/fs-drift_hostpath.yaml @@ -0,0 +1,32 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fs-drift-hostpath-benchmark + namespace: benchmark-operator +spec: + test_user: homer_simpson + # to separate this test run from everyone else's + clustername: test_ci + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + # where elastic search is running + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-fs-drift + metadata: + collection: false + hostpath: /mnt/vda1/fs_drift + workload: + name: fs-drift + args: + worker_pods: 1 + threads: 5 + max_file_size_kb: 4 + max_files: 1000 + duration: 240 + debug: true + response_times: true diff --git a/e2e/fs-drift/fs_drift.yaml b/e2e/fs-drift/fs_drift.yaml new file mode 100644 index 000000000..fd168c2f5 --- /dev/null +++ b/e2e/fs-drift/fs_drift.yaml @@ -0,0 +1,31 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fs-drift-benchmark + namespace: benchmark-operator +spec: + test_user: homer_simpson + # to separate this test run from everyone else's + clustername: test_ci + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + # where elastic search is running + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-fs-drift + metadata: + collection: true + workload: + name: fs-drift + args: + worker_pods: 1 + threads: 5 + max_file_size_kb: 4 + max_files: 1000 + duration: 240 + debug: true + response_times: true diff --git a/e2e/fs-drift/fs_drift_hostpath.yaml b/e2e/fs-drift/fs_drift_hostpath.yaml new file mode 100644 index 000000000..ea5bb6d43 --- /dev/null +++ b/e2e/fs-drift/fs_drift_hostpath.yaml @@ -0,0 +1,32 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fs-drift-hostpath-benchmark + namespace: benchmark-operator +spec: + test_user: homer_simpson + # to separate this test run from everyone else's + clustername: test_ci + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + # where elastic search is running + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-fs-drift + metadata: + collection: false + hostpath: /mnt/vda1/fs_drift + workload: + name: fs-drift + args: + worker_pods: 1 + threads: 5 + max_file_size_kb: 4 + max_files: 1000 + duration: 240 + debug: true + response_times: true diff --git a/e2e/hammerdb/hammerdb.yaml b/e2e/hammerdb/hammerdb.yaml new file mode 100644 index 000000000..f72f833db --- /dev/null +++ b/e2e/hammerdb/hammerdb.yaml @@ -0,0 +1,70 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: hammerdb + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-hammerdb + metadata: + collection: true + workload: + name: "hammerdb" + args: + # image: "quay.io/test/hammerdb:latest" + db_type: "mssql" + timed_test: true + test_type: "tpc-c" + db_init: false + db_benchmark: true + db_server: "mssql-deployment.sql-server" + db_port: "1433" + db_warehouses: 1 + db_num_workers: 1 + db_user: "SA" + db_pass: "s3curePasswordString" + db_name: "tpcc" + transactions: 1000 + raiseerror: "false" + keyandthink: "false" + driver: "timed" + rampup: 1 + runtime: 1 + allwarehouse: false + timeprofile: false + async_scale: false + async_client: 10 + async_verbose: false + async_delay: 1000 + samples: 1 + # database specific variables + # mssql: + db_mssql_tcp: "true" + db_mssql_azure: "false" + db_mssql_authentication: "windows" + db_mssql_linux_authent: "sql" + db_mssql_odbc_driver: "ODBC Driver 13 for SQL Server" + db_mssql_linux_odbc: "ODBC Driver 17 for SQL Server" + db_mssql_imdb: "false" + db_mssql_bucket: 1 + db_mssql_durability: "SCHEMA_AND_DATA" + db_mssql_checkpoint: "false" + # mysql: + db_mysql_storage_engine: "innodb" + db_mysql_partition: "false" + # postgresql + db_postgresql_superuser: "SA" + db_postgresql_superuser_pass: "s3curePasswordString" + db_postgresql_defaultdbase: "postgres" + db_postgresql_vacuum: "false" + db_postgresql_dritasnap: "false" + db_postgresql_oracompat: "false" + db_postgresql_storedprocs: "false" + debug: true diff --git a/e2e/hammerdb/sql-server.yaml b/e2e/hammerdb/sql-server.yaml new file mode 100644 index 000000000..1649d7fad --- /dev/null +++ b/e2e/hammerdb/sql-server.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: sql-server +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mssql-deployment + namespace: sql-server +spec: + replicas: 1 + selector: + matchLabels: + app: mssql + template: + metadata: + labels: + app: mssql + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: mssql + image: quay.io/cloud-bulldozer/mssql:latest + ports: + - containerPort: 1433 + resources: + requests: + memory: "2048Mi" +--- +apiVersion: v1 +kind: Service +metadata: + name: mssql-deployment + namespace: sql-server +spec: + selector: + app: mssql + ports: + - protocol: TCP + port: 1433 + targetPort: 1433 diff --git a/e2e/helpers.bash b/e2e/helpers.bash new file mode 100644 index 000000000..69cf525a4 --- /dev/null +++ b/e2e/helpers.bash @@ -0,0 +1,91 @@ +# vi: ft=bash + +uuid="" +suuid="" +ARTIFACTS_DIR=artifacts +NAMESPACE=benchmark-operator + + +basic_setup() { + export PROMETHEUS_TOKEN=$(oc sa get-token -n openshift-monitoring prometheus-k8s) +} + +basic_teardown() { + kubectl_exec delete benchmark ${CR_NAME} --ignore-not-found +} + +check_es() { + if [[ -n ${indexes} ]]; then + for index in "${indexes[@]}"; do + echo "Looking for documents with uuid: ${uuid} in index ${index}" + documents=$(curl -sS ${ES_SERVER}/${index}/_search?q=uuid.keyword:${uuid} | jq .hits.total.value) + if [[ ${documents} -le 0 ]]; then + die "${documents} documents found in index ${index}" + fi + done + else + echo "Looking for documents with uuid: ${uuid} in index ${ES_INDEX}" + documents=$(curl -sS ${ES_SERVER}/${ES_INDEX}/_search?q=uuid.keyword:${uuid} | jq .hits.total.value) + if [[ ${documents} -le 0 ]]; then + die "${documents} documents found in index ${ES_INDEX}" + fi + fi +} + +get_uuid() { + echo "Waiting for UUID from ${1}" + local timeout=300 + while [[ ${timeout} -gt 0 ]]; do + uuid=$(kubectl_exec get benchmarks ${1} -o jsonpath="{.status.uuid}") + if [[ -n ${uuid} ]]; then + suuid=$(kubectl_exec get benchmarks ${1} -o jsonpath="{.status.suuid}") + return + fi + sleep 1 + timeout=$((timeout - 1)) + done + die "Timeout waiting for uuid from benchmark ${1}" +} + +check_benchmark() { + local timeout=${1} + while [[ ${timeout} -gt 0 ]]; do + if [[ $(kubectl_exec get benchmark/${CR_NAME} -o jsonpath={.status.complete}) == 'true' ]]; then + break + fi + if [[ ${timeout} -lt 0 ]]; then + die "Timeout waiting for benchmark/${CR_NAME} to complete" + fi + sleep 10 + timeout=$((timeout - 10)) + done + local state=$(kubectl_exec get benchmark/${CR_NAME} -o jsonpath={.status.state}) + if [[ ${state} != "Complete" ]]; then + die "Benchmark state: ${state}" + fi +} + +die() { + printf "\nError message: ${1}\n" + local TEST_ARTIFACTS=${ARTIFACTS_DIR}/${CR_NAME} + local LOGS_FLAGS="--tail=-1 --all-containers --prefix" + mkdir -p ${TEST_ARTIFACTS} + echo "Dumping logs at ${TEST_ARTIFACTS}" + kubectl_exec get benchmark ${CR_NAME} -o yaml --ignore-not-found > ${TEST_ARTIFACTS}/${CR_NAME}.yaml + kubectl_exec logs deployment/benchmark-controller-manager --tail=-1 -c manager > ${ARTIFACTS_DIR}/benchmark-controller-manager.log + for pod in $(kubectl_exec get pod -l benchmark-uuid=${uuid} -o custom-columns="name:.metadata.name" --no-headers); do + log_file=${TEST_ARTIFACTS}/${pod}.log + echo "Saving log from pod ${pod} in ${log_file}" + kubectl_exec logs ${LOGS_FLAGS} ${pod} > ${log_file} + done + false +} + +get_benchmark_name() { + benchmark_file=$1 + yq e '.metadata.name' $benchmark_file +} + +kubectl_exec() { + kubectl -n ${NAMESPACE} $@ +} diff --git a/e2e/image_pull/image_pull.yaml b/e2e/image_pull/image_pull.yaml new file mode 100644 index 000000000..add809f5d --- /dev/null +++ b/e2e/image_pull/image_pull.yaml @@ -0,0 +1,18 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: image-pull + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + index_name: image-pull + workload: + name: image_pull + args: + pod_count: 2 + timeout: 60 + retries: 1 + image_list: + - docker://quay.io/cloud-bulldozer/backpack + - docker://quay.io/cloud-bulldozer/hello-openshift diff --git a/e2e/iperf3/iperf3.yaml b/e2e/iperf3/iperf3.yaml new file mode 100644 index 000000000..d0bae2507 --- /dev/null +++ b/e2e/iperf3/iperf3.yaml @@ -0,0 +1,32 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: iperf3-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + workload: + name: iperf3 + args: + pairs: 1 + hostnetwork: false + port: 5201 + transmit_type: time + transmit_value: 60 + omit_start: 0 + length_buffer: 128K + window_size: 64k + ip_tos: 0 + mss: 900 + streams: 1 + extra_options_server: '--forceflush ' + extra_options_client: '-R ' diff --git a/e2e/kube-burner/cluster-density.yaml b/e2e/kube-burner/cluster-density.yaml new file mode 100644 index 000000000..d0c37f0b0 --- /dev/null +++ b/e2e/kube-burner/cluster-density.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-cluster-density + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + prometheus: + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + workload: cluster-density + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 1 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: metrics-aggregated.yaml + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/e2e/kube-burner/concurrent-builds.yaml b/e2e/kube-burner/concurrent-builds.yaml new file mode 100644 index 000000000..c0a850eac --- /dev/null +++ b/e2e/kube-burner/concurrent-builds.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-concurrent-builds + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: false + workload: + name: kube-burner + args: + workload: concurrent-builds + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 1 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: metrics-aggregated.yaml + app: django + source_strat_env: PIP_INDEX_URL + source_strat_from: python + source_strat_from_version: latest + post_commit_script: "./manage.py test" + build_image_stream: django-psql-example + git_url: https://github.com/sclorg/django-ex.git + build_image: image-registry.openshift-image-registry.svc:5000/svt-django/django-psql-example + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/e2e/kube-burner/configmap-cfg.yaml b/e2e/kube-burner/configmap-cfg.yaml new file mode 100644 index 000000000..9c2579c5d --- /dev/null +++ b/e2e/kube-burner/configmap-cfg.yaml @@ -0,0 +1,79 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-burner-config + namespace: benchmark-operator +data: + alerts.yml: "- expr: avg_over_time(histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m]))[5m:]) + > 0.01\n description: 5 minutes avg. etcd fsync latency on {{$labels.pod}} higher + than 10ms {{$value}}\n severity: warning\n\n- expr: avg_over_time(histogram_quantile(0.99, + rate(etcd_network_peer_round_trip_time_seconds_bucket[5m]))[5m:]) > 0.1\n description: + 5 minutes avg. etcd network peer round trip on {{$labels.pod}} higher than 100ms + {{$value}}\n severity: warning \n" + config.yml: | + --- + global: + writeToFile: false + metricsDirectory: collected-metrics + indexerConfig: + enabled: true + esServers: [{{ .ES_SERVER }}] + insecureSkipVerify: true + defaultIndex: ripsaw-kube-burner + type: elastic + measurements: + - name: podLatency + esIndex: ripsaw-kube-burner + + jobs: + - name: basic-job + jobIterations: 5 + qps: 20 + burst: 20 + namespacedIterations: false + namespace: basic-job + waitWhenFinished: true + podWait: false + objects: + + - objectTemplate: pod.yml + replicas: 1 + inputVars: + containerImage: gcr.io/google_containers/pause-amd64:3.0 + metrics.yml: | + - query: count(kube_namespace_created) + metricName: namespaceCount + instant: true + + - query: sum(kube_pod_status_phase{}) by (phase) + metricName: podStatusCount + instant: true + + - query: count(kube_secret_info{}) + metricName: secretCount + instant: true + + - query: count(kube_deployment_labels{}) + metricName: deploymentCount + instant: true + + - query: count(kube_configmap_info{}) + metricName: configmapCount + instant: true + pod.yml: | + kind: Pod + apiVersion: v1 + metadata: + name: basic-pod-{{.Iteration}} + spec: + nodeSelector: + node-role.kubernetes.io/worker: "" + containers: + - name: basic-container + image: {{.containerImage}} + ports: + - containerPort: 8080 + protocol: TCP + imagePullPolicy: IfNotPresent + securityContext: + privileged: false diff --git a/e2e/kube-burner/configmap.yaml b/e2e/kube-burner/configmap.yaml new file mode 100644 index 000000000..f74a6587f --- /dev/null +++ b/e2e/kube-burner/configmap.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-configmap-cfg + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: false + workload: + name: kube-burner + args: + extra_env_vars: + ES_SERVER: ${ES_SERVER} + configmap: kube-burner-config + cleanup: true + pin_server: {"node-role.kubernetes.io/worker": ""} + image: quay.io/cloud-bulldozer/kube-burner:latest + log_level: info + step: 30s + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/e2e/kube-burner/max-namespaces.yaml b/e2e/kube-burner/max-namespaces.yaml new file mode 100644 index 000000000..08283dd26 --- /dev/null +++ b/e2e/kube-burner/max-namespaces.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-max-namespaces + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: false + workload: + name: kube-burner + args: + workload: max-namespaces + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 1 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: metrics-aggregated.yaml + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/e2e/kube-burner/max-services.yaml b/e2e/kube-burner/max-services.yaml new file mode 100644 index 000000000..49b94b950 --- /dev/null +++ b/e2e/kube-burner/max-services.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-max-services + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: false + workload: + name: kube-burner + args: + workload: max-services + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 1 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: metrics-aggregated.yaml + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/e2e/kube-burner/node-density-cni-networkpolicy.yaml b/e2e/kube-burner/node-density-cni-networkpolicy.yaml new file mode 100644 index 000000000..3825bbe82 --- /dev/null +++ b/e2e/kube-burner/node-density-cni-networkpolicy.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-node-density-cni-networkpolicy + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: false + workload: + name: kube-burner + args: + workload: node-density-cni-policy + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 2 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: metrics.yaml + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/e2e/kube-burner/node-density-cni.yaml b/e2e/kube-burner/node-density-cni.yaml new file mode 100644 index 000000000..3778945fc --- /dev/null +++ b/e2e/kube-burner/node-density-cni.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-node-density-cni + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: false + workload: + name: kube-burner + args: + workload: node-density-cni + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 2 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: metrics.yaml + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/e2e/kube-burner/node-density-heavy.yaml b/e2e/kube-burner/node-density-heavy.yaml new file mode 100644 index 000000000..3faedb7f1 --- /dev/null +++ b/e2e/kube-burner/node-density-heavy.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-node-density-heavy + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: false + workload: + name: kube-burner + args: + workload: node-density-heavy + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 2 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: metrics.yaml + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/e2e/kube-burner/node-density.yaml b/e2e/kube-burner/node-density.yaml new file mode 100644 index 000000000..ac8be5ef4 --- /dev/null +++ b/e2e/kube-burner/node-density.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-node-density + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: false + workload: + name: kube-burner + args: + workload: node-density + cleanup: true + default_index: ripsaw-kube-burner + job_iterations: 10 + pin_server: {"node-role.kubernetes.io/worker": ""} + wait_when_finished: true + pod_wait: false + image: quay.io/cloud-bulldozer/kube-burner:latest + qps: 25 + burst: 50 + log_level: info + verify_objects: true + error_on_verify: true + step: 30s + metrics_profile: metrics.yaml + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/e2e/log_generator/log_generator.yaml b/e2e/log_generator/log_generator.yaml new file mode 100644 index 000000000..146496d78 --- /dev/null +++ b/e2e/log_generator/log_generator.yaml @@ -0,0 +1,18 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: log-generator + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + index_name: log-generator + workload: + name: log_generator + args: + pod_count: 1 + size: 512 + messages_per_second: 1 + duration: 1 + es_url: ${ES_SERVER} + timeout: 600 diff --git a/e2e/pgbench/pgbench.yaml b/e2e/pgbench/pgbench.yaml new file mode 100644 index 000000000..63be13ad3 --- /dev/null +++ b/e2e/pgbench/pgbench.yaml @@ -0,0 +1,37 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: pgbench-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-pgbench + metadata: + collection: true + workload: + name: "pgbench" + args: + timeout: 5 + clients: + - 1 + - 2 + threads: 1 + transactions: 10 + cmd_flags: '' + init_cmd_flags: '' + scaling_factor: 1 + samples: 2 + num_databases_pattern: 'all' + databases: + - host: ${POSTGRES_IP} + user: ci + password: ci + db_name: cidb + debug: true diff --git a/e2e/pgbench/postgres.yaml b/e2e/pgbench/postgres.yaml new file mode 100644 index 000000000..6c739e8c0 --- /dev/null +++ b/e2e/pgbench/postgres.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-config + namespace: benchmark-operator + labels: + app: postgres +data: + POSTGRES_DB: cidb + POSTGRES_USER: ci + POSTGRES_PASSWORD: ci + PGDATA: /var/lib/postgresql/data/pgdata +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: benchmark-operator +spec: + selector: + matchLabels: + app: postgres + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:10.4 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 5432 + envFrom: + - configMapRef: + name: postgres-config diff --git a/e2e/run_tests_ci.sh b/e2e/run_tests_ci.sh new file mode 100755 index 000000000..1c36a0f75 --- /dev/null +++ b/e2e/run_tests_ci.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +select_tests(){ + for f in $(git diff --name-only origin/master); do + found=false + while read -r t; do + regex=$(echo ${t} | cut -d : -f 1) + test_filter=$(echo ${t} | cut -d : -f 2) + # File found in test_map + if echo $f | grep -qE ${regex}; then + found=true + # Check if test was previously added + if ! echo "${BATS_TESTS}" | grep -q "${test_filter}"; then + if [[ ${BATS_TESTS} != "" ]]; then + BATS_TESTS+="|" + fi + BATS_TESTS+="${test_filter}" + fi + break + fi + done < e2e/test_map + # If one of the modified files is not present in the test_map, we exit the loop and + # run all tests + if [[ ${found} == "false" ]]; then + echo "File ${f} not found in test_map, running all tests" + BATS_TESTS=. + break + fi + done + export BATS_TESTS +} + +if [[ -z ${BATS_TESTS} ]]; then + select_tests +fi +make e2e-tests diff --git a/e2e/scale_openshift/scale_down.yaml b/e2e/scale_openshift/scale_down.yaml new file mode 100644 index 000000000..1091cd9cc --- /dev/null +++ b/e2e/scale_openshift/scale_down.yaml @@ -0,0 +1,23 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: scale-down + namespace: benchmark-operator +spec: + metadata: + collection: true + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: openshift-cluster-timings + workload: + name: scale_openshift + args: + scale: 0 + poll_interval: 2 + debug: true diff --git a/e2e/scale_openshift/scale_up.yaml b/e2e/scale_openshift/scale_up.yaml new file mode 100644 index 000000000..6096f8c97 --- /dev/null +++ b/e2e/scale_openshift/scale_up.yaml @@ -0,0 +1,23 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: scale-up + namespace: benchmark-operator +spec: + metadata: + collection: true + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: openshift-cluster-timings + workload: + name: scale_openshift + args: + scale: 1 + poll_interval: 1 + debug: true diff --git a/e2e/smallfile/smallfile.yaml b/e2e/smallfile/smallfile.yaml new file mode 100644 index 000000000..9a4752de1 --- /dev/null +++ b/e2e/smallfile/smallfile.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: smallfile-benchmark + namespace: benchmark-operator +spec: + test_user: homer_simpson + # to separate this test run from everyone else's + clustername: test_ci + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-smallfile + metadata: + collection: true + workload: + name: smallfile + args: + clients: 2 + operation: ["create", "read", "delete", "cleanup"] + threads: 1 + file_size: 2 + record_size: 1 + xattr_size: 50 + pause: 0 + auto_pause: y + stonewall: n + finish: y + prefix: abc + hash_into_dirs: true + same_dir: y + incompressible: y + verify_read: y + xattr_count: 10 + fsync: false + files_per_dir: 2000 + dirs_per_dir: 4 + files: 100000 + cleanup_delay_usec_per_file: 20 + debug: true diff --git a/e2e/smallfile/smallfile_hostpath.yaml b/e2e/smallfile/smallfile_hostpath.yaml new file mode 100644 index 000000000..465adb6d5 --- /dev/null +++ b/e2e/smallfile/smallfile_hostpath.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: smallfile-hostpath + namespace: benchmark-operator +spec: + test_user: homer_simpson + # to separate this test run from everyone else's + clustername: test_ci + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-smallfile + metadata: + collection: false + hostpath: /mnt/vda1/smallfile + workload: + name: smallfile + args: + clients: 1 + operation: ["create", "read", "append", "delete"] + threads: 1 + file_size: 0 + files: 100000 + debug: true diff --git a/e2e/stressng/stressng.yaml b/e2e/stressng/stressng.yaml new file mode 100644 index 000000000..608b5c2c7 --- /dev/null +++ b/e2e/stressng/stressng.yaml @@ -0,0 +1,34 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: stressng + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-stressng + metadata: + collection: true + workload: + name: "stressng" + args: + # general options + runtype: "parallel" + timeout: "10" + instances: 1 + # nodeselector: + # cpu stressor options + cpu_stressors: "1" + cpu_percentage: "100" + # vm stressor option + vm_stressors: "1" + vm_bytes: "128M" + # mem stressor options + mem_stressors: "1" + debug: true diff --git a/e2e/sysbench/sysbench.yaml b/e2e/sysbench/sysbench.yaml new file mode 100644 index 000000000..e05ae5638 --- /dev/null +++ b/e2e/sysbench/sysbench.yaml @@ -0,0 +1,23 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: sysbench-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + workload: + name: sysbench + args: + tests: + - name: cpu + parameters: + cpu-max-prime: 200 diff --git a/e2e/test_map b/e2e/test_map new file mode 100644 index 000000000..e45350257 --- /dev/null +++ b/e2e/test_map @@ -0,0 +1,21 @@ +kustomization\.yaml|Makefile|\.md$|servicemesh|benchmark_state|LICENSE|PROJECT|^charts:skip +api_load:api-load +backpackbackpack +byowl:byowl +flent:flent +fs-drift:fs-drift +hammerdb:hammerdb +image_pull:image_pull +iperf3:iperf3 +fio/|fio_distributed/|ceph_osd_cache_drop|kernel_cache_drop:fio +kube-burner:kube-burner +log_generator:log_generator +pgbench:pgbench +scale_openshift:scale_openshift +smallfile:smallfile +stressng:stressng +sysbench:sysbench +testpmd:testpmd +uperf:uperf +vegeta:vegeta +ycsb:ycsb diff --git a/e2e/uperf/uperf.yaml b/e2e/uperf/uperf.yaml new file mode 100644 index 000000000..c9706f627 --- /dev/null +++ b/e2e/uperf/uperf.yaml @@ -0,0 +1,44 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-burstable + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf + args: + hostnetwork: false + serviceip: false + pin: false + multus: + enabled: false + samples: 2 + pair: 1 + test_types: + - stream + - rr + - bidirec + protos: + - tcp + - udp + - sctp + sizes: + - 1024 + - [1024, 512] + nthrs: + - 1 + - 2 + runtime: 2 + debug: true diff --git a/e2e/uperf/uperf_hostnetwork.yaml b/e2e/uperf/uperf_hostnetwork.yaml new file mode 100644 index 000000000..4c655639b --- /dev/null +++ b/e2e/uperf/uperf_hostnetwork.yaml @@ -0,0 +1,38 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-hostnetwork + namespace: benchmark-operator +spec: + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-uperf + metadata: + collection: false + cleanup: false + workload: + name: uperf + args: + hostnetwork: true + serviceip: false + pin: false + multus: + enabled: false + samples: 1 + pair: 1 + test_types: + - stream + protos: + - tcp + sizes: + - 1024 + nthrs: + - 1 + runtime: 2 + debug: true diff --git a/e2e/uperf/uperf_networkpolicy.yaml b/e2e/uperf/uperf_networkpolicy.yaml new file mode 100644 index 000000000..1f5bfd4df --- /dev/null +++ b/e2e/uperf/uperf_networkpolicy.yaml @@ -0,0 +1,39 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-networkpolicy + namespace: benchmark-operator +spec: + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-uperf + metadata: + collection: false + cleanup: false + workload: + name: uperf + args: + hostnetwork: false + serviceip: false + networkpolicy: true + pin: false + multus: + enabled: false + samples: 1 + pair: 1 + test_types: + - stream + protos: + - tcp + sizes: + - 1024 + nthrs: + - 1 + runtime: 2 + debug: true diff --git a/e2e/uperf/uperf_resources.yaml b/e2e/uperf/uperf_resources.yaml new file mode 100644 index 000000000..288d2bb3a --- /dev/null +++ b/e2e/uperf/uperf_resources.yaml @@ -0,0 +1,45 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-resources + namespace: benchmark-operator +spec: + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-uperf + metadata: + collection: false + cleanup: false + workload: + name: uperf + args: + client_resources: + requests: + cpu: 100m + memory: 100Mi + server_resources: + requests: + cpu: 100m + memory: 100Mi + hostnetwork: false + serviceip: false + multus: + enabled: false + samples: 1 + pair: 1 + test_types: + - stream + protos: + - tcp + sizes: + - 1024 + nthrs: + - 1 + runtime: 2 + debug: true diff --git a/e2e/uperf/uperf_serviceip.yaml b/e2e/uperf/uperf_serviceip.yaml new file mode 100644 index 000000000..4df1a4fdb --- /dev/null +++ b/e2e/uperf/uperf_serviceip.yaml @@ -0,0 +1,39 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-serviceip + namespace: benchmark-operator +spec: + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-uperf + metadata: + collection: false + cleanup: false + workload: + name: uperf + args: + hostnetwork: false + serviceip: true + multus: + enabled: false + samples: 1 + pair: 1 + test_types: + - stream + protos: + - tcp + - udp + - sctp + sizes: + - 512 + nthrs: + - 1 + runtime: 2 + debug: true diff --git a/e2e/uperf/uperf_serviceip_nodeport.yaml b/e2e/uperf/uperf_serviceip_nodeport.yaml new file mode 100644 index 000000000..e0175d9b2 --- /dev/null +++ b/e2e/uperf/uperf_serviceip_nodeport.yaml @@ -0,0 +1,39 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-serviceip-nodeport + namespace: benchmark-operator +spec: + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-uperf + metadata: + collection: false + cleanup: false + workload: + name: uperf + args: + hostnetwork: false + serviceip: true + servicetype: "nodeport" + multus: + enabled: false + samples: 1 + pair: 2 + test_types: + - stream + protos: + - tcp + - udp + sizes: + - 512 + nthrs: + - 1 + runtime: 2 + debug: true diff --git a/e2e/vegeta/vegeta.yaml b/e2e/vegeta/vegeta.yaml new file mode 100644 index 000000000..93cb362c8 --- /dev/null +++ b/e2e/vegeta/vegeta.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: vegeta-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-vegeta + metadata: + collection: true + workload: + name: vegeta + args: + clients: 4 + targets: + - name: 2w-ka + urls: + - GET https://1.1.1.1 + - GET http://1.1.1.1 + samples: 1 + workers: 2 + duration: 5 + keepalive: true + - name: 2w-noka + urls: + - GET https://1.1.1.1 + workers: 2 + duration: 5 + debug: true diff --git a/e2e/vegeta/vegeta_hostnetwork.yaml b/e2e/vegeta/vegeta_hostnetwork.yaml new file mode 100644 index 000000000..771a11b4b --- /dev/null +++ b/e2e/vegeta/vegeta_hostnetwork.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: vegeta-benchmark-hostnetwork + namespace: benchmark-operator +spec: + system_metrics: + collection: false + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-vegeta + metadata: + collection: false + workload: + name: vegeta + args: + hostnetwork: true + clients: 2 + targets: + - name: 2w-ka + urls: + - GET https://1.1.1.1 + samples: 1 + workers: 2 + duration: 5 + keepalive: true + debug: true diff --git a/e2e/ycsb/mongo.yaml b/e2e/ycsb/mongo.yaml new file mode 100644 index 000000000..622911bef --- /dev/null +++ b/e2e/ycsb/mongo.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: mongo + labels: + name: mongo +spec: + ports: + - port: 27017 + targetPort: 27017 + clusterIP: None + selector: + role: mongo +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: mongo +spec: + selector: + matchLabels: + role: mongo + serviceName: "mongo" + replicas: 1 + selector: + matchLabels: + role: mongo + template: + metadata: + labels: + role: mongo + environment: test + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: mongo + image: mongo + command: ["/bin/sh"] + args: ["-c", "mkdir -p /tmp/data/db; mongod --bind_ip 0.0.0.0 --dbpath /tmp/data/db"] + ports: + - containerPort: 27017 diff --git a/e2e/ycsb/ycsb-mongo.yaml b/e2e/ycsb/ycsb-mongo.yaml new file mode 100644 index 000000000..5f03033e6 --- /dev/null +++ b/e2e/ycsb/ycsb-mongo.yaml @@ -0,0 +1,30 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: ycsb-mongo-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + metrics_profile: node-metrics.yml + elasticsearch: + url: ${ES_SERVER} + index_name: ripsaw-ycsb + metadata: + collection: true + workload: + name: ycsb + args: + infra: mongodb + driver: mongodb + recordcount: 100 + operationcount: 100 + requestdistribution: zipfian + workloads: + - workloada + options_load: '-p mongodb.url="mongodb://mongo-0.mongo/ycsb?"' #passed as is to ycsb when loading database + options_run: '-p mongodb.url="mongodb://mongo-0.mongo/ycsb?" -threads 10 -target 100' + debug: true diff --git a/group_vars/all.yml b/group_vars/all.yml deleted file mode 100644 index 940081af5..000000000 --- a/group_vars/all.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -operator_namespace: '{{ meta.namespace }}' -clustername: 'myk8scluster' -kernel_cache_drop_svc_port: 9222 diff --git a/hack/install-bats.sh b/hack/install-bats.sh new file mode 100755 index 000000000..9f9219571 --- /dev/null +++ b/hack/install-bats.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -e + +if [[ "$(type -t bats)" != "" ]]; then + # bats is already installed. + exit 0 +fi + +buildDir=$(mktemp -d) +git clone https://github.com/bats-core/bats-core $buildDir + +pushd $buildDir +pwd +git reset --hard ${VERSION} +echo "Installing bats to /usr/local (requires root)" +sudo ./install.sh /usr/local +popd + +rm -rf $buildDir diff --git a/hack/tag_name.sh b/hack/tag_name.sh new file mode 100755 index 000000000..0ff164f41 --- /dev/null +++ b/hack/tag_name.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +if [[ -z $(git branch --show-current) ]]; then + git describe --tags --abbrev=0 +else + git branch --show-current | sed 's/master/latest/g' +fi diff --git a/image_resources/centos8-appstream.repo b/image_resources/centos8-appstream.repo index 5f2902c0f..a20e22a86 100644 --- a/image_resources/centos8-appstream.repo +++ b/image_resources/centos8-appstream.repo @@ -1,5 +1,17 @@ -[centos8-appstream] -name=CentOS-8-Appstream -baseurl=http://mirror.centos.org/centos/8/AppStream/x86_64/os/ +[centos8-appstream-x86_64] +name=CentOS-8-Appstream-x86_64 +baseurl=http://vault.centos.org/centos/8-stream/AppStream/x86_64/os/ +enabled=0 +gpgcheck=0 + +[centos8-appstream-aarch64] +name=CentOS-8-Appstream-aarch64 +baseurl=http://vault.centos.org/centos/8-stream/AppStream/aarch64/os/ +enabled=0 +gpgcheck=0 + +[centos8-appstream-ppc64le] +name=CentOS-8-Appstream-ppc64le +baseurl=http://vault.centos.org/centos/8-stream/AppStream/ppc64le/os/ enabled=0 gpgcheck=0 diff --git a/meta/main.yml b/meta/main.yml deleted file mode 100644 index 80a3c730f..000000000 --- a/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ -collections: -- community.kubernetes -- operator_sdk.util diff --git a/playbook.yml b/playbook.yml deleted file mode 100644 index 8cf814a6d..000000000 --- a/playbook.yml +++ /dev/null @@ -1,95 +0,0 @@ -- hosts: localhost - collections: - - operator_sdk.util - gather_facts: no - tasks: - - - include_role: - name: "kernel_cache_drop" - when: workload.args.drop_cache_kernel is defined - - - name: Get update from Cerberus if connected - block: - - include_role: - name: "cerberus" - - when: cerberus_url is defined and cerberus_url != "" - - - - name: Get state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: cr_state - - - name: Capture operator information - k8s_facts: - kind: Pod - api_version: v1 - namespace: "{{ operator_namespace }}" - label_selectors: - - name = benchmark-operator - register: bo - - - debug: - msg: "{{ cr_state }}" - - - name: Set Workload UUID - block: - - include_role: - name: "uuid" - - - name: Setting the uuid for the benchmark - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - uuid: "{{ uuid }}" - complete: false - suuid: "{{ trunc_uuid }}" - metadata: "not collected" - cerberus: "not connected" - - when: workload is defined and (cr_state.resources[0].status is not defined or cr_state.resources[0].status.uuid is not defined) - - - name: Run Workload - block: - - - set_fact: - uuid: "{{ cr_state.resources[0].status.uuid }}" - trunc_uuid: "{{ cr_state.resources[0].status.suuid }}" - - - block: - - - include_role: - name: backpack - when: metadata is defined and not metadata.targeted | default('true') | bool - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - metadata: "Collecting" - when: metadata is defined and metadata.targeted | default('true') | bool and not cr_state.resources[0].status.state is defined - - when: metadata is defined and metadata.collection | default('false') | bool and (cr_state.resources[0].status.metadata is defined and cr_state.resources[0].status.metadata != "Complete") - - - block: - - - include_role: - name: "common" - - - include_role: - name: "{{ workload.name }}" - vars: - workload_args: "{{ workload.args }}" - - when: metadata is not defined or not metadata.collection | default('false') | bool or (cr_state.resources[0].status.metadata is defined and cr_state.resources[0].status.metadata == "Complete") or metadata.targeted | default('true') | bool - - when: cr_state is defined and cr_state.resources[0].status is defined and not cr_state.resources[0].status.complete|bool and (cr_state.resources[0].status.state is not defined or cr_state.resources[0].status.state != "Error") diff --git a/playbooks/benchmark.yml b/playbooks/benchmark.yml new file mode 100644 index 000000000..b60b2b8ab --- /dev/null +++ b/playbooks/benchmark.yml @@ -0,0 +1,120 @@ +- hosts: localhost + collections: + - operator_sdk.util + gather_facts: no + pre_tasks: + + - name: Get current state + k8s_info: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: '{{ ansible_operator_meta.name }}' + namespace: '{{ operator_namespace }}' + register: benchmark_state + tasks: + + - include_role: + name: "ceph_osd_cache_drop" + when: workload.args.drop_cache_rook_ceph is defined + + - include_role: + name: "kernel_cache_drop" + when: workload.args.drop_cache_kernel is defined + + - name: Capture operator information + k8s_info: + kind: Pod + api_version: v1 + namespace: "{{ operator_namespace }}" + label_selectors: + - control-plane = controller-manager + register: bo + + + - name: Set Workload UUID + block: + - include_role: + name: "uuid" + + - name: Setting the uuid for the benchmark + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + uuid: "{{ uuid }}" + complete: false + suuid: "{{ trunc_uuid }}" + metadata: "not collected" + rescue: + - include_role: + name: benchmark_state + tasks_from: failure + + when: workload is defined and (benchmark_state.resources[0].status is not defined or benchmark_state.resources[0].status.uuid is not defined) + + + - set_fact: + uuid: "{{ benchmark_state.resources[0].status.uuid }}" + trunc_uuid: "{{ benchmark_state.resources[0].status.suuid }}" + when: + - benchmark_state.resources[0].status is defined + - benchmark_state.resources[0].status.uuid is defined + - benchmark_state.resources[0].status.uuid != "" + + - name: Run Workload + rescue: + - include_role: + name: benchmark_state + tasks_from: failure + block: + - block: + + - include_role: + name: backpack + when: metadata is defined and not metadata.targeted | default('true') | bool + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + metadata: "Collecting" + when: metadata is defined and metadata.targeted | default('true') | bool and not benchmark_state.resources[0].status.state is defined + + when: metadata is defined and metadata.collection | default('false') | bool and (benchmark_state.resources[0].status.metadata is defined and benchmark_state.resources[0].status.metadata != "Complete") + + - block: + + - include_role: + name: "common" + + - name: Set Building state + include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Building + when: benchmark_state.resources[0].status.state is not defined and workload.name != "backpack" + + - include_role: + name: "{{ workload.name }}" + vars: + workload_args: "{{ workload.args }}" + + when: metadata is not defined or not metadata.collection | default('false') | bool or (benchmark_state.resources[0].status.metadata is defined and benchmark_state.resources[0].status.metadata == "Complete") or metadata.targeted | default('true') | bool + + when: benchmark_state is defined and benchmark_state.resources[0].status is defined and not benchmark_state.resources[0].status.complete|bool and (benchmark_state.resources[0].status.state is not defined or benchmark_state.resources[0].status.state != "Error") + + - include_role: + name: system-metrics + + vars: + workload_args: "{{ workload.args }}" + when: + - benchmark_state.resources[0].status.state is defined + - benchmark_state.resources[0].status.state == "Complete" + - system_metrics.collection | bool + - workload.name not in ["kube-burner", "backpack"] diff --git a/playbooks/group_vars/all.yml b/playbooks/group_vars/all.yml new file mode 100644 index 000000000..d14efd36a --- /dev/null +++ b/playbooks/group_vars/all.yml @@ -0,0 +1,10 @@ +--- +operator_namespace: '{{ ansible_operator_meta.namespace }}' + +# for upstream kubernetes this might be rook-ceph +# FIXME: how do we automatically determine that we are in openshift? +rook_ceph_namespace: 'openshift-storage' + +clustername: 'myk8scluster' +kernel_cache_drop_svc_port: 9222 +ceph_cache_drop_svc_port: 9457 diff --git a/templates/metadata.yml.j2 b/playbooks/templates/metadata.yml.j2 similarity index 76% rename from templates/metadata.yml.j2 rename to playbooks/templates/metadata.yml.j2 index 5a7fd1381..2dcc6169e 100644 --- a/templates/metadata.yml.j2 +++ b/playbooks/templates/metadata.yml.j2 @@ -1,7 +1,5 @@ + serviceAccountName: benchmark-operator {% if metadata.collection is sameas true and metadata.targeted is sameas true %} -{% if metadata.serviceaccount != "default" %} - serviceAccountName: {{ metadata.serviceaccount }} -{% endif %} initContainers: - name: backpack image: {{ metadata.image }} @@ -19,9 +17,9 @@ {% if metadata.force is sameas true %} --force {% endif %} - --tags={{ metadata.stockpile_tags|default(["common", "k8s", "openshift"])|join(",") }} -{% if metadata.stockpile_skip_tags|length > 0 %} - --skip-tags={{ metadata.stockpile_skip_tags|join(",") }} + --tags={{ metadata.stockpileTags|default(["common", "k8s", "openshift"])|join(",") }} +{% if metadata.stockpileSkipTags|length > 0 %} + --skip-tags={{ metadata.stockpileSkipTags|join(",") }} {% endif %} {% if metadata.ssl is sameas true %} --sslskipverify True @@ -29,6 +27,7 @@ imagePullPolicy: Always securityContext: privileged: {{ metadata.privileged }} + readOnlyRootFilesystem: false runAsNonRoot: false env: - name: my_node_name diff --git a/playbooks/templates/metadata_pod.yml.j2 b/playbooks/templates/metadata_pod.yml.j2 new file mode 100644 index 000000000..d74178580 --- /dev/null +++ b/playbooks/templates/metadata_pod.yml.j2 @@ -0,0 +1,41 @@ + serviceAccountName: benchmark-operator +{% if metadata.collection is sameas true and metadata.targeted is sameas true %} + initContainers: + - name: backpack + image: {{ metadata.image }} + command: ["/bin/sh", "-c"] + args: + - > + python3 + stockpile-wrapper.py + -s={{ elasticsearch.url }} + -u={{ uuid }} + -n=${my_node_name} + -N=${my_pod_name} + --redisip={{ bo.resources[0].status.podIP }} + --redisport=6379 +{% if metadata.force is sameas true %} + --force +{% endif %} + --tags={{ metadata.stockpileTags|default(["common", "k8s", "openshift"])|join(",") }} +{% if metadata.stockpileSkipTags|length > 0 %} + --skip-tags={{ metadata.stockpileSkipTags|join(",") }} +{% endif %} +{% if metadata.ssl is sameas true %} + --sslskipverify True +{% endif %} + imagePullPolicy: Always + securityContext: + privileged: {{ metadata.privileged }} + readOnlyRootFilesystem: false + runAsNonRoot: false + env: + - name: my_node_name + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: my_pod_name + valueFrom: + fieldRef: + fieldPath: metadata.name +{% endif %} diff --git a/requirements.yml b/requirements.yml index 8a661f8b9..8c8123ba9 100644 --- a/requirements.yml +++ b/requirements.yml @@ -1,5 +1,6 @@ --- collections: - name: community.kubernetes - version: "<1.0.0" - - operator_sdk.util + version: "2.0.1" + - name: operator_sdk.util + version: "0.5.0" diff --git a/resources/backpack_role.yaml b/resources/backpack_role.yaml deleted file mode 100644 index 59e81076d..000000000 --- a/resources/backpack_role.yaml +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: backpack_role -rules: -- apiGroups: - - "*" - resources: - - "*" - verbs: - - get - - list - - watch -- apiGroups: - - policy - resources: - - podsecuritypolicies - verbs: - - use - resourceNames: - - privileged -- apiGroups: - - security.openshift.io - resourceNames: - - privileged - resources: - - securitycontextconstraints - verbs: - - use ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: backpack-view - namespace: my-ripsaw ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: binding-backpack_role - namespace: my-ripsaw -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: backpack_role -subjects: -- kind: ServiceAccount - name: backpack-view - namespace: my-ripsaw diff --git a/resources/crds/ripsaw_v1alpha1_hammerdb_cr.yaml b/resources/crds/ripsaw_v1alpha1_hammerdb_cr.yaml deleted file mode 100644 index 8be445f15..000000000 --- a/resources/crds/ripsaw_v1alpha1_hammerdb_cr.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: hammerdb-benchmark-example - namespace: my-ripsaw -spec: - # where elastic search is running - elasticsearch: - url: http://my.elasticsearch.server:80 - verify_cert: false - parallel: false - # clustername: myk8scluster - # test_user: ripsaw - workload: - name: hammerdb - args: - ## 'initialize' enables the schema creation in the db if it's not done already - ## 'benchmark' enables the actual benchmarking using the parameters further down below - db_init: true - db_benchmark: true - ## Standard options used for all operations, initialization and benchmark - db_server: "mssql-deployment.sql-server" - db_port: "1443" - db_user: "SA" - db_pass: "s3curePasswordString" - db_warehouses: 1 - db_num_workers: 1 - db_tcp: "true" - - ## Workload specific settings - ## runtime and transactions are mutually exclusive, transactions will override runtime - transactions: 100000 - runtime: 30 # corresponds to runtimer in the tcl script (in minutes) - rampup: 1 # corresponds to mssqls_rampup in the tcl script (minutes) - samples: 3 # needs to be implemented - #clients: 4 # corresponds to virtual users (vu) in the tcl script, this should be the maximum number of clients which we approach logarithmically (sp?) - diff --git a/resources/crds/ripsaw_v1alpha1_hammerdb_index_cr.yaml b/resources/crds/ripsaw_v1alpha1_hammerdb_index_cr.yaml deleted file mode 100644 index 93b743cca..000000000 --- a/resources/crds/ripsaw_v1alpha1_hammerdb_index_cr.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: hammerdb-benchmark-example - namespace: my-ripsaw -spec: - # where elastic search is running - elasticsearch: - url: http://my.elasticsearch.server:80 - verify_cert: false - parallel: false - # clustername: myk8scluster - # test_user: ripsaw - workload: - # cleanup: true - name: hammerdb - args: - db_server: false - db_port: false - warehouses: 1 - workers: 1 - protocol: "tcp" - runtime: 30 diff --git a/resources/crds/ripsaw_v1alpha1_smallfile_cr.yaml b/resources/crds/ripsaw_v1alpha1_smallfile_cr.yaml deleted file mode 100644 index 57e09ad41..000000000 --- a/resources/crds/ripsaw_v1alpha1_smallfile_cr.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: example-benchmark - namespace: my-ripsaw -spec: - test_user: homer_simpson - clustername: aws-2009-09-10 - elasticsearch: - url: http://my.elasticsearch.server:9200 - workload: - name: smallfile - args: - clients: 1 - samples: 1 - operation: ["create", "read"] - threads: 5 - file_size: 64 - files: 100 - storageclass: rook-ceph-block - storagesize: 2Gi diff --git a/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml b/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml deleted file mode 100644 index bf2606875..000000000 --- a/resources/crds/ripsaw_v1alpha1_uperf_cr.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 -kind: Benchmark -metadata: - name: uperf-benchmark - namespace: my-ripsaw -spec: - clustername: myk8scluster - elasticsearch: - url: http://es.server.com:80 - #test_user: username_to_attach_to_metadata - workload: - # cleanup: true - name: uperf - args: - serviceip: false - hostnetwork: false - networkpolicy: false - pin: false - multus: - enabled: false - pin_server: "node-0" - pin_client: "node-1" - samples: 1 - kind: pod - pair: 1 - test_types: - - stream - protos: - - tcp - sizes: - - 16384 - nthrs: - - 1 - runtime: 30 diff --git a/resources/kernel-cache-drop-daemonset.yaml b/resources/kernel-cache-drop-daemonset.yaml index fbf9c4b30..dd1a43ab6 100644 --- a/resources/kernel-cache-drop-daemonset.yaml +++ b/resources/kernel-cache-drop-daemonset.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: kernel-cache-dropper - namespace: my-ripsaw + namespace: benchmark-operator labels: app: kernel-cache-dropper spec: @@ -14,6 +14,8 @@ spec: labels: name: kernel-cache-dropper spec: + nodeSelector: + kernel-cache-dropper: "yes" serviceAccountName: benchmark-operator #tolerations: #- key: node-role.kubernetes.io/master @@ -29,7 +31,6 @@ spec: value: "{{ kernel_cache_drop_svc_port }}" command: ["/usr/bin/python3"] args: ["/opt/kernel_cache_drop/kernel-cache-drop-websvc.py"] - imagePullPolicy: Always securityContext: privileged: true # we don't need all the same volumes as the toolbox pod diff --git a/resources/kube-burner-role.yml b/resources/kube-burner-role.yml deleted file mode 100644 index 32bb9d57c..000000000 --- a/resources/kube-burner-role.yml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-burner - namespace: my-ripsaw ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: binding-kube-burner - namespace: my-ripsaw -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: -- kind: ServiceAccount - name: kube-burner - namespace: my-ripsaw diff --git a/resources/namespace.yaml b/resources/namespace.yaml deleted file mode 100644 index 0908738fc..000000000 --- a/resources/namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: my-ripsaw diff --git a/resources/operator.yaml b/resources/operator.yaml deleted file mode 100644 index f49ac96f0..000000000 --- a/resources/operator.yaml +++ /dev/null @@ -1,81 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: benchmark-operator - namespace: my-ripsaw -spec: - replicas: 1 - selector: - matchLabels: - name: benchmark-operator - template: - metadata: - labels: - name: benchmark-operator - spec: - tolerations: - - key: role - value: workload - effect: NoSchedule - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - preference: - matchExpressions: - - key: node-role.kubernetes.io/workload - operator: In - values: - - "" - serviceAccountName: benchmark-operator - containers: - - name: ansible - command: - - /usr/local/bin/ao-logs - - /tmp/ansible-operator/runner - - stdout - image: quay.io/benchmark-operator/benchmark-operator:master - imagePullPolicy: "Always" - volumeMounts: - - mountPath: /tmp/ansible-operator/runner - name: runner - readOnly: true - - name: benchmark-operator - image: quay.io/benchmark-operator/benchmark-operator:master - imagePullPolicy: Always - env: - - name: WATCH_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: OPERATOR_NAME - value: "benchmark-operator" - - name: WORKER_BENCHMARK_RIPSAW_CLOUDBULLDOZER_IO - value: "1" - - name: ANSIBLE_VERBOSITY - value: "4" - volumeMounts: - - mountPath: /tmp/ansible-operator/runner - name: runner - - name: redis-master - image: k8s.gcr.io/redis:v1 - env: - - name: MASTER - value: "true" - ports: - - containerPort: 6379 - resources: - limits: - cpu: "0.1" - volumeMounts: - - mountPath: /redis-master-data - name: data - volumes: - - name: data - emptyDir: {} - - name: runner - emptyDir: {} diff --git a/resources/scale_role.yaml b/resources/scale_role.yaml deleted file mode 100644 index 2d4ca4c76..000000000 --- a/resources/scale_role.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: scale_role -rules: -- apiGroups: - - "*" - resources: - - machines - - machinesets - - nodes - - infrastructures - verbs: - - '*' ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: scaler - namespace: my-ripsaw ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: scaler - namespace: my-ripsaw -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: scale_role -subjects: -- kind: ServiceAccount - name: scaler - namespace: my-ripsaw diff --git a/roles/api_load/tasks/main.yml b/roles/api_load/tasks/main.yml new file mode 100644 index 000000000..a9280456f --- /dev/null +++ b/roles/api_load/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- set_fact: + resources: + - "{{ role_path }}/templates/api_load.yml" + +- name: Generic workload - API load + include_role: + name: generic_workload diff --git a/roles/api_load/templates/api_load.yml b/roles/api_load/templates/api_load.yml new file mode 100644 index 000000000..905a13fcd --- /dev/null +++ b/roles/api_load/templates/api_load.yml @@ -0,0 +1,143 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: 'api-load-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' +spec: + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout | default(28800) }} + parallelism: {{ workload_args.pod_count | default(1) | int }} + template: + metadata: + labels: + app: api-load-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.tolerations is defined %} + tolerations: + - key: {{ workload_args.tolerations.key }} + value: {{ workload_args.tolerations.value }} + effect: {{ workload_args.tolerations.effect }} +{% endif %} +{% if workload_args.label is defined %} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ workload_args.label.key }} + operator: In + values: + - {{ workload_args.label.value }} +{% endif %} + containers: + - name: api-load + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/ocm-api-load:latest') }} + env: + - name: my_node_name + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: my_pod_name + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: uuid + value: "{{ uuid }}" + command: ["/bin/sh", "-c"] + args: + - > +{% if workload_args.override is defined %} + ocm-load-test {{ workload_args.override }}; +{% else %} + echo "Setting redis status"; + redis-cli -h {{ bo.resources[0].status.podIP }} SET "{{ uuid }}-status" "running"; +{% for key, value in workload_args.test_list.items() %} + ocm-load-test + --test-id={{ uuid }} + --gateway-url={{ workload_args.gateway_url }} + --ocm-token={{ workload_args.ocm_token }} + --duration={{ value.duration | default(workload_args.duration) }} + --rate={{ value.rate | default(workload_args.rate) }} + --output-path={{ workload_args.output_path | default('/tmp/results') }} + --test-names={{ key }} + --aws-access-key={{ workload_args.aws_access_key }} + --aws-access-secret={{ workload_args.aws_access_secret }} + --aws-account-id={{ workload_args.aws_account_id }} + --aws-region={{ workload_args.aws_region | default('us-west-2') }} +{% if value.ramp_type is defined %} + --ramp-type={{ value.ramp_type }} + --ramp-duration={{ value.ramp_duration | default(value.duration) }} + --ramp-steps={{ value.ramp_steps }} + --start-rate={{ value.ramp_start_rate }} + --end-rate={{ value.ramp_end_rate }} +{% endif %} + --elastic-server={{ elasticsearch.url }} + --elastic-index={{ elasticsearch.index_name }} +{% if elasticsearch.verify_cert %} + --elastic-insecure-skip-verify; +{% else %} + ; +{% endif %} + echo "Cooldown for {{ workload_args.cooldown | default(60) }}"; + sleep {{ workload_args.cooldown | default(60) }}; +{% endfor %} +{% endif %} + echo "Updating redis status"; + redis-cli -h {{ bo.resources[0].status.podIP }} SET "{{ uuid }}-status" "ready"; + volumeMounts: + - mountPath: /tmp/results + name: results + - name: data-collector + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/ocm-api-load:latest') }} + imagePullPolicy: Always + env: + - name: my_node_name + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: my_pod_name + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: uuid + value: "{{ uuid }}" + command: ["/bin/sh", "-c"] + args: + - > + echo "Getting redis status"; + status=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "{{ uuid }}-status"`; + while [ "$status" != "ready" ]; do + sleep {{ workload_args.sleep | default(360) }}; + echo "Testing readiness"; + status=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "{{ uuid }}-status"`; + done; +{% if workload_args.override is defined %} + echo "succeeded" +{% else %} +{% if snappy.url %} + echo "Uploading RAW files..."; + python automation.py upload --dir {{ workload_args.output_path | default('/tmp/results') }} --server {{ snappy.url }} --user {{ snappy.user }} --password {{ snappy.password }}; +{% endif %} +{% endif %} + volumeMounts: + - mountPath: /tmp/results + name: results + volumes: + - name: results + hostDisk: + path: /tmp/results + capacity: 10Gi + type: DiskOrCreate + restartPolicy: Never +{% include "metadata.yml.j2" %} diff --git a/roles/backpack/tasks/main.yml b/roles/backpack/tasks/main.yml index 549651f70..02e1248ce 100644 --- a/roles/backpack/tasks/main.yml +++ b/roles/backpack/tasks/main.yml @@ -1,33 +1,24 @@ --- -- name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - register: benchmark_state - - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: - state: "Metadata Collecting" complete: false metadata: "Collecting" when: benchmark_state.resources[0].status.state is not defined - name: Get benchmark state - k8s_facts: + k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" register: benchmark_state - name: Get DaemonSet state - k8s_facts: + k8s_info: api_version: apps/v1 kind: DaemonSet name: "backpack-{{ trunc_uuid }}" @@ -45,7 +36,7 @@ definition: "{{ lookup('template', 'backpack.yml') | from_yaml }}" - name: Get DaemonSet Status - k8s_facts: + k8s_info: api_version: apps/v1 kind: DaemonSet name: "backpack-{{ trunc_uuid }}" @@ -58,29 +49,10 @@ set_fact: backpack_cur_gen: "{{ my_daemonset | json_query('resources[].metadata.generation')|first }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Metadata Collecting" - complete: false - metadata: "Collecting" - when: backpack_orig_gen != backpack_cur_gen - - - name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - register: benchmark_state - - block: - name: Get initial pod list - k8s_facts: + k8s_info: kind: Pod namespace: "{{ operator_namespace }}" label_selectors: @@ -92,18 +64,28 @@ - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Complete + complete: true + metadata: Complete + when: workload.name == "backpack" + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: - state: null - complete: false - metadata: "Complete" + metadata: Complete + when: workload.name != "backpack" - name: Get benchmark state - k8s_facts: + k8s_info: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" register: benchmark_state diff --git a/roles/backpack/templates/backpack.yml b/roles/backpack/templates/backpack.yml index 48382e738..b9a2862f7 100755 --- a/roles/backpack/templates/backpack.yml +++ b/roles/backpack/templates/backpack.yml @@ -16,6 +16,7 @@ spec: labels: name: backpack-{{ trunc_uuid }} app: backpack-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} spec: tolerations: - key: node-role.kubernetes.io/master @@ -50,9 +51,9 @@ spec: {% if metadata.force is sameas true %} --force {% endif %} - --tags={{ metadata.stockpile_tags|default(["common", "k8s", "openshift"])|join(",") }} -{% if metadata.stockpile_skip_tags|length > 0 %} - --skip-tags={{ metadata.stockpile_skip_tags|join(",") }} + --tags={{ metadata.stockpileTags|default(["common", "k8s", "openshift"])|join(",") }} +{% if metadata.stockpileSkipTags|length > 0 %} + --skip-tags={{ metadata.stockpileSkipTags|join(",") }} {% endif %} {% if metadata.ssl is sameas true %} --sslskipverify True @@ -61,6 +62,7 @@ spec: imagePullPolicy: Always securityContext: privileged: {{ metadata.privileged }} + readOnlyRootFilesystem: false runAsNonRoot: false readinessProbe: exec: @@ -79,5 +81,5 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - serviceAccountName: {{ metadata.serviceaccount }} + serviceAccountName: benchmark-operator terminationGracePeriodSeconds: 30 diff --git a/roles/benchmark_state/tasks/completed.yml b/roles/benchmark_state/tasks/completed.yml new file mode 100644 index 000000000..1ad95db48 --- /dev/null +++ b/roles/benchmark_state/tasks/completed.yml @@ -0,0 +1,41 @@ +--- +- name: Get job status + k8s_info: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - benchmark-uuid = {{ uuid }} + register: job_state + +# Set Benchmark as Complete when all jobs succeed +- name: Setting benchmark state - Complete + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Complete + complete: true + when: job_state|json_query('resources[].status[].conditions[]')|selectattr('type', 'equalto', 'Complete')|list + +# Set Benchmark as failed when one of the jobs fails +- name: Setting benchmark state - Failed + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Failed + complete: true + when: job_state|json_query('resources[].status[].conditions[]')|selectattr('type', 'equalto', 'Failed')|list + +- name: Get current state + k8s_info: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: '{{ ansible_operator_meta.name }}' + namespace: '{{ operator_namespace }}' + register: benchmark_state diff --git a/roles/benchmark_state/tasks/failure.yml b/roles/benchmark_state/tasks/failure.yml new file mode 100644 index 000000000..cdbc1efea --- /dev/null +++ b/roles/benchmark_state/tasks/failure.yml @@ -0,0 +1,18 @@ +--- +# Removing {{ ansible_failed_task}} and {{ ansible_failed_result }} as they are being very unstable across ansible releases. +# Related issues: +# - https://stackoverflow.com/questions/41104241/ansible-failed-task-ansible-failed-result-variables-are-undefined-when-res +# - https://github.com/ansible/ansible/issues/64789 +- name: Failure State + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + uuid: "{{ uuid }}" + complete: True + state: Failed + message: "Benchmark failed, please check the logs for more details" + suuid: "{{ trunc_uuid }}" + metadata: "not collected" diff --git a/roles/benchmark_state/tasks/set_state.yml b/roles/benchmark_state/tasks/set_state.yml new file mode 100644 index 000000000..936f47a4d --- /dev/null +++ b/roles/benchmark_state/tasks/set_state.yml @@ -0,0 +1,17 @@ +--- +- name: Setting benchmark state - Building + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "{{ state }}" + +- name: Get current benchmark state + k8s_info: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: '{{ ansible_operator_meta.name }}' + namespace: '{{ operator_namespace }}' + register: benchmark_state diff --git a/roles/byowl/tasks/main.yml b/roles/byowl/tasks/main.yml index c9a3b7ce8..b161868cb 100644 --- a/roles/byowl/tasks/main.yml +++ b/roles/byowl/tasks/main.yml @@ -1,69 +1,8 @@ --- +- set_fact: + resources: + - "{{ role_path }}/templates/workload.yml" -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Update current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- block: - - - name: Start byowl job - k8s: - state: present - definition: "{{ lookup('template', 'workload.yml') | from_yaml }}" - when: workload_args.kind is not defined - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - complete: false - - when: resource_state.resources[0].status.state == "Starting" - -- block: - - - name: Waiting for pods to complete.... - k8s_facts: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = byowl-{{ trunc_uuid }} - register: client_pods - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "workload_args.clients == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" - - when: resource_state.resources[0].status.state == "Running" +- name: Generic workload - byowl + include_role: + name: generic_workload diff --git a/roles/byowl/templates/workload.yml b/roles/byowl/templates/workload.yml index e66dafebc..834b12509 100755 --- a/roles/byowl/templates/workload.yml +++ b/roles/byowl/templates/workload.yml @@ -1,18 +1,31 @@ --- +kind: Job apiVersion: batch/v1 -kind: "job" metadata: name: "byowl-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: byowl-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.specoptions is defined %} +{% for label, value in workload_args.specoptions.items() %} + {{ label }}: {{ value | to_json }} +{% endfor %} {% endif %} containers: - name: byowl @@ -20,7 +33,12 @@ spec: args: ["{{ workload_args.commands }}"] image: "{{ workload_args.image }}" imagePullPolicy: Always - restartPolicy: OnFailure +{% if workload_args.containeroptions is defined %} +{% for label, value in workload_args.containeroptions.items() %} + {{ label }}: {{ value | to_json }} +{% endfor %} +{% endif %} + restartPolicy: Never {% if workload_args.nodeselector is defined %} nodeSelector: {% for label, value in workload_args.nodeselector.items() %} diff --git a/resources/kernel-cache-drop-clusterrole.yaml b/roles/ceph_osd_cache_drop/ocs-cache-drop-clusterrole.yaml similarity index 72% rename from resources/kernel-cache-drop-clusterrole.yaml rename to roles/ceph_osd_cache_drop/ocs-cache-drop-clusterrole.yaml index 4fd0acd22..4d61914ec 100644 --- a/resources/kernel-cache-drop-clusterrole.yaml +++ b/roles/ceph_osd_cache_drop/ocs-cache-drop-clusterrole.yaml @@ -1,7 +1,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: benchmark-operator + name: benchmark-operator-openshift-storage rules: - apiGroups: - '' @@ -22,12 +22,12 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: benchmark-operator + name: benchmark-operator-openshift-storage roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: benchmark-operator + name: benchmark-operator-openshift-storage subjects: - kind: ServiceAccount name: benchmark-operator - namespace: my-ripsaw + namespace: openshift-storage diff --git a/resources/rook_ceph_drop_cache_pod.yaml b/roles/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml similarity index 58% rename from resources/rook_ceph_drop_cache_pod.yaml rename to roles/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml index a4ec8d5c1..b7ffaec7f 100644 --- a/resources/rook_ceph_drop_cache_pod.yaml +++ b/roles/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml @@ -2,21 +2,33 @@ apiVersion: v1 kind: Pod metadata: name: rook-ceph-osd-cache-drop - namespace: rook-ceph + # if you are using upstream rook-ceph, change the next line + #namespace: "{{ rook_ceph_namespace }}" + namespace: "openshift-storage" + labels: + app: rook-ceph-osd-cache-drop spec: containers: - name: rook-ceph-osd-cache-drop - image: quay.io/cloud-bulldozer/ceph-cache-dropper:latest + image: "quay.io/cloud-bulldozer/ceph-cache-dropper:latest" imagePullPolicy: Always command: [/bin/sh, -c] args: - - cd /opt/bohica/ceph-cache-dropper; python3 ./osd-cache-drop-websvc.py + - cd /opt/ceph_cache_drop; python3 ./osd-cache-drop-websvc.py env: - - name: ROOK_ADMIN_SECRET + - name: ROOK_CEPH_USERNAME valueFrom: secretKeyRef: - key: admin-secret name: rook-ceph-mon + key: ceph-username + - name: ROOK_CEPH_SECRET + valueFrom: + secretKeyRef: + name: rook-ceph-mon + key: ceph-secret + - name: ceph_cache_drop_port + #value: "{{ ceph_cache_drop_svc_port }}" + value: "9457" securityContext: privileged: true volumeMounts: diff --git a/roles/ceph_osd_cache_drop/tasks/main.yml b/roles/ceph_osd_cache_drop/tasks/main.yml new file mode 100644 index 000000000..aecda4722 --- /dev/null +++ b/roles/ceph_osd_cache_drop/tasks/main.yml @@ -0,0 +1,130 @@ +--- +- debug: + msg: "in ceph OSD cache dropper role" + +# First ensure that toolbox pod is running + +- name: check for existing rook-ceph cluster + k8s_info: + kind: Pod + namespace: "{{ rook_ceph_namespace }}" + label_selectors: + - app = rook-ceph-mgr + register: rook_ceph_mgr_exists + +- name: generate boolean for namespace existence + set_fact: + rook_ceph_cluster_exists: "{{ rook_ceph_mgr_exists.resources | length > 0 }}" + +- debug: + var: rook_ceph_cluster_exists + +- fail: + msg: "You are asking for Ceph cache with drop_cache_rook_ceph: true in CR, but there is no ceph cluster!" + when: not rook_ceph_cluster_exists + +#- name: get pod info of pre-existing ceph toolbox pod +# k8s_info: +# kind: Pod +# label_selectors: +# - app = rook-ceph-tools +# namespace: "{{ rook_ceph_namespace }}" +# register: ceph_toolbox_already_exists + +#- debug: +# var: ceph_toolbox_already_exists + +# FIXME: next commented-out task gets 403 unprivileged error +# workaround is to issue oc command (with admin privs) to do it +# oc patch OCSInitialization ocsinit -n openshift-storage --type json --patch +# '[{ "op": "replace", "path": "/spec/enableCephTools", "value": true }]' + +#- name: ensure Ceph OSD toolbox pod is started +# k8s: +# namespace: "{{ rook_ceph_namespace }}" +# name: ocsinit +# kind: OCSInitialization +# state: present +# apply: true +# resource_definition: +# spec: +# enableCephTools: true +# when: ceph_toolbox_already_exists.resources | length == 0 + +#- name: wait for toolbox pod to start +# shell: "python3 /opt/ansible/roles/ceph_osd_cache_drop/wait_for_pod.py 30 {{ rook_ceph_namespace }} ceph-toolbox" +# when: ceph_toolbox_already_exists.resources | length == 0 + +- name: get pod name of running ceph toolbox pod + k8s_info: + kind: Pod + label_selectors: + - app = rook-ceph-tools + namespace: "{{ rook_ceph_namespace }}" + register: ceph_toolbox + +#- debug: +# var: ceph_toolbox + +# this next var is referenced by the YAML that creates the cache dropper pod + +- name: put pod id into a var + set_fact: + rook_ceph_toolbox_pod: "{{ ceph_toolbox.resources[0].metadata.name }}" + +- debug: + var: rook_ceph_toolbox_pod + + +# now ensure that ceph cache dropper pod is started and we have its IP + +#- name: get pre-existing ceph cache dropper pod +# k8s_info: +# kind: Pod +# label_selectors: +# - app = rook-ceph-osd-cache-drop +# namespace: "{{ rook_ceph_namespace }}" +# register: drop_pod_already_exists + +#- debug: +# var: drop_pod_already_exists + +# FIXME: this gets 403 unprivileged error +# workaround is to issue oc create -f command (with admin privs) to do it +# you must substitute a real value for the jinja2 var in this template first + +#- name: start ceph OSD cache dropper +# k8s: +# definition: "{{ lookup('template', '/opt/ansible/roles/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml') | from_yaml }}" +# register: ceph_cache_dropper_start +# when: drop_pod_already_exists.resources | length == 0 +# +#- name: wait for cache dropper to start +# shell: "python3 /opt/ansible/roles/ceph_osd_cache_drop/wait_for_pod.py 30 {{ rook_ceph_namespace }} rook-ceph-osd-cache-drop" +# when: rook_ceph_cluster_exists and drop_pod_already_exists.resources | length == 0 + +- name: get cache dropper pod + k8s_info: + kind: Pod + label_selectors: + - app = rook-ceph-osd-cache-drop + namespace: "{{ rook_ceph_namespace }}" + register: ceph_osd_cache_drop_pod + +#- debug: +# var: ceph_osd_cache_drop_pod + +- name: put ip into a var + set_fact: + ceph_osd_cache_drop_pod_ip: "{{ ceph_osd_cache_drop_pod.resources[0].status.podIP }}" + +- debug: + var: ceph_osd_cache_drop_pod_ip + +- name: test IP + shell: "curl http://{{ ceph_osd_cache_drop_pod_ip }}:{{ ceph_cache_drop_svc_port }}/" + register: drop_pod_test + +#- debug: +# var: drop_pod_test + diff --git a/roles/ceph_osd_cache_drop/vars/main.yml b/roles/ceph_osd_cache_drop/vars/main.yml new file mode 100644 index 000000000..517af5d01 --- /dev/null +++ b/roles/ceph_osd_cache_drop/vars/main.yml @@ -0,0 +1,3 @@ +# when debugging set this var, for example: +#ceph_cache_drop_image: quay.io/bengland2/ceph-cache-dropper:debug + diff --git a/roles/ceph_osd_cache_drop/wait_for_pod.py b/roles/ceph_osd_cache_drop/wait_for_pod.py new file mode 100644 index 000000000..297631cd4 --- /dev/null +++ b/roles/ceph_osd_cache_drop/wait_for_pod.py @@ -0,0 +1,75 @@ +#!/usr/bin/python3 +# wait_for_pod.py - +# wait until toolbox pod and web service pod is in Running state for specified deploy +# see usage() for input params + +from kubernetes import client, config +from sys import argv, exit +from os import getenv +import time + +NOTOK = 1 +# may have to adjust this based on number of pods +poll_interval = 2.0 + +def usage(msg): + print('ERROR: %s' % msg) + print('usage: wait_for_pod.py timeout namespace pod-name-pattern') + exit(NOTOK) + + +# parse command line + +if len(argv) < 4: + usage('too few parameters') + +timeout_str = argv[1] +ns = argv[2] +pod_name_pattern = argv[3] + +# show 'em what we parsed + +print('timeout if pods not seen in %s sec' % timeout_str) +print('namespace: %s' % ns) +print('pod name pattern: %s' % pod_name_pattern) + +timeout = int(timeout_str) +if timeout <= poll_interval: + usage('timeout %d must be greater than poll interval %d' % + (timeout, poll_interval)) + +# wait for pods + +# cannot do this from inside cluster pod: config.load_kube_config() +if getenv('KUBECONFIG'): + config.load_kube_config() +else: + config.load_incluster_config() +v1 = client.CoreV1Api() + +print('waiting for pod...') +start_time = time.time() +matching_pods = 0 + +while True: + time.sleep(poll_interval) + now_time = time.time() + delta_time = now_time - start_time + if delta_time > timeout: + break + + ret = v1.list_pod_for_all_namespaces(watch=False) + matching_pods = 0 + for i in ret.items: + if i.metadata.namespace == ns and \ + i.metadata.generate_name.__contains__ (pod_name_pattern) and \ + i.status.phase == 'Running': + matching_pods += 1 + if matching_pods >= 1: + break + +if delta_time > timeout: + usage('timeout waiting for pods to reach running state') + +if matching_pods != 1: + usage('expected 1 pod, found %d pods' % matching_pods) diff --git a/roles/cerberus/tasks/main.yml b/roles/cerberus/tasks/main.yml deleted file mode 100644 index d7e50d607..000000000 --- a/roles/cerberus/tasks/main.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- name: Get status from Cerberus - uri: - url: "{{ cerberus_url }}" - return_content: yes - register: result - -- name: Update status if unhealthy - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - cerberus: "Connected" - state: "Error" - when: result.content == "False" - -- name: Update status if healthy - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - cerberus: "Connected" - when: result.content == "True" - diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index 1f1d60fd1..13f88b3c0 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -1,11 +1,11 @@ # common tasks across multiple roles go here - block: - name: Get Network Policy - k8s_facts: + k8s_info: kind: NetworkPolicy api_version: networking.k8s.io/v1 namespace: '{{ operator_namespace }}' - name: "{{ meta.name }}-networkpolicy-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-networkpolicy-{{ trunc_uuid }}" register: network_policy - name: Create Network policy if enabled diff --git a/roles/common/templates/networkpolicy.yml.j2 b/roles/common/templates/networkpolicy.yml.j2 index 6267d2196..f1b2703a0 100644 --- a/roles/common/templates/networkpolicy.yml.j2 +++ b/roles/common/templates/networkpolicy.yml.j2 @@ -1,14 +1,14 @@ kind: NetworkPolicy apiVersion: networking.k8s.io/v1 metadata: - name: "{{ meta.name }}-networkpolicy-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-networkpolicy-{{ trunc_uuid }}" namespace: '{{ operator_namespace }}' spec: podSelector: matchLabels: - type: "{{ meta.name }}-bench-server-{{ trunc_uuid }}" + type: "{{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }}" ingress: - from: - podSelector: matchLabels: - type: "{{ meta.name }}-bench-client-{{ trunc_uuid }}" + type: "{{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }}" diff --git a/roles/cyclictest/tasks/main.yml b/roles/cyclictest/tasks/main.yml index 86c426944..881967181 100644 --- a/roles/cyclictest/tasks/main.yml +++ b/roles/cyclictest/tasks/main.yml @@ -1,31 +1,4 @@ --- - -- name: Get current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state if changed - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - name: Add cyclic test scripts @@ -34,70 +7,26 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' + name: 'cyclictest-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: cyclictest.sh: "{{ lookup('file', 'cyclictest.sh') }}" functions.sh: "{{ lookup('file', 'functions.sh') }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: ConfigMaps Created - complete: false - - when: resource_state.resources[0].status.state == "Building" - -- block: - name: Start cyclitest job k8s: state: present definition: "{{ lookup('template', 'cyclictestjob.yaml') | from_yaml }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Benchmark Running - complete: false - - when: resource_state.resources[0].status.state == "ConfigMaps Created" - -- block: - - - name: Wait for benchmark to complete - k8s_facts: - kind: Job - api_version: batch/v1 - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' - namespace: "{{ operator_namespace }}" - register: cyclictest_status - - - name: Set Complete Status to benchmark - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Benchmark Complete - complete: true - when: cyclictest_status.resources[0].status.succeeded is defined and (cyclictest_status.resources[0].status.succeeded | int) > 0 + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - - name: Set failed state to benchmark - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Failed - complete: true - when: cyclictest_status.resources[0].status.failed is defined and (cyclictest_status.resources[0].status.failed | int) > 0 + when: benchmark_state.resources[0].status.state == "Building" - when: resource_state.resources[0].status.state == "Benchmark Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/cyclictest/templates/cyclictestjob.yaml b/roles/cyclictest/templates/cyclictestjob.yaml index 1652f68a4..a9a421523 100644 --- a/roles/cyclictest/templates/cyclictestjob.yaml +++ b/roles/cyclictest/templates/cyclictestjob.yaml @@ -1,28 +1,40 @@ apiVersion: batch/v1 kind: Job metadata: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "cyclictest-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: template: metadata: labels: app: cyclictest-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} ttlSecondsAfterFinished: 600 +{% if workload_args.node_selector is defined %} + nodeSelector: + '{{ workload_args.node_selector.split("=")[0] }}': '{{ workload_args.node_selector.split("=")[1] }}' +{% endif %} containers: - name: cyclictest image: {{ workload_args.image | default('quay.io/cloud-bulldozer/cyclictest:latest') }} command: ["/bin/sh", "-c"] - args: ["run_snafu --tool cyclictest -p /tmp/cyclictest.sh -u {{ uuid }} --user {{test_user | default("ripsaw")}}"] - imagePullPolicy: Always -{% if workload_args.nodeselector is defined %} - nodeSelector: - "{{ workload_args.nodeselector }}" + args: + - run_snafu --tool cyclictest -p /tmp/cyclictest.sh -u {{ uuid }} --user {{test_user | default("ripsaw")}} +{% if workload_args.debug is defined and workload_args.debug %} + -v {% endif %} + ; + imagePullPolicy: Always resources: requests: memory: {{ workload_args.pod.requests.memory }} @@ -37,11 +49,9 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es - value: "{{ elasticsearch.server }}" - - name: es_port - value: "{{ elasticsearch.port }}" + value: "{{ elasticsearch.url}}" - name: es_index value: "{{ elasticsearch.index_name | default("ripsaw-cyclictest") }}" - name: parallel @@ -80,8 +90,7 @@ spec: path: /dev/cpu_dma_latency - name: cyclictest-volume configMap: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "cyclictest-{{ trunc_uuid }}" defaultMode: 0555 restartPolicy: Never - serviceAccountName: benchmark-operator {% include "metadata.yml.j2" %} diff --git a/roles/fio_distributed/tasks/main.yml b/roles/fio_distributed/tasks/main.yml index 484ce865a..e9913a007 100644 --- a/roles/fio_distributed/tasks/main.yml +++ b/roles/fio_distributed/tasks/main.yml @@ -1,35 +1,9 @@ --- -- name: Get current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - If it has changed - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - name: Provide path for fio - PV set_fact: - fio_path: /mnt/pvc - when: workload_args.storageclass is defined + fio_path: /dev/xvda + when: workload_args.storageclass is defined - name: Provide path for fio - No PV set_fact: @@ -39,6 +13,10 @@ - name: Generate fio test k8s: definition: "{{ lookup('template', 'configmap.yml.j2') | from_yaml }}" + + - name: Generate fio prefill job + k8s: + definition: "{{ lookup('template', 'prefill-configmap.yml.j2') | from_yaml }}" - name: Create PVC(s) k8s: @@ -48,12 +26,11 @@ metadata: name: "claim-{{ item }}-{{ trunc_uuid }}" namespace: '{{ operator_namespace }}' - annotations: - volume.beta.kubernetes.io/storage-class: "{{ workload_args.storageclass }}" spec: accessModes: - "{{ workload_args.pvcaccessmode | default('ReadWriteOnce') }}" volumeMode: "{{ workload_args.pvcvolumemode | default('Filesystem') }}" + storageClassName: "{{ workload_args.storageclass }}" resources: requests: storage: "{{ workload_args.storagesize }}" @@ -72,18 +49,15 @@ when: workload_args.servers|default('1')|int > 0 and (workload.args.kind | default('pod')) == "vm" with_sequence: start=1 count={{ workload_args.servers|default('1')|int }} - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: StartingServers - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: StartingServers - when: resource_state.resources[0].status.state == "Building" + when: benchmark_state.resources[0].status.state == "Building" -- block : +- block: - name: Capture pod list k8s_info: @@ -94,20 +68,23 @@ - app = fio-benchmark-{{ trunc_uuid }} register: server_pods - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: StartingClient - complete: false - when : "workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status.podIP')|length) and workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length) and resource_state.resources[0].status.state == 'StartingServers'" + - name: Check IP addresses + set_fact: + check_ip: true + when: "( server_pods is defined ) and ( server_pods is mapping ) and (workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status.podIP')|length))" + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: StartingClient + when: "check_ip|default('false')|bool and workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status.podIP')|length) and workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length) and benchmark_state.resources[0].status.state == 'StartingServers'" - name: Create IP list and nodes set_fact: pod_details: "{{ pod_details|default({}) | combine({item.status.podIP: item.spec.nodeName}) }}" with_items: "{{ server_pods.resources }}" + when: "check_ip|default('false')|bool" when: (workload.args.kind | default('pod')) == "pod" @@ -116,26 +93,24 @@ - name: Capture pod list k8s_info: kind: VirtualMachineInstance - api_version: kubevirt.io/v1alpha3 + api_version: kubevirt.io/v1 namespace: '{{ operator_namespace }}' label_selectors: - app = fio-benchmark-{{ trunc_uuid }} register: server_pods - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: StartingClient - complete: false - when : "workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status.interfaces[].ipAddress')|length) and workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length) and resource_state.resources[0].status.state == 'StartingServers'" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: StartingClient + when : "workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status.interfaces[].ipAddress')|length) and workload_args.servers|default('1')|int == (server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length) and benchmark_state.resources[0].status.state == 'StartingServers'" - name: Create IP list and nodes set_fact: pod_details: "{{ pod_details|default({}) | combine({item.status.interfaces[0].ipAddress: item.status.nodeName}) }}" with_items: "{{ server_pods.resources }}" + when: benchmark_state.resources[0].status.state == "StartingClient" when: (workload.args.kind | default('pod')) == "vm" @@ -161,40 +136,51 @@ port: 8765 when: (workload.args.kind | default('pod')) == "vm" with_items: "[{% for ip in pod_details.keys() %}'{{ ip }}', {% endfor %}]" - - - name: Start FIO Client + when: benchmark_state.resources[0].status.state == "StartingClient" + +- block: + - name: Run Prefill Job k8s: - definition: "{{ lookup('template', 'client.yaml') | from_yaml }}" + definition: "{{ lookup('template', 'prefill-client.yaml.j2') | from_yaml }}" + register: fio_prefill_pod + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Prefilling + when: workload.args.prefill and benchmark_state.resources[0].status.state == "StartingClient" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" +- block: + - name: wait for db creation job to finish + k8s_info: + kind: Job + api_version: batch/v1 + name: 'fio-prefill-{{ trunc_uuid }}' namespace: "{{ operator_namespace }}" - status: - state: Running - complete: false + register: fio_prefill_pod - when: resource_state.resources[0].status.state == "StartingClient" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: StartBenchmark + when: fio_prefill_pod | json_query('resources[].status.succeeded') + when: workload.args.prefill and benchmark_state.resources[0].status.state == "Prefilling" - block: - - name: Capture Client pod list - k8s_info: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = fiod-client-{{ trunc_uuid }} - register: client_pods - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when : "client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length > 0 " + - name: Start FIO Client + k8s: + definition: "{{ lookup('template', 'client.yaml') | from_yaml }}" - when: resource_state.resources[0].status.state == "Running" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running + when: (benchmark_state.resources[0].status.state == "StartBenchmark" and workload.args.prefill) or (benchmark_state.resources[0].status.state == "StartingClient" and not workload.args.prefill) + +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/fio_distributed/templates/client.yaml b/roles/fio_distributed/templates/client.yaml index 551098952..51df759a2 100644 --- a/roles/fio_distributed/templates/client.yaml +++ b/roles/fio_distributed/templates/client.yaml @@ -5,11 +5,22 @@ metadata: name: 'fio-client-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: + benchmark-uuid: {{ uuid }} app: fiod-client-{{ trunc_uuid }} +{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -22,7 +33,7 @@ spec: - name: uuid value: "{{ uuid }}" - name: test_user - value: "{{ test_user | default("ripsaw") }}" + value: "{{ test_user | default('ripsaw') }}" - name: clustername value: "{{ clustername }}" - name: ceph_cache_drop_pod_ip @@ -31,11 +42,17 @@ spec: value: "{{ kcache_drop_pod_ips | default() }}" - name: KCACHE_DROP_PORT_NUM value: "{{ kernel_cache_drop_svc_port }}" -{% if elasticsearch is defined %} +{% if ceph_osd_cache_drop_pod_ip is defined %} + - name: ceph_osd_cache_drop_pod_ip + value: "{{ ceph_osd_cache_drop_pod_ip }}" + - name: CEPH_CACHE_DROP_PORT_NUM + value: "{{ ceph_cache_drop_svc_port }}" +{% endif %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index - value: "{{ elasticsearch.index_name | default("ripsaw-fio") }}" + value: "{{ elasticsearch.index_name | default('ripsaw-fio') }}" - name: es_verify_cert value: "{{ elasticsearch.verify_cert | default(true) }}" - name: parallel @@ -54,16 +71,6 @@ spec: command: ["/bin/sh", "-c"] args: - "cat /tmp/host/hosts; -{% if workload_args.prefill is defined and workload_args.prefill is sameas true %} - echo ***************Prefill*****************; - cat /tmp/fio/fiojob-prefill; - mkdir -p /tmp/fiod-{{ uuid }}/fiojob-prefill; - fio --client=/tmp/host/hosts /tmp/fio/fiojob-prefill --output-format=json; -{% if workload_args.post_prefill_sleep is defined %} - sleep {{ workload_args.post_prefill_sleep }}; -{% endif %} - echo ***********End of Prefill*************; -{% endif %} {% if workload_args.bsrange is defined %} {% set loopvar = workload_args.bsrange %} {% elif workload_args.bs is defined %} @@ -72,7 +79,12 @@ spec: {% for numjobs in workload_args.numjobs %} {% for i in loopvar %} {% for job in workload_args.jobs %} - cat /tmp/fio/fiojob-{{job}}-{{i}}-{{numjobs}}; mkdir -p /tmp/fiod-{{uuid}}/fiojob-{{job}}-{{i}}-{{numjobs}}; run_snafu -t fio -H /tmp/host/hosts -j /tmp/fio/fiojob-{{job}}-{{i}}-{{numjobs}} -s {{workload_args.samples}} -d /tmp/fiod-{{ uuid }}/fiojob-{{job}}-{{i}}-{{numjobs}}; + cat /tmp/fio/fiojob-{{job}}-{{i}}-{{numjobs}}; mkdir -p /tmp/fiod-{{uuid}}/fiojob-{{job}}-{{i}}-{{numjobs}}; + run_snafu -t fio -H /tmp/host/hosts -j /tmp/fio/fiojob-{{job}}-{{i}}-{{numjobs}} -s {{workload_args.samples}} -d /tmp/fiod-{{ uuid }}/fiojob-{{job}}-{{i}}-{{numjobs}} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; {% if workload_args.fio_json_to_log is defined and workload_args.fio_json_to_log is sameas true %} for fio_sample in $(seq 1 {{workload_args.samples}}); do echo START_FIO_JSON_OUTPUT_fiod-{{uuid}}_fiojob-{{job}}-{{i}}-{{numjobs}}_SAMPLE_$fio_sample; diff --git a/roles/fio_distributed/templates/configmap.yml.j2 b/roles/fio_distributed/templates/configmap.yml.j2 index f57de4649..95d15b4f7 100644 --- a/roles/fio_distributed/templates/configmap.yml.j2 +++ b/roles/fio_distributed/templates/configmap.yml.j2 @@ -5,10 +5,16 @@ metadata: name: fio-test-{{ trunc_uuid }} namespace: '{{ operator_namespace }}' data: +# FIXME: I don't think prefill works correctly for a list of numjobs values, only for 1 of them +{% for numjobs in workload_args.numjobs %} {% if workload_args.prefill is defined and workload_args.prefill is sameas true %} fiojob-prefill: | [global] +{% if workload_args.pvcvolumemode is defined and workload_args.pvcvolumemode == "Block" %} + filename={{fio_path}} +{% else %} directory={{fio_path}} +{% endif %} filename_format=f.\$jobnum.\$filenum clocksource=clock_gettime kb_base=1000 @@ -22,7 +28,7 @@ data: {% endif %} iodepth=1 direct=1 - numjobs=1 + numjobs={{numjobs}} [write] rw=write @@ -40,19 +46,26 @@ data: {% set loopvar = workload_args.bs %} {% set loopvar_str = 'bs' %} {% endif %} -{% for numjobs in workload_args.numjobs %} {% for i in loopvar %} {% for job in workload_args.jobs %} fiojob-{{job}}-{{i}}-{{numjobs}}: | [global] +{% if workload_args.pvcvolumemode is defined and workload_args.pvcvolumemode == "Block" %} + filename={{fio_path}} +{% else %} directory={{fio_path}} +{% endif %} filename_format=f.\$jobnum.\$filenum write_bw_log=fio write_iops_log=fio write_lat_log=fio +{% if workload_args.log_sample_rate is defined %} + log_avg_msec={{ workload_args.log_sample_rate }} +{% endif %} +{% if workload.log_hist_msec is defined %} write_hist_log=fio - log_avg_msec={{workload_args.log_sample_rate}} - log_hist_msec={{workload_args.log_sample_rate}} + log_hist_msec={{ workload_args.log_hist_msec }} +{% endif %} clocksource=clock_gettime kb_base=1000 unit_base=8 diff --git a/roles/fio_distributed/templates/prefill-client.yaml.j2 b/roles/fio_distributed/templates/prefill-client.yaml.j2 new file mode 100644 index 000000000..d33af6363 --- /dev/null +++ b/roles/fio_distributed/templates/prefill-client.yaml.j2 @@ -0,0 +1,58 @@ +--- +kind: Job +apiVersion: batch/v1 +metadata: + name: 'fio-prefill-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' +spec: + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} + template: + metadata: + labels: + benchmark-uuid: prefill-{{ uuid }} + app: fiod-prefill-{{ trunc_uuid }} +{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} + containers: + - name: fio-client + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/fio:latest') }} + imagePullPolicy: Always + command: ["/bin/sh", "-c"] + args: + - "cat /tmp/host/hosts; + echo ***************Prefill*****************; + cat /tmp/fio/fiojob-prefill; + mkdir -p /tmp/fiod-{{ uuid }}/fiojob-prefill; + fio --client=/tmp/host/hosts /tmp/fio/fiojob-prefill --output-format=json; +{% if workload_args.post_prefill_sleep is defined %} + sleep {{ workload_args.post_prefill_sleep }}; +{% endif %} + echo ***********End of Prefill*************" + volumeMounts: + - name: fio-volume + mountPath: "/tmp/fio" + - name: host-volume + mountPath: "/tmp/host" + volumes: + - name: fio-volume + configMap: + name: "fio-prefill-{{ trunc_uuid }}" + defaultMode: 0777 + - name: host-volume + configMap: + name: "fio-hosts-{{ trunc_uuid }}" + defaultMode: 0777 + restartPolicy: Never +{% include "metadata.yml.j2" %} diff --git a/roles/fio_distributed/templates/prefill-configmap.yml.j2 b/roles/fio_distributed/templates/prefill-configmap.yml.j2 new file mode 100644 index 000000000..bd368cb54 --- /dev/null +++ b/roles/fio_distributed/templates/prefill-configmap.yml.j2 @@ -0,0 +1,42 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fio-prefill-{{ trunc_uuid }} + namespace: '{{ operator_namespace }}' +data: +# FIXME: I don't think prefill works correctly for a list of numjobs values, only for 1 of them +{% for numjobs in workload_args.numjobs %} +{% if workload_args.prefill is defined and workload_args.prefill is sameas true %} + fiojob-prefill: | + [global] +{% if workload_args.pvcvolumemode is defined and workload_args.pvcvolumemode == "Block" %} + filename={{fio_path}} +{% else %} + directory={{fio_path}} +{% endif %} + filename_format=f.\$jobnum.\$filenum + clocksource=clock_gettime + kb_base=1000 + unit_base=8 + ioengine=libaio + size={{workload_args.filesize}} +{% if workload_args.prefill_bs is defined %} + bs={{workload_args.prefill_bs}} +{% else %} + bs=4096KiB +{% endif %} + iodepth=1 + direct=1 + numjobs={{numjobs}} + + [write] + rw=write + create_on_open=1 + fsync_on_close=1 +{% if workload_args.cmp_ratio is defined %} + buffer_compress_percentage={{ workload_args.cmp_ratio }} + buffer_pattern=0xdeadface +{% endif %} +{% endif %} +{% endfor %} diff --git a/roles/fio_distributed/templates/server_vm.yml.j2 b/roles/fio_distributed/templates/server_vm.yml.j2 index 1da5d7e66..8d42170b1 100644 --- a/roles/fio_distributed/templates/server_vm.yml.j2 +++ b/roles/fio_distributed/templates/server_vm.yml.j2 @@ -1,25 +1,44 @@ --- -apiVersion: kubevirt.io/v1alpha3 +apiVersion: kubevirt.io/v1 kind: VirtualMachineInstance metadata: name: 'fio-server-{{item | string}}-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' labels: app: fio-benchmark-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: domain: cpu: cores: {{ workload_args.vm_cores | default(1) }} devices: disks: - - name: registrydisk - - name: cloudinitdisk + - disk: + bus: {{ workload_args.vm_bus | default('virtio') }} + name: registrydisk + - disk: + bus: {{ workload_args.vm_bus | default('virtio') }} + name: cloudinitdisk {% if workload_args.storageclass is defined or workload_args.hostpath is defined %} - disk: - bus: virtio + bus: {{ workload_args.vm_bus | default('virtio') }} name: data-volume serial: data +{% else %} + - disk: + bus: {{ workload_args.vm_bus | default('virtio') }} + name: emptydisk + serial: data {% endif %} resources: requests: @@ -27,37 +46,23 @@ spec: volumes: - name: registrydisk containerDisk: - image: {{ workload_args.vm_image | default('kubevirt/fedora-cloud-container-disk-demo:latest') }} + image: {{ workload_args.vm_image | default('quay.io/kubevirt/fedora-container-disk-images:latest') }} - name: cloudinitdisk cloudInitNoCloud: userData: |- #cloud-config - password: fedora + password: ripsaw chpasswd: { expire: False } bootcmd: -{% if workload_args.storageclass is defined - or workload_args.hostpath is defined %} - "mkdir -p {{ fio_path }} || true" - - mkfs.ext4 /dev/disk/by-id/virtio-data - - "mount /dev/disk/by-id/virtio-data {{ fio_path }}" -{% endif %} + - "[ -e /dev/disk/by-id/*data ] && disk=$(shopt -s nullglob; basename /dev/disk/by-id/*data) && mkfs.ext4 /dev/disk/by-id/$disk && mount /dev/disk/by-id/$disk {{ fio_path }}" runcmd: -{% if workload_args.vm_image is undefined %} - - dnf install -y fio wget curl python3 python3-pip -{% endif %} - - cd /tmp -{% if workload_args.metadata is defined - and (workload_args.metadata.collection|default(false)) - and (workload_args.metadata.targeted|default(true)) %} - - export ES_SERVER={{ elasticsearch.url }} -{% if workload_args.vm_image is undefined %} - - wget https://raw.githubusercontent.com/cloud-bulldozer/bohica/master/stockpile-wrapper/stockpile-wrapper.py - - pip3 install elasticsearch-dsl openshift kubernetes redis -{% endif %} - # TODO: start stockpile-wrapper script - python3 stockpile-wrapper.py -s {{ elasticsearch.url }} -u {{ uuid }} -n "$my_node_name" -N "$my_pod_name" --redisip {{ bo.resources[0].status.podIP }} --redisport 6379 --force -{% endif %} - - fio --server + - dnf install -y podman + - "img=`podman create {{ workload_args.image | default('quay.io/cloud-bulldozer/fio:latest') }}`" + - fs=`podman mount $img` + - "mkdir -p $fs/{{ fio_path }} || true" + - "mount -o bind /{{ fio_path }} $fs/{{ fio_path }}" + - "chroot $fs bash -c 'cd /{{ fio_path }}; fio --server'" {% if workload_args.storageclass is defined %} - name: data-volume persistentVolumeClaim: @@ -65,7 +70,17 @@ spec: {% elif workload_args.hostpath is defined %} - name: data-volume hostDisk: - path: {{ workload_args.hostpath }} + path: "{{ workload_args.hostpath }}/fio-server-{{item | string}}-{{ trunc_uuid }}" capacity: {{ workload_args.storagesize | default("5Gi") }} type: DiskOrCreate +{% else %} + - name: emptydisk + emptyDisk: + capacity: {{ workload_args.storagesize | default("5Gi") }} {% endif %} +{% if workload_args.nodeselector is defined %} + nodeSelector: +{% for label, value in workload_args.nodeselector.items() %} + {{ label | replace ("_", "-" )}}: {{ value }} +{% endfor %} +{% endif %} diff --git a/roles/fio_distributed/templates/servers.yaml b/roles/fio_distributed/templates/servers.yaml index 213dacc6a..3afc7f567 100644 --- a/roles/fio_distributed/templates/servers.yaml +++ b/roles/fio_distributed/templates/servers.yaml @@ -1,71 +1,84 @@ --- -kind: Job -apiVersion: batch/v1 +kind: Pod +apiVersion: v1 metadata: name: 'fio-server-{{ item }}-benchmark-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + app: fio-benchmark-{{ trunc_uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: - ttlSecondsAfterFinished: 600 - template: - metadata: - labels: - app: fio-benchmark-{{ trunc_uuid }} - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - fio-benchmark-{{ trunc_uuid }} - topologyKey: "kubernetes.io/hostname" + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - fio-benchmark-{{ trunc_uuid }} + topologyKey: "kubernetes.io/hostname" {% if workload_args.runtime_class is defined %} - runtimeClassName: "{{ workload_args.runtime_class }}" + runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} - containers: - - name: fio-server + containers: + - name: fio-server {% if hostpath is defined %} - securityContext: - privileged: true + securityContext: + privileged: true {% endif %} - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/fio:latest') }} - imagePullPolicy: Always - ports: - - containerPort: 8765 - command: ["/bin/sh", "-c"] - args: - - "cd /tmp; fio --server" + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/fio:latest') }} + imagePullPolicy: Always + ports: + - containerPort: 8765 + command: ["/bin/sh", "-c"] + args: + - "cd /tmp; fio --server" {% if workload_args.storageclass is defined or hostpath is defined %} - volumeMounts: - - name: data-volume - mountPath: "{{ fio_path }}" +{% if workload_args.pvcvolumemode is defined and workload_args.pvcvolumemode == "Block" %} + volumeDevices: +{% else %} + volumeMounts: +{% endif %} + - name: data-volume +{% if workload_args.pvcvolumemode is defined and workload_args.pvcvolumemode == "Block" %} + devicePath: "{{ fio_path }}" +{% else %} + mountPath: "{{ fio_path }}" +{% endif %} {% endif %} - restartPolicy: Never - serviceAccountName: benchmark-operator + restartPolicy: Never {% if workload_args.nodeselector is defined %} - nodeSelector: + nodeSelector: {% for label, value in workload_args.nodeselector.items() %} - {{ label | replace ("_", "-" )}}: {{ value }} + {{ label | replace ("_", "-" )}}: {{ value }} {% endfor %} {% endif %} {% if workload_args.tolerations is defined %} - tolerations: - {{ workload_args.tolerations }} + tolerations: + {{ workload_args.tolerations }} {% endif %} {% if workload_args.storageclass is defined %} - volumes: - - name: data-volume - persistentVolumeClaim: - claimName: claim-{{ item }}-{{ trunc_uuid }} + volumes: + - name: data-volume + persistentVolumeClaim: + claimName: claim-{{ item }}-{{ trunc_uuid }} {% elif hostpath is defined %} - volumes: - - name: data-volume - hostPath: - path: {{ hostpath }} - type: DirectoryOrCreate + volumes: + - name: data-volume + hostPath: + path: {{ hostpath }} + type: DirectoryOrCreate {% endif %} -{% include "metadata.yml.j2" %} +{% include "metadata_pod.yml.j2" %} diff --git a/roles/flent/tasks/main.yml b/roles/flent/tasks/main.yml index 998a3f6c5..d167d35f9 100644 --- a/roles/flent/tasks/main.yml +++ b/roles/flent/tasks/main.yml @@ -1,40 +1,4 @@ --- - -- name: Get current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - If it has changed - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- name: Capture operator information - k8s_info: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - name = benchmark-operator - register: bo - - block: - name: Start Server(s) @@ -52,16 +16,13 @@ - type = flent-bench-server-{{ trunc_uuid }} register: server_pods - - name: Update resource state to starting servers - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Servers" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting Servers - when: resource_state.resources[0].status.state == "Building" + when: benchmark_state.resources[0].status.state == "Building" - block: @@ -74,18 +35,15 @@ - type = flent-bench-server-{{ trunc_uuid }} register: server_pods - - name: Update resource state to starting clients - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Clients" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting Clients when: "workload_args.pair|default('1')|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" - when: resource_state.resources[0].status.state == "Starting Servers" + when: benchmark_state.resources[0].status.state == "Starting Servers" - block: @@ -104,117 +62,70 @@ definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" with_items: "{{ server_pods.resources }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Waiting for Clients + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Waiting for Clients - when: resource_state.resources[0].status.state == "Starting Clients" + when: benchmark_state.resources[0].status.state == "Starting Clients" - block: + - name: Get client pod status + k8s_info: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = flent-bench-client-{{ trunc_uuid }} + register: client_pods - block: - - name: Get client pod status - k8s_info: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = flent-bench-client-{{ trunc_uuid }} - register: client_pods - - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Clients Running - when: "workload_args.pair|default('1')|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_pods | json_query('resources[].status.podIP')|length)" - - when: resource_state.resources[0].status.state == "Waiting for Clients" -- block: + - name: Signal workload + command: "redis-cli set start-{{ trunc_uuid }} true" - - name: Signal workload - command: "redis-cli set start true" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running + when: "workload_args.pair|default('1')|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_pods | json_query('resources[].status.podIP')|length)" - - name: Update resource state to Running - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Running" + when: benchmark_state.resources[0].status.state == "Waiting for Clients" - when: resource_state.resources[0].status.state == "Clients Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" - block: - - block: - - name: Waiting for pods to complete.... - k8s_info: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = flent-bench-client-{{ trunc_uuid }} - register: client_pods - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Cleanup - complete: false - when: "workload_args.pair|default('1')|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" - - when: resource_state.resources[0].status.state == "Running" -- block: + - name: Get Server Pods + k8s_info: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = flent-bench-server-{{ trunc_uuid }} + register: server_pods - - block: - - name: Get Server Pods - k8s_info: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = flent-bench-server-{{ trunc_uuid }} - register: server_pods - - - name: Pod names - to clean - set_fact: - clean_pods: | - [ - {% for item in server_pods.resources %} - "{{ item['metadata']['name'] }}", - {% endfor %} - ] - - - name: Cleanup run - k8s: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - state: absent - name: "{{ item }}" - with_items: "{{ clean_pods }}" - when: cleanup - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - - when: resource_state.resources[0].status.state == "Cleanup" + - name: Pod names - to clean + set_fact: + clean_pods: | + [ + {% for item in server_pods.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] + + - name: Cleanup run + k8s: + kind: pod + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_pods }}" + + when: benchmark_state.resources[0].status.complete and cleanup diff --git a/roles/flent/templates/server.yml.j2 b/roles/flent/templates/server.yml.j2 index 31f3754b8..9df8dfbe2 100644 --- a/roles/flent/templates/server.yml.j2 +++ b/roles/flent/templates/server.yml.j2 @@ -1,41 +1,44 @@ --- -kind: Job -apiVersion: batch/v1 +kind: Pod +apiVersion: v1 metadata: name: 'flent-server-{{ item }}-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' -spec: - ttlSecondsAfterFinished: 600 - backoffLimit: 0 - template: - metadata: - labels: - app : flent-bench-server-{{item}}-{{ trunc_uuid }} - type : flent-bench-server-{{ trunc_uuid }} + labels: + app : flent-bench-server-{{item}}-{{ trunc_uuid }} + type : flent-bench-server-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.multus.enabled is sameas true or workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: {% if workload_args.multus.enabled is sameas true %} - annotations: - k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} + k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} +{% endif %} +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} {% endif %} - spec: +spec: {% if workload_args.runtime_class is defined %} - runtimeClassName: "{{ workload_args.runtime_class }}" + runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} {% if workload_args.hostnetwork is sameas true %} - hostNetwork: true - serviceAccountName: benchmark-operator + hostNetwork: true {% endif %} - containers: - - name: benchmark - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/flent:latest') }} + containers: + - name: benchmark + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/flent:latest') }} {% if workload_args.server_resources is defined %} - resources: {{ workload_args.server_resources | to_json }} + resources: {{ workload_args.server_resources | to_json }} {% endif %} - imagePullPolicy: Always - command: ["/bin/sh","-c"] - args: ["netserver -D"] - restartPolicy: OnFailure + imagePullPolicy: Always + command: ["/bin/sh","-c"] + args: ["netserver -D"] + restartPolicy: OnFailure {% if workload_args.pin is sameas true %} - nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_server }}' + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_server }}' {% endif %} -{% include "metadata.yml.j2" %} +{% include "metadata_pod.yml.j2" %} diff --git a/roles/flent/templates/workload.yml.j2 b/roles/flent/templates/workload.yml.j2 index a6022efd5..d1b9af109 100644 --- a/roles/flent/templates/workload.yml.j2 +++ b/roles/flent/templates/workload.yml.j2 @@ -10,14 +10,22 @@ spec: labels: app: flent-bench-client-{{ trunc_uuid }} clientfor: {{ item.metadata.labels.app }} + benchmark-uuid: {{ uuid }} +{% if workload_args.multus.enabled is sameas true or workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: {% if workload_args.multus.enabled is sameas true %} - annotations: - k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.client }} + k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} +{% endif %} +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} {% endif %} spec: {% if workload_args.hostnetwork is sameas true %} hostNetwork : true - serviceAccountName: benchmark-operator {% endif %} affinity: podAntiAffinity: @@ -41,7 +49,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index @@ -67,9 +75,13 @@ spec: export hostnet={{workload_args.hostnetwork}}; cd /tmp/flent-test; while true; do - if [[ $(redis-cli -h {{bo.resources[0].status.podIP}} get start) =~ 'true' ]]; then + if [[ $(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{trunc_uuid}}) =~ 'true' ]]; then {% for test in workload_args.test_types %} - run_snafu --tool flent --ftest {{test}} -u {{ uuid }} -r $h --user {{test_user | default("ripsaw")}}; + run_snafu --tool flent --ftest {{test}} -u {{ uuid }} -r $h --user {{test_user | default("ripsaw")}} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; {% endfor %} else continue; @@ -77,14 +89,14 @@ spec: break; done; - redis-cli -h {{bo.resources[0].status.podIP}} set start false" + redis-cli -h {{bo.resources[0].status.podIP}} set start-{{trunc_uuid}} false" volumeMounts: - name: flent-tmp mountPath: "/tmp/flent-test" volumes: - name: flent-tmp emptyDir: {} - restartPolicy: OnFailure + restartPolicy: Never {% if workload_args.pin is sameas true %} nodeSelector: kubernetes.io/hostname: '{{ workload_args.pin_client }}' diff --git a/roles/fs-drift/tasks/main.yml b/roles/fs-drift/tasks/main.yml index 2faf68df1..dfb80e06e 100644 --- a/roles/fs-drift/tasks/main.yml +++ b/roles/fs-drift/tasks/main.yml @@ -1,31 +1,4 @@ --- - -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Update current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - set_fact: @@ -68,14 +41,6 @@ subscriber: "{{ lookup('template', 'subscriber.py.j2') }}" publisher: "{{ lookup('template', 'publisher.py.j2') }}" - - name: Capture operator information - k8s_facts: - kind: Pod - api_version: v1 - namespace: "{{ operator_namespace }}" - label_selectors: - - name = benchmark-operator - register: bo - name: Run PUBLISHER Pod k8s: @@ -83,7 +48,7 @@ kind: Job apiVersion: batch/v1 metadata: - name: "{{ meta.name }}-fs-drift-publisher-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-fs-drift-publisher-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: ttlSecondsAfterFinished: 600 @@ -94,12 +59,12 @@ spec: containers: - name: publisher-container - image: quay.io/cloud-bulldozer/fs-drift:master + image: "{{ workload_args.image | default('quay.io/cloud-bulldozer/fs-drift:latest') }}" tty: true command: ["/bin/sh", "-c"] workingDir: /root/fs-drift-master/ args: - - python /tmp/publisher {{bo.resources[0].status.podIP}} {{workload_args.worker_pods|default('1')|int}} + - python /tmp/publisher {{bo.resources[0].status.podIP}} {{workload_args.worker_pods|default('1')|int}} {{ trunc_uuid }} volumeMounts: - name: config-volume mountPath: "/tmp" @@ -116,36 +81,15 @@ with_sequence: start=1 count={{workload_args.worker_pods|default('1')|int}} when: workload_args.worker_pods|default('1')|int > 0 - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - complete: false - - when: resource_state.resources[0].status.state == "Starting" - -- block: - - - name: Waiting for pods to complete.... - k8s_facts: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = fs-drift-benchmark-{{ trunc_uuid }} - register: client_pods + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "workload_args.worker_pods|default('1')|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: benchmark_state.resources[0].status.state == "Building" - when: resource_state.resources[0].status.state == "Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/fs-drift/templates/other_parameters.yaml.j2 b/roles/fs-drift/templates/other_parameters.yaml.j2 index 69b580d86..d36352be7 100644 --- a/roles/fs-drift/templates/other_parameters.yaml.j2 +++ b/roles/fs-drift/templates/other_parameters.yaml.j2 @@ -47,5 +47,8 @@ gaussian_stddev: {{workload_args.gaussian_stddev}} {% if workload_args.create_stddevs_ahead is defined %} create_stddevs_ahead: {{workload_args.create_stddevs_ahead}} {% endif %} +{% if workload_args.response_times is defined %} +response_times: {{workload_args.response_times}} +{% endif %} diff --git a/roles/fs-drift/templates/publisher.py.j2 b/roles/fs-drift/templates/publisher.py.j2 index 9a31395be..a831cac48 100644 --- a/roles/fs-drift/templates/publisher.py.j2 +++ b/roles/fs-drift/templates/publisher.py.j2 @@ -8,16 +8,17 @@ import sys redis_host = sys.argv[1] number_of_pods = sys.argv[2] +trunc_uuid = sys.argv[3] try: r = redis.StrictRedis(host=redis_host, port=6379) count = 0 while int(count) != int(number_of_pods): - count = r.execute_command("PUBSUB NUMSUB fs-drift")[1] + count = r.execute_command(f"PUBSUB NUMSUB fs-drift-{trunc_uuid}")[1] print(count, file=sys.stdout) time.sleep(1) print("All Pods are ready to run. Triggering the run ........\(^_^)/", file=sys.stdout) - r.publish("fs-drift","run") + r.publish(f"fs-drift-{trunc_uuid}","run") except Exception as e: print("******Exception Occured*******") print(str(e)) diff --git a/roles/fs-drift/templates/subscriber.py.j2 b/roles/fs-drift/templates/subscriber.py.j2 index 4cf0d29e5..bd685c9ab 100644 --- a/roles/fs-drift/templates/subscriber.py.j2 +++ b/roles/fs-drift/templates/subscriber.py.j2 @@ -6,12 +6,13 @@ import sys import redis redis_host = sys.argv[1] +trunc_uuid = sys.argv[2] try: r = redis.StrictRedis(host=redis_host, port=6379) p = r.pubsub() - p.subscribe('fs-drift') + p.subscribe(f'fs-drift-{trunc_uuid}') STATE = True while STATE: diff --git a/roles/fs-drift/templates/workload_job.yml.j2 b/roles/fs-drift/templates/workload_job.yml.j2 index 4b1cc0ff2..bd51c5af0 100644 --- a/roles/fs-drift/templates/workload_job.yml.j2 +++ b/roles/fs-drift/templates/workload_job.yml.j2 @@ -2,7 +2,7 @@ kind: Job apiVersion: batch/v1 metadata: - name: "{{ meta.name }}-fs-drift-client-benchmark-{{ trunc_uuid }}-pod-{{ item }}" + name: "{{ ansible_operator_meta.name }}-client-{{ trunc_uuid }}-{{ item }}" namespace: "{{ operator_namespace }}" spec: ttlSecondsAfterFinished: 600 @@ -11,6 +11,13 @@ spec: metadata: labels: app: fs-drift-benchmark-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -29,12 +36,12 @@ spec: topologyKey: "kubernetes.io/hostname" restartPolicy: Never containers: - - name: benchmark-server + - name: fs-drift {% if hostpath is defined %} securityContext: privileged: true {% endif %} - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/fs-drift:master') }} + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/fs-drift:latest') }} # example of what to do when debugging image # image: quay.io/bengland2/fs-drift:latest imagePullPolicy: Always @@ -47,7 +54,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index @@ -74,16 +81,19 @@ spec: rm -rf ${TMPDIR}; mkdir -pv ${TMPDIR}/RESULTS; mkdir -pv {{fs_drift_path}}/fs_drift_test_data; - python /tmp/fs-drift/subscriber {{bo.resources[0].status.podIP}}; + python /tmp/fs-drift/subscriber {{bo.resources[0].status.podIP}} {{ trunc_uuid }}; cat /tmp/fs-drift/params; run_snafu --tool fs-drift +{% if workload_args.debug is defined and workload_args.debug %} + -v +{% endif %} --top {{fs_drift_path}}/fs_drift_test_data --dir ${TMPDIR}/RESULTS {% if workload_args.samples is defined %} --samples {{workload_args.samples}} {% endif %} - --yaml-input-file /tmp/fs-drift/params ; + --yaml-input-file /tmp/fs-drift/params; if [ $? = 0 ] ; then echo RUN STATUS DONE ; else echo FAIL ; exit 1 ; fi volumeMounts: - name: config-volume @@ -108,5 +118,4 @@ spec: type: DirectoryOrCreate {% endif %} restartPolicy: Never - serviceAccountName: benchmark-operator {% include "metadata.yml.j2" %} diff --git a/roles/generic_workload/tasks/main.yml b/roles/generic_workload/tasks/main.yml new file mode 100644 index 000000000..665656608 --- /dev/null +++ b/roles/generic_workload/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- block: + + - name: Create workload resources + k8s: + state: present + definition: "{{ lookup('template', item) }}" + loop: "{{ resources }}" + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running + + when: benchmark_state.resources[0].status.state == "Building" + +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/hammerdb/defaults/main.yml b/roles/hammerdb/defaults/main.yml new file mode 100644 index 000000000..c2be416b5 --- /dev/null +++ b/roles/hammerdb/defaults/main.yml @@ -0,0 +1,2 @@ +--- +resource_kind: "{{ workload.args.kind | default('pod') }}" diff --git a/roles/hammerdb/tasks/main.yaml b/roles/hammerdb/tasks/main.yaml index 115c86696..f5a1f07f3 100644 --- a/roles/hammerdb/tasks/main.yaml +++ b/roles/hammerdb/tasks/main.yaml @@ -1,43 +1,36 @@ --- -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - If it has changed - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state +- name: Create PVC(s) + k8s: + definition: + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: "claim-{{ trunc_uuid }}" + namespace: '{{ operator_namespace }}' + annotations: + volume.beta.kubernetes.io/storage-class: "{{ workload_args.client_vm.pvc_storageclass }}" + spec: + accessModes: + - "{{ workload_args.client_vm.pvc_pvcaccessmode | default('ReadWriteOnce') }}" + volumeMode: "{{ workload_args.client_vm.pvc_pvcvolumemode | default('Filesystem') }}" + resources: + requests: + storage: "{{ workload_args.client_vm.pvc_storagesize }}" + when: workload_args.client_vm.pvc is sameas true - block: - - name: template mysql db creation script - k8s: + - name: template mariadb db creation script + k8s: definition: apiVersion: v1 kind: ConfigMap - metadata: - name: '{{ meta.name }}-creator-{{ trunc_uuid }}' + metadata: + name: '{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' - data: - createdb.tcl: "{{ lookup('template', 'createdb_mysql.tcl.j2') }}" - when: (workload_args.db_type == "mysql") + data: + createdb.tcl: "{{ lookup('template', 'createdb_mariadb.tcl.j2') }}" + when: (workload_args.db_type == "mariadb") - name: template mssql db creation script k8s: @@ -45,11 +38,11 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-creator-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: createdb.tcl: "{{ lookup('template', 'createdb_mssql.tcl.j2') }}" - when: (workload_args.db_type == "mssql") + when: (workload_args.db_type == "mssql") - name: template postgresql db creation script k8s: @@ -57,23 +50,23 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-creator-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: createdb.tcl: "{{ lookup('template', 'createdb_pg.tcl.j2') }}" - when: (workload_args.db_type == "pg") + when: (workload_args.db_type == "pg") - - name: template mysql db workload script + - name: template mariadb db workload script k8s: definition: apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: - tpcc-workload-mysql.tcl: "{{ lookup('template', 'tpcc-workload-mysql.tcl.j2') }}" - when: (workload_args.db_type == "mysql") + tpcc-workload-mariadb.tcl: "{{ lookup('template', 'tpcc-workload-mariadb.tcl.j2') }}" + when: (workload_args.db_type == "mariadb") - name: template mssql db workload script k8s: @@ -81,11 +74,11 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: tpcc-workload-mssql.tcl: "{{ lookup('template', 'tpcc-workload-mssql.tcl.j2') }}" - when: (workload_args.db_type == "mssql") + when: (workload_args.db_type == "mssql") - name: template postgresql db workload script k8s: @@ -93,140 +86,212 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: tpcc-workload-pg.tcl: "{{ lookup('template', 'tpcc-workload-pg.tcl.j2') }}" - when: (workload_args.db_type == "pg") - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting DB - complete: false + when: (workload_args.db_type == "pg") + + - name: template mssql workload vm + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: '{{ ansible_operator_meta.name }}-mssql-workload-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' + data: + run_mssql_script.sh: "{{ lookup('template', 'db_mssql_workload_vm.sh.j2') }}" + when: workload_args.db_type == "mssql" and resource_kind == "vm" + + - name: template mariadb workload vm + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: '{{ ansible_operator_meta.name }}-mariadb-workload-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' + data: + run_mariadb_script.sh: "{{ lookup('template', 'db_mariadb_workload_vm.sh.j2') }}" + when: workload_args.db_type == "mariadb" and resource_kind == "vm" + + - name: template postgresql workload vm + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: '{{ ansible_operator_meta.name }}-postgres-workload-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' + data: + run_postgres_script.sh: "{{ lookup('template', 'db_postgres_workload_vm.sh.j2') }}" + when: workload_args.db_type == "pg" and resource_kind == "vm" + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting DB when: workload_args.db_init is defined and workload_args.db_init - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting DB Workload - complete: false + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting DB Workload when: workload_args.db_benchmark is defined and workload_args.db_benchmark and (workload_args.db_init is not defined or not workload_args.db_init) - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: ConfigMaps Created - complete: true + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: ConfigMaps Created when: (workload_args.db_benchmark is not defined or not workload_args.db_benchmark) and (workload_args.db_init is not defined or not workload_args.db_init) - when: resource_state.resources[0].status.state == "Building" + when: benchmark_state.resources[0].status.state == "Building" - block: - - name: start db creation job - k8s: - state: present - definition: "{{ lookup('template', 'db_creation.yml') | from_yaml }}" - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: DB Creating - complete: false + - block: + - name: start db creation job + k8s: + state: present + definition: "{{ lookup('template', 'db_creation.yml') | from_yaml }}" + + when: resource_kind == "pod" + + - block: + - name: set complete to false + command: "redis-cli set db-creation-{{trunc_uuid}} false" - when: resource_state.resources[0].status.state == "Starting DB" + - name: start mssql db creation vm job + k8s: + state: present + definition: "{{ lookup('template', 'db_creation_mssql_vm.yml') | from_yaml }}" + when: (workload_args.db_type == "mssql") + + - name: start mariadb db creation vm job + k8s: + state: present + definition: "{{ lookup('template', 'db_creation_mariadb_vm.yml') | from_yaml }}" + when: (workload_args.db_type == "mariadb") + + - name: start postgres db creation vm job + k8s: + state: present + definition: "{{ lookup('template', 'db_creation_postgres_vm.yml') | from_yaml }}" + when: (workload_args.db_type == "pg") + + when: resource_kind == "vm" + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: DB Creating + + when: benchmark_state.resources[0].status.state == "Starting DB" - block: - - name: wait for db creation job to finish - k8s_facts: - kind: Job - api_version: batch/v1 - name: '{{ meta.name }}-creator-{{ trunc_uuid }}' - namespace: "{{ operator_namespace }}" - register: hammerdb_creator_pod - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: + - block: + - name: wait for db creation job to finish + k8s_info: + kind: Job + api_version: batch/v1 + name: '{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}' + namespace: "{{ operator_namespace }}" + register: hammerdb_creator_pod + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: state: Starting DB Workload - complete: false - when: hammerdb_creator_pod | json_query('resources[].status.succeeded') and workload_args.db_benchmark is defined and workload_args.db_benchmark - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: + when: hammerdb_creator_pod | json_query('resources[].status.succeeded') and workload_args.db_benchmark is defined and workload_args.db_benchmark + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: state: DB Created - complete: true - when: hammerdb_creator_pod | json_query('resources[].status.succeeded') and (workload_args.db_benchmark is not defined or not workload_args.db_benchmark) + when: hammerdb_creator_pod | json_query('resources[].status.succeeded') and (workload_args.db_benchmark is not defined or not workload_args.db_benchmark) + when: resource_kind == "pod" + + - block: + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting DB Workload + + - name: get complete + command: "redis-cli get db-creation-{{trunc_uuid}}" + register: db_creation_status + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: DB Created + when: db_creation_status.stdout == "true" + when: resource_kind == "vm" + when: benchmark_state.resources[0].status.state == "DB Creating" - when: resource_state.resources[0].status.state == "DB Creating" - - block: + - name: set complete to false + command: "redis-cli set complete false" + when: resource_kind == "vm" + - name: start mssql db workload job k8s: state: present definition: "{{ lookup('template', 'db_mssql_workload.yml.j2') | from_yaml }}" - when: (workload_args.db_type == "mssql") + when: workload_args.db_type == "mssql" and resource_kind == "pod" - - name: start mysql db workload job + - name: start mariadb db workload job k8s: state: present - definition: "{{ lookup('template', 'db_mysql_workload.yml.j2') | from_yaml }}" - when: (workload_args.db_type == "mysql") + definition: "{{ lookup('template', 'db_mariadb_workload.yml.j2') | from_yaml }}" + when: workload_args.db_type == "mariadb" and resource_kind == "pod" - name: start postgresql db workload job k8s: state: present definition: "{{ lookup ('template', 'db_postgres_workload.yml.j2') | from_yaml }}" - when: (workload_args.db_type == "pg") + when: workload_args.db_type == "pg" and resource_kind == "pod" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: DB workload running - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - when: resource_state.resources[0].status.state == "Starting DB Workload" + when: benchmark_state.resources[0].status.state == "Starting DB Workload" - block: - - name: wait for db workload job to finish - k8s_facts: - kind: Job - api_version: batch/v1 - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' - namespace: "{{ operator_namespace }}" - register: hammerdb_workload_pod - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: DB Workload Complete - complete: true - when: "hammerdb_workload_pod | json_query('resources[].status.succeeded')" - - when: resource_state.resources[0].status.state == "DB workload running" + + - include_role: + name: benchmark_state + tasks_from: completed.yml + when: resource_kind == "pod" + + + - block: + + - name: get complete + command: "redis-cli get complete-{{ trunc_uuid }}" + register: complete_status + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Complete + complete: true + when: complete_status.stdout == "true" + when: resource_kind == "vm" + when: benchmark_state.resources[0].status.state == "Running" + diff --git a/roles/hammerdb/templates/createdb_mysql.tcl.j2 b/roles/hammerdb/templates/createdb_mariadb.tcl.j2 similarity index 92% rename from roles/hammerdb/templates/createdb_mysql.tcl.j2 rename to roles/hammerdb/templates/createdb_mariadb.tcl.j2 index 0c08a45eb..9cbc3e170 100644 --- a/roles/hammerdb/templates/createdb_mysql.tcl.j2 +++ b/roles/hammerdb/templates/createdb_mariadb.tcl.j2 @@ -11,6 +11,7 @@ dbset db mysql dbset bm TPC-C diset connection mysql_host {{workload_args.db_server}} diset connection mysql_port {{workload_args.db_port}} +diset connection mysql_socket {{workload_args.db_mysql_socket}} diset tpcc mysql_user {{workload_args.db_user}} diset tpcc mysql_pass {{workload_args.db_pass}} diset tpcc mysql_dbase {{workload_args.db_name}} @@ -22,4 +23,3 @@ puts "CREATING SCHEMA" buildschema wait_to_complete vwait forever - diff --git a/roles/hammerdb/templates/createdb_pg.tcl.j2 b/roles/hammerdb/templates/createdb_pg.tcl.j2 index 2cb6265d9..fed3d4273 100644 --- a/roles/hammerdb/templates/createdb_pg.tcl.j2 +++ b/roles/hammerdb/templates/createdb_pg.tcl.j2 @@ -11,8 +11,6 @@ dbset db pg dbset bm TPC-C diset connection pg_host {{workload_args.db_server}} diset connection pg_port {{workload_args.db_port}} -diset tpcc pg_user {{workload_args.db_user}} -diset tpcc pg_pass {{workload_args.db_pass}} diset tpcc pg_dbase {{workload_args.db_postgresql_defaultdbase}} diset tpcc pg_count_ware {{workload_args.db_warehouses}} diset tpcc pg_num_vu {{workload_args.db_num_workers}} @@ -21,4 +19,3 @@ puts "CREATING SCHEMA" buildschema wait_to_complete vwait forever - diff --git a/roles/hammerdb/templates/db_creation.yml b/roles/hammerdb/templates/db_creation.yml index bf3f844ed..5be9d77b8 100644 --- a/roles/hammerdb/templates/db_creation.yml +++ b/roles/hammerdb/templates/db_creation.yml @@ -1,20 +1,46 @@ +--- apiVersion: batch/v1 -kind: "job" +kind: Job metadata: - name: "{{ meta.name }}-creator-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: hammerdb_creator-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-creator-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_node }}' +{% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} containers: - name: hammerdb +{% if workload_args.resources is sameas true %} + resources: + requests: + cpu: {{ workload_args.requests_cpu }} + memory: {{ workload_args.requests_memory }} + limits: + cpu: {{ workload_args.limits_cpu }} + memory: {{ workload_args.limits_memory }} +{% endif %} command: ["/bin/sh", "-c"] args: ["/usr/local/bin/uid_entrypoint; cd /hammer; ./hammerdbcli auto /creator/createdb.tcl"] image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:latest') }} @@ -26,7 +52,7 @@ spec: volumes: - name: hammerdb-creator-volume configMap: - name: "{{ meta.name }}-creator-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}" defaultMode: 0640 - restartPolicy: OnFailure + restartPolicy: Never {% include "metadata.yml.j2" %} diff --git a/roles/hammerdb/templates/db_creation_mariadb_vm.yml b/roles/hammerdb/templates/db_creation_mariadb_vm.yml new file mode 100644 index 000000000..a28d83e91 --- /dev/null +++ b/roles/hammerdb/templates/db_creation_mariadb_vm.yml @@ -0,0 +1,131 @@ +--- +apiVersion: kubevirt.io/v1alpha3 +kind: VirtualMachineInstance +metadata: + name: {{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + labels: + app: hammerdb_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} +spec: + domain: + cpu: + sockets: {{ workload_args.client_vm.sockets }} + cores: {{ workload_args.client_vm.cores }} + threads: {{ workload_args.client_vm.threads }} + dedicatedCpuPlacement: {{ workload_args.client_vm.dedicatedcpuplacement }} +{% if 'hostpassthrough' in workload_args.client_vm.extra_options %} + model: host-passthrough +{% endif %} + devices: + disks: + - disk: + bus: virtio + name: containerdisk + - disk: + bus: virtio + name: cloudinitdisk + - disk: {} + name: hammerdb-creator-volume + # set serial + serial: CVLY623300HK240C + - disk: {} + name: hammerdb-workload-volume + # set serial + serial: CVLY623300HK240D + - disk: {} + name: hammerdb-mariadb-workload-volume + # set serial + serial: CVLY623300HK240E +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} + - disk: + bus: virtio + name: data-volume + serial: data +{% endif %} + interfaces: + - name: default + {{ workload_args.client_vm.network.front_end }}: {} + networkInterfaceMultiqueue: {{ workload_args.client_vm.network.multiqueue.enabled }} + machine: + type: "" + resources: + requests: + memory: {{ workload_args.client_vm.requests.memory }} + limits: + memory: {{ workload_args.client_vm.limits.memory }} + terminationGracePeriodSeconds: 0 +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_node }}' +{% endif %} + networks: + - name: default + pod: {} + volumes: + - name: containerdisk + containerDisk: + image: {{ workload_args.client_vm.image }} + - cloudInitNoCloud: + userData: |- + #cloud-config + password: centos + chpasswd: { expire: False } + bootcmd: + # mount the ConfigMap + - "mkdir /creator" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240C | cut -f1 -d' ') /creator" + - "mkdir /workload" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240D | cut -f1 -d' ') /workload" + - "mkdir /tmp/hammerdb-mariadb-test" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240E | cut -f1 -d' ') /tmp/hammerdb-mariadb-test" + runcmd: +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} + - "mkdir -p /var/lib/mysql || true" + - mkfs.ext4 /dev/disk/by-id/virtio-data + - "mount /dev/disk/by-id/virtio-data /var/lib/mysql" + - "chown -R mysql:mysql /var/lib/mysql" +{% endif %} +{% if workload_args.client_vm.network.multiqueue.enabled %} + - dnf install -y ethtool + - ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }} +{% endif %} + - dnf install -y redis + - systemctl start redis + - systemctl enable redis + - bash /tmp/hammerdb-mariadb-test/run_mariadb_script.sh + - redis-cli -h {{bo.resources[0].status.podIP}} set complete true + name: cloudinitdisk + - configMap: + name: "{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}" + name: hammerdb-creator-volume + - configMap: + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" + name: hammerdb-workload-volume + - configMap: + name: "{{ ansible_operator_meta.name }}-mariadb-workload-{{ trunc_uuid }}" + name: hammerdb-mariadb-workload-volume +{% if workload_args.client_vm.pvc is sameas true %} + - name: data-volume + persistentVolumeClaim: + claimName: claim-{{ trunc_uuid }} +{% elif workload_args.client_vm.hostpath is sameas true %} + - name: data-volume + hostDisk: + path: {{ workload_args.client_vm.hostpath_path }} + capacity: {{ workload_args.client_vm.hostpath_storagesize | default("10Gi") }} + type: DiskOrCreate +{% endif %} +status: {} diff --git a/roles/hammerdb/templates/db_creation_mssql_vm.yml b/roles/hammerdb/templates/db_creation_mssql_vm.yml new file mode 100644 index 000000000..3f4c34032 --- /dev/null +++ b/roles/hammerdb/templates/db_creation_mssql_vm.yml @@ -0,0 +1,132 @@ +--- +apiVersion: kubevirt.io/v1alpha3 +kind: VirtualMachineInstance +metadata: + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" + namespace: "{{ operator_namespace }}" + labels: + app: hammerdb_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} +spec: + domain: + cpu: + sockets: {{ workload_args.client_vm.sockets }} + cores: {{ workload_args.client_vm.cores }} + threads: {{ workload_args.client_vm.threads }} + dedicatedCpuPlacement: {{ workload_args.client_vm.dedicatedcpuplacement }} +{% if 'hostpassthrough' in workload_args.client_vm.extra_options %} + model: host-passthrough +{% endif %} + devices: + disks: + - disk: + bus: virtio + name: containerdisk + - disk: + bus: virtio + name: cloudinitdisk + - disk: {} + name: hammerdb-creator-volume + # set serial + serial: CVLY623300HK240C + - disk: {} + name: hammerdb-workload-volume + # set serial + serial: CVLY623300HK240D + - disk: {} + name: hammerdb-mssql-workload-volume + # set serial + serial: CVLY623300HK240E +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} + - disk: + bus: virtio + name: data-volume + serial: data +{% endif %} + interfaces: + - name: default + {{ workload_args.client_vm.network.front_end }}: {} + networkInterfaceMultiqueue: {{ workload_args.client_vm.network.multiqueue.enabled }} + machine: + type: "" + resources: + requests: + memory: {{ workload_args.client_vm.requests.memory }} + limits: + memory: {{ workload_args.client_vm.limits.memory }} + terminationGracePeriodSeconds: 0 +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_node }}' +{% endif %} + networks: + - name: default + pod: {} + volumes: + - name: containerdisk + containerDisk: + image: {{ workload_args.client_vm.image }} + - cloudInitNoCloud: + userData: |- + #cloud-config + password: centos + chpasswd: { expire: False } + bootcmd: + # mount the ConfigMap + - "mkdir /creator" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240C | cut -f1 -d' ') /creator" + - "mkdir /workload" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240D | cut -f1 -d' ') /workload" + - "mkdir /tmp/hammerdb-mssql-test" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240E | cut -f1 -d' ') /tmp/hammerdb-mssql-test" + runcmd: +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} + - "mkdir -p /var/opt/mssql || true" + - mkfs.ext4 /dev/disk/by-id/virtio-data + - "mount /dev/disk/by-id/virtio-data /var/opt/mssql" + - "chown -R mssql:mssql /var/opt/mssql" + - "chgrp mssql /var/opt/mssql" +{% endif %} +{% if workload_args.client_vm.network.multiqueue.enabled %} + - dnf install -y ethtool + - ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }} +{% endif %} + - dnf install -y redis + - systemctl start redis + - systemctl enable redis + - bash /tmp/hammerdb-mssql-test/run_mssql_script.sh + - redis-cli -h {{bo.resources[0].status.podIP}} set complete true + name: cloudinitdisk + - configMap: + name: "{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}" + name: hammerdb-creator-volume + - configMap: + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" + name: hammerdb-workload-volume + - configMap: + name: "{{ ansible_operator_meta.name }}-mssql-workload-{{ trunc_uuid }}" + name: hammerdb-mssql-workload-volume +{% if workload_args.client_vm.pvc is sameas true %} + - name: data-volume + persistentVolumeClaim: + claimName: claim-{{ trunc_uuid }} +{% elif workload_args.client_vm.hostpath is sameas true %} + - name: data-volume + hostDisk: + path: {{ workload_args.client_vm.hostpath_path }} + capacity: {{ workload_args.client_vm.hostpath_storagesize | default("10Gi") }} + type: DiskOrCreate +{% endif %} +status: {} diff --git a/roles/hammerdb/templates/db_creation_postgres_vm.yml b/roles/hammerdb/templates/db_creation_postgres_vm.yml new file mode 100644 index 000000000..102e4a0b5 --- /dev/null +++ b/roles/hammerdb/templates/db_creation_postgres_vm.yml @@ -0,0 +1,131 @@ +--- +apiVersion: kubevirt.io/v1alpha3 +kind: VirtualMachineInstance +metadata: + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" + namespace: "{{ operator_namespace }}" + labels: + app: hammerdb_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} +spec: + domain: + cpu: + sockets: {{ workload_args.client_vm.sockets }} + cores: {{ workload_args.client_vm.cores }} + threads: {{ workload_args.client_vm.threads }} + dedicatedCpuPlacement: {{ workload_args.client_vm.dedicatedcpuplacement }} +{% if 'hostpassthrough' in workload_args.client_vm.extra_options %} + model: host-passthrough +{% endif %} + devices: + disks: + - disk: + bus: virtio + name: containerdisk + - disk: + bus: virtio + name: cloudinitdisk + - disk: {} + name: hammerdb-creator-volume + # set serial + serial: CVLY623300HK240C + - disk: {} + name: hammerdb-workload-volume + # set serial + serial: CVLY623300HK240D + - disk: {} + name: hammerdb-postgres-workload-volume + # set serial + serial: CVLY623300HK240E +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} + - disk: + bus: virtio + name: data-volume + serial: data +{% endif %} + interfaces: + - name: default + {{ workload_args.client_vm.network.front_end }}: {} + networkInterfaceMultiqueue: {{ workload_args.client_vm.network.multiqueue.enabled }} + machine: + type: "" + resources: + requests: + memory: {{ workload_args.client_vm.requests.memory }} + limits: + memory: {{ workload_args.client_vm.limits.memory }} + terminationGracePeriodSeconds: 0 +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_node }}' +{% endif %} + networks: + - name: default + pod: {} + volumes: + - name: containerdisk + containerDisk: + image: {{ workload_args.client_vm.image }} + - cloudInitNoCloud: + userData: |- + #cloud-config + password: centos + chpasswd: { expire: False } + bootcmd: + # mount the ConfigMap + - "mkdir /creator" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240C | cut -f1 -d' ') /creator" + - "mkdir /workload" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240D | cut -f1 -d' ') /workload" + - "mkdir /tmp/hammerdb-postgres-test" + - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240E | cut -f1 -d' ') /tmp/hammerdb-postgres-test" +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} + - "mkdir -p /var/lib/pgsql || true" + - mkfs.ext4 /dev/disk/by-id/virtio-data + - "mount /dev/disk/by-id/virtio-data /var/lib/pgsql" + - "chown -R postgres:postgres /var/lib/pgsql" +{% endif %} + runcmd: +{% if workload_args.client_vm.network.multiqueue.enabled %} + - dnf install -y ethtool + - ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }} +{% endif %} + - dnf install -y redis + - systemctl start redis + - systemctl enable redis + - bash /tmp/hammerdb-postgres-test/run_postgres_script.sh + - redis-cli -h {{bo.resources[0].status.podIP}} set complete true + name: cloudinitdisk + - configMap: + name: "{{ ansible_operator_meta.name }}-creator-{{ trunc_uuid }}" + name: hammerdb-creator-volume + - configMap: + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" + name: hammerdb-workload-volume + - configMap: + name: "{{ ansible_operator_meta.name }}-postgres-workload-{{ trunc_uuid }}" + name: hammerdb-postgres-workload-volume +{% if workload_args.client_vm.pvc is sameas true %} + - name: data-volume + persistentVolumeClaim: + claimName: claim-{{ trunc_uuid }} +{% elif workload_args.client_vm.hostpath is sameas true %} + - name: data-volume + hostDisk: + path: {{ workload_args.client_vm.hostpath_path }} + capacity: {{ workload_args.client_vm.hostpath_storagesize | default("10Gi") }} + type: DiskOrCreate +{% endif %} +status: {} diff --git a/roles/hammerdb/templates/db_mysql_workload.yml.j2 b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 similarity index 63% rename from roles/hammerdb/templates/db_mysql_workload.yml.j2 rename to roles/hammerdb/templates/db_mariadb_workload.yml.j2 index 4888e36eb..f4e8b41ac 100644 --- a/roles/hammerdb/templates/db_mysql_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mariadb_workload.yml.j2 @@ -1,22 +1,47 @@ --- apiVersion: batch/v1 -kind: "job" +kind: Job metadata: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: hammerdb_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_node }}' +{% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} containers: - name: hammerdb - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:master') }} +{% if workload_args.resources is sameas true %} + resources: + requests: + cpu: {{ workload_args.requests_cpu | default("200m") }} + memory: {{ workload_args.requests_memory | default("100Mi") }} + limits: + cpu: {{ workload_args.limits_cpu }} + memory: {{ workload_args.limits_memory }} +{% endif %} + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:latest') }} imagePullPolicy: Always env: - name: uuid @@ -25,7 +50,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index @@ -44,9 +69,9 @@ spec: value: "{{ prometheus.prom_url | default() }}" {% endif %} command: ["/bin/sh", "-c"] - args: + args: - "/usr/local/bin/uid_entrypoint; - export db_type={{workload_args.db_type}}; + export db_type={{workload_args.db_type}}; export timed_test={{workload_args.timed_test}}; export db_server={{workload_args.db_server}}; export db_port={{workload_args.db_port}}; @@ -70,7 +95,14 @@ spec: export samples={{workload_args.samples}}; export db_mysql_storage_engine={{workload_args.db_mysql_storage_engine}}; export db_mysql_partition={{workload_args.db_mysql_partition}}; - cd /hammer; +{% if workload_args.es_custom_field is sameas true %} + export es_ocp_version={{workload_args.es_ocp_version}}; + export es_cnv_version={{workload_args.es_cnv_version}}; + export es_db_version={{workload_args.es_db_version}}; + export es_os_version={{workload_args.es_os_version}}; + export es_kind={{workload_args.es_kind}}; +{% endif %} + cd /hammer; run_snafu --tool hammerdb -u {{ uuid }}" volumeMounts: - name: hammerdb-workload-volume @@ -79,7 +111,8 @@ spec: volumes: - name: hammerdb-workload-volume configMap: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" defaultMode: 0640 - restartPolicy: OnFailure + restartPolicy: Never {% include "metadata.yml.j2" %} + diff --git a/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 b/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 new file mode 100644 index 000000000..62c07e3bd --- /dev/null +++ b/roles/hammerdb/templates/db_mariadb_workload_vm.sh.j2 @@ -0,0 +1,61 @@ +{% if elasticsearch.url %} +export es={{ elasticsearch.url }}; +export es_index={{ elasticsearch.index_name | default("ripsaw-hammerdb") }}; +export es_verify_cert={{ elasticsearch.verify_cert | default("true") }}; +export parallel={{ elasticsearch.parallel | default("false") }}; +export uuid={{uuid}}; +{% endif %} +{% if workload_args.client_vm.network.multiqueue.enabled %} +dnf install -y ethtool; +ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }}; +{% endif %} +dnf install -y epel-release git python3-pip; +pip3 install --upgrade pip; +pip3 install git+https://github.com/cloud-bulldozer/benchmark-wrapper; +export db_type={{workload_args.db_type}}; +export timed_test={{workload_args.timed_test}}; +export db_server={{workload_args.db_server}}; +export db_port={{workload_args.db_port}}; +export db_warehouses={{workload_args.db_warehouses}}; +export db_num_workers={{workload_args.db_num_workers}}; +export db_user={{workload_args.db_user}}; +export db_pass={{workload_args.db_pass}}; +export db_name={{workload_args.db_name}}; +export transactions={{workload_args.transactions}}; +export raiseerror={{workload_args.raiseerror}}; +export keyandthink={{workload_args.keyandthink}}; +export driver={{workload_args.driver}}; +export rampup={{workload_args.rampup}}; +export runtime={{workload_args.runtime}}; +export allwarehouse={{workload_args.allwarehouse}}; +export timeprofile={{workload_args.timeprofile}}; +export async_scale={{workload_args.async_scale}}; +export async_client={{workload_args.async_client}}; +export async_verbose={{workload_args.async_verbose}}; +export async_delay={{workload_args.async_delay}}; +export samples={{workload_args.samples}}; +export db_mysql_storage_engine={{workload_args.db_mysql_storage_engine}}; +export db_mysql_partition={{workload_args.db_mysql_partition}}; +{% if workload_args.es_custom_field is sameas true %} +export es_ocp_version={{workload_args.es_ocp_version}}; +export es_cnv_version={{workload_args.es_cnv_version}}; +export es_db_version={{workload_args.es_db_version}}; +export es_os_version={{workload_args.es_os_version}}; +export es_kind={{workload_args.es_kind}}; +{% endif %} +export HOME=/root; +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} +sudo systemctl stop mariadb.service; +echo "UPDATE mysql.user SET Password=PASSWORD('mysql') WHERE User='root';" > input; +echo "GRANT ALL ON *.* to root@'%' IDENTIFIED BY 'mysql';" >> input; +echo "flush privileges;" >> input; +echo "exit" >> input; +sudo systemctl start mariadb.service; +mysql -u root < input; +{% endif %} +cd /hammer; +./hammerdbcli auto /creator/createdb.tcl; +redis-cli set db-creation-{{trunc_uuid}} true; +run_snafu --tool hammerdb -u {{ uuid }}; +redis-cli -h {{bo.resources[0].status.podIP}} set complete-{{trunc_uuid}} true; diff --git a/roles/hammerdb/templates/db_mssql_workload.yml.j2 b/roles/hammerdb/templates/db_mssql_workload.yml.j2 index 84aeb5bab..1c9b011fa 100644 --- a/roles/hammerdb/templates/db_mssql_workload.yml.j2 +++ b/roles/hammerdb/templates/db_mssql_workload.yml.j2 @@ -1,22 +1,47 @@ --- apiVersion: batch/v1 -kind: "job" +kind: Job metadata: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: hammerdb_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_node }}' +{% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} containers: - name: hammerdb - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:master') }} +{% if workload_args.resources is sameas true %} + resources: + requests: + cpu: {{ workload_args.requests_cpu | default("200m") }} + memory: {{ workload_args.requests_memory | default("100Mi") }} + limits: + cpu: {{ workload_args.limits_cpu }} + memory: {{ workload_args.limits_memory }} +{% endif %} + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:latest') }} imagePullPolicy: Always env: - name: uuid @@ -25,7 +50,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index @@ -76,6 +101,13 @@ spec: export db_mssql_bucket={{workload_args.db_mssql_bucket}}; export db_mssql_durability={{workload_args.db_mssql_durability}}; export db_mssql_checkpoint={{workload_args.db_mssql_checkpoint}}; +{% if workload_args.es_custom_field is sameas true %} + export es_ocp_version={{workload_args.es_ocp_version}}; + export es_cnv_version={{workload_args.es_cnv_version}}; + export es_db_version={{workload_args.es_db_version}}; + export es_os_version={{workload_args.es_os_version}}; + export es_kind={{workload_args.es_kind}}; +{% endif %} cd /hammer; run_snafu --tool hammerdb -u {{ uuid }}" volumeMounts: @@ -85,7 +117,7 @@ spec: volumes: - name: hammerdb-workload-volume configMap: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" defaultMode: 0640 - restartPolicy: OnFailure + restartPolicy: Never {% include "metadata.yml.j2" %} diff --git a/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 new file mode 100644 index 000000000..b90a84cb3 --- /dev/null +++ b/roles/hammerdb/templates/db_mssql_workload_vm.sh.j2 @@ -0,0 +1,67 @@ +{% if elasticsearch.url %} +export es={{ elasticsearch.url }}; +export es_index={{ elasticsearch.index_name | default("ripsaw-hammerdb") }}; +export es_verify_cert={{ elasticsearch.verify_cert | default("true") }}; +export parallel={{ elasticsearch.parallel | default("false") }}; +export uuid={{uuid}}; +{% endif %} +{% if workload_args.client_vm.network.multiqueue.enabled %} +dnf install -y ethtool; +ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }}; +{% endif %} +dnf install -y epel-release git python3-pip; +pip3 install --upgrade pip; +pip3 install git+https://github.com/cloud-bulldozer/benchmark-wrapper; +export db_type={{workload_args.db_type}}; +export timed_test={{workload_args.timed_test}}; +export db_server={{workload_args.db_server}}; +export db_port={{workload_args.db_port}}; +export db_warehouses={{workload_args.db_warehouses}}; +export db_num_workers={{workload_args.db_num_workers}}; +export db_user={{workload_args.db_user}}; +export transactions={{workload_args.transactions}}; +export raiseerror={{workload_args.raiseerror}}; +export keyandthink={{workload_args.keyandthink}}; +export db_driver={{workload_args.driver}}; +export rampup={{workload_args.rampup}}; +export runtime={{workload_args.runtime}}; +export allwarehouse={{workload_args.allwarehouse}}; +export timeprofile={{workload_args.timeprofile}}; +export async_scale={{workload_args.async_scale}}; +export async_client={{workload_args.async_client}}; +export async_verbose={{workload_args.async_verbose}}; +export async_delay={{workload_args.async_delay}}; +export samples={{workload_args.samples}}; +export db_mssql_tcp={{workload_args.db_mssql_tcp}}; +export db_mssql_azure={{workload_args.db_mssql_azure}}; +export db_mssql_authentication={{workload_args.db_mssql_authentication}}; +export db_mssql_linux_authent={{workload_args.db_mssql_linux_authent}}; +export db_mssql_odbc_driver='{{workload_args.db_mssql_odbc_driver}}'; +export db_mssql_linux_odbc='{{workload_args.db_mssql_linux_odbc}}'; +export db_mssql_imdb={{workload_args.db_mssql_imdb}}; +export db_mssql_bucket={{workload_args.db_mssql_bucket}}; +export db_mssql_durability={{workload_args.db_mssql_durability}}; +export db_mssql_checkpoint={{workload_args.db_mssql_checkpoint}}; +{% if workload_args.es_custom_field is sameas true %} +export es_ocp_version={{workload_args.es_ocp_version}}; +export es_cnv_version={{workload_args.es_cnv_version}}; +export es_db_version={{workload_args.es_db_version}}; +export es_os_version={{workload_args.es_os_version}}; +export es_kind={{workload_args.es_kind}}; +{% endif %} +export HOME=/root; +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} +sudo systemctl stop mssql-server; +sudo ACCEPT_EULA='Y' MSSQL_PID='Enterprise' MSSQL_SA_PASSWORD='s3curePasswordString' /opt/mssql/bin/mssql-conf setup; +# Fixed firewall-cmd failure in cloud-init, for more details https://forums.centos.org/viewtopic.php?f=47&t=52162&p=220915 +sudo firewall-offline-cmd --add-port=1433/tcp; +sudo systemctl enable firewalld; +sudo systemctl start firewalld; +sudo systemctl start mssql-server; +{% endif %} +cd /hammer; +./hammerdbcli auto /creator/createdb.tcl; +redis-cli set db-creation-{{trunc_uuid}} true; +run_snafu --tool hammerdb -u {{ uuid }}; +redis-cli -h {{bo.resources[0].status.podIP}} set complete-{{trunc_uuid}} true; diff --git a/roles/hammerdb/templates/db_postgres_workload.yml.j2 b/roles/hammerdb/templates/db_postgres_workload.yml.j2 index 4f207e718..85941db7d 100644 --- a/roles/hammerdb/templates/db_postgres_workload.yml.j2 +++ b/roles/hammerdb/templates/db_postgres_workload.yml.j2 @@ -1,22 +1,47 @@ --- apiVersion: batch/v1 -kind: "job" +kind: Job metadata: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: hammerdb_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_node }}' +{% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} containers: - name: hammerdb - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:master') }} +{% if workload_args.resources is sameas true %} + resources: + requests: + cpu: {{ workload_args.requests_cpu | default("200m") }} + memory: {{ workload_args.requests_memory | default("100Mi") }} + limits: + cpu: {{ workload_args.limits_cpu }} + memory: {{ workload_args.limits_memory }} +{% endif %} + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/hammerdb:latest') }} imagePullPolicy: Always env: - name: uuid @@ -25,7 +50,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index @@ -44,8 +69,8 @@ spec: value: "{{ prometheus.prom_url | default() }}" {% endif %} command: ["/bin/sh", "-c"] - args: - - "/usr/local/bin/uid_entrypoint; + args: + - "/usr/local/bin/uid_entrypoint; export db_type={{workload_args.db_type}}; export timed_test={{workload_args.timed_test}}; export db_server={{workload_args.db_server}}; @@ -74,7 +99,14 @@ spec: export db_postgresql_dritasnap={{workload_args.db_postgresql_dritasnap}}; export db_postgresql_oracompat={{workload_args.db_postgresql_oracompat}}; export db_postgresql_storedprocs={{workload_args.db_postgresql_storedprocs}}; - cd /hammer; +{% if workload_args.es_custom_field is sameas true %} + export es_ocp_version={{workload_args.es_ocp_version}}; + export es_cnv_version={{workload_args.es_cnv_version}}; + export es_db_version={{workload_args.es_db_version}}; + export es_os_version={{workload_args.es_os_version}}; + export es_kind={{workload_args.es_kind}}; +{% endif %} + cd /hammer; run_snafu --tool hammerdb -u {{ uuid }}" volumeMounts: - name: hammerdb-workload-volume @@ -83,7 +115,7 @@ spec: volumes: - name: hammerdb-workload-volume configMap: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" defaultMode: 0640 - restartPolicy: OnFailure + restartPolicy: Never {% include "metadata.yml.j2" %} diff --git a/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 new file mode 100644 index 000000000..2065841d6 --- /dev/null +++ b/roles/hammerdb/templates/db_postgres_workload_vm.sh.j2 @@ -0,0 +1,70 @@ +{% if elasticsearch.url %} +export es={{ elasticsearch.url }}; +export es_index={{ elasticsearch.index_name | default("ripsaw-hammerdb") }}; +export es_verify_cert={{ elasticsearch.verify_cert | default("true") }}; +export parallel={{ elasticsearch.parallel | default("false") }}; +export uuid={{uuid}}; +{% endif %} +{% if workload_args.client_vm.network.multiqueue.enabled %} +dnf install -y ethtool; +ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }}; +{% endif %} +dnf install -y epel-release git python3-pip; +pip3 install --upgrade pip; +pip3 install git+https://github.com/cloud-bulldozer/benchmark-wrapper; +export db_type={{workload_args.db_type}}; +export timed_test={{workload_args.timed_test}}; +export db_server={{workload_args.db_server}}; +export db_port={{workload_args.db_port}}; +export db_warehouses={{workload_args.db_warehouses}}; +export db_num_workers={{workload_args.db_num_workers}}; +export db_user={{workload_args.db_user}}; +export db_name={{workload_args.db_name}}; +export transactions={{workload_args.transactions}}; +export raiseerror={{workload_args.raiseerror}}; +export keyandthink={{workload_args.keyandthink}}; +export driver={{workload_args.driver}}; +export rampup={{workload_args.rampup}}; +export runtime={{workload_args.runtime}}; +export allwarehouse={{workload_args.allwarehouse}}; +export timeprofile={{workload_args.timeprofile}}; +export async_scale={{workload_args.async_scale}}; +export async_client={{workload_args.async_client}}; +export async_verbose={{workload_args.async_verbose}}; +export async_delay={{workload_args.async_delay}}; +export samples={{workload_args.samples}}; +export db_postgresql_defaultdbase={{workload_args.db_postgresql_defaultdbase}}; +export db_postgresql_superuser={{workload_args.db_postgresql_superuser}}; +export db_postgresql_superuser_pass={{workload_args.db_postgresql_superuser_pass}}; +export db_postgresql_vacuum={{workload_args.db_postgresql_vacuum}}; +export db_postgresql_dritasnap={{workload_args.db_postgresql_dritasnap}}; +export db_postgresql_oracompat={{workload_args.db_postgresql_oracompat}}; +export db_postgresql_storedprocs={{workload_args.db_postgresql_storedprocs}}; +{% if workload_args.es_custom_field is sameas true %} +export es_ocp_version={{workload_args.es_ocp_version}}; +export es_cnv_version={{workload_args.es_cnv_version}}; +export es_db_version={{workload_args.es_db_version}}; +export es_os_version={{workload_args.es_os_version}}; +export es_kind={{workload_args.es_kind}}; +{% endif %} +export HOME=/root; +{% if workload_args.client_vm.pvc is sameas true + or workload_args.client_vm.hostpath is sameas true %} +sudo systemctl stop postgresql; +sudo postgresql-setup initdb; +/sbin/restorecon -v /var/lib/pgsql; +echo local all all trust > /var/lib/pgsql/data/pg_hba.conf; +echo host all all all trust >> /var/lib/pgsql/data/pg_hba.conf; +echo max_connections = 100 > /var/lib/pgsql/data/postgresql.conf +echo max_prepared_transactions = 0 >> /var/lib/pgsql/data/postgresql.conf +echo shared_buffers = 4096MB >> /var/lib/pgsql/data/postgresql.conf +echo autovacuum = off >> /var/lib/pgsql/data/postgresql.conf +sudo systemctl start postgresql; +echo "alter role postgres password 'postgres'" > input +psql -U postgres -d postgres < input +{% endif %} +cd /hammer; +./hammerdbcli auto /creator/createdb.tcl; +redis-cli set db-creation-{{trunc_uuid}} true; +run_snafu --tool hammerdb -u {{ uuid }}; +redis-cli -h {{bo.resources[0].status.podIP}} set complete-{{trunc_uuid}} true; diff --git a/roles/hammerdb/templates/tpcc-workload-mysql.tcl.j2 b/roles/hammerdb/templates/tpcc-workload-mariadb.tcl.j2 similarity index 97% rename from roles/hammerdb/templates/tpcc-workload-mysql.tcl.j2 rename to roles/hammerdb/templates/tpcc-workload-mariadb.tcl.j2 index 2dfae4066..4b1f0d15c 100644 --- a/roles/hammerdb/templates/tpcc-workload-mysql.tcl.j2 +++ b/roles/hammerdb/templates/tpcc-workload-mariadb.tcl.j2 @@ -43,7 +43,7 @@ diset tpcc mysql_async_scale {{workload_args.async_scale}} diset tpcc mysql_async_client {{workload_args.async_client}} diset tpcc mysql_async_verbose {{workload_args.async_verbose}} diset tpcc mysql_async_delay {{workload_args.async_delay}} - +diset connection mysql_socket {{workload_args.db_mysql_socket}} vuset logtotemp 1 loadscript diff --git a/roles/image_pull/tasks/main.yml b/roles/image_pull/tasks/main.yml new file mode 100644 index 000000000..ac199f0af --- /dev/null +++ b/roles/image_pull/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- set_fact: + resources: + - "{{ role_path }}/templates/image_pull.yml" + +- name: Generic workload - image_pull + include_role: + name: generic_workload diff --git a/roles/image_pull/templates/image_pull.yml b/roles/image_pull/templates/image_pull.yml new file mode 100644 index 000000000..0cc738231 --- /dev/null +++ b/roles/image_pull/templates/image_pull.yml @@ -0,0 +1,110 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: 'image-pull-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' +spec: + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} + parallelism: {{ workload_args.pod_count | default(1) | int }} + template: + metadata: + labels: + app: image-pull-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.tolerations is defined %} + tolerations: + - key: {{ workload_args.tolerations.key }} + value: {{ workload_args.tolerations.value }} + effect: {{ workload_args.tolerations.effect }} +{% endif %} +{% if workload_args.label is defined %} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ workload_args.label.key }} + operator: In + values: + - {{ workload_args.label.value }} +{% endif %} + containers: + - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/image_pull:latest') }} + name: pull-image + env: + - name: my_node_name + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: my_pod_name + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: uuid + value: "{{ uuid }}" + - name: test_user + value: "{{ test_user | default("ripsaw") }}" + - name: clustername + value: "{{ clustername }}" +{% if elasticsearch.url %} + - name: es + value: "{{ elasticsearch.url }}" + - name: es_index + value: "{{ elasticsearch.index_name | default("image-pull") }}" + - name: es_verify_cert + value: "{{ elasticsearch.verify_cert | default(true) }}" + - name: parallel + value: "{{ elasticsearch.parallel | default(false) }}" +{% endif %} +{% if prometheus is defined %} + - name: prom_es + value: "{{ prometheus.es_url }}" + - name: prom_parallel + value: "{{ prometheus.es_parallel | default(false) }}" + - name: prom_token + value: "{{ prometheus.prom_token | default() }}" + - name: prom_url + value: "{{ prometheus.prom_url | default() }}" +{% endif %} + command: ["/bin/sh", "-c"] + args: + - > +{% for my_image in workload_args.image_list %} + echo "Testing image: {{ my_image }}"; + echo "Waiting for all pods to be Ready"; + redis-cli -h {{ bo.resources[0].status.podIP }} INCR "image-pull-{{ trunc_uuid }}" > /dev/null 2>&1; + pods=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "image-pull-{{ trunc_uuid }}"`; + while [[ $pods != {{ workload_args.pod_count | default(1) | string }} ]]; do + pods=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "image-pull-{{ trunc_uuid }}"`; + sleep .5; + done; + sleep 2; + redis-cli -h {{ bo.resources[0].status.podIP }} DECR "image-pull-{{ trunc_uuid }}" > /dev/null 2>&1; + date; + run_snafu + --tool image_pull +{% if workload_args.debug is defined and workload_args.debug %} + -v +{% endif %} + -u "{{ uuid }}" + --pod-name ${my_pod_name} + --timeout "{{ workload_args.timeout | default(600) }}" + --pod-count {{ workload_args.pod_count | default(1) | int }} + --retries {{ workload_args.retries | default(0) | int }} + --image "{{ my_image }}"; +{% endfor %} + imagePullPolicy: Always + restartPolicy: Never +{% include "metadata.yml.j2" %} diff --git a/roles/iperf3/tasks/main.yml b/roles/iperf3/tasks/main.yml index 3889060d9..b722d0843 100644 --- a/roles/iperf3/tasks/main.yml +++ b/roles/iperf3/tasks/main.yml @@ -1,30 +1,4 @@ --- -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Update current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - name: Start Server(s) @@ -32,21 +6,18 @@ definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" with_sequence: start=1 count={{ workload_args.pairs|default('1')|int }} - - name: Update state to Starting Server - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Server" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting Server" - when: resource_state.resources[0].status.state == "Starting" + when: benchmark_state.resources[0].status.state == "Building" - block: - name: Wait for pods to be running.... - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -54,22 +25,19 @@ - app = iperf3-bench-server-{{ trunc_uuid }} register: server_pods - - name: Update state to Starting Clients - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Clients" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting Clients" when: "workload_args.pairs|default('1')|int == (server_pods | json_query('resources[].status.podIP')|length) and (server_pods | json_query('resources[].status.phase') is defined and workload_args.pairs|default('1')|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length)" - when: resource_state.resources[0].status.state == "Starting Server" + when: benchmark_state.resources[0].status.state == "Starting Server" - block: - name: Get server pods - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -82,21 +50,18 @@ definition: "{{ lookup('template', 'client.yml.j2') | from_yaml }}" with_items: "{{ server_pods.resources }}" - - name: Update state to Waiting for Clients - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Waiting for Clients" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Waiting for Clients" - when: resource_state.resources[0].status.state == "Starting Clients" + when: benchmark_state.resources[0].status.state == "Starting Clients" - block: - name: Wait for pods to be running.... - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -104,36 +69,16 @@ - app = iperf3-bench-client-{{ trunc_uuid }} register: client_pods - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running when: client_pods.resources|length > 0 - when: ( workload_args.pairs|default('1')|int > 0 ) and resource_state.resources[0].status.state == "Waiting for Clients" + when: ( workload_args.pairs|default('1')|int > 0 ) and benchmark_state.resources[0].status.state == "Waiting for Clients" -- block: - - name: Waiting for Jobs to complete.... - k8s_facts: - kind: Job - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = iperf3-bench-client-{{ trunc_uuid }} - register: client_jobs - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "workload_args.pairs|default('1')|int == (client_jobs | json_query('resources[].status.succeeded')|length)" - - when: resource_state.resources[0].status.state == "Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/iperf3/templates/client.yml.j2 b/roles/iperf3/templates/client.yml.j2 index cad36864f..9b6d29904 100644 --- a/roles/iperf3/templates/client.yml.j2 +++ b/roles/iperf3/templates/client.yml.j2 @@ -5,11 +5,22 @@ metadata: name: 'iperf3-client-{{ item.status.podIP }}-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: iperf3-bench-client-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: affinity: podAntiAffinity: @@ -28,7 +39,6 @@ spec: {% endif %} {% if workload_args.hostnetwork is sameas true %} hostNetwork: true - serviceAccountName: benchmark-operator {% endif %} containers: - name: benchmark @@ -36,8 +46,19 @@ spec: imagePullPolicy: Always command: ["/bin/sh", "-c"] args: - - "iperf3 -c {{ item.status.podIP }} -p {{ workload_args.port }} --{{ workload_args.transmit_type }} {{ workload_args.transmit_value }} -l {{ workload_args.length_buffer }} -P {{ workload_args.streams }} -w {{ workload_args.window_size }} -M {{ workload_args.mss }} -S {{ workload_args.ip_tos }} -O {{ workload_args.omit_start }} {{ workload_args.extra_options_client }}" - restartPolicy: OnFailure + - "iperf3 -c + {% if workload_args.extra_options_client is defined and '-6' in workload_args.extra_options_client %} {{ item.status.podIPs[-1].ip }} + {% else %} {{ item.status.podIP }} {% endif %} + {% if workload_args.port is defined %} -p {{ workload_args.port }} {% endif %} + {% if workload_args.transmit_type is defined and workload_args.transmit_value is defined %} --{{ workload_args.transmit_type }} {{ workload_args.transmit_value }} {% endif %} + {% if workload_args.length_buffer is defined %} -l {{ workload_args.length_buffer }} {% endif %} + {% if workload_args.streams is defined %} -P {{ workload_args.streams }} {% endif %} + {% if workload_args.window_size is defined %} -w {{ workload_args.window_size }} {% endif %} + {% if workload_args.mss is defined %} -M {{ workload_args.mss }} {% endif %} + {% if workload_args.ip_tos is defined %} -S {{ workload_args.ip_tos }} {% endif %} + {% if workload_args.omit_start is defined %} -O {{ workload_args.omit_start }} {% endif %} + {% if workload_args.extra_options_client is defined %} {{ workload_args.extra_options_client }} {% endif %}" + restartPolicy: Never {% if workload_args.pin_client is defined %} nodeSelector: kubernetes.io/hostname: '{{ workload_args.pin_client }}' diff --git a/roles/iperf3/templates/client_store.yml.j2 b/roles/iperf3/templates/client_store.yml.j2 deleted file mode 100644 index 2260547b2..000000000 --- a/roles/iperf3/templates/client_store.yml.j2 +++ /dev/null @@ -1,52 +0,0 @@ ---- -kind: Job -apiVersion: batch/v1 -metadata: - name: 'iperf3-client-{{ item.status.podIP }}-{{ trunc_uuid }}' - namespace: '{{ operator_namespace }}' -spec: - ttlSecondsAfterFinished: 600 - template: - metadata: - labels: - app: iperf3-bench-client-{{ trunc_uuid }} - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - iperf3-bench-server-{{ trunc_uuid }} - topologyKey: kubernetes.io/hostname -{% if workload_args.runtime_class is defined %} - runtimeClassName: "{{ workload_args.runtime_class }}" -{% endif %} -{% if workload_args.hostnetwork is sameas true %} - hostNetwork: true - serviceAccountName: benchmark-operator -{% endif %} - containers: - - name: benchmark - image: "quay.io/cloud-bulldozer/iperf3:latest" - imagePullPolicy: Always - command: ["/bin/sh", "-c"] - args: - - "mkdir -p {{results.path}}/iperf3-{{ uuid }}; - iperf3 -c {{ item.status.podIP }} -J -p {{ workload_args.port }} --{{ workload_args.transmit_type }} {{ workload_args.transmit_value }} -l {{ workload_args.length_buffer }} -P {{ workload_args.streams }} -w {{ workload_args.window_size }} -M {{ workload_args.mss }} -S {{ workload_args.ip_tos }} -O {{ workload_args.omit_start }} {{ workload_args.extra_options_client }} >> {{results.path}}/iperf3-{{ uuid }}/iperf_result.json" - volumeMounts: - - name: result-data - mountPath: "{{results.path}}" - volumes: - - name: result-data - persistentVolumeClaim: - claimName: result-volume -{% if workload_args.pin_client is defined %} - nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' -{% endif %} - restartPolicy: OnFailure diff --git a/roles/iperf3/templates/server.yml.j2 b/roles/iperf3/templates/server.yml.j2 index d982e2d61..2d4a1c754 100644 --- a/roles/iperf3/templates/server.yml.j2 +++ b/roles/iperf3/templates/server.yml.j2 @@ -1,45 +1,52 @@ --- -kind: Job -apiVersion: batch/v1 +kind: Pod +apiVersion: v1 metadata: name: 'iperf3-server-{{ item }}-{{ trunc_uuid }}' namespace: "{{ operator_namespace }}" + labels: + benchmark-uuid: {{ uuid }} + app: iperf3-bench-server-{{ trunc_uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: - ttlSecondsAfterFinished: 600 - template: - metadata: - labels: - app: iperf3-bench-server-{{ trunc_uuid }} - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - iperf3-bench-client-{{ trunc_uuid }} - topologyKey: kubernetes.io/hostname + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - iperf3-bench-client-{{ trunc_uuid }} + topologyKey: kubernetes.io/hostname {% if workload_args.runtime_class is defined %} - runtimeClassName: "{{ workload_args.runtime_class }}" + runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} {% if workload_args.hostnetwork is sameas true %} - hostNetwork: true - serviceAccountName: benchmark-operator + hostNetwork: true {% endif %} - containers: - - name: benchmark - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/iperf3:latest') }} - imagePullPolicy: Always - command: ["/bin/sh", "-c"] - args: - - "iperf3 -s -p {{ workload_args.port }} {{ workload_args.extra_options_server }}" - restartPolicy: OnFailure + containers: + - name: benchmark + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/iperf3:latest') }} + imagePullPolicy: Always + command: ["/bin/sh", "-c"] + args: + - "iperf3 -s + {% if workload_args.port is defined %} -p {{ workload_args.port }} {% endif %} + {% if workload_args.extra_options_server is defined %} {{ workload_args.extra_options_server }} {% endif %}" + restartPolicy: Never {% if workload_args.pin_server is defined %} - nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_server }}' + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_server }}' {% endif %} -{% include "metadata.yml.j2" %} +{% include "metadata_pod.yml.j2" %} diff --git a/roles/kernel_cache_drop/tasks/main.yml b/roles/kernel_cache_drop/tasks/main.yml index f8b486b2f..8b0ef4180 100644 --- a/roles/kernel_cache_drop/tasks/main.yml +++ b/roles/kernel_cache_drop/tasks/main.yml @@ -10,15 +10,15 @@ shell: "python3 /opt/ansible/roles/kernel_cache_drop/wait_for_daemonset.py 30 {{ operator_namespace }} kernel-cache-dropper" - name: get kernel cache dropper pods - k8s_facts: + k8s_info: kind: Pod label_selectors: - name = kernel-cache-dropper namespace: "{{ operator_namespace }}" register: kcache_drop_pod_list -- debug: - var: kcache_drop_pod_list +#- debug: +# var: kcache_drop_pod_list - name: put ip list into a var set_fact: diff --git a/roles/kube-burner/files/allow-http.yml b/roles/kube-burner/files/allow-http.yml new file mode 100644 index 000000000..dcda8c641 --- /dev/null +++ b/roles/kube-burner/files/allow-http.yml @@ -0,0 +1,12 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-{{.Replica}}-{{.Iteration}} +spec: + podSelector: + matchLabels: + name: webserver-{{.Replica}}-{{.Iteration}} + ingress: + - ports: + - protocol: TCP + port: 8080 diff --git a/roles/kube-burner/files/app-deployment.yml b/roles/kube-burner/files/app-deployment.yml index 026e4de77..6043f37fa 100644 --- a/roles/kube-burner/files/app-deployment.yml +++ b/roles/kube-burner/files/app-deployment.yml @@ -13,6 +13,7 @@ spec: containers: - name: perfapp image: quay.io/rsevilla/perfapp:latest + imagePullPolicy: Always readinessProbe: httpGet: path: /ready @@ -21,6 +22,14 @@ spec: failureThreshold: 1 timeoutSeconds: 60 initialDelaySeconds: 30 + livenessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: {{ .livenessPeriod }} + failureThreshold: 1 + timeoutSeconds: 15 + initialDelaySeconds: 30 ports: - containerPort: 8080 protocol: TCP @@ -37,7 +46,6 @@ spec: value: '5432' - name: POSTGRESQL_RETRY_INTERVAL value: '5' - imagePullPolicy: IfNotPresent securityContext: privileged: false restartPolicy: Always diff --git a/roles/kube-burner/files/buildconfig_triggers.yml b/roles/kube-burner/files/buildconfig_triggers.yml new file mode 100644 index 000000000..44c836469 --- /dev/null +++ b/roles/kube-burner/files/buildconfig_triggers.yml @@ -0,0 +1,39 @@ +--- +kind: BuildConfig +apiVersion: build.openshift.io/v1 +metadata: + name: {{.JobName}}-{{.Replica}} + annotations: + template.alpha.openshift.io/wait-for-ready: 'true' +spec: + nodeSelector: + {{.nodeSelectorKey}}: {{.nodeSelectorValue}} + triggers: + - type: GitHub + github: + secret: {{.JobName}}-{{.Replica}} + - type: ImageChange + - type: ConfigChange + source: + git: + uri: {{.gitUri}} + type: Git + strategy: + type: Source + sourceStrategy: +{{ if index . "sourceStratEnv" }} + env: + - name: {{.sourceStratEnv}} +{{ end }} + from: + kind: ImageStreamTag + name: {{.fromSource}}:{{.fromSourceVersion}} + namespace: 'openshift' +{{ if index . "postCommitScript" }} + postCommit: + script: {{.postCommitScript}} +{{ end }} + output: + to: + kind: ImageStreamTag + name: {{.imageStream}}-{{.Replica}}:latest diff --git a/roles/kube-burner/files/case2-networkpolicy.yaml b/roles/kube-burner/files/case2-networkpolicy.yaml new file mode 100644 index 000000000..1aefbfbaa --- /dev/null +++ b/roles/kube-burner/files/case2-networkpolicy.yaml @@ -0,0 +1,16 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: case2-{{.podselector_label}}{{.Replica}}-{{.ingress_label}} +spec: + podSelector: + matchLabels: + {{.podselector_label_num}}: {{.podselector_label}}-{{.Replica}} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + {{.ingress_label_num}}: {{.ingress_label}}-{{randInteger 1 5}} + diff --git a/roles/kube-burner/files/case3-networkpolicy.yaml b/roles/kube-burner/files/case3-networkpolicy.yaml new file mode 100644 index 000000000..9e7cee4f2 --- /dev/null +++ b/roles/kube-burner/files/case3-networkpolicy.yaml @@ -0,0 +1,17 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: case3-{{.podselector_label}}{{.Replica}}-{{.ingress_label}} +spec: + podSelector: + matchLabels: + {{.podselector_label_num}}: {{.podselector_label}}-{{.Replica}} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchExpressions: + - key: {{.ingress_label_num}} + operator: NotIn + values: [{{.ingress_label}}-{{randInteger 1 5}}] diff --git a/roles/kube-burner/files/cross-ns-networkpolicy.yaml b/roles/kube-burner/files/cross-ns-networkpolicy.yaml new file mode 100644 index 000000000..818e11999 --- /dev/null +++ b/roles/kube-burner/files/cross-ns-networkpolicy.yaml @@ -0,0 +1,18 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: cross-ns-{{randInteger 1 1000000}} +spec: + podSelector: + matchLabels: + label1: foo-{{randInteger 1 5}} + policyTypes: + - Ingress + ingress: + - from: + - podSelector: + matchLabels: + label1: foo-{{randInteger 1 5}} + namespaceSelector: + matchLabels: + ns_label: bar-{{randInteger 1 4}} diff --git a/roles/kube-burner/files/curl-deployment.yml b/roles/kube-burner/files/curl-deployment.yml new file mode 100644 index 000000000..ad415900a --- /dev/null +++ b/roles/kube-burner/files/curl-deployment.yml @@ -0,0 +1,46 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: curl-{{.Replica}}-{{.Iteration}} +spec: + template: + metadata: + labels: + name: curl-{{.Replica}}-{{.Iteration}} + spec: + nodeSelector: + {{.nodeSelectorKey}}: {{.nodeSelectorValue}} + containers: + - name: curlapp + image: quay.io/cloud-bulldozer/curl:latest + command: ["sleep", "inf"] + resources: + requests: + memory: "10Mi" + cpu: "10m" + env: + - name: WEBSERVER_HOSTNAME + value: webserver-{{.Replica}}-{{.Iteration}} + - name: WEBSERVER_PORT + value: "8080" + imagePullPolicy: IfNotPresent + securityContext: + privileged: false + startupProbe: + exec: + command: + - "/bin/sh" + - "-c" + - "curl ${WEBSERVER_HOSTNAME}:${WEBSERVER_PORT}" + periodSeconds: 1 + timeoutSeconds: 1 + failureThreshold: 600 + restartPolicy: Always + replicas: 1 + selector: + matchLabels: + name: curl-{{.Replica}}-{{.Iteration}} + triggers: + - type: ConfigChange + strategy: + type: RollingUpdate diff --git a/roles/kube-burner/files/deny-all.yml b/roles/kube-burner/files/deny-all.yml new file mode 100644 index 000000000..599bf171b --- /dev/null +++ b/roles/kube-burner/files/deny-all.yml @@ -0,0 +1,7 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: deny-by-default +spec: + podSelector: {} + ingress: [] diff --git a/roles/kube-burner/files/metrics-aggregated.yaml b/roles/kube-burner/files/metrics-aggregated.yaml index 7ca4445b0..10093b637 100644 --- a/roles/kube-burner/files/metrics-aggregated.yaml +++ b/roles/kube-burner/files/metrics-aggregated.yaml @@ -1,183 +1,209 @@ -metrics: - - query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|sdn|ovn-kubernetes|.*apiserver|authentication|.*controller-manager|.*scheduler)"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 - metricName: containerCPU-Masters +# API server +- query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb!~"WATCH", subresource!="log"}[2m])) by (verb,resource,subresource,instance,le)) > 0 + metricName: API99thLatency - - query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress)"}[2m]) * 100 and on (node) kube_node_role{role="worker"}) by (namespace, container)) > 0 - metricName: containerCPU-AggregatedWorkers +- query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH",subresource!="log"}[2m])) by (verb,instance,resource,code) > 0 + metricName: APIRequestRate - - query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring)"}[2m]) * 100 and on (node) kube_node_role{role="infra"}) by (namespace, container)) > 0 - metricName: containerCPU-AggregatedInfra +# P&F +- query: rate(apiserver_flowcontrol_dispatched_requests_total[2m]) > 0 + metricName: APIFlowControlDispatchedRequests - - query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{verb!="WATCH"}[5m])) by (le, verb)) > 0 - metricName: API99thLatency +- query: sum(apiserver_flowcontrol_current_executing_requests) by (instance, priority_level,flow_schema) > 0 + metricName: APIFlowControlCurrentExecutingRequests - - query: (sum(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler)"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 - metricName: containerMemory-Masters +- query: rate(apiserver_flowcontrol_rejected_requests_total[2m]) > 0 + metricName: APIFlowControlRejectedRequests - - query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress)"} and on (node) kube_node_role{role="worker"}) by (container, namespace) - metricName: containerMemory-AggregatedWorkers +- query: sum(apiserver_flowcontrol_current_inqueue_requests) by (instance, flow_schema, priority_level) > 0 + metricName: APIFlowControlInqueueRequests - - query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring)"} and on (node) kube_node_role{role="infra"}) by (container, namespace) - metricName: containerMemory-AggregatedInfra +- query: avg(apiserver_flowcontrol_request_concurrency_limit{}) by (priority_level) + metricName: APIFlowControlRequestConcurrencyLimit + instant: true - - query: avg(irate(process_cpu_seconds_total{service="kubelet",job="kubelet"}[2m]) * 100 and on (node) kube_node_role{role="worker"}) - metricName: KubeletCPU-AggregatedWorkers +# Containers & pod metrics +- query: (sum(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|image-registry|operator-lifecycle-manager)"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 + metricName: containerMemory-Masters - - query: avg(process_resident_memory_bytes{service="kubelet",job="kubelet"} and on (node) kube_node_role{role="worker"}) - metricName: KubeletMemory-AggregatedWorkers +- query: (sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|sdn|ovn-kubernetes|.*apiserver|authentication|.*controller-manager|.*scheduler|image-registry|operator-lifecycle-manager)"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="master"}) > 0 + metricName: containerCPU-Masters - - query: avg(irate(process_cpu_seconds_total{service="kubelet",job="crio"}[2m]) * 100 and on (node) kube_node_role{role="worker"}) - metricName: CrioCPU-AggregatedWorkers +- query: (sum(irate(container_cpu_usage_seconds_total{pod!="",container="prometheus",namespace="openshift-monitoring"}[2m]) * 100) by (container, pod, namespace, node) and on (node) kube_node_role{role="infra"}) > 0 + metricName: containerCPU-Prometheus - - query: avg(process_resident_memory_bytes{service="kubelet",job="crio"} and on (node) kube_node_role{role="worker"}) - metricName: CrioMemory-AggregatedWorkers +- query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress)"}[2m]) * 100 and on (node) kube_node_role{role="worker"}) by (namespace, container)) > 0 + metricName: containerCPU-AggregatedWorkers - - query: (sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)")) > 0 - metricName: nodeCPU-Masters +- query: (avg(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|operator-lifecycle-manager)"}[2m]) * 100 and on (node) kube_node_role{role="infra"}) by (namespace, container)) > 0 + metricName: containerCPU-AggregatedInfra - - query: (avg((sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)"))) by (mode)) > 0 - metricName: nodeCPU-AggregatedWorkers +- query: (sum(container_memory_rss{pod!="",namespace="openshift-monitoring",name!="",container="prometheus"}) by (container, pod, namespace, node) and on (node) kube_node_role{role="infra"}) > 0 + metricName: containerMemory-Prometheus - - query: (avg((sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)"))) by (mode)) > 0 - metricName: nodeCPU-AggregatedInfra +- query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress)"} and on (node) kube_node_role{role="worker"}) by (container, namespace) + metricName: containerMemory-AggregatedWorkers - - query: avg(node_memory_MemAvailable_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: nodeMemoryAvailable-Masters +- query: avg(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(sdn|ovn-kubernetes|ingress|monitoring|image-registry|operator-lifecycle-manager)"} and on (node) kube_node_role{role="infra"}) by (container, namespace) + metricName: containerMemory-AggregatedInfra - - query: avg(node_memory_MemAvailable_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) - metricName: nodeMemoryAvailable-AggregatedWorkers +# Node metrics +- query: (sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)")) > 0 + metricName: nodeCPU-Masters - - query: avg(node_memory_MemAvailable_bytes and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) - metricName: nodeMemoryAvailable-AggregatedInfra +- query: (avg((sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)"))) by (mode)) > 0 + metricName: nodeCPU-AggregatedWorkers - - query: avg(node_memory_Active_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: nodeMemoryActive-Masters +- query: (avg((sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)"))) by (mode)) > 0 + metricName: nodeCPU-AggregatedInfra - - query: avg(node_memory_Active_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) - metricName: nodeMemoryActive-AggregatedWorkers +- query: avg(node_memory_MemAvailable_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") + metricName: nodeMemoryAvailable-Masters - - query: avg(avg(node_memory_Active_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) - metricName: nodeMemoryActive-AggregatedInfra +- query: avg(node_memory_MemAvailable_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) + metricName: nodeMemoryAvailable-AggregatedWorkers - - query: avg(node_memory_Cached_bytes) by (instance) + avg(node_memory_Buffers_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: nodeMemoryCached+nodeMemoryBuffers-Masters +- query: avg(node_memory_MemAvailable_bytes and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) + metricName: nodeMemoryAvailable-AggregatedInfra - - query: avg(node_memory_Cached_bytes + node_memory_Buffers_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) - metricName: nodeMemoryCached+nodeMemoryBuffers-AggregatedWorkers +- query: avg(node_memory_MemTotal_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") + metricName: nodeMemoryTotal-Masters + instant: true - - query: avg(node_memory_Cached_bytes + node_memory_Buffers_bytes and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) - metricName: nodeMemoryCached+nodeMemoryBuffers-AggregatedInfra +- query: avg(node_memory_MemTotal_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) + metricName: nodeMemoryTotal-AggregatedWorkers + instant: true - - query: irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: rxNetworkBytes-Masters +- query: avg(node_memory_MemTotal_bytes and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) + metricName: nodeMemoryTotal-AggregatedInfra + instant: true - - query: avg(irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: rxNetworkBytes-AggregatedWorkers +- query: avg(node_memory_Cached_bytes) by (instance) + avg(node_memory_Buffers_bytes) by (instance) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") + metricName: nodeMemoryCached+nodeMemoryBuffers-Masters - - query: avg(irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: rxNetworkBytes-AggregatedInfra +- query: avg(node_memory_Cached_bytes + node_memory_Buffers_bytes and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) + metricName: nodeMemoryCached+nodeMemoryBuffers-AggregatedWorkers - - query: irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: txNetworkBytes-Masters +- query: avg(node_memory_Cached_bytes + node_memory_Buffers_bytes and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) + metricName: nodeMemoryCached+nodeMemoryBuffers-AggregatedInfra - - query: avg(irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: txNetworkBytes-AggregatedWorkers +- query: irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") + metricName: rxNetworkBytes-Masters - - query: avg(irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: txNetworkBytes-AggregatedInfra +- query: avg(irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: rxNetworkBytes-AggregatedWorkers - - query: (irate(node_network_receive_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)")) > 0 - metricName: rxDroppedPackets-Masters +- query: avg(irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: rxNetworkBytes-AggregatedInfra - - query: (avg(irate(node_network_receive_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device)) > 0 - metricName: rxDroppedPackets-AggregatedWorkers +- query: irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") + metricName: txNetworkBytes-Masters - - query: (avg(irate(node_network_receive_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device)) > 0 - metricName: rxDroppedPackets-AggregatedInfra +- query: avg(irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: txNetworkBytes-AggregatedWorkers - - query: irate(node_network_transmit_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: txDroppedPackets-Masters +- query: avg(irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: txNetworkBytes-AggregatedInfra - - query: (avg(irate(node_network_transmit_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device)) > 0 - metricName: txDroppedPackets-AggregatedWorkers +- query: rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") + metricName: nodeDiskWrittenBytes-Masters - - query: (avg(irate(node_network_transmit_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device)) > 0 - metricName: txDroppedPackets-AggregatedInfra +- query: avg(rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: nodeDiskWrittenBytes-AggregatedWorkers - - query: rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: nodeDiskWrittenBytes-Masters +- query: avg(rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: nodeDiskWrittenBytes-AggregatedInfra - - query: avg(rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: nodeDiskWrittenBytes-AggregatedWorkers +- query: rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") + metricName: nodeDiskReadBytes-Masters - - query: avg(rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: nodeDiskWrittenBytes-AggregatedInfra +- query: avg(rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: nodeDiskReadBytes-AggregatedWorkers - - query: rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="master"}, "instance", "$1", "node", "(.+)") - metricName: nodeDiskReadBytes-Masters +- query: avg(rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) + metricName: nodeDiskReadBytes-AggregatedInfra - - query: avg(rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="worker"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: nodeDiskReadBytes-AggregatedWorkers +- query: irate(container_runtime_crio_operations_latency_microseconds{operation_type="network_setup_pod"}[2m]) > 0 + metricName: containerNetworkSetupLatency - - query: avg(rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) and on (instance) label_replace(kube_node_role{role="infra"}, "instance", "$1", "node", "(.+)")) by (device) - metricName: nodeDiskReadBytes-AggregatedInfra +- query: irate(container_runtime_crio_operations_latency_microseconds{operation_type="network_setup_overall"}[2m]) > 0 + metricName: containerNetworkSetupOverallLatency - - query: sum(rate(etcd_server_leader_changes_seen_total[2m])) - metricName: etcdLeaderChangesRate +# Etcd metrics +- query: sum(rate(etcd_server_leader_changes_seen_total[2m])) + metricName: etcdLeaderChangesRate - - query: etcd_server_is_leader > 0 - metricName: etcdServerIsLeader +- query: etcd_server_is_leader > 0 + metricName: etcdServerIsLeader - - query: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m])) - metricName: 99thEtcdDiskBackendCommitDurationSeconds +- query: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m])) + metricName: 99thEtcdDiskBackendCommitDurationSeconds - - query: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m])) - metricName: 99thEtcdDiskWalFsyncDurationSeconds +- query: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m])) + metricName: 99thEtcdDiskWalFsyncDurationSeconds - - query: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) - metricName: 99thEtcdRoundTripTimeSeconds +- query: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) + metricName: 99thEtcdRoundTripTimeSeconds - - query: etcd_mvcc_db_total_size_in_bytes - metricName: etcdDBPhysicalSizeBytes +- query: etcd_mvcc_db_total_size_in_bytes + metricName: etcdDBPhysicalSizeBytes - - query: etcd_mvcc_db_total_size_in_use_in_bytes - metricName: etcdDBLogicalSizeBytes +- query: etcd_mvcc_db_total_size_in_use_in_bytes + metricName: etcdDBLogicalSizeBytes - - query: count(kube_namespace_created) - metricName: namespaceCount +- query: sum by (cluster_version)(etcd_cluster_version) + metricName: etcdVersion + instant: true - - query: sum(kube_pod_status_phase{}) by (phase) - metricName: podStatusCount +- query: sum(rate(etcd_object_counts{}[5m])) by (resource) > 0 + metricName: etcdObjectCount - - query: count(kube_secret_info{}) - metricName: secretCount +- query: histogram_quantile(0.99,sum(rate(etcd_request_duration_seconds_bucket[2m])) by (le,operation,apiserver)) > 0 + metricName: P99APIEtcdRequestLatency - - query: count(kube_deployment_labels{}) - metricName: deploymentCount +# Cluster metrics +- query: sum(kube_namespace_status_phase) by (phase) > 0 + metricName: namespaceCount - - query: count(kube_configmap_info{}) - metricName: configmapCount +- query: sum(kube_pod_status_phase{}) by (phase) + metricName: podStatusCount - - query: count(kube_service_info{}) - metricName: serviceCount +- query: count(kube_secret_info{}) + metricName: secretCount - - query: kube_node_role - metricName: nodeRoles - instant: true +- query: count(kube_deployment_labels{}) + metricName: deploymentCount - - query: sum(kube_node_status_condition{status="true"}) by (condition) - metricName: nodeStatus +- query: count(kube_configmap_info{}) + metricName: configmapCount - - query: (sum(rate(container_fs_writes_bytes_total{container!="",device!~".+dm.+"}[5m])) by (device, container, node) and on (node) kube_node_role{role="master"}) > 0 - metricName: containerDiskUsage +- query: count(kube_service_info{}) + metricName: serviceCount - - query: sum(rate(etcd_object_counts{}[5m])) by (resource) > 0 - metricName: etcdObjectCount +- query: kube_node_role + metricName: nodeRoles + instant: true - - query: cluster_version{type="completed"} - metricName: clusterVersion - instant: true +- query: sum(kube_node_status_condition{status="true"}) by (condition) + metricName: nodeStatus - - query: sum by (cluster_version)(etcd_cluster_version) - metricName: etcdVersion - instant: true +- query: (sum(rate(container_fs_writes_bytes_total{container!="",device!~".+dm.+"}[5m])) by (device, container, node) and on (node) kube_node_role{role="master"}) > 0 + metricName: containerDiskUsage + +- query: cluster_version{type="completed"} + metricName: clusterVersion + instant: true + +# Golang metrics + +- query: go_memstats_heap_alloc_bytes{job=~"apiserver|api|etcd"} + metricName: goHeapAllocBytes + +- query: go_memstats_heap_inuse_bytes{job=~"apiserver|api|etcd"} + metricName: goHeapInuseBytes + +- query: go_gc_duration_seconds{job=~"apiserver|api|etcd",quantile="1"} + metricName: goGCDurationSeconds + +- query: topk(10,ALERTS{severity!="none"}) + metricName: alerts diff --git a/roles/kube-burner/files/metrics.yaml b/roles/kube-burner/files/metrics.yaml index 452df2e8f..0db189309 100644 --- a/roles/kube-burner/files/metrics.yaml +++ b/roles/kube-burner/files/metrics.yaml @@ -1,111 +1,140 @@ -metrics: - - query: sum(irate(container_cpu_usage_seconds_total{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler)"}[2m]) * 100) by (pod, namespace, node) - metricName: podCPU +# API server +- query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver="kube-apiserver", verb!~"WATCH", subresource!="log"}[2m])) by (verb,resource,subresource,instance,le)) > 0 + metricName: API99thLatency - - query: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{verb!="WATCH"}[5m])) by (le, verb)) > 0 - metricName: API99thLatency +- query: sum(irate(apiserver_request_total{apiserver="kube-apiserver",verb!="WATCH",subresource!="log"}[2m])) by (verb,instance,resource,code) > 0 + metricName: APIRequestRate - - query: sum(container_memory_rss{name!="",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler)"}) by (pod, namespace, node) - metricName: podMemory +# P&F +- query: rate(apiserver_flowcontrol_dispatched_requests_total[2m]) > 0 + metricName: APIFlowControlDispatchedRequests - - query: sum(irate(process_cpu_seconds_total{service="kubelet",job="kubelet"}[2m]) * 100) by (node) and on (node) kube_node_role{role="worker"} - metricName: kubeletCPU +- query: sum(apiserver_flowcontrol_current_executing_requests) by (instance, priority_level,flow_schema) > 0 + metricName: APIFlowControlCurrentExecutingRequests - - query: sum(process_resident_memory_bytes{service="kubelet",job="kubelet"}) by (node) and on (node) kube_node_role{role="worker"} - metricName: kubeletMemory +- query: rate(apiserver_flowcontrol_rejected_requests_total[2m]) > 0 + metricName: APIFlowControlRejectedRequests - - query: sum(irate(process_cpu_seconds_total{service="kubelet",job="crio"}[2m]) * 100) by (node) and on (node) kube_node_role{role="worker"} - metricName: crioCPU +- query: sum(apiserver_flowcontrol_current_inqueue_requests) by (instance, flow_schema, priority_level) > 0 + metricName: APIFlowControlInqueueRequests - - query: sum(process_resident_memory_bytes{service="kubelet",job="crio"}) by (node) and on (node) kube_node_role{role="worker"} - metricName: crioMemory +- query: avg(apiserver_flowcontrol_request_concurrency_limit{}) by (priority_level) + metricName: APIFlowControlRequestConcurrencyLimit + instant: true - - query: sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) > 0 - metricName: nodeCPU +# Containers & pod metrics +- query: sum(irate(container_cpu_usage_seconds_total{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|image-registry|operator-lifecycle-manager)"}[2m]) * 100) by (container, pod, namespace, node) + metricName: containerCPU - - query: avg(node_memory_MemAvailable_bytes) by (instance) - metricName: nodeMemoryAvailable +- query: sum(container_memory_rss{name!="",container!="POD",namespace=~"openshift-(etcd|oauth-apiserver|.*apiserver|ovn-kubernetes|sdn|ingress|authentication|.*controller-manager|.*scheduler|monitoring|image-registry|operator-lifecycle-manager)"}) by (container, pod, namespace, node) + metricName: containerMemory - - query: avg(node_memory_Active_bytes) by (instance) - metricName: nodeMemoryActive +- query: (sum(rate(container_fs_writes_bytes_total{container!="",device!~".+dm.+"}[5m])) by (device, container, node) and on (node) kube_node_role{role="master"}) > 0 + metricName: containerDiskUsage - - query: avg(node_memory_Cached_bytes) by (instance) + avg(node_memory_Buffers_bytes) by (instance) - metricName: nodeMemoryCached+nodeMemoryBuffers +# Kubelet & CRI-O metrics +- query: sum(irate(process_cpu_seconds_total{service="kubelet",job="kubelet"}[2m]) * 100) by (node) and on (node) kube_node_role{role="worker"} + metricName: kubeletCPU - - query: irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) - metricName: rxNetworkBytes +- query: sum(process_resident_memory_bytes{service="kubelet",job="kubelet"}) by (node) and on (node) kube_node_role{role="worker"} + metricName: kubeletMemory - - query: irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) - metricName: txNetworkBytes +- query: sum(irate(process_cpu_seconds_total{service="kubelet",job="crio"}[2m]) * 100) by (node) and on (node) kube_node_role{role="worker"} + metricName: crioCPU - - query: irate(node_network_receive_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) - metricName: rxDroppedPackets +- query: sum(process_resident_memory_bytes{service="kubelet",job="crio"}) by (node) and on (node) kube_node_role{role="worker"} + metricName: crioMemory - - query: irate(node_network_transmit_drop_total{device=~"^(ens|eth|bond|team).*"}[2m]) - metricName: txDroppedPackets +- query: irate(container_runtime_crio_operations_latency_microseconds{operation_type="network_setup_pod"}[2m]) > 0 + metricName: containerNetworkSetupLatency - - query: rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) - metricName: nodeDiskWrittenBytes +- query: irate(container_runtime_crio_operations_latency_microseconds{operation_type="network_setup_overall"}[2m]) > 0 + metricName: containerNetworkSetupOverallLatency - - query: rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) - metricName: nodeDiskReadBytes +# Node metrics +- query: sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) > 0 + metricName: nodeCPU - - query: sum(rate(etcd_server_leader_changes_seen_total[2m])) - metricName: etcdLeaderChangesRate +- query: avg(node_memory_MemAvailable_bytes) by (instance) + metricName: nodeMemoryAvailable - - query: etcd_server_is_leader > 0 - metricName: etcdServerIsLeader +- query: avg(node_memory_MemTotal_bytes) by (instance) + metricName: nodeMemoryTotal + instant: true - - query: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m])) - metricName: 99thEtcdDiskBackendCommitDurationSeconds +- query: avg(node_memory_Cached_bytes) by (instance) + avg(node_memory_Buffers_bytes) by (instance) + metricName: nodeMemoryCached+nodeMemoryBuffers - - query: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m])) - metricName: 99thEtcdDiskWalFsyncDurationSeconds +- query: irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) + metricName: rxNetworkBytes - - query: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) - metricName: 99thEtcdRoundTripTimeSeconds +- query: irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) + metricName: txNetworkBytes - - query: etcd_mvcc_db_total_size_in_bytes - metricName: etcdDBPhysicalSizeBytes +- query: rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) + metricName: nodeDiskWrittenBytes - - query: etcd_mvcc_db_total_size_in_use_in_bytes - metricName: etcdDBLogicalSizeBytes +- query: rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) + metricName: nodeDiskReadBytes - - query: count(kube_namespace_created) - metricName: namespaceCount +# Etcd metrics +- query: sum(rate(etcd_server_leader_changes_seen_total[2m])) + metricName: etcdLeaderChangesRate - - query: sum(kube_pod_status_phase{}) by (phase) - metricName: podStatusCount +- query: etcd_server_is_leader > 0 + metricName: etcdServerIsLeader - - query: count(kube_secret_info{}) - metricName: secretCount +- query: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m])) + metricName: 99thEtcdDiskBackendCommitDurationSeconds - - query: count(kube_deployment_labels{}) - metricName: deploymentCount +- query: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m])) + metricName: 99thEtcdDiskWalFsyncDurationSeconds - - query: count(kube_configmap_info{}) - metricName: configmapCount +- query: histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) + metricName: 99thEtcdRoundTripTimeSeconds - - query: count(kube_service_info{}) - metricName: serviceCount +- query: etcd_mvcc_db_total_size_in_bytes + metricName: etcdDBPhysicalSizeBytes - - query: kube_node_role - metricName: nodeRoles - instant: true +- query: etcd_mvcc_db_total_size_in_use_in_bytes + metricName: etcdDBLogicalSizeBytes - - query: sum(kube_node_status_condition{status="true"}) by (condition) - metricName: nodeStatus +- query: sum(rate(etcd_object_counts{}[5m])) by (resource) > 0 + metricName: etcdObjectCount - - query: (sum(rate(container_fs_writes_bytes_total{container!="",device!~".+dm.+"}[5m])) by (device, container, node) and on (node) kube_node_role{role="master"}) > 0 - metricName: containerDiskUsage +- query: sum by (cluster_version)(etcd_cluster_version) + metricName: etcdVersion + instant: true - - query: sum(rate(etcd_object_counts{}[5m])) by (resource) > 0 - metricName: etcdObjectCount +# Cluster metrics +- query: sum(kube_namespace_status_phase) by (phase) > 0 + metricName: namespaceCount - - query: cluster_version{type="completed"} - metricName: clusterVersion - instant: true +- query: sum(kube_pod_status_phase{}) by (phase) + metricName: podStatusCount - - query: sum by (cluster_version)(etcd_cluster_version) - metricName: etcdVersion - instant: true +- query: count(kube_secret_info{}) + metricName: secretCount + +- query: count(kube_deployment_labels{}) + metricName: deploymentCount + +- query: count(kube_configmap_info{}) + metricName: configmapCount + +- query: count(kube_service_info{}) + metricName: serviceCount + +- query: kube_node_role + metricName: nodeRoles + instant: true + +- query: sum(kube_node_status_condition{status="true"}) by (condition) + metricName: nodeStatus + +- query: cluster_version{type="completed"} + metricName: clusterVersion + instant: true + +- query: topk(10,ALERTS{severity!="none"}) + metricName: alerts diff --git a/roles/kube-burner/files/networkpolicy-deny-all.yaml b/roles/kube-burner/files/networkpolicy-deny-all.yaml new file mode 100644 index 000000000..64548414e --- /dev/null +++ b/roles/kube-burner/files/networkpolicy-deny-all.yaml @@ -0,0 +1,8 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-all +spec: + podSelector: {} + policyTypes: + - Ingress diff --git a/roles/kube-burner/files/networkpolicy-ingress.yaml b/roles/kube-burner/files/networkpolicy-ingress.yaml new file mode 100644 index 000000000..c180e2736 --- /dev/null +++ b/roles/kube-burner/files/networkpolicy-ingress.yaml @@ -0,0 +1,13 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-from-openshift-ingress +spec: + ingress: + - from: + - namespaceSelector: + matchLabels: + network.openshift.io/policy-group: ingress + podSelector: {} + policyTypes: + - Ingress diff --git a/roles/kube-burner/files/networkpolicy-monitoring.yaml b/roles/kube-burner/files/networkpolicy-monitoring.yaml new file mode 100644 index 000000000..941162c3b --- /dev/null +++ b/roles/kube-burner/files/networkpolicy-monitoring.yaml @@ -0,0 +1,13 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-from-openshift-monitoring +spec: + ingress: + - from: + - namespaceSelector: + matchLabels: + network.openshift.io/policy-group: monitoring + podSelector: {} + policyTypes: + - Ingress diff --git a/roles/kube-burner/files/networkpolicy-same-ns.yaml b/roles/kube-burner/files/networkpolicy-same-ns.yaml new file mode 100644 index 000000000..0517b1767 --- /dev/null +++ b/roles/kube-burner/files/networkpolicy-same-ns.yaml @@ -0,0 +1,9 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-same-namespace +spec: + podSelector: + ingress: + - from: + - podSelector: {} diff --git a/roles/kube-burner/files/nginx-cross-ns.yaml b/roles/kube-burner/files/nginx-cross-ns.yaml new file mode 100644 index 000000000..2e1a1afa7 --- /dev/null +++ b/roles/kube-burner/files/nginx-cross-ns.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-{{.Replica}}-{{.Iteration}}-{{randInteger 0 1000000}} + labels: + label1: {{.label1}}-{{randInteger 1 5}} +spec: + containers: + - name: nginx + image: quay.io/openshift-scale/nginx:latest + ports: + - containerPort: 8080 diff --git a/roles/kube-burner/files/nginx.yaml b/roles/kube-burner/files/nginx.yaml new file mode 100644 index 000000000..35c14fb3a --- /dev/null +++ b/roles/kube-burner/files/nginx.yaml @@ -0,0 +1,14 @@ + +apiVersion: v1 +kind: Pod +metadata: + name: nginx-{{.Replica}}-{{.Iteration}}-{{randInteger 0 1000000}} + labels: + label1: {{.label1}}-{{.Replica}} + label2: {{.label2}}-{{.Replica}} +spec: + containers: + - name: nginx + image: quay.io/openshift-scale/nginx:latest + ports: + - containerPort: 8080 diff --git a/roles/kube-burner/files/webserver-deployment.yml b/roles/kube-burner/files/webserver-deployment.yml new file mode 100644 index 000000000..fa97b1e04 --- /dev/null +++ b/roles/kube-burner/files/webserver-deployment.yml @@ -0,0 +1,34 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: webserver-{{.Replica}}-{{.Iteration}} +spec: + template: + metadata: + labels: + name: webserver-{{.Replica}}-{{.Iteration}} + spec: + nodeSelector: + {{.nodeSelectorKey}}: {{.nodeSelectorValue}} + containers: + - name: webserver + image: quay.io/cloud-bulldozer/sampleapp:latest + resources: + requests: + memory: "10Mi" + cpu: "10m" + ports: + - containerPort: 8080 + protocol: TCP + imagePullPolicy: IfNotPresent + securityContext: + privileged: false + restartPolicy: Always + replicas: 1 + selector: + matchLabels: + name: webserver-{{.Replica}}-{{.Iteration}} + triggers: + - type: ConfigChange + strategy: + type: RollingUpdate diff --git a/roles/kube-burner/files/webserver-service.yml b/roles/kube-burner/files/webserver-service.yml new file mode 100644 index 000000000..a569151b8 --- /dev/null +++ b/roles/kube-burner/files/webserver-service.yml @@ -0,0 +1,12 @@ +kind: Service +apiVersion: v1 +metadata: + name: webserver-{{.Replica}}-{{.Iteration}} +spec: + selector: + name: webserver-{{.Replica}}-{{.Iteration}} + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 + type: ClusterIP diff --git a/roles/kube-burner/tasks/main.yml b/roles/kube-burner/tasks/main.yml index eb4792007..e245a9a67 100644 --- a/roles/kube-burner/tasks/main.yml +++ b/roles/kube-burner/tasks/main.yml @@ -1,40 +1,4 @@ --- - -- name: Get current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - If it has changed - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- name: Capture operator information - k8s_info: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - name = benchmark-operator - register: bo - - block: - name: Create metrics profile for remote configuration job @@ -50,136 +14,219 @@ when: - workload_args.remote_config is defined - workload_args.remote_config - - workload_args.remote_metrics_profile is not defined - - workload_args.remote_metrics_profile - - - name: Create cluster-density configmaps - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: kube-burner-config-{{ trunc_uuid }} - namespace: "{{ operator_namespace }}" - data: - config.yml: "{{ lookup('template', 'cluster-density.yml.j2')}}" - metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" - build.yml: "{{ lookup('file', 'build.yml')}}" - buildconfig.yml: "{{ lookup('file', 'buildconfig.yml')}}" - deployment.yml: "{{ lookup('file', 'deployment.yml')}}" - imagestream.yml: "{{ lookup('file', 'imagestream.yml')}}" - route.yml: "{{ lookup('file', 'route.yml')}}" - secret.yml: "{{ lookup('file', 'secret.yml')}}" - service.yml: "{{ lookup('file', 'service.yml')}}" - configmap.yml: "{{ lookup('file', 'configmap.yml')}}" - when: workload_args.workload == "cluster-density" - - - name: Create node-density or pod-density configmaps - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: kube-burner-config-{{ trunc_uuid }} - namespace: "{{ operator_namespace }}" - data: - config.yml: "{{ lookup('template', 'node-pod-density.yml.j2')}}" - metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics.yaml')) }}" - pod.yml: "{{ lookup('file', 'pod.yml')}}" - when: workload_args.workload == "node-density" or workload_args.workload == "pod-density" - - - name: Create node-density-heavy configmaps - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: kube-burner-config-{{ trunc_uuid }} - namespace: "{{ operator_namespace }}" - data: - config.yml: "{{ lookup('template', 'node-density-heavy.yml.j2')}}" - metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics.yaml')) }}" - app-deployment.yml: "{{ lookup('file', 'app-deployment.yml')}}" - postgres-deployment.yml: "{{ lookup('file', 'postgres-deployment.yml')}}" - postgres-service.yml: "{{ lookup('file', 'postgres-service.yml')}}" - when: workload_args.workload == "node-density-heavy" - - name: Create max-namespaces configmaps - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: kube-burner-config-{{ trunc_uuid }} - namespace: "{{ operator_namespace }}" - data: - config.yml: "{{ lookup('template', 'max-namespaces.yml.j2')}}" - metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" - app-deployment.yml: "{{ lookup('file', 'app-deployment.yml')}}" - postgres-deployment.yml: "{{ lookup('file', 'postgres-deployment.yml')}}" - postgres-service.yml: "{{ lookup('file', 'postgres-service.yml')}}" - secret.yml: "{{ lookup('file', 'secret.yml')}}" - when: workload_args.workload == "max-namespaces" - - - name: Create max-services configmaps - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: kube-burner-config-{{ trunc_uuid }} - namespace: "{{ operator_namespace }}" - data: - config.yml: "{{ lookup('template', 'max-services.yml.j2')}}" - metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" - simple-deployment.yml: "{{ lookup('file', 'simple-deployment.yml')}}" - service.yml: "{{ lookup('file', 'service.yml')}}" - when: workload_args.workload == "max-services" + - block: + + - name: Create cluster-density configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'cluster-density.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" + build.yml: "{{ lookup('file', 'build.yml')}}" + buildconfig.yml: "{{ lookup('file', 'buildconfig.yml')}}" + deployment.yml: "{{ lookup('file', 'deployment.yml')}}" + imagestream.yml: "{{ lookup('file', 'imagestream.yml')}}" + route.yml: "{{ lookup('file', 'route.yml')}}" + secret.yml: "{{ lookup('file', 'secret.yml')}}" + service.yml: "{{ lookup('file', 'service.yml')}}" + configmap.yml: "{{ lookup('file', 'configmap.yml')}}" + when: workload_args.workload == "cluster-density" + + - name: Create node-density or pod-density configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'node-pod-density.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics.yaml')) }}" + pod.yml: "{{ lookup('file', 'pod.yml')}}" + when: workload_args.workload == "node-density" or workload_args.workload == "pod-density" + + - name: Create node-density-heavy configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'node-density-heavy.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics.yaml')) }}" + app-deployment.yml: "{{ lookup('file', 'app-deployment.yml')}}" + postgres-deployment.yml: "{{ lookup('file', 'postgres-deployment.yml')}}" + postgres-service.yml: "{{ lookup('file', 'postgres-service.yml')}}" + when: workload_args.workload == "node-density-heavy" + + - name: Create node-density-cni configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'node-density-cni.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics.yaml')) }}" + curl-deployment.yml: "{{ lookup('file', 'curl-deployment.yml')}}" + webserver-deployment.yml: "{{ lookup('file', 'webserver-deployment.yml')}}" + webserver-service.yml: "{{ lookup('file', 'webserver-service.yml')}}" + when: workload_args.workload == "node-density-cni" + + - name: Create node-density-cni-networkpolicy configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'node-density-cni-networkpolicy.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics.yaml')) }}" + curl-deployment.yml: "{{ lookup('file', 'curl-deployment.yml')}}" + webserver-deployment.yml: "{{ lookup('file', 'webserver-deployment.yml')}}" + webserver-service.yml: "{{ lookup('file', 'webserver-service.yml')}}" + deny-all.yml: "{{ lookup('file', 'deny-all.yml')}}" + allow-http.yml: "{{ lookup('file', 'allow-http.yml')}}" + when: workload_args.workload == "node-density-cni-policy" + + - name: Create max-namespaces configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'max-namespaces.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" + app-deployment.yml: "{{ lookup('file', 'app-deployment.yml')}}" + postgres-deployment.yml: "{{ lookup('file', 'postgres-deployment.yml')}}" + postgres-service.yml: "{{ lookup('file', 'postgres-service.yml')}}" + secret.yml: "{{ lookup('file', 'secret.yml')}}" + when: workload_args.workload == "max-namespaces" + + - name: Create networkpolicy-multitenant configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'networkpolicy-multitenant.yml.j2')}}" + nginx.yaml: "{{ lookup('file', 'nginx.yaml')}}" + networkpolicy-ingress.yaml: "{{ lookup('file', 'networkpolicy-ingress.yaml')}}" + networkpolicy-monitoring.yaml: "{{ lookup('file', 'networkpolicy-monitoring.yaml')}}" + networkpolicy-same-ns.yaml: "{{ lookup('file', 'networkpolicy-same-ns.yaml')}}" + when: workload_args.workload == "networkpolicy-multitenant" + + - name: Create networkpolicy-case2 configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'networkpolicy-case2.yml.j2')}}" + nginx.yaml: "{{ lookup('file', 'nginx.yaml')}}" + case2-networkpolicy.yaml: "{{ lookup('file', 'case2-networkpolicy.yaml')}}" + networkpolicy-deny-all.yaml: "{{ lookup('file', 'networkpolicy-deny-all.yaml')}}" + when: workload_args.workload == "networkpolicy-case2" + + - name: Create networkpolicy-case3 configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'networkpolicy-case3.yml.j2')}}" + nginx.yaml: "{{ lookup('file', 'nginx.yaml')}}" + case3-networkpolicy.yaml: "{{ lookup('file', 'case3-networkpolicy.yaml')}}" + networkpolicy-deny-all.yaml: "{{ lookup('file', 'networkpolicy-deny-all.yaml')}}" + when: workload_args.workload == "networkpolicy-case3" + + + - name: Create networkpolicy-cross-ns configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'networkpolicy-cross-ns.yml.j2')}}" + nginx-cross-ns.yaml: "{{ lookup('file', 'nginx-cross-ns.yaml')}}" + cross-ns-networkpolicy.yaml: "{{ lookup('file', 'cross-ns-networkpolicy.yaml')}}" + networkpolicy-deny-all.yaml: "{{ lookup('file', 'networkpolicy-deny-all.yaml')}}" + when: workload_args.workload == "networkpolicy-cross-ns" + + - name: Create max-services configmaps + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'max-services.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" + simple-deployment.yml: "{{ lookup('file', 'simple-deployment.yml')}}" + service.yml: "{{ lookup('file', 'service.yml')}}" + + when: workload_args.workload == "max-services" + + - name: Create concurrent builds + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: kube-burner-config-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + config.yml: "{{ lookup('template', 'concurrent-builds.yml.j2')}}" + metrics.yaml: "{{ lookup('file', workload_args.metrics_profile|default('metrics-aggregated.yaml')) }}" + buildconfig_triggers.yml: "{{ lookup('file', 'buildconfig_triggers.yml')}}" + configmap.yml: "{{ lookup('file', 'configmap.yml')}}" + imagestream.yml: "{{ lookup('file', 'imagestream.yml')}}" + when: workload_args.workload == "concurrent-builds" + when: workload_args.configmap is not defined - name: Launching kube-burner job k8s: definition: "{{ lookup('template', 'kube-burner.yml.j2') | from_yaml }}" - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - - when: resource_state.resources[0].status.state == "Building" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running -- block: + when: benchmark_state.resources[0].status.state == "Building" - - name: Get job status - k8s_info: - kind: Job - api_version: v1 - namespace: '{{ operator_namespace }}' - name: kube-burner-{{ trunc_uuid }} - register: job_state - - - name: Set complete state to benchmark - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: job_state.resources[0].status.succeeded is defined and (job_state.resources[0].status.succeeded | int) > 0 - - - name: Set failed state to benchmark - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Failed - complete: true - when: job_state.resources[0].status.failed is defined and (job_state.resources[0].status.failed | int) > 0 - when: resource_state.resources[0].status.state == "Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/kube-burner/templates/concurrent-builds.yml.j2 b/roles/kube-burner/templates/concurrent-builds.yml.j2 new file mode 100644 index 000000000..09eb564c2 --- /dev/null +++ b/roles/kube-burner/templates/concurrent-builds.yml.j2 @@ -0,0 +1,51 @@ +--- +global: + writeToFile: false +{% if prometheus is defined and prometheus.prom_url is defined and prometheus.es_url != "" %} + indexerConfig: + enabled: true + esServers: ["{{ prometheus.es_url }}"] + insecureSkipVerify: true + defaultIndex: {{ workload_args.default_index|default("ripsaw-kube-burner") }} + type: elastic + measurements: + - name: podLatency + esIndex: {{ workload_args.default_index|default("ripsaw-kube-burner") }} +{% endif %} + +jobs: + - name: concurrent-builds + jobIterations: {{ workload_args.job_iterations }} + qps: {{ workload_args.qps|default(5) }} + burst: {{ workload_args.burst|default(10) }} + namespacedIterations: true + namespace: svt-{{ uuid }} + cleanup: {{ workload_args.cleanup|default(true) }} + waitWhenFinished: {{ workload_args.wait_when_finished|default(true) }} + podWait: {{ workload_args.pod_wait|default(false) }} +{% if wait_for is defined %} + waitFor: {{ wait_for|default(["Build"]) }} +{% endif %} + verifyObjects: {{ workload_args.verify_objects|default(true) }} + errorOnVerify: {{ workload_args.error_on_verify|default(false) }} + objects: + - objectTemplate: imagestream.yml + replicas: 1 + inputVars: + prefix: {{ workload_args.build_image_stream|default("") }} + image: {{ workload_args.build_image|default("") }} + - objectTemplate: buildconfig_triggers.yml + replicas: 1 + inputVars: + imageStream: {{ workload_args.build_image_stream|default("") }} + gitUri: {{ workload_args.git_url|default("") }} + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker")}} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" +{% if source_strat_env is defined and source_strat_env != "" %} + sourceStratEnv: {{ workload_args.source_strat_env|default("") }} +{% endif %} + fromSource: {{ workload_args.source_strat_from|default("") }} + fromSourceVersion: {{ workload_args.source_strat_from_version|default("latest") }} +{% if post_commit_script is defined and post_commit_script != "" %} + postCommitScript: {{ workload_args.post_commit_script|default("") }} +{% endif %} diff --git a/roles/kube-burner/templates/kube-burner.yml.j2 b/roles/kube-burner/templates/kube-burner.yml.j2 index 9e4c5485a..c86435621 100644 --- a/roles/kube-burner/templates/kube-burner.yml.j2 +++ b/roles/kube-burner/templates/kube-burner.yml.j2 @@ -11,13 +11,19 @@ spec: metadata: labels: app: kube-burner-benchmark-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} tolerations: {{ workload_args.tolerations|default([]) }} restartPolicy: Never - serviceAccountName: kube-burner nodeSelector: {% if workload_args.pin_server is defined and workload_args.pin_server is mapping %} {% for label, value in workload_args.pin_server.items() %} @@ -30,9 +36,19 @@ spec: - name: kube-burner image: {{ workload_args.image | default('quay.io/cloud-bulldozer/kube-burner:latest') }} imagePullPolicy: Always + securityContext: + readOnlyRootFilesystem: false env: - - name: prom_es - value: "{{ prometheus.es_url }}" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{% if workload_args.extra_env_vars is defined and workload_args.extra_env_vars is mapping %} +{% for name, value in workload_args.extra_env_vars.items() %} + - name: {{ name }} + value: "{{ value }}" +{% endfor %} +{% endif %} workingDir: /tmp/kube-burner command: ["/bin/sh", "-c"] args: @@ -40,6 +56,9 @@ spec: kube-burner init {% if workload_args.remote_config is defined and workload_args.remote_config %} -c {{ workload_args.remote_config }} +{% elif workload_args.configmap is defined and workload_args.configmap %} + --configmap={{ workload_args.configmap }} + --namespace=${POD_NAMESPACE} {% else %} -c config.yml {% endif %} @@ -47,13 +66,16 @@ spec: -u {{ prometheus.prom_url }} -t {{ prometheus.prom_token }} --step={{ workload_args.step|default("30s") }} +{% if workload_args.configmap is not defined %} -m {{ workload_args.remote_metrics_profile|default("metrics.yaml", true) }} {% if workload_args.remote_alert_profile is defined and workload_args.remote_alert_profile %} -a {{ workload_args.remote_alert_profile }} {% endif %} +{% endif %} {% endif %} --uuid={{ uuid }} --log-level={{ workload_args.log_level|default("info") }} +{% if workload_args.configmap is not defined %} volumeMounts: - name: kube-burner-config mountPath: /tmp/kube-burner @@ -61,4 +83,5 @@ spec: - name: kube-burner-config configMap: name: kube-burner-config-{{ trunc_uuid }} +{% endif %} {% include "metadata.yml.j2" %} diff --git a/roles/kube-burner/templates/max-namespaces.yml.j2 b/roles/kube-burner/templates/max-namespaces.yml.j2 index e612b88b5..2fa8b7b91 100644 --- a/roles/kube-burner/templates/max-namespaces.yml.j2 +++ b/roles/kube-burner/templates/max-namespaces.yml.j2 @@ -37,6 +37,7 @@ jobs: replicas: 5 inputVars: readinessPeriod: 10 + livenessPeriod: 10 nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" diff --git a/roles/kube-burner/templates/max-services.yml.j2 b/roles/kube-burner/templates/max-services.yml.j2 index 92d29851d..c3841aea6 100644 --- a/roles/kube-burner/templates/max-services.yml.j2 +++ b/roles/kube-burner/templates/max-services.yml.j2 @@ -14,7 +14,7 @@ global: {% endif %} jobs: - - name: max-services + - name: max-serv jobIterations: {{ workload_args.job_iterations }} qps: {{ workload_args.qps|default(5) }} burst: {{ workload_args.burst|default(10) }} @@ -30,11 +30,11 @@ jobs: - objectTemplate: simple-deployment.yml replicas: 1 inputVars: - name: max-services + name: max-serv nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" - objectTemplate: service.yml replicas: 1 inputVars: - name: max-services + name: max-serv diff --git a/roles/kube-burner/templates/networkpolicy-case2.yml.j2 b/roles/kube-burner/templates/networkpolicy-case2.yml.j2 new file mode 100644 index 000000000..8802f3cf9 --- /dev/null +++ b/roles/kube-burner/templates/networkpolicy-case2.yml.j2 @@ -0,0 +1,47 @@ +--- +global: + writeToFile: false +jobs: + - name: networkpolicy-case2 + jobIterations: {{ workload_args.job_iterations|default(400) }} + qps: {{ workload_args.qps|default(20) }} + burst: {{ workload_args.burst|default(20) }} + namespacedIterations: true + namespace: np-{{ uuid }} + podWait: {{ workload_args.pod_wait|default(false) }} + cleanup: {{ workload_args.cleanup|default(false) }} + waitWhenFinished: {{ workload_args.wait_when_finished|default(true) }} + verifyObjects: {{ workload_args.verify_objects|default(true) }} + errorOnVerify: {{ workload_args.error_on_verify|default(false) }} + objects: + +{% for num in range(5) %} + - objectTemplate: nginx.yaml + replicas: {{ workload_args.replicas|default(20) }} + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + label1: "foo" + label2: "bar" +{% endfor %} + + + - objectTemplate: case2-networkpolicy.yaml + replicas: {{ workload_args.replicas|default(20) }} + inputVars: + podselector_label_num: "label1" + podselector_label: "foo" + ingress_label_num: "label1" + ingress_label: "foo" + + - objectTemplate: case2-networkpolicy.yaml + replicas: {{ workload_args.replicas|default(20) }} + inputVars: + podselector_label_num: "label2" + podselector_label: "bar" + ingress_label_num: "label2" + ingress_label: "bar" + + - objectTemplate: networkpolicy-deny-all.yaml + replicas: 1 + diff --git a/roles/kube-burner/templates/networkpolicy-case3.yml.j2 b/roles/kube-burner/templates/networkpolicy-case3.yml.j2 new file mode 100644 index 000000000..132d93f2c --- /dev/null +++ b/roles/kube-burner/templates/networkpolicy-case3.yml.j2 @@ -0,0 +1,47 @@ +--- +global: + writeToFile: false +jobs: + - name: networkpolicy-case3 + jobIterations: {{ workload_args.job_iterations|default(400) }} + qps: {{ workload_args.qps|default(20) }} + burst: {{ workload_args.burst|default(20) }} + namespacedIterations: true + namespace: np-{{ uuid }} + podWait: {{ workload_args.pod_wait|default(false) }} + cleanup: {{ workload_args.cleanup|default(false) }} + waitWhenFinished: {{ workload_args.wait_when_finished|default(true) }} + verifyObjects: {{ workload_args.verify_objects|default(true) }} + errorOnVerify: {{ workload_args.error_on_verify|default(false) }} + objects: + +{% for num in range(5) %} + - objectTemplate: nginx.yaml + replicas: {{ workload_args.replicas|default(5) }} + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + label1: "foo" + label2: "bar" +{% endfor %} + + + - objectTemplate: case3-networkpolicy.yaml + replicas: {{ workload_args.replicas|default(5) }} + inputVars: + podselector_label_num: "label1" + podselector_label: "foo" + ingress_label_num: "label1" + ingress_label: "foo" + + - objectTemplate: case3-networkpolicy.yaml + replicas: {{ workload_args.replicas|default(5) }} + inputVars: + podselector_label_num: "label2" + podselector_label: "bar" + ingress_label_num: "label2" + ingress_label: "bar" + + - objectTemplate: networkpolicy-deny-all.yaml + replicas: 1 + diff --git a/roles/kube-burner/templates/networkpolicy-cross-ns.yml.j2 b/roles/kube-burner/templates/networkpolicy-cross-ns.yml.j2 new file mode 100644 index 000000000..88f23b395 --- /dev/null +++ b/roles/kube-burner/templates/networkpolicy-cross-ns.yml.j2 @@ -0,0 +1,34 @@ +--- +global: + writeToFile: false +jobs: + - name: networkpolicy-cross-ns + jobIterations: {{ workload_args.job_iterations|default(400) }} + qps: {{ workload_args.qps|default(20) }} + burst: {{ workload_args.burst|default(20) }} + namespacedIterations: true + namespace: np-{{ uuid }} + podWait: {{ workload_args.pod_wait|default(false) }} + cleanup: {{ workload_args.cleanup|default(false) }} + waitWhenFinished: {{ workload_args.wait_when_finished|default(true) }} + verifyObjects: {{ workload_args.verify_objects|default(true) }} + errorOnVerify: {{ workload_args.error_on_verify|default(false) }} + objects: + +{% for num in range(5) %} + - objectTemplate: nginx-cross-ns.yaml + replicas: {{ workload_args.replicas|default(1) }} + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + label1: "foo" + label2: "bar" +{% endfor %} + + + - objectTemplate: cross-ns-networkpolicy.yaml + replicas: {{ workload_args.replicas|default(5) }} + + - objectTemplate: networkpolicy-deny-all.yaml + replicas: 1 + diff --git a/roles/kube-burner/templates/networkpolicy-multitenant.yml.j2 b/roles/kube-burner/templates/networkpolicy-multitenant.yml.j2 new file mode 100644 index 000000000..83a8ac67c --- /dev/null +++ b/roles/kube-burner/templates/networkpolicy-multitenant.yml.j2 @@ -0,0 +1,35 @@ +--- +global: + writeToFile: false +jobs: + - name: networkpolicy-multitenant + jobIterations: {{ workload_args.job_iterations|default(400) }} + qps: {{ workload_args.qps|default(20) }} + burst: {{ workload_args.burst|default(20) }} + namespacedIterations: true + namespace: np-{{ uuid }} + podWait: {{ workload_args.pod_wait|default(false) }} + cleanup: {{ workload_args.cleanup|default(false) }} + waitWhenFinished: {{ workload_args.wait_when_finished|default(true) }} + verifyObjects: {{ workload_args.verify_objects|default(true) }} + errorOnVerify: {{ workload_args.error_on_verify|default(false) }} + objects: + +{% for num in range(5) %} + - objectTemplate: nginx.yaml + replicas: {{ workload_args.replicas|default(1) }} + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + label1: "foo" + label2: "bar" +{% endfor %} + + + - objectTemplate: networkpolicy-same-ns.yaml + replicas: 1 + - objectTemplate: networkpolicy-monitoring.yaml + replicas: 1 + - objectTemplate: networkpolicy-ingress.yaml + replicas: 1 + diff --git a/roles/kube-burner/templates/node-density-cni-networkpolicy.yml.j2 b/roles/kube-burner/templates/node-density-cni-networkpolicy.yml.j2 new file mode 100644 index 000000000..21f3085fb --- /dev/null +++ b/roles/kube-burner/templates/node-density-cni-networkpolicy.yml.j2 @@ -0,0 +1,64 @@ +--- +global: + writeToFile: false +{% if prometheus is defined and prometheus.prom_url is defined and prometheus.es_url != "" %} + indexerConfig: + enabled: true + esServers: ["{{ prometheus.es_url }}"] + insecureSkipVerify: true + defaultIndex: {{ workload_args.default_index|default("ripsaw-kube-burner") }} + type: elastic + measurements: + - name: podLatency + esIndex: {{ workload_args.default_index|default("ripsaw-kube-burner") }} +{% endif %} + + +jobs: + + - name: deny-all-policy + jobIterations: 1 + qps: 1 + burst: 1 + namespacedIterations: false + namespace: {{workload_args.workload}}-{{ uuid }} + cleanup: {{ workload_args.cleanup|default(true) }} + jobPause: 1m + objects: + + - objectTemplate: deny-all.yml + replicas: 1 + + - name: {{ workload_args.workload }} + jobIterations: {{ workload_args.job_iterations }} + qps: {{ workload_args.qps|default(5) }} + burst: {{ workload_args.burst|default(10) }} + namespacedIterations: false + namespace: {{workload_args.workload}}-{{ uuid }} + podWait: {{ workload_args.pod_wait|default(false) }} + waitWhenFinished: {{ workload_args.wait_when_finished|default(true) }} + verifyObjects: {{ workload_args.verify_objects|default(true) }} + errorOnVerify: {{ workload_args.error_on_verify|default(false) }} + preLoadImages: true + preLoadPeriod: 2m + objects: + + - objectTemplate: webserver-deployment.yml + replicas: 1 + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + + + - objectTemplate: webserver-service.yml + replicas: 1 + + - objectTemplate: allow-http.yml + replicas: 1 + + - objectTemplate: curl-deployment.yml + replicas: 1 + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + diff --git a/roles/kube-burner/templates/node-density-cni.yml.j2 b/roles/kube-burner/templates/node-density-cni.yml.j2 new file mode 100644 index 000000000..aa7a1f738 --- /dev/null +++ b/roles/kube-burner/templates/node-density-cni.yml.j2 @@ -0,0 +1,47 @@ +--- +global: + writeToFile: false +{% if prometheus is defined and prometheus.prom_url is defined and prometheus.es_url != "" %} + indexerConfig: + enabled: true + esServers: ["{{ prometheus.es_url }}"] + insecureSkipVerify: true + defaultIndex: {{ workload_args.default_index|default("ripsaw-kube-burner") }} + type: elastic + measurements: + - name: podLatency + esIndex: {{ workload_args.default_index|default("ripsaw-kube-burner") }} +{% endif %} + + +jobs: + - name: {{ workload_args.workload }} + jobIterations: {{ workload_args.job_iterations }} + qps: {{ workload_args.qps|default(5) }} + burst: {{ workload_args.burst|default(10) }} + namespacedIterations: false + namespace: {{workload_args.workload}}-{{ uuid }} + podWait: {{ workload_args.pod_wait|default(false) }} + cleanup: {{ workload_args.cleanup|default(true) }} + waitWhenFinished: {{ workload_args.wait_when_finished|default(true) }} + verifyObjects: {{ workload_args.verify_objects|default(true) }} + errorOnVerify: {{ workload_args.error_on_verify|default(false) }} + preLoadImages: true + preLoadPeriod: 2m + objects: + + - objectTemplate: webserver-deployment.yml + replicas: 1 + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + + - objectTemplate: webserver-service.yml + replicas: 1 + + - objectTemplate: curl-deployment.yml + replicas: 1 + inputVars: + nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} + nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" + diff --git a/roles/kube-burner/templates/node-density-heavy.yml.j2 b/roles/kube-burner/templates/node-density-heavy.yml.j2 index 8d1aeaf17..10584b18f 100644 --- a/roles/kube-burner/templates/node-density-heavy.yml.j2 +++ b/roles/kube-burner/templates/node-density-heavy.yml.j2 @@ -37,6 +37,7 @@ jobs: replicas: 1 inputVars: readinessPeriod: 10 + livenessPeriod: 10 nodeSelectorKey: {{ workload_args.node_selector.key|default("node-role.kubernetes.io/worker") }} nodeSelectorValue: "{{ workload_args.node_selector.value|default("") }}" diff --git a/roles/log_generator/tasks/main.yml b/roles/log_generator/tasks/main.yml new file mode 100644 index 000000000..5f39129f3 --- /dev/null +++ b/roles/log_generator/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- set_fact: + resources: + - "{{ role_path }}/templates/log_generator.yml" + +- name: Generic workload - log_generator + include_role: + name: generic_workload diff --git a/roles/log_generator/templates/log_generator.yml b/roles/log_generator/templates/log_generator.yml new file mode 100644 index 000000000..a3587fd7a --- /dev/null +++ b/roles/log_generator/templates/log_generator.yml @@ -0,0 +1,136 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: 'log-generator-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' +spec: + parallelism: {{ workload_args.pod_count | default(1) | int }} + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} + template: + metadata: + labels: + app: log-generator-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.tolerations is defined %} + tolerations: + - key: {{ workload_args.tolerations.key }} + value: {{ workload_args.tolerations.value }} + effect: {{ workload_args.tolerations.effect }} +{% endif %} +{% if workload_args.label is defined %} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ workload_args.label.key }} + operator: In + values: + - {{ workload_args.label.value }} +{% endif %} + containers: + - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/log_generator:latest') }} + name: log-generator + env: + - name: my_node_name + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: my_pod_name + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: uuid + value: "{{ uuid }}" + - name: test_user + value: "{{ test_user | default("ripsaw") }}" + - name: clustername + value: "{{ clustername }}" + - name: snafu_disable_logs + value: "{{ workload_args.snafu_disable_logs | default(false) }}" +{% if elasticsearch.url %} + - name: es + value: "{{ elasticsearch.url }}" + - name: es_index + value: "{{ elasticsearch.index_name | default("log-generator") }}" + - name: es_verify_cert + value: "{{ elasticsearch.verify_cert | default(true) }}" + - name: parallel + value: "{{ elasticsearch.parallel | default(false) }}" +{% endif %} +{% if prometheus is defined %} + - name: prom_es + value: "{{ prometheus.es_url }}" + - name: prom_parallel + value: "{{ prometheus.es_parallel | default(false) }}" + - name: prom_token + value: "{{ prometheus.prom_token | default() }}" + - name: prom_url + value: "{{ prometheus.prom_url | default() }}" +{% endif %} + command: ["/bin/sh", "-c"] + args: + - > +{% if workload_args.pod_count | default(1) | int > 1 %} + if [[ "${snafu_disable_logs}" == "False" ]]; then echo "Waiting for all pods to be Ready"; fi; + redis-cli -h {{ bo.resources[0].status.podIP }} INCR "log-generator-{{ trunc_uuid }}" > /dev/null 2>&1; + pods=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "log-generator-{{ trunc_uuid }}"`; + while [[ $pods != {{ workload_args.pod_count | int }} ]]; do + pods=`redis-cli -h {{ bo.resources[0].status.podIP }} GET "log-generator-{{ trunc_uuid }}"`; + sleep 1; + done; +{% endif %} + run_snafu + --tool log_generator +{% if workload_args.debug is defined and workload_args.debug %} + -v +{% endif %} + -u "{{ uuid }}" + --size "{{ workload_args.size | default(512) }}" +{% if workload_args.messages_per_second is defined %} + --messages-per-second "{{ workload_args.messages_per_second }}" +{% endif %} +{% if workload_args.messages_per_minute is defined %} + --messages-per-minute "{{ workload_args.messages_per_minute }}" +{% endif %} + --duration "{{ workload_args.duration | default(10) }}" +{% if workload_args.es_url is defined %} + --es-url "{{ workload_args.es_url }}" +{% endif %} +{% if workload_args.es_index is defined %} + --es-index "{{ workload_args.es_index }}" +{% endif %} +{% if workload_args.es_token is defined %} + --es-token "{{ workload_args.es_token }}" +{% endif %} +{% if workload_args.cloudwatch_log_group is defined %} + --cloudwatch-log-group "{{ workload_args.cloudwatch_log_group }}" + --aws-region "{{ workload_args.aws_region }}" + --aws-access-key "{{ workload_args.aws_access_key }}" + --aws-secret-key "{{ workload_args.aws_secret_key }}" +{% endif %} +{% if workload_args.kafka_bootstrap_server is defined and workload_args.kafka_topic is defined %} + --kafka-bootstrap-server "{{ workload_args.kafka_bootstrap_server }}" + --kafka-topic "{{ workload_args.kafka_topic }}" +{% if workload_args.kafka_check is defined and workload_args.kafka_check %} + --kafka-check +{% endif %} +{% endif %} + --pod-name ${my_pod_name} + --timeout "{{ workload_args.timeout | default(600) }}" + --pod-count {{ workload_args.pod_count | default(1) | int }} + imagePullPolicy: Always + restartPolicy: Never +{% include "metadata.yml.j2" %} diff --git a/roles/nighthawk/README.md b/roles/nighthawk/README.md new file mode 100644 index 000000000..ec509e75a --- /dev/null +++ b/roles/nighthawk/README.md @@ -0,0 +1,136 @@ +nighthawk-benchmark +========= + +HTTP benchmark workload. More details [here](https://github.com/envoyproxy/nighthawk). + +## Running Nighthawk + +Here are some sample CRs and instructions to run this workload. + +### Sample CR for testing +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: nighthawk-benchmark + namespace: benchmark-operator +spec: + clustername: myk8scluster + elasticsearch: + url: "http://es-instance.com:9200" + workload: + cleanup: true + name: nighthawk + args: + url: https://www.testurlhere.com + image: quay.io/vchalla/nighthawk:latest + terminations: ["http", "edge", "passthrough", "reencrypt"] + kind: pod + hostnetwork: false + number_of_routes: 3 + samples: 1 + concurrency: 8 + duration: 60 + connections: 80 + max_requests_per_connection: 50 + debug: true +``` +### Sample CR with default parameters +```yaml +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: nighthawk-benchmark + namespace: benchmark-operator +spec: + clustername: myk8scluster + elasticsearch: + url: "http://es-instance.com:9200" + workload: + cleanup: true + name: nighthawk + args: + terminations: ["http"] +``` + +### Parameters +`url` Target url to test. Works as a client server model with client-server pod pairs. +`image`: Image if needed to be overriden from the default one. +`terminations`: List of network terminations to be applied. +`kind`: Kind of machine to run workload on. For now only pod is supported. +`hostnetwork`: Will test the performance of the node the pod will run on. +`number_of_routes`: Number of routes to be created. +`samples`: Number of times to run the workload. +`concurrency`: The number of concurrent event loops that should be used. Specify 'auto' to "\ + "let Nighthawk leverage all vCPUs that have affinity to the Nighthawk process"\ + ". Note that increasing this results in an effective load multiplier combined"\ + " with the configured --rps and --connections values" +`duration`: The number of seconds that the test should be running. +`connections`: The maximum allowed number of concurrent connections per event loop. +`max_requests_per_connection`: Max requests per connection. +`debug`: To turn on debug logs. + +### Once the testing is done the results are published to elastic search as follows. +```yaml +"hits" : [ + { + "_index" : "ripsaw-nighthawk-results", + "_type" : "_doc", + "_id" : "4a2a5aa5c41a3ec53bdc5de9f9ea4a04f12cc8fa967fa2033b95f82e2ae356f0", + "_score" : 1.0, + "_source" : { + "concurrency" : 8, + "duration" : 60, + "connections" : 80, + "max_requests_per_connection" : 4294937295, + "rps" : 5, + "kind" : "pod", + "url" : "http://nighthawk-benchmark-route-http-2-benchmark-operator.apps.vchalla-perfscale.perfscale.devcluster.openshift.com", + "workload" : "nighthawk", + "uuid" : "e777d5e4-4b9e-5e90-a453-61e30caa9a5b", + "user" : "ripsaw", + "cluster_name" : "myk8scluster", + "targets" : [ + "http://nighthawk-benchmark-route-http-2-benchmark-operator.apps.vchalla-perfscale.perfscale.devcluster.openshift.com" + ], + "hostname" : "nighthawk-client-10.0.221.20-nginx-http-2-e777d5e4-pmxvt", + "requested_qps" : 40, + "throughput" : 39.98936405750067, + "status_codes_1xx" : 0, + "status_codes_2xx" : 2400, + "status_codes_3xx" : 0, + "status_codes_4xx" : 0, + "status_codes_5xx" : 0, + "p50_latency" : 3.188863, + "p75_latency" : 4.302335, + "p80_latency" : 5.629951, + "p90_latency" : 10.191871, + "p95_latency" : 13.958143, + "p99_latency" : 24.370175, + "p99_9_latency" : 39.522303, + "avg_latency" : 4.663531, + "timestamp" : "2022-08-18T19:57:57.357909411Z", + "bytes_in" : 1108800.0, + "bytes_out" : 331200.0, + "iteration" : 1, + "run_id" : "NA" + } + } + ] +``` +### And also you can verify the logs in your openshift namespace using the below script +``` +echo -n "`oc get all | grep -i -e 'completed' | cut -f 1 -d ' '`" > output.txt + +cat output.txt | awk '{print $0}' | while + read each +do + echo "client name: $each" + echo "client info" + echo -n "`oc get $each -o yaml | grep -i -e "clientfor" -e 'port' -A 2`" + echo -e "\n" + echo "client logs" + echo -n "`oc logs $each | tail -n 5`" + echo -e "\n\n" +done +``` \ No newline at end of file diff --git a/roles/nighthawk/defaults/main.yml b/roles/nighthawk/defaults/main.yml new file mode 100644 index 000000000..1ed4613ee --- /dev/null +++ b/roles/nighthawk/defaults/main.yml @@ -0,0 +1,3 @@ +--- +# defaults file for nighthawk +resource_kind: "{{ workload.args.kind | default('pod') }}" \ No newline at end of file diff --git a/roles/nighthawk/handlers/main.yml b/roles/nighthawk/handlers/main.yml new file mode 100644 index 000000000..5780811e8 --- /dev/null +++ b/roles/nighthawk/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for nighthawk \ No newline at end of file diff --git a/roles/nighthawk/meta/main.yml b/roles/nighthawk/meta/main.yml new file mode 100644 index 000000000..227ad9c34 --- /dev/null +++ b/roles/nighthawk/meta/main.yml @@ -0,0 +1,53 @@ +galaxy_info: + author: your name + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.9 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. + \ No newline at end of file diff --git a/roles/nighthawk/tasks/edge-tasks.yml b/roles/nighthawk/tasks/edge-tasks.yml new file mode 100644 index 000000000..a9fd8027f --- /dev/null +++ b/roles/nighthawk/tasks/edge-tasks.yml @@ -0,0 +1,21 @@ +--- +# tasks for edge termination type. + +- name: Create service for edge type server pods + k8s: + definition: "{{ lookup('template', 'http-service.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + vars: + type: edge + +- name: Create route for edge type server pods + k8s: + definition: "{{ lookup('template', 'edge-route.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + +- name: Start edge server types(s) + k8s: + definition: "{{ lookup('template', 'nginx-server.yml.j2') | from_yaml }}" + vars: + type: edge + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} \ No newline at end of file diff --git a/roles/nighthawk/tasks/http-tasks.yml b/roles/nighthawk/tasks/http-tasks.yml new file mode 100644 index 000000000..e60ea8d69 --- /dev/null +++ b/roles/nighthawk/tasks/http-tasks.yml @@ -0,0 +1,21 @@ +--- +# tasks for http termination type. + +- name: Create service for http type server pods + k8s: + definition: "{{ lookup('template', 'http-service.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + vars: + type: http + +- name: Create route for http type server pods + k8s: + definition: "{{ lookup('template', 'http-route.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + +- name: Start http server types(s) + k8s: + definition: "{{ lookup('template', 'nginx-server.yml.j2') | from_yaml }}" + vars: + type: http + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} \ No newline at end of file diff --git a/roles/nighthawk/tasks/main.yml b/roles/nighthawk/tasks/main.yml new file mode 100644 index 000000000..29a02cc67 --- /dev/null +++ b/roles/nighthawk/tasks/main.yml @@ -0,0 +1,228 @@ +--- +# tasks file for nighthawk + +- block: + + - include_tasks: http-tasks.yml + when: "'http' in {{ workload_args.terminations | default([]) }} or 'mix' in {{ workload_args.terminations | default([]) }}" + + - include_tasks: edge-tasks.yml + when: "'edge' in {{ workload_args.terminations | default([]) }} or 'mix' in {{ workload_args.terminations | default([]) }}" + + - include_tasks: passthrough-tasks.yml + when: "'passthrough' in {{ workload_args.terminations | default([]) }} or 'mix' in {{ workload_args.terminations | default([]) }}" + + - include_tasks: reencrypt-tasks.yml + when: "'reencrypt' in {{ workload_args.terminations | default([]) }} or 'mix' in {{ workload_args.terminations | default([]) }}" + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting Servers + + when: benchmark_state.resources[0].status.state == "Building" and resource_kind == "pod" + +- block: + + - name: Get server pods + k8s_info: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-nighthawk-server-{{ trunc_uuid }} + register: server_pods + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting Clients" + when: "workload_args.terminations|default([])|length * workload_args.number_of_routes|default(1)|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" + + when: benchmark_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod" + +- block: + + - name: Get server pod info + k8s_info: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-nighthawk-server-{{ trunc_uuid }} + register: server_pods + + - name: Get route info + k8s_info: + kind: Route + api_version: route.openshift.io/v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-nighthawk-route-{{ trunc_uuid }} + register: routes + + - block: + - name: Start Client(s) + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + with_items: "{{ server_pods.resources }}" + vars: + routerCanonicalName: "{{ routes.resources[0] | json_query('status.ingress[0].routerCanonicalHostname') }}" + when: ( server_pods.resources|length > 0 ) + + when: resource_kind == "pod" + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Waiting for Clients" + + when: benchmark_state.resources[0].status.state == "Starting Clients" + +- block: + + - block: + - name: Get client pod status + k8s_info: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = nighthawk-bench-client-{{ trunc_uuid }} + register: client_pods + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Clients Running + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Clients Running + when: "workload_args.terminations|default([])|length * workload_args.number_of_routes|default(1)|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_pods | json_query('resources[].status.podIP')|length)" + + when: resource_kind == "pod" + + when: benchmark_state.resources[0].status.state == "Waiting for Clients" + +- block: + + - name: Signal workload + command: "redis-cli set start-{{ trunc_uuid }} true" + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running + + when: benchmark_state.resources[0].status.state == "Clients Running" + +- block: + - block: + - name: Waiting for pods to complete.... + k8s_info: + kind: pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = nighthawk-bench-client-{{ trunc_uuid }} + register: client_pods + + - name: Check for client pod failures + include_role: + name: benchmark_state + tasks_from: failure + when: "(client_pods|json_query('resources[].status[]')|selectattr('phase','match','Failed')|list|length) > 0" + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Cleanup + when: "workload_args.terminations|default([])|length * workload_args.number_of_routes|default(1)|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: resource_kind == "pod" + + when: benchmark_state.resources[0].status.state == "Running" + +- block: + + - block: + - name: Get Server Jobs + k8s_info: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-nighthawk-job-{{ trunc_uuid }} + register: server_jobs + + - name: Get Server Pods + k8s_info: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-nighthawk-server-{{ trunc_uuid }} + register: server_pods + + - name: Server Job and Pod names - to clean + set_fact: + clean_jobs: | + [ + {% for item in server_jobs.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] + clean_pods: | + [ + {% for item in server_pods.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] + + - name: Cleanup server Job + k8s: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_jobs }}" + + - name: Cleanup server Pod + k8s: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_pods }}" + + when: resource_kind == "pod" and cleanup == True + + - block: + - name: Cleanup redis + command: "{{ item }}" + with_items: + - redis-cli del start-{{ trunc_uuid }} + when: resource_kind == "pod" + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Complete + complete: true + + when: benchmark_state.resources[0].status.state == "Cleanup" diff --git a/roles/nighthawk/tasks/passthrough-tasks.yml b/roles/nighthawk/tasks/passthrough-tasks.yml new file mode 100644 index 000000000..0cf6cfc8b --- /dev/null +++ b/roles/nighthawk/tasks/passthrough-tasks.yml @@ -0,0 +1,21 @@ +--- +# tasks for passthrough termination type. + +- name: Create service for passthrough type server pods + k8s: + definition: "{{ lookup('template', 'https-service.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + vars: + type: passthrough + +- name: Create route for passthrough type server pods + k8s: + definition: "{{ lookup('template', 'passthrough-route.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + +- name: Start passthrough server types(s) + k8s: + definition: "{{ lookup('template', 'nginx-server.yml.j2') | from_yaml }}" + vars: + type: passthrough + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} \ No newline at end of file diff --git a/roles/nighthawk/tasks/reencrypt-tasks.yml b/roles/nighthawk/tasks/reencrypt-tasks.yml new file mode 100644 index 000000000..44839f6e2 --- /dev/null +++ b/roles/nighthawk/tasks/reencrypt-tasks.yml @@ -0,0 +1,21 @@ +--- +# tasks for reencrypt termination type. + +- name: Create service for reencrypt type server pods + k8s: + definition: "{{ lookup('template', 'https-service.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + vars: + type: reencrypt + +- name: Create route for reencrypt type server pods + k8s: + definition: "{{ lookup('template', 'reencrypt-route.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} + +- name: Start reencrypt server types(s) + k8s: + definition: "{{ lookup('template', 'nginx-server.yml.j2') | from_yaml }}" + vars: + type: reencrypt + with_sequence: start=0 count={{ workload_args.number_of_routes | default(1) | int }} \ No newline at end of file diff --git a/roles/nighthawk/templates/edge-route.yml.j2 b/roles/nighthawk/templates/edge-route.yml.j2 new file mode 100644 index 000000000..b5e0fc873 --- /dev/null +++ b/roles/nighthawk/templates/edge-route.yml.j2 @@ -0,0 +1,15 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + type: {{ ansible_operator_meta.name }}-bench-nighthawk-route-{{ trunc_uuid }} + name: nighthawk-benchmark-route-edge-{{ item }} + namespace: "{{ operator_namespace }}" +spec: + port: + targetPort: http + tls: + termination: edge + to: + kind: Service + name: nighthawk-benchmark-service-edge-{{ item }} \ No newline at end of file diff --git a/roles/nighthawk/templates/http-route.yml.j2 b/roles/nighthawk/templates/http-route.yml.j2 new file mode 100644 index 000000000..4a5a390fd --- /dev/null +++ b/roles/nighthawk/templates/http-route.yml.j2 @@ -0,0 +1,13 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + type: {{ ansible_operator_meta.name }}-bench-nighthawk-route-{{ trunc_uuid }} + name: nighthawk-benchmark-route-http-{{ item }} + namespace: "{{ operator_namespace }}" +spec: + port: + targetPort: http + to: + kind: Service + name: nighthawk-benchmark-service-http-{{ item }} \ No newline at end of file diff --git a/roles/nighthawk/templates/http-service.yml.j2 b/roles/nighthawk/templates/http-service.yml.j2 new file mode 100644 index 000000000..6a9b0320c --- /dev/null +++ b/roles/nighthawk/templates/http-service.yml.j2 @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: nighthawk-benchmark + name: nighthawk-benchmark-service-{{ type }}-{{ item }} + namespace: "{{ operator_namespace }}" +spec: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: nginx-{{ type }}-{{ item }} + type: ClusterIP \ No newline at end of file diff --git a/roles/nighthawk/templates/https-service.yml.j2 b/roles/nighthawk/templates/https-service.yml.j2 new file mode 100644 index 000000000..fed4d1c25 --- /dev/null +++ b/roles/nighthawk/templates/https-service.yml.j2 @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: nighthawk-benchmark + name: nighthawk-benchmark-service-{{ type }}-{{ item }} + namespace: "{{ operator_namespace }}" +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + app: nginx-{{ type }}-{{ item }} + type: ClusterIP \ No newline at end of file diff --git a/roles/nighthawk/templates/nginx-server.yml.j2 b/roles/nighthawk/templates/nginx-server.yml.j2 new file mode 100644 index 000000000..21fc323a2 --- /dev/null +++ b/roles/nighthawk/templates/nginx-server.yml.j2 @@ -0,0 +1,58 @@ +--- +kind: Job +apiVersion: batch/v1 +metadata: + name: nighthawk-benchmark-server-{{ type }}-{{ item }} + namespace: '{{ operator_namespace }}' + labels: + run: nighthawk-benchmark-{{ type }}-{{ item }} + app: nginx + type: {{ ansible_operator_meta.name }}-bench-nighthawk-job-{{ trunc_uuid }} +spec: + ttlSecondsAfterFinished: 600 + backoffLimit: 0 + template: + metadata: + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: nighthawk + benchmark-operator-role: server + app: nginx-{{ type }}-{{ item }} + termination: {{ type }} + route: nighthawk-benchmark-route-{{ type }}-{{ item }} + type: {{ ansible_operator_meta.name }}-bench-nighthawk-server-{{ trunc_uuid }} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.hostnetwork is sameas true %} + hostNetwork: true + serviceAccountName: benchmark-operator +{% endif %} + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: type + operator: In + values: + - {{ ansible_operator_meta.name }}-bench-nighthawk-server-{{ trunc_uuid }} + topologyKey: kubernetes.io/hostname + containers: + - name: nginx + image: quay.io/cloud-bulldozer/nginx:latest + imagePullPolicy: Always + ports: + - containerPort: 8080 + resources: + requests: + memory: "10Mi" + cpu: "10m" + dnsPolicy: ClusterFirst + restartPolicy: OnFailure +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_server | default("unknown") }}' +{% endif %} +{% include "metadata.yml.j2" %} \ No newline at end of file diff --git a/roles/nighthawk/templates/passthrough-route.yml.j2 b/roles/nighthawk/templates/passthrough-route.yml.j2 new file mode 100644 index 000000000..1a4804cd6 --- /dev/null +++ b/roles/nighthawk/templates/passthrough-route.yml.j2 @@ -0,0 +1,15 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + type: {{ ansible_operator_meta.name }}-bench-nighthawk-route-{{ trunc_uuid }} + name: nighthawk-benchmark-route-passthrough-{{ item }} + namespace: "{{ operator_namespace }}" +spec: + port: + targetPort: https + tls: + termination: passthrough + to: + kind: Service + name: nighthawk-benchmark-service-passthrough-{{ item }} \ No newline at end of file diff --git a/roles/nighthawk/templates/reencrypt-route.yml.j2 b/roles/nighthawk/templates/reencrypt-route.yml.j2 new file mode 100644 index 000000000..78d1982fb --- /dev/null +++ b/roles/nighthawk/templates/reencrypt-route.yml.j2 @@ -0,0 +1,66 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + type: {{ ansible_operator_meta.name }}-bench-nighthawk-route-{{ trunc_uuid }} + name: nighthawk-benchmark-route-reencrypt-{{ item }} + namespace: "{{ operator_namespace }}" +spec: + port: + targetPort: https + tls: + termination: reencrypt + certificate: | + -----BEGIN CERTIFICATE----- + MIIDezCCAmOgAwIBAgIUIDE86s4g/jNL6iHVCeMKzsLL/rwwDQYJKoZIhvcNAQEL + BQAwTTELMAkGA1UEBhMCRVMxDzANBgNVBAgMBk1hZHJpZDEPMA0GA1UEBwwGTWFk + cmlkMRwwGgYDVQQKDBNEZWZhdWx0IENvbXBhbnkgTHRkMB4XDTIxMDEyMDExNDAw + NFoXDTIxMDIxOTExNDAwNFowTTELMAkGA1UEBhMCRVMxDzANBgNVBAgMBk1hZHJp + ZDEPMA0GA1UEBwwGTWFkcmlkMRwwGgYDVQQKDBNEZWZhdWx0IENvbXBhbnkgTHRk + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvb+XB/1moWr0MJZoSRN/ + MIvXyPM0pR0N8Gxfpk9LF0qHLX3I35BkugWZpua85HXmurbZgW12UpplTuA/wack + YjVsTjWNb/p4cQF/o1MIm5iWWCrZY6+Du52gYb36lHNU3Uv2flsNZRHulMq9Ptov + I01mtYpr/PujU5r+QN01kiswrC4Z7xip3VTRtn4L8VYXIDapNLw4YinNJXIUd+tS + YsEYG343E9dPXFbobpHi/6JNjrcs6oe4VKvM8pgeAPTM9dbLJtX/TFLAkPLoeIoT + R/TmP/y7EbZXdAQL2schIL4Nx0iHaLoLPF9mGL/1D/9skAHr3RRC8eAmKKVoZVee + qwIDAQABo1MwUTAdBgNVHQ4EFgQU3gxPTXzT++e2GtTEnkHNXWTw2KMwHwYDVR0j + BBgwFoAU3gxPTXzT++e2GtTEnkHNXWTw2KMwDwYDVR0TAQH/BAUwAwEB/zANBgkq + hkiG9w0BAQsFAAOCAQEAf+cvaTY02vA1BwW8K8kbWS//F+GFgJhFZ7fi2szVZjBk + JErJcCr8+lwY4GVytPglhO8zWDfMY+c4LKFchPN1LnVYb7v18xbNb7Smz02SQQDl + sXgTf/f+Fc+lwVRjtVNegfwqc5wZj2ZqBcXq0UnxIBBzXS9EL6czeOqW4gPy5bPa + KejQwkkULk6KqKGT2tp71BsMyXCM8fMQkvM6FxHPKNUR/GxxQoH4mTQlOgnqrdAc + 4GUhmQZc3IQyUHTUoxMw4BodsLJBH3kCqQ5dy/O3h3GTNzivZNRpItVI6HHjrN2s + cZm6iaKNoUXr3bv3ugZwe+1R4ISvw/FXS1pMxiD0YA== + -----END CERTIFICATE----- + key: | + -----BEGIN RSA PRIVATE KEY----- + MIIEpAIBAAKCAQEAvb+XB/1moWr0MJZoSRN/MIvXyPM0pR0N8Gxfpk9LF0qHLX3I + 35BkugWZpua85HXmurbZgW12UpplTuA/wackYjVsTjWNb/p4cQF/o1MIm5iWWCrZ + Y6+Du52gYb36lHNU3Uv2flsNZRHulMq9PtovI01mtYpr/PujU5r+QN01kiswrC4Z + 7xip3VTRtn4L8VYXIDapNLw4YinNJXIUd+tSYsEYG343E9dPXFbobpHi/6JNjrcs + 6oe4VKvM8pgeAPTM9dbLJtX/TFLAkPLoeIoTR/TmP/y7EbZXdAQL2schIL4Nx0iH + aLoLPF9mGL/1D/9skAHr3RRC8eAmKKVoZVeeqwIDAQABAoIBAGSKrlZ3ePgzGezc + 5alDAXQRxWcfJ1gOCyLH6e7PuTRAM1xxeAyuEBFZgk8jmBdeOcHZvWqNO9MNKH0g + 6eeMzwSS1i6ixaz+BO+sIZvDFZ6Mva0+Fy5xA9ZX8XGZHrumWONhqtzNFk3lsIt6 + 2cgCCFQmYTP0gr/r/mEAkZSBIi+udRIEIsR7T4iLw+H+MWwnXEbM9yke9Qvvvzyt + ULLWzJT9TrmW7teAvtKYx7D0dyJEXnM8zEeuDd+unzeV2uqrfptU34fjvuQLaUKk + Br02YeOvs7rSSkGReZNnt8ZXWYFssYs3Kf8s/5EkXYFPukK+8LjII9L8zFgviBMf + bIqnIYECgYEA6Ns1gkkWtnBmfzSGrFkM5ztQvuLwgaMRLnzsY7JQBnTjWFXTRAqu + pBtTaxPGEg0P1A4fRnjiSjV1BGzS5yUdyZSYTbc9jiILQwdayB97chQAE+hnVokU + 1zPqtCfiMJETOnV8fRNMBaDRHoBoe1l6va7G37P9NzNgWkpAj+MNKssCgYEA0JuL + b9IEk0BhOBaV0ROsMJnErsDNbESY4kVotVssZYL1U61jDfC7KbQMmW3frPtGWJi8 + HW99ulXpGvRmwrOhRtPaXzSBwpF8KTseAaRJmPmtJAFYhYRfMFTZWE2JKNGOhkws + olO6FKkEfgR+m/HtgIqeOdQ/nI1q192SbAHff6ECgYEA4sqN5ST2gB4dVgt8l2Ps + E1JMJH63rCt8UoDNY5SKKJ+zxZdhusWErsUGjCWoJnCeV/ShNWwLSieinvq2tvYJ + ewnFBPxRcZtqyI/jNUKkYslkAf+6lifRKoCgOXMW9CJ4Tdmbs94Vju3AfyqlmG3g + A9q0S7DsENVzJL1pADst2d0CgYEAxRALOsj1FX2d2XRMdsPUx9ya1lLAO+TZX/cd + oSTN3d9GjZOfnU2qIQ07Ub1frXN50rwGCPCHnv0FRjdW09sJIXWENqfNZNY2qmR0 + RizCccZ67yZuT0LrASdGYopsZakAsJFJINdjU50O51Srnfl+2Q0Zx5tftC5Lnjxr + 06g5T8ECgYAIqSiZU3HluraETpV4OD3XtOTnuEkCA/cd1msPCWsszgvA8Ij0PYjY + Wl4wlnBAz0mrZbdbJXZ5SqXRiwA+M/bckrODU5ZFiT8Fk5mJlvROqW5rLFIpnJKZ + cKYMl5I6JC6SQHcbCQ743Yohay5TyGmLFCMnCktx/lHXjZTk8ITY7g== + -----END RSA PRIVATE KEY----- + destinationCACertificate: "-----BEGIN CERTIFICATE-----\nMIIDbTCCAlWgAwIBAgIJAJR/jN0Oa+/rMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV\nBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMQswCQYDVQQHDAJOWTEcMBoGA1UE\nCgwTRGVmYXVsdCBDb21wYW55IEx0ZDAeFw0xNzAxMjQwODExMDJaFw0yNzAxMjIw\nODExMDJaME0xCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMQswCQYD\nVQQHDAJOWTEcMBoGA1UECgwTRGVmYXVsdCBDb21wYW55IEx0ZDCCASIwDQYJKoZI\nhvcNAQEBBQADggEPADCCAQoCggEBAMItGS9sSafyqBuOcQcQ5j7OQ0EwF9qOckhl\nfT8VzUbcOy8/L/w654MpLEa4O4Fiek3keE7SDWGVtGZWDvT9y1QUxPhkDWq1Y3rr\nyMelv1xRIyPVD7EEicga50flKe8CKd1U3D6iDQzq0uxZZ6I/VArXW/BZ4LfPauzN\n9EpCYyKq0fY7WRFIGouO9Wu800nxcHptzhLAgSpO97aaZ+V+jeM7n7fchRSNrpIR\nzPBl/lIBgCPJgkax0tcm4EIKIwlG+jXWc5mvV8sbT8rAv32HVuaP6NafyWXXP3H1\noBf2CQCcwuM0sM9ZeZ5JEDF/7x3eNtqSt1X9HjzVpQjiVBXY+E0CAwEAAaNQME4w\nHQYDVR0OBBYEFOXxMHAA1qaKWlP+gx8tKO2rQ81WMB8GA1UdIwQYMBaAFOXxMHAA\n1qaKWlP+gx8tKO2rQ81WMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEB\nAJAri7Pd0eSY/rvIIvAvjhDPvKt6gI5hJEUp+M3nWTWA/IhQFYutb9kkZGhbBeLj\nqneJa6XYKaCcUx6/N6Vvr3AFqVsbbubbejRpdpXldJC33QkwaWtTumudejxSon24\nW/ANN/3ILNJVMouspLRGkFfOYp3lq0oKAlNZ5G3YKsG0znAfqhAVtqCTG9RU24Or\nxzkEaCw8IY5N4wbjCS9FPLm7zpzdg/M3A/f/vrIoGdns62hzjzcp0QVTiWku74M8\nv7/XlUYYvXOvPQCCHgVjnAZlnjcxMTBbwtdwfxjAmdNTmFFpASnf0s3b287zQwVd\nIeSydalVtLm7rBRZ59/2DYo=\n-----END CERTIFICATE-----" + to: + kind: Service + name: nighthawk-benchmark-service-reencrypt-{{ item }} \ No newline at end of file diff --git a/roles/nighthawk/templates/workload.yml.j2 b/roles/nighthawk/templates/workload.yml.j2 new file mode 100644 index 000000000..e23289430 --- /dev/null +++ b/roles/nighthawk/templates/workload.yml.j2 @@ -0,0 +1,129 @@ +--- +kind: Job +apiVersion: batch/v1 +metadata: + name: 'nighthawk-client-{{item.status.hostIP}}-{{ item.metadata.labels.app }}-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' +spec: + template: + metadata: + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: nighthawk + benchmark-operator-role: client + app: nighthawk-bench-client-{{ trunc_uuid }} + clientfor: {{ item.metadata.labels.app }} + type: {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} +{% if workload_args.multus.enabled is sameas true %} + annotations: + k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.client | default("unknown") }} +{% endif %} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.hostnetwork is sameas true %} + hostNetwork: true + serviceAccountName: benchmark-operator +{% endif %} + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: type + operator: In + values: + - {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} + topologyKey: kubernetes.io/hostname + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ item.metadata.labels.app }} + topologyKey: kubernetes.io/hostname + containers: + - name: benchmark + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/nighthawk:latest') }} + env: + - name: uuid + value: "{{ uuid }}" + - name: test_user + value: "{{ test_user | default("ripsaw") }}" + - name: clustername + value: "{{ clustername }}" +{% if elasticsearch.url %} + - name: es + value: "{{ elasticsearch.url }}" + - name: es_index + value: "{{ elasticsearch.index_name | default("ripsaw-nighthawk") }}" + - name: parallel + value: "{{ elasticsearch.parallel | default(false) }}" + - name: es_verify_cert + value: "{{ elasticsearch.verify_cert | default(true) }}" +{% endif %} +{% if prometheus is defined %} + - name: prom_es + value: "{{ prometheus.es_url }}" + - name: prom_parallel + value: "{{ prometheus.es_parallel | default(false) }}" + - name: prom_token + value: "{{ prometheus.prom_token | default() }}" + - name: prom_url + value: "{{ prometheus.prom_url | default() }}" +{% endif %} + - name: client_node + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: server_node + value: "{{ workload_args.pin_server|default("unknown") }}" +{% if workload_args.client_resources is defined %} + resources: {{ workload_args.client_resources | to_json }} +{% endif %} + imagePullPolicy: Always + workingDir: /tmp + command: ["/bin/bash", "-c"] + args: + - | +{% if item.metadata.labels.termination|default("http") == "http" %} + protocol="http"; +{% else %} + protocol="https"; +{% endif %} +{% if workload_args.url is defined %} + url={{ workload_args.url | default("unknown") }} +{% else %} + url=$protocol://{{ item.metadata.labels.route }}-{{ operator_namespace }}.{{ routerCanonicalName.split('.')[1:] | join('.') }} +{% endif %} + exit_status=0; + while true; do + if [[ $(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{ trunc_uuid }}) =~ 'true' ]]; then + run_snafu -t nighthawk -s {{workload_args.samples|default(1)|int}} --resourcetype {{ resource_kind }} -u {{ uuid }} --user {{test_user | default("ripsaw")}} --url $url --concurrency {{ workload_args.concurrency|default(8)|int }} --duration {{ workload_args.duration|default(60)|int }} --connections {{ workload_args.connections|default(80)|int }} --max-requests-per-connection {{ workload_args.max_requests_per_connection|default(50)|int }} --create-archive --archive-file /tmp/nighthawk.archive \ +{% if workload_args.run_id is defined %} + --run-id {{workload_args.run_id}} \ +{% endif %} +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; + if [[ $? -ne 0 ]]; then + exit_status=1; + fi; + else + continue; + fi; + break; + done; + redis-cli -h {{bo.resources[0].status.podIP}} set start-{{ trunc_uuid }} false; + exit $exit_status; + restartPolicy: Never +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_client | default("unknown") }}' +{% endif %} diff --git a/roles/nighthawk/vars/main.yml b/roles/nighthawk/vars/main.yml new file mode 100644 index 000000000..f52cf6d81 --- /dev/null +++ b/roles/nighthawk/vars/main.yml @@ -0,0 +1,3 @@ +--- +# vars file for nighthawk +cleanup: true \ No newline at end of file diff --git a/roles/oslat/files/oslat.sh b/roles/oslat/files/oslat.sh index 53cc33300..3598d5f77 100755 --- a/roles/oslat/files/oslat.sh +++ b/roles/oslat/files/oslat.sh @@ -78,9 +78,9 @@ if [ "${USE_TASKSET}" == "true" ]; then prefix_cmd="taskset --cpu-list ${cyccore}" fi -echo "cmd to run: oslat --runtime ${RUNTIME} --rtprio ${RTPRIO} --cpu-list ${cyccore} --cpu-main-thread ${cpus[0]}" +echo "cmd to run: oslat --duration ${RUNTIME} --rtprio ${RTPRIO} --cpu-list ${cyccore} --cpu-main-thread ${cpus[0]}" -oslat --runtime ${RUNTIME} --rtprio ${RTPRIO} --cpu-list ${cyccore} --cpu-main-thread ${cpus[0]} +oslat --duration ${RUNTIME} --rtprio ${RTPRIO} --cpu-list ${cyccore} --cpu-main-thread ${cpus[0]} if [ "${DISABLE_CPU_BALANCE}" == "true" ]; then enable_balance diff --git a/roles/oslat/tasks/main.yml b/roles/oslat/tasks/main.yml index e2c3b7d4e..a043c7b97 100644 --- a/roles/oslat/tasks/main.yml +++ b/roles/oslat/tasks/main.yml @@ -1,103 +1,21 @@ --- - -- name: Get current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state if changed - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- block: - - - name: Add oslat test scripts - k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' - namespace: '{{ operator_namespace }}' - data: - oslat.sh: "{{ lookup('file', 'oslat.sh') }}" - functions.sh: "{{ lookup('file', 'functions.sh') }}" - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: ConfigMaps Created - complete: false - - when: resource_state.resources[0].status.state == "Building" - -- block: - - name: Start cyclitest job - k8s: - state: present - definition: "{{ lookup('template', 'oslatjob.yaml') | from_yaml }}" - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Benchmark Running - complete: false - - when: resource_state.resources[0].status.state == "ConfigMaps Created" - -- block: - - - name: Wait for benchmark to complete - k8s_facts: - kind: Job - api_version: batch/v1 - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' - namespace: "{{ operator_namespace }}" - register: oslat_status - - - name: Set Complete Status to benchmark - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Benchmark Complete - complete: true - when: oslat_status.resources[0].status.succeeded is defined and (oslat_status.resources[0].status.succeeded | int) > 0 - - - name: Set failed state to benchmark - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Failed - complete: true - when: oslat_status.resources[0].status.failed is defined and (oslat_status.resources[0].status.failed | int) > 0 - - when: resource_state.resources[0].status.state == "Benchmark Running" +- name: Add oslat test scripts + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: 'oslat-{{ trunc_uuid }}' + namespace: '{{ operator_namespace }}' + data: + oslat.sh: "{{ lookup('file', 'oslat.sh') }}" + functions.sh: "{{ lookup('file', 'functions.sh') }}" + when: benchmark_state.resources[0].status.state == "Building" + +- set_fact: + resources: + - "{{ role_path }}/templates/oslatjob.yaml" + +- name: Generic workload - oslat + include_role: + name: generic_workload diff --git a/roles/oslat/templates/oslatjob.yaml b/roles/oslat/templates/oslatjob.yaml index 7441527d3..3e506c919 100644 --- a/roles/oslat/templates/oslatjob.yaml +++ b/roles/oslat/templates/oslatjob.yaml @@ -1,28 +1,41 @@ apiVersion: batch/v1 kind: Job metadata: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "oslat-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: oslat-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} - ttlSecondsAfterFinished: 600 +{% if workload_args.node_selector is defined %} + nodeSelector: + '{{ workload_args.node_selector.split("=")[0] }}': '{{ workload_args.node_selector.split("=")[1] }}' +{% endif %} containers: - name: oslat image: {{ workload_args.image | default('quay.io/cloud-bulldozer/oslat:latest') }} command: ["/bin/sh", "-c"] - args: ["run_snafu --tool oslat -p /tmp/oslat.sh -u {{ uuid }} --user {{test_user | default("ripsaw")}}"] - imagePullPolicy: Always -{% if workload_args.nodeselector is defined %} - nodeSelector: - "{{ workload_args.nodeselector }}" + args: + - run_snafu --tool oslat -p /tmp/oslat.sh -u {{ uuid }} --user {{test_user | default("ripsaw")}} +{% if workload_args.debug is defined and workload_args.debug %} + -v {% endif %} + ; + imagePullPolicy: Always resources: requests: memory: {{ workload_args.pod.requests.memory }} @@ -37,11 +50,9 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es - value: "{{ elasticsearch.server }}" - - name: es_port - value: "{{ elasticsearch.port }}" + value: "{{ elasticsearch.url}}" - name: es_index value: "{{ elasticsearch.index_name | default("ripsaw-oslat") }}" - name: parallel @@ -80,8 +91,7 @@ spec: path: /dev/cpu_dma_latency - name: oslat-volume configMap: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "oslat-{{ trunc_uuid }}" defaultMode: 0555 restartPolicy: Never - serviceAccountName: benchmark-operator {% include "metadata.yml.j2" %} diff --git a/roles/pgbench/defaults/main.yml b/roles/pgbench/defaults/main.yml index ef4224582..5625a5c03 100644 --- a/roles/pgbench/defaults/main.yml +++ b/roles/pgbench/defaults/main.yml @@ -1,6 +1,6 @@ --- db_port: 5432 -pgb_base_image: "quay.io/cloud-bulldozer/pgbench:latest" +pgb_base_image: "{{ workload_args.image | default('quay.io/cloud-bulldozer/pgbench:latest') }}" timeout_min: 300 num_databases_pattern: "{{ workload_args.num_databases_pattern|default('all') }}" run_time: "{{ workload_args.run_time|default('') }}" diff --git a/roles/pgbench/tasks/check_clients.yml b/roles/pgbench/tasks/check_clients.yml index cbec31cfd..7d87b7028 100644 --- a/roles/pgbench/tasks/check_clients.yml +++ b/roles/pgbench/tasks/check_clients.yml @@ -3,12 +3,9 @@ shell: "redis-cli --raw llen pgb_client_ready" register: client_ready -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Run Workload" - complete: false +- include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Run Workload" when: client_ready.stdout|int == dbnum|int diff --git a/roles/pgbench/tasks/init.yml b/roles/pgbench/tasks/init.yml index f9a352a7b..28198f89c 100644 --- a/roles/pgbench/tasks/init.yml +++ b/roles/pgbench/tasks/init.yml @@ -2,11 +2,8 @@ - name: Initialize dbnum_item in redis command: "redis-cli set dbnum_item 0" -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Prep Workload" - complete: false +- include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Prep Workload" diff --git a/roles/pgbench/tasks/main.yml b/roles/pgbench/tasks/main.yml index 0ba50dd7b..80a89eae2 100644 --- a/roles/pgbench/tasks/main.yml +++ b/roles/pgbench/tasks/main.yml @@ -1,45 +1,10 @@ --- -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Init" - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Update current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- name: Capture benchmark operator information - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - name = benchmark-operator - register: bo - - name: Generate list of database counts include_tasks: generate_num_databases.yml - name: Init benchmark environment include_tasks: init.yml - when: resource_state.resources[0].status.state == "Init" + when: benchmark_state.resources[0].status.state == "Building" - name: Get dbnum_item from redis command: "redis-cli get dbnum_item" @@ -55,15 +20,15 @@ - name: Prep pgbench workload include_tasks: prep_workload.yml - when: resource_state.resources[0].status.state == "Prep Workload" + when: benchmark_state.resources[0].status.state == "Prep Workload" - name: Check pgbench clients include_tasks: check_clients.yml - when: resource_state.resources[0].status.state == "Check Clients" + when: benchmark_state.resources[0].status.state == "Check Clients" - name: Run pgbench workloads include_tasks: run_workload.yml - when: resource_state.resources[0].status.state == "Run Workload" + when: benchmark_state.resources[0].status.state == "Run Workload" #TODO: #- database passwords are currently provided in plain text in the CR diff --git a/roles/pgbench/tasks/prep_workload.yml b/roles/pgbench/tasks/prep_workload.yml index c361c1287..94a2bf679 100644 --- a/roles/pgbench/tasks/prep_workload.yml +++ b/roles/pgbench/tasks/prep_workload.yml @@ -7,15 +7,12 @@ - name: Setup pgbench test job(s) k8s: - definition: "{{ lookup('template', 'templates/workload.yml.j2') }}" + definition: "{{ lookup('template', 'workload.yml.j2') }}" with_indexed_items: "{{ workload_args.databases }}" when: item.0|int in range(dbnum|int) -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Check Clients" - complete: false +- include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Check Clients" diff --git a/roles/pgbench/tasks/run_workload.yml b/roles/pgbench/tasks/run_workload.yml index 5ef258178..05c9698a3 100644 --- a/roles/pgbench/tasks/run_workload.yml +++ b/roles/pgbench/tasks/run_workload.yml @@ -3,7 +3,7 @@ command: "redis-cli set pgb_start true" - name: Check for pods to complete - k8s_facts: + k8s_info: kind: pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -15,20 +15,17 @@ command: "redis-cli incr dbnum_item" when: "'Succeeded' in (pgbench_pods | json_query('resources[].status.phase'))" -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Prep Workload" - complete: false +- include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Prep Workload" when: "'Succeeded' in (pgbench_pods | json_query('resources[].status.phase')) and dbnum_item|int < (num_databases|length - 1)" - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: "Complete" diff --git a/roles/pgbench/templates/workload.yml.j2 b/roles/pgbench/templates/workload.yml.j2 index 7d804b165..c4eef932a 100644 --- a/roles/pgbench/templates/workload.yml.j2 +++ b/roles/pgbench/templates/workload.yml.j2 @@ -5,11 +5,19 @@ metadata: name: 'pgbench-{{ dbnum }}-dbs-client-{{ item.0|int + 1 }}-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: pgbench-client-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -32,7 +40,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: '{{ clustername }}' -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index @@ -73,7 +81,11 @@ spec: export sample_start_timestamp=$(date +%s.%3N); echo \"Begin test sample $i of {{ workload_args.samples }}...\"; export pgbench_opts=\"$pgbench_auth -c {{ clients }} -j {{ workload_args.threads }} {% if workload_args.transactions is defined and workload_args.transactions|int > 0 %} -t {{ workload_args.transactions }} {% elif run_time|int > 0 %} -T {{ run_time }} {% endif %} -s {{ workload_args.scaling_factor }} {{ cmd_flags }} {{ item.1.db_name }}\"; - run_snafu --tool pgbench -r $i; + run_snafu --tool pgbench -r $i \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; done; {% endfor %} else @@ -82,7 +94,7 @@ spec: break; done; fi" - restartPolicy: OnFailure + restartPolicy: Never {% if item.1.pin_node is defined and item.1.pin_node %} nodeSelector: kubernetes.io/hostname: '{{ item.1.pin_node }}' diff --git a/roles/scale_openshift/tasks/main.yml b/roles/scale_openshift/tasks/main.yml index 3ebfa25aa..e680236d9 100644 --- a/roles/scale_openshift/tasks/main.yml +++ b/roles/scale_openshift/tasks/main.yml @@ -1,66 +1,8 @@ --- -- name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - register: benchmark_state +- set_fact: + resources: + - "{{ role_path }}/templates/scale.yml" -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Scale Pod" - complete: false - when: benchmark_state.resources[0].status.state is not defined - -- name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - register: benchmark_state - -- block: - - - name: Create scale pod - k8s: - state: present - definition: "{{ lookup('template', 'scale.yml') }}" - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Scaling Cluster" - - when: benchmark_state.resources[0].status.state == "Starting Scale Pod" - -- block: - - - name: Waiting for scaling pod to complete - k8s_facts: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = scale-{{ trunc_uuid }} - register: scale_pod - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "scale_pod|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length > 0" - - when: benchmark_state.resources[0].status.state == "Scaling Cluster" +- name: Generic workload - scale_openshift + include_role: + name: generic_workload diff --git a/roles/scale_openshift/templates/scale.yml b/roles/scale_openshift/templates/scale.yml index e7e8eb750..4af503f55 100755 --- a/roles/scale_openshift/templates/scale.yml +++ b/roles/scale_openshift/templates/scale.yml @@ -5,11 +5,19 @@ metadata: name: 'scale-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: scale-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -42,7 +50,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index @@ -62,10 +70,24 @@ spec: - name: prom_url value: "{{ prometheus.prom_url | default() }}" {% endif %} - command: ["/bin/sh", "-c"] +{% if workload_args.aws is defined %} + - name: AWS_ACCESS_KEY_ID + value: "{{ workload_args.aws.access_key_id | default() }}" + - name: AWS_SECRET_ACCESS_KEY + value: "{{ workload_args.aws.secret_access_key | default() }}" + - name: AWS_DEFAULT_REGION + value: "{{ workload_args.aws.default_region | default() }}" +{% endif %} + command: ["/bin/sh", "-e", "-c"] args: - - "run_snafu --tool scale --scale {{workload_args.scale}} -u {{uuid}} --user {{test_user|default("ripsaw")}} --incluster true --poll_interval {{workload_args.poll_interval|default(5)}}; + - "run_snafu --tool scale --scale {{workload_args.scale}} -u {{uuid}} --user {{test_user|default("ripsaw")}} --incluster true --poll_interval {{workload_args.poll_interval|default(5)}} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} +{% if workload_args.rosa is defined %} + --rosa-cluster {{workload_args.rosa.cluster_name}} --rosa-env {{workload_args.rosa.env}} --rosa-token {{workload_args.rosa.token}} \ +{% endif %} + ; sleep {{post_sleep|default(0)}}" - serviceAccountName: {{workload_args.serviceaccount|default("default")}} restartPolicy: Never {% include "metadata.yml.j2" %} diff --git a/roles/servicemesh/tasks/main.yml b/roles/servicemesh/tasks/main.yml index 65e45eb1a..da2099e4a 100644 --- a/roles/servicemesh/tasks/main.yml +++ b/roles/servicemesh/tasks/main.yml @@ -1,25 +1,17 @@ --- -- name: Get current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - name: Define variables set_fact: servicemesh: "{{ servicemesh_defaults | combine(workload_args, recursive=True) }}" - controlplane_namespace: "{{ meta.name }}-controlplane-{{ trunc_uuid }}" - workload_namespace: "{{ meta.name }}-workload-{{ trunc_uuid }}" + controlplane_namespace: "{{ ansible_operator_meta.name }}-controlplane-{{ trunc_uuid }}" + workload_namespace: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" owner: - apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" - uid: "{{ resource_state.resources[0].metadata.uid }}" + uid: "{{ benchmark_state.resources[0].metadata.uid }}" -- when: resource_state.resources[0].status.state is not defined +- when: benchmark_state.resources[0].status.state is not defined block: - name: Create control plane namespace k8s: @@ -59,38 +51,25 @@ apiVersion: hyperfoil.io/v1alpha2 kind: Hyperfoil metadata: - name: '{{ meta.name }}-hyperfoil-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-hyperfoil-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' ownerReferences: "{{ owner }}" spec: version: "{{ servicemesh.hyperfoil_version }}" - - name: Set state to Init - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Init" - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Init -# Note: register is invoked even if the task is skipped; therefore we'll update the state non-conditionally -- name: Update current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- when: resource_state.resources[0].status.state == "Init" +- when: benchmark_state.resources[0].status.state == "Init" block: - name: Check Hyperfoil-status k8s_info: api_version: hyperfoil.io/v1alpha2 kind: Hyperfoil - name: "{{ meta.name }}-hyperfoil-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-hyperfoil-{{ trunc_uuid }}" namespace: '{{ operator_namespace }}' register: hf_state @@ -98,31 +77,19 @@ k8s_info: api_version: maistra.io/v1 kind: ServiceMeshControlPlane - name: '{{ meta.name }}' + name: '{{ ansible_operator_meta.name }}' namespace: "{{ controlplane_namespace }}" register: smcp_state - set_fact: smcp_ready: "{{ smcp_state | json_query('resources[0].status.conditions[]') | selectattr('type','match','Ready') | selectattr('status', 'match', 'True') | list | length > 0 }}" - - name: Progress to deploy + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Deploy when: hf_state.resources[0].status.status == "Ready" and smcp_ready - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Deploy" - complete: false - -- name: Update current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - name: Check ingress route k8s_info: @@ -136,9 +103,9 @@ when: ingress_route.resources | length > 0 set_fact: #TODO: maybe use trunc_uuid in the hostname? - app_route: "{{ meta.name }}-app-{{ trunc_uuid}}{{ ingress_route.resources[0].status.ingress[0].host | regex_replace('^[^.]*', '')}}" + app_route: "{{ ansible_operator_meta.name }}-app-{{ trunc_uuid}}{{ ingress_route.resources[0].status.ingress[0].host | regex_replace('^[^.]*', '')}}" -- when: resource_state.resources[0].status.state == "Deploy" +- when: benchmark_state.resources[0].status.state == "Deploy" block: - name: Generate certificates command: openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /tmp/app.key -out /tmp/app.crt -subj "/CN={{ app_route }}/O=app" @@ -195,26 +162,14 @@ ready_containers: "{{ pods.resources | map(attribute='status.containerStatuses') | flatten | selectattr('ready', 'equalto', true) | list | length }}" expected_containers: "{{ 2 * servicemesh.deployments * servicemesh.pods_per_deployment }}" - - name: Progress to ready + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Ready when: ready_containers == expected_containers - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Ready" - complete: false - -- name: Update current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state -- when: resource_state.resources[0].status.state == "Ready" +- when: benchmark_state.resources[0].status.state == "Ready" block: - name: "Expand test template into config map" k8s: @@ -223,7 +178,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: "{{ meta.name }}-test-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-test-{{ trunc_uuid }}" namespace: '{{ operator_namespace }}' ownerReferences: "{{ owner }}" data: @@ -236,7 +191,7 @@ kind: Pod namespace: '{{ operator_namespace }}' label_selectors: - - app = {{ meta.name }}-hyperfoil-{{ trunc_uuid }} + - app = {{ ansible_operator_meta.name }}-hyperfoil-{{ trunc_uuid }} - role = controller register: hf_pod @@ -244,65 +199,20 @@ k8s: definition: "{{ lookup('template', 'job.yaml.j2') | from_yaml }}" - - name: Progress to running - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Running" - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running -- name: Update current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- when: resource_state.resources[0].status.state == "Running" +- when: benchmark_state.resources[0].status.state == "Running" block: - - name: Get job status - k8s_info: - api_version: v1 - kind: Job - namespace: '{{ operator_namespace }}' - name: "{{ meta.name }}-{{ trunc_uuid }}" - register: job_state - - - name: Complete benchmark - when: job_state.resources[0].status.succeeded is defined and (job_state.resources[0].status.succeeded | int) > 0 - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Complete" - complete: true - - - name: Fail benchmark - when: job_state.resources[0].status.failed is defined and (job_state.resources[0].status.failed | int) > 0 - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Failed" - complete: true - -- name: Update current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state + - name: Set completed state + include_role: + name: benchmark_state + tasks_from: completed.yml -- when: resource_state.resources[0].status.state == "Complete" and cleanup +- when: benchmark_state.resources[0].status.state == "Complete" and cleanup block: - name: Delete Hyperfoil resource k8s: @@ -311,7 +221,7 @@ apiVersion: hyperfoil.io/v1alpha2 kind: Hyperfoil metadata: - name: '{{ meta.name }}-hyperfoil-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-hyperfoil-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' - name: Delete config map @@ -321,7 +231,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-test-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-test-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' - name: Delete workload namespace diff --git a/roles/servicemesh/templates/job.yaml.j2 b/roles/servicemesh/templates/job.yaml.j2 index e0fff579f..190169afd 100644 --- a/roles/servicemesh/templates/job.yaml.j2 +++ b/roles/servicemesh/templates/job.yaml.j2 @@ -1,7 +1,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: "{{ meta.name }}-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-{{ trunc_uuid }}" namespace: '{{ operator_namespace }}' ownerReferences: "{{ owner }}" spec: @@ -9,7 +9,13 @@ spec: template: metadata: labels: - app: "{{ meta.name }}-{{ trunc_uuid }}" + app: "{{ ansible_operator_meta.name }}-{{ trunc_uuid }}" +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -31,4 +37,4 @@ spec: - name: files configMap: defaultMode: 493 - name: "{{ meta.name }}-test-{{ trunc_uuid }}" \ No newline at end of file + name: "{{ ansible_operator_meta.name }}-test-{{ trunc_uuid }}" diff --git a/roles/servicemesh/templates/smcp.yaml.j2 b/roles/servicemesh/templates/smcp.yaml.j2 index 7babdc7b1..77892ed09 100644 --- a/roles/servicemesh/templates/smcp.yaml.j2 +++ b/roles/servicemesh/templates/smcp.yaml.j2 @@ -1,7 +1,7 @@ apiVersion: maistra.io/v1 kind: ServiceMeshControlPlane metadata: - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ controlplane_namespace }}" spec: istio: diff --git a/roles/smallfile/tasks/main.yml b/roles/smallfile/tasks/main.yml index 46b49efd3..801a92afb 100644 --- a/roles/smallfile/tasks/main.yml +++ b/roles/smallfile/tasks/main.yml @@ -1,31 +1,4 @@ --- -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- name: Update current state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - set_fact: @@ -68,15 +41,6 @@ subscriber: "{{ lookup('template', 'subscriber.py.j2') }}" publisher: "{{ lookup('template', 'publisher.py.j2') }}" - - name: Capture operator information - k8s_facts: - kind: Pod - api_version: v1 - namespace: "{{ operator_namespace }}" - label_selectors: - - name = benchmark-operator - register: bo - - name: Run PUBLISHER Pod k8s: definition: @@ -94,12 +58,12 @@ spec: containers: - name: publisher-container - image: "{{ workload_args.image | default('quay.io/cloud-bulldozer/smallfile:master') }}" + image: "{{ workload_args.image | default('quay.io/cloud-bulldozer/smallfile:latest') }}" tty: true command: ["/bin/sh", "-c"] workingDir: /root/smallfile-master/ args: - - python /tmp/publisher {{bo.resources[0].status.podIP}} {{workload_args.clients|default('1')|int}} + - python /tmp/publisher {{bo.resources[0].status.podIP}} {{workload_args.clients|default('1')|int}} {{trunc_uuid}} volumeMounts: - name: config-volume mountPath: "/tmp" @@ -117,36 +81,15 @@ with_sequence: start=1 count={{ workload_args.clients|default('1')|int }} when: workload_args.clients|default('1')|int > 0 - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - complete: false - - when: resource_state.resources[0].status.state == "Starting" - -- block: - - - name: Waiting for pods to complete.... - k8s_facts: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = smallfile-benchmark-{{ trunc_uuid }} - register: client_pods + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "workload_args.clients|default('1')|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: benchmark_state.resources[0].status.state == "Building" - when: resource_state.resources[0].status.state == "Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/smallfile/templates/publisher.py.j2 b/roles/smallfile/templates/publisher.py.j2 index 23f41dac1..b393cfc72 100644 --- a/roles/smallfile/templates/publisher.py.j2 +++ b/roles/smallfile/templates/publisher.py.j2 @@ -8,16 +8,17 @@ import sys redis_host = sys.argv[1] number_of_pods = sys.argv[2] +uuid = sys.argv[3] try: r = redis.StrictRedis(host=redis_host, port=6379) count = 0 while int(count) != int(number_of_pods): - count = r.execute_command("PUBSUB NUMSUB smallfile")[1] + count = r.execute_command(f"PUBSUB NUMSUB smallfile-{uuid}")[1] print(count, file=sys.stdout) time.sleep(1) print("All Pods are ready to run. Triggering the run ........\(^_^)/", file=sys.stdout) - r.publish("smallfile","run") + r.publish(f"smallfile-{uuid}","run") except Exception as e: print("******Exception Occured*******") print(str(e)) diff --git a/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 b/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 index 5540a5fcb..666e8153b 100644 --- a/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 +++ b/roles/smallfile/templates/smallfilejob.other_parameters.yaml.j2 @@ -1,6 +1,12 @@ - - top: {{ smallfile_path }}/smallfile_test_data + +{% if workload_args.auto_pause is defined %} +auto-pause: {{ workload_args.auto_pause }} +{% endif %} + +{% if workload_args.clients is defined %} +total-hosts: {{ workload_args.clients }} +{% endif %} {% if workload_args.threads is defined %} threads: {{ workload_args.threads }} @@ -15,81 +21,86 @@ files: {{ workload_args.files }} {% endif %} {% if workload_args.fsync is defined %} -fsync: y +fsync: {{ workload_args.fsync }} {% endif %} -{% if workload_args.response_times is defined %} -response-times: true -{% endif %} +# response-times is always set to Y so that +# benchmark-wrapper can collect response time data -{% if workload_args.network_sync_dir is defined %} -network-sync-dir: {{workload_args.network_sync_dir}} -{% endif %} +# network-sync-dir is never used with benchmark-operator +# since it uses redis to synchronize pods and elastic to return data {% if workload_args.files_per_dir is defined %} -files-per-dir: {{workload_args.files_per_dir}} +files-per-dir: {{ workload_args.files_per_dir }} {% endif %} {% if workload_args.dirs_per_dir is defined %} -dirs-per-dir: {{workload_args.dirs_per_dir}} +dirs-per-dir: {{ workload_args.dirs_per_dir }} {% endif %} {% if workload_args.record_size is defined %} -record-size: {{workload_args.record_size}} +record-size: {{ workload_args.record_size }} {% endif %} {% if workload_args.file_size_distribution is defined %} -file-size-distribution: {{workload_args.file_size_distribution}} +file-size-distribution: {{ workload_args.file_size_distribution }} {% endif %} {% if workload_args.xattr_size is defined %} -xattr-size: {{workload_args.xattr_size}} +xattr-size: {{ workload_args.xattr_size }} {% endif %} {% if workload_args.xattr_count is defined %} -xattr-count: {{workload_args.xattr_count}} +xattr-count: {{ workload_args.xattr_count }} {% endif %} {% if workload_args.pause is defined %} -pause: {{workload_args.pause}} +pause: {{ workload_args.pause }} {% endif %} {% if workload_args.stonewall is defined %} -stonewall: {{workload_args.stonewall}} +stonewall: {{ workload_args.stonewall }} {% endif %} {% if workload_args.finish is defined %} -finish: {{workload_args.finish}} +finish: {{ workload_args.finish }} {% endif %} {% if workload_args.prefix is defined %} -prefix: {{workload_args.prefix}} +prefix: {{ workload_args.prefix }} +{% endif %} + +{% if workload_args.suffix is defined %} +suffix: {{ workload_args.suffix }} {% endif %} {% if workload_args.hash_into_dirs is defined %} -hash-into-dirs: {{workload_args.hash_into_dirs}} +hash-into-dirs: {{ workload_args.hash_into_dirs }} {% endif %} {% if workload_args.same_dir is defined %} -same-dir: {{workload_args.same_dir}} +same-dir: {{ workload_args.same_dir }} {% endif %} {% if workload_args.verbose is defined %} -verbose: {{workload_args.verbose}} +verbose: {{ workload_args.verbose }} {% endif %} -{% if workload_args.permute_host_dirs is defined %} -permute-host-dirs: true -{% endif %} +# permute-host-dirs is not applicable to K8S pods {% if workload_args.record_ctime_size is defined %} -record-ctime-size: true +record-ctime-size: {{ workload_args.record_ctime_size }} {% endif %} {% if workload_args.verify_read is defined %} -verify-read: true +verify-read: {{ workload_args.verify_read }} {% endif %} {% if workload_args.incompressible is defined %} -incompressible: true +incompressible: {{ workload_args.incompressible }} {% endif %} + +{% if workload_args.cleanup_delay_usec_per_file is defined %} +cleanup-delay-usec-per-file: {{ workload_args.cleanup_delay_usec_per_file }} +{% endif %} + diff --git a/roles/smallfile/templates/subscriber.py.j2 b/roles/smallfile/templates/subscriber.py.j2 index b69babf06..08b7c7dba 100644 --- a/roles/smallfile/templates/subscriber.py.j2 +++ b/roles/smallfile/templates/subscriber.py.j2 @@ -6,12 +6,13 @@ import sys import redis redis_host = sys.argv[1] +uuid = sys.argv[2] try: r = redis.StrictRedis(host=redis_host, port=6379) p = r.pubsub() - p.subscribe('smallfile') + p.subscribe(f'smallfile-{uuid}') STATE = True while STATE: diff --git a/roles/smallfile/templates/workload_job.yml.j2 b/roles/smallfile/templates/workload_job.yml.j2 index 5a6c06efb..1be1b1958 100644 --- a/roles/smallfile/templates/workload_job.yml.j2 +++ b/roles/smallfile/templates/workload_job.yml.j2 @@ -5,12 +5,19 @@ metadata: name: "smallfile-client-{{ item }}-benchmark-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: - ttlSecondsAfterFinished: 600 backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: smallfile-benchmark-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -29,12 +36,12 @@ spec: topologyKey: "kubernetes.io/hostname" restartPolicy: Never containers: - - name: benchmark-server + - name: smallfile {% if hostpath is defined %} securityContext: privileged: true {% endif %} - image: {{ workload_args.image | default('quay.io/cloud-bulldozer/smallfile:master') }} + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/smallfile:latest') }} imagePullPolicy: Always command: ["/bin/sh", "-c"] workingDir: /opt/snafu @@ -42,20 +49,28 @@ spec: - name: uuid value: "{{ uuid }}" - name: test_user - value: "{{ test_user | default("ripsaw") }}" + value: "{{ test_user | default('ripsaw') }}" - name: clustername - value: "{{ clustername }}" -{% if kcache_drop_pod_ips is defined and item | int == 1 %} + value: "{{ clustername | default('mycluster') }}" +{% if item | int == 1 %} +{% if kcache_drop_pod_ips is defined %} - name: kcache_drop_pod_ips - value: "{{ kcache_drop_pod_ips | default() }}" + value: "{{ kcache_drop_pod_ips }}" - name: KCACHE_DROP_PORT_NUM value: "{{ kernel_cache_drop_svc_port }}" {% endif %} -{% if elasticsearch is defined %} +{% if ceph_osd_cache_drop_pod_ip is defined %} + - name: ceph_osd_cache_drop_pod_ip + value: "{{ ceph_osd_cache_drop_pod_ip }}" + - name: CEPH_CACHE_DROP_PORT_NUM + value: "{{ ceph_cache_drop_svc_port }}" +{% endif %} +{% endif %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index - value: "{{ elasticsearch.index_name | default("ripsaw-smallfile") }}" + value: "{{ elasticsearch.index_name | default('ripsaw-smallfile') }}" - name: parallel value: "{{ elasticsearch.parallel | default(false) }}" - name: es_verify_cert @@ -78,21 +93,28 @@ spec: - name: clients value: "{{ workload_args.clients|default('1')|int }}" args: - - python /tmp/smallfile/subscriber {{bo.resources[0].status.podIP}}; + - python /tmp/smallfile/subscriber {{ bo.resources[0].status.podIP }} {{ trunc_uuid }}; export TMPDIR=/tmp - arr=$(python <<< "print(','.join({{workload_args.operation}}))"); - rm -rf {{smallfile_path}}/RESULTS ; - mkdir -p {{smallfile_path}}/RESULTS; - mkdir -p {{smallfile_path}}/smallfile_test_data; + arr=$(python <<< "print(','.join({{ workload_args.operation }}))"); + rm -rf {{ smallfile_path }}/RESULTS ; + mkdir -p {{ smallfile_path }}/RESULTS; + mkdir -p {{ smallfile_path }}/smallfile_test_data; run_snafu {% if workload_args.samples is defined %} - --samples {{workload_args.samples}} + --samples {{ workload_args.samples }} +{% endif %} + --tool smallfile + --operations $arr + --top {{ smallfile_path }}/smallfile_test_data + --dir /var/tmp/RESULTS + --yaml-input-file /tmp/smallfile/smallfilejob +{% if workload_args.debug is defined and workload_args.debug %} + -v +{% endif %} + ; +{% if workload_args.debug is defined and workload_args.debug %} + ls /tmp ; cat /tmp/invoke*log ; {% endif %} - --tool smallfile - --operations $arr - --top {{smallfile_path}}/smallfile_test_data - --dir /var/tmp/RESULTS - --yaml-input-file /tmp/smallfile/smallfilejob ; if [ $? = 0 ] ; then echo RUN STATUS DONE ; else exit 1 ; fi volumeMounts: - name: config-volume @@ -110,7 +132,7 @@ spec: {% if workload_args.storageclass is defined %} - name: data-volume persistentVolumeClaim: - claimName: "claim{{item}}-{{ trunc_uuid }}" + claimName: "claim{{ item }}-{{ trunc_uuid }}" {% elif hostpath is defined %} - name: data-volume hostPath: @@ -118,5 +140,4 @@ spec: type: DirectoryOrCreate {% endif %} restartPolicy: Never - serviceAccountName: benchmark-operator {% include "metadata.yml.j2" %} diff --git a/roles/stressng/tasks/main.yaml b/roles/stressng/tasks/main.yaml index 060906c06..bb1e72f66 100644 --- a/roles/stressng/tasks/main.yaml +++ b/roles/stressng/tasks/main.yaml @@ -1,30 +1,4 @@ --- -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - If it has changed - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - name: template stressng config file @@ -33,25 +7,22 @@ apiVersion: v1 kind: ConfigMap metadata: - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' + name: '{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' data: jobfile: "{{ lookup ('template', 'jobfile.j2') }}" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: ConfigMaps Created - complete: false - - when: resource_state.resources[0].status.state == "Building" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: ConfigMaps Created + when: benchmark_state.resources[0].status.state == "Building" - block: + - name: set complete to false - command: "redis-cli set complete false" + command: "redis-cli set complete-{{ trunc_uuid }} false" - name: start stressng workload job k8s: @@ -65,50 +36,36 @@ register: jobs when: resource_kind == "vm" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Benchmark running - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - when: resource_state.resources[0].status.state == "ConfigMaps Created" + when: benchmark_state.resources[0].status.state == "ConfigMaps Created" - block: - - block: - - name: wait for benchmark job to finish - k8s_facts: - kind: Job - api_version: batch/v1 - name: '{{ meta.name }}-workload-{{ trunc_uuid }}' - namespace: "{{ operator_namespace }}" - register: stressng_workload_pod - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Benchmark Complete - complete: true - when: stressng_workload_pod | json_query('resources[].status.succeeded') + - include_role: + name: benchmark_state + tasks_from: completed.yml when: resource_kind == "pod" + - block: + - name: get complete - command: "redis-cli get complete" + command: "redis-cli get complete-{{ trunc_uuid }}" register: complete_status + - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: - state: Benchmark Complete + state: Complete complete: true when: complete_status.stdout == "true" when: resource_kind == "vm" - when: resource_state.resources[0].status.state == "Benchmark running" + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/stressng/templates/jobfile.j2 b/roles/stressng/templates/jobfile.j2 index a296f322c..42dd72019 100644 --- a/roles/stressng/templates/jobfile.j2 +++ b/roles/stressng/templates/jobfile.j2 @@ -7,6 +7,7 @@ timeout {{ workload_args.timeout}} # stop each stressor after 60 seconds {% if workload_args.cpu_stressors is defined %} cpu {{ workload_args.cpu_stressors }} # cpu stressor cpu-load {{ workload_args.cpu_percentage }} # percentage to which the cpu gets loaded +cpu-method {{ workload_args.cpu_method }} # cpu test method {% endif %} # vm stressor diff --git a/roles/stressng/templates/stressng_workload.yml.j2 b/roles/stressng/templates/stressng_workload.yml.j2 index dfb9eb309..24841e56f 100644 --- a/roles/stressng/templates/stressng_workload.yml.j2 +++ b/roles/stressng/templates/stressng_workload.yml.j2 @@ -1,17 +1,31 @@ --- apiVersion: batch/v1 -kind: "job" +kind: Job metadata: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: parallelism: {{ workload_args.instances }} - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: stressng_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: stressng +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: +{% if workload_args.pin is sameas true %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_node }}' +{% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} @@ -20,6 +34,15 @@ spec: {% endif %} containers: - name: stressng +{% if workload_args.resources is sameas true %} + resources: + requests: + cpu: {{ workload_args.requests_cpu }} + memory: {{ workload_args.requests_memory }} + limits: + cpu: {{ workload_args.limits_cpu }} + memory: {{ workload_args.limits_memory }} +{% endif %} image: {{ workload_args.image | default('quay.io/cloud-bulldozer/stressng:latest') }} imagePullPolicy: Always env: @@ -53,7 +76,7 @@ spec: - name: mem_stressors value: "{{workload_args.mem_stressors}}" {% endif %} -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index @@ -74,7 +97,12 @@ spec: value: "{{ prometheus.prom_url | default() }}" {% endif %} command: ["/bin/sh", "-c"] - args: ["run_snafu --tool stressng -j /workload/jobfile -u {{ uuid }}"] + args: + - run_snafu --tool stressng -j /workload/jobfile -u {{ uuid }} +{% if workload_args.debug is defined and workload_args.debug %} + -v +{% endif %} + ; volumeMounts: - name: stressng-workload-volume mountPath: "/workload" @@ -82,8 +110,7 @@ spec: volumes: - name: stressng-workload-volume configMap: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" defaultMode: 0660 - restartPolicy: OnFailure + restartPolicy: Never {% include "metadata.yml.j2" %} - diff --git a/roles/stressng/templates/stressng_workload_vm.yml.j2 b/roles/stressng/templates/stressng_workload_vm.yml.j2 index 7545aa84d..994ae5709 100644 --- a/roles/stressng/templates/stressng_workload_vm.yml.j2 +++ b/roles/stressng/templates/stressng_workload_vm.yml.j2 @@ -2,8 +2,19 @@ apiVersion: kubevirt.io/v1alpha3 kind: VirtualMachineInstance metadata: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" + labels: + app: stressng_workload-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-workload-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: stressng +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: domain: cpu: @@ -40,7 +51,7 @@ spec: terminationGracePeriodSeconds: 0 {% if workload_args.pin is sameas true %} nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' + kubernetes.io/hostname: '{{ workload_args.pin_node }}' {% endif %} networks: - name: default @@ -60,15 +71,24 @@ spec: - "mkdir /tmp/stressng-test/jobfile" - "mount /dev/$(lsblk --nodeps -no name,serial | grep CVLY623300HK240D | cut -f1 -d' ') /tmp/stressng-test" runcmd: +{% if elasticsearch.url %} + - export es={{ elasticsearch.url }}; + - export es_index={{ elasticsearch.index_name | default("ripsaw-stressng") }}; + - export es_verify_cert={{ elasticsearch.verify_cert | default("true") }}; + - export parallel={{ elasticsearch.parallel | default("false") }}; + - export uuid={{uuid}}; +{% endif %} {% if workload_args.client_vm.network.multiqueue.enabled %} - dnf install -y ethtool - ethtool -L eth0 combined {{ workload_args.client_vm.network.multiqueue.queues }} {% endif %} - - dnf install -y stress-ng redis git - - pip3 install git+https://github.com/cloud-bulldozer/benchmark-wrapper + - dnf install -y python3-pip + - dnf install -y git redis stress-ng + - pip install git+https://github.com/cloud-bulldozer/benchmark-wrapper - run_snafu --tool stressng -j /tmp/stressng-test/jobfile -u {{ uuid }} + - redis-cli -h {{bo.resources[0].status.podIP}} set complete-{{ trunc_uuid }} true name: cloudinitdisk - configMap: - name: "{{ meta.name }}-workload-{{ trunc_uuid }}" + name: "{{ ansible_operator_meta.name }}-workload-{{ trunc_uuid }}" name: stressng-workload-volume status: {} diff --git a/roles/sysbench/tasks/main.yml b/roles/sysbench/tasks/main.yml index 9edd25fbb..5c594b790 100644 --- a/roles/sysbench/tasks/main.yml +++ b/roles/sysbench/tasks/main.yml @@ -1,31 +1,4 @@ --- - -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Update current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - block: - name: template sysbench script k8s: @@ -37,6 +10,24 @@ namespace: '{{ operator_namespace }}' data: sysbenchScript: "{{ lookup('template', 'sysbench.sh.j2') }}" + - name: Create PVC + k8s: + definition: + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: "claim-sysbench-{{ trunc_uuid }}" + namespace: '{{ operator_namespace }}' + annotations: + volume.beta.kubernetes.io/storage-class: "{{ workload_args.storageclass }}" + spec: + accessModes: + - "{{ workload_args.pvcaccessmode | default('ReadWriteOnce') }}" + volumeMode: "{{ workload_args.pvcvolumemode | default('Filesystem') }}" + resources: + requests: + storage: "{{ workload_args.storagesize }}" + when: workload_args.storageclass is defined - name: Start sysbench job k8s: @@ -50,36 +41,15 @@ definition: "{{ lookup('template', 'workload_vm.yml') | from_yaml }}" when: workload_args.kind is defined and workload_args.kind == "vm" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - complete: false - - when: resource_state.resources[0].status.state == "Starting" - -- block: - - - name: Waiting for pods to complete.... - k8s_facts: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = sysbench-{{ trunc_uuid }} - register: client_pods + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "1 == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: benchmark_state.resources[0].status.state == "Building" - when: resource_state.resources[0].status.state == "Running" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/sysbench/templates/sysbench.sh.j2 b/roles/sysbench/templates/sysbench.sh.j2 index 9aba92636..17f96af0a 100755 --- a/roles/sysbench/templates/sysbench.sh.j2 +++ b/roles/sysbench/templates/sysbench.sh.j2 @@ -1,4 +1,12 @@ #!/bin/bash +{% macro merge_params(global, local) -%} +{% for key, val in global.items() %} + {% if key not in local %} + {% set dummy = local.__setitem__(key, val) %} + {% endif %} +{% endfor %} +{%- endmacro -%} + {% for test in workload_args.tests %} {% if test.name == "cpu" or test.name == "memory" %} sysbench --test={{test.name}} {% for opt,value in test.parameters.items() %} --{{opt}}={{value}} {% endfor %} run @@ -6,11 +14,34 @@ sysbench --test={{test.name}} {% for opt,value in test.parameters.items() %} --{ {% if 'file-total-size' not in test.parameters %} {% set dummy = test['parameters'].__setitem__("file-total-size", "1G") %} {% endif %} -sysbench --test=fileio {% for opt,value in test.parameters.items() %} --{{opt}}={{value}} {% endfor %} prepare + +{% if 'prepare_parameters' in test %} +{% set dummy = merge_params(test.parameters, test.prepare_parameters) %} +{% set prep_params = test.prepare_parameters %} +{% else %} +{% set prep_params = test.parameters %} +{% endif %} +sysbench --test=fileio {% for opt,value in prep_params.items() %} {% if value is not none %} --{{opt}}={{value}} {% endif %} {% endfor %} prepare + {% if 'file-test-mode' not in test.parameters %} {% set dummy = test['parameters'].__setitem__("file-test-mode", "rndrw") %} {% endif %} -sysbench --test=fileio {% for opt,value in test.parameters.items() %} --{{opt}}={{value}} {% endfor %} run -sysbench --test=fileio --file-total-size={{test['parameters']['file-total-size']}} cleanup + +{% if 'run_parameters' in test %} +{% set dummy = merge_params(test.parameters, test.run_parameters) %} +{% set run_params = test.run_parameters %} +{% else %} +{% set run_params = test.parameters %} +{% endif %} +sysbench --test=fileio {% for opt,value in run_params.items() %} {% if value is not none %} --{{opt}}={{value}} {% endif %}{% endfor %} run + +{% if 'cleanup_parameters' in test %} +{% set dummy = merge_params(test.parameters, test.cleanup_parameters) %} +{% set clean_params = test.cleanup_parameters %} +{% else %} +{% set clean_params = test.parameters %} +{% endif %} +sysbench --test=fileio {% for opt, value in clean_params.items() %} {% if value is not none %} --{{opt}}={{value}} {% endif %}{% endfor %} cleanup + {% endif %} {% endfor %} diff --git a/roles/sysbench/templates/workload.yml b/roles/sysbench/templates/workload.yml index 404310d84..cc5443bcb 100644 --- a/roles/sysbench/templates/workload.yml +++ b/roles/sysbench/templates/workload.yml @@ -6,12 +6,24 @@ metadata: name: "sysbench-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" spec: - ttlSecondsAfterFinished: 600 + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: sysbench-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: +{% if workload_args.pin_node is defined %} + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_node }}' +{% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} @@ -28,10 +40,15 @@ spec: mountPath: "/opt/sysbench" volumes: - name: sysbench-runtime +{% if workload_args.storageclass is defined %} + persistentVolumeClaim: + claimName: claim-sysbench-{{ trunc_uuid }} +{% else %} emptyDir: {} +{% endif %} - name: sysbench-volume configMap: name: "sysbench-config-{{ trunc_uuid }}" defaultMode: 0777 - restartPolicy: OnFailure + restartPolicy: Never {% include "metadata.yml.j2" %} diff --git a/roles/sysbench/templates/workload_vm.yml b/roles/sysbench/templates/workload_vm.yml index 148e49c97..5f03efd73 100644 --- a/roles/sysbench/templates/workload_vm.yml +++ b/roles/sysbench/templates/workload_vm.yml @@ -7,6 +7,13 @@ metadata: namespace: "{{ operator_namespace }}" labels: kubevirt-vm: vm-sysbench-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: domain: devices: diff --git a/roles/system-metrics/tasks/main.yml b/roles/system-metrics/tasks/main.yml new file mode 100644 index 000000000..fd642228f --- /dev/null +++ b/roles/system-metrics/tasks/main.yml @@ -0,0 +1,56 @@ +--- + +- block: + + - name: Create kube-burner configmap + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: system-metrics-collector-{{ trunc_uuid }} + namespace: "{{ operator_namespace }}" + data: + index.yml: "{{ lookup('template', 'index.yml')}}" + node-metrics.yml: "{{ lookup('template', 'node-metrics.yml')}}" + + - name: Launching kube-burner job to index system-metrics + k8s: + definition: "{{ lookup('template', 'kube-burner.yml.j2') | from_yaml }}" + + - name: Get job status + k8s_info: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + name: system-metrics-collector-{{ trunc_uuid }} + register: job_state + + - name: Set collecting state to system-metrics benchmark + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + system_metrics: Collecting + + when: benchmark_state.resources[0].status.system_metrics == "Not collected" + +- name: Get job status + k8s_info: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + name: system-metrics-collector-{{ trunc_uuid }} + register: job_state + +- name: Set collected state to system-metrics benchmark + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + system_metrics: Collected + when: job_state.resources[0].status.succeeded is defined and (job_state.resources[0].status.succeeded | int) > 0 diff --git a/roles/system-metrics/templates/index.yml b/roles/system-metrics/templates/index.yml new file mode 100644 index 000000000..8dd80c9d1 --- /dev/null +++ b/roles/system-metrics/templates/index.yml @@ -0,0 +1,9 @@ +--- +global: + writeToFile: false + indexerConfig: + enabled: true + esServers: ["{{ system_metrics.es_url }}"] + insecureSkipVerify: true + defaultIndex: "{{ system_metrics.index_name }}" + type: elastic diff --git a/roles/system-metrics/templates/kube-burner.yml.j2 b/roles/system-metrics/templates/kube-burner.yml.j2 new file mode 100644 index 000000000..10a7aa2ad --- /dev/null +++ b/roles/system-metrics/templates/kube-burner.yml.j2 @@ -0,0 +1,48 @@ +--- +kind: Job +apiVersion: batch/v1 +metadata: + name: system-metrics-collector-{{ trunc_uuid }} + namespace: {{ operator_namespace }} +spec: + backoffLimit: 0 + activeDeadlineSeconds: 3600 + template: + metadata: + labels: + app: system-metrics-collector-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} + restartPolicy: Never + containers: + - name: kube-burner + image: {{ system_metrics.image }} + imagePullPolicy: Always + workingDir: /tmp/kube-burner + command: ["/bin/sh", "-c"] + args: + - > + kube-burner index + --uuid={{ uuid }} + -c index.yml + -u {{ system_metrics.prom_url }} + -t {{ system_metrics.prom_token }} + --start={{ (benchmark_state.resources[0].metadata.creationTimestamp|to_datetime('%Y-%m-%dT%H:%M:%SZ')).strftime('%s') }} + -m {{ system_metrics.metrics_profile|default("node-metrics.yml") }} + --step={{ system_metrics.step }} + volumeMounts: + - name: system-metrics-collector + mountPath: /tmp/kube-burner + volumes: + - name: system-metrics-collector + configMap: + name: system-metrics-collector-{{ trunc_uuid }} diff --git a/roles/system-metrics/templates/node-metrics.yml b/roles/system-metrics/templates/node-metrics.yml new file mode 100644 index 000000000..eeb1128b9 --- /dev/null +++ b/roles/system-metrics/templates/node-metrics.yml @@ -0,0 +1,31 @@ +- query: sum(irate(node_cpu_seconds_total[2m])) by (mode,instance) > 0 + metricName: nodeCPU + +- query: avg(node_memory_MemAvailable_bytes) by (instance) + metricName: nodeMemoryAvailable + +- query: avg(node_memory_Active_bytes) by (instance) + metricName: nodeMemoryActive + +- query: avg(node_memory_Cached_bytes) by (instance) + avg(node_memory_Buffers_bytes) by (instance) + metricName: nodeMemoryCached+nodeMemoryBuffers + +- query: irate(node_network_receive_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) + metricName: rxNetworkBytes + +- query: irate(node_network_transmit_bytes_total{device=~"^(ens|eth|bond|team).*"}[2m]) + metricName: txNetworkBytes + +- query: rate(node_disk_written_bytes_total{device!~"^(dm|rb).*"}[2m]) + metricName: nodeDiskWrittenBytes + +- query: rate(node_disk_read_bytes_total{device!~"^(dm|rb).*"}[2m]) + metricName: nodeDiskReadBytes + +- query: kube_node_role + metricName: nodeRoles + instant: true + +- query: cluster_version{type="completed"} + metricName: clusterVersion + instant: true diff --git a/roles/testpmd/tasks/main.yml b/roles/testpmd/tasks/main.yml index b7b58642d..af96c95d4 100644 --- a/roles/testpmd/tasks/main.yml +++ b/roles/testpmd/tasks/main.yml @@ -1,30 +1,4 @@ --- -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Starting - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Update current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - - name: "Parse testpmd network" include_tasks: 10_network_parser.yml loop: "{{ workload_args.networks.testpmd }}" @@ -33,7 +7,7 @@ include_tasks: 20_mac_parser.yml loop: "{{ workload_args.networks.testpmd }}" loop_control: - loop_var: network + loop_var: network vars: net_type: testpmd @@ -41,7 +15,7 @@ include_tasks: 20_mac_parser.yml loop: "{{ workload_args.networks.trex }}" loop_control: - loop_var: network + loop_var: network vars: net_type: trex @@ -51,21 +25,18 @@ k8s: definition: "{{ lookup('template', 'testpmd.yml.j2') | from_yaml }}" - - name: Update state to Starting Server - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting TestPMD" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting TestPMD" - when: resource_state.resources[0].status.state == "Starting" + when: benchmark_state.resources[0].status.state == "Building" - block: - name: Wait for pods to be running.... - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -73,18 +44,15 @@ - app = testpmd-application-pod-{{ trunc_uuid }} register: application_pods - - name: Update state to Running - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting TRex" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting TRex" when: - "(application_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length) == 1" - when: resource_state.resources[0].status.state == "Starting TestPMD" + when: benchmark_state.resources[0].status.state == "Starting TestPMD" - name: Set trex network fact set_fact: @@ -97,7 +65,7 @@ - block: - name: Get TestPMD pod info - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -110,21 +78,18 @@ definition: "{{ lookup('template', 'trex.yml.j2') | from_yaml }}" with_items: "{{ application_pods.resources }}" - - name: Update state to Waiting for TRex - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Waiting for TRex" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Waiting for TRex" - when: resource_state.resources[0].status.state == "Starting TRex" + when: benchmark_state.resources[0].status.state == "Starting TRex" - block: - name: Wait for pods to be running.... - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -132,36 +97,16 @@ - app = trex-traffic-gen-pod-{{ trunc_uuid }} register: trex_pods - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running when: trex_pods.resources|length > 0 - when: resource_state.resources[0].status.state == "Waiting for TRex" - -- block: - - name: Waiting for Jobs to complete.... - k8s_facts: - kind: Job - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = trex-traffic-gen-pod-{{ trunc_uuid }} - register: trex_pods - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: "(trex_pods | json_query('resources[].status.succeeded')|length) == 1" - when: resource_state.resources[0].status.state == "Running" + when: benchmark_state.resources[0].status.state == "Waiting for TRex" +- include_role: + name: benchmark_state + tasks_from: completed.yml + when: benchmark_state.resources[0].status.state == "Running" diff --git a/roles/testpmd/templates/testpmd.yml.j2 b/roles/testpmd/templates/testpmd.yml.j2 index bfc9e38bd..e07b9edc3 100644 --- a/roles/testpmd/templates/testpmd.yml.j2 +++ b/roles/testpmd/templates/testpmd.yml.j2 @@ -1,85 +1,82 @@ # https://github.com/rh-nfv-int/testpmd-operator -kind: Job -apiVersion: batch/v1 +kind: Pod +apiVersion: v1 metadata: name: "testpmd-application-pod-{{ trunc_uuid }}" namespace: "{{ operator_namespace }}" -spec: - template: - metadata: - labels: - app: "testpmd-application-pod-{{ trunc_uuid }}" - annotations: - k8s.v1.cni.cncf.io/networks: '[ + labels: + app: "testpmd-application-pod-{{ trunc_uuid }}" + benchmark-uuid: {{ uuid }} + annotations: + maclist_trex: "{{ maclist_trex|list|join(',') }}" + maclist_testpmd: "{{ maclist_testpmd|list|join(',') }}" + k8s.v1.cni.cncf.io/networks: '[ {% set i = namespace(value=0) %} {% for network in workload_args.networks.testpmd %} {% set outer_loop = loop %} {% for count in range(network.count) %} - { - "name": "{{ network.name }}", - "mac": "{{ maclist_testpmd[i.value] }}", - "namespace": "{{ operator_namespace }}" - }{% if loop.index < network.count or outer_loop.index < network_name_list|length %},{% endif %} + { + "name": "{{ network.name }}", + "mac": "{{ maclist_testpmd[i.value] }}", + "namespace": "{{ operator_namespace }}" + }{% if loop.index < network.count or outer_loop.index < network_name_list|length %},{% endif %} {% set i.value = i.value + 1 %} {% endfor %} {% endfor %} - ]' - maclist_trex: "{{ maclist_trex|list|join(',') }}" - maclist_testpmd: "{{ maclist_testpmd|list|join(',') }}" - spec: + ]' +spec: {% if workload_args.pin is sameas true %} - nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_testpmd }}' + nodeSelector: + kubernetes.io/hostname: '{{ workload_args.pin_testpmd }}' {% endif %} - terminationGracePeriodSeconds: 10 - containers: - - name: testpmd - command: ["/bin/sh"] - args: - - "-c" - - echo "set promisc all off" >> /tmp/testpmd-cmdline.txt && testpmd -l $(cat /sys/fs/cgroup/cpuset/cpuset.cpus) $(PCI=""; IFS=',' read -r -a NETWORK_ARRAY <<< "$NETWORK_NAME_LIST"; for item in "${NETWORK_ARRAY[@]}"; do IFS='/' read -r -a RES_ARRAY <<< "$item"; NAME="PCIDEVICE_OPENSHIFT_IO_${RES_ARRAY[1]^^}";IFS=',' read -r -a PCI_ARRAY <<< "${!NAME}";for pci_item in "${PCI_ARRAY[@]}"; do PCI+=" -w ${pci_item} ";done;done;echo $PCI) --socket-mem {{ workload_args.socket_memory | default('1024,0') }} -n {{ workload_args.memory_channels | default(4) }} --proc-type auto --file-prefix pg -- {% if workload_args.disable_rss | default(true) %} --disable-rss {% endif %} --nb-cores={{ workload_args.forwarding_cores | default(4) }} --rxq={{ workload_args.rx_queues | default(1) }} --txq={{ workload_args.tx_queues | default(1) }} --rxd={{ workload_args.rx_descriptors | default(1024) }} --txd={{ workload_args.tx_descriptors | default(1024) }} --auto-start {% for mac in maclist_trex %} --eth-peer={{ loop.index -1 }},{{ mac }} {% endfor %} --forward-mode={{ workload_args.forward_mode | default('mac') }} --stats-period {{ workload_args.stats_period | default(1) }} --cmdline-file /tmp/testpmd-cmdline.txt - image: "{{ workload_args.image_testpmd | default('registry.redhat.io/openshift4/dpdk-base-rhel8:v4.6') }}" - imagePullPolicy: "{{ workload_args.image_pull_policy | default('Always') }}" - securityContext: + terminationGracePeriodSeconds: 10 + containers: + - name: testpmd + command: ["/bin/sh"] + args: + - "-c" + - echo "set promisc all off" >> /tmp/testpmd-cmdline.txt && testpmd -l $(cat /sys/fs/cgroup/cpuset/cpuset.cpus) $(PCI=""; IFS=',' read -r -a NETWORK_ARRAY <<< "$NETWORK_NAME_LIST"; for item in "${NETWORK_ARRAY[@]}"; do IFS='/' read -r -a RES_ARRAY <<< "$item"; NAME="PCIDEVICE_OPENSHIFT_IO_${RES_ARRAY[1]^^}";IFS=',' read -r -a PCI_ARRAY <<< "${!NAME}";for pci_item in "${PCI_ARRAY[@]}"; do PCI+=" -w ${pci_item} ";done;done;echo $PCI) --socket-mem {{ workload_args.socket_memory | default('1024,0') }} -n {{ workload_args.memory_channels | default(4) }} --proc-type auto --file-prefix pg -- {% if workload_args.disable_rss | default(true) %} --disable-rss {% endif %} --nb-cores={{ workload_args.forwarding_cores | default(4) }} --rxq={{ workload_args.rx_queues | default(1) }} --txq={{ workload_args.tx_queues | default(1) }} --rxd={{ workload_args.rx_descriptors | default(1024) }} --txd={{ workload_args.tx_descriptors | default(1024) }} --auto-start {% for mac in maclist_trex %} --eth-peer={{ loop.index -1 }},{{ mac }} {% endfor %} --forward-mode={{ workload_args.forward_mode | default('mac') }} --stats-period {{ workload_args.stats_period | default(1) }} --cmdline-file /tmp/testpmd-cmdline.txt + image: "{{ workload_args.image_testpmd | default('registry.redhat.io/openshift4/dpdk-base-rhel8:v4.6') }}" + imagePullPolicy: "{{ workload_args.image_pull_policy | default('Always') }}" + securityContext: {% if workload_args.privileged %} - privileged: true + privileged: true {% else %} - capabilities: - add: ["IPC_LOCK", "NET_ADMIN"] + capabilities: + add: ["IPC_LOCK", "NET_ADMIN"] {% endif %} - resources: - limits: - hugepages-1Gi: {{ workload_args.pod_hugepage_1gb_count | default('4Gi') }} - memory: {{ workload_args.pod_memory | default('1000Mi') }} - cpu: {{ workload_args.pod_cpu | default(6) }} + resources: + limits: + hugepages-1Gi: {{ workload_args.pod_hugepage_1gb_count | default('4Gi') }} + memory: {{ workload_args.pod_memory | default('1000Mi') }} + cpu: {{ workload_args.pod_cpu | default(6) }} {% for key, value in network_resources.items() %} - {{ key }}: {{ value }} + {{ key }}: {{ value }} {% endfor %} - requests: - hugepages-1Gi: {{ workload_args.pod_hugepage_1gb_count | default('4Gi') }} - memory: {{ workload_args.pod_memory | default('1000Mi') }} - cpu: {{ workload_args.pod_cpu | default(6) }} + requests: + hugepages-1Gi: {{ workload_args.pod_hugepage_1gb_count | default('4Gi') }} + memory: {{ workload_args.pod_memory | default('1000Mi') }} + cpu: {{ workload_args.pod_cpu | default(6) }} {% for key, value in network_resources.items() %} - {{ key }}: {{ value }} + {{ key }}: {{ value }} {% endfor %} - volumeMounts: - - name: hugepage - mountPath: /dev/hugepages - env: - - name: NETWORK_NAME_LIST - value: "{{ network_resources.keys()|list|join(',') }}" + volumeMounts: + - name: hugepage + mountPath: /dev/hugepages + env: + - name: NETWORK_NAME_LIST + value: "{{ network_resources.keys()|list|join(',') }}" {% if workload_args.environments is defined %} {% for key, value in workload_args.environments.items() %} - - name: {{ key }} - value: "{{ value }}" + - name: {{ key }} + value: "{{ value }}" {% endfor %} {% endif %} - volumes: - - name: hugepage - emptyDir: - medium: HugePages - restartPolicy: OnFailure - serviceAccountName: benchmark-operator -{% include "metadata.yml.j2" %} + volumes: + - name: hugepage + emptyDir: + medium: HugePages + restartPolicy: Never +{% include "metadata_pod.yml.j2" %} diff --git a/roles/testpmd/templates/trex.yml.j2 b/roles/testpmd/templates/trex.yml.j2 index a7c15f29f..54e0ec28b 100644 --- a/roles/testpmd/templates/trex.yml.j2 +++ b/roles/testpmd/templates/trex.yml.j2 @@ -10,6 +10,7 @@ spec: metadata: labels: app: "trex-traffic-gen-pod-{{ trunc_uuid }}" + benchmark-uuid: {{ uuid }} annotations: k8s.v1.cni.cncf.io/networks: '[ {% set i = namespace(value=0) %} @@ -47,10 +48,9 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: trex - command: ["/bin/sh"] + command: ["/bin/sh", "-c"] args: - - "-c" - - "run_snafu --tool trex --uuid {{ trunc_uuid }} --user {{ test_user | default('snafu') }}" + - run_snafu --tool trex --uuid {{ trunc_uuid }} --user {{ test_user | default('snafu') }} {% if workload_args.debug is defined and workload_args.debug %} -v {% endif %}; image: "{{ workload_args.image_trex | default('quay.io/cloud-bulldozer/trex:latest') }}" imagePullPolicy: "{{ workload_args.image_pull_policy | default('Always') }}" securityContext: @@ -78,6 +78,8 @@ spec: volumeMounts: - name: hugepage mountPath: /dev/hugepages + - name: modules + mountPath: /lib/modules env: - name: CLUSTERNAME value: "{{ clustername }}" @@ -111,7 +113,7 @@ spec: value: "{{ value }}" {% endfor %} {% endif %} -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index @@ -133,7 +135,9 @@ spec: - name: hugepage emptyDir: medium: HugePages - restartPolicy: OnFailure - serviceAccountName: benchmark-operator + - name: modules + hostPath: + path: /lib/modules + restartPolicy: Never {% include "metadata.yml.j2" %} diff --git a/roles/uperf-scale/README.md b/roles/uperf-scale/README.md new file mode 100644 index 000000000..225dd44b9 --- /dev/null +++ b/roles/uperf-scale/README.md @@ -0,0 +1,38 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/roles/uperf-scale/defaults/main.yml b/roles/uperf-scale/defaults/main.yml new file mode 100644 index 000000000..6561363be --- /dev/null +++ b/roles/uperf-scale/defaults/main.yml @@ -0,0 +1,9 @@ +--- +# defaults file for uperf-scale +resource_kind: "{{ workload.args.kind | default('pod') }}" +uperf-scale: + proto: tcp + test_type: stream + nthr: 1 + size: 1024 + runtime: 60 \ No newline at end of file diff --git a/roles/uperf-scale/handlers/main.yml b/roles/uperf-scale/handlers/main.yml new file mode 100644 index 000000000..e0cd86577 --- /dev/null +++ b/roles/uperf-scale/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for uperf-scale diff --git a/roles/uperf-scale/meta/main.yml b/roles/uperf-scale/meta/main.yml new file mode 100644 index 000000000..c572acc9f --- /dev/null +++ b/roles/uperf-scale/meta/main.yml @@ -0,0 +1,52 @@ +galaxy_info: + author: your name + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.1 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. diff --git a/roles/uperf-scale/tasks/cleanup.yml b/roles/uperf-scale/tasks/cleanup.yml new file mode 100644 index 000000000..4be469fc2 --- /dev/null +++ b/roles/uperf-scale/tasks/cleanup.yml @@ -0,0 +1,75 @@ +--- + +- block: + ### kind + # Cleanup servers, but leave clients around mostly for further examining of results. + - name: Get Server Jobs + k8s_info: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_jobs + + - name: Get Server Pods + k8s_info: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_pods + + - name: Server Job and Pod names - to clean + set_fact: + clean_jobs: | + [ + {% for item in server_jobs.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] + clean_pods: | + [ + {% for item in server_pods.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] + + - name: Cleanup server Job + k8s: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_jobs }}" + + - name: Cleanup server Pod + k8s: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_pods }}" + + when: resource_kind == "pod" and cleanup == True + +- block: + - name: Cleanup redis + command: "{{ item }}" + with_items: + - redis-cli del num_completion-{{trunc_uuid}} + - redis-cli del start-{{trunc_uuid}} + when: resource_kind == "pod" + +- operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Complete + complete: true + \ No newline at end of file diff --git a/roles/uperf-scale/tasks/init.yml b/roles/uperf-scale/tasks/init.yml new file mode 100644 index 000000000..253243328 --- /dev/null +++ b/roles/uperf-scale/tasks/init.yml @@ -0,0 +1,21 @@ +--- + +- name: Clear start flag + command: "redis-cli set start-{{trunc_uuid}} 0" + +- name: Clear num_completion + command: "redis-cli set num_completion-{{trunc_uuid}} 0" + +- name: Init node and pod indices in benchmark context + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + pod_hi_idx: "{{pod_hi_idx}}" + pod_low_idx: "{{pod_low_idx}}" + node_hi_idx: "{{node_hi_idx}}" + node_low_idx: "{{node_low_idx}}" + node_idx: "{{node_low_idx}}" + pod_idx: "{{pod_low_idx}}" diff --git a/roles/uperf-scale/tasks/main.yml b/roles/uperf-scale/tasks/main.yml new file mode 100644 index 000000000..6a0ac6389 --- /dev/null +++ b/roles/uperf-scale/tasks/main.yml @@ -0,0 +1,43 @@ +--- + +- include_tasks: setup.yml + +- include_tasks: start_server.yml + when: benchmark_state.resources[0].status.state == "Building" + +- block: + + - include_tasks: wait_server_ready.yml + when: benchmark_state.resources[0].status.state == "Starting Servers" + + - include_tasks: start_client.yml + when: benchmark_state.resources[0].status.state == "Starting Clients" + + - include_tasks: wait_client_ready.yml + when: benchmark_state.resources[0].status.state == "Waiting for Clients" + + # LOOP BEGIN + # This loop iterates density_range[] and node_range[] for "scale" mode + + - include_tasks: run_a_set.yml + when: benchmark_state.resources[0].status.state == "Clients Running" + + - include_tasks: wait_set_done.yml + when: benchmark_state.resources[0].status.state == "Set Running" + + + - include_tasks: next_set.yml + when: benchmark_state.resources[0].status.state == "Run Next Set" + # will loop back to "Client Running" state, or FALLTHRU to "Running" + # state below and finish + + # LOOP END + + - include_tasks: wait_client_done.yml + when: benchmark_state.resources[0].status.state == "Running" + + - include_tasks: cleanup.yml + when: benchmark_state.resources[0].status.state == "Cleanup" + + when: resource_kind == "pod" + \ No newline at end of file diff --git a/roles/uperf-scale/tasks/next_set.yml b/roles/uperf-scale/tasks/next_set.yml new file mode 100644 index 000000000..c8c74f07b --- /dev/null +++ b/roles/uperf-scale/tasks/next_set.yml @@ -0,0 +1,104 @@ +--- +# +# This module logically implements an RE-ENTRANT nested "for" loops; +# +# with_items: +# range (node_low_idx, node_hi_idx) +# range (pod_low_idx, pod_hi_idx) +# +# Each iteration executes one item, and each re-entrance +# continues where it left off. +# +- block: + - name: Read previous node_idx and pod_idx + set_fact: + all_run_done: False + inc: "{{workload_args.step_size|default('add1')}}" + amount: 0 + pod_idx: "{{benchmark_state.resources[0].status.pod_idx|int}}" + node_idx: "{{benchmark_state.resources[0].status.node_idx|int}}" + + - name: Extract add amount + set_fact: + amount: "{{ inc | regex_replace('[^0-9]', '') }}" + inc: add + when: "'add' in inc" + + - name: Increment pod_idx + set_fact: + pod_idx: "{%-if inc=='add' -%}{{pod_idx|int+amount|int}} + {%-elif inc=='log2' -%}{{(pod_idx|int*2)+1}} + {%-else -%}{{pod_idx|int+1}} + {% endif %}" + - block: + # + # This block starts a new node loop + # + - name: Increment node_idx + set_fact: + node_idx: "{%- if inc=='add' -%}{{node_idx|int+amount|int}} + {%- elif inc=='log2' -%}{{(node_idx|int *2)+1}} + {%- else -%}{{node_idx|int+1}} + {% endif %}" + + - name: Check node loop for ending condition + set_fact: + all_run_done: True + when: "node_idx|int > benchmark_state.resources[0].status.node_hi_idx|int" + + # + # Reset pod_idx AFTER node_idx tasks above, else cond change + # causes it to skip node_idx tasks + # + - name: Reset pod_idx to pod_low_idx + set_fact: + pod_idx: "{{benchmark_state.resources[0].status.pod_low_idx}}" + + when: "pod_idx|int > benchmark_state.resources[0].status.pod_hi_idx|int" + + - block: + # + # All done + # + - name: Unpause pods to complete + command: "redis-cli set start-{{trunc_uuid}} done" + + - name: Change state to proceed to exit + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Running + + when: all_run_done == True + + - block: + # + # More round(s) to run. + # + - name: Send redis restart signal + command: "redis-cli set start-{{trunc_uuid}} restart" + + - name: Reset redis num_completion + command: "redis-cli set num_completion-{{trunc_uuid}} 0" + + - name: Change state to run next round + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Clients Running + pod_idx: "{{pod_idx}}" + node_idx: "{{node_idx}}" + + when: all_run_done == False + + when: resource_kind == "pod" + +# +# No block - Scale mode support is N/A +# diff --git a/roles/uperf-scale/tasks/run_a_set.yml b/roles/uperf-scale/tasks/run_a_set.yml new file mode 100644 index 000000000..ca0e44f76 --- /dev/null +++ b/roles/uperf-scale/tasks/run_a_set.yml @@ -0,0 +1,26 @@ +--- + +- block: + # + # Entry Condition: + # 1. A previous task has set 'node_idx' and 'pod_idx' in benchmark ctx + # 2. All cliest are polling redis for 'start-node_idx-pod_idx' to start + # Output: Clients with node_idx <= redis node_idx && pod_idx <= redis pod_ix will run + # + + - name: Signal group to run + command: "redis-cli set start-{{trunc_uuid}} true-{{benchmark_state.resources[0].status.node_idx|int}}-{{benchmark_state.resources[0].status.pod_idx|int}}" + + - name: Update state to "Set Running" + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Set Running + when: resource_kind == "pod" + +# +# No kind - It has not been adapted to Scale mode +# diff --git a/roles/uperf-scale/tasks/setup.yml b/roles/uperf-scale/tasks/setup.yml new file mode 100644 index 000000000..6b8d29513 --- /dev/null +++ b/roles/uperf-scale/tasks/setup.yml @@ -0,0 +1,79 @@ +--- + # + # Scale mode logic where client and server pods are spreaded + # across all eligible nodes + # +- name: List Nodes Labeled as Workers + k8s_info: + api_version: v1 + kind: Node + label_selectors: + - "node-role.kubernetes.io/worker=" + register: node_list + no_log: True + +- name: Isolate Worker Role Hostnames + set_fact: + worker_node_list: "{{ node_list | json_query('resources[].metadata.labels.\"kubernetes.io/hostname\"') | list }}" + +- name: List Nodes Labeled with {{ workload_args.exclude_label }} + k8s_info: + api_version: v1 + kind: Node + label_selectors: + - '{{ item }}' + with_items: "{{ workload_args.exclude_labels }}" + register: exclude_node_list + when: workload_args.exclude_labels is defined and workload_args.exclude_labels | length > 0 + +- name: Isolate Worker Role Hostnames for label {{ workload_args.exclude_label }} + set_fact: + worker_node_exclude_list: "{{ exclude_node_list | json_query('results[].resources[].metadata.name') }}" + +- name: Exclude labeled nodes + set_fact: + worker_node_list: "{{ worker_node_list | difference(worker_node_exclude_list) }}" + when: workload_args.exclude_labels is defined and workload_args.exclude_labels | length > 0 + # + # Compute node and pod limits using CR params while taking into account + # of the actual number of nodes available in the system + # +- name: init pod and node low/hi idx + set_fact: + pod_low_idx: "{{ workload_args.density_range[0] | default('1')|int - 1 }}" + pod_hi_idx: "{{ workload_args.density_range[1] | default('1')|int - 1 }}" + node_low_idx: "{{ workload_args.node_range[0] | default('1')|int - 1 }}" + node_hi_idx: "{{ workload_args.node_range[1] | default('1')|int - 1 }}" + # + # Next sanity check and massage the indices if necessary. + # We shall complete gracefully and not iterate wildly. + # +- name: Adjust node_hi_idx if cluster has less nodes + set_fact: + node_hi_idx: "{{ worker_node_list|length| default('0')|int -1 }}" + when: "node_hi_idx|int >= worker_node_list|length| default('0')|int " + +- name: Adjust node_low_idx if necessary + set_fact: + node_low_idx: "{{node_hi_idx|int}}" + when: "node_low_idx|int > node_hi_idx|int" + +- name: Adjust pod_low_idx if necessary + set_fact: + pod_low_idx: "{{pod_hi_idx|int}}" + when: "pod_low_idx|int > pod_hi_idx|int" + +- name: Record num server pods using new worker_node_list + # in Scale mode, num server pods = num_node * number_pod + set_fact: + num_server_pods: "{{ (node_hi_idx|int+1) * (pod_hi_idx|int+1) }}" + +- name: Capture ServiceIP + k8s_info: + kind: Service + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: serviceip + when: workload_args.serviceip is defined and workload_args.serviceip diff --git a/roles/uperf-scale/tasks/start_client.yml b/roles/uperf-scale/tasks/start_client.yml new file mode 100644 index 000000000..2794df285 --- /dev/null +++ b/roles/uperf-scale/tasks/start_client.yml @@ -0,0 +1,63 @@ +--- + +- name: Get pod info + k8s_info: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_pods + +- name: set pod_sequence + set_fact: + pod_sequence: "{{ (num_server_pods|int - 1) if (workload_args.serviceip|default(False) == True and workload_args.servicetype | default('clusterip') == 'nodeport') else (pod_hi_idx|int) }}" + +- name: Generate uperf-scale xml files + k8s: + definition: "{{ lookup('template', 'configmap.yml.j2') | from_yaml }}" + with_sequence: start=0 end={{ pod_sequence|int }} + +- block: + ### kind + - name: Start Client(s) w/o serviceIP + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + vars: + resource_item: "{{ server_pods.resources }}" + when: + - ( workload_args.serviceip|default(False) == False and server_pods.resources|length > 0 ) + + + - name: Start Client(s) with nodeport serviceIP + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + vars: + resource_item: "{{ server_pods.resources }}" + when: + - ( workload_args.serviceip|default(False) == True and server_pods.resources|length > 0 and + workload_args.servicetype | default("clusterip") == "nodeport" ) + + # + # Each server annotates a "node_idx". Each peer client will + # derive its affinity according the 'colocate' variable. + # + + - name: Start Client(s) with serviceIP + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + vars: + resource_item: "{{ serviceip.resources }}" + when: + - workload_args.serviceip|default(False) == True and serviceip.resources|length > 0 + - workload_args.servicetype | default("clusterip") != "nodeport" + when: resource_kind == "pod" + + +- operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Waiting for Clients diff --git a/roles/uperf-scale/tasks/start_server.yml b/roles/uperf-scale/tasks/start_server.yml new file mode 100644 index 000000000..440e4989c --- /dev/null +++ b/roles/uperf-scale/tasks/start_server.yml @@ -0,0 +1,59 @@ +--- + +- block: + ### kind + - include_tasks: init.yml + + - name: Create service for server pods + k8s: + definition: "{{ lookup('template', 'service.yml.j2') | from_yaml }}" + vars: + pod_sequence: "{{ pod_hi_idx|int +1 }}" + node_sequence: "{{ node_hi_idx|int +1 }}" + + when: + - workload_args.serviceip is defined and workload_args.serviceip + - ( workload_args.servicetype | default("clusterip") == "clusterip" ) or + ( workload_args.servicetype | default("clusterip") == "nodeport" ) + + - name: Create metal LB service for server pods + k8s: + definition: "{{ lookup('template', 'service_metallb.yml.j2') | from_yaml }}" + vars: + pod_sequence: "{{ pod_hi_idx|int +1 }}" + node_sequence: "{{ node_hi_idx|int +1 }}" + + when: + - workload_args.serviceip is defined and workload_args.serviceip + - workload_args.servicetype | default("clusterip") == "metallb" + + - name: Start Server(s) - total = eligible nodes * density + k8s: + definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}" + vars: + pod_sequence: "{{ pod_hi_idx|int +1 }}" + node_sequence: "{{ node_hi_idx|int +1 }}" + + # + # Each server annotates a "node_idx" which will allow its peer client + # to derive its affinity according the 'colocate' variable + # + - name: Wait for pods to be running.... + k8s_info: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_pods + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Starting Servers" + + when: resource_kind == "pod" diff --git a/roles/uperf-scale/tasks/wait_client_done.yml b/roles/uperf-scale/tasks/wait_client_done.yml new file mode 100644 index 000000000..c3f8cec57 --- /dev/null +++ b/roles/uperf-scale/tasks/wait_client_done.yml @@ -0,0 +1,28 @@ +--- +- block: + ### kind + - name: Waiting for pods to complete.... + k8s_info: + kind: pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = uperf-scale-bench-client-{{ trunc_uuid }} + register: client_pods + + - name: Check for client pod failures + include_role: + name: benchmark_state + tasks_from: failure + when: "(client_pods|json_query('resources[].status[]')|selectattr('phase','match','Failed')|list|length) > 0" + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Cleanup + complete: false + when: "num_server_pods|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" + when: resource_kind == "pod" diff --git a/roles/uperf-scale/tasks/wait_client_ready.yml b/roles/uperf-scale/tasks/wait_client_ready.yml new file mode 100644 index 000000000..b9afe2099 --- /dev/null +++ b/roles/uperf-scale/tasks/wait_client_ready.yml @@ -0,0 +1,26 @@ +--- + +- block: + ### kind + + - name: Get client pod status + k8s_info: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = uperf-scale-bench-client-{{ trunc_uuid }} + register: client_pods + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Clients Running + when: "num_server_pods|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and num_server_pods|int == (client_pods | json_query('resources[].status.podIP')|length)" + + when: resource_kind == "pod" + \ No newline at end of file diff --git a/roles/uperf-scale/tasks/wait_server_ready.yml b/roles/uperf-scale/tasks/wait_server_ready.yml new file mode 100644 index 000000000..6006a9561 --- /dev/null +++ b/roles/uperf-scale/tasks/wait_server_ready.yml @@ -0,0 +1,24 @@ +--- +- block: + ### kind + + - name: Get server pods + k8s_info: + kind: Pod + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_pods + + - name: Update resource state + operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: "Starting Clients" + when: "num_server_pods|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" + + when: resource_kind == "pod" diff --git a/roles/uperf-scale/tasks/wait_set_done.yml b/roles/uperf-scale/tasks/wait_set_done.yml new file mode 100644 index 000000000..f82d0ccbf --- /dev/null +++ b/roles/uperf-scale/tasks/wait_set_done.yml @@ -0,0 +1,28 @@ +--- + +- block: + - block: + ### kind + - name: read pod completion count + command: "redis-cli get num_completion-{{trunc_uuid}}" + register: num_completion + + - operator_sdk.util.k8s_status: + api_version: ripsaw.cloudbulldozer.io/v1alpha1 + kind: Benchmark + name: "{{ ansible_operator_meta.name }}" + namespace: "{{ operator_namespace }}" + status: + state: Run Next Set + + when: "num_completion.stdout|int == ((benchmark_state.resources[0].status.node_idx|int +1) * (benchmark_state.resources[0].status.pod_idx|int +1))" + + when: resource_kind == "pod" + + ### no kind block - Run a "set" is not yet supported + + when: resource_kind == "pod" + +# +# No kind block - It has not been adapted to scale mode yet. +# diff --git a/roles/uperf-scale/templates/configmap.yml.j2 b/roles/uperf-scale/templates/configmap.yml.j2 new file mode 100644 index 000000000..2f8862913 --- /dev/null +++ b/roles/uperf-scale/templates/configmap.yml.j2 @@ -0,0 +1,66 @@ +--- +{% set control_port = 30000 %} +apiVersion: v1 +kind: ConfigMap +metadata: + name: uperf-scale-test-{{ item }}-{{ trunc_uuid }} + namespace: '{{ operator_namespace }}' +data: +{% for test in workload_args.test_types %} +{% for proto in workload_args.protos %} +{% for size in workload_args.sizes %} +{% if size is iterable %} +{% set wsize = size[0] %} +{% set rsize = size[1] %} +{% else %} +{% set wsize = size %} +{% set rsize = size %} +{% endif %} +{% for nthr in workload_args.nthrs %} + uperf-scale-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} : | + + + {% if ( 'rr' == test ) %} + + + + + + + + + + + + {% endif %} + {% if ( 'stream' == test or 'bidirec' == test ) %} + + + + + + + + + + + + {% endif %} + {% if ( 'maerts' == test or 'bidirec' == test ) %} + + + + + + + + + + + + {% endif %} + +{% endfor %} +{% endfor %} +{% endfor %} +{% endfor %} diff --git a/roles/uperf-scale/templates/configmap_script.yml.j2 b/roles/uperf-scale/templates/configmap_script.yml.j2 new file mode 100644 index 000000000..462087f8e --- /dev/null +++ b/roles/uperf-scale/templates/configmap_script.yml.j2 @@ -0,0 +1,60 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: uperf-scale-run-script-{{item.status.interfaces[0].ipAddress}}-{{ trunc_uuid }} + namespace: '{{ operator_namespace }}' +data: + run_script.sh : | + export h={{item.status.interfaces[0].ipAddress}} +{% if elasticsearch.url %} + export es={{ elasticsearch.url }} + export es_index={{ elasticsearch.index_name | default("ripsaw-uperf") }} + export es_verify_cert={{ elasticsearch.verify_cert | default(true) }} + export parallel={{ elasticsearch.parallel | default(false) }} + export uuid={{uuid}} +{% endif %} +{% if test_user is defined %} + export test_user={{test_user}} +{% endif %} + export clustername={{clustername}} + export hostnet={{workload_args.hostnetwork}} + export ips=$(hostname -I) + echo "Setting ready, waiting for signal to start..." + redis-cli -h {{bo.resources[0].status.podIP}} setnx clients-{{trunc_uuid}} 0 + redis-cli -h {{bo.resources[0].status.podIP}} incr clients-{{trunc_uuid}} + while true; do + BO_START=$(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{ trunc_uuid }}) + CLIENTS_READY=$(redis-cli -h {{bo.resources[0].status.podIP}} get clients-{{ trunc_uuid }}) + SERVERS_READY=$(redis-cli -h {{bo.resources[0].status.podIP}} get {{ trunc_uuid }}) + if [[ ("${BO_START}" =~ 'true') && ("${CLIENTS_READY}" == "${SERVERS_READY}") ]]; then +{% for test in workload_args.test_types %} +{% for proto in workload_args.protos %} +{% for size in workload_args.sizes %} +{% if size is iterable %} +{% set wsize = size[0] %} +{% set rsize = size[1] %} +{% else %} +{% set wsize = size %} +{% set rsize = size %} +{% endif %} +{% for nthr in workload_args.nthrs %} + cat /tmp/uperf-scale-test/uperf-scale-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}}; + run_snafu --tool uperf -w /tmp/uperf-scale-test/uperf-scale-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; +{% endfor %} +{% endfor %} +{% endfor %} +{% endfor %} + else + sleep 0.1; + continue; + fi; + break; + done; + redis-cli -h {{bo.resources[0].status.podIP}} set start-{{trunc_uuid}} false + redis-cli -h {{bo.resources[0].status.podIP}} del clients-{{trunc_uuid}} + redis-cli -h {{bo.resources[0].status.podIP}} set complete-{{trunc_uuid}} true diff --git a/roles/uperf-scale/templates/server.yml.j2 b/roles/uperf-scale/templates/server.yml.j2 new file mode 100644 index 000000000..e86111921 --- /dev/null +++ b/roles/uperf-scale/templates/server.yml.j2 @@ -0,0 +1,98 @@ +--- +{% set control_port = 30000 %} +{% set counter = { + 'counter': 0, + } +%} +{% macro increment(dct, key, inc=1)%} + {% if dct.update({key: dct[key] + inc}) %} {% endif %} +{% endmacro %} +apiVersion: v1 +kind: List +metadata: {} +items: +{% macro job_template(item, node_idx_item='') %} + - kind: Job + apiVersion: batch/v1 + metadata: +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + name: 'uperf-scale-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }}' +{% else %} + name: 'uperf-scale-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }}' +{% endif %} + namespace: "{{ operator_namespace }}" +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} + spec: + ttlSecondsAfterFinished: 600 + backoffLimit: 0 + template: + metadata: + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf-scale + benchmark-operator-role: server +{% if (workload_args.serviceip is defined and workload_args.serviceip and ( workload_args.servicetype | default("clusterip") == "nodeport" )) %} + index: "{{ counter["counter"] }}" +{% set port_index = counter["counter"] %} +{{ increment(counter, 'counter') }} +{% else %} +{% set port_index = item %} + index: "{{ item }}" +{% endif %} +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + annotations: +{% if workload_args.multus.enabled is sameas true %} + k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} +{% endif %} + node_idx: '{{ node_idx_item }}' + pod_idx: '{{ item }}' + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.hostnetwork is sameas true %} + hostNetwork: true +{% endif %} + containers: + - name: benchmark + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} +{% if workload_args.server_resources is defined %} + resources: {{ workload_args.server_resources | to_json }} +{% endif %} + imagePullPolicy: Always + command: ["/bin/sh","-c"] + args: ["uperf -s -v -P {{ control_port|int + (0 if (workload_args.serviceip is defined and workload_args.serviceip and ( workload_args.servicetype | default("clusterip") == "clusterip" )) else (port_index * 100)) }}"] + restartPolicy: Never + nodeSelector: + kubernetes.io/hostname: '{{ worker_node_list[node_idx_item] }}' + +{% if workload_args.serviceip is sameas true %} + securityContext: + sysctls: + - name: net.ipv4.ip_local_port_range + value: 30000 32011 +{% endif %} +{% filter indent(width=4, first=True) %} +{% include "metadata.yml.j2" %} +{% endfilter %} +{% endmacro %} +{% for node_idx_item in range(node_sequence|int) %} +{% for item in range(pod_sequence|int) %} +{{ job_template(item,node_idx_item) }} +{% endfor %} +{% endfor %} + diff --git a/roles/uperf-scale/templates/service.yml.j2 b/roles/uperf-scale/templates/service.yml.j2 new file mode 100644 index 000000000..6fe3de922 --- /dev/null +++ b/roles/uperf-scale/templates/service.yml.j2 @@ -0,0 +1,94 @@ +--- +{% set control_port = 30000 %} +{% set counter = { + 'counter': 0, + } +%} +{% macro increment(dct, key, inc=1)%} + {% if dct.update({key: dct[key] + inc}) %} {% endif %} +{% endmacro %} +apiVersion: v1 +kind: List +metadata: {} +items: +{% macro job_template(item, node_idx_item='') %} + - kind: Service + apiVersion: v1 + metadata: +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + name: uperf-scale-service-{{worker_node_list[node_idx_item] | truncate(26,true,'') | replace('.','-')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + name: uperf-scale-service-{{worker_node_list[node_idx_item] | truncate(27,true,'') | replace('.','-')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf-scale +{% if (workload_args.serviceip is defined and workload_args.serviceip and ( workload_args.servicetype | default("clusterip") == "nodeport" )) %} + index: "{{ counter["counter"] }}" +{% set port_index = counter["counter"] %} +{{ increment(counter, 'counter') }} +{% else %} +{% set port_index = item %} + index: "{{ item }}" +{% endif %} +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + annotations: + node_idx: '{{ node_idx_item }}' + pod_idx: '{{ item }}' + spec: + selector: +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-scale-bench-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-scale-bench-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} +{% if workload_args.servicetype | default("clusterip") == "nodeport" %} + type: NodePort + ports: + - name: uperf-scale + port: {{ control_port|int + port_index * 100 }} + targetPort: {{ control_port|int + port_index * 100 }} + nodePort: {{ control_port|int + port_index * 100 }} + protocol: TCP +{% for num in range(1,12,1) %} + - name: uperf-scale-control-tcp-{{ control_port|int + port_index * 100 + num }} + port: {{ control_port|int + port_index * 100 + num }} + targetPort: {{ control_port|int + port_index * 100 + num }} + nodePort: {{ control_port|int + port_index * 100 + num }} + protocol: TCP + - name: uperf-scale-control-udp-{{ control_port|int + port_index * 100 + num }} + port: {{ control_port|int + port_index * 100 + num }} + targetPort: {{ control_port|int + port_index * 100 + num }} + nodePort: {{ control_port|int + port_index * 100 + num }} + protocol: UDP +{% endfor %} +{% else %} + type: ClusterIP + ports: + - name: uperf-scale + port: 30000 + targetPort: 30000 + protocol: TCP +{% for num in range(30001,30012,1) %} + - name: uperf-scale-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: TCP + - name: uperf-scale-control-udp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: UDP +{% endfor %} +{% endif %} +{% endmacro %} +{% for node_idx_item in range(node_sequence|int) %} +{% for item in range(pod_sequence|int) %} +{{ job_template(item,node_idx_item) }} +{% endfor %} +{% endfor %} diff --git a/roles/uperf-scale/templates/service_metallb.yml.j2 b/roles/uperf-scale/templates/service_metallb.yml.j2 new file mode 100644 index 000000000..1ea0a1af4 --- /dev/null +++ b/roles/uperf-scale/templates/service_metallb.yml.j2 @@ -0,0 +1,96 @@ +--- +apiVersion: v1 +kind: List +metadata: {} +items: +{% macro job_template(item, node_idx_item='') %} + - kind: Service + apiVersion: v1 + metadata: +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + name: uperf-scale-service-{{worker_node_list[node_idx_item] | truncate(26,true,'') | replace('.','-')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + name: uperf-scale-service-{{worker_node_list[node_idx_item] | truncate(27,true,'') | replace('.','-')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf-scale + index: "{{ item }}" +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + annotations: + node_idx: '{{ node_idx_item }}' + pod_idx: '{{ item }}' + metallb.universe.tf/address-pool: '{{ workload_args.metallb.addresspool | default("addresspool-l2") }}' + metallb.universe.tf/allow-shared-ip: uperf-scale-service-{{ item }}-{{ trunc_uuid }} + spec: + selector: +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-scale-bench-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-scale-bench-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + externalTrafficPolicy: '{{ workload_args.metallb.service_etp | default("Cluster") }}' + type: LoadBalancer + ports: + - name: uperf-scale + port: 30000 + targetPort: 30000 + protocol: TCP +{% for num in range(30001,30012,1) %} + - name: uperf-scale-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: TCP +{% endfor %} + - kind: Service + apiVersion: v1 + metadata: + name: uperf-scale-service-{{ item }}-{{ trunc_uuid }}-udp + namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf-scale + index: "{{ item }}" +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-scale-bench-server-{{ worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }}-udp + annotations: + node_idx: '{{ node_idx_item }}' + pod_idx: '{{ item }}' + metallb.universe.tf/address-pool: '{{ workload_args.metallb.addresspool | default("addresspool-l2") }}' + metallb.universe.tf/allow-shared-ip: uperf-scale-service-{{ item }}-{{ trunc_uuid }} + spec: + selector: +{% if (worker_node_list[node_idx_item]|length>27) and (worker_node_list[node_idx_item][26] == '.') %} + app: uperf-scale-bench-server-{{worker_node_list[node_idx_item] | truncate(26,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% else %} + app: uperf-scale-bench-server-{{worker_node_list[node_idx_item] | truncate(27,true,'')}}-{{ item }}-{{ trunc_uuid }} +{% endif %} + externalTrafficPolicy: '{{ workload_args.metallb.service_etp | default("Cluster") }}' + type: LoadBalancer + ports: + - name: uperf-scale + port: 30000 + targetPort: 30000 + protocol: UDP +{% for num in range(30001,30012,1) %} + - name: uperf-scale-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: UDP +{% endfor %} +{% endmacro %} +{% for node_idx_item in range(node_sequence|int) %} +{% for item in range(pod_sequence|int) %} +{{ job_template(item,node_idx_item) }} +{% endfor %} +{% endfor %} diff --git a/roles/uperf-scale/templates/workload.yml.j2 b/roles/uperf-scale/templates/workload.yml.j2 new file mode 100644 index 000000000..54adf9117 --- /dev/null +++ b/roles/uperf-scale/templates/workload.yml.j2 @@ -0,0 +1,239 @@ +--- +{% set control_port = 30000 %} +apiVersion: v1 +kind: List +metadata: {} +items: +{% for item in resource_item %} + - kind: Job + apiVersion: batch/v1 + metadata: +{% if workload_args.serviceip is sameas true %} +{% if workload_args.servicetype | default("clusterip") == "nodeport" %} + name: 'uperf-scale-client-{{item.status.hostIP}}-{{ item.metadata.labels.index|int }}-{{ trunc_uuid }}' +{% elif workload_args.servicetype | default("clusterip") == "metallb" or workload_args.servicetype | default("clusterip") == "loadbalancer" %} + name: 'uperf-scale-client-{{item.status.loadBalancer.ingress[0].ip}}-{{ trunc_uuid }}' +{% else %} + name: 'uperf-scale-client-{{item.spec.clusterIP}}-{{ trunc_uuid }}' +{% endif %} +{% else %} + name: 'uperf-scale-client-{{item.status.podIP}}-{{ trunc_uuid }}' +{% endif %} + namespace: '{{ operator_namespace }}' +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} + spec: + template: + metadata: + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf-scale + benchmark-operator-role: client + app: uperf-scale-bench-client-{{ trunc_uuid }} + clientfor: {{ item.metadata.labels.app }} + type: {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} +{% if workload_args.multus.enabled is sameas true %} + annotations: + k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.client }} +{% endif %} + spec: +{% if workload_args.runtime_class is defined %} + runtimeClassName: "{{ workload_args.runtime_class }}" +{% endif %} +{% if workload_args.hostnetwork is sameas true %} + hostNetwork: true + serviceAccountName: benchmark-operator +{% endif %} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ item.metadata.labels.app }} + topologyKey: kubernetes.io/hostname + containers: + - name: benchmark + image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} + env: + - name: uuid + value: "{{ uuid }}" + - name: test_user + value: "{{ test_user | default("ripsaw") }}" + - name: clustername + value: "{{ clustername }}" +{% if elasticsearch.url %} + - name: es + value: "{{ elasticsearch.url }}" + - name: es_index + value: "{{ elasticsearch.index_name | default("ripsaw-uperf") }}" + - name: parallel + value: "{{ elasticsearch.parallel | default(false) }}" + - name: es_verify_cert + value: "{{ elasticsearch.verify_cert | default(true) }}" +{% endif %} +{% if prometheus is defined %} + - name: prom_es + value: "{{ prometheus.es_url }}" + - name: prom_parallel + value: "{{ prometheus.es_parallel | default(false) }}" + - name: prom_token + value: "{{ prometheus.prom_token | default() }}" + - name: prom_url + value: "{{ prometheus.prom_url | default() }}" +{% endif %} + - name: client_node + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: server_node + value: "{{ item.spec.nodeName|default("unknown") }}" +{% if workload_args.client_resources is defined %} + resources: {{ workload_args.client_resources | to_json }} +{% endif %} + imagePullPolicy: Always + command: ["/bin/sh", "-c"] + args: +{% if workload_args.serviceip is sameas true %} + - "export serviceip=true; +{% if workload_args.servicetype | default("clusterip") == "nodeport" %} + export h={{item.status.hostIP}}; + export servicetype={{workload_args.servicetype}}; +{% elif workload_args.servicetype | default("clusterip") == "metallb" or workload_args.servicetype | default("clusterip") == "loadbalancer" %} + export h={{item.status.loadBalancer.ingress[0].ip}}; + export servicetype={{workload_args.servicetype}}; +{% else %} + export h={{item.spec.clusterIP}}; + export servicetype={{workload_args.servicetype | default("clusterip")}}; +{% endif %} +{% else %} +{% if workload_args.multus.client is defined %} + - "export multus_client={{workload_args.multus.client}}; + export h={{ (item['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; +{% else %} + - "export h={{item.status.podIP}}; +{% endif %} +{% endif %} + export port=30000 +{% if (workload_args.colocate is defined) %} + export colocate={{ workload_args.colocate}}; +{% endif %} +{% if workload_args.step_size is defined %} + export stepsize={{ workload_args.step_size }}; +{% endif %} +{% if workload_args.node_range is defined %} + export node_range='{{ workload_args.node_range[0] }}_{{ workload_args.node_range[1] }}'; +{% endif %} +{% if workload_args.density_range is defined %} + export density_range='{{ workload_args.density_range[0] }}_{{ workload_args.density_range[1] }}'; +{% endif %} +{% if workload_args.networkpolicy is defined %} + export networkpolicy={{workload_args.networkpolicy}}; +{% endif %} + export hostnet={{workload_args.hostnetwork}}; + export my_node_idx={{ (item['metadata']['annotations']['node_idx'] | from_json) }}; + export my_pod_idx={{ (item['metadata']['annotations']['pod_idx'] | from_json) }}; + export ips=$(hostname -I); + export num_pairs=1 + export node_count=0; + export pod_count=0; + node_limit=0; + pod_limit=0; + exit_status=0 + STR=''; + while true; do + STR=$(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{trunc_uuid}}); + state=$(echo $STR | cut -f1 -d-); + if [[ $state =~ 'true' ]]; then + node_limit=$(echo $STR | cut -f2 -d-); + pod_limit=$(echo $STR | cut -f3 -d-); + if [[ $my_node_idx -gt $node_limit || $my_pod_idx -gt $pod_limit ]]; then + sleep 0.5; continue; + fi; + + echo 'uperf-scale-run-context num_node=' $((node_limit+1)) 'density=' $((pod_limit+1)) 'my_node_idx=' $my_node_idx 'my_pod_idx=' $my_pod_idx; + node_count=$((node_limit+1)); + pod_count=$((pod_limit+1)); + num_pairs=$((pod_limit+1)); + +{% for test in workload_args.test_types %} +{% for proto in workload_args.protos %} +{% for size in workload_args.sizes %} +{% if size is iterable %} +{% set wsize = size[0] %} +{% set rsize = size[1] %} +{% else %} +{% set wsize = size %} +{% set rsize = size %} +{% endif %} +{% for nthr in workload_args.nthrs %} + cat /tmp/uperf-scale-test/uperf-scale-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}}; + run_snafu --tool uperf -w /tmp/uperf-scale-test/uperf-scale-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ +{% if workload_args.run_id is defined %} + --run-id {{workload_args.run_id}} \ +{% endif %} +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; + if [[ $? -ne 0 ]]; then + exit_status=1; + fi; +{% endfor %} +{% endfor %} +{% endfor %} +{% endfor %} + redis-cli -h {{bo.resources[0].status.podIP}} incr num_completion-{{trunc_uuid}}; + while true; do + state=$(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{trunc_uuid}}); + if [[ $state =~ 'restart' ]]; then + break; + elif [[ $state =~ 'done' ]]; then + break; + else + sleep 0.5; continue; + fi; + done; + if [[ $state =~ 'restart' ]]; then + sleep 0.5; continue; + fi; + + elif [[ $state =~ 'done' ]]; then + break; + else + sleep 0.5; continue; + fi; + break; + done; + exit $exit_status; + " + volumeMounts: + - name: config-volume + mountPath: "/tmp/uperf-scale-test" + volumes: + - name: config-volume + configMap: + name: uperf-scale-test-{{ item.metadata.labels.index|int }}-{{ trunc_uuid }} + restartPolicy: Never +{% if workload_args.colocate is sameas true %} + nodeSelector: + # client node same as server node + kubernetes.io/hostname: "{{ worker_node_list[item['metadata']['annotations']['node_idx'] | from_json] }}" +{% else %} + nodeSelector: + # skew client node one position to the right in the worker_node_list + kubernetes.io/hostname: "{{ worker_node_list[ (1+(item['metadata']['annotations']['node_idx'] | from_json)) % (worker_node_list|length)] }}" +{% endif %} + +{% endfor %} diff --git a/roles/uperf-scale/vars/main.yml b/roles/uperf-scale/vars/main.yml new file mode 100644 index 000000000..ddec85769 --- /dev/null +++ b/roles/uperf-scale/vars/main.yml @@ -0,0 +1,11 @@ +--- +# vars file for uperf-scale +cleanup: false +worker_node_list: [] +pod_low_idx: "0" +pod_hi_idx: "0" +node_low_idx: "0" +node_hi_idx: "0" +node_idx: "0" +pod_idx: "0" +all_run_done: false \ No newline at end of file diff --git a/roles/uperf/README.md b/roles/uperf/README.md index f8dad4e9c..71fc153b7 100644 --- a/roles/uperf/README.md +++ b/roles/uperf/README.md @@ -1,5 +1,4 @@ uperf-benchmark ========= -Network dataplane workload - +Network dataplane workload \ No newline at end of file diff --git a/roles/uperf/defaults/main.yml b/roles/uperf/defaults/main.yml index 9f9e3e09c..7e3bad17e 100644 --- a/roles/uperf/defaults/main.yml +++ b/roles/uperf/defaults/main.yml @@ -1,8 +1,9 @@ --- +# defaults file for uperf-scale resource_kind: "{{ workload.args.kind | default('pod') }}" uperf: proto: tcp test_type: stream nthr: 1 size: 1024 - runtime: 60 + runtime: 60 \ No newline at end of file diff --git a/roles/uperf/handlers/main.yml b/roles/uperf/handlers/main.yml index de0f0626f..572a9a4de 100644 --- a/roles/uperf/handlers/main.yml +++ b/roles/uperf/handlers/main.yml @@ -1,2 +1,2 @@ --- -# handlers file for bench +# handlers file for uperf \ No newline at end of file diff --git a/roles/uperf/meta/main.yml b/roles/uperf/meta/main.yml index 5d50bf41b..227ad9c34 100644 --- a/roles/uperf/meta/main.yml +++ b/roles/uperf/meta/main.yml @@ -1,34 +1,26 @@ galaxy_info: author: your name - description: your description + description: your role description company: your company (optional) # If the issue tracker for your role is not on github, uncomment the # next line and provide a value # issue_tracker_url: http://example.com/issue/tracker - # Some suggested licenses: - # - BSD (default) + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) # - MIT - # - GPLv2 - # - GPLv3 - # - Apache - # - CC-BY - license: license (GPLv2, CC-BY, etc) + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) - min_ansible_version: 2.4 + min_ansible_version: 2.9 # If this a Container Enabled role, provide the minimum Ansible Container version. # min_ansible_container_version: - # Optionally specify the branch Galaxy will use when accessing the GitHub - # repo for this role. During role install, if no tags are available, - # Galaxy will use this branch. During import Galaxy will access files on - # this branch. If Travis integration is configured, only notifications for this - # branch will be accepted. Otherwise, in all cases, the repo's default branch - # (usually master) will be used. - #github_branch: - # # Provide a list of supported platforms, and for each platform a list of versions. # If you don't wish to enumerate all versions for a particular platform, use 'all'. @@ -57,4 +49,5 @@ galaxy_info: dependencies: [] # List your role dependencies here, one per line. Be sure to remove the '[]' above, - # if you add dependencies to this list. \ No newline at end of file + # if you add dependencies to this list. + \ No newline at end of file diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml index 5e5fafefb..0c530c6e6 100644 --- a/roles/uperf/tasks/main.yml +++ b/roles/uperf/tasks/main.yml @@ -1,47 +1,12 @@ --- -- name: Get current state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get current state - If it has changed - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- name: Capture operator information - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - name = benchmark-operator - register: bo - - name: Capture ServiceIP - k8s_facts: + k8s_info: kind: Service api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: serviceip when: workload_args.serviceip is defined and workload_args.serviceip @@ -51,7 +16,18 @@ k8s: definition: "{{ lookup('template', 'service.yml.j2') | from_yaml }}" with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} - when: workload_args.serviceip is defined and workload_args.serviceip + when: + - workload_args.serviceip is defined and workload_args.serviceip + - ( workload_args.servicetype | default("clusterip") == "clusterip" ) or + ( workload_args.servicetype | default("clusterip") == "nodeport" ) + + - name: Create metal LB service for server pods + k8s: + definition: "{{ lookup('template', 'service_metallb.yml.j2') | from_yaml }}" + with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} + when: + - workload_args.serviceip is defined and workload_args.serviceip + - workload_args.servicetype | default("clusterip") == "metallb" - name: Start Server(s) k8s: @@ -59,25 +35,13 @@ register: servers with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} - - name: Wait for pods to be running.... - k8s_facts: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} - register: server_pods + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting Servers - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Servers" - - when: resource_state.resources[0].status.state == "Building" and resource_kind == "pod" + when: benchmark_state.resources[0].status.state == "Building" and resource_kind == "pod" - block: @@ -87,119 +51,117 @@ register: servers with_sequence: start=0 count={{ workload_args.pair | default('1')|int }} - - name: Wait for vms to be running.... - k8s_facts: - kind: VirtualMachineInstance - api_version: kubevirt.io/v1alpha3 - namespace: '{{ operator_namespace }}' - label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} - register: server_vms - - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Servers" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Starting Servers - when: resource_state.resources[0].status.state == "Building" and resource_kind == "vm" + when: benchmark_state.resources[0].status.state == "Building" and resource_kind == "vm" - block: - name: Get server pods - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_pods - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Clients" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting Clients" when: "workload_args.pair|default('1')|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length" - when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod" + when: benchmark_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod" - block: - name: Wait for vms to be running.... - k8s_facts: + k8s_info: kind: VirtualMachineInstance api_version: kubevirt.io/v1alpha3 namespace: '{{ operator_namespace }}' label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_vms - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Clients" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting Clients" when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - name: blocking client from running uperf - command: "redis-cli set start false" + command: "redis-cli set start-{{ trunc_uuid }} false" with_items: "{{ server_vms.resources }}" when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" - when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm" + when: benchmark_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm" and workload_args.pair|default('1')|int|int == 1 - block: - name: Get pod info - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_pods + + - name: set pod_sequence + set_fact: + pod_sequence: "{{ workload_args.pair|int - 1}}" - name: Generate uperf xml files k8s: definition: "{{ lookup('template', 'configmap.yml.j2') | from_yaml }}" + with_sequence: start=0 end={{ pod_sequence|int }} - block: - name: Start Client(s) k8s: definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" with_items: "{{ server_pods.resources }}" - when: workload_args.serviceip is defined and not workload_args.serviceip|default('false') and server_pods.resources|length > 0 + when: ( workload_args.serviceip|default(False) == False and server_pods.resources|length > 0 ) + + - name: Start Client(s) with nodeport serviceIP + k8s: + definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" + with_items: "{{ server_pods.resources }}" + when: + - ( workload_args.serviceip|default(False) == True and server_pods.resources|length > 0 and + workload_args.servicetype | default("clusterip") == "nodeport" ) - name: Start Client(s) - ServiceIP k8s: definition: "{{ lookup('template', 'workload.yml.j2') | from_yaml }}" with_items: "{{ serviceip.resources }}" - when: workload_args.serviceip is defined and workload_args.serviceip and serviceip.resources|length > 0 + when: + - workload_args.serviceip|default(False) == True and serviceip.resources|length > 0 + - workload_args.servicetype | default("clusterip") != "nodeport" when: resource_kind == "pod" - block: - name: Wait for vms to be running.... - k8s_facts: + k8s_info: kind: VirtualMachineInstance api_version: kubevirt.io/v1alpha3 namespace: '{{ operator_namespace }}' label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_vms + - name: Generate uperf test files k8s: definition: "{{ lookup('template', 'configmap_script.yml.j2') | from_yaml }}" @@ -208,26 +170,24 @@ - name: Start Client(s) k8s: definition: "{{ lookup('template', 'workload_vm.yml.j2') | from_yaml }}" - with_items: "{{ server_vms.resources }}" + with_indexed_items: "{{ server_vms.resources }}" when: server_vms.resources|length > 0 when: resource_kind == "vm" - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Waiting for Clients + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Waiting for Clients" - when: resource_state.resources[0].status.state == "Starting Clients" + when: benchmark_state.resources[0].status.state == "Starting Clients" - block: - block: - name: Get client pod status - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -239,10 +199,15 @@ operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Clients Running + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Clients Running when: "workload_args.pair|default('1')|int == client_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_pods | json_query('resources[].status.podIP')|length)" when: resource_kind == "pod" @@ -250,46 +215,45 @@ - block: - name: set complete to false - command: "redis-cli set complete false" + command: "redis-cli set complete-{{ trunc_uuid }} false" - - name: Get count of clients ready - command: "redis-cli get clients-{{ trunc_uuid }}" - register: clients_ready_count + - name: Get client vm status + k8s_info: + kind: VirtualMachineInstance + api_version: kubevirt.io/v1alpha3 + namespace: '{{ operator_namespace }}' + label_selectors: + - app = uperf-bench-client-{{ trunc_uuid }} + register: client_vms - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Clients Running - when: "workload_args.pair|default('1')|int == clients_ready_count.stdout|int" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Clients Running + when: "workload_args.pair|default('1')|int == client_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int == (client_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)" when: resource_kind == "vm" - when: resource_state.resources[0].status.state == "Waiting for Clients" + when: benchmark_state.resources[0].status.state == "Waiting for Clients" - block: - name: Signal workload - command: "redis-cli set start true" + command: "redis-cli set start-{{ trunc_uuid }} true" - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Running" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Running - when: resource_state.resources[0].status.state == "Clients Running" + when: benchmark_state.resources[0].status.state == "Clients Running" - block: - block: - name: Waiting for pods to complete.... - k8s_facts: + k8s_info: kind: pod api_version: v1 namespace: '{{ operator_namespace }}' @@ -297,50 +261,65 @@ - app = uperf-bench-client-{{ trunc_uuid }} register: client_pods - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Cleanup - complete: false + - name: Check for client pod failures + include_role: + name: benchmark_state + tasks_from: failure + when: "(client_pods|json_query('resources[].status[]')|selectattr('phase','match','Failed')|list|length) > 0" + + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Cleanup when: "workload_args.pair|default('1')|int == (client_pods|json_query('resources[].status[]')|selectattr('phase','match','Succeeded')|list|length)" when: resource_kind == "pod" - block: - name: get complete - command: "redis-cli get complete" + command: "redis-cli get complete-{{ trunc_uuid }}" register: complete_status - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Cleanup - complete: false + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: Cleanup when: complete_status.stdout == "true" when: resource_kind == "vm" - when: resource_state.resources[0].status.state == "Running" + when: benchmark_state.resources[0].status.state == "Running" - block: - block: + - name: Get Server Jobs + k8s_info: + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + label_selectors: + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + register: server_jobs + - name: Get Server Pods - k8s_facts: + k8s_info: kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' label_selectors: - - type = uperf-bench-server-{{ trunc_uuid }} + - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} register: server_pods - - name: Pod names - to clean + - name: Server Job and Pod names - to clean set_fact: + clean_jobs: | + [ + {% for item in server_jobs.resources %} + "{{ item['metadata']['name'] }}", + {% endfor %} + ] clean_pods: | [ {% for item in server_pods.resources %} @@ -348,30 +327,41 @@ {% endfor %} ] - - name: Cleanup run + - name: Cleanup server Job k8s: - kind: pod + kind: Job + api_version: v1 + namespace: '{{ operator_namespace }}' + state: absent + name: "{{ item }}" + with_items: "{{ clean_jobs }}" + + - name: Cleanup server Pod + k8s: + kind: Pod api_version: v1 namespace: '{{ operator_namespace }}' state: absent name: "{{ item }}" with_items: "{{ clean_pods }}" - when: cleanup - when: resource_kind == "pod" - - name: delete redis keys - command: "redis-cli del {{ item }}" - loop: - - "{{ trunc_uuid }}" - - "clients-{{ trunc_uuid }}" + when: resource_kind == "pod" and cleanup == True + + - block: + - name: Cleanup redis + command: "{{ item }}" + with_items: + - redis-cli del complete + - redis-cli del start-{{ trunc_uuid }} + when: resource_kind == "pod" - operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: state: Complete complete: true - when: resource_state.resources[0].status.state == "Cleanup" + when: benchmark_state.resources[0].status.state == "Cleanup" diff --git a/roles/uperf/templates/configmap.yml.j2 b/roles/uperf/templates/configmap.yml.j2 index f892a65e6..1ace4c5ef 100644 --- a/roles/uperf/templates/configmap.yml.j2 +++ b/roles/uperf/templates/configmap.yml.j2 @@ -1,8 +1,9 @@ --- +{% set control_port = 30000 %} apiVersion: v1 kind: ConfigMap metadata: - name: uperf-test-{{ trunc_uuid }} + name: uperf-test-{{ item }}-{{ trunc_uuid }} namespace: '{{ operator_namespace }}' data: {% for test in workload_args.test_types %} @@ -22,7 +23,7 @@ data: {% if ( 'rr' == test ) %} - + @@ -35,7 +36,7 @@ data: {% if ( 'stream' == test or 'bidirec' == test ) %} - + @@ -48,7 +49,7 @@ data: {% if ( 'maerts' == test or 'bidirec' == test ) %} - + @@ -62,4 +63,4 @@ data: {% endfor %} {% endfor %} {% endfor %} -{% endfor %} +{% endfor %} \ No newline at end of file diff --git a/roles/uperf/templates/configmap_script.yml.j2 b/roles/uperf/templates/configmap_script.yml.j2 index 278f51da4..2977a4f55 100644 --- a/roles/uperf/templates/configmap_script.yml.j2 +++ b/roles/uperf/templates/configmap_script.yml.j2 @@ -7,15 +7,13 @@ metadata: data: run_script.sh : | export h={{item.status.interfaces[0].ipAddress}} -{% if elasticsearch is defined %} -{% if elasticsearch.url is defined %} +{% if elasticsearch.url %} export es={{ elasticsearch.url }} export es_index={{ elasticsearch.index_name | default("ripsaw-uperf") }} export es_verify_cert={{ elasticsearch.verify_cert | default(true) }} export parallel={{ elasticsearch.parallel | default(false) }} export uuid={{uuid}} {% endif %} -{% endif %} {% if test_user is defined %} export test_user={{test_user}} {% endif %} @@ -26,7 +24,7 @@ data: redis-cli -h {{bo.resources[0].status.podIP}} setnx clients-{{trunc_uuid}} 0 redis-cli -h {{bo.resources[0].status.podIP}} incr clients-{{trunc_uuid}} while true; do - BO_START=$(redis-cli -h {{bo.resources[0].status.podIP}} get start) + BO_START=$(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{ trunc_uuid }}) CLIENTS_READY=$(redis-cli -h {{bo.resources[0].status.podIP}} get clients-{{ trunc_uuid }}) SERVERS_READY=$(redis-cli -h {{bo.resources[0].status.podIP}} get {{ trunc_uuid }}) if [[ ("${BO_START}" =~ 'true') && ("${CLIENTS_READY}" == "${SERVERS_READY}") ]]; then @@ -42,7 +40,11 @@ data: {% endif %} {% for nthr in workload_args.nthrs %} cat /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}}; - run_snafu --tool uperf -w /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}}; + run_snafu --tool uperf -w /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; {% endfor %} {% endfor %} {% endfor %} @@ -53,6 +55,6 @@ data: fi; break; done; - redis-cli -h {{bo.resources[0].status.podIP}} set start false + redis-cli -h {{bo.resources[0].status.podIP}} set start-{{trunc_uuid}} false redis-cli -h {{bo.resources[0].status.podIP}} del clients-{{trunc_uuid}} - redis-cli -h {{bo.resources[0].status.podIP}} set complete true + redis-cli -h {{bo.resources[0].status.podIP}} set complete-{{trunc_uuid}} true \ No newline at end of file diff --git a/roles/uperf/templates/server.yml.j2 b/roles/uperf/templates/server.yml.j2 index d46a66799..2002509ef 100644 --- a/roles/uperf/templates/server.yml.j2 +++ b/roles/uperf/templates/server.yml.j2 @@ -1,17 +1,31 @@ --- +{% set control_port = 30000 %} kind: Job apiVersion: batch/v1 metadata: name: 'uperf-server-{{ item }}-{{ trunc_uuid }}' namespace: "{{ operator_namespace }}" +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: ttlSecondsAfterFinished: 600 backoffLimit: 0 template: metadata: labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf + benchmark-operator-role: server app: uperf-bench-server-{{item}}-{{ trunc_uuid }} - type: uperf-bench-server-{{ trunc_uuid }} + index: "{{item}}" + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} {% if workload_args.multus.enabled is sameas true %} annotations: k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.server}} @@ -22,8 +36,17 @@ spec: {% endif %} {% if workload_args.hostnetwork is sameas true %} hostNetwork: true - serviceAccountName: benchmark-operator {% endif %} + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: type + operator: In + values: + - {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + topologyKey: kubernetes.io/hostname containers: - name: benchmark image: {{ workload_args.image | default('quay.io/cloud-bulldozer/uperf:latest') }} @@ -32,8 +55,8 @@ spec: {% endif %} imagePullPolicy: Always command: ["/bin/sh","-c"] - args: ["uperf -s -v -P 20000"] - restartPolicy: OnFailure + args: ["uperf -s -v -P {{ control_port|int + (0 if (workload_args.serviceip is defined and workload_args.serviceip and ( workload_args.servicetype | default("clusterip") == "clusterip" )) else (item|int * 100)) }}"] + restartPolicy: Never {% if workload_args.pin is sameas true %} nodeSelector: kubernetes.io/hostname: '{{ workload_args.pin_server }}' @@ -42,6 +65,6 @@ spec: securityContext: sysctls: - name: net.ipv4.ip_local_port_range - value: 20000 20011 + value: 30000 32011 {% endif %} {% include "metadata.yml.j2" %} diff --git a/roles/uperf/templates/server_vm.yml.j2 b/roles/uperf/templates/server_vm.yml.j2 index 28c7d21f1..3a7bfb583 100644 --- a/roles/uperf/templates/server_vm.yml.j2 +++ b/roles/uperf/templates/server_vm.yml.j2 @@ -5,8 +5,20 @@ metadata: name: 'uperf-server-{{ item }}-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf + benchmark-operator-role: server app : uperf-bench-server-{{ item }}-{{ trunc_uuid }} - type : uperf-bench-server-{{ trunc_uuid }} + type : {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: domain: cpu: @@ -61,6 +73,6 @@ spec: - dnf install -y uperf redis git - redis-cli -h {{ bo.resources[0].status.podIP }} setnx {{ trunc_uuid }} 0 - redis-cli -h {{ bo.resources[0].status.podIP }} incr {{ trunc_uuid }} - - uperf -s -v -P 20000 + - uperf -s -v -P 30000 name: cloudinitdisk -status: {} +status: {} \ No newline at end of file diff --git a/roles/uperf/templates/service.yml.j2 b/roles/uperf/templates/service.yml.j2 index 89177dad0..b255a936f 100644 --- a/roles/uperf/templates/service.yml.j2 +++ b/roles/uperf/templates/service.yml.j2 @@ -1,21 +1,54 @@ --- +{% set control_port = 30000 %} kind: Service apiVersion: v1 metadata: name: uperf-service-{{ item }}-{{ trunc_uuid }} namespace: '{{ operator_namespace }}' labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} - type: uperf-bench-server-{{ trunc_uuid }} + index: "{{item}}" + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} spec: selector: app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} +{% if workload_args.servicetype | default("clusterip") == "nodeport" %} + type: NodePort ports: - name: uperf - port: 20000 - targetPort: 20000 + port: {{ control_port|int + item|int * 100 }} + targetPort: {{ control_port|int + item|int * 100 }} + nodePort: {{ control_port|int + item|int * 100 }} protocol: TCP -{% for num in range(20001,20012,1) %} +{% for num in range(1,12,1) %} + - name: uperf-control-tcp-{{ control_port|int + item|int * 100 + num }} + port: {{ control_port|int + item|int * 100 + num }} + targetPort: {{ control_port|int + item|int * 100 + num }} + nodePort: {{ control_port|int + item|int * 100 + num }} + protocol: TCP + - name: uperf-control-udp-{{ control_port|int + item|int * 100 + num }} + port: {{ control_port|int + item|int * 100 + num }} + targetPort: {{ control_port|int + item|int * 100 + num }} + nodePort: {{ control_port|int + item|int * 100 + num }} + protocol: UDP +{% if "sctp" in workload_args.protos %} + - name: uperf-control-sctp-{{ control_port|int + item|int * 100 + num }} + port: {{ control_port|int + item|int * 100 + num }} + targetPort: {{ control_port|int + item|int * 100 + num }} + nodePort: {{ control_port|int + item|int * 100 + num }} + protocol: SCTP +{% endif %} +{% endfor %} +{% else %} + type: ClusterIP + ports: + - name: uperf + port: 30000 + targetPort: 30000 + protocol: TCP +{% for num in range(30001,30012,1) %} - name: uperf-control-tcp-{{num}} port: {{num}} targetPort: {{num}} @@ -24,4 +57,11 @@ spec: port: {{num}} targetPort: {{num}} protocol: UDP +{% if "sctp" in workload_args.protos %} + - name: uperf-control-sctp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: SCTP +{% endif %} {% endfor %} +{% endif %} \ No newline at end of file diff --git a/roles/uperf/templates/service_metallb.yml.j2 b/roles/uperf/templates/service_metallb.yml.j2 new file mode 100644 index 000000000..6ff7d8410 --- /dev/null +++ b/roles/uperf/templates/service_metallb.yml.j2 @@ -0,0 +1,94 @@ +--- +kind: Service +apiVersion: v1 +metadata: + name: uperf-service-{{ item }}-{{ trunc_uuid }} + namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf + app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} + index: "{{ item }}" + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + annotations: + metallb.universe.tf/address-pool: '{{ workload_args.metallb.addresspool | default("addresspool-l2") }}' + metallb.universe.tf/allow-shared-ip: uperf-service-{{ item }}-{{ trunc_uuid }} +spec: + selector: + app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} + externalTrafficPolicy: '{{ workload_args.metallb.service_etp | default("Cluster") }}' + type: LoadBalancer + ports: + - name: uperf + port: 30000 + targetPort: 30000 + protocol: TCP +{% for num in range(30001,30012,1) %} + - name: uperf-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: TCP +{% endfor %} +--- +kind: Service +apiVersion: v1 +metadata: + name: uperf-service-{{ item }}-{{ trunc_uuid }}-udp + namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf + app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} + index: "{{ item }}" + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }}-udp + annotations: + metallb.universe.tf/address-pool: '{{ workload_args.metallb.addresspool | default("addresspool-l2") }}' + metallb.universe.tf/allow-shared-ip: uperf-service-{{ item }}-{{ trunc_uuid }} +spec: + selector: + app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} + externalTrafficPolicy: '{{ workload_args.metallb.service_etp | default("Cluster") }}' + type: LoadBalancer + ports: + - name: uperf + port: 30000 + targetPort: 30000 + protocol: UDP +{% for num in range(30001,30012,1) %} + - name: uperf-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: UDP +{% endfor %} +--- +{% if "sctp" in workload_args.protos %} +kind: Service +apiVersion: v1 +metadata: + name: uperf-service-{{ item }}-{{ trunc_uuid }}-sctp + namespace: '{{ operator_namespace }}' + labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf + app: uperf-bench-server-{{ item }}-{{ trunc_uuid }}} + type: {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }}-udp + annotations: + metallb.universe.tf/address-pool: '{{ workload_args.metallb.addresspool | default("addresspool-l2") }}' + metallb.universe.tf/allow-shared-ip: uperf-service-{{ item }}-{{ trunc_uuid }} +spec: + selector: + app: uperf-bench-server-{{ item }}-{{ trunc_uuid }} + externalTrafficPolicy: '{{ workload_args.metallb.service_etp | default("Cluster") }}' + type: LoadBalancer + ports: + - name: uperf + port: 30000 + targetPort: 30000 + protocol: SCTP +{% for num in range(30001,30012,1) %} + - name: uperf-control-tcp-{{num}} + port: {{num}} + targetPort: {{num}} + protocol: SCTP +{% endfor %} +{% endif %} \ No newline at end of file diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2 index 493b4b7a9..fdcf783ab 100644 --- a/roles/uperf/templates/workload.yml.j2 +++ b/roles/uperf/templates/workload.yml.j2 @@ -1,20 +1,39 @@ --- +{% set control_port = 30000 %} kind: Job apiVersion: batch/v1 metadata: {% if workload_args.serviceip is sameas true %} +{% if workload_args.servicetype | default("clusterip") == "nodeport" %} + name: 'uperf-client-{{item.status.hostIP}}-{{ item.metadata.labels.index|int }}-{{ trunc_uuid }}' +{% elif workload_args.servicetype | default("clusterip") == "metallb" or workload_args.servicetype | default("clusterip") == "loadbalancer" %} + name: 'uperf-client-{{item.status.loadBalancer.ingress[0].ip}}-{{ trunc_uuid }}' +{% else %} name: 'uperf-client-{{item.spec.clusterIP}}-{{ trunc_uuid }}' +{% endif %} {% else %} name: 'uperf-client-{{item.status.podIP}}-{{ trunc_uuid }}' {% endif %} namespace: '{{ operator_namespace }}' +{% if workload_args.annotations is defined or workload_args.server_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.server_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: template: metadata: labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf + benchmark-operator-role: client app: uperf-bench-client-{{ trunc_uuid }} clientfor: {{ item.metadata.labels.app }} - type: uperf-bench-client-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} {% if workload_args.multus.enabled is sameas true %} annotations: k8s.v1.cni.cncf.io/networks: {{ workload_args.multus.client }} @@ -28,6 +47,25 @@ spec: serviceAccountName: benchmark-operator {% endif %} affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: type + operator: In + values: + - {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} + topologyKey: kubernetes.io/hostname + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: type + operator: In + values: + - {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }} + topologyKey: topology.kubernetes.io/zone podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 @@ -49,7 +87,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index @@ -74,7 +112,7 @@ spec: fieldRef: fieldPath: spec.nodeName - name: server_node - value: "{{ uperf.pin_server|default("unknown") }}" + value: "{{ item.spec.nodeName | default("unknown") }}" {% if workload_args.client_resources is defined %} resources: {{ workload_args.client_resources | to_json }} {% endif %} @@ -83,23 +121,34 @@ spec: args: {% if workload_args.serviceip is sameas true %} - "export serviceip=true; - export h={{item.spec.clusterIP}}; +{% if workload_args.servicetype | default("clusterip") == "nodeport" %} + export h={{item.status.hostIP}}; + export servicetype={{workload_args.servicetype}}; +{% elif workload_args.servicetype | default("clusterip") == "metallb" or workload_args.servicetype | default("clusterip") == "loadbalancer" %} + export h={{item.status.loadBalancer.ingress[0].ip}}; + export servicetype={{workload_args.servicetype}}; +{% else %} + export h={{item.spec.clusterIP}}; + export servicetype={{workload_args.servicetype | default("clusterip")}}; +{% endif %} {% else %} {% if workload_args.multus.client is defined %} - "export multus_client={{workload_args.multus.client}}; - export h={{ (item['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; + export h={{ (item['metadata']['annotations']['k8s.v1.cni.cncf.io/networks-status'] | from_json)[1]['ips'][0] }}; {% else %} - "export h={{item.status.podIP}}; {% endif %} {% endif %} + export port={{ control_port + (0 if (workload_args.serviceip is defined and workload_args.serviceip and ( workload_args.servicetype | default("clusterip") == "clusterip" )) else (item.metadata.labels.index|int * 100))}} {% if workload_args.networkpolicy is defined %} export networkpolicy={{workload_args.networkpolicy}}; {% endif %} export hostnet={{workload_args.hostnetwork}}; + export num_pairs={{workload_args.pair| default('1')}}; export ips=$(hostname -I); - export num_pairs={{workload_args.pair}}; + exit_status=0; while true; do - if [[ $(redis-cli -h {{bo.resources[0].status.podIP}} get start) =~ 'true' ]]; then + if [[ $(redis-cli -h {{bo.resources[0].status.podIP}} get start-{{ trunc_uuid }}) =~ 'true' ]]; then {% for test in workload_args.test_types %} {% for proto in workload_args.protos %} {% for size in workload_args.sizes %} @@ -112,11 +161,17 @@ spec: {% endif %} {% for nthr in workload_args.nthrs %} cat /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}}; + run_snafu --tool uperf -w /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}} \ {% if workload_args.run_id is defined %} - run_snafu --tool uperf --run-id {{workload_args.run_id}} -w /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}}; -{% else %} - run_snafu --tool uperf -w /tmp/uperf-test/uperf-{{test}}-{{proto}}-{{wsize}}-{{rsize}}-{{nthr}} -s {{workload_args.samples}} --resourcetype {{resource_kind}} -u {{ uuid }} --user {{test_user | default("ripsaw")}}; + --run-id {{workload_args.run_id}} \ +{% endif %} +{% if workload_args.debug is defined and workload_args.debug %} + -v \ {% endif %} + ; + if [[ $? -ne 0 ]]; then + exit_status=1; + fi; {% endfor %} {% endfor %} {% endfor %} @@ -126,17 +181,17 @@ spec: fi; break; done; - redis-cli -h {{bo.resources[0].status.podIP}} set start false" + redis-cli -h {{bo.resources[0].status.podIP}} set start-{{ trunc_uuid }} false; + exit $exit_status;" volumeMounts: - name: config-volume mountPath: "/tmp/uperf-test" volumes: - name: config-volume configMap: - name: uperf-test-{{ trunc_uuid }} - restartPolicy: OnFailure + name: uperf-test-{{ item.metadata.labels.index|int }}-{{ trunc_uuid }} + restartPolicy: Never {% if workload_args.pin is sameas true %} nodeSelector: - kubernetes.io/hostname: '{{ workload_args.pin_client }}' + kubernetes.io/hostname: '{{ workload_args.pin_client }}' {% endif %} -{% include "metadata.yml.j2" %} diff --git a/roles/uperf/templates/workload_vm.yml.j2 b/roles/uperf/templates/workload_vm.yml.j2 index ca0fac148..35808f20a 100644 --- a/roles/uperf/templates/workload_vm.yml.j2 +++ b/roles/uperf/templates/workload_vm.yml.j2 @@ -2,11 +2,23 @@ apiVersion: kubevirt.io/v1alpha3 kind: VirtualMachineInstance metadata: - name: 'uperf-client-{{item.status.interfaces[0].ipAddress}}-{{ trunc_uuid }}' + name: 'uperf-client-{{item.1.status.interfaces[0].ipAddress}}-{{ trunc_uuid }}' namespace: '{{ operator_namespace }}' labels: + benchmark-uuid: {{ uuid }} + benchmark-operator-workload: uperf + benchmark-operator-role: client app: uperf-bench-client-{{ trunc_uuid }} - type: uperf-bench-client-{{ trunc_uuid }} + type: {{ ansible_operator_meta.name }}-bench-client-{{ trunc_uuid }} +{% if workload_args.annotations is defined or workload_args.client_annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% for annotation, value in workload_args.client_annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: domain: cpu: @@ -78,9 +90,9 @@ spec: - bash /tmp/uperf_script/run_script.sh name: cloudinitdisk - configMap: - name: uperf-test-{{ trunc_uuid }} + name: uperf-test-{{ item.0 }}-{{ trunc_uuid }} name: app-config-disk - configMap: - name: uperf-run-script-{{item.status.interfaces[0].ipAddress}}-{{ trunc_uuid }} + name: uperf-run-script-{{item.1.status.interfaces[0].ipAddress}}-{{ trunc_uuid }} name: run-config-disk -status: {} +status: {} \ No newline at end of file diff --git a/roles/uperf/vars/main.yml b/roles/uperf/vars/main.yml index 82fc9c23f..fc9dc0bf4 100644 --- a/roles/uperf/vars/main.yml +++ b/roles/uperf/vars/main.yml @@ -1,3 +1,3 @@ --- -# vars file for bench -cleanup: true +# vars file for uperf +cleanup: true \ No newline at end of file diff --git a/roles/uuid/tasks/main.yml b/roles/uuid/tasks/main.yml index 4ff8d2ec7..2ba7e2c58 100644 --- a/roles/uuid/tasks/main.yml +++ b/roles/uuid/tasks/main.yml @@ -5,6 +5,3 @@ - name: set the truncated uuid set_fact: trunc_uuid={{ uuid.split("-")[0] }} - -- debug: - msg: " The results of current execution of the workload {{ workload.name }} will be associated with uuid: {{ uuid }}" diff --git a/roles/vegeta/tasks/main.yml b/roles/vegeta/tasks/main.yml index 2505e6f89..3edfa084f 100644 --- a/roles/vegeta/tasks/main.yml +++ b/roles/vegeta/tasks/main.yml @@ -1,82 +1,15 @@ --- +- name: Create vegeta clients + k8s: + definition: "{{ lookup('template', 'vegeta.yml.j2') | from_yaml }}" + loop: "{{ range(0, workload_args.clients|default(1)|int)|list }}" + when: benchmark_state.resources[0].status.state == "Building" -- name: Get current state - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state +- set_fact: + resources: + - "{{ role_path }}/templates/targets.yml.j2" -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined -- name: Get current state - If it has changed - k8s_info: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: '{{ meta.name }}' - namespace: '{{ operator_namespace }}' - register: resource_state - -- name: Capture operator information - k8s_info: - kind: Pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - name = benchmark-operator - register: bo - -- block: - - name: Create targets configmap - k8s: - definition: "{{ lookup('template', 'targets.yml.j2') | from_yaml }}" - - - name: Create vegeta clients - k8s: - definition: "{{ lookup('template', 'vegeta.yml.j2') | from_yaml }}" - loop: "{{ range(0, workload_args.clients|default(1)|int)|list }}" - - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Running - - when: resource_state.resources[0].status.state == "Building" - -- block: - - - name: Waiting for pods to complete.... - k8s_info: - kind: pod - api_version: v1 - namespace: '{{ operator_namespace }}' - label_selectors: - - app = vegeta-benchmark-{{ trunc_uuid }} - field_selectors: - - status.phase=Succeeded - register: client_pods - - - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Complete - complete: true - when: workload_args.clients|default(1)|int == client_pods.resources|length - - when: resource_state.resources[0].status.state == "Running" +- name: Generic workload - vegeta + include_role: + name: generic_workload diff --git a/roles/vegeta/templates/vegeta.yml.j2 b/roles/vegeta/templates/vegeta.yml.j2 index 199a8f748..279d05dfa 100644 --- a/roles/vegeta/templates/vegeta.yml.j2 +++ b/roles/vegeta/templates/vegeta.yml.j2 @@ -5,17 +5,29 @@ metadata: name: vegeta-{{ item }}-{{ trunc_uuid }} namespace: {{ operator_namespace }} spec: + backoffLimit: 0 + activeDeadlineSeconds: {{ workload_args.job_timeout|default(3600) }} template: metadata: labels: app: vegeta-benchmark-{{ trunc_uuid }} + benchmark-uuid: {{ uuid }} +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: +{% if workload_args.node_selector is defined %} + nodeSelector: + '{{ workload_args.node_selector.split("=")[0] }}': '{{ workload_args.node_selector.split("=")[1] }}' +{% endif %} {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" {% endif %} {% if workload_args.hostnetwork is sameas true %} hostNetwork : true - serviceAccountName: benchmark-operator {% endif %} affinity: podAntiAffinity: @@ -39,7 +51,7 @@ spec: value: "{{ test_user | default("ripsaw") }}" - name: clustername value: "{{ clustername }}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index @@ -65,17 +77,17 @@ spec: args: - | {% for t in workload_args.targets %} - redis-cli -h {{ bo.resources[0].status.podIP }} set vegeta-{{ item }}-{{ trunc_uuid }} ready - ready=0 - while [[ ${ready} == 0 ]]; do - for client in $(seq 0 {{ workload_args.clients-1 }}); do - [[ `redis-cli -h {{ bo.resources[0].status.podIP }} get vegeta-${client}-{{ trunc_uuid }}` != "ready" ]] && ready=0 && break - ready=1 - done + redis-cli -h {{ bo.resources[0].status.podIP }} incr vegeta-{{ trunc_uuid }} + echo "Waiting for {{ workload_args.clients-1 }} clients to be ready" + while [[ $(redis-cli -h {{ bo.resources[0].status.podIP }} get vegeta-{{ trunc_uuid }}) != {{ workload_args.clients }} ]]; do + sleep 0.1 done - sleep 0.1 - redis-cli -h {{ bo.resources[0].status.podIP }} set vegeta-{{ item }}-{{ trunc_uuid }} notready - run_snafu --tool vegeta --targets /tmp/vegeta/{{ t.name|replace(" ","") }} -u ${uuid} -d {{ t.duration }} --user ${test_user} -w {{ t.workers|default(1) }} -s {{ t.samples|default(1) }} {{ "--keepalive" if t.keepalive|default(false) }} + run_snafu --tool vegeta --targets /tmp/vegeta/{{ t.name|replace(" ","") }} -u ${uuid} -d {{ t.duration }} --user ${test_user} -w {{ t.workers|default(1) }} -s {{ t.samples|default(1) }} {{ "--keepalive" if t.keepalive|default(false) }} \ +{% if workload_args.debug is defined and workload_args.debug %} + -v \ +{% endif %} + ; + redis-cli -h {{ bo.resources[0].status.podIP }} decr vegeta-{{ trunc_uuid }} {% endfor %} volumeMounts: - name: targets-volume @@ -84,7 +96,7 @@ spec: - name: targets-volume configMap: name: vegeta-targets-{{ trunc_uuid }} - restartPolicy: OnFailure + restartPolicy: Never {% if workload_args.nodeselector is defined %} nodeSelector: {{ nodeselector|to_json }} {% endif %} diff --git a/roles/ycsb/tasks/main.yml b/roles/ycsb/tasks/main.yml index a5be7b1a3..85aba2c6b 100644 --- a/roles/ycsb/tasks/main.yml +++ b/roles/ycsb/tasks/main.yml @@ -1,30 +1,4 @@ --- -- name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - register: resource_state - -- operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: Building - complete: false - when: resource_state.resources[0].status.state is not defined - -- name: Get benchmark state - k8s_facts: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - register: resource_state - - block: - name: setting ycsb_workload_load var @@ -40,27 +14,21 @@ definition: "{{ lookup('template', 'ycsb_load.yaml') | from_yaml }}" when: workload_args.loaded is undefined or not workload_args.loaded - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Running Load" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Running Load" when: workload_args.loaded is undefined or not workload_args.loaded|default('false') - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting First Workload" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting First Workload" when: workload_args.loaded is defined and workload_args.loaded|default('false') - when: resource_state.resources[0].status.state == "Building" + when: benchmark_state.resources[0].status.state == "Building" - block: @@ -69,7 +37,7 @@ ycsb_workload_load: "workloada" - name: Wait for Load Job to Succeed... - k8s_facts: + k8s_info: kind: Job api_version: batch/v1 name: 'ycsb-data-load-job-{{ ycsb_workload_load }}-{{ trunc_uuid }}' @@ -78,51 +46,35 @@ - name = 'ycsb-load-{{ trunc_uuid }}' register: ycsb_load_pod - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting First Workload" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting First Workload" when: "ycsb_load_pod | json_query('resources[].status.succeeded')" - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Complete" - when: workload_args.workloads is undefined and ycsb_load_pod | json_query('resources[].status.succeeded') - - when: resource_state.resources[0].status.state == "Running Load" + when: benchmark_state.resources[0].status.state == "Running Load" - block: - name: Add the workload list length to redis - command: "redis-cli set {{ meta.name }}-{{ uuid }}-ycsb {{ workload_args.workloads|length }}" + command: "redis-cli set {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb {{ workload_args.workloads|length }}" - name: Add the first workload index to redis - command: "redis-cli set {{ meta.name }}-{{ uuid }}-ycsb-current 0" + command: "redis-cli set {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb-current 0" - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Workload" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting Workload" - when: resource_state.resources[0].status.state == "Starting First Workload" + when: benchmark_state.resources[0].status.state == "Starting First Workload" - block: - name: Get current workload index - command: "redis-cli get {{ meta.name }}-{{ uuid }}-ycsb-current" + command: "redis-cli get {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb-current" register: wrkload - name: set ycsb_workload variable @@ -137,21 +89,18 @@ k8s: definition: "{{ lookup('template', 'ycsb_run.yaml') | from_yaml }}" - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Running Workload" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Running Workload" - when: resource_state.resources[0].status.state == "Starting Workload" + when: benchmark_state.resources[0].status.state == "Starting Workload" - block: - name: Get current workload from redis - command: "redis-cli get {{ meta.name }}-{{ uuid }}-ycsb-current" + command: "redis-cli get {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb-current" register: wrkload - name: set ycsb_workload variable @@ -159,7 +108,7 @@ ycsb_workload: "{{ workload_args.workloads[wrkload.stdout|int] }}" - name: Wait for YCSB Workload Job to Succeed... - k8s_facts: + k8s_info: kind: Job api_version: batch/v1 name: 'ycsb-bench-job-{{ ycsb_workload }}-{{ trunc_uuid }}' @@ -168,26 +117,23 @@ - name = 'ycsb-run-{{ trunc_uuid }}' register: ycsb_bench - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Workload Complete" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Workload Complete" when: "ycsb_bench | json_query('resources[].status.succeeded')" - when: resource_state.resources[0].status.state == "Running Workload" + when: benchmark_state.resources[0].status.state == "Running Workload" - block: - name: Get current workload from redis - command: "redis-cli get {{ meta.name }}-{{ uuid }}-ycsb-current" + command: "redis-cli get {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb-current" register: current_workload - name: Get list length of workloads from redis - command: "redis-cli get {{ meta.name }}-{{ uuid }}-ycsb" + command: "redis-cli get {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb" register: workload_list - name: Iterate index @@ -195,26 +141,24 @@ new_workload_index: "{{ current_workload.stdout|int + 1 }}" - name: Update current workload item in redis - command: "redis-cli set {{ meta.name }}-{{ uuid }}-ycsb-current {{ new_workload_index }}" + command: "redis-cli set {{ ansible_operator_meta.name }}-{{ uuid }}-ycsb-current {{ new_workload_index }}" - - name: Update resource state - operator_sdk.util.k8s_status: - api_version: ripsaw.cloudbulldozer.io/v1alpha1 - kind: Benchmark - name: "{{ meta.name }}" - namespace: "{{ operator_namespace }}" - status: - state: "Starting Workload" + - include_role: + name: benchmark_state + tasks_from: set_state + vars: + state: "Starting Workload" when: workload_list.stdout != new_workload_index - name: Update resource state operator_sdk.util.k8s_status: api_version: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark - name: "{{ meta.name }}" + name: "{{ ansible_operator_meta.name }}" namespace: "{{ operator_namespace }}" status: - state: "Complete" + state: Complete + complete: true when: workload_list.stdout == new_workload_index - when: resource_state.resources[0].status.state == "Workload Complete" + when: benchmark_state.resources[0].status.state == "Workload Complete" diff --git a/roles/ycsb/templates/ycsb_load.yaml b/roles/ycsb/templates/ycsb_load.yaml index 9ef0407b5..78d13ad21 100644 --- a/roles/ycsb/templates/ycsb_load.yaml +++ b/roles/ycsb/templates/ycsb_load.yaml @@ -7,10 +7,16 @@ metadata: spec: template: metadata: - name: '{{ meta.name }}-ycsb-data-load' + name: '{{ ansible_operator_meta.name }}-ycsb-data-load' namespace: '{{ operator_namespace }}' labels: name: 'ycsb-load-{{ trunc_uuid }}' +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -32,7 +38,7 @@ spec: value: "{{workload_args.operationcount}}" - name: uuid value: "{{uuid}}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index @@ -54,7 +60,12 @@ spec: {% endif %} imagePullPolicy: Always command: ["/bin/sh", "-c"] - args: ['run_snafu --tool ycsb -r 1 -l -d {{ workload_args.driver }} -u {{ uuid }} --user {{ test_user | default("ripsaw") }} -w {{ ycsb_workload_load }} -x "{{ workload_args.options_load }}"'] + args: + - run_snafu --tool ycsb +{% if workload_args.debug is defined and workload_args.debug %} + -v +{% endif %} + -r 1 -l -d {{ workload_args.driver }} -u {{ uuid }} --user {{ test_user | default("ripsaw") }} -w {{ ycsb_workload_load }} -x "{{ workload_args.options_load }}"; volumeMounts: - name: config-volume mountPath: "/tmp/ycsb" diff --git a/roles/ycsb/templates/ycsb_run.yaml b/roles/ycsb/templates/ycsb_run.yaml index 9711d0af3..f4397a401 100644 --- a/roles/ycsb/templates/ycsb_run.yaml +++ b/roles/ycsb/templates/ycsb_run.yaml @@ -7,10 +7,16 @@ metadata: spec: template: metadata: - name: '{{ meta.name }}-ycsb-bench' + name: '{{ ansible_operator_meta.name }}-ycsb-bench' namespace: '{{ operator_namespace }}' labels: name: 'ycsb-run-{{ trunc_uuid }}' +{% if workload_args.annotations is defined %} + annotations: +{% for annotation, value in workload_args.annotations.items() %} + "{{annotation}}": "{{value}}" +{% endfor %} +{% endif %} spec: {% if workload_args.runtime_class is defined %} runtimeClassName: "{{ workload_args.runtime_class }}" @@ -32,7 +38,7 @@ spec: value: "{{workload_args.operationcount}}" - name: uuid value: "{{uuid}}" -{% if elasticsearch is defined %} +{% if elasticsearch.url %} - name: es value: "{{ elasticsearch.url }}" - name: es_index @@ -54,7 +60,12 @@ spec: {% endif %} imagePullPolicy: Always command: ["/bin/sh", "-c"] - args: ['run_snafu --tool ycsb -r 1 -d {{ workload_args.driver }} -w {{ ycsb_workload }} -u {{ uuid }} --user {{ test_user | default("ripsaw") }} -x "{{ workload_args.options_run }}"'] + args: + - run_snafu --tool ycsb +{% if workload_args.debug is defined and workload_args.debug %} + -v +{% endif %} + -r 1 -d {{ workload_args.driver }} -w {{ ycsb_workload }} -u {{ uuid }} --user {{ test_user | default("ripsaw") }} -x "{{ workload_args.options_run }}"; volumeMounts: - name: config-volume mountPath: "/tmp/ycsb" diff --git a/run_test.sh b/run_test.sh deleted file mode 100755 index 5bc2b7e06..000000000 --- a/run_test.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -set -x - -ci_dir=$1 -ci_test=`echo $1 | sed 's/-/_/g'` -retries=3 - -figlet $ci_test - -# Random short sleep to avoid command collision -sleep $[ ( $RANDOM % 10 )]s - -cd $ci_dir -source tests/common.sh - -count=0 -start_time=`date` - -# Run the test up to a max of $retries -while [ $count -le $retries ] -do - wait_clean - - # Test ci - if /bin/bash tests/$ci_test.sh >> $ci_test.out 2>&1 - then - # if the test passes update the results and complete - end_time=`date` - duration=`date -ud@$(($(date -ud"$end_time" +%s)-$(date -ud"$start_time" +%s))) +%T` - echo "$ci_dir: Successful" - echo "$ci_dir: Successful" > ci_results - echo " " > results.xml - echo "$ci_test | Pass | $count | $duration" > results.markdown - count=$retries - else - # if the test failed check if we have done the max retries - if [ $count -lt $retries ] - then - echo "$ci_dir: Failed. Retrying" - echo "$ci_dir: Failed. Retrying" >> $ci_test.out - else - end_time=`date` - duration=`date -ud@$(($(date -ud"$end_time" +%s)-$(date -ud"$start_time" +%s))) +%T` - echo "$ci_dir: Failed retry" - echo "$ci_dir: Failed" > ci_results - echo " " > results.xml - echo " - " >> results.xml - echo "$ci_test | Fail | $count | $duration" > results.markdown - echo "Logs for "$ci_dir - - # Display the error log since we have failed to pass - cat $ci_test.out - fi - fi - ((count++)) -done -wait_clean diff --git a/test.sh b/test.sh deleted file mode 100755 index 4b37ddb78..000000000 --- a/test.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env bash -set -x - -# The maximum number of concurrent tests to run at one time (0 for unlimited). This can be overriden with -p -max_concurrent=3 - -# Presetting test_choice to be blank. -test_choice='' - -while getopts p:t:s: flag -do - case "${flag}" in - p) max_concurrent=${OPTARG};; - t) test_choice=${OPTARG};; - s) ES_SERVER=${OPTARG};; - esac -done - -source tests/common.sh - -eStatus=0 - -git_diff_files="$(git diff remotes/origin/master --name-only)" - -if [[ $test_choice != '' ]]; then - echo "Running for requested tests" - populate_test_list "${test_choice}" -elif [[ `echo "${git_diff_files}" | grep -cv /` -gt 0 || `echo "${git_diff_files}" | grep -E "^(templates|build|deploy|group_vars|resources|tests/common.sh|roles/uuid)"` ]]; then - echo "Running full test" - cp tests/test_list tests/iterate_tests -else - echo "Running specific tests" - populate_test_list "${git_diff_files}" -fi - -test_list="$(cat tests/iterate_tests)" -echo "running test suit consisting of ${test_list}" - -if [[ ${test_list} == "" ]]; then - echo "No tests to run" - echo "Results for "$JOB_NAME > results.markdown - echo "No tests to run" >> results.markdown - exit 0 -fi - -# Massage the names into something that is acceptable for a namespace -sed 's/\.sh//g' tests/iterate_tests | sort | uniq > tests/my_tests -sed -i 's/_/-/g' tests/my_tests - -# Prep the results.xml file -echo ' - - ' > results.xml - -# Prep the results.markdown file -echo "Results for "$JOB_NAME > results.markdown -echo "" >> results.markdown -echo 'Test | Result | Retries| Duration (HH:MM:SS)' >> results.markdown -echo '-----|--------|--------|---------' >> results.markdown - -# Create a "gold" directory based off the current branch -mkdir gold - -# Generate uuid -NEW_UUID=$(uuidgen) -UUID=${NEW_UUID%-*} - -#Tag name -tag_name="${NODE_NAME:-master}" - -sed -i "s#ES_SERVER#$ES_SERVER#g" tests/test_crs/* -sed -i "s/sql-server/sql-server-$UUID/g" tests/mssql.yaml tests/test_crs/valid_hammerdb.yaml tests/test_hammerdb.sh -sed -i "s/benchmarks.ripsaw.cloudbulldozer.io/benchmarks-$UUID.ripsaw.cloudbulldozer.io/g" resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -sed -i "s/kind: Benchmark/kind: Benchmark-$UUID/g" resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -sed -i "s/listKind: BenchmarkList/listKind: BenchmarkList-$UUID/g" resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -sed -i "s/plural: benchmarks/plural: benchmarks-$UUID/g" resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -sed -i "s/singular: benchmark/singular: benchmark-$UUID/g" resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml -sed -i "s/benchmarks/benchmarks-$UUID/g" tests/common.sh -sed -i "s/kind: Benchmark/kind: Benchmark-$UUID/g" tests/test_crs/*.yaml -sed -i "s/kind: Benchmark/kind: Benchmark-$UUID/g" playbook.yml -sed -i "s/kind: Benchmark/kind: Benchmark-$UUID/g" watches.yaml -sed -i "s/backpack_role/backpack_role-$UUID/g" resources/backpack_role.yaml -sed -i "s/my-ripsaw/my-ripsaw-$UUID-test-fiod/g" resources/kernel-cache-drop-daemonset.yaml -grep -Rl "kind: Benchmark" roles/ | xargs sed -i "s/kind: Benchmark/kind: Benchmark-$UUID/g" -sed -i "s| image: quay.io/benchmark-operator/benchmark-operator:master*| image: $image_location/$image_account/benchmark-operator:$tag_name # |" resources/operator.yaml - -cp -pr * gold/ - -# Create individual directories for each test -for ci_dir in `cat tests/my_tests` -do - mkdir $ci_dir - cp -pr gold/* $ci_dir/ - cd $ci_dir/ - # Edit the namespaces so we can run in parallel - sed -i "s/my-ripsaw/my-ripsaw-$UUID-$ci_dir/g" `grep -Rl my-ripsaw` - sed -i "s/benchmark-operator-role/benchmark-operator-role-$UUID-$ci_dir/g" `grep -Rl benchmark-operator-role` - cd .. -done - -# Update the operator image -update_operator_image - -# Run scale test first if it is in the test list -scale_test="false" -if [[ `grep test-scale-openshift tests/my_tests` ]] -then - scale_test="true" - sed -i '/test-scale-openshift/d' tests/my_tests - ./run_test.sh test-scale-openshift -fi - -# Run tests in parallel up to $max_concurrent at a time. -parallel -n 1 -a tests/my_tests -P $max_concurrent ./run_test.sh -if [[ $scale_test == "true" ]] -then - echo "test-scale-openshift" >> tests/my_tests -fi - -# Update and close JUnit test results.xml and markdown file -for test_dir in `cat tests/my_tests` -do - cat $test_dir/results.xml >> results.xml - cat $test_dir/results.markdown >> results.markdown - cat $test_dir/ci_results >> ci_results -done - -# Get number of successes/failures -testcount=`wc -l ci_results` -success=`grep Successful ci_results | awk -F ":" '{print $1}'` -failed=`grep Failed ci_results | awk -F ":" '{print $1}'` -failcount=`grep -c Failed ci_results` -echo "CI tests that passed: "$success -echo "CI tests that failed: "$failed -echo "Smoke test: Complete" - -echo " -" >> results.xml - -sed -i "s/NUMTESTS/$testcount/g" results.xml -sed -i "s/NUMFAILURES/$failcount/g" results.xml - -if [ `grep -c Failed ci_results` -gt 0 ] -then - eStatus=1 -fi - -# Clean up our created directories -rm -rf gold test-* ci_results - -exit $eStatus diff --git a/tests/check_es.py b/tests/check_es.py old mode 100644 new mode 100755 diff --git a/tests/common.sh b/tests/common.sh old mode 100644 new mode 100755 index b7d9a9d7e..ac0a29fee --- a/tests/common.sh +++ b/tests/common.sh @@ -14,6 +14,7 @@ function populate_test_list { # Check for changes in roles if [[ $(echo ${item} | grep 'roles/fs-drift') ]]; then echo "test_fs_drift.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'roles/uperf') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/uperf-scale') ]]; then echo "test_uperf_scale.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'roles/fio_distributed') ]]; then echo "test_fiod.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'roles/iperf3') ]]; then echo "test_iperf3.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'roles/byowl') ]]; then echo "test_byowl.sh" >> tests/iterate_tests; fi @@ -28,7 +29,8 @@ function populate_test_list { if [[ $(echo ${item} | grep 'roles/scale_openshift') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'roles/kube-burner') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'roles/flent') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi - + if [[ $(echo ${item} | grep 'roles/log_generator') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'roles/image_pull') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi # Check for changes in cr files if [[ $(echo ${item} | grep 'valid_backpack*') ]]; then echo "test_backpack.sh" >> tests/iterate_tests; fi @@ -41,13 +43,15 @@ function populate_test_list { if [[ $(echo ${item} | grep 'valid_smallfile*') ]]; then echo "test_smallfile.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_sysbench*') ]]; then echo "test_sysbench.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_uperf*') ]]; then echo "test_uperf.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_uperf_scale*') ]]; then echo "test_uperf_scale.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_ycsb*') ]]; then echo "test_ycsb.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_vegeta*') ]]; then echo "test_vegeta.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_stressng*') ]]; then echo "test_stressng.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_scale*') ]]; then echo "test_scale_openshift.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_kube-burner*') ]]; then echo "test_kubeburner.sh" >> tests/iterate_tests; fi if [[ $(echo ${item} | grep 'valid_flent*') ]]; then echo "test_flent.sh" >> tests/iterate_tests; fi - + if [[ $(echo ${item} | grep 'valid_log_generator*') ]]; then echo "test_log_generator.sh" >> tests/iterate_tests; fi + if [[ $(echo ${item} | grep 'valid_image_pull*') ]]; then echo "test_image_pull.sh" >> tests/iterate_tests; fi # Check for changes in test scripts test_check=`echo $item | awk -F / '{print $2}'` @@ -56,23 +60,27 @@ function populate_test_list { done } +function delete_benchmark { + kubectl delete -f $1 --ignore-not-found=true +} + function wait_clean { if [[ `kubectl get benchmarks.ripsaw.cloudbulldozer.io --all-namespaces` ]] then - kubectl delete benchmarks -n my-ripsaw --all --ignore-not-found + echo "skipping cleanup" fi - kubectl delete namespace my-ripsaw --ignore-not-found } # The argument is 'timeout in seconds' function get_uuid () { - sleep_time=$1 + sleep_time=20 + benchmark_name=$1 sleep $sleep_time counter=0 counter_max=6 uuid="False" until [ $uuid != "False" ] ; do - uuid=$(kubectl -n my-ripsaw get benchmarks -o jsonpath='{.items[0].status.uuid}') + uuid=$(kubectl -n benchmark-operator get benchmark/$benchmark_name -o jsonpath='{.status.uuid}') if [ -z $uuid ]; then sleep $sleep_time uuid="False" @@ -95,7 +103,7 @@ function get_pod () { pod_name="False" until [ $pod_name != "False" ] ; do sleep $sleep_time - pod_name=$(kubectl get pods -l $1 --namespace ${3:-my-ripsaw} -o name | cut -d/ -f2) + pod_name=$(kubectl get pods -l $1 --namespace ${3:-benchmark-operator} -o name | cut -d/ -f2) if [ -z $pod_name ]; then pod_name="False" fi @@ -117,7 +125,7 @@ function pod_count () { export $1 until [ $pod_count == $2 ] ; do sleep $sleep_time - pod_count=$(kubectl get pods -n my-ripsaw -l $1 -o name | wc -l) + pod_count=$(kubectl get pods -n benchmark-operator -l $1 -o name | wc -l) if [ -z $pod_count ]; then pod_count=0 fi @@ -131,53 +139,15 @@ function pod_count () { } function apply_operator { - operator_requirements - BENCHMARK_OPERATOR_IMAGE=${BENCHMARK_OPERATOR_IMAGE:-"quay.io/benchmark-operator/benchmark-operator:master"} - cat resources/operator.yaml | \ - sed 's#quay.io/benchmark-operator/benchmark-operator:master#'$BENCHMARK_OPERATOR_IMAGE'#' | \ - kubectl apply -f - - kubectl wait --for=condition=available "deployment/benchmark-operator" -n my-ripsaw --timeout=300s + tag_name="${NODE_NAME:-master}" + make image-build image-push deploy IMG=$image_location/$image_account/benchmark-operator:$tag_name + kubectl wait --for=condition=available "deployment/benchmark-controller-manager" -n benchmark-operator --timeout=300s } function delete_operator { - kubectl delete -f resources/operator.yaml + make undeploy } -function marketplace_setup { - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/01_namespace.yaml - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/02_catalogsourceconfig.crd.yaml - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/03_operatorsource.crd.yaml - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/04_service_account.yaml - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/05_role.yaml - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/06_role_binding.yaml - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/07_upstream_operatorsource.cr.yaml - kubectl apply -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/08_operator.yaml -} - -function marketplace_cleanup { - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/07_upstream_operatorsource.cr.yaml - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/08_operator.yaml - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/06_role_binding.yaml - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/05_role.yaml - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/04_service_account.yaml - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/03_operatorsource.crd.yaml - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/02_catalogsourceconfig.crd.yaml - kubectl delete -f https://raw.githubusercontent.com/operator-framework/operator-marketplace/master/deploy/upstream/01_namespace.yaml -} - -function operator_requirements { - kubectl apply -f resources/namespace.yaml - kubectl apply -f deploy - kubectl apply -f resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml - kubectl -n my-ripsaw get roles - kubectl -n my-ripsaw get rolebindings - kubectl -n my-ripsaw get podsecuritypolicies - kubectl -n my-ripsaw get serviceaccounts - kubectl -n my-ripsaw get serviceaccount benchmark-operator -o yaml - kubectl -n my-ripsaw get role benchmark-operator -o yaml - kubectl -n my-ripsaw get rolebinding benchmark-operator -o yaml - kubectl -n my-ripsaw get podsecuritypolicy privileged -o yaml -} function backpack_requirements { kubectl apply -f resources/backpack_role.yaml @@ -185,48 +155,21 @@ function backpack_requirements { then if [[ `oc get securitycontextconstraints.security.openshift.io` ]] then - oc adm policy -n my-ripsaw add-scc-to-user privileged -z benchmark-operator - oc adm policy -n my-ripsaw add-scc-to-user privileged -z backpack-view + oc adm policy -n benchmark-operator add-scc-to-user privileged -z benchmark-operator + oc adm policy -n benchmark-operator add-scc-to-user privileged -z backpack-view fi fi } -function create_operator { - operator_requirements - apply_operator -} - -function cleanup_resources { - echo "Exiting after cleanup of resources" - kubectl delete -f resources/crds/ripsaw_v1alpha1_ripsaw_crd.yaml - kubectl delete -f deploy -} function cleanup_operator_resources { delete_operator - cleanup_resources wait_clean } -function update_operator_image { - tag_name="${NODE_NAME:-master}" - if operator-sdk build $image_location/$image_account/benchmark-operator:$tag_name --image-builder podman; then - # In case we have issues uploading to quay we will retry a few times - for i in {1..3}; do - podman push $image_location/$image_account/benchmark-operator:$tag_name && break - echo "Could not upload image to quay. Exiting" - exit 1 - done - else - echo "Could not build image. Exiting" - exit 1 - fi - sed -i "s| image: quay.io/benchmark-operator/benchmark-operator:master*| image: $image_location/$image_account/benchmark-operator:$tag_name # |" resources/operator.yaml -} - function check_log(){ for i in {1..10}; do - if kubectl logs -f $1 --namespace my-ripsaw | grep -q $2 ; then + if kubectl logs -f $1 --namespace benchmark-operator | grep -q $2 ; then break; else sleep 10 @@ -234,8 +177,53 @@ function check_log(){ done } +function get_benchmark_name() { + benchmark_file=$1 + echo $(yq e '.metadata.name' $benchmark_file) +} + +function get_benchmark_state() { + benchmark_name=$1 + kubectl -n benchmark-operator get benchmark/$benchmark_name -o jsonpath={.status.state} +} + +function wait_for_benchmark() { + benchmark_name=$1 + desired_state=$2 + until [[ $(get_benchmark_state $benchmark_name) == "$desired_state" ]]; do + if [[ $(get_benchmark_state $benchmark_name) == "Failed" ]]; then + echo "Benchmark $benchmark_name failed" + return 1 + fi + sleep 5 + done +} + +function check_benchmark_for_desired_state(){ + benchmark_name=$1 + desired_state=$2 + timeout=${3:-500s} + export -f wait_for_benchmark + export -f get_benchmark_state + if ! timeout -k $timeout $timeout bash -c "wait_for_benchmark $benchmark_name $desired_state" + then + echo "Timeout exceeded for: "$benchmark_name + + counter=3 + until [ $counter -gt $# ] + do + echo "Logs from "${@:$counter} + kubectl -n benchmark-operator logs --tail=40 ${@:$counter} + counter=$(( counter+1 )) + done + return 1 + fi + echo $(kubectl -n benchmark-operator get benchmark/$benchmark_name -o jsonpath={.status.state}) + +} + # Takes 2 or more arguments: 'command to run', 'time to wait until true' -# Any additional arguments will be passed to kubectl -n my-ripsaw logs to provide logging if a timeout occurs +# Any additional arguments will be passed to kubectl -n benchmark-operator logs to provide logging if a timeout occurs function wait_for() { if ! timeout -k $2 $2 $1 then @@ -245,7 +233,7 @@ function wait_for() { until [ $counter -gt $# ] do echo "Logs from "${@:$counter} - kubectl -n my-ripsaw logs --tail=40 ${@:$counter} + kubectl -n benchmark-operator logs --tail=40 ${@:$counter} counter=$(( counter+1 )) done return 1 @@ -263,11 +251,11 @@ function error { echo "Error caught. Dumping logs before exiting" echo "Benchmark operator Logs" - kubectl -n my-ripsaw logs --tail=40 -l name=benchmark-operator -c benchmark-operator - echo "Ansible sidecar Logs" - kubectl -n my-ripsaw logs -l name=benchmark-operator -c ansible + kubectl -n benchmark-operator logs --tail=200 -l control-plane=controller-manager -c manager } + + function wait_for_backpack() { echo "Waiting for backpack to complete before starting benchmark test" @@ -276,10 +264,10 @@ function wait_for_backpack() { max_count=60 while [[ $count -lt $max_count ]] do - if [[ `kubectl -n my-ripsaw get daemonsets backpack-$uuid` ]] + if [[ `kubectl -n benchmark-operator get daemonsets backpack-$uuid` ]] then - desired=`kubectl -n my-ripsaw get daemonsets backpack-$uuid | grep -v NAME | awk '{print $2}'` - ready=`kubectl -n my-ripsaw get daemonsets backpack-$uuid | grep -v NAME | awk '{print $4}'` + desired=`kubectl -n benchmark-operator get daemonsets backpack-$uuid | grep -v NAME | awk '{print $2}'` + ready=`kubectl -n benchmark-operator get daemonsets backpack-$uuid | grep -v NAME | awk '{print $4}'` if [[ $desired -eq $ready ]] then echo "Backpack complete. Starting benchmark" diff --git a/tests/full_test_file_trigger b/tests/full_test_file_trigger old mode 100644 new mode 100755 index 6af13e9b3..7859a425c --- a/tests/full_test_file_trigger +++ b/tests/full_test_file_trigger @@ -1,8 +1,4 @@ build/Dockerfile -deploy/10_service_account.yaml -deploy/20_role.yaml -deploy/30_role_binding.yaml -deploy/40_cluster_role_kubevirt.yaml group_vars/all.yml resources/operator.yaml tests/common.sh diff --git a/tests/mssql.yaml b/tests/mssql.yaml old mode 100644 new mode 100755 index e83de4e39..1649d7fad --- a/tests/mssql.yaml +++ b/tests/mssql.yaml @@ -11,7 +11,7 @@ metadata: namespace: sql-server spec: replicas: 1 - selector: + selector: matchLabels: app: mssql template: @@ -41,4 +41,3 @@ spec: - protocol: TCP port: 1433 targetPort: 1433 - diff --git a/tests/test_backpack.sh b/tests/test_backpack.sh deleted file mode 100644 index 0d3707133..000000000 --- a/tests/test_backpack.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - echo "Cleaning up backpack" - kubectl delete -f resources/backpack_role.yaml - wait_clean -} - -trap error ERR -trap finish EXIT - -function functional_test_backpack { - wait_clean - apply_operator - backpack_requirements - kubectl apply -f $1 - long_uuid=$(get_uuid 20) - uuid=${long_uuid:0:8} - - if [[ $1 == "tests/test_crs/valid_backpack_daemonset.yaml" ]] - then - wait_for_backpack $uuid - else - byowl_pod=$(get_pod "app=byowl-$uuid" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized pods/$byowl_pod --timeout=500s" "500s" $byowl_pod - wait_for "kubectl -n my-ripsaw wait --for=condition=complete -l app=byowl-$uuid jobs --timeout=500s" "500s" $byowl_pod - fi - - indexes="cpu_vulnerabilities-metadata cpuinfo-metadata dmidecode-metadata k8s_nodes-metadata lspci-metadata meminfo-metadata sysctl-metadata ocp_network_operator-metadata ocp_install_config-metadata ocp_kube_apiserver-metadata ocp_dns-metadata ocp_kube_controllermanager-metadata" - if check_es "${long_uuid}" "${indexes}" - then - echo "Backpack test: Success" - else - echo "Failed to find data in ES" - exit 1 - fi -} - -figlet $(basename $0) -functional_test_backpack tests/test_crs/valid_backpack_daemonset.yaml -functional_test_backpack tests/test_crs/valid_backpack_init.yaml diff --git a/tests/test_byowl.sh b/tests/test_byowl.sh old mode 100644 new mode 100755 index 974b2bc65..b210c9f44 --- a/tests/test_byowl.sh +++ b/tests/test_byowl.sh @@ -10,25 +10,22 @@ function finish { fi echo "Cleaning up byowl" - kubectl delete -f tests/test_crs/valid_byowl.yaml - delete_operator + wait_clean } trap error ERR trap finish EXIT function functional_test_byowl { - wait_clean - apply_operator - kubectl apply -f tests/test_crs/valid_byowl.yaml - long_uuid=$(get_uuid 20) - uuid=${long_uuid:0:8} + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + cr=tests/test_crs/valid_byowl.yaml - byowl_pod=$(get_pod "app=byowl-$uuid" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized pods/$byowl_pod --timeout=500s" "500s" $byowl_pod - wait_for "kubectl -n my-ripsaw wait --for=condition=complete -l app=byowl-$uuid jobs --timeout=300s" "300s" $byowl_pod - kubectl -n my-ripsaw logs "$byowl_pod" | grep "Test" + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - + check_benchmark_for_desired_state $benchmark_name Complete 500s echo "BYOWL test: Success" + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_crs/configmap-cfg.yaml b/tests/test_crs/configmap-cfg.yaml new file mode 100644 index 000000000..9c2579c5d --- /dev/null +++ b/tests/test_crs/configmap-cfg.yaml @@ -0,0 +1,79 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-burner-config + namespace: benchmark-operator +data: + alerts.yml: "- expr: avg_over_time(histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m]))[5m:]) + > 0.01\n description: 5 minutes avg. etcd fsync latency on {{$labels.pod}} higher + than 10ms {{$value}}\n severity: warning\n\n- expr: avg_over_time(histogram_quantile(0.99, + rate(etcd_network_peer_round_trip_time_seconds_bucket[5m]))[5m:]) > 0.1\n description: + 5 minutes avg. etcd network peer round trip on {{$labels.pod}} higher than 100ms + {{$value}}\n severity: warning \n" + config.yml: | + --- + global: + writeToFile: false + metricsDirectory: collected-metrics + indexerConfig: + enabled: true + esServers: [{{ .ES_SERVER }}] + insecureSkipVerify: true + defaultIndex: ripsaw-kube-burner + type: elastic + measurements: + - name: podLatency + esIndex: ripsaw-kube-burner + + jobs: + - name: basic-job + jobIterations: 5 + qps: 20 + burst: 20 + namespacedIterations: false + namespace: basic-job + waitWhenFinished: true + podWait: false + objects: + + - objectTemplate: pod.yml + replicas: 1 + inputVars: + containerImage: gcr.io/google_containers/pause-amd64:3.0 + metrics.yml: | + - query: count(kube_namespace_created) + metricName: namespaceCount + instant: true + + - query: sum(kube_pod_status_phase{}) by (phase) + metricName: podStatusCount + instant: true + + - query: count(kube_secret_info{}) + metricName: secretCount + instant: true + + - query: count(kube_deployment_labels{}) + metricName: deploymentCount + instant: true + + - query: count(kube_configmap_info{}) + metricName: configmapCount + instant: true + pod.yml: | + kind: Pod + apiVersion: v1 + metadata: + name: basic-pod-{{.Iteration}} + spec: + nodeSelector: + node-role.kubernetes.io/worker: "" + containers: + - name: basic-container + image: {{.containerImage}} + ports: + - containerPort: 8080 + protocol: TCP + imagePullPolicy: IfNotPresent + securityContext: + privileged: false diff --git a/tests/test_crs/configmap.yaml b/tests/test_crs/configmap.yaml new file mode 100644 index 000000000..e6ee8b971 --- /dev/null +++ b/tests/test_crs/configmap.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: kube-burner-configmap-cfg + namespace: benchmark-operator +spec: + elasticsearch: + url: ${ES_SERVER} + metadata: + collection: true + prometheus: + es_url: ${ES_SERVER} + prom_token: ${PROMETHEUS_TOKEN} + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + workload: + name: kube-burner + args: + extra_env_vars: + ES_SERVER: ${ES_SERVER} + configmap: kube-burner-config + cleanup: true + pin_server: {"node-role.kubernetes.io/worker": ""} + image: quay.io/cloud-bulldozer/kube-burner:latest + log_level: info + step: 30s + node_selector: + key: node-role.kubernetes.io/worker + value: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/tests/test_crs/valid_api_load.yaml b/tests/test_crs/valid_api_load.yaml new file mode 100644 index 000000000..d075ada49 --- /dev/null +++ b/tests/test_crs/valid_api_load.yaml @@ -0,0 +1,17 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: api-load + namespace: benchmark-operator +spec: + elasticsearch: + url: ES_SERVER + index_name: "api-load" + snappy: + url: SNAPPY_SERVER + user: realuser + password: realpassword + workload: + name: api_load + args: + override: version diff --git a/tests/test_crs/valid_backpack_daemonset.yaml b/tests/test_crs/valid_backpack_daemonset.yaml index c67788eeb..97fb74314 100644 --- a/tests/test_crs/valid_backpack_daemonset.yaml +++ b/tests/test_crs/valid_backpack_daemonset.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: backpack - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: ES_SERVER diff --git a/tests/test_crs/valid_backpack_init.yaml b/tests/test_crs/valid_backpack_init.yaml index 2842f769e..6886d3faa 100644 --- a/tests/test_crs/valid_backpack_init.yaml +++ b/tests/test_crs/valid_backpack_init.yaml @@ -2,7 +2,7 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: backpack - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: ES_SERVER diff --git a/tests/test_crs/valid_byowl.yaml b/tests/test_crs/valid_byowl.yaml index 9f1260357..621d08dc8 100644 --- a/tests/test_crs/valid_byowl.yaml +++ b/tests/test_crs/valid_byowl.yaml @@ -2,8 +2,14 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: byowl-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER metadata: diff --git a/tests/test_crs/valid_fiod.yaml b/tests/test_crs/valid_fiod.yaml index 1bb26a04b..9df7dfe1c 100644 --- a/tests/test_crs/valid_fiod.yaml +++ b/tests/test_crs/valid_fiod.yaml @@ -1,9 +1,15 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: fiod-benchmark + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-fio @@ -33,6 +39,7 @@ spec: filesize: 10MiB log_sample_rate: 2000 storagesize: 16Mi + debug: true ####################################### # EXPERT AREA - MODIFY WITH CAUTION # ####################################### diff --git a/tests/test_crs/valid_fiod_bsrange.yaml b/tests/test_crs/valid_fiod_bsrange.yaml index a580eccf5..2c5cdbb26 100644 --- a/tests/test_crs/valid_fiod_bsrange.yaml +++ b/tests/test_crs/valid_fiod_bsrange.yaml @@ -1,9 +1,15 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: fiod-bsrange-benchmark + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-fio @@ -32,7 +38,9 @@ spec: write_ramp_time: 1 filesize: 10MiB log_sample_rate: 2000 + log_hist_msec: 3000 storagesize: 16Mi + debug: true ####################################### # EXPERT AREA - MODIFY WITH CAUTION # ####################################### diff --git a/tests/test_crs/valid_fiod_hostpath.yaml b/tests/test_crs/valid_fiod_hostpath.yaml index ec7d9573d..cc56c4cbd 100644 --- a/tests/test_crs/valid_fiod_hostpath.yaml +++ b/tests/test_crs/valid_fiod_hostpath.yaml @@ -1,9 +1,15 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: fiod-hostpath-benchmark + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-fio @@ -20,14 +26,11 @@ spec: prefill: true jobs: - write - - read - - randwrite - randread - randrw bs: - 4KiB numjobs: - - 1 - 2 iodepth: 2 read_runtime: 6 @@ -35,7 +38,7 @@ spec: read_ramp_time: 1 write_ramp_time: 1 filesize: 10MiB - log_sample_rate: 2000 + debug: false ####################################### # EXPERT AREA - MODIFY WITH CAUTION # ####################################### diff --git a/tests/test_crs/valid_fiod_ocs_cache_drop.yaml b/tests/test_crs/valid_fiod_ocs_cache_drop.yaml new file mode 100644 index 000000000..2ebba6a0e --- /dev/null +++ b/tests/test_crs/valid_fiod_ocs_cache_drop.yaml @@ -0,0 +1,85 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: fiod-ocs-benchmark + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-fio + metadata: + collection: true + cleanup: false + workload: + name: "fio_distributed" + args: + drop_cache_rook_ceph: true + drop_cache_kernel: true + samples: 2 + servers: 2 + jobs: + - randwrite + - randread + - randrw + bs: + - 4KiB + numjobs: + - 2 + iodepth: 1 + read_runtime: 10 + write_runtime: 10 + read_ramp_time: 1 + write_ramp_time: 1 + filesize: 10MiB + log_sample_rate: 2000 + storagesize: 16Mi + debug: false +####################################### +# EXPERT AREA - MODIFY WITH CAUTION # +####################################### + job_params: + - jobname_match: write + params: + - time_based=1 + - fsync_on_close=1 + - create_on_open=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} + - jobname_match: read + params: + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: rw + params: + - rwmixread=50 + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: readwrite + params: + - rwmixread=50 + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: randread + params: + - time_based=1 + - runtime={{ workload_args.read_runtime }} + - ramp_time={{ workload_args.read_ramp_time }} + - jobname_match: randwrite + params: + - time_based=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} + - jobname_match: randrw + params: + - time_based=1 + - runtime={{ workload_args.write_runtime }} + - ramp_time={{ workload_args.write_ramp_time }} diff --git a/tests/test_crs/valid_flent.yaml b/tests/test_crs/valid_flent.yaml index 3fff6fd72..4d17d4ce4 100644 --- a/tests/test_crs/valid_flent.yaml +++ b/tests/test_crs/valid_flent.yaml @@ -2,8 +2,14 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: flent-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-flent @@ -22,3 +28,4 @@ spec: test_types: - tcp_download runtime: 2 + debug: true diff --git a/tests/test_crs/valid_flent_resources.yaml b/tests/test_crs/valid_flent_resources.yaml index e0ec9ba8f..bef6ba7ab 100644 --- a/tests/test_crs/valid_flent_resources.yaml +++ b/tests/test_crs/valid_flent_resources.yaml @@ -1,10 +1,16 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: flent-benchmark - namespace: my-ripsaw + name: flent-resources-benchmark + namespace: benchmark-operator spec: clustername: myk8scluster + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-flent @@ -30,3 +36,4 @@ spec: test_types: - tcp_download runtime: 2 + debug: true diff --git a/tests/test_crs/valid_fs_drift.yaml b/tests/test_crs/valid_fs_drift.yaml index 84d237fcc..c5aa35b3d 100644 --- a/tests/test_crs/valid_fs_drift.yaml +++ b/tests/test_crs/valid_fs_drift.yaml @@ -1,15 +1,18 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: fs-drift-benchmark + namespace: benchmark-operator spec: - test_user: homer_simpson - # to separate this test run from everyone else's - clustername: test_ci + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: "http://es-instance.com:9200" + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml # where elastic search is running elasticsearch: - url: ES_SERVER + url: "http://es-instance.com:9200" index_name: ripsaw-fs-drift metadata: collection: true @@ -20,4 +23,6 @@ spec: threads: 5 max_file_size_kb: 4 max_files: 1000 - duration: 15 + duration: 240 + response_times: Y + debug: true diff --git a/tests/test_crs/valid_fs_drift_hostpath.yaml b/tests/test_crs/valid_fs_drift_hostpath.yaml index 650cbce58..66267425d 100644 --- a/tests/test_crs/valid_fs_drift_hostpath.yaml +++ b/tests/test_crs/valid_fs_drift_hostpath.yaml @@ -1,13 +1,15 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: fs-drift-hostpath-benchmark + namespace: benchmark-operator spec: - test_user: homer_simpson - # to separate this test run from everyone else's - clustername: test_ci - # where elastic search is running + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-fs-drift @@ -21,4 +23,6 @@ spec: threads: 5 max_file_size_kb: 4 max_files: 1000 - duration: 15 + duration: 240 + response_times: true + debug: true diff --git a/tests/test_crs/valid_hammerdb.yaml b/tests/test_crs/valid_hammerdb.yaml index 1c975cee6..6c3d1b7f3 100644 --- a/tests/test_crs/valid_hammerdb.yaml +++ b/tests/test_crs/valid_hammerdb.yaml @@ -2,8 +2,14 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: hammerdb - namespace: my-ripsaw + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-hammerdb @@ -12,7 +18,7 @@ spec: workload: name: "hammerdb" args: - # image: "quay.io/mkarg/hammerdb:latest" + # image: "quay.io/test/hammerdb:latest" db_type: "mssql" timed_test: true test_type: "tpc-c" @@ -61,3 +67,4 @@ spec: db_postgresql_dritasnap: "false" db_postgresql_oracompat: "false" db_postgresql_storedprocs: "false" + debug: true diff --git a/tests/test_crs/valid_image_pull.yaml b/tests/test_crs/valid_image_pull.yaml new file mode 100644 index 000000000..c39cf5e63 --- /dev/null +++ b/tests/test_crs/valid_image_pull.yaml @@ -0,0 +1,18 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: image-pull + namespace: benchmark-operator +spec: + elasticsearch: + url: ES_SERVER + index_name: "image-pull" + workload: + name: image_pull + args: + pod_count: 2 + timeout: 100 + retries: 1 + image_list: + - docker://quay.io/cloud-bulldozer/backpack + - docker://quay.io/cloud-bulldozer/fio diff --git a/tests/test_crs/valid_iperf3.yaml b/tests/test_crs/valid_iperf3.yaml index 5ac1faac3..d0ebe5b91 100644 --- a/tests/test_crs/valid_iperf3.yaml +++ b/tests/test_crs/valid_iperf3.yaml @@ -1,9 +1,15 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: iperf3-benchmark + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER metadata: diff --git a/tests/test_crs/valid_kube-burner.yaml b/tests/test_crs/valid_kube-burner.yaml index d6d19172a..476067b96 100644 --- a/tests/test_crs/valid_kube-burner.yaml +++ b/tests/test_crs/valid_kube-burner.yaml @@ -3,12 +3,12 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: kube-burner-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: elasticsearch: url: ES_SERVER metadata: - collection: true + collection: false prometheus: es_url: ES_SERVER prom_token: PROMETHEUS_TOKEN @@ -32,6 +32,14 @@ spec: error_on_verify: true step: 30s metrics_profile: METRICS_PROFILE + app: django + source_strat_env: PIP_INDEX_URL + source_strat_from: python + source_strat_from_version: latest + post_commit_script: "./manage.py test" + build_image_stream: django-psql-example + git_url: https://github.com/sclorg/django-ex.git + build_image: image-registry.openshift-image-registry.svc:5000/svt-django/django-psql-example node_selector: key: node-role.kubernetes.io/worker value: diff --git a/tests/test_crs/valid_log_generator.yaml b/tests/test_crs/valid_log_generator.yaml new file mode 100644 index 000000000..72b926db2 --- /dev/null +++ b/tests/test_crs/valid_log_generator.yaml @@ -0,0 +1,18 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: log-generator + namespace: benchmark-operator +spec: + elasticsearch: + url: ES_SERVER + index_name: log-generator + workload: + name: log_generator + args: + pod_count: 1 + size: 512 + messages_per_second: 1 + duration: 1 + es_url: ES_SERVER + timeout: 600 diff --git a/tests/test_crs/valid_pgbench.yaml b/tests/test_crs/valid_pgbench.yaml index 6862b19d7..908a4e58b 100644 --- a/tests/test_crs/valid_pgbench.yaml +++ b/tests/test_crs/valid_pgbench.yaml @@ -2,8 +2,14 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: pgbench-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-pgbench @@ -28,3 +34,4 @@ spec: user: ci password: ci db_name: cidb + debug: true diff --git a/tests/test_crs/valid_scale_down.yaml b/tests/test_crs/valid_scale_down.yaml index fb24e9666..539d45906 100644 --- a/tests/test_crs/valid_scale_down.yaml +++ b/tests/test_crs/valid_scale_down.yaml @@ -1,9 +1,17 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: scale - namespace: my-ripsaw + name: scale-down + namespace: benchmark-operator spec: + metadata: + collection: true + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: openshift-cluster-timings @@ -11,6 +19,5 @@ spec: name: scale_openshift args: scale: 0 - serviceaccount: scaler poll_interval: 2 - + debug: true diff --git a/tests/test_crs/valid_scale_up.yaml b/tests/test_crs/valid_scale_up.yaml index f3ce103a4..302437fd8 100644 --- a/tests/test_crs/valid_scale_up.yaml +++ b/tests/test_crs/valid_scale_up.yaml @@ -1,9 +1,17 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: scale - namespace: my-ripsaw + name: scale-up + namespace: benchmark-operator spec: + metadata: + collection: true + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: openshift-cluster-timings @@ -11,6 +19,5 @@ spec: name: scale_openshift args: scale: 1 - serviceaccount: scaler poll_interval: 2 - + debug: true diff --git a/tests/test_crs/valid_servicemesh.yaml b/tests/test_crs/valid_servicemesh.yaml index 7f7b93147..c41fab396 100644 --- a/tests/test_crs/valid_servicemesh.yaml +++ b/tests/test_crs/valid_servicemesh.yaml @@ -1,8 +1,8 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: servicemesh-benchmark + namespace: benchmark-operator spec: workload: name: servicemesh diff --git a/tests/test_crs/valid_smallfile.yaml b/tests/test_crs/valid_smallfile.yaml index f15d3bbea..f37b85731 100644 --- a/tests/test_crs/valid_smallfile.yaml +++ b/tests/test_crs/valid_smallfile.yaml @@ -1,12 +1,18 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: smallfile-benchmark + namespace: benchmark-operator spec: test_user: homer_simpson # to separate this test run from everyone else's clustername: test_ci + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml # where elastic search is running elasticsearch: url: ES_SERVER @@ -17,7 +23,24 @@ spec: name: smallfile args: clients: 2 - operation: ["create", "read", "append", "delete"] + operation: ["create", "read", "delete", "cleanup"] threads: 1 - file_size: 0 + file_size: 2 + record_size: 1 + xattr_size: 50 + pause: 0 + auto_pause: y + stonewall: n + finish: y + prefix: abc + hash_into_dirs: true + same_dir: y + incompressible: y + verify_read: y + xattr_count: 10 + fsync: false + files_per_dir: 2000 + dirs_per_dir: 4 files: 100000 + cleanup_delay_usec_per_file: 20 + debug: true diff --git a/tests/test_crs/valid_smallfile_hostpath.yaml b/tests/test_crs/valid_smallfile_hostpath.yaml index 51604ab4b..daf1c273b 100644 --- a/tests/test_crs/valid_smallfile_hostpath.yaml +++ b/tests/test_crs/valid_smallfile_hostpath.yaml @@ -2,12 +2,18 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: smallfile-hostpath-benchmark + namespace: benchmark-operator spec: test_user: homer_simpson # to separate this test run from everyone else's clustername: test_ci + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-smallfile @@ -22,3 +28,4 @@ spec: threads: 1 file_size: 0 files: 100000 + debug: true diff --git a/tests/test_crs/valid_stressng.yaml b/tests/test_crs/valid_stressng.yaml index be0fc3468..220004b18 100644 --- a/tests/test_crs/valid_stressng.yaml +++ b/tests/test_crs/valid_stressng.yaml @@ -2,8 +2,14 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: stressng - namespace: my-ripsaw + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-stressng @@ -25,4 +31,4 @@ spec: vm_bytes: "128M" # mem stressor options mem_stressors: "1" - + debug: true diff --git a/tests/test_crs/valid_sysbench.yaml b/tests/test_crs/valid_sysbench.yaml index 28c4ff235..9bd1aca1d 100644 --- a/tests/test_crs/valid_sysbench.yaml +++ b/tests/test_crs/valid_sysbench.yaml @@ -1,9 +1,15 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: sysbench-benchmark + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER metadata: diff --git a/tests/test_crs/valid_uperf.yaml b/tests/test_crs/valid_uperf.yaml index fcc5061f2..a0ff00c5e 100644 --- a/tests/test_crs/valid_uperf.yaml +++ b/tests/test_crs/valid_uperf.yaml @@ -1,9 +1,15 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: uperf + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-uperf @@ -36,3 +42,4 @@ spec: - 1 - 2 runtime: 2 + debug: true diff --git a/tests/test_crs/valid_uperf_networkpolicy.yaml b/tests/test_crs/valid_uperf_networkpolicy.yaml index a3a95317e..54cb87e1f 100644 --- a/tests/test_crs/valid_uperf_networkpolicy.yaml +++ b/tests/test_crs/valid_uperf_networkpolicy.yaml @@ -1,9 +1,15 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: uperf-networkpolicy + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-uperf @@ -37,3 +43,4 @@ spec: - 1 - 2 runtime: 2 + debug: true diff --git a/tests/test_crs/valid_uperf_resources.yaml b/tests/test_crs/valid_uperf_resources.yaml index 435d80e95..da3f60392 100644 --- a/tests/test_crs/valid_uperf_resources.yaml +++ b/tests/test_crs/valid_uperf_resources.yaml @@ -1,9 +1,15 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: uperf-resources + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-uperf @@ -43,3 +49,4 @@ spec: - 1 - 2 runtime: 2 + debug: true diff --git a/tests/test_crs/valid_uperf_scale.yaml b/tests/test_crs/valid_uperf_scale.yaml new file mode 100644 index 000000000..76473c9f1 --- /dev/null +++ b/tests/test_crs/valid_uperf_scale.yaml @@ -0,0 +1,46 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-scale + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf-scale + args: + hostnetwork: false + serviceip: false + multus: + enabled: false + samples: 2 + kind: pod + test_types: + - stream + - rr + - bidirec + protos: + - tcp + - udp + sizes: + - 1024 + - [1024, 512] + nthrs: + - 1 + - 2 + runtime: 2 + colocate: false + density_range: [1, 1] + node_range: [1, 5] + step_size: add1 + debug: true diff --git a/tests/test_crs/valid_uperf_scale_networkpolicy.yaml b/tests/test_crs/valid_uperf_scale_networkpolicy.yaml new file mode 100644 index 000000000..5538e6eec --- /dev/null +++ b/tests/test_crs/valid_uperf_scale_networkpolicy.yaml @@ -0,0 +1,47 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-scale-networkpolicy + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf-scale + args: + hostnetwork: false + serviceip: false + networkpolicy: true + multus: + enabled: false + samples: 2 + kind: pod + test_types: + - stream + - rr + - bidirec + protos: + - tcp + - udp + sizes: + - 1024 + - 512 + nthrs: + - 1 + - 2 + runtime: 2 + colocate: false + density_range: [1, 1] + node_range: [1, 5] + step_size: add1 + debug: true diff --git a/tests/test_crs/valid_uperf_scale_resources.yaml b/tests/test_crs/valid_uperf_scale_resources.yaml new file mode 100644 index 000000000..92574fd3d --- /dev/null +++ b/tests/test_crs/valid_uperf_scale_resources.yaml @@ -0,0 +1,53 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-scale-resources + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf-scale + args: + client_resources: + requests: + cpu: 100m + memory: 100Mi + server_resources: + requests: + cpu: 100m + memory: 100Mi + hostnetwork: true + serviceip: false + multus: + enabled: false + samples: 2 + kind: pod + test_types: + - stream + - rr + protos: + - tcp + - udp + sizes: + - 1024 + - 512 + nthrs: + - 1 + - 2 + runtime: 2 + colocate: false + density_range: [1, 1] + node_range: [1, 1] + step_size: add1 + debug: true diff --git a/tests/test_crs/valid_uperf_scale_serviceip_nodeport.yaml b/tests/test_crs/valid_uperf_scale_serviceip_nodeport.yaml new file mode 100644 index 000000000..52b1f3f7e --- /dev/null +++ b/tests/test_crs/valid_uperf_scale_serviceip_nodeport.yaml @@ -0,0 +1,45 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-scale-serviceip-nodeport + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf-scale + args: + hostnetwork: false + serviceip: true + servicetype: "nodeport" + multus: + enabled: false + samples: 2 + kind: pod + test_types: + - stream + - rr + protos: + - tcp + - udp + sizes: + - 1024 + - 512 + nthrs: + - 1 + runtime: 2 + colocate: false + density_range: [1, 1] + node_range: [1, 5] + step_size: add1 + debug: true diff --git a/tests/test_crs/valid_uperf_scale_servieip.yaml b/tests/test_crs/valid_uperf_scale_servieip.yaml new file mode 100644 index 000000000..f3c6b5e7c --- /dev/null +++ b/tests/test_crs/valid_uperf_scale_servieip.yaml @@ -0,0 +1,45 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-scale-serviceip + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf-scale + args: + hostnetwork: false + serviceip: true + multus: + enabled: false + samples: 1 + kind: pod + test_types: + - stream + - rr + protos: + - tcp + - udp + sizes: + - 1024 + - 512 + nthrs: + - 1 + - 2 + runtime: 2 + colocate: false + density_range: [1, 1] + node_range: [1, 5] + step_size: add1 + debug: true diff --git a/tests/test_crs/valid_uperf_serviceip.yaml b/tests/test_crs/valid_uperf_serviceip.yaml index e03d5ea39..9837bbe7c 100644 --- a/tests/test_crs/valid_uperf_serviceip.yaml +++ b/tests/test_crs/valid_uperf_serviceip.yaml @@ -1,9 +1,15 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: - name: example-benchmark - namespace: my-ripsaw + name: uperf-serviceip + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-uperf @@ -35,3 +41,4 @@ spec: - 1 - 2 runtime: 2 + debug: true diff --git a/tests/test_crs/valid_uperf_serviceip_nodeport.yaml b/tests/test_crs/valid_uperf_serviceip_nodeport.yaml new file mode 100644 index 000000000..9ddb1fe73 --- /dev/null +++ b/tests/test_crs/valid_uperf_serviceip_nodeport.yaml @@ -0,0 +1,44 @@ +apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 +kind: Benchmark +metadata: + name: uperf-serviceip-nodeport + namespace: benchmark-operator +spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml + elasticsearch: + url: ES_SERVER + index_name: ripsaw-uperf + metadata: + collection: true + cleanup: false + workload: + name: uperf + args: + hostnetwork: false + serviceip: true + servicetype: "nodeport" + pin: false + pin_server: "node-0" + pin_client: "node-1" + multus: + enabled: false + samples: 2 + pair: 2 + test_types: + - stream + - rr + protos: + - tcp + - udp + sizes: + - 1024 + - 512 + nthrs: + - 1 + runtime: 2 + debug: true diff --git a/tests/test_crs/valid_vegeta.yaml b/tests/test_crs/valid_vegeta.yaml index ff69405dd..4d9e79f5e 100644 --- a/tests/test_crs/valid_vegeta.yaml +++ b/tests/test_crs/valid_vegeta.yaml @@ -3,8 +3,14 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: vegeta-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-vegeta @@ -13,13 +19,13 @@ spec: workload: name: vegeta args: - clients: 2 + clients: 3 targets: - name: 2w-ka urls: - GET https://1.1.1.1 - GET http://1.1.1.1 - samples: 1 + samples: 3 workers: 2 duration: 5 keepalive: true @@ -28,3 +34,4 @@ spec: - GET https://1.1.1.1 workers: 2 duration: 5 + debug: true diff --git a/tests/test_crs/valid_vegeta_hostnetwork.yaml b/tests/test_crs/valid_vegeta_hostnetwork.yaml index 50f4fe0f2..6d3f1ecb8 100644 --- a/tests/test_crs/valid_vegeta_hostnetwork.yaml +++ b/tests/test_crs/valid_vegeta_hostnetwork.yaml @@ -3,8 +3,14 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: vegeta-benchmark-hostnetwork - namespace: my-ripsaw + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-vegeta @@ -24,3 +30,4 @@ spec: workers: 2 duration: 5 keepalive: true + debug: true diff --git a/tests/test_crs/valid_ycsb-mongo.yaml b/tests/test_crs/valid_ycsb-mongo.yaml index 4fd9f4b38..3e8a56cda 100644 --- a/tests/test_crs/valid_ycsb-mongo.yaml +++ b/tests/test_crs/valid_ycsb-mongo.yaml @@ -2,8 +2,14 @@ apiVersion: ripsaw.cloudbulldozer.io/v1alpha1 kind: Benchmark metadata: name: ycsb-mongo-benchmark - namespace: my-ripsaw + namespace: benchmark-operator spec: + system_metrics: + collection: true + prom_url: https://prometheus-k8s.openshift-monitoring.svc.cluster.local:9091 + es_url: ES_SERVER + prom_token: PROMETHEUS_TOKEN + metrics_profile: node-metrics.yml elasticsearch: url: ES_SERVER index_name: ripsaw-ycsb @@ -21,3 +27,4 @@ spec: - workloada options_load: '-p mongodb.url="mongodb://mongo-0.mongo/ycsb?"' #passed as is to ycsb when loading database options_run: '-p mongodb.url="mongodb://mongo-0.mongo/ycsb?" -threads 10 -target 100' + debug: true diff --git a/tests/test_fiod.sh b/tests/test_fiod.sh index f063e4450..78fb7d8af 100755 --- a/tests/test_fiod.sh +++ b/tests/test_fiod.sh @@ -10,7 +10,6 @@ function finish { fi echo "Cleaning up fio" - kubectl delete -f resources/kernel-cache-drop-clusterrole.yaml --ignore-not-found wait_clean } @@ -19,20 +18,23 @@ trap finish EXIT function functional_test_fio { wait_clean - apply_operator - kubectl apply -f resources/kernel-cache-drop-clusterrole.yaml test_name=$1 cr=$2 + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr echo "Performing: ${test_name}" - kubectl apply -f ${cr} - long_uuid=$(get_uuid 20) + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} pod_count "app=fio-benchmark-$uuid" 2 300 - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized -l app=fio-benchmark-$uuid pods --timeout=300s" "300s" fio_pod=$(get_pod "app=fiod-client-$uuid" 300) - wait_for "kubectl wait --for=condition=Initialized pods/$fio_pod -n my-ripsaw --timeout=500s" "500s" $fio_pod - wait_for "kubectl wait --for=condition=complete -l app=fiod-client-$uuid jobs -n my-ripsaw --timeout=700s" "700s" $fio_pod + check_benchmark_for_desired_state $benchmark_name Complete 1200s + kubectl -n benchmark-operator logs $fio_pod > /tmp/$fio_pod.log + + + indexes="ripsaw-fio-results ripsaw-fio-log ripsaw-fio-analyzed-result" if check_es "${long_uuid}" "${indexes}" @@ -40,13 +42,26 @@ function functional_test_fio { echo "${test_name} test: Success" else echo "Failed to find data for ${test_name} in ES" - kubectl logs "$fio_pod" -n my-ripsaw + kubectl logs "$fio_pod" -n benchmark-operator exit 1 fi + delete_benchmark $cr } figlet $(basename $0) kubectl label nodes -l node-role.kubernetes.io/worker= kernel-cache-dropper=yes --overwrite functional_test_fio "Fio distributed" tests/test_crs/valid_fiod.yaml +openshift_storage_present=$(oc get namespace | awk '/openshift-storage/' | wc -l) +if [ $openshift_storage_present -gt 0 ] ; then + oc patch OCSInitialization ocsinit -n openshift-storage --type json --patch \ + '[{ "op": "replace", "path": "/spec/enableCephTools", "value": true }]' + drop_cache_pods=$(oc -n openshift-storage get pod | awk '/drop/' | awk '/unning/' | wc -l) + if [ $drop_cache_pods -eq 0 ] ; then + oc create -f roles/ceph_osd_cache_drop/rook_ceph_drop_cache_pod.yaml + kubectl wait --for=condition=Initialized pods/rook-ceph-osd-cache-drop -n openshift-storage --timeout=100s + fi + sleep 5 + functional_test_fio "Fio cache drop" tests/test_crs/valid_fiod_ocs_cache_drop.yaml +fi functional_test_fio "Fio distributed - bsrange" tests/test_crs/valid_fiod_bsrange.yaml functional_test_fio "Fio hostpath distributed" tests/test_crs/valid_fiod_hostpath.yaml diff --git a/tests/test_flent.sh b/tests/test_flent.sh index 0dda96d87..84d059cfe 100755 --- a/tests/test_flent.sh +++ b/tests/test_flent.sh @@ -17,21 +17,17 @@ trap error ERR trap finish EXIT function functional_test_flent { - wait_clean - apply_operator test_name=$1 cr=$2 + delete_benchmark $cr echo "Performing: ${test_name}" - kubectl apply -f ${cr} - long_uuid=$(get_uuid 20) + benchmark_name=$(get_benchmark_name $cr) + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} - pod_count "type=flent-bench-server-$uuid" 1 900 - flent_server_pod=$(get_pod "app=flent-bench-server-0-$uuid" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized -l app=flent-bench-server-0-$uuid pods --timeout=300s" "300s" $flent_server_pod - flent_client_pod=$(get_pod "app=flent-bench-client-$uuid" 900) - wait_for "kubectl wait -n my-ripsaw --for=condition=Initialized pods/$flent_client_pod --timeout=500s" "500s" $flent_client_pod - wait_for "kubectl wait -n my-ripsaw --for=condition=complete -l app=flent-bench-client-$uuid jobs --timeout=500s" "500s" $flent_client_pod + check_benchmark_for_desired_state $benchmark_name Complete 800s index="ripsaw-flent-results" if check_es "${long_uuid}" "${index}" @@ -39,9 +35,10 @@ function functional_test_flent { echo "${test_name} test: Success" else echo "Failed to find data for ${test_name} in ES" - kubectl logs "$flent_client_pod" -n my-ripsaw + kubectl logs "$flent_client_pod" -n benchmark-operator exit 1 fi + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_fs_drift.sh b/tests/test_fs_drift.sh index 6ef75b81b..e62af6af7 100755 --- a/tests/test_fs_drift.sh +++ b/tests/test_fs_drift.sh @@ -18,41 +18,29 @@ trap error ERR trap finish EXIT function functional_test_fs_drift { - wait_clean - apply_operator test_name=$1 cr=$2 + delete_benchmark $cr echo "Performing: ${test_name}" - kubectl apply -f ${cr} - long_uuid=$(get_uuid 20) + benchmark_name=$(get_benchmark_name $cr) + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} - - count=0 - while [[ $count -lt 24 ]]; do - if [[ `kubectl get pods -l app=fs-drift-benchmark-$uuid --namespace my-ripsaw -o name | cut -d/ -f2 | grep client` ]]; then - fsdrift_pod=$(kubectl get pods -l app=fs-drift-benchmark-$uuid --namespace my-ripsaw -o name | cut -d/ -f2 | grep client) - count=30 - fi - if [[ $count -ne 30 ]]; then - sleep 5 - count=$((count + 1)) - fi - done - echo fsdrift_pod $fs_drift_pod - wait_for "kubectl wait --for=condition=Initialized pods/$fsdrift_pod -n my-ripsaw --timeout=500s" "500s" $fsdrift_pod - wait_for "kubectl wait --for=condition=complete -l app=fs-drift-benchmark-$uuid jobs -n my-ripsaw --timeout=100s" "200s" $fsdrift_pod - + fsdrift_pod=$(get_pod "app=fs-drift-benchmark-$uuid" 300) + check_benchmark_for_desired_state $benchmark_name Complete 600s indexes="ripsaw-fs-drift-results ripsaw-fs-drift-rsptimes ripsaw-fs-drift-rates-over-time" if check_es "${long_uuid}" "${indexes}" then echo "${test_name} test: Success" else echo "Failed to find data for ${test_name} in ES" - kubectl logs "$fsdrift_pod" -n my-ripsaw + kubectl logs "$fsdrift_pod" -n benchmark-operator exit 1 fi + delete_benchmark $cr } figlet $(basename $0) functional_test_fs_drift "fs-drift" tests/test_crs/valid_fs_drift.yaml -functional_test_fs_drift "fs-drift hostpath" tests/test_crs/valid_fs_drift_hostpath.yaml +functional_test_fs_drift "fs-drift-hostpath" tests/test_crs/valid_fs_drift_hostpath.yaml diff --git a/tests/test_hammerdb.sh b/tests/test_hammerdb.sh index 7da818483..78c7aa145 100755 --- a/tests/test_hammerdb.sh +++ b/tests/test_hammerdb.sh @@ -13,24 +13,24 @@ function initdb_pod { function finish { echo "Cleaning up hammerdb" kubectl delete -f tests/mssql.yaml - kubectl delete -f tests/test_crs/valid_hammerdb.yaml - delete_operator + wait_clean } trap finish EXIT function functional_test_hammerdb { - wait_clean - apply_operator initdb_pod - kubectl apply -f tests/test_crs/valid_hammerdb.yaml - long_uuid=$(get_uuid 20) + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + cr=tests/test_crs/valid_hammerdb.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} # Wait for the workload pod to run the actual workload hammerdb_workload_pod=$(get_pod "app=hammerdb_workload-$uuid" 300) - kubectl wait --for=condition=Initialized "pods/$hammerdb_workload_pod" --namespace my-ripsaw --timeout=400s - kubectl wait --for=condition=complete -l app=hammerdb_workload-$uuid --namespace my-ripsaw jobs --timeout=500s + check_benchmark_for_desired_state $benchmark_name Complete 900s index="ripsaw-hammerdb-results" if check_es "${long_uuid}" "${index}" @@ -38,9 +38,10 @@ function functional_test_hammerdb { echo "Hammerdb test: Success" else echo "Failed to find data for HammerDB test in ES" - kubectl logs "$hammerdb_workload_pod" --namespace my-ripsaw + kubectl logs "$hammerdb_workload_pod" --namespace benchmark-operator exit 1 fi + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_image_pull.sh b/tests/test_image_pull.sh new file mode 100755 index 000000000..07a47309b --- /dev/null +++ b/tests/test_image_pull.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up Image Pull Test" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_image_pull { + test_name=$1 + cr=$2 + delete_benchmark $cr + echo "Performing: ${test_name}" + kubectl apply -f ${cr} + benchmark_name=$(get_benchmark_name $cr) + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + check_benchmark_for_desired_state $benchmark_name Complete 500s + + index="image-pull-results" + if check_es "${long_uuid}" "${index}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_image_pull "Image Pull" tests/test_crs/valid_image_pull.yaml diff --git a/tests/test_iperf3.sh b/tests/test_iperf3.sh old mode 100644 new mode 100755 index accdb960d..18efecbe0 --- a/tests/test_iperf3.sh +++ b/tests/test_iperf3.sh @@ -10,30 +10,31 @@ function finish { fi echo "Cleaning up iperf3" - kubectl delete -f tests/test_crs/valid_iperf3.yaml - delete_operator + wait_clean } trap error ERR trap finish EXIT function functional_test_iperf { - wait_clean - apply_operator echo "Performing iperf3: ${1}" - sed -e "s/hostnetwork:.*/${1}/g" tests/test_crs/valid_iperf3.yaml | kubectl apply -f - - long_uuid=$(get_uuid 20) + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + cr=tests/test_crs/valid_iperf3.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" -e "s/hostnetwork:.*/${1}/g" $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} iperf_server_pod=$(get_pod "app=iperf3-bench-server-$uuid" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized -l app=iperf3-bench-server-$uuid pods --timeout=300s" "300s" $iperf_server_pod + wait_for "kubectl -n benchmark-operator wait --for=condition=Initialized -l app=iperf3-bench-server-$uuid pods --timeout=300s" "300s" $iperf_server_pod iperf_client_pod=$(get_pod "app=iperf3-bench-client-$uuid" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized pods/$iperf_client_pod --timeout=500s" "500s" $iperf_client_pod - wait_for "kubectl -n my-ripsaw wait --for=condition=complete -l app=iperf3-bench-client-$uuid jobs --timeout=100s" "100s" $iperf_client_pod + check_benchmark_for_desired_state $benchmark_name Complete 600s sleep 5 # ensuring that iperf actually ran and we can access metrics - kubectl logs "$iperf_client_pod" --namespace my-ripsaw | grep "iperf Done." + kubectl logs "$iperf_client_pod" --namespace benchmark-operator | grep "iperf Done." echo "iperf ${1}: Success" + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_kubeburner.sh b/tests/test_kubeburner.sh deleted file mode 100755 index 536f36576..000000000 --- a/tests/test_kubeburner.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -set -xeEo pipefail - -source tests/common.sh - -function finish { - if [ $? -eq 1 ] && [ $ERRORED != "true" ] - then - error - fi - - [[ $check_logs == 1 ]] && kubectl logs -l app=kube-burner-benchmark-$uuid -n my-ripsaw - echo "Cleaning up kube-burner" - kubectl delete -f resources/kube-burner-role.yml --ignore-not-found - kubectl delete ns -l kube-burner-uuid=${long_uuid} - wait_clean -} - - -trap error ERR -trap finish EXIT - -function functional_test_kubeburner { - workload_name=$1 - metrics_profile=$2 - token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) - cr=tests/test_crs/valid_kube-burner.yaml - check_logs=0 - wait_clean - apply_operator - kubectl apply -f resources/kube-burner-role.yml - echo "Performing kube-burner: ${workload_name}" - sed -e "s/WORKLOAD/${workload_name}/g" -e "s/PROMETHEUS_TOKEN/${token}/g" -e "s/METRICS_PROFILE/${metrics_profile}/g" ${cr} | kubectl apply -f - - long_uuid=$(get_uuid 20) - uuid=${long_uuid:0:8} - - pod_count "app=kube-burner-benchmark-$uuid" 1 900 - check_logs=1 - wait_for "kubectl wait -n my-ripsaw --for=condition=complete -l app=kube-burner-benchmark-$uuid jobs --timeout=500s" "500s" - - index="ripsaw-kube-burner" - if check_es "${long_uuid}" "${index}" - then - echo "kube-burner ${workload_name}: Success" - else - echo "Failed to find data for kube-burner ${workload_name} in ES" - exit 1 - fi - kubectl delete ns -l kube-burner-uuid=${long_uuid} -} - -figlet $(basename $0) -functional_test_kubeburner cluster-density metrics-aggregated.yaml -functional_test_kubeburner node-density metrics.yaml -functional_test_kubeburner node-density-heavy metrics.yaml -functional_test_kubeburner max-namespaces metrics-aggregated.yaml -functional_test_kubeburner max-services metrics-aggregated.yaml diff --git a/tests/test_list b/tests/test_list old mode 100644 new mode 100755 index 3825b39de..10491480b --- a/tests/test_list +++ b/tests/test_list @@ -1,4 +1,3 @@ -test_backpack.sh test_ycsb.sh test_fiod.sh test_fs-drift.sh @@ -7,10 +6,9 @@ test_pgbench.sh test_iperf3.sh test_sysbench.sh test_uperf.sh -test_byowl.sh test_hammerdb.sh test_vegeta.sh test_scale_openshift.sh test_stressng.sh -test_kubeburner.sh test_flent.sh +test_log_generator.sh diff --git a/tests/test_log_generator.sh b/tests/test_log_generator.sh new file mode 100755 index 000000000..3d394f981 --- /dev/null +++ b/tests/test_log_generator.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up Log Generator Test" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_log_generator { + test_name=$1 + cr=$2 + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + echo "Performing: ${test_name}" + kubectl apply -f ${cr} + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + + log_gen_pod=$(get_pod "app=log-generator-$uuid" 300) + check_benchmark_for_desired_state $benchmark_name Complete 800s + + index="log-generator-results" + if check_es "${long_uuid}" "${index}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + kubectl logs "$log_gen_pod" -n benchmark-operator + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_log_generator "Log Generator" tests/test_crs/valid_log_generator.yaml diff --git a/tests/test_pgbench.sh b/tests/test_pgbench.sh old mode 100644 new mode 100755 index 7c589ca63..d8d33a249 --- a/tests/test_pgbench.sh +++ b/tests/test_pgbench.sh @@ -18,15 +18,13 @@ trap finish EXIT # Note we don't test persistent storage here function functional_test_pgbench { - wait_clean - apply_operator # stand up postgres deployment cat << EOF | kubectl apply -f - apiVersion: v1 kind: ConfigMap metadata: name: postgres-config - namespace: my-ripsaw + namespace: benchmark-operator labels: app: postgres data: @@ -40,7 +38,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: postgres - namespace: my-ripsaw + namespace: benchmark-operator spec: selector: matchLabels: @@ -70,18 +68,19 @@ EOF counter=0 until [[ $postgres_ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ || $counter -eq 10 ]]; do let counter+=1 - postgres_ip=$(kubectl get pod -n my-ripsaw $postgres_pod --template={{.status.podIP}}) + postgres_ip=$(kubectl get pod -n benchmark-operator $postgres_pod --template={{.status.podIP}}) sleep 2 done # deploy the test CR with the postgres pod IP - sed s/host:/host:\ ${postgres_ip}/ tests/test_crs/valid_pgbench.yaml | kubectl apply -f - - long_uuid=$(get_uuid 20) + cr=tests/test_crs/valid_pgbench.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed s/host:/host:\ ${postgres_ip}/ $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} pgbench_pod=$(get_pod "app=pgbench-client-$uuid" 300) - wait_for "kubectl wait --for=condition=Initialized pods/$pgbench_pod -n my-ripsaw --timeout=360s" "360s" $pgbench_pod - wait_for "kubectl wait --for=condition=Ready pods/$pgbench_pod -n my-ripsaw --timeout=60s" "60s" $pgbench_pod - wait_for "kubectl wait --for=condition=Complete jobs -l app=pgbench-client-$uuid -n my-ripsaw --timeout=300s" "300s" $pgbench_pod + check_benchmark_for_desired_state $benchmark_name Complete 500s index="ripsaw-pgbench-summary ripsaw-pgbench-raw" if check_es "${long_uuid}" "${index}" @@ -89,9 +88,10 @@ EOF echo "pgbench test: Success" else echo "Failed to find data for PGBench in ES" - kubectl logs -n my-ripsaw $pgbench_pod + kubectl logs -n benchmark-operator $pgbench_pod exit 1 fi + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_scale_openshift.sh b/tests/test_scale_openshift.sh index e0b26b0d8..f30bff4c3 100755 --- a/tests/test_scale_openshift.sh +++ b/tests/test_scale_openshift.sh @@ -18,33 +18,28 @@ trap error ERR trap finish EXIT function functional_test_scale_openshift { - wait_clean - apply_operator test_name=$1 cr=$2 - - # Apply scale role and service account - kubectl apply -f resources/scale_role.yaml - + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) echo "Performing: ${test_name}" - kubectl apply -f ${cr} - long_uuid=$(get_uuid 20) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} scale_pod=$(get_pod "app=scale-$uuid" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized -l app=scale-$uuid pods --timeout=300s" "300s" $scale_pod - wait_for "kubectl wait -n my-ripsaw --for=condition=complete -l app=scale-$uuid jobs --timeout=500s" "500s" $scale_pod - + check_benchmark_for_desired_state $benchmark_name Complete 500s index="openshift-cluster-timings" if check_es "${long_uuid}" "${index}" then echo "${test_name} test: Success" else echo "Failed to find data for ${test_name} in ES" - kubectl logs "$scale_pod" -n my-ripsaw + kubectl logs "$scale_pod" -n benchmark-operator exit 1 fi - kubectl delete -f ${cr} + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_servicemesh.sh b/tests/test_servicemesh.sh index 8b878fe7d..6f5bf18d0 100755 --- a/tests/test_servicemesh.sh +++ b/tests/test_servicemesh.sh @@ -14,17 +14,18 @@ function finish { uuid=${long_uuid:0:8} kubectl delete --ignore-not-found=true namespace example-benchmark-controlplane-$uuid kubectl delete --ignore-not-found=true namespace example-benchmark-workload-$uuid - kubectl delete -f tests/test_crs/valid_servicemesh.yaml - delete_operator + wait_clean } trap error ERR trap finish EXIT function functional_test_servicemesh { - apply_operator - kubectl apply -f tests/test_crs/valid_servicemesh.yaml - long_uuid=$(get_uuid 20) + cr=tests/test_crs/valid_servicemesh.yaml + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr + kubectl apply -f $cr + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} # wait until the job appears @@ -32,24 +33,25 @@ function functional_test_servicemesh { max_count=60 while [[ $count -lt $max_count ]] do - if kubectl get --namespace my-ripsaw job example-benchmark-$uuid; then + if kubectl get --namespace benchmark-operator job example-benchmark-$uuid; then break fi sleep 15 count=$((count + 1)) done - wait_for "kubectl -n my-ripsaw wait --for=condition=complete jobs --timeout=300s example-benchmark-$uuid" "300s" + wait_for "kubectl -n benchmark-operator wait --for=condition=complete jobs --timeout=300s example-benchmark-$uuid" "300s" job_pod=$(get_pod app=example-benchmark-$uuid 30) # ensuring that uperf actually ran and we can access metrics - INFO=$(kubectl logs $job_pod --namespace my-ripsaw | jq .info) + INFO=$(kubectl logs $job_pod --namespace benchmark-operator | jq .info) if [ -n "$INFO" ]; then echo "Successful: $INFO" else echo "Failed to verify benchmark results" exit 1 fi + delete_benchmark $cr } functional_test_servicemesh diff --git a/tests/test_smallfile.sh b/tests/test_smallfile.sh index 30852b199..b921adfb2 100755 --- a/tests/test_smallfile.sh +++ b/tests/test_smallfile.sh @@ -10,7 +10,6 @@ function finish { fi echo "Cleaning up smallfile" - kubectl delete -f resources/kernel-cache-drop-clusterrole.yaml --ignore-not-found wait_clean } @@ -19,30 +18,16 @@ trap error ERR trap finish EXIT function functional_test_smallfile { - wait_clean - apply_operator - kubectl apply -f resources/kernel-cache-drop-clusterrole.yaml test_name=$1 cr=$2 + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) echo "Performing: ${test_name}" - kubectl apply -f ${cr} - long_uuid=$(get_uuid 20) + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} - - count=0 - while [[ $count -lt 24 ]]; do - if [[ `kubectl get pods -l app=smallfile-benchmark-$uuid --namespace my-ripsaw -o name | cut -d/ -f2 | grep client` ]]; then - smallfile_pod=$(kubectl get pods -l app=smallfile-benchmark-$uuid --namespace my-ripsaw -o name | cut -d/ -f2 | grep client) - count=30 - fi - if [[ $count -ne 30 ]]; then - sleep 5 - count=$((count + 1)) - fi - done - echo "smallfile_pod ${smallfile_pod}" - wait_for "kubectl wait --for=condition=Initialized -l app=smallfile-benchmark-$uuid pods --namespace my-ripsaw --timeout=500s" "500s" - wait_for "kubectl wait --for=condition=complete -l app=smallfile-benchmark-$uuid jobs --namespace my-ripsaw --timeout=100s" "100s" + check_benchmark_for_desired_state $benchmark_name Complete 500s indexes="ripsaw-smallfile-results ripsaw-smallfile-rsptimes" if check_es "${long_uuid}" "${indexes}" @@ -50,11 +35,9 @@ function functional_test_smallfile { echo "${test_name} test: Success" else echo "Failed to find data for ${test_name} in ES" - for pod in ${smallfile_pod}; do - kubectl logs ${pod} --namespace my-ripsaw | grep "RUN STATUS" - done exit 1 fi + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_stressng.sh b/tests/test_stressng.sh index 9fa07c10b..d9c98c05c 100755 --- a/tests/test_stressng.sh +++ b/tests/test_stressng.sh @@ -10,23 +10,24 @@ function finish { error fi echo "Cleaning up stressng" - kubectl delete -f tests/test_crs/valid_stressng.yaml - delete_operator + wait_clean } trap finish EXIT function functional_test_stressng { - wait_clean - apply_operator - kubectl apply -f tests/test_crs/valid_stressng.yaml - long_uuid=$(get_uuid 20) + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + cr=tests/test_crs/valid_stressng.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} # Wait for the workload pod to run the actual workload stressng_workload_pod=$(get_pod "app=stressng_workload-$uuid" 300) - kubectl wait --for=condition=Initialized "pods/$stressng_workload_pod" --namespace my-ripsaw --timeout=400s - kubectl wait --for=condition=complete -l app=stressng_workload-$uuid --namespace my-ripsaw jobs --timeout=500s + check_benchmark_for_desired_state $benchmark_name Complete 500s + index="ripsaw-stressng-results" if check_es "${long_uuid}" "${index}" @@ -34,9 +35,11 @@ function functional_test_stressng { echo "StressNG test: Success" else echo "Failed to find data for StressNG test in ES" - kubectl logs "$stressng_workload_pod" --namespace my-ripsaw + kubectl logs "$stressng_workload_pod" --namespace benchmark-operator exit 1 fi + + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_sysbench.sh b/tests/test_sysbench.sh old mode 100644 new mode 100755 index a0cac0470..78571fe2c --- a/tests/test_sysbench.sh +++ b/tests/test_sysbench.sh @@ -10,28 +10,27 @@ function finish { fi echo "Cleaning up sysbench" - kubectl delete -f tests/test_crs/valid_sysbench.yaml - delete_operator + wait_clean } trap error ERR trap finish EXIT function functional_test_sysbench { - wait_clean - apply_operator - kubectl apply -f tests/test_crs/valid_sysbench.yaml - long_uuid=$(get_uuid 20) + cr=tests/test_crs/valid_sysbench.yaml + delete_benchmark $cr + kubectl apply -f $cr + benchmark_name=$(get_benchmark_name $cr) + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} sysbench_pod=$(get_pod "app=sysbench-$uuid" 300) - wait_for "kubectl wait --for=condition=Initialized pods/$sysbench_pod --namespace my-ripsaw --timeout=500s" "500s" $sysbench_pod - # Higher timeout as it takes longer - wait_for "kubectl wait --for=condition=complete -l app=sysbench-$uuid --namespace my-ripsaw jobs" "300s" $sysbench_pod + check_benchmark_for_desired_state $benchmark_name Complete 800s # sleep isn't needed as the sysbench is kind: job so once it's complete we can access logs # ensuring the run has actually happened - kubectl logs "$sysbench_pod" --namespace my-ripsaw | grep "execution time" + kubectl logs "$sysbench_pod" --namespace benchmark-operator | grep "execution time" echo "Sysbench test: Success" + delete_benchmark $cr } figlet $(basename $0) diff --git a/tests/test_uperf.sh b/tests/test_uperf.sh index 65abcc0f5..025d2499f 100755 --- a/tests/test_uperf.sh +++ b/tests/test_uperf.sh @@ -17,21 +17,18 @@ trap error ERR trap finish EXIT function functional_test_uperf { - wait_clean - apply_operator + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) test_name=$1 cr=$2 + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr echo "Performing: ${test_name}" - kubectl apply -f ${cr} - long_uuid=$(get_uuid 20) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} + check_benchmark_for_desired_state $benchmark_name Complete 1800s + - pod_count "type=uperf-bench-server-$uuid" 1 900 - uperf_server_pod=$(get_pod "app=uperf-bench-server-0-$uuid" 300) - wait_for "kubectl -n my-ripsaw wait --for=condition=Initialized -l app=uperf-bench-server-0-$uuid pods --timeout=300s" "300s" $uperf_server_pod - uperf_client_pod=$(get_pod "app=uperf-bench-client-$uuid" 900) - wait_for "kubectl wait -n my-ripsaw --for=condition=Initialized pods/$uperf_client_pod --timeout=500s" "500s" $uperf_client_pod - wait_for "kubectl wait -n my-ripsaw --for=condition=complete -l app=uperf-bench-client-$uuid jobs --timeout=500s" "500s" $uperf_client_pod index="ripsaw-uperf-results" if check_es "${long_uuid}" "${index}" @@ -39,13 +36,14 @@ function functional_test_uperf { echo "${test_name} test: Success" else echo "Failed to find data for ${test_name} in ES" - kubectl logs "$uperf_client_pod" -n my-ripsaw exit 1 fi + delete_benchmark $cr } figlet $(basename $0) functional_test_uperf "Uperf without resources definition" tests/test_crs/valid_uperf.yaml functional_test_uperf "Uperf with ServiceIP" tests/test_crs/valid_uperf_serviceip.yaml +functional_test_uperf "Uperf with NodePort ServiceIP" tests/test_crs/valid_uperf_serviceip_nodeport.yaml functional_test_uperf "Uperf with resources definition and hostNetwork" tests/test_crs/valid_uperf_resources.yaml functional_test_uperf "Uperf with networkpolicy" tests/test_crs/valid_uperf_networkpolicy.yaml diff --git a/tests/test_uperf_scale.sh b/tests/test_uperf_scale.sh new file mode 100644 index 000000000..c44eacc79 --- /dev/null +++ b/tests/test_uperf_scale.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -xeEo pipefail + +source tests/common.sh + +function finish { + if [ $? -eq 1 ] && [ $ERRORED != "true" ] + then + error + fi + + echo "Cleaning up Uperf" + wait_clean +} + +trap error ERR +trap finish EXIT + +function functional_test_uperf { + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + test_name=$1 + cr=$2 + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr + echo "Performing: ${test_name}" + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) + uuid=${long_uuid:0:8} + check_benchmark_for_desired_state $benchmark_name Complete 1800s + + + + index="ripsaw-uperf-results" + if check_es "${long_uuid}" "${index}" + then + echo "${test_name} test: Success" + else + echo "Failed to find data for ${test_name} in ES" + exit 1 + fi + delete_benchmark $cr +} + +figlet $(basename $0) +functional_test_uperf "Uperf scale without resources definition" tests/test_crs/valid_uperf_scale.yaml +functional_test_uperf "Uperf scale with ServiceIP" tests/test_crs/valid_uperf_scale_serviceip.yaml +functional_test_uperf "Uperf scale with NodePort ServiceIP" tests/test_crs/valid_uperf_scale_serviceip_nodeport.yaml +functional_test_uperf "Uperf scale with resources definition and hostNetwork" tests/test_crs/valid_uperf_scale_resources.yaml +functional_test_uperf "Uperf scale with networkpolicy" tests/test_crs/valid_uperf_scale_networkpolicy.yaml diff --git a/tests/test_vegeta.sh b/tests/test_vegeta.sh old mode 100644 new mode 100755 index 57bd144b1..ca3d19a19 --- a/tests/test_vegeta.sh +++ b/tests/test_vegeta.sh @@ -9,7 +9,7 @@ function finish { error fi - [[ $check_logs == 1 ]] && kubectl logs -l app=vegeta-benchmark-$uuid -n my-ripsaw + [[ $check_logs == 1 ]] && kubectl logs -l app=vegeta-benchmark-$uuid -n benchmark-operator echo "Cleaning up vegeta" wait_clean } @@ -20,17 +20,16 @@ trap finish EXIT function functional_test_vegeta { check_logs=0 - wait_clean - apply_operator test_name=$1 cr=$2 + benchmark_name=$(get_benchmark_name $cr) + delete_benchmark $cr echo "Performing: ${test_name}" - kubectl apply -f ${cr} - long_uuid=$(get_uuid 20) + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" ${cr} | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} - - pod_count "app=vegeta-benchmark-$uuid" 2 900 - wait_for "kubectl wait -n my-ripsaw --for=condition=complete -l app=vegeta-benchmark-$uuid jobs --timeout=500s" "500s" + check_benchmark_for_desired_state $benchmark_name Complete 500s check_logs=1 index="ripsaw-vegeta-results" @@ -41,6 +40,8 @@ function functional_test_vegeta { echo "Failed to find data for ${test_name} in ES" exit 1 fi + delete_benchmark $cr + } figlet $(basename $0) diff --git a/tests/test_ycsb.sh b/tests/test_ycsb.sh old mode 100644 new mode 100755 index e2aa69ecd..dc113fca4 --- a/tests/test_ycsb.sh +++ b/tests/test_ycsb.sh @@ -23,7 +23,7 @@ apiVersion: v1 kind: Service metadata: name: mongo - namespace: my-ripsaw + namespace: benchmark-operator labels: name: mongo spec: @@ -39,7 +39,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: mongo - namespace: my-ripsaw + namespace: benchmark-operator spec: selector: matchLabels: @@ -64,13 +64,16 @@ spec: ports: - containerPort: 27017 EOF - kubectl apply -f tests/test_crs/valid_ycsb-mongo.yaml - long_uuid=$(get_uuid 20) + token=$(oc -n openshift-monitoring sa get-token prometheus-k8s) + cr=tests/test_crs/valid_ycsb-mongo.yaml + delete_benchmark $cr + benchmark_name=$(get_benchmark_name $cr) + sed -e "s/PROMETHEUS_TOKEN/${token}/g" $cr | kubectl apply -f - + long_uuid=$(get_uuid $benchmark_name) uuid=${long_uuid:0:8} ycsb_load_pod=$(get_pod "name=ycsb-load-$uuid" 300) - wait_for "kubectl wait --for=condition=Initialized pods/$ycsb_load_pod -n my-ripsaw --timeout=500s" "500s" $ycsb_load_pod - wait_for "kubectl wait --for=condition=Complete jobs -l name=ycsb-load-$uuid -n my-ripsaw --timeout=300s" "300s" $ycsb_load_pod + check_benchmark_for_desired_state $benchmark_name Complete 500s indexes="ripsaw-ycsb-summary ripsaw-ycsb-results" if check_es "${long_uuid}" "${indexes}" @@ -78,11 +81,11 @@ EOF echo "ycsb test: Success" else echo "Failed to find data for ${test_name} in ES" - kubectl logs -n my-ripsaw $ycsb_load_pod + kubectl logs -n benchmark-operator $ycsb_load_pod exit 1 fi + delete_benchmark $cr } figlet $(basename $0) -apply_operator functional_test_ycsb diff --git a/watches.yaml b/watches.yaml index d3dffda1e..fa0f84f97 100644 --- a/watches.yaml +++ b/watches.yaml @@ -1,7 +1,11 @@ --- +# Use the 'create api' subcommand to add watches to this file. - version: v1alpha1 group: ripsaw.cloudbulldozer.io kind: Benchmark - playbook: /opt/ansible/playbook.yml + playbook: ./playbooks/benchmark.yml reconcilePeriod: 3s - manageStatus: false + manageStatus: true + snakeCaseParameters: false + # FIXME: Specify the role or playbook for this resource. +#+kubebuilder:scaffold:watch