From 1d5d128eb90ff670fa2d5192fc6c3dae5a8b120d Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Wed, 15 Apr 2020 23:36:47 +0200 Subject: [PATCH 01/31] Custom profiling commands & flamegraphs Allows running custom profiling command on the nodes via bcc-tools and perf, and obtains flamegraphs based on that. See the updated README.md in perf/benchmark for examples and more information. Signed-off-by: Otto van der Schaaf --- perf/benchmark/README.md | 24 ++++++ perf/benchmark/runner/runner.py | 90 +++++++++++++++++++- perf/benchmark/templates/fortio.yaml | 54 +++++++++++- perf/benchmark/values.yaml | 4 +- perf/docker/Dockerfile.profiling | 21 +++++ perf/docker/perf/setup-node-for-profiling.sh | 42 +++++++++ 6 files changed, 231 insertions(+), 4 deletions(-) create mode 100644 perf/docker/Dockerfile.profiling create mode 100644 perf/docker/perf/setup-node-for-profiling.sh diff --git a/perf/benchmark/README.md b/perf/benchmark/README.md index 411cb91acd..22f18cad10 100644 --- a/perf/benchmark/README.md +++ b/perf/benchmark/README.md @@ -141,12 +141,36 @@ optional arguments: --no_clientsidecar do not run clientsidecar-only for all --bothsidecar run both clientsidecar and serversidecar --no_sidecar do not run clientsidecar and serversidecar + --custom_profiling_command + runs a custom profiling commands on the nodes for the client and server, + and produces a flamegraph based on that. + Example on-cpu profile using bcc tools for the envoy sidecar proxy: + --custom_profiling_command=\"profile-bpfcc -df {duration} -p {sidecar_pid}\" + - runner.py will replace {duration} with whatever was specified for --duration. + - runner.py will replace {sidecar_pid} with the actual process id of the envoy + sidecar process. + --custom_profiling_name + filename prefix for the result of any --custom_profiling_command ``` Note: - `runner.py` will run all combinations of the parameters given. However, in order to reduce ambiguity when generating the graph, it would be better to change one parameter at a time and fix other parameters - if you want to run with `--perf` flag to generate a flame graph, please make sure you have the permission to gather perf data, please refer to step 2 of this [README](https://github.com/istio/tools/tree/master/perf/benchmark/flame#setup-perf-tool) +- if you want to run with `--custom_profiling_command`, `profilingMode` must be set to `true` in `values.yaml`. Doing so will set up the client and server pods to run the perf/profiling container. It's worth noting that this container runs `--priviledged`, and that `hostIPC` and `hostPID` will also be enabled, +weakening security. Resulting flamegraphs will be written to `flame/flameoutput`. +- sample sidecar profiling commands for `--custom_profiling_command`: + - "profile-bpfcc -df {duration} -p {sidecar_pid}" sidecar on-cpu profile + - "offcputime-bpfcc -df {duration} -p {sidecar_pid}" sidecar off-cpu profile + - "offwaketime-bpfcc -df {duration} -p {sidecar_pid}" sidecar offwaktime profile + - "wakeuptime-bpfcc -f -p {sidecar_pid} {duration}" sidecar wakeuptime profile + - "perf record -F 99 -a -g -p {sidecar_pid} -- sleep {duration} && perf script | ~/FlameGraph/stackcollapse-perf.pl | c++filt -n" on-cpu perf-generated profile + - "stackcount-bpfcc c:*alloc* -df -D {duration} -p {sidecar_pid}" profile calls to `*alloc*` +- It's also possible to run machine-wide profiling, for example: + - "profile-bpfcc -df {duration}" for obtaining a machine-wide on-cpu flamegraph. + - See http://www.brendangregg.com/FlameGraphs/ for more examples and information. +- Enabling `profilingMode` in `values.yaml` will also bring up and expose Prometheus's `node_exporter` at the configured port (default: 9100), + accessible over http via `/metrics. For example: diff --git a/perf/benchmark/runner/runner.py b/perf/benchmark/runner/runner.py index 36680b9b77..13fb9f5978 100644 --- a/perf/benchmark/runner/runner.py +++ b/perf/benchmark/runner/runner.py @@ -22,8 +22,11 @@ import shlex import uuid import sys + from subprocess import getoutput from urllib.parse import urlparse +from threading import Thread +from time import sleep import yaml from fortio import METRICS_START_SKIP_DURATION, METRICS_END_SKIP_DURATION @@ -114,7 +117,9 @@ def __init__( ingress=None, mesh="istio", cacert=None, - load_gen_type="fortio"): + load_gen_type="fortio", + custom_profiling_command=None, + custom_profiling_name="default-profile"): self.run_id = str(uuid.uuid4()).partition('-')[0] self.headers = headers self.conn = conn @@ -127,6 +132,8 @@ def __init__( self.r = "0.00005" self.telemetry_mode = telemetry_mode self.perf_record = perf_record + self.custom_profiling_command = custom_profiling_command + self.custom_profiling_name = custom_profiling_name self.server = pod_info("-lapp=" + server, namespace=self.ns) self.client = pod_info("-lapp=" + client, namespace=self.ns) self.additional_args = additional_args @@ -245,6 +252,70 @@ def run(self, headers, conn, qps, size, duration): headers_cmd = self.generate_headers_cmd(headers) fortio_cmd = self.generate_fortio_cmd(headers_cmd, conn, qps, duration, grpc, cacert_arg, labels) + def run_profiling_in_background(exec_cmd, podname, filename_prefix, profiling_command): + filename = "{filename_prefix}-{podname}".format( + filename_prefix=filename_prefix, podname=podname) + profiler_cmd = "{exec_cmd} \"{profiling_command} > {filename}.profile\"".format( + profiling_command=profiling_command, + exec_cmd=exec_cmd, + filename=filename + ) + # Run the profile collection tool, and wait for it to finish. + process = subprocess.Popen(shlex.split(profiler_cmd)) + process.wait() + # Next we feed the profiling data to the flamegraphing script. + flamegraph_cmd = "{exec_cmd} \"./FlameGraph/flamegraph.pl --title='{profiling_command} Flame Graph' < {filename}.profile > {filename}.svg\"".format( + exec_cmd=exec_cmd, + profiling_command=profiling_command, + filename=filename + ) + process = subprocess.Popen(shlex.split(flamegraph_cmd)) + process.wait() + # Lastly copy the resulting flamegraph out of the container + kubectl_cp(podname + ":{filename}.svg".format(filename=filename), + "flame/flameoutput/{filename}.svg".format(filename=filename), "perf") + + threads = [] + + if self.custom_profiling_command: + # We run any custom profiling command on both pods, as one runs on each node we're interested in. + for pod in [self.client.name, self.server.name]: + exec_cmd_on_pod = "kubectl exec -n {namespace} {podname} -c perf -it -- bash -c ".format( + namespace=os.environ.get("NAMESPACE", "twopods"), + podname=pod + ) + + # Wait for node_exporter to run, which indicates the profiling initialization container has finished initializing. + # once the init probe is supported, move this to a http probe instead in fortio.yaml + ne_pid = "" + attempts = 0 + while ne_pid == "" and attempts < 60: + ne_pid = getoutput("{exec_cmd} \"pgrep 'node_exporter'\"".format(exec_cmd=exec_cmd_on_pod)).strip() + attempts = attempts + 1 + print(".") + sleep(1) + + # Find side car process id's in case the profiling command needs it. + sidecar_ppid = getoutput("{exec_cmd} \"pgrep -f 'pilot-agent proxy sidecar'\"".format(exec_cmd=exec_cmd_on_pod)).strip() + sidecar_pid = getoutput("{exec_cmd} \"pgrep -P {sidecar_ppid}\"".format(exec_cmd=exec_cmd_on_pod, sidecar_ppid=sidecar_ppid)).strip() + profiling_command = self.custom_profiling_command.format( + duration=self.duration, sidecar_pid=sidecar_pid) + threads.append(Thread(target=run_profiling_in_background, args=[ + exec_cmd_on_pod, pod, self.custom_profiling_name, profiling_command])) + + for thread in threads: + thread.start() + + if self.run_ingress: + print('-------------- Running in ingress mode --------------') + kubectl_exec(self.client.name, self.ingress(fortio_cmd)) + if self.perf_record: + run_perf( + self.mesh, + self.server.name, + labels + "_srv_ingress", + duration=40) + if self.run_baseline: self.execute_sidecar_mode("baseline", self.load_gen_type, fortio_cmd, self.nosidecar, labels, "") @@ -267,6 +338,11 @@ def run(self, headers, conn, qps, size, duration): labels + "_srv_ingress", duration=40) + if len(threads) > 0: + if self.custom_profiling_command: + for thread in threads: + thread.join() + print("background profiler thread finished - flamegraphs are available in flame/flameoutput") PERFCMD = "/usr/lib/linux-tools/4.4.0-131-generic/perf" FLAMESH = "flame.sh" @@ -365,7 +441,9 @@ def run_perf_test(args): mesh=args.mesh, telemetry_mode=args.telemetry_mode, cacert=args.cacert, - load_gen_type=args.load_gen_type) + load_gen_type=args.load_gen_type, + custom_profiling_command=args.custom_profiling_command, + custom_profiling_name=args.custom_profiling_name) if fortio.duration <= min_duration: print("Duration must be greater than {min_duration}".format( @@ -425,6 +503,14 @@ def get_parser(): "--perf", help="also run perf and produce flame graph", default=False) + parser.add_argument( + "--custom_profiling_command", + help="Run custom profiling commands on the nodes for the client and server, and produce a flamegraph based on their outputs. E.g. --custom_profiling_command=\"/usr/share/bcc/tools/profile -df 40\"", + default=False) + parser.add_argument( + "--custom_profiling_name", + help="Name to be added to the flamegraph resulting from --custom_profiling_command", + default="default-profile") parser.add_argument( "--ingress", help="run traffic through ingress, should be a valid URL", diff --git a/perf/benchmark/templates/fortio.yaml b/perf/benchmark/templates/fortio.yaml index 342b32932b..851e07d575 100644 --- a/perf/benchmark/templates/fortio.yaml +++ b/perf/benchmark/templates/fortio.yaml @@ -40,7 +40,11 @@ spec: protocol: TCP - name: grpc-pinga port: 8076 +{{- if $.Values.profilingMode }} + - name: node-exporter + port: 9100 protocol: TCP +{{- end }} selector: app: {{ $.name }} {{- if $.V.expose }} @@ -98,7 +102,7 @@ spec: config.linkerd.io/skip-inbound-ports: "8077" {{- end }} # exclude inbound ports of the uncaptured container - traffic.sidecar.istio.io/excludeInboundPorts: "8076,8077,8078" + traffic.sidecar.istio.io/excludeInboundPorts: "8076,8077,8078,{{ $.Values.nodeExporterPort }}" sidecar.istio.io/proxyCPU: {{ $.Values.proxy.cpu }} sidecar.istio.io/proxyMemory: {{ $.Values.proxy.memory }} labels: @@ -118,9 +122,30 @@ spec: - "fortioclient" {{- end }} topologyKey: "kubernetes.io/hostname" +{{- if $.Values.profilingMode }} + hostIPC: true + hostPID: true +{{- end }} volumes: - name: shared-data emptyDir: {} +{{- if $.Values.profilingMode }} + - name: sys + hostPath: + path: /sys + - name: lsb-release + hostPath: + path: /etc/lsb-release + - name: modules-generated + hostPath: + path: /var/cache/kernel/modules + - name: headers-generated + hostPath: + path: /var/cache/kernel/headers + - name: usr-host + hostPath: + path: /usr +{{- end }} containers: - name: captured securityContext: @@ -152,6 +177,33 @@ spec: args: - /bin/sleep - infinity +{{- if $.Values.profilingMode }} + - name: perf + image: {{ $.Values.perfImage }} + imagePullPolicy: Always + securityContext: + privileged: true + capabilities: + add: + - SYS_ADMIN + - SYS_PTRACE + command: ["/bin/bash"] + args: ["-c", "./setup-node-for-profiling.sh :{{ $.Values.nodeExporterPort }}"] + ports: + - containerPort: {{ $.Values.nodeExporterPort }} + protocol: TCP + volumeMounts: + - mountPath: /sys + name: sys + - mountPath: /etc/lsb-release.host + name: lsb-release + - mountPath: /lib/modules + name: modules-generated + - mountPath: /usr/src + name: headers-generated + - mountPath: /usr-host + name: usr-host +{{- end }} - name: uncaptured securityContext: runAsUser: 1 diff --git a/perf/benchmark/values.yaml b/perf/benchmark/values.yaml index 4fa9b98b37..d65562e7bd 100644 --- a/perf/benchmark/values.yaml +++ b/perf/benchmark/values.yaml @@ -43,5 +43,7 @@ client: # client overrides cert: false interceptionMode: REDIRECT - +profilingMode: true +perfImage: oschaaf/istio-tools:profiling +nodeExporterPort: 9100 namespace: "" diff --git a/perf/docker/Dockerfile.profiling b/perf/docker/Dockerfile.profiling new file mode 100644 index 0000000000..5ff46b75cc --- /dev/null +++ b/perf/docker/Dockerfile.profiling @@ -0,0 +1,21 @@ +FROM ubuntu:18.04 + +WORKDIR /root + +COPY perf/setup-node-for-profiling.sh setup-node-for-profiling.sh + +RUN apt update && \ + apt install -y git gcc make curl wget libelf-dev bc bpfcc-tools \ + bison flex \ + libdw-dev systemtap-sdt-dev libunwind-dev libaudit-dev \ + libssl-dev libslang2-dev libgtk2.0-dev libperl-dev python-dev && \ + chmod +x setup-node-for-profiling.sh && \ + wget -qO- https://github.com/prometheus/node_exporter/releases/download/v0.18.1/node_exporter-0.18.1.linux-amd64.tar.gz | tar -C . -xvzf - && \ + cp node_exporter-*/node_exporter /usr/bin/ && \ + rm -rf node_exporter-* && \ + git clone --depth=1 https://github.com/BrendanGregg/FlameGraph && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf /tmp/* + +CMD ["setup-node-for-profiling.sh"] + diff --git a/perf/docker/perf/setup-node-for-profiling.sh b/perf/docker/perf/setup-node-for-profiling.sh new file mode 100644 index 0000000000..62c9fad986 --- /dev/null +++ b/perf/docker/perf/setup-node-for-profiling.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -ex + +USR_SRC="/usr/src" +KERNEL_VERSION="$(uname -r)" +CHROMEOS_RELEASE_VERSION="$(grep 'CHROMEOS_RELEASE_VERSION' /etc/lsb-release.host | cut -d '=' -f 2)" + +build_kernel() +{ + # Build the headers + cd "${WORKING_DIR}" + zcat /proc/config.gz > .config + make ARCH=x86 oldconfig > /dev/null + make ARCH=x86 prepare > /dev/null + + # Build perf + cd tools/perf/ + make ARCH=x86 > /dev/null + mv perf /usr/sbin/ +} + +prepare_node() +{ + WORKING_DIR="/linux-lakitu-${CHROMEOS_RELEASE_VERSION}" + SOURCES_DIR="${USR_SRC}/linux-lakitu-${CHROMEOS_RELEASE_VERSION}" + mkdir -p "${WORKING_DIR}" + curl -s "https://storage.googleapis.com/cos-tools/${CHROMEOS_RELEASE_VERSION}/kernel-src.tar.gz" \ + | tar -xzf - -C "${WORKING_DIR}" + build_kernel + rm -rf "${USR_SRC}${WORKING_DIR}" + mv "${WORKING_DIR}" "${USR_SRC}" +} + +prepare_node +mkdir -p "/lib/modules/${KERNEL_VERSION}" +ln -sf "${SOURCES_DIR}" "/lib/modules/${KERNEL_VERSION}/source" +ln -sf "${SOURCES_DIR}" "/lib/modules/${KERNEL_VERSION}/build" + +# fire up the node exporter process, listening at the passed in address:port +node_exporter --web.listen-address $1 + From 70b6e52c321daf12de787cbcd4cb6fa153ab8cca Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 16 Apr 2020 11:05:32 +0200 Subject: [PATCH 02/31] Docker linting, fix TODO Signed-off-by: Otto van der Schaaf --- perf/benchmark/runner/runner.py | 18 ++++-------------- perf/docker/Dockerfile.profiling | 4 +++- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/perf/benchmark/runner/runner.py b/perf/benchmark/runner/runner.py index 13fb9f5978..edde72c7fb 100644 --- a/perf/benchmark/runner/runner.py +++ b/perf/benchmark/runner/runner.py @@ -252,9 +252,9 @@ def run(self, headers, conn, qps, size, duration): headers_cmd = self.generate_headers_cmd(headers) fortio_cmd = self.generate_fortio_cmd(headers_cmd, conn, qps, duration, grpc, cacert_arg, labels) - def run_profiling_in_background(exec_cmd, podname, filename_prefix, profiling_command): - filename = "{filename_prefix}-{podname}".format( - filename_prefix=filename_prefix, podname=podname) + def run_profiling_in_background(exec_cmd, podname, filename_prefix, profiling_command, labels): + filename = "{filename_prefix}-{labels}-{podname}".format( + filename_prefix=filename_prefix, labels=labels, podname=podname) profiler_cmd = "{exec_cmd} \"{profiling_command} > {filename}.profile\"".format( profiling_command=profiling_command, exec_cmd=exec_cmd, @@ -301,21 +301,11 @@ def run_profiling_in_background(exec_cmd, podname, filename_prefix, profiling_co profiling_command = self.custom_profiling_command.format( duration=self.duration, sidecar_pid=sidecar_pid) threads.append(Thread(target=run_profiling_in_background, args=[ - exec_cmd_on_pod, pod, self.custom_profiling_name, profiling_command])) + exec_cmd_on_pod, pod, self.custom_profiling_name, profiling_command, labels])) for thread in threads: thread.start() - if self.run_ingress: - print('-------------- Running in ingress mode --------------') - kubectl_exec(self.client.name, self.ingress(fortio_cmd)) - if self.perf_record: - run_perf( - self.mesh, - self.server.name, - labels + "_srv_ingress", - duration=40) - if self.run_baseline: self.execute_sidecar_mode("baseline", self.load_gen_type, fortio_cmd, self.nosidecar, labels, "") diff --git a/perf/docker/Dockerfile.profiling b/perf/docker/Dockerfile.profiling index 5ff46b75cc..8c234dafc9 100644 --- a/perf/docker/Dockerfile.profiling +++ b/perf/docker/Dockerfile.profiling @@ -4,8 +4,10 @@ WORKDIR /root COPY perf/setup-node-for-profiling.sh setup-node-for-profiling.sh +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + RUN apt update && \ - apt install -y git gcc make curl wget libelf-dev bc bpfcc-tools \ + apt-get install -y git gcc make curl wget libelf-dev bc bpfcc-tools \ bison flex \ libdw-dev systemtap-sdt-dev libunwind-dev libaudit-dev \ libssl-dev libslang2-dev libgtk2.0-dev libperl-dev python-dev && \ From 81d4d7a1de5430ff67b8478b5de934959014db5b Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 16 Apr 2020 11:22:41 +0200 Subject: [PATCH 03/31] Docker lint tweak Signed-off-by: Otto van der Schaaf --- perf/docker/Dockerfile.profiling | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perf/docker/Dockerfile.profiling b/perf/docker/Dockerfile.profiling index 8c234dafc9..b93f35828a 100644 --- a/perf/docker/Dockerfile.profiling +++ b/perf/docker/Dockerfile.profiling @@ -6,8 +6,8 @@ COPY perf/setup-node-for-profiling.sh setup-node-for-profiling.sh SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN apt update && \ - apt-get install -y git gcc make curl wget libelf-dev bc bpfcc-tools \ +RUN apt-get update && \ + apt-get install -y --no-install-recommends git gcc make curl wget libelf-dev bc bpfcc-tools \ bison flex \ libdw-dev systemtap-sdt-dev libunwind-dev libaudit-dev \ libssl-dev libslang2-dev libgtk2.0-dev libperl-dev python-dev && \ From ee62a802603bafeb468815b7cff8ac26f01896dd Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 16 Apr 2020 13:43:10 +0200 Subject: [PATCH 04/31] Add perf label to flamegraph filename Signed-off-by: Otto van der Schaaf --- perf/benchmark/runner/runner.py | 90 +++++++++++-------- perf/docker/Dockerfile.profiling | 5 +- .../setup-node-for-profiling.sh | 3 + 3 files changed, 59 insertions(+), 39 deletions(-) rename perf/docker/{perf => profiling}/setup-node-for-profiling.sh (94%) diff --git a/perf/benchmark/runner/runner.py b/perf/benchmark/runner/runner.py index edde72c7fb..d1478b5eea 100644 --- a/perf/benchmark/runner/runner.py +++ b/perf/benchmark/runner/runner.py @@ -238,6 +238,29 @@ def generate_fortio_cmd(self, headers_cmd, conn, qps, duration, grpc, cacert_arg return fortio_cmd + def run_profiling_in_background(self, exec_cmd, podname, filename_prefix, profiling_command, labels): + filename = "{filename_prefix}-{labels}-{podname}".format( + filename_prefix=filename_prefix, labels=labels, podname=podname) + profiler_cmd = "{exec_cmd} \"{profiling_command} > {filename}.profile\"".format( + profiling_command=profiling_command, + exec_cmd=exec_cmd, + filename=filename + ) + # Run the profile collection tool, and wait for it to finish. + process = subprocess.Popen(shlex.split(profiler_cmd)) + process.wait() + # Next we feed the profiling data to the flamegraphing script. + flamegraph_cmd = "{exec_cmd} \"./FlameGraph/flamegraph.pl --title='{profiling_command} Flame Graph' < {filename}.profile > {filename}.svg\"".format( + exec_cmd=exec_cmd, + profiling_command=profiling_command, + filename=filename + ) + process = subprocess.Popen(shlex.split(flamegraph_cmd)) + process.wait() + # Lastly copy the resulting flamegraph out of the container + kubectl_cp(podname + ":{filename}.svg".format(filename=filename), + "flame/flameoutput/{filename}.svg".format(filename=filename), "perf") + def run(self, headers, conn, qps, size, duration): labels = self.generate_test_labels(conn, qps, size) @@ -251,29 +274,31 @@ def run(self, headers, conn, qps, size, duration): headers_cmd = self.generate_headers_cmd(headers) fortio_cmd = self.generate_fortio_cmd(headers_cmd, conn, qps, duration, grpc, cacert_arg, labels) + perf_label = "" + sidecar_mode = "" + sidecar_mode_func = None + + if self.run_baseline: + sidecar_mode = "baseline" + sidecar_mode_func = self.nosidecar + + if self.run_serversidecar: + perf_label = "_srv_serveronly" + sidecar_mode = "server sidecar" + sidecar_mode_func = self.serversidecar - def run_profiling_in_background(exec_cmd, podname, filename_prefix, profiling_command, labels): - filename = "{filename_prefix}-{labels}-{podname}".format( - filename_prefix=filename_prefix, labels=labels, podname=podname) - profiler_cmd = "{exec_cmd} \"{profiling_command} > {filename}.profile\"".format( - profiling_command=profiling_command, - exec_cmd=exec_cmd, - filename=filename - ) - # Run the profile collection tool, and wait for it to finish. - process = subprocess.Popen(shlex.split(profiler_cmd)) - process.wait() - # Next we feed the profiling data to the flamegraphing script. - flamegraph_cmd = "{exec_cmd} \"./FlameGraph/flamegraph.pl --title='{profiling_command} Flame Graph' < {filename}.profile > {filename}.svg\"".format( - exec_cmd=exec_cmd, - profiling_command=profiling_command, - filename=filename - ) - process = subprocess.Popen(shlex.split(flamegraph_cmd)) - process.wait() - # Lastly copy the resulting flamegraph out of the container - kubectl_cp(podname + ":{filename}.svg".format(filename=filename), - "flame/flameoutput/{filename}.svg".format(filename=filename), "perf") + if self.run_clientsidecar: + perf_label = "_srv_clientonly" + sidecar_mode = "client sidecar" + sidecar_mode_func = self.clientsidecar + + if self.run_bothsidecar: + perf_label = "_srv_bothsidecars" + sidecar_mode = "both sidecar" + sidecar_mode_func = self.bothsidecar + + if self.run_ingress: + perf_label = "_srv_ingress" threads = [] @@ -300,33 +325,24 @@ def run_profiling_in_background(exec_cmd, podname, filename_prefix, profiling_co sidecar_pid = getoutput("{exec_cmd} \"pgrep -P {sidecar_ppid}\"".format(exec_cmd=exec_cmd_on_pod, sidecar_ppid=sidecar_ppid)).strip() profiling_command = self.custom_profiling_command.format( duration=self.duration, sidecar_pid=sidecar_pid) - threads.append(Thread(target=run_profiling_in_background, args=[ - exec_cmd_on_pod, pod, self.custom_profiling_name, profiling_command, labels])) + threads.append(Thread(target=self.run_profiling_in_background, args=[ + exec_cmd_on_pod, pod, self.custom_profiling_name, profiling_command, labels + perf_label])) for thread in threads: thread.start() - if self.run_baseline: - self.execute_sidecar_mode("baseline", self.load_gen_type, fortio_cmd, self.nosidecar, labels, "") - - if self.run_serversidecar: - self.execute_sidecar_mode("server sidecar", self.load_gen_type, fortio_cmd, self.serversidecar, labels, "_srv_serveronly") - - if self.run_clientsidecar: - self.execute_sidecar_mode("client sidecar", self.load_gen_type, fortio_cmd, self.clientsidecar, labels, "_srv_clientonly") - - if self.run_bothsidecar: - self.execute_sidecar_mode("both sidecar", self.load_gen_type, fortio_cmd, self.bothsidecar, labels, "_srv_bothsidecars") - if self.run_ingress: print('-------------- Running in ingress mode --------------') kubectl_exec(self.client.name, self.ingress(fortio_cmd)) + if self.perf_record: run_perf( self.mesh, self.server.name, - labels + "_srv_ingress", + labels + perf_label, duration=40) + else: + self.execute_sidecar_mode(sidecar_mode, self.load_gen_type, fortio_cmd, sidecar_mode_func, labels, perf_label) if len(threads) > 0: if self.custom_profiling_command: diff --git a/perf/docker/Dockerfile.profiling b/perf/docker/Dockerfile.profiling index b93f35828a..6f361bba29 100644 --- a/perf/docker/Dockerfile.profiling +++ b/perf/docker/Dockerfile.profiling @@ -2,16 +2,17 @@ FROM ubuntu:18.04 WORKDIR /root -COPY perf/setup-node-for-profiling.sh setup-node-for-profiling.sh +COPY profiling/setup-node-for-profiling.sh setup-node-for-profiling.sh SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN apt-get update && \ apt-get install -y --no-install-recommends git gcc make curl wget libelf-dev bc bpfcc-tools \ - bison flex \ + bison flex ca-certificates \ libdw-dev systemtap-sdt-dev libunwind-dev libaudit-dev \ libssl-dev libslang2-dev libgtk2.0-dev libperl-dev python-dev && \ chmod +x setup-node-for-profiling.sh && \ + update-ca-certificates && \ wget -qO- https://github.com/prometheus/node_exporter/releases/download/v0.18.1/node_exporter-0.18.1.linux-amd64.tar.gz | tar -C . -xvzf - && \ cp node_exporter-*/node_exporter /usr/bin/ && \ rm -rf node_exporter-* && \ diff --git a/perf/docker/perf/setup-node-for-profiling.sh b/perf/docker/profiling/setup-node-for-profiling.sh similarity index 94% rename from perf/docker/perf/setup-node-for-profiling.sh rename to perf/docker/profiling/setup-node-for-profiling.sh index 62c9fad986..df98357142 100644 --- a/perf/docker/perf/setup-node-for-profiling.sh +++ b/perf/docker/profiling/setup-node-for-profiling.sh @@ -37,6 +37,9 @@ mkdir -p "/lib/modules/${KERNEL_VERSION}" ln -sf "${SOURCES_DIR}" "/lib/modules/${KERNEL_VERSION}/source" ln -sf "${SOURCES_DIR}" "/lib/modules/${KERNEL_VERSION}/build" +sysctl kernel.perf_event_paranoid=-1 +sysctl kernel.kptr_restrict=0 + # fire up the node exporter process, listening at the passed in address:port node_exporter --web.listen-address $1 From 689021b4cea4ab3a0886141cbe2c0650b45cd771 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 16 Apr 2020 23:35:51 +0200 Subject: [PATCH 05/31] Deduplicate redundant functionality As --custom_profiling_command offers a superset of --perf, reimplement the old --perf flag using that and eliminate the redundant code. Signed-off-by: Otto van der Schaaf --- perf/benchmark/flame/README.md | 63 ++++---------- perf/benchmark/flame/flame.sh | 48 ----------- perf/benchmark/flame/get_perfdata.sh | 42 --------- perf/benchmark/flame/get_proxy_perf.sh | 77 ----------------- perf/benchmark/runner/runner.py | 115 +++++++++---------------- 5 files changed, 60 insertions(+), 285 deletions(-) delete mode 100755 perf/benchmark/flame/flame.sh delete mode 100755 perf/benchmark/flame/get_perfdata.sh delete mode 100755 perf/benchmark/flame/get_proxy_perf.sh diff --git a/perf/benchmark/flame/README.md b/perf/benchmark/flame/README.md index 0d7f6e3120..2ad0e8a3a2 100644 --- a/perf/benchmark/flame/README.md +++ b/perf/benchmark/flame/README.md @@ -8,61 +8,36 @@ 1. Colors are arbitrary. 1. Function names are sorted left to right. -This document shows how to gather performance data from within the `istio-proxy` container. +This document shows how to gather performance data from via the `perf` container. -## Setup Perf tool +## Setup the perf container -Flame graphs are created from data collected using linux `perf_events` by the `perf` tool. +Enable `profilingMode` in [values.yaml](../values.yaml). This will end up adding the perf +container to the server and client pods, which both will be running on separate nodes. -1. Ensure that `perf` is installed within the container. - Since `istio-proxy` container does not allow installation of new packages, build a new docker image. +Flame graphs are created from data collected using linux `perf_events` by the `perf` and [BCC tools](https://github.com/iovisor/bcc). - ```plain - FROM gcr.io/istio-release/proxyv2:release-1.0-20180810-09-15 - # Install fpm tool - RUN sudo apt-get update && \ - sudo apt-get -qqy install linux-tools-generic - ``` +## Obtaining flame graphs - Build image and push docker image and use it in your deployment by adding the following annotation. +Flame graphs can be produced via `runner.py`, and will be stored in `flame/flameoutput`. - ```plain - "sidecar.istio.io/proxyImag" : - ``` +A few sample command lines. `{duration}` will be replaced by +whatever was passed for `--duration` to runner.py. `{sidecar_pid}` will +be replaced by `runner.py` with the process id of the Envoy sidecar. - This step will go away once the default debug image contains `perf` and related tools. +It is valid to omit `{sidecar_pid}` in `--custom_profiling_command`. +This may be useful for machine-wide profiling or arbitrary processes. -1. Ensure that you can run `perf record` - Running `perf record` from container requires the host to permit this activity. This is done by running the following command on the vm host. - For example, if you are running on a GKE cluster, you should `ssh` to the node using the command: +```bash +runner/runner.py --conn 20 --qps 10000 --duration 100 --custom_profiling_command="profile-bpfcc -df {duration} -p {sidecar_pid}" --custom_profiling_name="bcc-oncputime-sidecar" - ```bash - gcloud compute ssh gke-perf-test-default-pool-xxxxxx - ``` +runner/runner.py --conn 20 --qps 10000 --duration 100 --serversidecar --custom_profiling_command="offcputime-bpfcc -df {duration} -p {sidecar_pid}" --custom_profiling_name="bcc-offcputime-sidecar" - Then run the following command: +runner/runner.py --conn 20 --qps 10000 --duration 100 --serversidecar --custom_profiling_command="offwaketime-bpfcc -df {duration} -p {sidecar_pid}" --custom_profiling_name="bcc-offwaketime-sidecar" - ```bash - sudo sysctl kernel.perf_event_paranoid=-1 - sudo sysctl kernel.kptr_restrict=0 - ``` +runner/runner.py --conn 20 --qps 10000 --duration 100 --serversidecar --custom_profiling_command="wakeuptime-bpfcc -f -p {sidecar_pid} {duration}" --custom_profiling_name="bcc-wakeuptime-sidecar" - This setting is very permissive so it must be used with care. +runner/runner.py --conn 20 --qps 10000 --duration 100 --serversidecar --custom_profiling_command="perf record -F 99 -g -p {sidecar_pid} -- sleep {duration} && perf script | ~/FlameGraph/stackcollapse-perf.pl | c++filt -n" --custom_profiling_name="perf-oncputime-sidecar" +``` - If running perf still gives error:```You may not have permission to collect stats. Consider tweaking /proc/sys/kernel/perf_event_paranoid:``` - after running above commands, try ssh into node and run the container with --privileged flag. - -1. Run [`get_proxy_perf.sh`](get_proxy_perf.sh) to get the profiling svg. The following command collects samples at `177Hz` for `20s`. The svg file should be created under `flameoutput` dir - - ```plain - ./get_proxy_perf.sh -p svc05-0-7-564865d756-pvjhn -n service-graph05 -s 177 -t 20 - ... - [ perf record: Woken up 1 times to write data ] - [ perf record: Captured and wrote 0.061 MB /etc/istio/proxy/perf.data (74 samples) ] - - Wrote /etc/istio/proxy/perf.data.perf - ... - generating svg file svc05-0-7-564865d756-pvjhn-2020-01-29-22-34-19.perf - ... - ``` diff --git a/perf/benchmark/flame/flame.sh b/perf/benchmark/flame/flame.sh deleted file mode 100755 index ad09de7263..0000000000 --- a/perf/benchmark/flame/flame.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# Copyright Istio Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -WD=$(dirname "${0}") -WD=$(cd "${WD}" && pwd) - -FLAMEDIR="${WD}/FlameGraph" - -if ! command -v c++filt > /dev/null; then - echo "Install c++filt to demangle symbols" - exit 1 -fi - -cd "${WD}" || exit 1 - -if [[ ! -d ${FLAMEDIR} ]]; then - echo "Cloning FlameGraph repo in ${WD}" - git clone https://github.com/brendangregg/FlameGraph -fi - -# Given output of `perf script` produce a flamegraph -FILE=${1:?"perf script output"} -FILENAME=$(basename "${FILE}") -BASE=$(echo "${FILENAME}" | cut -d '.' -f 1) -SVGNAME="${BASE}.svg" - -mkdir -p "${WD}/flameoutput" -"${FLAMEDIR}/stackcollapse-perf.pl" "${FILE}" | c++filt -n | "${FLAMEDIR}/flamegraph.pl" --cp > "./flameoutput/${SVGNAME}" - -echo "Wrote ${SVGNAME}" -if [[ -n "${BUCKET}" ]];then - gsutil cp "${SVGNAME}" "${BUCKET}" -fi diff --git a/perf/benchmark/flame/get_perfdata.sh b/perf/benchmark/flame/get_perfdata.sh deleted file mode 100755 index 4881935ffd..0000000000 --- a/perf/benchmark/flame/get_perfdata.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Copyright Istio Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -WD=$(dirname "${0}") -WD=$(cd "${WD}" && pwd) - -FILENAME=${1:?"perffilename"} -DURATION=${2:?"duration"} -FREQ=${3:-"99"} - -PID=$(pgrep envoy) - -# This is specific to the kernel version -# example: /usr/lib/linux-tools-4.4.0-131/perf -# provided by `linux-tools-generic` -PERFDIR=$(find /usr/lib -name 'linux-tools-*' -type d | head -n 1) -if [[ -z "${PERFDIR}" ]]; then - echo "Missing perf tool. Install apt-get install linux-tools-generic" - exit 1 -fi - -PERF="${PERFDIR}/perf" - -"${PERF}" record -o "${WD}/${FILENAME}" -F "${FREQ}" -p "${PID}" -g -- sleep "${DURATION}" -"${PERF}" script -i "${WD}/${FILENAME}" --demangle > "${WD}/${FILENAME}.perf" - -echo "Wrote ${WD}/${FILENAME}.perf" diff --git a/perf/benchmark/flame/get_proxy_perf.sh b/perf/benchmark/flame/get_proxy_perf.sh deleted file mode 100755 index dd0f10e2b2..0000000000 --- a/perf/benchmark/flame/get_proxy_perf.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -# Copyright Istio Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -function usage() { - echo "usage: - ./get_proxy_perf.sh -p -n -s -t