From 1cfa1be8245b90ce98d7132baba89de0f1620e0d Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Thu, 12 Oct 2023 21:36:23 -0400 Subject: [PATCH 01/33] Sidecar/init containers, 23.1, core example update --- examples/2regions-hrr/meshrr-core.yaml | 71 +++++++++++++++++++++----- meshrr/Dockerfile | 39 ++++++-------- meshrr/juniper.conf.j2 | 19 +++++++ meshrr/render_config.py | 7 ++- meshrr/requirements.txt | 42 +++++++-------- meshrr/{runit-init.sh => run.sh} | 41 +++++++++------ meshrr/update_peers.py | 4 +- 7 files changed, 146 insertions(+), 77 deletions(-) rename meshrr/{runit-init.sh => run.sh} (60%) diff --git a/examples/2regions-hrr/meshrr-core.yaml b/examples/2regions-hrr/meshrr-core.yaml index 38081d5..cc4c534 100644 --- a/examples/2regions-hrr/meshrr-core.yaml +++ b/examples/2regions-hrr/meshrr-core.yaml @@ -47,16 +47,43 @@ spec: dnsPolicy: ClusterFirst terminationGracePeriodSeconds: 30 volumes: - - name: crpd-license - secret: - secretName: crpd-license - items: - - key: junos_sfnt.lic - path: junos_sfnt.lic + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} + initContainers: + - name: meshrr-init + image: meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license + - name: AUTONOMOUS_SYSTEM + value: '65000' + - name: ENCRYPTED_ROOT_PW + value: >- + $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 + - name: MESH_SERVICE_NAME + value: meshrr-core + - name: MESHRR_CLIENTRANGE + value: 0/0 containers: - - name: meshrr-core - image: /meshrr:latest - imagePullPolicy: Always + - name: crpd + image: crpd:23.2R1.13 + imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 initialDelaySeconds: 15 @@ -94,13 +121,33 @@ spec: - name: MESHRR_CLIENTRANGE value: 0/0 volumeMounts: - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' + - name: config + mountPath: /config/ securityContext: allowPrivilegeEscalation: true privileged: true runAsNonRoot: false + - name: meshrr + image: meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["sidecar"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: AUTONOMOUS_SYSTEM + value: '65000' + - name: ENCRYPTED_ROOT_PW + value: >- + $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 + - name: MESH_SERVICE_NAME + value: meshrr-core + - name: MESHRR_CLIENTRANGE + value: 0/0 imagePullSecrets: - name: regcred affinity: diff --git a/meshrr/Dockerfile b/meshrr/Dockerfile index 01f0797..906c858 100644 --- a/meshrr/Dockerfile +++ b/meshrr/Dockerfile @@ -1,4 +1,4 @@ -# Copyright (c) Juniper Networks, Inc., 2020. All rights reserved. +# Copyright (c) Juniper Networks, Inc., 2023. All rights reserved. # # Notice and Disclaimer: This code is licensed to you under the MIT License (the # "License"). You may not use this code except in compliance with the License. @@ -27,32 +27,25 @@ # components is subject to the terms and conditions of the respective license as # noted in the Third-Party source code file. -FROM crpd:22.4R1.10 as prebuild +FROM alpine:3.18 as meshrr-prebuild -WORKDIR /root +WORKDIR /opt/meshrr -COPY requirements.txt /root/ +COPY requirements.txt /opt/meshrr/ -# This blocks cron startup if not changed -RUN printf '#!/bin/sh\nexit 0\n' > /usr/sbin/policy-rc.d +RUN apk add --update --no-cache python3 py3-pip openssh \ + && pip3 install -r requirements.txt \ + && mkdir -p /config /secret/ssh -# Install requirements. Note that cron must be reconfigured later to create user. -RUN apt-get update \ - && apt-get -y --no-install-recommends install cron python3-minimal python3-setuptools python3-pip \ - && rm -fr /var/lib/apt/lists/* \ - && pip3 install wheel \ - && pip3 install -r requirements.txt +FROM meshrr-prebuild -FROM prebuild +COPY render_config.py /opt/meshrr/ +COPY run.sh /opt/meshrr/ +COPY juniper.conf.j2 /opt/meshrr/ +COPY update_peers.py /opt/meshrr/ -COPY render_config.py /root/ -COPY runit-init.sh /sbin/ -COPY juniper.conf.j2 /root/ -COPY update_peers.py /root/ +RUN chmod +x /opt/meshrr/run.sh +RUN chmod +x /opt/meshrr/update_peers.py +RUN chmod +x /opt/meshrr/render_config.py -RUN chmod +x /sbin/runit-init.sh -RUN chmod +x /root/update_peers.py -RUN chmod +x /root/render_config.py - -ENTRYPOINT [ "/sbin/runit-init.sh" ] -STOPSIGNAL 35 +ENTRYPOINT [ "/opt/meshrr/run.sh" ] \ No newline at end of file diff --git a/meshrr/juniper.conf.j2 b/meshrr/juniper.conf.j2 index 34af20b..9a561c7 100644 --- a/meshrr/juniper.conf.j2 +++ b/meshrr/juniper.conf.j2 @@ -2,8 +2,27 @@ system { root-authentication { encrypted-password "{{ ENCRYPTED_ROOT_PW }}"; ## SECRET-DATA } + login { + user meshrr { + class super-user; + uid 100; + authentication { + ssh-ed25519 "ssh-ed25519 TBD"; ## SECRET-DATA + } + } + } /* Required for configuration persistence */ no-compress-configuration-files; + services { + netconf { + ssh; + } + } + license { + keys { + key "{{ LICENSE_KEY }}"; + } + } processes { routing { bgp { diff --git a/meshrr/render_config.py b/meshrr/render_config.py index 7e0d70c..df7c85c 100755 --- a/meshrr/render_config.py +++ b/meshrr/render_config.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright (c) Juniper Networks, Inc., 2020. All rights reserved. +# Copyright (c) Juniper Networks, Inc., 2023. All rights reserved. # # Notice and Disclaimer: This code is licensed to you under the MIT License (the # "License"). You may not use this code except in compliance with the License. @@ -47,6 +47,9 @@ template = Template(args.inputfile.read()) configvars = dict() + configvars.update({"LICENSE_KEY": getenv("LICENSE_KEY", None)}) + if not configvars["LICENSE_KEY"]: + raise (Exception("LICENSE_KEY is not set.")) configvars.update({"ENCRYPTED_ROOT_PW": getenv("ENCRYPTED_ROOT_PW", None)}) if not configvars["ENCRYPTED_ROOT_PW"]: raise (Exception("ENCRYPTED_ROOT_PW is not set.")) @@ -63,7 +66,7 @@ # Default to Route Reflector Mode configvars.update({"MESHRR_MODE": getenv("MESHRR_MODE", "routereflector")}) - # Default AS Range for Route Server mode is 65001 to 65000 + # Default AS Range for Route Server mode is 65001 to 65500 configvars.update({"MESHRR_ASRANGE": getenv("MESHRR_ASRANGE", "65001-65500")}) # Default to inet unicast only configvars.update({"MESHRR_FAMILY_INET": getenv("MESHRR_FAMILY_INET", "true")}) diff --git a/meshrr/requirements.txt b/meshrr/requirements.txt index 2982694..daccf97 100644 --- a/meshrr/requirements.txt +++ b/meshrr/requirements.txt @@ -1,26 +1,22 @@ -bcrypt==3.2.0 -cffi==1.15.1 +bcrypt==4.0.1 +cffi==1.16.0 cryptography==41.0.4 -dnspython==2.0.0 -importlib-resources==3.3.0 -Jinja2==2.11.3 -junos-eznc==2.6.6 -lxml==4.9.1 -MarkupSafe==1.1.1 +dnspython==2.4.2 +Jinja2==3.1.2 +junos-eznc==2.6.8 +lxml==4.9.3 +MarkupSafe==2.1.3 ncclient==0.6.13 -netaddr==0.8.0 -paramiko==2.10.1 -pip==22.3.1 -pycparser==2.20 -PyNaCl==1.4.0 -pyparsing==2.4.7 +netaddr==0.9.0 +packaging==23.1 +paramiko==3.3.1 +pycparser==2.21 +PyNaCl==1.5.0 +pyparsing==3.0.9 pyserial==3.5 -python-dotenv==0.15.0 -PyYAML>=6.0.1 -scp==0.13.3 -setuptools==65.5.1 -six==1.15.0 -transitions==0.8.5 -wheel==0.38.1 -yamlordereddictloader==0.4.0 -zipp==3.4.0 \ No newline at end of file +python-dotenv==1.0.0 +PyYAML==6.0.1 +scp==0.14.5 +six==1.16.0 +transitions==0.9.0 +yamlordereddictloader==0.4.2 \ No newline at end of file diff --git a/meshrr/runit-init.sh b/meshrr/run.sh similarity index 60% rename from meshrr/runit-init.sh rename to meshrr/run.sh index e2f9188..bf7c7f9 100644 --- a/meshrr/runit-init.sh +++ b/meshrr/run.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # Copyright (c) Juniper Networks, Inc., 2020. All rights reserved. # @@ -33,20 +33,31 @@ set -e printenv > /etc/envvars -# Overwrite with existing configuration if it exists. -if [ -f "/config/juniper.conf" ]; then - sed "s/^\(.\+router-id\) [0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+/\1 {{ POD_IP }};/" /config/juniper.conf > juniper.conf.j2 +if [ $# -eq 0 ]; then + echo "One of the following arguments required: init, sidecar" + exit 1 +elif [ $1 = 'init' ]; then + echo "Initializing pod" + # Overwrite with existing configuration if it exists. + if [ -f "/config/juniper.conf" ]; then + echo "Existing configuration detected; overwriting pod IP only." + sed "s/^\(.\+router-id\) [0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+/\1 {{ POD_IP }};/" /config/juniper.conf > juniper.conf.j2 + else + echo "Initializing fresh configuration from template." + fi + # Generate a fresh SSH key and apply to configuration template. + ssh-keygen -q -t ed25519 -f /secret/ssh/id_ed25519 -P "" + PUBKEY=`cat \/secret\/ssh\/id_ed25519.pub | tr -d '\r\n'` + sed -i "/user meshrr/,/SECRET-DATA/ s~ssh-ed25519.*~ssh-ed25519 \"$PUBKEY\"; ## SECRET-DATA~" juniper.conf.j2 + ./render_config.py -i juniper.conf.j2 -o /config/juniper.conf +elif [ $1 = 'sidecar' ]; then + echo "Initializing peer maintenance every ${UPDATE_SECONDS:=30} seconds." + while true; do + ./update_peers.py + sleep ${UPDATE_SECONDS} + done +else + echo "Invalid argument: $1" fi -./render_config.py -i juniper.conf.j2 -o /config/juniper.conf -# Install crontab. Expect non-zero status for crontab -l -line="* * * * * /root/update_peers.py >> /var/log/update_peers 2>&1" -set +e - -# crontab group doesn't seem to be created in docker build. -dpkg-reconfigure cron -(crontab -l 2>/dev/null; echo "$line" ) | crontab - -set -e - -exec /sbin/runit-init 0 \ No newline at end of file diff --git a/meshrr/update_peers.py b/meshrr/update_peers.py index 2410ef6..fc89b1a 100644 --- a/meshrr/update_peers.py +++ b/meshrr/update_peers.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright (c) Juniper Networks, Inc., 2020. All rights reserved. +# Copyright (c) Juniper Networks, Inc., 2023. All rights reserved. # # Notice and Disclaimer: This code is licensed to you under the MIT License (the # "License"). You may not use this code except in compliance with the License. @@ -186,7 +186,7 @@ def update_peergroup(cu, group_name, service_name, max_peers=None): service_root_domain = getenv("SERVICE_ROOT_DOMAIN", "svc.cluster.local") # Open a connection to the device - dev = Device() + dev = Device(host="127.0.0.1",user="meshrr",ssh_private_key_file="/secret/ssh/id_ed25519") dev.open() with ConfigUpdate(dev, mode="private") as cu: From 6a6345229f2638e26a2b13abc8b648e3e0223fe8 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Thu, 12 Oct 2023 22:40:40 -0400 Subject: [PATCH 02/33] Clean up unused env vars --- examples/2regions-hrr/meshrr-core.yaml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/examples/2regions-hrr/meshrr-core.yaml b/examples/2regions-hrr/meshrr-core.yaml index cc4c534..0482239 100644 --- a/examples/2regions-hrr/meshrr-core.yaml +++ b/examples/2regions-hrr/meshrr-core.yaml @@ -106,20 +106,6 @@ spec: - name: bgp containerPort: 179 protocol: TCP - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 0/0 volumeMounts: - name: config mountPath: /config/ @@ -139,11 +125,6 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - name: MESH_SERVICE_NAME value: meshrr-core - name: MESHRR_CLIENTRANGE From b78d01ffd66c3b6005620a51c2fe3ea1c96ada0c Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Fri, 13 Oct 2023 09:28:11 -0400 Subject: [PATCH 03/33] Distinct overrides/ dir for ro/configmap mounts --- meshrr/run.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/meshrr/run.sh b/meshrr/run.sh index bf7c7f9..f4a137c 100644 --- a/meshrr/run.sh +++ b/meshrr/run.sh @@ -48,6 +48,10 @@ elif [ $1 = 'init' ]; then # Generate a fresh SSH key and apply to configuration template. ssh-keygen -q -t ed25519 -f /secret/ssh/id_ed25519 -P "" PUBKEY=`cat \/secret\/ssh\/id_ed25519.pub | tr -d '\r\n'` + # If /opt/meshrr/overrides/juniper.conf.j2 exists, overwrite the default juniper.conf.j2 location. + if test -f overrides/juniper.conf.j2; then + cp overrides/juniper.conf.j2 juniper.conf.j2 + fi sed -i "/user meshrr/,/SECRET-DATA/ s~ssh-ed25519.*~ssh-ed25519 \"$PUBKEY\"; ## SECRET-DATA~" juniper.conf.j2 ./render_config.py -i juniper.conf.j2 -o /config/juniper.conf elif [ $1 = 'sidecar' ]; then From 1902cdd3710ba3111338f2321a3e7f4414b9d11e Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Fri, 13 Oct 2023 09:29:24 -0400 Subject: [PATCH 04/33] 2regions-hrr ds/lothlorien-a for sidecar/init --- examples/2regions-hrr/meshrr-lothlorien.yaml | 84 +++++++++++++------ .../templates/lothlorien-config.j2 | 19 +++++ 2 files changed, 76 insertions(+), 27 deletions(-) diff --git a/examples/2regions-hrr/meshrr-lothlorien.yaml b/examples/2regions-hrr/meshrr-lothlorien.yaml index 5dda168..4db8db9 100644 --- a/examples/2regions-hrr/meshrr-lothlorien.yaml +++ b/examples/2regions-hrr/meshrr-lothlorien.yaml @@ -117,12 +117,10 @@ spec: dnsPolicy: ClusterFirst terminationGracePeriodSeconds: 30 volumes: - - name: crpd-license - secret: - secretName: crpd-license - items: - - key: junos_sfnt.lic - path: junos_sfnt.lic + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} - configMap: defaultMode: 256 items: @@ -131,11 +129,46 @@ spec: path: juniper.conf.j2 name: lothlorien-config optional: false - name: config + name: override-config + initContainers: + - name: meshrr-init + image: meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - mountPath: /opt/meshrr/overrides/juniper.conf.j2 + name: override-config + readOnly: true + subPath: juniper.conf.j2 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license + - name: AUTONOMOUS_SYSTEM + value: '65000' + - name: ENCRYPTED_ROOT_PW + value: >- + $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 + - name: MESH_SERVICE_NAME + value: meshrr-lothlorien + - name: UPSTREAM_SERVICE_NAME + value: meshrr-core + - name: MESHRR_CLIENTRANGE + value: 0/0 containers: - - name: meshrr-lothlorien-a - image: /meshrr:latest - imagePullPolicy: Always + - name: crpd + image: crpd:23.2R1.13 + imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 initialDelaySeconds: 15 @@ -159,34 +192,31 @@ spec: containerPort: 179 protocol: TCP hostIP: 172.19.1.1 + volumeMounts: + - name: config + mountPath: /config/ + securityContext: + allowPrivilegeEscalation: true + privileged: true + runAsNonRoot: false + - name: meshrr + image: meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["sidecar"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - name: MESH_SERVICE_NAME value: meshrr-lothlorien - name: UPSTREAM_SERVICE_NAME value: meshrr-core - name: MESHRR_CLIENTRANGE value: 0/0 - volumeMounts: - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - - mountPath: /root/juniper.conf.j2 - name: config - readOnly: true - subPath: juniper.conf.j2 - securityContext: - allowPrivilegeEscalation: true - privileged: true - runAsNonRoot: false imagePullSecrets: - name: regcred affinity: diff --git a/examples/2regions-hrr/templates/lothlorien-config.j2 b/examples/2regions-hrr/templates/lothlorien-config.j2 index a01e44e..4f41286 100644 --- a/examples/2regions-hrr/templates/lothlorien-config.j2 +++ b/examples/2regions-hrr/templates/lothlorien-config.j2 @@ -2,6 +2,25 @@ system { root-authentication { encrypted-password "{{ ENCRYPTED_ROOT_PW }}"; ## SECRET-DATA } + login { + user meshrr { + class super-user; + uid 100; + authentication { + ssh-ed25519 "ssh-ed25519 TBD"; ## SECRET-DATA + } + } + } + services { + netconf { + ssh; + } + } + license { + keys { + key "{{ LICENSE_KEY }}"; + } + } processes { routing { bgp { From db5f8b901cff1fac6f532f013f80b93b4f1a9072 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Fri, 13 Oct 2023 10:27:14 -0400 Subject: [PATCH 05/33] 2regions-hrr ds/lothlorien-b for sidecar/init --- examples/2regions-hrr/meshrr-lothlorien.yaml | 86 +++++++++++++------- meshrr/run.sh | 2 +- 2 files changed, 59 insertions(+), 29 deletions(-) diff --git a/examples/2regions-hrr/meshrr-lothlorien.yaml b/examples/2regions-hrr/meshrr-lothlorien.yaml index 4db8db9..f42b3d0 100644 --- a/examples/2regions-hrr/meshrr-lothlorien.yaml +++ b/examples/2regions-hrr/meshrr-lothlorien.yaml @@ -261,12 +261,10 @@ spec: dnsPolicy: ClusterFirst terminationGracePeriodSeconds: 30 volumes: - - name: crpd-license - secret: - secretName: crpd-license - items: - - key: junos_sfnt.lic - path: junos_sfnt.lic + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} - configMap: defaultMode: 256 items: @@ -275,11 +273,46 @@ spec: path: juniper.conf.j2 name: lothlorien-config optional: false - name: config + name: override-config + initContainers: + - name: meshrr-init + image: meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - mountPath: /opt/meshrr/overrides/juniper.conf.j2 + name: override-config + readOnly: true + subPath: juniper.conf.j2 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license + - name: AUTONOMOUS_SYSTEM + value: '65000' + - name: ENCRYPTED_ROOT_PW + value: >- + $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 + - name: MESH_SERVICE_NAME + value: meshrr-lothlorien + - name: UPSTREAM_SERVICE_NAME + value: meshrr-core + - name: MESHRR_CLIENTRANGE + value: 0/0 containers: - - name: meshrr-lothlorien-b - image: /meshrr:latest - imagePullPolicy: Always + - name: crpd + image: crpd:23.2R1.13 + imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 initialDelaySeconds: 15 @@ -303,34 +336,31 @@ spec: containerPort: 179 protocol: TCP hostIP: 172.19.1.2 + volumeMounts: + - name: config + mountPath: /config/ + securityContext: + allowPrivilegeEscalation: true + privileged: true + runAsNonRoot: false + - name: meshrr + image: meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["sidecar"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - name: MESH_SERVICE_NAME value: meshrr-lothlorien - name: UPSTREAM_SERVICE_NAME value: meshrr-core - name: MESHRR_CLIENTRANGE - value: 172.18.0.0/16 - volumeMounts: - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - - mountPath: /root/juniper.conf.j2 - name: config - readOnly: true - subPath: juniper.conf.j2 - securityContext: - allowPrivilegeEscalation: true - privileged: true - runAsNonRoot: false + value: 0/0 imagePullSecrets: - name: regcred affinity: diff --git a/meshrr/run.sh b/meshrr/run.sh index f4a137c..c9d9f78 100644 --- a/meshrr/run.sh +++ b/meshrr/run.sh @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright (c) Juniper Networks, Inc., 2020. All rights reserved. +# Copyright (c) Juniper Networks, Inc., 2023. All rights reserved. # # Notice and Disclaimer: This code is licensed to you under the MIT License (the # "License"). You may not use this code except in compliance with the License. From c56dc0ea750bad60d9953f442487b7881ab7badd Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Fri, 13 Oct 2023 11:10:06 -0400 Subject: [PATCH 06/33] Updated image paths --- examples/2regions-hrr/meshrr-core.yaml | 6 +++--- examples/2regions-hrr/meshrr-lothlorien.yaml | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/2regions-hrr/meshrr-core.yaml b/examples/2regions-hrr/meshrr-core.yaml index 0482239..9f1a807 100644 --- a/examples/2regions-hrr/meshrr-core.yaml +++ b/examples/2regions-hrr/meshrr-core.yaml @@ -53,7 +53,7 @@ spec: emptyDir: {} initContainers: - name: meshrr-init - image: meshrr:v0.2 + image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["init"] volumeMounts: @@ -82,7 +82,7 @@ spec: value: 0/0 containers: - name: crpd - image: crpd:23.2R1.13 + image: localhost/juniper/crpd:23.2R1.13 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 @@ -114,7 +114,7 @@ spec: privileged: true runAsNonRoot: false - name: meshrr - image: meshrr:v0.2 + image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["sidecar"] volumeMounts: diff --git a/examples/2regions-hrr/meshrr-lothlorien.yaml b/examples/2regions-hrr/meshrr-lothlorien.yaml index f42b3d0..2161219 100644 --- a/examples/2regions-hrr/meshrr-lothlorien.yaml +++ b/examples/2regions-hrr/meshrr-lothlorien.yaml @@ -132,7 +132,7 @@ spec: name: override-config initContainers: - name: meshrr-init - image: meshrr:v0.2 + image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["init"] volumeMounts: @@ -167,7 +167,7 @@ spec: value: 0/0 containers: - name: crpd - image: crpd:23.2R1.13 + image: localhost/juniper/crpd:23.2R1.13 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 @@ -200,7 +200,7 @@ spec: privileged: true runAsNonRoot: false - name: meshrr - image: meshrr:v0.2 + image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["sidecar"] volumeMounts: @@ -276,7 +276,7 @@ spec: name: override-config initContainers: - name: meshrr-init - image: meshrr:v0.2 + image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["init"] volumeMounts: @@ -311,7 +311,7 @@ spec: value: 0/0 containers: - name: crpd - image: crpd:23.2R1.13 + image: localhost/juniper/crpd:23.2R1.13 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 @@ -344,7 +344,7 @@ spec: privileged: true runAsNonRoot: false - name: meshrr - image: meshrr:v0.2 + image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["sidecar"] volumeMounts: From 5232bbddf377153e5ecc2f56c299f1cfca0c362a Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Fri, 13 Oct 2023 12:20:29 -0400 Subject: [PATCH 07/33] Don't restart meshrr sidecars pending crpd --- examples/2regions-hrr/meshrr-core.yaml | 7 +++ examples/2regions-hrr/meshrr-lothlorien.yaml | 14 ++++++ meshrr/Dockerfile | 2 + meshrr/connect_wait.py | 49 ++++++++++++++++++++ meshrr/run.sh | 2 + 5 files changed, 74 insertions(+) create mode 100644 meshrr/connect_wait.py diff --git a/examples/2regions-hrr/meshrr-core.yaml b/examples/2regions-hrr/meshrr-core.yaml index 9f1a807..982670f 100644 --- a/examples/2regions-hrr/meshrr-core.yaml +++ b/examples/2regions-hrr/meshrr-core.yaml @@ -117,6 +117,13 @@ spec: image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 volumeMounts: - name: ssh-id mountPath: /secret/ssh/ diff --git a/examples/2regions-hrr/meshrr-lothlorien.yaml b/examples/2regions-hrr/meshrr-lothlorien.yaml index 2161219..a21dc91 100644 --- a/examples/2regions-hrr/meshrr-lothlorien.yaml +++ b/examples/2regions-hrr/meshrr-lothlorien.yaml @@ -203,6 +203,13 @@ spec: image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 volumeMounts: - name: ssh-id mountPath: /secret/ssh/ @@ -347,6 +354,13 @@ spec: image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 volumeMounts: - name: ssh-id mountPath: /secret/ssh/ diff --git a/meshrr/Dockerfile b/meshrr/Dockerfile index 906c858..f4333c8 100644 --- a/meshrr/Dockerfile +++ b/meshrr/Dockerfile @@ -43,8 +43,10 @@ COPY render_config.py /opt/meshrr/ COPY run.sh /opt/meshrr/ COPY juniper.conf.j2 /opt/meshrr/ COPY update_peers.py /opt/meshrr/ +COPY connect_wait.py /opt/meshrr/ RUN chmod +x /opt/meshrr/run.sh +RUN chmod +x /opt/meshrr/connect_wait.py RUN chmod +x /opt/meshrr/update_peers.py RUN chmod +x /opt/meshrr/render_config.py diff --git a/meshrr/connect_wait.py b/meshrr/connect_wait.py new file mode 100644 index 0000000..db4de0b --- /dev/null +++ b/meshrr/connect_wait.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +# Copyright (c) Juniper Networks, Inc., 2023. All rights reserved. +# +# Notice and Disclaimer: This code is licensed to you under the MIT License (the +# "License"). You may not use this code except in compliance with the License. +# This code is not an official Juniper product. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Third-Party Code: This code may depend on other components under separate +# copyright notice and license terms. Your use of the source code for those +# components is subject to the terms and conditions of the respective license as +# noted in the Third-Party source code file. + +from jnpr.junos import Device +from jnpr.junos.exception import ConnectTimeoutError,ConnectRefusedError +from time import sleep + +if __name__ == "__main__": + # Attempt to open a connection to the device + dev = Device(host="127.0.0.1",user="meshrr",ssh_private_key_file="/secret/ssh/id_ed25519") + while True: + try: + dev.open() + break + except ConnectTimeoutError: + print("connect_wait.py: Connection timed out; retrying.") + except ConnectRefusedError: + print("connect_wait.py: Connection refused; retrying.") + sleep(1) + # Create /tmp/connected-to-crpd to inform startup probes + open("/tmp/connected-to-crpd","x") \ No newline at end of file diff --git a/meshrr/run.sh b/meshrr/run.sh index c9d9f78..78e68fa 100644 --- a/meshrr/run.sh +++ b/meshrr/run.sh @@ -55,6 +55,8 @@ elif [ $1 = 'init' ]; then sed -i "/user meshrr/,/SECRET-DATA/ s~ssh-ed25519.*~ssh-ed25519 \"$PUBKEY\"; ## SECRET-DATA~" juniper.conf.j2 ./render_config.py -i juniper.conf.j2 -o /config/juniper.conf elif [ $1 = 'sidecar' ]; then + # Wait for cRPD container to become available for netconf. + ./connect_wait.py echo "Initializing peer maintenance every ${UPDATE_SECONDS:=30} seconds." while true; do ./update_peers.py From 760c0dd85d17e6deffcfcf0a121a8345a1e46d2c Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Mon, 16 Oct 2023 10:27:36 -0400 Subject: [PATCH 08/33] Mirkwood on LBs and multiple containers --- examples/2regions-hrr/meshrr-lothlorien.yaml | 2 - examples/2regions-hrr/meshrr-mirkwood.yaml | 260 ++++++++++++++---- .../2regions-hrr/templates/mirkwood-config.j2 | 19 ++ 3 files changed, 219 insertions(+), 62 deletions(-) diff --git a/examples/2regions-hrr/meshrr-lothlorien.yaml b/examples/2regions-hrr/meshrr-lothlorien.yaml index a21dc91..7b81cd7 100644 --- a/examples/2regions-hrr/meshrr-lothlorien.yaml +++ b/examples/2regions-hrr/meshrr-lothlorien.yaml @@ -191,7 +191,6 @@ spec: - name: bgp containerPort: 179 protocol: TCP - hostIP: 172.19.1.1 volumeMounts: - name: config mountPath: /config/ @@ -342,7 +341,6 @@ spec: - name: bgp containerPort: 179 protocol: TCP - hostIP: 172.19.1.2 volumeMounts: - name: config mountPath: /config/ diff --git a/examples/2regions-hrr/meshrr-mirkwood.yaml b/examples/2regions-hrr/meshrr-mirkwood.yaml index a45789b..da5e992 100644 --- a/examples/2regions-hrr/meshrr-mirkwood.yaml +++ b/examples/2regions-hrr/meshrr-mirkwood.yaml @@ -1,4 +1,74 @@ --- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: meshrr-mirkwood + namespace: metallb +spec: + addresses: + - 172.19.2.1/32 + - 172.19.2.2/32 + autoAssign: false +--- +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: meshrr-mirkwood + namespace: metallb +spec: + ipAddressPools: + - meshrr-mirkwood +--- +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + name: meshrr-mirkwood-a + selfLink: /api/v1/namespaces/default/services/meshrr-mirkwood-a + annotations: + metallb.universe.tf/address-pool: meshrr-mirkwood +spec: + ports: + - name: bgp + port: 179 + protocol: TCP + targetPort: bgp + selector: + app: meshrr + meshrr_region_mirkwood: "true" + redundancy_group: a + sessionAffinity: None + type: LoadBalancer + loadBalancerIP: 172.19.2.1 + externalTrafficPolicy: Local +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + name: meshrr-mirkwood-b + selfLink: /api/v1/namespaces/default/services/meshrr-mirkwood-b + annotations: + metallb.universe.tf/address-pool: meshrr-mirkwood +spec: + ports: + - name: bgp + port: 179 + protocol: TCP + targetPort: bgp + selector: + app: meshrr + meshrr_region_mirkwood: "true" + redundancy_group: b + sessionAffinity: None + type: LoadBalancer + loadBalancerIP: 172.19.2.2 + externalTrafficPolicy: Local +status: + loadBalancer: {} +--- apiVersion: v1 kind: Service metadata: @@ -48,12 +118,10 @@ spec: dnsPolicy: ClusterFirst terminationGracePeriodSeconds: 30 volumes: - - name: crpd-license - secret: - secretName: crpd-license - items: - - key: junos_sfnt.lic - path: junos_sfnt.lic + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} - configMap: defaultMode: 256 items: @@ -62,11 +130,46 @@ spec: path: juniper.conf.j2 name: mirkwood-config optional: false - name: config + name: override-config + initContainers: + - name: meshrr-init + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - mountPath: /opt/meshrr/overrides/juniper.conf.j2 + name: override-config + readOnly: true + subPath: juniper.conf.j2 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license + - name: AUTONOMOUS_SYSTEM + value: '65000' + - name: ENCRYPTED_ROOT_PW + value: >- + $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 + - name: MESH_SERVICE_NAME + value: meshrr-mirkwood + - name: UPSTREAM_SERVICE_NAME + value: meshrr-core + - name: MESHRR_CLIENTRANGE + value: 0/0 containers: - - name: meshrr-mirkwood-a - image: /meshrr:latest - imagePullPolicy: Always + - name: crpd + image: localhost/juniper/crpd:23.2R1.13 + imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 initialDelaySeconds: 15 @@ -89,36 +192,38 @@ spec: - name: bgp containerPort: 179 protocol: TCP - hostIP: 172.19.2.1 - hostPort: 179 + volumeMounts: + - name: config + mountPath: /config/ + securityContext: + allowPrivilegeEscalation: true + privileged: true + runAsNonRoot: false + - name: meshrr + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - name: MESH_SERVICE_NAME value: meshrr-mirkwood - name: UPSTREAM_SERVICE_NAME value: meshrr-core - name: MESHRR_CLIENTRANGE - value: 172.18.0.0/16 - volumeMounts: - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - - mountPath: /root/juniper.conf.j2 - name: config - readOnly: true - subPath: juniper.conf.j2 - securityContext: - allowPrivilegeEscalation: true - privileged: true - runAsNonRoot: false + value: 0/0 imagePullSecrets: - name: regcred affinity: @@ -163,12 +268,10 @@ spec: dnsPolicy: ClusterFirst terminationGracePeriodSeconds: 30 volumes: - - name: crpd-license - secret: - secretName: crpd-license - items: - - key: junos_sfnt.lic - path: junos_sfnt.lic + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} - configMap: defaultMode: 256 items: @@ -177,11 +280,46 @@ spec: path: juniper.conf.j2 name: mirkwood-config optional: false - name: config + name: override-config + initContainers: + - name: meshrr-init + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - mountPath: /opt/meshrr/overrides/juniper.conf.j2 + name: override-config + readOnly: true + subPath: juniper.conf.j2 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license + - name: AUTONOMOUS_SYSTEM + value: '65000' + - name: ENCRYPTED_ROOT_PW + value: >- + $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 + - name: MESH_SERVICE_NAME + value: meshrr-mirkwood + - name: UPSTREAM_SERVICE_NAME + value: meshrr-core + - name: MESHRR_CLIENTRANGE + value: 0/0 containers: - - name: meshrr-mirkwood-b - image: /meshrr:latest - imagePullPolicy: Always + - name: crpd + image: localhost/juniper/crpd:23.2R1.13 + imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 initialDelaySeconds: 15 @@ -204,36 +342,38 @@ spec: - name: bgp containerPort: 179 protocol: TCP - hostIP: 172.19.2.2 - hostPort: 179 + volumeMounts: + - name: config + mountPath: /config/ + securityContext: + allowPrivilegeEscalation: true + privileged: true + runAsNonRoot: false + - name: meshrr + image: ghcr.io/juniper/meshrr:v0.2 + imagePullPolicy: IfNotPresent + args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - name: MESH_SERVICE_NAME value: meshrr-mirkwood - name: UPSTREAM_SERVICE_NAME value: meshrr-core - name: MESHRR_CLIENTRANGE - value: 172.18.0.0/16 - volumeMounts: - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - - mountPath: /root/juniper.conf.j2 - name: config - readOnly: true - subPath: juniper.conf.j2 - securityContext: - allowPrivilegeEscalation: true - privileged: true - runAsNonRoot: false + value: 0/0 imagePullSecrets: - name: regcred affinity: diff --git a/examples/2regions-hrr/templates/mirkwood-config.j2 b/examples/2regions-hrr/templates/mirkwood-config.j2 index a01e44e..4f41286 100644 --- a/examples/2regions-hrr/templates/mirkwood-config.j2 +++ b/examples/2regions-hrr/templates/mirkwood-config.j2 @@ -2,6 +2,25 @@ system { root-authentication { encrypted-password "{{ ENCRYPTED_ROOT_PW }}"; ## SECRET-DATA } + login { + user meshrr { + class super-user; + uid 100; + authentication { + ssh-ed25519 "ssh-ed25519 TBD"; ## SECRET-DATA + } + } + } + services { + netconf { + ssh; + } + } + license { + keys { + key "{{ LICENSE_KEY }}"; + } + } processes { routing { bgp { From e594c4e71eeec104a5ae5727d2359ace5f25efb0 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Mon, 16 Oct 2023 10:29:28 -0400 Subject: [PATCH 09/33] Image pull cleanup --- examples/2regions-hrr/meshrr-lothlorien.yaml | 8 ++------ examples/2regions-hrr/meshrr-mirkwood.yaml | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/examples/2regions-hrr/meshrr-lothlorien.yaml b/examples/2regions-hrr/meshrr-lothlorien.yaml index 7b81cd7..05a9d00 100644 --- a/examples/2regions-hrr/meshrr-lothlorien.yaml +++ b/examples/2regions-hrr/meshrr-lothlorien.yaml @@ -168,7 +168,7 @@ spec: containers: - name: crpd image: localhost/juniper/crpd:23.2R1.13 - imagePullPolicy: IfNotPresent + imagePullPolicy: Never livenessProbe: failureThreshold: 3 initialDelaySeconds: 15 @@ -223,8 +223,6 @@ spec: value: meshrr-core - name: MESHRR_CLIENTRANGE value: 0/0 - imagePullSecrets: - - name: regcred affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -318,7 +316,7 @@ spec: containers: - name: crpd image: localhost/juniper/crpd:23.2R1.13 - imagePullPolicy: IfNotPresent + imagePullPolicy: Never livenessProbe: failureThreshold: 3 initialDelaySeconds: 15 @@ -373,8 +371,6 @@ spec: value: meshrr-core - name: MESHRR_CLIENTRANGE value: 0/0 - imagePullSecrets: - - name: regcred affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/examples/2regions-hrr/meshrr-mirkwood.yaml b/examples/2regions-hrr/meshrr-mirkwood.yaml index da5e992..b00bd42 100644 --- a/examples/2regions-hrr/meshrr-mirkwood.yaml +++ b/examples/2regions-hrr/meshrr-mirkwood.yaml @@ -169,7 +169,7 @@ spec: containers: - name: crpd image: localhost/juniper/crpd:23.2R1.13 - imagePullPolicy: IfNotPresent + imagePullPolicy: Never livenessProbe: failureThreshold: 3 initialDelaySeconds: 15 @@ -224,8 +224,6 @@ spec: value: meshrr-core - name: MESHRR_CLIENTRANGE value: 0/0 - imagePullSecrets: - - name: regcred affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -319,7 +317,7 @@ spec: containers: - name: crpd image: localhost/juniper/crpd:23.2R1.13 - imagePullPolicy: IfNotPresent + imagePullPolicy: Never livenessProbe: failureThreshold: 3 initialDelaySeconds: 15 @@ -374,8 +372,6 @@ spec: value: meshrr-core - name: MESHRR_CLIENTRANGE value: 0/0 - imagePullSecrets: - - name: regcred affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: From a368f0f2717e9cb43470b0403cefd53010e3f803 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Tue, 17 Oct 2023 12:49:48 -0400 Subject: [PATCH 10/33] Restructuring --- LICENSE | 2 +- meshrr/Dockerfile | 8 +- meshrr/LICENSE | 21 ++++++ meshrr/defaults/juniper-evpnrs.conf.j2 | 100 +++++++++++++++++++++++++ meshrr/defaults/juniper-ipv4rr.conf.j2 | 100 +++++++++++++++++++++++++ meshrr/run.sh | 25 ++++--- 6 files changed, 243 insertions(+), 13 deletions(-) create mode 100644 meshrr/LICENSE create mode 100644 meshrr/defaults/juniper-evpnrs.conf.j2 create mode 100644 meshrr/defaults/juniper-ipv4rr.conf.j2 diff --git a/LICENSE b/LICENSE index f0d93c9..3e5398d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) Juniper Networks, Inc. 2020 +Copyright (c) Juniper Networks, Inc. 2023 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/meshrr/Dockerfile b/meshrr/Dockerfile index f4333c8..195a5b5 100644 --- a/meshrr/Dockerfile +++ b/meshrr/Dockerfile @@ -34,16 +34,18 @@ WORKDIR /opt/meshrr COPY requirements.txt /opt/meshrr/ RUN apk add --update --no-cache python3 py3-pip openssh \ - && pip3 install -r requirements.txt \ - && mkdir -p /config /secret/ssh + && pip3 install -r requirements.txt FROM meshrr-prebuild +RUN mkdir -p /config /secret/ssh conf + COPY render_config.py /opt/meshrr/ COPY run.sh /opt/meshrr/ -COPY juniper.conf.j2 /opt/meshrr/ +COPY defaults/ /opt/meshrr/defaults/ COPY update_peers.py /opt/meshrr/ COPY connect_wait.py /opt/meshrr/ +COPY ../LICENSE /opt/meshrr/ RUN chmod +x /opt/meshrr/run.sh RUN chmod +x /opt/meshrr/connect_wait.py diff --git a/meshrr/LICENSE b/meshrr/LICENSE new file mode 100644 index 0000000..3e5398d --- /dev/null +++ b/meshrr/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Juniper Networks, Inc. 2023 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/meshrr/defaults/juniper-evpnrs.conf.j2 b/meshrr/defaults/juniper-evpnrs.conf.j2 new file mode 100644 index 0000000..9a561c7 --- /dev/null +++ b/meshrr/defaults/juniper-evpnrs.conf.j2 @@ -0,0 +1,100 @@ +system { + root-authentication { + encrypted-password "{{ ENCRYPTED_ROOT_PW }}"; ## SECRET-DATA + } + login { + user meshrr { + class super-user; + uid 100; + authentication { + ssh-ed25519 "ssh-ed25519 TBD"; ## SECRET-DATA + } + } + } + /* Required for configuration persistence */ + no-compress-configuration-files; + services { + netconf { + ssh; + } + } + license { + keys { + key "{{ LICENSE_KEY }}"; + } + } + processes { + routing { + bgp { + rib-sharding; + update-threading; + } + } + } +} +routing-options { + autonomous-system {{ AUTONOMOUS_SYSTEM }}; + router-id {{ POD_IP }}; +} +groups { + MESHRR { + {% if MESHRR_MODE == 'routeserver' %}policy-options { + as-list MESHRR-RSCLIENTS members {{ MESHRR_ASRANGE }}; + } + {% endif %}protocols { + bgp { + {% if MESHRR_FAMILY_INET != "false" %}family inet { + unicast { + nexthop-resolution { + no-resolution; + } + no-install; + } + }{% endif %} + {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { + signaling { + nexthop-resolution { + no-resolution; + } + no-install; + } + }{% endif %} + group MESHRR-MESH { + type internal + {% if MESHRR_FAMILY_INET != "false" %}family inet { + unicast; + }{% endif %} + {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { + signaling; + }{% endif %} + } + group MESHRR-CLIENTS { + {% if MESHRR_MODE == 'routeserver' %}type external; + multihop { + ttl 10; + no-nexthop-change; + } + {% if MESHRR_FAMILY_INET != "false" %}family inet { + unicast; + }{% endif %} + {% if MESHRR_FAMILY_EVPN %}family evpn { + signaling; + }{% endif %} + peer-as-list MESHRR-RSCLIENTS;{% else %}type internal; + cluster {{ POD_IP }};{% endif %} + allow {{ MESHRR_CLIENTRANGE }}; + }{% if UPSTREAM_SERVICE_NAME is defined and UPSTREAM_SERVICE_NAME is not none %} + group MESHRR-UPSTREAM { + type internal; + {% if MESHRR_FAMILY_INET != "false" %}family inet { + unicast; + }{% endif %} + {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { + signaling; + }{% endif %} + }{% endif %} + } + } + } +} +apply-groups MESHRR; diff --git a/meshrr/defaults/juniper-ipv4rr.conf.j2 b/meshrr/defaults/juniper-ipv4rr.conf.j2 new file mode 100644 index 0000000..9a561c7 --- /dev/null +++ b/meshrr/defaults/juniper-ipv4rr.conf.j2 @@ -0,0 +1,100 @@ +system { + root-authentication { + encrypted-password "{{ ENCRYPTED_ROOT_PW }}"; ## SECRET-DATA + } + login { + user meshrr { + class super-user; + uid 100; + authentication { + ssh-ed25519 "ssh-ed25519 TBD"; ## SECRET-DATA + } + } + } + /* Required for configuration persistence */ + no-compress-configuration-files; + services { + netconf { + ssh; + } + } + license { + keys { + key "{{ LICENSE_KEY }}"; + } + } + processes { + routing { + bgp { + rib-sharding; + update-threading; + } + } + } +} +routing-options { + autonomous-system {{ AUTONOMOUS_SYSTEM }}; + router-id {{ POD_IP }}; +} +groups { + MESHRR { + {% if MESHRR_MODE == 'routeserver' %}policy-options { + as-list MESHRR-RSCLIENTS members {{ MESHRR_ASRANGE }}; + } + {% endif %}protocols { + bgp { + {% if MESHRR_FAMILY_INET != "false" %}family inet { + unicast { + nexthop-resolution { + no-resolution; + } + no-install; + } + }{% endif %} + {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { + signaling { + nexthop-resolution { + no-resolution; + } + no-install; + } + }{% endif %} + group MESHRR-MESH { + type internal + {% if MESHRR_FAMILY_INET != "false" %}family inet { + unicast; + }{% endif %} + {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { + signaling; + }{% endif %} + } + group MESHRR-CLIENTS { + {% if MESHRR_MODE == 'routeserver' %}type external; + multihop { + ttl 10; + no-nexthop-change; + } + {% if MESHRR_FAMILY_INET != "false" %}family inet { + unicast; + }{% endif %} + {% if MESHRR_FAMILY_EVPN %}family evpn { + signaling; + }{% endif %} + peer-as-list MESHRR-RSCLIENTS;{% else %}type internal; + cluster {{ POD_IP }};{% endif %} + allow {{ MESHRR_CLIENTRANGE }}; + }{% if UPSTREAM_SERVICE_NAME is defined and UPSTREAM_SERVICE_NAME is not none %} + group MESHRR-UPSTREAM { + type internal; + {% if MESHRR_FAMILY_INET != "false" %}family inet { + unicast; + }{% endif %} + {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { + signaling; + }{% endif %} + }{% endif %} + } + } + } +} +apply-groups MESHRR; diff --git a/meshrr/run.sh b/meshrr/run.sh index 78e68fa..a1a0afc 100644 --- a/meshrr/run.sh +++ b/meshrr/run.sh @@ -41,19 +41,26 @@ elif [ $1 = 'init' ]; then # Overwrite with existing configuration if it exists. if [ -f "/config/juniper.conf" ]; then echo "Existing configuration detected; overwriting pod IP only." - sed "s/^\(.\+router-id\) [0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+/\1 {{ POD_IP }};/" /config/juniper.conf > juniper.conf.j2 - else - echo "Initializing fresh configuration from template." + sed "s/^\(.\+router-id\) [0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+/\1 {{ POD_IP }};/" /config/juniper.conf > conf/juniper.conf.j2 + elif ! test -f conf/juniper.conf.j2; then + if [[ "$MESHRR_MODE" == 'evpnrs' ]] || [[ "$MESHRR_MODE" == 'routeserver' ]] ; then + template='juniper-evpnrs.conf.j2' + elif [[ "$MESHRR_MODE" == 'ipv4rr' ]] || [[ "$MESHRR_MODE" == 'routereflector' ]] || [[ -z $MESHRR_MODE ]]; then + template='juniper-ipv4rr.conf.j2' + else + echo "Invalid MESHRR_MODE set. Must be empty or one of [ 'evpnrs' 'ipv4rr' ]." + exit 1 + fi + echo "Initializing fresh configuration from defaults/${template}." + cp defaults/${template} conf/juniper.conf.j2 + else # Don't overwrite template since it's been explicitly mounted. + echo "Custom configuration provided; using conf/juniper.conf.j2 template." fi # Generate a fresh SSH key and apply to configuration template. ssh-keygen -q -t ed25519 -f /secret/ssh/id_ed25519 -P "" PUBKEY=`cat \/secret\/ssh\/id_ed25519.pub | tr -d '\r\n'` - # If /opt/meshrr/overrides/juniper.conf.j2 exists, overwrite the default juniper.conf.j2 location. - if test -f overrides/juniper.conf.j2; then - cp overrides/juniper.conf.j2 juniper.conf.j2 - fi - sed -i "/user meshrr/,/SECRET-DATA/ s~ssh-ed25519.*~ssh-ed25519 \"$PUBKEY\"; ## SECRET-DATA~" juniper.conf.j2 - ./render_config.py -i juniper.conf.j2 -o /config/juniper.conf + sed -i "/user meshrr/,/SECRET-DATA/ s~ssh-ed25519.*~ssh-ed25519 \"$PUBKEY\"; ## SECRET-DATA~" conf/juniper.conf.j2 + ./render_config.py -i conf/juniper.conf.j2 -o /config/juniper.conf elif [ $1 = 'sidecar' ]; then # Wait for cRPD container to become available for netconf. ./connect_wait.py From 32a8bc17227c39a553beb3a14a6663e8496f4475 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Thu, 19 Oct 2023 14:18:31 -0400 Subject: [PATCH 11/33] Initial YAML-driven config; 2region-hrr core only --- examples/2regions-hrr/meshrr-core.yaml | 53 +++++-- meshrr/Dockerfile | 1 + meshrr/config.py | 78 +++++++++++ meshrr/connect_wait.py | 3 +- meshrr/defaults/juniper-evpnrs.conf.j2 | 66 ++++----- meshrr/defaults/juniper-ipv4rr.conf.j2 | 64 +++------ meshrr/defaults/meshrr.conf.yml | 22 +++ meshrr/render_config.py | 31 ++-- meshrr/update_peers.py | 187 +++++++++++-------------- 9 files changed, 276 insertions(+), 229 deletions(-) create mode 100755 meshrr/config.py create mode 100644 meshrr/defaults/meshrr.conf.yml diff --git a/examples/2regions-hrr/meshrr-core.yaml b/examples/2regions-hrr/meshrr-core.yaml index 982670f..9100740 100644 --- a/examples/2regions-hrr/meshrr-core.yaml +++ b/examples/2regions-hrr/meshrr-core.yaml @@ -1,5 +1,26 @@ --- apiVersion: v1 +kind: ConfigMap +metadata: + name: meshrr-core-conf +data: + meshrr.conf.yml: |+ + encrypted_root_pw: NOLOGIN + asn: "65000" + mode: ipv4rr + bgpgroups: + - name: MESHRR-MESH + type: mesh + source: + sourcetype: dns + hostname: meshrr-core + - name: MESHRR-CLIENTS + type: subtractive + prefixes: + - 10.42.0.0/16 + +--- +apiVersion: v1 kind: Service metadata: creationTimestamp: null @@ -51,6 +72,15 @@ spec: emptyDir: {} - name: ssh-id emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 + items: + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-core-conf + optional: false initContainers: - name: meshrr-init image: ghcr.io/juniper/meshrr:v0.2 @@ -61,6 +91,10 @@ spec: mountPath: /secret/ssh/ - name: config mountPath: /config/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml env: - name: POD_IP valueFrom: @@ -71,15 +105,6 @@ spec: secretKeyRef: name: crpd-license key: crpd-license - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 0/0 containers: - name: crpd image: localhost/juniper/crpd:23.2R1.13 @@ -127,17 +152,15 @@ spec: volumeMounts: - name: ssh-id mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: MESH_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 0/0 - imagePullSecrets: - - name: regcred affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/meshrr/Dockerfile b/meshrr/Dockerfile index 195a5b5..b9ed877 100644 --- a/meshrr/Dockerfile +++ b/meshrr/Dockerfile @@ -40,6 +40,7 @@ FROM meshrr-prebuild RUN mkdir -p /config /secret/ssh conf +COPY config.py /opt/meshrr/ COPY render_config.py /opt/meshrr/ COPY run.sh /opt/meshrr/ COPY defaults/ /opt/meshrr/defaults/ diff --git a/meshrr/config.py b/meshrr/config.py new file mode 100755 index 0000000..30874ec --- /dev/null +++ b/meshrr/config.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +# Copyright (c) Juniper Networks, Inc., 2023. All rights reserved. +# +# Notice and Disclaimer: This code is licensed to you under the MIT License (the +# "License"). You may not use this code except in compliance with the License. +# This code is not an official Juniper product. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Third-Party Code: This code may depend on other components under separate +# copyright notice and license terms. Your use of the source code for those +# components is subject to the terms and conditions of the respective license as +# noted in the Third-Party source code file. + +import yaml +from yaml import SafeLoader +from os import getenv +from dotenv import load_dotenv +from os.path import isfile + +load_dotenv(dotenv_path="/etc/envvars") + +class Meshrrconfig: + def __init__(self): + if not isfile('/opt/meshrr/conf/meshrr.conf.yml'): + # Start from default config + with open('/opt/meshrr/defaults/meshrr.conf.yml') as d: + c = yaml.load(d,Loader=SafeLoader) + + # Copy in defined environment variables. + c.update({ + 'encrypted_root_pw': getenv("ENCRYPTED_ROOT_PW", c['encrypted_root_pw']), + 'asn': getenv("AUTONOMOUS_SYSTEM", c['asn']), + 'mode': getenv("MESHRR_MODE", c['mode']), + }) + + # Save to live config + with open('/opt/meshrr/conf/meshrr.conf.yml','w') as w: + yaml.dump(c,w) + with open('/opt/meshrr/conf/meshrr.conf.yml') as f: + self._config = yaml.load(f,Loader=SafeLoader) + + self.encrypted_root_pw = self._config['encrypted_root_pw'] + self.asn = self._config['asn'] + self.mode = self._config['mode'] + + self.bgpgroups_mesh = list() + self.bgpgroups_subtractive = list() + for group in self._config['bgpgroups']: + if 'type' not in group or 'name' not in group: + raise Exception("All bgpgroups require 'type' and 'name' definitions. Offending bgpgroup: "+str(group)) + elif group['type'].casefold() == 'mesh': + if 'source' not in group: + raise Exception("Mesh bgpgroups require 'source.hostname' definitions. Offending bgpgroup: "+str(group)) + self.bgpgroups_mesh.append(group) + elif group['type'].casefold() == 'subtractive': + if 'prefixes' not in group: + raise(Exception("Subtractive bgpgroups require 'source.hostname' definitions. Offending bgpgroup: "+str(group))) + self.bgpgroups_subtractive.append(group) + else: + raise(Exception("Invalid `type` in bgpgroup: "+str(group))) \ No newline at end of file diff --git a/meshrr/connect_wait.py b/meshrr/connect_wait.py index db4de0b..bf8ce82 100644 --- a/meshrr/connect_wait.py +++ b/meshrr/connect_wait.py @@ -46,4 +46,5 @@ print("connect_wait.py: Connection refused; retrying.") sleep(1) # Create /tmp/connected-to-crpd to inform startup probes - open("/tmp/connected-to-crpd","x") \ No newline at end of file + open("/tmp/connected-to-crpd","x") + \ No newline at end of file diff --git a/meshrr/defaults/juniper-evpnrs.conf.j2 b/meshrr/defaults/juniper-evpnrs.conf.j2 index 9a561c7..c94aa22 100644 --- a/meshrr/defaults/juniper-evpnrs.conf.j2 +++ b/meshrr/defaults/juniper-evpnrs.conf.j2 @@ -1,6 +1,6 @@ system { root-authentication { - encrypted-password "{{ ENCRYPTED_ROOT_PW }}"; ## SECRET-DATA + encrypted-password "{{ encrypted_root_pw }}"; ## SECRET-DATA } login { user meshrr { @@ -33,66 +33,50 @@ system { } } routing-options { - autonomous-system {{ AUTONOMOUS_SYSTEM }}; + autonomous-system {{ asn }}; router-id {{ POD_IP }}; } groups { MESHRR { - {% if MESHRR_MODE == 'routeserver' %}policy-options { - as-list MESHRR-RSCLIENTS members {{ MESHRR_ASRANGE }}; + policy-options { + {%- for bgpgroup in bgpgroups_subtractive %} + as-list MESHRR:{{ bgpgroup.name }} members [ {% for asrange in bgpgroup.asranges %}{{ asrange }} {% endfor %}]; + {%- endfor %} } - {% endif %}protocols { + protocols { bgp { - {% if MESHRR_FAMILY_INET != "false" %}family inet { - unicast { - nexthop-resolution { - no-resolution; - } - no-install; - } - }{% endif %} - {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { + family evpn { signaling { nexthop-resolution { no-resolution; } no-install; } - }{% endif %} - group MESHRR-MESH { + } + }{%- for bgpgroup in bgpgroups_mesh %} + /* Mesh {% if bgpgroup.max_peers is defined %}max_peers:{{ bgpgroup.max_peers }} {% endif %}group from {{ bgpgroup.source.sourcetype }}:{{ bgpgroup.source.hostname }} */ + group {{ bgpgroup.name }} { type internal - {% if MESHRR_FAMILY_INET != "false" %}family inet { - unicast; - }{% endif %} - {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { + family evpn { signaling; - }{% endif %} + } } - group MESHRR-CLIENTS { - {% if MESHRR_MODE == 'routeserver' %}type external; + {%- endfor %} + {%- for bgpgroup in bgpgroups_subtractive %} + /* Subtractive group from {{ bgpgroup.prefixes }} */ + group {{ bgpgroup.name }} { + type external; multihop { ttl 10; no-nexthop-change; } - {% if MESHRR_FAMILY_INET != "false" %}family inet { - unicast; - }{% endif %} - {% if MESHRR_FAMILY_EVPN %}family evpn { - signaling; - }{% endif %} - peer-as-list MESHRR-RSCLIENTS;{% else %}type internal; - cluster {{ POD_IP }};{% endif %} - allow {{ MESHRR_CLIENTRANGE }}; - }{% if UPSTREAM_SERVICE_NAME is defined and UPSTREAM_SERVICE_NAME is not none %} - group MESHRR-UPSTREAM { - type internal; - {% if MESHRR_FAMILY_INET != "false" %}family inet { - unicast; - }{% endif %} - {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { + family evpn { signaling; - }{% endif %} - }{% endif %} + } + peer-as-list MESHRR:{{ bgpgroup.name }}; + allow [ {% for prefix in bgpgroup.prefixes %}{{ prefix }} {% endfor %}]; + } + {%- endfor %} } } } diff --git a/meshrr/defaults/juniper-ipv4rr.conf.j2 b/meshrr/defaults/juniper-ipv4rr.conf.j2 index 9a561c7..0a2553e 100644 --- a/meshrr/defaults/juniper-ipv4rr.conf.j2 +++ b/meshrr/defaults/juniper-ipv4rr.conf.j2 @@ -1,6 +1,6 @@ system { root-authentication { - encrypted-password "{{ ENCRYPTED_ROOT_PW }}"; ## SECRET-DATA + encrypted-password "{{ encrypted_root_pw }}"; ## SECRET-DATA } login { user meshrr { @@ -33,66 +33,38 @@ system { } } routing-options { - autonomous-system {{ AUTONOMOUS_SYSTEM }}; + autonomous-system {{ asn }}; router-id {{ POD_IP }}; } groups { MESHRR { - {% if MESHRR_MODE == 'routeserver' %}policy-options { - as-list MESHRR-RSCLIENTS members {{ MESHRR_ASRANGE }}; - } - {% endif %}protocols { + protocols { bgp { - {% if MESHRR_FAMILY_INET != "false" %}family inet { + family inet { unicast { nexthop-resolution { no-resolution; } no-install; } - }{% endif %} - {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { - signaling { - nexthop-resolution { - no-resolution; - } - no-install; - } - }{% endif %} - group MESHRR-MESH { + } + {%- for bgpgroup in bgpgroups_mesh %} + /* Mesh {% if bgpgroup.max_peers is defined %}max_peers:{{ bgpgroup.max_peers }} {% endif %}group from {{ bgpgroup.source.sourcetype }}:{{ bgpgroup.source.hostname }} */ + group {{ bgpgroup.name }} { type internal - {% if MESHRR_FAMILY_INET != "false" %}family inet { + family inet { unicast; - }{% endif %} - {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { - signaling; - }{% endif %} - } - group MESHRR-CLIENTS { - {% if MESHRR_MODE == 'routeserver' %}type external; - multihop { - ttl 10; - no-nexthop-change; } - {% if MESHRR_FAMILY_INET != "false" %}family inet { - unicast; - }{% endif %} - {% if MESHRR_FAMILY_EVPN %}family evpn { - signaling; - }{% endif %} - peer-as-list MESHRR-RSCLIENTS;{% else %}type internal; - cluster {{ POD_IP }};{% endif %} - allow {{ MESHRR_CLIENTRANGE }}; - }{% if UPSTREAM_SERVICE_NAME is defined and UPSTREAM_SERVICE_NAME is not none %} - group MESHRR-UPSTREAM { + } + {%- endfor %} + {%- for bgpgroup in bgpgroups_subtractive %} + /* Subtractive group from {{ bgpgroup.prefixes }} */ + group {{ bgpgroup.name }} { type internal; - {% if MESHRR_FAMILY_INET != "false" %}family inet { - unicast; - }{% endif %} - {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { - signaling; - }{% endif %} - }{% endif %} + cluster {{ POD_IP }}; + allow [ {% for prefix in bgpgroup.prefixes %}{{ prefix }} {% endfor %}]; + } + {%- endfor %} } } } diff --git a/meshrr/defaults/meshrr.conf.yml b/meshrr/defaults/meshrr.conf.yml new file mode 100644 index 0000000..fc2ad49 --- /dev/null +++ b/meshrr/defaults/meshrr.conf.yml @@ -0,0 +1,22 @@ +encrypted_root_pw: NOLOGIN +asn: "65000" +mode: ipv4rr +bgpgroups: +- name: MESHRR-MESH + type: mesh + source: + sourcetype: dns + hostname: meshrr # FQDN for svc required if not in same namespace +- name: MESHRR-CLIENTS + type: subtractive # Prefixes in multiple external-subtractive groups must not overlap + prefixes: + - 0.0.0.0/0 + # For routeserver use case, an AS range is needed; we don't set this for RR use case. + # asranges: + # - 65001-65500 +- name: MESHRR-UPSTREAM + type: mesh + source: + sourcetype: dns + hostname: meshrr.core.svc.cluster.local # FQDN required if svc not in same namespace + max_peers: 2 diff --git a/meshrr/render_config.py b/meshrr/render_config.py index df7c85c..7164ed7 100755 --- a/meshrr/render_config.py +++ b/meshrr/render_config.py @@ -31,9 +31,12 @@ from argparse import ArgumentParser, FileType from os import getenv - from jinja2 import Template +from config import Meshrrconfig + +mconf=Meshrrconfig() + if __name__ == "__main__": parser = ArgumentParser( @@ -47,30 +50,22 @@ template = Template(args.inputfile.read()) configvars = dict() + + # Populate variables to be used in templates that must come from env configvars.update({"LICENSE_KEY": getenv("LICENSE_KEY", None)}) if not configvars["LICENSE_KEY"]: raise (Exception("LICENSE_KEY is not set.")) - configvars.update({"ENCRYPTED_ROOT_PW": getenv("ENCRYPTED_ROOT_PW", None)}) - if not configvars["ENCRYPTED_ROOT_PW"]: - raise (Exception("ENCRYPTED_ROOT_PW is not set.")) - configvars.update({"AUTONOMOUS_SYSTEM": getenv("AUTONOMOUS_SYSTEM", None)}) - if not configvars["AUTONOMOUS_SYSTEM"]: - raise (Exception("AUTONOMOUS_SYSTEM is not set.")) configvars.update({"POD_IP": getenv("POD_IP", None)}) if not configvars["POD_IP"]: raise (Exception("POD_IP is not set.")) - configvars.update({"MESHRR_CLIENTRANGE": getenv("MESHRR_CLIENTRANGE", None)}) - if not configvars["MESHRR_CLIENTRANGE"]: - raise (Exception("MESHRR_CLIENTRANGE is not set.")) - configvars.update({"UPSTREAM_SERVICE_NAME": getenv("UPSTREAM_SERVICE_NAME", None)}) - # Default to Route Reflector Mode - configvars.update({"MESHRR_MODE": getenv("MESHRR_MODE", "routereflector")}) - # Default AS Range for Route Server mode is 65001 to 65500 - configvars.update({"MESHRR_ASRANGE": getenv("MESHRR_ASRANGE", "65001-65500")}) - # Default to inet unicast only - configvars.update({"MESHRR_FAMILY_INET": getenv("MESHRR_FAMILY_INET", "true")}) - configvars.update({"MESHRR_FAMILY_EVPN": getenv("MESHRR_FAMILY_EVPN", "false")}) + # Populate variables to be used in termplates that will come from Meshrrconfig / YAML + configvars.update({ + "encrypted_root_pw": mconf.encrypted_root_pw, + "asn": mconf.asn, + "bgpgroups_mesh": mconf.bgpgroups_mesh, + "bgpgroups_subtractive": mconf.bgpgroups_subtractive + }) crpd_config = template.render(configvars) args.outputfile.write(crpd_config) diff --git a/meshrr/update_peers.py b/meshrr/update_peers.py index fc89b1a..108bd8e 100644 --- a/meshrr/update_peers.py +++ b/meshrr/update_peers.py @@ -33,13 +33,18 @@ from random import randrange from dns import resolver -from dotenv import load_dotenv from jnpr.junos import Device from jnpr.junos.utils.config import Config -from netaddr import IPNetwork, IPAddress, IPSet, cidr_exclude, cidr_merge +from netaddr import IPSet -load_dotenv(dotenv_path="/etc/envvars") +from config import Meshrrconfig +mconf=Meshrrconfig() + +# Load necessary environment variables +pod_ip = getenv("POD_IP") +if not pod_ip: + raise (Exception("POD_IP environment variable not set")) class ConfigUpdate(Config): # Child class to maintain lists of selected peers in managed groups @@ -62,7 +67,7 @@ def initiate_group(self, group_name, force=False): else: return False - def get_allowed_peers(self, allowed_base=getenv("MESHRR_CLIENTRANGE")): + def get_allowed_peers(self, bgpgroup): """ Returns the allowed_base minus any explicitly configured managed peers This is required, otherwise configured peers may end up in the wrong group. @@ -70,28 +75,70 @@ def get_allowed_peers(self, allowed_base=getenv("MESHRR_CLIENTRANGE")): """ disallowed = IPSet(cu.get_all_selected_peers()) - allowed = [IPNetwork(allowed_base)] - for disallowed_ip in disallowed: - for network in allowed: - if disallowed_ip in network: - allowed.remove(network) - allowed.extend(cidr_exclude(network, disallowed_ip)) - break - allowed = cidr_merge(allowed) + allowed = IPSet(bgpgroup['prefixes']) + allowed = allowed-disallowed return allowed - def commit_peerupdate(self, **kwargs): - allowed = self.get_allowed_peers() - allowed_string = " ".join([str(network) for network in allowed]) + def update_bgpgroup_mesh(self, bgpgroup): + detected_peers = list() + + # Get peers for DNS based BGP group [DEFAULT] + if 'sourcetype' not in bgpgroup['source'] or bgpgroup['source']['sourcetype'].casefold() == 'dns': + try: + result = resolver.resolve( + bgpgroup['source']['hostname'], "A", search=True + ) + except (resolver.NXDOMAIN, resolver.NoAnswer) as err: + print(err.msg, f"- Skipping processing of {bgpgroup['name']}.") + return + + for r in result: + if r.address != pod_ip: + detected_peers.append(r.address) + else: + raise(Exception(f"Invalid source type for group {bgpgroup['name']}: {bgpgroup['source']['sourcetype']}")) + + # Identify peers that should be active after this commit. + # Currently configured peers that are still detected via DNS are prioritized. + # Additional peers will be selected at random from detected_peers until `max_peers` are selected. + configured_peers = self.get_selected_peers(bgpgroup['name']) + selected_peers = list() + for peer_ip in configured_peers: + if 'max_peers' not in bgpgroup or len(selected_peers) < bgpgroup['max_peers']: + if peer_ip in detected_peers: + selected_peers.append(peer_ip) + detected_peers.remove(peer_ip) + else: + # The peer group is full with selected_peers. No need to continue this loop. + break + while len(detected_peers) and ( + 'max_peers' not in bgpgroup or len(selected_peers) < bgpgroup['max_peers'] + ): + # Add a peer at random from those detected to fill to max_peers. + selected_peers.append(detected_peers.pop(randrange(len(detected_peers)))) + + # Compare detected_peers (up-to-date list) with configured_peers. + # Add detected_peers not in configured_peers. + # Remove configured_peers not in detected_peers. + peers_to_add = set(selected_peers) - set(configured_peers) + for peer_ip in peers_to_add: + self.add_selected_peer(bgpgroup['name'], peer_ip) + + peers_to_remove = set(configured_peers) - set(selected_peers) + for peer_ip in peers_to_remove: + self.remove_selected_peer(bgpgroup['name'], peer_ip) + + def update_bgpgroup_subtractive(self, bgpgroup): + allowed = self.get_allowed_peers(bgpgroup) + allowed_string = " ".join([str(network) for network in allowed.iter_cidrs()]) self.load( - f"delete groups MESHRR protocols bgp group MESHRR-CLIENTS allow", + f"delete groups MESHRR protocols bgp group {bgpgroup['name']} allow", format="set", ) self.load( - f"set groups MESHRR protocols bgp group MESHRR-CLIENTS allow [ {allowed_string} ]", + f"set groups MESHRR protocols bgp group {bgpgroup['name']} allow [ {allowed_string} ]", format="set", ) - self.commit() def get_selected_peers(self, group_name): self.initiate_group(group_name) @@ -99,8 +146,8 @@ def get_selected_peers(self, group_name): def get_all_selected_peers(self): result = list() - for group in self.__selected_peers: - result.extend(self.__selected_peers[group]) + for groupname in self.__selected_peers: + result.extend(self.__selected_peers[groupname]) return result def add_selected_peer(self, group_name, peer_ip): @@ -120,101 +167,25 @@ def remove_selected_peer(self, group_name, peer_ip): ) -def update_peergroup(cu, group_name, service_name, max_peers=None): - detected_peers = list() - try: - result = resolver.resolve( - f"{service_name}.{kube_namespace}.{service_root_domain}", "A" - ) - except (resolver.NXDOMAIN, resolver.NoAnswer) as err: - print(err.msg, f"- Skipping processing of {group_name}.") - return cu - - for r in result: - if r.address != pod_ip: - detected_peers.append(r.address) - - # Identify peers that should be active after this commit. - # Currently configured peers that are still detected via DNS are prioritized. - # Additional peers will be selected at random from detected_peers until `max_peers` are selected. - configured_peers = cu.get_selected_peers(group_name) - selected_peers = list() - for peer_ip in configured_peers: - if max_peers is None or len(selected_peers) < max_peers: - if peer_ip in detected_peers: - selected_peers.append(peer_ip) - detected_peers.remove(peer_ip) - else: - # The peer group is full with selected_peers. No need to continue this loop. - break - while len(detected_peers) and ( - max_peers is None or len(selected_peers) < max_peers - ): - selected_peers.append(detected_peers.pop(randrange(len(detected_peers)))) - - # Compare detected_peers (up-to-date list) with configured_peers. - # Add detected_peers not in configured_peers. - # Remove configured_peers not in detected_peers. - peers_to_add = set(selected_peers) - set(configured_peers) - for peer_ip in peers_to_add: - cu.add_selected_peer(group_name, peer_ip) - - peers_to_remove = set(configured_peers) - set(selected_peers) - for peer_ip in peers_to_remove: - cu.remove_selected_peer(group_name, peer_ip) - return cu - - if __name__ == "__main__": - # Load necessary environment variables - pod_ip = getenv("POD_IP") - if not pod_ip: - raise (Exception("POD_IP environment variable not set")) - mesh_service_name = getenv("MESH_SERVICE_NAME") - - upstream_service_name = getenv("UPSTREAM_SERVICE_NAME") - - if not mesh_service_name and not upstream_service_name: - raise ( - Exception( - "MESH_SERVICE_NAME and UPSTREAM_SERVICE_NAME environment variables not set" - ) - ) - - kube_namespace = getenv("KUBE_NAMESPACE", "default") - service_root_domain = getenv("SERVICE_ROOT_DOMAIN", "svc.cluster.local") + # Confirm that a mesh BGP group has been defined. + if not mconf.bgpgroups_mesh: + raise(Exception("No mesh BGP groups defined.")) # Open a connection to the device dev = Device(host="127.0.0.1",user="meshrr",ssh_private_key_file="/secret/ssh/id_ed25519") dev.open() with ConfigUpdate(dev, mode="private") as cu: - - groups = list() - if mesh_service_name: - groups.append( - { - "name": "MESHRR-MESH", - "service_name": mesh_service_name, - "max_peers": None, - } + for group in mconf.bgpgroups_mesh: + cu.update_bgpgroup_mesh( + bgpgroup=group ) - - if upstream_service_name: - groups.append( - { - "name": "MESHRR-UPSTREAM", - "service_name": upstream_service_name, - "max_peers": 2, - } - ) - - for group in groups: - cu = update_peergroup( - cu, - group_name=group["name"], - service_name=group["service_name"], - max_peers=group["max_peers"], + for group in mconf.bgpgroups_subtractive: + cu.update_bgpgroup_subtractive( + bgpgroup=group ) if cu.diff(): - cu.commit_peerupdate() + print("Writing changes:") + cu.pdiff() + cu.commit() From c7b096363b4af2ca021c0dd272742d22bcf88237 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Thu, 19 Oct 2023 15:21:19 -0400 Subject: [PATCH 12/33] bgpgroups to j2 as dict --- meshrr/config.py | 15 ++++++++++++++- meshrr/defaults/juniper-evpnrs.conf.j2 | 14 +++++++------- meshrr/defaults/juniper-ipv4rr.conf.j2 | 10 +++++----- meshrr/render_config.py | 4 ++-- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/meshrr/config.py b/meshrr/config.py index 30874ec..e771705 100755 --- a/meshrr/config.py +++ b/meshrr/config.py @@ -75,4 +75,17 @@ def __init__(self): raise(Exception("Subtractive bgpgroups require 'source.hostname' definitions. Offending bgpgroup: "+str(group))) self.bgpgroups_subtractive.append(group) else: - raise(Exception("Invalid `type` in bgpgroup: "+str(group))) \ No newline at end of file + raise(Exception("Invalid `type` in bgpgroup: "+str(group))) + + def get_bgpgroups_dict(self, grouptype): + if grouptype == 'mesh': + bgpgroups = self.bgpgroups_mesh + elif grouptype == 'subtractive': + bgpgroups = self.bgpgroups_subtractive + else: + raise Exception("Invalid grouptype: ${grouptype}") + retval = dict() + for bgpgroup in bgpgroups: + retval.update({bgpgroup.pop('name'): bgpgroup}) + return retval + \ No newline at end of file diff --git a/meshrr/defaults/juniper-evpnrs.conf.j2 b/meshrr/defaults/juniper-evpnrs.conf.j2 index c94aa22..b1495c7 100644 --- a/meshrr/defaults/juniper-evpnrs.conf.j2 +++ b/meshrr/defaults/juniper-evpnrs.conf.j2 @@ -40,7 +40,7 @@ groups { MESHRR { policy-options { {%- for bgpgroup in bgpgroups_subtractive %} - as-list MESHRR:{{ bgpgroup.name }} members [ {% for asrange in bgpgroup.asranges %}{{ asrange }} {% endfor %}]; + as-list MESHRR:{{ bgpgroup }} members [ {% for asrange in bgpgroups_subtractive[bgpgroup].asranges %}{{ asrange }} {% endfor %}]; {%- endfor %} } protocols { @@ -54,8 +54,8 @@ groups { } } }{%- for bgpgroup in bgpgroups_mesh %} - /* Mesh {% if bgpgroup.max_peers is defined %}max_peers:{{ bgpgroup.max_peers }} {% endif %}group from {{ bgpgroup.source.sourcetype }}:{{ bgpgroup.source.hostname }} */ - group {{ bgpgroup.name }} { + /* Mesh {% if bgpgroups_mesh[bgpgroup].max_peers is defined %}max_peers:{{ bgpgroups_mesh[bgpgroup].max_peers }} {% endif %}group from {{ bgpgroups_mesh[bgpgroup].source.sourcetype }}:{{ bgpgroups_mesh[bgpgroup].source.hostname }} */ + group {{ bgpgroup }} { type internal family evpn { signaling; @@ -63,8 +63,8 @@ groups { } {%- endfor %} {%- for bgpgroup in bgpgroups_subtractive %} - /* Subtractive group from {{ bgpgroup.prefixes }} */ - group {{ bgpgroup.name }} { + /* Subtractive group from {{ bgpgroups_subtractive[bgpgroup].prefixes }} */ + group {{ bgpgroup }} { type external; multihop { ttl 10; @@ -73,8 +73,8 @@ groups { family evpn { signaling; } - peer-as-list MESHRR:{{ bgpgroup.name }}; - allow [ {% for prefix in bgpgroup.prefixes %}{{ prefix }} {% endfor %}]; + peer-as-list MESHRR:{{ bgpgroup }}; + allow [ {% for prefix in bgpgroups_subtractive[bgpgroup].prefixes %}{{ prefix }} {% endfor %}]; } {%- endfor %} } diff --git a/meshrr/defaults/juniper-ipv4rr.conf.j2 b/meshrr/defaults/juniper-ipv4rr.conf.j2 index 0a2553e..8dbad29 100644 --- a/meshrr/defaults/juniper-ipv4rr.conf.j2 +++ b/meshrr/defaults/juniper-ipv4rr.conf.j2 @@ -49,8 +49,8 @@ groups { } } {%- for bgpgroup in bgpgroups_mesh %} - /* Mesh {% if bgpgroup.max_peers is defined %}max_peers:{{ bgpgroup.max_peers }} {% endif %}group from {{ bgpgroup.source.sourcetype }}:{{ bgpgroup.source.hostname }} */ - group {{ bgpgroup.name }} { + /* Mesh {% if bgpgroups_mesh[bgpgroup].max_peers is defined %}max_peers:{{ bgpgroups_mesh[bgpgroup].max_peers }} {% endif %}group from {{ bgpgroups_mesh[bgpgroup].source.sourcetype }}:{{ bgpgroups_mesh[bgpgroup].source.hostname }} */ + group {{ bgpgroup }} { type internal family inet { unicast; @@ -58,11 +58,11 @@ groups { } {%- endfor %} {%- for bgpgroup in bgpgroups_subtractive %} - /* Subtractive group from {{ bgpgroup.prefixes }} */ - group {{ bgpgroup.name }} { + /* Subtractive group from {{ bgpgroups_subtractive[bgpgroup].prefixes }} */ + group {{ bgpgroup }} { type internal; cluster {{ POD_IP }}; - allow [ {% for prefix in bgpgroup.prefixes %}{{ prefix }} {% endfor %}]; + allow [ {% for prefix in bgpgroups_subtractive[bgpgroup].prefixes %}{{ prefix }} {% endfor %}]; } {%- endfor %} } diff --git a/meshrr/render_config.py b/meshrr/render_config.py index 7164ed7..6558a22 100755 --- a/meshrr/render_config.py +++ b/meshrr/render_config.py @@ -63,8 +63,8 @@ configvars.update({ "encrypted_root_pw": mconf.encrypted_root_pw, "asn": mconf.asn, - "bgpgroups_mesh": mconf.bgpgroups_mesh, - "bgpgroups_subtractive": mconf.bgpgroups_subtractive + "bgpgroups_mesh": mconf.get_bgpgroups_dict('mesh'), + "bgpgroups_subtractive": mconf.get_bgpgroups_dict('subtractive') }) crpd_config = template.render(configvars) From cee1c7a5ac7dde8c1276a80375be730480305b61 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Thu, 19 Oct 2023 15:48:46 -0400 Subject: [PATCH 13/33] Mirkwood example and configmap compatibility --- examples/2regions-hrr/meshrr-mirkwood.yaml | 98 ++++++++++++------- .../2regions-hrr/templates/mirkwood-config.j2 | 6 +- meshrr/run.sh | 5 +- 3 files changed, 68 insertions(+), 41 deletions(-) diff --git a/examples/2regions-hrr/meshrr-mirkwood.yaml b/examples/2regions-hrr/meshrr-mirkwood.yaml index b00bd42..0015a19 100644 --- a/examples/2regions-hrr/meshrr-mirkwood.yaml +++ b/examples/2regions-hrr/meshrr-mirkwood.yaml @@ -1,4 +1,30 @@ --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: meshrr-mirkwood-conf +data: + meshrr.conf.yml: |+ + encrypted_root_pw: NOLOGIN + asn: "65000" + mode: ipv4rr + bgpgroups: + - name: MESHRR-MESH + type: mesh + source: + sourcetype: dns + hostname: meshrr-mirkwood + - name: MESHRR-CLIENTS + type: subtractive + prefixes: + - 0.0.0.0/0 + - name: MESHRR-UPSTREAM + type: mesh + source: + sourcetype: dns + hostname: meshrr-core + max_peers: 2 +--- apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: @@ -122,6 +148,15 @@ spec: emptyDir: {} - name: ssh-id emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 + items: + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-mirkwood-conf + optional: false - configMap: defaultMode: 256 items: @@ -141,7 +176,11 @@ spec: mountPath: /secret/ssh/ - name: config mountPath: /config/ - - mountPath: /opt/meshrr/overrides/juniper.conf.j2 + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + - mountPath: /opt/meshrr/conf/juniper.conf.j2 name: override-config readOnly: true subPath: juniper.conf.j2 @@ -155,17 +194,6 @@ spec: secretKeyRef: name: crpd-license key: crpd-license - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-mirkwood - - name: UPSTREAM_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 0/0 containers: - name: crpd image: localhost/juniper/crpd:23.2R1.13 @@ -213,17 +241,15 @@ spec: volumeMounts: - name: ssh-id mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: MESH_SERVICE_NAME - value: meshrr-mirkwood - - name: UPSTREAM_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 0/0 affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -270,6 +296,15 @@ spec: emptyDir: {} - name: ssh-id emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 + items: + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-mirkwood-conf + optional: false - configMap: defaultMode: 256 items: @@ -289,7 +324,11 @@ spec: mountPath: /secret/ssh/ - name: config mountPath: /config/ - - mountPath: /opt/meshrr/overrides/juniper.conf.j2 + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + - mountPath: /opt/meshrr/conf/juniper.conf.j2 name: override-config readOnly: true subPath: juniper.conf.j2 @@ -303,17 +342,6 @@ spec: secretKeyRef: name: crpd-license key: crpd-license - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-mirkwood - - name: UPSTREAM_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 0/0 containers: - name: crpd image: localhost/juniper/crpd:23.2R1.13 @@ -361,17 +389,15 @@ spec: volumeMounts: - name: ssh-id mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: MESH_SERVICE_NAME - value: meshrr-mirkwood - - name: UPSTREAM_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 0/0 affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/examples/2regions-hrr/templates/mirkwood-config.j2 b/examples/2regions-hrr/templates/mirkwood-config.j2 index 4f41286..5ec7d92 100644 --- a/examples/2regions-hrr/templates/mirkwood-config.j2 +++ b/examples/2regions-hrr/templates/mirkwood-config.j2 @@ -1,6 +1,6 @@ system { root-authentication { - encrypted-password "{{ ENCRYPTED_ROOT_PW }}"; ## SECRET-DATA + encrypted-password "{{ encrypted_root_pw }}"; ## SECRET-DATA } login { user meshrr { @@ -40,7 +40,7 @@ groups { group MESHRR-CLIENTS { type internal; cluster {{ POD_IP }}; - allow {{ MESHRR_CLIENTRANGE }}; + allow [ {% for prefix in bgpgroups_subtractive['MESHRR-CLIENTS'].prefixes %}{{ prefix }} {% endfor %}]; }{% if UPSTREAM_SERVICE_NAME is not none %} group MESHRR-UPSTREAM { type internal; @@ -68,7 +68,7 @@ policy-options { community INREGION-PREFERRED members 65000:102; } routing-options { - autonomous-system {{ AUTONOMOUS_SYSTEM }}; + autonomous-system {{ asn }}; router-id {{ POD_IP }}; } protocols { diff --git a/meshrr/run.sh b/meshrr/run.sh index a1a0afc..ab4d8a0 100644 --- a/meshrr/run.sh +++ b/meshrr/run.sh @@ -59,8 +59,9 @@ elif [ $1 = 'init' ]; then # Generate a fresh SSH key and apply to configuration template. ssh-keygen -q -t ed25519 -f /secret/ssh/id_ed25519 -P "" PUBKEY=`cat \/secret\/ssh\/id_ed25519.pub | tr -d '\r\n'` - sed -i "/user meshrr/,/SECRET-DATA/ s~ssh-ed25519.*~ssh-ed25519 \"$PUBKEY\"; ## SECRET-DATA~" conf/juniper.conf.j2 - ./render_config.py -i conf/juniper.conf.j2 -o /config/juniper.conf + # Cannot overwrite in place as this may be a mounted configmap + sed "/user meshrr/,/SECRET-DATA/ s~ssh-ed25519.*~ssh-ed25519 \"$PUBKEY\"; ## SECRET-DATA~" conf/juniper.conf.j2 > /config/juniper.conf.j2 + ./render_config.py -i /config/juniper.conf.j2 -o /config/juniper.conf elif [ $1 = 'sidecar' ]; then # Wait for cRPD container to become available for netconf. ./connect_wait.py From 5fd84d4b267ebcd1a20c4299760614acce9157de Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Thu, 19 Oct 2023 15:54:30 -0400 Subject: [PATCH 14/33] pylint --- meshrr/update_peers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meshrr/update_peers.py b/meshrr/update_peers.py index 108bd8e..986c430 100644 --- a/meshrr/update_peers.py +++ b/meshrr/update_peers.py @@ -49,16 +49,16 @@ class ConfigUpdate(Config): # Child class to maintain lists of selected peers in managed groups - def __init__(self, dev, mode=None, **kwargs): + def __init__(self, device, mode=None, **kwargs): self.__selected_peers = dict() - super().__init__(dev, mode, **kwargs) + super().__init__(device, mode, **kwargs) def initiate_group(self, group_name, force=False): """Initiate the group from live configuration only if it's not inventoried yet. Override with force=True""" if group_name not in self.__selected_peers or force: # Get the list of neighbors in the group - filter = f"MESHRR{group_name}" - data = dev.rpc.get_config(filter_xml=filter) + xmlfilter = f"MESHRR{group_name}" + data = dev.rpc.get_config(filter_xml=xmlfilter) configured_peers = data.xpath( f"groups/protocols/bgp/group[name='{group_name}']/neighbor/name/text()" ) From 962099615cfbe2602076fd3066155009c14943c5 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Thu, 19 Oct 2023 16:08:26 -0400 Subject: [PATCH 15/33] Lothlorien example w/ YAML --- examples/2regions-hrr/meshrr-lothlorien.yaml | 98 ++++++++++++------- .../templates/lothlorien-config.j2 | 10 +- .../2regions-hrr/templates/mirkwood-config.j2 | 4 +- 3 files changed, 69 insertions(+), 43 deletions(-) diff --git a/examples/2regions-hrr/meshrr-lothlorien.yaml b/examples/2regions-hrr/meshrr-lothlorien.yaml index 05a9d00..824188e 100644 --- a/examples/2regions-hrr/meshrr-lothlorien.yaml +++ b/examples/2regions-hrr/meshrr-lothlorien.yaml @@ -1,4 +1,30 @@ --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: meshrr-lothlorien-conf +data: + meshrr.conf.yml: |+ + encrypted_root_pw: NOLOGIN + asn: "65000" + mode: ipv4rr + bgpgroups: + - name: MESHRR-MESH + type: mesh + source: + sourcetype: dns + hostname: meshrr-lothlorien + - name: MESHRR-CLIENTS + type: subtractive + prefixes: + - 0.0.0.0/0 + - name: MESHRR-UPSTREAM + type: mesh + source: + sourcetype: dns + hostname: meshrr-core + max_peers: 2 +--- apiVersion: metallb.io/v1beta1 kind: IPAddressPool metadata: @@ -121,6 +147,15 @@ spec: emptyDir: {} - name: ssh-id emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 + items: + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-lothlorien-conf + optional: false - configMap: defaultMode: 256 items: @@ -140,7 +175,11 @@ spec: mountPath: /secret/ssh/ - name: config mountPath: /config/ - - mountPath: /opt/meshrr/overrides/juniper.conf.j2 + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + - mountPath: /opt/meshrr/conf/juniper.conf.j2 name: override-config readOnly: true subPath: juniper.conf.j2 @@ -154,17 +193,6 @@ spec: secretKeyRef: name: crpd-license key: crpd-license - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-lothlorien - - name: UPSTREAM_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 0/0 containers: - name: crpd image: localhost/juniper/crpd:23.2R1.13 @@ -212,17 +240,15 @@ spec: volumeMounts: - name: ssh-id mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: MESH_SERVICE_NAME - value: meshrr-lothlorien - - name: UPSTREAM_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 0/0 affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -269,6 +295,15 @@ spec: emptyDir: {} - name: ssh-id emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 + items: + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-lothlorien-conf + optional: false - configMap: defaultMode: 256 items: @@ -288,7 +323,11 @@ spec: mountPath: /secret/ssh/ - name: config mountPath: /config/ - - mountPath: /opt/meshrr/overrides/juniper.conf.j2 + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + - mountPath: /opt/meshrr/conf/juniper.conf.j2 name: override-config readOnly: true subPath: juniper.conf.j2 @@ -302,17 +341,6 @@ spec: secretKeyRef: name: crpd-license key: crpd-license - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-lothlorien - - name: UPSTREAM_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 0/0 containers: - name: crpd image: localhost/juniper/crpd:23.2R1.13 @@ -360,17 +388,15 @@ spec: volumeMounts: - name: ssh-id mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: MESH_SERVICE_NAME - value: meshrr-lothlorien - - name: UPSTREAM_SERVICE_NAME - value: meshrr-core - - name: MESHRR_CLIENTRANGE - value: 0/0 affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/examples/2regions-hrr/templates/lothlorien-config.j2 b/examples/2regions-hrr/templates/lothlorien-config.j2 index 4f41286..25a2502 100644 --- a/examples/2regions-hrr/templates/lothlorien-config.j2 +++ b/examples/2regions-hrr/templates/lothlorien-config.j2 @@ -1,6 +1,6 @@ system { root-authentication { - encrypted-password "{{ ENCRYPTED_ROOT_PW }}"; ## SECRET-DATA + encrypted-password "{{ encrypted_root_pw }}"; ## SECRET-DATA } login { user meshrr { @@ -40,12 +40,12 @@ groups { group MESHRR-CLIENTS { type internal; cluster {{ POD_IP }}; - allow {{ MESHRR_CLIENTRANGE }}; - }{% if UPSTREAM_SERVICE_NAME is not none %} + allow [ {% for prefix in bgpgroups_subtractive['MESHRR-CLIENTS'].prefixes %}{{ prefix }} {% endfor %}]; + } group MESHRR-UPSTREAM { type internal; export UPSTREAM-OUT; - }{% endif %} + } } } } @@ -68,7 +68,7 @@ policy-options { community INREGION-PREFERRED members 65000:102; } routing-options { - autonomous-system {{ AUTONOMOUS_SYSTEM }}; + autonomous-system {{ asn }}; router-id {{ POD_IP }}; } protocols { diff --git a/examples/2regions-hrr/templates/mirkwood-config.j2 b/examples/2regions-hrr/templates/mirkwood-config.j2 index 5ec7d92..25a2502 100644 --- a/examples/2regions-hrr/templates/mirkwood-config.j2 +++ b/examples/2regions-hrr/templates/mirkwood-config.j2 @@ -41,11 +41,11 @@ groups { type internal; cluster {{ POD_IP }}; allow [ {% for prefix in bgpgroups_subtractive['MESHRR-CLIENTS'].prefixes %}{{ prefix }} {% endfor %}]; - }{% if UPSTREAM_SERVICE_NAME is not none %} + } group MESHRR-UPSTREAM { type internal; export UPSTREAM-OUT; - }{% endif %} + } } } } From fff16aa625bb4abfc507f59b20edc6e966565ad2 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" Date: Thu, 19 Oct 2023 21:16:55 -0400 Subject: [PATCH 16/33] Documentation updates --- README.md | 113 ++++++++++--------------- meshrr/defaults/juniper-evpnrs.conf.j2 | 4 + meshrr/defaults/juniper-ipv4rr.conf.j2 | 4 + meshrr/juniper.conf.j2 | 100 ---------------------- 4 files changed, 52 insertions(+), 169 deletions(-) delete mode 100644 meshrr/juniper.conf.j2 diff --git a/README.md b/README.md index af8eddb..9bd20ba 100644 --- a/README.md +++ b/README.md @@ -17,95 +17,70 @@ At this time and in the project's raw form, *meshrr* should not be considered fo ## Instructions ### Prerequisites + 1. An operational Kubernetes cluster with sufficient resources for the topology you wish to build. -2. A *private* container registry accessible to your Kubernetes cluster. - - You'll need to be logged in to your registry using `docker login` to push the image you'll build. - - You'll need to store your registry credentials in a secret in your cluster to pull from this registry. In this project, all examples use a secret named `regcred`. There are a few ways you can do this. - 1. If you already have a simple means of generating the secret manifest (e.g. using `doctl`), you can do this in one line: - ``` - doctl registry kubernetes-manifest --name regcred | kubectl apply -f - - ``` - 2. You can generate the secret manually with all the parameters: - ``` - kubectl create secret docker-registry regcred \ - --docker-server= --docker-username= \ - --docker-password= --docker-email= - ``` - 3. Any number of [other reasonable approaches](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/). - -3. A cRPD license for the number of nodes you wish to deploy. At the time of writing, Juniper offers [free trial licenses](https://www.juniper.net/us/en/dm/crpd-trial/). Standard licenses are limited to 16 BGP peers and 4M RIB entries. - -### Quickstart -1. (If required) modify [`juniper.conf.j2`](meshrr/juniper.conf.j2) -2. Build and push your image. **Do not push to a public registry.** +2. The cRPD software. The current tested version is **23.2R1.13**. +2. A cRPD license for the number of nodes you wish to deploy. At the time of writing, Juniper offers [free trial licenses](https://www.juniper.net/us/en/dm/crpd-trial/). Standard licenses are limited to 16 BGP peers and 4M RIB entries. - ```bash - docker build -t meshrr - docker push - ``` +### Usage - e.g. - ```bash - docker build meshrr -t registry.example.com/meshrr/meshrr:latest - docker push registry.example.com/meshrr/meshrr:latest - ``` -3. Either: +1. (If required) copy a configuration file template from [`the default templates`](meshrr/defaults/) and edit it to your liking. + +2. Either: 1. Pick an example topology from [`examples`](examples/) and modify the YAML files as required for your topology. Details for how to use examples and reasonable modifications are below in the [Examples](#Examples) section. 2. Create your own YAML files if you need a completely custom topology. -4. Populate the YAML files with the required information. You will need to, at a minimum, replace the following: - 1. Names - 1. Service - 2. Labels (if following the 2regions-hrr example, your regions probably are not in Middle Earth) + +3. Populate the Kubernetes manifest YAML files with the required information. You will need to, at a minimum, replace the following: + 1. Names of elements 2. [Environment Variables](#Environment-Variables) - 3. Licensing mechanism. Examples here currently use a secret mounted as a volume mapped to `/config/license/safenet/junos_sfnt.lic`. This may be appropriate for bundle licenses where it is appropriate to use the same license file for many similar devices in a deployment or daemonset. You can create this using: + 3. Licensing mechanism. Examples here currently use a secret exposed as an environment variable in the meshrr-init container which will populate the license into the config. This may be appropriate for bundle licenses where it is appropriate to use the same license file for many similar devices in a deployment or daemonset. You can create this using: ``` - kubectl create secret generic crpd-license --from-file=junos_sfnt.lic= + kubectl create secret generic crpd-license --from-file=crpd-license= ``` - 4. Custom configuration Jinja2 templates loaded into ConfigMaps and mapped as volumes. See [Examples](#Examples). - 5. Port mapping IP addresses (`hostIP`). No `hostIP` must be specified for instances only accessible within the cluster. Detailed strategy information to be defined in [Examples](#Examples). -5. Apply appropriate labels to the nodes: + Note that `` must point to a file that contains the singular license line and not an entire license file. + 4. (If required) Custom configuration Jinja2 templates loaded into ConfigMaps and mapped as volumes. See [Examples](#Examples). +4. (If required - e.g., for [2regions-hrr](examples/2regions-hrr/) where only certain nodes should host certain clusters of RRs) Apply appropriate labels to the nodes: ```bash kubectl label nodes = = ``` -6. Apply your configuration: +5. Apply your configuration: ```bash kubectl [-n namespace] apply -f kubectl [-n namespace] apply -f ``` ### Environment Variables -| Variable | Required? | Description | -| --------------------- | --------- | ------------------------------------------------------------ | -| POD_IP | Yes | The pod's IP address. Should be set by Kubernetes (`valueFrom: fieldRef: fieldPath: status.podIP`) | -| MESH_SERVICE_NAME | Yes* | The name of the mesh service. Set to the name of the headless Kubernetes service used for mesh BGP neighbor discovery. *Usually, a `MESH_SERVICE_NAME` is desirable. However, it may be skipped if there is an `UPSTREAM_SERVICE_NAME` in cases such as unmeshed regions learning routes from upstream HRRs. | -| UPSTREAM_SERVICE_NAME | No | The name of the upstream service. Set to the name of the headless Kubernetes service used for upstream BGP neighbor discovery. Defaults to `None`. | -| KUBE_NAMESPACE | No | Optional name of the Kubernetes namespace. Defaults to `default`. | -| ENCRYPTED_ROOT_PW | Yes | Encrypted ($6) root password for cRPD | -| AUTONOMOUS_SYSTEM | Yes | ASN for the router. | -| MESHRR_CLIENTRANGE | Yes | Range to allow. Currently, this accepts only one CIDR block. Format: `network/mask-length` | -| MESHRR_MODE | No | `routereflector` or `routeserver`. Defaults to `routereflector`. | -| MESHRR_ASRANGE | No | Range of ASNs to allow for `routeserver` mode. Defaults to `65001-65500` | -| MESHRR_FAMILY_INET | No | `true` or `false`. Defaults to `true` | -| MESHRR_FAMILY_EVPN | No | `true` or `false`. Defaults to `false` | -| SERVICE_ROOT_DOMAIN | No | Defaults to `svc.cluster.local`. You probably don't need to change this. | - -## Methodology -- Build container image based on crpd. - - Requires additional packages installed via `apt-get`: - - cron - - python3 - - Builds crontab - - Sets up `runit-init.sh` -- `runit-init.sh` initializes the environment: - - Saves environment variables, including the necessary pod's IP address, to `/etc/envvars` - - Sets up the cRPD configuration based on the template `juniper.conf.template` - - Calls `render_config.py` to create configuration file from Jinja2 template. -- `update_peers.py` called every minute via cron. - - Uses a Kubernetes headless service DNS A records to detect peers. + +| Variable | Required? | Description | +| -------------- | --------- | ------------------------------------------------------------ | +| LICENSE_KEY | Yes | Range to allow. Currently, this accepts only one CIDR block. Format: `network/mask-length` | +| POD_IP | Yes | The pod's IP address. Must be set by Kubernetes manifest in all pod templates for all meshrr containers. This does not need to be set for the cRPD containers. (`valueFrom: fieldRef: fieldPath: status.podIP`) | +| UPDATE_SECONDS | No | Frequency in seconds that `meshrr` container will attempt to update `crpd` container with changes to peers. (Default: 30) | + + +## Containers + +- Init Container - `meshrr-init`: + - `run.sh` with arg `init` + - Creates configuration from default template or mounted template or derives from existing `/config/juniper.conf` if pod uses persistent storage. +- Container - `crpd`: + - Unmodified cRPD image running Juniper cRPD. +- Container - `meshrr`: + - Conducts periodic BGP peer configuration changes on `crpd` container via Netconf. + +## BGP Group Types + +- `mesh` + - Discovers peers and connects to all or a limited number of them. + - Currently, the only BGP peer discovery mode is `dns`, which uses a Kubernetes headless service DNS A records to detect peers. New peers are added to config, removed peers are removed from config. - - Only occurs once a minute, so, given BGP timers, assume pod readiness 100 seconds from initiation. + - Supports a `max_peers` setting, which limits the number of peers added in this group. This is suitable for connections to a higher tier in a hierarchical route reflector / route server topology. +- `subtractive` + - This can be seen as a "wildcard". This is suitable for an environment in which not all peers are strictly defined and uses Junos BGP group `allow` config to permit connections from a range. + - The `allow` config is dynamically generated based on the list of all prefixes in the meshrr configuration with all peers from any mesh groups removed. ## Examples + - [2regions-hrr](examples/2regions-hrr) - Hierarchicial route reflectors broken into two regions with a single core region unifying them. - Reachability via static routes and Kubernetes NodeIP Services referencing additional loopbacks on the Kubernetes nodes. diff --git a/meshrr/defaults/juniper-evpnrs.conf.j2 b/meshrr/defaults/juniper-evpnrs.conf.j2 index b1495c7..9dfcc05 100644 --- a/meshrr/defaults/juniper-evpnrs.conf.j2 +++ b/meshrr/defaults/juniper-evpnrs.conf.j2 @@ -1,8 +1,10 @@ system { + /* Required, though use of a variable is optional */ root-authentication { encrypted-password "{{ encrypted_root_pw }}"; ## SECRET-DATA } login { + /* Required for meshrr authentication */ user meshrr { class super-user; uid 100; @@ -15,9 +17,11 @@ system { no-compress-configuration-files; services { netconf { + /* Required for meshrr access */ ssh; } } + /* Required for licensing unless using an alternative approach. */ license { keys { key "{{ LICENSE_KEY }}"; diff --git a/meshrr/defaults/juniper-ipv4rr.conf.j2 b/meshrr/defaults/juniper-ipv4rr.conf.j2 index 8dbad29..7f4131b 100644 --- a/meshrr/defaults/juniper-ipv4rr.conf.j2 +++ b/meshrr/defaults/juniper-ipv4rr.conf.j2 @@ -1,8 +1,10 @@ system { + /* Required, though use of a variable is optional */ root-authentication { encrypted-password "{{ encrypted_root_pw }}"; ## SECRET-DATA } login { + /* Required for meshrr authentication */ user meshrr { class super-user; uid 100; @@ -15,9 +17,11 @@ system { no-compress-configuration-files; services { netconf { + /* Required for meshrr access */ ssh; } } + /* Required for licensing unless using an alternative approach. */ license { keys { key "{{ LICENSE_KEY }}"; diff --git a/meshrr/juniper.conf.j2 b/meshrr/juniper.conf.j2 deleted file mode 100644 index 9a561c7..0000000 --- a/meshrr/juniper.conf.j2 +++ /dev/null @@ -1,100 +0,0 @@ -system { - root-authentication { - encrypted-password "{{ ENCRYPTED_ROOT_PW }}"; ## SECRET-DATA - } - login { - user meshrr { - class super-user; - uid 100; - authentication { - ssh-ed25519 "ssh-ed25519 TBD"; ## SECRET-DATA - } - } - } - /* Required for configuration persistence */ - no-compress-configuration-files; - services { - netconf { - ssh; - } - } - license { - keys { - key "{{ LICENSE_KEY }}"; - } - } - processes { - routing { - bgp { - rib-sharding; - update-threading; - } - } - } -} -routing-options { - autonomous-system {{ AUTONOMOUS_SYSTEM }}; - router-id {{ POD_IP }}; -} -groups { - MESHRR { - {% if MESHRR_MODE == 'routeserver' %}policy-options { - as-list MESHRR-RSCLIENTS members {{ MESHRR_ASRANGE }}; - } - {% endif %}protocols { - bgp { - {% if MESHRR_FAMILY_INET != "false" %}family inet { - unicast { - nexthop-resolution { - no-resolution; - } - no-install; - } - }{% endif %} - {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { - signaling { - nexthop-resolution { - no-resolution; - } - no-install; - } - }{% endif %} - group MESHRR-MESH { - type internal - {% if MESHRR_FAMILY_INET != "false" %}family inet { - unicast; - }{% endif %} - {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { - signaling; - }{% endif %} - } - group MESHRR-CLIENTS { - {% if MESHRR_MODE == 'routeserver' %}type external; - multihop { - ttl 10; - no-nexthop-change; - } - {% if MESHRR_FAMILY_INET != "false" %}family inet { - unicast; - }{% endif %} - {% if MESHRR_FAMILY_EVPN %}family evpn { - signaling; - }{% endif %} - peer-as-list MESHRR-RSCLIENTS;{% else %}type internal; - cluster {{ POD_IP }};{% endif %} - allow {{ MESHRR_CLIENTRANGE }}; - }{% if UPSTREAM_SERVICE_NAME is defined and UPSTREAM_SERVICE_NAME is not none %} - group MESHRR-UPSTREAM { - type internal; - {% if MESHRR_FAMILY_INET != "false" %}family inet { - unicast; - }{% endif %} - {% if MESHRR_FAMILY_EVPN == "true" %}family evpn { - signaling; - }{% endif %} - }{% endif %} - } - } - } -} -apply-groups MESHRR; From eaf3c36874c289b54f49a21ded634fdf4f21fb71 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" Date: Thu, 19 Oct 2023 21:19:56 -0400 Subject: [PATCH 17/33] Minor doc update --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9bd20ba..6c79a38 100644 --- a/README.md +++ b/README.md @@ -51,11 +51,11 @@ At this time and in the project's raw form, *meshrr* should not be considered fo ### Environment Variables -| Variable | Required? | Description | -| -------------- | --------- | ------------------------------------------------------------ | -| LICENSE_KEY | Yes | Range to allow. Currently, this accepts only one CIDR block. Format: `network/mask-length` | -| POD_IP | Yes | The pod's IP address. Must be set by Kubernetes manifest in all pod templates for all meshrr containers. This does not need to be set for the cRPD containers. (`valueFrom: fieldRef: fieldPath: status.podIP`) | -| UPDATE_SECONDS | No | Frequency in seconds that `meshrr` container will attempt to update `crpd` container with changes to peers. (Default: 30) | +| Variable | Required for | Optional for | Description | +| -------------- | ------------------- | ------------ | ------------------------------------------------------------ | +| LICENSE_KEY | meshrr-init | | Range to allow. Currently, this accepts only one CIDR block. Format: `network/mask-length` | +| POD_IP | meshrr-init, meshrr | | The pod's IP address. Must be set by Kubernetes manifest in all pod templates for all meshrr containers. This does not need to be set for the cRPD containers. (`valueFrom: fieldRef: fieldPath: status.podIP`) | +| UPDATE_SECONDS | | meshrr | Frequency in seconds that `meshrr` container will attempt to update `crpd` container with changes to peers. (Default: 30) | ## Containers From eaaa2f1cba9376256609dc3408046e7115ea1966 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" Date: Thu, 19 Oct 2023 22:55:55 -0400 Subject: [PATCH 18/33] 2regions-hrr docs update --- examples/2regions-hrr/README.md | 102 +++++++++---------------- examples/2regions-hrr/meshrr-core.yaml | 2 +- 2 files changed, 39 insertions(+), 65 deletions(-) diff --git a/examples/2regions-hrr/README.md b/examples/2regions-hrr/README.md index d6baca2..c616203 100644 --- a/examples/2regions-hrr/README.md +++ b/examples/2regions-hrr/README.md @@ -9,24 +9,21 @@ * **Redundancy groups and anycast addressing:** * Each node is assigned to redundancy group `a` or `b`. * For each region with neighbors outside the cluster, separate DaemonSets are created for `a` and `b`, each with a unique IP address for that [meshrr_region:redundancy_group] combination. This IP address is used for iBGP peering with neighbors outside the cluster. - * For Lothlorien: - * Kubernetes nodes run MetalLB. - * MetalLB eBGP peers to each connected router on a loopback with the same IP address (10.0.0.0). - * meshrr pods are assigned to a Kubernetes service using the MetalLB load balancer. This service is configured with `externalTrafficPolicy: local` and MetalLB announces the /32 of the service only from nodes with a corresponding meshrr pod, which will act as a route reflector for routers outside the cluster. - * This is the preferred method of distributing routes to the route reflector as it dynamically advertises based upon where route reflectors exist. - * For Mirkwood: - * Each /32 is assigned to the loopback interface of *every* node in the [meshrr_region:redundancy_group] combination. - * Routers connecting to the node have a static route to the /32 that is redistributed into the IGP. - * This method is discouraged but illustrates an alternative option. It provides no assurance that the route reflector is operational before advertising routes, as it relies on static routes from the connected router. Such a method could be enhanced using event scripts or similar, but the method used in Lothlorien provides more native dynamic routing. + * Kubernetes nodes run MetalLB. + * MetalLB eBGP peers to each connected router on a loopback with the same IP address (10.0.0.0). + * meshrr pods are assigned to a Kubernetes service using the MetalLB load balancer. This service is configured with `externalTrafficPolicy: local` and MetalLB announces the /32 of the service only from nodes with a corresponding meshrr pod, which will act as a route reflector for routers outside the cluster. + * This is the preferred method of distributing routes to the route reflector as it dynamically advertises based upon where route reflectors exist. It is possible, however, to use loopbacks with NodeIPs assigned to pods, but this lacks a native means of failover. * In the above topology: - | Node Region | MESHRR_REGION(s) | Redundancy Group | Loopback Address(es) | MetalLB IP(s) | - | ----------- | -------------------------- | ---------------- | ---------------------- | ------------- | - | Lothlorien | lothlorien | a | | 172.19.1.1 | - | Lothlorien | lothlorien | b | | 172.19.1.2 | - | Core | core, mirkwood, lothlorien | a | 172.19.2.1 | 172.19.1.1 | - | Core | core, mirkwood, lothlorien | b | 172.19.2.2 | 172.19.1.2 | - | Mirkwood | mirkwood | a | 172.19.2.1 | | - | Mirkwood | mirkwood | b | 172.19.2.2 | | + | Node | MESHRR_REGION(s) | Redundancy Group | MetalLB IP(s) | + | -------------- | -------------------- | ---------------- | ---------------------- | + | lothlorien-vm1 | core, lothlorien | a | 172.19.1.1 | + | lothlorien-vm2 | core, lothlorien | b | 172.19.1.2 | + | lothlorien-vm3 | mirkwood, lothlorien | a | 172.19.1.1, 172.19.2.1 | + | lothlorien-vm4 | mirkwood, lothlorien | b | 172.19.1.2, 172.19.2.2 | + | mirkwood-vm1 | core, mirkwood | a | 172.19.2.1 | + | mrikwood-vm2 | mirkwood | b | 172.19.2.2 | + + ### Usage 1. Follow the instructions in [Quickstart](../../README.md#Quickstart) using the example YAML files in [examples/2regions-hrr](.). @@ -37,7 +34,7 @@ sudo ip address add 172.19.1.1 dev lo ``` -3. Configure the Lothlorien routers connected to the nodes with: +3. Configure the routers connected to the nodes with: * The anycast peering address for MetalLB to peer to on the loopback * Hardcoded router ID (to ensure that the anycast peering address does not become the router ID) * BGP peering @@ -56,56 +53,20 @@ set policy-options policy-statement FILTER-RRLBPEER then reject set policy-options community no-advertise members no-advertise set policy-options policy-statement NO-ADVERTISE then community add no-advertise - set policy-options policy-statement REDISTRIBUTE-RRS from protocol bgp route-filter 172.19.1.0/24 prefix-length-range /32-/32 + set policy-options policy-statement REDISTRIBUTE-RRS from protocol bgp route-filter 172.19.0.0/16 prefix-length-range /32-/32 set policy-options policy-statement REDISTRIBUTE-RRS then accept ``` -4. Configure the Mirkwood routers connected to the nodes with static routes redistributed into your IGP for the node loopback addresses. -Configuration on the router servicing a Mirkwood A node may look like: - - ##### Junos - ```junos - routing-options { - static { - route 172.19.2.1/32 next-hop ; - } - } - policy-options { - policy-statement RRSTATIC-TO-ISIS { - from { - protocol static; - route-filter 172.19.2.1/32 exact; - } - then accept; - } - } - protocols { - isis { - export RRSTATIC-TO-ISIS; - } - } - ``` +4. Modify configuration templates as necessary. [`meshrr/juniper.conf.j2`](../../meshrr/defaults/juniper-ipv4rr.conf.j2) will be loaded to all instances by default, but customizations on a per-deployment/per-daemonset basis should be performed in most cases files (see [`core-config.j2`](templates/core-config.j2), [`mirkwood-config.j2`](templates/mirkwood-config.j2), and [`lothlorien-config.j2`](templates/lothlorien-config.j2). +Apply these configuration templates as ConfigSets for any cases that require customization as so: - ##### IOS-XR - ```iox-xr - router static - address-family ipv4 unicast - 172.19.2.1/32 - ! - route-policy STATIC-TO-ISIS - if destination in (172.19.2.1/32) then - pass - endif - end-policy - ! - router isis ISIS - redistribute static level-2 route-policy STATIC-TO-ISIS - ! + ```bash + k create configmap core-config \ + --from-file=config=examples/2regions-hrr/templates/core-config.j2 \ + -o yaml --dry-run=client | + k apply -f - ``` -5. Modify configuration templates as necessary. [`meshrr/juniper.conf.j2`](../../meshrr/juniper.conf.j2) will be loaded to all instances by default, but customizations on a per-deployment/per-daemonset basis should be performed on other J2 files (see [`mirkwood-config.j2`](templates/mirkwood-config.j2) and [`lothlorien-config.j2`](templates/lothlorien-config.j2). -Apply these configuration templates as ConfigSets for any cases that require customization as so: - ```bash k create configmap mirkwood-config \ --from-file=config=examples/2regions-hrr/templates/mirkwood-config.j2 \ @@ -120,6 +81,19 @@ Apply these configuration templates as ConfigSets for any cases that require cus k apply -f - ``` - These ConfigMaps are mounted as volumes in the corresponding DaemonSets. + These ConfigMaps are mounted as volumes in the corresponding Deployments/DaemonSets. -6. Modify the YAML files to your needs. At the least, `` will need to be replaced to reference your private registry. Load the YAML files for the DaemonSets and Services into Kubernetes as per [Quickstart](../../README.md#Quickstart). \ No newline at end of file +5. Label the nodes. You may need to adjust the node names. + ```bash + kubectl label nodes lothlorien-vm1 meshrr_region_core="true" meshrr_region_lothlorien="true" redundancy_group=a + kubectl label nodes lothlorien-vm2 meshrr_region_core="true" meshrr_region_lothlorien="true" redundancy_group=b + kubectl label nodes lothlorien-vm3 meshrr_region_lothlorien="true" meshrr_region_mirkwood="true" redundancy_group=a + kubectl label nodes lothlorien-vm4 meshrr_region_lothlorien="true" meshrr_region_mirkwood="true" redundancy_group=b + kubectl label nodes mirkwood-vm1 meshrr_region_core="true" meshrr_region_mirkwood="true" redundancy_group=a + kubectl label nodes mirkwood-vm2 meshrr_region_mirkwood="true" redundancy_group=b + ``` + +6. Apply the Kubernetes manifests: + ```bash + k apply -f bgppeer-global.yml meshrr-core.yaml meshrr-lothlorien.yaml meshrr-mirkwood.yaml + ``` \ No newline at end of file diff --git a/examples/2regions-hrr/meshrr-core.yaml b/examples/2regions-hrr/meshrr-core.yaml index 9100740..d1cd104 100644 --- a/examples/2regions-hrr/meshrr-core.yaml +++ b/examples/2regions-hrr/meshrr-core.yaml @@ -53,7 +53,7 @@ spec: selector: matchLabels: app: meshrr - replicas: 2 + replicas: 3 strategy: type: RollingUpdate rollingUpdate: From 4d302e15144b3bf88bc062a30604c9309e26e7dc Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Fri, 20 Oct 2023 09:46:01 -0400 Subject: [PATCH 19/33] Timestamps --- meshrr/connect_wait.py | 6 ++++-- meshrr/update_peers.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/meshrr/connect_wait.py b/meshrr/connect_wait.py index bf8ce82..b262c73 100644 --- a/meshrr/connect_wait.py +++ b/meshrr/connect_wait.py @@ -31,6 +31,8 @@ from jnpr.junos import Device from jnpr.junos.exception import ConnectTimeoutError,ConnectRefusedError + +from datetime import datetime from time import sleep if __name__ == "__main__": @@ -41,9 +43,9 @@ dev.open() break except ConnectTimeoutError: - print("connect_wait.py: Connection timed out; retrying.") + print(f"[{datetime.now()}]","connect_wait.py: Connection timed out; retrying.") except ConnectRefusedError: - print("connect_wait.py: Connection refused; retrying.") + print(f"[{datetime.now()}]","connect_wait.py: Connection refused; retrying.") sleep(1) # Create /tmp/connected-to-crpd to inform startup probes open("/tmp/connected-to-crpd","x") diff --git a/meshrr/update_peers.py b/meshrr/update_peers.py index 986c430..bd428fa 100644 --- a/meshrr/update_peers.py +++ b/meshrr/update_peers.py @@ -32,6 +32,7 @@ from os import getenv from random import randrange +from datetime import datetime from dns import resolver from jnpr.junos import Device from jnpr.junos.utils.config import Config @@ -89,7 +90,7 @@ def update_bgpgroup_mesh(self, bgpgroup): bgpgroup['source']['hostname'], "A", search=True ) except (resolver.NXDOMAIN, resolver.NoAnswer) as err: - print(err.msg, f"- Skipping processing of {bgpgroup['name']}.") + print(f"[{datetime.now()}]", err.msg, f"Skipping processing of {bgpgroup['name']}.") return for r in result: @@ -186,6 +187,6 @@ def remove_selected_peer(self, group_name, peer_ip): bgpgroup=group ) if cu.diff(): - print("Writing changes:") + print(f"[{datetime.now()}] Peer change detected. Writing changes:") cu.pdiff() cu.commit() From 15eb30c9c7c8abd4f38adca09a0306bb2d8fcf3a Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Fri, 20 Oct 2023 16:02:43 -0400 Subject: [PATCH 20/33] GitHub Action to push container image --- .github/workflows/publish-image.yml | 53 +++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/publish-image.yml diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml new file mode 100644 index 0000000..357ec34 --- /dev/null +++ b/.github/workflows/publish-image.yml @@ -0,0 +1,53 @@ +# +name: Publish container image + +on: + push: + release: + types: [published] + tags: + - 'latest' + - 'v[0-9]+' + - 'v[0-9]+.[0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+' + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + # + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: "{{defaultContext}}:meshrr" + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file From f78a33283630c99af43c09e05647cb7ce4d24695 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Mon, 23 Oct 2023 10:31:23 -0400 Subject: [PATCH 21/33] Docs update --- examples/2regions-hrr/README.md | 36 +++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/examples/2regions-hrr/README.md b/examples/2regions-hrr/README.md index c616203..6b2cbad 100644 --- a/examples/2regions-hrr/README.md +++ b/examples/2regions-hrr/README.md @@ -26,15 +26,12 @@ ### Usage -1. Follow the instructions in [Quickstart](../../README.md#Quickstart) using the example YAML files in [examples/2regions-hrr](.). - -2. Create the necessary loopback IPs on each of the nodes based on redundancy group and region. Internal-only MESHRR_REGIONs do not require a configured loopback IP. - - ```bash - sudo ip address add 172.19.1.1 dev lo +1. Review the instructions in [the project's main README](../../README.md#Usage). Create a license secret: + ``` + kubectl create secret generic crpd-license --from-file=crpd-license= ``` -3. Configure the routers connected to the nodes with: +2. Configure the routers connected to the nodes with: * The anycast peering address for MetalLB to peer to on the loopback * Hardcoded router ID (to ensure that the anycast peering address does not become the router ID) * BGP peering @@ -57,7 +54,7 @@ set policy-options policy-statement REDISTRIBUTE-RRS then accept ``` -4. Modify configuration templates as necessary. [`meshrr/juniper.conf.j2`](../../meshrr/defaults/juniper-ipv4rr.conf.j2) will be loaded to all instances by default, but customizations on a per-deployment/per-daemonset basis should be performed in most cases files (see [`core-config.j2`](templates/core-config.j2), [`mirkwood-config.j2`](templates/mirkwood-config.j2), and [`lothlorien-config.j2`](templates/lothlorien-config.j2). +3. Modify configuration templates as necessary. [`meshrr/juniper.conf.j2`](../../meshrr/defaults/juniper-ipv4rr.conf.j2) will be loaded to all instances by default, but customizations on a per-deployment/per-daemonset basis should be performed in most cases files (see [`core-config.j2`](templates/core-config.j2), [`mirkwood-config.j2`](templates/mirkwood-config.j2), and [`lothlorien-config.j2`](templates/lothlorien-config.j2). Apply these configuration templates as ConfigSets for any cases that require customization as so: ```bash @@ -83,7 +80,7 @@ Apply these configuration templates as ConfigSets for any cases that require cus These ConfigMaps are mounted as volumes in the corresponding Deployments/DaemonSets. -5. Label the nodes. You may need to adjust the node names. +4. Label the nodes. You may need to adjust the node names. ```bash kubectl label nodes lothlorien-vm1 meshrr_region_core="true" meshrr_region_lothlorien="true" redundancy_group=a kubectl label nodes lothlorien-vm2 meshrr_region_core="true" meshrr_region_lothlorien="true" redundancy_group=b @@ -93,7 +90,24 @@ Apply these configuration templates as ConfigSets for any cases that require cus kubectl label nodes mirkwood-vm2 meshrr_region_mirkwood="true" redundancy_group=b ``` -6. Apply the Kubernetes manifests: +5. Apply the Kubernetes manifests: ```bash k apply -f bgppeer-global.yml meshrr-core.yaml meshrr-lothlorien.yaml meshrr-mirkwood.yaml - ``` \ No newline at end of file + ``` + + See [Manifests and Objects Used](#Manifests-and-Objects-Used) for detail on what this does. + +### Manifests and Objects Used + +* [`bgppeer-global.yml`]() + * `BGPPeer/asn65000-global-lo1`: Defines a single BGP peer that will be used from every node in the Kubernetes cluster out every interface. In this example, we use 10.0.0.0/32 as an anycast IP on every router that connects to the Kubernetes cluster. This must not be advertised in any routing protocol. +* [`meshrr-core.yaml`]() + * `ConfigMap/meshrr-core-conf`: Contains the YAML to configure meshrr for the Core Deployment. + * `Service/meshrr-core`: Headless service used for inter-pod BGP service discovery. + * `Deployment/meshrr-core`: Defines the Deployment for the meshrr Core region. +* [`meshrr-lothlorien.yaml`]() and [`meshrr-mirkwood.yaml`]() + * `ConfigMap/meshrr--conf`: Contains the YAML to configure meshrr for both the DaemonSets. + * `IPAddressPool/meshrr-`: The address pool used for both the A side and B side load balancers in . + * `BGPAdvertisement/meshrr-`: Configures MetalLB to advertise the load balancer addresses when a pod is reachable via a LoadBalancer service on the node. + * `Service/meshrr--`: Configures a MetalLB LoadBalancer to connect external route reflector clients to a containerized route reflector in redundancy group . Note that `externalTrafficPolicy: Local` is of importance here to ensure that traffic to the service does not traverse the Kubernetes cluster to reach a Pod but only is advertised via BGP if a pod in the service exists on the node. + * `DaemonSet/meshrr--`: The DaemonSet for the redundancy group nodes in . Note the use of DaemonSets vs Deployments: Deployments ensure that a certain number of pods exist within the cluster. DaemonSets ensure that a pod exists on each node within the cluster that matches the parameters defined in `nodeAffinity`. From 4fddeb56cdac81c0ec7d2aa95f9e051e1e0dca9e Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Mon, 23 Oct 2023 11:18:24 -0400 Subject: [PATCH 22/33] Minor docs update --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6c79a38..dfd51ca 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,10 @@ At this time and in the project's raw form, *meshrr* should not be considered fo - [Introduction](#introduction) - [Instructions](#instructions) - [Prerequisites](#prerequisites) - - [Quickstart](#quickstart) + - [Usage](#usage) - [Environment Variables](#environment-variables) - - [Methodology](#methodology) + - [Containers](#containers) + - [BGP Group Types](#bgp-group-types) - [Examples](#examples) ## Instructions @@ -53,7 +54,7 @@ At this time and in the project's raw form, *meshrr* should not be considered fo | Variable | Required for | Optional for | Description | | -------------- | ------------------- | ------------ | ------------------------------------------------------------ | -| LICENSE_KEY | meshrr-init | | Range to allow. Currently, this accepts only one CIDR block. Format: `network/mask-length` | +| LICENSE_KEY | meshrr-init | | License key to be used for the cRPD container; expected to be a single line. | | POD_IP | meshrr-init, meshrr | | The pod's IP address. Must be set by Kubernetes manifest in all pod templates for all meshrr containers. This does not need to be set for the cRPD containers. (`valueFrom: fieldRef: fieldPath: status.podIP`) | | UPDATE_SECONDS | | meshrr | Frequency in seconds that `meshrr` container will attempt to update `crpd` container with changes to peers. (Default: 30) | From f6d5556fc35d4f4b310350c2a7c34e1ba6b26e72 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:40:32 -0400 Subject: [PATCH 23/33] Docs updates --- README.md | 17 ++++++++++++++++- examples/2regions-hrr/README.md | 21 +++++++-------------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index dfd51ca..612acd1 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ At this time and in the project's raw form, *meshrr* should not be considered fo - [Containers](#containers) - [BGP Group Types](#bgp-group-types) - [Examples](#examples) + - [Example Commands](#example-commands) ## Instructions @@ -87,4 +88,18 @@ At this time and in the project's raw form, *meshrr* should not be considered fo - Reachability via static routes and Kubernetes NodeIP Services referencing additional loopbacks on the Kubernetes nodes. - [load-balanced-route-servers](examples/load-balanced-route-servers) - EVPN route servers deployed in a full iBGP mesh with each other serving eBGP peers. Intended to scale DCI for multi-region deployment. - - Reachability for external devices achieved through use of MetalLB in BGP mode. \ No newline at end of file + - Reachability for external devices achieved through use of MetalLB in BGP mode. + +## Example Commands + +| Command | Description | +| ------------------------------------------------------------ | ------------------------------------------------------------ | +| `kubectl [-n NAMESPACE] get pods -o wide` | List pods and the nodes on which they run | +| `kubectl [-n NAMESPACE] exec -it POD -c crpd -- cli` | Access the CLI of cRPD | +| `kubectl [-n NAMESPACE] exec POD -c crpd -- cli show bgp summary` | See the `show bgp summary` output of a pod | +| `kubectl [-n NAMESPACE] exec POD -c crpd -- cli show bgp group summary \|except \"Allow\|orlonger\|^Default\|^$\"` | See the status of the neighbor groups of a pod | +| `kubectl [-n NAMESPACE] logs [-f] POD -c meshrr` | View the logs from the meshrr sidecar container. `-f` will follow the logs. | +| `kubectl [-n NAMESPACE] delete pod POD` | Delete POD. Because pods should be created by DaemonSet, StatefulSet, or Deployment, a new pod should be recreated in its place; in this context, this may be considered functionally more similar to a "restart" than to a "delete". | + + + diff --git a/examples/2regions-hrr/README.md b/examples/2regions-hrr/README.md index 6b2cbad..30b3090 100644 --- a/examples/2regions-hrr/README.md +++ b/examples/2regions-hrr/README.md @@ -7,7 +7,7 @@ * Within a region, all cRPDs are fully meshed via iBGP to provide maximum visibility within the region. * All cRPDs in a region other than `core` have BGP peerings with up to 2 `core` cRPDs. The `core` cRPDs serve as route reflectors for the non-core regions. (The limit of 2 is hard coded on upstream peer groups.) * **Redundancy groups and anycast addressing:** - * Each node is assigned to redundancy group `a` or `b`. + * Each node is assigned to redundancy group `a` or `b`. (In a production environment, two separate Kubernetes clusters may be desirable.) * For each region with neighbors outside the cluster, separate DaemonSets are created for `a` and `b`, each with a unique IP address for that [meshrr_region:redundancy_group] combination. This IP address is used for iBGP peering with neighbors outside the cluster. * Kubernetes nodes run MetalLB. * MetalLB eBGP peers to each connected router on a loopback with the same IP address (10.0.0.0). @@ -54,28 +54,21 @@ set policy-options policy-statement REDISTRIBUTE-RRS then accept ``` -3. Modify configuration templates as necessary. [`meshrr/juniper.conf.j2`](../../meshrr/defaults/juniper-ipv4rr.conf.j2) will be loaded to all instances by default, but customizations on a per-deployment/per-daemonset basis should be performed in most cases files (see [`core-config.j2`](templates/core-config.j2), [`mirkwood-config.j2`](templates/mirkwood-config.j2), and [`lothlorien-config.j2`](templates/lothlorien-config.j2). +3. Modify configuration templates as necessary. [`meshrr/juniper.conf.j2`](../../meshrr/defaults/juniper-ipv4rr.conf.j2) will be loaded to all instances by default, but customizations on a per-deployment/per-daemonset basis should be performed in most cases files (see [`mirkwood-config.j2`](templates/mirkwood-config.j2) and [`lothlorien-config.j2`](templates/lothlorien-config.j2)). Apply these configuration templates as ConfigSets for any cases that require customization as so: ```bash - k create configmap core-config \ - --from-file=config=examples/2regions-hrr/templates/core-config.j2 \ - -o yaml --dry-run=client | - k apply -f - - ``` - - ```bash - k create configmap mirkwood-config \ + kubectl create configmap mirkwood-config \ --from-file=config=examples/2regions-hrr/templates/mirkwood-config.j2 \ -o yaml --dry-run=client | - k apply -f - + kubectl apply -f - ``` ```bash - k create configmap lothlorien-config \ + kubectl create configmap lothlorien-config \ --from-file=config=examples/2regions-hrr/templates/lothlorien-config.j2 \ -o yaml --dry-run=client | - k apply -f - + kubectl apply -f - ``` These ConfigMaps are mounted as volumes in the corresponding Deployments/DaemonSets. @@ -92,7 +85,7 @@ Apply these configuration templates as ConfigSets for any cases that require cus 5. Apply the Kubernetes manifests: ```bash - k apply -f bgppeer-global.yml meshrr-core.yaml meshrr-lothlorien.yaml meshrr-mirkwood.yaml + k apply -f examples/2regions-hrr/bgppeer-global.yml -f examples/2regions-hrr/meshrr-core.yaml -f examples/2regions-hrr/meshrr-lothlorien.yaml -f examples/2regions-hrr/meshrr-mirkwood.yaml ``` See [Manifests and Objects Used](#Manifests-and-Objects-Used) for detail on what this does. From d24f0aacba63dc31bbe59b49f70b0d76901df7c4 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Tue, 31 Oct 2023 09:26:01 -0400 Subject: [PATCH 24/33] Container image tag normalization --- .github/workflows/publish-image.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 357ec34..9848b7d 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -3,13 +3,15 @@ name: Publish container image on: push: - release: - types: [published] + branches: + - 'main' + - 'next' tags: - - 'latest' - - 'v[0-9]+' - - 'v[0-9]+.[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+' + - 'v*' + pull_request: + branches: + - 'main' + - 'next' # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: @@ -41,6 +43,13 @@ jobs: uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + # set latest tag for default branch + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. From f8788f3edf9c11dcd49f9a5b8690ef30f66e26fd Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:10:17 -0400 Subject: [PATCH 25/33] Add cRPD image import instructions --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 612acd1..39f9c6f 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,14 @@ At this time and in the project's raw form, *meshrr* should not be considered fo ### Prerequisites 1. An operational Kubernetes cluster with sufficient resources for the topology you wish to build. -2. The cRPD software. The current tested version is **23.2R1.13**. -2. A cRPD license for the number of nodes you wish to deploy. At the time of writing, Juniper offers [free trial licenses](https://www.juniper.net/us/en/dm/crpd-trial/). Standard licenses are limited to 16 BGP peers and 4M RIB entries. +2. The cRPD software. The current tested version is **23.2R1.13**. The software must be available via a private repository or preloaded onto all nodes it may be run on. If using k3s, this can be accomplished with `k3s ctr images import junos-routing-crpd-docker-amd64-23.2R1.13.tar`. + - If the import fails, you may need to convert the tarfile into a format that can be imported. You can do so with + ```sh + docker load -i junos-routing-crpd-docker-amd64-23.2R1.13.tgz \ + && docker image tag crpd:23.2R1.13 localhost/juniper/crpd:23.2R1.13 \ + && docker image save localhost/juniper/crpd:23.2R1.13 --output=junos-routing-crpd-docker-amd64-23.2R1.13.tar + ``` +3. A cRPD license for the number of nodes you wish to deploy. At the time of writing, Juniper offers [free trial licenses](https://www.juniper.net/us/en/dm/crpd-trial/). Standard licenses are limited to 16 BGP peers and 4M RIB entries. ### Usage From 28b44775827c6b3ed3ac3d5b57314c8bd964bea7 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Thu, 2 Nov 2023 13:53:19 -0400 Subject: [PATCH 26/33] Fix Junos config syntax error for evpnrs --- meshrr/defaults/juniper-evpnrs.conf.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/meshrr/defaults/juniper-evpnrs.conf.j2 b/meshrr/defaults/juniper-evpnrs.conf.j2 index 9dfcc05..107839f 100644 --- a/meshrr/defaults/juniper-evpnrs.conf.j2 +++ b/meshrr/defaults/juniper-evpnrs.conf.j2 @@ -56,7 +56,6 @@ groups { } no-install; } - } }{%- for bgpgroup in bgpgroups_mesh %} /* Mesh {% if bgpgroups_mesh[bgpgroup].max_peers is defined %}max_peers:{{ bgpgroups_mesh[bgpgroup].max_peers }} {% endif %}group from {{ bgpgroups_mesh[bgpgroup].source.sourcetype }}:{{ bgpgroups_mesh[bgpgroup].source.hostname }} */ group {{ bgpgroup }} { From 0d94cce415f92443a60f029447de34cb87986913 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Thu, 2 Nov 2023 14:02:21 -0400 Subject: [PATCH 27/33] Update A side evpnrs use case --- .../bgppeer-global.yml | 10 +- .../meshrr-core.service.yml | 44 ++-- .../routeserver-1-a.ss.yml | 208 --------------- .../routeserver-a.ss.yml | 247 ++++++++++++++++++ 4 files changed, 274 insertions(+), 235 deletions(-) delete mode 100644 examples/load-balanced-route-servers/routeserver-1-a.ss.yml create mode 100644 examples/load-balanced-route-servers/routeserver-a.ss.yml diff --git a/examples/load-balanced-route-servers/bgppeer-global.yml b/examples/load-balanced-route-servers/bgppeer-global.yml index b86c96e..6c3312f 100644 --- a/examples/load-balanced-route-servers/bgppeer-global.yml +++ b/examples/load-balanced-route-servers/bgppeer-global.yml @@ -1,10 +1,10 @@ apiVersion: metallb.io/v1beta2 kind: BGPPeer metadata: - name: asn100-global-lo100 + name: global-lo1 namespace: metallb spec: - myASN: 65000 - peerASN: 100 - peerAddress: 10.0.0.0 - + myASN: 4259840003 + peerASN: 4259840002 + peerAddress: 192.168.255.0 + ebgpMultiHop: True diff --git a/examples/load-balanced-route-servers/meshrr-core.service.yml b/examples/load-balanced-route-servers/meshrr-core.service.yml index aa2d3b7..2bdf018 100644 --- a/examples/load-balanced-route-servers/meshrr-core.service.yml +++ b/examples/load-balanced-route-servers/meshrr-core.service.yml @@ -12,26 +12,26 @@ spec: protocol: TCP targetPort: bgp selector: - app: routeserver + app: meshrr-evpnrs type: ClusterIP ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: routeserver-mgt - namespace: metallb -spec: - addresses: - - 192.168.18.0/24 - autoAssign: false ---- -apiVersion: metallb.io/v1beta1 -kind: L2Advertisement -metadata: - name: routeserver-mgt - namespace: metallb -spec: - ipAddressPools: - - routeserver-mgt - interfaces: - - ens3 +# --- +# apiVersion: metallb.io/v1beta1 +# kind: IPAddressPool +# metadata: +# name: routeserver-mgt +# namespace: metallb +# spec: +# addresses: +# - 192.168.18.0/24 +# autoAssign: false +# --- +# apiVersion: metallb.io/v1beta1 +# kind: L2Advertisement +# metadata: +# name: routeserver-mgt +# namespace: metallb +# spec: +# ipAddressPools: +# - routeserver-mgt +# interfaces: +# - ens3 diff --git a/examples/load-balanced-route-servers/routeserver-1-a.ss.yml b/examples/load-balanced-route-servers/routeserver-1-a.ss.yml deleted file mode 100644 index 8de943d..0000000 --- a/examples/load-balanced-route-servers/routeserver-1-a.ss.yml +++ /dev/null @@ -1,208 +0,0 @@ ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: routeserver-1-a - namespace: metallb -spec: - addresses: - - 10.0.0.1/32 - autoAssign: false ---- -apiVersion: metallb.io/v1beta1 -kind: BGPAdvertisement -metadata: - name: routeserver-1-a - namespace: metallb -spec: - ipAddressPools: - - routeserver-1-a - communities: - - 65000:0 ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-1-a - annotations: - metallb.universe.tf/address-pool: routeserver-1-a -spec: - ports: - - name: bgp - port: 179 - protocol: TCP - targetPort: bgp - selector: - app: routeserver - region: "1" - side: a - type: LoadBalancer - externalTrafficPolicy: Local ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-1-a-0-mgt - annotations: - metallb.universe.tf/address-pool: routeserver-mgt -spec: - ports: - - name: ssh - port: 22 - protocol: TCP - targetPort: ssh - - name: netconf - port: 830 - protocol: TCP - targetPort: netconf - selector: - statefulset.kubernetes.io/pod-name: routeserver-1-a-0 - type: LoadBalancer - loadBalancerIP: 192.168.18.3 - externalTrafficPolicy: Local ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-1-a-1-mgt - annotations: - metallb.universe.tf/address-pool: routeserver-mgt -spec: - ports: - - name: ssh - port: 22 - protocol: TCP - targetPort: ssh - - name: netconf - port: 830 - protocol: TCP - targetPort: netconf - selector: - statefulset.kubernetes.io/pod-name: routeserver-1-a-1 - type: LoadBalancer - loadBalancerIP: 192.168.18.6 - externalTrafficPolicy: Local ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: routeserver-1-a - labels: - app: routeserver - region: "1" - side: a -spec: - serviceName: routeserver - podManagementPolicy: "Parallel" - selector: - matchLabels: - app: routeserver - region: "1" - side: a - replicas: 1 - template: - metadata: - labels: - app: routeserver - region: "1" - side: a - spec: - volumes: - - name: crpd-license - secret: - secretName: crpd-license - items: - - key: junos_sfnt.lic - path: junos_sfnt.lic - containers: - - name: crpd - image: /meshrr:latest - imagePullPolicy: Always - livenessProbe: - failureThreshold: 3 - initialDelaySeconds: 60 - periodSeconds: 5 - successThreshold: 1 - tcpSocket: - port: bgp - timeoutSeconds: 3 - readinessProbe: - failureThreshold: 5 - initialDelaySeconds: 30 - periodSeconds: 5 - successThreshold: 2 - tcpSocket: - port: bgp - timeoutSeconds: 3 - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - ports: - - name: bgp - containerPort: 179 - protocol: TCP - - name: ssh - containerPort: 22 - protocol: TCP - - name: netconf - containerPort: 830 - protocol: TCP - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-core - - name: MESHRR_MODE - value: routeserver - - name: MESHRR_ASRANGE - value: 65001-65535 - - name: MESHRR_CLIENTRANGE - value: 10.0/16 - - name: MESHRR_FAMILY_EVPN - value: "true" - - name: MESHRR_FAMILY_INET - value: "false" - volumeMounts: - - name: config - mountPath: /config - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - securityContext: - allowPrivilegeEscalation: true - privileged: true - runAsNonRoot: false - imagePullSecrets: - - name: regcred - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: side - operator: In - values: - - "a" - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: region - operator: In - values: - - "1" - volumeClaimTemplates: - - metadata: - name: config - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi - diff --git a/examples/load-balanced-route-servers/routeserver-a.ss.yml b/examples/load-balanced-route-servers/routeserver-a.ss.yml new file mode 100644 index 0000000..697441e --- /dev/null +++ b/examples/load-balanced-route-servers/routeserver-a.ss.yml @@ -0,0 +1,247 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: meshrr-conf +data: + meshrr.conf.yml: |+ + encrypted_root_pw: NOLOGIN + asn: "65000.1" + mode: evpnrs + bgpgroups: + - name: MESHRR-CORE + type: mesh + source: + sourcetype: dns + hostname: meshrr-core + - name: MESHRR-CLIENTS + type: subtractive + prefixes: + - 192.168.0.0/16 + asranges: + - 65001-65535 +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: routeserver-a + namespace: metallb +spec: + addresses: + - 192.168.255.1/32 + autoAssign: false +--- +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: routeserver-a + namespace: metallb +spec: + ipAddressPools: + - routeserver-a + communities: + - 65000:0 +--- +apiVersion: v1 +kind: Service +metadata: + name: routeserver-a + annotations: + metallb.universe.tf/address-pool: routeserver-a +spec: + ports: + - name: bgp + port: 179 + protocol: TCP + targetPort: bgp + selector: + app: meshrr-evpnrs + side: a + type: LoadBalancer + externalTrafficPolicy: Local +# --- +# apiVersion: v1 +# kind: Service +# metadata: +# name: routeserver-1-a-0-mgt +# annotations: +# metallb.universe.tf/address-pool: routeserver-mgt +# spec: +# ports: +# - name: ssh +# port: 22 +# protocol: TCP +# targetPort: ssh +# - name: netconf +# port: 830 +# protocol: TCP +# targetPort: netconf +# selector: +# statefulset.kubernetes.io/pod-name: routeserver-1-a-0 +# type: LoadBalancer +# loadBalancerIP: 192.168.18.3 +# externalTrafficPolicy: Local +# --- +# apiVersion: v1 +# kind: Service +# metadata: +# name: routeserver-1-a-1-mgt +# annotations: +# metallb.universe.tf/address-pool: routeserver-mgt +# spec: +# ports: +# - name: ssh +# port: 22 +# protocol: TCP +# targetPort: ssh +# - name: netconf +# port: 830 +# protocol: TCP +# targetPort: netconf +# selector: +# statefulset.kubernetes.io/pod-name: routeserver-1-a-1 +# type: LoadBalancer +# loadBalancerIP: 192.168.18.6 +# externalTrafficPolicy: Local +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: routeserver-a + labels: + app: meshrr-evpnrs + side: a +spec: + serviceName: routeserver + podManagementPolicy: "Parallel" + selector: + matchLabels: + app: meshrr-evpnrs + side: a + replicas: 2 + template: + metadata: + labels: + app: meshrr-evpnrs + side: a + spec: + volumes: + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 + items: + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-conf + optional: false + initContainers: + - name: meshrr-init + image: ghcr.io/juniper/meshrr:next + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license + - name: MESHRR_MODE + value: evpnrs + containers: + - name: crpd + image: localhost/juniper/crpd:23.2R1.13 + imagePullPolicy: Never + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 60 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: bgp + timeoutSeconds: 3 + readinessProbe: + failureThreshold: 5 + initialDelaySeconds: 30 + periodSeconds: 5 + successThreshold: 2 + tcpSocket: + port: bgp + timeoutSeconds: 3 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + ports: + - name: bgp + containerPort: 179 + protocol: TCP + - name: ssh + containerPort: 22 + protocol: TCP + - name: netconf + containerPort: 830 + protocol: TCP + volumeMounts: + - name: config + mountPath: /config/ + securityContext: + allowPrivilegeEscalation: true + privileged: true + runAsNonRoot: false + - name: meshrr + image: ghcr.io/juniper/meshrr:next + imagePullPolicy: IfNotPresent + args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: side + operator: In + values: + - "a" + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - meshrr-evpnrs + topologyKey: kubernetes.io/hostname From 9b62680636aa8d4a43f77085f351133200b51dd7 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Thu, 2 Nov 2023 14:11:27 -0400 Subject: [PATCH 28/33] b side evpnrs --- .../routeserver-b.ss.yml | 203 ++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 examples/load-balanced-route-servers/routeserver-b.ss.yml diff --git a/examples/load-balanced-route-servers/routeserver-b.ss.yml b/examples/load-balanced-route-servers/routeserver-b.ss.yml new file mode 100644 index 0000000..93c113b --- /dev/null +++ b/examples/load-balanced-route-servers/routeserver-b.ss.yml @@ -0,0 +1,203 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: meshrr-conf +data: + meshrr.conf.yml: |+ + encrypted_root_pw: NOLOGIN + asn: "65000.1" + mode: evpnrs + bgpgroups: + - name: MESHRR-CORE + type: mesh + source: + sourcetype: dns + hostname: meshrr-core + - name: MESHRR-CLIENTS + type: subtractive + prefixes: + - 192.168.0.0/16 + asranges: + - 65001-65535 +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: routeserver-b + namespace: metallb +spec: + addresses: + - 192.168.255.2/32 + autoAssign: false +--- +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: routeserver-b + namespace: metallb +spec: + ipAddressPools: + - routeserver-b + communities: + - 65000:0 +--- +apiVersion: v1 +kind: Service +metadata: + name: routeserver-b + annotations: + metallb.universe.tf/address-pool: routeserver-b +spec: + ports: + - name: bgp + port: 179 + protocol: TCP + targetPort: bgp + selector: + app: meshrr-evpnrs + side: b + type: LoadBalancer + externalTrafficPolicy: Local +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: routeserver-b + labels: + app: meshrr-evpnrs + side: b +spec: + serviceName: routeserver + podManagementPolicy: "Parallel" + selector: + matchLabels: + app: meshrr-evpnrs + side: b + replicas: 2 + template: + metadata: + labels: + app: meshrr-evpnrs + side: b + spec: + volumes: + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 + items: + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-conf + optional: false + initContainers: + - name: meshrr-init + image: ghcr.io/juniper/meshrr:next + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license + - name: MESHRR_MODE + value: evpnrs + containers: + - name: crpd + image: localhost/juniper/crpd:23.2R1.13 + imagePullPolicy: Never + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 60 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: bgp + timeoutSeconds: 3 + readinessProbe: + failureThreshold: 5 + initialDelaySeconds: 30 + periodSeconds: 5 + successThreshold: 2 + tcpSocket: + port: bgp + timeoutSeconds: 3 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + ports: + - name: bgp + containerPort: 179 + protocol: TCP + - name: ssh + containerPort: 22 + protocol: TCP + - name: netconf + containerPort: 830 + protocol: TCP + volumeMounts: + - name: config + mountPath: /config/ + securityContext: + allowPrivilegeEscalation: true + privileged: true + runAsNonRoot: false + - name: meshrr + image: ghcr.io/juniper/meshrr:next + imagePullPolicy: IfNotPresent + args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: side + operator: In + values: + - "b" + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - meshrr-evpnrs + topologyKey: kubernetes.io/hostname From 1811035e65baefb5229c2fbe56b9d4d9f1e58ba4 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Thu, 2 Nov 2023 22:31:23 -0400 Subject: [PATCH 29/33] Remove regions from routeserver example --- .../routeserver-1-b.ss.yml | 186 ---------------- .../routeserver-2-a.ss.yml | 208 ------------------ .../routeserver-2-b.ss.yml | 186 ---------------- 3 files changed, 580 deletions(-) delete mode 100644 examples/load-balanced-route-servers/routeserver-1-b.ss.yml delete mode 100644 examples/load-balanced-route-servers/routeserver-2-a.ss.yml delete mode 100644 examples/load-balanced-route-servers/routeserver-2-b.ss.yml diff --git a/examples/load-balanced-route-servers/routeserver-1-b.ss.yml b/examples/load-balanced-route-servers/routeserver-1-b.ss.yml deleted file mode 100644 index da96dff..0000000 --- a/examples/load-balanced-route-servers/routeserver-1-b.ss.yml +++ /dev/null @@ -1,186 +0,0 @@ ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: routeserver-1-b - namespace: metallb -spec: - addresses: - - 10.0.0.2/32 - autoAssign: false ---- -apiVersion: metallb.io/v1beta1 -kind: BGPAdvertisement -metadata: - name: routeserver-1-b - namespace: metallb -spec: - ipAddressPools: - - routeserver-1-b - communities: - - 65000:0 ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-1-b - annotations: - metallb.universe.tf/address-pool: routeserver-1-b -spec: - ports: - - name: bgp - port: 179 - protocol: TCP - targetPort: bgp - selector: - app: routeserver - region: "1" - side: b - type: LoadBalancer - externalTrafficPolicy: Local ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-1-b-0-mgt - annotations: - metallb.universe.tf/address-pool: routeserver-mgt -spec: - ports: - - name: ssh - port: 22 - protocol: TCP - targetPort: ssh - - name: netconf - port: 830 - protocol: TCP - targetPort: netconf - selector: - statefulset.kubernetes.io/pod-name: routeserver-1-b-0 - type: LoadBalancer - loadBalancerIP: 192.168.18.4 - externalTrafficPolicy: Local ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: routeserver-1-b - labels: - app: routeserver - region: "1" - side: b -spec: - serviceName: routeserver - podManagementPolicy: "Parallel" - selector: - matchLabels: - app: routeserver - region: "1" - side: b - replicas: 1 - template: - metadata: - labels: - app: routeserver - region: "1" - side: b - spec: - volumes: - - name: crpd-license - secret: - secretName: crpd-license - items: - - key: junos_sfnt.lic - path: junos_sfnt.lic - containers: - - name: crpd - image: /meshrr:latest - imagePullPolicy: Always - livenessProbe: - failureThreshold: 3 - initialDelaySeconds: 60 - periodSeconds: 5 - successThreshold: 1 - tcpSocket: - port: bgp - timeoutSeconds: 3 - readinessProbe: - failureThreshold: 5 - initialDelaySeconds: 30 - periodSeconds: 5 - successThreshold: 2 - tcpSocket: - port: bgp - timeoutSeconds: 3 - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - ports: - - name: bgp - containerPort: 179 - protocol: TCP - - name: ssh - containerPort: 22 - protocol: TCP - - name: netconf - containerPort: 830 - protocol: TCP - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-core - - name: MESHRR_MODE - value: routeserver - - name: MESHRR_ASRANGE - value: 65001-65535 - - name: MESHRR_CLIENTRANGE - value: 10.0/16 - - name: MESHRR_FAMILY_EVPN - value: "true" - - name: MESHRR_FAMILY_INET - value: "false" - volumeMounts: - - name: config - mountPath: /config - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - securityContext: - allowPrivilegeEscalation: true - privileged: true - runAsNonRoot: false - imagePullSecrets: - - name: regcred - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: side - operator: In - values: - - "b" - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: region - operator: In - values: - - "1" - volumeClaimTemplates: - - metadata: - name: config - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi - diff --git a/examples/load-balanced-route-servers/routeserver-2-a.ss.yml b/examples/load-balanced-route-servers/routeserver-2-a.ss.yml deleted file mode 100644 index c715d18..0000000 --- a/examples/load-balanced-route-servers/routeserver-2-a.ss.yml +++ /dev/null @@ -1,208 +0,0 @@ ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: routeserver-2-a - namespace: metallb -spec: - addresses: - - 10.0.0.3/32 - autoAssign: false ---- -apiVersion: metallb.io/v1beta1 -kind: BGPAdvertisement -metadata: - name: routeserver-2-a - namespace: metallb -spec: - ipAddressPools: - - routeserver-2-a - communities: - - 65000:0 ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-2-a - annotations: - metallb.universe.tf/address-pool: routeserver-2-a -spec: - ports: - - name: bgp - port: 179 - protocol: TCP - targetPort: bgp - selector: - app: routeserver - region: "2" - side: a - type: LoadBalancer - externalTrafficPolicy: Local ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-2-a-0-mgt - annotations: - metallb.universe.tf/address-pool: routeserver-mgt -spec: - ports: - - name: ssh - port: 22 - protocol: TCP - targetPort: ssh - - name: netconf - port: 830 - protocol: TCP - targetPort: netconf - selector: - statefulset.kubernetes.io/pod-name: routeserver-2-a-0 - type: LoadBalancer - loadBalancerIP: 192.168.18.5 - externalTrafficPolicy: Local ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-2-a-1-mgt - annotations: - metallb.universe.tf/address-pool: routeserver-mgt -spec: - ports: - - name: ssh - port: 22 - protocol: TCP - targetPort: ssh - - name: netconf - port: 830 - protocol: TCP - targetPort: netconf - selector: - statefulset.kubernetes.io/pod-name: routeserver-2-a-1 - type: LoadBalancer - loadBalancerIP: 192.168.18.6 - externalTrafficPolicy: Local ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: routeserver-2-a - labels: - app: routeserver - region: "2" - side: a -spec: - serviceName: routeserver - podManagementPolicy: "Parallel" - selector: - matchLabels: - app: routeserver - region: "2" - side: a - replicas: 1 - template: - metadata: - labels: - app: routeserver - region: "2" - side: a - spec: - volumes: - - name: crpd-license - secret: - secretName: crpd-license - items: - - key: junos_sfnt.lic - path: junos_sfnt.lic - containers: - - name: crpd - image: /meshrr:latest - imagePullPolicy: Always - livenessProbe: - failureThreshold: 3 - initialDelaySeconds: 60 - periodSeconds: 5 - successThreshold: 1 - tcpSocket: - port: bgp - timeoutSeconds: 3 - readinessProbe: - failureThreshold: 5 - initialDelaySeconds: 30 - periodSeconds: 5 - successThreshold: 2 - tcpSocket: - port: bgp - timeoutSeconds: 3 - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - ports: - - name: bgp - containerPort: 179 - protocol: TCP - - name: ssh - containerPort: 22 - protocol: TCP - - name: netconf - containerPort: 830 - protocol: TCP - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-core - - name: MESHRR_MODE - value: routeserver - - name: MESHRR_ASRANGE - value: 65001-65535 - - name: MESHRR_CLIENTRANGE - value: 10.0/16 - - name: MESHRR_FAMILY_EVPN - value: "true" - - name: MESHRR_FAMILY_INET - value: "false" - volumeMounts: - - name: config - mountPath: /config - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - securityContext: - allowPrivilegeEscalation: true - privileged: true - runAsNonRoot: false - imagePullSecrets: - - name: regcred - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: side - operator: In - values: - - "a" - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: region - operator: In - values: - - "2" - volumeClaimTemplates: - - metadata: - name: config - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi - diff --git a/examples/load-balanced-route-servers/routeserver-2-b.ss.yml b/examples/load-balanced-route-servers/routeserver-2-b.ss.yml deleted file mode 100644 index 60b69a3..0000000 --- a/examples/load-balanced-route-servers/routeserver-2-b.ss.yml +++ /dev/null @@ -1,186 +0,0 @@ ---- -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: routeserver-2-b - namespace: metallb -spec: - addresses: - - 10.0.0.4/32 - autoAssign: false ---- -apiVersion: metallb.io/v1beta1 -kind: BGPAdvertisement -metadata: - name: routeserver-2-b - namespace: metallb -spec: - ipAddressPools: - - routeserver-2-b - communities: - - 65000:0 ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-2-b - annotations: - metallb.universe.tf/address-pool: routeserver-2-b -spec: - ports: - - name: bgp - port: 179 - protocol: TCP - targetPort: bgp - selector: - app: routeserver - region: "2" - side: b - type: LoadBalancer - externalTrafficPolicy: Local ---- -apiVersion: v1 -kind: Service -metadata: - name: routeserver-2-b-0-mgt - annotations: - metallb.universe.tf/address-pool: routeserver-mgt -spec: - ports: - - name: ssh - port: 22 - protocol: TCP - targetPort: ssh - - name: netconf - port: 830 - protocol: TCP - targetPort: netconf - selector: - statefulset.kubernetes.io/pod-name: routeserver-2-b-0 - type: LoadBalancer - loadBalancerIP: 192.168.18.7 - externalTrafficPolicy: Local ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: routeserver-2-b - labels: - app: routeserver - region: "2" - side: b -spec: - serviceName: routeserver - podManagementPolicy: "Parallel" - selector: - matchLabels: - app: routeserver - region: "2" - side: b - replicas: 1 - template: - metadata: - labels: - app: routeserver - region: "2" - side: b - spec: - volumes: - - name: crpd-license - secret: - secretName: crpd-license - items: - - key: junos_sfnt.lic - path: junos_sfnt.lic - containers: - - name: crpd - image: /meshrr:latest - imagePullPolicy: Always - livenessProbe: - failureThreshold: 3 - initialDelaySeconds: 60 - periodSeconds: 5 - successThreshold: 1 - tcpSocket: - port: bgp - timeoutSeconds: 3 - readinessProbe: - failureThreshold: 5 - initialDelaySeconds: 30 - periodSeconds: 5 - successThreshold: 2 - tcpSocket: - port: bgp - timeoutSeconds: 3 - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - ports: - - name: bgp - containerPort: 179 - protocol: TCP - - name: ssh - containerPort: 22 - protocol: TCP - - name: netconf - containerPort: 830 - protocol: TCP - env: - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: AUTONOMOUS_SYSTEM - value: '65000' - - name: ENCRYPTED_ROOT_PW - value: >- - $6$4XN/d$QdZrrRZNL2MtGXqKuZ/mr1S5tH46eyBKTKeWGVugxfqCGjKKqH2LhP8yNoqfYwVNvc0xsd0JSl6s9epsSPB7M1 - - name: MESH_SERVICE_NAME - value: meshrr-core - - name: MESHRR_MODE - value: routeserver - - name: MESHRR_ASRANGE - value: 65001-65535 - - name: MESHRR_CLIENTRANGE - value: 10.0/16 - - name: MESHRR_FAMILY_EVPN - value: "true" - - name: MESHRR_FAMILY_INET - value: "false" - volumeMounts: - - name: config - mountPath: /config - - name: crpd-license - mountPath: /config/license/safenet/ - subPath: '' - securityContext: - allowPrivilegeEscalation: true - privileged: true - runAsNonRoot: false - imagePullSecrets: - - name: regcred - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: side - operator: In - values: - - "b" - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 1 - preference: - matchExpressions: - - key: region - operator: In - values: - - "2" - volumeClaimTemplates: - - metadata: - name: config - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi - From 6d4bb97f7d902f38c2b79f2ca59abfd75df56045 Mon Sep 17 00:00:00 2001 From: Jason Rokeach <1076569+jrokeach@users.noreply.github.com> Date: Tue, 7 Nov 2023 10:52:03 -0500 Subject: [PATCH 30/33] 3 client groups (#24) * Base 3clientgroups example * Junos config cleanup --- examples/3clientgroups/bgppeer-global.yml | 10 + .../3clientgroups/meshrr-3clientgroups.yaml | 513 ++++++++++++++++++ .../meshrr-defaultonly-clients.yml | 26 + .../meshrr-fulltable-clients.yml | 25 + .../meshrr-partialtable-clients.yml | 25 + 5 files changed, 599 insertions(+) create mode 100644 examples/3clientgroups/bgppeer-global.yml create mode 100644 examples/3clientgroups/meshrr-3clientgroups.yaml create mode 100644 examples/3clientgroups/meshrr-defaultonly-clients.yml create mode 100644 examples/3clientgroups/meshrr-fulltable-clients.yml create mode 100644 examples/3clientgroups/meshrr-partialtable-clients.yml diff --git a/examples/3clientgroups/bgppeer-global.yml b/examples/3clientgroups/bgppeer-global.yml new file mode 100644 index 0000000..acd6e75 --- /dev/null +++ b/examples/3clientgroups/bgppeer-global.yml @@ -0,0 +1,10 @@ +apiVersion: metallb.io/v1beta2 +kind: BGPPeer +metadata: + name: global-lo1 + namespace: metallb +spec: + myASN: 65001 + peerASN: 65000 + peerAddress: 10.0.0.0 + ebgpMultiHop: True diff --git a/examples/3clientgroups/meshrr-3clientgroups.yaml b/examples/3clientgroups/meshrr-3clientgroups.yaml new file mode 100644 index 0000000..d9e5cbb --- /dev/null +++ b/examples/3clientgroups/meshrr-3clientgroups.yaml @@ -0,0 +1,513 @@ +--- +apiVersion: v1 +data: + config: | + system { + /* Required, though use of a variable is optional */ + root-authentication { + encrypted-password "{{ encrypted_root_pw }}"; ## SECRET-DATA + } + login { + /* Required for meshrr authentication */ + user meshrr { + class super-user; + uid 100; + authentication { + ssh-ed25519 "ssh-ed25519 TBD"; ## SECRET-DATA + } + } + } + services { + netconf { + ssh; + } + } + /* Required for licensing unless using an alternative approach. */ + license { + keys { + key "{{ LICENSE_KEY }}"; + } + } + processes { + routing { + bgp { + rib-sharding; + update-threading; + } + } + } + } + groups { + MESHRR { + protocols { + bgp { + /* Mesh {% if bgpgroups_mesh['MESHRR-MESH'].max_peers is defined %}max_peers:{{ bgpgroups_mesh['MESHRR-MESH'].max_peers }} {% endif %}group from {{ bgpgroups_mesh['MESHRR-MESH'].source.sourcetype }}:{{ bgpgroups_mesh['MESHRR-MESH'].source.hostname }} */ + group MESHRR-MESH { + type internal + } + /* Mesh {% if bgpgroups_mesh['MESHRR-FULLTABLE-CLIENTS'].max_peers is defined %}max_peers:{{ bgpgroups_mesh['MESHRR-FULLTABLE-CLIENTS'].max_peers }} {% endif %}group from {{ bgpgroups_mesh['MESHRR-FULLTABLE-CLIENTS'].source.sourcetype }}:{{ bgpgroups_mesh['MESHRR-FULLTABLE-CLIENTS'].source.hostname }} */ + group MESHRR-FULLTABLE-CLIENTS { + type internal; + passive; + cluster {{ POD_IP }}; + } + /* Mesh {% if bgpgroups_mesh['MESHRR-PARTIALTABLE-CLIENTS'].max_peers is defined %}max_peers:{{ bgpgroups_mesh['MESHRR-PARTIALTABLE-CLIENTS'].max_peers }} {% endif %}group from {{ bgpgroups_mesh['MESHRR-PARTIALTABLE-CLIENTS'].source.sourcetype }}:{{ bgpgroups_mesh['MESHRR-PARTIALTABLE-CLIENTS'].source.hostname }} */ + group MESHRR-PARTIALTABLE-CLIENTS { + type internal; + passive; + export PARTIALTABLE-OUT; + cluster {{ POD_IP }}; + } + /* Mesh {% if bgpgroups_mesh['MESHRR-DEFAULTONLY-CLIENTS'].max_peers is defined %}max_peers:{{ bgpgroups_mesh['MESHRR-DEFAULTONLY-CLIENTS'].max_peers }} {% endif %}group from {{ bgpgroups_mesh['MESHRR-DEFAULTONLY-CLIENTS'].source.sourcetype }}:{{ bgpgroups_mesh['MESHRR-DEFAULTONLY-CLIENTS'].source.hostname }} */ + group MESHRR-DEFAULTONLY-CLIENTS { + type internal; + passive; + export DEFAULTONLY-OUT; + cluster {{ POD_IP }}; + } + } + } + } + } + policy-options { + policy-statement DEFAULTONLY-OUT { + from { + route-filter 0.0.0.0/0 exact; + } + then accept; + } + policy-statement PARTIALTABLE-OUT { + term SAME-AS { + from { + as-path-calc-length 0 orlower; + } + then accept; + } + term DEFAULT { + from { + route-filter 0.0.0.0/0 exact; + } + then accept; + } + } + } + routing-options { + autonomous-system {{ asn }}; + router-id {{ POD_IP }}; + } + protocols { + bgp { + apply-groups MESHRR; + family inet { + unicast { + nexthop-resolution { + no-resolution; + } + no-install; + } + } + } + } +kind: ConfigMap +metadata: + creationTimestamp: null + name: 3clientgroups-config +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: meshrr-3clientgroups-conf +data: + meshrr.conf.yml: |+ + encrypted_root_pw: NOLOGIN + asn: "65000" + mode: ipv4rr + bgpgroups: + - name: MESHRR-MESH + type: mesh + source: + sourcetype: dns + hostname: meshrr-3clientgroups + - name: MESHRR-FULLTABLE-CLIENTS + type: mesh + source: + sourcetype: dns + hostname: meshrr-fulltable-clients + - name: MESHRR-PARTIALTABLE-CLIENTS + type: mesh + source: + sourcetype: dns + hostname: meshrr-partialtable-clients + - name: MESHRR-DEFAULTONLY-CLIENTS + type: mesh + source: + sourcetype: dns + hostname: meshrr-defaultonly-clients +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: meshrr-3clientgroups + namespace: metallb +spec: + addresses: + - 172.19.1.1/32 + - 172.19.1.2/32 + autoAssign: false +--- +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: meshrr-3clientgroups + namespace: metallb +spec: + ipAddressPools: + - meshrr-3clientgroups +--- +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + name: meshrr-3clientgroups-a + annotations: + metallb.universe.tf/address-pool: meshrr-3clientgroups +spec: + ports: + - name: bgp + port: 179 + protocol: TCP + targetPort: bgp + selector: + app: meshrr + redundancy_group: a + sessionAffinity: None + type: LoadBalancer + loadBalancerIP: 172.19.1.1 + externalTrafficPolicy: Local +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + name: meshrr-3clientgroups-b + annotations: + metallb.universe.tf/address-pool: meshrr-3clientgroups +spec: + ports: + - name: bgp + port: 179 + protocol: TCP + targetPort: bgp + selector: + app: meshrr + redundancy_group: b + sessionAffinity: None + type: LoadBalancer + loadBalancerIP: 172.19.1.2 + externalTrafficPolicy: Local +status: + loadBalancer: {} +--- +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + name: meshrr-3clientgroups +spec: + clusterIP: None + ports: + - name: bgp + port: 179 + protocol: TCP + targetPort: bgp + selector: + app: meshrr + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: meshrr-3clientgroups-a + labels: + app: meshrr + redundancy_group: a +spec: + selector: + matchLabels: + app: meshrr + minReadySeconds: 5 + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + app: meshrr + redundancy_group: a + spec: + dnsPolicy: ClusterFirst + terminationGracePeriodSeconds: 30 + volumes: + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 + items: + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-3clientgroups-conf + optional: false + - configMap: + defaultMode: 256 + items: + - key: config + mode: 256 + path: juniper.conf.j2 + name: 3clientgroups-config + optional: false + name: override-config + initContainers: + - name: meshrr-init + image: ghcr.io/juniper/meshrr:next + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + - mountPath: /opt/meshrr/conf/juniper.conf.j2 + name: override-config + readOnly: true + subPath: juniper.conf.j2 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license + containers: + - name: crpd + image: localhost/juniper/crpd:23.2R1.13 + imagePullPolicy: Never + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 2 + successThreshold: 1 + tcpSocket: + port: bgp + timeoutSeconds: 3 + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 2 + successThreshold: 2 + tcpSocket: + port: bgp + timeoutSeconds: 3 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + ports: + - name: bgp + containerPort: 179 + protocol: TCP + volumeMounts: + - name: config + mountPath: /config/ + securityContext: + allowPrivilegeEscalation: true + privileged: true + runAsNonRoot: false + - name: meshrr + image: ghcr.io/juniper/meshrr:next + imagePullPolicy: IfNotPresent + args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: redundancy_group + operator: In + values: + - a + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: meshrr-3clientgroups-b + labels: + app: meshrr + redundancy_group: b +spec: + selector: + matchLabels: + app: meshrr + minReadySeconds: 5 + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + app: meshrr + redundancy_group: b + spec: + dnsPolicy: ClusterFirst + terminationGracePeriodSeconds: 30 + volumes: + - name: config + emptyDir: {} + - name: ssh-id + emptyDir: {} + - name: meshrr-conf + configMap: + defaultMode: 256 + items: + - key: meshrr.conf.yml + mode: 256 + path: meshrr.conf.yml + name: meshrr-3clientgroups-conf + optional: false + - configMap: + defaultMode: 256 + items: + - key: config + mode: 256 + path: juniper.conf.j2 + name: 3clientgroups-config + optional: false + name: override-config + initContainers: + - name: meshrr-init + image: ghcr.io/juniper/meshrr:next + imagePullPolicy: IfNotPresent + args: ["init"] + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: config + mountPath: /config/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + - mountPath: /opt/meshrr/conf/juniper.conf.j2 + name: override-config + readOnly: true + subPath: juniper.conf.j2 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LICENSE_KEY + valueFrom: + secretKeyRef: + name: crpd-license + key: crpd-license + containers: + - name: crpd + image: localhost/juniper/crpd:23.2R1.13 + imagePullPolicy: Never + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 2 + successThreshold: 1 + tcpSocket: + port: bgp + timeoutSeconds: 3 + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 2 + successThreshold: 2 + tcpSocket: + port: bgp + timeoutSeconds: 3 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + ports: + - name: bgp + containerPort: 179 + protocol: TCP + volumeMounts: + - name: config + mountPath: /config/ + securityContext: + allowPrivilegeEscalation: true + privileged: true + runAsNonRoot: false + - name: meshrr + image: ghcr.io/juniper/meshrr:next + imagePullPolicy: IfNotPresent + args: ["sidecar"] + startupProbe: + exec: + command: + - cat + - /tmp/connected-to-crpd + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - name: ssh-id + mountPath: /secret/ssh/ + - name: meshrr-conf + mountPath: /opt/meshrr/conf/meshrr.conf.yml + readOnly: true + subPath: meshrr.conf.yml + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: redundancy_group + operator: In + values: + - b \ No newline at end of file diff --git a/examples/3clientgroups/meshrr-defaultonly-clients.yml b/examples/3clientgroups/meshrr-defaultonly-clients.yml new file mode 100644 index 0000000..2fc7715 --- /dev/null +++ b/examples/3clientgroups/meshrr-defaultonly-clients.yml @@ -0,0 +1,26 @@ +--- +kind: "Service" +apiVersion: "v1" +metadata: + name: "meshrr-defaultonly-clients" +spec: + clusterIP: None + ports: + - name: "bgp" + protocol: "TCP" + port: 179 + targetPort: 179 +--- +kind: "Endpoints" +apiVersion: "v1" +metadata: + name: "meshrr-defaultonly-clients" +subsets: + - addresses: + - ip: "192.0.2.4" + - ip: "192.0.2.5" + - ip: "192.0.2.56" + ports: + - + port: 179 + name: "bgp" \ No newline at end of file diff --git a/examples/3clientgroups/meshrr-fulltable-clients.yml b/examples/3clientgroups/meshrr-fulltable-clients.yml new file mode 100644 index 0000000..fb66040 --- /dev/null +++ b/examples/3clientgroups/meshrr-fulltable-clients.yml @@ -0,0 +1,25 @@ +--- +kind: "Service" +apiVersion: "v1" +metadata: + name: "meshrr-fulltable-clients" +spec: + clusterIP: None + ports: + - name: "bgp" + protocol: "TCP" + port: 179 + targetPort: 179 +--- +kind: "Endpoints" +apiVersion: "v1" +metadata: + name: "meshrr-fulltable-clients" +subsets: + - addresses: + - ip: "1.2.3.4" + - ip: "4.5.6.7" + ports: + - + port: 179 + name: "bgp" \ No newline at end of file diff --git a/examples/3clientgroups/meshrr-partialtable-clients.yml b/examples/3clientgroups/meshrr-partialtable-clients.yml new file mode 100644 index 0000000..18dfce6 --- /dev/null +++ b/examples/3clientgroups/meshrr-partialtable-clients.yml @@ -0,0 +1,25 @@ +--- +kind: "Service" +apiVersion: "v1" +metadata: + name: "meshrr-partialtable-clients" +spec: + clusterIP: None + ports: + - name: "bgp" + protocol: "TCP" + port: 179 + targetPort: 179 +--- +kind: "Endpoints" +apiVersion: "v1" +metadata: + name: "meshrr-partialtable-clients" +subsets: + - addresses: + - ip: "2.3.4.5" + - ip: "6.7.8.9" + ports: + - + port: 179 + name: "bgp" \ No newline at end of file From 6bb064ffcddc2be576e19f55049ca9f07572b864 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:07:08 -0500 Subject: [PATCH 31/33] Routeservers readme update --- .../load-balanced-route-servers/README.md | 88 ++++++++++++++++--- 1 file changed, 74 insertions(+), 14 deletions(-) diff --git a/examples/load-balanced-route-servers/README.md b/examples/load-balanced-route-servers/README.md index 32dd914..23d9e62 100644 --- a/examples/load-balanced-route-servers/README.md +++ b/examples/load-balanced-route-servers/README.md @@ -1,15 +1,13 @@ # Example: Fully-meshed Multi-region Route Servers ## Description -* This topology has two regions, `1` and `2`. * cRPD is used as route servers for EVPN connectivity to minimize peering requirements in a many-DC environment. -* The two regions have route servers fully meshed together. -* Route servers, and Kubernetes nodes, are defined as "a" or "b" side. -* These route servers are deployed as StatefulSets for configuration persistence. +* Route servers, and Kubernetes nodes, are defined as "a" or "b" side. In this example, they exist within a single Kubernetes cluster using labels to differentiate between nodes. However, it is feasible to structure this with two entirely separate Kubernetes clusters. +* These route servers are deployed as StatefulSets; these could be used to provide for configuration persistence but do not in this example. * **Redundancy groups and anycast addressing:** * Each node is assigned to redundancy group ("side") `a` or `b`. - * cRPD pods are scheduled via StatefulSets, and will only be scheduled on a node of their appropriate side. They will prefer to be scheduled on a node of their region. With minor modification to the manifest, backup region(s) can be configured to avoid them being scheduled to just any side-compliant node. - * In this example, the same IP address is configured as a loopback on each device connecting to a k8s node. This enables to use of only one MetalLB `BGPPeer` manifest globally. + * cRPD pods are scheduled via StatefulSets, and will only be scheduled on a node of their appropriate side. + * In this example, the same IP address is configured as a loopback on each device connecting to a k8s node. Serving as an anycast address, this enables to use of only one MetalLB `BGPPeer` manifest globally. * MetalLB is used to provide external addressing for BGP connectivity to the cRPD route servers, as well as load balancing if the StatefulSet's `replicas` setting is greater than 1 (or multiple deployments provide endpoints for the same service). * MetalLB is also used in this example to provide external addressing for traditional management connectivity (SSH) to the cRPD route servers. * `LoadBalancer` services use a `Local` `externalTrafficPolicy`. This is important to ensure that: @@ -17,12 +15,7 @@ * cRPD's view of the peer shows the physical router's peering IP address (not critical for function but important for operational clarity) ## Requirements -* Kubernetes environment with at least two nodes (one `side=a` and one `side=b`) and with coredns active. For this example, we used microk8s. -* Available storage class. Longhorn was used for this example: - ```zsh - helm repo add longhorn https://charts.longhorn.io - helm install longhorn longhorn/longhorn --namespace longhorn-system --create-namespace --version 1.4.0 --set defaultSettings.defaultDataPath="/longhorn" --set csi.kubeletRootDir="/var/snap/microk8s/common/var/lib/kubelet" - ``` +* Kubernetes environment with at least two nodes (one `side=a` and one `side=b`) and with coredns active. This example has been tested on k3s. * MetalLB: ```zsh helm repo add metallb https://metallb.github.io/metallb @@ -40,7 +33,74 @@ Configuration on the router may look like: - ##### Junos + ##### Junos - If RS is in a DC fabric + ```junos + interfaces { + lo0 { + unit 0 { + family inet { + # Primary loopback address + address 192.168.1.3/32 { + primary; + preferred; + } + # MetalLB Anycast Peer + address 192.168.255.0/32/32; + } + } + } + ge-0/0/2 { + description "meshrr-kube1 eth1"; + unit 0 { + family inet { + address 172.16.1.12/31 + } + } + } + } + policy-options { + policy-statement ADVERTISE-LOOPBACKS { + term DENY-MESHRR-LB { + from { + route-filter 192.168.255.0/32 exact; + } + then reject; + } + from { + family inet; + protocol direct; + interface lo0.0; + } + then accept; + } + policy-statement ADVERTISE-MESHRR { + from { + family inet; + protocol direct; + interface ge-0/0/2.0; + } + then accept; + } + } + protocols { + bgp { + group MESHRR-LB { + type external; + local-address 192.168.255.0; + ttl 1; + family inet { + unicast; + } + peer-as 65000.3; + local-as 65000.2 private; + allow 172.16.0.0/16; + } + export [ ADVERTISE-LOOPBACKS ADVERTISE-MESHRR ]; + } + } + ``` + + ##### Junos - If RS is on the MPLS network in a VRF ```junos interfaces lo0 { unit 100 { @@ -70,7 +130,7 @@ } ``` - ##### IOS-XR + ##### IOS-XR - If RS is on the MPLS network in a VRF ```ios-xr interface Loopback100 vrf DCI From 5cab89c65d5b24469bdfdc22ceb3cd1f55b9fd01 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Fri, 5 Jan 2024 15:59:26 -0500 Subject: [PATCH 32/33] Removed references to regions from RS example --- examples/load-balanced-route-servers/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/load-balanced-route-servers/README.md b/examples/load-balanced-route-servers/README.md index 23d9e62..340c129 100644 --- a/examples/load-balanced-route-servers/README.md +++ b/examples/load-balanced-route-servers/README.md @@ -1,4 +1,4 @@ -# Example: Fully-meshed Multi-region Route Servers +# Example: Fully-meshed Route Servers ## Description * cRPD is used as route servers for EVPN connectivity to minimize peering requirements in a many-DC environment. @@ -165,8 +165,8 @@ - `Service/meshrr-core` - Provides a headless service coordinating the meshrr function of automatically forming full-mesh iBGP peerings between route servers. Also includes the IPAddressPool and corresponding L2Advertisement for the management network connectivity of the route servers. (In a lab environment, a single L2 domain for cRPD management was sufficient.) - `metallb-bgppeer-global.yml` - `bgppeers.metallb.io/asn100-global-lo100` - Peers MetalLB to the loopback deployed for on each router connecting to the Kubernetes cluster. -- `routeserver--.yml` - - `ipaddresspools.metallb.io/routeserver--` - Creates a pool containing the single address for the service per region per side. `autoAssign: false` ensures that the address is not allocated unless specifically requested by the service. - - `bgpadvertisements.metallb.io/routeserver--` - Advertises the address to all peers (by default) from all nodes that host and endpoint for the service. - - `Service/routeserver--` - Allocates the address based on the pool defined previously and uses it as an external address load balancing BGP to all healthy pods matching the criteria. - - `Deployment/routeserver--` - Creates a deployment of the service for the region and side. In the `routerserver-1-b` example, `replicas: 2`, but for most production deployments, 1 should be sufficient and operationally simpler. \ No newline at end of file +- `routeserver-.ss.yml` + - `ipaddresspools.metallb.io/routeserver-` - Creates a pool containing the single address for the service per side. `autoAssign: false` ensures that the address is not allocated unless specifically requested by the service. + - `bgpadvertisements.metallb.io/routeserver-` - Advertises the address to all peers (by default) from all nodes that host and endpoint for the service. + - `Service/routeserver-` - Allocates the address based on the pool defined previously and uses it as an external address load balancing BGP to all healthy pods matching the criteria. + - `StatefulSet/routeserver-` - Creates a StatefulSet of the service for the side. In the `routerserver-b` example, `replicas: 2`, but for most production deployments, 1 should be sufficient and operationally simpler. \ No newline at end of file From d14afeae41481fd69b0c0084a3746e35e224a817 Mon Sep 17 00:00:00 2001 From: "Jason R. Rokeach" <1076569+jrokeach@users.noreply.github.com> Date: Fri, 5 Jan 2024 16:04:54 -0500 Subject: [PATCH 33/33] Update manifests to v0.2 image tag --- examples/3clientgroups/meshrr-3clientgroups.yaml | 8 ++++---- examples/load-balanced-route-servers/routeserver-a.ss.yml | 4 ++-- examples/load-balanced-route-servers/routeserver-b.ss.yml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/3clientgroups/meshrr-3clientgroups.yaml b/examples/3clientgroups/meshrr-3clientgroups.yaml index d9e5cbb..f213033 100644 --- a/examples/3clientgroups/meshrr-3clientgroups.yaml +++ b/examples/3clientgroups/meshrr-3clientgroups.yaml @@ -278,7 +278,7 @@ spec: name: override-config initContainers: - name: meshrr-init - image: ghcr.io/juniper/meshrr:next + image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["init"] volumeMounts: @@ -338,7 +338,7 @@ spec: privileged: true runAsNonRoot: false - name: meshrr - image: ghcr.io/juniper/meshrr:next + image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["sidecar"] startupProbe: @@ -420,7 +420,7 @@ spec: name: override-config initContainers: - name: meshrr-init - image: ghcr.io/juniper/meshrr:next + image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["init"] volumeMounts: @@ -480,7 +480,7 @@ spec: privileged: true runAsNonRoot: false - name: meshrr - image: ghcr.io/juniper/meshrr:next + image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["sidecar"] startupProbe: diff --git a/examples/load-balanced-route-servers/routeserver-a.ss.yml b/examples/load-balanced-route-servers/routeserver-a.ss.yml index 697441e..f8b61d4 100644 --- a/examples/load-balanced-route-servers/routeserver-a.ss.yml +++ b/examples/load-balanced-route-servers/routeserver-a.ss.yml @@ -141,7 +141,7 @@ spec: optional: false initContainers: - name: meshrr-init - image: ghcr.io/juniper/meshrr:next + image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["init"] volumeMounts: @@ -205,7 +205,7 @@ spec: privileged: true runAsNonRoot: false - name: meshrr - image: ghcr.io/juniper/meshrr:next + image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["sidecar"] startupProbe: diff --git a/examples/load-balanced-route-servers/routeserver-b.ss.yml b/examples/load-balanced-route-servers/routeserver-b.ss.yml index 93c113b..56ee83e 100644 --- a/examples/load-balanced-route-servers/routeserver-b.ss.yml +++ b/examples/load-balanced-route-servers/routeserver-b.ss.yml @@ -97,7 +97,7 @@ spec: optional: false initContainers: - name: meshrr-init - image: ghcr.io/juniper/meshrr:next + image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["init"] volumeMounts: @@ -161,7 +161,7 @@ spec: privileged: true runAsNonRoot: false - name: meshrr - image: ghcr.io/juniper/meshrr:next + image: ghcr.io/juniper/meshrr:v0.2 imagePullPolicy: IfNotPresent args: ["sidecar"] startupProbe: