-
Notifications
You must be signed in to change notification settings - Fork 186
273 lines (266 loc) · 14.8 KB
/
build_and_test.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
name: "Build and Test"
on:
- pull_request
- workflow_dispatch
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Build sidecar
uses: docker/build-push-action@v4
with:
push: false
outputs: type=docker,dest=/tmp/k8s-sidecar.tar
tags: "kiwigrid/k8s-sidecar:testing"
- name: Prepare dummy server static resources
run: |
cp test/kubelogo.png test/server/static/
- name: Build dummy server
uses: docker/build-push-action@v4
with:
context: "test/server"
push: false
outputs: type=docker,dest=/tmp/dummy-server.tar
tags: "dummy-server:1.0.0"
- name: Upload artifacts
uses: actions/upload-artifact@v3
with:
name: images
path: /tmp/*.tar
test:
needs:
- build
strategy:
matrix:
# see https://github.com/kubernetes-sigs/kind/releases for supported k8s versions per kind version
k8s:
- maj_min: v1.21
digest: sha256:8a4e9bb3f415d2bb81629ce33ef9c76ba514c14d707f9797a01e3216376ba093
- maj_min: v1.22
digest: sha256:f5b2e5698c6c9d6d0adc419c0deae21a425c07d81bbf3b6a6834042f25d4fba2
- maj_min: v1.23
digest: sha256:fbb92ac580fce498473762419df27fa8664dbaa1c5a361b5957e123b4035bdcf
- maj_min: v1.24
digest: sha256:ea292d57ec5dd0e2f3f5a2d77efa246ac883c051ff80e887109fabefbd3125c7
- maj_min: v1.25
digest: sha256:9d0a62b55d4fe1e262953be8d406689b947668626a357b5f9d0cfbddbebbc727
- maj_min: v1.26
digest: sha256:15ae92d507b7d4aec6e8920d358fc63d3b980493db191d7327541fbaaed1f789
- maj_min: v1.27
digest: sha256:3700c811144e24a6c6181065265f69b9bf0b437c45741017182d7c82b908918f
- maj_min: v1.28
digest: sha256:b7e1cf6b2b729f604133c667a6be8aab6f4dde5bb042c1891ae248d9154f665b
- maj_min: v1.29
digest: sha256:a0cc28af37cf39b019e2b448c54d1a3f789de32536cb5a5db61a49623e527144
name: "Test on k8s ${{ matrix.k8s.maj_min }}"
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Create k8s Kind Cluster
# make sure the k8s versions match the kind version of the action version 🤯
uses: helm/[email protected]
with:
node_image: kindest/node@${{ matrix.k8s.digest }}
config: test/kind-config.yaml
cluster_name: sidecar-testing
wait: 5m
- name: Download artifact
uses: actions/download-artifact@v3
with:
name: images
path: /tmp
- name: Install Helm
run: |
wget https://get.helm.sh/helm-v3.16.1-linux-amd64.tar.gz
tar -zxf helm-v3.16.1-linux-amd64.tar.gz
mv linux-amd64/helm /usr/local/bin/helm
- name: Install cert-manager
run: |
helm repo add jetstack https://charts.jetstack.io
helm repo update
helm install cert-manager jetstack/cert-manager --version v1.11.0 --set installCRDs=true
- name: Load images into kind cluster
run: |
kind load image-archive /tmp/k8s-sidecar.tar --name sidecar-testing
kind load image-archive /tmp/dummy-server.tar --name sidecar-testing
- name: Install Sidecar and Dummy Server
run: |
wait_for_pod_ready() {
while [[ $(kubectl get pods $1 -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do echo "waiting for pod '$1' to become ready..." && sleep 5; done
echo "Pod '$1' ready."
}
echo "Installing sidecar..."
kubectl apply -f "test/resources/sidecar.yaml"
sleep 10
kubectl get pods
wait_for_pod_ready "sidecar"
wait_for_pod_ready "sidecar-basicauth-args"
wait_for_pod_ready "sidecar-5xx"
wait_for_pod_ready "sidecar-tls"
wait_for_pod_ready "sidecar-pythonscript"
wait_for_pod_ready "sidecar-pythonscript-logfile"
wait_for_pod_ready "sidecar-logtofile-pythonscript"
wait_for_pod_ready "dummy-server-pod"
- name: Install Configmaps and Secrets
run: |
wait_for_pod_log() {
while [[ $(kubectl logs $1 | grep $2) == "" ]]; do echo "waiting 5 more seconds for '$2' to appear in logs of pod '$1'..." && sleep 5; done
echo "Pod '$1' logs contains '$2'"
}
# because the sidecar pods signal ready state before we actually opened up all watching subprocesses, we wait some more time
sleep 20
echo "Installing resources..."
kubectl apply -f "test/resources/resources.yaml"
pods=("sidecar" "sidecar-basicauth-args" "sidecar-5xx" "sidecar-pythonscript" "sidecar-pythonscript-logfile")
resources=("sample-configmap" "sample-secret-binary" "absolute-configmap" "relative-configmap" "change-dir-configmap" "similar-configmap-secret" "url-configmap-500" "url-configmap-basic-auth" "sample-configmap")
for p in ${pods[*]}; do
for r in ${resources[*]}; do
wait_for_pod_log $p $r
done
done
wait_for_pod_log "sidecar-tls" "tls-url-configmap-200"
# 10 more seconds after the last thing appeared in the logs.
sleep 10
- name: Retrieve pod logs
run: |
mkdir /tmp/logs
kubectl logs sidecar > /tmp/logs/sidecar.log
kubectl logs sidecar-basicauth-args > /tmp/logs/sidecar-basicauth-args.log
kubectl logs sidecar-5xx > /tmp/logs/sidecar-5xx.log
kubectl logs sidecar-pythonscript > /tmp/logs/sidecar-pythonscript.log
kubectl logs sidecar-pythonscript-logfile > /tmp/logs/sidecar-pythonscript-logfile.log
kubectl logs dummy-server-pod > /tmp/logs/dummy-server.log
kubectl logs sidecar-tls > /tmp/logs/sidecar-tls.log
#For Debug
cat /tmp/logs/sidecar-tls.log
- name: Upload artifacts (pod logs)
uses: actions/upload-artifact@v3
with:
name: pod-logs_${{ matrix.k8s.maj_min }}
path: /tmp/logs/*
- name: Download expected files from cluster
run: |
mkdir /tmp/sidecar
mkdir /tmp/sidecar-tls
mkdir /tmp/sidecar-5xx
echo "Downloading resource files from sidecar..."
kubectl cp sidecar:/tmp/hello.world /tmp/sidecar/hello.world
kubectl cp sidecar:/tmp/cm-kubelogo.png /tmp/sidecar/cm-kubelogo.png
kubectl cp sidecar:/tmp/secret-kubelogo.png /tmp/sidecar/secret-kubelogo.png
kubectl cp sidecar:/tmp/url-downloaded-kubelogo.png /tmp/sidecar/url-downloaded-kubelogo.png
# script also generates into '/tmp'
kubectl cp sidecar:/tmp/script_result /tmp/sidecar/script_result
# absolute path in configmap points to /tmp in 'absolute-configmap'
kubectl cp sidecar:/tmp/absolute/absolute.txt /tmp/sidecar/absolute.txt
kubectl cp sidecar:/tmp/relative/relative.txt /tmp/sidecar/relative.txt
kubectl cp sidecar:/tmp/orig-dir/change-dir.txt /tmp/sidecar/change-dir.txt
kubectl cp sidecar:/tmp/500.txt /tmp/sidecar/500.txt || true
kubectl cp sidecar:/tmp/secured.txt /tmp/sidecar/secured.txt
kubectl cp sidecar:/tmp/similar-configmap.txt /tmp/sidecar/similar-configmap.txt
kubectl cp sidecar:/tmp/similar-secret.txt /tmp/sidecar/similar-secret.txt
echo "Downloading resource files from sidecar-basicauth-args pod"
kubectl cp sidecar-basicauth-args:/tmp/secured.txt /tmp/sidecar-basicauth-args/secured.txt
echo "Downloading resource files from sidecar-5xx..."
kubectl cp sidecar-5xx:/tmp-5xx/hello.world /tmp/sidecar-5xx/hello.world
kubectl cp sidecar-5xx:/tmp-5xx/cm-kubelogo.png /tmp/sidecar-5xx/cm-kubelogo.png
kubectl cp sidecar-5xx:/tmp-5xx/secret-kubelogo.png /tmp/sidecar-5xx/secret-kubelogo.png
kubectl cp sidecar-5xx:/tmp-5xx/url-downloaded-kubelogo.png /tmp/sidecar-5xx/url-downloaded-kubelogo.png
# script also generates into '/tmp'
kubectl cp sidecar-5xx:/tmp/script_result /tmp/sidecar-5xx/script_result
# absolute path in configmap points to /tmp in 'absolute-configmap'
kubectl cp sidecar-5xx:/tmp/absolute/absolute.txt /tmp/sidecar-5xx/absolute.txt
kubectl cp sidecar-5xx:/tmp-5xx/relative/relative.txt /tmp/sidecar-5xx/relative.txt
kubectl cp sidecar-5xx:/tmp-5xx/orig-dir/change-dir.txt /tmp/sidecar-5xx/change-dir.txt
kubectl cp sidecar-5xx:/tmp-5xx/500.txt /tmp/sidecar-5xx/500.txt
kubectl cp sidecar-5xx:/tmp-5xx/secured.txt /tmp/sidecar-5xx/secured.txt
kubectl cp sidecar-5xx:/tmp-5xx/similar-configmap.txt /tmp/sidecar-5xx/similar-configmap.txt
kubectl cp sidecar-5xx:/tmp-5xx/similar-secret.txt /tmp/sidecar-5xx/similar-secret.txt
echo "Downloading resource files from sidecar-tls pod"
kubectl cp sidecar-tls:/tmp/200-tls.txt /tmp/sidecar-tls/200-tls.txt
# For Debug
cat /tmp/sidecar-tls/200-tls.txt
- name: Upload artifacts (expected files from cluster)
uses: actions/upload-artifact@v3
with:
name: expected-files_${{ matrix.k8s.maj_min }}
path: |
/tmp/sidecar/**
/tmp/sidecar-tls/**
/tmp/sidecar-5xx/**
- name: Update Configmaps and Secrets
run: |
sleep 5
current_time=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
wait_for_pod_log() {
while [[ $(kubectl logs $1 --since-time ${current_time} | grep $2) == "" ]]; do echo "waiting 5 more seconds for '$2' to appear in logs of pod '$1'..." && sleep 5; done
echo "Pod '$1' logs contains '$2'"
}
echo "Updating resources..."
kubectl apply -f "test/resources/change_resources.yaml"
pods=("sidecar" "sidecar-5xx")
resources=("sample-configmap" "sample-secret-binary" "absolute-configmap" "relative-configmap" "change-dir-configmap" "similar-configmap-secret" "url-configmap-500" "url-configmap-basic-auth" "sample-configmap")
for p in ${pods[*]}; do
for r in ${resources[*]}; do
wait_for_pod_log $p $r
done
done
# 20 more seconds after the last thing appeared in the logs.
sleep 20
- name: Verify sidecar files after initial sync
run: |
echo -n "Hello World!" | diff - /tmp/sidecar/hello.world &&
diff test/kubelogo.png /tmp/sidecar/cm-kubelogo.png &&
diff test/kubelogo.png /tmp/sidecar/secret-kubelogo.png &&
diff test/kubelogo.png /tmp/sidecar/url-downloaded-kubelogo.png &&
echo -n "This absolutely exists" | diff - /tmp/sidecar/absolute.txt &&
echo -n "This relatively exists" | diff - /tmp/sidecar/relative.txt &&
echo -n "This change-dir exists" | diff - /tmp/sidecar/change-dir.txt &&
echo -n "I'm very similar" | diff - /tmp/sidecar/similar-configmap.txt &&
echo -n "I'm very similar" | diff - /tmp/sidecar/similar-secret.txt &&
echo -n "allowed" | diff - /tmp/sidecar/secured.txt &&
[ ! -f /tmp/sidecar-tls/200-tls.txt ] && echo "No 200-tls.txt file created" &&
echo -n "200" | diff - /tmp/sidecar-tls/200-tls.txt &&
[ ! -f /tmp/sidecar/500.txt ] && echo "No 5xx file created" &&
ls /tmp/sidecar/script_result
- name: Verify sidecar-basicauth-args pod file after initial sync
run: |
echo -n "allowed" | diff - /tmp/sidecar-basicauth-args/secured.txt
- name: Verify sidecar-5xx files after initial sync
run: |
echo -n '{"detail":"Not authenticated"}' | diff - /tmp/sidecar-5xx/secured.txt &&
echo -n "Hello World!" | diff - /tmp/sidecar-5xx/hello.world &&
diff test/kubelogo.png /tmp/sidecar-5xx/cm-kubelogo.png &&
diff test/kubelogo.png /tmp/sidecar-5xx/secret-kubelogo.png &&
diff test/kubelogo.png /tmp/sidecar-5xx/url-downloaded-kubelogo.png &&
echo -n "This absolutely exists" | diff - /tmp/sidecar-5xx/absolute.txt &&
echo -n "This relatively exists" | diff - /tmp/sidecar-5xx/relative.txt &&
echo -n "This change-dir exists" | diff - /tmp/sidecar-5xx/change-dir.txt &&
echo -n "I'm very similar" | diff - /tmp/sidecar-5xx/similar-configmap.txt &&
echo -n "I'm very similar" | diff - /tmp/sidecar-5xx/similar-secret.txt &&
echo -n "500" | diff - /tmp/sidecar-5xx/500.txt &&
ls /tmp/sidecar-5xx/script_result
- name: Verify sidecar-python logs after initial sync
run: |
# Make sure to update this number this when adding or removing configmap or secrets
# For log to a file, Need to consider Jobs "Install Configmaps and Secrets" and "Update Configmaps and Secrets"
# Total is (9 + 7)
test $(cat /tmp/logs/sidecar-pythonscript.log | grep "Hello from python script!" | wc -l) = "9" &&
test $(cat /tmp/logs/sidecar-pythonscript-logfile.log | grep "Hello from python script!" | wc -l) = "9" &&
kubectl exec sidecar-logtofile-pythonscript -- sh -c "test -e /opt/logs/sidecar.log" &&
test $(kubectl exec sidecar-logtofile-pythonscript -- sh -c 'cat /opt/logs/sidecar.log | grep "Hello from python script!" | wc -l') = "16"
- name: Verify sidecar files after update
run: |
kubectl exec sidecar -- sh -c "ls /tmp/" &&
kubectl exec sidecar -- sh -c "! test -e /tmp/hello.world" && kubectl exec sidecar -- sh -c "test -e /tmp/change-hello.world" &&
kubectl exec sidecar -- sh -c "! test -e /tmp/cm-kubelogo.png" && kubectl exec sidecar -- sh -c "test -e /tmp/change-cm-kubelogo.png" &&
kubectl exec sidecar -- sh -c "! test -e /tmp/secret-kubelogo.png" && kubectl exec sidecar -- sh -c "test -e /tmp/change-secret-kubelogo.png" &&
kubectl exec sidecar -- sh -c "! test -e /tmp/absolute/absolute.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/absolute/change-absolute.txt" &&
kubectl exec sidecar -- sh -c "! test -e /tmp/relative/relative.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/relative/change-relative.txt" &&
kubectl exec sidecar -- sh -c "! test -e /tmp/orig-dir/change-dir.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/new-dir/change-dir.txt" &&
kubectl exec sidecar -- sh -c "! test -e /tmp/similar-configmap.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/change-similar-configmap.txt" &&
kubectl exec sidecar -- sh -c "! test -e /tmp/similar-secret.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/change-similar-secret.txt"