forked from vitessio/vitess
-
Notifications
You must be signed in to change notification settings - Fork 3
328 lines (287 loc) · 14.3 KB
/
upgrade_downgrade_test_backups_manual.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
name: Upgrade Downgrade Testing - Backups - Manual
on:
push:
pull_request:
concurrency:
group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing - Backups - Manual')
cancel-in-progress: true
jobs:
get_previous_release:
if: always()
name: Get Previous Release - Backups - Manual
runs-on:
group: vitess-ubuntu20
outputs:
previous_release: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }}
steps:
- name: Check out to HEAD
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set output with latest release branch
id: output-previous-release-ref
run: |
previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}})
echo $previous_release_ref
echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT
# This job usually execute in ± 20 minutes
upgrade_downgrade_test_manual:
timeout-minutes: 40
if: always() && (needs.get_previous_release.result == 'success')
name: Run Upgrade Downgrade Test - Backups - Manual
runs-on:
group: vitess-ubuntu20
needs:
- get_previous_release
steps:
- name: Check if workflow needs to be skipped
id: skip-workflow
run: |
skip='false'
if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then
skip='true'
fi
echo Skip ${skip}
echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT
# Checkout to this build's commit
- name: Checkout to commit's code
if: steps.skip-workflow.outputs.skip-workflow == 'false'
uses: actions/checkout@v3
- name: Check for changes in relevant files
if: steps.skip-workflow.outputs.skip-workflow == 'false'
uses: frouioui/paths-filter@main
id: changes
with:
token: ''
filters: |
end_to_end:
- 'go/**'
- 'go/**/*.go'
- 'test.go'
- 'Makefile'
- 'build.env'
- 'go.sum'
- 'go.mod'
- 'proto/*.proto'
- 'tools/**'
- 'config/**'
- 'bootstrap.sh'
- '.github/workflows/upgrade_downgrade_test_backups_manual.yml'
- name: Set up older Go
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/setup-go@v3
with:
go-version: 1.18.7
- name: Set up python
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/setup-python@v4
- name: Tune the OS
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
# TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185
- name: Add the current IP address, long hostname and short hostname record to /etc/hosts file
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
# DON'T FORGET TO REMOVE CODE ABOVE WHEN ISSUE IS ADRESSED!
- name: Get base dependencies
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
sudo DEBIAN_FRONTEND="noninteractive" apt-get update
# Uninstall any previously installed MySQL first
sudo systemctl stop apparmor
sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common
sudo apt-get -y autoremove
sudo apt-get -y autoclean
sudo deluser mysql
sudo rm -rf /var/lib/mysql
sudo rm -rf /etc/mysql
# Install MySQL 8.0
## Temporarily pin the MySQL version at 8.0.28
#sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29
#wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb
#echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections
#sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
#sudo apt-get update
#sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client
####
wget -c https://cdn.mysql.com/archives/mysql-8.0/mysql-common_8.0.28-1ubuntu20.04_amd64.deb \
https://cdn.mysql.com/archives/mysql-8.0/mysql-community-client-core_8.0.28-1ubuntu20.04_amd64.deb \
https://cdn.mysql.com/archives/mysql-8.0/mysql-community-client-plugins_8.0.28-1ubuntu20.04_amd64.deb \
https://cdn.mysql.com/archives/mysql-8.0/mysql-client_8.0.28-1ubuntu20.04_amd64.deb \
https://cdn.mysql.com/archives/mysql-8.0/mysql-community-server-core_8.0.28-1ubuntu20.04_amd64.deb \
https://cdn.mysql.com/archives/mysql-8.0/mysql-community-server_8.0.28-1ubuntu20.04_amd64.deb \
https://cdn.mysql.com/archives/mysql-8.0/mysql-community-client_8.0.28-1ubuntu20.04_amd64.deb
sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y ./mysql-*.deb
# Install everything else we need, and configure
sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata grep
sudo service mysql stop
sudo service etcd stop
sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile"
# install JUnit report formatter
go install github.com/vitessio/go-junit-report@HEAD
wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
sudo apt-get install -y gnupg2
sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
sudo apt-get update
sudo apt-get install percona-xtrabackup-24
# Checkout to the last release of Vitess
- name: Checkout to the other version's code (${{ needs.get_previous_release.outputs.previous_release }})
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/checkout@v3
with:
ref: ${{ needs.get_previous_release.outputs.previous_release }}
- name: Get dependencies for the last release
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
go mod download
- name: Building last release's binaries
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env
make build
mkdir -p /tmp/vitess-build-other/
cp -R bin /tmp/vitess-build-other/
rm -Rf bin/*
- name: Set up current Go
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/setup-go@v3
with:
go-version: 1.19.12
# Checkout to this build's commit
- name: Checkout to commit's code
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/checkout@v3
- name: Get dependencies for this commit
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
go mod download
- name: Run make minimaltools
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
make minimaltools
- name: Building the binaries for this commit
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env
make build
mkdir -p /tmp/vitess-build-current/
cp -R bin /tmp/vitess-build-current/
# We create a sharded Vitess cluster following the local example.
# We also insert a few rows in our three tables.
- name: Create the example Vitess cluster with all components using version N
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env ; cd examples/local
./backups/start_cluster.sh
# Taking a backup
- name: Take a backup of all the shards
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 2
run: |
source build.env ; cd examples/local
./backups/take_backups.sh
# We insert more data in every table after the backup.
# When we restore the backup made in the previous step, we do not want to see the rows we are about to insert now.
# The initial number of rows for each table is:
# - customer: 5
# - product: 2
# - corder: 5
# We shall see the same number of rows after restoring the backup.
- name: Insert more data after the backup
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env ; cd examples/local ; source ./env.sh
echo "insert into customer(email) values('[email protected]');" | mysql
echo "insert into product(sku, description, price) values('SKU-1009', 'description', 89);" | mysql
echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1009', 100);" | mysql
# Stop all the tablets and remove their data
- name: Stop tablets
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env ; cd examples/local
./backups/stop_tablets.sh
# We downgrade: we use the version N-1 of vttablet
- name: Downgrade - Swap binaries, use VTTablet N-1
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env
rm -f $PWD/bin/vttablet
cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
vttablet --version
# Starting the tablets again, they will automatically start restoring the last backup.
- name: Start new tablets and restore
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env ; cd examples/local
./backups/restart_tablets.sh
# give enough time to the tablets to restore the backup
sleep 60
# Count the number of rows in each table to make sure the restoration is successful.
- name: Assert the number of rows in every table
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env ; cd examples/local ; source ./env.sh
echo "select count(sku) from product;" | mysql 2>&1| grep 2
echo "select count(email) from customer;" | mysql 2>&1| grep 5
echo "select count(sku) from corder;" | mysql 2>&1| grep 5
# We insert one more row in every table.
- name: Insert more rows in the tables
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env ; cd examples/local ; source ./env.sh
echo "insert into customer(email) values('[email protected]');" | mysql
echo "insert into product(sku, description, price) values('SKU-1011', 'description', 111);" | mysql
echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1011', 111);" | mysql
# Taking a second backup of the cluster.
- name: Take a second backup of all the shards
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 2
run: |
source build.env ; cd examples/local
./backups/take_backups.sh
# Stopping the tablets so we can perform the upgrade.
- name: Stop tablets
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env ; cd examples/local
./backups/stop_tablets.sh
# We upgrade: we swap binaries and use the version N of the tablet.
- name: Upgrade - Swap binaries, use VTTablet N
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env
rm -f $PWD/bin/vttablet
cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
vttablet --version
# Starting the tablets again and restoring the previous backup.
- name: Start new tablets and restore
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env ; cd examples/local
./backups/restart_tablets.sh
# give enough time to the tablets to restore the backup
sleep 60
# We count the number of rows in every table to check that the restore step was successful.
- name: Assert the number of rows in every table
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env ; cd examples/local ; source ./env.sh
echo "select count(sku) from product;" | mysql 2>&1| grep 3
echo "select count(email) from customer;" | mysql 2>&1| grep 6
echo "select count(sku) from corder;" | mysql 2>&1| grep 6
- name: Stop the Vitess cluster
if: always() && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env ; cd examples/local
./401_teardown.sh || true