Skip to content

Commit

Permalink
systemd: improve the time synchronisation process at boot
Browse files Browse the repository at this point in the history
Improve the reliability and robustness of time synchronisation
during boot.

A new time-set.target has been added (as per systemd versions
since v242). The timeinit-rtc and timeinit-timestamp services
have been added to the time-set.target.

Chrony is now part of the time-sync.target and has a dependency on
the time-set.target. New services have been added to the
time-sync.target to ensure that system time is synchronised with a
remote source before the time-sync.target is reached.

The timesync-remote service is a blocking service that will only
terminate if:
a) NTP synchronisation has been achieved (via chrony), or
b) time is successfully updated using ntpdate, or
c) time is successfully updated from an HTTPS header (using curl).

The timesync-https service is a background service that will
periodically attempt to update the system time from an HTTPS header.
The service will terminate if NTP synchronisation has been achieved
(via chrony). This service provides a time synchronisation backup in
cases where access to NTP port 123 is blocked.

The openvpn service now pulls in the time-sync.target and will only
start after initial time synchronisation has completed as described
above.

Change-type: minor
Connects-to: #1852 #1776 #1337
Signed-off-by: Mark Corbin <mark@balena.io>
  • Loading branch information
markcorbinuk committed Aug 18, 2020
1 parent abdd15e commit 3c652f2
Show file tree
Hide file tree
Showing 12 changed files with 287 additions and 11 deletions.
Original file line number Diff line number Diff line change
@@ -1,6 +1,20 @@
# Copyright 2018-2020 Balena Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

[Unit]
Description=OpenVPN
Requires=prepare-openvpn.service bind-etc-openvpn.service
Requires=prepare-openvpn.service bind-etc-openvpn.service time-sync.target
After=syslog.target network.target prepare-openvpn.service bind-etc-openvpn.service time-sync.target
ConditionFileNotEmpty=/etc/openvpn/openvpn.conf

Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
[Unit]
Before=time-sync.target
Wants=time-sync.target var-volatile-lib.service
After=var-volatile-lib.service
Wants=time-set.target var-volatile-lib.service
After=time-set.target var-volatile-lib.service

[Service]
Type=simple
Restart=always
RestartSec=10s
ExecStart=
ExecStart=/usr/sbin/chronyd -s -d

[Install]
WantedBy=time-sync.target
20 changes: 19 additions & 1 deletion meta-balena-common/recipes-core/systemd/timeinit.bb
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2018 Resinio Ltd.
# Copyright 2018-2020 Balena Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -20,6 +20,12 @@ SRC_URI = " \
file://timeinit-rtc.service \
file://timeinit-timestamp.service \
file://timeinit-timestamp.sh \
file://time-set.target \
file://time-sync.target.conf \
file://timesync-remote.service \
file://timesync-remote.sh \
file://timesync-https.service \
file://timesync-https.sh \
"
S = "${WORKDIR}"

Expand All @@ -28,6 +34,8 @@ inherit allarch systemd
SYSTEMD_SERVICE_${PN} = " \
timeinit-rtc.service \
timeinit-timestamp.service \
timesync-remote.service \
timesync-https.service \
"

do_install() {
Expand All @@ -36,4 +44,14 @@ do_install() {
install -m 0775 ${WORKDIR}/timeinit-timestamp.sh ${D}${bindir}
install -c -m 0644 ${WORKDIR}/timeinit-timestamp.service ${D}${systemd_unitdir}/system
install -c -m 0644 ${WORKDIR}/timeinit-rtc.service ${D}${systemd_unitdir}/system
install -c -m 0644 ${WORKDIR}/time-set.target ${D}${systemd_unitdir}/system

# Update time-sync.target to want the new time-set.target
install -d -m 0755 ${D}/${sysconfdir}/systemd/system/time-sync.target.d
install -m 0644 ${WORKDIR}/time-sync.target.conf ${D}/${sysconfdir}/systemd/system/time-sync.target.d/

install -m 0775 ${WORKDIR}/timesync-remote.sh ${D}${bindir}
install -c -m 0644 ${WORKDIR}/timesync-remote.service ${D}${systemd_unitdir}/system
install -m 0775 ${WORKDIR}/timesync-https.sh ${D}${bindir}
install -c -m 0644 ${WORKDIR}/timesync-https.service ${D}${systemd_unitdir}/system
}
13 changes: 13 additions & 0 deletions meta-balena-common/recipes-core/systemd/timeinit/time-set.target
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# SPDX-License-Identifier: LGPL-2.1+
#
# This file is part of systemd.
#
# systemd is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.

[Unit]
Description=System Time Set
Documentation=man:systemd.special(7)
RefuseManualStart=yes
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[Unit]
After=time-set.target
Wants=time-set.target
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2018 Resinio Ltd.
# Copyright 2018-2020 Balena Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -18,12 +18,12 @@ ConditionVirtualization=!docker
ConditionPathExists=/dev/rtc
DefaultDependencies=no
After=systemd-udev-settle.service
Before=chronyd.service timeinit-timestamp.service
Before=chronyd.service timeinit-timestamp.service time-set.target

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/sbin/hwclock --hctosys

[Install]
WantedBy=time-sync.target
WantedBy=time-set.target
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2018 Resinio Ltd.
# Copyright 2018-2020 Balena Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -15,12 +15,12 @@
[Unit]
Description=Initialize system clock from build timestamp
DefaultDependencies=no
Before=chronyd.service
Before=chronyd.service time-set.target

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/bin/sh /usr/bin/timeinit-timestamp.sh

[Install]
WantedBy=time-sync.target
WantedBy=time-set.target
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/sh
#
# Copyright 2018 Resinio Ltd.
# Copyright 2018-2020 Balena Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -23,6 +23,8 @@ if [ ! -f $TIMESTAMP ]; then
exit 1
fi

echo "[INFO] Checking build time."

SYS_TIME=$(date -u "+%4Y%2m%2d%2H%2M%2S")
BUILD_TIME=$(cat $TIMESTAMP)

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Copyright 2020 Balena Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

[Unit]
Description=Initialize system clock from a secure website
DefaultDependencies=no
After=chronyd.service network.target timesync-remote.service
Before=time-sync.target

[Service]
Type=simple
RemainAfterExit=yes
ExecStart=/bin/sh /usr/bin/timesync-https.sh

[Install]
WantedBy=time-sync.target
64 changes: 64 additions & 0 deletions meta-balena-common/recipes-core/systemd/timeinit/timesync-https.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
#!/bin/sh
#
# Copyright 2020 Balena Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

trap 'exit' INT

# Synchronisation period in seconds
SLEEP_INTERVAL=1800

HTTPS_SERVER=https://balena-cloud.com


function set_system_date {
local DATETIME="$(echo "$1" | awk '{string=substr($0, 5, 8); print string;}')"
local YEAR="$(echo "$1" | awk '{string=substr($0, 1, 4); print string;}')"
local SEC="$(echo "$1" | awk '{string=substr($0, 13, 2); print string;}')"
date -u "${DATETIME}${YEAR}.${SEC}" > /dev/null 2>&1
}

function exit_service {
echo "[INFO] Exiting HTTPS time synchronisation service."
exit 0
}


echo "[INFO] Starting HTTPS time synchronisation service."

while [ true ]; do
# Test for NTP synchronisation
echo "[INFO] Checking NTP synchronisation."
chronyc waitsync 1 > /dev/null 2>&1
RESULT=$?

if [ $RESULT -eq 0 ]; then
echo "[INFO] NTP service synchronised."
exit_service
fi

SYS_TIME=$(date -u "+%4Y%2m%2d%2H%2M%2S")
SERVER_TIME_STRING=$(curl --silent -m5 -k -I "$HTTPS_SERVER" --stderr - | grep -i Date: | sed -e 's/[Dd]ate: //')

echo "[INFO] Checking HTTPS header time."
if [ ! -z "$SERVER_TIME_STRING" ]; then
SERVER_TIME=$(date "+%4Y%2m%2d%2H%2M%2S" -d "$SERVER_TIME_STRING")
if [ "$SYS_TIME" -lt "$SERVER_TIME" ]; then
set_system_date $SERVER_TIME
echo "[INFO] Time synchronised via HTTPS."
fi
fi

sleep $SLEEP_INTERVAL
done
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Copyright 2020 Balena Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

[Unit]
Description=Initialize system clock from a remote source
DefaultDependencies=no
ConditionPathExists=!/dev/rtc
ConditionPathExists=!/var/lib/chrony/drift
Wants=network-online.target
After=chronyd.service network-online.target
Before=time-sync.target

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/bin/sh /usr/bin/timesync-remote.sh

[Install]
WantedBy=time-sync.target
102 changes: 102 additions & 0 deletions meta-balena-common/recipes-core/systemd/timeinit/timesync-remote.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
#!/bin/sh
#
# Copyright 2020 Balena Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

trap 'exit' INT

BURST_COUNT=5
BURST_WAITSYNC_DELAY=3
NTPDATE_SERVER=0.resinio.pool.ntp.org
HTTPS_SERVER=https://balena-cloud.com


BURST_COUNTER=1

function set_system_date {
local DATETIME="$(echo "$1" | awk '{string=substr($0, 5, 8); print string;}')"
local YEAR="$(echo "$1" | awk '{string=substr($0, 1, 4); print string;}')"
local SEC="$(echo "$1" | awk '{string=substr($0, 13, 2); print string;}')"
date -u "${DATETIME}${YEAR}.${SEC}" > /dev/null 2>&1
}

function exit_service {
echo "[INFO] Exiting remote time synchronisation service."
exit 0
}


echo "[INFO] Starting remote time synchronisation service."

while [ true ]; do
# Test for NTP synchronisation
echo "[INFO] Checking NTP synchronisation."
chronyc waitsync 1 > /dev/null 2>&1
RESULT=$?

if [ $RESULT -eq 0 ]; then
echo "[INFO] NTP service synchronised."
exit_service
fi

# Run an NTP burst every BURST_COUNT times round the loop
if [ $BURST_COUNTER -eq $BURST_COUNT ]; then
echo "[INFO] Starting NTP burst sequence."
# Wait for any active bursts to complete
echo -n "[INFO] Waiting for active bursts to complete..."
BURST_HOSTS=$(chronyc activity | grep burst.*online | awk '{ print $1 }')
if [ ! -z "$BURST_HOSTS" ]; then
until [ $BURST_HOSTS -eq 0 ]; do
sleep 5
BURST_HOSTS=$(chronyc activity | grep burst.*online | awk '{ print $1 }')
if [ -z "$BURST_HOSTS" ]; then
break
fi
done
fi
echo "done."
echo "[INFO] Triggering NTP burst."
chronyc burst 4/8 > /dev/null 2>&1
chronyc waitsync $BURST_WAITSYNC_DELAY > /dev/null 2>&1
RESULT=$?
if [ $RESULT -eq 0 ]; then
echo "[INFO] NTP service synchronised."
exit_service
fi
BURST_COUNTER=0
fi

# Run ntpdate
echo "[INFO] Running ntpdate."
ntpdate $NTPDATE_SERVER > /dev/null 2>&1
RESULT=$?
if [ $RESULT -eq 0 ]; then
echo "[INFO] Time synchronised via ntpdate."
exit_service
fi

SYS_TIME=$(date -u "+%4Y%2m%2d%2H%2M%2S")
SERVER_TIME_STRING=$(curl --silent -m5 -k -I "$HTTPS_SERVER" --stderr - | grep -i Date: | sed -e 's/[Dd]ate: //')

echo "[INFO] Checking HTTPS header time."
if [ ! -z "$SERVER_TIME_STRING" ]; then
SERVER_TIME=$(date "+%4Y%2m%2d%2H%2M%2S" -d "$SERVER_TIME_STRING")
if [ "$SYS_TIME" -lt "$SERVER_TIME" ]; then
set_system_date $SERVER_TIME
echo "[INFO] Time synchronised via HTTPS."
exit_service
fi
fi
let BURST_COUNTER=BURST_COUNTER+1
done

0 comments on commit 3c652f2

Please sign in to comment.