diff --git a/.travis.yml b/.travis.yml index 871b947c..c8a72337 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,4 +36,5 @@ install: - ./autogen.sh && ./configure && make script: + - ./scripts/kata-env-setup.sh - cd ${TRAVIS_BUILD_DIR} && hack/test-cmd.sh diff --git a/scripts/install_agent.sh b/scripts/install_agent.sh new file mode 100755 index 00000000..450234e7 --- /dev/null +++ b/scripts/install_agent.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# +# Copyright (c) 2017-2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +cidir=$(dirname "$0") + +source "${cidir}/lib.sh" + +clone_build_and_install "github.com/kata-containers/agent" diff --git a/scripts/install_kata_image.sh b/scripts/install_kata_image.sh new file mode 100755 index 00000000..d262a2a4 --- /dev/null +++ b/scripts/install_kata_image.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# +# Copyright (c) 2017-2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail + +cidir=$(dirname "$0") + +source /etc/os-release +source "${cidir}/lib.sh" + +AGENT_INIT=${AGENT_INIT:-no} +TEST_INITRD=${TEST_INITRD:-no} + +TMP_DIR= +ROOTFS_DIR= + +PACKAGED_IMAGE="kata-containers-image" +IMG_PATH="/usr/share/kata-containers" +IMG_NAME="kata-containers.img" + +agent_path="${GOPATH}/src/github.com/kata-containers/agent" + +IMG_MOUNT_DIR= +LOOP_DEVICE= + +# Build Kata agent +bash -f "${cidir}/install_agent.sh" +agent_commit=$(git --work-tree="${agent_path}" --git-dir="${agent_path}/.git" log --format=%h -1 HEAD) + +cleanup() { + [ -d "${ROOTFS_DIR}" ] && [[ "${ROOTFS_DIR}" = *"rootfs"* ]] && sudo rm -rf "${ROOTFS_DIR}" + [ -d "${TMP_DIR}" ] && rm -rf "${TMP_DIR}" + if [ -n "${IMG_MOUNT_DIR}" ] && mount | grep -q "${IMG_MOUNT_DIR}"; then + sudo umount "${IMG_MOUNT_DIR}" + fi + if [ -d "${IMG_MOUNT_DIR}" ]; then + rm -rf "${IMG_MOUNT_DIR}" + fi + if [ -n "${LOOP_DEVICE}" ]; then + sudo losetup -d "${LOOP_DEVICE}" + fi +} + +trap cleanup EXIT + +get_packaged_agent_version() { + version=$(ls "$IMG_PATH" | grep "$PACKAGED_IMAGE" | cut -d'_' -f4 | cut -d'.' -f1) + if [ -z "$version" ]; then + die "unknown agent version" + fi + echo "$version" +} + +install_packaged_image() { + if [ "$ID" == "ubuntu" ]; then + sudo -E apt install -y "$PACKAGED_IMAGE" + elif [ "$ID" == "fedora" ]; then + sudo -E dnf install -y "$PACKAGED_IMAGE" + elif [ "$ID" == "centos" ]; then + sudo -E yum install -y "$PACKAGED_IMAGE" + else + die "Linux distribution not supported" + fi +} + +update_agent() { + pushd "$agent_path" + + LOOP_DEVICE=$(sudo losetup -f --show "${IMG_PATH}/${IMG_NAME}") + IMG_MOUNT_DIR=$(mktemp -d -t kata-image-mount.XXXXXXXXXX) + sudo partprobe "$LOOP_DEVICE" + sudo mount "${LOOP_DEVICE}p1" "$IMG_MOUNT_DIR" + + echo "Old agent version:" + "${IMG_MOUNT_DIR}/usr/bin/kata-agent" --version + + echo "Install new agent" + sudo -E PATH="$PATH" bash -c "make install DESTDIR=$IMG_MOUNT_DIR" + installed_version=$("${IMG_MOUNT_DIR}/usr/bin/kata-agent" --version) + echo "New agent version: $installed_version" + + popd + installed_version=${installed_version##k*-} + [[ "${installed_version}" == *"${current_version}"* ]] +} + +build_image() { + TMP_DIR=$(mktemp -d -t kata-image-install.XXXXXXXXXX) + readonly ROOTFS_DIR="${TMP_DIR}/rootfs" + export ROOTFS_DIR + + image_type=$(get_version "assets.image.meta.image-type") + OSBUILDER_DISTRO="alpine" + + osbuilder_repo="github.com/kata-containers/osbuilder" + + # Clone os-builder repository + go get -d "${osbuilder_repo}" || true + + (cd "${GOPATH}/src/${osbuilder_repo}/rootfs-builder" && \ + sudo -E AGENT_INIT="${AGENT_INIT}" AGENT_VERSION="${agent_commit}" \ + GOPATH="$GOPATH" USE_DOCKER=true ./rootfs.sh "${OSBUILDER_DISTRO}") + + # Build the image + if [ x"${TEST_INITRD}" == x"yes" ]; then + pushd "${GOPATH}/src/${osbuilder_repo}/initrd-builder" + sudo -E AGENT_INIT="${AGENT_INIT}" USE_DOCKER=true ./initrd_builder.sh "$ROOTFS_DIR" + image_name="kata-containers-initrd.img" + else + pushd "${GOPATH}/src/${osbuilder_repo}/image-builder" + sudo -E AGENT_INIT="${AGENT_INIT}" USE_DOCKER=true ./image_builder.sh "$ROOTFS_DIR" + image_name="kata-containers.img" + fi + + # Install the image + commit=$(git log --format=%h -1 HEAD) + date=$(date +%Y-%m-%d-%T.%N%z) + image="kata-containers-${date}-osbuilder-${commit}-agent-${agent_commit}" + + sudo install -o root -g root -m 0640 -D ${image_name} "/usr/share/kata-containers/${image}" + (cd /usr/share/kata-containers && sudo ln -sf "$image" ${image_name}) + + popd +} + +main() { + install_yq || build_image +} + +main diff --git a/scripts/install_kata_kernel.sh b/scripts/install_kata_kernel.sh new file mode 100755 index 00000000..cbb234ad --- /dev/null +++ b/scripts/install_kata_kernel.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# +# Copyright (c) 2017-2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Currently we will use this repository until this issue is solved +# See https://github.com/kata-containers/packaging/issues/1 + +set -e + +cidir=$(dirname "$0") +source "${cidir}/lib.sh" +source "/etc/os-release" + +repo_name="packaging" +repo_owner="kata-containers" +kata_kernel_dir="/usr/share/kata-containers" +kernel_arch="$(arch)" +get_kernel_url="https://cdn.kernel.org/pub/linux/kernel" +tmp_dir="$(mktemp -d)" +hypervisor="kvm" +packaged_kernel="kata-linux-container" + +download_repo() { + pushd ${tmp_dir} + git clone --depth 1 https://github.com/${repo_owner}/${repo_name} + popd +} + +get_current_kernel_version() { + kernel_version=$(get_version "assets.kernel.version") + echo "${kernel_version/v/}" +} + +get_kata_config_version() { + pushd "${tmp_dir}/${repo_name}" >> /dev/null + kata_config_version=$(cat kernel/kata_config_version) + popd >> /dev/null + echo "${kata_config_version}" +} + +get_packaged_kernel_version() { + if [ "$ID" == "ubuntu" ]; then + kernel_version=$(sudo apt-cache madison $packaged_kernel | awk '{print $3}' | cut -d'-' -f1) + elif [ "$ID" == "fedora" ]; then + kernel_version=$(sudo dnf --showduplicate list ${packaged_kernel}.${kernel_arch} | awk '/'$packaged_kernel'/ {print $2}' | cut -d'-' -f1) + elif [ "$ID" == "centos" ]; then + kernel_version=$(sudo yum --showduplicate list $packaged_kernel | awk '/'$packaged_kernel'/ {print $2}' | cut -d'-' -f1) + fi + + if [ -z "$kernel_version" ]; then + die "unknown kernel version" + else + echo "${kernel_version}" + fi + +} + +# download the linux kernel, first argument is the kernel version +download_kernel() { + kernel_version=$1 + pushd $tmp_dir + kernel_tar_file="linux-${kernel_version}.tar.xz" + kernel_url="${get_kernel_url}/v$(echo $kernel_version | cut -f1 -d.).x/${kernel_tar_file}" + curl -LOk ${kernel_url} + tar -xf ${kernel_tar_file} + popd +} + +# build the linux kernel, first argument is the kernel version +build_and_install_kernel() { + kernel_version=$1 + pushd ${tmp_dir} + kernel_config_file=$(realpath ${repo_name}/kernel/configs/[${kernel_arch}]*_kata_${hypervisor}_* | tail -1) + kernel_patches=$(realpath ${repo_name}/kernel/patches/*) + kernel_src_dir="linux-${kernel_version}" + pushd ${kernel_src_dir} + cp ${kernel_config_file} .config + for p in ${kernel_patches}; do patch -p1 < $p; done + make -s ARCH=${kernel_arch} oldconfig > /dev/null + if [ $CI == "true" ]; then + make ARCH=${kernel_arch} -j$(nproc) + else + make ARCH=${kernel_arch} + fi + sudo mkdir -p ${kata_kernel_dir} + sudo cp -a "$(realpath arch/${kernel_arch}/boot/bzImage)" "${kata_kernel_dir}/vmlinuz.container" + sudo cp -a "$(realpath vmlinux)" "${kata_kernel_dir}/vmlinux.container" + popd + popd +} + +install_packaged_kernel(){ + if [ "$ID" == "ubuntu" ]; then + sudo apt install -y "$packaged_kernel" + elif [ "$ID" == "fedora" ]; then + sudo dnf install -y "$packaged_kernel" + elif [ "$ID" == "centos" ]; then + sudo yum install -y "$packaged_kernel" + else + die "Unrecognized distro" + fi +} + +cleanup() { + rm -rf "${tmp_dir}" +} + +main() { + download_repo + kernel_version="$(get_current_kernel_version)" + kata_config_version="$(get_kata_config_version)" + current_kernel_version="${kernel_version}.${kata_config_version}" + + download_kernel ${kernel_version} + build_and_install_kernel ${kernel_version} + cleanup +} + +main diff --git a/scripts/install_qemu.sh b/scripts/install_qemu.sh new file mode 100755 index 00000000..0827e4e4 --- /dev/null +++ b/scripts/install_qemu.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# +# Copyright (c) 2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +cidir=$(dirname "$0") +source "${cidir}/lib.sh" +source /etc/os-release + +CURRENT_QEMU_COMMIT=$(get_version "assets.hypervisor.qemu-lite.commit") +PACKAGED_QEMU="qemu-lite" +QEMU_ARCH=$(arch) + +get_packaged_qemu_commit() { + if [ "$ID" == "ubuntu" ]; then + qemu_commit=$(sudo apt-cache madison $PACKAGED_QEMU \ + | awk '{print $3}' | cut -d'-' -f1 | cut -d'.' -f4) + elif [ "$ID" == "fedora" ]; then + qemu_commit=$(sudo dnf --showduplicate list ${PACKAGED_QEMU}.${QEMU_ARCH} \ + | awk '/'$PACKAGED_QEMU'/ {print $2}' | cut -d'-' -f1 | cut -d'.' -f4) + elif [ "$ID" == "centos" ]; then + qemu_commit=$(sudo yum --showduplicate list $PACKAGED_QEMU \ + | awk '/'$PACKAGED_QEMU'/ {print $2}' | cut -d'-' -f1 | cut -d'.' -f4) + fi + + if [ -z "$qemu_commit" ]; then + die "unknown qemu commit" + else + echo "${qemu_commit}" + fi +} + +install_packaged_qemu() { + if [ "$ID" == "ubuntu" ]; then + sudo apt install -y "$PACKAGED_QEMU" + elif [ "$ID" == "fedora" ]; then + sudo dnf install -y "$PACKAGED_QEMU" + elif [ "$ID" == "centos" ]; then + sudo yum install -y "$PACKAGED_QEMU" + else + die "Unrecognized distro" + fi +} + +build_and_install_qemu() { + QEMU_REPO=$(get_version "assets.hypervisor.qemu-lite.url") + # Remove 'https://' from the repo url to be able to clone the repo using 'go get' + QEMU_REPO=${QEMU_REPO/https:\/\//} + PACKAGING_REPO="github.com/kata-containers/packaging" + QEMU_CONFIG_SCRIPT="${GOPATH}/src/${PACKAGING_REPO}/scripts/configure-hypervisor.sh" + + go get -d "${QEMU_REPO}" || true + go get -d "$PACKAGING_REPO" || true + + pushd "${GOPATH}/src/${QEMU_REPO}" + git fetch + git checkout "$CURRENT_QEMU_COMMIT" + [ -d "capstone" ] || git clone https://github.com/qemu/capstone.git capstone + [ -d "ui/keycodemapdb" ] || git clone https://github.com/qemu/keycodemapdb.git ui/keycodemapdb + + echo "Build Qemu" + "${QEMU_CONFIG_SCRIPT}" "qemu" | xargs ./configure + make -j $(nproc) + + echo "Install Qemu" + sudo -E make install + + # Add link from /usr/local/bin to /usr/bin + sudo ln -sf $(command -v qemu-system-${QEMU_ARCH}) "/usr/bin/qemu-lite-system-${QEMU_ARCH}" + popd +} + +main() { + packaged_qemu_commit=$(get_packaged_qemu_commit) + short_current_qemu_commit=${CURRENT_QEMU_COMMIT:0:10} + build_and_install_qemu +} + +main diff --git a/scripts/kata-env-setup.sh b/scripts/kata-env-setup.sh new file mode 100755 index 00000000..a2429c32 --- /dev/null +++ b/scripts/kata-env-setup.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +cidir=$(dirname "$0") + +source "${cidir}/lib.sh" + +clone_repo "github.com/kata-containers/runtime" + +${cidir}/install_kata_image.sh && ${cidir}/install_kata_kernel.sh && ${cidir}/install_qemu.sh diff --git a/scripts/lib.sh b/scripts/lib.sh new file mode 100755 index 00000000..a7012bc3 --- /dev/null +++ b/scripts/lib.sh @@ -0,0 +1,166 @@ +#!/bin/bash +# +# Copyright (c) 2017-2018 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -e + +export KATA_RUNTIME=${KATA_RUNTIME:-cc} + +# If we fail for any reason a message will be displayed +die(){ + msg="$*" + echo "ERROR: $msg" >&2 + exit 1 +} + +info() { + echo -e "INFO: $*" +} + +function clone_repo() { + github_project="$1" + project_dir="${GOPATH}/src/${github_project}" + + echo "Retrieve repository ${github_project}" + go get -d ${github_project} || true +} + +function clone_and_build() { + github_project="$1" + make_target="$2" + project_dir="${GOPATH}/src/${github_project}" + + echo "Retrieve repository ${github_project}" + go get -d ${github_project} || true + + # fixme: once tool to parse and get branches from github is + # completed, add it here to fetch branches under testing + + pushd ${project_dir} + + echo "Build ${github_project}" + if [ ! -f Makefile ]; then + echo "Run autogen.sh to generate Makefile" + bash -f autogen.sh + fi + + make + + popd +} + +function clone_build_and_install() { + clone_and_build $1 $2 + pushd "${GOPATH}/src/${1}" + echo "Install repository ${1}" + sudo -E PATH=$PATH KATA_RUNTIME=${KATA_RUNTIME} make install + popd +} + +function install_yq() { + GOPATH=${GOPATH:-${HOME}/go} + local yq_path="${GOPATH}/bin/yq" + local yq_pkg="github.com/mikefarah/yq" + [ -x "${GOPATH}/bin/yq" ] && return + + case "$(arch)" in + "aarch64") + goarch=arm64 + ;; + + "x86_64") + goarch=amd64 + ;; + "*") + echo "Arch $(arch) not supported" + exit + ;; + esac + + mkdir -p "${GOPATH}/bin" + + # Workaround to get latest release from github (to not use github token). + # Get the redirection to latest release on github. + yq_latest_url=$(curl -Ls -o /dev/null -w %{url_effective} "https://${yq_pkg}/releases/latest") + # The redirected url should include the latest release version + # https://github.com/mikefarah/yq/releases/tag/ + yq_version=$(basename "${yq_latest_url}") + + + local yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_linux_${goarch}" + curl -o "${yq_path}" -L ${yq_url} + chmod +x ${yq_path} +} + +function get_version(){ + dependency="$1" + GOPATH=${GOPATH:-${HOME}/go} + # This is needed in order to retrieve the version for qemu-lite + install_yq >&2 + runtime_repo="github.com/kata-containers/runtime" + runtime_repo_dir="$GOPATH/src/${runtime_repo}" + versions_file="${runtime_repo_dir}/versions.yaml" + mkdir -p "$(dirname ${runtime_repo_dir})" + [ -d "${runtime_repo_dir}" ] || git clone --quiet https://${runtime_repo}.git "${runtime_repo_dir}" + [ ! -f "$versions_file" ] && { echo >&2 "ERROR: cannot find $versions_file"; exit 1; } + result=$("${GOPATH}/bin/yq" read "$versions_file" "$dependency") + [ "$result" = "null" ] && result="" + echo "$result" +} + + +function apply_depends_on() { + pushd "${GOPATH}/src/${kata_repo}" + label_lines=$(git log --format=%s%b master.. | grep "Depends-on:" || true) + if [ "${label_lines}" == "" ]; then + popd + return 0 + fi + + nb_lines=$(echo "${label_lines}" | wc -l) + + repos_found=() + for i in $(seq 1 "${nb_lines}") + do + label_line=$(echo "${label_lines}" | sed "${i}q;d") + label_str=$(echo "${label_line}" | awk '{print $2}') + repo=$(echo "${label_str}" | cut -d'#' -f1) + if [[ "${repos_found[@]}" =~ "${repo}" ]]; then + echo "Repository $repo was already defined in a 'Depends-on:' tag." + echo "Only one repository per tag is allowed." + return 1 + fi + repos_found+=("$repo") + pr_id=$(echo "${label_str}" | cut -d'#' -f2) + + echo "This PR depends on repository: ${repo} and pull request: ${pr_id}" + if [ ! -d "${GOPATH}/src/${repo}" ]; then + go get -d "$repo" || true + fi + + pushd "${GOPATH}/src/${repo}" + echo "Fetching pull request: ${pr_id} for repository: ${repo}" + git fetch origin "pull/${pr_id}/head" && git checkout FETCH_HEAD && git rebase origin/master + popd + done + + popd +} + +function waitForProcess(){ + wait_time="$1" + sleep_time="$2" + cmd="$3" + while [ "$wait_time" -gt 0 ]; do + if eval "$cmd"; then + return 0 + else + sleep "$sleep_time" + wait_time=$((wait_time-sleep_time)) + fi + done + return 1 +}