qemu-arm: Add support for TravisCI

.circleci/config.yml

@@ -147,7 +147,9 @@ workflows:
     jobs:
      - linux-linux
      - linux-netbsd
-     - qemu-arm-linux
-     - qemu-arm-netbsd
+     # FIXME: Forked version of QEMU is needed
+     # - qemu-arm-linux
+     # TODO: Investigate netbsd support for qemu-arm
+     # - qemu-arm-netbsd
      - freebsd-linux
      - freebsd-netbsd

.travis.yml

@@ -1,8 +1,59 @@
-language: c
+language: nix
 
-compiler:
-  - gcc
-  - clang
+env:
+  global:
+    # XXX: Nix on Travis uses the master branch of the nixpkgs repo, meaning we
+    # need to pin nixpkgs to a specific version if we want to avoid frequent
+    # rebuilds of dependant packages.
+    #
+    # TODO: <nixpkgs> should ideally point to the latest stable release.
+    # However, we currently use the unstable release (@ 2019-03-03) since we
+    # depend on NixOS/nixpkgs#52146, which hasn't been merged into stable yet.
+    # It should be possible to use stable once 19.03 is out.
+    - NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/34aa254f9ebf5899636a9927ceefbc9df80230f4.tar.gz
+
+matrix:
+  include:
+    - name: "gcc-netbsd"
+      env: BUILD_TARGET=frankenlibc-gcc
+    - name: "clang-netbsd"
+      env: BUILD_TARGET=frankenlibc-clang
+    - name: "lkl-musl"
+      env: BUILD_TARGET=frankenlibc-lkl
+    - name: "raspi2-linux"
+      env: BUILD_TARGET=frankenlibc-arm
+
+git:
+  # XXX: Submodules have to be fetched manually since it frequently times out.
+  submodules: false
+
+cache:
+  timeout: 1000
+  directories:
+    - $HOME/nix.store
+    # XXX: Beware. Desperate attempt to cache the lkl repository.
+    - $TRAVIS_BUILD_DIR/.git/modules
+
+before_cache:
+  - mkdir -p $HOME/nix.store
+  - |
+    nix copy --to file://$HOME/nix.store -f default.nix \
+      "$BUILD_TARGET".buildInputs \
+      "$BUILD_TARGET".nativeBuildInputs \
+      "$BUILD_TARGET".depsBuildBuild
+
+before_install:
+  # XXX: Shallow clone of submodules is possible with newer version of git, but
+  # is tricky (https://stackoverflow.com/questions/2144406/how-to-make-shallow-git-submodules)
+  - travis_wait 30 git submodule update --init --recursive
+  - mkdir -p $HOME/.config/nix
+  - |
+    # XXX: We write the configuration line-by-line, since Travis doesn't seem
+    # to recognize heredoc delimiters for some reason.
+    echo "substituters = file://$HOME/nix.store https://cache.nixos.org/" \
+      >> ~/.config/nix/nix.conf
+    echo "require-sigs = false" \
+      >> ~/.config/nix/nix.conf
 
 script:
-  - ./build.sh
+  - nix-build -A "$BUILD_TARGET"

build.sh

@@ -535,6 +535,7 @@ ${INSTALL-install} ${RUMP}/bin/rexec ${BINDIR}
 rumpkernel_install_extra_libs
 
 ${INSTALL-install} ${RUMP}/lib/*.o ${OUTDIR}/lib
+[ -f ${RUMP}/lib/link.ld ] && ${INSTALL-install} ${RUMP}/lib/link.ld ${OUTDIR}/lib
 [ -f ${RUMP}/lib/libg.a ] && ${INSTALL-install} ${RUMP}/lib/libg.a ${OUTDIR}/lib
 ${INSTALL-install} ${RUMPOBJ}/explode/libc.a ${OUTDIR}/lib
 

default.nix

@@ -0,0 +1,39 @@
+{ pkgs ? import <nixpkgs> {} }:
+
+with pkgs;
+
+let
+  callPackage = lib.callPackageWith (pkgs // self);
+
+  keepDebugInfo = stdenv: stdenv // {
+    mkDerivation = args: stdenv.mkDerivation (args // {
+      dontStrip = true;
+    });
+  };
+
+  self = {
+    frankenlibc-gcc = callPackage ./nix/frankenlibc.nix {
+      stdenv = overrideCC stdenv gcc6;
+    };
+
+    frankenlibc-clang = callPackage ./nix/frankenlibc.nix {
+      inherit (llvmPackages_38) stdenv;
+    };
+
+    frankenlibc-lkl = callPackage ./nix/frankenlibc.nix {
+      rumpKernel = "linux";
+    };
+
+    frankenlibc-arm = callPackage ./nix/frankenlibc.nix {
+      stdenv = pkgsCross.arm-embedded.buildPackages.stdenv;
+      rumpKernel = "linux";
+    };
+
+    qemu-circle = callPackage ./nix/qemu-circle.nix { };
+
+    # XXX: We can't use enableDebugging from nixpkgs, since the linux kernel
+    # refuses to compile with -O0 flags.
+    enableDebugging = pkg: pkg.override { stdenv = keepDebugInfo pkg.stdenv; };
+  };
+
+in self

nix/frankenlibc.nix

@@ -0,0 +1,58 @@
+{ stdenv, lib, buildPackages, bc, e2fsprogs, nettools, python, which, zlib
+, gcc-arm-embedded, qemu-circle
+, rumpKernel ? "netbsd"
+, doCheck ? true }:
+
+assert rumpKernel == "netbsd" || rumpKernel == "linux";
+
+let
+  inherit (stdenv) buildPlatform hostPlatform targetPlatform;
+
+  rumpCC =
+    # XXX: We can't build with pkgsCross.arm-embedded.cc because newlib-nano
+    # is required instead of newlib. See NixOS/nixpkgs#51907 for details.
+    if targetPlatform.isAarch32 then gcc-arm-embedded
+    else assert hostPlatform == targetPlatform; stdenv.cc;
+
+in
+stdenv.mkDerivation {
+  name = "frankenlibc";
+
+  src = lib.cleanSource ../.;
+
+  depsBuildBuild = [ zlib ]
+    ++ lib.optional (buildPlatform != targetPlatform) buildPackages.stdenv.cc;
+  depsBuildTarget = [ rumpCC ];
+  nativeBuildInputs = [ bc e2fsprogs nettools python which ];
+  buildInputs = [ rumpCC ]
+    ++ lib.optional targetPlatform.isAarch32 qemu-circle;
+
+  preConfigure = lib.optionalString targetPlatform.isAarch32 ''
+    export CC="${gcc-arm-embedded}/bin/arm-none-eabi-gcc"
+    export NM="${gcc-arm-embedded}/bin/arm-none-eabi-nm"
+    export AR="${gcc-arm-embedded}/bin/arm-none-eabi-ar"
+    export OBJCOPY="${gcc-arm-embedded}/bin/arm-none-eabi-objcopy"
+  '';
+
+  buildFlags = [
+    "-q" # XXX: Travis terminates builds with excessive log ouput
+    "-k " rumpKernel
+  ] ++ lib.optional targetPlatform.isAarch32 "qemu-arm"
+    # XXX: We perform the check phase here because it is tricky to setup outside
+    # build.sh.
+    ++ lib.optional (!(doCheck && buildPlatform == hostPlatform)) "notest";
+
+  buildPhase = ''
+    mkdir -p $out
+    ./build.sh -d $out $buildFlags
+  '';
+
+  installPhase = ''
+    mkdir -p $out/libexec/tools
+    mv rumpobj/tests $out/libexec
+    mv $out/bin/rump.* $out/libexec/tools || true
+  '';
+
+  enableParallelBuilding = true;
+  inherit doCheck;
+}

nix/qemu-circle.nix

@@ -0,0 +1,41 @@
+{ stdenv, fetchurl, fetchFromGitHub, overrideCC, qemu, gcc6, SDL }:
+
+let
+  qemu-circle = qemu.override {
+    hostCpuTargets = [ "arm-softmmu" ];
+    pulseSupport = false;
+    sdlSupport = false; SDL2 = SDL;
+    gtkSupport = false;
+    vncSupport = false;
+    smartcardSupport = false;
+    spiceSupport = false;
+    xenSupport = false;
+    cephSupport = false;
+    openGLSupport = false;
+    virglSupport = false;
+    smbdSupport = false;
+  };
+
+  diffForCommit = commit: "https://github.com/qemu/qemu/commit/${commit}.diff";
+in
+  qemu-circle.overrideAttrs (old: rec {
+    name = "qemu-circle-${version}";
+    version = "2.4.1";
+    sha256 = "19936r6x7wal09zh0f1s5y32v4k4z5nmnfb2jf0padmff3808jl9";
+    src = fetchFromGitHub {
+      owner = "rsta2";
+      repo = "qemu";
+      rev = "7a24a5a051ea83529b057fd18c76ca40eb717392";
+      fetchSubmodules = true;
+      inherit sha256;
+    };
+
+    patches = builtins.tail old.patches ++ [
+      (fetchurl {
+        url = diffForCommit "75e5b70e6b5dcc4f2219992d7cffa462aa406af0";
+        sha256 = "0a9a09xy4iy6gnmhlbgj28pvzxwp7h165s3rvpc9n6b7lbm0dibj";
+      })
+    ];
+
+    stdenv = overrideCC stdenv gcc6;
+  })

platform/qemu-arm/Makefile

@@ -9,6 +9,7 @@ CRTFILES=$(CRTDIR)/startup.o $(CRTDIR)/crt0.o
 CRTFILES+=$(CRTDIR)/crti.o $(CRTDIR)/crtn.o
 CRTFILES+=$(CRTDIR)/crtbegin.o $(CRTDIR)/crtbeginT.o
 CRTFILES+=$(CRTDIR)/crtend.o
+CRTFILES+=$(CRTDIR)/link.ld
 OBJ=$(CRTFILES)
 LIBG=$(CRTDIR)/libg.a
 OBJ+=$(LIBG)
@@ -38,7 +39,7 @@ OBJ+=$(OBJDIR)/libcircle.o
 OBJ+=$(OBJDIR)/libcircleusb.o
 OBJ+=$(OBJDIR)/libcircleinput.o
 OBJ+=$(OBJDIR)/libcirclefs.o
-HEADERS=$(CRTDIR) $(OBJDIR)
+HEADERS=$(CRTDIR) $(OBJDIR) circle
 INCLUDES=-I../include -I../../franken/include -I.
 CPPFLAGS+=-nostdinc $(INCLUDES)
 CFLAGS+=-fno-stack-protector
@@ -73,6 +74,9 @@ $(CRTDIR)/crtbeginT.o:		$(CRTDIR)
 $(CRTDIR)/crtend.o:		$(CRTDIR)
 				printf "\n" | $(COMPILE.c) -o $@ -x c -
 
+$(CRTDIR)/link.ld:		link.ld $(CRTDIR)
+				$(INSTALL) -m644 $< $@
+
 $(LIBG):			$(CRTDIR)
 				printf "\n" | $(COMPILE.c) -o $(OBJDIR)/g.o -x c -
 				$(AR) cr $@ $(OBJDIR)/g.o
@@ -157,6 +161,10 @@ $(OBJDIR)/libcircleinput.o:	circle/lib/input/libcircleinput.o $(HEADERS)
 $(OBJDIR)/libcirclefs.o:	circle/lib/fs/libcirclefs.o $(HEADERS)
 				$(INSTALL) $< $@
 
+circle: circle/lib/startup.o circle/lib/libcircle.o
+circle: circle/lib/usb/libcircleusb.o circle/lib/input/libcircleinput.o
+circle: circle/lib/fs/libcirclefs.o
+
 circle/lib/startup.o:
 	$(MAKE) -C circle/lib $(notdir $@) $(CIRCLE_FLAGS)
 

platform/qemu-arm/platform.sh

@@ -14,7 +14,7 @@ appendvar EXTRA_AFLAGS "-march=armv7-a -marm -mfpu=neon-vfpv4 -D__SOFTFP__"
 # this compiler is very fussy, planning to fix these issues at some point
 appendvar EXTRA_CWARNFLAGS "-Wno-error"
 
-LINKSCRIPT="${PWD}/platform/qemu-arm/link.ld"
+LINKSCRIPT="${OUTDIR}/lib/link.ld"
 EXTRA_LDSCRIPT="-T ${LINKSCRIPT}"
 EXTRA_LDSCRIPT_CC="-Wl,-T,${LINKSCRIPT}"
 

tools/qemu-arm.sh

@@ -1,20 +1,31 @@
 #!/bin/bash
 
-LKL_TAP="${LKL_TAP:-tap0}"
+LKL_TAP="${LKL_TAP:-none}"
 LKL_DEBUG="${LKL_DEBUG:-1}"
 LKL_QEMU_CMDLINE=
 BIN_PATH="$1"
 
 main() {
-  sudo env PATH="$PATH" \
+  if [[ "$LKL_TAP" != "none" ]]; then
+    sudo env PATH="$PATH" \
+      qemu-system-arm \
+        $LKL_QEMU_ARGS \
+        -M raspi2 \
+        -nographic \
+        -device usb-net,vlan=0 \
+        -net tap,vlan=0,ifname="$LKL_TAP",script=no,downscript=no \
+        -append "$(build_qemu_cmdline "$@")" \
+        -semihosting -kernel "$BIN_PATH"
+  else
     qemu-system-arm \
       $LKL_QEMU_ARGS \
       -M raspi2 \
       -nographic \
       -device usb-net,vlan=0 \
-      -net tap,vlan=0,ifname="$LKL_TAP",script=no,downscript=no \
+      -net user,vlan=0 \
       -append "$(build_qemu_cmdline "$@")" \
       -semihosting -kernel "$BIN_PATH"
+  fi
 }
 
 build_qemu_cmdline() {