diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1377554 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.swp diff --git a/README.md b/README.md index c1ed1cc..cb46a63 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# Minimal boot linux coreutils bash glibc +# Minimal build & boot linux kernel + coreutils (via busybox), openssh & iputils ![qemu showing boot into minimal linux](./img/boot-qemu-example.png) -Goal is to have mimimal but realistic node with network capabilities build in (`ip`) and bootstrapped (similar to [alpine's netbot](https://boot.alpinelinux.org/) ([example](https://github.com/KarmaComputing/server-bootstrap/blob/494089caa2c88bbf37a739aa96561231d5847be5/.github/workflows/build-alpine-netboot-image-zfs.yml#L1))). [Clearlinux](https://github.com/clearlinux/distribution) is interesting. This is using glibc over musl (which alpine uses). +Goal is to have mimimal but realistic node with network capabilities build in (`ip`) and bootstrapped (similar to [alpine's netbot](https://boot.alpinelinux.org/) ([example](https://github.com/KarmaComputing/server-bootstrap/blob/494089caa2c88bbf37a739aa96561231d5847be5/.github/workflows/build-alpine-netboot-image-zfs.yml#L1))). [Clearlinux](https://github.com/clearlinux/distribution) is interesting. This is using musl (which alpine uses) rather than glibc to support staticaly built binaries more easily. ## The process in a nutshell @@ -12,6 +12,9 @@ Goal is to have mimimal but realistic node with network capabilities build in (` - Write an `init` script - Build all the binaries + `init` script into a initramfs - Run/boot with qemu +- Rememebr busybox needs to be static build (see .config) + - dropbear needs at least a `dropbear_rsa_host_key` key config or will not start see [gist](https://gist.github.com/mad4j/7983719) + - Prefering openssh for end user compatability (statically built) @@ -19,31 +22,16 @@ TODO: add [iproute2](https://github.com/iproute2/iproute2) for minimal routing. ## Things you need to build -- git clone & build GNU `bash` is in it's own repo -- git clone & build coreutils (`ls` , `date` , `touch` etc(`mount` is not in here- who knew) -- To get `mount` you need to build util-linux https://en.wikipedia.org/wiki/Util-linux -- is `ls` failing, did you forget to include `/usr/lib/x86_64-linux-gnu/libcap.so.2`? See `mkchroot.sh` helper from https://landley.net/writing/rootfs-programming.html - -- Don't forget to `mount` linux virtual filesystems (oh wait, did you forget to build util-linux into your init? - -Example built rootfs: -``` -ls -lh rootfs.cpio.gz --rw-rw-r-- 1 chris chris 16M Dec 8 23:40 rootfs.cpio.gz -``` +See [./build-all.sh](./build-all.sh) # What does this repo not include (yet) -- Full clone/compile instructions - Automated ci -### How do I build statically coreutils, do I even need to? +### How do I build statically coreutils, do I even need to? -See https://lists.gnu.org/archive/html/coreutils/2019-04/msg00001.html +See https://lists.gnu.org/archive/html/coreutils/2019-04/msg00001.html switched to using `musl`. -## `ls` , `cat` and `date` etc won't run without libc! - -git clone https://sourceware.org/git/glibc.git # Reading See also @@ -52,3 +40,17 @@ https://wiki.gentoo.org/wiki/Custom_Initramfs https://unix.stackexchange.com/a/305406 https://landley.net/writing/rootfs-howto.html https://landley.net/writing/rootfs-programming.html +- https://unix.stackexchange.com/questions/193066/how-to-unlock-account-for-public-key-ssh-authorization-but-not-for-password-aut +- https://stackoverflow.com/a/79151188 +- https://z49x2vmq.github.io/2020/12/24/linux-tiny-qemu/ + +> "Stuff like this is slowly becoming a lost art" [src](https://www.linuxquestions.org/questions/linux-general-1/bin-bash-as-primary-init-4175543547/#post5367386) ooopse. + + +TODO: kernel inital ram disk support https://stackoverflow.com/questions/14430551/qemu-boot-error-swapper-used-greatest-stack-depth +TODO READ: https://bbs.archlinux.org/viewtopic.php?pid=1378903#p1378903 + +## Notes + +> "busybox qemu /bin/sh: can't access tty; job control turned off" +> https://github.com/brgl/busybox/blob/master/shell/cttyhack.c diff --git a/build-all.sh b/build-all.sh index ce363da..5f947a7 100755 --- a/build-all.sh +++ b/build-all.sh @@ -2,5 +2,42 @@ set -exu +# We will +# +# - Empty ./build-dir (to build from scratch) +# - Build linux kernel +# - Build busybox +# - Build musl +# - Build openssh statically (using musl) +# - iproute2 is built into busybox + +SCRIPT_START_DIR=$PWD +BUILD_DIR=$(realpath -s ./build-dir) +rm -rf "$BUILD_DIR" +mkdir -p "$BUILD_DIR"/build-artifacts +BUILD_ARTIFACTS_DIR="$BUILD_DIR"/build-artifacts + +cd "$BUILD_DIR" + +# Build linux kernel +BUILD_ARTIFACTS_DIR=$BUILD_ARTIFACTS_DIR ../build-kernel.sh + +cd "$BUILD_DIR" +# Build busybox +BUILD_ARTIFACTS_DIR=$BUILD_ARTIFACTS_DIR ../build-busybox.sh + +# Build musl +../build-musl.sh +cd "$BUILD_DIR" + +# Build openssh +BUILD_ARTIFACTS_DIR=$BUILD_ARTIFACTS_DIR ../build-openssh-statically.sh +cd "$BUILD_DIR" + +cd "$SCRIPT_START_DIR" + ./create-init.sh ./build-rootfs.sh + +echo If all is wel, you\'re now good to run the vm in qemu +echo see ./run-qemu.sh diff --git a/build-busybox.sh b/build-busybox.sh new file mode 100755 index 0000000..24e419b --- /dev/null +++ b/build-busybox.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -eux + +INITAL_WORKING_DIR=$PWD + +git clone git://git.busybox.net/busybox + +cd busybox +git checkout 1_37_0 +make defconfig +echo CONFIG_STATIC=y >> .config +sed -i 's/CONFIG_TC=y/# CONFIG_TC is not set/g' .config +make -j$(nproc) + +mkdir "$BUILD_ARTIFACTS_DIR"/busybox +cp ./busybox "$BUILD_ARTIFACTS_DIR"/busybox + +cd $INITAL_WORKING_DIR diff --git a/build-coreutils.sh b/build-coreutils.sh deleted file mode 100755 index 705483f..0000000 --- a/build-coreutils.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -x - -INITAL_WORKING_DIR=$PWD - -cd coreutils -git checkout v9.5 -./bootstrap -./configure LDFLAGS="-static" -make -find src/ -executable -maxdepth 1 - - -cd $INITAL_WORKING_DIR diff --git a/build-kernel.sh b/build-kernel.sh new file mode 100755 index 0000000..8fec055 --- /dev/null +++ b/build-kernel.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -eux + +INITAL_WORKING_DIR=$PWD + +mkdir linux-kernel +cd linux-kernel +wget https://www.kernel.org/pub/linux/kernel/v6.x/linux-6.9.tar.xz + +tar xf linux-6.9.tar.xz +cd linux-6.9 +mkdir -p $BUILD_ARTIFACTS_DIR/linux-kernel +make defconfig +make -j$(nproc) +cp ./arch/x86_64/boot/bzImage $BUILD_ARTIFACTS_DIR/linux-kernel + + +cd $INITAL_WORKING_DIR diff --git a/build-libc.sh b/build-libc.sh deleted file mode 100755 index 3c38af2..0000000 --- a/build-libc.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -set -xu - -SCRATCH_SPACE_DIR=$1 - -INITAL_WORKING_DIR=$PWD - -git clone https://sourceware.org/git/glibc.git -cd glibc -git checkout release/2.40/master -mkdir glibc-build -cd glibc-build -mkdir out -../configure --prefix=$(pwd)/out -make -make install - -# Copy built libc objects/files over into scratch space -cp -r -n ./out/* $SCRATCH_SPACE_DIR - -cd $INITAL_WORKING_DIR diff --git a/build-musl.sh b/build-musl.sh new file mode 100755 index 0000000..1e3eccf --- /dev/null +++ b/build-musl.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Via https://wiki.musl-libc.org/getting-started.html + +set -xu + +INITAL_WORKING_DIR=$PWD + +git clone git://git.musl-libc.org/musl +cd musl/ +git checkout v1.2.5 +./configure --prefix=$HOME/musl --exec-prefix=$HOME/bin --syslibdir=$HOME/musl/lib --disable-shared + +make +make install + + +cd $INITAL_WORKING_DIR diff --git a/build-openssh-statically.sh b/build-openssh-statically.sh new file mode 100755 index 0000000..ccc7b94 --- /dev/null +++ b/build-openssh-statically.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# 1. clone openssh-portable +git clone https://github.com/openssh/openssh-portable +cd openssh-portable +git checkout V_9_9_P1 +autoconf + +# Note this uses only the **experimental** internal (reduced) cryp algos +# built-into openssh. TODO actuall include libcrypto +CC="musl-gcc -static" ./configure --prefix=/usr/bin --sysconfdir=/etc/ssh --without-zlib --without-openssl +make -j$(nproc) + +# Copy over openssh binaries to build artifacts dir +mkdir -p $BUILD_ARTIFACTS_DIR/openssh + +echo $PWD + +for sshUtility in $(find ./ -maxdepth 1 -type f -executable | grep -E -v '(\.sh|\.in|\.rc|\.sub|\.sample|\.status|\.guess|configure|fixpaths|install-sh|mkinstalldirs|fixalgorithms)'); do + echo Copying over "$sshUtility" + cp "$sshUtility" $BUILD_ARTIFACTS_DIR/openssh + done + +cp $(find ./ -name sshd_config) $BUILD_ARTIFACTS_DIR/openssh + diff --git a/create-init.sh b/create-init.sh index d09067b..8135a0a 100755 --- a/create-init.sh +++ b/create-init.sh @@ -1,8 +1,38 @@ -#!/bin/bash +#!/bin/sh set -eux -echo "#!/usr/bin/bash" > scratch-space/init -echo "exec /usr/bin/bash" >> scratch-space/init -chmod +x scratch-space/init +INIT_FILE_PATH=scratch-space/init + +echo "#!/bin/busybox sh" > $INIT_FILE_PATH +echo 'echo YOLOOooooooooooooooooooooooooooo' >> $INIT_FILE_PATH +echo 'echo YOLOOooooooooooooooooooooooooooo' >> $INIT_FILE_PATH +echo 'echo YOLOOooooooooooooooooooooooooooo' >> $INIT_FILE_PATH +echo 'echo YOLOOooooooooooooooooooooooooooo' >> $INIT_FILE_PATH +echo 'echo YOLOOooooooooooooooooooooooooooo' >> $INIT_FILE_PATH +echo 'mount -t sysfs sysfs /sys' >> $INIT_FILE_PATH +echo 'mount -t proc proc /proc' >> $INIT_FILE_PATH +# (sshd needs openpty: No such file or directory ) +echo 'mount -t devtmpfs udev /dev' >> $INIT_FILE_PATH +echo 'mkdir /dev/pts' >> $INIT_FILE_PATH +echo 'mount -t devpts devpts /dev/pts' >> $INIT_FILE_PATH +echo 'sysctl -w kernel.printk="2 4 1 7"' >> $INIT_FILE_PATH +echo 'chown -R root:root /var/empty' >> $INIT_FILE_PATH +echo 'chmod -R 400 /var/empty' >> $INIT_FILE_PATH + +echo 'echo Bringing up loopback interface' >> $INIT_FILE_PATH +echo 'ip link set lo up' >> $INIT_FILE_PATH +echo 'ip addr show lo' >> $INIT_FILE_PATH + +echo 'echo Generating ssh host keys' >> $INIT_FILE_PATH +echo 'ssh-keygen -A' >> $INIT_FILE_PATH +echo 'ls -l /etc/ssh' >> $INIT_FILE_PATH + +echo 'echo Starting sshd' >> $INIT_FILE_PATH +echo '/usr/bin/sshd -E ssh_log' >> $INIT_FILE_PATH + +# Curious? See https://github.com/brgl/busybox/blob/master/shell/cttyhack.c +echo 'setsid cttyhack /bin/sh' >> $INIT_FILE_PATH + +chmod +x $INIT_FILE_PATH diff --git a/create-scratch-space.sh b/create-scratch-space.sh new file mode 100755 index 0000000..8abcda8 --- /dev/null +++ b/create-scratch-space.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +set -eux +INITAL_WORKING_DIR=$PWD +SCRATCH_DIR=./scratch-space +BUILD_ARTIFACTS_FOLDER=../build-dir/build-artifacts + +rm -rf "$SCRATCH_DIR" +mkdir "$SCRATCH_DIR" + +cd $SCRATCH_DIR + + +mkdir bin dev proc sys etc root usr var +mkdir -p usr/bin/libexec # (sshd-session by (default?) compiles into /usr/bin/libexec +mkdir -p etc/ssh +mkdir -p var/run # (otherwise sshd cannot write its pid file) + +# Crate users/groups + +echo 'root:x:0:' > ./etc/group + +# Copy over busybox +cp "$BUILD_ARTIFACTS_FOLDER"/busybox/busybox ./bin +cd ./bin +for utility in $(./busybox --list); do + ln -s ./busybox ./$utility +done +cd - + +# ssh/sshd etc bootstraping + +# Copy over default sshd_config config +cp "$BUILD_ARTIFACTS_FOLDER"/openssh/sshd_config ./etc/ssh/sshd_config + +for sshUtility in $(find "$BUILD_ARTIFACTS_FOLDER"/openssh -maxdepth 1 -type f -executable | grep -E -v '(\.sh|\.in|\.rc|\.sub|\.sample|\.status|\.guess|configure|fixpaths|install-sh|mkinstalldirs|fixalgorithms)'); do + + echo Copying over "$sshUtility" + cp "$sshUtility" ./usr/bin +done + mv ./usr/bin/sshd-session ./usr/bin/libexec + +# Bootstrap ssh users/config setup + +cd - && cd ../ +echo $PWD + +# Layout minimal user accounts +echo 'root:x:0:0:root:/root:/bin/sh' > ./etc/passwd +# Without sshd user, you get 'Privilege separation user sshd does not exist' +echo 'sshd:x:128:65534::/run/sshd:/usr/sbin/nologin' >> ./etc/passwd + +echo 'root:*:19216:0:99999:7:::' > ./etc/shadow + +echo 'echo 'root:x:0:' > ./etc/groups' +mkdir var/empty # TODO Missing privilege separation directory: /var/empty (sshd wants it) +# NOTE ownership of /var/empty is altered during init + + +# TODO generate host keys (ssh-keygen -A) + +cd $INITAL_WORKING_DIR + +./create-init.sh diff --git a/mkchroot.sh b/mkchroot.sh deleted file mode 100644 index f97fe53..0000000 --- a/mkchroot.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh - -# Credit: -# https://landley.net/writing/rootfs-programming.html - -function mkchroot -{ - [ $# -lt 2 ] && return - - dest=$1 - shift - for i in "$@" - do - # Get an absolute path for the file - [ "${i:0:1}" == "/" ] || i=$(which $i) - # Skip files that already exist at target. - [ -f "$dest/$i" ] && continue - if [ -e "$i" ] - then - # Create destination path - d=`echo "$i" | grep -o '.*/'` && - mkdir -p "$dest/$d" && - # Copy file - cat "$i" > "$dest/$i" && - chmod +x "$dest/$i" - else - echo "Not found: $i" - fi - # Recursively copy shared libraries' shared libraries. - mkchroot "$dest" $(ldd "$i" | egrep -o '/.* ') - done -} - -mkchroot "$@" diff --git a/run-qemu.sh b/run-qemu.sh index 4e9517b..2c922b9 100755 --- a/run-qemu.sh +++ b/run-qemu.sh @@ -2,4 +2,4 @@ set -eux -qemu-system-x86_64 -kernel vmlinuz-lts -initrd rootfs.cpio.gz --append "init=/usr/bin/init" +qemu-system-x86_64 -kernel ./build-dir/linux-kernel/linux-6.9/arch/x86/boot/bzImage -initrd rootfs.cpio.gz --append "console=ttyS0 init=/init" -nographic # -icount 10,align=on