Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable NixOS cross-compiling from aarch64-linux to x86_64-linux and vice versa #363935

Closed
wants to merge 44 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
fd8c319
maltego: 4.8.0 -> 4.8.1
r-ryantm Dec 5, 2024
7889853
python3Packages.dirsearch: init at 0.4.3
quantenzitrone Oct 22, 2024
d1b8c0e
Enable full qemu emulation with TCG fallback when cross compiling ima…
prinzdezibel Dec 4, 2024
391d3af
Undo python fixes and move them to separate branch.
prinzdezibel Dec 5, 2024
cc00e08
--amend
prinzdezibel Dec 6, 2024
b86a278
Enable dot file globbing
prinzdezibel Dec 8, 2024
36f25b0
Merge remote-tracking branch 'origin'
prinzdezibel Dec 8, 2024
971d7ef
Enable cross compiling of network manager plugin
prinzdezibel Dec 9, 2024
a020629
Merge branch 'NixOS:master' into master
prinzdezibel Dec 9, 2024
290667b
Merge branch 'NixOS:master' into master
prinzdezibel Dec 9, 2024
44e5d71
Revert "Undo python fixes and move them to separate branch."
prinzdezibel Dec 9, 2024
61abf83
Merge branch 'NixOS:master' into master
prinzdezibel Dec 10, 2024
a448965
Merge branch 'NixOS:master' into master
prinzdezibel Dec 10, 2024
7f38563
Merge branch 'NixOS:master' into master
prinzdezibel Dec 10, 2024
2187ab6
Linting: Remove trailing whitespace.
prinzdezibel Dec 10, 2024
a503b27
Merge branch 'NixOS:master' into master
prinzdezibel Dec 10, 2024
687a068
Merge branch 'NixOS:master' into master
prinzdezibel Dec 10, 2024
56fe166
Remove wrong comment
prinzdezibel Dec 10, 2024
06796c0
Merge branch 'NixOS:master' into master
prinzdezibel Dec 10, 2024
60e6d80
Merge branch 'NixOS:master' into master
prinzdezibel Dec 10, 2024
a5d2112
Merge branch 'NixOS:master' into master
prinzdezibel Dec 10, 2024
652cd86
Merge branch 'NixOS:master' into master
prinzdezibel Dec 11, 2024
76749a8
Merge branch 'NixOS:master' into master
prinzdezibel Dec 11, 2024
0c69a62
Merge branch 'NixOS:master' into master
prinzdezibel Dec 11, 2024
3c44bfd
Update pkgs/by-name/ma/maturin/package.nix
prinzdezibel Dec 11, 2024
6df8f8f
Update nixos/lib/make-disk-image.nix
prinzdezibel Dec 11, 2024
b4e4ffa
Update pkgs/build-support/rust/hooks/default.nix
prinzdezibel Dec 12, 2024
c34cffc
formatting
prinzdezibel Dec 12, 2024
509d5a4
Enable mounting of loop devices in VM
prinzdezibel Dec 27, 2024
4d7484f
Ensure that file allocation tables always use 32 bit
prinzdezibel Dec 27, 2024
98a7953
Merge branch 'crosscompile_make-disk-image' into staging
prinzdezibel Jan 2, 2025
c8c8a28
use virtiofsd from buildpkgs, not hostpkgs.
Mic92 Dec 5, 2024
772864e
Merge remote-tracking branch 'upstream/staging' into staging
prinzdezibel Jan 2, 2025
4cd09f7
remove python include
prinzdezibel Jan 2, 2025
dc8aee4
Linting
prinzdezibel Jan 2, 2025
61af462
Merge branch 'NixOS:staging' into staging
prinzdezibel Jan 2, 2025
8fb8415
nixfmt
prinzdezibel Jan 2, 2025
d8d2cf2
Linting
prinzdezibel Jan 2, 2025
a0575df
nixfmt
prinzdezibel Jan 2, 2025
ff2dc78
Merge branch 'staging'
prinzdezibel Jan 2, 2025
7ec0c1e
fix merge conflict
prinzdezibel Jan 3, 2025
99702ad
Merge remote-tracking branch 'upstream/staging' into staging
prinzdezibel Jan 3, 2025
21a1070
downgrade gpg bc of build error
prinzdezibel Jan 3, 2025
469acb0
Merge branch 'staging'
prinzdezibel Jan 3, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 26 additions & 13 deletions nixos/lib/make-disk-image.nix
Original file line number Diff line number Diff line change
Expand Up @@ -383,9 +383,10 @@ let
nixos-enter
nix
systemdMinimal
coreutils
findutils
]
++ lib.optional deterministic gptfdisk
++ stdenv.initialPath
);

# I'm preserving the line below because I'm going to search for it across nixpkgs to consolidate
Expand Down Expand Up @@ -602,6 +603,7 @@ let
}

echo "copying staging root to image..."
shopt -s dotglob
cptofs -p ${lib.optionalString (partitionTableType != "none") "-P ${rootPartition}"} \
-t ${fsType} \
-i $diskImage \
Expand Down Expand Up @@ -634,8 +636,18 @@ let
echo "file ${format}-image $out/${filename}" >> $out/nix-support/hydra-build-products
'';

buildSystem = pkgs.stdenv.buildPlatform.system;
hostSystem = pkgs.stdenv.hostPlatform.system;
crossPkgs =
if (hostSystem != buildSystem) then
import ../.. # nixpkgs
{ system = hostSystem; }
else
pkgs;

buildImage = pkgs.vmTools.runInLinuxVM (
pkgs.runCommand name

crossPkgs.runCommand name
{
preVM = prepareImage + lib.optionalString touchEFIVars createEFIVars;
buildInputs = with pkgs; [
Expand All @@ -661,6 +673,7 @@ let
''
export PATH=${binPath}:$PATH


rootDisk=${if partitionTableType != "none" then "/dev/vda${rootPartition}" else "/dev/vda"}

# It is necessary to set root filesystem unique identifier in advance, otherwise
Expand All @@ -680,26 +693,26 @@ let
# Create the ESP and mount it. Unlike e2fsprogs, mkfs.vfat doesn't support an
# '-E offset=X' option, so we can't do this outside the VM.
${lib.optionalString (partitionTableType == "efi" || partitionTableType == "hybrid") ''
mkdir -p /mnt/boot
mkfs.vfat -n ESP /dev/vda1
mount /dev/vda1 /mnt/boot
mkdir -p $mountPoint/boot
mkfs.vfat -F 32 -n ESP /dev/vda1
mount /dev/vda1 $mountPoint/boot

${lib.optionalString touchEFIVars "mount -t efivarfs efivarfs /sys/firmware/efi/efivars"}
''}
${lib.optionalString (partitionTableType == "efixbootldr") ''
mkdir -p /mnt/{boot,efi}
mkfs.vfat -n ESP /dev/vda1
mkfs.vfat -n BOOT /dev/vda2
mount /dev/vda1 /mnt/efi
mount /dev/vda2 /mnt/boot
mkdir -p $mountPoint/{boot,efi}
mkfs.vfat -F 32 -n ESP /dev/vda1
mkfs.vfat -F 32 -n BOOT /dev/vda2
mount /dev/vda1 $mountPoint/efi
mount /dev/vda2 $mountPoint/boot

${lib.optionalString touchEFIVars "mount -t efivarfs efivarfs /sys/firmware/efi/efivars"}
''}

# Install a configuration.nix
mkdir -p /mnt/etc/nixos
mkdir -p $mountPoint/etc/nixos
${lib.optionalString (configFile != null) ''
cp ${configFile} /mnt/etc/nixos/configuration.nix
cp ${configFile} $mountPoint/etc/nixos/configuration.nix
''}

${lib.optionalString installBootLoader ''
Expand Down Expand Up @@ -741,7 +754,7 @@ let
fi
done

umount -R /mnt
umount -R $mountPoint

# Make sure resize2fs works. Note that resize2fs has stricter criteria for resizing than a normal
# mount, so the `-c 0` and `-i 0` don't affect it. Setting it to `now` doesn't produce deterministic
Expand Down
8 changes: 7 additions & 1 deletion pkgs/build-support/rust/hooks/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,17 @@
, rust
, rustc
, stdenv
, python3

# This confusingly-named parameter indicates the *subdirectory of
# `target/` from which to copy the build artifacts. It is derived
# from a stdenv platform (or a JSON file).
, target ? stdenv.hostPlatform.rust.cargoShortTarget
}:

let
inherit (python3) pythonOnBuildForHost;
pythonInterpreter = pythonOnBuildForHost.interpreter;
in
{
cargoBuildHook = callPackage ({ }:
makeSetupHook {
Expand Down Expand Up @@ -76,6 +80,7 @@
};
} ./cargo-setup-hook.sh) {};


maturinBuildHook = callPackage ({ pkgsHostTarget }:
makeSetupHook {
name = "maturin-build-hook.sh";
Expand All @@ -86,6 +91,7 @@
];
substitutions = {
inherit (rust.envVars) rustTargetPlatformSpec setEnv;
pythonInterpreter = lib.versions.majorMinor pythonOnBuildForHost.version;
};
} ./maturin-build-hook.sh) {};

Expand Down
1 change: 1 addition & 0 deletions pkgs/build-support/rust/hooks/maturin-build-hook.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ maturinBuildHook() {

local flagsArray=(
"--jobs=$NIX_BUILD_CORES"
"--interpreter=@pythonInterpreter@"
"--offline"
"--target" "@rustTargetPlatformSpec@"
"--manylinux" "off"
Expand Down
36 changes: 29 additions & 7 deletions pkgs/build-support/vm/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
, img ? pkgs.stdenv.hostPlatform.linux-kernel.target
, storeDir ? builtins.storeDir
, rootModules ?
[ "virtio_pci" "virtio_mmio" "virtio_blk" "virtio_balloon" "virtio_rng" "ext4" "unix" "virtiofs" "crc32c_generic" ]
[ "virtio_pci" "virtio_mmio" "virtio_blk" "virtio_balloon" "virtio_rng" "ext4" "unix" "virtiofs" "crc32c_generic" "iso9660" "loop"]
}:

let
Expand All @@ -16,7 +16,7 @@ in
rec {
qemu-common = import ../../../nixos/lib/qemu-common.nix { inherit lib pkgs; };

qemu = buildPackages.qemu_kvm;
qemu = buildPackages.qemu;

modulesClosure = pkgs.makeModulesClosure {
inherit kernel rootModules;
Expand Down Expand Up @@ -161,6 +161,27 @@ rec {
];
};

# Switch standard build environment to target host platform
stage2Stdenv =
let
buildSystem = pkgs.stdenv.buildPlatform.system;
hostSystem = pkgs.stdenv.hostPlatform.system;
in if (buildSystem == hostSystem) then stdenv else
let
targetArch = if ( hostSystem == "x86_64-linux") then
"gnu64" else
"aarch64-multiplatform";
in
pkgs.stdenv.override {
buildPlatform = pkgs.stdenv.hostPlatform;
cc = null;
preHook = "";
allowedRequisites = null;
initialPath = [pkgs.pkgsCross.${targetArch}.busybox];
shell = "${pkgs.pkgsCross.${targetArch}.bash}/bin/bash";
extraNativeBuildInputs = [];
}
;

stage2Init = writeScript "vm-run-stage2" ''
#! ${bash}/bin/sh
Expand All @@ -178,7 +199,8 @@ rec {
export PATH=/empty
cd "$NIX_BUILD_TOP"

source $stdenv/setup
source ${stage2Stdenv}/setup
export stdenv=${stage2Stdenv}

if ! test -e /bin/sh; then
${coreutils}/bin/mkdir -p /bin
Expand Down Expand Up @@ -217,7 +239,6 @@ rec {
fi
'';


qemuCommandLinux = ''
${if (customQemu != null) then customQemu else (qemu-common.qemuBinary qemu)} \
-nographic -no-reboot \
Expand All @@ -233,7 +254,6 @@ rec {
$QEMU_OPTS
'';


vmRunCommand = qemuCommand: writeText "vm-run" ''
${coreutils}/bin/mkdir xchg
export > xchg/saved-env
Expand All @@ -260,9 +280,11 @@ rec {
# the -K option to preserve the temporary build directory).
cat > ./run-vm <<EOF
#! ${bash}/bin/sh

''${diskImage:+diskImage=$diskImage}
${pkgs.virtiofsd}/bin/virtiofsd --xattr --socket-path virtio-store.sock --sandbox none --shared-dir "${storeDir}" &
${pkgs.virtiofsd}/bin/virtiofsd --xattr --socket-path virtio-xchg.sock --sandbox none --shared-dir xchg &
${buildPackages.virtiofsd}/bin/virtiofsd --xattr --socket-path virtio-store.sock --sandbox none --shared-dir "${storeDir}" &
${buildPackages.virtiofsd}/bin/virtiofsd --xattr --socket-path virtio-xchg.sock --sandbox none --shared-dir xchg &

${qemuCommand}
EOF

Expand Down
10 changes: 6 additions & 4 deletions pkgs/by-name/ma/maturin/package.nix
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,12 @@ rustPlatform.buildRustPackage rec {

cargoHash = "sha256-xC0zCDGE0AynMUc5qSkgOYB3JqOWDlc8oz4yi6cGCJs=";

buildInputs = lib.optionals stdenv.hostPlatform.isDarwin [
darwin.apple_sdk.frameworks.Security
libiconv
];
buildInputs =
[ python3 ]
++ lib.optionals stdenv.hostPlatform.isDarwin [
darwin.apple_sdk.frameworks.Security
libiconv
];

# Requires network access, fails in sandbox.
doCheck = false;
Expand Down
3 changes: 3 additions & 0 deletions pkgs/by-name/ne/networkmanager-iodine/package.nix
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ stdenv.mkDerivation {
libnma
];

# configure.ac:53: warning: macro 'AM_GLIB_GNU_GETTEXT' not found in library
strictDeps = false;

configureFlags = [
"--with-gnome=${if withGnome then "yes" else "no"}"
"--localstatedir=/" # needed for the management socket under /run/NetworkManager
Expand Down
3 changes: 2 additions & 1 deletion pkgs/by-name/op/openfortivpn/package.nix
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
, openssl
, ppp
, systemd
, glib
, withSystemd ? lib.meta.availableOn stdenv.hostPlatform systemd
, withPpp ? stdenv.hostPlatform.isLinux
}:
Expand All @@ -27,7 +28,7 @@ stdenv.mkDerivation rec {
--replace '$(DESTDIR)$(confdir)' /tmp
'';

nativeBuildInputs = [ autoreconfHook pkg-config ];
nativeBuildInputs = [ autoreconfHook pkg-config glib ];

buildInputs = [
openssl
Expand Down
1 change: 1 addition & 0 deletions pkgs/development/python-modules/rpds-py/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ buildPythonPackage rec {
rustPlatform.maturinBuildHook
cargo
rustc
#python3
];

buildInputs = lib.optionals stdenv.hostPlatform.isDarwin [ libiconv ];
Expand Down
4 changes: 2 additions & 2 deletions pkgs/tools/security/gnupg/24.nix
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,11 @@ assert guiSupport -> !enableMinimal;

stdenv.mkDerivation rec {
pname = "gnupg";
version = "2.4.7";
version = "2.4.5";

src = fetchurl {
url = "mirror://gnupg/gnupg/${pname}-${version}.tar.bz2";
hash = "sha256-eyRwbk2n4OOwbKBoIxAnQB8jgQLEHJCWMTSdzDuF60Y=";
hash = "sha256-9o99ddBssWNcM2002ESvl0NsP2TqFLy3yGl4L5b0Qnc=";
};

depsBuildBuild = [ buildPackages.stdenv.cc ];
Expand Down
Loading