From d68c5e3b1769c41a6812865a352ae87ff35e1534 Mon Sep 17 00:00:00 2001 From: Harald Hoyer Date: Fri, 29 Mar 2024 13:49:25 +0100 Subject: [PATCH] Fix and improve the nix flake (#512) * fix(flake): `public` assets were moved to `kitsune/assets` Signed-off-by: Harald Hoyer * feat(flake): use mold linker Signed-off-by: Harald Hoyer * fix(flake): fix build dependency for openssl Native deps need the tools which run on the build host (when cross compiling). Normal build deps are the libs linked to the binary. OPENSSL_NO_VENDOR ensures that openssl-sys uses the system lib. Remove the rest of the workarounds for openssl. Signed-off-by: Harald Hoyer * fix(flake): proper `mkYarnPackage` Signed-off-by: Harald Hoyer * feat(flake): add overlay test Signed-off-by: Harald Hoyer * fix(flake): skip more tests Signed-off-by: Harald Hoyer * feat(flake): use the crane nix lib https://crane.dev/ enables caching of artifacts and potentially reducing build time. Even End to End(E2E) testing could be added: https://crane.dev/examples/end-to-end-testing.html Signed-off-by: Harald Hoyer * ci(flake): add nix github action Signed-off-by: Harald Hoyer * ci(flake): disable `cargo check` in nix flake Left as a separate commit, so it is easy to revert. Signed-off-by: Harald Hoyer * ci(flake): run nixci in debug build by default normal flake produces release code still. Signed-off-by: Harald Hoyer * ci(flake): build heavy stuff sequentially and skip running nixci. Signed-off-by: Harald Hoyer * feat(flake): add `mrf-tool` and `kitsune-job-runner` Signed-off-by: Harald Hoyer --------- Signed-off-by: Harald Hoyer --- .github/workflows/nix.yml | 44 +++++++++++++++ .gitignore | 2 + flake.lock | 38 +++++++++++++ flake.nix | 112 +++++++++++++++++++++++--------------- overlay.nix | 2 +- test-overlay/flake.nix | 27 +++++++++ 6 files changed, 180 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/nix.yml create mode 100644 test-overlay/flake.nix diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml new file mode 100644 index 000000000..694809eca --- /dev/null +++ b/.github/workflows/nix.yml @@ -0,0 +1,44 @@ +name: Nix checks + +on: + merge_group: + pull_request: + push: + branches: + - main + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + RUST_LOG: "debug" + RUSTFLAGS: "-C debuginfo=0" + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@v4 + - uses: DeterminateSystems/magic-nix-cache-action@main + - run: nix flake check -L --show-trace --keep-going --impure + + build: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@v4 + - uses: DeterminateSystems/magic-nix-cache-action@main + + - name: nix build main + run: nix build --override-input debugBuild github:boolean-option/true/6ecb49143ca31b140a5273f1575746ba93c3f698 -L .#main + - name: nix build cli + run: nix build --override-input debugBuild github:boolean-option/true/6ecb49143ca31b140a5273f1575746ba93c3f698 -L .#cli + - name: nix build frontend + run: nix build --override-input debugBuild github:boolean-option/true/6ecb49143ca31b140a5273f1575746ba93c3f698 -L .#frontend + - name: nix build mrf-tool + run: nix build --override-input debugBuild github:boolean-option/true/6ecb49143ca31b140a5273f1575746ba93c3f698 -L .#mrf-tool + + - name: nix check overlay + run: cd test-overlay && nix build --no-write-lock-file -L .#kitsune + diff --git a/.gitignore b/.gitignore index e21b772ff..21db890ff 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ target-analyzer /result /.devenv /.pre-commit-config.yaml +/test-overlay/flake.lock +/test-overlay/result \ No newline at end of file diff --git a/flake.lock b/flake.lock index 16503c7a5..b4d584edb 100644 --- a/flake.lock +++ b/flake.lock @@ -24,6 +24,42 @@ "type": "github" } }, + "crane": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1711407199, + "narHash": "sha256-A/nB4j3JHL51ztlMQdfKw6y8tUJJzai3bLsZUEEaBxY=", + "owner": "ipetkov", + "repo": "crane", + "rev": "7e468a455506f2e65550e08dfd45092f0857a009", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, + "debugBuild": { + "locked": { + "lastModified": 1657739266, + "narHash": "sha256-vLy8GQr0noEcoA+jX24FgUVBA/poV36zDWAUChN3hIY=", + "owner": "boolean-option", + "repo": "false", + "rev": "d06b4794a134686c70a1325df88a6e6768c6b212", + "type": "github" + }, + "original": { + "owner": "boolean-option", + "repo": "false", + "rev": "d06b4794a134686c70a1325df88a6e6768c6b212", + "type": "github" + } + }, "devenv": { "inputs": { "cachix": "cachix", @@ -527,6 +563,8 @@ }, "root": { "inputs": { + "crane": "crane", + "debugBuild": "debugBuild", "devenv": "devenv", "flake-utils": "flake-utils_4", "nixpkgs": "nixpkgs_2", diff --git a/flake.nix b/flake.nix index cada6ffe8..29ed38fd3 100644 --- a/flake.nix +++ b/flake.nix @@ -15,96 +15,113 @@ }; url = "github:oxalica/rust-overlay"; }; + + crane = { + url = "github:ipetkov/crane"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + + # The premise is this is the "default" and if you want to do a debug build, + # pass it in as an arg. + # like so `nix build --override-input debugBuild github:boolean-option/true` + debugBuild.url = "github:boolean-option/false/d06b4794a134686c70a1325df88a6e6768c6b212"; }; - outputs = { self, devenv, flake-utils, nixpkgs, rust-overlay, ... } @ inputs: - flake-utils.lib.eachDefaultSystem + outputs = { self, devenv, flake-utils, nixpkgs, rust-overlay, crane, ... } @ inputs: + (flake-utils.lib.eachDefaultSystem (system: let + features = "--all-features"; overlays = [ (import rust-overlay) ]; pkgs = import nixpkgs { inherit overlays system; }; + stdenv = pkgs.stdenvAdapters.useMoldLinker pkgs.stdenv; rustPlatform = pkgs.makeRustPlatform { cargo = pkgs.rust-bin.stable.latest.minimal; rustc = pkgs.rust-bin.stable.latest.minimal; + inherit stdenv; }; - baseDependencies = with pkgs; [ + + craneLib = (crane.mkLib pkgs).overrideToolchain pkgs.rust-bin.stable.latest.minimal; + buildInputs = with pkgs; [ openssl - pkg-config - protobuf sqlite zlib ]; - cargoConfig = builtins.fromTOML (builtins.readFile ./.cargo/config.toml); # TODO: Set the target CPU conditionally - cargoToml = builtins.fromTOML (builtins.readFile ./Cargo.toml); + nativeBuildInputs = with pkgs; [ + protobuf + pkg-config + rustPlatform.bindgenHook + ]; + src = pkgs.lib.cleanSourceWith { src = pkgs.lib.cleanSource ./.; filter = name: type: let baseName = baseNameOf (toString name); in !(baseName == "flake.lock" || pkgs.lib.hasSuffix ".nix" baseName); }; - version = cargoToml.workspace.package.version; - basePackage = { - inherit version src; + commonArgs = { + inherit src stdenv buildInputs nativeBuildInputs; + + strictDeps = true; meta = { description = "ActivityPub-federated microblogging"; homepage = "https://joinkitsune.org"; }; - cargoLock = { - lockFile = ./Cargo.lock; - allowBuiltinFetchGit = true; - }; + OPENSSL_NO_VENDOR = 1; + NIX_OUTPATH_USED_AS_RANDOM_SEED = "aaaaaaaaaa"; + cargoExtraArgs = "--locked ${features}"; + } // (pkgs.lib.optionalAttrs inputs.debugBuild.value { + # do a debug build, as `dev` is the default debug profile + CARGO_PROFILE = "dev"; + }); - nativeBuildInputs = baseDependencies; - - PKG_CONFIG_PATH = "${pkgs.openssl.dev}/lib/pkgconfig"; # Not sure why this is broken but it is - RUSTFLAGS = builtins.concatStringsSep " " cargoConfig.build.rustflags; # Oh god help. - - checkFlags = [ - # Depend on creating an HTTP client and that reads from the systems truststore - # Because nix is fully isolated, these types of tests fail - # - # Some (most?) of these also depend on the network? Not good?? - "--skip=activitypub::fetcher::test::federation_allow" - "--skip=activitypub::fetcher::test::federation_deny" - "--skip=activitypub::fetcher::test::fetch_actor" - "--skip=activitypub::fetcher::test::fetch_note" - "--skip=resolve::post::test::parse_mentions" - "--skip=webfinger::test::fetch_qarnax_ap_id" - "--skip=basic_request" - "--skip=json_request" - ]; - }; + cargoToml = builtins.fromTOML (builtins.readFile ./Cargo.toml); + version = cargoToml.workspace.package.version; + + cargoArtifacts = craneLib.buildDepsOnly (commonArgs // { + pname = "kitsune-workspace"; + src = craneLib.cleanCargoSource src; + }); in { formatter = pkgs.nixpkgs-fmt; packages = rec { - # Hack to make latest devenv work - devenv-up = self.devShells.${system}.default.config.procfileScript; - default = main; - cli = rustPlatform.buildRustPackage (basePackage // { + cli = craneLib.buildPackage (commonArgs // { pname = "kitsune-cli"; - cargoBuildFlags = "-p kitsune-cli"; + cargoExtraArgs = commonArgs.cargoExtraArgs + " --bin kitsune-cli"; + inherit cargoArtifacts; + doCheck = false; + }); + + mrf-tool = craneLib.buildPackage (commonArgs // { + pname = "mrf-tool"; + cargoExtraArgs = commonArgs.cargoExtraArgs + " --bin mrf-tool"; + inherit cargoArtifacts; + doCheck = false; }); - main = rustPlatform.buildRustPackage (basePackage // { + main = craneLib.buildPackage (commonArgs // rec { pname = "kitsune"; - buildFeatures = [ "meilisearch" "oidc" ]; - cargoBuildFlags = "-p kitsune"; + cargoExtraArgs = commonArgs.cargoExtraArgs + " --bin kitsune --bin kitsune-job-runner"; + inherit cargoArtifacts; + doCheck = false; }); frontend = pkgs.mkYarnPackage { inherit version; - + packageJSON = "${src}/kitsune-fe/package.json"; + yarnLock = "${src}/kitsune-fe/yarn.lock"; src = "${src}/kitsune-fe"; buildPhase = '' + export HOME=$(mktemp -d) yarn --offline build ''; @@ -131,7 +148,7 @@ rust-bin.stable.latest.default ] ++ - baseDependencies; + buildInputs ++ nativeBuildInputs; enterShell = '' export PG_HOST=127.0.0.1 @@ -171,5 +188,12 @@ default = kitsune; kitsune = (import ./module.nix); }; + }) // { + nixci.default = { + debug = { + dir = "."; + overrideInputs.debugBuild = "github:boolean-option/true/6ecb49143ca31b140a5273f1575746ba93c3f698"; + }; + }; }; } diff --git a/overlay.nix b/overlay.nix index de5a13eb2..5fc2c0ca9 100644 --- a/overlay.nix +++ b/overlay.nix @@ -9,7 +9,7 @@ in installPhase = '' mkdir -p $out cp -R ${packages.main}/bin $out - cp -R ${packages.main.src}/public $out + cp -R ${packages.main.src}/kitsune/assets $out/public cp -R ${packages.frontend}/dist $out/kitsune-fe ''; }; diff --git a/test-overlay/flake.nix b/test-overlay/flake.nix new file mode 100644 index 000000000..15b4eb0ec --- /dev/null +++ b/test-overlay/flake.nix @@ -0,0 +1,27 @@ +{ + inputs = { + kitsune-overlay.url = "./.."; + kitsune-overlay.inputs.debugBuild.follows = "debugBuild"; + nixpkgs.follows = "kitsune-overlay/nixpkgs"; + flake-utils.follows = "kitsune-overlay/flake-utils"; + debugBuild.url = "github:boolean-option/true/6ecb49143ca31b140a5273f1575746ba93c3f698"; + }; + outputs = { self, flake-utils, nixpkgs, kitsune-overlay, ... } @ inputs: + flake-utils.lib.eachDefaultSystem + (system: + let + overlays = [ kitsune-overlay.overlays.default ]; + pkgs = import nixpkgs { + inherit overlays system; + }; + in + { + formatter = pkgs.nixpkgs-fmt; + packages = rec { + default = kitsune; + inherit (pkgs) kitsune; + inherit (pkgs) kitsune-cli; + }; + } + ); +}