Fix and improve the nix flake (#512)

* fix(flake): `public` assets were moved to `kitsune/assets` Signed-off-by: Harald Hoyer <[email protected]> * feat(flake): use mold linker Signed-off-by: Harald Hoyer <[email protected]> * fix(flake): fix build dependency for openssl Native deps need the tools which run on the build host (when cross compiling). Normal build deps are the libs linked to the binary. OPENSSL_NO_VENDOR ensures that openssl-sys uses the system lib. Remove the rest of the workarounds for openssl. Signed-off-by: Harald Hoyer <[email protected]> * fix(flake): proper `mkYarnPackage` Signed-off-by: Harald Hoyer <[email protected]> * feat(flake): add overlay test Signed-off-by: Harald Hoyer <[email protected]> * fix(flake): skip more tests Signed-off-by: Harald Hoyer <[email protected]> * feat(flake): use the crane nix lib https://crane.dev/ enables caching of artifacts and potentially reducing build time. Even End to End(E2E) testing could be added: https://crane.dev/examples/end-to-end-testing.html Signed-off-by: Harald Hoyer <[email protected]> * ci(flake): add nix github action Signed-off-by: Harald Hoyer <[email protected]> * ci(flake): disable `cargo check` in nix flake Left as a separate commit, so it is easy to revert. Signed-off-by: Harald Hoyer <[email protected]> * ci(flake): run nixci in debug build by default normal flake produces release code still. Signed-off-by: Harald Hoyer <[email protected]> * ci(flake): build heavy stuff sequentially and skip running nixci. Signed-off-by: Harald Hoyer <[email protected]> * feat(flake): add `mrf-tool` and `kitsune-job-runner` Signed-off-by: Harald Hoyer <[email protected]> --------- Signed-off-by: Harald Hoyer <[email protected]>
kitsune-soc · Mar 29, 2024 · d68c5e3 · d68c5e3
1 parent d45e8f6
commit d68c5e3
Show file tree

Hide file tree

Showing 6 changed files with 180 additions and 45 deletions.
diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml
@@ -0,0 +1,44 @@
+name: Nix checks
+
+on:
+  merge_group:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_LOG: "debug"
+  RUSTFLAGS: "-C debuginfo=0"
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: DeterminateSystems/nix-installer-action@v4
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: nix flake check -L --show-trace --keep-going --impure
+
+  build:
+    needs: check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: DeterminateSystems/nix-installer-action@v4
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+
+      - name: nix build main
+        run: nix build --override-input debugBuild github:boolean-option/true/6ecb49143ca31b140a5273f1575746ba93c3f698 -L .#main
+      - name: nix build cli
+        run: nix build --override-input debugBuild github:boolean-option/true/6ecb49143ca31b140a5273f1575746ba93c3f698 -L .#cli
+      - name: nix build frontend
+        run: nix build --override-input debugBuild github:boolean-option/true/6ecb49143ca31b140a5273f1575746ba93c3f698 -L .#frontend
+      - name: nix build mrf-tool
+        run: nix build --override-input debugBuild github:boolean-option/true/6ecb49143ca31b140a5273f1575746ba93c3f698 -L .#mrf-tool
+
+      - name: nix check overlay
+        run: cd test-overlay && nix build --no-write-lock-file -L .#kitsune
+
diff --git a/.gitignore b/.gitignore
@@ -25,3 +25,5 @@ target-analyzer
 /result
 /.devenv
 /.pre-commit-config.yaml
+/test-overlay/flake.lock
+/test-overlay/result
diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -15,96 +15,113 @@
       };
       url = "github:oxalica/rust-overlay";
     };
+
+    crane = {
+      url = "github:ipetkov/crane";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
+    # The premise is this is the "default" and if you want to do a debug build,
+    # pass it in as an arg.
+    # like so `nix build --override-input debugBuild github:boolean-option/true`
+    debugBuild.url = "github:boolean-option/false/d06b4794a134686c70a1325df88a6e6768c6b212";
   };
-  outputs = { self, devenv, flake-utils, nixpkgs, rust-overlay, ... } @ inputs:
-    flake-utils.lib.eachDefaultSystem
+  outputs = { self, devenv, flake-utils, nixpkgs, rust-overlay, crane, ... } @ inputs:
+    (flake-utils.lib.eachDefaultSystem
       (system:
         let
+          features = "--all-features";
           overlays = [ (import rust-overlay) ];
           pkgs = import nixpkgs {
             inherit overlays system;
           };
+          stdenv = pkgs.stdenvAdapters.useMoldLinker pkgs.stdenv;
           rustPlatform = pkgs.makeRustPlatform {
             cargo = pkgs.rust-bin.stable.latest.minimal;
             rustc = pkgs.rust-bin.stable.latest.minimal;
+            inherit stdenv;
           };
-          baseDependencies = with pkgs; [
+
+          craneLib = (crane.mkLib pkgs).overrideToolchain pkgs.rust-bin.stable.latest.minimal;
+          buildInputs = with pkgs; [
             openssl
-            pkg-config
-            protobuf
             sqlite
             zlib
           ];
 
-          cargoConfig = builtins.fromTOML (builtins.readFile ./.cargo/config.toml); # TODO: Set the target CPU conditionally
-          cargoToml = builtins.fromTOML (builtins.readFile ./Cargo.toml);
+          nativeBuildInputs = with pkgs; [
+            protobuf
+            pkg-config
+            rustPlatform.bindgenHook
+          ];
+
           src = pkgs.lib.cleanSourceWith {
             src = pkgs.lib.cleanSource ./.;
             filter = name: type:
               let baseName = baseNameOf (toString name);
               in !(baseName == "flake.lock" || pkgs.lib.hasSuffix ".nix" baseName);
           };
-          version = cargoToml.workspace.package.version;
 
-          basePackage = {
-            inherit version src;
+          commonArgs = {
+            inherit src stdenv buildInputs nativeBuildInputs;
+
+            strictDeps = true;
 
             meta = {
               description = "ActivityPub-federated microblogging";
               homepage = "https://joinkitsune.org";
             };
 
-            cargoLock = {
-              lockFile = ./Cargo.lock;
-              allowBuiltinFetchGit = true;
-            };
+            OPENSSL_NO_VENDOR = 1;
+            NIX_OUTPATH_USED_AS_RANDOM_SEED = "aaaaaaaaaa";
+            cargoExtraArgs = "--locked ${features}";
+          } // (pkgs.lib.optionalAttrs inputs.debugBuild.value {
+            # do a debug build, as `dev` is the default debug profile
+            CARGO_PROFILE = "dev";
+          });
 
-            nativeBuildInputs = baseDependencies;
-
-            PKG_CONFIG_PATH = "${pkgs.openssl.dev}/lib/pkgconfig"; # Not sure why this is broken but it is
-            RUSTFLAGS = builtins.concatStringsSep " " cargoConfig.build.rustflags; # Oh god help.
-
-            checkFlags = [
-              # Depend on creating an HTTP client and that reads from the systems truststore
-              # Because nix is fully isolated, these types of tests fail
-              #
-              # Some (most?) of these also depend on the network? Not good??
-              "--skip=activitypub::fetcher::test::federation_allow"
-              "--skip=activitypub::fetcher::test::federation_deny"
-              "--skip=activitypub::fetcher::test::fetch_actor"
-              "--skip=activitypub::fetcher::test::fetch_note"
-              "--skip=resolve::post::test::parse_mentions"
-              "--skip=webfinger::test::fetch_qarnax_ap_id"
-              "--skip=basic_request"
-              "--skip=json_request"
-            ];
-          };
+          cargoToml = builtins.fromTOML (builtins.readFile ./Cargo.toml);
+          version = cargoToml.workspace.package.version;
+
+          cargoArtifacts = craneLib.buildDepsOnly (commonArgs // {
+            pname = "kitsune-workspace";
+            src = craneLib.cleanCargoSource src;
+          });
         in
         {
           formatter = pkgs.nixpkgs-fmt;
           packages = rec {
-            # Hack to make latest devenv work
-            devenv-up = self.devShells.${system}.default.config.procfileScript;
-
             default = main;
 
-            cli = rustPlatform.buildRustPackage (basePackage // {
+            cli = craneLib.buildPackage (commonArgs // {
               pname = "kitsune-cli";
-              cargoBuildFlags = "-p kitsune-cli";
+              cargoExtraArgs = commonArgs.cargoExtraArgs + " --bin kitsune-cli";
+              inherit cargoArtifacts;
+              doCheck = false;
+            });
+
+            mrf-tool = craneLib.buildPackage (commonArgs // {
+              pname = "mrf-tool";
+              cargoExtraArgs = commonArgs.cargoExtraArgs + " --bin mrf-tool";
+              inherit cargoArtifacts;
+              doCheck = false;
             });
 
-            main = rustPlatform.buildRustPackage (basePackage // {
+            main = craneLib.buildPackage (commonArgs // rec {
               pname = "kitsune";
-              buildFeatures = [ "meilisearch" "oidc" ];
-              cargoBuildFlags = "-p kitsune";
+              cargoExtraArgs = commonArgs.cargoExtraArgs + " --bin kitsune --bin kitsune-job-runner";
+              inherit cargoArtifacts;
+              doCheck = false;
             });
 
             frontend = pkgs.mkYarnPackage {
               inherit version;
-
+              packageJSON = "${src}/kitsune-fe/package.json";
+              yarnLock = "${src}/kitsune-fe/yarn.lock";
               src = "${src}/kitsune-fe";
 
               buildPhase = ''
+                export HOME=$(mktemp -d)
                 yarn --offline build
               '';
 
@@ -131,7 +148,7 @@
                     rust-bin.stable.latest.default
                   ]
                   ++
-                  baseDependencies;
+                  buildInputs ++ nativeBuildInputs;
 
                   enterShell = ''
                     export PG_HOST=127.0.0.1
@@ -171,5 +188,12 @@
         default = kitsune;
         kitsune = (import ./module.nix);
       };
+    }) // {
+      nixci.default = {
+        debug = {
+          dir = ".";
+          overrideInputs.debugBuild = "github:boolean-option/true/6ecb49143ca31b140a5273f1575746ba93c3f698";
+        };
+      };
     };
 }
diff --git a/overlay.nix b/overlay.nix
@@ -9,7 +9,7 @@ in
     installPhase = ''
       mkdir -p $out
       cp -R ${packages.main}/bin $out
-      cp -R ${packages.main.src}/public $out
+      cp -R ${packages.main.src}/kitsune/assets $out/public
       cp -R ${packages.frontend}/dist $out/kitsune-fe
     '';
   };

diff --git a/test-overlay/flake.nix b/test-overlay/flake.nix
@@ -0,0 +1,27 @@
+{
+  inputs = {
+    kitsune-overlay.url = "./..";
+    kitsune-overlay.inputs.debugBuild.follows = "debugBuild";
+    nixpkgs.follows = "kitsune-overlay/nixpkgs";
+    flake-utils.follows = "kitsune-overlay/flake-utils";
+    debugBuild.url = "github:boolean-option/true/6ecb49143ca31b140a5273f1575746ba93c3f698";
+  };
+  outputs = { self, flake-utils, nixpkgs, kitsune-overlay, ... } @ inputs:
+    flake-utils.lib.eachDefaultSystem
+      (system:
+        let
+          overlays = [ kitsune-overlay.overlays.default ];
+          pkgs = import nixpkgs {
+            inherit overlays system;
+          };
+        in
+        {
+          formatter = pkgs.nixpkgs-fmt;
+          packages = rec {
+            default = kitsune;
+            inherit (pkgs) kitsune;
+            inherit (pkgs) kitsune-cli;
+          };
+        }
+      );
+}