diff --git a/.envrc b/.envrc new file mode 100644 index 0000000000..3550a30f2d --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f2b958afc2..f12c659158 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,17 @@ version: 2 updates: + + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: daily + time: '00:00' + timezone: UTC + open-pull-requests-limit: 10 + commit-message: + prefix: "chore" + include: "scope" + - package-ecosystem: cargo directory: '/' schedule: diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml new file mode 100644 index 0000000000..73d8018626 --- /dev/null +++ b/.github/workflows/update-flake-lock.yml @@ -0,0 +1,19 @@ +name: update-flake-lock +on: + workflow_dispatch: # allows manual triggering + schedule: + - cron: '0 0 * * 0' # runs weekly on Sunday at 00:00 + +jobs: + lockfile: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Install Nix + uses: cachix/install-nix-action@v16 + with: + extra_nix_config: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + - name: Update flake.lock + uses: DeterminateSystems/update-flake-lock@v8 diff --git a/.gitignore b/.gitignore index 21a72d8228..14122c376c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ book /target /vendor -.idea/ -.vscode/ +/result +/.idea/ +/.vscode/ +/.direnv/ +/.pre-commit-config.yaml diff --git a/flake.lock b/flake.lock index 2e0f462d62..f9c2630ab8 100644 --- a/flake.lock +++ b/flake.lock @@ -1,12 +1,68 @@ { "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1644229661, + "narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "3cecb5b042f7f209c56ffd8371b2711a290ec797", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1646480205, + "narHash": "sha256-kekOlTlu45vuK2L9nq8iVN17V3sB0WWPqTTW3a2SQG0=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "bff2832ec341cf30acb3a4d3e2e7f1f7b590116a", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, + "naersk": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1647605581, + "narHash": "sha256-56AaI1Zqgxuu+b8o3RbQBP2wtdn3/mQ23eLnkxOtHm8=", + "owner": "yusdacra", + "repo": "naersk", + "rev": "7882e303c4904664194236c921365b4d6b86b9bc", + "type": "github" + }, + "original": { + "owner": "yusdacra", + "ref": "feat/cargolock-git-deps", + "repo": "naersk", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1637595801, - "narHash": "sha256-LkIMwVFKCuEqidaUdg8uxwpESAXjsPo4oCz3eJ7RaRw=", + "lastModified": 1647297614, + "narHash": "sha256-ulGq3W5XsrBMU/u5k9d4oPy65pQTkunR4HKKtTq0RwY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "263ef4cc4146c9fab808085487438c625d4426a9", + "rev": "73ad5f9e147c0d2a2061f1d4bd91e05078dc0b58", "type": "github" }, "original": { @@ -16,24 +72,59 @@ "type": "github" } }, + "pre-commit-hooks": { + "inputs": { + "flake-utils": [ + "flake-utils" + ], + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1646153636, + "narHash": "sha256-AlWHMzK+xJ1mG267FdT8dCq/HvLCA6jwmx2ZUy5O8tY=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "b6bc0b21e1617e2b07d8205e7fae7224036dfa4b", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, "root": { "inputs": { + "flake-utils": "flake-utils", + "gitignore": "gitignore", + "naersk": "naersk", "nixpkgs": "nixpkgs", - "utils": "utils" + "pre-commit-hooks": "pre-commit-hooks", + "rust-overlay": "rust-overlay" } }, - "utils": { + "rust-overlay": { + "inputs": { + "flake-utils": [ + "flake-utils" + ], + "nixpkgs": [ + "nixpkgs" + ] + }, "locked": { - "lastModified": 1624650440, - "narHash": "sha256-BIwYENLvbSTNYdyeVJihwi7G6bXVGrPNoOgvJ1Z1SlY=", - "owner": "kreisys", - "repo": "flake-utils", - "rev": "026014b92b548ef130a7aebc84a3894f791370fc", + "lastModified": 1647570584, + "narHash": "sha256-wQ/xUt2yU+ncx0JQMMQDcFoWLY2xo0Vo5CxIGuxMdmY=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "00ac40d23ec331be4fd143b8cd3e9142e3bcbe9b", "type": "github" }, "original": { - "owner": "kreisys", - "repo": "flake-utils", + "owner": "oxalica", + "repo": "rust-overlay", "type": "github" } } diff --git a/flake.nix b/flake.nix index 23a453912d..c917d469cd 100644 --- a/flake.nix +++ b/flake.nix @@ -1,110 +1,196 @@ { - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; - utils.url = "github:kreisys/flake-utils"; - }; - - nixConfig = { - extra-substituters = [ - "https://hydra.iohk.io" - "https://vit-ops.cachix.org" - ]; - extra-trusted-public-keys = [ - "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" - "vit-ops.cachix.org-1:LY84nIKdW7g1cvhJ6LsupHmGtGcKAlUXo+l1KByoDho=" - ]; - }; - - outputs = { self, nixpkgs, utils }: - let - workspaceCargo = builtins.fromTOML (builtins.readFile ./Cargo.toml); - inherit (workspaceCargo.workspace) members; - in utils.lib.simpleFlake { - inherit nixpkgs; - systems = [ "x86_64-linux" "aarch64-linux" ]; - preOverlays = [ ]; - overlay = final: prev: - let inherit (prev) lib; - in (lib.listToAttrs (lib.forEach members (member: - lib.nameValuePair member (prev.rustPlatform.buildRustPackage { - inherit ((builtins.fromTOML - (builtins.readFile (./. + "/${member}/Cargo.toml"))).package) - name version; - src = ./.; - cargoSha256 = "0hzzsgbyj1wjw8aiywzdbccmzlcbxcq361xln21q118xk16xzlzp"; - buildFeatures = lib.optional (member == "jormungandr") "prometheus-metrics"; - nativeBuildInputs = with final; [ pkg-config protobuf rustfmt ]; - buildInputs = with final; [ openssl ]; - PROTOC = "${final.protobuf}/bin/protoc"; - PROTOC_INCLUDE = "${final.protobuf}/include"; - })))) // { - jormungandr-entrypoint = let - script = final.writeShellScriptBin "entrypoint" '' - set -exuo pipefail - - ulimit -n 1024 - - nodeConfig="$NOMAD_TASK_DIR/node-config.json" - runConfig="$NOMAD_TASK_DIR/running.json" - runYaml="$NOMAD_TASK_DIR/running.yaml" - name="jormungandr" + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + inputs.flake-utils.url = "github:numtide/flake-utils"; + inputs.gitignore.url = "github:hercules-ci/gitignore.nix"; + inputs.gitignore.inputs.nixpkgs.follows = "nixpkgs"; + inputs.pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix"; + inputs.pre-commit-hooks.inputs.nixpkgs.follows = "nixpkgs"; + inputs.pre-commit-hooks.inputs.flake-utils.follows = "flake-utils"; + inputs.rust-overlay.url = "github:oxalica/rust-overlay"; + inputs.rust-overlay.inputs.flake-utils.follows = "flake-utils"; + inputs.rust-overlay.inputs.nixpkgs.follows = "nixpkgs"; + # XXX: https://github.com/nix-community/naersk/pull/167 + #inputs.naersk.url = "github:nix-community/naersk"; + inputs.naersk.url = "github:yusdacra/naersk/feat/cargolock-git-deps"; + inputs.naersk.inputs.nixpkgs.follows = "nixpkgs"; + nixConfig.extra-substituters = [ + "https://hydra.iohk.io" + "https://vit-ops.cachix.org" + ]; + nixConfig.extra-trusted-public-keys = [ + "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" + "vit-ops.cachix.org-1:LY84nIKdW7g1cvhJ6LsupHmGtGcKAlUXo+l1KByoDho=" + ]; + + outputs = { + self, + nixpkgs, + flake-utils, + gitignore, + pre-commit-hooks, + rust-overlay, + naersk, + }: + flake-utils.lib.eachSystem + [ + flake-utils.lib.system.x86_64-linux + flake-utils.lib.system.aarch64-linux + ] + ( + system: let + readTOML = file: builtins.fromTOML (builtins.readFile file); + workspaceCargo = readTOML ./Cargo.toml; + + pkgs = import nixpkgs { + inherit system; + overlays = [(import rust-overlay)]; + }; + + rust = let + _rust = pkgs.rust-bin.stable.latest.default.override { + extensions = [ + "rust-src" + "rust-analysis" + "rls-preview" + "rustfmt-preview" + "clippy-preview" + ]; + }; + in + pkgs.buildEnv { + name = _rust.name; + inherit (_rust) meta; + buildInputs = [pkgs.makeWrapper]; + paths = [_rust]; + pathsToLink = ["/" "/bin"]; + # XXX: This is needed because cargo and clippy commands need to + # also be aware of other binaries in order to work properly. + # https://github.com/cachix/pre-commit-hooks.nix/issues/126 + postBuild = '' + for i in $out/bin/*; do + wrapProgram "$i" --prefix PATH : "$out/bin" + done + ''; + }; + + naersk-lib = naersk.lib."${system}".override { + cargo = rust; + rustc = rust; + }; + + mkPackage = name: let + pkgCargo = readTOML ./${name}/Cargo.toml; + cargoOptions = + [ + "--package" + name + ] + ++ (pkgs.lib.optionals (name == "jormungandr") [ + "--features" + "prometheus-metrics" + ]); + in + naersk-lib.buildPackage { + root = gitignore.lib.gitignoreSource self; + + cargoBuildOptions = x: x ++ cargoOptions; + cargoTestOptions = x: x ++ cargoOptions; + + PROTOC = "${pkgs.protobuf}/bin/protoc"; + PROTOC_INCLUDE = "${pkgs.protobuf}/include"; + + nativeBuildInputs = with pkgs; [ + pkg-config + protobuf + rustfmt + ]; + + buildInputs = with pkgs; [ + openssl + ]; + }; + + workspace = + builtins.listToAttrs + ( + builtins.map + (name: { + inherit name; + value = mkPackage name; + }) + workspaceCargo.workspace.members + ); + + jormungandr-entrypoint = let + script = + pkgs.writeShellScriptBin "entrypoint" + '' + set -exuo pipefail + + ulimit -n 1024 + + nodeConfig="$NOMAD_TASK_DIR/node-config.json" + runConfig="$NOMAD_TASK_DIR/running.json" + runYaml="$NOMAD_TASK_DIR/running.yaml" + name="jormungandr" + + chmod u+rwx -R "$NOMAD_TASK_DIR" || true + + function convert () { chmod u+rwx -R "$NOMAD_TASK_DIR" || true + cp "$nodeConfig" "$runConfig" + remarshal --if json --of yaml "$runConfig" > "$runYaml" + } + + if [ "$RESET" = "true" ]; then + echo "RESET is given, will start from scratch..." + rm -rf "$STORAGE_DIR" + elif [ -d "$STORAGE_DIR" ]; then + echo "$STORAGE_DIR found, not restoring from backup..." + else + echo "$STORAGE_DIR not found, restoring backup..." + + restic restore latest \ + --verbose=5 \ + --no-lock \ + --tag "$NAMESPACE" \ + --target / \ + || echo "couldn't restore backup, continue startup procedure..." + fi + + set +x + echo "waiting for $REQUIRED_PEER_COUNT peers" + until [ "$(jq -e -r '.p2p.trusted_peers | length' < "$nodeConfig" || echo 0)" -ge $REQUIRED_PEER_COUNT ]; do + sleep 1 + done + set -x - function convert () { - chmod u+rwx -R "$NOMAD_TASK_DIR" || true - cp "$nodeConfig" "$runConfig" - remarshal --if json --of yaml "$runConfig" > "$runYaml" - } - - if [ "$RESET" = "true" ]; then - echo "RESET is given, will start from scratch..." - rm -rf "$STORAGE_DIR" - elif [ -d "$STORAGE_DIR" ]; then - echo "$STORAGE_DIR found, not restoring from backup..." - else - echo "$STORAGE_DIR not found, restoring backup..." - - restic restore latest \ - --verbose=5 \ - --no-lock \ - --tag "$NAMESPACE" \ - --target / \ - || echo "couldn't restore backup, continue startup procedure..." - fi - - set +x - echo "waiting for $REQUIRED_PEER_COUNT peers" - until [ "$(jq -e -r '.p2p.trusted_peers | length' < "$nodeConfig" || echo 0)" -ge $REQUIRED_PEER_COUNT ]; do - sleep 1 - done - set -x - - convert - - if [ -n "$PRIVATE" ]; then - echo "Running with node with secrets..." - exec jormungandr \ - --storage "$STORAGE_DIR" \ - --config "$NOMAD_TASK_DIR/running.yaml" \ - --genesis-block $NOMAD_TASK_DIR/block0.bin/block0.bin \ - --secret $NOMAD_SECRETS_DIR/bft-secret.yaml \ - "$@" || true - else - echo "Running with follower node..." - exec jormungandr \ - --storage "$STORAGE_DIR" \ - --config "$NOMAD_TASK_DIR/running.yaml" \ - --genesis-block $NOMAD_TASK_DIR/block0.bin/block0.bin \ - "$@" || true - fi - ''; - in final.symlinkJoin { - name = "entrypoint"; - paths = with final; [ - jormungandr - script + convert + if [ -n "$PRIVATE" ]; then + echo "Running with node with secrets..." + exec jormungandr \ + --storage "$STORAGE_DIR" \ + --config "$NOMAD_TASK_DIR/running.yaml" \ + --genesis-block $NOMAD_TASK_DIR/block0.bin/block0.bin \ + --secret $NOMAD_SECRETS_DIR/bft-secret.yaml \ + "$@" || true + else + echo "Running with follower node..." + exec jormungandr \ + --storage "$STORAGE_DIR" \ + --config "$NOMAD_TASK_DIR/running.yaml" \ + --genesis-block $NOMAD_TASK_DIR/block0.bin/block0.bin \ + "$@" || true + fi + ''; + in + pkgs.symlinkJoin { + name = "entrypoint"; + paths = + [script workspace.jormungandr] + ++ (with pkgs; [ bashInteractive coreutils curl @@ -114,7 +200,6 @@ gnugrep gnused htop - jormungandr jq lsof netcat @@ -129,20 +214,54 @@ utillinux vim yq - ]; + ]); + }; + + pre-commit = pre-commit-hooks.lib.${system}.run { + src = self; + hooks = { + alejandra = { + enable = true; + }; + rustfmt = { + enable = true; + entry = pkgs.lib.mkForce "${rust}/bin/cargo-fmt fmt -- --check --color always"; }; }; + }; - packages = { jormungandr, jcli, jormungandr-entrypoint }@pkgs: pkgs; + warnToUpdateNix = pkgs.lib.warn "Consider updating to Nix > 2.7 to remove this warning!"; + in rec { + packages = { + inherit (workspace) jormungandr jcli; + inherit jormungandr-entrypoint; + default = workspace.jormungandr; + }; - devShell = - { mkShell, rustc, cargo, pkg-config, openssl, protobuf, rustfmt }: - mkShell { - PROTOC = "${protobuf}/bin/protoc"; - PROTOC_INCLUDE = "${protobuf}/include"; - buildInputs = [ rustc cargo pkg-config openssl protobuf rustfmt ]; + devShells.default = pkgs.mkShell { + PROTOC = "${pkgs.protobuf}/bin/protoc"; + PROTOC_INCLUDE = "${pkgs.protobuf}/include"; + buildInputs = + [rust] + ++ (with pkgs; [ + pkg-config + openssl + protobuf + ]); + shellHook = + pre-commit.shellHook + + '' + echo "=== Jormungandr development shell ===" + echo "Info: Git hooks can be installed using \`pre-commit install\`" + ''; }; - hydraJobs = { jormungandr, jcli, jormungandr-entrypoint }@pkgs: pkgs; - }; + checks.pre-commit = pre-commit; + + hydraJobs = packages; + + defaultPackage = warnToUpdateNix packages.default; + devShell = warnToUpdateNix devShells.default; + } + ); }