From 467de5590881baf22eb425baa3571bef82e00112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Fran=C3=A7a?= Date: Mon, 20 Nov 2023 14:36:24 +0000 Subject: [PATCH] Delete unused Docker files + misc updates (#36) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What ❔ - Removing old Docker files. They are no longer used. - Updated PR template. - Updated docs. - Updated code owners. ## Why ❔ Just cleaning up the repo. --- .dockerignore | 3 - .github/CODEOWNERS | 9 +- .github/pull_request_template.md | 2 +- .github/workflows/docker_build.yaml | 57 ----------- composes/generate/Cargo.lock | 7 -- composes/generate/Cargo.toml | 8 -- composes/generate/src/main.rs | 145 ---------------------------- docker/Dockerfile | 22 ----- docker/compose.Dockerfile | 31 ------ docker/localenv.Dockerfile | 61 ------------ docs/architecture.md | 9 +- docs/launch.md | 16 +-- 12 files changed, 12 insertions(+), 358 deletions(-) delete mode 100644 .dockerignore delete mode 100644 .github/workflows/docker_build.yaml delete mode 100644 composes/generate/Cargo.lock delete mode 100644 composes/generate/Cargo.toml delete mode 100644 composes/generate/src/main.rs delete mode 100644 docker/Dockerfile delete mode 100644 docker/compose.Dockerfile delete mode 100644 docker/localenv.Dockerfile diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 05e62c70..00000000 --- a/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -**/.git -node/target -tools/target diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cffd80d3..b77c7568 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,9 +1,12 @@ CODEOWNERS @brunoffranca -/node/actors/consensus/ @brunoffranca -/node/actors/executor/ @brunoffranca @pompon0 +/node/actors/consensus/ @brunoffranca @moshababo +/node/actors/executor/ @brunoffranca @pompon0 @slowli /node/actors/network/ @pompon0 /node/actors/sync_blocks/ @slowli /node/libs/concurrency/ @pompon0 -/node/libs/crypto/ @brunoffranca \ No newline at end of file +/node/libs/crypto/ @brunoffranca +/node/libs/protobuf/ @pompon0 +/node/libs/protobuf_build/ @pompon0 +/node/libs/storage/ @slowli \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 304b0d3a..6dc78a03 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,4 @@ -# What ❔ +## What ❔ diff --git a/.github/workflows/docker_build.yaml b/.github/workflows/docker_build.yaml deleted file mode 100644 index 08735179..00000000 --- a/.github/workflows/docker_build.yaml +++ /dev/null @@ -1,57 +0,0 @@ -name: Build and push Docker image - -on: - push: - branches: [ "main" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - build-push-image: - name: Build and push Docker image - runs-on: [matterlabs-ci-runner] - steps: - - uses: actions/checkout@v3 - - - name: Set outputs - id: vars - run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - - name: Login to GAR - run: | - gcloud auth configure-docker us-docker.pkg.dev -q - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Generate build ID for Flux Image Automation - id: set_version - run: | - if [[ ${{ github.ref }} =~ "refs/tags" ]]; then - echo "image_tag_suffix=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - else - sha=$(git rev-parse --short HEAD) - ts=$(date +%s%N | cut -b1-13) - echo "image_tag_suffix=${sha}-${ts}" >> $GITHUB_OUTPUT - fi - - name: Log in to Docker Hub - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push - id: docker_build_push_tag - uses: docker/build-push-action@v3 - with: - context: . - file: docker/Dockerfile - push: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) }} - tags: | - "matterlabs/zksync-bft:latest" - "matterlabs/zksync-bft:${{ steps.set_version.outputs.image_tag_suffix }}" - "us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zksync-bft:latest" - "us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zksync-bft:${{ steps.set_version.outputs.image_tag_suffix }}" - - name: Print image digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/composes/generate/Cargo.lock b/composes/generate/Cargo.lock deleted file mode 100644 index 41cb2aea..00000000 --- a/composes/generate/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "generate" -version = "0.1.0" diff --git a/composes/generate/Cargo.toml b/composes/generate/Cargo.toml deleted file mode 100644 index 22de7aa4..00000000 --- a/composes/generate/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "generate" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] diff --git a/composes/generate/src/main.rs b/composes/generate/src/main.rs deleted file mode 100644 index 5a4f66e3..00000000 --- a/composes/generate/src/main.rs +++ /dev/null @@ -1,145 +0,0 @@ -use std::env; -use std::fs::File; -use std::io::prelude::*; - -const START: &str = r#" -# this file is auto generated by generate.py -# do not edit it manually - -version: '3.9' -services: -"#; - -const END: &str = r#" -networks: - intrnl: - ipam: - driver: default - config: - - subnet: "192.168.92.0/24" - -# This Docker Compose file defines a multi-container application with four services: -# app0, app1, app2, and app3. The file specifies how the services should be built, -# configured, and run. - -# This app0 ... app3 are the nodes of the blockchain network. Each node is a container -# running a blockchain node. The nodes are connected to each other via a TCP protocol. - -# This specifies that the app0...3 service should be built using the Dockerfile located -# in the composes directory, with the node argument set to node0...3. The service should -# also expose port 3333..3336 on the host and map it to port 3333..3336 on the container. -# The command field specifies the command that should be run when the container is started, -# which in this case is ./executor 0..3. The networks field specifies that the service -# should be connected to the internal network and should be assigned the IP address 192.168.92.21...24. -"#; - -const LOOP_BODY: &str = r#" - app#i: - build: - context: .. - dockerfile: ./docker/compose.Dockerfile - args: - node: node#i - port: #port - ports: - - '#port:#port' - command: ./executor #i --ci-mode - environment: - - 'RUST_LOG=DEBUG' - networks: - intrnl: - ipv4_address: '#ip' - volumes: - - ../logs/node#i:/usr/src/myapp/artifacts/node#i/logs -"#; - -fn get_ip_by_i(index: usize) -> String { - format!("192.168.92.{}", index + 21) -} - -fn get_port_by_i(index: usize) -> String { - format!("{}", index + 3333) -} - -fn get_loop_body(size: usize) -> String { - let mut body = String::new(); - for i in 0..size { - body += &LOOP_BODY - .replace("#ip", &get_ip_by_i(i)) - .replace("#port", &get_port_by_i(i)) - .replace("#i", &i.to_string()); - } - body -} - -fn get_compose_file(size: usize) -> String { - format!("{}{}{}", START, get_loop_body(size), END) -} - -fn sizes() -> Vec { - let small: Vec = (1..4).map(|i| 3 * i + 1).collect(); - let big: Vec = (1..4).map(|i| 3 * 5 * i + 1).collect(); - let possible: Vec = (1..334).map(|i| 3 * i + 1).collect(); - let pow2: Vec = (1..7) - .filter_map(|i| { - let num = 2usize.pow(i as u32); - if possible.contains(&num) { - Some(num) - } else { - None - } - }) - .collect(); - - let mut sizes: Vec = small; - sizes.extend(big); - sizes.extend(pow2); - sizes.sort(); - - sizes -} - -fn save_compose_file(size: usize) { - let file_path = format!("../../composes/docker-compose.local{}.yml", size); - let mut file = File::create(&file_path).expect("Failed to create file"); - let compose_file = get_compose_file(size); - file.write_all(compose_file.as_bytes()) - .expect("Failed to write to file"); - println!("saved {}", file_path); -} - -fn main() { - let args: Vec = env::args().collect(); - - if args.len() < 2 { - println!("usage: cargo run "); - std::process::exit(1); - } - - let what_to_do = &args[1]; - - if what_to_do == "all" { - for i in sizes() { - save_compose_file(i); - } - std::process::exit(0); - } - - let size = match args[1].parse::() { - Ok(value) => value, - Err(_) => { - println!("Invalid size argument"); - std::process::exit(1); - } - }; - - let supported_sizes = (1..=67).step_by(3).collect::>(); - - if !supported_sizes.contains(&size) { - println!("size {} is not supported", size); - std::process::exit(1); - } - - save_compose_file(size); - std::process::exit(0); -} diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 75838ea9..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -# syntax=docker/dockerfile:experimental -FROM rust:1.71-buster as builder - -RUN apt update -RUN apt install -y clang - -WORKDIR /usr/src/zksync-bft/ -COPY . . - -WORKDIR /usr/src/zksync-bft/node/ -RUN cargo build --release --target-dir ./ - -FROM debian:buster-slim -EXPOSE 3333 -EXPOSE 3334 -EXPOSE 3335 -EXPOSE 3336 -COPY --from=builder --chmod=+x /usr/src/zksync-bft/node/release/executor /usr/bin/executor - -ENV RUST_LOG=DEBUG - -CMD ["/usr/bin/executor"] diff --git a/docker/compose.Dockerfile b/docker/compose.Dockerfile deleted file mode 100644 index b7c8d786..00000000 --- a/docker/compose.Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM rust:1.71 - -# Define a build-time argument called "node" -ARG node - -# Set the working directory to /usr/src/myapp/artifacts -# Directory structure: -# artifacts -# node0..4 -# config.toml -# private_key.toml -# executor -WORKDIR /usr/src/myapp/artifacts - -# Copy the contents of the node0..4 artifacts directory from the "localenvc" image -# into the current working directory's -COPY --from=localenvc /usr/src/myapp/node/artifacts/$node/* $node/ - -# Make the executor binary in the node0..4 directory executable -RUN chmod +x $node/executor - -# Expose public ports 3333, 3334, 3335, and 3336 where node tcp servers will be listening -EXPOSE $port - -# Set the working directory to the directory specified by the "node" argument -# it will be node0..4 -WORKDIR /usr/src/myapp/artifacts/$node/ -RUN mkdir /usr/src/myapp/artifacts/$node/logs/ - -# You can ignore this command. In docker-compose.yml file we have specified the different command -CMD ["./executor", "0"] diff --git a/docker/localenv.Dockerfile b/docker/localenv.Dockerfile deleted file mode 100644 index 426293ba..00000000 --- a/docker/localenv.Dockerfile +++ /dev/null @@ -1,61 +0,0 @@ -# Start from the official Rust image with version 1.71 -# and give it the alias "localenvc" -FROM rust:1.71 as localenvc - -ARG nodes - -WORKDIR /usr/src/myapp - -# Copy the current directory (.) into the working directory of the image -COPY . . - -# Install -# clang package required for rocksdb -# cmake package required for capnp -RUN apt update -RUN apt install -y clang -RUN apt install -y cmake - -# Build tools crate and create the config files -WORKDIR /usr/src/myapp/node -RUN cargo run -p tools --bin localnet_config -- --nodes=$nodes - -# Build binary file in release mode and create a main release binary -WORKDIR /usr/src/myapp/node -RUN cargo build -p tools --bin executor --release - -# Create the artifacts directory -WORKDIR /usr/src/myapp/node/ -RUN i=0; \ - while [ $i -lt $nodes ]; do \ - mkdir -p "artifacts/node$i"; \ - i=$(expr $i + 1); \ - done - -# Copy the binary file to the artifacts directory -RUN i=0; \ - while [ $i -lt $nodes ]; do \ - cp target/release/executor "artifacts/node$i/"; \ - i=$(expr $i + 1); \ - done - -# Copy the config file to the artifacts directory -RUN i=0; \ - while [ $i -lt $nodes ]; do \ - cp ../configs/localnet/node$i/* "artifacts/node$i/"; \ - i=$(expr $i + 1); \ - done - -# Check config files for each node -WORKDIR /usr/src/myapp/node/artifacts/ -RUN i=0; \ - while [ $i -lt $nodes ]; do \ - chmod +x "node$i/executor"; \ - cd "node$i/"; \ - ./executor $i --verify-config; \ - cd "../"; \ - i=$(expr $i + 1); \ - done - -# Report succes -CMD ["echo", "Done"] diff --git a/docs/architecture.md b/docs/architecture.md index 15fc4cd5..a9bbfc63 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -24,9 +24,9 @@ The library crates are just crates that have some basic functionality that can b - the `crypto` crate, which provides several cryptographic primitives. -- the `roles` crate, which implements the types necessary for each role in the network. We have just two roles: `Node` and `Validator`. +- the `protobuf` and `protbuf_build` crates, which contains all the code to create the protobuf schemas used by the other crates. -- the `schema` crate, which contains all the protobuf schemas used by the other crates. +- the `roles` crate, which implements the types necessary for each role in the network. We have just two roles: `Node` and `Validator`. - the `storage` crate is responsible for storing the current state of the system and providing an interface to access the stored data. It is a key component of the system that ensures the persistence of data and the ability to retrieve it when needed. @@ -36,8 +36,6 @@ The library crates are just crates that have some basic functionality that can b This section provides a physical map of folders & files in this repository. -- `/composes` & `/docker`: Project docker files. - - `/infrastructure`: Infrastructure scripts that are needed to test the zkSync Era Consensus Layer. - `/node` @@ -53,8 +51,9 @@ This section provides a physical map of folders & files in this repository. - `/concurrency`: Crate with essential primitives for structured concurrency. - `/crypto`: Cryptographic primitives used by the other crates. + - `/protobuf`: Code generated from protobuf schema files and utilities for serialization. + - `/protobuf_build`: Generates rust code from the proto files. - `/roles`: Essential types for the different node roles. - - `/schema`: Protobuf schemas used by the other crates. - `/storage`: Storage layer for the node. - `/utils`: Collection of small utilities. diff --git a/docs/launch.md b/docs/launch.md index 42940b7f..05f06c1b 100644 --- a/docs/launch.md +++ b/docs/launch.md @@ -12,18 +12,4 @@ sysctl -w net.ipv4.tcp_slow_start_after_idle=0 ## Running as an application -TBD - -## Running as a local testnet (OUTDATED) - -You can see localenv.Dockerfile where we build all local configurations and prepare binary file for node - -``` -DOCKER_BUILDKIT=1 docker image build --no-cache --build-arg nodes=16 . -f docker/localenv.Dockerfile -t localenvc -``` - -Using docker compose we start 16 nodes - -``` -docker compose -f composes/docker-compose.local16.yml up -``` +TBD \ No newline at end of file