Update various things

Cargo.toml

@@ -6,41 +6,39 @@ edition = "2018"
 
 # TODO: Go through dependencies and disable unused features to improve compile time
 [dependencies]
-env_logger = "0.7.1"
+env_logger = "0.8.1"
 log = "0.4.8"
 serde = { version = "1.0.98", features = ["derive"] }
 serde_json = "1.0.40"
 snafu = "0.6.0"
 clap = "2.33"
-crossbeam-channel = "0.4.0"
 chrono = "0.4.9"
 toml = "0.5.3"
 hex = "0.4.0"
 url = { version = "2.1.0", features = ["serde"] }
 
 # Web
-# TODO: Update `actix-rt` and `futures` once `actix-web` 2.0 is released
-actix-web = "1.0.5"
-actix-files = "0.1.6"
-actix-rt = "0.2.4"
-futures = "0.1.28"
+actix-web = "3.1.0"
+actix-files = "0.4.0"
+actix-rt = "1.1.1"
+futures = "0.3.6"
 
 # GitHub
-hubcaps = { git = "https://github.com/softprops/hubcaps.git" }
-# TODO: Update to 0.16 once jsonwebtoken v7 is released:
-# https://github.com/Keats/jsonwebtoken/pull/91
-ring = "0.14.6"
+hubcaps = "0.6.2"
+ring = "0.16.15"
 
 # OpenStack
-openstack = "0.3.0"
+openstack = "0.4.0"
 fallible-iterator = "0.2.0"
 waiter = "0.1.1"
 
 # Worker
-ssh2 = "0.6.0"
-reqwest = "0.9.20"
+ssh2 = "0.8.2"
+reqwest = { version = "0.10.8", features = ["blocking"] }
+crossbeam-utils = "0.8.0"
 
 # pcap
+# TODO: Other pcap library?
 pcap-file = "1.1.1"
 etherparse = "0.9.0"
 byteorder = "1.3.2"

Dockerfile

@@ -1,9 +1,9 @@
 FROM debian:buster
-EXPOSE 8080
+EXPOSE 9999
 VOLUME /config
 RUN apt-get update && apt-get --yes install python-openstackclient libssl1.1 ca-certificates && apt-get clean
 COPY target/release/ixy-ci /ixy-ci
-COPY runner/target/release/runner /runner-bin
+COPY runner-bin /runner-bin
 ENV RUST_BACKTRACE 1
 ENV RUST_LOG info,ixy_ci=trace
 CMD ["/ixy-ci", "--config", "/config/config.toml"]

Makefile

@@ -2,6 +2,9 @@
 start: config.toml runner-bin
 	env RUST_LOG=info,ixy_ci=trace cargo run --release
 
+.PHONY: runner-bin
 runner-bin:
-	cd runner && cargo build --release && cp target/release/runner ../runner-bin
+	cd runner && \
+		cargo build --target x86_64-unknown-linux-musl --release && \
+		cp target/x86_64-unknown-linux-musl/release/runner ../runner-bin
 

README.md

@@ -47,13 +47,15 @@ in the [ixy pktgen example](https://github.com/emmericp/ixy/blob/master/src/app/
 These instructions are only needed when you want to deploy your own instance of ixy-ci.
 
 ### OpenStack
+- ixy-ci assumes a dedicated OpenStack project with no other users.
 - Network `pktgen-fwd`
     - Port `pktgen`
     - Port `fwd-in`
 - Network `fwd-pcap`
     - Port `fwd-out`
     - Port `pcap`
-- Network require a default subnet for port creation to succeed
+- Networks require a subnet for port creation/attachment to succeed (disable DHCP)
+    - This may be incorrect since it seems to work without a subnet throught the `openstack` CLI
 - Disable "Port Security" on all ports
 - Create a keypair for ixy-ci to SSH into the spawned VMs
 
@@ -80,15 +82,56 @@ token_ (GitHub / Setting / Developer settings) with access to the `public_repo`
 ### Deploy with Docker
 
 ```
-cargo build --release
-cd runner; cargo build --release; cd -
+make # Ctrl+C once running
 docker build . -t ixy-ci
 docker volume create ixy-ci-config
 cp ~/.ssh/id_rsa /var/lib/docker/volumes/ixy-ci-config/_data/
 cp config.toml.example /var/lib/docker/volumes/ixy-ci-config/_data/config.toml
 docker run --mount source=ixy-ci-config,target=/config -p 127.0.0.1:9999:8080 --restart always -d --name ixy-ci ixy-ci
 ```
 
+## MicroStack
+For local development you can use [MicroStack](https://microstack.run/) which deploys OpenStack
+on your machine. (in a snap so without affecting the rest of your system too much)
+
+Here are complete instructions for getting up and running:
+```
+# Setup MicroStack
+sudo snap install microstack --devmode --beta
+sudo microstack.init --auto --control
+# Get password for "admin"; Web UI is at http://10.20.20.1
+sudo snap get microstack config.credentials.keystone-password
+
+# Import Debian 10 image
+wget https://cdimage.debian.org/cdimage/openstack/current-10/debian-10-openstack-amd64.qcow2
+microstack.openstack image create --container-format bare --disk-format qcow2 --file debian-10-openstack-amd64.qcow2 debian-10-openstack-amd64
+rm debian-10-openstack-amd64.qcow2
+
+# Create virtual networks
+microstack.openstack network create pktgen-fwd --disable-port-security
+microstack.openstack port create pktgen --network pktgen-fwd
+microstack.openstack port create fwd-in --network pktgen-fwd
+
+microstack.openstack network create fwd-pcap --disable-port-security
+microstack.openstack port create fwd-out --network fwd-pcap
+microstack.openstack port create pcap --network fwd-pcap
+
+# Import SSH public key
+microstack.openstack keypair create ixy-ci --public-key ~/.ssh/id_rsa.pub
+
+# Adjust config.toml:
+# Set the openstack password (see command above)
+# Set the GitHub API token
+# Adjust the path to the SSH private key
+
+# After a reboot you can start MicroStack again with:
+sudo snap start microstack
+
+# Remove MicroStack again when you're done
+sudo snap remove microstack --purge
+# Restart computer to also get rid of left over virtual interfaces etc.
+```
+
 ## TODO
 - Only allow configured users to start tests (to prevent abuse)
 - Do more stuff concurrently once async/await is ready (also trussh instead of libssh2)

config.toml.example

@@ -1,45 +1,49 @@
 bind_address = "0.0.0.0:9999"
-public_url = "https://ci.ixy.rs"
+public_url = "http://localhost:9999"
 # How many jobs can be in the queue at most
 job_queue_size = 10
 log_directory = "logs"
 
 [github]
 bot_name = "ixy-ci"
-api_token = "redacted"
+api_token = "INSERT_HERE"
 
 # Names are case-sensitive
 [github.webhook_secrets]
-"emmericp/ixy" = "redacted"
-"ixy-languages/ixy.rs" = "redacted"
+# "emmericp/ixy" = "redacted"
+# "ixy-languages/ixy.rs" = "redacted"
 
 [openstack]
 # VM flavor to be used
-flavor ="lrz.medium"
+flavor ="m1.medium"
 # VM base image to be used
-image ="Debian-10-buster"
+image ="debian-10-openstack-amd64"
 # SSH login name for the selected base image
 ssh_login = "debian"
+# Network which has access to the Internet (for fetching code etc.)
+internet_network = "test"
+# Pool from which floating IPs will be allocated for the spawned VMs
+floating_ip_pool = "external"
 # SSH keypair to be used
 keypair ="ixy-ci"
 # Path to the private key corresponding to the selected keypair
 private_key_path ="/home/ixy-ci/.ssh/id_rsa"
 
 # Exported from OpenStack (except project domain which must be queried manually)
-auth_url = "https://cc.lrz.de:5000/v3"
-user_name = "user"
-user_domain = "ADS"
-password = "redacted"
-project_name = "project"
-project_domain = "project_domain"
+auth_url = "http://10.20.20.1:5000/v3/"
+user_name = "admin"
+user_domain = "Default"
+password = "INSERT_HERE"
+project_name = "admin"
+project_domain = "Default"
 
 [test]
 # How many packets are captured for the test
-packets = 100_000
+packets = 500_000
 
 # PCI addresses of the VirtIO NICs
 [test.pci_addresses]
-pktgen = "0000:00:06.0"
-fwd_src = "0000:00:06.0"
-fwd_dst = "0000:00:07.0"
-pcap = "0000:00:06.0"
+pktgen = "0000:00:08.0"
+fwd_src = "0000:00:08.0"
+fwd_dst = "0000:00:09.0"
+pcap = "0000:00:08.0"

runner/src/main.rs

@@ -5,6 +5,9 @@ use std::sync::Arc;
 use std::time::Duration;
 use std::{env, thread};
 
+// Use the `x86_64-unknown-linux-musl` for a completely static binary (otherwise rustc may
+// dynamically stuff from newer glibc versions)
+
 fn main() -> Result<()> {
     let mut args = env::args().skip(1);
     match args.next() {

rustfmt.toml

@@ -0,0 +1 @@
+merge_imports = true

src/config.rs

@@ -1,8 +1,10 @@
-use std::collections::HashMap;
-use std::convert::TryFrom;
-use std::fmt::{self, Display, Formatter};
-use std::net::SocketAddr;
-use std::path::PathBuf;
+use std::{
+    collections::HashMap,
+    convert::TryFrom,
+    fmt::{self, Display, Formatter},
+    net::SocketAddr,
+    path::PathBuf,
+};
 
 use serde::Deserialize;
 use url::Url;
@@ -34,9 +36,11 @@ pub struct GitHubConfig {
 pub struct OpenStackConfig {
     pub flavor: String,
     pub image: String,
+    pub internet_network: String,
+    pub floating_ip_pool: String,
+    pub ssh_login: String,
     pub keypair: String,
     pub private_key_path: PathBuf,
-    pub ssh_login: String,
 
     // OpenStack API
     pub auth_url: String,