Merge branch 'main' into webapp

.gitignore

@@ -4,5 +4,7 @@ Cargo.lock
 
 stderr.txt
 stdout.txt
-
-node_modules/
+node_modules/
+/target
+rustup-init.exe
+src/main_worked.rs

Cargo.toml

@@ -0,0 +1,44 @@
+[workspace]
+resolver = "2"
+members = ["crankshaft", "tes", "wdl-runtime"]
+
+[workspace.dependencies]
+anyhow = "1.0.86"
+async-trait = "0.1.82"
+bollard = "0.17.1"
+bytes = "1.7.1"
+chrono = { version = "0.4.38", features = ["serde"] }
+clap = { version = "4.5.16", features = ["derive"] }
+config = "0.14.0"
+dirs = "5.0.1"
+futures = "0.3.30"
+id-arena = "2.2.1"
+indexmap = "2.5.0"
+indicatif = "0.17.8"
+nonempty = "0.10.0"
+ordered-float = "4.2.2"
+paste = "1.0.15"
+petgraph = "0.6.5"
+rand = "0.8.5"
+random_word = { version = "0.4.3", features = ["en"] }
+regex = "1.10.6"
+reqwest = "0.12.7"
+reqwest-middleware = "0.3.3"
+reqwest-retry = "0.6.1"
+serde = { version = "1.0.209", features = ["derive"] }
+serde_json = "1.0.128"
+serde_yaml = "0.8"                                                                 # Optional, if you want YAML support
+string-interner = "0.17.0"
+tar = "0.4.41"
+tempfile = "3.12.0"
+tes = { path = "../tes", version = "0.1.0" }
+tokio = { version = "1.40.0", features = ["full", "time"] }
+toml = "0.8.19"
+tracing = "0.1.40"
+tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+url = "2.5.2"
+wdl-analysis = { git = "https://github.com/peterhuene/wdl", branch = "hackathon" }
+wdl-ast = { git = "https://github.com/peterhuene/wdl", branch = "hackathon", features = ["codespan"] }
+wdl-grammar = { git = "https://github.com/peterhuene/wdl", branch = "hackathon", features = ["codespan"] }
+codespan-reporting = "0.11.1"
+colored = "2.1.0"

crankshaft/Cargo.toml

@@ -4,29 +4,39 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-async-trait = "0.1.82"
-bollard = "0.17.1"
-bytes = "1.7.1"
-clap = { version = "4.5.16", features = ["derive"] }
-config = "0.14.0"
-dirs = "5.0.1"
-futures = "0.3.30"
-indexmap = "2.5.0"
-nonempty = "0.10.0"
-paste = "1.0.15"
-rand = "0.8.5"
-random_word = { version = "0.4.3", features = ["en"] }
-regex = "1.10.6"
-reqwest = "0.12.7"
-serde = { version = "1.0.209", features = ["derive"] }
-serde_json = "1.0.128"
-tempfile = "3.12.0"
+indicatif = { workspace = true }
 tes = { path = "../tes", version = "0.1.0" }
-tokio = { version = "1.40.0", features = ["full", "time"] }
-toml = "0.8.19"
-tracing = "0.1.40"
-tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
-url = "2.5.2"
+wdl-runtime = { path = "../wdl-runtime", version = "0.1.0" }
+async-trait = { workspace = true }
+bollard = { workspace = true }
+bytes = { workspace = true }
+clap = { workspace = true }
+config = { workspace = true }
+dirs = { workspace = true }
+futures = { workspace = true }
+indexmap = { workspace = true }
+nonempty = { workspace = true }
+paste = { workspace = true }
+rand = { workspace = true }
+random_word = { workspace = true }
+regex = { workspace = true }
+reqwest = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+tempfile = { workspace = true }
+serde_yaml = { workspace = true }
+tokio = { workspace = true }
+toml = { workspace = true }
+tracing = { workspace = true }
+tracing-subscriber = { workspace = true }
+url = { workspace = true }
+wdl-grammar = { workspace = true }
+wdl-ast = { workspace = true }
+wdl-analysis = { workspace = true }
+anyhow = { workspace = true }
+tar.workspace = true
+codespan-reporting = { workspace = true }
+colored = { workspace = true }
 
 [lints.rust]
 missing_docs = "warn"

crankshaft/README.md

@@ -0,0 +1,92 @@
+# Crankshaft - A Simple Task Runner CLI
+
+Crankshaft is a command-line interface (CLI) tool designed to execute tasks defined in JSON or YAML files. It allows users to automate commands via the system shell, making it easy to run predefined scripts or repetitive tasks.
+
+## Features
+
+- Supports task definitions in both JSON and YAML formats.
+- Executes commands directly through the system's shell.
+- Compatible with Windows and Unix-like systems( Linux, macOS).
+
+## Requirements
+
+- Rust installed on your system. Install Rust from [rust-lang.org](https://www.rust-lang.org/).
+- A compatible shell available.
+
+## Setup
+
+### 1. Clone the Repository
+
+```
+git clone <https://github.com/stjude-biohackathon/KIDS24-team15.git>
+cd KIDS24-team15
+```
+### 2. Build the Project
+
+Compile the project using Cargo, Rust's package manager:
+
+```
+cargo build
+
+```
+### How to Use
+
+### 1. Create a Task File
+
+- Define your task in a JSON or YAML file. Each task file should include a name and a command to execute.
+
+- Example JSON file (example_task.json)
+
+{
+    "name": "Sample Task",
+    "command": "echo Hello, json file input!"
+}
+
+- Example YAML file (example_task.yaml)
+
+name: Sample Task
+command: echo Hello, yaml file!
+
+### 2 Run a Task
+
+- Use the following command to run a task defined in your JSON or YAML file:
+
+```
+cargo run --bin crankshaft -- <path_to_task_file>
+```
+
+- Example Command
+
+```
+cargo run --bin crankshaft -- ./example_task.json
+
+```
+### 3 Expected Output
+
+The command specified in the task file will be executed via the system shell.
+
+For the examples provided, the expected output should be "Hello, Json file input!". when using json file.
+
+### Troubleshooting
+- No Output: Verify that the command in your task file works independently in your terminal.
+- JSON/YAML Errors: Ensure that your task files are correctly formatted with valid JSON or YAML syntax.
+- Command Execution Errors: Check that the shell command is valid and your system environment is set up correctly.
+
+### References
+
+- 1. Rust programming language: <https://www.rust-lang.org/>
+- 2. Rust Libraries Used:
+	- Clap (Command-line Argument Parser) <https://github.com/clap-rs/clap> 
+	- Serde (Serialization and Deserialization) <https://github.com/serde-rs/serde>
+- 3. Workflow Description Language (WDL): <https://github.com/openwdl/wdl>
+
+### Contributors
+
+- Peter Huene <https://github.com/peterhuene> 
+- Clay McLeod <dhttps://github.com/claymcleod>
+- John McGuigan <https://github.com/jrm5100 >
+- Andrew Frantz <https://github.com/a-frantz>
+- Braden Everson <https://github.com/BradenEverson>
+- Jared Andrews <https://github.com/j-andrews7>
+- Michael Gattas <https://github.com/michaelgattas>
+- Suchitra Chavan <https://github.com/schavan023>

crankshaft/demo.json

@@ -0,0 +1,4 @@
+{
+    "url": "https://www.encodeproject.org/files/ENCFF863PGO/@@download/ENCFF863PGO.bam",
+    "count": 100000
+}

crankshaft/demo.wdl

@@ -0,0 +1,22 @@
+## This is a demonstration of evaluating WDL and running a task using crankshaft.
+
+version 1.2
+
+task samtools_flagstat {
+    input {
+        String url
+        Int count = 100000
+    }
+
+    command <<<
+        samtools flagstat <(wget -O - -q '~{url}' | samtools view -h | head -n ~{count})
+    >>>
+
+    requirements {
+        container: "quay.io/biocontainers/samtools:1.19.2--h50ea8bc_0"
+    }
+
+    output {
+        String stats = read_string(stdout())
+    }
+}

crankshaft/examples/docker.rs

@@ -1,8 +1,13 @@
 //! An example for runner a task using the Docker backend service.
+use std::io::Write;
 
+use crankshaft::engine::task::input;
 use crankshaft::engine::task::Execution;
+use crankshaft::engine::task::Input;
 use crankshaft::engine::Engine;
 use crankshaft::engine::Task;
+use tempfile::NamedTempFile;
+use tracing::info;
 use tracing_subscriber::fmt;
 use tracing_subscriber::layer::SubscriberExt as _;
 use tracing_subscriber::util::SubscriberInitExt as _;
@@ -17,31 +22,49 @@ async fn main() {
 
     let mut engine = Engine::default();
 
+    let mut temp_file = NamedTempFile::new().unwrap();
+    writeln!(temp_file, "Hello, world from an input").unwrap();
+
+    // Get the path to the temp file
+    let temp_path = temp_file.path().to_path_buf();
+    let input = Input::builder()
+        .contents(temp_path)
+        .path("/volA/test_input.txt")
+        .r#type(input::Type::File)
+        .try_build()
+        .unwrap();
+
     let task = Task::builder()
         .name("my-example-task")
-        .unwrap()
         .description("a longer description")
-        .unwrap()
-        .extend_executors(vec![Execution::builder()
-            .image("ubuntu")
-            .args(&[String::from("echo"), String::from("'hello, world!'")])
-            .stdout("stdout.txt")
-            .stderr("stderr.txt")
-            .try_build()
-            .unwrap()])
-        .unwrap()
+        .extend_inputs(vec![input])
+        .extend_executions(vec![
+            Execution::builder()
+                .image("ubuntu")
+                .args(&[
+                    String::from("bash"),
+                    String::from("-c"),
+                    String::from("ls /volA"),
+                ])
+                .try_build()
+                .unwrap(),
+            Execution::builder()
+                .image("ubuntu")
+                .args(&[String::from("cat"), String::from("/volA/test_input.txt")])
+                .try_build()
+                .unwrap(),
+        ])
         .extend_volumes(vec!["/volA".to_string(), "/volB".to_string()])
-        .unwrap()
         .try_build()
         .unwrap();
 
     let receivers = (0..10)
-        .map(|_| engine.submit(task.clone()).callback)
+        .map(|_| engine.submit("docker", task.clone()).callback)
         .collect::<Vec<_>>();
 
     engine.run().await;
 
     for rx in receivers {
-        println!("Reply: {:?}", rx.await.unwrap());
+        info!(runner = "Docker", reply = ?rx.await.unwrap());
     }
 }

crankshaft/examples/full.rs

@@ -0,0 +1,80 @@
+//! An example for runner that uses multiple backends at the same time.
+
+use crankshaft::engine::config::Config;
+use crankshaft::engine::service::runner::backend::config::BackendType;
+use crankshaft::engine::service::runner::backend::generic::GenericBackend;
+use crankshaft::engine::service::runner::backend::tes::TesBackend;
+use crankshaft::engine::task::Execution;
+use crankshaft::engine::Engine;
+use crankshaft::engine::Task;
+use tracing::info;
+use tracing_subscriber::fmt;
+use tracing_subscriber::layer::SubscriberExt as _;
+use tracing_subscriber::util::SubscriberInitExt as _;
+use tracing_subscriber::EnvFilter;
+
+/// The environment variable name for the token.
+const TOKEN_ENV_NAME: &str = "TOKEN";
+
+#[tokio::main]
+async fn main() {
+    tracing_subscriber::registry()
+        .with(fmt::layer())
+        .with(EnvFilter::from_default_env())
+        .init();
+
+    let token = std::env::var(TOKEN_ENV_NAME).ok();
+
+    let url = std::env::args().nth(1).expect("no url provided");
+    let config = Config::new(std::env::args().nth(2).expect("no config provided"))
+        .expect("could not load from config file")
+        .backends
+        .into_iter()
+        .find(|backend| matches!(backend.kind, BackendType::Generic(_)))
+        .expect("at least one generic backend config to be present in the config");
+
+    let mut engine = Engine::empty()
+        .with_docker(false)
+        .expect("docker daemon to be alive and reachable")
+        .with_backend("tes", TesBackend::new(url, token))
+        .with_backend(
+            "lsf",
+            GenericBackend::try_from(config)
+                .expect("parsing the backend configuration")
+                .to_runner(),
+        );
+
+    let task = Task::builder()
+        .name("my-example-task")
+        .description("a longer description")
+        .extend_executions(vec![Execution::builder()
+            .image("ubuntu")
+            .args(&[String::from("echo"), String::from("'hello, world!'")])
+            .stdout("stdout.txt")
+            .stderr("stderr.txt")
+            .try_build()
+            .unwrap()])
+        .try_build()
+        .unwrap();
+
+    let mut receivers = Vec::new();
+    let runners = engine.runners().map(|s| s.to_owned()).collect::<Vec<_>>();
+
+    for runner in &runners {
+        if runner == "lsf" {
+            continue;
+        }
+
+        info!("creating jobs within {runner}");
+
+        for _ in 0..10 {
+            receivers.push(engine.submit(runner, task.clone()).callback);
+        }
+    }
+
+    engine.run().await;
+
+    for rx in receivers {
+        info!(reply = ?rx.await.unwrap());
+    }
+}