From 26f5078b0988b456d566d23cd2c7db25d9b4c1de Mon Sep 17 00:00:00 2001 From: Conor Griffin Date: Wed, 21 Feb 2018 13:27:58 +0000 Subject: [PATCH 01/19] Change worker to report task id with result. --- master/src/common/task.rs | 28 +++++++++++++++++++++++++-- master/src/worker_management/state.rs | 7 +++++++ proto/worker.proto | 4 ++++ worker/src/operations/map.rs | 6 ++++++ worker/src/operations/reduce.rs | 8 ++++++++ 5 files changed, 51 insertions(+), 2 deletions(-) diff --git a/master/src/common/task.rs b/master/src/common/task.rs index 8b6333b2..2dda72a8 100644 --- a/master/src/common/task.rs +++ b/master/src/common/task.rs @@ -66,14 +66,17 @@ impl Task { let mut map_task_input = pb::MapTaskInput::new(); map_task_input.set_input_locations(RepeatedField::from_vec(input_locations)); + let id = Uuid::new_v4().to_string(); + let mut map_request = pb::PerformMapRequest::new(); map_request.set_input(map_task_input); map_request.set_mapper_file_path(binary_path.into()); + map_request.set_task_id(id.clone()); Task { task_type: TaskType::Map, job_id: job_id.into(), - id: Uuid::new_v4().to_string(), + id: id, map_request: Some(map_request), reduce_request: None, @@ -132,6 +135,8 @@ impl Task { input_files: Vec, output_directory: S, ) -> Self { + let id = Uuid::new_v4().to_string(); + let mut reduce_request = pb::PerformReduceRequest::new(); reduce_request.set_partition(input_partition); for input_file in input_files { @@ -139,11 +144,12 @@ impl Task { } reduce_request.set_reducer_file_path(binary_path.into()); reduce_request.set_output_directory(output_directory.into()); + reduce_request.set_task_id(id.clone()); Task { task_type: TaskType::Reduce, job_id: job_id.into(), - id: Uuid::new_v4().to_string(), + id: id, map_request: None, reduce_request: Some(reduce_request), @@ -255,6 +261,24 @@ impl StateHandling for Task { self.id = serde_json::from_value(data["id"].clone()).chain_err( || "Unable to convert id", )?; + + match self.task_type { + TaskType::Map => { + let mut map_request = self.map_request.clone().chain_err( + || "Map Request should exist", + )?; + map_request.task_id = self.id.clone(); + self.map_request = Some(map_request); + } + TaskType::Reduce => { + let mut reduce_request = self.reduce_request.clone().chain_err( + || "Reduce Request should exist", + )?; + reduce_request.task_id = self.id.clone(); + self.reduce_request = Some(reduce_request); + } + } + self.map_output_files = serde_json::from_value(data["map_output_files"].clone()) .chain_err(|| "Unable to convert map_output_files")?; self.assigned_worker_id = serde_json::from_value(data["assigned_worker_id"].clone()) diff --git a/master/src/worker_management/state.rs b/master/src/worker_management/state.rs index 1081c945..171ae83d 100644 --- a/master/src/worker_management/state.rs +++ b/master/src/worker_management/state.rs @@ -120,6 +120,10 @@ impl State { ) })?; + if scheduled_task.id != reduce_result.task_id { + return Err("Task id does not match expected task id.".into()); + } + scheduled_task.status = TaskStatus::Complete; scheduled_task.cpu_time = reduce_result.get_cpu_time(); return Ok(scheduled_task); @@ -142,6 +146,9 @@ impl State { ) })?; + if scheduled_task.id != map_result.task_id { + return Err("Task id does not match expected task id.".into()); + } let worker = self.workers.get_mut(&map_result.worker_id).chain_err(|| { format!("Worker with ID {} not found.", map_result.worker_id) diff --git a/proto/worker.proto b/proto/worker.proto index 1120c4c2..cb629311 100644 --- a/proto/worker.proto +++ b/proto/worker.proto @@ -71,6 +71,7 @@ message MapResult { uint64 cpu_time = 4; // Details on the error if the status is failure string failure_details = 5; + string task_id = 6; } message ReduceResult { @@ -81,6 +82,7 @@ message ReduceResult { uint64 cpu_time = 3; // Details on the error if the status is failure string failure_details = 4; + string task_id = 5; } // ScheduleOperationService is used by the master to schedule Map and Reduce operations on the @@ -118,6 +120,7 @@ message PerformMapRequest { MapTaskInput input = 1; // The path to the binary that performs the map operation. string mapper_file_path = 2; + string task_id = 3; } message PerformReduceRequest { @@ -130,6 +133,7 @@ message PerformReduceRequest { string reducer_file_path = 3; // The path to the output directory. string output_directory = 4; + string task_id = 5; } //////////////////////////////////////////////////////////////////////////////// diff --git a/worker/src/operations/map.rs b/worker/src/operations/map.rs index 534affd7..02e83945 100644 --- a/worker/src/operations/map.rs +++ b/worker/src/operations/map.rs @@ -154,6 +154,7 @@ fn combine_map_results( master_interface_arc: &Arc, initial_cpu_time: u64, output_dir: &str, + task_id: &str, ) -> Result<()> { let partition_map; { @@ -187,6 +188,7 @@ fn combine_map_results( map_result.set_map_results(map_results); map_result.set_status(pb::ResultStatus::SUCCESS); map_result.set_cpu_time(operation_handler::get_cpu_time() - initial_cpu_time); + map_result.set_task_id(task_id.to_owned()); if let Err(err) = send_map_result(master_interface_arc, map_result) { log_map_operation_err(err, operation_state_arc); @@ -226,6 +228,7 @@ fn process_map_result( master_interface_arc: &Arc, initial_cpu_time: u64, output_dir: &str, + task_id: &str, ) { { // The number of operations waiting will be 0 if we have already processed one @@ -276,6 +279,7 @@ fn process_map_result( master_interface_arc, initial_cpu_time, output_dir, + task_id, ); if let Err(err) = result { @@ -361,6 +365,7 @@ fn internal_perform_map( let operation_state_arc_clone = Arc::clone(operation_state_arc); let master_interface_arc_clone = Arc::clone(master_interface_arc); let output_path_str: String = (*output_path.to_string_lossy()).to_owned(); + let task_id = map_options.task_id.clone(); thread::spawn(move || { let result = map_operation_thread_impl(&map_input_document, child); @@ -371,6 +376,7 @@ fn internal_perform_map( &master_interface_arc_clone, initial_cpu_time, &output_path_str, + &task_id, ); }); } diff --git a/worker/src/operations/reduce.rs b/worker/src/operations/reduce.rs index 49476f87..03e1d770 100644 --- a/worker/src/operations/reduce.rs +++ b/worker/src/operations/reduce.rs @@ -40,6 +40,7 @@ struct ReduceInput { struct ReduceOptions { reducer_file_path: String, output_directory: String, + task_id: String, } /// `ReduceOperationQueue` is a struct used for storing and executing a queue of `ReduceOperations`. @@ -269,6 +270,7 @@ fn internal_perform_reduce( let reduce_options = ReduceOptions { reducer_file_path: reduce_request.get_reducer_file_path().to_owned(), output_directory: reduce_request.get_output_directory().to_owned(), + task_id: reduce_request.get_task_id().to_owned(), }; run_reduce_queue( @@ -287,11 +289,13 @@ fn handle_reduce_queue_finished( operation_state_arc: &Arc>, master_interface_arc: &Arc, initial_cpu_time: u64, + task_id: &str, ) { if let Some(err) = reduce_err { let mut response = pb::ReduceResult::new(); response.set_status(pb::ResultStatus::FAILURE); response.set_failure_details(operation_handler::failure_details_from_error(&err)); + response.set_task_id(task_id.to_owned()); let result = send_reduce_result(master_interface_arc, response); if let Err(err) = result { @@ -303,6 +307,8 @@ fn handle_reduce_queue_finished( let mut response = pb::ReduceResult::new(); response.set_status(pb::ResultStatus::SUCCESS); response.set_cpu_time(operation_handler::get_cpu_time() - initial_cpu_time); + response.set_task_id(task_id.to_owned()); + let result = send_reduce_result(master_interface_arc, response); match result { @@ -353,6 +359,7 @@ fn run_reduce_queue( &operation_state_arc, &master_interface_arc, initial_cpu_time, + &reduce_options.task_id, ); }); } @@ -391,6 +398,7 @@ mod tests { let reduce_options = ReduceOptions { output_directory: "foo".to_owned(), reducer_file_path: "bar".to_owned(), + task_id: "task1".to_owned(), }; let data_abstraction_layer: Arc = From 69a17c290916e52bb9fd8c0111e650b522945977 Mon Sep 17 00:00:00 2001 From: Conor Griffin Date: Wed, 21 Feb 2018 16:27:41 +0000 Subject: [PATCH 02/19] Add task start time and end time. --- master/src/common/task.rs | 41 ++++++++++++ master/src/main.rs | 4 +- master/src/scheduling/mod.rs | 2 +- master/src/scheduling/scheduler.rs | 66 +++++++++---------- master/src/scheduling/state.rs | 51 +++++++++++++- master/src/worker_management/state.rs | 15 ++--- .../src/worker_management/worker_manager.rs | 36 +++++----- 7 files changed, 149 insertions(+), 66 deletions(-) diff --git a/master/src/common/task.rs b/master/src/common/task.rs index 2dda72a8..44743969 100644 --- a/master/src/common/task.rs +++ b/master/src/common/task.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use protobuf::repeated::RepeatedField; +use chrono::prelude::*; use serde_json; use uuid::Uuid; @@ -55,6 +56,9 @@ pub struct Task { // Number of times this task has failed in the past. pub failure_count: u16, pub failure_details: Option, + + pub time_started: Option>, + pub time_completed: Option>, } impl Task { @@ -90,6 +94,9 @@ impl Task { failure_count: 0, failure_details: None, + + time_started: None, + time_completed: None, } } @@ -163,6 +170,9 @@ impl Task { failure_count: 0, failure_details: None, + + time_started: None, + time_completed: None, } } @@ -244,6 +254,15 @@ impl StateHandling for Task { } }; + let time_started = match self.time_started { + Some(timestamp) => timestamp.timestamp(), + None => -1, + }; + let time_completed = match self.time_completed { + Some(timestamp) => timestamp.timestamp(), + None => -1, + }; + Ok(json!({ "task_type": self.task_type, "request": request, @@ -254,6 +273,8 @@ impl StateHandling for Task { "status": self.status, "failure_count": self.failure_count, "failure_details": self.failure_details, + "time_started": time_started, + "time_completed": time_completed, })) } @@ -291,6 +312,26 @@ impl StateHandling for Task { self.failure_details = serde_json::from_value(data["failure_details"].clone()) .chain_err(|| "Unable to convert failure_details")?; + let time_started: i64 = serde_json::from_value(data["time_started"].clone()) + .chain_err(|| "Unable to convert time_started")?; + self.time_started = match time_started { + -1 => None, + _ => Some(DateTime::::from_utc( + NaiveDateTime::from_timestamp(time_started, 0), + Utc, + )), + }; + + let time_completed: i64 = serde_json::from_value(data["time_completed"].clone()) + .chain_err(|| "Unable to convert time_completed")?; + self.time_completed = match time_completed { + -1 => None, + _ => Some(DateTime::::from_utc( + NaiveDateTime::from_timestamp(time_completed, 0), + Utc, + )), + }; + Ok(()) } } diff --git a/master/src/main.rs b/master/src/main.rs index 6cb2511e..02425e74 100644 --- a/master/src/main.rs +++ b/master/src/main.rs @@ -47,7 +47,7 @@ use std::path::Path; use std::str::FromStr; use errors::*; -use scheduling::{TaskProcessorImpl, Scheduler, run_task_result_loop}; +use scheduling::{TaskProcessorImpl, Scheduler, run_task_update_loop}; use util::init_logger; use util::data_layer::{AbstractionLayer, NullAbstractionLayer, NFSAbstractionLayer}; use worker_communication::WorkerInterfaceImpl; @@ -121,7 +121,7 @@ fn run() -> Result<()> { run_health_check_loop(Arc::clone(&worker_manager)); // Startup scheduler loop - run_task_result_loop( + run_task_update_loop( Arc::clone(&map_reduce_scheduler), &Arc::clone(&worker_manager), ); diff --git a/master/src/scheduling/mod.rs b/master/src/scheduling/mod.rs index 2d9d688a..2350fc12 100644 --- a/master/src/scheduling/mod.rs +++ b/master/src/scheduling/mod.rs @@ -1,5 +1,5 @@ pub use self::scheduler::Scheduler; -pub use self::scheduler::run_task_result_loop; +pub use self::scheduler::run_task_update_loop; pub use self::task_processor::TaskProcessor; pub use self::task_processor::TaskProcessorImpl; diff --git a/master/src/scheduling/scheduler.rs b/master/src/scheduling/scheduler.rs index e863f102..1fad1341 100644 --- a/master/src/scheduling/scheduler.rs +++ b/master/src/scheduling/scheduler.rs @@ -2,11 +2,9 @@ use std::thread; use std::collections::HashMap; use std::sync::{Mutex, Arc}; -use chrono::Utc; use serde_json; -use cerberus_proto::mapreduce as pb; -use common::{Task, Job}; +use common::{Task, TaskStatus, Job}; use errors::*; use scheduling::state::{ScheduledJob, State}; use scheduling::task_processor::TaskProcessor; @@ -85,6 +83,30 @@ impl Scheduler { Ok(()) } + fn process_in_progress_task(&self, task: &Task) -> Result<()> { + let mut state = self.state.lock().unwrap(); + state + .update_job_started( + &task.job_id, + task.time_started.chain_err( + || "Time started expected to exist.", + )?, + ) + .chain_err(|| "Error updating job start time.")?; + + state.update_task(task.to_owned()).chain_err( + || "Error updating task info.", + ) + } + + pub fn process_updated_task(&self, task: &Task) -> Result<()> { + if task.status == TaskStatus::Complete || task.status == TaskStatus::Failed { + self.process_completed_task(task) + } else { + self.process_in_progress_task(task) + } + } + /// Schedule a [`Task`](common::Task) to be executed. fn schedule_task(&self, task: Task) { let worker_manager = Arc::clone(&self.worker_manager); @@ -107,8 +129,6 @@ impl Scheduler { } info!("Starting job with ID {}.", job.id); - job.status = pb::Status::IN_PROGRESS; - job.time_started = Some(Utc::now()); let scheduled_job = ScheduledJob { job: job, @@ -123,7 +143,7 @@ impl Scheduler { pub fn get_job_queue_size(&self) -> u32 { let state = self.state.lock().unwrap(); - state.get_in_progress_job_count() + state.get_job_queue_size() } // Returns a count of workers registered with the master. @@ -131,39 +151,17 @@ impl Scheduler { self.worker_manager.get_available_workers() } - /// `modify_job_status` modifies a job status to IN_QUEUE if no progress has been made on it - /// yet. - fn modify_job_status(&self, job: &mut Job) { - if job.status == pb::Status::IN_PROGRESS && job.map_tasks_completed == 0 && - !self.worker_manager.is_job_in_progress(&job.id) - { - // If the job is not in progress, it's status should be queued. - // This check is needed because the scheduler activates all jobs immediately. - job.status = pb::Status::IN_QUEUE; - } - } - pub fn get_mapreduce_status(&self, mapreduce_id: &str) -> Result { let state = self.state.lock().unwrap(); - let mut job = state.get_job(mapreduce_id).chain_err( + state.get_job(mapreduce_id).chain_err( || "Error getting map reduce status.", - )?; - - self.modify_job_status(&mut job); - - Ok(job) + ) } /// `get_mapreduce_client_status` returns a vector of `Job`s for a given client. pub fn get_mapreduce_client_status(&self, client_id: &str) -> Vec { let state = self.state.lock().unwrap(); - let mut jobs = state.get_jobs(client_id); - - for mut job in &mut jobs { - self.modify_job_status(&mut job); - } - - jobs + state.get_jobs(client_id) } } @@ -191,14 +189,14 @@ impl state::SimpleStateHandling for Scheduler { } -pub fn run_task_result_loop(scheduler: Arc, worker_manager: &Arc) { - let reciever = worker_manager.get_result_reciever(); +pub fn run_task_update_loop(scheduler: Arc, worker_manager: &Arc) { + let reciever = worker_manager.get_update_reciever(); thread::spawn(move || loop { let receiver = reciever.lock().unwrap(); match receiver.recv() { Err(e) => error!("Error processing task results: {}", e), Ok(task) => { - let result = scheduler.process_completed_task(&task); + let result = scheduler.process_updated_task(&task); if let Err(e) = result { output_error(&e); } diff --git a/master/src/scheduling/state.rs b/master/src/scheduling/state.rs index a177f094..0d89ddeb 100644 --- a/master/src/scheduling/state.rs +++ b/master/src/scheduling/state.rs @@ -100,6 +100,27 @@ impl State { Ok(()) } + pub fn update_job_started(&mut self, job_id: &str, time_started: DateTime) -> Result<()> { + let scheduled_job = match self.scheduled_jobs.get_mut(job_id) { + Some(scheduled_job) => scheduled_job, + None => return Err(format!("Job with ID {} is not found.", &job_id).into()), + }; + + if scheduled_job.job.status == pb::Status::IN_QUEUE { + scheduled_job.job.status = pb::Status::IN_PROGRESS; + } + + if let Some(job_time_started) = scheduled_job.job.time_started { + if time_started < job_time_started { + scheduled_job.job.time_started = Some(time_started); + } + } else { + scheduled_job.job.time_started = Some(time_started); + } + + Ok(()) + } + pub fn get_job(&self, job_id: &str) -> Result { match self.scheduled_jobs.get(job_id) { Some(scheduled_job) => Ok(scheduled_job.job.clone()), @@ -171,6 +192,23 @@ impl State { Ok(()) } + pub fn update_task(&mut self, task: Task) -> Result<()> { + let scheduled_job = match self.scheduled_jobs.get_mut(&task.job_id) { + Some(scheduled_job) => scheduled_job, + None => return Err(format!("Job with ID {} is not found.", task.job_id).into()), + }; + + if !scheduled_job.tasks.contains_key(&task.id) { + return Err( + format!("Task with ID {} is does not exist.", &task.id).into(), + ); + } + + scheduled_job.tasks.insert(task.id.to_owned(), task); + + Ok(()) + } + // Returns if reduce tasks are required for a given job. pub fn reduce_tasks_required(&self, job_id: &str) -> Result { let scheduled_job = match self.scheduled_jobs.get(job_id) { @@ -186,6 +224,13 @@ impl State { // Adds the information for a completed task and updates the job. pub fn add_completed_task(&mut self, task: Task) -> Result<()> { + self.update_job_started( + &task.job_id, + task.time_started.chain_err( + || "Time started is expected to exist.", + )?, + ).chain_err(|| "Error adding completed task.")?; + let scheduled_job = match self.scheduled_jobs.get_mut(&task.job_id) { Some(scheduled_job) => scheduled_job, None => return Err(format!("Job with ID {} is not found.", &task.job_id).into()), @@ -219,10 +264,12 @@ impl State { Ok(()) } - pub fn get_in_progress_job_count(&self) -> u32 { + pub fn get_job_queue_size(&self) -> u32 { let mut job_count = 0; for scheduled_job in self.scheduled_jobs.values() { - if scheduled_job.job.status == pb::Status::IN_PROGRESS { + if scheduled_job.job.status == pb::Status::IN_PROGRESS || + scheduled_job.job.status == pb::Status::IN_QUEUE + { job_count += 1; } } diff --git a/master/src/worker_management/state.rs b/master/src/worker_management/state.rs index 171ae83d..7fe8a498 100644 --- a/master/src/worker_management/state.rs +++ b/master/src/worker_management/state.rs @@ -40,15 +40,6 @@ impl State { workers } - pub fn is_job_in_progress(&self, job_id: &str) -> bool { - for task in self.assigned_tasks.values() { - if task.job_id == job_id { - return true; - } - } - false - } - // Returns a list of worker not currently assigned a task sorted by most recent health checks. pub fn get_available_workers(&self) -> Vec { let mut workers = Vec::new(); @@ -125,6 +116,7 @@ impl State { } scheduled_task.status = TaskStatus::Complete; + scheduled_task.time_completed = Some(Utc::now()); scheduled_task.cpu_time = reduce_result.get_cpu_time(); return Ok(scheduled_task); } @@ -165,6 +157,7 @@ impl State { ); } scheduled_task.status = TaskStatus::Complete; + scheduled_task.time_completed = Some(Utc::now()); scheduled_task.cpu_time = map_result.get_cpu_time(); return Ok(scheduled_task); @@ -195,6 +188,7 @@ impl State { if assigned_task.failure_count > MAX_TASK_FAILURE_COUNT { assigned_task.status = TaskStatus::Failed; + assigned_task.time_completed = Some(Utc::now()); } else { assigned_task.status = TaskStatus::Queued; self.task_queue.push_front(assigned_task.clone()); @@ -271,6 +265,9 @@ impl State { }; scheduled_task.status = TaskStatus::InProgress; + if scheduled_task.time_started == None { + scheduled_task.time_started = Some(Utc::now()); + } scheduled_task.assigned_worker_id = worker.worker_id.to_owned(); let task = scheduled_task.clone(); diff --git a/master/src/worker_management/worker_manager.rs b/master/src/worker_management/worker_manager.rs index e99ecadd..d9b148eb 100644 --- a/master/src/worker_management/worker_manager.rs +++ b/master/src/worker_management/worker_manager.rs @@ -30,8 +30,8 @@ pub struct WorkerManager { state: Arc>, worker_interface: Arc, - task_result_sender: Mutex>, - task_result_reciever: Arc>>, + task_update_sender: Mutex>, + task_update_reciever: Arc>>, } impl WorkerManager { @@ -41,13 +41,13 @@ impl WorkerManager { state: Arc::new(Mutex::new(State::new())), worker_interface: worker_interface, - task_result_sender: Mutex::new(sender), - task_result_reciever: Arc::new(Mutex::new(receiver)), + task_update_sender: Mutex::new(sender), + task_update_reciever: Arc::new(Mutex::new(receiver)), } } - pub fn get_result_reciever(&self) -> Arc>> { - Arc::clone(&self.task_result_reciever) + pub fn get_update_reciever(&self) -> Arc>> { + Arc::clone(&self.task_update_reciever) } // Returns a count of workers registered with the master. @@ -56,12 +56,6 @@ impl WorkerManager { state.get_worker_count() } - // Returns if a there are any tasks assigned to workers for a given job. - pub fn is_job_in_progress(&self, job_id: &str) -> bool { - let state = self.state.lock().unwrap(); - state.is_job_in_progress(job_id) - } - pub fn register_worker(&self, worker: Worker) -> Result<()> { let mut state = self.state.lock().unwrap(); @@ -99,8 +93,8 @@ impl WorkerManager { ); if task.status == TaskStatus::Complete || task.status == TaskStatus::Failed { - let task_result_sender = self.task_result_sender.lock().unwrap(); - task_result_sender.send(task).chain_err( + let task_update_sender = self.task_update_sender.lock().unwrap(); + task_update_sender.send(task).chain_err( || "Error processing reduce result.", )?; } @@ -127,8 +121,8 @@ impl WorkerManager { ); if task.status == TaskStatus::Complete || task.status == TaskStatus::Failed { - let task_result_sender = self.task_result_sender.lock().unwrap(); - task_result_sender.send(task).chain_err( + let task_update_sender = self.task_update_sender.lock().unwrap(); + task_update_sender.send(task).chain_err( || "Error processing map result.", )?; } @@ -274,9 +268,15 @@ impl WorkerManager { if let Some(task) = task_option { let task_assignment_future = match task.task_type { - TaskType::Map => self.assign_map_task(worker_id, task), - TaskType::Reduce => self.assign_reduce_task(worker_id, task), + TaskType::Map => self.assign_map_task(worker_id, task.clone()), + TaskType::Reduce => self.assign_reduce_task(worker_id, task.clone()), }; + + let task_update_sender = self.task_update_sender.lock().unwrap(); + if let Err(err) = task_update_sender.send(task) { + error!("Error updating scheduler on task status: {}", err); + } + task_assignment_futures.push(task_assignment_future); } else { break; From 533b0419bcbb6d928ad9e2710d5d514dcf52d6b2 Mon Sep 17 00:00:00 2001 From: Conor Griffin Date: Wed, 21 Feb 2018 19:13:46 +0000 Subject: [PATCH 03/19] Reduce number of arguments to worker functions. --- worker/src/operations/map.rs | 100 +++++++-------------- worker/src/operations/operation_handler.rs | 41 ++++----- worker/src/operations/reduce.rs | 58 +++++------- 3 files changed, 74 insertions(+), 125 deletions(-) diff --git a/worker/src/operations/map.rs b/worker/src/operations/map.rs index 02e83945..d5812f2c 100644 --- a/worker/src/operations/map.rs +++ b/worker/src/operations/map.rs @@ -15,9 +15,9 @@ use cerberus_proto::worker as pb; use master_interface::MasterInterface; use super::io; use super::operation_handler; +use super::operation_handler::OperationResources; use super::state::OperationState; use util::output_error; -use util::data_layer::AbstractionLayer; const WORKER_OUTPUT_DIRECTORY: &str = "/tmp/cerberus/"; @@ -102,13 +102,11 @@ fn map_operation_thread_impl( pub fn perform_map( map_options: &pb::PerformMapRequest, - operation_state_arc: &Arc>, - master_interface_arc: &Arc, - data_abstraction_layer_arc: &Arc, + resources: &OperationResources, output_dir_uuid: &str, ) -> Result<()> { { - let mut operation_state = operation_state_arc.lock().unwrap(); + let mut operation_state = resources.operation_state.lock().unwrap(); operation_state.initial_cpu_time = operation_handler::get_cpu_time(); } @@ -128,21 +126,15 @@ pub fn perform_map( debug!("Input files: {:?}", input_files); - if operation_handler::get_worker_status(operation_state_arc) == pb::WorkerStatus::BUSY { + if operation_handler::get_worker_status(&resources.operation_state) == pb::WorkerStatus::BUSY { warn!("Map operation requested while worker is busy"); return Err("Worker is busy.".into()); } - operation_handler::set_busy_status(operation_state_arc); - - let result = internal_perform_map( - map_options, - operation_state_arc, - master_interface_arc, - data_abstraction_layer_arc, - output_dir_uuid, - ); + operation_handler::set_busy_status(&resources.operation_state); + + let result = internal_perform_map(map_options, resources, output_dir_uuid); if let Err(err) = result { - log_map_operation_err(err, operation_state_arc); + log_map_operation_err(err, &resources.operation_state); return Err("Error starting map operation.".into()); } @@ -150,15 +142,14 @@ pub fn perform_map( } fn combine_map_results( - operation_state_arc: &Arc>, - master_interface_arc: &Arc, + resources: &OperationResources, initial_cpu_time: u64, output_dir: &str, task_id: &str, ) -> Result<()> { let partition_map; { - let operation_state = operation_state_arc.lock().unwrap(); + let operation_state = resources.operation_state.lock().unwrap(); partition_map = operation_state.intermediate_map_results.clone(); } @@ -180,7 +171,7 @@ fn combine_map_results( } { - let mut operation_state = operation_state_arc.lock().unwrap(); + let mut operation_state = resources.operation_state.lock().unwrap(); operation_state.add_intermediate_files(intermediate_files); } @@ -190,24 +181,19 @@ fn combine_map_results( map_result.set_cpu_time(operation_handler::get_cpu_time() - initial_cpu_time); map_result.set_task_id(task_id.to_owned()); - if let Err(err) = send_map_result(master_interface_arc, map_result) { - log_map_operation_err(err, operation_state_arc); + if let Err(err) = send_map_result(&resources.master_interface, map_result) { + log_map_operation_err(err, &resources.operation_state); } else { info!("Map operation completed sucessfully."); - operation_handler::set_complete_status(operation_state_arc); + operation_handler::set_complete_status(&resources.operation_state); } Ok(()) } -fn process_map_operation_error( - err: Error, - operation_state_arc: &Arc>, - master_interface_arc: &Arc, - initial_cpu_time: u64, -) { +fn process_map_operation_error(err: Error, resources: &OperationResources, initial_cpu_time: u64) { { - let mut operation_state = operation_state_arc.lock().unwrap(); + let mut operation_state = resources.operation_state.lock().unwrap(); operation_state.waiting_map_operations = 0; } @@ -216,16 +202,15 @@ fn process_map_operation_error( map_result.set_cpu_time(operation_handler::get_cpu_time() - initial_cpu_time); map_result.set_failure_details(operation_handler::failure_details_from_error(&err)); - if let Err(err) = send_map_result(master_interface_arc, map_result) { + if let Err(err) = send_map_result(&resources.master_interface, map_result) { error!("Could not send map operation failed: {}", err); } - log_map_operation_err(err, operation_state_arc); + log_map_operation_err(err, &resources.operation_state); } fn process_map_result( result: Result, - operation_state_arc: &Arc>, - master_interface_arc: &Arc, + resources: &OperationResources, initial_cpu_time: u64, output_dir: &str, task_id: &str, @@ -233,7 +218,7 @@ fn process_map_result( { // The number of operations waiting will be 0 if we have already processed one // that has failed. - let operation_state = operation_state_arc.lock().unwrap(); + let operation_state = resources.operation_state.lock().unwrap(); if operation_state.waiting_map_operations == 0 { return; } @@ -244,7 +229,7 @@ fn process_map_result( let (finished, parse_failed) = { let mut parse_failed = false; - let mut operation_state = operation_state_arc.lock().unwrap(); + let mut operation_state = resources.operation_state.lock().unwrap(); for (partition, value) in map_result { let vec = operation_state .intermediate_map_results @@ -269,36 +254,19 @@ fn process_map_result( Error::from_kind(ErrorKind::Msg( "Failed to parse map operation output".to_owned(), )), - operation_state_arc, - master_interface_arc, + resources, initial_cpu_time, ); } else if finished { - let result = combine_map_results( - operation_state_arc, - master_interface_arc, - initial_cpu_time, - output_dir, - task_id, - ); + let result = combine_map_results(resources, initial_cpu_time, output_dir, task_id); if let Err(err) = result { - process_map_operation_error( - err, - operation_state_arc, - master_interface_arc, - initial_cpu_time, - ); + process_map_operation_error(err, resources, initial_cpu_time); } } } Err(err) => { - process_map_operation_error( - err, - operation_state_arc, - master_interface_arc, - initial_cpu_time, - ); + process_map_operation_error(err, resources, initial_cpu_time); } } } @@ -306,9 +274,7 @@ fn process_map_result( // Internal implementation for performing a map task. fn internal_perform_map( map_options: &pb::PerformMapRequest, - operation_state_arc: &Arc>, - master_interface_arc: &Arc, - data_abstraction_layer_arc: &Arc, + resources: &OperationResources, output_dir_uuid: &str, ) -> Result<()> { let mut output_path = PathBuf::new(); @@ -324,7 +290,7 @@ fn internal_perform_map( let initial_cpu_time; { - let mut operation_state = operation_state_arc.lock().unwrap(); + let mut operation_state = resources.operation_state.lock().unwrap(); operation_state.waiting_map_operations = input_locations.len(); operation_state.intermediate_map_results = HashMap::new(); @@ -332,10 +298,11 @@ fn internal_perform_map( } for input_location in input_locations { - let map_input_value = io::read_location(data_abstraction_layer_arc, input_location) + let map_input_value = io::read_location(&resources.data_abstraction_layer, input_location) .chain_err(|| "unable to open input file")?; - let absolute_path = data_abstraction_layer_arc + let absolute_path = resources + .data_abstraction_layer .absolute_path(Path::new(map_options.get_mapper_file_path())) .chain_err(|| "Unable to get absolute path")?; let child = Command::new(absolute_path) @@ -362,18 +329,17 @@ fn internal_perform_map( return Err("Could not convert map input to bson::Document.".into()); } - let operation_state_arc_clone = Arc::clone(operation_state_arc); - let master_interface_arc_clone = Arc::clone(master_interface_arc); let output_path_str: String = (*output_path.to_string_lossy()).to_owned(); let task_id = map_options.task_id.clone(); + let resources = resources.clone(); + thread::spawn(move || { let result = map_operation_thread_impl(&map_input_document, child); process_map_result( result, - &operation_state_arc_clone, - &master_interface_arc_clone, + &resources, initial_cpu_time, &output_path_str, &task_id, diff --git a/worker/src/operations/operation_handler.rs b/worker/src/operations/operation_handler.rs index b4759261..92c6e0ef 100644 --- a/worker/src/operations/operation_handler.rs +++ b/worker/src/operations/operation_handler.rs @@ -14,6 +14,14 @@ use super::map; use super::reduce; use super::state::OperationState; +/// `OperationResources` is used to hold resources passed to map and reduce operation functions. +#[derive(Clone)] +pub struct OperationResources { + pub operation_state: Arc>, + pub master_interface: Arc, + pub data_abstraction_layer: Arc, +} + /// `OperationHandler` is used for executing Map and Reduce operations queued by the Master pub struct OperationHandler { operation_state: Arc>, @@ -111,19 +119,16 @@ impl OperationHandler { &self, map_options: pb::PerformMapRequest, ) -> impl Future { - let operation_state_arc = Arc::clone(&self.operation_state); - let master_interface_arc = Arc::clone(&self.master_interface); - let data_abstraction_layer_arc = Arc::clone(&self.data_abstraction_layer); + let resources = OperationResources { + operation_state: Arc::clone(&self.operation_state), + master_interface: Arc::clone(&self.master_interface), + data_abstraction_layer: Arc::clone(&self.data_abstraction_layer), + }; + let output_dir_uuid = self.output_dir_uuid.clone(); future::lazy(move || { - let result = map::perform_map( - &map_options, - &operation_state_arc, - &master_interface_arc, - &data_abstraction_layer_arc, - &output_dir_uuid, - ); + let result = map::perform_map(&map_options, &resources, &output_dir_uuid); future::result(result) }) @@ -133,20 +138,16 @@ impl OperationHandler { &self, reduce_request: pb::PerformReduceRequest, ) -> impl Future { - let operation_state_arc = Arc::clone(&self.operation_state); - let master_interface_arc = Arc::clone(&self.master_interface); - let data_abstraction_layer_arc = Arc::clone(&self.data_abstraction_layer); + let resources = OperationResources { + operation_state: Arc::clone(&self.operation_state), + master_interface: Arc::clone(&self.master_interface), + data_abstraction_layer: Arc::clone(&self.data_abstraction_layer), + }; let output_dir_uuid = self.output_dir_uuid.clone(); future::lazy(move || { - let result = reduce::perform_reduce( - &reduce_request, - &operation_state_arc, - master_interface_arc, - data_abstraction_layer_arc, - &output_dir_uuid, - ); + let result = reduce::perform_reduce(&reduce_request, &resources, &output_dir_uuid); future::result(result) }) diff --git a/worker/src/operations/reduce.rs b/worker/src/operations/reduce.rs index 03e1d770..fc7743c6 100644 --- a/worker/src/operations/reduce.rs +++ b/worker/src/operations/reduce.rs @@ -11,6 +11,7 @@ use errors::*; use master_interface::MasterInterface; use super::io; use super::operation_handler; +use super::operation_handler::OperationResources; use super::state::OperationState; use util::output_error; use util::data_layer::AbstractionLayer; @@ -220,13 +221,11 @@ fn create_reduce_operations( // Public api for performing a reduce task. pub fn perform_reduce( reduce_request: &pb::PerformReduceRequest, - operation_state_arc: &Arc>, - master_interface_arc: Arc, - data_abstraction_layer_arc: Arc, + resources: &OperationResources, output_uuid: &str, ) -> Result<()> { { - let mut operation_state = operation_state_arc.lock().unwrap(); + let mut operation_state = resources.operation_state.lock().unwrap(); operation_state.initial_cpu_time = operation_handler::get_cpu_time(); } @@ -235,21 +234,15 @@ pub fn perform_reduce( reduce_request.reducer_file_path ); - if operation_handler::get_worker_status(operation_state_arc) == pb::WorkerStatus::BUSY { + if operation_handler::get_worker_status(&resources.operation_state) == pb::WorkerStatus::BUSY { warn!("Reduce operation requested while worker is busy"); return Err("Worker is busy.".into()); } - operation_handler::set_busy_status(operation_state_arc); + operation_handler::set_busy_status(&resources.operation_state); - let result = internal_perform_reduce( - reduce_request, - Arc::clone(operation_state_arc), - master_interface_arc, - data_abstraction_layer_arc, - output_uuid, - ); + let result = internal_perform_reduce(reduce_request, resources, output_uuid); if let Err(err) = result { - log_reduce_operation_err(err, operation_state_arc); + log_reduce_operation_err(err, &resources.operation_state); return Err("Error starting reduce operation.".into()); } @@ -259,9 +252,7 @@ pub fn perform_reduce( // Internal implementation for performing a reduce task. fn internal_perform_reduce( reduce_request: &pb::PerformReduceRequest, - operation_state_arc: Arc>, - master_interface_arc: Arc, - data_abstraction_layer_arc: Arc, + resources: &OperationResources, output_uuid: &str, ) -> Result<()> { let reduce_operations = create_reduce_operations(reduce_request, output_uuid) @@ -273,21 +264,14 @@ fn internal_perform_reduce( task_id: reduce_request.get_task_id().to_owned(), }; - run_reduce_queue( - reduce_options, - operation_state_arc, - reduce_operations, - master_interface_arc, - data_abstraction_layer_arc, - ); + run_reduce_queue(reduce_options, resources, reduce_operations); Ok(()) } fn handle_reduce_queue_finished( reduce_err: Option, - operation_state_arc: &Arc>, - master_interface_arc: &Arc, + resources: &OperationResources, initial_cpu_time: u64, task_id: &str, ) { @@ -297,28 +281,28 @@ fn handle_reduce_queue_finished( response.set_failure_details(operation_handler::failure_details_from_error(&err)); response.set_task_id(task_id.to_owned()); - let result = send_reduce_result(master_interface_arc, response); + let result = send_reduce_result(&resources.master_interface, response); if let Err(err) = result { error!("Error sending reduce failed: {}", err); } - log_reduce_operation_err(err, operation_state_arc); + log_reduce_operation_err(err, &resources.operation_state); } else { let mut response = pb::ReduceResult::new(); response.set_status(pb::ResultStatus::SUCCESS); response.set_cpu_time(operation_handler::get_cpu_time() - initial_cpu_time); response.set_task_id(task_id.to_owned()); - let result = send_reduce_result(master_interface_arc, response); + let result = send_reduce_result(&resources.master_interface, response); match result { Ok(_) => { - operation_handler::set_complete_status(operation_state_arc); + operation_handler::set_complete_status(&resources.operation_state); info!("Reduce operation completed sucessfully."); } Err(err) => { error!("Error sending reduce result: {}", err); - operation_handler::set_failed_status(operation_state_arc); + operation_handler::set_failed_status(&resources.operation_state); } } } @@ -326,12 +310,11 @@ fn handle_reduce_queue_finished( fn run_reduce_queue( reduce_options: ReduceOptions, - operation_state_arc: Arc>, + resources: &OperationResources, reduce_operations: Vec, - master_interface_arc: Arc, - data_abstraction_layer_arc: Arc, ) { - let initial_cpu_time = operation_state_arc.lock().unwrap().initial_cpu_time; + let initial_cpu_time = resources.operation_state.lock().unwrap().initial_cpu_time; + let resources = resources.clone(); thread::spawn(move || { let mut reduce_err = None; @@ -345,7 +328,7 @@ fn run_reduce_queue( } else { let result = reduce_queue.perform_next_reduce_operation( &reduce_options, - &data_abstraction_layer_arc, + &resources.data_abstraction_layer, ); if let Err(err) = result { reduce_err = Some(err); @@ -356,8 +339,7 @@ fn run_reduce_queue( handle_reduce_queue_finished( reduce_err, - &operation_state_arc, - &master_interface_arc, + &resources, initial_cpu_time, &reduce_options.task_id, ); From 70afe6ea72bc50ced46e38391efec04c3ce9c46c Mon Sep 17 00:00:00 2001 From: Darragh Griffin Date: Wed, 21 Feb 2018 22:13:59 +0000 Subject: [PATCH 04/19] Minor change to worker health check retry logic Add a larger wait time between retries and don't spam the output trying to make health check requests when the worker is already disconnected from the master. --- worker/src/main.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/worker/src/main.rs b/worker/src/main.rs index eb6ea801..9f32f698 100644 --- a/worker/src/main.rs +++ b/worker/src/main.rs @@ -61,6 +61,7 @@ use util::data_layer::{AbstractionLayer, NullAbstractionLayer, NFSAbstractionLay const WORKER_REGISTRATION_RETRIES: u16 = 5; const MAX_HEALTH_CHECK_FAILURES: u16 = 10; const MAIN_LOOP_SLEEP_MS: u64 = 3000; +const RECONNECT_FAILED_WAIT_MS: u64 = 5000; const WORKER_REGISTRATION_RETRY_WAIT_DURATION_MS: u64 = 1000; // Setting the port to 0 means a random available port will be selected const DEFAULT_PORT: &str = "0"; @@ -143,20 +144,21 @@ fn run() -> Result<()> { loop { thread::sleep(time::Duration::from_millis(MAIN_LOOP_SLEEP_MS)); - if let Err(err) = operation_handler.update_worker_status() { - error!("Could not send updated worker status to master: {}", err); - current_health_check_failures += 1; - } else { - current_health_check_failures = 0; - } - if current_health_check_failures >= MAX_HEALTH_CHECK_FAILURES { if let Err(err) = register_worker(&*master_interface, &local_addr) { error!("Failed to re-register worker after disconnecting: {}", err); + thread::sleep(time::Duration::from_millis(RECONNECT_FAILED_WAIT_MS)); } else { info!("Successfully re-registered with master after being disconnected."); current_health_check_failures = 0; } + } else { + if let Err(err) = operation_handler.update_worker_status() { + error!("Could not send updated worker status to master: {}", err); + current_health_check_failures += 1; + } else { + current_health_check_failures = 0; + } } if !srv.is_alive() { From 6b0733bacfddaa4c9addfa9b7dfee8e6dbb38616 Mon Sep 17 00:00:00 2001 From: Darragh Griffin Date: Sat, 3 Mar 2018 22:38:45 +0000 Subject: [PATCH 05/19] Add support for combiner in libcerberus --- libcerberus/examples/distributed-grep.rs | 15 +-- libcerberus/examples/end-to-end.rs | 15 +-- libcerberus/examples/word-counter.rs | 10 +- libcerberus/src/combiner.rs | 42 ++++++++ libcerberus/src/intermediate.rs | 27 ++++++ libcerberus/src/io.rs | 38 +++++--- libcerberus/src/lib.rs | 6 +- libcerberus/src/reducer.rs | 36 ++----- libcerberus/src/runner.rs | 117 +++++++++++++++++++---- libcerberus/src/serialise.rs | 54 +++++++++-- 10 files changed, 275 insertions(+), 85 deletions(-) create mode 100644 libcerberus/src/combiner.rs create mode 100644 libcerberus/src/intermediate.rs diff --git a/libcerberus/examples/distributed-grep.rs b/libcerberus/examples/distributed-grep.rs index 6bcdcba0..96bf8458 100644 --- a/libcerberus/examples/distributed-grep.rs +++ b/libcerberus/examples/distributed-grep.rs @@ -44,7 +44,7 @@ impl Map for GrepMapper { struct GrepReducer; impl Reduce for GrepReducer { type Value = String; - fn reduce(&self, input: ReduceInputKV, mut emitter: E) -> Result<()> + fn reduce(&self, input: IntermediateInputKV, mut emitter: E) -> Result<()> where E: EmitFinal, { @@ -66,12 +66,13 @@ fn run() -> Result<()> { let matches = cerberus::parse_command_line(); - let registry = UserImplRegistryBuilder::new() - .mapper(&grep_mapper) - .reducer(&grep_reducer) - .partitioner(&grep_partitioner) - .build() - .chain_err(|| "Error building UserImplRegistry.")?; + let registry = + UserImplRegistryBuilder::::new() + .mapper(&grep_mapper) + .reducer(&grep_reducer) + .partitioner(&grep_partitioner) + .build() + .chain_err(|| "Error building UserImplRegistry.")?; cerberus::run(&matches, ®istry) } diff --git a/libcerberus/examples/end-to-end.rs b/libcerberus/examples/end-to-end.rs index db76e307..ada789fb 100644 --- a/libcerberus/examples/end-to-end.rs +++ b/libcerberus/examples/end-to-end.rs @@ -22,7 +22,7 @@ impl Map for TestMapper { struct TestReducer; impl Reduce for TestReducer { type Value = String; - fn reduce(&self, input: ReduceInputKV, mut emitter: E) -> Result<()> + fn reduce(&self, input: IntermediateInputKV, mut emitter: E) -> Result<()> where E: EmitFinal, { @@ -69,12 +69,13 @@ fn run() -> Result<()> { let matches = cerberus::parse_command_line(); - let registry = UserImplRegistryBuilder::new() - .mapper(&test_mapper) - .reducer(&test_reducer) - .partitioner(&test_partitioner) - .build() - .chain_err(|| "Error building UserImplRegistry.")?; + let registry = + UserImplRegistryBuilder::::new() + .mapper(&test_mapper) + .reducer(&test_reducer) + .partitioner(&test_partitioner) + .build() + .chain_err(|| "Error building UserImplRegistry.")?; cerberus::run(&matches, ®istry) } diff --git a/libcerberus/examples/word-counter.rs b/libcerberus/examples/word-counter.rs index 3a3fae04..5d619790 100644 --- a/libcerberus/examples/word-counter.rs +++ b/libcerberus/examples/word-counter.rs @@ -29,7 +29,7 @@ impl Map for WordCountMapper { struct WordCountReducer; impl Reduce for WordCountReducer { type Value = u64; - fn reduce(&self, input: ReduceInputKV, mut emitter: E) -> Result<()> + fn reduce(&self, input: IntermediateInputKV, mut emitter: E) -> Result<()> where E: EmitFinal, { @@ -55,8 +55,12 @@ fn run() -> Result<()> { let matches = cerberus::parse_command_line(); - let registry = UserImplRegistryBuilder::new() - .mapper(&wc_mapper) + let registry = UserImplRegistryBuilder::< + WordCountMapper, + WordCountReducer, + HashPartitioner, + NullCombiner, + >::new().mapper(&wc_mapper) .reducer(&wc_reducer) .partitioner(&wc_partitioner) .build() diff --git a/libcerberus/src/combiner.rs b/libcerberus/src/combiner.rs new file mode 100644 index 00000000..7af779e6 --- /dev/null +++ b/libcerberus/src/combiner.rs @@ -0,0 +1,42 @@ +use emitter::EmitIntermediate; +use errors::*; +use intermediate::IntermediateInputKV; +use serde::Serialize; +use serde::de::DeserializeOwned; + +/// The `Combine` trait defines a function for performing a combine operation. +/// +/// The output types are decided by the implementation of this trait. +/// +/// # Arguments +/// +/// * `input` - A `IntermediateInputKV` containing the input data for the combine operation. +/// * `emitter` - A struct implementing the `EmitIntermediate` trait, +/// provided by the combine runner. +/// +/// # Outputs +/// +/// An empty result used for returning an error. Outputs of the reduce operation are sent out +/// through the `emitter`. +pub trait Combine +where + V: Default + Serialize + DeserializeOwned, +{ + fn combine(&self, input: IntermediateInputKV, emitter: E) -> Result<()> + where + E: EmitIntermediate; +} + +// A null implementation for `Combine` as this is optional component. +pub struct NullCombiner; +impl Combine for NullCombiner +where + V: Default + Serialize + DeserializeOwned, +{ + fn combine(&self, _input: IntermediateInputKV, _emitter: E) -> Result<()> + where + E: EmitIntermediate, + { + Ok(()) + } +} diff --git a/libcerberus/src/intermediate.rs b/libcerberus/src/intermediate.rs new file mode 100644 index 00000000..456d5b31 --- /dev/null +++ b/libcerberus/src/intermediate.rs @@ -0,0 +1,27 @@ +use serde::Serialize; + +/// The `IntermediateInputKV` is a struct for passing input data to a `Reduce` or `Combine`. +/// +/// `IntermediateInputKV` is a thin wrapper around a `(String, Vec)`, +/// used for creating a clearer API. +/// It can be constructed normally or using `IntermediateInputKV::new()`. +#[derive(Debug, Default, Deserialize, PartialEq)] +pub struct IntermediateInputKV +where + V: Default + Serialize, +{ + pub key: String, + pub values: Vec, +} + +impl IntermediateInputKV +where + V: Default + Serialize, +{ + pub fn new(key: String, values: Vec) -> Self { + IntermediateInputKV { + key: key, + values: values, + } + } +} diff --git a/libcerberus/src/io.rs b/libcerberus/src/io.rs index 62b0c1a7..d4324e4a 100644 --- a/libcerberus/src/io.rs +++ b/libcerberus/src/io.rs @@ -1,11 +1,11 @@ use bson; use errors::*; use mapper::MapInputKV; -use reducer::ReduceInputKV; +use intermediate::IntermediateInputKV; use serde::Serialize; use serde::de::DeserializeOwned; use serde_json; -use serialise::{FinalOutputObject, IntermediateOutputObject}; +use serialise::{FinalOutputObject, IntermediateOutputObject, IntermediateOutputPair}; use std::io::{Read, Write}; /// `read_map_input` reads bytes from a source and returns a `MapInputKV`. @@ -23,11 +23,11 @@ pub fn read_map_input(source: &mut R) -> Result { Ok(map_input) } -/// `read_reduce_input` reads a string from a source and returns a `ReduceInputKV`. +/// `read_intermediate_input` reads a string from a source and returns a `IntermediateInputKV`. /// /// It attempts to parse the string from the input source as JSON and returns an `errors::Error` if /// the attempt fails. -pub fn read_reduce_input(source: &mut R) -> Result> +pub fn read_intermediate_input(source: &mut R) -> Result> where R: Read, V: Default + Serialize + DeserializeOwned, @@ -40,13 +40,29 @@ where warn!("bytes_read is 0"); } let result = serde_json::from_str(input_string.as_str()).chain_err( - || "Error parsing input JSON to ReduceInputKV.", + || "Error parsing input JSON to IntermediateInputKV.", )?; Ok(result) } -/// `write_map_output` attempts to serialise an `IntermediateOutputObject` to a given sink. -pub fn write_map_output( +/// `write_intermediate_pair` attempts to serialise an `IntermediateOutputPair` to a given sink. +pub fn write_intermediate_pair( + sink: &mut W, + output: &IntermediateOutputPair, +) -> Result<()> +where + W: Write, + K: Default + Serialize, + V: Default + Serialize, +{ + serde_json::to_writer(sink, &output).chain_err( + || "Error writing to sink.", + )?; + Ok(()) +} + +/// `write_intermediate_output` attempts to serialise an `IntermediateOutputObject` to a given sink. +pub fn write_intermediate_output( sink: &mut W, output: &IntermediateOutputObject, ) -> Result<()> @@ -117,12 +133,12 @@ mod tests { fn read_valid_reduce_input_kv() { let test_string = r#"{"key":"foo","values":["bar","baz"]}"#; let mut cursor = Cursor::new(test_string); - let expected_result = ReduceInputKV { + let expected_result = IntermediateInputKV { key: "foo".to_owned(), values: vec!["bar".to_owned(), "baz".to_owned()], }; - let result: ReduceInputKV = read_reduce_input(&mut cursor).unwrap(); + let result: IntermediateInputKV = read_intermediate_input(&mut cursor).unwrap(); assert_eq!(expected_result, result); } @@ -133,7 +149,7 @@ mod tests { let test_string = ""; let mut cursor = Cursor::new(test_string); - let _: ReduceInputKV = read_reduce_input(&mut cursor).unwrap(); + let _: IntermediateInputKV = read_intermediate_input(&mut cursor).unwrap(); } #[test] @@ -162,7 +178,7 @@ mod tests { let output_vector: Vec = Vec::new(); let mut cursor = Cursor::new(output_vector); - write_map_output(&mut cursor, &test_object).unwrap(); + write_intermediate_output(&mut cursor, &test_object).unwrap(); let output_string = String::from_utf8(cursor.into_inner()).unwrap(); assert_eq!(expected_json_string, output_string); diff --git a/libcerberus/src/lib.rs b/libcerberus/src/lib.rs index 825a040a..ba723e14 100644 --- a/libcerberus/src/lib.rs +++ b/libcerberus/src/lib.rs @@ -25,18 +25,22 @@ mod errors { } } +pub mod combiner; pub mod emitter; pub mod io; +pub mod intermediate; pub mod mapper; pub mod partition; pub mod reducer; pub mod runner; pub mod serialise; +pub use combiner::{Combine, NullCombiner}; pub use errors::*; pub use emitter::{EmitIntermediate, EmitPartitionedIntermediate, EmitFinal}; +pub use intermediate::IntermediateInputKV; pub use mapper::{Map, MapInputKV}; pub use partition::{HashPartitioner, Partition, PartitionInputPairs}; -pub use reducer::{Reduce, ReduceInputKV}; +pub use reducer::Reduce; pub use runner::*; pub use serialise::{FinalOutputObject, IntermediateOutputObject}; diff --git a/libcerberus/src/reducer.rs b/libcerberus/src/reducer.rs index ec480cd3..9ae8ddc4 100644 --- a/libcerberus/src/reducer.rs +++ b/libcerberus/src/reducer.rs @@ -1,40 +1,16 @@ use emitter::EmitFinal; use errors::*; +use intermediate::IntermediateInputKV; use serde::Serialize; use serde::de::DeserializeOwned; -/// The `ReduceInputKV` is a struct for passing input data to a `Reduce`. -/// -/// `ReduceInputKV` is a thin wrapper around a `(String, Vec)`, used for creating a clearer API. -/// It can be constructed normally or using `ReduceInputKV::new()`. -#[derive(Debug, Default, Deserialize, PartialEq)] -pub struct ReduceInputKV -where - V: Default + Serialize, -{ - pub key: String, - pub values: Vec, -} - -impl ReduceInputKV -where - V: Default + Serialize, -{ - pub fn new(key: String, values: Vec) -> Self { - ReduceInputKV { - key: key, - values: values, - } - } -} - /// The `Reduce` trait defines a function for performing a reduce operation. /// /// The output types are decided by the implementation of this trait. /// /// # Arguments /// -/// * `input` - A `ReduceInputKV` containing the input data for the reduce operation. +/// * `input` - A `IntermediateInputKV` containing the input data for the reduce operation. /// * `emitter` - A struct implementing the `EmitFinal` trait, provided by the reduce runner. /// /// # Outputs @@ -43,7 +19,7 @@ where /// through the `emitter`. pub trait Reduce { type Value: Default + Serialize + DeserializeOwned; - fn reduce(&self, input: ReduceInputKV, emitter: E) -> Result<()> + fn reduce(&self, input: IntermediateInputKV, emitter: E) -> Result<()> where E: EmitFinal; } @@ -56,7 +32,7 @@ mod tests { struct TestReducer; impl Reduce for TestReducer { type Value = String; - fn reduce(&self, input: ReduceInputKV, mut emitter: E) -> Result<()> + fn reduce(&self, input: IntermediateInputKV, mut emitter: E) -> Result<()> where E: EmitFinal, { @@ -71,7 +47,7 @@ mod tests { #[test] fn test_reducer_test_strings() { let test_vector = vec!["foo".to_owned(), "bar".to_owned()]; - let test_kv = ReduceInputKV::new("test_vector".to_owned(), test_vector); + let test_kv = IntermediateInputKV::new("test_vector".to_owned(), test_vector); let mut sink: Vec = Vec::new(); let reducer = TestReducer; @@ -86,7 +62,7 @@ mod tests { fn reduce_input_kv_construction() { let test_vector = vec!["foo".to_owned(), "bar".to_owned()]; - let test_kv = ReduceInputKV::new("test_vector".to_owned(), test_vector); + let test_kv = IntermediateInputKV::new("test_vector".to_owned(), test_vector); assert_eq!("foo", test_kv.values[0]); assert_eq!("bar", test_kv.values[1]); diff --git a/libcerberus/src/runner.rs b/libcerberus/src/runner.rs index 2b00d11b..1da0f16e 100644 --- a/libcerberus/src/runner.rs +++ b/libcerberus/src/runner.rs @@ -2,6 +2,8 @@ use std::io::{stdin, stdout}; use chrono::prelude::*; use clap::{App, ArgMatches, SubCommand}; +use serde::Serialize; +use serde::de::DeserializeOwned; use uuid::Uuid; use emitter::IntermediateVecEmitter; @@ -10,77 +12,95 @@ use io::*; use mapper::Map; use partition::{Partition, PartitionInputPairs}; use reducer::Reduce; +use combiner::Combine; use serialise::{FinalOutputObject, FinalOutputObjectEmitter, IntermediateOutputObject, - IntermediateOutputObjectEmitter}; + IntermediateOutputPair, IntermediateOutputObjectEmitter, + IntermediateOutputPairEmitter}; use super::VERSION; /// `UserImplRegistry` tracks the user's implementations of Map, Reduce, etc. /// /// The user should use the `UserImplRegistryBuilder` to create this and then pass it in to `run`. -pub struct UserImplRegistry<'a, M, R, P> +pub struct UserImplRegistry<'a, M, R, P, C> where M: Map + 'a, R: Reduce + 'a, P: Partition + 'a, + C: Combine + 'a, { mapper: &'a M, reducer: &'a R, partitioner: &'a P, + combiner: Option<&'a C>, } /// `UserImplRegistryBuilder` is used to create a `UserImplRegistry`. -pub struct UserImplRegistryBuilder<'a, M, R, P> +pub struct UserImplRegistryBuilder<'a, M, R, P, C> where M: Map + 'a, R: Reduce + 'a, P: Partition + 'a, + C: Combine + 'a, { mapper: Option<&'a M>, reducer: Option<&'a R>, partitioner: Option<&'a P>, + combiner: Option<&'a C>, } -impl<'a, M, R, P> Default for UserImplRegistryBuilder<'a, M, R, P> +impl<'a, M, R, P, C> Default for UserImplRegistryBuilder<'a, M, R, P, C> where M: Map + 'a, R: Reduce + 'a, - P: Partition + 'a, + P: Partition + + 'a, + C: Combine + 'a, { - fn default() -> UserImplRegistryBuilder<'a, M, R, P> { + fn default() -> UserImplRegistryBuilder<'a, M, R, P, C> { UserImplRegistryBuilder { mapper: None, reducer: None, partitioner: None, + combiner: None, } } } -impl<'a, M, R, P> UserImplRegistryBuilder<'a, M, R, P> +impl<'a, M, R, P, C> UserImplRegistryBuilder<'a, M, R, P, C> where M: Map + 'a, R: Reduce + 'a, P: Partition + 'a, + C: Combine + 'a, { - pub fn new() -> UserImplRegistryBuilder<'a, M, R, P> { + pub fn new() -> UserImplRegistryBuilder<'a, M, R, P, C> { Default::default() } - pub fn mapper(&mut self, mapper: &'a M) -> &mut UserImplRegistryBuilder<'a, M, R, P> { + pub fn mapper(&mut self, mapper: &'a M) -> &mut UserImplRegistryBuilder<'a, M, R, P, C> { self.mapper = Some(mapper); self } - pub fn reducer(&mut self, reducer: &'a R) -> &mut UserImplRegistryBuilder<'a, M, R, P> { + pub fn reducer(&mut self, reducer: &'a R) -> &mut UserImplRegistryBuilder<'a, M, R, P, C> { self.reducer = Some(reducer); self } - pub fn partitioner(&mut self, partitioner: &'a P) -> &mut UserImplRegistryBuilder<'a, M, R, P> { + pub fn partitioner( + &mut self, + partitioner: &'a P, + ) -> &mut UserImplRegistryBuilder<'a, M, R, P, C> { self.partitioner = Some(partitioner); self } - pub fn build(&self) -> Result> { + pub fn combiner(&mut self, combiner: &'a C) -> &mut UserImplRegistryBuilder<'a, M, R, P, C> { + self.combiner = Some(combiner); + self + } + + pub fn build(&self) -> Result> { let mapper = self.mapper.chain_err( || "Error building UserImplRegistry: No Mapper provided", )?; @@ -95,6 +115,7 @@ where mapper: mapper, reducer: reducer, partitioner: partitioner, + combiner: self.combiner, }) } } @@ -111,6 +132,8 @@ pub fn parse_command_line<'a>() -> ArgMatches<'a> { .version(VERSION.unwrap_or("unknown")) .subcommand(SubCommand::with_name("map")) .subcommand(SubCommand::with_name("reduce")) + .subcommand(SubCommand::with_name("combine")) + .subcommand(SubCommand::with_name("has-combine")) .subcommand(SubCommand::with_name("sanity-check")); app.get_matches() } @@ -121,15 +144,37 @@ pub fn parse_command_line<'a>() -> ArgMatches<'a> { /// /// `matches` - The output of the `parse_command_line` function. /// `registry` - The output of the `register_mapper_reducer` function. -pub fn run(matches: &ArgMatches, registry: &UserImplRegistry) -> Result<()> +pub fn run(matches: &ArgMatches, registry: &UserImplRegistry) -> Result<()> where M: Map, R: Reduce, P: Partition, + C: Combine, { match matches.subcommand_name() { - Some("map") => Ok(run_map(registry.mapper, registry.partitioner)?), - Some("reduce") => Ok(run_reduce(registry.reducer)?), + Some("map") => { + run_map(registry.mapper, registry.partitioner).chain_err( + || "Error running map", + )?; + Ok(()) + } + Some("reduce") => { + run_reduce(registry.reducer).chain_err( + || "Error running reduce", + )?; + Ok(()) + } + Some("combiner") => { + let combiner = registry.combiner.chain_err( + || "Attempt to run combine command when combiner is not implemented", + )?; + run_combine(combiner).chain_err(|| "Error running combine")?; + Ok(()) + } + Some("has-combine") => { + run_has_combine(registry.combiner); + Ok(()) + } Some("sanity-check") => { run_sanity_check(); Ok(()) @@ -169,16 +214,15 @@ where ) .chain_err(|| "Error partitioning map output")?; - write_map_output(&mut sink, &output_object).chain_err( - || "Error writing map output to stdout.", - )?; + write_intermediate_output(&mut sink, &output_object) + .chain_err(|| "Error writing map output to stdout.")?; Ok(()) } fn run_reduce(reducer: &R) -> Result<()> { let mut source = stdin(); let mut sink = stdout(); - let input_kv = read_reduce_input(&mut source).chain_err( + let input_kv = read_intermediate_input(&mut source).chain_err( || "Error getting input to reduce.", )?; let mut output_object = FinalOutputObject::::default(); @@ -193,6 +237,41 @@ fn run_reduce(reducer: &R) -> Result<()> { Ok(()) } +fn run_combine(combiner: &C) -> Result<()> +where + V: Default + Serialize + DeserializeOwned, + C: Combine, +{ + let mut source = stdin(); + let mut sink = stdout(); + let input_kv = read_intermediate_input(&mut source).chain_err( + || "Error getting input to reduce.", + )?; + let mut output_object = IntermediateOutputPair::::default(); + + combiner + .combine( + input_kv, + IntermediateOutputPairEmitter::new(&mut output_object), + ) + .chain_err(|| "Error running combine operation.")?; + + write_intermediate_pair(&mut sink, &output_object) + .chain_err(|| "Error writing combine output to stdout.")?; + Ok(()) +} + +fn run_has_combine(combiner: Option<&C>) +where + V: Default + Serialize + DeserializeOwned, + C: Combine, +{ + match combiner { + Some(_) => println!("yes"), + None => println!("no"), + } +} + fn run_sanity_check() { println!("sanity located"); } diff --git a/libcerberus/src/serialise.rs b/libcerberus/src/serialise.rs index 593cb691..ac5947cb 100644 --- a/libcerberus/src/serialise.rs +++ b/libcerberus/src/serialise.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use serde::Serialize; -use emitter::{EmitPartitionedIntermediate, EmitFinal}; +use emitter::{EmitIntermediate, EmitFinal, EmitPartitionedIntermediate}; use errors::*; /// `IntermediateOutputPair` is a struct representing an intermediate key-value pair as outputted @@ -35,7 +35,46 @@ pub struct FinalOutputObject { pub values: Vec, } -/// A struct implementing `EmitIntermediate` which emits to an `IntermediateOutputObject`. +/// A struct implementing `EmitIntermediate` which emits to an `IntermediateOutputPair`. +pub struct IntermediateOutputPairEmitter<'a, K, V> +where + K: Default + Serialize + 'a, + V: Default + Serialize + 'a, +{ + sink: &'a mut IntermediateOutputPair, +} + +impl<'a, K, V> IntermediateOutputPairEmitter<'a, K, V> +where + K: Default + Serialize, + V: Default + Serialize, +{ + /// Constructs a new `IntermediateOutputPairEmitter` with a mutable reference to a given + /// `IntermediateOutputPair`. + /// + /// # Arguments + /// + /// * `sink` - A mutable reference to the `IntermediateOutputPair` + /// to receive the emitted values. + pub fn new(sink: &'a mut IntermediateOutputPair) -> Self { + IntermediateOutputPairEmitter { sink: sink } + } +} + +impl<'a, K, V> EmitIntermediate for IntermediateOutputPairEmitter<'a, K, V> +where + K: Default + Serialize, + V: Default + Serialize, +{ + fn emit(&mut self, key: K, value: V) -> Result<()> { + self.sink.key = key; + self.sink.value = value; + Ok(()) + } +} + +/// A struct implementing `EmitPartitionedIntermediate` +/// which emits to an `IntermediateOutputObject`. pub struct IntermediateOutputObjectEmitter<'a, K, V> where K: Default + Serialize + 'a, @@ -54,7 +93,8 @@ where /// /// # Arguments /// - /// * `sink` - A mutable reference to the `IntermediateOutputObject` to receive the emitted values. + /// * `sink` - A mutable reference to the `IntermediateOutputObject` + /// to receive the emitted values. pub fn new(sink: &'a mut IntermediateOutputObject) -> Self { IntermediateOutputObjectEmitter { sink: sink } } @@ -136,12 +176,12 @@ mod tests { let output = IntermediateOutputObject { partitions: partitions }; let mut output_set = HashSet::new(); - let expected_json_string1 = + let expected_string1 = r#"{"partitions":{"0":[{"key":"foo_intermediate","value":"bar"},{"key":"foo_intermediate","value":"baz"}],"1":[{"key":"foo_intermediate2","value":"bar"}]}}"#; - let expected_json_string2 = + let expected_string2 = r#"{"partitions":{"1":[{"key":"foo_intermediate2","value":"bar"}],"0":[{"key":"foo_intermediate","value":"bar"},{"key":"foo_intermediate","value":"baz"}]}}"#; - output_set.insert(expected_json_string1.to_owned()); - output_set.insert(expected_json_string2.to_owned()); + output_set.insert(expected_string1.to_owned()); + output_set.insert(expected_string2.to_owned()); let json_string = serde_json::to_string(&output).unwrap(); From 30d56e411118096573ea3eec7e423d96137712d1 Mon Sep 17 00:00:00 2001 From: Ryan Connell Date: Sun, 4 Mar 2018 15:11:00 +0000 Subject: [PATCH 06/19] Add Job Cancellation API to Master and CLI --- cli/src/main.rs | 1 + cli/src/parser.rs | 11 ++++++++++ cli/src/runner.rs | 22 ++++++++++++++++++++ master/src/scheduling/scheduler.rs | 16 +++++++++++++++ master/src/server/client_service.rs | 32 +++++++++++++++++++++++++++++ proto/mapreduce.proto | 15 ++++++++++++++ 6 files changed, 97 insertions(+) diff --git a/cli/src/main.rs b/cli/src/main.rs index 25c6f45d..88a226f8 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -51,6 +51,7 @@ fn run() -> Result<()> { println!("Getting Cluster Status..."); runner::cluster_status(&client) } + ("cancel", sub) => runner::cancel(&client, sub), ("status", Some(sub)) => runner::status(&client, sub), _ => Err(matches.usage().into()), } diff --git a/cli/src/parser.rs b/cli/src/parser.rs index 50db8a4e..004e8244 100644 --- a/cli/src/parser.rs +++ b/cli/src/parser.rs @@ -42,6 +42,17 @@ pub fn parse_command_line<'a>() -> ArgMatches<'a> { .required(false), ), ) + .subcommand( + SubCommand::with_name("cancel") + .about("Cancels a running or queued MapReduce. If no id is provided it will cancel the most recently schceduled job") + .arg( + Arg::with_name("id") + .short("i") + .help("ID of the MapReduce to cancel") + .required(false) + .takes_value(true), + ), + ) .subcommand(SubCommand::with_name("cluster_status").about( "Status of the cluster", )) diff --git a/cli/src/runner.rs b/cli/src/runner.rs index 701fbc81..86c7ce23 100644 --- a/cli/src/runner.rs +++ b/cli/src/runner.rs @@ -136,6 +136,28 @@ pub fn cluster_status(client: &grpc_pb::MapReduceServiceClient) -> Result<()> { Ok(()) } +pub fn cancel(client: &grpc_pb::MapReduceServiceClient, args: Option<&ArgMatches>) -> Result<()> { + let mut req = pb::MapReduceCancelRequest::new(); + req.set_client_id(get_client_id().chain_err(|| "Error getting client id")?); + if let Some(sub) = args { + if let Some(id) = sub.value_of("id") { + req.set_mapreduce_id(id.to_owned()); + } + } + + let resp = client + .cancel_map_reduce(RequestOptions::new(), req) + .wait() + .chain_err(|| "Failed to cancel MapReduce")? + .1; + + println!( + "Succesfully cancelled MapReduce with ID: {}", + resp.mapreduce_id + ); + Ok(()) +} + pub fn status(client: &grpc_pb::MapReduceServiceClient, matches: &ArgMatches) -> Result<()> { let mut req = pb::MapReduceStatusRequest::new(); req.set_client_id(get_client_id()?); diff --git a/master/src/scheduling/scheduler.rs b/master/src/scheduling/scheduler.rs index 1fad1341..0146b05e 100644 --- a/master/src/scheduling/scheduler.rs +++ b/master/src/scheduling/scheduler.rs @@ -141,6 +141,11 @@ impl Scheduler { ) } + // TODO(rhino): Continue this + pub fn cancel_job(&self, _job_id: &str) -> Result<()> { + Ok(()) + } + pub fn get_job_queue_size(&self) -> u32 { let state = self.state.lock().unwrap(); state.get_job_queue_size() @@ -163,6 +168,17 @@ impl Scheduler { let state = self.state.lock().unwrap(); state.get_jobs(client_id) } + + pub fn get_most_recent_client_job_id(&self, client_id: &str) -> Result { + let state = self.state.lock().unwrap(); + let jobs = state.get_jobs(client_id); + + if jobs.is_empty() { + return Err("No jobs queued for client".into()); + } + + Ok(jobs[0].id.clone()) + } } impl state::SimpleStateHandling for Scheduler { diff --git a/master/src/server/client_service.rs b/master/src/server/client_service.rs index 721da462..aad75c14 100644 --- a/master/src/server/client_service.rs +++ b/master/src/server/client_service.rs @@ -10,6 +10,7 @@ use util::data_layer::AbstractionLayer; use cerberus_proto::mapreduce as pb; use cerberus_proto::mapreduce_grpc as grpc_pb; +const JOB_CANCEL_ERROR: &str = "Unable to cancel mapreduce job"; const JOB_SCHEDULE_ERROR: &str = "Unable to schedule mapreduce job"; const JOB_RETRIEVAL_ERROR: &str = "Unable to retrieve mapreduce jobs"; const MISSING_JOB_IDS: &str = "No client_id or mapreduce_id provided"; @@ -112,6 +113,37 @@ impl grpc_pb::MapReduceService for ClientService { SingleResponse::completed(response) } + fn cancel_map_reduce( + &self, + _: RequestOptions, + req: pb::MapReduceCancelRequest, + ) -> SingleResponse { + let mut response = pb::MapReduceCancelResponse::new(); + let job_id = { + if !req.mapreduce_id.is_empty() { + req.mapreduce_id + } else { + match self.scheduler.get_most_recent_client_job_id(&req.client_id) { + Err(err) => { + output_error(&err.chain_err(|| "Error cancelling MapReduce.")); + return SingleResponse::err(Error::Other("No jobs found for this client")); + } + Ok(job_id) => job_id, + } + } + }; + response.set_mapreduce_id(job_id.clone()); + + println!("Attempting to cancel MapReduce: {}", job_id); + let result = self.scheduler.cancel_job(job_id.as_ref()); + if let Err(err) = result { + output_error(&err.chain_err(|| "Error cancelling MapReduce")); + return SingleResponse::err(Error::Other(JOB_CANCEL_ERROR)); + } + + SingleResponse::completed(response) + } + fn cluster_status( &self, _: RequestOptions, diff --git a/proto/mapreduce.proto b/proto/mapreduce.proto index b1c5004c..d4e34a04 100644 --- a/proto/mapreduce.proto +++ b/proto/mapreduce.proto @@ -14,6 +14,8 @@ service MapReduceService { // Gets the status of the map reduce. rpc MapReduceStatus (MapReduceStatusRequest) returns (MapReduceStatusResponse); + // Attempts to cancel a running or scheduled map reduce. + rpc CancelMapReduce (MapReduceCancelRequest) returns (MapReduceCancelResponse); // Cluster Status checks. rpc ClusterStatus (EmptyMessage) returns (ClusterStatusResponse); @@ -92,3 +94,16 @@ message ClusterStatusResponse { int64 workers = 1; int64 queue_size = 2; } + +message MapReduceCancelRequest { + // ID of the client. + string client_id = 1; + // ID of the map reduce to cancel. If this is not provided then the most recently + // scheduled MapReduce for the given client will be cancelled. + string mapreduce_id = 2; +} + +message MapReduceCancelResponse { + // ID of the canceled map reduce + string mapreduce_id = 1; +} From e4f84099b55aad6e74bd3acceb4291c7111004f4 Mon Sep 17 00:00:00 2001 From: Ryan Connell Date: Sun, 4 Mar 2018 19:45:59 +0000 Subject: [PATCH 07/19] Pick most recent job based on time_requested --- master/src/scheduling/scheduler.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/master/src/scheduling/scheduler.rs b/master/src/scheduling/scheduler.rs index 0146b05e..5e7592d1 100644 --- a/master/src/scheduling/scheduler.rs +++ b/master/src/scheduling/scheduler.rs @@ -177,7 +177,15 @@ impl Scheduler { return Err("No jobs queued for client".into()); } - Ok(jobs[0].id.clone()) + let mut latest_job = &jobs[0]; + for i in 1..jobs.len() { + let job = &jobs[i]; + if job.time_requested > latest_job.time_requested { + latest_job = job; + } + } + + Ok(latest_job.id.clone()) } } From 23baa9465907036dc797fd00d6821e3ba90e12bb Mon Sep 17 00:00:00 2001 From: Darragh Griffin Date: Sun, 4 Mar 2018 22:51:58 +0000 Subject: [PATCH 08/19] Add support in worker for running combine operation --- worker/src/operations/combine.rs | 136 +++++++++++++++++++++ worker/src/operations/map.rs | 5 + worker/src/operations/mod.rs | 1 + worker/src/operations/operation_handler.rs | 3 + worker/src/operations/reduce.rs | 4 +- 5 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 worker/src/operations/combine.rs diff --git a/worker/src/operations/combine.rs b/worker/src/operations/combine.rs new file mode 100644 index 00000000..f5c7671c --- /dev/null +++ b/worker/src/operations/combine.rs @@ -0,0 +1,136 @@ +use std::collections::HashMap; +use std::io::Write; +use std::path::Path; +use std::process::{Command, Stdio}; + +use serde_json; + +use errors::*; +use super::operation_handler::OperationResources; + +fn check_has_combine(resources: &OperationResources) -> Result { + let absolute_path = resources + .data_abstraction_layer + .absolute_path(Path::new(&resources.binary_path)) + .chain_err(|| "Unable to get absolute path")?; + let output = Command::new(absolute_path) + .arg("has-combine") + .output() + .chain_err(|| "Error running MapReduce binary.")?; + let output_str = String::from_utf8(output.stdout).unwrap(); + + let has_combine = "yes" == output_str.trim(); + + Ok(has_combine) +} + +fn do_combine_operation( + resources: &OperationResources, + key: &str, + values: Vec, +) -> Result { + let combine_input = json!({ + "key": key.clone().to_owned(), + "values": values, + }); + let combine_input_str = serde_json::to_string(&combine_input).chain_err( + || "Error seralizing combine operation input.", + )?; + + let absolute_binary_path = resources + .data_abstraction_layer + .absolute_path(Path::new(&resources.binary_path)) + .chain_err(|| "Unable to get absolute path")?; + let mut child = Command::new(absolute_binary_path) + .arg("combine") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .chain_err(|| "Failed to start combine operation process.")?; + + if let Some(stdin) = child.stdin.as_mut() { + stdin.write_all(combine_input_str.as_bytes()).chain_err( + || "Error writing to payload stdin.", + )?; + } else { + return Err("Error accessing stdin of payload binary.".into()); + } + + let output = child.wait_with_output().chain_err( + || "Error waiting for payload result.", + )?; + + let output_str = String::from_utf8(output.stdout).chain_err( + || "Error accessing payload output.", + )?; + + let combine_result: serde_json::Value = serde_json::from_str(&output_str).chain_err( + || "Error parsing reduce results.", + )?; + + Ok(combine_result) +} + +fn run_combine(resources: &OperationResources) -> Result<()> { + let partition_map; + { + let operation_state = resources.operation_state.lock().unwrap(); + partition_map = operation_state.intermediate_map_results.clone(); + } + + let mut new_partition_map = HashMap::new(); + for (partition, values) in (&partition_map).iter() { + let mut kv_map: HashMap> = HashMap::new(); + + for pair in values { + let key = pair["key"].as_str().chain_err( + || "Error parsing map output key to run combine.", + )?; + + let value = pair["value"].clone(); + if value.is_null() { + return Err("Error parsing map output value to run combine.".into()); + } + + let vec = kv_map.entry(key.to_string()).or_insert_with(Vec::new); + vec.push(value); + } + + let mut partition_results: Vec = Vec::new(); + + for (key, values) in (&kv_map).iter() { + if values.len() > 1 { + let pair = do_combine_operation(resources, key, values.clone()) + .chain_err(|| "Failed to run combine operation.")?; + + partition_results.push(pair); + } else if let Some(value) = values.first() { + partition_results.push(json!({ + "key": key, + "value": value + })); + } + } + + new_partition_map.insert(*partition, partition_results); + } + + let mut operation_state = resources.operation_state.lock().unwrap(); + operation_state.intermediate_map_results = new_partition_map; + + Ok(()) +} + +/// Optionally run a combine operation if it's implemented by the MapReduce binary. +pub fn optional_run_combine(resources: &OperationResources) -> Result<()> { + let has_combine = check_has_combine(resources).chain_err( + || "Error running has-combine command.", + )?; + + if has_combine { + return run_combine(resources); + } + + Ok(()) +} diff --git a/worker/src/operations/map.rs b/worker/src/operations/map.rs index d5812f2c..b3102709 100644 --- a/worker/src/operations/map.rs +++ b/worker/src/operations/map.rs @@ -13,6 +13,7 @@ use uuid::Uuid; use errors::*; use cerberus_proto::worker as pb; use master_interface::MasterInterface; +use super::combine; use super::io; use super::operation_handler; use super::operation_handler::OperationResources; @@ -147,6 +148,10 @@ fn combine_map_results( output_dir: &str, task_id: &str, ) -> Result<()> { + combine::optional_run_combine(resources).chain_err( + || "Error running combine operation.", + )?; + let partition_map; { let operation_state = resources.operation_state.lock().unwrap(); diff --git a/worker/src/operations/mod.rs b/worker/src/operations/mod.rs index 3b6627b5..c8c69dd4 100644 --- a/worker/src/operations/mod.rs +++ b/worker/src/operations/mod.rs @@ -1,3 +1,4 @@ +mod combine; pub mod io; mod map; mod reduce; diff --git a/worker/src/operations/operation_handler.rs b/worker/src/operations/operation_handler.rs index 92c6e0ef..ba2af7ad 100644 --- a/worker/src/operations/operation_handler.rs +++ b/worker/src/operations/operation_handler.rs @@ -20,6 +20,7 @@ pub struct OperationResources { pub operation_state: Arc>, pub master_interface: Arc, pub data_abstraction_layer: Arc, + pub binary_path: String, } /// `OperationHandler` is used for executing Map and Reduce operations queued by the Master @@ -123,6 +124,7 @@ impl OperationHandler { operation_state: Arc::clone(&self.operation_state), master_interface: Arc::clone(&self.master_interface), data_abstraction_layer: Arc::clone(&self.data_abstraction_layer), + binary_path: map_options.get_mapper_file_path().to_string(), }; let output_dir_uuid = self.output_dir_uuid.clone(); @@ -142,6 +144,7 @@ impl OperationHandler { operation_state: Arc::clone(&self.operation_state), master_interface: Arc::clone(&self.master_interface), data_abstraction_layer: Arc::clone(&self.data_abstraction_layer), + binary_path: reduce_request.get_reducer_file_path().to_string(), }; let output_dir_uuid = self.output_dir_uuid.clone(); diff --git a/worker/src/operations/reduce.rs b/worker/src/operations/reduce.rs index fc7743c6..2cd29073 100644 --- a/worker/src/operations/reduce.rs +++ b/worker/src/operations/reduce.rs @@ -187,12 +187,12 @@ fn create_reduce_operations( if let serde_json::Value::Array(ref pairs) = parsed_value { for pair in pairs { let key = pair["key"].as_str().chain_err( - || "Error parsing reduce input.", + || "Error parsing reduce input key.", )?; let value = pair["value"].clone(); if value.is_null() { - return Err("Error parsing reduce input.".into()); + return Err("Error parsing reduce input value.".into()); } let reduce_array = reduce_map.entry(key.to_owned()).or_insert_with(Vec::new); From f5e097e3b0b0f95167742106b141c8ec2e9e76bf Mon Sep 17 00:00:00 2001 From: Darragh Griffin Date: Mon, 5 Mar 2018 16:21:42 +0000 Subject: [PATCH 09/19] Make it so user code does not have to use NullCombiner directly --- libcerberus/examples/distributed-grep.rs | 13 ++++++------- libcerberus/examples/end-to-end.rs | 13 ++++++------- libcerberus/examples/word-counter.rs | 8 ++------ libcerberus/src/lib.rs | 2 +- libcerberus/src/runner.rs | 17 ++++++++++++++++- 5 files changed, 31 insertions(+), 22 deletions(-) diff --git a/libcerberus/examples/distributed-grep.rs b/libcerberus/examples/distributed-grep.rs index 96bf8458..cce82db2 100644 --- a/libcerberus/examples/distributed-grep.rs +++ b/libcerberus/examples/distributed-grep.rs @@ -66,13 +66,12 @@ fn run() -> Result<()> { let matches = cerberus::parse_command_line(); - let registry = - UserImplRegistryBuilder::::new() - .mapper(&grep_mapper) - .reducer(&grep_reducer) - .partitioner(&grep_partitioner) - .build() - .chain_err(|| "Error building UserImplRegistry.")?; + let registry = UserImplRegistryBuilder::new_no_combiner() + .mapper(&grep_mapper) + .reducer(&grep_reducer) + .partitioner(&grep_partitioner) + .build() + .chain_err(|| "Error building UserImplRegistry.")?; cerberus::run(&matches, ®istry) } diff --git a/libcerberus/examples/end-to-end.rs b/libcerberus/examples/end-to-end.rs index ada789fb..78caf9c8 100644 --- a/libcerberus/examples/end-to-end.rs +++ b/libcerberus/examples/end-to-end.rs @@ -69,13 +69,12 @@ fn run() -> Result<()> { let matches = cerberus::parse_command_line(); - let registry = - UserImplRegistryBuilder::::new() - .mapper(&test_mapper) - .reducer(&test_reducer) - .partitioner(&test_partitioner) - .build() - .chain_err(|| "Error building UserImplRegistry.")?; + let registry = UserImplRegistryBuilder::new_no_combiner() + .mapper(&test_mapper) + .reducer(&test_reducer) + .partitioner(&test_partitioner) + .build() + .chain_err(|| "Error building UserImplRegistry.")?; cerberus::run(&matches, ®istry) } diff --git a/libcerberus/examples/word-counter.rs b/libcerberus/examples/word-counter.rs index 5d619790..b34a0e1b 100644 --- a/libcerberus/examples/word-counter.rs +++ b/libcerberus/examples/word-counter.rs @@ -55,12 +55,8 @@ fn run() -> Result<()> { let matches = cerberus::parse_command_line(); - let registry = UserImplRegistryBuilder::< - WordCountMapper, - WordCountReducer, - HashPartitioner, - NullCombiner, - >::new().mapper(&wc_mapper) + let registry = UserImplRegistryBuilder::new_no_combiner() + .mapper(&wc_mapper) .reducer(&wc_reducer) .partitioner(&wc_partitioner) .build() diff --git a/libcerberus/src/lib.rs b/libcerberus/src/lib.rs index ba723e14..36b37046 100644 --- a/libcerberus/src/lib.rs +++ b/libcerberus/src/lib.rs @@ -35,7 +35,7 @@ pub mod reducer; pub mod runner; pub mod serialise; -pub use combiner::{Combine, NullCombiner}; +pub use combiner::Combine; pub use errors::*; pub use emitter::{EmitIntermediate, EmitPartitionedIntermediate, EmitFinal}; pub use intermediate::IntermediateInputKV; diff --git a/libcerberus/src/runner.rs b/libcerberus/src/runner.rs index 1da0f16e..81e25d92 100644 --- a/libcerberus/src/runner.rs +++ b/libcerberus/src/runner.rs @@ -12,7 +12,7 @@ use io::*; use mapper::Map; use partition::{Partition, PartitionInputPairs}; use reducer::Reduce; -use combiner::Combine; +use combiner::{Combine, NullCombiner}; use serialise::{FinalOutputObject, FinalOutputObjectEmitter, IntermediateOutputObject, IntermediateOutputPair, IntermediateOutputObjectEmitter, IntermediateOutputPairEmitter}; @@ -120,6 +120,21 @@ where } } +impl<'a, M, R, P> UserImplRegistryBuilder<'a, M, R, P, NullCombiner> +where + M: Map + 'a, + R: Reduce + 'a, + P: Partition< + M::Key, + M::Value, + > + + 'a, +{ + pub fn new_no_combiner() -> UserImplRegistryBuilder<'a, M, R, P, NullCombiner> { + Default::default() + } +} + /// `parse_command_line` uses `clap` to parse the command-line arguments passed to the payload. /// /// The output of this function is required by the `run` function, to decide what subcommand to From a46b94117c570a42c5e37f64289463196ba29b9e Mon Sep 17 00:00:00 2001 From: Darragh Griffin Date: Mon, 5 Mar 2018 18:37:07 +0000 Subject: [PATCH 10/19] Add combiner to word-counter example and let combine return multiple values --- libcerberus/examples/word-counter.rs | 22 ++++++++++++++- libcerberus/src/combiner.rs | 25 ++++------------- libcerberus/src/io.rs | 10 ++----- libcerberus/src/reducer.rs | 5 ++-- libcerberus/src/runner.rs | 41 ++++++++++++++++++---------- libcerberus/src/serialise.rs | 29 ++++++++------------ worker/src/operations/combine.rs | 34 +++++++++++++++++++---- worker/src/operations/map.rs | 12 +++++++- worker/src/operations/reduce.rs | 12 +++++++- 9 files changed, 122 insertions(+), 68 deletions(-) diff --git a/libcerberus/examples/word-counter.rs b/libcerberus/examples/word-counter.rs index b34a0e1b..eea4556e 100644 --- a/libcerberus/examples/word-counter.rs +++ b/libcerberus/examples/word-counter.rs @@ -44,6 +44,24 @@ impl Reduce for WordCountReducer { } } +pub struct WordCountCombiner; +impl Combine for WordCountCombiner { + fn combine(&self, input: IntermediateInputKV, mut emitter: E) -> Result<()> + where + E: EmitFinal, + { + let mut total: u64 = 0; + for val in input.values { + total += val; + } + emitter.emit(total).chain_err(|| { + format!("Error emitting value {:?}.", total) + })?; + + Ok(()) + } +} + fn run() -> Result<()> { env_logger::init().chain_err( || "Failed to initialise logging.", @@ -52,13 +70,15 @@ fn run() -> Result<()> { let wc_mapper = WordCountMapper; let wc_reducer = WordCountReducer; let wc_partitioner = HashPartitioner::new(MAP_OUTPUT_PARTITIONS); + let wc_combiner = WordCountCombiner; let matches = cerberus::parse_command_line(); - let registry = UserImplRegistryBuilder::new_no_combiner() + let registry = UserImplRegistryBuilder::new() .mapper(&wc_mapper) .reducer(&wc_reducer) .partitioner(&wc_partitioner) + .combiner(&wc_combiner) .build() .chain_err(|| "Error building UserImplRegistry.")?; diff --git a/libcerberus/src/combiner.rs b/libcerberus/src/combiner.rs index 7af779e6..d1a745d9 100644 --- a/libcerberus/src/combiner.rs +++ b/libcerberus/src/combiner.rs @@ -1,9 +1,10 @@ -use emitter::EmitIntermediate; -use errors::*; -use intermediate::IntermediateInputKV; use serde::Serialize; use serde::de::DeserializeOwned; +use emitter::EmitFinal; +use errors::*; +use intermediate::IntermediateInputKV; + /// The `Combine` trait defines a function for performing a combine operation. /// /// The output types are decided by the implementation of this trait. @@ -11,7 +12,7 @@ use serde::de::DeserializeOwned; /// # Arguments /// /// * `input` - A `IntermediateInputKV` containing the input data for the combine operation. -/// * `emitter` - A struct implementing the `EmitIntermediate` trait, +/// * `emitter` - A struct implementing the `EmitFinal` trait, /// provided by the combine runner. /// /// # Outputs @@ -24,19 +25,5 @@ where { fn combine(&self, input: IntermediateInputKV, emitter: E) -> Result<()> where - E: EmitIntermediate; -} - -// A null implementation for `Combine` as this is optional component. -pub struct NullCombiner; -impl Combine for NullCombiner -where - V: Default + Serialize + DeserializeOwned, -{ - fn combine(&self, _input: IntermediateInputKV, _emitter: E) -> Result<()> - where - E: EmitIntermediate, - { - Ok(()) - } + E: EmitFinal; } diff --git a/libcerberus/src/io.rs b/libcerberus/src/io.rs index d4324e4a..689c0bcc 100644 --- a/libcerberus/src/io.rs +++ b/libcerberus/src/io.rs @@ -5,7 +5,7 @@ use intermediate::IntermediateInputKV; use serde::Serialize; use serde::de::DeserializeOwned; use serde_json; -use serialise::{FinalOutputObject, IntermediateOutputObject, IntermediateOutputPair}; +use serialise::{FinalOutputObject, IntermediateOutputObject}; use std::io::{Read, Write}; /// `read_map_input` reads bytes from a source and returns a `MapInputKV`. @@ -45,14 +45,10 @@ where Ok(result) } -/// `write_intermediate_pair` attempts to serialise an `IntermediateOutputPair` to a given sink. -pub fn write_intermediate_pair( - sink: &mut W, - output: &IntermediateOutputPair, -) -> Result<()> +/// `write_intermediate_vector` attempts to serialise an `Vec` to a given sink. +pub fn write_intermediate_vector(sink: &mut W, output: &Vec) -> Result<()> where W: Write, - K: Default + Serialize, V: Default + Serialize, { serde_json::to_writer(sink, &output).chain_err( diff --git a/libcerberus/src/reducer.rs b/libcerberus/src/reducer.rs index 9ae8ddc4..bbde7c95 100644 --- a/libcerberus/src/reducer.rs +++ b/libcerberus/src/reducer.rs @@ -1,8 +1,9 @@ +use serde::Serialize; +use serde::de::DeserializeOwned; + use emitter::EmitFinal; use errors::*; use intermediate::IntermediateInputKV; -use serde::Serialize; -use serde::de::DeserializeOwned; /// The `Reduce` trait defines a function for performing a reduce operation. /// diff --git a/libcerberus/src/runner.rs b/libcerberus/src/runner.rs index 81e25d92..6cfac36b 100644 --- a/libcerberus/src/runner.rs +++ b/libcerberus/src/runner.rs @@ -6,16 +6,16 @@ use serde::Serialize; use serde::de::DeserializeOwned; use uuid::Uuid; -use emitter::IntermediateVecEmitter; +use combiner::Combine; +use emitter::{EmitFinal, IntermediateVecEmitter}; use errors::*; use io::*; +use intermediate::IntermediateInputKV; use mapper::Map; use partition::{Partition, PartitionInputPairs}; use reducer::Reduce; -use combiner::{Combine, NullCombiner}; use serialise::{FinalOutputObject, FinalOutputObjectEmitter, IntermediateOutputObject, - IntermediateOutputPair, IntermediateOutputObjectEmitter, - IntermediateOutputPairEmitter}; + IntermediateOutputObjectEmitter, VecEmitter}; use super::VERSION; /// `UserImplRegistry` tracks the user's implementations of Map, Reduce, etc. @@ -120,6 +120,22 @@ where } } +/// A null implementation for `Combine` as this is optional component. +/// This should not be used by user code. +pub struct NullCombiner; +impl Combine for NullCombiner +where + V: Default + Serialize + DeserializeOwned, +{ + fn combine(&self, _input: IntermediateInputKV, _emitter: E) -> Result<()> + where + E: EmitFinal, + { + Err("This code should never run".into()) + } +} + +/// Construct a UserImplRegistryBuilder that does not need a `Combine` implementation impl<'a, M, R, P> UserImplRegistryBuilder<'a, M, R, P, NullCombiner> where M: Map + 'a, @@ -179,7 +195,7 @@ where )?; Ok(()) } - Some("combiner") => { + Some("combine") => { let combiner = registry.combiner.chain_err( || "Attempt to run combine command when combiner is not implemented", )?; @@ -198,8 +214,7 @@ where eprintln!("{}", matches.usage()); Ok(()) } - // This won't ever be reached, due to clap checking invalid commands before this. - _ => Ok(()), + Some(other) => Err(format!("Unknown command {}", other).into()), } } @@ -260,18 +275,16 @@ where let mut source = stdin(); let mut sink = stdout(); let input_kv = read_intermediate_input(&mut source).chain_err( - || "Error getting input to reduce.", + || "Error getting input to combine.", )?; - let mut output_object = IntermediateOutputPair::::default(); + + let mut output_object = Vec::::new(); combiner - .combine( - input_kv, - IntermediateOutputPairEmitter::new(&mut output_object), - ) + .combine(input_kv, VecEmitter::new(&mut output_object)) .chain_err(|| "Error running combine operation.")?; - write_intermediate_pair(&mut sink, &output_object) + write_intermediate_vector(&mut sink, &output_object) .chain_err(|| "Error writing combine output to stdout.")?; Ok(()) } diff --git a/libcerberus/src/serialise.rs b/libcerberus/src/serialise.rs index ac5947cb..64293b74 100644 --- a/libcerberus/src/serialise.rs +++ b/libcerberus/src/serialise.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use serde::Serialize; -use emitter::{EmitIntermediate, EmitFinal, EmitPartitionedIntermediate}; +use emitter::{EmitFinal, EmitPartitionedIntermediate}; use errors::*; /// `IntermediateOutputPair` is a struct representing an intermediate key-value pair as outputted @@ -35,40 +35,35 @@ pub struct FinalOutputObject { pub values: Vec, } -/// A struct implementing `EmitIntermediate` which emits to an `IntermediateOutputPair`. -pub struct IntermediateOutputPairEmitter<'a, K, V> +/// A struct implementing `EmitFinal` which emits to a `Vec` of intermediate values. +pub struct VecEmitter<'a, V> where - K: Default + Serialize + 'a, V: Default + Serialize + 'a, { - sink: &'a mut IntermediateOutputPair, + sink: &'a mut Vec, } -impl<'a, K, V> IntermediateOutputPairEmitter<'a, K, V> +impl<'a, V> VecEmitter<'a, V> where - K: Default + Serialize, V: Default + Serialize, { - /// Constructs a new `IntermediateOutputPairEmitter` with a mutable reference to a given - /// `IntermediateOutputPair`. + /// Constructs a new `VecEmitter` with a mutable reference to a given `Vec`. /// /// # Arguments /// - /// * `sink` - A mutable reference to the `IntermediateOutputPair` + /// * `sink` - A mutable reference to the `Vec` /// to receive the emitted values. - pub fn new(sink: &'a mut IntermediateOutputPair) -> Self { - IntermediateOutputPairEmitter { sink: sink } + pub fn new(sink: &'a mut Vec) -> Self { + VecEmitter { sink: sink } } } -impl<'a, K, V> EmitIntermediate for IntermediateOutputPairEmitter<'a, K, V> +impl<'a, V> EmitFinal for VecEmitter<'a, V> where - K: Default + Serialize, V: Default + Serialize, { - fn emit(&mut self, key: K, value: V) -> Result<()> { - self.sink.key = key; - self.sink.value = value; + fn emit(&mut self, value: V) -> Result<()> { + self.sink.push(value); Ok(()) } } diff --git a/worker/src/operations/combine.rs b/worker/src/operations/combine.rs index f5c7671c..349bf465 100644 --- a/worker/src/operations/combine.rs +++ b/worker/src/operations/combine.rs @@ -45,7 +45,7 @@ fn do_combine_operation( .arg("combine") .stdin(Stdio::piped()) .stdout(Stdio::piped()) - .stderr(Stdio::null()) + .stderr(Stdio::piped()) .spawn() .chain_err(|| "Failed to start combine operation process.")?; @@ -65,11 +65,21 @@ fn do_combine_operation( || "Error accessing payload output.", )?; - let combine_result: serde_json::Value = serde_json::from_str(&output_str).chain_err( - || "Error parsing reduce results.", + let stderr_str = String::from_utf8(output.stderr).chain_err( + || "Error accessing payload output.", + )?; + + if !stderr_str.is_empty() { + return Err( + format!("MapReduce binary failed with stderr:\n {}", stderr_str).into(), + ); + } + + let combine_results: serde_json::Value = serde_json::from_str(&output_str).chain_err( + || "Error parsing combine results.", )?; - Ok(combine_result) + Ok(combine_results) } fn run_combine(resources: &OperationResources) -> Result<()> { @@ -101,10 +111,22 @@ fn run_combine(resources: &OperationResources) -> Result<()> { for (key, values) in (&kv_map).iter() { if values.len() > 1 { - let pair = do_combine_operation(resources, key, values.clone()) + let results = do_combine_operation(resources, key, values.clone()) .chain_err(|| "Failed to run combine operation.")?; - partition_results.push(pair); + if let serde_json::Value::Array(ref values) = results { + for value in values { + partition_results.push(json!({ + "key": key, + "value": value + })); + } + } else { + partition_results.push(json!({ + "key": key, + "value": results + })); + } } else if let Some(value) = values.first() { partition_results.push(json!({ "key": key, diff --git a/worker/src/operations/map.rs b/worker/src/operations/map.rs index b3102709..acad38e0 100644 --- a/worker/src/operations/map.rs +++ b/worker/src/operations/map.rs @@ -94,6 +94,16 @@ fn map_operation_thread_impl( || "Error accessing payload output.", )?; + let stderr_str = String::from_utf8(output.stderr).chain_err( + || "Error accessing payload output.", + )?; + + if !stderr_str.is_empty() { + return Err( + format!("MapReduce binary failed with stderr:\n {}", stderr_str).into(), + ); + } + let map_results = parse_map_results(&output_str).chain_err( || "Error parsing map results.", )?; @@ -314,7 +324,7 @@ fn internal_perform_map( .arg("map") .stdin(Stdio::piped()) .stdout(Stdio::piped()) - .stderr(Stdio::null()) + .stderr(Stdio::piped()) .spawn() .chain_err(|| "Failed to start map operation process.")?; diff --git a/worker/src/operations/reduce.rs b/worker/src/operations/reduce.rs index 2cd29073..5cd95bae 100644 --- a/worker/src/operations/reduce.rs +++ b/worker/src/operations/reduce.rs @@ -72,6 +72,16 @@ fn run_reducer( || "Error accessing payload output.", )?; + let stderr_str = String::from_utf8(output.stderr).chain_err( + || "Error accessing payload output.", + )?; + + if !stderr_str.is_empty() { + return Err( + format!("MapReduce binary failed with stderr:\n {}", stderr_str).into(), + ); + } + let reduce_results: serde_json::Value = serde_json::from_str(&output_str).chain_err( || "Error parsing reduce results.", )?; @@ -110,7 +120,7 @@ impl ReduceOperationQueue { .arg("reduce") .stdin(Stdio::piped()) .stdout(Stdio::piped()) - .stderr(Stdio::null()) + .stderr(Stdio::piped()) .spawn() .chain_err(|| "Failed to start reduce operation process.")?; From 99c97aaf8da61ae579c741c9f7bdf8b937dc23ce Mon Sep 17 00:00:00 2001 From: Darragh Griffin Date: Wed, 28 Feb 2018 22:16:16 +0000 Subject: [PATCH 11/19] Change worker manager to allow a task to be ran more than once at a time --- master/src/worker_management/state.rs | 193 +++++++++++++------------- 1 file changed, 95 insertions(+), 98 deletions(-) diff --git a/master/src/worker_management/state.rs b/master/src/worker_management/state.rs index 7fe8a498..6ebf89b8 100644 --- a/master/src/worker_management/state.rs +++ b/master/src/worker_management/state.rs @@ -16,11 +16,10 @@ const MAX_TASK_ASSIGNMENT_FAILURE: u16 = 5; pub struct State { // A map of worker id to worker. workers: HashMap, + // A map of task id to task. + tasks: HashMap, // Tasks that are in the queue to be ran. - task_queue: VecDeque, - // Tasks that are currently running on a worker. - // A map of worker id to ScheduledTask. - assigned_tasks: HashMap, + task_queue: VecDeque, } impl State { @@ -88,28 +87,20 @@ impl State { worker.operation_status = operation_status; worker.status_last_updated = Utc::now(); - if !worker.current_task_id.is_empty() && - operation_status != pb::OperationStatus::IN_PROGRESS - { - if let Some(assigned_task) = self.assigned_tasks.remove(worker_id) { - self.task_queue.push_front(assigned_task); - } - worker.current_task_id = String::new(); - } - Ok(()) } pub fn process_reduce_task_result(&mut self, reduce_result: &pb::ReduceResult) -> Result { if reduce_result.status == pb::ResultStatus::SUCCESS { - let mut scheduled_task = self.assigned_tasks - .remove(reduce_result.get_worker_id()) - .chain_err(|| { + let mut scheduled_task = self.tasks.remove(reduce_result.get_task_id()).chain_err( + || { format!( - "Task not found for Worker with ID {}.", - reduce_result.get_worker_id(), + "Task with ID {} not found, Worker ID: {}.", + reduce_result.get_task_id(), + reduce_result.get_worker_id() ) - })?; + }, + )?; if scheduled_task.id != reduce_result.task_id { return Err("Task id does not match expected task id.".into()); @@ -123,20 +114,20 @@ impl State { self.task_failed( reduce_result.get_worker_id(), + reduce_result.get_task_id(), reduce_result.get_failure_details(), ).chain_err(|| "Error marking task as failed") } pub fn process_map_task_result(&mut self, map_result: &pb::MapResult) -> Result { if map_result.status == pb::ResultStatus::SUCCESS { - let mut scheduled_task = self.assigned_tasks - .remove(&map_result.worker_id) - .chain_err(|| { - format!( - "Task not found for Worker with ID {}.", - map_result.worker_id, - ) - })?; + let mut scheduled_task = self.tasks.remove(map_result.get_task_id()).chain_err(|| { + format!( + "Task with ID {} not found, Worker ID: {}.", + map_result.get_task_id(), + map_result.get_worker_id() + ) + })?; if scheduled_task.id != map_result.task_id { return Err("Task id does not match expected task id.".into()); @@ -163,22 +154,31 @@ impl State { return Ok(scheduled_task); } - self.task_failed(map_result.get_worker_id(), map_result.get_failure_details()) - .chain_err(|| "Error marking task as failed") + self.task_failed( + map_result.get_worker_id(), + map_result.get_task_id(), + map_result.get_failure_details(), + ).chain_err(|| "Error marking task as failed") } - fn task_failed(&mut self, worker_id: &str, failure_details: &str) -> Result<(Task)> { + fn task_failed( + &mut self, + worker_id: &str, + task_id: &str, + failure_details: &str, + ) -> Result<(Task)> { let worker = self.workers.get_mut(worker_id).chain_err(|| { format!("Worker with ID {} not found.", worker_id) })?; worker.current_task_id = String::new(); - let mut assigned_task = self.assigned_tasks.remove(worker_id).chain_err(|| { + let mut assigned_task = self.tasks.remove(task_id).chain_err(|| { format!( - "Task not found for Worker with ID {}.", - worker_id, - ) + "Task with ID {} not found, Worker ID: {}.", + task_id, + worker_id + ) })?; if failure_details != "" { @@ -191,7 +191,11 @@ impl State { assigned_task.time_completed = Some(Utc::now()); } else { assigned_task.status = TaskStatus::Queued; - self.task_queue.push_front(assigned_task.clone()); + self.tasks.insert( + task_id.clone().to_owned(), + assigned_task.clone(), + ); + self.task_queue.push_front(task_id.clone().to_owned()); } Ok(assigned_task) @@ -220,80 +224,79 @@ impl State { } pub fn remove_worker(&mut self, worker_id: &str) -> Result<()> { - if !self.workers.contains_key(worker_id) { - return Err(format!("Worker with ID {} not found.", worker_id).into()); - } + if let Some(worker) = self.workers.remove(worker_id) { - // If this worker is a assigned a task, requeue the task. - if let Some(assigned_task) = self.assigned_tasks.remove(worker_id) { - self.task_queue.push_front(assigned_task); + // If this worker is a assigned a task, requeue the task. + if !worker.current_task_id.is_empty() { + self.task_queue.push_front(worker.current_task_id); + } + } else { + return Err(format!("Worker with ID {} not found.", worker_id).into()); } - self.workers.remove(worker_id); Ok(()) } pub fn add_task(&mut self, task: Task) { - self.task_queue.push_back(task); + self.tasks.insert(task.id.clone(), task.clone()); + self.task_queue.push_back(task.id); } // Unassign a task assigned to a worker and put the task back in the queue. pub fn unassign_worker(&mut self, worker_id: &str) -> Result<()> { - if let Some(assigned_task) = self.assigned_tasks.remove(worker_id) { - self.task_queue.push_front(assigned_task); - } - let worker = self.workers.get_mut(worker_id).chain_err(|| { format!("Worker with ID {} not found.", worker_id) })?; + if !worker.current_task_id.is_empty() { + self.task_queue.push_front(worker.current_task_id.clone()); + } + worker.current_task_id = String::new(); Ok(()) } - // Tries to assign a worker the next task in the queue. - // Returns the task if one exists. - pub fn try_assign_worker_task(&mut self, worker_id: &str) -> Result<(Option)> { + // Assigns a given task_id to a worker + fn assign_worker_task(&mut self, worker_id: &str, task_id: &str) -> Result<(Task)> { + let assigned_task = { + if let Some(scheduled_task) = self.tasks.get_mut(task_id) { + scheduled_task.status = TaskStatus::InProgress; + if scheduled_task.time_started == None { + scheduled_task.time_started = Some(Utc::now()); + } + scheduled_task.assigned_worker_id = worker_id.clone().to_owned(); + + scheduled_task.clone() + } else { + return Err(format!("Could not get Task with ID {}", task_id).into()); + } + }; + let worker = self.workers.get_mut(worker_id).chain_err(|| { format!("Worker with ID {} not found.", worker_id) })?; + worker.current_task_id = task_id.clone().to_owned(); + + Ok(assigned_task) + } - let mut scheduled_task = match self.task_queue.pop_front() { - Some(scheduled_task) => scheduled_task, + // Tries to assign a worker the next task in the queue. + // Returns the task if one exists. + pub fn try_assign_worker_task(&mut self, worker_id: &str) -> Result<(Option)> { + let scheduled_task_id = match self.task_queue.pop_front() { + Some(scheduled_task_id) => scheduled_task_id, None => return Ok(None), }; - scheduled_task.status = TaskStatus::InProgress; - if scheduled_task.time_started == None { - scheduled_task.time_started = Some(Utc::now()); + match self.assign_worker_task(worker_id, &scheduled_task_id) { + Ok(task) => Ok(Some(task)), + Err(err) => Err(err), } - scheduled_task.assigned_worker_id = worker.worker_id.to_owned(); - - let task = scheduled_task.clone(); - worker.current_task_id = scheduled_task.id.to_owned(); - self.assigned_tasks.insert( - worker_id.to_owned(), - scheduled_task, - ); - - Ok(Some(task)) } pub fn has_task(&self, task_id: &str) -> bool { - for task in self.assigned_tasks.values() { - if task.id == task_id { - return true; - } - } - - for task in &self.task_queue { - if task.id == task_id { - return true; - } - } - - false + self.tasks.contains_key(task_id) } pub fn increment_failed_task_assignments(&mut self, worker_id: &str) -> Result<()> { @@ -341,20 +344,15 @@ impl state::StateHandling for State { )?); } - let mut task_queue_json: Vec = Vec::new(); - for task in &self.task_queue { - task_queue_json.push(task.dump_state().chain_err(|| "Error dumping task state.")?); - } - - let mut assigned_tasks_json: Vec = Vec::new(); - for task in self.assigned_tasks.values() { - assigned_tasks_json.push(task.dump_state().chain_err(|| "Error dumping task state.")?); + let mut tasks_json: Vec = Vec::new(); + for task in self.tasks.values() { + tasks_json.push(task.dump_state().chain_err(|| "Error dumping task state.")?); } Ok(json!({ "workers": json!(workers_json), - "task_queue": json!(task_queue_json), - "assigned_tasks": json!(assigned_tasks_json), + "task_queue": json!(self.task_queue), + "tasks": json!(tasks_json), })) } @@ -371,28 +369,27 @@ impl state::StateHandling for State { } if let serde_json::Value::Array(ref task_queue) = data["task_queue"] { - for task in task_queue { - let task = Task::new_from_json(task.clone()).chain_err( - || "Unable to create Task from json.", + for task_id in task_queue { + let task_id_str: String = serde_json::from_value(task_id.clone()).chain_err( + || "Error processing task_id in tasks queue", )?; - self.task_queue.push_back(task); + + self.task_queue.push_front(task_id_str); } } else { return Err("Error processing tasks queue.".into()); } - if let serde_json::Value::Array(ref tasks_array) = data["assigned_tasks"] { + if let serde_json::Value::Array(ref tasks_array) = data["tasks"] { for task in tasks_array { let task = Task::new_from_json(task.clone()).chain_err( || "Unable to create Task from json.", )?; - self.assigned_tasks.insert( - task.assigned_worker_id.to_owned(), - task, - ); + debug!("Loaded task from state:\n {:?}", task); + self.tasks.insert(task.id.to_owned(), task); } } else { - return Err("Error processing assigned tasks array.".into()); + return Err("Error processing tasks array.".into()); } Ok(()) From f4468e893abcbdcb9bd57347237927116c6f9ea9 Mon Sep 17 00:00:00 2001 From: Ryan Connell Date: Tue, 6 Mar 2018 22:45:16 +0000 Subject: [PATCH 12/19] Add Job & Task Cancel functionality Jobs and Tasks can now be cancelled while in-progress or enqueued. --- cli/src/runner.rs | 1 + master/src/common/job.rs | 3 + master/src/common/task.rs | 1 + master/src/common/worker.rs | 5 ++ master/src/scheduling/scheduler.rs | 25 ++++++++- master/src/scheduling/state.rs | 18 ++++++ master/src/server/client_service.rs | 4 ++ .../worker_communication/worker_interface.rs | 15 +++++ master/src/worker_management/state.rs | 56 +++++++++++++++++++ .../src/worker_management/worker_manager.rs | 31 ++++++++++ proto/mapreduce.proto | 1 + proto/worker.proto | 9 +++ worker/src/operations/map.rs | 25 +++++++++ worker/src/operations/operation_handler.rs | 15 +++++ worker/src/operations/reduce.rs | 17 ++++++ worker/src/operations/state.rs | 10 ++++ worker/src/server/master_service.rs | 12 ++++ 17 files changed, 246 insertions(+), 2 deletions(-) diff --git a/cli/src/runner.rs b/cli/src/runner.rs index 86c7ce23..b97e37ed 100644 --- a/cli/src/runner.rs +++ b/cli/src/runner.rs @@ -202,6 +202,7 @@ fn print_table(rep: &pb::MapReduceReport) { } pb::Status::IN_QUEUE => format!("IN_QUEUE ({})", rep.get_queue_length()), pb::Status::FAILED => format!("FAILED\n{}", rep.get_failure_details()).to_owned(), + pb::Status::CANCELLED => "CANCELLED".to_owned(), }; table!(["MRID", id], ["Status", status], ["Output", output]).printstd(); diff --git a/master/src/common/job.rs b/master/src/common/job.rs index 321c1aba..dc406a5a 100644 --- a/master/src/common/job.rs +++ b/master/src/common/job.rs @@ -77,6 +77,7 @@ pub enum SerializableJobStatus { IN_QUEUE, FAILED, UNKNOWN, + CANCELLED, } impl Job { @@ -208,6 +209,7 @@ impl Job { SerializableJobStatus::IN_QUEUE => pb::Status::IN_QUEUE, SerializableJobStatus::FAILED => pb::Status::FAILED, SerializableJobStatus::UNKNOWN => pb::Status::UNKNOWN, + SerializableJobStatus::CANCELLED => pb::Status::CANCELLED, } } @@ -218,6 +220,7 @@ impl Job { pb::Status::IN_QUEUE => SerializableJobStatus::IN_QUEUE, pb::Status::FAILED => SerializableJobStatus::FAILED, pb::Status::UNKNOWN => SerializableJobStatus::UNKNOWN, + pb::Status::CANCELLED => SerializableJobStatus::CANCELLED, } } } diff --git a/master/src/common/task.rs b/master/src/common/task.rs index 44743969..bc8f081f 100644 --- a/master/src/common/task.rs +++ b/master/src/common/task.rs @@ -16,6 +16,7 @@ pub enum TaskStatus { InProgress, Complete, Failed, + Cancelled, } #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] diff --git a/master/src/common/worker.rs b/master/src/common/worker.rs index ca683a0c..91c27d08 100644 --- a/master/src/common/worker.rs +++ b/master/src/common/worker.rs @@ -24,6 +24,7 @@ pub enum OperationStatus { IN_PROGRESS, COMPLETE, FAILED, + CANCELLED, UNKNOWN, } @@ -36,6 +37,7 @@ pub struct Worker { pub status_last_updated: DateTime, pub current_task_id: String, + pub last_cancelled_task_id: Option, pub worker_id: String, pub task_assignments_failed: u16, @@ -55,6 +57,7 @@ impl Worker { status_last_updated: Utc::now(), current_task_id: String::new(), + last_cancelled_task_id: None, worker_id: Uuid::new_v4().to_string(), task_assignments_failed: 0, @@ -66,6 +69,7 @@ impl Worker { OperationStatus::IN_PROGRESS => pb::OperationStatus::IN_PROGRESS, OperationStatus::COMPLETE => pb::OperationStatus::COMPLETE, OperationStatus::FAILED => pb::OperationStatus::FAILED, + OperationStatus::CANCELLED => pb::OperationStatus::CANCELLED, OperationStatus::UNKNOWN => pb::OperationStatus::UNKNOWN, } } @@ -75,6 +79,7 @@ impl Worker { pb::OperationStatus::IN_PROGRESS => OperationStatus::IN_PROGRESS, pb::OperationStatus::COMPLETE => OperationStatus::COMPLETE, pb::OperationStatus::FAILED => OperationStatus::FAILED, + pb::OperationStatus::CANCELLED => OperationStatus::CANCELLED, pb::OperationStatus::UNKNOWN => OperationStatus::UNKNOWN, } } diff --git a/master/src/scheduling/scheduler.rs b/master/src/scheduling/scheduler.rs index 5e7592d1..dad15532 100644 --- a/master/src/scheduling/scheduler.rs +++ b/master/src/scheduling/scheduler.rs @@ -141,8 +141,29 @@ impl Scheduler { ) } - // TODO(rhino): Continue this - pub fn cancel_job(&self, _job_id: &str) -> Result<()> { + pub fn cancel_job(&self, job_id: &str) -> Result<()> { + { + let mut state = self.state.lock().unwrap(); + state.cancel_job(job_id).chain_err(|| { + format!("Unable to cancel job with ID: {}", job_id) + })?; + } + + let workers = self.worker_manager + .get_workers_running_job(job_id) + .chain_err(|| { + format!("Unable to get list of workers running job {}", job_id) + })?; + + self.worker_manager + .remove_queued_tasks_for_job(job_id) + .chain_err(|| "Unable to remove queued task from state")?; + + self.worker_manager + .cancel_workers_tasks(workers) + .chain_err(|| "Unable to cancel task on workers")?; + + println!("Succesfully cancelled job {}", job_id); Ok(()) } diff --git a/master/src/scheduling/state.rs b/master/src/scheduling/state.rs index 0d89ddeb..daab111b 100644 --- a/master/src/scheduling/state.rs +++ b/master/src/scheduling/state.rs @@ -121,6 +121,24 @@ impl State { Ok(()) } + pub fn cancel_job(&mut self, job_id: &str) -> Result<()> { + let scheduled_job = match self.scheduled_jobs.get_mut(job_id) { + Some(job) => job, + None => return Err(format!("Job with ID {} was not found.", &job_id).into()), + }; + + let job_status = scheduled_job.job.status; + if !(job_status == pb::Status::IN_PROGRESS || job_status == pb::Status::IN_QUEUE) { + return Err( + format!("Unable to cancel job. Expected IN_PROGRESS or IN_QUEUE job. Got {:?}", + job_status).into(), + ); + } + + scheduled_job.job.status = pb::Status::CANCELLED; + Ok(()) + } + pub fn get_job(&self, job_id: &str) -> Result { match self.scheduled_jobs.get(job_id) { Some(scheduled_job) => Ok(scheduled_job.job.clone()), diff --git a/master/src/server/client_service.rs b/master/src/server/client_service.rs index aad75c14..ceb663bf 100644 --- a/master/src/server/client_service.rs +++ b/master/src/server/client_service.rs @@ -205,6 +205,10 @@ mod tests { ) -> Result<()> { Ok(()) } + + fn cancel_task(&self, _request: wpb::CancelTaskRequest, _worker_id: &str) -> Result<()> { + Ok(()) + } } fn create_scheduler() -> Scheduler { diff --git a/master/src/worker_communication/worker_interface.rs b/master/src/worker_communication/worker_interface.rs index 9eb510b9..38b0d5d8 100644 --- a/master/src/worker_communication/worker_interface.rs +++ b/master/src/worker_communication/worker_interface.rs @@ -18,6 +18,7 @@ pub trait WorkerInterface: Sync + Send { fn schedule_map(&self, request: pb::PerformMapRequest, worker_id: &str) -> Result<()>; fn schedule_reduce(&self, request: pb::PerformReduceRequest, worker_id: &str) -> Result<()>; + fn cancel_task(&self, request: pb::CancelTaskRequest, worker_id: &str) -> Result<()>; } #[derive(Default)] @@ -97,4 +98,18 @@ impl WorkerInterface for WorkerInterfaceImpl { Err(NO_CLIENT_FOUND_ERR.into()) } } + + fn cancel_task(&self, request: pb::CancelTaskRequest, worker_id: &str) -> Result<()> { + let clients = self.clients.read().unwrap(); + + if let Some(client) = clients.get(worker_id) { + client + .cancel_task(RequestOptions::new(), request) + .wait() + .chain_err(|| "Failed to cancel task")?; + Ok(()) + } else { + Err(NO_CLIENT_FOUND_ERR.into()) + } + } } diff --git a/master/src/worker_management/state.rs b/master/src/worker_management/state.rs index 6ebf89b8..fede9933 100644 --- a/master/src/worker_management/state.rs +++ b/master/src/worker_management/state.rs @@ -27,6 +27,16 @@ impl State { Default::default() } + pub fn remove_queued_tasks_for_job(&mut self, job_id: &str) -> Result<()> { + let mut new_task_queue = self.task_queue.clone(); + new_task_queue.retain(|t| match self.tasks.get_mut(&t.clone()) { + Some(task) => task.job_id != job_id, + None => false, + }); + self.task_queue = new_task_queue; + Ok(()) + } + pub fn get_worker_count(&self) -> u32 { self.workers.len() as u32 } @@ -106,6 +116,17 @@ impl State { return Err("Task id does not match expected task id.".into()); } + let worker = self.workers.get(&reduce_result.worker_id).chain_err(|| { + format!("Worker with ID {} not found.", reduce_result.worker_id) + })?; + + if let Some(task_id) = worker.last_cancelled_task_id.clone() { + if task_id == reduce_result.task_id { + scheduled_task.status = TaskStatus::Cancelled; + return Ok(scheduled_task); + } + } + scheduled_task.status = TaskStatus::Complete; scheduled_task.time_completed = Some(Utc::now()); scheduled_task.cpu_time = reduce_result.get_cpu_time(); @@ -137,6 +158,13 @@ impl State { format!("Worker with ID {} not found.", map_result.worker_id) })?; + if let Some(task_id) = worker.last_cancelled_task_id.clone() { + if task_id == map_result.task_id { + scheduled_task.status = TaskStatus::Cancelled; + return Ok(scheduled_task); + } + } + for (partition, output_file) in map_result.get_map_results() { scheduled_task.map_output_files.insert( *partition, @@ -201,6 +229,21 @@ impl State { Ok(assigned_task) } + pub fn get_workers_running_job(&self, job_id: &str) -> Result> { + let mut worker_ids = Vec::new(); + + let workers = self.get_workers(); + for worker in workers { + if let Some(task) = self.tasks.get(&worker.current_task_id) { + if task.job_id == job_id { + worker_ids.push(worker.worker_id.clone()); + } + } + } + + Ok(worker_ids) + } + // Mark that a given worker has returned a result for it's task. pub fn set_worker_operation_completed( &mut self, @@ -295,6 +338,19 @@ impl State { } } + // Clears the workers current_task_id and returns the previous value. + pub fn cancel_task_for_worker(&mut self, worker_id: &str) -> Result { + let worker = self.workers.get_mut(worker_id).chain_err(|| { + format!("Worker with ID {} not found.", worker_id) + })?; + + let previous_task_id = worker.current_task_id.clone(); + worker.last_cancelled_task_id = Some(worker.current_task_id.clone()); + worker.current_task_id = String::new(); + + Ok(previous_task_id) + } + pub fn has_task(&self, task_id: &str) -> bool { self.tasks.contains_key(task_id) } diff --git a/master/src/worker_management/worker_manager.rs b/master/src/worker_management/worker_manager.rs index d9b148eb..145ed0c9 100644 --- a/master/src/worker_management/worker_manager.rs +++ b/master/src/worker_management/worker_manager.rs @@ -148,6 +148,27 @@ impl WorkerManager { } } + pub fn cancel_workers_tasks(&self, workers: Vec) -> Result<()> { + let mut state = self.state.lock().unwrap(); + + for worker_id in workers { + // Clear the task from the worker so that we can ignore it's result. + let task_id = state.cancel_task_for_worker(&worker_id).chain_err(|| { + format!("Error cancelling task on worker: {}", worker_id) + })?; + + // Create a request to cancel the task the worker is currently running. + let mut cancel_request = pb::CancelTaskRequest::new(); + cancel_request.task_id = task_id; + + // Tell the worker to cancel what it's doing. + self.worker_interface + .cancel_task(cancel_request, &worker_id) + .chain_err(|| "Error telling worker to cancel task")?; + } + Ok(()) + } + fn remove_workers(&self, workers: Vec) { let mut state = self.state.lock().unwrap(); for worker_id in workers { @@ -311,6 +332,16 @@ impl WorkerManager { let state = self.state.lock().unwrap(); state.has_task(task_id) } + + pub fn remove_queued_tasks_for_job(&self, job_id: &str) -> Result<()> { + let mut state = self.state.lock().unwrap(); + state.remove_queued_tasks_for_job(job_id) + } + + pub fn get_workers_running_job(&self, job_id: &str) -> Result> { + let state = self.state.lock().unwrap(); + state.get_workers_running_job(job_id) + } } impl state::SimpleStateHandling for WorkerManager { diff --git a/proto/mapreduce.proto b/proto/mapreduce.proto index d4e34a04..bb9c3984 100644 --- a/proto/mapreduce.proto +++ b/proto/mapreduce.proto @@ -71,6 +71,7 @@ enum Status { IN_PROGRESS = 2; IN_QUEUE = 3; FAILED = 4; + CANCELLED = 5; }; message MapReduceReport { diff --git a/proto/worker.proto b/proto/worker.proto index cb629311..5ed32fe4 100644 --- a/proto/worker.proto +++ b/proto/worker.proto @@ -45,6 +45,7 @@ enum OperationStatus { IN_PROGRESS = 1; COMPLETE = 2; FAILED = 3; + CANCELLED = 4; }; message UpdateStatusRequest { @@ -95,6 +96,9 @@ service ScheduleOperationService { // Tells a worker to start a reduce operation for a certain partition of the map output. // An error will be returned if the operation is not valid or the worker is busy. rpc PerformReduce (PerformReduceRequest) returns (EmptyMessage); + + // Used by the master to tell a worker to stop working on it's current task. + rpc CancelTask (CancelTaskRequest) returns (EmptyMessage); } // Message specifying where one part of the input for a map task is located. @@ -136,6 +140,11 @@ message PerformReduceRequest { string task_id = 5; } +message CancelTaskRequest { + // The id of the task the worker should cancel. + string task_id = 1; +} + //////////////////////////////////////////////////////////////////////////////// // IntermediateDataService allows reduce worker to get intemediate data from workers. diff --git a/worker/src/operations/map.rs b/worker/src/operations/map.rs index acad38e0..3a641c3b 100644 --- a/worker/src/operations/map.rs +++ b/worker/src/operations/map.rs @@ -239,6 +239,18 @@ fn process_map_result( } } + // If we have cancelled the current task then we should avoid processing the map results. + { + let cancelled = { + let operation_state = resources.operation_state.lock().unwrap(); + operation_state.task_cancelled(task_id) + }; + if cancelled { + operation_handler::set_cancelled_status(&resources.operation_state); + return; + } + } + match result { Ok(map_result) => { let (finished, parse_failed) = { @@ -313,6 +325,19 @@ fn internal_perform_map( } for input_location in input_locations { + // Make sure the job hasn't been cancelled before continuing. + { + let cancelled = { + let operation_state = resources.operation_state.lock().unwrap(); + operation_state.task_cancelled(&map_options.task_id) + }; + if cancelled { + operation_handler::set_cancelled_status(&resources.operation_state); + println!("Succesfully cancelled task: {}", map_options.task_id); + return Ok(()); + } + } + let map_input_value = io::read_location(&resources.data_abstraction_layer, input_location) .chain_err(|| "unable to open input file")?; diff --git a/worker/src/operations/operation_handler.rs b/worker/src/operations/operation_handler.rs index ba2af7ad..77902082 100644 --- a/worker/src/operations/operation_handler.rs +++ b/worker/src/operations/operation_handler.rs @@ -65,6 +65,14 @@ pub fn set_failed_status(operation_state_arc: &Arc>) { ); } +pub fn set_cancelled_status(operation_state_arc: &Arc>) { + set_operation_handler_status( + operation_state_arc, + pb::WorkerStatus::AVAILABLE, + pb::OperationStatus::CANCELLED, + ); +} + pub fn set_busy_status(operation_state_arc: &Arc>) { let mut operation_state = operation_state_arc.lock().unwrap(); @@ -156,6 +164,13 @@ impl OperationHandler { }) } + pub fn cancel_task(&self, request: pb::CancelTaskRequest) -> Result<()> { + let mut operation_state = self.operation_state.lock().unwrap(); + operation_state.last_cancelled_task_id = Some(request.task_id.clone()); + + Ok(()) + } + pub fn update_worker_status(&self) -> Result<()> { let worker_status = self.get_worker_status(); let operation_status = self.get_worker_operation_status(); diff --git a/worker/src/operations/reduce.rs b/worker/src/operations/reduce.rs index 5cd95bae..86747767 100644 --- a/worker/src/operations/reduce.rs +++ b/worker/src/operations/reduce.rs @@ -336,6 +336,23 @@ fn run_reduce_queue( if reduce_queue.is_queue_empty() { break; } else { + + // Make sure the job hasn't been cancelled before continuing. + { + let cancelled = { + let operation_state = resources.operation_state.lock().unwrap(); + operation_state.task_cancelled(&reduce_options.task_id) + }; + if cancelled { + operation_handler::set_cancelled_status(&resources.operation_state); + println!( + "Succesfully cancelled reduce task: {}", + &reduce_options.task_id + ); + return; + } + } + let result = reduce_queue.perform_next_reduce_operation( &reduce_options, &resources.data_abstraction_layer, diff --git a/worker/src/operations/state.rs b/worker/src/operations/state.rs index e2370b41..adf18d16 100644 --- a/worker/src/operations/state.rs +++ b/worker/src/operations/state.rs @@ -19,6 +19,8 @@ pub struct OperationState { pub waiting_map_operations: usize, pub intermediate_map_results: HashMap>, + + pub last_cancelled_task_id: Option, } impl OperationState { @@ -30,10 +32,18 @@ impl OperationState { intermediate_file_store: Vec::new(), waiting_map_operations: 0, intermediate_map_results: HashMap::new(), + last_cancelled_task_id: None, } } pub fn add_intermediate_files(&mut self, files: Vec) { self.intermediate_file_store.extend(files.into_iter()); } + + pub fn task_cancelled(&self, task_id: &str) -> bool { + match self.last_cancelled_task_id.clone() { + Some(id) => task_id == id, + None => false, + } + } } diff --git a/worker/src/server/master_service.rs b/worker/src/server/master_service.rs index e22f3c4d..2d22f113 100644 --- a/worker/src/server/master_service.rs +++ b/worker/src/server/master_service.rs @@ -41,4 +41,16 @@ impl grpc_pb::ScheduleOperationService for ScheduleOperationService { Err(err) => SingleResponse::err(Error::Panic(err.to_string())), } } + + fn cancel_task( + &self, + _o: RequestOptions, + cancel_request: pb::CancelTaskRequest, + ) -> SingleResponse { + let cancel_task_result = self.operation_handler.cancel_task(cancel_request); + match cancel_task_result { + Ok(_) => SingleResponse::completed(pb::EmptyMessage::new()), + Err(err) => SingleResponse::err(Error::Panic(err.to_string())), + } + } } From 27cf88bff17d1cd362d28101c82c4dc39d14c6e4 Mon Sep 17 00:00:00 2001 From: Ryan Connell Date: Wed, 7 Mar 2018 22:11:53 +0000 Subject: [PATCH 13/19] Add Priority for jobs and tasks --- cli/src/parser.rs | 11 +++- cli/src/runner.rs | 14 ++++ master/src/common/job.rs | 13 ++++ master/src/common/mod.rs | 1 + master/src/common/task.rs | 78 +++++++++++++++++++++-- master/src/scheduling/task_processor.rs | 8 ++- master/src/worker_management/state.rs | 85 ++++++++++++++++++------- proto/mapreduce.proto | 3 + 8 files changed, 180 insertions(+), 33 deletions(-) diff --git a/cli/src/parser.rs b/cli/src/parser.rs index 004e8244..6197f3b2 100644 --- a/cli/src/parser.rs +++ b/cli/src/parser.rs @@ -40,7 +40,16 @@ pub fn parse_command_line<'a>() -> ArgMatches<'a> { .takes_value(true) .default_value("") .required(false), - ), + ) + .arg( + Arg::with_name("priority") + .long("priority") + .short("p") + .help("Priority of the MapReduce") + .takes_value(true) + .default_value("1") + .required(false), + ) ) .subcommand( SubCommand::with_name("cancel") diff --git a/cli/src/runner.rs b/cli/src/runner.rs index b97e37ed..794a84d4 100644 --- a/cli/src/runner.rs +++ b/cli/src/runner.rs @@ -104,11 +104,25 @@ pub fn run(client: &grpc_pb::MapReduceServiceClient, matches: &ArgMatches) -> Re || "Invalid binary path.", )?; + let priority_str = matches.value_of("priority").unwrap_or("1"); + let priority: u32 = match priority_str.parse() { + Ok(val) => val, + Err(err) => { + println!( + "Error occured while converting '{}' to a u32: {}", + priority_str, + err + ); + 1 + } + }; + let mut req = pb::MapReduceRequest::new(); req.set_binary_path(binary.to_owned()); req.set_input_directory(input.to_owned()); req.set_client_id(get_client_id()?); req.set_output_directory(output.to_owned()); + req.set_priority(priority); let res = client .perform_map_reduce(RequestOptions::new(), req) diff --git a/master/src/common/job.rs b/master/src/common/job.rs index dc406a5a..7976795f 100644 --- a/master/src/common/job.rs +++ b/master/src/common/job.rs @@ -24,6 +24,8 @@ pub struct JobOptions { pub output_directory: Option, /// Determines if paths should be validated. Should only be disabled during tests. pub validate_paths: bool, + /// Priority that should be applied to all tasks for the job. + pub priority: u32, } impl From for JobOptions { @@ -38,6 +40,7 @@ impl From for JobOptions { None }, validate_paths: true, + priority: other.priority, } } } @@ -51,6 +54,8 @@ pub struct Job { pub input_directory: String, pub output_directory: String, + pub priority: u32, + pub status: pb::Status, pub status_details: Option, @@ -104,6 +109,8 @@ impl Job { input_directory: input_directory, output_directory: output_directory, + priority: options.priority, + status: pb::Status::IN_QUEUE, status_details: None, @@ -237,6 +244,9 @@ impl StateHandling for Job { output_directory: serde_json::from_value(data["output_directory"].clone()) .chain_err(|| "Unable to convert output dir")?, validate_paths: false, + priority: serde_json::from_value(data["priority"].clone()).chain_err( + || "Unable to convert priority", + )?, }; let mut job = Job::new_no_validate(options).chain_err( @@ -264,6 +274,8 @@ impl StateHandling for Job { "input_directory": self.input_directory, "output_directory": self.output_directory, + "priority": self.priority, + "status": self.get_serializable_status(), "status_details": self.status_details, @@ -367,6 +379,7 @@ mod tests { input_directory: "/tmp/input/".to_owned(), output_directory: Some("/tmp/output/".to_owned()), validate_paths: false, + priority: 1, }).unwrap(); assert_eq!("/tmp/input/output/", job1.output_directory); diff --git a/master/src/common/mod.rs b/master/src/common/mod.rs index 22d43921..c98ee55c 100644 --- a/master/src/common/mod.rs +++ b/master/src/common/mod.rs @@ -14,4 +14,5 @@ pub use self::job::JobOptions; pub use self::task::Task; pub use self::task::TaskStatus; pub use self::task::TaskType; +pub use self::task::PriorityTask; pub use self::worker::Worker; diff --git a/master/src/common/task.rs b/master/src/common/task.rs index bc8f081f..9f54f61c 100644 --- a/master/src/common/task.rs +++ b/master/src/common/task.rs @@ -1,8 +1,11 @@ use std::collections::HashMap; +use std::cmp::Ordering; +use std::result::Result as std_result; use protobuf::repeated::RepeatedField; use chrono::prelude::*; use serde_json; +use serde::ser::{Serialize, Serializer, SerializeStruct}; use uuid::Uuid; use errors::*; @@ -40,6 +43,8 @@ pub struct Task { pub job_id: String, pub id: String, + pub job_priority: u32, + // This will only exist if TaskType is Map. pub map_request: Option, // This will only exist if TaskType is Reduce. @@ -67,6 +72,7 @@ impl Task { job_id: S, binary_path: S, input_locations: Vec, + job_priority: u32, ) -> Self { let mut map_task_input = pb::MapTaskInput::new(); map_task_input.set_input_locations(RepeatedField::from_vec(input_locations)); @@ -83,6 +89,8 @@ impl Task { job_id: job_id.into(), id: id, + job_priority: job_priority, + map_request: Some(map_request), reduce_request: None, @@ -126,7 +134,10 @@ impl Task { }) .collect(); - let mut task = Task::new_map_task(id, binary_path, input_locations_pb); + let job_priority: u32 = serde_json::from_value(data["job_priority"].clone()) + .chain_err(|| "Unable to convert job priority")?; + + let mut task = Task::new_map_task(id, binary_path, input_locations_pb, job_priority); // Update the state. task.load_state(data).chain_err( @@ -142,6 +153,7 @@ impl Task { input_partition: u64, input_files: Vec, output_directory: S, + job_priority: u32, ) -> Self { let id = Uuid::new_v4().to_string(); @@ -159,6 +171,8 @@ impl Task { job_id: job_id.into(), id: id, + job_priority: job_priority, + map_request: None, reduce_request: Some(reduce_request), @@ -192,8 +206,17 @@ impl Task { .chain_err(|| "Unable to convert binary_path")?; let output_dir: String = serde_json::from_value(request_data["output_directory"].clone()) .chain_err(|| "Unable to convert output_directory")?; - let mut task = - Task::new_reduce_task(id, binary_path, input_partition, input_files, output_dir); + + let job_priority: u32 = serde_json::from_value(data["job_priority"].clone()) + .chain_err(|| "Unable to convert job priority")?; + let mut task = Task::new_reduce_task( + id, + binary_path, + input_partition, + input_files, + output_dir, + job_priority, + ); task.load_state(data).chain_err( || "Unable to load Task from state", @@ -276,6 +299,7 @@ impl StateHandling for Task { "failure_details": self.failure_details, "time_started": time_started, "time_completed": time_completed, + "job_priority": self.job_priority, })) } @@ -337,6 +361,42 @@ impl StateHandling for Task { } } +#[derive(Eq, PartialEq)] +pub struct PriorityTask { + pub id: String, + pub priority: u32, +} + +impl PriorityTask { + pub fn new(id: String, priority: u32) -> Self { + PriorityTask { id, priority } + } +} + +impl Ord for PriorityTask { + fn cmp(&self, other: &Self) -> Ordering { + self.priority.cmp(&other.priority) + } +} + +impl PartialOrd for PriorityTask { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Serialize for PriorityTask { + fn serialize(&self, serializer: S) -> std_result + where + S: Serializer, + { + let mut state = serializer.serialize_struct("PriorityTask", 2)?; + state.serialize_field("id", &self.id)?; + state.serialize_field("priority", &self.priority)?; + state.end() + } +} + #[cfg(test)] mod tests { use super::*; @@ -348,7 +408,7 @@ mod tests { input_location.set_start_byte(0); input_location.set_end_byte(0); - let map_task = Task::new_map_task("map-1", "/tmp/bin", vec![input_location]); + let map_task = Task::new_map_task("map-1", "/tmp/bin", vec![input_location], 1); let reduce_task = Task::new_reduce_task( "reduce-1", @@ -356,6 +416,7 @@ mod tests { 0, vec!["/tmp/input/".to_owned()], "/tmp/output/", + 1, ); assert_eq!(map_task.task_type, TaskType::Map); @@ -369,7 +430,7 @@ mod tests { input_location.set_start_byte(0); input_location.set_end_byte(0); - let map_task = Task::new_map_task("map-1", "/tmp/bin", vec![input_location]); + let map_task = Task::new_map_task("map-1", "/tmp/bin", vec![input_location], 1); assert_eq!(map_task.job_id, "map-1"); } @@ -380,7 +441,7 @@ mod tests { input_location.set_start_byte(0); input_location.set_end_byte(0); - let map_task = Task::new_map_task("map-1", "/tmp/bin", vec![input_location]); + let map_task = Task::new_map_task("map-1", "/tmp/bin", vec![input_location], 1); let map_request = map_task.map_request.unwrap(); assert_eq!("/tmp/bin", map_request.get_mapper_file_path()); @@ -397,6 +458,7 @@ mod tests { 0, vec!["/tmp/input/file1".to_owned(), "/tmp/input/file2".to_owned()], "/tmp/output/", + 1, ); let reduce_request = reduce_task.reduce_request.unwrap(); @@ -415,7 +477,7 @@ mod tests { input_location.set_start_byte(0); input_location.set_end_byte(0); - let mut map_task = Task::new_map_task("map-1", "/tmp/bin", vec![input_location]); + let mut map_task = Task::new_map_task("map-1", "/tmp/bin", vec![input_location], 1); map_task.map_output_files.insert( 0, @@ -439,6 +501,7 @@ mod tests { 0, vec!["/tmp/input/inter_mediate".to_owned()], "/tmp/output/", + 1, ); // Assert assigned worker id starts as an empty string. assert_eq!(reduce_task.assigned_worker_id, ""); @@ -455,6 +518,7 @@ mod tests { 0, vec!["/tmp/input/inter_mediate".to_owned()], "/tmp/output/", + 1, ); // Assert that the default status for a task is Queued. assert_eq!(reduce_task.status, TaskStatus::Queued); diff --git a/master/src/scheduling/task_processor.rs b/master/src/scheduling/task_processor.rs index 2412e0fa..e121e723 100644 --- a/master/src/scheduling/task_processor.rs +++ b/master/src/scheduling/task_processor.rs @@ -77,6 +77,7 @@ impl TaskProcessorImpl { job.id.as_str(), job.binary_path.as_str(), map_task_file_info.input_locations.clone(), + job.priority, )); *map_task_file_info = MapTaskFileInformation { @@ -127,6 +128,7 @@ impl TaskProcessor for TaskProcessorImpl { job.id.as_str(), job.binary_path.as_str(), map_task_file_info.input_locations, + job.priority, )); } @@ -152,6 +154,7 @@ impl TaskProcessor for TaskProcessorImpl { reduce_partition, reduce_input, job.output_directory.as_str(), + job.priority, )); } @@ -277,7 +280,8 @@ mod tests { input_location.set_start_byte(0); input_location.set_end_byte(0); - let mut map_task1 = Task::new_map_task("map-1", "/tmp/bin", vec![input_location.clone()]); + let mut map_task1 = + Task::new_map_task("map-1", "/tmp/bin", vec![input_location.clone()], 1); map_task1.map_output_files.insert( 0, "/tmp/output/1".to_owned(), @@ -287,7 +291,7 @@ mod tests { "/tmp/output/2".to_owned(), ); - let mut map_task2 = Task::new_map_task("map-2", "/tmp/bin", vec![input_location]); + let mut map_task2 = Task::new_map_task("map-2", "/tmp/bin", vec![input_location], 1); map_task2.map_output_files.insert( 0, "/tmp/output/3".to_owned(), diff --git a/master/src/worker_management/state.rs b/master/src/worker_management/state.rs index fede9933..d67d6081 100644 --- a/master/src/worker_management/state.rs +++ b/master/src/worker_management/state.rs @@ -1,25 +1,30 @@ use std::collections::HashMap; -use std::collections::VecDeque; +use std::collections::BinaryHeap; use chrono::prelude::*; use serde_json; use cerberus_proto::worker as pb; -use common::{Task, TaskStatus, Worker}; +use common::{PriorityTask, Task, TaskStatus, Worker}; use errors::*; use state; const MAX_TASK_FAILURE_COUNT: u16 = 10; const MAX_TASK_ASSIGNMENT_FAILURE: u16 = 5; +// Priority for different task types. +const DEFAULT_TASK_PRIORITY: u32 = 10; +const REQUEUED_TASK_PRIORITY: u32 = 15; +const FAILED_TASK_PRIORITY: u32 = 20; + #[derive(Default)] pub struct State { // A map of worker id to worker. workers: HashMap, // A map of task id to task. tasks: HashMap, - // Tasks that are in the queue to be ran. - task_queue: VecDeque, + // Prioritised list of tasks that are in the queue to be ran. + priority_task_queue: BinaryHeap, } impl State { @@ -28,12 +33,17 @@ impl State { } pub fn remove_queued_tasks_for_job(&mut self, job_id: &str) -> Result<()> { - let mut new_task_queue = self.task_queue.clone(); - new_task_queue.retain(|t| match self.tasks.get_mut(&t.clone()) { - Some(task) => task.job_id != job_id, - None => false, - }); - self.task_queue = new_task_queue; + let mut new_priority_queue: BinaryHeap = BinaryHeap::new(); + + for priority_task in self.priority_task_queue.drain() { + if let Some(task) = self.tasks.get(&priority_task.id.clone()) { + if task.job_id != job_id { + new_priority_queue.push(priority_task); + } + } + } + + self.priority_task_queue = new_priority_queue; Ok(()) } @@ -223,7 +233,10 @@ impl State { task_id.clone().to_owned(), assigned_task.clone(), ); - self.task_queue.push_front(task_id.clone().to_owned()); + self.priority_task_queue.push(PriorityTask::new( + task_id.to_owned().clone(), + FAILED_TASK_PRIORITY * assigned_task.job_priority, + )); } Ok(assigned_task) @@ -271,7 +284,13 @@ impl State { // If this worker is a assigned a task, requeue the task. if !worker.current_task_id.is_empty() { - self.task_queue.push_front(worker.current_task_id); + let assigned_task = self.tasks.get(&worker.current_task_id).chain_err( + || "Unable to get worker task", + )?; + self.priority_task_queue.push(PriorityTask::new( + worker.current_task_id, + REQUEUED_TASK_PRIORITY * assigned_task.job_priority, + )); } } else { return Err(format!("Worker with ID {} not found.", worker_id).into()); @@ -282,7 +301,10 @@ impl State { pub fn add_task(&mut self, task: Task) { self.tasks.insert(task.id.clone(), task.clone()); - self.task_queue.push_back(task.id); + self.priority_task_queue.push(PriorityTask::new( + task.id, + DEFAULT_TASK_PRIORITY * task.job_priority, + )); } // Unassign a task assigned to a worker and put the task back in the queue. @@ -292,7 +314,10 @@ impl State { })?; if !worker.current_task_id.is_empty() { - self.task_queue.push_front(worker.current_task_id.clone()); + self.priority_task_queue.push(PriorityTask::new( + worker.current_task_id.clone(), + REQUEUED_TASK_PRIORITY, + )); } worker.current_task_id = String::new(); @@ -327,8 +352,15 @@ impl State { // Tries to assign a worker the next task in the queue. // Returns the task if one exists. pub fn try_assign_worker_task(&mut self, worker_id: &str) -> Result<(Option)> { - let scheduled_task_id = match self.task_queue.pop_front() { - Some(scheduled_task_id) => scheduled_task_id, + let scheduled_task_id = match self.priority_task_queue.pop() { + Some(priority_task) => { + println!( + "Popped off task {} with priority {}", + priority_task.id, + priority_task.priority + ); + priority_task.id + } None => return Ok(None), }; @@ -407,7 +439,7 @@ impl state::StateHandling for State { Ok(json!({ "workers": json!(workers_json), - "task_queue": json!(self.task_queue), + "priority_task_queue": json!(self.priority_task_queue), "tasks": json!(tasks_json), })) } @@ -424,13 +456,20 @@ impl state::StateHandling for State { return Err("Error processing workers array.".into()); } - if let serde_json::Value::Array(ref task_queue) = data["task_queue"] { - for task_id in task_queue { - let task_id_str: String = serde_json::from_value(task_id.clone()).chain_err( - || "Error processing task_id in tasks queue", - )?; + if let serde_json::Value::Array(ref priority_task_queue) = data["priority_task_queue"] { + for priority_task in priority_task_queue { + let task_id_str: String = + serde_json::from_value(priority_task["id"].clone()) + .chain_err(|| "Error processing task_id in priority task queue")?; + + let priority: u32 = + serde_json::from_value(priority_task["priority"].clone()) + .chain_err(|| "Error processing priority in priority task queue")?; - self.task_queue.push_front(task_id_str); + self.priority_task_queue.push(PriorityTask::new( + task_id_str, + priority, + )); } } else { return Err("Error processing tasks queue.".into()); diff --git a/proto/mapreduce.proto b/proto/mapreduce.proto index bb9c3984..3a9711ac 100644 --- a/proto/mapreduce.proto +++ b/proto/mapreduce.proto @@ -39,6 +39,9 @@ message MapReduceRequest { // making the request. Generated on a single machine for now // TODO(voy): Convert to unique identity once we have authentication. string client_id = 4; + + // Priority of the MapReduce + uint32 priority = 5; } // Response from the master about the map reduce. From 833ba92b17bf55a1420913dadf052e6a42f0c9b1 Mon Sep 17 00:00:00 2001 From: Ryan Connell Date: Fri, 9 Mar 2018 21:42:24 +0000 Subject: [PATCH 14/19] Addressed Conor's Comments --- cli/src/parser.rs | 3 +-- cli/src/runner.rs | 23 +++++++++++++++++------ master/src/common/task.rs | 16 +--------------- master/src/worker_management/state.rs | 25 ++++++------------------- 4 files changed, 25 insertions(+), 42 deletions(-) diff --git a/cli/src/parser.rs b/cli/src/parser.rs index 6197f3b2..4485e9ce 100644 --- a/cli/src/parser.rs +++ b/cli/src/parser.rs @@ -45,9 +45,8 @@ pub fn parse_command_line<'a>() -> ArgMatches<'a> { Arg::with_name("priority") .long("priority") .short("p") - .help("Priority of the MapReduce") + .help("Priority of the MapReduce. Valid values are 1 to 10") .takes_value(true) - .default_value("1") .required(false), ) ) diff --git a/cli/src/runner.rs b/cli/src/runner.rs index 794a84d4..d8b51b39 100644 --- a/cli/src/runner.rs +++ b/cli/src/runner.rs @@ -17,6 +17,8 @@ use errors::*; const CLIENT_ID_DIR: &str = ".local/share/"; // Client ID file name. const CLIENT_ID_FILE: &str = "cerberus"; +// Default priority applied to jobs. +const DEFAULT_PRIORITY: &str = "3"; fn verify_valid_path(path_str: &str) -> Result { let path = Path::new(path_str); @@ -104,18 +106,27 @@ pub fn run(client: &grpc_pb::MapReduceServiceClient, matches: &ArgMatches) -> Re || "Invalid binary path.", )?; - let priority_str = matches.value_of("priority").unwrap_or("1"); + let priority_str = matches.value_of("priority").unwrap_or(DEFAULT_PRIORITY); let priority: u32 = match priority_str.parse() { Ok(val) => val, Err(err) => { - println!( - "Error occured while converting '{}' to a u32: {}", - priority_str, - err + return Err( + format!( + "Error occured while converting '{}' to a u32: {}", + priority_str, + err + ).into(), ); - 1 } }; + if priority < 1 || priority > 10 { + return Err( + format!( + "Priority can only be between 1 and 10. {} is not in this range", + priority + ).into(), + ); + } let mut req = pb::MapReduceRequest::new(); req.set_binary_path(binary.to_owned()); diff --git a/master/src/common/task.rs b/master/src/common/task.rs index 9f54f61c..8686b591 100644 --- a/master/src/common/task.rs +++ b/master/src/common/task.rs @@ -1,11 +1,9 @@ use std::collections::HashMap; use std::cmp::Ordering; -use std::result::Result as std_result; use protobuf::repeated::RepeatedField; use chrono::prelude::*; use serde_json; -use serde::ser::{Serialize, Serializer, SerializeStruct}; use uuid::Uuid; use errors::*; @@ -361,7 +359,7 @@ impl StateHandling for Task { } } -#[derive(Eq, PartialEq)] +#[derive(Eq, PartialEq, Serialize, Deserialize)] pub struct PriorityTask { pub id: String, pub priority: u32, @@ -385,18 +383,6 @@ impl PartialOrd for PriorityTask { } } -impl Serialize for PriorityTask { - fn serialize(&self, serializer: S) -> std_result - where - S: Serializer, - { - let mut state = serializer.serialize_struct("PriorityTask", 2)?; - state.serialize_field("id", &self.id)?; - state.serialize_field("priority", &self.priority)?; - state.end() - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/master/src/worker_management/state.rs b/master/src/worker_management/state.rs index d67d6081..fc32b351 100644 --- a/master/src/worker_management/state.rs +++ b/master/src/worker_management/state.rs @@ -313,10 +313,13 @@ impl State { format!("Worker with ID {} not found.", worker_id) })?; + let assigned_task = self.tasks.get(&worker.current_task_id).chain_err( + || "Unable to get worker task", + )?; if !worker.current_task_id.is_empty() { self.priority_task_queue.push(PriorityTask::new( worker.current_task_id.clone(), - REQUEUED_TASK_PRIORITY, + REQUEUED_TASK_PRIORITY * assigned_task.job_priority, )); } @@ -456,24 +459,8 @@ impl state::StateHandling for State { return Err("Error processing workers array.".into()); } - if let serde_json::Value::Array(ref priority_task_queue) = data["priority_task_queue"] { - for priority_task in priority_task_queue { - let task_id_str: String = - serde_json::from_value(priority_task["id"].clone()) - .chain_err(|| "Error processing task_id in priority task queue")?; - - let priority: u32 = - serde_json::from_value(priority_task["priority"].clone()) - .chain_err(|| "Error processing priority in priority task queue")?; - - self.priority_task_queue.push(PriorityTask::new( - task_id_str, - priority, - )); - } - } else { - return Err("Error processing tasks queue.".into()); - } + self.priority_task_queue = serde_json::from_value(data["priority_task_queue"].clone()) + .chain_err(|| "Error processing priority task queue")?; if let serde_json::Value::Array(ref tasks_array) = data["tasks"] { for task in tasks_array { From 9b55ca13190703713df98fc1cdf1b363450aab17 Mon Sep 17 00:00:00 2001 From: Darragh Griffin Date: Mon, 12 Mar 2018 23:43:09 +0000 Subject: [PATCH 15/19] Add basic cluster dashboard This commit adds the basic structure and functionality for the master dashboard. Currently this dashboard just shows the current information for the cluster and can not be interacted with. The design of the dashboard is also very basic but can be improved upon in future commits. The dashboard is served by the master using the Iron web framework. --- Cargo.lock | 252 ++++++++++++++++++ master/Cargo.toml | 8 + master/build.rs | 31 +++ master/content/dashboard.js | 102 +++++++ master/content/index.html | 37 +++ master/content/stylesheet.css | 98 +++++++ master/src/common/job.rs | 2 +- master/src/common/worker.rs | 4 +- master/src/dashboard/mod.rs | 4 + master/src/dashboard/server.rs | 134 ++++++++++ master/src/main.rs | 19 ++ master/src/parser.rs | 8 + master/src/scheduling/scheduler.rs | 24 ++ master/src/scheduling/state.rs | 11 +- master/src/server/mod.rs | 1 + master/src/worker_management/state.rs | 8 + .../src/worker_management/worker_manager.rs | 51 +++- 17 files changed, 784 insertions(+), 10 deletions(-) create mode 100644 master/build.rs create mode 100644 master/content/dashboard.js create mode 100644 master/content/index.html create mode 100644 master/content/stylesheet.css create mode 100644 master/src/dashboard/mod.rs create mode 100644 master/src/dashboard/server.rs diff --git a/Cargo.lock b/Cargo.lock index ca84bd15..a8316251 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,6 +50,15 @@ dependencies = [ "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "base64" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "base64" version = "0.9.0" @@ -177,6 +186,14 @@ dependencies = [ "uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "conduit-mime-types" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "csv" version = "0.15.0" @@ -225,6 +242,15 @@ dependencies = [ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "error" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "error-chain" version = "0.11.0" @@ -233,6 +259,11 @@ dependencies = [ "backtrace 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "fs_extra" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "fuchsia-zircon" version = "0.3.3" @@ -319,6 +350,11 @@ dependencies = [ "winutil 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "httparse" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "httpbis" version = "0.4.1" @@ -338,6 +374,34 @@ dependencies = [ "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "hyper" +version = "0.10.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "httparse 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "mime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "idna" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-normalization 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "iovec" version = "0.1.2" @@ -347,6 +411,23 @@ dependencies = [ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "iron" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "conduit-mime-types 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "error 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.10.13 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "modifier 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "itoa" version = "0.3.4" @@ -361,6 +442,11 @@ dependencies = [ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "language-tags" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "lazy_static" version = "0.2.11" @@ -424,18 +510,28 @@ dependencies = [ "clap 2.26.2 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "grpc 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "iron 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "mount 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "protobuf 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "router 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", + "staticfile 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "util 0.4.0", "uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "matches" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "memchr" version = "0.1.11" @@ -460,6 +556,14 @@ dependencies = [ "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "mime" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "mio" version = "0.6.13" @@ -506,6 +610,20 @@ dependencies = [ "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "modifier" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "mount" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "iron 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sequence_trie 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "net2" version = "0.2.31" @@ -571,6 +689,19 @@ dependencies = [ "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "percent-encoding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "plugin" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "prettytable-rs" version = "0.6.7" @@ -702,6 +833,21 @@ dependencies = [ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "route-recognizer" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "router" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "iron 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "route-recognizer 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rust-crypto" version = "0.2.36" @@ -772,6 +918,11 @@ name = "semver-parser" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "sequence_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "serde" version = "1.0.27" @@ -818,6 +969,17 @@ name = "slab" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "staticfile" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "iron 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mount 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "strsim" version = "0.6.0" @@ -986,6 +1148,45 @@ dependencies = [ "tokio-io 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "traitobject" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "typeable" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "typemap" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unsafe-any 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicase" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "version_check 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unicode-width" version = "0.1.4" @@ -1004,6 +1205,24 @@ dependencies = [ "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "unsafe-any" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "url" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "utf8-ranges" version = "0.1.3" @@ -1037,6 +1256,11 @@ name = "vec_map" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "version_check" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "void" version = "1.0.2" @@ -1119,6 +1343,7 @@ dependencies = [ "checksum atty 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "8352656fd42c30a0c3c89d26dea01e3b77c0ab2af18230835c15e2e13cd51859" "checksum backtrace 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ebbbf59b1c43eefa8c3ede390fcc36820b4999f7914104015be25025e0d62af2" "checksum backtrace-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "44585761d6161b0f57afc49482ab6bd067e4edef48c12a152c237eb0203f7661" +"checksum base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96434f987501f0ed4eb336a411e0631ecd1afa11574fe148587adc4ff96143c9" "checksum base64 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "229d032f1a99302697f10b27167ae6d03d49d032e6a8e2550e8d3fc13356d2b4" "checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" "checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf" @@ -1129,13 +1354,16 @@ dependencies = [ "checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de" "checksum chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7c20ebe0b2b08b0aeddba49c609fe7957ba2e33449882cb186a180bc60682fa9" "checksum clap 2.26.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3451e409013178663435d6f15fdb212f14ee4424a3d74f979d081d0a66b6f1f2" +"checksum conduit-mime-types 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "95ca30253581af809925ef68c2641cc140d6183f43e12e0af4992d53768bd7b8" "checksum csv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7ef22b37c7a51c564a365892c012dc0271221fdcc64c69b19ba4d6fa8bd96d9c" "checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab" "checksum encode_unicode 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c088ec0ed2282dcd054f2c124c0327f953563e6c75fdc6ff5141779596289830" "checksum env_logger 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "15abd780e45b3ea4f76b4e9a26ff4843258dd8a3eed2775a0e7368c2e7936c2f" "checksum env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3ddf21e73e016298f5cb37d6ef8e8da8e39f91f9ec8b0df44b7deb16a9f8cd5b" "checksum errno 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1e2b2decb0484e15560df3210cf0d78654bb0864b2c138977c07e377a1bae0e2" +"checksum error 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "a6e606f14042bb87cc02ef6a14db6c90ab92ed6f62d87e69377bc759fd7987cc" "checksum error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff511d5dc435d703f4971bc399647c9bc38e20cb41452e3b9feb4765419ed3f3" +"checksum fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5f2a4a2034423744d2cc7ca2068453168dcdb82c438419e639a26bd87839c674" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "0bab5b5e94f5c31fc764ba5dd9ad16568aae5d4825538c01d6bca680c9bf94a7" @@ -1146,10 +1374,15 @@ dependencies = [ "checksum grpc-rust 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "795c3f654333803efdab3a51f228456d08981731ddfb753741090a13783833a1" "checksum hex 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d6a22814455d41612f41161581c2883c0c6a1c41852729b17d5ed88f01e153aa" "checksum hostname 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "58fab6e177434b0bb4cd344a4dabaa5bd6d7a8d792b1885aebcae7af1091d1cb" +"checksum httparse 1.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c2f407128745b78abc95c0ffbe4e5d37427fdc0d45470710cfef8c44522a2e37" "checksum httpbis 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73dde194d197eaa5b49bb58d212ed4a99a713cf47fc77a9d0471b76c0f73fcf0" +"checksum hyper 0.10.13 (registry+https://github.com/rust-lang/crates.io-index)" = "368cb56b2740ebf4230520e2b90ebb0461e69034d85d1945febd9b3971426db2" +"checksum idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "014b298351066f1512874135335d62a789ffe78a9974f94b43ed5621951eaf7d" "checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08" +"checksum iron 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2440ae846e7a8c7f9b401db8f6e31b4ea5e7d3688b91761337da7e054520c75b" "checksum itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8324a32baf01e2ae060e9de58ed0bc2320c9a2833491ee36cd3b4c414de4db8c" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +"checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" "checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" "checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d" "checksum lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a6f08839bc70ef4a3fe1d566d5350f519c5912ea86be0df1740a7d247c7fc0ef" @@ -1159,13 +1392,17 @@ dependencies = [ "checksum local-ip 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1199ecb36eea49b2b849800de0a72dbdf6d8b69046765c2c78c76fdb96440d34" "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" "checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2" +"checksum matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "100aabe6b8ff4e4a7e32c1c13523379802df0772b82466207ac25b013f193376" "checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20" "checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" "checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" +"checksum mime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" "checksum mio 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "7da01a5e23070d92d99b1ecd1cd0af36447c6fd44b0fe283c2db199fa136724f" "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" "checksum mocktopus 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9dc5fbcabea8244edabd101547b0981f9a9cee2aa89fcccbbbe6ff9c59c0a262" "checksum mocktopus_macros 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1aefe0f9a78f54eb38430e506f499165b36505f377ec1dc22e74c17e739f194e" +"checksum modifier 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "41f5c9112cb662acd3b204077e0de5bc66305fa8df65c8019d5adb10e9ab6e58" +"checksum mount 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "32245731923cd096899502fc4c4317cfd09f121e80e73f7f576cf3777a824256" "checksum net2 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)" = "3a80f842784ef6c9a958b68b7516bc7e35883c614004dd94959a4dca1b716c09" "checksum nom 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf51a729ecf40266a2368ad335a5fdde43471f545a967109cd62146ecf8b66ff" "checksum num 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e" @@ -1174,6 +1411,8 @@ dependencies = [ "checksum num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" "checksum num-traits 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e7de20f146db9d920c45ee8ed8f71681fd9ade71909b48c3acbd766aa504cf10" "checksum num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c51a3322e4bca9d212ad9a158a02abc6934d005490c054a2778df73a70aa0a30" +"checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" +"checksum plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a6a0dc3910bc8db877ffed8e457763b317cf880df4ae19109b9f77d277cf6e0" "checksum prettytable-rs 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "34dc1f4f6dddab3bf008ecfd4fd2a631b585fbf0af123f34c1324f51a034ff5f" "checksum procinfo 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6ab1427f3d2635891f842892dda177883dca0639e05fe66796a62c9d2f23b49c" "checksum protobuf 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bec26e67194b7d991908145fdf21b7cae8b08423d96dcb9e860cd31f854b9506" @@ -1189,6 +1428,8 @@ dependencies = [ "checksum regex-syntax 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f9ec002c35e86791825ed294b50008eea9ddfc8def4420124fbc6b08db834957" "checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e" "checksum remove_dir_all 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b5d2f806b0fcdabd98acd380dc8daef485e22bcb7cddc811d1337967f2528cf5" +"checksum route-recognizer 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3255338088df8146ba63d60a9b8e3556f1146ce2973bc05a75181a42ce2256" +"checksum router 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9b1797ff166029cb632237bb5542696e54961b4cf75a324c6f05c9cf0584e4e" "checksum rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a" "checksum rustc-demangle 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f312457f8a4fa31d3581a6f423a70d6c33a10b95291985df55f1ff670ec10ce8" "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" @@ -1199,12 +1440,14 @@ dependencies = [ "checksum semver 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a3186ec9e65071a2095434b1f5bb24838d4e8e130f584c790f6033c79943537" "checksum semver-parser 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d8fff3c9c5a54636ab95acd8c1349926e04cb1eb8cd70b5adced8a1d1f703a67" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum sequence_trie 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c915714ca833b1d4d6b8f6a9d72a3ff632fe45b40a8d184ef79c81bec6327eed" "checksum serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)" = "db99f3919e20faa51bb2996057f5031d8685019b5a06139b1ce761da671b8526" "checksum serde_derive 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)" = "f4ba7591cfe93755e89eeecdbcc668885624829b020050e6aec99c2a03bd3fd0" "checksum serde_derive_internals 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6e03f1c9530c3fb0a0a5c9b826bdd9246a5921ae995d75f512ac917fc4dd55b5" "checksum serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c9db7266c7d63a4c4b7fe8719656ccdd51acf1bed6124b174f933b009fb10bcb" "checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23" "checksum slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fdeff4cd9ecff59ec7e3744cbca73dfe5ac35c2aedb2cfba8a1c715a18912e9d" +"checksum staticfile 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "31493480e073d52522a94cdf56269dd8eb05f99549effd1826b0271690608878" "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" "checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" "checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" @@ -1223,13 +1466,22 @@ dependencies = [ "checksum tokio-io 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b9532748772222bf70297ec0e2ad0f17213b4a7dd0e6afb68e0a0768f69f4e4f" "checksum tokio-timer 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6131e780037787ff1b3f8aad9da83bca02438b72277850dd6ad0d455e0e20efc" "checksum tokio-tls-api 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "a47b126d55dc5f9407eeba619a305ca70a8ba8f7d6b813bfaf93e924afe27ce1" +"checksum traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" +"checksum typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1410f6f91f21d1612654e7cc69193b0334f909dcf2c790c4826254fbb86f8887" +"checksum typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "653be63c80a3296da5551e1bfd2cca35227e13cdd08c6668903ae2f4f77aa1f6" +"checksum unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" +"checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +"checksum unicode-normalization 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "51ccda9ef9efa3f7ef5d91e8f9b83bbe6955f9bf86aec89d5cce2c874625920f" "checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f" "checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +"checksum unsafe-any 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f30360d7979f5e9c6e6cea48af192ea8fab4afb3cf72597154b8f08935bc9c7f" +"checksum url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f808aadd8cfec6ef90e4a14eb46f24511824d1ac596b9682703c87056c8678b7" "checksum utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1ca13c08c41c9c3e04224ed9ff80461d97e121589ff27c753a16cb10830ae0f" "checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" "checksum uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bcc7e3b898aa6f6c08e5295b6c89258d1331e9ac578cc992fb818759951bdc22" "checksum vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "887b5b631c2ad01628bbbaa7dd4c869f80d3186688f8d0b6f58774fbe324988c" +"checksum version_check 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6b772017e347561807c1aa192438c5fd74242a670a6cffacc40f2defd1dc069d" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04e3bd221fcbe8a271359c04f21a76db7d0c6028862d1bb5512d85e1e2eb5bb3" diff --git a/master/Cargo.toml b/master/Cargo.toml index e475b6c4..bd7922c4 100644 --- a/master/Cargo.toml +++ b/master/Cargo.toml @@ -1,6 +1,7 @@ [package] name = "master" version = "0.4.0" +build = "build.rs" authors = ["Cerberus Authors "] [dependencies] @@ -12,10 +13,17 @@ error-chain = "0.11.0" futures = "0.1" futures-cpupool = "0.1" grpc = "0.2.1" +iron = "0.5.1" log = "0.3" +mount = "0.3" protobuf = "1.4.1" +router = "0.5.1" serde = "1.0" serde_derive = "1.0" serde_json = "1.0" +staticfile = "0.4.0" util = { path = "../util" } uuid = { version = "0.5", features = ["v4"] } + +[build-dependencies] +fs_extra = "1.1.0" diff --git a/master/build.rs b/master/build.rs new file mode 100644 index 00000000..246675dc --- /dev/null +++ b/master/build.rs @@ -0,0 +1,31 @@ +extern crate fs_extra; + +use std::env; +use std::fs; +use std::path::PathBuf; + +use fs_extra::dir; + +fn main() { + let build_out_dir = env::var("OUT_DIR").unwrap(); + let mut out_path = PathBuf::from(build_out_dir); + out_path.pop(); + out_path.pop(); + out_path.pop(); + + let mut previous_dir = out_path.clone(); + previous_dir.push("content"); + + if previous_dir.exists() { + fs::remove_dir_all(previous_dir).unwrap(); + } + + fs::create_dir_all(out_path.clone()).unwrap(); + + let mut options = dir::CopyOptions::new(); + options.overwrite = true; + + let source_dir = PathBuf::from("content"); + + dir::copy(source_dir, out_path, &options).unwrap(); +} diff --git a/master/content/dashboard.js b/master/content/dashboard.js new file mode 100644 index 00000000..7a31b6bf --- /dev/null +++ b/master/content/dashboard.js @@ -0,0 +1,102 @@ +function createCard(parent) { + var infoBox = $("
") + .addClass("card") + .appendTo(parent); + + var container = $("
") + .addClass("container") + .appendTo(infoBox); + + return container; +} + +function addProperty(name, value, container) { + $("") + .addClass("ui-all") + .text(name + ": " + value) + .appendTo(container); +} + +function updateWorkersList() { + var workersBox = $("#workers"); + + $.ajax({ + url: "/api/workers", + dataType: "json", + success: function(workers) { + workersBox.empty(); + + workers.forEach(function(workerInfo) { + var container = createCard(workersBox); + + addProperty("Worker ID", workerInfo.worker_id, container); + addProperty("Status", workerInfo.status, container); + addProperty("Operation Status", workerInfo.operation_status, container); + addProperty("Current Task ID", workerInfo.current_task_id, container); + addProperty("Task Assignments Failed", workerInfo.task_assignments_failed, container); + }); + } + }); +} + +function updateJobsList() { + var jobsBox = $("#jobs"); + + $.ajax({ + url: "/api/jobs", + dataType: "json", + success: function(jobs) { + jobsBox.empty(); + + jobs.forEach(function(jobsInfo) { + var container = createCard(tasksBox); + + addProperty("Job ID", jobsInfo.job_id, container); + addProperty("Client ID", jobsInfo.client_id, container); + addProperty("Binary", jobsInfo.binary_path, container); + addProperty("Input", jobsInfo.input_directory, container); + addProperty("Output", jobsInfo.output_directory, container); + addProperty("Status", jobsInfo.status, container); + var mapTasksText = jobsInfo.map_tasks_completed + "/" + jobsInfo.map_tasks_total; + addProperty("Map Tasks Completed", mapTasksText, container); + var reduceTasksText = jobsInfo.reduce_tasks_completed + "/" + jobsInfo.reduce_tasks_total; + addProperty("Reduce Tasks Completed", reduceTasksText, container); + }); + } + }); +} + +function updateTasksList() { + var tasksBox = $("#tasks"); + + $.ajax({ + url: "/api/tasks", + dataType: "json", + success: function(tasks) { + tasksBox.empty(); + + tasks.forEach(function(taskInfo) { + var container = createCard(tasksBox); + + addProperty("Task ID", taskInfo.task_id, container); + addProperty("Job ID", taskInfo.job_id, container); + addProperty("Task Type", taskInfo.task_type, container); + addProperty("Assigned Worker ID", taskInfo.assigned_worker_id, container); + addProperty("Status", taskInfo.status, container); + addProperty("Failure Count", taskInfo.failure_count, container); + }); + } + }); +} + +function updateFunction() { + updateWorkersList(); + updateJobsList(); + updateTasksList(); +} + +$(document).ready(function() { + updateFunction(); + + setInterval(updateFunction, 2000); +}); diff --git a/master/content/index.html b/master/content/index.html new file mode 100644 index 00000000..c63e3526 --- /dev/null +++ b/master/content/index.html @@ -0,0 +1,37 @@ + + + + + + + + + + + +
+

Cluster Dashboard

+
+ +
Workers +
+
+ +
+
+
Jobs +
+
+
Tasks +
+
+
+
+ +
+ Cerberus MapReduce Project on Github +
+ + + + diff --git a/master/content/stylesheet.css b/master/content/stylesheet.css new file mode 100644 index 00000000..5ddefe6b --- /dev/null +++ b/master/content/stylesheet.css @@ -0,0 +1,98 @@ +* { + box-sizing: border-box; +} + +body { + font-family: Arial, Helvetica, sans-serif; +} + +.header { + background-color: #2196f3; + color: #fff; + font-size: 20px; + padding: 1px; + text-align: center; +} + +.card { + background-color: #8bc34a; + border-radius: 5px; + box-shadow: 0 4px 8px 0 rgba(0, 0, 0, .2); + color: #212121; + transition: .3s; + width: 45%; +} + +.card:hover { + box-shadow: 0 8px 16px 0 rgba(0, 0, 0, .2); +} + +.container { + display: table; + padding: 2px 16px; +} + +.lower { + height: 45%; +} + +.row { + height: 100%; +} + +.ui-all { + color: #fff; + display: block; + text-decoration: none; +} + +.box { + background-color: #bbdefb; + color: #212121; + float: left; + font-size: 20px; + font-weight: bold; + height: 40%; + overflow: auto; + padding: 10px; + text-align: center; + width: 100%; +} + +.box.container { + padding-top: 10px; +} + +.column.container { + padding-top: 10px; +} + +.column { + background-color: #ffc107; + color: #212121; + float: left; + font-weight: bold; + height: 500px; + overflow: auto; + padding: 10px; + text-align: center; + width: 50%; +} + +.row:after { + clear: both; + content: ''; + display: table; +} + +.footer { + background-color: #ffecb3; + padding: 10px; + text-align: center; +} + +@media (max-width: 600px) { + .column { + width: 100%; + } +} diff --git a/master/src/common/job.rs b/master/src/common/job.rs index 7976795f..5706d5a7 100644 --- a/master/src/common/job.rs +++ b/master/src/common/job.rs @@ -220,7 +220,7 @@ impl Job { } } - fn get_serializable_status(&self) -> SerializableJobStatus { + pub fn get_serializable_status(&self) -> SerializableJobStatus { match self.status { pb::Status::DONE => SerializableJobStatus::DONE, pb::Status::IN_PROGRESS => SerializableJobStatus::IN_PROGRESS, diff --git a/master/src/common/worker.rs b/master/src/common/worker.rs index 91c27d08..5a6802a5 100644 --- a/master/src/common/worker.rs +++ b/master/src/common/worker.rs @@ -74,7 +74,7 @@ impl Worker { } } - fn get_serializable_operation_status(&self) -> OperationStatus { + pub fn get_serializable_operation_status(&self) -> OperationStatus { match self.operation_status { pb::OperationStatus::IN_PROGRESS => OperationStatus::IN_PROGRESS, pb::OperationStatus::COMPLETE => OperationStatus::COMPLETE, @@ -91,7 +91,7 @@ impl Worker { } } - fn get_serializable_worker_status(&self) -> WorkerStatus { + pub fn get_serializable_worker_status(&self) -> WorkerStatus { match self.status { pb::WorkerStatus::AVAILABLE => WorkerStatus::AVAILABLE, pb::WorkerStatus::BUSY => WorkerStatus::BUSY, diff --git a/master/src/dashboard/mod.rs b/master/src/dashboard/mod.rs new file mode 100644 index 00000000..98aa56c8 --- /dev/null +++ b/master/src/dashboard/mod.rs @@ -0,0 +1,4 @@ +// The dashboard module contains code used for serving the cluster dashboard web page. +pub mod server; + +pub use self::server::DashboardServer; diff --git a/master/src/dashboard/server.rs b/master/src/dashboard/server.rs new file mode 100644 index 00000000..c512e725 --- /dev/null +++ b/master/src/dashboard/server.rs @@ -0,0 +1,134 @@ +use std::path::Path; +use std::sync::Arc; + +use iron; +use iron::prelude::*; +use mount::Mount; +use router::Router; +use staticfile::Static; + +use errors::*; +use scheduling::Scheduler; +use worker_management::WorkerManager; +use util::output_error; + +#[derive(Clone)] +struct ApiHandler { + scheduler_arc: Arc, + worker_manager_arc: Arc, +} + +impl ApiHandler { + /// Returns information about the `Tasks` which are currently in progress. + fn tasks(&self, _req: &mut Request) -> Result { + let tasks_info = self.worker_manager_arc.get_tasks_info().chain_err( + || "Failed to get tasks info", + )?; + + Ok(Response::with((iron::status::Ok, tasks_info.to_string()))) + } + + /// Returns information about the `Workers` currently registered with the cluster. + fn workers(&self, _req: &mut Request) -> Result { + let workers_info = self.worker_manager_arc.get_workers_info().chain_err( + || "Failed to get workers info", + )?; + + Ok(Response::with((iron::status::Ok, workers_info.to_string()))) + } + + /// Returns information about the `Jobs` currently running on the cluster. + fn jobs(&self, _req: &mut Request) -> Result { + let jobs_info = self.scheduler_arc.get_jobs_info().chain_err( + || "Failed to get jobs info", + )?; + + Ok(Response::with((iron::status::Ok, jobs_info.to_string()))) + } + + fn handle_endpoint(&self, endpoint: &str, req: &mut Request) -> IronResult { + let result = { + match endpoint { + "tasks" => self.tasks(req), + "workers" => self.workers(req), + "jobs" => self.jobs(req), + _ => Err("Invalid endpoint".into()), + } + }; + + match result { + Ok(response) => Ok(response), + Err(err) => { + let chained_err = err.chain_err(|| "Error serving Cluster Dashboard request"); + output_error(&chained_err); + Err(IronError::new(chained_err, iron::status::BadRequest)) + } + } + } +} + +impl iron::Handler for ApiHandler { + fn handle(&self, req: &mut Request) -> IronResult { + let endpoint: String = { + match req.extensions.get::() { + Some(params) => { + match params.find("endpoint") { + Some(endpoint) => endpoint.to_string(), + None => { + return Err(IronError::new( + Error::from_kind( + ErrorKind::Msg("No API endpoint found in request".into()), + ), + iron::status::BadRequest, + )); + } + } + } + None => { + return Err(IronError::new( + Error::from_kind( + ErrorKind::Msg("Failed to get Router for request".into()), + ), + iron::status::InternalServerError, + )); + } + } + }; + + self.handle_endpoint(&endpoint, req) + } +} + +pub struct DashboardServer { + pub iron_server: iron::Listening, +} + +impl DashboardServer { + pub fn new( + serving_addr: &str, + scheduler_arc: Arc, + worker_manager_arc: Arc, + ) -> Result { + let handler = ApiHandler { + scheduler_arc: scheduler_arc, + worker_manager_arc: worker_manager_arc, + }; + + let mut router = Router::new(); + router.get("/:endpoint", handler.clone(), "api"); + router.get("/:endpoint/*", handler, "api_query"); + + let mut mount = Mount::new(); + + mount + .mount("/api/", router) + .mount("/dashboard", Static::new(Path::new("content/index.html"))) + .mount("/", Static::new(Path::new("content/"))); + + Ok(DashboardServer { + iron_server: Iron::new(mount).http(serving_addr).chain_err( + || "Failed to start cluster dashboard server.", + )?, + }) + } +} diff --git a/master/src/main.rs b/master/src/main.rs index 02425e74..4fafab00 100644 --- a/master/src/main.rs +++ b/master/src/main.rs @@ -10,14 +10,18 @@ extern crate error_chain; extern crate futures; extern crate futures_cpupool; extern crate grpc; +extern crate iron; #[macro_use] extern crate log; +extern crate mount; extern crate protobuf; +extern crate router; #[macro_use] extern crate serde_derive; extern crate serde; #[macro_use] extern crate serde_json; +extern crate staticfile; extern crate uuid; extern crate util; @@ -34,6 +38,7 @@ mod errors { } mod common; +mod dashboard; mod scheduling; mod state; mod worker_communication; @@ -46,6 +51,7 @@ use std::{thread, time}; use std::path::Path; use std::str::FromStr; +use dashboard::DashboardServer; use errors::*; use scheduling::{TaskProcessorImpl, Scheduler, run_task_update_loop}; use util::init_logger; @@ -59,6 +65,7 @@ const MAIN_LOOP_SLEEP_MS: u64 = 100; const DUMP_LOOP_MS: u64 = 5000; const DEFAULT_PORT: &str = "8081"; const DEFAULT_DUMP_DIR: &str = "/var/lib/cerberus"; +const DEFAULT_DASHBOARD_ADDRESS: &str = "127.0.0.1:3000"; fn run() -> Result<()> { println!("Cerberus Master!"); @@ -126,11 +133,23 @@ fn run() -> Result<()> { &Arc::clone(&worker_manager), ); + let dashboard_address = matches.value_of("dashboard-address").unwrap_or( + DEFAULT_DASHBOARD_ADDRESS, + ); + let mut dashboard = DashboardServer::new( + &dashboard_address, + Arc::clone(&map_reduce_scheduler), + Arc::clone(&worker_manager), + ).chain_err(|| "Failed to create cluster dashboard server.")?; + let mut count = 0; loop { thread::sleep(time::Duration::from_millis(MAIN_LOOP_SLEEP_MS)); if !srv.is_alive() { + dashboard.iron_server.close().chain_err( + || "Failed to close dashboard server", + )?; return Err("GRPC server unexpectedly died".into()); } diff --git a/master/src/parser.rs b/master/src/parser.rs index 27bb8342..7077ed58 100644 --- a/master/src/parser.rs +++ b/master/src/parser.rs @@ -48,5 +48,13 @@ pub fn parse_command_line<'a>() -> ArgMatches<'a> { .takes_value(true) .required(false), ) + .arg( + Arg::with_name("dashboard-address") + .long("dashboard-address") + .short("d") + .help("The address to serve the cluster dashboard to") + .takes_value(true) + .required(false), + ) .get_matches() } diff --git a/master/src/scheduling/scheduler.rs b/master/src/scheduling/scheduler.rs index dad15532..92f8e301 100644 --- a/master/src/scheduling/scheduler.rs +++ b/master/src/scheduling/scheduler.rs @@ -208,6 +208,30 @@ impl Scheduler { Ok(latest_job.id.clone()) } + + pub fn get_jobs_info(&self) -> Result { + let state = self.state.lock().unwrap(); + + let mut results_vec = Vec::new(); + for job in state.get_all_jobs() { + let job_info = json!({ + "job_id": job.id, + "client_id": job.client_id, + "binary_path": job.binary_path, + "input_directory": job.input_directory, + "output_directory": job.output_directory, + "status": job.get_serializable_status(), + "map_tasks_completed": job.map_tasks_completed, + "map_tasks_total": job.map_tasks_total, + "reduce_tasks_completed": job.reduce_tasks_completed, + "reduce_tasks_total": job.reduce_tasks_total, + }); + + results_vec.push(job_info); + } + + Ok(json!(results_vec)) + } } impl state::SimpleStateHandling for Scheduler { diff --git a/master/src/scheduling/state.rs b/master/src/scheduling/state.rs index daab111b..0c017cde 100644 --- a/master/src/scheduling/state.rs +++ b/master/src/scheduling/state.rs @@ -146,6 +146,15 @@ impl State { } } + pub fn get_all_jobs(&self) -> Vec<&Job> { + let mut jobs = Vec::new(); + for scheduled_job in self.scheduled_jobs.values() { + jobs.push(&scheduled_job.job); + } + + jobs + } + pub fn get_jobs(&self, client_id: &str) -> Vec { let mut jobs = Vec::new(); for scheduled_job in self.scheduled_jobs.values() { @@ -259,7 +268,7 @@ impl State { match task.task_type { TaskType::Map => { scheduled_job.job.map_tasks_completed += 1; - } + } TaskType::Reduce => { scheduled_job.job.reduce_tasks_completed += 1; if scheduled_job.job.reduce_tasks_completed == diff --git a/master/src/server/mod.rs b/master/src/server/mod.rs index cd3043e6..4b05d65b 100644 --- a/master/src/server/mod.rs +++ b/master/src/server/mod.rs @@ -5,6 +5,7 @@ pub use self::client_service::ClientService; pub use self::worker_service::WorkerService; use grpc; + use cerberus_proto::{mapreduce_grpc, worker_grpc}; use errors::*; diff --git a/master/src/worker_management/state.rs b/master/src/worker_management/state.rs index fc32b351..78d9a9e0 100644 --- a/master/src/worker_management/state.rs +++ b/master/src/worker_management/state.rs @@ -59,6 +59,14 @@ impl State { workers } + pub fn get_tasks(&self) -> Vec<&Task> { + let mut tasks = Vec::new(); + for task in self.tasks.values() { + tasks.push(task) + } + tasks + } + // Returns a list of worker not currently assigned a task sorted by most recent health checks. pub fn get_available_workers(&self) -> Vec { let mut workers = Vec::new(); diff --git a/master/src/worker_management/worker_manager.rs b/master/src/worker_management/worker_manager.rs index 145ed0c9..050e6e45 100644 --- a/master/src/worker_management/worker_manager.rs +++ b/master/src/worker_management/worker_manager.rs @@ -342,6 +342,45 @@ impl WorkerManager { let state = self.state.lock().unwrap(); state.get_workers_running_job(job_id) } + + pub fn get_workers_info(&self) -> Result { + let state = self.state.lock().unwrap(); + + let mut results_vec = Vec::new(); + for worker in state.get_workers() { + let worker_info = json!({ + "worker_id": worker.worker_id, + "status": worker.get_serializable_worker_status(), + "operation_status": worker.get_serializable_operation_status(), + "current_task_id": worker.current_task_id, + "task_assignments_failed": worker.task_assignments_failed, + }); + + results_vec.push(worker_info); + } + + Ok(json!(results_vec)) + } + + pub fn get_tasks_info(&self) -> Result { + let state = self.state.lock().unwrap(); + + let mut results_vec = Vec::new(); + for task in state.get_tasks() { + let task_info = json!({ + "task_id": task.id, + "job_id": task.job_id, + "task_type": task.task_type, + "assigned_worker_id": task.assigned_worker_id, + "status": task.status, + "failure_count": task.failure_count, + }); + + results_vec.push(task_info); + } + + Ok(json!(results_vec)) + } } impl state::SimpleStateHandling for WorkerManager { @@ -362,9 +401,9 @@ impl state::SimpleStateHandling for WorkerManager { for worker in workers { let add_client_result = self.worker_interface.add_client(worker); if let Err(e) = add_client_result { - output_error(&e.chain_err(|| { - format!("Unable to reconnect to worker {}", worker.worker_id) - })); + output_error(&e.chain_err( + || format!("Unable to reconnect to worker {}", worker.worker_id), + )); workers_to_remove.push(worker.worker_id.to_owned()); } } @@ -396,9 +435,9 @@ fn handle_task_assignment_result( ) { let worker_id = assignment_result.worker_id.clone(); if let Err(err) = assignment_result.result { - output_error(&err.chain_err(|| { - format!("Error assigning task to worker {}", worker_id) - })); + output_error(&err.chain_err( + || format!("Error assigning task to worker {}", worker_id), + )); let result = worker_manager.handle_task_assignment_failure(&worker_id); if let Err(err) = result { output_error(&err.chain_err(|| "Error handling task assignment failure.")); From 38c755743ebe719c5cf86adf2bada691ae534b13 Mon Sep 17 00:00:00 2001 From: Darragh Griffin Date: Tue, 13 Mar 2018 15:30:11 +0000 Subject: [PATCH 16/19] Clean up CSS and dashboard --- master/content/dashboard.js | 15 +++++--- master/content/index.html | 10 ++--- master/content/stylesheet.css | 72 ++++++++++++++++++++++++++++++----- 3 files changed, 77 insertions(+), 20 deletions(-) diff --git a/master/content/dashboard.js b/master/content/dashboard.js index 7a31b6bf..d7fe0c20 100644 --- a/master/content/dashboard.js +++ b/master/content/dashboard.js @@ -7,14 +7,17 @@ function createCard(parent) { .addClass("container") .appendTo(infoBox); - return container; + var table = $("") + .addClass("stats-table") + .appendTo(container); + + return table; } function addProperty(name, value, container) { - $("") - .addClass("ui-all") - .text(name + ": " + value) - .appendTo(container); + var row = $("").appendTo(container); + $("
").text(name).appendTo(row); + $("").text(value).appendTo(row); } function updateWorkersList() { @@ -49,7 +52,7 @@ function updateJobsList() { jobsBox.empty(); jobs.forEach(function(jobsInfo) { - var container = createCard(tasksBox); + var container = createCard(jobsBox); addProperty("Job ID", jobsInfo.job_id, container); addProperty("Client ID", jobsInfo.client_id, container); diff --git a/master/content/index.html b/master/content/index.html index c63e3526..fc335273 100644 --- a/master/content/index.html +++ b/master/content/index.html @@ -14,16 +14,16 @@

Cluster Dashboard

Workers -
+
-
Jobs -
+
Jobs +
-
Tasks -
+
Tasks +
diff --git a/master/content/stylesheet.css b/master/content/stylesheet.css index 5ddefe6b..f58941e8 100644 --- a/master/content/stylesheet.css +++ b/master/content/stylesheet.css @@ -2,23 +2,60 @@ box-sizing: border-box; } +html, body { + height: 100%; + margin: 0; + width: 100%; +} + body { font-family: Arial, Helvetica, sans-serif; } +.stats-table { + border-collapse: collapse; + width: 100%; +} + +td, th { + border: 1px solid #ddd; + font-weight: normal; + padding: 8px; +} + +td:first-child { + font-weight: bold; +} + +tr:nth-child(even) { + background-color: #f2f2f2; +} + +tr:nth-child(odd) { + background-color: #ffffff; +} + +tr:hover { + background-color: #ddd; +} + .header { background-color: #2196f3; color: #fff; font-size: 20px; + height: 70px; padding: 1px; text-align: center; } .card { - background-color: #8bc34a; + background-color: #bbdefb; border-radius: 5px; box-shadow: 0 4px 8px 0 rgba(0, 0, 0, .2); color: #212121; + float: left; + margin-bottom: 20px; + margin-right: 20px; transition: .3s; width: 45%; } @@ -28,12 +65,16 @@ body { } .container { - display: table; - padding: 2px 16px; + padding: 8px 16px; +} + +.container-grid { + display: auto; + padding: 8px 16px; } .lower { - height: 45%; + height: 50%; } .row { @@ -47,7 +88,10 @@ body { } .box { - background-color: #bbdefb; + background-color: #ffffff; + border-color: #bbdefb; + border-style: double; + border-width: medium; color: #212121; float: left; font-size: 20px; @@ -68,11 +112,14 @@ body { } .column { - background-color: #ffc107; + background-color: #ffffff; + border-color: #bbdefb; + border-style: double; + border-width: medium; color: #212121; float: left; font-weight: bold; - height: 500px; + height: 100%; overflow: auto; padding: 10px; text-align: center; @@ -86,11 +133,18 @@ body { } .footer { - background-color: #ffecb3; - padding: 10px; + background-color: #b3e5fc; + padding-bottom: 10px; + padding-top: 10px; text-align: center; } +a:link { + color: #FFFFFF; + font-weight: bold; + padding-top: 10px; +} + @media (max-width: 600px) { .column { width: 100%; From 5dc9ac5246cbd1bd7d2391832caf7ba099e22fbf Mon Sep 17 00:00:00 2001 From: Conor Griffin Date: Tue, 6 Mar 2018 21:55:34 +0000 Subject: [PATCH 17/19] Add protos for distributed file system. --- master/src/main.rs | 9 ++-- master/src/server/filesystem_service.rs | 54 +++++++++++++++++++ master/src/server/mod.rs | 12 ++++- proto/build.rs | 1 + proto/filesystem.proto | 71 +++++++++++++++++++++++++ proto/src/lib.rs | 3 ++ 6 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 master/src/server/filesystem_service.rs create mode 100644 proto/filesystem.proto diff --git a/master/src/main.rs b/master/src/main.rs index 4fafab00..a530d2fa 100644 --- a/master/src/main.rs +++ b/master/src/main.rs @@ -58,7 +58,7 @@ use util::init_logger; use util::data_layer::{AbstractionLayer, NullAbstractionLayer, NFSAbstractionLayer}; use worker_communication::WorkerInterfaceImpl; use worker_management::{WorkerManager, run_health_check_loop, run_task_assigment_loop}; -use server::{Server, ClientService, WorkerService}; +use server::{Server, ClientService, FileSystemService, WorkerService}; use state::StateHandler; const MAIN_LOOP_SLEEP_MS: u64 = 100; @@ -102,9 +102,12 @@ fn run() -> Result<()> { // Cli to Master Communications let client_service = ClientService::new( Arc::clone(&map_reduce_scheduler), - data_abstraction_layer_arc, + Arc::clone(&data_abstraction_layer_arc), ); - let srv = Server::new(port, client_service, worker_service) + + let file_system_service = FileSystemService::new(data_abstraction_layer_arc); + + let srv = Server::new(port, client_service, worker_service, file_system_service) .chain_err(|| "Error building grpc server.")?; let state_handler = StateHandler::new( diff --git a/master/src/server/filesystem_service.rs b/master/src/server/filesystem_service.rs new file mode 100644 index 00000000..f8367c5a --- /dev/null +++ b/master/src/server/filesystem_service.rs @@ -0,0 +1,54 @@ +use std::sync::Arc; + +use grpc::{SingleResponse, RequestOptions}; + +use util::data_layer::AbstractionLayer; + +use cerberus_proto::filesystem as pb; +use cerberus_proto::filesystem_grpc as grpc_pb; + +/// `FileSystemService` recieves communication from a clients and workers in relation to the +/// distributed file system. +pub struct FileSystemService { + // file_system_manager: Arc, + data_abstraction_layer: Arc, +} + +impl FileSystemService { + pub fn new(data_abstraction_layer: Arc) -> Self { + FileSystemService { + // file_system_manager: file_system_manager, + data_abstraction_layer: data_abstraction_layer, + } + } +} + +impl grpc_pb::FileSystemMasterService for FileSystemService { + fn upload_file( + &self, + _: RequestOptions, + req: pb::UploadFileRequest, + ) -> SingleResponse { + SingleResponse::completed(pb::EmptyMessage::new()) + } + + fn download_file( + &self, + _: RequestOptions, + req: pb::DownloadFileRequest, + ) -> SingleResponse { + let response = pb::DownloadFileResponse::new(); + + SingleResponse::completed(response) + } + + fn get_file_location( + &self, + _: RequestOptions, + req: pb::FileLocationRequest, + ) -> SingleResponse { + let response = pb::FileLocationResponse::new(); + + SingleResponse::completed(response) + } +} diff --git a/master/src/server/mod.rs b/master/src/server/mod.rs index 4b05d65b..3af910b0 100644 --- a/master/src/server/mod.rs +++ b/master/src/server/mod.rs @@ -1,12 +1,14 @@ pub mod client_service; +pub mod filesystem_service; pub mod worker_service; pub use self::client_service::ClientService; +pub use self::filesystem_service::FileSystemService; pub use self::worker_service::WorkerService; use grpc; -use cerberus_proto::{mapreduce_grpc, worker_grpc}; +use cerberus_proto::{filesystem_grpc, mapreduce_grpc, worker_grpc}; use errors::*; const GRPC_THREAD_POOL_SIZE: usize = 8; @@ -20,6 +22,7 @@ impl Server { port: u16, client_service: ClientService, worker_service: WorkerService, + file_system_service: FileSystemService, ) -> Result { let mut server_builder = grpc::ServerBuilder::new_plain(); server_builder.http.set_port(port); @@ -37,6 +40,13 @@ impl Server { worker_service, )); + // Register the FileSystemService + server_builder.add_service( + filesystem_grpc::FileSystemMasterServiceServer::new_service_def( + file_system_service, + ), + ); + Ok(Server { server: server_builder.build().chain_err( || "Error building grpc server", diff --git a/proto/build.rs b/proto/build.rs index fb90890b..722cf6ad 100644 --- a/proto/build.rs +++ b/proto/build.rs @@ -51,4 +51,5 @@ fn compile(proto_name: &str) { fn main() { compile("mapreduce"); compile("worker"); + compile("filesystem"); } diff --git a/proto/filesystem.proto b/proto/filesystem.proto new file mode 100644 index 00000000..8eb32c52 --- /dev/null +++ b/proto/filesystem.proto @@ -0,0 +1,71 @@ +syntax = "proto3"; + +package filesystem; + +// File System Master service is responsible for recieving files uploaded to the file system +// and processing requests for file locations from workers. +service FileSystemMasterService { + rpc UploadFile (UploadFileRequest) returns (EmptyMessage); + rpc DownloadFile (DownloadFileRequest) returns (DownloadFileResponse); + rpc GetFileLocation (FileLocationRequest) returns (FileLocationResponse); +} + +// Empty message where there is nothing to be send or replied with. +message EmptyMessage {} + +message UploadFileRequest { + string file_path = 1; + string client_id = 2; + uint64 start_byte = 3; + bytes data = 4; +} + +message DownloadFileRequest { + string file_path = 1; + string client_id = 2; + uint64 start_byte = 3; + uint64 end_byte = 4; +} + +message DownloadFileResponse { + bytes data = 1; +} + +message FileLocationRequest { + string file_path = 1; + uint64 start_byte = 2; + uint64 end_byte = 3; +} + +message FileLocation { + string worker_address = 1; + uint64 start_byte = 2; + uint64 end_byte = 3; +} + +message FileLocationResponse { + repeated FileLocation file_locations = 1; +} + +// File System Worker service is responsible for recieving distributed files from the master +// and allowing other workers to read the files stored on that worker. +service FileSystemWorkerService { + rpc StoreFile (StoreFileRequest) returns (EmptyMessage); + rpc ReadFile (ReadFileRequest) returns (ReadFileResponse); +} + +message StoreFileRequest { + string file_path = 1; + uint64 start_byte = 2; + bytes data = 3; +} + +message ReadFileRequest { + string file_path = 1; + uint64 start_byte = 2; + uint64 end_byte = 3; +} + +message ReadFileResponse { + bytes data = 1; +} diff --git a/proto/src/lib.rs b/proto/src/lib.rs index 0e036464..eb15d4c5 100644 --- a/proto/src/lib.rs +++ b/proto/src/lib.rs @@ -3,6 +3,9 @@ extern crate grpc_rust; extern crate protobuf; extern crate tls_api; +pub mod filesystem; +pub mod filesystem_grpc; + pub mod mapreduce; pub mod mapreduce_grpc; From 2947cbf2a09c8d1a54d2249464a03e90b4058ea9 Mon Sep 17 00:00:00 2001 From: Conor Griffin Date: Thu, 8 Mar 2018 00:40:25 +0000 Subject: [PATCH 18/19] Distributed file system version 1 --- Cargo.lock | 6 + cli/Cargo.toml | 1 + cli/src/main.rs | 3 + cli/src/parser.rs | 40 +++ cli/src/runner.rs | 238 ++++++++++++++++- libcerberus/src/io.rs | 2 +- libcerberus/src/runner.rs | 2 +- master/src/common/job.rs | 14 +- master/src/main.rs | 61 ++++- master/src/parser.rs | 10 +- master/src/scheduling/scheduler.rs | 3 +- master/src/scheduling/task_processor.rs | 189 ++++++++------ master/src/server/filesystem_service.rs | 82 ++++-- master/src/worker_management/state.rs | 9 +- .../src/worker_management/worker_manager.rs | 20 +- proto/filesystem.proto | 44 ++-- util/Cargo.toml | 5 + util/src/data_layer/abstraction_layer.rs | 12 +- util/src/data_layer/nfs_layer.rs | 81 ++++-- util/src/data_layer/null_layer.rs | 43 ++- .../distributed_file_layer.rs | 211 +++++++++++++++ .../filesystem_manager.rs | 247 ++++++++++++++++++ .../filesystem_master_interface.rs | 130 +++++++++ .../filesystem_worker_interface.rs | 112 ++++++++ .../local_file_manager.rs | 239 +++++++++++++++++ util/src/distributed_filesystem/mod.rs | 15 ++ .../worker_info_provider.rs | 8 + util/src/lib.rs | 6 + worker/src/main.rs | 76 ++++-- worker/src/operations/combine.rs | 15 +- worker/src/operations/io.rs | 44 +--- worker/src/operations/map.rs | 9 +- worker/src/operations/operation_handler.rs | 2 +- worker/src/operations/reduce.rs | 2 +- worker/src/parser.rs | 9 + worker/src/server/filesystem_service.rs | 82 ++++++ worker/src/server/master_service.rs | 2 +- worker/src/server/mod.rs | 13 + 38 files changed, 1835 insertions(+), 252 deletions(-) create mode 100644 util/src/distributed_filesystem/distributed_file_layer.rs create mode 100644 util/src/distributed_filesystem/filesystem_manager.rs create mode 100644 util/src/distributed_filesystem/filesystem_master_interface.rs create mode 100644 util/src/distributed_filesystem/filesystem_worker_interface.rs create mode 100644 util/src/distributed_filesystem/local_file_manager.rs create mode 100644 util/src/distributed_filesystem/mod.rs create mode 100644 util/src/distributed_filesystem/worker_info_provider.rs create mode 100644 worker/src/server/filesystem_service.rs diff --git a/Cargo.lock b/Cargo.lock index a8316251..414cea05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -183,6 +183,7 @@ dependencies = [ "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "grpc 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "prettytable-rs 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", + "util 0.4.0", "uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1237,10 +1238,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "util" version = "0.4.0" dependencies = [ + "cerberus-proto 0.4.0", "chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "grpc 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "protobuf 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index bb1388fe..9e167fdd 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -11,3 +11,4 @@ error-chain = "0.11.0" grpc = "0.2.1" prettytable-rs = "^0.6" uuid = { version = "0.5", features = ["v4"] } +util = { path = "../util" } diff --git a/cli/src/main.rs b/cli/src/main.rs index 88a226f8..0dd224cd 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -7,6 +7,7 @@ extern crate grpc; #[macro_use] extern crate prettytable; extern crate uuid; +extern crate util; extern crate cerberus_proto; @@ -53,6 +54,8 @@ fn run() -> Result<()> { } ("cancel", sub) => runner::cancel(&client, sub), ("status", Some(sub)) => runner::status(&client, sub), + ("upload", Some(sub)) => runner::upload(&master_addr, sub), + ("download", Some(sub)) => runner::download(&master_addr, sub), _ => Err(matches.usage().into()), } } diff --git a/cli/src/parser.rs b/cli/src/parser.rs index 4485e9ce..10d3eaab 100644 --- a/cli/src/parser.rs +++ b/cli/src/parser.rs @@ -61,6 +61,46 @@ pub fn parse_command_line<'a>() -> ArgMatches<'a> { .takes_value(true), ), ) + .subcommand( + SubCommand::with_name("upload") + .about("Uploads a file or directory to the cluster distributed filesystem") + .arg( + Arg::with_name("local_path") + .short("l") + .long("local_path") + .help("Path of the file or directory on the local machine") + .required(true) + .takes_value(true), + ) + .arg( + Arg::with_name("remote_path") + .short("r") + .long("remote_path") + .help("Path of the file or directory on the cluster. The local file path will be used if this is not provided") + .required(false) + .takes_value(true), + ), + ) + .subcommand( + SubCommand::with_name("download") + .about("Downloads a file or directory from the cluster distributed filesystem") + .arg( + Arg::with_name("remote_path") + .short("r") + .long("remote_path") + .help("Path of the file or directory on the distributed filesystem") + .required(true) + .takes_value(true), + ) + .arg( + Arg::with_name("local_path") + .short("l") + .long("local_path") + .help("Directory or file to store the file or directory") + .required(true) + .takes_value(true), + ), + ) .subcommand(SubCommand::with_name("cluster_status").about( "Status of the cluster", )) diff --git a/cli/src/runner.rs b/cli/src/runner.rs index d8b51b39..a55f65f5 100644 --- a/cli/src/runner.rs +++ b/cli/src/runner.rs @@ -1,7 +1,11 @@ use std::env; use std::fs; +use std::fs::{DirEntry, File}; use std::io::prelude::{Read, Write}; -use std::path::Path; +use std::io::BufReader; +use std::net::SocketAddr; +use std::sync::Arc; +use std::path::{Path, PathBuf}; use chrono::Local; use clap::ArgMatches; @@ -12,6 +16,9 @@ use cerberus_proto::mapreduce as pb; use cerberus_proto::mapreduce_grpc as grpc_pb; use cerberus_proto::mapreduce_grpc::MapReduceService; // do not use use errors::*; +use util::data_layer::AbstractionLayer; +use util::distributed_filesystem::{NetworkFileSystemMasterInterface, FileSystemMasterInterface, + DFSAbstractionLayer, LocalFileManager}; // Directory of client ID in the users home directory. const CLIENT_ID_DIR: &str = ".local/share/"; @@ -19,6 +26,7 @@ const CLIENT_ID_DIR: &str = ".local/share/"; const CLIENT_ID_FILE: &str = "cerberus"; // Default priority applied to jobs. const DEFAULT_PRIORITY: &str = "3"; +const DFS_FILE_DIRECTORY: &str = "/tmp/cerberus/dfs/"; fn verify_valid_path(path_str: &str) -> Result { let path = Path::new(path_str); @@ -183,6 +191,234 @@ pub fn cancel(client: &grpc_pb::MapReduceServiceClient, args: Option<&ArgMatches Ok(()) } +fn get_local_files(path: &Path) -> Result> { + let mut files = Vec::new(); + + if path.is_dir() { + let entries = fs::read_dir(path).chain_err( + || "Unable to read local file directroy", + )?; + + for entry in entries { + let entry: DirEntry = entry.chain_err(|| "Error reading input directory")?; + let entry_path = Path::new("/").join(entry.path()); + if entry_path.is_file() { + files.push(entry_path.to_string_lossy().to_string()); + } + } + } else { + files.push(path.to_string_lossy().to_string()); + } + + Ok(files) +} + +fn upload_local_file( + master_interface: &NetworkFileSystemMasterInterface, + local_path: &str, + remote_path: &str, +) -> Result<()> { + println!("Uploading File {} to Cluster", local_path); + //TODO(conor): Improve this function to allow for files that can not be kept in memory. + + let file = File::open(local_path).chain_err(|| { + format!("unable to open file {}", local_path) + })?; + + let mut buf_reader = BufReader::new(file); + let mut data = Vec::new(); + buf_reader.read_to_end(&mut data).chain_err(|| { + format!("unable to read content of {}", local_path) + })?; + + master_interface + .upload_file_chunk(remote_path, 0, data) + .chain_err(|| "Error uploading file chunk.")?; + + Ok(()) +} + +fn get_upload_file_path(local_path: &str, remote_path: Option<&str>) -> Result { + let remote_path_str = match remote_path { + Some(rstr) => rstr, + None => return Ok(local_path.to_owned()), + }; + + let remote_path = Path::new(remote_path_str); + if remote_path.is_file() { + return Ok(remote_path_str.to_owned()); + } + + let local_file_name = Path::new(&local_path).file_name().chain_err( + || "Error getting file name to upload", + )?; + + let remote_path = remote_path.join(local_file_name); + Ok(remote_path.to_string_lossy().to_string()) +} + +pub fn upload(master_addr: &SocketAddr, args: &ArgMatches) -> Result<()> { + println!("Uploading File(s) to Cluster..."); + + let local_path = args.value_of("local_path").chain_err( + || "Local path can not be empty", + )?; + + let remote_path = args.value_of("remote_path"); + + let local_files = get_local_files(Path::new(local_path)).chain_err( + || "Error getting files to uplaod to cluster", + )?; + + if local_files.is_empty() { + return Err( + "No local file found to upload. Is the directory empty?".into(), + ); + } else if let Some(remote_path_str) = remote_path { + if local_files.len() > 1 { + let remote_path = Path::new(remote_path_str); + if remote_path.is_file() { + return Err( + "Remote path must be directory when uploading more than one file.".into(), + ); + } + } + } + + let master_interface = + NetworkFileSystemMasterInterface::new(*master_addr) + .chain_err(|| "Error creating distributed filesystem master interface")?; + + for local_path in local_files { + let remote_path_str = get_upload_file_path(&local_path, remote_path).chain_err( + || "Error getting file path to upload", + )?; + + upload_local_file(&master_interface, &local_path, &remote_path_str) + .chain_err(|| "Error uploading local file")?; + } + + Ok(()) +} + +fn get_files_to_download( + data_layer: &DFSAbstractionLayer, + remote_path: &str, +) -> Result> { + let mut files = Vec::new(); + + if data_layer.is_file(Path::new(remote_path)).chain_err( + || "Error checking is file", + )? + { + files.push(remote_path.to_string()); + } else { + let dir_entries = data_layer.read_dir(Path::new(remote_path)).chain_err( + || "Error reading directory", + )?; + + for entry in dir_entries { + files.push(entry.to_string_lossy().to_string()); + } + } + + Ok(files) +} + +fn download_file( + data_layer: &DFSAbstractionLayer, + remote_path: &str, + local_path: &str, +) -> Result<()> { + println!("Downloading file {} to {}", remote_path, local_path); + + let remote_path = Path::new(remote_path); + let file_length = data_layer.get_file_length(remote_path).chain_err( + || "Error getting file length", + )?; + + let file_data = data_layer + .read_file_location(remote_path, 0 /* Start btye */, file_length) + .chain_err(|| "Error reading file")?; + + let mut local_directory = Path::new(local_path).to_path_buf(); + local_directory.pop(); + fs::create_dir_all(local_directory).chain_err( + || "Error creating new local directory", + )?; + + let mut file = File::create(local_path).chain_err(|| { + format!("unable to create file {}", local_path) + })?; + + file.write_all(&file_data).chain_err(|| { + format!( + "unable to write content to {}", + local_path, + ) + })?; + + Ok(()) +} + +fn get_download_file_path(local_path_str: &str, remote_path: &str) -> Result { + let local_path = Path::new(local_path_str); + + if local_path.is_dir() { + let file_name = Path::new(&remote_path).file_name().chain_err( + || "Error getting file name to download", + )?; + + let local_path = local_path.join(file_name); + return Ok(local_path.to_string_lossy().to_string()); + } + + Ok(local_path_str.to_owned()) +} + +pub fn download(master_addr: &SocketAddr, args: &ArgMatches) -> Result<()> { + println!("Downloading File(s) from Cluster..."); + + let remote_path = args.value_of("remote_path").chain_err( + || "Remote path can not be empty", + )?; + + let local_path = args.value_of("local_path").chain_err( + || "Local path can not be empty", + )?; + + let master_interface = + NetworkFileSystemMasterInterface::new(*master_addr) + .chain_err(|| "Error creating distributed filesystem master interface")?; + + let mut path_buf = PathBuf::new(); + path_buf.push(DFS_FILE_DIRECTORY); + let local_file_manager = Arc::new(LocalFileManager::new(path_buf)); + let data_layer = DFSAbstractionLayer::new(local_file_manager, Box::new(master_interface)); + + let files = get_files_to_download(&data_layer, remote_path).chain_err( + || "Error getting files to download", + )?; + + if files.is_empty() { + return Err("No files found to download".into()); + } else if files.len() > 1 && Path::new(local_path).is_file() { + return Err( + "Local Path must be directory to download multiple files".into(), + ); + } + + for remote_file_path in files { + let download_path = get_download_file_path(local_path, &remote_file_path) + .chain_err(|| "Error getting download path")?; + + download_file(&data_layer, &remote_file_path, &download_path) + .chain_err(|| "Error downloading file")?; + } + + Ok(()) +} + pub fn status(client: &grpc_pb::MapReduceServiceClient, matches: &ArgMatches) -> Result<()> { let mut req = pb::MapReduceStatusRequest::new(); req.set_client_id(get_client_id()?); diff --git a/libcerberus/src/io.rs b/libcerberus/src/io.rs index 689c0bcc..84cb8b75 100644 --- a/libcerberus/src/io.rs +++ b/libcerberus/src/io.rs @@ -46,7 +46,7 @@ where } /// `write_intermediate_vector` attempts to serialise an `Vec` to a given sink. -pub fn write_intermediate_vector(sink: &mut W, output: &Vec) -> Result<()> +pub fn write_intermediate_vector(sink: &mut W, output: &[V]) -> Result<()> where W: Write, V: Default + Serialize, diff --git a/libcerberus/src/runner.rs b/libcerberus/src/runner.rs index 6cfac36b..baecbc22 100644 --- a/libcerberus/src/runner.rs +++ b/libcerberus/src/runner.rs @@ -135,7 +135,7 @@ where } } -/// Construct a UserImplRegistryBuilder that does not need a `Combine` implementation +/// Construct a `UserImplRegistryBuilder` that does not need a `Combine` implementation impl<'a, M, R, P> UserImplRegistryBuilder<'a, M, R, P, NullCombiner> where M: Map + 'a, diff --git a/master/src/common/job.rs b/master/src/common/job.rs index 5706d5a7..91ff8fab 100644 --- a/master/src/common/job.rs +++ b/master/src/common/job.rs @@ -156,10 +156,7 @@ impl Job { )?; if !is_dir { return Err( - format!( - "Input directory does not exist: {:?}", - data_abstraction_layer.absolute_path(input_path) - ).into(), + format!("Input directory does not exist: {:?}", input_path).into(), ); } @@ -168,12 +165,7 @@ impl Job { || "Error checking if path is a file", )?; if !is_file { - return Err( - format!( - "Binary does not exist: {:?}", - data_abstraction_layer.absolute_path(binary_path) - ).into(), - ); + return Err(format!("Binary does not exist: {:?}", binary_path).into()); } // Binary exists, so run sanity-check on it to verify that it's a libcerberus binary. @@ -186,7 +178,7 @@ impl Job { ) -> Result<()> { let binary_path = Path::new(&self.binary_path); let absolute_path = data_abstraction_layer - .absolute_path(binary_path) + .get_local_file(binary_path) .chain_err(|| "unable to get absolute path")?; let child = Command::new(absolute_path) .arg("sanity-check") diff --git a/master/src/main.rs b/master/src/main.rs index a530d2fa..1b0a13ad 100644 --- a/master/src/main.rs +++ b/master/src/main.rs @@ -48,14 +48,19 @@ mod parser; use std::sync::Arc; use std::{thread, time}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::str::FromStr; +use clap::ArgMatches; use dashboard::DashboardServer; + use errors::*; use scheduling::{TaskProcessorImpl, Scheduler, run_task_update_loop}; use util::init_logger; use util::data_layer::{AbstractionLayer, NullAbstractionLayer, NFSAbstractionLayer}; +use util::distributed_filesystem::{LocalFileManager, DFSAbstractionLayer, + LocalFileSystemMasterInterface, FileSystemManager, + WorkerInfoProvider}; use worker_communication::WorkerInterfaceImpl; use worker_management::{WorkerManager, run_health_check_loop, run_task_assigment_loop}; use server::{Server, ClientService, FileSystemService, WorkerService}; @@ -66,6 +71,43 @@ const DUMP_LOOP_MS: u64 = 5000; const DEFAULT_PORT: &str = "8081"; const DEFAULT_DUMP_DIR: &str = "/var/lib/cerberus"; const DEFAULT_DASHBOARD_ADDRESS: &str = "127.0.0.1:3000"; +const DFS_FILE_DIRECTORY: &str = "/tmp/cerberus/dfs/"; + +fn get_data_abstraction_layer( + matches: &ArgMatches, + worker_info_provider: &Arc, +) -> (Arc, Option>) { + let data_abstraction_layer: Arc; + let filesystem_manager: Option>; + + let nfs_path = matches.value_of("nfs"); + let dfs = matches.is_present("dfs"); + if let Some(path) = nfs_path { + data_abstraction_layer = Arc::new(NFSAbstractionLayer::new(Path::new(path))); + filesystem_manager = None; + } else if dfs { + let mut storage_dir = PathBuf::new(); + storage_dir.push(DFS_FILE_DIRECTORY); + let local_file_manager_arc = Arc::new(LocalFileManager::new(storage_dir)); + let file_manager_arc = Arc::new(FileSystemManager::new(Arc::clone(worker_info_provider))); + + let master_interface = Box::new(LocalFileSystemMasterInterface::new( + Arc::clone(&file_manager_arc), + )); + + data_abstraction_layer = Arc::new(DFSAbstractionLayer::new( + Arc::clone(&local_file_manager_arc), + master_interface, + )); + + filesystem_manager = Some(file_manager_arc); + } else { + data_abstraction_layer = Arc::new(NullAbstractionLayer::new()); + filesystem_manager = None; + } + + (data_abstraction_layer, filesystem_manager) +} fn run() -> Result<()> { println!("Cerberus Master!"); @@ -82,17 +124,16 @@ fn run() -> Result<()> { DEFAULT_DUMP_DIR, ); - let nfs_path = matches.value_of("nfs"); - let data_abstraction_layer_arc: Arc = match nfs_path { - Some(path) => Arc::new(NFSAbstractionLayer::new(Path::new(path))), - None => Arc::new(NullAbstractionLayer::new()), - }; + let worker_interface = Arc::new(WorkerInterfaceImpl::new()); + let worker_manager = Arc::new(WorkerManager::new(worker_interface)); + let worker_info_provider = Arc::clone(&worker_manager) as Arc; + + let (data_abstraction_layer_arc, filesystem_manager) = + get_data_abstraction_layer(&matches, &worker_info_provider); let task_processor = Arc::new(TaskProcessorImpl::new( Arc::clone(&data_abstraction_layer_arc), )); - let worker_interface = Arc::new(WorkerInterfaceImpl::new()); - let worker_manager = Arc::new(WorkerManager::new(worker_interface)); let map_reduce_scheduler = Arc::new(Scheduler::new(Arc::clone(&worker_manager), task_processor)); @@ -105,7 +146,7 @@ fn run() -> Result<()> { Arc::clone(&data_abstraction_layer_arc), ); - let file_system_service = FileSystemService::new(data_abstraction_layer_arc); + let file_system_service = FileSystemService::new(filesystem_manager); let srv = Server::new(port, client_service, worker_service, file_system_service) .chain_err(|| "Error building grpc server.")?; @@ -140,7 +181,7 @@ fn run() -> Result<()> { DEFAULT_DASHBOARD_ADDRESS, ); let mut dashboard = DashboardServer::new( - &dashboard_address, + dashboard_address, Arc::clone(&map_reduce_scheduler), Arc::clone(&worker_manager), ).chain_err(|| "Failed to create cluster dashboard server.")?; diff --git a/master/src/parser.rs b/master/src/parser.rs index 7077ed58..09df4c39 100644 --- a/master/src/parser.rs +++ b/master/src/parser.rs @@ -53,7 +53,15 @@ pub fn parse_command_line<'a>() -> ArgMatches<'a> { .long("dashboard-address") .short("d") .help("The address to serve the cluster dashboard to") - .takes_value(true) + .takes_value(true), + ) + .arg( + Arg::with_name("dfs") + .long("dfs") + .help( + "Makes the master run using the distributed file system for data access.", + ) + .takes_value(false) .required(false), ) .get_matches() diff --git a/master/src/scheduling/scheduler.rs b/master/src/scheduling/scheduler.rs index 92f8e301..2446868d 100644 --- a/master/src/scheduling/scheduler.rs +++ b/master/src/scheduling/scheduler.rs @@ -199,8 +199,7 @@ impl Scheduler { } let mut latest_job = &jobs[0]; - for i in 1..jobs.len() { - let job = &jobs[i]; + for job in jobs.iter().skip(1) { if job.time_requested > latest_job.time_requested { latest_job = job; } diff --git a/master/src/scheduling/task_processor.rs b/master/src/scheduling/task_processor.rs index e121e723..9c266cbe 100644 --- a/master/src/scheduling/task_processor.rs +++ b/master/src/scheduling/task_processor.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::io::{BufRead, BufReader}; +use std::cmp::max; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -8,12 +8,14 @@ use common::{Job, Task}; use util::data_layer::AbstractionLayer; use errors::*; -const MEGA_BYTE: usize = 1000 * 1000; -const MAP_INPUT_SIZE: usize = MEGA_BYTE * 64; +const MEGA_BYTE: u64 = 1000 * 1000; +const MAP_INPUT_SIZE: u64 = MEGA_BYTE * 64; +const CLOSEST_ENDLINE_STEP: u64 = 1000; +const NEWLINE: u8 = 0x0A; -struct MapTaskFileInformation { - task_num: u32, - bytes_to_write: usize, +#[derive(Clone)] +struct MapTaskInformation { + bytes_remaining: u64, input_locations: Vec, } @@ -37,97 +39,133 @@ impl TaskProcessorImpl { TaskProcessorImpl { data_abstraction_layer: data_abstraction_layer } } - /// `read_input_file` reads a given input file and splits it into chunks for map tasks. - fn read_input_file( + /// `get_closest_endline` files the endline closest to the end of a given range for input file + /// splitting. + fn get_closest_endline( &self, - job: &Job, - map_task_file_info: &mut MapTaskFileInformation, input_file_path: &PathBuf, - map_tasks: &mut Vec, - ) -> Result<()> { - let input_path_str = input_file_path.to_str().ok_or("Invalid input file path.")?; - - let mut input_location = pb::InputLocation::new(); - input_location.set_input_path(input_path_str.to_owned()); - input_location.set_start_byte(0); - - let input_file = self.data_abstraction_layer - .open_file(input_file_path) - .chain_err(|| "Error opening input file.")?; - - let mut bytes_read: usize = 0; - - let buf_reader = BufReader::new(input_file); - for line in buf_reader.lines() { - let mut read_str = line.chain_err(|| "Error reading Map input.")?; - bytes_read += read_str.len() + 1; // Add one byte for newline character + start_byte: u64, + end_byte: u64, + ) -> Result { + let mut current_byte = end_byte; + while current_byte > start_byte { + let new_current_byte = max(start_byte, current_byte - CLOSEST_ENDLINE_STEP); + + let bytes = self.data_abstraction_layer + .read_file_location(input_file_path, new_current_byte, current_byte) + .chain_err(|| "Error getting closest newline")?; + + current_byte = new_current_byte; + + for i in (0..bytes.len()).rev() { + if bytes[i] == NEWLINE { + // Start next chunk after newline. + return Ok(current_byte + (i as u64) + 1); + } + } + } - input_location.set_end_byte(bytes_read as u64); + // Could not find a newline, just split at the end of the chunk. + Ok(end_byte) + } - let ammount_read: usize = read_str.len(); - if ammount_read > map_task_file_info.bytes_to_write { - map_task_file_info.input_locations.push(input_location); + /// `read_input_file` reads a given input file and splits it into chunks for map tasks. + /// If a file can fit into one map task, it will not be split. + fn read_input_file(&self, input_file_path: &PathBuf) -> Result> { + let input_path_str = input_file_path.to_str().ok_or("Invalid input file path.")?; - input_location = pb::InputLocation::new(); - input_location.set_input_path(input_path_str.to_owned()); - input_location.set_start_byte(bytes_read as u64 + 1); - input_location.set_end_byte(bytes_read as u64 + 1); + let mut input_locations = Vec::new(); - map_tasks.push(Task::new_map_task( - job.id.as_str(), - job.binary_path.as_str(), - map_task_file_info.input_locations.clone(), - job.priority, - )); + let mut start_byte: u64 = 0; + let end_byte: u64 = self.data_abstraction_layer + .get_file_length(input_file_path) + .chain_err(|| "Error reading input file")?; - *map_task_file_info = MapTaskFileInformation { - task_num: map_task_file_info.task_num + 1, - bytes_to_write: MAP_INPUT_SIZE, + while end_byte - start_byte > MAP_INPUT_SIZE { + let new_start_byte = + self.get_closest_endline(input_file_path, start_byte, start_byte + MAP_INPUT_SIZE) + .chain_err(|| "Error reading input file")?; + start_byte = new_start_byte; - input_locations: Vec::new(), - }; - } else { - map_task_file_info.bytes_to_write -= ammount_read; - } + let mut input_location = pb::InputLocation::new(); + input_location.set_input_path(input_path_str.to_owned()); + input_location.set_start_byte(start_byte); + input_location.set_end_byte(new_start_byte); + input_locations.push(input_location); } - if input_location.start_byte != input_location.end_byte { - map_task_file_info.input_locations.push(input_location); + if start_byte != end_byte { + let mut input_location = pb::InputLocation::new(); + input_location.set_input_path(input_path_str.to_owned()); + input_location.set_start_byte(start_byte); + input_location.set_end_byte(end_byte); + input_locations.push(input_location); } - Ok(()) + Ok(input_locations) } -} -impl TaskProcessor for TaskProcessorImpl { - fn create_map_tasks(&self, job: &Job) -> Result> { - let mut map_tasks = Vec::new(); + /// `get_map_task_infos` reads a directory and creates a set of `MapTaskFileInformations` + fn get_map_task_infos(&self, input_directory: &Path) -> Result> { + let mut map_task_infos = Vec::new(); - let mut map_task_file_info = MapTaskFileInformation { - task_num: 1, - bytes_to_write: MAP_INPUT_SIZE, + let mut map_task_info = MapTaskInformation { + bytes_remaining: MAP_INPUT_SIZE, input_locations: Vec::new(), }; let paths = self.data_abstraction_layer - .read_dir(Path::new(&job.input_directory)) + .read_dir(input_directory) .chain_err(|| "Unable to read directory")?; for path in paths { if self.data_abstraction_layer .is_file(path.as_path()) .chain_err(|| "Failed to check if path is a file")? { - self.read_input_file(job, &mut map_task_file_info, &path, &mut map_tasks) - .chain_err(|| "Error reading input file.")?; + let input_locations = self.read_input_file(&path).chain_err( + || "Error reading input file.", + )?; + + for input_location in input_locations { + let bytes_to_read = input_location.end_byte - input_location.start_byte; + if bytes_to_read > map_task_info.bytes_remaining { + map_task_infos.push(map_task_info); + + map_task_info = MapTaskInformation { + bytes_remaining: MAP_INPUT_SIZE, + input_locations: Vec::new(), + }; + } + + map_task_info.bytes_remaining -= bytes_to_read; + map_task_info.input_locations.push(input_location); + } } } - if map_task_file_info.bytes_to_write != MAP_INPUT_SIZE { + if map_task_info.bytes_remaining != MAP_INPUT_SIZE { + map_task_infos.push(map_task_info); + } + + Ok(map_task_infos) + } +} + +impl TaskProcessor for TaskProcessorImpl { + fn create_map_tasks(&self, job: &Job) -> Result> { + let map_task_infos = self.get_map_task_infos(Path::new(&job.input_directory)) + .chain_err(|| "Error creating map tasks")?; + + // TODO(conor): Consider adding together any map tasks that can be combined here. + + let mut map_tasks = Vec::new(); + + for map_task_info in map_task_infos { map_tasks.push(Task::new_map_task( job.id.as_str(), job.binary_path.as_str(), - map_task_file_info.input_locations, + map_task_info.input_locations, job.priority, )); } @@ -169,6 +207,7 @@ mod tests { use std::io::{Read, Write}; use std::collections::HashSet; use std::fs; + use std::fs::File; use common::{JobOptions, TaskType}; use util::data_layer::NullAbstractionLayer; @@ -185,17 +224,11 @@ mod tests { input_path2.push("input-2"); fs::create_dir_all(test_path.clone()).unwrap(); - let mut input_file1 = task_processor - .data_abstraction_layer - .create_file(input_path1.as_path().clone()) - .unwrap(); + let mut input_file1 = File::create(&input_path1.as_path()).unwrap(); input_file1 .write_all(b"this is the first test file") .unwrap(); - let mut input_file2 = task_processor - .data_abstraction_layer - .create_file(input_path2.as_path().clone()) - .unwrap(); + let mut input_file2 = File::create(&input_path2.as_path()).unwrap(); input_file2 .write_all(b"this is the second test file") .unwrap(); @@ -224,20 +257,14 @@ mod tests { let path = perform_map_req.get_input().get_input_locations()[0] .input_path .clone(); - let mut input_file0 = task_processor - .data_abstraction_layer - .open_file(&Path::new(&path)) - .unwrap(); + let mut input_file0 = File::open(&path).unwrap(); let mut map_input0 = String::new(); input_file0.read_to_string(&mut map_input0).unwrap(); let path = perform_map_req.get_input().get_input_locations()[1] .input_path .clone(); - let mut input_file1 = task_processor - .data_abstraction_layer - .open_file(&Path::new(&path)) - .unwrap(); + let mut input_file1 = File::open(&path).unwrap(); let mut map_input1 = String::new(); input_file1.read_to_string(&mut map_input1).unwrap(); diff --git a/master/src/server/filesystem_service.rs b/master/src/server/filesystem_service.rs index f8367c5a..ca928cbd 100644 --- a/master/src/server/filesystem_service.rs +++ b/master/src/server/filesystem_service.rs @@ -1,25 +1,25 @@ use std::sync::Arc; -use grpc::{SingleResponse, RequestOptions}; - -use util::data_layer::AbstractionLayer; +use grpc::{SingleResponse, Error, RequestOptions}; use cerberus_proto::filesystem as pb; use cerberus_proto::filesystem_grpc as grpc_pb; +use util::distributed_filesystem::FileSystemManager; +use util::output_error; + +const NOT_DISTRIBUTED_FILESYSTEM: &str = "Master is not running in distributed filesytem configuration"; +const UPLOAD_FILE_ERROR: &str = "Error processing upload file request"; +const FILE_LOCATION_ERROR: &str = "Error processing file location request"; /// `FileSystemService` recieves communication from a clients and workers in relation to the /// distributed file system. pub struct FileSystemService { - // file_system_manager: Arc, - data_abstraction_layer: Arc, + filesystem_manager: Option>, } impl FileSystemService { - pub fn new(data_abstraction_layer: Arc) -> Self { - FileSystemService { - // file_system_manager: file_system_manager, - data_abstraction_layer: data_abstraction_layer, - } + pub fn new(filesystem_manager: Option>) -> Self { + FileSystemService { filesystem_manager: filesystem_manager } } } @@ -29,26 +29,68 @@ impl grpc_pb::FileSystemMasterService for FileSystemService { _: RequestOptions, req: pb::UploadFileRequest, ) -> SingleResponse { + info!("Processing upload file request for {}", req.file_path); + + let filesystem_manager = match self.filesystem_manager { + Some(ref filesystem_manager) => filesystem_manager, + None => { + error!("Master is not running in distributed filesystem mode."); + return SingleResponse::err(Error::Other(NOT_DISTRIBUTED_FILESYSTEM)); + } + }; + + if let Err(err) = filesystem_manager.upload_file_chunk( + &req.file_path, + req.start_byte, + &req.data, + ) + { + output_error(&err.chain_err(|| "Error processing upload file request.")); + return SingleResponse::err(Error::Other(UPLOAD_FILE_ERROR)); + } + SingleResponse::completed(pb::EmptyMessage::new()) } - fn download_file( + fn get_file_location( &self, _: RequestOptions, - req: pb::DownloadFileRequest, - ) -> SingleResponse { - let response = pb::DownloadFileResponse::new(); + req: pb::FileLocationRequest, + ) -> SingleResponse { + let filesystem_manager = match self.filesystem_manager { + Some(ref filesystem_manager) => filesystem_manager, + None => { + error!("Master is not running in distributed filesystem mode."); + return SingleResponse::err(Error::Other(NOT_DISTRIBUTED_FILESYSTEM)); + } + }; - SingleResponse::completed(response) + let file_location_result = + filesystem_manager.get_file_location(&req.file_path, req.start_byte, req.end_byte); + + match file_location_result { + Ok(file_location) => SingleResponse::completed(file_location), + Err(err) => { + output_error(&err.chain_err(|| "Error processing file location request.")); + SingleResponse::err(Error::Other(FILE_LOCATION_ERROR)) + } + } } - fn get_file_location( + fn get_file_info( &self, _: RequestOptions, - req: pb::FileLocationRequest, - ) -> SingleResponse { - let response = pb::FileLocationResponse::new(); + req: pb::FileInfoRequest, + ) -> SingleResponse { + let filesystem_manager = match self.filesystem_manager { + Some(ref filesystem_manager) => filesystem_manager, + None => { + error!("Master is not running in distributed filesystem mode."); + return SingleResponse::err(Error::Other(NOT_DISTRIBUTED_FILESYSTEM)); + } + }; - SingleResponse::completed(response) + let file_info_response = filesystem_manager.get_file_info(&req.file_path); + SingleResponse::completed(file_info_response) } } diff --git a/master/src/worker_management/state.rs b/master/src/worker_management/state.rs index 78d9a9e0..0e76a885 100644 --- a/master/src/worker_management/state.rs +++ b/master/src/worker_management/state.rs @@ -237,10 +237,7 @@ impl State { assigned_task.time_completed = Some(Utc::now()); } else { assigned_task.status = TaskStatus::Queued; - self.tasks.insert( - task_id.clone().to_owned(), - assigned_task.clone(), - ); + self.tasks.insert(task_id.to_owned(), assigned_task.clone()); self.priority_task_queue.push(PriorityTask::new( task_id.to_owned().clone(), FAILED_TASK_PRIORITY * assigned_task.job_priority, @@ -344,7 +341,7 @@ impl State { if scheduled_task.time_started == None { scheduled_task.time_started = Some(Utc::now()); } - scheduled_task.assigned_worker_id = worker_id.clone().to_owned(); + scheduled_task.assigned_worker_id = worker_id.to_owned(); scheduled_task.clone() } else { @@ -355,7 +352,7 @@ impl State { let worker = self.workers.get_mut(worker_id).chain_err(|| { format!("Worker with ID {} not found.", worker_id) })?; - worker.current_task_id = task_id.clone().to_owned(); + worker.current_task_id = task_id.to_owned(); Ok(assigned_task) } diff --git a/master/src/worker_management/worker_manager.rs b/master/src/worker_management/worker_manager.rs index 050e6e45..7cc164eb 100644 --- a/master/src/worker_management/worker_manager.rs +++ b/master/src/worker_management/worker_manager.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; +use std::net::SocketAddr; use std::{thread, time}; use std::sync::{Arc, Mutex}; @@ -17,6 +19,7 @@ use state; use state::StateHandling; use worker_communication::WorkerInterface; use worker_management::state::State; +use util::distributed_filesystem::WorkerInfoProvider; const HEALTH_CHECK_LOOP_MS: u64 = 100; const TIME_BEFORE_WORKER_TASK_REASSIGNMENT_S: i64 = 60; @@ -28,14 +31,14 @@ const ASSIGN_TASK_LOOP_MS: u64 = 100; /// It also handles worker health checking and stores the list of `Worker`s currently registered. pub struct WorkerManager { state: Arc>, - worker_interface: Arc, + worker_interface: Arc, task_update_sender: Mutex>, task_update_reciever: Arc>>, } impl WorkerManager { - pub fn new(worker_interface: Arc) -> Self { + pub fn new(worker_interface: Arc) -> Self { let (sender, receiver) = channel(); WorkerManager { state: Arc::new(Mutex::new(State::new())), @@ -417,6 +420,19 @@ impl state::SimpleStateHandling for WorkerManager { } } +impl WorkerInfoProvider for WorkerManager { + fn get_workers(&self) -> HashMap { + let mut worker_map = HashMap::new(); + let state = self.state.lock().unwrap(); + let workers = state.get_workers(); + for worker in workers { + worker_map.insert(worker.worker_id.to_owned(), worker.address); + } + + worker_map + } +} + pub fn run_health_check_loop(worker_manager: Arc) { thread::spawn(move || loop { thread::sleep(time::Duration::from_millis(HEALTH_CHECK_LOOP_MS)); diff --git a/proto/filesystem.proto b/proto/filesystem.proto index 8eb32c52..77db964b 100644 --- a/proto/filesystem.proto +++ b/proto/filesystem.proto @@ -6,8 +6,8 @@ package filesystem; // and processing requests for file locations from workers. service FileSystemMasterService { rpc UploadFile (UploadFileRequest) returns (EmptyMessage); - rpc DownloadFile (DownloadFileRequest) returns (DownloadFileResponse); rpc GetFileLocation (FileLocationRequest) returns (FileLocationResponse); + rpc GetFileInfo (FileInfoRequest) returns (FileInfoResponse); } // Empty message where there is nothing to be send or replied with. @@ -15,36 +15,40 @@ message EmptyMessage {} message UploadFileRequest { string file_path = 1; - string client_id = 2; - uint64 start_byte = 3; - bytes data = 4; -} - -message DownloadFileRequest { - string file_path = 1; - string client_id = 2; - uint64 start_byte = 3; - uint64 end_byte = 4; -} - -message DownloadFileResponse { - bytes data = 1; + uint64 start_byte = 2; + bytes data = 3; } message FileLocationRequest { string file_path = 1; uint64 start_byte = 2; + // Gets the locations for the entire file if end_byte is 0 uint64 end_byte = 3; } -message FileLocation { - string worker_address = 1; - uint64 start_byte = 2; - uint64 end_byte = 3; +message FileChunk { + uint64 start_byte = 1; + uint64 end_byte = 2; + repeated string worker_address = 3; } message FileLocationResponse { - repeated FileLocation file_locations = 1; + repeated FileChunk chunks = 1; +} + +message FileInfoRequest { + string file_path = 1; +} + +message FileInfoResponse { + bool exists = 1; + bool is_file = 2; + + // File length if file, number of files in directory if directory. + uint64 length = 3; + + // Files in the directory, if it is a directory. + repeated string children = 4; } // File System Worker service is responsible for recieving distributed files from the master diff --git a/util/Cargo.toml b/util/Cargo.toml index dcd2551a..9be84e13 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -4,7 +4,12 @@ version = "0.4.0" authors = ["Cerberus Authors "] [dependencies] +cerberus-proto = { path = "../proto" } chrono = "0.4" env_logger = "0.4.3" error-chain = "0.11.0" +grpc = "0.2.1" log = "0.3.8" +protobuf = "1.4.1" +rand = "0.4" +uuid = { version = "0.5", features = ["v4"] } diff --git a/util/src/data_layer/abstraction_layer.rs b/util/src/data_layer/abstraction_layer.rs index 9222bda8..226fe972 100644 --- a/util/src/data_layer/abstraction_layer.rs +++ b/util/src/data_layer/abstraction_layer.rs @@ -1,16 +1,18 @@ -use std::fs::File; use std::path::{Path, PathBuf}; use errors::*; pub trait AbstractionLayer { - fn open_file(&self, path: &Path) -> Result; + fn get_file_length(&self, path: &Path) -> Result; - fn create_file(&self, path: &Path) -> Result; + fn read_file_location(&self, path: &Path, start_byte: u64, end_byte: u64) -> Result>; - fn absolute_path(&self, path: &Path) -> Result; + fn write_file(&self, path: &Path, data: &[u8]) -> Result<()>; - fn abstracted_path(&self, path: &Path) -> Result; + /// `get_local_file` returns a filepath of the given file on the local machine. If the file can + /// not be accessed on the local machine already, it will retrive the file and create it on the + /// local machine. + fn get_local_file(&self, path: &Path) -> Result; fn read_dir(&self, path: &Path) -> Result>; diff --git a/util/src/data_layer/nfs_layer.rs b/util/src/data_layer/nfs_layer.rs index b7277639..54af00a1 100644 --- a/util/src/data_layer/nfs_layer.rs +++ b/util/src/data_layer/nfs_layer.rs @@ -1,5 +1,6 @@ use std::fs::{File, DirEntry}; use std::fs; +use std::io::{Read, Write, Seek, SeekFrom}; use std::path::{Path, PathBuf}; use errors::*; @@ -14,21 +15,15 @@ impl NFSAbstractionLayer { pub fn new(nfs_path: &Path) -> Self { NFSAbstractionLayer { nfs_path: PathBuf::from(nfs_path) } } -} - -impl AbstractionLayer for NFSAbstractionLayer { - fn open_file(&self, path: &Path) -> Result { - let file_path = self.absolute_path(path).chain_err(|| "Unable to get path")?; - debug!("Opening file: {}", file_path.to_string_lossy()); - File::open(file_path.clone()).chain_err(|| format!("unable to open file {:?}", file_path)) - } - fn create_file(&self, path: &Path) -> Result { - let file_path = self.absolute_path(path).chain_err(|| "Unable to get path")?; - debug!("Creating file: {}", file_path.to_string_lossy()); - File::create(file_path.clone()).chain_err(|| { - format!("unable to create file {:?}", file_path) - }) + fn abstracted_path(&self, path: &Path) -> Result { + if path.starts_with(self.nfs_path.clone()) { + let abstracted_path = path.strip_prefix(self.nfs_path.as_path()).chain_err( + || "Unable to strip prefix from path", + )?; + return Ok(PathBuf::from(abstracted_path)); + } + Ok(PathBuf::from(path)) } fn absolute_path(&self, path: &Path) -> Result { @@ -44,14 +39,56 @@ impl AbstractionLayer for NFSAbstractionLayer { Ok(self.nfs_path.join(relative_path)) } - fn abstracted_path(&self, path: &Path) -> Result { - if path.starts_with(self.nfs_path.clone()) { - let abstracted_path = path.strip_prefix(self.nfs_path.as_path()).chain_err( - || "Unable to strip prefix from path", - )?; - return Ok(PathBuf::from(abstracted_path)); - } - Ok(PathBuf::from(path)) + fn open_file(&self, path: &Path) -> Result { + let file_path = self.absolute_path(path).chain_err(|| "Unable to get path")?; + debug!("Opening file: {}", file_path.to_string_lossy()); + File::open(file_path.clone()).chain_err(|| format!("unable to open file {:?}", file_path)) + } +} + +impl AbstractionLayer for NFSAbstractionLayer { + fn get_file_length(&self, path: &Path) -> Result { + debug!("Getting file length: {:?}", path); + + let file_path = self.absolute_path(path).chain_err(|| "Unable to get path")?; + let metadata = fs::metadata(file_path).chain_err( + || "Error getting metadata", + )?; + + Ok(metadata.len()) + } + + fn read_file_location(&self, path: &Path, start_byte: u64, end_byte: u64) -> Result> { + debug!("Reading file: {:?}", path); + + let mut file = self.open_file(path)?; + file.seek(SeekFrom::Start(start_byte)).chain_err(|| { + format!("Error reading file {:?}", path) + })?; + + let mut bytes = vec![0; (end_byte - start_byte) as usize]; + file.read_exact(&mut bytes).chain_err(|| { + format!("Error reading file {:?}", path) + })?; + + Ok(bytes) + } + + fn write_file(&self, path: &Path, data: &[u8]) -> Result<()> { + let file_path = self.absolute_path(path).chain_err(|| "Unable to get path")?; + debug!("Writing file: {}", file_path.to_string_lossy()); + let mut file = File::create(file_path.clone()).chain_err(|| { + format!("unable to create file {:?}", file_path) + })?; + + file.write_all(data).chain_err(|| { + format!("unable to write content to {:?}", file_path) + }) + } + + + fn get_local_file(&self, path: &Path) -> Result { + self.absolute_path(path) } fn read_dir(&self, path: &Path) -> Result> { diff --git a/util/src/data_layer/null_layer.rs b/util/src/data_layer/null_layer.rs index 96c82a70..2b21fe7f 100644 --- a/util/src/data_layer/null_layer.rs +++ b/util/src/data_layer/null_layer.rs @@ -1,5 +1,6 @@ use std::fs::{File, DirEntry}; use std::fs; +use std::io::{Read, Write, Seek, SeekFrom}; use std::path::{Path, PathBuf}; use errors::*; @@ -14,24 +15,50 @@ impl NullAbstractionLayer { pub fn new() -> Self { NullAbstractionLayer } -} -impl AbstractionLayer for NullAbstractionLayer { fn open_file(&self, path: &Path) -> Result { debug!("Opening file: {}", path.to_string_lossy()); File::open(&path).chain_err(|| format!("unable to open file {:?}", path)) } +} - fn create_file(&self, path: &Path) -> Result { - debug!("Creating file: {}", path.to_string_lossy()); - File::create(&path).chain_err(|| format!("unable to create file {:?}", path)) +impl AbstractionLayer for NullAbstractionLayer { + fn get_file_length(&self, path: &Path) -> Result { + debug!("Getting file length: {:?}", path); + + let metadata = fs::metadata(path).chain_err(|| "Error getting metadata")?; + + Ok(metadata.len()) } - fn absolute_path(&self, path: &Path) -> Result { - Ok(PathBuf::from(path)) + fn read_file_location(&self, path: &Path, start_byte: u64, end_byte: u64) -> Result> { + debug!("Reading file: {:?}", path); + + let mut file = self.open_file(path)?; + file.seek(SeekFrom::Start(start_byte)).chain_err(|| { + format!("Error reading file {:?}", path) + })?; + + let mut bytes = vec![0; (end_byte - start_byte) as usize]; + file.read_exact(&mut bytes).chain_err(|| { + format!("Error reading file {:?}", path) + })?; + + Ok(bytes) + } + + fn write_file(&self, path: &Path, data: &[u8]) -> Result<()> { + debug!("Writing file: {}", path.to_string_lossy()); + let mut file = File::create(&path).chain_err(|| { + format!("unable to create file {:?}", path) + })?; + + file.write_all(data).chain_err(|| { + format!("unable to write content to {:?}", path) + }) } - fn abstracted_path(&self, path: &Path) -> Result { + fn get_local_file(&self, path: &Path) -> Result { Ok(PathBuf::from(path)) } diff --git a/util/src/distributed_filesystem/distributed_file_layer.rs b/util/src/distributed_filesystem/distributed_file_layer.rs new file mode 100644 index 00000000..f398b8e6 --- /dev/null +++ b/util/src/distributed_filesystem/distributed_file_layer.rs @@ -0,0 +1,211 @@ +use std::cmp::{max, min}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use rand::random; + +use data_layer::AbstractionLayer; +use distributed_filesystem::{LocalFileManager, FileSystemMasterInterface, + FileSystemWorkerInterface}; +use errors::*; + +pub struct DFSAbstractionLayer { + local_file_manager: Arc, + master_interface: Box, + worker_interface: FileSystemWorkerInterface, +} + +impl DFSAbstractionLayer { + pub fn new( + local_file_manager: Arc, + master_interface: Box, + ) -> Self { + DFSAbstractionLayer { + local_file_manager: local_file_manager, + master_interface: master_interface, + worker_interface: FileSystemWorkerInterface::new(), + } + } + + fn get_remote_data(&self, file_path: &str, start_byte: u64, end_byte: u64) -> Result> { + let mut data = Vec::new(); + + let file_location = self.master_interface + .get_file_location(file_path, start_byte, end_byte) + .chain_err(|| "Error getting remote data.")?; + + for chunk in file_location.chunks.iter() { + if chunk.worker_address.len() == 0 { + return Err( + "Error getting remote data: Incomplete file location information.".into(), + ); + } + + let random_val = random::() % chunk.worker_address.len(); + let worker_addr = &chunk.worker_address[random_val]; + + let remote_data = self.worker_interface + .read_file( + worker_addr, + file_path, + max(start_byte, chunk.start_byte), + min(end_byte, chunk.end_byte), + ) + .chain_err(|| "Error getting remote data")?; + data.extend(remote_data); + } + + Ok(data) + } +} + +impl AbstractionLayer for DFSAbstractionLayer { + fn get_file_length(&self, path: &Path) -> Result { + debug!("Getting file length: {:?}", path); + + let file_info = self.master_interface + .get_file_info(&path.to_string_lossy()) + .chain_err(|| "Error getting file length")?; + + if !file_info.exists || !file_info.is_file { + return Err("File does not exist".into()); + } + + Ok(file_info.length) + } + + fn read_file_location(&self, path: &Path, start_byte: u64, end_byte: u64) -> Result> { + debug!("Reading file: {:?}", path); + + let local_file_chunks = self.local_file_manager + .read_local_file(&path.to_string_lossy(), start_byte, end_byte) + .chain_err(|| "Error reading local file")?; + + let mut data = Vec::new(); + let mut on_local_chunk = 0; + let mut on_byte = start_byte; + while on_byte < end_byte { + while on_local_chunk < local_file_chunks.len() && + local_file_chunks[on_local_chunk].start_byte < on_byte + { + on_local_chunk += 1; + } + if on_local_chunk < local_file_chunks.len() { + if on_byte == local_file_chunks[on_local_chunk].start_byte { + data.extend(local_file_chunks[on_local_chunk].data.clone()); + on_byte = local_file_chunks[on_local_chunk].end_byte; + } else { + let new_on_byte = local_file_chunks[on_local_chunk].start_byte; + let remote_data = + self.get_remote_data(&path.to_string_lossy(), on_byte, new_on_byte) + .chain_err(|| "Error reading remote data.")?; + data.extend(remote_data); + on_byte = new_on_byte; + } + } else { + let remote_data = self.get_remote_data(&path.to_string_lossy(), on_byte, end_byte) + .chain_err(|| "Error reading remote data.")?; + data.extend(remote_data); + on_byte = end_byte; + } + } + + Ok(data) + } + + fn write_file(&self, path: &Path, data: &[u8]) -> Result<()> { + debug!("Writing file: {:?}", path); + + let mut data_vec = Vec::new(); + data_vec.extend_from_slice(data); + + self.master_interface + .upload_file_chunk(&path.to_string_lossy(), 0, data_vec) + .chain_err(|| "Error writing file") + } + + fn get_local_file(&self, path: &Path) -> Result { + debug!("Getting local file: {:?}", path); + + if let Some(local_file_path) = + self.local_file_manager.get_local_file( + &path.to_string_lossy(), + ) + { + return Ok(PathBuf::from(local_file_path)); + } + + // TODO(conor): Improve this function to work for files that can not fit in memory. + let file_length = self.get_file_length(path).chain_err( + || "Error getting file length", + )?; + let data = self.read_file_location(path, 0, file_length).chain_err( + || "Error getting local file", + )?; + + let local_file_path = self.local_file_manager + .write_local_file(&path.to_string_lossy(), &data) + .chain_err(|| "Error writing local file")?; + + Ok(PathBuf::from(local_file_path)) + } + + fn read_dir(&self, path: &Path) -> Result> { + debug!("Reading directory {:?}", path); + + let file_info = self.master_interface + .get_file_info(&path.to_string_lossy()) + .chain_err(|| "Error reading directory")?; + + if !file_info.exists || file_info.is_file { + return Err("Directory does not exist".into()); + } + + let mut dir_info = Vec::new(); + for file_path in file_info.children.iter() { + let mut path_buf = PathBuf::new(); + path_buf.push(file_path); + dir_info.push(path_buf); + } + + Ok(dir_info) + } + + fn is_file(&self, path: &Path) -> Result<(bool)> { + let file_info = self.master_interface + .get_file_info(&path.to_string_lossy()) + .chain_err(|| "Error checking is file")?; + + if !file_info.exists { + return Err("File does not exist".into()); + } + + Ok(file_info.is_file) + } + + fn is_dir(&self, path: &Path) -> Result<(bool)> { + let file_info = self.master_interface + .get_file_info(&path.to_string_lossy()) + .chain_err(|| "Error checking is directory")?; + + if !file_info.exists { + return Err("Directory does not exist".into()); + } + + Ok(!file_info.is_file) + } + + fn exists(&self, path: &Path) -> Result<(bool)> { + let file_info = self.master_interface + .get_file_info(&path.to_string_lossy()) + .chain_err(|| "Error checking file exists")?; + + Ok(file_info.exists) + } + + fn create_dir_all(&self, _path: &Path) -> Result<()> { + // Creating a directory is not needed for the distributed file system, can write to any + // path. + Ok(()) + } +} diff --git a/util/src/distributed_filesystem/filesystem_manager.rs b/util/src/distributed_filesystem/filesystem_manager.rs new file mode 100644 index 00000000..d8300bf2 --- /dev/null +++ b/util/src/distributed_filesystem/filesystem_manager.rs @@ -0,0 +1,247 @@ +use std::collections::HashMap; +use std::cmp::max; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::{Arc, RwLock}; + +use protobuf::repeated::RepeatedField; +use rand::random; + +use cerberus_proto::filesystem as pb; +use errors::*; +use distributed_filesystem::{FileSystemWorkerInterface, WorkerInfoProvider}; +use logging::output_error; + +const MIN_DISTRIBUTION_LEVEL: usize = 2; +const MAX_DISTRIBUTION_LEVEL: usize = 3; +const MAX_DISTRIBUTION_FAILURES: usize = 3; + +struct FileChunk { + start_byte: u64, + end_byte: u64, + + // The worker ids of the workers this chunk is stored on. + workers: Vec, +} + +struct FileInfo { + length: u64, + chunks: Vec, +} + +struct DirInfo { + // List of files in the directory + children: Vec, +} + +pub struct FileSystemManager { + file_info_map: RwLock>, + dir_info_map: RwLock>, + worker_interface: FileSystemWorkerInterface, + worker_info_provider: Arc, +} + +impl FileSystemManager { + pub fn new(worker_info_provider: Arc) -> Self { + FileSystemManager { + file_info_map: RwLock::new(HashMap::new()), + dir_info_map: RwLock::new(HashMap::new()), + worker_interface: FileSystemWorkerInterface::new(), + worker_info_provider: worker_info_provider, + } + } + + fn get_distribution_level(&self, worker_count: usize) -> usize { + let mut distribution_level = worker_count / 2; + + if distribution_level > MAX_DISTRIBUTION_LEVEL { + distribution_level = MAX_DISTRIBUTION_LEVEL; + } else if distribution_level < MIN_DISTRIBUTION_LEVEL { + distribution_level = MIN_DISTRIBUTION_LEVEL; + } + distribution_level + } + + fn get_random_worker(&self, workers: &HashMap) -> String { + let workers_vec: Vec<&String> = workers.keys().collect(); + + let random_val = random::() % workers_vec.len(); + + workers_vec[random_val].to_owned() + } + + fn update_file_info(&self, file_path: &str, file_chunk: FileChunk) { + let mut file_info_map = self.file_info_map.write().unwrap(); + + let file_info = file_info_map.entry(file_path.to_owned()).or_insert( + FileInfo { + length: 0, + chunks: Vec::new(), + }, + ); + + file_info.length = max(file_info.length, file_chunk.end_byte); + file_info.chunks.push(file_chunk); + } + + fn update_dir_info(&self, file_path: &str) { + let mut dir_path = PathBuf::from(file_path); + dir_path.pop(); + let dir_path_str = dir_path.to_string_lossy().to_string() + "/"; + + let mut dir_info_map = self.dir_info_map.write().unwrap(); + + let dir_info = dir_info_map.entry(dir_path_str).or_insert(DirInfo { + children: Vec::new(), + }); + + dir_info.children.push(file_path.to_owned()); + } + + pub fn upload_file_chunk(&self, file_path: &str, start_byte: u64, data: &[u8]) -> Result<()> { + let mut active_workers = self.worker_info_provider.get_workers(); + + if active_workers.len() < MIN_DISTRIBUTION_LEVEL { + return Err("Not enough workers in cluster".into()); + } + + let mut used_workers = Vec::new(); + + let distribution_level = self.get_distribution_level(active_workers.len()); + let mut distribution_count = 0; + let mut distribution_failures = 0; + while distribution_count < distribution_level && + distribution_failures < MAX_DISTRIBUTION_FAILURES + { + let worker = self.get_random_worker(&active_workers); + let worker_addr = active_workers + .get(&worker) + .chain_err(|| "Error getting worker address")? + .to_owned(); + + let mut data_vec = Vec::new(); + data_vec.extend_from_slice(data); + let result = self.worker_interface.store_file( + &format!("{}", worker_addr), + file_path, + start_byte, + data_vec, + ); + + match result { + Ok(_) => { + distribution_count += 1; + active_workers.remove(&worker); + used_workers.push(worker); + } + Err(err) => { + output_error(&err.chain_err( + || format!("Error storing file on worker {}", worker), + )); + distribution_failures += 1; + if distribution_count + active_workers.len() > distribution_level { + active_workers.remove(&worker); + } + } + } + } + + let file_chunk = FileChunk { + start_byte: start_byte, + end_byte: start_byte + (data.len() as u64), + workers: used_workers, + }; + + self.update_file_info(file_path, file_chunk); + self.update_dir_info(file_path); + + Ok(()) + } + + fn convert_file_chunk( + &self, + chunk: &FileChunk, + active_workers: &HashMap, + ) -> pb::FileChunk { + let mut pb_chunk = pb::FileChunk::new(); + pb_chunk.start_byte = chunk.start_byte; + pb_chunk.end_byte = chunk.end_byte; + + for worker in &chunk.workers { + if let Some(worker_addr) = active_workers.get(worker) { + pb_chunk.worker_address.push(format!("{}", worker_addr)); + } + } + + pb_chunk + } + + // Gets the locations for the entire file if end_byte is 0 + pub fn get_file_location( + &self, + file_path: &str, + start_byte: u64, + end_byte: u64, + ) -> Result { + let file_info_map = self.file_info_map.read().unwrap(); + match file_info_map.get(file_path) { + Some(file_info) => { + let active_workers = self.worker_info_provider.get_workers(); + + let mut response = pb::FileLocationResponse::new(); + if end_byte == 0 { + for chunk in &file_info.chunks { + let pb_chunk = self.convert_file_chunk(chunk, &active_workers); + response.chunks.push(pb_chunk); + } + } else { + for chunk in &file_info.chunks { + // Check if chunk is in the range. + if (start_byte >= chunk.start_byte && start_byte < chunk.end_byte) || + (chunk.start_byte >= start_byte && chunk.start_byte < end_byte) + { + let pb_chunk = self.convert_file_chunk(chunk, &active_workers); + response.chunks.push(pb_chunk); + } + } + } + + Ok(response) + } + None => Err(format!("No file info found for {}", file_path).into()), + } + } + + pub fn get_file_info(&self, file_path: &str) -> pb::FileInfoResponse { + { + let file_info_map = self.file_info_map.read().unwrap(); + if let Some(file_info) = file_info_map.get(file_path) { + // Is file + let mut response = pb::FileInfoResponse::new(); + response.set_exists(true); + response.set_is_file(true); + response.set_length(file_info.length); + return response; + } + } + + { + let dir_info_map = self.dir_info_map.read().unwrap(); + if let Some(dir_info) = dir_info_map.get(file_path) { + // Is Dir + // Is file + let mut response = pb::FileInfoResponse::new(); + response.set_exists(true); + response.set_is_file(false); + response.set_length(dir_info.children.len() as u64); + response.set_children(RepeatedField::from_vec(dir_info.children.clone())); + return response; + } + + } + + let mut response = pb::FileInfoResponse::new(); + response.set_exists(false); + response + } +} diff --git a/util/src/distributed_filesystem/filesystem_master_interface.rs b/util/src/distributed_filesystem/filesystem_master_interface.rs new file mode 100644 index 00000000..519b69da --- /dev/null +++ b/util/src/distributed_filesystem/filesystem_master_interface.rs @@ -0,0 +1,130 @@ +use std::net::SocketAddr; +use std::sync::Arc; + +use errors::*; +use grpc::RequestOptions; + +use cerberus_proto::filesystem as pb; +use cerberus_proto::filesystem_grpc as grpc_pb; +use cerberus_proto::filesystem_grpc::FileSystemMasterService; // Importing methods, don't use directly +use distributed_filesystem::FileSystemManager; + +pub trait FileSystemMasterInterface { + fn upload_file_chunk(&self, file_path: &str, start_byte: u64, data: Vec) -> Result<()>; + + fn get_file_location( + &self, + file_path: &str, + start_byte: u64, + end_byte: u64, + ) -> Result; + + fn get_file_info(&self, file_path: &str) -> Result; +} + +/// `NetworkFileSystemMasterInterface` is used to perform file system operations on the master and get +/// information about the distributed filesystem from the master. +pub struct NetworkFileSystemMasterInterface { + client: grpc_pb::FileSystemMasterServiceClient, +} + +impl NetworkFileSystemMasterInterface { + pub fn new(master_addr: SocketAddr) -> Result { + let client = grpc_pb::FileSystemMasterServiceClient::new_plain( + &master_addr.ip().to_string(), + master_addr.port(), + Default::default(), + ).chain_err(|| "Error building FileSystemMasterService client.")?; + + Ok(NetworkFileSystemMasterInterface { client: client }) + } +} + +impl FileSystemMasterInterface for NetworkFileSystemMasterInterface { + fn upload_file_chunk(&self, file_path: &str, start_byte: u64, data: Vec) -> Result<()> { + let mut upload_file_req = pb::UploadFileRequest::new(); + upload_file_req.file_path = file_path.to_owned(); + upload_file_req.start_byte = start_byte; + upload_file_req.data = data; + + self.client + .upload_file(RequestOptions::new(), upload_file_req) + .wait() + .chain_err(|| "Failed to upload file chunk")?; + + Ok(()) + } + + // Gets the locations for the entire file if end_byte is 0 + fn get_file_location( + &self, + file_path: &str, + start_byte: u64, + end_byte: u64, + ) -> Result { + let mut file_location_req = pb::FileLocationRequest::new(); + file_location_req.file_path = file_path.to_owned(); + file_location_req.start_byte = start_byte; + file_location_req.end_byte = end_byte; + + let response = self.client + .get_file_location(RequestOptions::new(), file_location_req) + .wait() + .chain_err(|| "Failed to get file location")? + .1; + + Ok(response) + } + + fn get_file_info(&self, file_path: &str) -> Result { + let mut file_info_req = pb::FileInfoRequest::new(); + file_info_req.file_path = file_path.to_owned(); + + let response = self.client + .get_file_info(RequestOptions::new(), file_info_req) + .wait() + .chain_err(|| "Failed to get file info")? + .1; + + Ok(response) + } +} + +/// `LocalFileSystemMasterInterface` is used to perform file system operations on the master. +pub struct LocalFileSystemMasterInterface { + filesystem_manager: Arc, +} + +impl LocalFileSystemMasterInterface { + pub fn new(filesystem_manager: Arc) -> Self { + LocalFileSystemMasterInterface { filesystem_manager: filesystem_manager } + } +} + +impl FileSystemMasterInterface for LocalFileSystemMasterInterface { + fn upload_file_chunk(&self, file_path: &str, start_byte: u64, data: Vec) -> Result<()> { + self.filesystem_manager.upload_file_chunk( + file_path, + start_byte, + &data, + ) + } + + // Gets the locations for the entire file if end_byte is 0 + fn get_file_location( + &self, + file_path: &str, + start_byte: u64, + end_byte: u64, + ) -> Result { + self.filesystem_manager.get_file_location( + file_path, + start_byte, + end_byte, + ) + } + + fn get_file_info(&self, file_path: &str) -> Result { + Ok(self.filesystem_manager.get_file_info(file_path)) + } +} diff --git a/util/src/distributed_filesystem/filesystem_worker_interface.rs b/util/src/distributed_filesystem/filesystem_worker_interface.rs new file mode 100644 index 00000000..5c97e7c6 --- /dev/null +++ b/util/src/distributed_filesystem/filesystem_worker_interface.rs @@ -0,0 +1,112 @@ + +use std::collections::HashMap; +use std::net::SocketAddr; +use std::str::FromStr; +use std::sync::RwLock; + +use grpc::RequestOptions; + +use cerberus_proto::filesystem as pb; +use cerberus_proto::filesystem_grpc as grpc_pb; +use cerberus_proto::filesystem_grpc::FileSystemWorkerService; // Importing methods, don't use directly +use errors::*; + +const NO_CLIENT_FOUND_ERR: &str = "No client found for this worker"; + +#[derive(Default)] +pub struct FileSystemWorkerInterface { + clients: RwLock>, +} + +/// `FileSystemWorkerInterface` is used to perform file system operations on workers. +impl FileSystemWorkerInterface { + pub fn new() -> Self { + Default::default() + } + + // Creates a client for a given worker address if one does not already exist. + fn create_client_if_required(&self, worker_addr: &str) -> Result<()> { + { + let clients = self.clients.read().unwrap(); + if clients.contains_key(worker_addr) { + return Ok(()); + } + } + + let worker_socket_addr = SocketAddr::from_str(worker_addr).chain_err( + || "Invalid worker address", + )?; + + let client = grpc_pb::FileSystemWorkerServiceClient::new_plain( + &worker_socket_addr.ip().to_string(), + worker_socket_addr.port(), + Default::default(), + ).chain_err(|| "Error building client for worker")?; + + let mut clients = self.clients.write().unwrap(); + + clients.insert(worker_addr.to_owned(), client); + Ok(()) + } + + pub fn store_file( + &self, + worker_addr: &str, + file_path: &str, + start_byte: u64, + data: Vec, + ) -> Result<()> { + self.create_client_if_required(worker_addr).chain_err( + || "Error creating client", + )?; + + let mut request = pb::StoreFileRequest::new(); + request.set_file_path(file_path.to_owned()); + request.set_start_byte(start_byte); + request.set_data(data); + + let clients = self.clients.read().unwrap(); + + if let Some(client) = clients.get(worker_addr) { + client + .store_file(RequestOptions::new(), request) + .wait() + .chain_err(|| "Failed to store file")?; + return Ok(()); + } else { + // This should never happen. + return Err(NO_CLIENT_FOUND_ERR.into()); + } + } + + pub fn read_file( + &self, + worker_addr: &str, + file_path: &str, + start_byte: u64, + end_byte: u64, + ) -> Result> { + self.create_client_if_required(worker_addr).chain_err( + || "Error creating client", + )?; + + let mut request = pb::ReadFileRequest::new(); + request.set_file_path(file_path.to_owned()); + request.set_start_byte(start_byte); + request.set_end_byte(end_byte); + + let clients = self.clients.read().unwrap(); + + if let Some(client) = clients.get(worker_addr) { + let response = client + .read_file(RequestOptions::new(), request) + .wait() + .chain_err(|| "Failed to read file")? + .1; + return Ok(response.data); + } else { + // This should never happen. + return Err(NO_CLIENT_FOUND_ERR.into()); + } + } +} diff --git a/util/src/distributed_filesystem/local_file_manager.rs b/util/src/distributed_filesystem/local_file_manager.rs new file mode 100644 index 00000000..b87271b1 --- /dev/null +++ b/util/src/distributed_filesystem/local_file_manager.rs @@ -0,0 +1,239 @@ +use std::collections::HashMap; +use std::cmp::{max, min}; +use std::fs::{File, OpenOptions}; +use std::fs; +use std::io::{Read, Write, Seek, SeekFrom}; +use std::sync::Mutex; +use std::path::PathBuf; +use std::os::unix::fs::OpenOptionsExt; + +use uuid::Uuid; + +use errors::*; + +const COMPLETE_SUB_DIR: &str = "complete"; + +struct FileChunk { + local_file_path: PathBuf, + start_byte: u64, + end_byte: u64, +} + +pub struct ReadFileChunk { + pub start_byte: u64, + pub end_byte: u64, + pub data: Vec, +} + +pub struct LocalFileManager { + local_file_map: Mutex>>, + complete_file_map: Mutex>, + storage_directory: PathBuf, +} + +impl LocalFileManager { + pub fn new(storage_directory: PathBuf) -> Self { + LocalFileManager { + local_file_map: Mutex::new(HashMap::new()), + complete_file_map: Mutex::new(HashMap::new()), + storage_directory: storage_directory, + } + } + + pub fn store_file_chunk(&self, file_path: &str, start_byte: u64, data: &[u8]) -> Result<()> { + let mut storage_path = PathBuf::new(); + storage_path.push(self.storage_directory.clone()); + + fs::create_dir_all(&storage_path).chain_err( + || "Failed to create storage directory", + )?; + + let file_name = Uuid::new_v4().to_string(); + storage_path.push(file_name); + + info!( + "Storing file chunk {} ({} -> {}) to {}", + file_path, + start_byte, + start_byte + data.len() as u64, + storage_path.to_string_lossy() + ); + + let mut file = File::create(storage_path.clone()).chain_err( + || "Unable to create file", + )?; + file.write_all(data).chain_err(|| "Unable to write data")?; + + let mut local_file_map = self.local_file_map.lock().unwrap(); + let chunks = local_file_map.entry(file_path.to_owned()).or_insert_with( + Vec::new, + ); + + let file_chunk = FileChunk { + local_file_path: storage_path, + start_byte: start_byte, + end_byte: start_byte + (data.len() as u64), + }; + chunks.push(file_chunk); + + Ok(()) + } + + pub fn get_local_file(&self, file_path: &str) -> Option { + let complete_file_map = self.complete_file_map.lock().unwrap(); + complete_file_map.get(file_path).map(|s| s.to_owned()) + } + + pub fn write_local_file(&self, file_path: &str, data: &[u8]) -> Result { + let mut storage_path = PathBuf::new(); + storage_path.push(self.storage_directory.clone()); + storage_path.push(COMPLETE_SUB_DIR); + + fs::create_dir_all(&storage_path).chain_err( + || "Failed to create storage directory", + )?; + + let file_name = Uuid::new_v4().to_string(); + storage_path.push(file_name); + + let mut options = OpenOptions::new(); + options.read(true); + options.write(true); + options.truncate(true); + options.create(true); + options.mode(0o777); + + let mut file = options.open(storage_path.clone()).chain_err( + || "Unable to create file", + )?; + file.write_all(data).chain_err(|| "Unable to write data")?; + + let mut complete_file_map = self.complete_file_map.lock().unwrap(); + complete_file_map.insert( + file_path.to_owned(), + storage_path.to_string_lossy().to_string(), + ); + + Ok(storage_path.to_string_lossy().to_string()) + } + + /// `read_file_chunk` reads a single file chunk known to exist requested by another worker. + pub fn read_file_chunk( + &self, + file_path: &str, + start_byte: u64, + end_byte: u64, + ) -> Result> { + let local_file_map = self.local_file_map.lock().unwrap(); + let stored_chunks = match local_file_map.get(file_path) { + Some(stored_chunks) => stored_chunks, + None => { + return Err( + format!("No stored file chunks found for {}", file_path).into(), + ) + } + }; + + let mut file_chunk = None; + for chunk in stored_chunks { + if chunk.start_byte <= start_byte && chunk.end_byte >= end_byte { + file_chunk = Some(chunk); + break; + } + } + + if let Some(chunk) = file_chunk { + let bytes_to_read = end_byte - start_byte; + let mut bytes = vec![0u8; bytes_to_read as usize]; + + let mut file = File::open(chunk.local_file_path.clone()).chain_err(|| { + format!("Error opening file chunk {:?}", chunk.local_file_path) + })?; + + file.seek(SeekFrom::Start(start_byte - chunk.start_byte)) + .chain_err(|| { + format!("Error reading file chunk {:?}", chunk.local_file_path) + })?; + + file.read_exact(&mut bytes).chain_err(|| { + format!("Error reading file chunk {:?}", chunk.local_file_path) + })?; + + return Ok(bytes); + } + Err( + format!("Stored file chunk not found for {}", file_path).into(), + ) + } + + /// `read_local_file` reads the portion of the requested file that is stored localy and + /// returns this. Returns a Vector for the unlikely case where the stored file is split across + /// multiple chunks. + pub fn read_local_file( + &self, + file_path: &str, + mut start_byte: u64, + end_byte: u64, + ) -> Result> { + let mut file_chunks = Vec::new(); + + let local_file_map = self.local_file_map.lock().unwrap(); + + if let Some(stored_chunks) = local_file_map.get(file_path) { + while start_byte < end_byte { + let mut best: u64 = end_byte + 1; + let mut best_chunk = 0; + for (i, chunk) in stored_chunks.iter().enumerate() { + if chunk.end_byte > start_byte && chunk.start_byte < end_byte { + // Stored chunk starts at or after the requested section. + if chunk.start_byte >= start_byte { + let distance = chunk.start_byte - start_byte; + if distance < best { + best = distance; + best_chunk = i; + } + } else { + // Stored chunk starts before the requested section and continues into + // the requested section. + best = 0; + best_chunk = i; + } + } + } + + if best >= end_byte { + break; + } else { + let chunk = &stored_chunks[best_chunk]; + + start_byte = max(start_byte, chunk.start_byte); + let bytes_to_read = min(end_byte - start_byte, chunk.end_byte - start_byte); + let mut bytes = vec![0u8; bytes_to_read as usize]; + + let mut file = File::open(chunk.local_file_path.clone()).chain_err(|| { + format!("Error opening file chunk {:?}", chunk.local_file_path) + })?; + file.seek(SeekFrom::Start(start_byte - chunk.start_byte)) + .chain_err(|| { + format!("Error reading file chunk {:?}", chunk.local_file_path) + })?; + + file.read_exact(&mut bytes).chain_err(|| { + format!("Error reading file chunk {:?}", chunk.local_file_path) + })?; + + let file_chunk = ReadFileChunk { + start_byte: start_byte, + end_byte: start_byte + bytes_to_read, + data: bytes, + }; + file_chunks.push(file_chunk); + + start_byte += bytes_to_read; + } + } + } + + Ok(file_chunks) + } +} diff --git a/util/src/distributed_filesystem/mod.rs b/util/src/distributed_filesystem/mod.rs new file mode 100644 index 00000000..e4b97605 --- /dev/null +++ b/util/src/distributed_filesystem/mod.rs @@ -0,0 +1,15 @@ +mod distributed_file_layer; +mod filesystem_manager; +mod filesystem_master_interface; +mod filesystem_worker_interface; +mod local_file_manager; +mod worker_info_provider; + +pub use self::distributed_file_layer::DFSAbstractionLayer; +pub use self::filesystem_manager::FileSystemManager; +pub use self::filesystem_master_interface::FileSystemMasterInterface; +pub use self::filesystem_master_interface::NetworkFileSystemMasterInterface; +pub use self::filesystem_master_interface::LocalFileSystemMasterInterface; +pub use self::filesystem_worker_interface::FileSystemWorkerInterface; +pub use self::local_file_manager::LocalFileManager; +pub use self::worker_info_provider::WorkerInfoProvider; diff --git a/util/src/distributed_filesystem/worker_info_provider.rs b/util/src/distributed_filesystem/worker_info_provider.rs new file mode 100644 index 00000000..17d6c279 --- /dev/null +++ b/util/src/distributed_filesystem/worker_info_provider.rs @@ -0,0 +1,8 @@ + +use std::collections::HashMap; +use std::net::SocketAddr; + +pub trait WorkerInfoProvider { + // Returns a map of Worker Id -> Worker Address + fn get_workers(&self) -> HashMap; +} diff --git a/util/src/lib.rs b/util/src/lib.rs index 96ee41e9..26817a05 100644 --- a/util/src/lib.rs +++ b/util/src/lib.rs @@ -1,9 +1,14 @@ +extern crate cerberus_proto; extern crate chrono; extern crate env_logger; #[macro_use] extern crate error_chain; +extern crate grpc; #[macro_use] extern crate log; +extern crate protobuf; +extern crate rand; +extern crate uuid; pub mod errors { error_chain!{} @@ -11,6 +16,7 @@ pub mod errors { pub mod logging; pub mod data_layer; +pub mod distributed_filesystem; pub use logging::init_logger; pub use logging::output_error; diff --git a/worker/src/main.rs b/worker/src/main.rs index 9f32f698..4dccf196 100644 --- a/worker/src/main.rs +++ b/worker/src/main.rs @@ -49,14 +49,18 @@ use std::{thread, time}; use std::net::SocketAddr; use std::str::FromStr; use std::sync::Arc; -use std::path::Path; +use std::path::{Path, PathBuf}; + +use clap::ArgMatches; use errors::*; use master_interface::MasterInterface; use operations::OperationHandler; -use server::{Server, ScheduleOperationService, IntermediateDataService}; +use server::{Server, ScheduleOperationService, IntermediateDataService, FileSystemService}; use util::init_logger; use util::data_layer::{AbstractionLayer, NullAbstractionLayer, NFSAbstractionLayer}; +use util::distributed_filesystem::{LocalFileManager, DFSAbstractionLayer, + NetworkFileSystemMasterInterface}; const WORKER_REGISTRATION_RETRIES: u16 = 5; const MAX_HEALTH_CHECK_FAILURES: u16 = 10; @@ -67,6 +71,7 @@ const WORKER_REGISTRATION_RETRY_WAIT_DURATION_MS: u64 = 1000; const DEFAULT_PORT: &str = "0"; const DEFAULT_MASTER_ADDR: &str = "[::]:8081"; const DEFAULT_WORKER_IP: &str = "[::]"; +const DFS_FILE_DIRECTORY: &str = "/tmp/cerberus/dfs/"; fn register_worker(master_interface: &MasterInterface, address: &SocketAddr) -> Result<()> { let mut retries = WORKER_REGISTRATION_RETRIES; @@ -92,6 +97,44 @@ fn register_worker(master_interface: &MasterInterface, address: &SocketAddr) -> Ok(()) } +type AbstractionLayerArc = Arc; + +fn get_data_abstraction_layer( + master_addr: SocketAddr, + matches: &ArgMatches, +) -> Result<(AbstractionLayerArc, Option>)> { + let data_abstraction_layer: Arc; + let local_file_manager: Option>; + + let nfs_path = matches.value_of("nfs"); + let dfs = matches.is_present("dfs"); + if let Some(path) = nfs_path { + data_abstraction_layer = Arc::new(NFSAbstractionLayer::new(Path::new(path))); + local_file_manager = None; + } else if dfs { + let mut storage_dir = PathBuf::new(); + storage_dir.push(DFS_FILE_DIRECTORY); + let local_file_manager_arc = Arc::new(LocalFileManager::new(storage_dir)); + + let master_interface = Box::new( + NetworkFileSystemMasterInterface::new(master_addr) + .chain_err(|| "Error creating filesystem master interface.")?, + ); + + data_abstraction_layer = Arc::new(DFSAbstractionLayer::new( + Arc::clone(&local_file_manager_arc), + master_interface, + )); + + local_file_manager = Some(local_file_manager_arc); + } else { + data_abstraction_layer = Arc::new(NullAbstractionLayer::new()); + local_file_manager = None; + } + + Ok((data_abstraction_layer, local_file_manager)) +} + fn run() -> Result<()> { println!("Cerberus Worker!"); init_logger().chain_err(|| "Failed to initialise logging.")?; @@ -107,11 +150,9 @@ fn run() -> Result<()> { || "Error creating master interface.", )?); - let nfs_path = matches.value_of("nfs"); - let data_abstraction_layer: Arc = match nfs_path { - Some(path) => Arc::new(NFSAbstractionLayer::new(Path::new(path))), - None => Arc::new(NullAbstractionLayer::new()), - }; + let (data_abstraction_layer, local_file_manager) = + get_data_abstraction_layer(master_addr, &matches) + .chain_err(|| "Error creating data abstraction layer.")?; let operation_handler = Arc::new(OperationHandler::new( Arc::clone(&master_interface), @@ -120,8 +161,15 @@ fn run() -> Result<()> { let scheduler_service = ScheduleOperationService::new(Arc::clone(&operation_handler)); let interm_data_service = IntermediateDataService; - let srv = Server::new(port, scheduler_service, interm_data_service) - .chain_err(|| "Can't create server")?; + let filesystem_service = FileSystemService::new(local_file_manager); + + let srv = Server::new( + port, + scheduler_service, + interm_data_service, + filesystem_service, + ).chain_err(|| "Can't create server")?; + let local_ip_addr = matches.value_of("ip").unwrap_or(DEFAULT_WORKER_IP); let local_addr = SocketAddr::from_str(&format!( @@ -152,13 +200,11 @@ fn run() -> Result<()> { info!("Successfully re-registered with master after being disconnected."); current_health_check_failures = 0; } + } else if let Err(err) = operation_handler.update_worker_status() { + error!("Could not send updated worker status to master: {}", err); + current_health_check_failures += 1; } else { - if let Err(err) = operation_handler.update_worker_status() { - error!("Could not send updated worker status to master: {}", err); - current_health_check_failures += 1; - } else { - current_health_check_failures = 0; - } + current_health_check_failures = 0; } if !srv.is_alive() { diff --git a/worker/src/operations/combine.rs b/worker/src/operations/combine.rs index 349bf465..a0437199 100644 --- a/worker/src/operations/combine.rs +++ b/worker/src/operations/combine.rs @@ -11,7 +11,7 @@ use super::operation_handler::OperationResources; fn check_has_combine(resources: &OperationResources) -> Result { let absolute_path = resources .data_abstraction_layer - .absolute_path(Path::new(&resources.binary_path)) + .get_local_file(Path::new(&resources.binary_path)) .chain_err(|| "Unable to get absolute path")?; let output = Command::new(absolute_path) .arg("has-combine") @@ -27,10 +27,10 @@ fn check_has_combine(resources: &OperationResources) -> Result { fn do_combine_operation( resources: &OperationResources, key: &str, - values: Vec, + values: &[serde_json::Value], ) -> Result { let combine_input = json!({ - "key": key.clone().to_owned(), + "key": key.to_owned(), "values": values, }); let combine_input_str = serde_json::to_string(&combine_input).chain_err( @@ -39,7 +39,7 @@ fn do_combine_operation( let absolute_binary_path = resources .data_abstraction_layer - .absolute_path(Path::new(&resources.binary_path)) + .get_local_file(Path::new(&resources.binary_path)) .chain_err(|| "Unable to get absolute path")?; let mut child = Command::new(absolute_binary_path) .arg("combine") @@ -111,8 +111,9 @@ fn run_combine(resources: &OperationResources) -> Result<()> { for (key, values) in (&kv_map).iter() { if values.len() > 1 { - let results = do_combine_operation(resources, key, values.clone()) - .chain_err(|| "Failed to run combine operation.")?; + let results = do_combine_operation(resources, key, values).chain_err( + || "Failed to run combine operation.", + )?; if let serde_json::Value::Array(ref values) = results { for value in values { @@ -144,7 +145,7 @@ fn run_combine(resources: &OperationResources) -> Result<()> { Ok(()) } -/// Optionally run a combine operation if it's implemented by the MapReduce binary. +/// Optionally run a combine operation if it's implemented by the `MapReduce` binary. pub fn optional_run_combine(resources: &OperationResources) -> Result<()> { let has_combine = check_has_combine(resources).chain_err( || "Error running has-combine command.", diff --git a/worker/src/operations/io.rs b/worker/src/operations/io.rs index 51f7bcb2..2a4946cf 100644 --- a/worker/src/operations/io.rs +++ b/worker/src/operations/io.rs @@ -1,6 +1,6 @@ use std::fs::File; -use std::io::{BufReader, Read, Seek, SeekFrom, Write}; -use std::path::{Path, PathBuf}; +use std::io::{BufReader, Read, Write}; +use std::path::Path; use std::str; use std::sync::Arc; @@ -18,31 +18,11 @@ pub fn read_location( data_abstraction_layer: &Arc, input_location: &pb::InputLocation, ) -> Result { - let mut path = PathBuf::new(); - path.push(input_location.get_input_path()); + let path = Path::new(input_location.get_input_path()); - let file = data_abstraction_layer.open_file(&path).chain_err(|| { - format!("unable to open file {}", input_location.get_input_path()) - })?; - - let mut buf_reader = BufReader::new(file); - buf_reader - .seek(SeekFrom::Start(input_location.start_byte as u64)) - .chain_err(|| { - format!( - "could not seek file {} to position {}", - input_location.get_input_path(), - input_location.start_byte - ) - })?; - - let mut buffer = vec![0; (input_location.end_byte - input_location.start_byte) as usize]; - buf_reader.read(&mut buffer[..]).chain_err(|| { - format!( - "unable to read content of {}", - input_location.get_input_path() - ) - })?; + let buffer: Vec = data_abstraction_layer + .read_file_location(path, input_location.start_byte, input_location.end_byte) + .chain_err(|| "Error reading file location.")?; let value = str::from_utf8(&buffer).chain_err(|| { format!("Invalid string in file {}", input_location.get_input_path()) @@ -75,17 +55,11 @@ pub fn write>( path: P, data: &[u8], ) -> Result<()> { - let mut file = data_abstraction_layer_arc - .create_file(path.as_ref()) + data_abstraction_layer_arc + .write_file(path.as_ref(), data) .chain_err(|| { - format!("unable to create file {}", path.as_ref().to_string_lossy()) + format!("Unable to write file {}", path.as_ref().to_string_lossy()) })?; - file.write_all(data).chain_err(|| { - format!( - "unable to write content to {}", - path.as_ref().to_string_lossy() - ) - })?; Ok(()) } diff --git a/worker/src/operations/map.rs b/worker/src/operations/map.rs index 3a641c3b..358c7923 100644 --- a/worker/src/operations/map.rs +++ b/worker/src/operations/map.rs @@ -338,12 +338,19 @@ fn internal_perform_map( } } + info!( + "Running map task for {} ({} - > {})", + input_location.input_path, + input_location.start_byte, + input_location.end_byte + ); + let map_input_value = io::read_location(&resources.data_abstraction_layer, input_location) .chain_err(|| "unable to open input file")?; let absolute_path = resources .data_abstraction_layer - .absolute_path(Path::new(map_options.get_mapper_file_path())) + .get_local_file(Path::new(map_options.get_mapper_file_path())) .chain_err(|| "Unable to get absolute path")?; let child = Command::new(absolute_path) .arg("map") diff --git a/worker/src/operations/operation_handler.rs b/worker/src/operations/operation_handler.rs index 77902082..73fd7cb6 100644 --- a/worker/src/operations/operation_handler.rs +++ b/worker/src/operations/operation_handler.rs @@ -164,7 +164,7 @@ impl OperationHandler { }) } - pub fn cancel_task(&self, request: pb::CancelTaskRequest) -> Result<()> { + pub fn cancel_task(&self, request: &pb::CancelTaskRequest) -> Result<()> { let mut operation_state = self.operation_state.lock().unwrap(); operation_state.last_cancelled_task_id = Some(request.task_id.clone()); diff --git a/worker/src/operations/reduce.rs b/worker/src/operations/reduce.rs index 86747767..852da0c0 100644 --- a/worker/src/operations/reduce.rs +++ b/worker/src/operations/reduce.rs @@ -114,7 +114,7 @@ impl ReduceOperationQueue { data_abstraction_layer_arc: &Arc, ) -> Result<()> { let absolute_path = data_abstraction_layer_arc - .absolute_path(Path::new(&reduce_options.reducer_file_path)) + .get_local_file(Path::new(&reduce_options.reducer_file_path)) .chain_err(|| "Unable to get absolute path")?; let child = Command::new(absolute_path) .arg("reduce") diff --git a/worker/src/parser.rs b/worker/src/parser.rs index 4983d5b6..a92ebda0 100644 --- a/worker/src/parser.rs +++ b/worker/src/parser.rs @@ -39,5 +39,14 @@ pub fn parse_command_line<'a>() -> ArgMatches<'a> { .takes_value(true) .required(false), ) + .arg( + Arg::with_name("dfs") + .long("dfs") + .help( + "Makes the worker run using the distributed file system for data access.", + ) + .takes_value(false) + .required(false), + ) .get_matches() } diff --git a/worker/src/server/filesystem_service.rs b/worker/src/server/filesystem_service.rs new file mode 100644 index 00000000..848870af --- /dev/null +++ b/worker/src/server/filesystem_service.rs @@ -0,0 +1,82 @@ +use std::sync::Arc; + +use grpc::{SingleResponse, Error, RequestOptions}; + +use cerberus_proto::filesystem as pb; +use cerberus_proto::filesystem_grpc as grpc_pb; +use util::distributed_filesystem::LocalFileManager; +use util::output_error; + +const NOT_DISTRIBUTED_FILESYSTEM: &str = "Worker is not running in distributed filesytem configuration"; +const STORE_FILE_ERROR: &str = "Error processing store file request"; +const READ_FILE_ERROR: &str = "Error processing read file request"; + +/// `FileSystemService` recieves communication from the master and other workers in relation to the +/// distributed file system. +pub struct FileSystemService { + local_file_manager: Option>, +} + +impl FileSystemService { + pub fn new(local_file_manager: Option>) -> Self { + FileSystemService { local_file_manager: local_file_manager } + } +} + +impl grpc_pb::FileSystemWorkerService for FileSystemService { + fn store_file( + &self, + _: RequestOptions, + req: pb::StoreFileRequest, + ) -> SingleResponse { + info!("Processing store file request for {}", req.file_path); + + let local_file_manager = match self.local_file_manager { + Some(ref local_file_manager) => local_file_manager, + None => { + error!("Worker is not running in distributed filesystem mode."); + return SingleResponse::err(Error::Other(NOT_DISTRIBUTED_FILESYSTEM)); + } + }; + + if let Err(err) = local_file_manager.store_file_chunk( + &req.file_path, + req.start_byte, + &req.data, + ) + { + output_error(&err.chain_err(|| "Error processing store file request.")); + return SingleResponse::err(Error::Other(STORE_FILE_ERROR)); + } + + SingleResponse::completed(pb::EmptyMessage::new()) + } + + fn read_file( + &self, + _: RequestOptions, + req: pb::ReadFileRequest, + ) -> SingleResponse { + let local_file_manager = match self.local_file_manager { + Some(ref local_file_manager) => local_file_manager, + None => { + error!("Worker is not running in distributed filesystem mode."); + return SingleResponse::err(Error::Other(NOT_DISTRIBUTED_FILESYSTEM)); + } + }; + + let read_file_result = + local_file_manager.read_file_chunk(&req.file_path, req.start_byte, req.end_byte); + match read_file_result { + Ok(bytes) => { + let mut response = pb::ReadFileResponse::new(); + response.data = bytes; + SingleResponse::completed(response) + } + Err(err) => { + output_error(&err.chain_err(|| "Error processing read file request.")); + SingleResponse::err(Error::Other(READ_FILE_ERROR)) + } + } + } +} diff --git a/worker/src/server/master_service.rs b/worker/src/server/master_service.rs index 2d22f113..2bf8b5fb 100644 --- a/worker/src/server/master_service.rs +++ b/worker/src/server/master_service.rs @@ -47,7 +47,7 @@ impl grpc_pb::ScheduleOperationService for ScheduleOperationService { _o: RequestOptions, cancel_request: pb::CancelTaskRequest, ) -> SingleResponse { - let cancel_task_result = self.operation_handler.cancel_task(cancel_request); + let cancel_task_result = self.operation_handler.cancel_task(&cancel_request); match cancel_task_result { Ok(_) => SingleResponse::completed(pb::EmptyMessage::new()), Err(err) => SingleResponse::err(Error::Panic(err.to_string())), diff --git a/worker/src/server/mod.rs b/worker/src/server/mod.rs index dd0627d3..1a4c0005 100644 --- a/worker/src/server/mod.rs +++ b/worker/src/server/mod.rs @@ -3,13 +3,18 @@ mod intermediate_data_service; /// `master_service` is responsible for handing data incoming from the master. mod master_service; +/// `filesystem_service` is responsible for handling requests related to the distributed file +/// system. +mod filesystem_service; pub use self::intermediate_data_service::IntermediateDataService; pub use self::master_service::ScheduleOperationService; +pub use self::filesystem_service::FileSystemService; use std::net::SocketAddr; use grpc; use cerberus_proto::worker_grpc; +use cerberus_proto::filesystem_grpc; use errors::*; const GRPC_THREAD_POOL_SIZE: usize = 10; @@ -23,6 +28,7 @@ impl Server { port: u16, scheduler_service: ScheduleOperationService, interm_data_service: IntermediateDataService, + filesystem_service: FileSystemService, ) -> Result { let mut server_builder = grpc::ServerBuilder::new_plain(); server_builder.http.set_port(port); @@ -43,6 +49,13 @@ impl Server { ), ); + // Register FileSystemService + server_builder.add_service( + filesystem_grpc::FileSystemWorkerServiceServer::new_service_def( + filesystem_service, + ), + ); + Ok(Server { server: server_builder.build().chain_err( || "Error building gRPC server", From c1b33a08df23325ece65b8dc2902ede5928b134e Mon Sep 17 00:00:00 2001 From: Conor Griffin Date: Thu, 15 Mar 2018 10:41:25 +0000 Subject: [PATCH 19/19] Release 0.5.0 --- Cargo.lock | 436 +++++++++++++++++++++++++++-------------- cli/Cargo.toml | 2 +- libcerberus/Cargo.toml | 2 +- master/Cargo.toml | 2 +- proto/Cargo.toml | 2 +- util/Cargo.toml | 2 +- worker/Cargo.toml | 2 +- 7 files changed, 300 insertions(+), 148 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 414cea05..1f860677 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,12 +19,20 @@ name = "ansi_term" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "arrayvec" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "atty" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -36,8 +44,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "backtrace-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-demangle 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-demangle 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -46,8 +54,8 @@ name = "backtrace-sys" version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -87,12 +95,12 @@ dependencies = [ "chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "hostname 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", "rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -112,12 +120,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.4" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "cerberus" -version = "0.3.0" +version = "0.5.0" dependencies = [ "bson 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -125,22 +133,22 @@ dependencies = [ "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", "uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "cerberus-proto" -version = "0.4.0" +version = "0.5.0" dependencies = [ "grpc 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "grpc-compiler 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "grpc-rust 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "protobuf 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "protoc-rust 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "protobuf 1.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "protoc-rust 1.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "tls-api 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -164,7 +172,7 @@ version = "2.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -175,15 +183,15 @@ dependencies = [ [[package]] name = "cli" -version = "0.3.0" +version = "0.5.0" dependencies = [ - "cerberus-proto 0.4.0", + "cerberus-proto 0.5.0", "chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.26.2 (registry+https://github.com/rust-lang/crates.io-index)", "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "grpc 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "prettytable-rs 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", - "util 0.4.0", + "util 0.5.0", "uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -195,6 +203,36 @@ dependencies = [ "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "crossbeam-deque" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-epoch 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-utils" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "csv" version = "0.15.0" @@ -230,7 +268,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -239,7 +277,7 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -309,11 +347,11 @@ dependencies = [ "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "httpbis 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "protobuf 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "protobuf 1.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "tls-api 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", "tls-api-stub 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-core 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-tls-api 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -322,7 +360,7 @@ name = "grpc-compiler" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "protobuf 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "protobuf 1.4.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -347,7 +385,7 @@ name = "hostname" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "winutil 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -365,11 +403,11 @@ dependencies = [ "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "net2 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", "tls-api 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", "tls-api-stub 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-core 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-timer 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-tls-api 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -408,7 +446,7 @@ name = "iovec" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -465,7 +503,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "libc" -version = "0.2.36" +version = "0.2.39" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -504,9 +542,9 @@ dependencies = [ [[package]] name = "master" -version = "0.4.0" +version = "0.5.0" dependencies = [ - "cerberus-proto 0.4.0", + "cerberus-proto 0.5.0", "chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.26.2 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -518,13 +556,13 @@ dependencies = [ "iron 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mount 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "protobuf 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "protobuf 1.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "router 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", "staticfile 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "util 0.4.0", + "util 0.5.0", "uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -538,7 +576,7 @@ name = "memchr" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -546,7 +584,7 @@ name = "memchr" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -554,9 +592,14 @@ name = "memchr" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "memoffset" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "mime" version = "0.2.6" @@ -567,7 +610,7 @@ dependencies = [ [[package]] name = "mio" -version = "0.6.13" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -575,11 +618,11 @@ dependencies = [ "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "net2 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", - "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -589,7 +632,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "net2 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -627,16 +670,19 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "nodrop" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "nom" version = "2.2.1" @@ -649,7 +695,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-integer 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", "num-iter 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -657,7 +703,7 @@ name = "num-integer" version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-traits 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -666,20 +712,12 @@ version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-integer 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "num-traits" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num-traits 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-traits" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -687,7 +725,7 @@ name = "num_cpus" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -708,7 +746,7 @@ name = "prettytable-rs" version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "atty 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "csv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "encode_unicode 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -716,20 +754,28 @@ dependencies = [ "unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "proc-macro2" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "procinfo" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "nom 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "protobuf" -version = "1.4.3" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -737,7 +783,7 @@ dependencies = [ [[package]] name = "protoc" -version = "1.4.3" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -745,11 +791,11 @@ dependencies = [ [[package]] name = "protoc-rust" -version = "1.4.3" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "protobuf 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "protoc 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "protobuf 1.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "protoc 1.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -758,13 +804,21 @@ name = "quote" version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "quote" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rand" version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -774,7 +828,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -805,12 +859,12 @@ dependencies = [ [[package]] name = "regex" -version = "0.2.6" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -822,8 +876,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "regex-syntax" -version = "0.4.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "remove_dir_all" @@ -855,7 +912,7 @@ version = "0.2.36" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", @@ -863,7 +920,7 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -873,10 +930,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "rustc_version" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "semver 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -886,7 +943,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "scoped-tls" -version = "0.1.0" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "scopeguard" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -899,7 +961,7 @@ dependencies = [ [[package]] name = "semver" -version = "0.6.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -926,38 +988,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde" -version = "1.0.27" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde_derive" -version = "1.0.27" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive_internals 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive_internals 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.12.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_derive_internals" -version = "0.19.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", - "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.12.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_json" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -996,6 +1059,16 @@ dependencies = [ "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "syn" +version = "0.12.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "synom" version = "0.11.3" @@ -1028,7 +1101,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1037,7 +1110,7 @@ name = "termion" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1057,7 +1130,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1082,7 +1155,7 @@ name = "time" version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1104,24 +1177,52 @@ dependencies = [ "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tokio" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-threadpool 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tokio-core" -version = "0.1.12" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "mio 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", - "scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "scoped-tls 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-executor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "tokio-io" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1129,6 +1230,32 @@ dependencies = [ "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "tokio-reactor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-deque 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tokio-timer" version = "0.1.2" @@ -1145,8 +1272,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "tls-api 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-core 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1167,6 +1294,11 @@ dependencies = [ "unsafe-any 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ucd-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unicase" version = "1.4.2" @@ -1198,6 +1330,11 @@ name = "unicode-xid" version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unreachable" version = "1.0.0" @@ -1236,15 +1373,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "util" -version = "0.4.0" +version = "0.5.0" dependencies = [ - "cerberus-proto 0.4.0", + "cerberus-proto 0.5.0", "chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "grpc 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "protobuf 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "protobuf 1.4.4 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1311,25 +1448,25 @@ dependencies = [ [[package]] name = "worker" -version = "0.4.0" +version = "0.5.0" dependencies = [ "bson 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "cerberus-proto 0.4.0", + "cerberus-proto 0.5.0", "clap 2.26.2 (registry+https://github.com/rust-lang/crates.io-index)", "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", "grpc 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)", "local-ip 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mocktopus 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "procinfo 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "protobuf 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", + "protobuf 1.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", "tls-api 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", - "util 0.4.0", + "util 0.5.0", "uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1346,7 +1483,8 @@ dependencies = [ "checksum aho-corasick 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ca972c2ea5f742bfce5687b9aef75506a764f61d37f8f649047846a9686ddb66" "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4" "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" -"checksum atty 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "8352656fd42c30a0c3c89d26dea01e3b77c0ab2af18230835c15e2e13cd51859" +"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" +"checksum atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "af80143d6f7608d746df1520709e5d141c96f240b0e62b0aa41bdfb53374d9d4" "checksum backtrace 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ebbbf59b1c43eefa8c3ede390fcc36820b4999f7914104015be25025e0d62af2" "checksum backtrace-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "44585761d6161b0f57afc49482ab6bd067e4edef48c12a152c237eb0203f7661" "checksum base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96434f987501f0ed4eb336a411e0631ecd1afa11574fe148587adc4ff96143c9" @@ -1356,11 +1494,14 @@ dependencies = [ "checksum bson 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97a88dc284b4f4fa28e8d874a5fde53291290bace25f68f3e7bbd6653afc1a50" "checksum byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "652805b7e73fada9d85e9a6682a4abd490cb52d96aeecc12e33a0de34dfd0d23" "checksum bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1b7db437d718977f6dc9b2e3fd6fc343c02ac6b899b73fdd2179163447bd9ce9" -"checksum cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "deaf9ec656256bb25b404c51ef50097207b9cbb29c933d31f92cae5a8a0ffee0" +"checksum cc 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "87f38f122db5615319a985757e526c00161d924d19b71a0f3e80c52bab1adcf6" "checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de" "checksum chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7c20ebe0b2b08b0aeddba49c609fe7957ba2e33449882cb186a180bc60682fa9" "checksum clap 2.26.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3451e409013178663435d6f15fdb212f14ee4424a3d74f979d081d0a66b6f1f2" "checksum conduit-mime-types 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "95ca30253581af809925ef68c2641cc140d6183f43e12e0af4992d53768bd7b8" +"checksum crossbeam-deque 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c1bdc73742c36f7f35ebcda81dbb33a7e0d33757d03a06d9ddca762712ec5ea2" +"checksum crossbeam-epoch 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2066d2bac93369853e53064ae60d7eab9e8edba77be4ad81bc7628be30aa7960" +"checksum crossbeam-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2760899e32a1d58d5abb31129f8fae5de75220bc2176e77ff7c627ae45c918d9" "checksum csv 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7ef22b37c7a51c564a365892c012dc0271221fdcc64c69b19ba4d6fa8bd96d9c" "checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab" "checksum encode_unicode 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c088ec0ed2282dcd054f2c124c0327f953563e6c75fdc6ff5141779596289830" @@ -1392,7 +1533,7 @@ dependencies = [ "checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" "checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d" "checksum lazycell 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a6f08839bc70ef4a3fe1d566d5350f519c5912ea86be0df1740a7d247c7fc0ef" -"checksum libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "1e5d97d6708edaa407429faa671b942dc0f2727222fb6b6539bf1db936e4b121" +"checksum libc 0.2.39 (registry+https://github.com/rust-lang/crates.io-index)" = "f54263ad99207254cf58b5f701ecb432c717445ea2ee8af387334bdd1a03fdff" "checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd" "checksum linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "70fb39025bc7cdd76305867c4eccf2f2dcf6e9a57f5b21a93e1c2d86cd03ec9e" "checksum local-ip 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1199ecb36eea49b2b849800de0a72dbdf6d8b69046765c2c78c76fdb96440d34" @@ -1402,60 +1543,65 @@ dependencies = [ "checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20" "checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" "checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" +"checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3" "checksum mime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" -"checksum mio 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "7da01a5e23070d92d99b1ecd1cd0af36447c6fd44b0fe283c2db199fa136724f" +"checksum mio 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)" = "6d771e3ef92d58a8da8df7d6976bfca9371ed1de6619d9d5a5ce5b1f29b85bfe" "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" "checksum mocktopus 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9dc5fbcabea8244edabd101547b0981f9a9cee2aa89fcccbbbe6ff9c59c0a262" "checksum mocktopus_macros 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1aefe0f9a78f54eb38430e506f499165b36505f377ec1dc22e74c17e739f194e" "checksum modifier 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "41f5c9112cb662acd3b204077e0de5bc66305fa8df65c8019d5adb10e9ab6e58" "checksum mount 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "32245731923cd096899502fc4c4317cfd09f121e80e73f7f576cf3777a824256" -"checksum net2 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)" = "3a80f842784ef6c9a958b68b7516bc7e35883c614004dd94959a4dca1b716c09" +"checksum net2 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)" = "9044faf1413a1057267be51b5afba8eb1090bd2231c693664aa1db716fe1eae0" +"checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" "checksum nom 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf51a729ecf40266a2368ad335a5fdde43471f545a967109cd62146ecf8b66ff" "checksum num 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e" "checksum num-integer 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f8d26da319fb45674985c78f1d1caf99aa4941f785d384a2ae36d0740bc3e2fe" "checksum num-iter 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "4b226df12c5a59b63569dd57fafb926d91b385dfce33d8074a412411b689d593" -"checksum num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" -"checksum num-traits 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e7de20f146db9d920c45ee8ed8f71681fd9ade71909b48c3acbd766aa504cf10" +"checksum num-traits 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3c2bd9b9d21e48e956b763c9f37134dc62d9e95da6edb3f672cacb6caf3cd3" "checksum num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c51a3322e4bca9d212ad9a158a02abc6934d005490c054a2778df73a70aa0a30" "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" "checksum plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "1a6a0dc3910bc8db877ffed8e457763b317cf880df4ae19109b9f77d277cf6e0" "checksum prettytable-rs 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "34dc1f4f6dddab3bf008ecfd4fd2a631b585fbf0af123f34c1324f51a034ff5f" +"checksum proc-macro2 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cd07deb3c6d1d9ff827999c7f9b04cdfd66b1b17ae508e14fe47b620f2282ae0" "checksum procinfo 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6ab1427f3d2635891f842892dda177883dca0639e05fe66796a62c9d2f23b49c" -"checksum protobuf 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bec26e67194b7d991908145fdf21b7cae8b08423d96dcb9e860cd31f854b9506" -"checksum protoc 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "5379c34ea2f9c69b99e6f25f6d0e6619876195ae7a3dcaf69f66bdb6c2e4dceb" -"checksum protoc-rust 1.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e211a7f56b2d020a59d483f652cfdfa6fd42e37bf544c0231e373807aa316c45" +"checksum protobuf 1.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a06aeffc36f6abfaf86e6b5b350c5ab4ec81ed3d95215c17730e0e744dfdd2c5" +"checksum protoc 1.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "37b952735ccc55f22ddb6d60fb4d7ada863e973a868447d86afb6e3743ecf28c" +"checksum protoc-rust 1.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "70c45a2989a4486bd8ef3d1d127ee1af4cbda71e0c9698ea7173e2a3a2a4f13a" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" +"checksum quote 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1eca14c727ad12702eb4b6bfb5a232287dcf8385cb8ca83a3eeaf6519c44c408" "checksum rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "15a732abf9d20f0ad8eeb6f909bf6868722d9a06e1e50802b6a70351f40b4eb1" "checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" "checksum redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "0d92eecebad22b767915e4d529f89f28ee96dbbf5a4810d2b844373f136417fd" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum regex 0.1.80 (registry+https://github.com/rust-lang/crates.io-index)" = "4fd4ace6a8cf7860714a2c2280d6c1f7e6a413486c13298bbc86fd3da019402f" -"checksum regex 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "5be5347bde0c48cfd8c3fdc0766cdfe9d8a755ef84d620d6794c778c91de8b2b" +"checksum regex 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bde64a9b799f85750f6469fd658cff5fce8d910a7d510858a1f9d15ca9f023bf" "checksum regex-syntax 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f9ec002c35e86791825ed294b50008eea9ddfc8def4420124fbc6b08db834957" -"checksum regex-syntax 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e" +"checksum regex-syntax 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b2550876c31dc914696a6c2e01cbce8afba79a93c8ae979d2fe051c0230b3756" "checksum remove_dir_all 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b5d2f806b0fcdabd98acd380dc8daef485e22bcb7cddc811d1337967f2528cf5" "checksum route-recognizer 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3255338088df8146ba63d60a9b8e3556f1146ce2973bc05a75181a42ce2256" "checksum router 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9b1797ff166029cb632237bb5542696e54961b4cf75a324c6f05c9cf0584e4e" "checksum rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a" -"checksum rustc-demangle 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f312457f8a4fa31d3581a6f423a70d6c33a10b95291985df55f1ff670ec10ce8" +"checksum rustc-demangle 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "11fb43a206a04116ffd7cfcf9bcb941f8eb6cc7ff667272246b0a1c74259a3cb" "checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda" -"checksum rustc_version 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9743a7670d88d5d52950408ecdb7c71d8986251ab604d4689dd2ca25c9bca69" +"checksum rustc_version 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a54aa04a10c68c1c4eacb4337fd883b435997ede17a9385784b990777686b09a" "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" -"checksum scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f417c22df063e9450888a7561788e9bd46d3bb3c1466435b4eccb903807f147d" +"checksum scoped-tls 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8674d439c964889e2476f474a3bf198cc9e199e77499960893bac5de7e9218a4" +"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" "checksum semver 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ca1c06afc03e8a202bc5c1db01524cceb7e1b1ca062d959d83a61b20d76e394e" -"checksum semver 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a3186ec9e65071a2095434b1f5bb24838d4e8e130f584c790f6033c79943537" +"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver-parser 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d8fff3c9c5a54636ab95acd8c1349926e04cb1eb8cd70b5adced8a1d1f703a67" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" "checksum sequence_trie 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c915714ca833b1d4d6b8f6a9d72a3ff632fe45b40a8d184ef79c81bec6327eed" -"checksum serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)" = "db99f3919e20faa51bb2996057f5031d8685019b5a06139b1ce761da671b8526" -"checksum serde_derive 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)" = "f4ba7591cfe93755e89eeecdbcc668885624829b020050e6aec99c2a03bd3fd0" -"checksum serde_derive_internals 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6e03f1c9530c3fb0a0a5c9b826bdd9246a5921ae995d75f512ac917fc4dd55b5" -"checksum serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c9db7266c7d63a4c4b7fe8719656ccdd51acf1bed6124b174f933b009fb10bcb" +"checksum serde 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)" = "c73f63e08b33f6e59dfb3365b009897ebc3a3edc4af6e4f3ce8e483cf3d80ce7" +"checksum serde_derive 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)" = "9cd9e89b8be5b611971734eaf887f1da0ce1a5b51491f04b09fe855649a84f3b" +"checksum serde_derive_internals 0.20.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a79b781fe5c4a7037a10a485249a499ea02927046360afe7e04885aad2f9c10c" +"checksum serde_json 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "fab6c4d75bedcf880711c85e39ebf8ccc70d0eba259899047ec5d7436643ee17" "checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23" "checksum slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fdeff4cd9ecff59ec7e3744cbca73dfe5ac35c2aedb2cfba8a1c715a18912e9d" "checksum staticfile 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "31493480e073d52522a94cdf56269dd8eb05f99549effd1826b0271690608878" "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" "checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" +"checksum syn 0.12.14 (registry+https://github.com/rust-lang/crates.io-index)" = "8c5bc2d6ff27891209efa5f63e9de78648d7801f085e4653701a692ce938d6fd" "checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" "checksum tempdir 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f73eebdb68c14bcb24aef74ea96079830e7fa7b31a6106e42ea7ee887c1e134e" "checksum term 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "fa63644f74ce96fbeb9b794f66aff2a52d601cbd5e80f4b97123e3899f4570f1" @@ -1468,18 +1614,24 @@ dependencies = [ "checksum time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "a15375f1df02096fb3317256ce2cee6a1f42fc84ea5ad5fc8c421cfe40c73098" "checksum tls-api 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "a2424af93d25301cead84cc8225f119918e63d44ee494a0209338dcf7d1eec18" "checksum tls-api-stub 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "f8cf9d37d193ad3e78efb7c85b6c00311d90ed32755a16c62080d1711c1d99fb" -"checksum tokio-core 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "52b4e32d8edbf29501aabb3570f027c6ceb00ccef6538f4bddba0200503e74e8" -"checksum tokio-io 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b9532748772222bf70297ec0e2ad0f17213b4a7dd0e6afb68e0a0768f69f4e4f" +"checksum tokio 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "490c5ff233997a62649c0a7b523b25a1cc6fab1389b3faed0d72e8bdcef7b0ad" +"checksum tokio-core 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "b630092b9a89f5c4eedfe9d022a0c16fb111ed2403a38f985bd1ca68b6b762d3" +"checksum tokio-executor 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46108c2aca0eb4b9a883bf37a28d122ca70f5318446f59f729cd1ff78a0bb5fb" +"checksum tokio-io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "6af9eb326f64b2d6b68438e1953341e00ab3cf54de7e35d92bfc73af8555313a" +"checksum tokio-reactor 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f21d00eb356854d502b81776cec931d12771e4ed6d198478d23ffd38c19279af" +"checksum tokio-threadpool 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "19a8656c45ae7893c9090ac5c98749e7ff904932973fabd541463f82628efacb" "checksum tokio-timer 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6131e780037787ff1b3f8aad9da83bca02438b72277850dd6ad0d455e0e20efc" "checksum tokio-tls-api 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "a47b126d55dc5f9407eeba619a305ca70a8ba8f7d6b813bfaf93e924afe27ce1" "checksum traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" "checksum typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1410f6f91f21d1612654e7cc69193b0334f909dcf2c790c4826254fbb86f8887" "checksum typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "653be63c80a3296da5551e1bfd2cca35227e13cdd08c6668903ae2f4f77aa1f6" +"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" "checksum unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" "checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" "checksum unicode-normalization 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "51ccda9ef9efa3f7ef5d91e8f9b83bbe6955f9bf86aec89d5cce2c874625920f" "checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f" "checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" +"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" "checksum unsafe-any 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f30360d7979f5e9c6e6cea48af192ea8fab4afb3cf72597154b8f08935bc9c7f" "checksum url 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f808aadd8cfec6ef90e4a14eb46f24511824d1ac596b9682703c87056c8678b7" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 9e167fdd..2a8fb8a9 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cli" -version = "0.3.0" +version = "0.5.0" authors = ["Cerberus Authors "] [dependencies] diff --git a/libcerberus/Cargo.toml b/libcerberus/Cargo.toml index 8bf3d98a..2f41bbf4 100644 --- a/libcerberus/Cargo.toml +++ b/libcerberus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cerberus" -version = "0.3.0" +version = "0.5.0" authors = ["Cerberus Authors "] [dependencies] diff --git a/master/Cargo.toml b/master/Cargo.toml index bd7922c4..8306a712 100644 --- a/master/Cargo.toml +++ b/master/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "master" -version = "0.4.0" +version = "0.5.0" build = "build.rs" authors = ["Cerberus Authors "] diff --git a/proto/Cargo.toml b/proto/Cargo.toml index 3eb77141..f6468c30 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cerberus-proto" -version = "0.4.0" +version = "0.5.0" build = "build.rs" [dependencies] diff --git a/util/Cargo.toml b/util/Cargo.toml index 9be84e13..785e02b1 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "util" -version = "0.4.0" +version = "0.5.0" authors = ["Cerberus Authors "] [dependencies] diff --git a/worker/Cargo.toml b/worker/Cargo.toml index d46f9392..64151dd3 100644 --- a/worker/Cargo.toml +++ b/worker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "worker" -version = "0.4.0" +version = "0.5.0" authors = ["Cerberus Authors "] [dependencies]