[WIP] Composite EC2 Plugin for Metadata

.gitignore

@@ -1,5 +1,6 @@
 /.ansible/galaxy-roles
 /.vagrant
+/.vscode
 
 /target
 **/*.rs.bk

Cargo.toml

@@ -5,11 +5,17 @@ authors = ["Naftuli Kay <[email protected]>"]
 edition = "2018"
 
 [dependencies]
+actix-rt = "0.2"
+actix-web = "1.0"
+crossbeam = "0.7"
+futures = "0.1"
+http = "0.1"
+lazy_static = "1.3"
 log = "0.4"
 log4rs = "0.8"
 num_cpus = "1.10"
 parking_lot = "0.8"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 structopt = "0.2"
-tera = "1.0.0-beta.9"
+tera = "1.0.0-beta.9"

Makefile

@@ -0,0 +1,27 @@
+#!/usr/bin/make -f
+
+EXEC_NAME:=jinjer
+RUST_RELEASE_TARGET:=x86_64-unknown-linux-musl
+
+release:
+	@cargo build --release --bin $(EXEC_NAME) --target $(RUST_RELEASE_TARGET)
+
+strip:
+	@strip target/$(RUST_RELEASE_TARGET)/release/$(EXEC_NAME)
+
+deploy: release strip
+ifndef DEPLOY_HOST
+$(error DEPLOY_HOST is not set.)
+endif
+
+	@echo "Deploying binary to $(DEPLOY_HOST)..."
+	@scp -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+		target/$(RUST_RELEASE_TARGET)/release/$(EXEC_NAME) \
+		$(DEPLOY_HOST):bin/
+
+shell:
+ifndef DEPLOY_HOST
+$(error DEPLOY_HOST is not set.)
+endif
+	@ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+		$(DEPLOY_HOST)

README.md

@@ -84,7 +84,7 @@ Licensed at your discretion under either:
  - [MIT License](./LICENSE-MIT)
 
  [docker]: https://cloud.docker.com/repository/docker/naftulikay/jinjer
- [docker.svg]: https://img.shields.io/docker/build/naftulikay/jinjer.svg
+ [docker.svg]: https://img.shields.io/docker/cloud/build/naftulikay/jinjer.svg
  [tera]: https://tera.netlify.com/
  [travis]: https://travis-ci.org/naftulikay/jinjer
  [travis.svg]: https://travis-ci.org/naftulikay/jinjer.svg?branch=master

src/facts.rs

@@ -1,16 +1,19 @@
 pub mod plugins;
 
+use actix_rt::System;
+
+use futures::Future;
+
 use serde_json::Map;
 use serde_json::Value;
 
 use std::collections::HashMap;
-
 use std::default::Default;
-
 use std::io;
 
 pub type FactSet = Map<String, Value>;
 
+/// A registry for synchronous plugins.
 pub type PluginRegistry = HashMap<String, Box<dyn FactPlugin>>;
 
 pub struct Facts {
@@ -19,31 +22,39 @@ pub struct Facts {
 
 impl Facts {
     pub fn new() -> Self {
-        Self {
-            plugins: PluginRegistry::new(),
-        }
+        Self { plugins: PluginRegistry::default() }
     }
 
     /// Discover all available facts via the registered fact plugins.
     ///
     /// This operation will take some time as it fetches facts from all available providers.
     pub fn discover(&self) -> FactSet {
-        let mut r = FactSet::new();
-
-        for (name, plugin) in self.plugins.iter() {
-            match plugin.discover() {
-                Ok(f) => {
-                    r.insert(name.to_string(), Value::Object(f));
-                }
-                Err(e) => log::warn!("Discovery failed for {}: {}", name, e),
-            }
+        let mut result = FactSet::new();
+
+        // FIXME this should be parallelized using a stream map over the futures
+        for (name, plugin) in &self.plugins {
+            log::info!("Executing asynchronous plugin {}", name);
+
+            match System::new("jinjer").block_on(plugin.discover()) {
+                Ok(v) => {
+                    log::info!("Successfully determined {} facts.", name);
+                    result.insert(name.to_string(), Value::Object(v));
+                },
+                Err(e) => {
+                    match e.kind() {
+                        io::ErrorKind::NotFound => log::debug!("Unable to resolve {} facts.", name),
+                        _ => log::warn!("Unable to resolve facts for {}: {:?}", name, e),
+                    };
+                },
+            };
         }
 
-        r
+        result
     }
 
+    /// Register a plugin.
     pub fn register(&mut self, id: &str, plugin: Box<dyn FactPlugin>) {
-        self.plugins.insert(id.to_string(), plugin);
+        self.plugins.insert(id.to_string(), plugin.into());
     }
 }
 
@@ -53,13 +64,15 @@ impl Default for Facts {
 
         // install plugins
         f.register("basic", Box::new(plugins::basic::BasicPlugin::new()));
+        f.register("ec2", Box::new(plugins::ec2::Ec2Plugin::new()));
         f.register("env", Box::new(plugins::env::EnvPlugin::new()));
 
         f
     }
 }
 
+/// A plugin which discovers facts asynchronously.
 pub trait FactPlugin {
-    /// Execute the plugin, discovering and returning all available facts.
-    fn discover(&self) -> Result<FactSet, io::Error>;
+    /// Execute the plugin, yielding a future that can be polled for readiness.
+    fn discover(&self) -> Box<dyn Future<Item=FactSet, Error=io::Error>>;
 }

src/facts/plugins.rs

@@ -1,2 +1,3 @@
 pub mod basic;
+pub mod ec2;
 pub mod env;

src/facts/plugins/basic.rs

@@ -1,6 +1,8 @@
 use crate::facts::FactPlugin;
 use crate::facts::FactSet;
 
+use futures::Future;
+
 use log;
 
 use num_cpus;
@@ -9,7 +11,6 @@ use serde_json::Number;
 use serde_json::Value;
 
 use std::default::Default;
-
 use std::io;
 
 /// A fact plugin providing a basic set of system facts.
@@ -28,29 +29,31 @@ impl Default for BasicPlugin {
 }
 
 impl FactPlugin for BasicPlugin {
-    fn discover(&self) -> Result<FactSet, io::Error> {
-        log::info!("Discovering basic facts...");
-
-        let mut f = FactSet::new();
-
-        f.insert(
-            "cpu_cores".to_string(),
-            Value::Object({
-                let mut p = FactSet::new();
-
-                p.insert(
-                    "logical".to_string(),
-                    Value::Number(Number::from(num_cpus::get())),
-                );
-                p.insert(
-                    "physical".to_string(),
-                    Value::Number(Number::from(num_cpus::get_physical())),
-                );
-
-                p
-            }),
-        );
-
-        Ok(f)
+    fn discover(&self) -> Box<Future<Item = FactSet, Error = io::Error>> {
+        Box::new(futures::lazy(|| {
+            log::info!("Discovering basic facts...");
+
+            let mut f = FactSet::new();
+
+            f.insert(
+                "cpu_cores".to_string(),
+                Value::Object({
+                    let mut p = FactSet::new();
+
+                    p.insert(
+                        "logical".to_string(),
+                        Value::Number(Number::from(num_cpus::get())),
+                    );
+                    p.insert(
+                        "physical".to_string(),
+                        Value::Number(Number::from(num_cpus::get_physical())),
+                    );
+
+                    p
+                }),
+            );
+
+            Ok(f)
+        }))
     }
 }

src/facts/plugins/ec2.rs

@@ -0,0 +1,68 @@
+mod metadata;
+
+use actix_web::client::Client;
+use actix_web::http::header::HeaderValue;
+
+use futures::Future;
+
+use crate::facts::FactPlugin;
+use crate::facts::FactSet;
+
+use http::StatusCode;
+
+use log;
+
+use std::default::Default;
+
+use std::env;
+
+use std::io;
+
+lazy_static! {
+    pub static ref EC2_METADATA_URL: String = format!(
+        "http://{}/",
+        env::var("EC2_METADATA_HOST").unwrap_or("169.254.169.254".to_string())
+    );
+}
+
+/// A fact plugin for EC2-related data.
+///
+/// This plugin will produce EC2 metadata and EC2 tag facts.
+pub struct Ec2Plugin;
+
+impl Ec2Plugin {
+    pub fn new() -> Self {
+        Self::default()
+    }
+}
+
+impl Default for Ec2Plugin {
+    fn default() -> Self {
+        Self
+    }
+}
+
+impl FactPlugin for Ec2Plugin {
+    fn discover(&self) -> Box<Future<Item = FactSet, Error = io::Error>> {
+        let f = Client::default()
+            .get(EC2_METADATA_URL.as_str())
+            .header("User-Agent", "jinjer")
+            .send()
+            .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("{:?}", e)))
+            .map(|resp| {
+                log::debug!("Reached the EC2 metadata server: {:?}", resp);
+
+                if resp.status() != StatusCode::OK {
+                    log::debug!(
+                        "Reached the EC2 metadata service, but received an error response."
+                    );
+
+                    // return Err(io::Error::new(io::ErrorKind::NotFound, "HTTP error reaching EC2 metadata service."));
+                }
+
+                FactSet::new()
+            });
+
+        Box::new(f)
+    }
+}

src/facts/plugins/ec2/metadata.rs

@@ -0,0 +1,65 @@
+use actix_rt::System;
+use actix_web::client::Client;
+
+use crate::facts::FactPlugin;
+use crate::facts::FactSet;
+
+use futures::future;
+use futures::Future;
+
+use log;
+
+use std::default::Default;
+
+use std::env;
+
+use std::io;
+
+lazy_static! {
+    static ref METADATA_URL: String = format!(
+        "http://{}/latest/meta-data/",
+        env::var("EC2_METADATA_HOST").unwrap_or("169.254.169.254".to_string())
+    );
+}
+
+/// An EC2 instance metadata fact plugin.
+///
+/// This plugin harvests all available facts present in the latest API revision of the EC2 metadata
+/// service.
+pub struct Ec2MetadataPlugin;
+
+impl Ec2MetadataPlugin {
+    pub fn new() -> Self {
+        Self::default()
+    }
+}
+
+impl Default for Ec2MetadataPlugin {
+    fn default() -> Self {
+        Self
+    }
+}
+
+impl FactPlugin for Ec2MetadataPlugin {
+    fn discover(&self) -> Box<Future<Item = FactSet, Error = io::Error>> {
+        Box::new(future::lazy(|| {
+            log::info!("Discovering EC2 metadata facts...");
+
+            System::new("ec2-metadata")
+                .block_on(futures::lazy(|| {
+                    Client::default()
+                        .get(METADATA_URL.as_str())
+                        .header("User-Agent", "jinjer")
+                        .send()
+                        .map_err(|e| log::error!("Error: {:?}", e))
+                        .and_then(|response| {
+                            log::info!("Response: {:?}", response);
+                            Ok(())
+                        })
+                }))
+                .unwrap();
+
+            Ok(FactSet::new())
+        }))
+    }
+}