New output model (#11)

add new data output model

fix travis windows build

Author: binh-vu

.travis/install-dependencies-x86_64.sh

@@ -37,3 +37,10 @@ else
 
     echo ${MINICONDA_VERSION} > ${MINICONDA_HOME}/travis-${MINICONDA_VERSION}-20191015.txt
 fi
+
+##########################
+# INSTALL RUST for Travis Windows
+if [ "${TRAVIS_OS_NAME}" = "windows" ]; then
+    echo "Install x86_64-pc-windows-msvc target on windows"
+    rustup target add x86_64-pc-windows-msvc
+fi

drepr/Cargo.lock

@@ -1,7 +1,7 @@
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [package]
 name = "engine"
-version = "1.0.5"   # ___PKG_VERSION___: DO NOT MODIFY the version here. Update it via version_manager.py!
+version = "1.0.6"   # ___PKG_VERSION___: DO NOT MODIFY the version here. Update it via version_manager.py!
 authors = ["Binh Vu <[email protected]>"]
 edition = "2018"
 
@@ -30,6 +30,7 @@ fancy-regex = "0.1.0"
 lazy_static = "1.3.0"
 
 [dependencies.cpython]
-version = "0.2.1"
+#version = "0.3.1"
+git = "https://github.com/binh-vu/rust-cpython"
 # Note: enable this feature generating error when running test or building binary package. See README.md for more information
 #features = ["extension-module"]

drepr/engine/Cargo.toml

@@ -67,18 +67,18 @@ impl<'a> ClassMapPlan<'a> {
         }
         GraphNode::LiteralNode(n) => {
           literal_props.push(LiteralProp {
-            predicate_id: desc.semantic_model.get_edge(class_id, n.node_id).unwrap().edge_id,
+            predicate_id: eid,
             value: n.val.clone(),
           });
         }
         GraphNode::ClassNode(n) => {
           let attribute = &desc.attributes[class2subj[n.node_id]];
-          let predicate_id = desc.semantic_model.get_edge(class_id, n.node_id).unwrap().edge_id;
+          let predicate_id = eid;
           // a class node is optional if all of its properties are optional
           let is_target_optional = desc.semantic_model.outgoing_edges[n.node_id]
             .iter().all(|&eid| edges_optional[eid]);
           let alignments = inference.get_alignments(subj, attribute.id);
-          
+
           let prop = if n.is_blank_node(&desc.semantic_model) {
             ObjectProp::BlankObject(BlankObject {
               attribute,
@@ -106,7 +106,7 @@ impl<'a> ClassMapPlan<'a> {
                 .collect::<HashSet<_>>(),
             })
           };
-          
+
           if removed_edges[predicate_id] {
             buffered_object_props.push(prop);
           } else {

drepr/engine/src/execution_plans/classes_map_plan/class_map_plan.rs

@@ -82,7 +82,7 @@ impl<'a> ClassesMapExecutionPlan<'a> {
 
       let write_mode = WriteMode::create(
         !cls_plan.is_optional(), cls_plan.subject.is_unique(),
-          match &cls_plan.subject {
+        match &cls_plan.subject {
             Subject::BlankSubject(_) => Some(true),
             Subject::InternalIDSubject(s) => if s.missing_values.len() > 0 && s.is_optional {
               // uri is missing sometime so we have to use blank node
@@ -98,7 +98,7 @@ impl<'a> ClassesMapExecutionPlan<'a> {
               // always uri
               Some(false)
             },
-          }, obj_blank_or_uri);
+        }, obj_blank_or_uri);
 
       class_write_modes[cls_plan.class_id] = write_mode;
     }

drepr/engine/src/execution_plans/classes_map_plan/mod.rs

@@ -72,7 +72,7 @@ impl<'a> Subject<'a> {
       // always true for blank
       Subject::BlankSubject(_) => true,
       Subject::InternalIDSubject(s) => s.attr.unique,
-      Subject::ExternalIDSubject(s) => s.real_id.0.unique
+      Subject::ExternalIDSubject(s) => s.real_id.0.unique,
     }
   }
 }

drepr/engine/src/execution_plans/classes_map_plan/subject.rs

@@ -7,7 +7,7 @@ use crate::executors::classes_map::generic_algo::generic_class_map;
 use crate::executors::preprocessing::exec_preprocessing;
 use crate::lang::{Description, Resource};
 use crate::writers::stream_writer::{OutputFormat};
-use crate::writers::stream_writer::{GraphJSONWriter, TTLStreamWriter};
+use crate::writers::stream_writer::{GraphJSONWriter, TTLStreamWriter, GraphPyWriter};
 use crate::writers::stream_writer::stream_writer::{StreamWriterResult, WriteResult};
 use crate::execution_plans::classes_map_plan::class_map_plan::ClassMapExecStrategy;
 #[cfg(feature = "enable-exec-macro-cls-map")]
@@ -55,6 +55,17 @@ pub fn classes_map(resource_files: &[PhysicalResource], desc: &Description, plan
           }
         };
         readers.push(reader);
+      },
+      Resource::NPDict(_) => {
+        let reader = match &resource_files[i] {
+          PhysicalResource::File(fpath) => {
+            Box::new(JSONRAReader::from_file(fpath))
+          }
+          PhysicalResource::String(content) => {
+            Box::new(JSONRAReader::from_str(content))
+          }
+        };
+        readers.push(reader);
       }
       _ => unimplemented!()
     }
@@ -74,12 +85,18 @@ pub fn classes_map(resource_files: &[PhysicalResource], desc: &Description, plan
             &format!("{}.edge", fpath),
             &desc.semantic_model))
         }
-        PhysicalOutput::String { format: OutputFormat::TTL } => {
+        PhysicalOutput::File { fpath: _, format: OutputFormat::GraphPy } => {
+          unimplemented!()
+        }
+        PhysicalOutput::Memory { format: OutputFormat::TTL } => {
           Box::new(TTLStreamWriter::write2str(&desc.semantic_model))
         }
-        PhysicalOutput::String { format: OutputFormat::GraphJSON } => {
+        PhysicalOutput::Memory { format: OutputFormat::GraphJSON } => {
           Box::new(GraphJSONWriter::write2str(&desc.semantic_model))
         }
+        PhysicalOutput::Memory { format: OutputFormat::GraphPy } => {
+          Box::new(GraphPyWriter::write2mem(&desc.semantic_model))
+        }
       };
 
       writer.begin();

drepr/engine/src/executors/classes_map/mod.rs

@@ -20,8 +20,8 @@ pub enum PhysicalResource {
 pub enum PhysicalOutput {
   #[serde(rename = "file")]
   File { fpath: String, format: OutputFormat },
-  #[serde(rename = "string")]
-  String { format: OutputFormat },
+  #[serde(rename = "memory")]
+  Memory { format: OutputFormat }
 }
 
 #[derive(Deserialize, Serialize, Debug, Clone)]
@@ -45,7 +45,7 @@ impl Executor {
 //    let edges_optional = vec![true; self.description.semantic_model.edges.len()];
     let output_format = match &self.output {
       PhysicalOutput::File { fpath: _, format } => format,
-      PhysicalOutput::String { format } => format
+      PhysicalOutput::Memory { format } => format
     };
     let exec_plan = ClassesMapExecutionPlan::new(&self.description, output_format, &self.edges_optional);
     ExecutionPlan::ClassesMap(exec_plan)
@@ -56,7 +56,7 @@ impl PhysicalOutput {
   pub fn get_format(&self) -> &OutputFormat {
     match self {
       PhysicalOutput::File { fpath: _, format } => format,
-      PhysicalOutput::String { format } => format
+      PhysicalOutput::Memory { format } => format
     }
   }
 }

drepr/engine/src/executors/mod.rs

@@ -28,7 +28,7 @@ impl<'a, F> FilterFunc<'a, F>
           }
         }
         StepExpr::Range(r) => {
-          let mut end = r.get_end(reader.len());
+          let end = r.get_end(reader.len());
           let mut idx = [Index::Idx(r.start)];
           for i in (r.start..end).step_by(r.step).rev() {
             idx[0] = Index::Idx(i);

drepr/engine/src/functions/filter_func.rs

@@ -10,8 +10,10 @@ pub enum Resource {
   JSON(usize),
   #[serde(rename = "spreadsheet")]
   Spreadsheet(usize),
-  #[serde(rename = "netcdf")]
-  NetCDF(usize),
+  #[serde(rename = "netcdf4")]
+  NetCDF4(usize),
+  #[serde(rename = "np-dict")]
+  NPDict(usize)
 }
 
 #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]

drepr/engine/src/lang/resource.rs

@@ -65,8 +65,7 @@ impl ClassNode {
 
     format!("{}{}", short_lbl, self.node_id)
   }
-  
-  
+
   pub fn is_blank_node(&self, sm: &SemanticModel) -> bool {
     for &e in &sm.outgoing_edges[self.node_id] {
       if sm.edges[e].rel_label == DREPR_URI {