Introduce JIT code generation (#1849)

Cargo.toml

@@ -20,6 +20,7 @@ members = [
     "datafusion",
     "datafusion-common",
     "datafusion-expr",
+    "datafusion-jit",
     "datafusion-physical-expr",
     "datafusion-cli",
     "datafusion-examples",

datafusion-common/Cargo.toml

@@ -35,6 +35,7 @@ path = "src/lib.rs"
 [features]
 avro = ["avro-rs"]
 pyarrow = ["pyo3"]
+jit = ["cranelift-module"]
 
 [dependencies]
 arrow = { version = "9.0.0", features = ["prettyprint"] }
@@ -43,3 +44,4 @@ avro-rs = { version = "0.13", features = ["snappy"], optional = true }
 pyo3 = { version = "0.15", optional = true }
 sqlparser = "0.14"
 ordered-float = "2.10"
+cranelift-module = { version = "0.81.1", optional = true }

datafusion-common/src/error.rs

@@ -25,6 +25,8 @@ use std::result;
 use arrow::error::ArrowError;
 #[cfg(feature = "avro")]
 use avro_rs::Error as AvroError;
+#[cfg(feature = "jit")]
+use cranelift_module::ModuleError;
 use parquet::errors::ParquetError;
 use sqlparser::parser::ParserError;
 
@@ -69,6 +71,9 @@ pub enum DataFusionError {
     /// Errors originating from outside DataFusion's core codebase.
     /// For example, a custom S3Error from the crate datafusion-objectstore-s3
     External(GenericError),
+    #[cfg(feature = "jit")]
+    /// Error occurs during code generation
+    JITError(ModuleError),
 }
 
 impl From<io::Error> for DataFusionError {
@@ -112,6 +117,13 @@ impl From<ParserError> for DataFusionError {
     }
 }
 
+#[cfg(feature = "jit")]
+impl From<ModuleError> for DataFusionError {
+    fn from(e: ModuleError) -> Self {
+        DataFusionError::JITError(e)
+    }
+}
+
 impl From<GenericError> for DataFusionError {
     fn from(err: GenericError) -> Self {
         DataFusionError::External(err)
@@ -152,6 +164,10 @@ impl Display for DataFusionError {
             DataFusionError::External(ref desc) => {
                 write!(f, "External error: {}", desc)
             }
+            #[cfg(feature = "jit")]
+            DataFusionError::JITError(ref desc) => {
+                write!(f, "JIT error: {}", desc)
+            }
         }
     }
 }
@@ -196,3 +212,10 @@ mod test {
         Ok(())
     }
 }
+
+#[macro_export]
+macro_rules! internal_err {
+    ($($arg:tt)*) => {
+        Err(DataFusionError::Internal(format!($($arg)*)))
+    };
+}

datafusion-jit/Cargo.toml

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "datafusion-jit"
+description = "DataFusion is an in-memory query engine that uses Apache Arrow as the memory model"
+version = "7.0.0"
+homepage = "https://github.com/apache/arrow-datafusion"
+repository = "https://github.com/apache/arrow-datafusion"
+readme = "../README.md"
+authors = ["Apache Arrow <[email protected]>"]
+license = "Apache-2.0"
+keywords = [ "arrow", "query", "sql" ]
+edition = "2021"
+rust-version = "1.58"
+
+[lib]
+name = "datafusion_jit"
+path = "src/lib.rs"
+
+[features]
+jit = []
+
+[dependencies]
+datafusion-common = { path = "../datafusion-common", version = "7.0.0", features = ["jit"] }
+cranelift = "0.81.1"
+cranelift-module = "0.81.1"
+cranelift-jit = "0.81.1"
+cranelift-native = "0.81.1"
+parking_lot = "0.12"