Add to_utc_timestamp support in more dialects (#420)

* Add support for to_utc_timestamp in more dialects * Use setup-pixi action * Manual pixi installation * again
vega · Nov 16, 2023 · a27ed7b · a27ed7b
1 parent 0b5c122
commit a27ed7b
Show file tree

Hide file tree

Showing 9 changed files with 349 additions and 75 deletions.
diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml
@@ -8,8 +8,10 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi
-        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
+          cache: true
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -37,14 +39,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi (linux / osx)
-        if: ${{ runner.os != 'Windows' }}
-        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
-      - name: Install pixi (windows)
-        if: ${{ runner.os == 'Windows' }}
-        run: |
-          iwr -useb https://pixi.sh/install.ps1 | iex
-          echo "${HOME}/AppData/Local/pixi/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -71,8 +68,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi (linux / osx)
-        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -158,10 +156,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi
-        run: |
-          iwr -useb https://pixi.sh/install.ps1 | iex
-          echo "${HOME}/AppData/Local/pixi/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -186,8 +183,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi
-        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -212,8 +210,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi
-        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -239,8 +238,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi (linux / osx)
-        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -265,8 +265,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi (linux / osx)
-        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -401,8 +402,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi on linux
-        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -452,8 +454,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi
-        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -560,10 +563,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi
-        run: |
-          iwr -useb https://pixi.sh/install.ps1 | iex
-          echo "${HOME}/AppData/Local/pixi/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -593,8 +595,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi
-        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -623,8 +626,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi
-        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:

diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
@@ -8,8 +8,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi
-        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -36,8 +37,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi
-        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -64,8 +66,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi
-        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -94,10 +97,17 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
+# # Using pixi action here results in error on cleanup:
+# # EBUSY: resource busy or locked, unlink 'D:\a\vegafusion\vegafusion\.pixi\env\Library\lib\jvm\lib\modules'
+#      - uses: prefix-dev/[email protected]
+#        with:
+#          pixi-version: v0.7.0
+# # So use manual install for now
       - name: Install pixi
         run: |
           iwr -useb https://pixi.sh/install.ps1 | iex
-          echo "${HOME}/AppData/Local/pixi/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "${HOME}\.pixi\bin"
+          echo "${HOME}\.pixi\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
       - name: Cache
         uses: actions/cache@v3
         with:
@@ -129,8 +139,9 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
-      - name: Install pixi
-        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.7.0
       - name: Cache
         uses: actions/cache@v3
         with:

diff --git a/vegafusion-sql/Cargo.toml b/vegafusion-sql/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2021"
 description = "VegaFusion SQL dialect generation and connection implementations"
 
 [features]
-datafusion-conn = [ "datafusion", "tempfile", "reqwest", "reqwest-retry", "reqwest-middleware", "vegafusion-datafusion-udfs", "object_store", "url"]
+datafusion-conn = [ "datafusion", "tempfile", "reqwest", "reqwest-retry", "reqwest-middleware", "vegafusion-datafusion-udfs", "object_store", "url", "vegafusion-common/object_store",]
 
 [dependencies]
 async-trait = "0.1.73"

diff --git a/vegafusion-sql/src/compile/scalar.rs b/vegafusion-sql/src/compile/scalar.rs
@@ -1,12 +1,19 @@
 use crate::compile::data_type::ToSqlDataType;
+use crate::compile::expr::ToSqlExpr;
 use crate::dialect::Dialect;
-use arrow::datatypes::DataType;
+use arrow::datatypes::{DataType, TimeUnit};
 use datafusion_common::scalar::ScalarValue;
+use datafusion_common::DFSchema;
+use datafusion_expr::{
+    expr, lit, ColumnarValue, Expr, ReturnTypeFunction, ScalarFunctionImplementation, ScalarUDF,
+    Signature, Volatility,
+};
 use sqlparser::ast::{
     Expr as SqlExpr, Function as SqlFunction, FunctionArg as SqlFunctionArg, FunctionArgExpr,
     Ident, ObjectName as SqlObjectName, Value as SqlValue,
 };
 use std::ops::Add;
+use std::sync::Arc;
 use vegafusion_common::error::{Result, VegaFusionError};
 
 pub trait ToSqlScalar {
@@ -167,28 +174,28 @@ impl ToSqlScalar for ScalarValue {
             )),
             ScalarValue::TimestampSecond(v, _) => {
                 if let Some(v) = v {
-                    Ok(ms_to_timestamp(v * 1000))
+                    Ok(ms_to_timestamp(v * 1000, dialect)?)
                 } else {
                     Ok(SqlExpr::Value(SqlValue::Null))
                 }
             }
             ScalarValue::TimestampMillisecond(v, _) => {
                 if let Some(v) = v {
-                    Ok(ms_to_timestamp(*v))
+                    Ok(ms_to_timestamp(*v, dialect)?)
                 } else {
                     Ok(SqlExpr::Value(SqlValue::Null))
                 }
             }
             ScalarValue::TimestampMicrosecond(v, _) => {
                 if let Some(v) = v {
-                    Ok(ms_to_timestamp(v / 1000))
+                    Ok(ms_to_timestamp(v / 1000, dialect)?)
                 } else {
                     Ok(SqlExpr::Value(SqlValue::Null))
                 }
             }
             ScalarValue::TimestampNanosecond(v, _) => {
                 if let Some(v) = v {
-                    Ok(ms_to_timestamp(v / 1000000))
+                    Ok(ms_to_timestamp(v / 1000000, dialect)?)
                 } else {
                     Ok(SqlExpr::Value(SqlValue::Null))
                 }
@@ -245,24 +252,26 @@ impl ToSqlScalar for ScalarValue {
     }
 }
 
-fn ms_to_timestamp(v: i64) -> SqlExpr {
-    let function_ident = Ident {
-        value: "epoch_ms_to_utc_timestamp".to_string(),
-        quote_style: None,
-    };
-
-    let v_ms_expr = SqlExpr::Value(SqlValue::Number(v.to_string(), false));
-
-    let args = vec![SqlFunctionArg::Unnamed(FunctionArgExpr::Expr(v_ms_expr))];
+fn ms_to_timestamp(v: i64, dialect: &Dialect) -> Result<SqlExpr> {
+    // Hack to recursively transform the epoch_ms_to_utc_timestamp
+    let return_type: ReturnTypeFunction =
+        Arc::new(move |_| Ok(Arc::new(DataType::Timestamp(TimeUnit::Millisecond, None))));
+    let signature: Signature = Signature::exact(vec![DataType::Int64], Volatility::Immutable);
+    let scalar_fn: ScalarFunctionImplementation = Arc::new(move |_args: &[ColumnarValue]| {
+        panic!("Placeholder UDF implementation should not be called")
+    });
 
-    SqlExpr::Function(SqlFunction {
-        name: SqlObjectName(vec![function_ident]),
-        args,
-        over: None,
-        distinct: false,
-        special: false,
-        order_by: Default::default(),
+    let udf = ScalarUDF::new(
+        "epoch_ms_to_utc_timestamp",
+        &signature,
+        &return_type,
+        &scalar_fn,
+    );
+    Expr::ScalarUDF(expr::ScalarUDF {
+        fun: Arc::new(udf),
+        args: vec![lit(v)],
     })
+    .to_sql(dialect, &DFSchema::empty())
 }
 
 fn date32_to_date(days: &Option<i32>, dialect: &Dialect) -> Result<SqlExpr> {