Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Return scalar result when all inputs are constants in map and make_map #11461

Merged
merged 9 commits into from
Jul 15, 2024
Merged

Return scalar result when all inputs are constants in map and make_map #11461

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
a584b9e
return scalar result when all inputs are constants.
Rachelint Jul 14, 2024
b2ba98e
support convert map array to scalar.
Rachelint Jul 14, 2024
ae1f0f2
disable the const evaluate for Map type before impl its hash calculat…
Rachelint Jul 14, 2024
0956bcd
add tests in map.slt.
Rachelint Jul 14, 2024
48314ae
improve error return.
Rachelint Jul 14, 2024
1217e96
fix error.
Rachelint Jul 14, 2024
46e57ea
fix remove unused import.
Rachelint Jul 14, 2024
6c8c699
remove duplicated testcase.
Rachelint Jul 14, 2024
3a6d37a
remove inline.
Rachelint Jul 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion datafusion/common/src/scalar/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2678,7 +2678,10 @@ impl ScalarValue {
DataType::Duration(TimeUnit::Nanosecond) => {
typed_cast!(array, index, DurationNanosecondArray, DurationNanosecond)?
}

DataType::Map(_, _) => {
let a = array.slice(index, 1);
Self::Map(Arc::new(a.as_map().to_owned()))
}
other => {
return _not_impl_err!(
"Can't create a scalar from array of type \"{other:?}\""
Expand Down
35 changes: 31 additions & 4 deletions datafusion/functions/src/core/map.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,22 @@ use datafusion_common::{exec_err, internal_err, ScalarValue};
use datafusion_common::{not_impl_err, Result};
use datafusion_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility};

/// Check if we can evaluate the expr to constant directly.
///
/// # Example
/// ```sql
/// SELECT make_map('type', 'test') from test
/// ```
/// We can evaluate the result of `make_map` directly.
#[inline]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any specific reason for using inline over here ?

Copy link
Contributor Author

@Rachelint Rachelint Jul 15, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just a personal habit to mark the very short function, and try to reduce the function call cost(it will be really inlined or not... determinded by the compiler).
Will it cause some problems that I haven't noticed before?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since it more of a suggestion to complier, i dont think this should cause problems. TIL: Inlining small functions can provide small speed wins.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think there is rare case for inline. I perfer not to inline unless the benmark shows that compiler fail to do the good job for us

https://matklad.github.io/2021/07/09/inline-in-rust.html

Copy link
Contributor Author

@Rachelint Rachelint Jul 15, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seem inline here is actually unnecessary according to the article mentioned by @jayzhan211 , removed.

fn can_evaluate_to_const(args: &[ColumnarValue]) -> bool {
args.iter()
.all(|arg| matches!(arg, ColumnarValue::Scalar(_)))
}

fn make_map(args: &[ColumnarValue]) -> Result<ColumnarValue> {
let can_evaluate_to_const = can_evaluate_to_const(args);

let (key, value): (Vec<_>, Vec<_>) = args
.chunks_exact(2)
.map(|chunk| {
Expand Down Expand Up @@ -58,7 +73,7 @@ fn make_map(args: &[ColumnarValue]) -> Result<ColumnarValue> {
Ok(value) => value,
Err(e) => return internal_err!("Error concatenating values: {}", e),
};
make_map_batch_internal(key, value)
make_map_batch_internal(key, value, can_evaluate_to_const)
}

fn make_map_batch(args: &[ColumnarValue]) -> Result<ColumnarValue> {
Expand All @@ -68,9 +83,12 @@ fn make_map_batch(args: &[ColumnarValue]) -> Result<ColumnarValue> {
args.len()
);
}

let can_evaluate_to_const = can_evaluate_to_const(args);

let key = get_first_array_ref(&args[0])?;
let value = get_first_array_ref(&args[1])?;
make_map_batch_internal(key, value)
make_map_batch_internal(key, value, can_evaluate_to_const)
}

fn get_first_array_ref(columnar_value: &ColumnarValue) -> Result<ArrayRef> {
Expand All @@ -85,7 +103,11 @@ fn get_first_array_ref(columnar_value: &ColumnarValue) -> Result<ArrayRef> {
}
}

fn make_map_batch_internal(keys: ArrayRef, values: ArrayRef) -> Result<ColumnarValue> {
fn make_map_batch_internal(
keys: ArrayRef,
values: ArrayRef,
can_evaluate_to_const: bool,
) -> Result<ColumnarValue> {
if keys.null_count() > 0 {
return exec_err!("map key cannot be null");
}
Expand Down Expand Up @@ -124,8 +146,13 @@ fn make_map_batch_internal(keys: ArrayRef, values: ArrayRef) -> Result<ColumnarV
.add_buffer(entry_offsets_buffer)
.add_child_data(entry_struct.to_data())
.build()?;
let map_array = Arc::new(MapArray::from(map_data));

Ok(ColumnarValue::Array(Arc::new(MapArray::from(map_data))))
Ok(if can_evaluate_to_const {
ColumnarValue::Scalar(ScalarValue::try_from_array(map_array.as_ref(), 0)?)
} else {
ColumnarValue::Array(map_array)
})
}

#[derive(Debug)]
Expand Down
27 changes: 25 additions & 2 deletions datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -656,12 +656,35 @@ impl<'a> ConstEvaluator<'a> {
} else {
// Non-ListArray
match ScalarValue::try_from_array(&a, 0) {
Ok(s) => ConstSimplifyResult::Simplified(s),
Ok(s) => {
// TODO: support the optimization for `Map` type after support impl hash for it
if matches!(&s, ScalarValue::Map(_)) {
ConstSimplifyResult::SimplifyRuntimeError(
DataFusionError::NotImplemented("Const evaluate for Map type is still not supported".to_string()),
expr,
)
} else {
ConstSimplifyResult::Simplified(s)
}
}
Err(err) => ConstSimplifyResult::SimplifyRuntimeError(err, expr),
}
}
}
ColumnarValue::Scalar(s) => ConstSimplifyResult::Simplified(s),
ColumnarValue::Scalar(s) => {
// TODO: support the optimization for `Map` type after support impl hash for it
if matches!(&s, ScalarValue::Map(_)) {
ConstSimplifyResult::SimplifyRuntimeError(
DataFusionError::NotImplemented(
"Const evaluate for Map type is still not supported"
.to_string(),
),
expr,
)
} else {
ConstSimplifyResult::Simplified(s)
}
}
}
}
}
Expand Down
92 changes: 92 additions & 0 deletions datafusion/sqllogictest/test_files/map.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,3 +212,95 @@ SELECT map(column5, column6) FROM t;
# {k1:1, k2:2}
# {k3: 3}
# {k5: 5}

query error
SELECT map(column5, column6) FROM t;
# TODO: support array value
# ----
# {k1:1, k2:2}
# {k3: 3}
# {k5: 5}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
query error
SELECT map(column5, column6) FROM t;
# TODO: support array value
# ----
# {k1:1, k2:2}
# {k3: 3}
# {k5: 5}

I think this test is the same as the above one. We can remove it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed


query ?
SELECT MAKE_MAP('POST', 41, 'HEAD', 33, 'PATCH', 30, 'OPTION', 29, 'GET', 27, 'PUT', 25, 'DELETE', 24) AS method_count from t;
----
{POST: 41, HEAD: 33, PATCH: 30, OPTION: 29, GET: 27, PUT: 25, DELETE: 24}
{POST: 41, HEAD: 33, PATCH: 30, OPTION: 29, GET: 27, PUT: 25, DELETE: 24}
{POST: 41, HEAD: 33, PATCH: 30, OPTION: 29, GET: 27, PUT: 25, DELETE: 24}

query I
SELECT MAKE_MAP('POST', 41, 'HEAD', 33)['POST'] from t;
----
41
41
41

query ?
SELECT MAKE_MAP('POST', 41, 'HEAD', 33, 'PATCH', null) from t;
----
{POST: 41, HEAD: 33, PATCH: }
{POST: 41, HEAD: 33, PATCH: }
{POST: 41, HEAD: 33, PATCH: }

query ?
SELECT MAKE_MAP('POST', null, 'HEAD', 33, 'PATCH', null) from t;
----
{POST: , HEAD: 33, PATCH: }
{POST: , HEAD: 33, PATCH: }
{POST: , HEAD: 33, PATCH: }

query ?
SELECT MAKE_MAP(1, null, 2, 33, 3, null) from t;
----
{1: , 2: 33, 3: }
{1: , 2: 33, 3: }
{1: , 2: 33, 3: }

query ?
SELECT MAKE_MAP([1,2], ['a', 'b'], [3,4], ['b']) from t;
----
{[1, 2]: [a, b], [3, 4]: [b]}
{[1, 2]: [a, b], [3, 4]: [b]}
{[1, 2]: [a, b], [3, 4]: [b]}

query ?
SELECT MAP(['POST', 'HEAD', 'PATCH'], [41, 33, 30]) from t;
----
{POST: 41, HEAD: 33, PATCH: 30}
{POST: 41, HEAD: 33, PATCH: 30}
{POST: 41, HEAD: 33, PATCH: 30}

query ?
SELECT MAP(['POST', 'HEAD', 'PATCH'], [41, 33, null]) from t;
----
{POST: 41, HEAD: 33, PATCH: }
{POST: 41, HEAD: 33, PATCH: }
{POST: 41, HEAD: 33, PATCH: }

query ?
SELECT MAP([[1,2], [3,4]], ['a', 'b']) from t;
----
{[1, 2]: a, [3, 4]: b}
{[1, 2]: a, [3, 4]: b}
{[1, 2]: a, [3, 4]: b}

query ?
SELECT MAP(make_array('POST', 'HEAD', 'PATCH'), make_array(41, 33, 30)) from t;
----
{POST: 41, HEAD: 33, PATCH: 30}
{POST: 41, HEAD: 33, PATCH: 30}
{POST: 41, HEAD: 33, PATCH: 30}

query ?
SELECT MAP(arrow_cast(make_array('POST', 'HEAD', 'PATCH'), 'FixedSizeList(3, Utf8)'), arrow_cast(make_array(41, 33, 30), 'FixedSizeList(3, Int64)')) from t;
----
{POST: 41, HEAD: 33, PATCH: 30}
{POST: 41, HEAD: 33, PATCH: 30}
{POST: 41, HEAD: 33, PATCH: 30}

query ?
SELECT MAP(arrow_cast(make_array('POST', 'HEAD', 'PATCH'), 'LargeList(Utf8)'), arrow_cast(make_array(41, 33, 30), 'LargeList(Int64)')) from t;
----
{POST: 41, HEAD: 33, PATCH: 30}
{POST: 41, HEAD: 33, PATCH: 30}
{POST: 41, HEAD: 33, PATCH: 30}