add page

datafusion/core/src/bin/print_functions_docs.rs

@@ -17,8 +17,8 @@
 
 use datafusion::execution::SessionStateDefaults;
 use datafusion_expr::{
-    aggregate_doc_sections, scalar_doc_sections, window_doc_sections, AggregateUDF,
-    DocSection, Documentation, ScalarUDF, WindowUDF,
+    aggregate_doc_sections, scalar_doc_sections, special_doc_sections,
+    window_doc_sections, AggregateUDF, DocSection, Documentation, ScalarUDF, WindowUDF,
 };
 use hashbrown::HashSet;
 use itertools::Itertools;
@@ -35,7 +35,7 @@ fn main() {
 
     if args.len() != 2 {
         panic!(
-            "Usage: {} type (one of 'aggregate', 'scalar', 'window')",
+            "Usage: {} type (one of 'aggregate', 'scalar', 'special', 'window')",
             args[0]
         );
     }
@@ -44,6 +44,7 @@ fn main() {
     let docs = match function_type.as_str() {
         "aggregate" => print_aggregate_docs(),
         "scalar" => print_scalar_docs(),
+        "special" => print_special_docs(),
         "window" => print_window_docs(),
         _ => {
             panic!("Unknown function type: {}", function_type)
@@ -73,6 +74,17 @@ fn print_scalar_docs() -> String {
     print_docs(providers, scalar_doc_sections::doc_sections())
 }
 
+fn print_special_docs() -> String {
+    let mut providers: Vec<Box<dyn DocProvider>> = vec![];
+
+    // Iterates through the default_scalar_functions to retrieve the special functions
+    for f in SessionStateDefaults::default_scalar_functions() {
+        providers.push(Box::new(f.as_ref().clone()));
+    }
+
+    print_docs(providers, special_doc_sections::doc_sections())
+}
+
 fn print_window_docs() -> String {
     let mut providers: Vec<Box<dyn DocProvider>> = vec![];
 

datafusion/expr/src/lib.rs

@@ -35,6 +35,7 @@ mod table_source;
 mod udaf;
 mod udf;
 mod udf_docs;
+mod udsf;
 mod udwf;
 
 pub mod conditional_expressions;
@@ -96,6 +97,7 @@ pub use udaf::{
 };
 pub use udf::{scalar_doc_sections, ScalarUDF, ScalarUDFImpl};
 pub use udf_docs::{DocSection, Documentation, DocumentationBuilder};
+pub use udsf::special_doc_sections;
 pub use udwf::{window_doc_sections, ReversedUDWF, WindowUDF, WindowUDFImpl};
 pub use window_frame::{WindowFrame, WindowFrameBound, WindowFrameUnits};
 

datafusion/expr/src/udf.rs

@@ -767,7 +767,6 @@ pub mod scalar_doc_sections {
             DOC_SECTION_DATETIME,
             DOC_SECTION_ARRAY,
             DOC_SECTION_STRUCT,
-            DOC_SECTION_MAP,
             DOC_SECTION_HASHING,
             DOC_SECTION_OTHER,
         ]
@@ -826,12 +825,6 @@ The following regular expression functions are supported:"#,
         description: None,
     };
 
-    pub const DOC_SECTION_MAP: DocSection = DocSection {
-        include: true,
-        label: "Map Functions",
-        description: None,
-    };
-
     pub const DOC_SECTION_HASHING: DocSection = DocSection {
         include: true,
         label: "Hashing Functions",

datafusion/expr/src/udsf.rs

@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! [`SpecialUDF`]: Special User Defined Functions
+
+pub mod special_doc_sections {
+    use crate::DocSection;
+
+    pub fn doc_sections() -> Vec<DocSection> {
+        vec![DOC_SECTION_MAP]
+    }
+
+    pub const DOC_SECTION_MAP: DocSection = DocSection {
+        include: true,
+        label: "Map Functions",
+        description: None,
+    };
+}

datafusion/functions-nested/src/map.rs

@@ -17,7 +17,7 @@
 
 use std::any::Any;
 use std::collections::{HashSet, VecDeque};
-use std::sync::Arc;
+use std::sync::{Arc, OnceLock};
 
 use arrow::array::ArrayData;
 use arrow_array::{Array, ArrayRef, MapArray, OffsetSizeTrait, StructArray};
@@ -27,7 +27,10 @@ use arrow_schema::{DataType, Field, SchemaBuilder};
 use datafusion_common::utils::{fixed_size_list_to_arrays, list_to_arrays};
 use datafusion_common::{exec_err, Result, ScalarValue};
 use datafusion_expr::expr::ScalarFunction;
-use datafusion_expr::{ColumnarValue, Expr, ScalarUDFImpl, Signature, Volatility};
+use datafusion_expr::special_doc_sections::DOC_SECTION_MAP;
+use datafusion_expr::{
+    ColumnarValue, Documentation, Expr, ScalarUDFImpl, Signature, Volatility,
+};
 
 use crate::make_array::make_array;
 
@@ -238,7 +241,50 @@ impl ScalarUDFImpl for MapFunc {
     fn invoke(&self, args: &[ColumnarValue]) -> Result<ColumnarValue> {
         make_map_batch(args)
     }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        Some(get_map_doc())
+    }
 }
+
+static DOCUMENTATION: OnceLock<Documentation> = OnceLock::new();
+
+fn get_map_doc() -> &'static Documentation {
+    DOCUMENTATION.get_or_init(|| {
+    Documentation::builder()
+        .with_doc_section(DOC_SECTION_MAP)
+        .with_description(
+            "Returns an Arrow map with the specified key-value pairs.",
+        )
+        .with_syntax_example("map(key, value)\nmap(key: value)")
+        .with_sql_example(
+            r#"```sql
+SELECT MAP(['POST', 'HEAD', 'PATCH'], [41, 33, null]);
+----
+{POST: 41, HEAD: 33, PATCH: }
+
+SELECT MAP([[1,2], [3,4]], ['a', 'b']);
+----
+{[1, 2]: a, [3, 4]: b}
+
+SELECT MAP { 'a': 1, 'b': 2 };
+----
+{a: 1, b: 2}
+```"#,
+        )
+        .with_argument(
+            "key",
+            "Expression to be used for key. Can be a constant, column, or function, any combination of arithmetic or string operators, or a named expression of the previously listed.",
+        )
+        .with_argument(
+            "value",
+            "Expression to be used for value. Can be a constant, column, or function, any combination of arithmetic or string operators, or a named expression of the previously listed.",
+        )
+        .build()
+        .unwrap()
+})
+}
+
 fn get_element_type(data_type: &DataType) -> Result<&DataType> {
     match data_type {
         DataType::List(element) => Ok(element.data_type()),

dev/update_function_docs.sh

@@ -298,3 +298,50 @@ npx [email protected] --write "$TARGET_FILE"
 
 echo "'$TARGET_FILE' successfully updated!"
 
+TARGET_FILE="docs/source/user-guide/sql/special_functions_new.md"
+PRINT_SPECIAL_FUNCTION_DOCS_COMMAND="cargo run --manifest-path datafusion/core/Cargo.toml --bin print_functions_docs -- special"
+
+echo "Inserting header"
+cat <<'EOF' > "$TARGET_FILE"
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+<!---
+This file was generated by the dev/update_function_docs.sh script.
+Do not edit it manually as changes will be overwritten.
+Instead, edit the SpecialUDFImpl's documentation() function to
+update documentation for an individual UDF or the
+dev/update_function_docs.sh file for updating surrounding text.
+-->
+
+# Special Functions (NEW)
+
+Note: this documentation is in the process of being migrated to be [automatically created from the codebase].
+
+[automatically created from the codebase]: https://github.com/apache/datafusion/issues/12740
+
+EOF
+
+echo "Running CLI and inserting special function docs table"
+$PRINT_SPECIAL_FUNCTION_DOCS_COMMAND >> "$TARGET_FILE"
+
+echo "Running prettier"
+npx [email protected] --write "$TARGET_FILE"
+
+echo "'$TARGET_FILE' successfully updated!"

docs/source/user-guide/sql/special_functions_new.md

@@ -0,0 +1,66 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+<!---
+This file was generated by the dev/update_function_docs.sh script.
+Do not edit it manually as changes will be overwritten.
+Instead, edit the SpecialUDFImpl's documentation() function to
+update documentation for an individual UDF or the
+dev/update_function_docs.sh file for updating surrounding text.
+-->
+
+# Special Functions (NEW)
+
+Note: this documentation is in the process of being migrated to be [automatically created from the codebase].
+
+[automatically created from the codebase]: https://github.com/apache/datafusion/issues/12740
+
+## Map Functions
+
+- [map](#map)
+
+### `map`
+
+Returns an Arrow map with the specified key-value pairs.
+
+```
+map(key, value)
+map(key: value)
+```
+
+#### Arguments
+
+- **key**: Expression to be used for key. Can be a constant, column, or function, any combination of arithmetic or string operators, or a named expression of the previously listed.
+- **value**: Expression to be used for value. Can be a constant, column, or function, any combination of arithmetic or string operators, or a named expression of the previously listed.
+
+#### Example
+
+```sql
+SELECT MAP(['POST', 'HEAD', 'PATCH'], [41, 33, null]);
+----
+{POST: 41, HEAD: 33, PATCH: }
+
+SELECT MAP([[1,2], [3,4]], ['a', 'b']);
+----
+{[1, 2]: a, [3, 4]: b}
+
+SELECT MAP { 'a': 1, 'b': 2 };
+----
+{a: 1, b: 2}
+```