wip: sqlness

Signed-off-by: Ruihang Xia <[email protected]>

src/datatypes/src/error.rs

@@ -232,6 +232,12 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+    #[snafu(display("Invalid skip index option: {}", msg))]
+    InvalidSkipIndexOption {
+        msg: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 impl ErrorExt for Error {
@@ -252,7 +258,8 @@ impl ErrorExt for Error {
             | InvalidPrecisionOrScale { .. }
             | InvalidJson { .. }
             | InvalidVector { .. }
-            | InvalidFulltextOption { .. } => StatusCode::InvalidArguments,
+            | InvalidFulltextOption { .. }
+            | InvalidSkipIndexOption { .. } => StatusCode::InvalidArguments,
 
             ValueExceedsPrecision { .. }
             | CastType { .. }

src/datatypes/src/schema.rs

@@ -28,7 +28,7 @@ use snafu::{ensure, ResultExt};
 use crate::error::{self, DuplicateColumnSnafu, Error, ProjectArrowSchemaSnafu, Result};
 use crate::prelude::ConcreteDataType;
 pub use crate::schema::column_schema::{
-    ColumnSchema, FulltextAnalyzer, FulltextOptions, Metadata,
+    ColumnSchema, FulltextAnalyzer, FulltextOptions, Metadata, SkipIndexOptions,
     COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE, COLUMN_FULLTEXT_OPT_KEY_ANALYZER,
     COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE, COLUMN_SKIP_INDEX_OPT_KEY_GRANULARITY,
     COLUMN_SKIP_INDEX_OPT_KEY_TYPE, COMMENT_KEY, FULLTEXT_KEY, INVERTED_INDEX_KEY, TIME_INDEX_KEY,

src/datatypes/src/schema/column_schema.rs

@@ -39,6 +39,8 @@ const DEFAULT_CONSTRAINT_KEY: &str = "greptime:default_constraint";
 pub const FULLTEXT_KEY: &str = "greptime:fulltext";
 /// Key used to store whether the column has inverted index in arrow field's metadata.
 pub const INVERTED_INDEX_KEY: &str = "greptime:inverted_index";
+/// Key used to store skip options in arrow field's metadata.
+pub const SKIP_KEY: &str = "greptime:skip";
 
 /// Keys used in fulltext options
 pub const COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE: &str = "enable";
@@ -49,6 +51,8 @@ pub const COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE: &str = "case_sensitive";
 pub const COLUMN_SKIP_INDEX_OPT_KEY_GRANULARITY: &str = "granularity";
 pub const COLUMN_SKIP_INDEX_OPT_KEY_TYPE: &str = "type";
 
+pub const DEFAULT_GRANULARITY: u32 = 10240;
+
 /// Schema of a column, used as an immutable struct.
 #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct ColumnSchema {
@@ -302,6 +306,34 @@ impl ColumnSchema {
         );
         Ok(())
     }
+
+    /// Retrieves the skip options for the column.
+    pub fn skip_options(&self) -> Result<Option<SkipIndexOptions>> {
+        match self.metadata.get(SKIP_KEY) {
+            None => Ok(None),
+            Some(json) => {
+                let options =
+                    serde_json::from_str(json).context(error::DeserializeSnafu { json })?;
+                Ok(Some(options))
+            }
+        }
+    }
+
+    pub fn with_skip_options(mut self, options: SkipIndexOptions) -> Result<Self> {
+        self.metadata.insert(
+            SKIP_KEY.to_string(),
+            serde_json::to_string(&options).context(error::SerializeSnafu)?,
+        );
+        Ok(self)
+    }
+
+    pub fn set_skip_options(&mut self, options: &SkipIndexOptions) -> Result<()> {
+        self.metadata.insert(
+            SKIP_KEY.to_string(),
+            serde_json::to_string(options).context(error::SerializeSnafu)?,
+        );
+        Ok(())
+    }
 }
 
 /// Column extended type set in column schema's metadata.
@@ -499,6 +531,76 @@ impl fmt::Display for FulltextAnalyzer {
     }
 }
 
+/// Skip options for a column.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, Visit, VisitMut)]
+#[serde(rename_all = "kebab-case")]
+pub struct SkipIndexOptions {
+    /// The granularity of the skip index.
+    pub granularity: u32,
+    /// The type of the skip index.
+    #[serde(default)]
+    pub index_type: SkipIndexType,
+}
+
+impl fmt::Display for SkipIndexOptions {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "granularity={}", self.granularity)?;
+        write!(f, ", index_type={}", self.index_type)?;
+        Ok(())
+    }
+}
+
+/// Skip index types.
+#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, Visit, VisitMut)]
+pub enum SkipIndexType {
+    #[default]
+    BloomFilter,
+}
+
+impl fmt::Display for SkipIndexType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            SkipIndexType::BloomFilter => write!(f, "BLOOM"),
+        }
+    }
+}
+
+impl TryFrom<HashMap<String, String>> for SkipIndexOptions {
+    type Error = Error;
+
+    fn try_from(options: HashMap<String, String>) -> Result<Self> {
+        // Parse granularity with default value 1
+        let granularity = match options.get(COLUMN_SKIP_INDEX_OPT_KEY_GRANULARITY) {
+            Some(value) => value.parse::<u32>().map_err(|_| {
+                error::InvalidSkipIndexOptionSnafu {
+                    msg: format!("Invalid granularity: {value}, expected: positive integer"),
+                }
+                .build()
+            })?,
+            None => DEFAULT_GRANULARITY,
+        };
+
+        // Parse index type with default value BloomFilter
+        let index_type = match options.get(COLUMN_SKIP_INDEX_OPT_KEY_TYPE) {
+            Some(typ) => match typ.to_ascii_uppercase().as_str() {
+                "BLOOM" => SkipIndexType::BloomFilter,
+                _ => {
+                    return error::InvalidSkipIndexOptionSnafu {
+                        msg: format!("Invalid index type: {typ}, expected: 'BLOOM'"),
+                    }
+                    .fail();
+                }
+            },
+            None => SkipIndexType::default(),
+        };
+
+        Ok(SkipIndexOptions {
+            granularity,
+            index_type,
+        })
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::sync::Arc;

src/sql/src/error.rs

@@ -326,6 +326,13 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Failed to set skip index option"))]
+    SetSkipIndexOption {
+        source: datatypes::error::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Datatype error: {}", source))]
     Datatype {
         source: datatypes::error::Error,
@@ -375,7 +382,7 @@ impl ErrorExt for Error {
             ConvertSqlValue { .. } | ConvertValue { .. } => StatusCode::Unsupported,
 
             PermissionDenied { .. } => StatusCode::PermissionDenied,
-            SetFulltextOption { .. } => StatusCode::Unexpected,
+            SetFulltextOption { .. } | SetSkipIndexOption { .. } => StatusCode::Unexpected,
         }
     }
 

src/sql/src/statements.rs

@@ -58,7 +58,8 @@ use crate::error::{
     self, ColumnTypeMismatchSnafu, ConvertSqlValueSnafu, ConvertToGrpcDataTypeSnafu,
     ConvertValueSnafu, DatatypeSnafu, InvalidCastSnafu, InvalidSqlValueSnafu, InvalidUnaryOpSnafu,
     ParseSqlValueSnafu, Result, SerializeColumnDefaultConstraintSnafu, SetFulltextOptionSnafu,
-    TimestampOverflowSnafu, UnsupportedDefaultValueSnafu, UnsupportedUnaryOpSnafu,
+    SetSkipIndexOptionSnafu, TimestampOverflowSnafu, UnsupportedDefaultValueSnafu,
+    UnsupportedUnaryOpSnafu,
 };
 use crate::statements::create::Column;
 pub use crate::statements::option_map::OptionMap;
@@ -513,6 +514,12 @@ pub fn column_to_schema(
             .context(SetFulltextOptionSnafu)?;
     }
 
+    if let Some(options) = column.extensions.build_skip_index_options()? {
+        column_schema = column_schema
+            .with_skip_options(options)
+            .context(SetSkipIndexOptionSnafu)?;
+    }
+
     Ok(column_schema)
 }
 

src/sql/src/statements/create.rs

@@ -16,15 +16,15 @@ use std::collections::HashMap;
 use std::fmt::{Display, Formatter};
 
 use common_catalog::consts::FILE_ENGINE;
-use datatypes::schema::FulltextOptions;
+use datatypes::schema::{FulltextOptions, SkipIndexOptions};
 use itertools::Itertools;
 use serde::Serialize;
 use snafu::ResultExt;
 use sqlparser::ast::{ColumnOptionDef, DataType, Expr, Query};
 use sqlparser_derive::{Visit, VisitMut};
 
 use crate::ast::{ColumnDef, Ident, ObjectName, Value as SqlValue};
-use crate::error::{Result, SetFulltextOptionSnafu};
+use crate::error::{Result, SetFulltextOptionSnafu, SetSkipIndexOptionSnafu};
 use crate::statements::statement::Statement;
 use crate::statements::OptionMap;
 
@@ -182,6 +182,15 @@ impl ColumnExtensions {
         let options: HashMap<String, String> = options.clone().into_map();
         Ok(Some(options.try_into().context(SetFulltextOptionSnafu)?))
     }
+
+    pub fn build_skip_index_options(&self) -> Result<Option<SkipIndexOptions>> {
+        let Some(options) = self.skip_index_options.as_ref() else {
+            return Ok(None);
+        };
+
+        let options: HashMap<String, String> = options.clone().into_map();
+        Ok(Some(options.try_into().context(SetSkipIndexOptionSnafu)?))
+    }
 }
 
 #[derive(Debug, PartialEq, Eq, Clone, Visit, VisitMut, Serialize)]

tests/cases/standalone/common/create/create_with_skip_index.result

@@ -0,0 +1,33 @@
+create table
+    skipping_table (
+        ts timestamp time index,
+        id string skip,
+        `name` string skip
+        with
+            (granularity = 8192),
+    );
+
+Affected Rows: 0
+
+show
+create table
+    skipping_table;
+
++----------------+-----------------------------------------------+
+| Table          | Create Table                                  |
++----------------+-----------------------------------------------+
+| skipping_table | CREATE TABLE IF NOT EXISTS "skipping_table" ( |
+|                |   "ts" TIMESTAMP(3) NOT NULL,                 |
+|                |   "id" STRING NULL,                           |
+|                |   "name" STRING NULL,                         |
+|                |   TIME INDEX ("ts")                           |
+|                | )                                             |
+|                |                                               |
+|                | ENGINE=mito                                   |
+|                |                                               |
++----------------+-----------------------------------------------+
+
+drop table skipping_table;
+
+Affected Rows: 0
+

tests/cases/standalone/common/create/create_with_skip_index.sql

@@ -0,0 +1,14 @@
+create table
+    skipping_table (
+        ts timestamp time index,
+        id string skip,
+        `name` string skip
+        with
+            (granularity = 8192),
+    );
+
+show
+create table
+    skipping_table;
+
+drop table skipping_table;