Merge branch '4paradigm:main' into add_apiserver_test

4paradigm · Jan 24, 2024 · 8b7f625 · 8b7f625
2 parents ba3ff59 + b2f85fb
commit 8b7f625
Show file tree

Hide file tree

Showing 351 changed files with 7,165 additions and 2,007 deletions.
diff --git a/.github/workflows/udf-doc.yml b/.github/workflows/udf-doc.yml
@@ -7,8 +7,8 @@ on:
   pull_request:
     paths:
       - '.github/workflows/udf-doc.yml'
-      - 'hybridse/tools/documentation/**'
-      - 'hybridse/src/cmd/export_udf_info.cc'
+      - 'include/**'
+      - 'hybridse/**'
   workflow_dispatch:
 
 jobs:

diff --git a/README.md b/README.md
@@ -157,11 +157,11 @@ We really appreciate the contribution from our community.
   ![wechat](docs/en/about/images/wechat.png)  
 
 ## 12. Publications
-
+- [PECJ: Stream Window Join on Disorder Data Streams with Proactive Error Compensation](https://tonyskyzeng.github.io/downloads/PECJ_TR.pdf). Xianzhi Zeng, Shuhao Zhang, Hongbin Zhong, Hao Zhang, Mian Lu, Zhao Zheng, and Yuqiang Chen. International Conference on Management of Data (SIGMOD/PODS) 2024.
 - [Principles and Practices of Real-Time Feature Computing Platforms for ML](https://cacm.acm.org/magazines/2023/7/274061-principles-and-practices-of-real-time-feature-computing-platforms-for-ml/fulltext). Hao Zhang, Jun Yang, Cheng Chen, Siqi Wang, Jiashu Li, and Mian Lu. 2023. Communications of the ACM 66, 7 (July 2023), 77–78.
 - [Scalable Online Interval Join on Modern Multicore Processors in OpenMLDB](docs/paper/scale_oij_icde2023.pdf). Hao Zhang, Xianzhi Zeng, Shuhao Zhang, Xinyi Liu, Mian Lu, and Zhao Zheng. In 2023 IEEE 39rd International Conference on Data Engineering (ICDE) 2023. [[code]](https://github.com/4paradigm/OpenMLDB/tree/stream)
 - [FEBench: A Benchmark for Real-Time Relational Data Feature Extraction](https://github.com/decis-bench/febench/blob/main/report/febench.pdf). Xuanhe Zhou, Cheng Chen, Kunyi Li, Bingsheng He, Mian Lu, Qiaosheng Liu, Wei Huang, Guoliang Li, Zhao Zheng, Yuqiang Chen. International Conference on Very Large Data Bases (VLDB) 2023. [[code]](https://github.com/decis-bench/febench).
--  [A System for Time Series Feature Extraction in Federated Learning](https://dl.acm.org/doi/pdf/10.1145/3511808.3557176). Siqi Wang, Jiashu Li, Mian Lu, Zhao Zheng, Yuqiang Chen, and Bingsheng He. 2022. In Proceedings of the 31st ACM International Conference on Information & Knowledge Management (CIKM) 2022. [[code]](https://github.com/4paradigm/tsfe).
+- [A System for Time Series Feature Extraction in Federated Learning](https://dl.acm.org/doi/pdf/10.1145/3511808.3557176). Siqi Wang, Jiashu Li, Mian Lu, Zhao Zheng, Yuqiang Chen, and Bingsheng He. 2022. In Proceedings of the 31st ACM International Conference on Information & Knowledge Management (CIKM) 2022. [[code]](https://github.com/4paradigm/tsfe).
 - [Optimizing in-memory database engine for AI-powered on-line decision augmentation using persistent memory](http://vldb.org/pvldb/vol14/p799-chen.pdf). Cheng Chen, Jun Yang, Mian Lu, Taize Wang, Zhao Zheng, Yuqiang Chen, Wenyuan Dai, Bingsheng He, Weng-Fai Wong, Guoan Wu, Yuping Zhao, and Andy Rudoff. International Conference on Very Large Data Bases (VLDB) 2021.
 
 ## 13. [The User List](https://github.com/4paradigm/OpenMLDB/discussions/707)

diff --git a/README_cn.md b/README_cn.md
@@ -144,7 +144,7 @@ OpenMLDB 的整体架构设计是为了达到特征平台从开发到部署的
   ![wechat](docs/zh/about/images/wechat.png)  
 
 ## 12. 学术论文
-
+- [PECJ: Stream Window Join on Disorder Data Streams with Proactive Error Compensation](https://tonyskyzeng.github.io/downloads/PECJ_TR.pdf). Xianzhi Zeng, Shuhao Zhang, Hongbin Zhong, Hao Zhang, Mian Lu, Zhao Zheng, and Yuqiang Chen. International Conference on Management of Data (SIGMOD/PODS) 2024.
 - [Principles and Practices of Real-Time Feature Computing Platforms for ML](https://cacm.acm.org/magazines/2023/7/274061-principles-and-practices-of-real-time-feature-computing-platforms-for-ml/fulltext). Hao Zhang, Jun Yang, Cheng Chen, Siqi Wang, Jiashu Li, and Mian Lu. 2023. Communications of the ACM 66, 7 (July 2023), 77–78.
 - [Scalable Online Interval Join on Modern Multicore Processors in OpenMLDB](docs/paper/scale_oij_icde2023.pdf). Hao Zhang, Xianzhi Zeng, Shuhao Zhang, Xinyi Liu, Mian Lu, and Zhao Zheng. In 2023 IEEE 39rd International Conference on Data Engineering (ICDE) 2023. [[code]](https://github.com/4paradigm/OpenMLDB/tree/stream)
 - [FEBench: A Benchmark for Real-Time Relational Data Feature Extraction](https://github.com/decis-bench/febench/blob/main/report/febench.pdf). Xuanhe Zhou, Cheng Chen, Kunyi Li, Bingsheng He, Mian Lu, Qiaosheng Liu, Wei Huang, Guoliang Li, Zhao Zheng, Yuqiang Chen. International Conference on Very Large Data Bases (VLDB) 2023. [[code]](https://github.com/decis-bench/febench).

diff --git a/cases/function/join/test_lastjoin_simple.yaml b/cases/function/join/test_lastjoin_simple.yaml
@@ -1067,4 +1067,4 @@ cases:
       rows:
         - [ "aa",2,131,1590738990000 ]
         - [ "bb",21,NULL,NULL ]
-        - [ "dd", 41, NULL, NULL ]
+        - [ "dd", 41, NULL, NULL ]
diff --git a/cases/plan/back_quote_identifier.yaml b/cases/plan/back_quote_identifier.yaml
@@ -131,12 +131,10 @@ cases:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: a-1
           |  |    +-column_type: int32
-          |  |    +-NOT NULL: 0
           |  +-1:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: b-1
           |  |    +-column_type: string
-          |  |    +-NOT NULL: 0
           |  +-2:
           |    +-node[kColumnIndex]
           |      +-keys: [a-1, b-1]

diff --git a/cases/plan/const_query.yaml b/cases/plan/const_query.yaml
@@ -39,3 +39,36 @@ cases:
     mode: request-unsupport
     sql: |
       select int(NULL) as c1, bigint(NULL) as c2, float(NULL) as c3, double(NULL) as c4, timestamp(NULL) as c5, date(NULL) as c6, string(NULL) as c7;
+
+  - id: map_data_type
+    mode: request-unsupport
+    desc: access map value with []operator
+    sql: |
+      select map(1, 2)[1]
+    expect:
+      node_tree_str: |
+        +-node[kQuery]: kQuerySelect
+          +-distinct_opt: false
+          +-where_expr: null
+          +-group_expr_list: null
+          +-having_expr: null
+          +-order_expr_list: null
+          +-limit: null
+          +-select_list[list]:
+          |  +-0:
+          |    +-node[kResTarget]
+          |      +-val:
+          |      |  map(1, 2)[1]
+          |      +-name: <nil>
+          +-tableref_list: []
+          +-window_list: []
+      plan_tree_str: |
+        +-[kQueryPlan]
+          +-[kProjectPlan]
+            +-table: <nil>
+            +-project_list_vec[list]:
+              +-[kProjectList]
+                +-projects on table [list]:
+                  +-[kProjectNode]
+                    +-[0]map(1, 2)[1]: map(1, 2)[1]
+         null
diff --git a/cases/plan/create.yaml b/cases/plan/create.yaml
@@ -163,12 +163,10 @@ cases:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: a
           |  |    +-column_type: int32
-          |  |    +-NOT NULL: 0
           |  +-1:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: b
           |  |    +-column_type: string
-          |  |    +-NOT NULL: 0
           |  +-2:
           |    +-node[kColumnIndex]
           |      +-keys: [a, b]
@@ -218,12 +216,10 @@ cases:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: a
           |  |    +-column_type: int16
-          |  |    +-NOT NULL: 0
           |  +-1:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: b
           |  |    +-column_type: float
-          |  |    +-NOT NULL: 0
           |  +-2:
           |    +-node[kColumnIndex]
           |      +-keys: [a]
@@ -274,12 +270,10 @@ cases:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: a
           |  |    +-column_type: int32
-          |  |    +-NOT NULL: 0
           |  +-1:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: b
           |  |    +-column_type: timestamp
-          |  |    +-NOT NULL: 0
           |  +-2:
           |    +-node[kColumnIndex]
           |      +-keys: [a]
@@ -627,12 +621,10 @@ cases:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: a
           |  |    +-column_type: int32
-          |  |    +-NOT NULL: 0
           |  +-1:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: b
           |  |    +-column_type: timestamp
-          |  |    +-NOT NULL: 0
           |  +-2:
           |    +-node[kColumnIndex]
           |      +-keys: [a]
@@ -685,33 +677,27 @@ cases:
           |  +-0:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: column1
-          |  |    +-column_type: int32
-          |  |    +-NOT NULL: 1
+          |  |    +-column_type: int32 NOT NULL
           |  +-1:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: column2
-          |  |    +-column_type: int16
-          |  |    +-NOT NULL: 1
+          |  |    +-column_type: int16 NOT NULL
           |  +-2:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: column5
-          |  |    +-column_type: string
-          |  |    +-NOT NULL: 1
+          |  |    +-column_type: string NOT NULL
           |  +-3:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: column6
-          |  |    +-column_type: string
-          |  |    +-NOT NULL: 1
+          |  |    +-column_type: string NOT NULL
           |  +-4:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: std_ts
-          |  |    +-column_type: timestamp
-          |  |    +-NOT NULL: 1
+          |  |    +-column_type: timestamp NOT NULL
           |  +-5:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: std_date
-          |  |    +-column_type: date
-          |  |    +-NOT NULL: 1
+          |  |    +-column_type: date NOT NULL
           |  +-6:
           |    +-node[kColumnIndex]
           |      +-keys: [column2]
@@ -743,33 +729,27 @@ cases:
           |  +-0:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: column1
-          |  |    +-column_type: int32
-          |  |    +-NOT NULL: 1
+          |  |    +-column_type: int32 NOT NULL
           |  +-1:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: column2
-          |  |    +-column_type: int16
-          |  |    +-NOT NULL: 1
+          |  |    +-column_type: int16 NOT NULL
           |  +-2:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: column5
-          |  |    +-column_type: string
-          |  |    +-NOT NULL: 1
+          |  |    +-column_type: string NOT NULL
           |  +-3:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: column6
-          |  |    +-column_type: string
-          |  |    +-NOT NULL: 1
+          |  |    +-column_type: string NOT NULL
           |  +-4:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: std_ts
-          |  |    +-column_type: timestamp
-          |  |    +-NOT NULL: 1
+          |  |    +-column_type: timestamp NOT NULL
           |  +-5:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: std_date
-          |  |    +-column_type: date
-          |  |    +-NOT NULL: 1
+          |  |    +-column_type: date NOT NULL
           |  +-6:
           |    +-node[kColumnIndex]
           |      +-keys: [column2]
@@ -796,17 +776,11 @@ cases:
           |  +-0:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: column1
-          |  |    +-column_type: int32
-          |  |    +-NOT NULL: 0
-          |  |    +-default_value:
-          |  |      +-expr[primary]
-          |  |        +-value: 1
-          |  |        +-type: int32
+          |  |    +-column_type: int32 DEFAULT 1
           |  +-1:
           |    +-node[kColumnDesc]
           |      +-column_name: column2
           |      +-column_type: int32
-          |      +-NOT NULL: 0
           +-table_option_list: []
   - id: 27
     desc: Column default value with explicit type
@@ -824,20 +798,11 @@ cases:
           |  +-0:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: column1
-          |  |    +-column_type: string
-          |  |    +-NOT NULL: 0
-          |  |    +-default_value:
-          |  |      +-expr[cast]
-          |  |        +-cast_type: string
-          |  |        +-expr:
-          |  |          +-expr[primary]
-          |  |            +-value: 1
-          |  |            +-type: int32
+          |  |    +-column_type: string DEFAULT string(1)
           |  +-1:
           |    +-node[kColumnDesc]
           |      +-column_name: column3
           |      +-column_type: int32
-          |      +-NOT NULL: 0
           +-table_option_list: []
   - id: 28
     desc: Create table with database.table
@@ -856,12 +821,10 @@ cases:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: column1
           |  |    +-column_type: string
-          |  |    +-NOT NULL: 0
           |  +-1:
           |    +-node[kColumnDesc]
           |      +-column_name: column3
           |      +-column_type: int32
-          |      +-NOT NULL: 0
           +-table_option_list: []
   - id: 29
     desc: create index with db name prefix
@@ -898,12 +861,10 @@ cases:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: column1
           |  |    +-column_type: int32
-          |  |    +-NOT NULL: 0
           |  +-1:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: column2
           |  |    +-column_type: timestamp
-          |  |    +-NOT NULL: 0
           |  +-2:
           |    +-node[kColumnIndex]
           |      +-keys: [column1]
@@ -934,12 +895,10 @@ cases:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: a
           |  |    +-column_type: int32
-          |  |    +-NOT NULL: 0
           |  +-1:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: b
           |  |    +-column_type: timestamp
-          |  |    +-NOT NULL: 0
           |  +-2:
           |    +-node[kColumnIndex]
           |      +-keys: [a]
@@ -1049,12 +1008,10 @@ cases:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: column1
           |  |    +-column_type: int32
-          |  |    +-NOT NULL: 0
           |  +-1:
           |  |  +-node[kColumnDesc]
           |  |    +-column_name: column2
           |  |    +-column_type: timestamp
-          |  |    +-NOT NULL: 0
           |  +-2:
           |    +-node[kColumnIndex]
           |      +-keys: [column1]
@@ -1068,3 +1025,45 @@ cases:
             +-0:
               +-node[kCompressType]
                 +-compress_type: snappy
+  - id: 35
+    desc: Create table with array & map type
+    sql: |
+      create table t1 (id int,
+        member ARRAY <STRING> NOT NULL,
+        attrs MAP <STRING, INT NOT NULL > NOT NULL);
+    expect:
+      node_tree_str: |
+        +-node[CREATE]
+          +-table: t1
+          +-IF NOT EXIST: 0
+          +-column_desc_list[list]:
+          |  +-0:
+          |  |  +-node[kColumnDesc]
+          |  |    +-column_name: id
+          |  |    +-column_type: int32
+          |  +-1:
+          |  |  +-node[kColumnDesc]
+          |  |    +-column_name: member
+          |  |    +-column_type: array<string> NOT NULL
+          |  +-2:
+          |    +-node[kColumnDesc]
+          |      +-column_name: attrs
+          |      +-column_type: map<string, int32 NOT NULL> NOT NULL
+          +-table_option_list: []
+      plan_tree_str: |
+        +-[kCreatePlan]
+          +-table: t1
+          +-column_desc_list[list]:
+          |  +-0:
+          |  |  +-node[kColumnDesc]
+          |  |    +-column_name: id
+          |  |    +-column_type: int32
+          |  +-1:
+          |  |  +-node[kColumnDesc]
+          |  |    +-column_name: member
+          |  |    +-column_type: array<string> NOT NULL
+          |  +-2:
+          |    +-node[kColumnDesc]
+          |      +-column_name: attrs
+          |      +-column_type: map<string, int32 NOT NULL> NOT NULL
+          +-table_option_list: []
diff --git a/cases/plan/simple_query.yaml b/cases/plan/simple_query.yaml
@@ -644,3 +644,4 @@ cases:
                   +-[kTablePlan]
                     +-table: t
               +-alias: t1
+
Original file line number	Diff line number	Diff line change
Expand Up		@@ -644,3 +644,4 @@ cases:
		+-[kTablePlan]
		+-table: t
		+-alias: t1