Skip to content

Commit

Permalink
优化增强解析collate信息 alibaba#5822
Browse files Browse the repository at this point in the history 
优化增强解析collate信息 alibaba#5822
  • Loading branch information
@lizongbo
lizongbo committed May 4, 2024
1 parent 2496258 commit 1c00d90
Show file tree
Hide file tree
Showing 9 changed files with 273 additions and 31 deletions.
14 changes: 13 additions & 1 deletion core/src/main/java/com/alibaba/druid/sql/ast/expr/SQLCharExpr.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@
public class SQLCharExpr extends SQLTextLiteralExpr implements SQLValuableExpr, Comparable<SQLCharExpr> {
public static final SQLDataType DATA_TYPE = new SQLCharacterDataType("char");

protected String collate;

public SQLCharExpr() {
}

Expand All @@ -40,6 +42,14 @@ public SQLCharExpr(String text, SQLObject parent) {
this.parent = parent;
}

public String getCollate() {
return collate;
}

public void setCollate(String collate) {
this.collate = collate;
}

public void output(StringBuilder buf) {
this.accept(new SQLASTOutputVisitor(buf));
}
Expand All @@ -59,7 +69,9 @@ public String toString() {
}

public SQLCharExpr clone() {
return new SQLCharExpr(this.text);
SQLCharExpr expr = new SQLCharExpr(this.text);
expr.setCollate(collate);
return expr;
}

public SQLDataType computeDataType() {
Expand Down
18 changes: 16 additions & 2 deletions core/src/main/java/com/alibaba/druid/sql/ast/expr/SQLIdentifierExpr.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ public final class SQLIdentifierExpr extends SQLExprImpl implements SQLName, Com
private SQLObject resolvedColumn;
private SQLObject resolvedOwnerObject;

protected String collate;

public SQLIdentifierExpr() {
}

Expand Down Expand Up @@ -69,6 +71,14 @@ public void setName(String name) {
}
}

public String getCollate() {
return collate;
}

public void setCollate(String collate) {
this.collate = collate;
}

public long nameHashCode64() {
return hashCode64();
}
Expand All @@ -77,7 +87,11 @@ public long nameHashCode64() {
public long hashCode64() {
if (hashCode64 == 0
&& name != null) {
hashCode64 = FnvHash.hashCode64(name);
if (collate != null) {
hashCode64 = FnvHash.hashCode64(name + collate);
} else {
hashCode64 = FnvHash.hashCode64(name);
}
}
return hashCode64;
}
Expand Down Expand Up @@ -124,7 +138,7 @@ public SQLIdentifierExpr clone() {
if (hint != null) {
x.hint = hint.clone();
}

x.collate = collate;
return x;
}

Expand Down
27 changes: 25 additions & 2 deletions core/src/main/java/com/alibaba/druid/sql/dialect/mysql/parser/MySqlExprParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1890,7 +1890,19 @@ public boolean parseTableOptions(List<SQLAssignItem> assignItems, SQLDDLStatemen
if (lexer.token() == Token.EQ) {
lexer.nextToken();
}
assignItem = new SQLAssignItem(new SQLIdentifierExpr("CHARACTER SET"), expr());
if (lexer.token() == Token.IDENTIFIER) {
assignItem = new SQLAssignItem(new SQLIdentifierExpr("CHARACTER SET"), new SQLIdentifierExpr(lexer.stringVal()));
lexer.nextToken();
} else if (lexer.token() == Token.LITERAL_ALIAS || lexer.token() == Token.LITERAL_CHARS) {
String charset = lexer.stringVal();
if (charset.startsWith("\"")) {
charset = charset.substring(1, charset.length() - 1);
}
assignItem = new SQLAssignItem(new SQLIdentifierExpr("CHARACTER SET"), new SQLCharExpr(charset));
lexer.nextToken();
} else {
assignItem = new SQLAssignItem(new SQLIdentifierExpr("CHARACTER SET"), expr());
}
} else if (lexer.identifierEquals(FnvHash.Constants.COLLATE)) {
lexer.nextToken();
if (lexer.token() == Token.EQ) {
Expand All @@ -1904,8 +1916,19 @@ public boolean parseTableOptions(List<SQLAssignItem> assignItems, SQLDDLStatemen
accept(Token.SET);
if (lexer.token() == Token.EQ) {
lexer.nextToken();
} if (lexer.token() == Token.IDENTIFIER) {
assignItem = new SQLAssignItem(new SQLIdentifierExpr("CHARACTER SET"), new SQLIdentifierExpr(lexer.stringVal()));
lexer.nextToken();
} else if (lexer.token() == Token.LITERAL_ALIAS || lexer.token() == Token.LITERAL_CHARS) {
String charset = lexer.stringVal();
if (charset.startsWith("\"")) {
charset = charset.substring(1, charset.length() - 1);
}
assignItem = new SQLAssignItem(new SQLIdentifierExpr("CHARACTER SET"), new SQLCharExpr(charset));
lexer.nextToken();
} else {
assignItem = new SQLAssignItem(new SQLIdentifierExpr("CHARACTER SET"), expr());
}
assignItem = new SQLAssignItem(new SQLIdentifierExpr("CHARACTER SET"), expr());
} else if (hash == FnvHash.Constants.DATA ||
lexer.token() == Token.INDEX) {
// {DATA|INDEX} DIRECTORY [=] 'absolute path to directory'
Expand Down
21 changes: 20 additions & 1 deletion core/src/main/java/com/alibaba/druid/sql/dialect/postgresql/visitor/PGOutputVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2647,7 +2647,26 @@ public boolean visit(SQLCharExpr x, boolean parameterized) {
if (x instanceof PGCharExpr && ((PGCharExpr) x).isCSytle()) {
print('E');
}
printChars(x.getText());
if (x.getCollate() != null) {
String collate = x.getCollate();
if (x.isParenthesized()) {
print('(');
}
printChars(x.getText());
print(" COLLATE ");
if (collate.startsWith("'") || collate.endsWith("\"")) {
print(collate);
} else {
print('\'');
print(collate);
print('\'');
}
if (x.isParenthesized()) {
print(')');
}
} else {
printChars(x.getText());
}

return false;
}
Expand Down
42 changes: 36 additions & 6 deletions core/src/main/java/com/alibaba/druid/sql/parser/SQLExprParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import com.alibaba.druid.sql.parser.Lexer.SavePoint;
import com.alibaba.druid.util.FnvHash;
import com.alibaba.druid.util.HexBin;
import com.alibaba.druid.util.JdbcUtils;
import com.alibaba.druid.util.MySqlUtils;
import com.alibaba.druid.util.StringUtils;

Expand Down Expand Up @@ -441,8 +442,18 @@ public SQLExpr primary() {
}

lexer.nextToken();

if (hash_lower == FnvHash.Constants.TRY_CAST) {
if (lexer.identifierEquals("COLLATE")) {
acceptIdentifier("COLLATE");
String collateValue = lexer.stringVal();
if (lexer.token == Token.IDENTIFIER || lexer.token == Token.LITERAL_ALIAS || lexer.token == Token.LITERAL_CHARS) {
SQLIdentifierExpr identifierExpr = new SQLIdentifierExpr(ident);
identifierExpr.setCollate(collateValue);
lexer.nextToken();
sqlExpr = identifierExpr;
} else {
throw new ParserException("syntax error. " + lexer.info());
}
} else if (hash_lower == FnvHash.Constants.TRY_CAST) {
accept(Token.LPAREN);
SQLCastExpr cast = new SQLCastExpr();
cast.setTry(true);
Expand Down Expand Up @@ -607,7 +618,26 @@ public SQLExpr primary() {
break;
case LITERAL_CHARS: {
sqlExpr = new SQLCharExpr(lexer.stringVal());

if (JdbcUtils.isPgsqlDbType(dbType)) {
Lexer.SavePoint savePoint = lexer.mark();
lexer.nextToken();
if (lexer.token() == Token.IDENTIFIER) {
String collate = lexer.stringVal();
if (collate.equalsIgnoreCase("collate")) {
lexer.nextToken();
String collateValue = lexer.stringVal();
if (lexer.token == Token.IDENTIFIER || lexer.token == Token.LITERAL_ALIAS || lexer.token == Token.LITERAL_CHARS) {
((SQLCharExpr) sqlExpr).setCollate(lexer.stringVal());
} else {
throw new ParserException("syntax error. " + lexer.info());
}
} else {
lexer.reset(savePoint);
}
} else {
lexer.reset(savePoint);
}
}
if (DbType.mysql == dbType) {
lexer.nextTokenValue();

Expand Down Expand Up @@ -5899,8 +5929,8 @@ public SQLSelectItem parseSelectItem() {
expr = new SQLIdentifierExpr(ident);
}
} else if (lexer.identifierEquals(FnvHash.Constants.COLLATE)
&& dbType == DbType.mysql
&& lexer.stringVal().charAt(0) != '`'
&& (JdbcUtils.isMysqlDbType(dbType) || JdbcUtils.isPgsqlDbType(dbType))
&& lexer.stringVal().charAt(0) != '`'
) {
lexer.nextToken();
String collate = lexer.stringVal();
Expand All @@ -5909,7 +5939,7 @@ public SQLSelectItem parseSelectItem() {
SQLBinaryOpExpr binaryExpr = new SQLBinaryOpExpr(
new SQLIdentifierExpr(ident),
SQLBinaryOperator.COLLATE,
new SQLIdentifierExpr(collate), DbType.mysql
new SQLIdentifierExpr(collate), dbType
);

expr = binaryExpr;
Expand Down
10 changes: 10 additions & 0 deletions core/src/main/java/com/alibaba/druid/sql/parser/SQLSelectParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -662,6 +662,16 @@ public void parseWhere(SQLSelectQueryBlock queryBlock) {
} else {
identExpr = new SQLIdentifierExpr(ident, hash_lower);
}
} else if (lexer.identifierEquals("COLLATE")) {
acceptIdentifier("COLLATE");
String collateValue = lexer.stringVal();
if (lexer.token == Token.IDENTIFIER || lexer.token == Token.LITERAL_ALIAS || lexer.token == Token.LITERAL_CHARS) {
identExpr = new SQLIdentifierExpr(ident);
((SQLIdentifierExpr) identExpr).setCollate(collateValue);
lexer.nextToken();
} else {
throw new ParserException("syntax error. " + lexer.info());
}
} else {
identExpr = new SQLIdentifierExpr(ident, hash_lower);
}
Expand Down
26 changes: 11 additions & 15 deletions core/src/main/java/com/alibaba/druid/sql/visitor/SQLASTOutputVisitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1059,23 +1059,9 @@ private void visitBinaryLeft(SQLExpr left, SQLBinaryOperator op) {
}
} else if (left instanceof SQLBetweenExpr) {
SQLBetweenExpr betweenExpr = (SQLBetweenExpr) left;
boolean quote;
if (betweenExpr.isNot()) {
quote = op.priority <= SQLBinaryOperator.Equality.priority;
} else {
quote = op.priority < SQLBinaryOperator.Equality.priority;
}
// if (quote) {
// print("(aaa");
// }
visit(betweenExpr);
// if (quote) {
// print(')');
// }
} else if (left instanceof SQLNotExpr) {
// print('(');
printExpr(left);
// print(')');
} else if (left instanceof SQLUnaryExpr) {
SQLUnaryExpr unary = (SQLUnaryExpr) left;

Expand Down Expand Up @@ -1436,6 +1422,17 @@ public boolean visit(SQLIdentifierExpr x) {
print('(');
}
printName0(x.getName());
if (x.getCollate() != null) {
String collate = x.getCollate();
print(" COLLATE ");
if (collate.startsWith("'") || collate.endsWith("\"")) {
print(collate);
} else {
print('\'');
print(collate);
print('\'');
}
}
if (x.isParenthesized()) {
print(')');
}
Expand Down Expand Up @@ -5800,7 +5797,6 @@ public boolean visit(SQLAlterTableStatement x) {
if (x.isNotClustered()) {
print0(ucase ? " NOT CLUSTERED" : " not clustered");
}

return false;
}

Expand Down
42 changes: 38 additions & 4 deletions core/src/test/java/com/alibaba/druid/bvt/sql/mysql/alterTable/MySqlAlterTableTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,22 @@
*/
package com.alibaba.druid.bvt.sql.mysql.alterTable;

import java.util.List;

import com.alibaba.druid.DbType;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLStatement;
import com.alibaba.druid.sql.dialect.mysql.parser.MySqlStatementParser;
import com.alibaba.druid.sql.parser.SQLParserUtils;
import com.alibaba.druid.sql.parser.SQLStatementParser;
import com.alibaba.druid.sql.parser.Token;

import junit.framework.TestCase;
import org.junit.Assert;


public class MySqlAlterTableTest extends TestCase {

public void test_alter_0() throws Exception {
String sql = "ALTER TABLE `test`.`tb1` CHANGE COLUMN `fname` `fname1` VARCHAR(45) NULL DEFAULT NULL ;";
MySqlStatementParser parser = new MySqlStatementParser(sql);
Expand All @@ -40,7 +47,8 @@ public void test_alter_1() throws Exception {
parser.match(Token.EOF);

Assert.assertEquals("ALTER TABLE `test`.`tb1`\n\tCHARACTER SET = utf8 COLLATE = utf8_general_ci;", SQLUtils.toMySqlString(stmt));
Assert.assertEquals("alter table `test`.`tb1`\n\tcharacter set = utf8 collate = utf8_general_ci;", SQLUtils.toMySqlString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
Assert.assertEquals("alter table `test`.`tb1`\n\tcharacter set = utf8 collate = utf8_general_ci;",
SQLUtils.toMySqlString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
}

public void test_alter_2() throws Exception {
Expand Down Expand Up @@ -81,13 +89,39 @@ public void test_alter_5() throws Exception {

public void test_alter_6() throws Exception {
String sql = "ALTER TABLE `test`.`tb1` DEFAULT CHARACTER SET utf8 COLLATE = utf8_general_ci ;";

MySqlStatementParser parser = new MySqlStatementParser(sql);
SQLStatement stmt = parser.parseStatementList().get(0);
parser.match(Token.EOF);

System.out.println(stmt.toString());
//System.out.println(SQLUtils.toMySqlString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
Assert.assertEquals("ALTER TABLE `test`.`tb1`\n\tCHARACTER SET = utf8 COLLATE utf8_general_ci;", SQLUtils.toMySqlString(stmt));
Assert.assertEquals("alter table `test`.`tb1`\n\tcharacter set = utf8 collate utf8_general_ci;", SQLUtils.toMySqlString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
Assert.assertEquals("ALTER TABLE `test`.`tb1`\n\tCHARACTER SET = utf8 COLLATE = utf8_general_ci;", SQLUtils.toMySqlString(stmt));
Assert.assertEquals("alter table `test`.`tb1`\n\tcharacter set = utf8 collate = utf8_general_ci;",
SQLUtils.toMySqlString(stmt, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION));
}

public void test_alter_7() throws Exception {
for (String sql : new String[]{
"ALTER TABLE \n"
+ " `test` CHARACTER SET utf8mb4 COLLATE utf8mb4_bin;",
"ALTER TABLE \n"
+ " `test` CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_bin';",
"ALTER TABLE \n"
+ " `test` CHARACTER SET \"utf8mb4\" COLLATE \"utf8mb4_bin\";",
"ALTER TABLE \n"
+ " `test` default CHARACTER SET utf8mb4 COLLATE =utf8mb4_bin",
}) {
System.out.println("原始的sql===" + sql);
SQLStatementParser parser1 = SQLParserUtils.createSQLStatementParser(sql, DbType.mysql);
List<SQLStatement> statementList1 = parser1.parseStatementList();
String sqleNew = statementList1.get(0).toString();
System.out.println("生成的sql===" + sqleNew);
SQLStatementParser parser2 = SQLParserUtils.createSQLStatementParser(sqleNew, DbType.mariadb);
List<SQLStatement> statementList2 = parser2.parseStatementList();
String sqleNew2 = statementList2.get(0).toString();
System.out.println("再次解析生成的sql===" + sqleNew2);
assertEquals(sqleNew, sqleNew2);
}

}
}
Loading

0 comments on commit 1c00d90

Please sign in to comment.