Pongzu/works

.gitignore

@@ -4,5 +4,4 @@ vendor
 
 
 # for GoLand user
-.idea/**/workspace.xml
-.idea/**/tasks.xmlq
+.idea

README.md

@@ -85,7 +85,7 @@ ORDER BY
 ## Installation
 
 ```bash
-run git clone and go build -o sqlfmt 
+go get github.com/kanmu/go-sqlfmt/cmd/sqlfmt
 ```
 ## Usage
 

go.mod

@@ -1,5 +1,9 @@
 module github.com/kanmu/go-sqlfmt
 
-go 1.13
+go 1.14
 
-require github.com/pkg/errors v0.8.1
+require (
+	github.com/pkg/errors v0.9.1
+	github.com/stretchr/testify v1.6.1
+	golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2
+)

go.sum

@@ -1,2 +1,30 @@
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2 h1:FD4wDsP+CQUqh2V12OBOt90pLHVToe58P++fUu3ggV4=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

sqlfmt.go

@@ -45,8 +45,7 @@ func visitFile(path string, info os.FileInfo, err error) error {
 		err = processFile(path, nil, os.Stdout)
 	}
 	if err != nil {
-		processError(errors.Wrap(err, "visit file failed"))
-
+		log.Fatal(err)
 	}
 	return nil
 }
@@ -111,7 +110,7 @@ func sqlfmtMain() {
 			log.Fatal("can not use -w while using pipeline")
 		}
 		if err := processFile("<standard input>", os.Stdin, os.Stdout); err != nil {
-			processError(errors.Wrap(err, "processFile failed"))
+			log.Fatal(err)
 		}
 		return
 	}
@@ -120,18 +119,18 @@ func sqlfmtMain() {
 		path := flag.Arg(i)
 		switch dir, err := os.Stat(path); {
 		case err != nil:
-			processError(err)
+			log.Fatal(err)
 		case dir.IsDir():
 			walkDir(path)
 		default:
 			info, err := os.Stat(path)
 			if err != nil {
-				processError(err)
+				log.Fatal(err)
 			}
 			if isGoFile(info) {
 				err = processFile(path, nil, os.Stdout)
 				if err != nil {
-					processError(err)
+					log.Fatal(err)
 				}
 			}
 		}
@@ -169,12 +168,3 @@ func diff(b1, b2 []byte) (data []byte, err error) {
 	}
 	return
 }
-
-func processError(err error) {
-	switch err.(type) {
-	case *sqlfmt.FormatError:
-		log.Println(err)
-	default:
-		log.Fatal(err)
-	}
-}

sqlfmt/ast.go

@@ -1,50 +1,78 @@
 package sqlfmt
 
 import (
-	"fmt"
 	"go/ast"
 	"go/token"
+	"golang.org/x/tools/go/ast/astutil"
 	"log"
-	"strings"
-
-	"github.com/kanmu/go-sqlfmt/sqlfmt/parser/group"
-)
-
-// sqlfmt retrieves all strings from "Query" and "QueryRow" and "Exec" functions in .go file
-const (
-	QUERY    = "Query"
-	QUERYROW = "QueryRow"
-	EXEC     = "Exec"
 )
 
-// replaceAst replace ast node with formatted SQL statement
-func replaceAst(f *ast.File, fset *token.FileSet, options *Options) {
+// Replace replace ast node with formatted SQL statement
+func Replace(f *ast.File, options *Options) {
 	ast.Inspect(f, func(n ast.Node) bool {
-		if x, ok := n.(*ast.CallExpr); ok {
-			if fun, ok := x.Fun.(*ast.SelectorExpr); ok {
-				funcName := fun.Sel.Name
-				if funcName == QUERY || funcName == QUERYROW || funcName == EXEC {
-					// not for parsing url.Query
-					if len(x.Args) > 0 {
-						if arg, ok := x.Args[0].(*ast.BasicLit); ok {
-							sqlStmt := arg.Value
-							if !strings.HasPrefix(sqlStmt, "`") {
-								return true
-							}
-							src := strings.Trim(sqlStmt, "`")
-							res, err := Format(src, options)
-							if err != nil {
-								log.Println(fmt.Sprintf("Format failed at %s: %v", fset.Position(arg.Pos()), err))
-								return true
-							}
-							// FIXME
-							// more elegant
-							arg.Value = "`" + res + strings.Repeat(group.WhiteSpace, options.Distance) + "`"
-						}
-					}
-				}
+		sql, found := findSQL(n)
+		if found {
+			res, err := Format(sql, options)
+			if err != nil {
+				log.Println(err)
+
+				// XXX for debugging
+				log.Println(sql)
+			} else {
+				replace(n, res)
 			}
 		}
 		return true
 	})
 }
+
+func replace(n ast.Node, sql string) {
+	replaceFunc := func(cr *astutil.Cursor) bool {
+		cr.Replace(&ast.BasicLit{
+			Kind:  token.STRING,
+			Value: sql,
+		})
+		return true
+	}
+	astutil.Apply(n, replaceFunc, nil)
+}
+
+func findSQL(n ast.Node) (string, bool) {
+	ce, ok := n.(*ast.CallExpr)
+	if !ok {
+		return "", false
+	}
+	se, ok := ce.Fun.(*ast.SelectorExpr)
+	if !ok {
+		return "", false
+	}
+
+	// check func name
+	ok = validateFuncName(se.Sel.Name)
+	if !ok {
+		return "", false
+	}
+
+	// check length of the parameter
+	// this is not for parsing "url.Query()"
+	// FIXME: very adhoc
+	if len(ce.Args) == 0 {
+		return "", false
+	}
+
+	// SQL statement should appear in the first parameter
+	arg, ok := ce.Args[0].(*ast.BasicLit)
+	if !ok {
+		return "", false
+	}
+	return arg.Value, true
+}
+
+// go-sqlfmt only formats the value passed as the parameter of "Exec(string, ... any type)", "Query(string, ... any type)" and "QueryRow(string, ... any type)"
+func validateFuncName(name string) bool {
+	switch name {
+	case "Exec", "Query", "QueryRow":
+		return true
+	}
+	return false
+}

sqlfmt/format.go

@@ -4,31 +4,42 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
-	"strings"
-
 	"github.com/kanmu/go-sqlfmt/sqlfmt/lexer"
 	"github.com/kanmu/go-sqlfmt/sqlfmt/parser"
 	"github.com/kanmu/go-sqlfmt/sqlfmt/parser/group"
 	"github.com/pkg/errors"
+	"strings"
 )
 
-// Format formats src in 3 steps
-// 1: tokenize src
-// 2: parse tokens by SQL clause group
-// 3: for each clause group (Reindenter), add indentation or new line in the correct position
+// Format parse tokens, and build
 func Format(src string, options *Options) (string, error) {
-	t := lexer.NewTokenizer(src)
-	tokens, err := t.GetTokens()
+	// 最初と最後を取り除く的な事をする ` or "
+	tokens, err := lexer.Tokenize(src)
 	if err != nil {
 		return src, errors.Wrap(err, "Tokenize failed")
 	}
 
-	rs, err := parser.ParseTokens(tokens)
+	exprs, err := parser.ParseTokens(tokens)
 	if err != nil {
 		return src, errors.Wrap(err, "ParseTokens failed")
 	}
 
-	res, err := getFormattedStmt(rs, options.Distance)
+	/*
+	        var buf &bytes.Buffer{}
+	        for _, expr := range exprs {
+	            stmt, err = expr.Build(options)
+	            // err
+	            buf.WriteString(stmt)
+	        }
+
+	        res := buf.String()
+			if bytes.Compare(src, res) {
+	            // 崩れたよ！って教えるやつ
+				return err
+			}
+			return res
+	 */
+	res, err := getFormattedStmt(exprs, options.Distance)
 	if err != nil {
 		return src, errors.Wrap(err, "getFormattedStmt failed")
 	}

sqlfmt/lexer/token.go

@@ -90,6 +90,9 @@ const (
 
 	QUOTEAREA
 	SURROUNDING
+	COLON
+	DOUBLECOLON
+	SPACE
 )
 
 // TokenType is an alias type that represents a kind of token
@@ -110,7 +113,7 @@ func (t Token) IncrementIndentLevel(lev int) {}
 // end keywords of each clause
 var (
 	EndOfSelect      = []TokenType{FROM, UNION, EOF}
-	EndOfCase        = []TokenType{END}
+	EndOfCase        = []TokenType{END, EOF}
 	EndOfFrom        = []TokenType{WHERE, INNER, OUTER, LEFT, RIGHT, JOIN, NATURAL, CROSS, ORDER, GROUP, UNION, OFFSET, LIMIT, FETCH, EXCEPT, INTERSECT, EOF, ENDPARENTHESIS}
 	EndOfJoin        = []TokenType{WHERE, ORDER, GROUP, LIMIT, OFFSET, FETCH, ANDGROUP, ORGROUP, LEFT, RIGHT, INNER, OUTER, NATURAL, CROSS, UNION, EXCEPT, INTERSECT, EOF, ENDPARENTHESIS}
 	EndOfWhere       = []TokenType{GROUP, ORDER, LIMIT, OFFSET, FETCH, ANDGROUP, OR, UNION, EXCEPT, INTERSECT, RETURNING, EOF, ENDPARENTHESIS}
@@ -120,16 +123,17 @@ var (
 	EndOfHaving      = []TokenType{LIMIT, OFFSET, FETCH, ORDER, UNION, EXCEPT, INTERSECT, EOF, ENDPARENTHESIS}
 	EndOfOrderBy     = []TokenType{LIMIT, FETCH, OFFSET, UNION, EXCEPT, INTERSECT, EOF, ENDPARENTHESIS}
 	EndOfLimitClause = []TokenType{UNION, EXCEPT, INTERSECT, EOF, ENDPARENTHESIS}
-	EndOfParenthesis = []TokenType{ENDPARENTHESIS}
-	EndOfTieClause   = []TokenType{SELECT}
+	EndOfParenthesis = []TokenType{ENDPARENTHESIS, EOF}
+	// 微妙
+	EndOfTieClause   = []TokenType{SELECT, EOF}
 	EndOfUpdate      = []TokenType{WHERE, SET, RETURNING, EOF}
 	EndOfSet         = []TokenType{WHERE, RETURNING, EOF}
 	EndOfReturning   = []TokenType{EOF}
 	EndOfDelete      = []TokenType{WHERE, FROM, EOF}
 	EndOfInsert      = []TokenType{VALUES, EOF}
 	EndOfValues      = []TokenType{UPDATE, RETURNING, EOF}
-	EndOfFunction    = []TokenType{ENDPARENTHESIS}
-	EndOfTypeCast    = []TokenType{ENDPARENTHESIS}
+	EndOfFunction    = []TokenType{ENDPARENTHESIS, EOF}
+	EndOfTypeCast    = []TokenType{ENDPARENTHESIS, EOF}
 	EndOfLock        = []TokenType{EOF}
 	EndOfWith        = []TokenType{EOF}
 )
@@ -186,4 +190,4 @@ func (t Token) IsNeedNewLineBefore() bool {
 // IsKeyWordInSelect returns true if token is a keyword in select group
 func (t Token) IsKeyWordInSelect() bool {
 	return t.Type == SELECT || t.Type == EXISTS || t.Type == DISTINCT || t.Type == DISTINCTROW || t.Type == INTO || t.Type == AS || t.Type == GROUP || t.Type == ORDER || t.Type == BY || t.Type == ON || t.Type == RETURNING || t.Type == SET || t.Type == UPDATE
-}
+}