From 2db1f2b944070d7a4e1634a6a84bb5a0ad3ffb1b Mon Sep 17 00:00:00 2001 From: jefjin <64047032@qq.com> Date: Wed, 22 Jun 2016 11:45:00 +0800 Subject: [PATCH 1/3] support range shard table based on default value support range shard table based on default value --- config/config.go | 1 + etc/ks.yaml | 1 + proxy/router/numkey.go | 6 +++--- proxy/router/router.go | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/config/config.go b/config/config.go index d372d718..a2d19ba2 100644 --- a/config/config.go +++ b/config/config.go @@ -66,6 +66,7 @@ type ShardConfig struct { Nodes []string `yaml:"nodes"` Locations []int `yaml:"locations"` Type string `yaml:"type"` + TableRowBase int `yaml:"table_row_base"` TableRowLimit int `yaml:"table_row_limit"` DateRange []string `yaml:"date_range"` } diff --git a/etc/ks.yaml b/etc/ks.yaml index 63ff5a07..99cf0c25 100755 --- a/etc/ks.yaml +++ b/etc/ks.yaml @@ -87,6 +87,7 @@ schema : type: range nodes: [node1, node2] locations: [4,4] + table_row_base: 2000 table_row_limit: 10000 - table: test_shard_year diff --git a/proxy/router/numkey.go b/proxy/router/numkey.go index 4c9e21b9..5d2dfb0a 100644 --- a/proxy/router/numkey.go +++ b/proxy/router/numkey.go @@ -47,7 +47,7 @@ func (kr NumKeyRange) String() string { return fmt.Sprintf("{Start: %d, End: %d}", kr.Start, kr.End) } -func ParseNumSharding(Locations []int, TableRowLimit int) ([]NumKeyRange, error) { +func ParseNumSharding(Locations []int, TableRowLimit int, TableRowBase int) ([]NumKeyRange, error) { tableCount := 0 length := len(Locations) @@ -57,8 +57,8 @@ func ParseNumSharding(Locations []int, TableRowLimit int) ([]NumKeyRange, error) ranges := make([]NumKeyRange, tableCount) for i := 0; i < tableCount; i++ { - ranges[i].Start = int64(i * TableRowLimit) - ranges[i].End = int64((i + 1) * TableRowLimit) + ranges[i].Start = int64(TableRowBase + i*TableRowLimit) + ranges[i].End = int64(TableRowBase + (i+1)*TableRowLimit) } return ranges, nil } diff --git a/proxy/router/router.go b/proxy/router/router.go index 5f1e1cc3..9cac3769 100644 --- a/proxy/router/router.go +++ b/proxy/router/router.go @@ -233,7 +233,7 @@ func parseShard(r *Rule, cfg *config.ShardConfig) error { case HashRuleType: r.Shard = &HashShard{ShardNum: len(r.TableToNode)} case RangeRuleType: - rs, err := ParseNumSharding(cfg.Locations, cfg.TableRowLimit) + rs, err := ParseNumSharding(cfg.Locations, cfg.TableRowLimit, cfg.TableRowBase) if err != nil { return err } From 7f28ca59d9f128bee5ef4a11ee2dd4f4149bbd2e Mon Sep 17 00:00:00 2001 From: jefjin <64047032@qq.com> Date: Thu, 14 Jul 2016 14:46:01 +0800 Subject: [PATCH 2/3] add build script --- DEBIAN/conffiles | 1 + DEBIAN/control | 4 +++ DEBIAN/mkrootfs | 14 ++++++++ DEBIAN/postinst | 2 ++ DEBIAN/postrm | 2 ++ DEBIAN/preinst | 2 ++ DEBIAN/prerm | 2 ++ Makefile | 22 ++++++------ Makefile.in | 15 ++++++++ build.sh | 14 ++++++++ cmd/kingshard/main.go | 23 ++++++++++++ config/config.go | 2 ++ dev.sh | 4 --- etc/ks.yaml | 79 ++++++++++++------------------------------ etc/unshard.yaml | 2 ++ proxy/router/numkey.go | 6 ++-- proxy/router/router.go | 32 +++++++++++++++-- sqlparser/sql.go | 4 +-- ucdbproxy.sh | 47 +++++++++++++++++++++++++ 19 files changed, 198 insertions(+), 79 deletions(-) create mode 100644 DEBIAN/conffiles create mode 100644 DEBIAN/control create mode 100644 DEBIAN/mkrootfs create mode 100644 DEBIAN/postinst create mode 100644 DEBIAN/postrm create mode 100644 DEBIAN/preinst create mode 100644 DEBIAN/prerm create mode 100644 Makefile.in create mode 100644 build.sh create mode 100644 ucdbproxy.sh diff --git a/DEBIAN/conffiles b/DEBIAN/conffiles new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/DEBIAN/conffiles @@ -0,0 +1 @@ + diff --git a/DEBIAN/control b/DEBIAN/control new file mode 100644 index 00000000..f739f14d --- /dev/null +++ b/DEBIAN/control @@ -0,0 +1,4 @@ +Package: +Version: +Maintainer: +Description: UC DB Proxy Server. diff --git a/DEBIAN/mkrootfs b/DEBIAN/mkrootfs new file mode 100644 index 00000000..fc11291b --- /dev/null +++ b/DEBIAN/mkrootfs @@ -0,0 +1,14 @@ +#!/bin/sh +#make root file system for debian pakcage +#EnvirmentVar: DEB_ROOTFS, DEB_CTRLDIR + +mkdir -p -m 755 "${DEB_ROOTFS}" || exit 1 +mkdir -p -m 755 "${DEB_ROOTFS}"/${UC_ROOT}/sbin || exit 2 +install -m 755 "bin/ucdbproxy" "${DEB_ROOTFS}"/${UC_ROOT}/sbin || exit 3 +mkdir -p -m 755 "${DEB_ROOTFS}/${UC_ROOT}/etc" || exit 4 +mkdir -p -m 755 "${DEB_ROOTFS}/${UC_ROOT}/etc/init.d" || exit 5 +mkdir -p -m 755 "${DEB_ROOTFS}/${UC_ROOT}/share" || exit 6 +install -m 644 "etc/ks.yaml" "${DEB_ROOTFS}/${UC_ROOT}/etc" || exit 7 +install -m 744 "ucdbproxy.sh" "${DEB_ROOTFS}/${UC_ROOT}/etc/init.d" || exit 8 +chown -R root:root "${DEB_ROOTFS}/${UC_ROOT}" || exit 9 + diff --git a/DEBIAN/postinst b/DEBIAN/postinst new file mode 100644 index 00000000..13f47935 --- /dev/null +++ b/DEBIAN/postinst @@ -0,0 +1,2 @@ +#!/bin/sh + diff --git a/DEBIAN/postrm b/DEBIAN/postrm new file mode 100644 index 00000000..13f47935 --- /dev/null +++ b/DEBIAN/postrm @@ -0,0 +1,2 @@ +#!/bin/sh + diff --git a/DEBIAN/preinst b/DEBIAN/preinst new file mode 100644 index 00000000..13f47935 --- /dev/null +++ b/DEBIAN/preinst @@ -0,0 +1,2 @@ +#!/bin/sh + diff --git a/DEBIAN/prerm b/DEBIAN/prerm new file mode 100644 index 00000000..13f47935 --- /dev/null +++ b/DEBIAN/prerm @@ -0,0 +1,2 @@ +#!/bin/sh + diff --git a/Makefile b/Makefile index 7b82518e..cb9d2382 100755 --- a/Makefile +++ b/Makefile @@ -1,15 +1,13 @@ -all: build +TOPDIR=../../../../../ +include ${TOPDIR}/build-scripts/Macros.mk + +DEB_PACKAGE=ucdbproxy +DEB_VERSION=1.0.0.0 +#DEB_MAINTAINER= +#DEB_DESCRIPTION= +DEB_DEPENDS= + +include ${TOPDIR}/build-scripts/deb.mk -build: kingshard -kingshard: - go tool yacc -o ./sqlparser/sql.go ./sqlparser/sql.y - gofmt -w ./sqlparser/sql.go - @bash genver.sh - go build -o ./bin/kingshard ./cmd/kingshard -clean: - @rm -rf bin - @rm -f ./sqlparser/y.output ./sqlparser/sql.go -test: - go test ./go/... -race diff --git a/Makefile.in b/Makefile.in new file mode 100644 index 00000000..7b82518e --- /dev/null +++ b/Makefile.in @@ -0,0 +1,15 @@ +all: build + +build: kingshard + +kingshard: + go tool yacc -o ./sqlparser/sql.go ./sqlparser/sql.y + gofmt -w ./sqlparser/sql.go + @bash genver.sh + go build -o ./bin/kingshard ./cmd/kingshard +clean: + @rm -rf bin + @rm -f ./sqlparser/y.output ./sqlparser/sql.go + +test: + go test ./go/... -race diff --git a/build.sh b/build.sh new file mode 100644 index 00000000..fd79ebe5 --- /dev/null +++ b/build.sh @@ -0,0 +1,14 @@ +#!/bin/bash +export GOBIN=/usr/local/go/bin +export GOARCH=amd64 +export GOROOT=/usr/local/go +export GOOS=linux + +source ./dev.sh + +go tool yacc -o ./sqlparser/sql.go ./sqlparser/sql.y +gofmt -w ./sqlparser/sql.go +source genver.sh +go build -o ./bin/ucdbproxy ./cmd/kingshard + + diff --git a/cmd/kingshard/main.go b/cmd/kingshard/main.go index fc7230aa..4ea48319 100644 --- a/cmd/kingshard/main.go +++ b/cmd/kingshard/main.go @@ -49,6 +49,28 @@ const banner string = ` /____/ ` +func writePID(cfg *config.Config) { + pidpath := cfg.PidPath + if pidpath == "" { + fmt.Println("WritePid failed, please confirm that the pidfile configuration item are correctly filed") + os.Exit(-1) + } + + fp, err := os.Create(pidpath) + if err != nil { + fmt.Println("WritePID failed, create pidfile failed %s", err.Error()) + os.Exit(-1) + } + defer fp.Close() + + pid := os.Getpid() + _, err = fp.WriteString(fmt.Sprintf("%d", pid)) + if err != nil { + fmt.Println("WritePID failed, %s", err.Error()) + os.Exit(-1) + } +} + func main() { fmt.Print(banner) runtime.GOMAXPROCS(runtime.NumCPU()) @@ -117,6 +139,7 @@ func main() { svr.Close() }() + writePID(cfg) svr.Run() } diff --git a/config/config.go b/config/config.go index d372d718..1b10ac22 100644 --- a/config/config.go +++ b/config/config.go @@ -23,6 +23,7 @@ import ( //整个config文件对应的结构 type Config struct { + PidPath string `yaml:"pidfile"` Addr string `yaml:"addr"` User string `yaml:"user"` Password string `yaml:"password"` @@ -66,6 +67,7 @@ type ShardConfig struct { Nodes []string `yaml:"nodes"` Locations []int `yaml:"locations"` Type string `yaml:"type"` + TableRowBase int `yaml:"table_row_base"` TableRowLimit int `yaml:"table_row_limit"` DateRange []string `yaml:"date_range"` } diff --git a/dev.sh b/dev.sh index bc3ea6e7..41fb4e78 100755 --- a/dev.sh +++ b/dev.sh @@ -8,11 +8,8 @@ if [[ "$VTTOP" == "${VTTOP/\/src\/github.com\/flike\/kingshard/}" ]]; then fi export GOTOP=$VTTOP - function prepend_path() { - # $1 path variable - # $2 path to add if [ -d "$2" ] && [[ ":$1:" != *":$2:"* ]]; then echo "$2:$1" else @@ -21,4 +18,3 @@ function prepend_path() } export GOPATH=$(prepend_path $GOPATH $VTROOT) - diff --git a/etc/ks.yaml b/etc/ks.yaml index 63ff5a07..451a15a4 100755 --- a/etc/ks.yaml +++ b/etc/ks.yaml @@ -1,16 +1,18 @@ +pidfile : /uc/etc/ucdbproxy.pid + # server listen addr addr : 0.0.0.0:9696 # server user and password -user : kingshard -password : kingshard +user : ucdbproxy +password : ucdbproxy # if set log_path, the sql log will write into log_path/sql.log,the system log # will write into log_path/sys.log #log_path : /Users/flike/log # log level[debug|info|warn|error],default error -log_level : debug +log_level : info # if set log_sql(on|off) off,the sql log will not output log_sql: on @@ -27,7 +29,7 @@ log_sql: on # the charset of kingshard, if you don't set this item # the default charset of kingshard is utf8. -#proxy_charset: gbk +proxy_charset: utf8mb4 # node is an agenda for real remote mysql server. nodes : @@ -35,11 +37,11 @@ nodes : name : node1 # default max conns for mysql server - max_conns_limit : 32 + max_conns_limit : 128 # all mysql in a node must have the same user and password - user : kingshard - password : kingshard + user : root + password : quanshi # master represents a real mysql master server master : 127.0.0.1:3306 @@ -47,62 +49,27 @@ nodes : # slave represents a real mysql salve server,and the number after '@' is # read load weight of this slave. #slave : 192.168.59.101:3307@2,192.168.59.101:3307@3 - down_after_noalive : 32 -- - name : node2 - - # default max conns for mysql server - max_conns_limit : 32 - - # all mysql in a node must have the same user and password - user : kingshard - password : kingshard - - # master represents a real mysql master server - master : 192.168.59.103:3307 - - # slave represents a real mysql salve server - slave : - - # down mysql after N seconds noalive - # 0 will no down - down_after_noalive: 32 + #down_after_noalive : 32 # schema defines sharding rules, the db is the sharding table database. schema : - db : kingshard - nodes: [node1,node2] + db : statusnet + nodes: [node1] default: node1 shard: - - table: test_shard_hash - key: id - nodes: [node1, node2] - type: hash - locations: [4,4] - - - - table: test_shard_range + table: user_group key: id type: range - nodes: [node1, node2] - locations: [4,4] + nodes: [node1] + locations: [4] + table_row_base: 2000 table_row_limit: 10000 - - table: test_shard_year - key: ctime - type: date_year - nodes: [node1,node2] - date_range: [2015-2016,2017-2018] - - - table: test_shard_month - key: dtime - type: date_month - nodes: [node1,node2] - date_range: [201603-201605,201609-201612] - - - table: test_shard_day - key: mtime - type: date_day - nodes: [node1,node2] - date_range: [20160306-20160307,20160308-20160309] + table: group_member + key: group_id + type: range + nodes: [node1] + locations: [4] + table_row_base: 2000 + table_row_limit: 10000 diff --git a/etc/unshard.yaml b/etc/unshard.yaml index c0ec3c58..3a9b20bb 100644 --- a/etc/unshard.yaml +++ b/etc/unshard.yaml @@ -1,3 +1,5 @@ +pidfile : /uc/etc/ucdbsql.pid + # server listen addr addr : 0.0.0.0:9696 diff --git a/proxy/router/numkey.go b/proxy/router/numkey.go index 4c9e21b9..5d2dfb0a 100644 --- a/proxy/router/numkey.go +++ b/proxy/router/numkey.go @@ -47,7 +47,7 @@ func (kr NumKeyRange) String() string { return fmt.Sprintf("{Start: %d, End: %d}", kr.Start, kr.End) } -func ParseNumSharding(Locations []int, TableRowLimit int) ([]NumKeyRange, error) { +func ParseNumSharding(Locations []int, TableRowLimit int, TableRowBase int) ([]NumKeyRange, error) { tableCount := 0 length := len(Locations) @@ -57,8 +57,8 @@ func ParseNumSharding(Locations []int, TableRowLimit int) ([]NumKeyRange, error) ranges := make([]NumKeyRange, tableCount) for i := 0; i < tableCount; i++ { - ranges[i].Start = int64(i * TableRowLimit) - ranges[i].End = int64((i + 1) * TableRowLimit) + ranges[i].Start = int64(TableRowBase + i*TableRowLimit) + ranges[i].End = int64(TableRowBase + (i+1)*TableRowLimit) } return ranges, nil } diff --git a/proxy/router/router.go b/proxy/router/router.go index 5f1e1cc3..e4455962 100644 --- a/proxy/router/router.go +++ b/proxy/router/router.go @@ -233,7 +233,7 @@ func parseShard(r *Rule, cfg *config.ShardConfig) error { case HashRuleType: r.Shard = &HashShard{ShardNum: len(r.TableToNode)} case RangeRuleType: - rs, err := ParseNumSharding(cfg.Locations, cfg.TableRowLimit) + rs, err := ParseNumSharding(cfg.Locations, cfg.TableRowLimit, cfg.TableRowBase) if err != nil { return err } @@ -490,6 +490,7 @@ func (r *Router) rewriteSelectSql(plan *Plan, node *sqlparser.Select, tableIndex var prefix string //rewrite select expr for _, expr := range node.SelectExprs { + switch v := expr.(type) { case *sqlparser.StarExpr: //for shardTable.*,need replace table into shardTable_xxxx. @@ -535,6 +536,7 @@ func (r *Router) rewriteSelectSql(plan *Plan, node *sqlparser.Select, tableIndex buf.Fprintf("%s%v", prefix, n) } } + buf.Fprintf(" from ") switch v := (node.From[0]).(type) { case *sqlparser.AliasedTableExpr: @@ -570,7 +572,32 @@ func (r *Router) rewriteSelectSql(plan *Plan, node *sqlparser.Select, tableIndex tableIndex, ) } - buf.Fprintf(" %s %v", v.Join, v.RightExpr) + + if ate_r, ok_r := (v.RightExpr).(*sqlparser.AliasedTableExpr); ok_r { + if len(ate_r.As) != 0 { + fmt.Fprintf(buf, " %s %s_%04d as %s", + sqlparser.AST_LEFT_JOIN, + sqlparser.String(ate_r.Expr), + tableIndex, + string(ate_r.As), + ) + } else { + fmt.Fprintf(buf, " %s %s_%04d", + sqlparser.AST_LEFT_JOIN, + sqlparser.String(ate_r.Expr), + tableIndex, + ) + } + } else { + fmt.Fprintf(buf, " %s %s_%04d", + sqlparser.AST_LEFT_JOIN, + sqlparser.String(v.RightExpr), + tableIndex, + ) + } + + //buf.Fprintf(" %s %v", v.Join, v.RightExpr) + //golog.Info("router", "sqlparser.JoinTableExpr and v.Join, v.RightExpr", buf.String(), 0) if v.On != nil { buf.Fprintf(" on %v", v.On) } @@ -580,6 +607,7 @@ func (r *Router) rewriteSelectSql(plan *Plan, node *sqlparser.Select, tableIndex tableIndex, ) } + //append other tables prefix = ", " for i := 1; i < len(node.From); i++ { diff --git a/sqlparser/sql.go b/sqlparser/sql.go index df168c84..14ee2cd3 100644 --- a/sqlparser/sql.go +++ b/sqlparser/sql.go @@ -1,9 +1,9 @@ -//line ./sqlparser/sql.y:20 +//line ./sqlparser/sql.y:19 package sqlparser import __yyfmt__ "fmt" -//line ./sqlparser/sql.y:20 +//line ./sqlparser/sql.y:21 import "bytes" func SetParseTree(yylex interface{}, stmt Statement) { diff --git a/ucdbproxy.sh b/ucdbproxy.sh new file mode 100644 index 00000000..30788d77 --- /dev/null +++ b/ucdbproxy.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# +# chkconfig: 2345 64 36 +# description: ucdbproxy startup scripts +# +ucdbproxy_root=/uc/sbin +# each config file for one instance +configs="/uc/etc/ks.yaml" +pidfile="/uc/etc/ucdbproxy.pid" +nohupfile="/uc/etc/nohup.ucdbproxy.out" + +ulimit -c unlimited +echo 1 > /proc/sys/fs/suid_dumpable +echo "/uc/share/%e-%p-%s-%t.core" >/proc/sys/kernel/core_pattern + +ulimit -n 1024000 + +start() { + test -f $pidfile && echo "ucdbproxy is running" + nohup $ucdbproxy_root/ucdbproxy -config $configs >> $nohupfile & + sleep 1 + test -f $pidfile && echo "Start ucdbproxy success" +} + +stop() { + kill -9 `cat $pidfile` + test -f $pidfile && rm $pidfile + echo "Stop ucdbproxy success"; +} + +# See how we were called. +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart) + stop + start + ;; + *) + echo $"Usage: $0 {start|stop|restart}" + ;; +esac +exit 0 From 059bde5abe3c75e2b9b82828a22b05495edf957b Mon Sep 17 00:00:00 2001 From: jefjin <64047032@qq.com> Date: Thu, 14 Jul 2016 14:52:33 +0800 Subject: [PATCH 3/3] fix conflict --- etc/ks.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/etc/ks.yaml b/etc/ks.yaml index b401a488..451a15a4 100755 --- a/etc/ks.yaml +++ b/etc/ks.yaml @@ -61,13 +61,8 @@ schema : table: user_group key: id type: range -<<<<<<< HEAD nodes: [node1] locations: [4] -======= - nodes: [node1, node2] - locations: [4,4] ->>>>>>> origin/master table_row_base: 2000 table_row_limit: 10000 -