diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..68334cf --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 + +updates: + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "daily" \ No newline at end of file diff --git a/.github/workflows/hygeine.yml b/.github/workflows/hygeine.yml new file mode 100644 index 0000000..61bf5ff --- /dev/null +++ b/.github/workflows/hygeine.yml @@ -0,0 +1,21 @@ +name: hygeiene + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + golangci: + # Linting job + # https://github.com/golangci/golangci-lint-action + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version + version: v1.52.1 \ No newline at end of file diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..a979c4c --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,22 @@ +name: security + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + # Static security scan using gosec + # https://github.com/securego/gosec + gosec: + runs-on: ubuntu-latest + env: + GO111MODULE: on + steps: + - name: Checkout Source + uses: actions/checkout@v3 + - name: Run Gosec Security Scanner + uses: securego/gosec@master + with: + args: ./... \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..9785d53 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,47 @@ +# Go test workflow +name: test + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.22 + + - name: Install project dependencies + run: | + go mod download + + - name: Build App + run: make build-app + + go-test: + outputs: + COVERAGE: ${{ steps.unit.outputs.coverage }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.22 + + - name: Install project dependencies + run: | + go mod download + + - name: Run Tests + id: unit + run: | + make test diff --git a/.golangci.yml b/.golangci.yml index a92da5b..4948dcf 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -116,7 +116,7 @@ linters: disable-all: true enable: ## enabled by default - - errcheck # checking for unchecked errors, these unchecked errors can be critical bugs in some cases + #- errcheck # checking for unchecked errors, these unchecked errors can be critical bugs in some cases - gosimple # specializes in simplifying a code - ineffassign # detects when assignments to existing variables are not used - staticcheck # is a go vet on steroids, applying a ton of static analysis checks @@ -133,10 +133,10 @@ linters: - errname # checks that sentinel errors are prefixed with the Err and error types are suffixed with the Error - errorlint # finds code that will cause problems with the error wrapping scheme introduced in Go 1.13 - execinquery # checks query string in Query function which reads your Go src files and warning it finds - - exhaustive # checks exhaustiveness of enum switch statements + #- exhaustive # checks exhaustiveness of enum switch statements - exportloopref # checks for pointers to enclosing loop variables - forbidigo # forbids identifiers - - funlen # tool for detection of long functions + #- funlen # tool for detection of long functions - gocheckcompilerdirectives # validates go compiler directive comments (//go:) - gochecknoinits # checks that no init functions are present in Go code - gocognit # computes and checks the cognitive complexity of functions @@ -144,16 +144,16 @@ linters: - gocritic # provides diagnostics that check for bugs, performance and style issues - gocyclo # computes and checks the cyclomatic complexity of functions - goimports # in addition to fixing imports, goimports also formats your code in the same style as gofmt - # - gomnd # detects magic numbers + #- gomnd # detects magic numbers - gomodguard # allow and block lists linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations - goprintffuncname # checks that printf-like functions are named with f at the end - gosec # inspects source code for security problems - - lll # reports long lines + #- lll # reports long lines - loggercheck # checks key value pairs for common logger libraries (kitlog,klog,logr,zap) - makezero # finds slice declarations with non-zero initial length - misspell # Ensures real english is used within strings - nakedret # finds naked returns in functions greater than a specified function length - - nestif # reports deeply nested if statements + #- nestif # reports deeply nested if statements - nilerr # finds the code that returns nil even if it checks that the error is not nil - nilnil # checks that there is no simultaneous return of nil error and an invalid value - noctx # finds sending http request without context.Context diff --git a/bench b/bench deleted file mode 100644 index e69de29..0000000 diff --git a/cmd/client/main.go b/cmd/client/rand/main.go similarity index 79% rename from cmd/client/main.go rename to cmd/client/rand/main.go index 3511a42..065d0ac 100644 --- a/cmd/client/main.go +++ b/cmd/client/rand/main.go @@ -4,14 +4,9 @@ import ( "bufio" "context" "crypto/rand" - "fmt" - "github.com/denzelpenzel/nyx/cmd/client/textprot" - "github.com/denzelpenzel/nyx/internal/common" - "github.com/denzelpenzel/nyx/internal/logging" - "github.com/denzelpenzel/nyx/internal/utils" - "github.com/urfave/cli" - "go.uber.org/zap" + "errors" "io" + "log" "math/big" "net" "os" @@ -19,6 +14,13 @@ import ( "runtime/pprof" "sync" "time" + + "github.com/DenzelPenzel/nyx/cmd/client/textprot" + "github.com/DenzelPenzel/nyx/internal/common" + "github.com/DenzelPenzel/nyx/internal/logging" + "github.com/DenzelPenzel/nyx/internal/utils" + "github.com/urfave/cli" + "go.uber.org/zap" ) var ( @@ -104,7 +106,7 @@ func run(c *cli.Context) { opsPerTask := numOps / numCmds / numWorkers - fmt.Printf("Running %v ops total with:\n"+ + log.Printf("Running %v ops total with:\n"+ "\t%v workers\n"+ "\ttotal commands %v\n"+ "\tusing the %v protocol\n"+ @@ -131,7 +133,7 @@ func run(c *cli.Context) { connWg.Add(1) conn, err := utils.Connect(httpAddr) if err != nil { - fmt.Printf("Failed connect to %s error: %s\n", httpAddr.String(), err) + log.Printf("Failed connect to %s error: %s\n", httpAddr.String(), err) i-- connWg.Add(-1) continue @@ -154,31 +156,31 @@ func run(c *cli.Context) { hits[m.op] = append(hits[m.op], int(m.duration)) } - metricPool.Put(m) + metricPool.Put(m) //nolint:staticcheck //check pointer-like issue } for i, op := range allOps { if i == 0 { - fmt.Println("===========Metrics===========") + log.Println("===========Metrics===========") } renderStats("hits", op, hits[op]) renderStats("misses", op, misses[op]) - fmt.Println("=============================") + log.Println("=============================") } stats.Done() }() - fmt.Println("Generate testing tasks...") + log.Println("Generate testing tasks...") tasksWg.Wait() - fmt.Println("Tasks generation done") + log.Println("Tasks generation done") close(tasks) - fmt.Println("Start tasks execution...") + log.Println("Start tasks execution...") connWg.Wait() - fmt.Println("Execution done") + log.Println("Execution done") close(metrics) stats.Wait() @@ -223,8 +225,8 @@ func execute(conn net.Conn, connWg *sync.WaitGroup, tasks <-chan *Task, metrics if err != nil { if !common.IsMiss(err) { // socket is closed - if err == io.EOF { - fmt.Printf("Failed to execute request: %s, key: %s, error: %v\n", item.Cmd, item.Key, err) + if errors.Is(err, io.EOF) { + log.Printf("Failed to execute request: %s, key: %s, error: %v\n", item.Cmd, item.Key, err) return } } @@ -249,17 +251,17 @@ func genData(cmd Op) []byte { func renderStats(t string, op Op, data []int) { if len(data) == 0 { - fmt.Printf("\nNo %s %s\n", op.String(), t) + log.Printf("\nNo %s %s\n", op.String(), t) return } s := GetStats(data) - fmt.Printf("%s %s (n = %d)\n", op.String(), t, len(data)) - fmt.Printf("Min: %fms\n", s.Min) - fmt.Printf("Max: %fms\n", s.Max) - fmt.Printf("Avg: %fms\n", s.Avg) - fmt.Printf("p50: %fms\n", s.P50) - fmt.Printf("p75: %fms\n", s.P75) - fmt.Printf("p90: %fms\n", s.P90) - fmt.Printf("p95: %fms\n", s.P95) - fmt.Printf("p99: %fms\n", s.P99) + log.Printf("%s %s (n = %d)\n", op.String(), t, len(data)) + log.Printf("Min: %fms\n", s.Min) + log.Printf("Max: %fms\n", s.Max) + log.Printf("Avg: %fms\n", s.Avg) + log.Printf("p50: %fms\n", s.P50) + log.Printf("p75: %fms\n", s.P75) + log.Printf("p90: %fms\n", s.P90) + log.Printf("p95: %fms\n", s.P95) + log.Printf("p99: %fms\n", s.P99) } diff --git a/cmd/client/rand/stats.go b/cmd/client/rand/stats.go new file mode 100644 index 0000000..abb7d19 --- /dev/null +++ b/cmd/client/rand/stats.go @@ -0,0 +1,60 @@ +package main + +import ( + "math" + "slices" + "sort" +) + +const ( + msFactor = 1000000 +) + +type Stats struct { + Avg float64 + Min float64 + Max float64 + P50 float64 + P75 float64 + P90 float64 + P95 float64 + P99 float64 +} + +func GetStats(data []int) Stats { + if len(data) == 0 { + return Stats{} + } + + sort.Ints(data) + minv, maxv := slices.Min(data), slices.Max(data) + + return Stats{ + Avg: avg(data) / msFactor, + Min: float64(minv / msFactor), + Max: float64(maxv / msFactor), + P50: percent(data, 0.5) / msFactor, + P75: percent(data, 0.75) / msFactor, + P90: percent(data, 0.9) / msFactor, + P95: percent(data, 0.95) / msFactor, + P99: percent(data, 0.99) / msFactor, + } +} + +func avg(data []int) float64 { + r := float64(0) + for _, d := range data { + r += float64(d) + } + return r / float64(len(data)) +} + +func percent(data []int, p float64) float64 { + idx := pIdx(len(data), p) + return float64(data[idx]) +} + +func pIdx(datalen int, p float64) int { + w := math.Ceil(float64(datalen) * p) + return int(min(w, float64(datalen-1))) +} diff --git a/cmd/client/stats.go b/cmd/client/stats.go deleted file mode 100644 index b1e9042..0000000 --- a/cmd/client/stats.go +++ /dev/null @@ -1,149 +0,0 @@ -package main - -import ( - "fmt" - "math" - "slices" - "sort" -) - -const ( - msFactor = 1000000 - numBuckets = 100 - - maxHeight = 50 -) - -type Stats struct { - Avg float64 - Min float64 - Max float64 - P50 float64 - P75 float64 - P90 float64 - P95 float64 - P99 float64 -} - -func GetStats(data []int) Stats { - if len(data) == 0 { - return Stats{} - } - - sort.Ints(data) - minv, maxv := slices.Min(data), slices.Max(data) - - return Stats{ - Avg: avg(data) / msFactor, - Min: float64(minv / msFactor), - Max: float64(maxv / msFactor), - P50: percent(data, 0.5) / msFactor, - P75: percent(data, 0.75) / msFactor, - P90: percent(data, 0.9) / msFactor, - P95: percent(data, 0.95) / msFactor, - P99: percent(data, 0.99) / msFactor, - } -} - -func PrintStats(data []int) { - if len(data) == 0 { - return - } - - p99Idx := pIdx(len(data), 0.99) - data = data[:p99Idx] - - buckets := make([]int, numBuckets) - minv := data[0] - maxv := data[len(data)-1] - step := float64(maxv-minv) / numBuckets - prevCutIdx := 0 - maxBucket := 0 - maxBucketIdx := 0 - - for i := 0; i < numBuckets; i++ { - cut := float64(minv) + step*float64(i+1) - count := 0 - j := prevCutIdx - - for ; j < len(data) && float64(data[j]) < cut; j++ { - count++ - } - - prevCutIdx = j - buckets[i] = count - - if count > maxBucket { - maxBucket = count - maxBucketIdx = i - } - } - - topBucketFmt := fmt.Sprintf("%%%vd\n", maxBucketIdx+3) - topPointerRow := make([]rune, numBuckets) - - for i := 1; i < numBuckets-1; i++ { - topPointerRow[i] = ' ' - } - - topPointerRow[maxBucketIdx+1] = 'v' - - heightRatio := float64(maxHeight) / float64(maxBucket) - for i := 0; i < len(buckets); i++ { - buckets[i] = int(math.Min(float64(maxHeight), math.Ceil(float64(buckets[i])*heightRatio))) - } - - var hist []rune - for i := maxHeight; i >= 0; i-- { - for j := 0; j < numBuckets; j++ { - if i == 0 { - hist = append(hist, '=') - } else if buckets[j] == i { - hist = append(hist, '|') - buckets[j]-- - } else { - hist = append(hist, ' ') - } - } - - hist = append(hist, '\n') - } - - gmin := float64(minv) / msFactor - gmax := float64(maxv) / msFactor - gmid := gmin + (gmax-gmin)/2 - - pointerRow := make([]rune, numBuckets) - - for i := 1; i < numBuckets-1; i++ { - pointerRow[i] = ' ' - } - - pointerRow[0] = '^' - pointerRow[len(pointerRow)/2] = '^' - pointerRow[len(pointerRow)-1] = '^' - - fmt.Printf(topBucketFmt, maxBucket) - fmt.Println(string(topPointerRow)) - fmt.Print(string(hist)) - fmt.Println(string(pointerRow)) - fmt.Printf("%.4fms %.4fms %.4fms\n", gmin, gmid, gmax) -} - -func avg(data []int) float64 { - r := float64(0) - for _, d := range data { - r += float64(d) - } - return r / float64(len(data)) -} - -func percent(data []int, p float64) float64 { - idx := pIdx(len(data), p) - return float64(data[idx]) -} - -func pIdx(datalen int, p float64) int { - w := math.Ceil(float64(datalen) * p) - return int(min(w, float64(datalen-1))) -} diff --git a/cmd/client/textprot/textprot.go b/cmd/client/textprot/textprot.go index 06ffcb4..9f5e3b9 100644 --- a/cmd/client/textprot/textprot.go +++ b/cmd/client/textprot/textprot.go @@ -2,8 +2,9 @@ package textprot import ( "bufio" + "crypto/rand" "fmt" - "math/rand/v2" + "math/big" "strings" ) @@ -189,13 +190,18 @@ func (t TextProt) Delete(rw *bufio.ReadWriter, key []byte) error { func (t TextProt) Touch(rw *bufio.ReadWriter, key []byte) error { strKey := string(key) - if _, err := fmt.Fprintf(rw, "touch %s %v\r\n", strKey, rand.IntN(maxTTL)); err != nil { + ttl, err := rand.Int(rand.Reader, big.NewInt(int64(maxTTL))) + if err != nil { + return err + } + + if _, err := fmt.Fprintf(rw, "touch %s %v\r\n", strKey, ttl); err != nil { return err } rw.Flush() - _, err := rw.ReadString('\n') + _, err = rw.ReadString('\n') if err != nil { return err } diff --git a/cmd/nyx/main.go b/cmd/nyx/main.go index c1135a1..bafb16a 100644 --- a/cmd/nyx/main.go +++ b/cmd/nyx/main.go @@ -2,12 +2,13 @@ package main import ( "context" - "github.com/denzelpenzel/nyx/internal/app" - "github.com/denzelpenzel/nyx/internal/config" - "github.com/denzelpenzel/nyx/internal/logging" + "os" + + "github.com/DenzelPenzel/nyx/internal/app" + "github.com/DenzelPenzel/nyx/internal/config" + "github.com/DenzelPenzel/nyx/internal/logging" "github.com/urfave/cli" "go.uber.org/zap" - "os" ) func main() { diff --git a/e2e/node.txt b/e2e/node.txt index 39429f6..7773b93 100644 --- a/e2e/node.txt +++ b/e2e/node.txt @@ -4,11 +4,11 @@ // "context" // "encoding/json" // "fmt" -// "github.com/denzelpenzel/nyx/internal/common" -// "github.com/denzelpenzel/nyx/internal/config" -// "github.com/denzelpenzel/nyx/internal/db" -// "github.com/denzelpenzel/nyx/internal/dyport" -// "github.com/denzelpenzel/nyx/internal/utils" +// "github.com/DenzelPenzel/nyx/internal/common" +// "github.com/DenzelPenzel/nyx/internal/config" +// "github.com/DenzelPenzel/nyx/internal/db" +// "github.com/DenzelPenzel/nyx/internal/dyport" +// "github.com/DenzelPenzel/nyx/internal/utils" // "io" // "net/http" // "net/url" diff --git a/go.mod b/go.mod index 88febd9..27e835c 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/denzelpenzel/nyx +module github.com/DenzelPenzel/nyx go 1.22.0 diff --git a/internal/app/app.go b/internal/app/app.go index e8cc108..bec8403 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -2,15 +2,16 @@ package app import ( "context" - "github.com/denzelpenzel/nyx/internal/config" - "github.com/denzelpenzel/nyx/internal/db" - "github.com/denzelpenzel/nyx/internal/logging" - "github.com/denzelpenzel/nyx/internal/nyx" - "github.com/denzelpenzel/nyx/internal/server" - "go.uber.org/zap" "os" "os/signal" "syscall" + + "github.com/DenzelPenzel/nyx/internal/config" + "github.com/DenzelPenzel/nyx/internal/db" + "github.com/DenzelPenzel/nyx/internal/logging" + "github.com/DenzelPenzel/nyx/internal/nyx" + "github.com/DenzelPenzel/nyx/internal/server" + "go.uber.org/zap" ) // Application ... Pessimism app struct @@ -35,13 +36,13 @@ func NewFastCacheApp(ctx context.Context, cfg *config.Config) (*Application, fun // Start ... Starts the application func (a *Application) Start() error { - // Run metrics server - // a.metrics.Run() + // Open metrics server + // a.metrics.Open() - // Run the API server + // Open the API server go server.ListenAndServe(a.ctx, a.l, a.db, nyx.NewNyx) - // if err := a.server.Run(); err != nil { + // if err := a.server.Open(); err != nil { // return err // } diff --git a/internal/common/common.go b/internal/common/common.go index cbea820..6e1f84b 100644 --- a/internal/common/common.go +++ b/internal/common/common.go @@ -16,6 +16,24 @@ const ( const VersionString = "0.1" +type DBHandler interface { + Set(req SetRequest) error + Add(req SetRequest) error + Replace(req SetRequest) error + Append(req SetRequest) error + Prepend(req SetRequest) error + Delete(req DeleteRequest) error + Touch(req TouchRequest) error + Get(req GetRequest) error + GetE(req GetRequest) error + Gat(req GATRequest) error + Noop(req NoopRequest) error + Quit(req QuitRequest) error + Version(req VersionRequest) error + Unknown(req Request) error + Error(req Request, reqType RequestType, err error) +} + // Common metrics used across packages var ( ErrBadRequest = errors.New("CLIENT_ERROR bad request") @@ -85,12 +103,6 @@ const ( // different protocols. This means it can also be the accumulation of many GETQ commands. RequestGet - // RequestGat is a get-and-touch operation that retrieves the information while updating the TTL - RequestGat - - // RequestGetE is a custom get which returns the TTL remaining with the data - RequestGetE - // RequestSet is to insert a new piece of data unconditionally. What that means is different // depending on L1 / L2 handling. RequestSet diff --git a/internal/config/config.go b/internal/config/config.go index 62dff25..5286f51 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,11 +1,12 @@ package config import ( - "github.com/denzelpenzel/nyx/internal/common" - "github.com/denzelpenzel/nyx/internal/utils" - "github.com/urfave/cli" "net" "time" + + "github.com/DenzelPenzel/nyx/internal/common" + "github.com/DenzelPenzel/nyx/internal/utils" + "github.com/urfave/cli" ) type DBConfig struct { diff --git a/internal/db/db.go b/internal/db/db.go index a0263d7..bd254b3 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -3,11 +3,6 @@ package db import ( "context" "fmt" - "github.com/denzelpenzel/nyx/internal/common" - "github.com/denzelpenzel/nyx/internal/config" - "github.com/denzelpenzel/nyx/internal/db/store" - "github.com/denzelpenzel/nyx/internal/logging" - "go.uber.org/zap" "net" "os" "runtime/debug" @@ -15,6 +10,12 @@ import ( "sync" "sync/atomic" "time" + + "github.com/DenzelPenzel/nyx/internal/common" + "github.com/DenzelPenzel/nyx/internal/config" + "github.com/DenzelPenzel/nyx/internal/db/store" + "github.com/DenzelPenzel/nyx/internal/logging" + "go.uber.org/zap" ) type DB interface { @@ -50,7 +51,7 @@ type db struct { } func NewDB(ctx context.Context, cfg *config.DBConfig) (DB, error) { - c := &db{ + d := &db{ ctx: ctx, slaveAddr: cfg.Backup, } @@ -59,15 +60,15 @@ func NewDB(ctx context.Context, cfg *config.DBConfig) (DB, error) { if err != nil { return nil, err } - c.store = s + // save the store pointer + d.store = s debug.SetGCPercent(20) - atomic.StoreUint64(&c.getCnt, 0) - atomic.StoreUint64(&c.setCnt, 0) - - c.stats = &stats{} + atomic.StoreUint64(&d.getCnt, 0) + atomic.StoreUint64(&d.setCnt, 0) + d.stats = &stats{} - return c, nil + return d, nil } func (c *db) Set(cmd common.SetRequest) error { diff --git a/internal/db/db_test.go b/internal/db/db_test.go index 6d35bdb..f9172ed 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -4,13 +4,14 @@ import ( "bytes" "context" "encoding/binary" - "github.com/denzelpenzel/nyx/internal/common" - "github.com/denzelpenzel/nyx/internal/config" - "github.com/denzelpenzel/nyx/internal/db" - "github.com/denzelpenzel/nyx/internal/utils" - "github.com/stretchr/testify/require" "os" "testing" + + "github.com/DenzelPenzel/nyx/internal/common" + "github.com/DenzelPenzel/nyx/internal/config" + "github.com/DenzelPenzel/nyx/internal/db" + "github.com/DenzelPenzel/nyx/internal/utils" + "github.com/stretchr/testify/require" ) func openDB() (db.DB, func(), error) { diff --git a/internal/db/store/bucket.go b/internal/db/store/bucket.go index 4cc547c..aae6a4a 100644 --- a/internal/db/store/bucket.go +++ b/internal/db/store/bucket.go @@ -2,10 +2,11 @@ package store import ( "errors" - "github.com/denzelpenzel/nyx/internal/common" - "github.com/google/btree" "slices" "strings" + + "github.com/DenzelPenzel/nyx/internal/common" + "github.com/google/btree" ) // Str implements the Item interface for strings. diff --git a/internal/db/store/bucket_test.go b/internal/db/store/bucket_test.go index 3c3ccba..dced928 100644 --- a/internal/db/store/bucket_test.go +++ b/internal/db/store/bucket_test.go @@ -1,9 +1,10 @@ package store import ( - "github.com/stretchr/testify/require" "os" "testing" + + "github.com/stretchr/testify/require" ) var bucketDirName = "-db-test-bucket--tmp-" diff --git a/internal/db/store/shard/header.go b/internal/db/store/shard/header.go index aba15aa..9d0769a 100644 --- a/internal/db/store/shard/header.go +++ b/internal/db/store/shard/header.go @@ -4,8 +4,9 @@ import ( "encoding/binary" "errors" "fmt" - "github.com/denzelpenzel/nyx/internal/utils" "io" + + "github.com/DenzelPenzel/nyx/internal/utils" ) const ( diff --git a/internal/db/store/shard/shard.go b/internal/db/store/shard/shard.go index bedaa25..95991d2 100644 --- a/internal/db/store/shard/shard.go +++ b/internal/db/store/shard/shard.go @@ -5,13 +5,14 @@ import ( "encoding/binary" "errors" "fmt" - "github.com/denzelpenzel/nyx/internal/common" - "github.com/denzelpenzel/nyx/internal/utils" - "github.com/spaolacci/murmur3" "io" "os" "sync" "time" + + "github.com/DenzelPenzel/nyx/internal/common" + "github.com/DenzelPenzel/nyx/internal/utils" + "github.com/spaolacci/murmur3" ) type Shard struct { @@ -24,193 +25,180 @@ type Shard struct { var forceExit bool -func getShardVer(file *os.File) (int, error) { - b := make([]byte, 2) - n, err := file.Read(b) +// upgrade ... upgrade the file format +func (s *Shard) upgrade(ver int, name string) error { + var newFile *os.File + newName := name + ".new" + newFile, err := os.OpenFile(newName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.FileMode(0644)) if err != nil { - return -1, err + return err } - if n != 2 { - return -1, errors.New("short file") + // write the new version header + _, err = newFile.Write([]byte{versionMarker, currentShardVer}) + if err != nil { + return err } - if b[0] == versionMarker { - if b[1] == 0 || b[1] == deleted { - return 0, nil + seek := uint32(2) + oldSizeHead := sizeHeaders[ver] + sizeDiff := sizeHead - oldSizeHead + + for { + header, err := readHeader(s.f, ver) + if err != nil { + newFile.Close() + return err } - return int(b[1]), nil - } - if b[1] == 0 || b[1] == deleted { - return 0, nil - } - return -1, nil -} + if header == nil { + break + } + oldSizeData := (1 << header.sizeByte) - oldSizeHead + sizeb, size := utils.NextPowerOf2(sizeHead + uint32(header.keyLength) + header.valLength) + header.sizeByte = sizeb -func (s *Shard) Run(name string) error { - s.Lock() - defer s.Unlock() + b := make([]byte, size+sizeDiff) + writeHeader(b, header) + n, err := s.f.Read(b[sizeHead : sizeHead+oldSizeData]) + if err != nil { + return err + } + if n != int(oldSizeData) { + return fmt.Errorf("wrong shart len: %d", n) + } - forceExit = false + if header.status == deleted || (header.expire != 0 && int64(header.expire) < time.Now().Unix()) { + continue + } - f, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, os.FileMode(0644)) + startPos := int(sizeHead) + int(header.valLength) + endPos := int(sizeHead) + int(header.keyLength) + int(header.valLength) + h := murmur3.Sum32WithSeed(b[startPos:endPos], 0) + + s.mapping[h] = Encode(seek, header.sizeByte, header.expire) + n, err = newFile.Write(b[0:size]) + if err != nil { + return err + } + seek += uint32(n) + } + // close old file + err = s.f.Close() if err != nil { return err } - - err = f.Sync() + // rewrite the ref to the new file + s.f = newFile + // remove the old file from the disk + err = os.Remove(name) if err != nil { return err } + // rename the new file name to the old file name + err = os.Rename(newName, name) + if err != nil { + return err + } + return nil +} - s.f = f - s.mapping = make(map[uint32]uint64) - s.remapping = make(map[uint32]byte) - - if fi, err := s.f.Stat(); err == nil { - // create a new file - if fi.Size() == 0 { - // write shard info to the file - _, err = s.f.Write([]byte{versionMarker, currentShardVer}) - if err != nil { - return err - } - return nil +func (s *Shard) writeHeader(ver int, offset uint32) error { + for { + header, err := readHeader(s.f, ver) + if err != nil { + return err + } + if header == nil { + break + } + _, err = s.f.Seek(int64(header.valLength), 1) + if err != nil { + return err } - // read file - var seek uint32 - ver, err := getShardVer(s.f) + // read key + key, err := s.readKey(header.keyLength) if err != nil { return err } - if ver < 0 || ver > currentShardVer { - return errors.New("unknown Shard version in file " + name) + shift := 1 << header.sizeByte + // skip empty tail + res, err := s.f.Seek(int64(shift-int(header.keyLength)-int(header.valLength)-int(sizeHead)), 1) + if err != nil { + return err } - if ver == 0 { - _, err = s.f.Seek(0, 0) - if err != nil { - return err - } + if header.status != deleted && (header.expire == 0 || int64(header.expire) >= time.Now().Unix()) { + h := murmur3.Sum32WithSeed(key, 0) + s.mapping[h] = Encode(offset, header.sizeByte, header.expire) } else { - seek = 2 + s.remapping[offset] = header.sizeByte } - if ver > currentShardVer { - return fmt.Errorf("unsupported Shard: name: %s version: %d", name, ver) - } + offset = uint32(res) + } - if ver < currentShardVer { - var newFile *os.File - newName := name + ".new" - newFile, err := os.OpenFile(newName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.FileMode(0644)) - if err != nil { - return err - } - // write version - _, err = newFile.Write([]byte{versionMarker, currentShardVer}) - if err != nil { - return err - } - seek = 2 - oldSizeHead := sizeHeaders[ver] - sizeDiff := sizeHead - oldSizeHead - for { - header, err := readHeader(s.f, ver) - if err != nil { - newFile.Close() - return err - } - if header == nil { - break - } - oldSizeData := (1 << header.sizeByte) - oldSizeHead - sizeb, size := utils.NextPowerOf2(sizeHead + uint32(header.keyLength) + header.valLength) - header.sizeByte = sizeb - - b := make([]byte, size+sizeDiff) - writeHeader(b, header) - n, err := s.f.Read(b[sizeHead : sizeHead+oldSizeData]) - if err != nil { - return err - } - if n != int(oldSizeData) { - return fmt.Errorf("wrong shart len: %d", n) - } + return nil +} - if header.status == deleted || (header.expire != 0 && int64(header.expire) < time.Now().Unix()) { - continue - } +func (s *Shard) Open(name string) error { + s.Lock() + defer s.Unlock() - startPos := int(sizeHead) + int(header.valLength) - endPos := int(sizeHead) + int(header.valLength) + int(header.keyLength) - h := murmur3.Sum32WithSeed(b[startPos:endPos], 0) + forceExit = false - s.mapping[h] = Encode(seek, header.sizeByte, header.expire) - n, err = newFile.Write(b[0:size]) - if err != nil { - return err - } - seek += uint32(n) - } + f, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, os.FileMode(0644)) + if err != nil { + return err + } - err = s.f.Close() - if err != nil { - return err - } + err = f.Sync() + if err != nil { + return err + } - s.f = newFile - err = os.Remove(name) - if err != nil { - return err - } + s.f = f + s.mapping = make(map[uint32]uint64) + s.remapping = make(map[uint32]byte) + fi, err := s.f.Stat() - err = os.Rename(newName, name) - if err != nil { - return err - } - return nil - } + if err != nil { + return err + } - for { - header, err := readHeader(s.f, ver) - if err != nil { - return err - } - if header == nil { - break - } + // create a new file + if fi.Size() == 0 { + // write shard info to the file + _, err = s.f.Write([]byte{versionMarker, currentShardVer}) + if err != nil { + return err + } + return nil + } - _, err = s.f.Seek(int64(header.valLength), 1) - if err != nil { - return err - } + // read file + var offset uint32 + ver, err := getFileVer(s.f) + if err != nil { + return err + } - // read key - key, err := s.readKey(header.keyLength) - if err != nil { - return err - } - shift := 1 << header.sizeByte - // skip empty tail - res, err := s.f.Seek(int64(shift-int(header.keyLength)-int(header.valLength)-int(sizeHead)), 1) // skip empty tail - if err != nil { - return err - } + if ver < 0 || ver > currentShardVer { + return errors.New("unknown shard version in file " + name) + } - if header.status != deleted && (header.expire == 0 || int64(header.expire) >= time.Now().Unix()) { - h := murmur3.Sum32WithSeed(key, 0) - s.mapping[h] = Encode(seek, header.sizeByte, header.expire) - } else { - s.remapping[seek] = header.sizeByte - } + if ver == 0 { + s.f.Seek(0, 0) + } else { + offset = 2 + } - seek = uint32(res) - } + if ver < currentShardVer { + return s.upgrade(ver, name) } - return nil + + return s.writeHeader(ver, offset) } func (s *Shard) readKey(keyLen uint16) ([]byte, error) { @@ -355,7 +343,7 @@ func (s *Shard) write(k, v []byte, h, expire uint32) error { if pos < 0 { // append to the end of file - pos, err = s.f.Seek(0, 2) + pos, _ = s.f.Seek(0, 2) } _, err = s.f.WriteAt(b, pos) @@ -441,12 +429,6 @@ func (s *Shard) get(k []byte, h uint32) ([]byte, *Header, error) { return nil, nil, common.ErrKeyNotFound } -func (s *Shard) length() int { - s.RLock() - defer s.RUnlock() - return len(s.mapping) -} - func (s *Shard) Close() error { forceExit = true s.Lock() diff --git a/internal/db/store/shard/utils.go b/internal/db/store/shard/utils.go index 9f80f8d..61adfc7 100644 --- a/internal/db/store/shard/utils.go +++ b/internal/db/store/shard/utils.go @@ -1,5 +1,10 @@ package shard +import ( + "errors" + "os" +) + func Encode(addr uint32, size byte, expire uint32) uint64 { return uint64(addr)<<32 | uint64(size)<<24 | uint64(expire)>>9 } @@ -13,3 +18,24 @@ func Decode(key uint64) (uint32, byte, uint32) { } return add, size, expire } + +func getFileVer(file *os.File) (int, error) { + b := make([]byte, 2) + n, err := file.Read(b) + if err != nil { + return -1, err + } + if n != 2 { + return -1, errors.New("short file") + } + if b[0] == versionMarker { + if b[1] == 0 || b[1] == deleted { + return 0, nil + } + return int(b[1]), nil + } + if b[1] == 0 || b[1] == deleted { + return 0, nil + } + return -1, nil +} diff --git a/internal/db/store/store.go b/internal/db/store/store.go index 48c90ab..8bc21fc 100644 --- a/internal/db/store/store.go +++ b/internal/db/store/store.go @@ -4,15 +4,18 @@ import ( "compress/gzip" "errors" "fmt" - "github.com/denzelpenzel/nyx/internal/common" - "github.com/denzelpenzel/nyx/internal/db/store/shard" - "github.com/denzelpenzel/nyx/internal/interval" - "github.com/google/btree" - "github.com/spaolacci/murmur3" "io" "os" "sync" "time" + + "github.com/DenzelPenzel/nyx/internal/common" + "github.com/DenzelPenzel/nyx/internal/db/store/shard" + "github.com/DenzelPenzel/nyx/internal/interval" + "github.com/DenzelPenzel/nyx/internal/logging" + "github.com/google/btree" + "github.com/spaolacci/murmur3" + "go.uber.org/zap" ) type Store struct { @@ -77,13 +80,6 @@ func ShardsTotal(shards int) OptStore { } } -func ShardPrefix(prefix string) OptStore { - return func(s *Store) error { - s.prefix = prefix - return nil - } -} - // SyncInterval - how often fsync do, default 0 - OS will do it func SyncInterval(interv time.Duration) OptStore { return func(s *Store) error { @@ -104,12 +100,16 @@ func SyncInterval(interv time.Duration) OptStore { func ExpireInterval(interv time.Duration) OptStore { return func(s *Store) error { + logger := logging.NoContext() s.expireInterval = interv if interv > 0 { s.expInterv = interval.SetInterval(func(_ time.Time) { err := s.shards[s.expireShardSeq].ExpireKeys(interv) if err != nil { - fmt.Printf("Error expire:%s\n", err) + logger.Warn("Error expire shard", + zap.Int("shard seq", s.expireShardSeq), + zap.Error(err), + ) } s.expireShardSeq++ if s.expireShardSeq >= s.shardsCount { @@ -143,7 +143,7 @@ func Open(opts ...OptStore) (*Store, error) { stopWorkers := false s.shards = make([]shard.Shard, s.shardsCount) - shChan := make(chan int, s.shardsCount) + shardsChan := make(chan int, s.shardsCount) errChan := make(chan error, 4) var wg sync.WaitGroup @@ -151,19 +151,17 @@ func Open(opts ...OptStore) (*Store, error) { for i := 0; i < 4; i++ { wg.Add(1) go func() { - for i := range shChan { + for i := range shardsChan { if stopWorkers { break } - var filename string if s.prefix != "" { filename = fmt.Sprintf("%s/%s-%d", s.dir, s.prefix, i) } else { filename = fmt.Sprintf("%s/%d", s.dir, i) } - - err := s.shards[i].Run(filename) + err := s.shards[i].Open(filename) if err != nil { errChan <- err stopWorkers = true @@ -175,11 +173,10 @@ func Open(opts ...OptStore) (*Store, error) { } for i := range s.shards { - shChan <- i + shardsChan <- i } - close(shChan) - + close(shardsChan) wg.Wait() if len(errChan) > 0 { @@ -187,8 +184,6 @@ func Open(opts ...OptStore) (*Store, error) { return s, err } - s.btree = btree.New(32) - return s, nil } @@ -276,7 +271,6 @@ func (s *Store) Count() int { // Close ... close related shards func (s *Store) Close() error { - errStr := "" if s.syncInterval > 0 { s.interv.Clear() } @@ -286,13 +280,9 @@ func (s *Store) Close() error { for i := range s.shards { err := s.shards[i].Close() if err != nil { - errStr += err.Error() + "\r\n" return err } } - if errStr != "" { - return errors.New(errStr) - } return nil } diff --git a/internal/db/store/store_test.go b/internal/db/store/store_test.go index c10c762..b92e928 100644 --- a/internal/db/store/store_test.go +++ b/internal/db/store/store_test.go @@ -4,15 +4,15 @@ import ( "bytes" "encoding/binary" "errors" - "fmt" - "github.com/denzelpenzel/nyx/internal/common" - "github.com/denzelpenzel/nyx/internal/utils" - "github.com/spaolacci/murmur3" - "github.com/stretchr/testify/require" - "github.com/tidwall/lotsa" "os" "runtime" "testing" + + "github.com/DenzelPenzel/nyx/internal/common" + "github.com/DenzelPenzel/nyx/internal/utils" + "github.com/spaolacci/murmur3" + "github.com/stretchr/testify/require" + "github.com/tidwall/lotsa" ) var dirName = "-db-tmp-test-" @@ -113,18 +113,19 @@ func Test_ReadAfterClose(t *testing.T) { require.Equal(t, 1, s.Count()) } -func Test_HashCollision(_ *testing.T) { +func Test_HashCollision(t *testing.T) { mapping := make(map[uint32]int, 100_000_000) + colCnt := 0 for i := 0; i < 100_000_000; i++ { k1 := make([]byte, 8) binary.BigEndian.PutUint64(k1, uint64(i)) h := murmur3.Sum32WithSeed(k1, 0) - if val, ok := mapping[h]; ok { - println("collision", h, i, val) - return + if _, ok := mapping[h]; ok { + colCnt++ } mapping[h] = i } + require.Equal(t, 0, colCnt) } func Test_ManyKeysOp(t *testing.T) { @@ -142,8 +143,6 @@ func Test_ManyKeysOp(t *testing.T) { var ms runtime.MemStats runtime.ReadMemStats(&ms) - fmt.Printf("Alloc = %v MiB Total = %v MiB\n", ms.Alloc/1024/1024, ms.TotalAlloc/1024/1024) - fmt.Print("Test Set key/val operation: ") collCnt := 0 lotsa.Ops(n, workers, func(i, _ int) { b := make([]byte, 8) @@ -159,26 +158,22 @@ func Test_ManyKeysOp(t *testing.T) { }) runtime.ReadMemStats(&ms) - fmt.Printf("Alloc = %v MiB Total = %v MiB Coll=%d\n", ms.Alloc/1024/1024, ms.TotalAlloc/1024/1024, collCnt) - fmt.Print("Test Get operation: ") + lotsa.Ops(n, workers, func(i, _ int) { b, err := s.Get(keys[i]) if err != nil { - fmt.Printf("failed to get the key %s\n", string(keys[i])) panic(err) } v := binary.BigEndian.Uint64(b) if uint64(i) != v { - fmt.Printf("wrong key value, key: %s val: %d\n", string(keys[i]), v) panic("wrong value") } }) runtime.ReadMemStats(&ms) - fmt.Printf("Alloc = %v MiB Total = %v MiB\n", ms.Alloc/1024/1024, ms.TotalAlloc/1024/1024) - fmt.Print("Test Delete operation: ") + lotsa.Ops(n, workers, func(i, _ int) { _, err := s.Delete(keys[i]) // skip the ErrKeyNotFound issue check diff --git a/internal/logging/logging.go b/internal/logging/logging.go index eedb048..2b1db3d 100644 --- a/internal/logging/logging.go +++ b/internal/logging/logging.go @@ -3,7 +3,8 @@ package logging import ( "context" "encoding/json" - "github.com/denzelpenzel/nyx/internal/common" + + "github.com/DenzelPenzel/nyx/internal/common" "go.uber.org/zap" "go.uber.org/zap/buffer" "go.uber.org/zap/zapcore" diff --git a/internal/nyx/nyx.go b/internal/nyx/nyx.go index 3ccd052..27df8e2 100644 --- a/internal/nyx/nyx.go +++ b/internal/nyx/nyx.go @@ -1,39 +1,21 @@ package nyx import ( - "github.com/denzelpenzel/nyx/internal/common" - "github.com/denzelpenzel/nyx/internal/db" - "github.com/denzelpenzel/nyx/internal/proto" + "github.com/DenzelPenzel/nyx/internal/common" + "github.com/DenzelPenzel/nyx/internal/db" + "github.com/DenzelPenzel/nyx/internal/proto" ) type HandlerConst func() db.DB -type NConst func(d db.DB, res proto.Responder) NyxDB - -type NyxDB interface { - Set(req common.SetRequest) error - Add(req common.SetRequest) error - Replace(req common.SetRequest) error - Append(req common.SetRequest) error - Prepend(req common.SetRequest) error - Delete(req common.DeleteRequest) error - Touch(req common.TouchRequest) error - Get(req common.GetRequest) error - GetE(req common.GetRequest) error - Gat(req common.GATRequest) error - Noop(req common.NoopRequest) error - Quit(req common.QuitRequest) error - Version(req common.VersionRequest) error - Unknown(req common.Request) error - Error(req common.Request, reqType common.RequestType, err error) -} +type NConst func(d db.DB, res proto.Responder) common.DBHandler type Nyx struct { db db.DB res proto.Responder } -func NewNyx(d db.DB, res proto.Responder) NyxDB { +func NewNyx(d db.DB, res proto.Responder) common.DBHandler { return &Nyx{ db: d, res: res, diff --git a/internal/proto/proto.go b/internal/proto/proto.go index 18e8010..9cfba1b 100644 --- a/internal/proto/proto.go +++ b/internal/proto/proto.go @@ -2,7 +2,8 @@ package proto import ( "bufio" - "github.com/denzelpenzel/nyx/internal/common" + + "github.com/DenzelPenzel/nyx/internal/common" ) type RequestParser interface { diff --git a/internal/proto/textprot/components.go b/internal/proto/textprot/components.go index 09fb234..95a54a4 100644 --- a/internal/proto/textprot/components.go +++ b/internal/proto/textprot/components.go @@ -2,7 +2,8 @@ package textprot import ( "bufio" - "github.com/denzelpenzel/nyx/internal/proto" + + "github.com/DenzelPenzel/nyx/internal/proto" ) // Components ... Holder for all the different protocol components in the textprot package diff --git a/internal/proto/textprot/parser.go b/internal/proto/textprot/parser.go index f678d44..6178c4c 100644 --- a/internal/proto/textprot/parser.go +++ b/internal/proto/textprot/parser.go @@ -3,12 +3,13 @@ package textprot import ( "bufio" "errors" - "github.com/denzelpenzel/nyx/internal/common" "io" "log" "strconv" "strings" "time" + + "github.com/DenzelPenzel/nyx/internal/common" ) type ParserText struct { diff --git a/internal/proto/textprot/respond.go b/internal/proto/textprot/respond.go index bcc9c4c..67c2a15 100644 --- a/internal/proto/textprot/respond.go +++ b/internal/proto/textprot/respond.go @@ -4,7 +4,8 @@ import ( "bufio" "errors" "fmt" - "github.com/denzelpenzel/nyx/internal/common" + + "github.com/DenzelPenzel/nyx/internal/common" ) type ResponderText struct { diff --git a/internal/server/listen.go b/internal/server/listen.go index d8d3f70..6cd0766 100644 --- a/internal/server/listen.go +++ b/internal/server/listen.go @@ -4,16 +4,17 @@ import ( "bufio" "context" "errors" - "github.com/denzelpenzel/nyx/internal/db" - "github.com/denzelpenzel/nyx/internal/logging" - "github.com/denzelpenzel/nyx/internal/nyx" - "github.com/denzelpenzel/nyx/internal/proto" - "github.com/denzelpenzel/nyx/internal/proto/textprot" - "go.uber.org/zap" "io" "log" "net" "time" + + "github.com/DenzelPenzel/nyx/internal/db" + "github.com/DenzelPenzel/nyx/internal/logging" + "github.com/DenzelPenzel/nyx/internal/nyx" + "github.com/DenzelPenzel/nyx/internal/proto" + "github.com/DenzelPenzel/nyx/internal/proto/textprot" + "go.uber.org/zap" ) // ListenConst is a constructor function for listener implementations @@ -36,7 +37,11 @@ func (l *tcpListener) Accept() (net.Conn, error) { } func (l *tcpListener) Configure(conn net.Conn) (net.Conn, error) { - tcpRemote := conn.(*net.TCPConn) + tcpRemote, ok := conn.(*net.TCPConn) + + if !ok { + return nil, errors.New("wrong conn type") + } if err := tcpRemote.SetKeepAlive(true); err != nil { return conn, err diff --git a/internal/server/server.go b/internal/server/server.go index a0a1843..0c19d7d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1,17 +1,18 @@ package server import ( + "errors" "fmt" - "github.com/denzelpenzel/nyx/internal/common" - "github.com/denzelpenzel/nyx/internal/logging" - "github.com/denzelpenzel/nyx/internal/nyx" - "github.com/denzelpenzel/nyx/internal/proto" - "github.com/denzelpenzel/nyx/internal/utils" - "go.uber.org/zap" "io" + + "github.com/DenzelPenzel/nyx/internal/common" + "github.com/DenzelPenzel/nyx/internal/logging" + "github.com/DenzelPenzel/nyx/internal/proto" + "github.com/DenzelPenzel/nyx/internal/utils" + "go.uber.org/zap" ) -type SrvConst func(conns []io.Closer, rp proto.RequestParser, n nyx.NyxDB) Server +type SrvConst func(conns []io.Closer, rp proto.RequestParser, n common.DBHandler) Server type Server interface { Loop() @@ -19,11 +20,11 @@ type Server interface { type DefaultServer struct { rp proto.RequestParser - n nyx.NyxDB + n common.DBHandler conns []io.Closer } -func NewServer(conns []io.Closer, rp proto.RequestParser, n nyx.NyxDB) Server { +func NewServer(conns []io.Closer, rp proto.RequestParser, n common.DBHandler) Server { return &DefaultServer{ n: n, rp: rp, @@ -36,7 +37,7 @@ func (s *DefaultServer) Loop() { defer func() { if r := recover(); r != nil { - if r != io.EOF { + if err, ok := r.(error); ok && !errors.Is(err, io.EOF) { logger.Fatal("recover from runtime panic", zap.Any("recover", r), zap.String("path", utils.IdentifyPanic()), diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 638de59..3bb7453 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -1,12 +1,13 @@ package server_test import ( - "github.com/denzelpenzel/nyx/internal/common" - "github.com/denzelpenzel/nyx/internal/server" - "github.com/stretchr/testify/require" "io" "runtime" "testing" + + "github.com/DenzelPenzel/nyx/internal/common" + "github.com/DenzelPenzel/nyx/internal/server" + "github.com/stretchr/testify/require" ) type ioCloserWrp struct { diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 52113e0..37d7ca7 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -41,7 +41,6 @@ func GetTCPAddr(addr string) (*net.TCPAddr, error) { func TempDir(name string) string { path, err := os.MkdirTemp("", name) - fmt.Printf("db path: %s\n", path) if err != nil { panic("failed to create temp dir") } diff --git a/makefile b/makefile index e22c39b..78442ef 100644 --- a/makefile +++ b/makefile @@ -33,7 +33,7 @@ run-app: .PHONY: test test: - go test ./internal/... -timeout $(TEST_LIMIT) + go test -v ./internal/... -timeout $(TEST_LIMIT) .PHONY: lint lint: @@ -50,16 +50,16 @@ lint: gosec: @echo "$(GREEN) Running security scan with gosec...$(COLOR_END)" - gosec ./... + gosec -exclude G104,G304 ./... # ============================================================================== # Load testing -client-load-test: - @echo "$(BLUE)» run client load tests... $(COLOR_END)" - @CGO_ENABLED=1 go build -a -o bin/client ./cmd/client +client-rand-load-test: + @echo "$(BLUE)» run client rand load tests... $(COLOR_END)" + @CGO_ENABLED=1 go build -a -o bin/rand ./cmd/client/rand @echo "Binary successfully built" - @./bin/client -num-ops 100000 -num-workers 10 + @./bin/rand -num-ops 100000 -num-workers 10 curl-set: curl -X POST "localhost:4001/api/set" -H "Authorization: Bearer 123" -H "Content-Type: application/json" -d '{"key": "foo", "value": "bar", "exp": 3600}'