Merge pull request #17 from insolar/PENV-244-add-nightly-ci

PENV-244: add nightly ci jobs and clean Makefile
insolar · May 13, 2020 · ed29ba6 · ed29ba6
2 parents 821de8b + 40f3b56
commit ed29ba6
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 119 deletions.
diff --git a/.github/workflows/build_canceller.yaml b/.github/workflows/build_canceller.yaml
diff --git a/.github/workflows/common.yaml b/.github/workflows/common.yaml
@@ -1,9 +1,8 @@
 name: common
 on:
-  push:
-    branches:
-      - master
   pull_request:
+  schedule:
+    - cron: '0 2 * * *'
 env:
   GO_VERSION: 1.12.15
   CI_GOMAXPROCS: 0
@@ -15,6 +14,12 @@ jobs:
     env:
       GOPATH: /home/runner/work/mainnet/mainnet/go
     steps:
+      - name: set nightly run params
+        if: github.event_name == 'schedule'
+        run: |
+          echo "::set-env name=TEST_COUNT::300"
+          echo "::set-env name=TEST_PARALLEL::10"
+          echo "::set-env name=TEST_ARGS::-race -failfast -timeout 240m"
       - name: checkout code
         uses: actions/checkout@master
         with:
@@ -39,6 +44,12 @@ jobs:
     env:
       GOPATH: /home/runner/work/mainnet/mainnet/go
     steps:
+      - name: set nightly run params
+        if: github.event_name == 'schedule'
+        run: |
+          echo "::set-env name=TEST_COUNT::20"
+          echo "::set-env name=TEST_PARALLEL::1"
+          echo "::set-env name=TEST_ARGS::-race -timeout 240m"
       - name: checkout code
         uses: actions/checkout@master
         with:
@@ -98,7 +109,7 @@ jobs:
         run: echo "::add-path::${{env.GOPATH}}/bin"
       - name: generate codecov
         run: |
-          make ci-test-with-coverage
+          make test-with-coverage
           content=`cat coverage.txt | ./scripts/dev/cover-filter.sh`
           echo "$content" | tee $GITHUB_WORKSPACE/coverage.txt
           cat ./coverage.txt | wc
@@ -115,6 +126,12 @@ jobs:
     env:
       GOPATH: /home/runner/work/mainnet/mainnet/go
     steps:
+      - name: set nightly run params
+        if: github.event_name == 'schedule'
+        run: |
+          echo "::set-env name=TEST_COUNT::50"
+          echo "::set-env name=TEST_PARALLEL::10"
+          echo "::set-env name=TEST_ARGS::-race -timeout 240m"
       - name: checkout code
         uses: actions/checkout@master
         with:
@@ -149,6 +166,12 @@ jobs:
     env:
       GOPATH: /home/runner/work/mainnet/mainnet/go
     steps:
+      - name: set nightly run params
+        if: github.event_name == 'schedule'
+        run: |
+          echo "::set-env name=TEST_COUNT::50"
+          echo "::set-env name=TEST_PARALLEL::10"
+          echo "::set-env name=TEST_ARGS::-race -timeout 240m"
       - name: checkout code
         uses: actions/checkout@master
         with:

diff --git a/Makefile b/Makefile
@@ -15,17 +15,12 @@ GOBUILD ?= go build
 GOTEST ?= go test
 
 TEST_COUNT ?= 1
-FUNCTEST_COUNT ?= 1
+TEST_PARALLEL ?= 1
+TEST_ARGS ?= -timeout 1200s
+GOMAXPROCS ?= 0
 TESTED_PACKAGES ?= $(shell go list ${ALL_PACKAGES})
 COVERPROFILE ?= coverage.txt
-TEST_ARGS ?= -timeout 1200s
-TEST_ARGS_NIGHTLY ?= -timeout 1000m
 BUILD_TAGS ?=
-LONG_TESTS_SCRIPT_PATH = ./scripts/dev/run-tests-with-repeat.sh
-
-CI_GOMAXPROCS ?= 8
-CI_TEST_ARGS ?= -p 4
-CI_LONG_TIMEOUT ?= "1*60*60" # one hour
 
 BUILD_NUMBER := $(TRAVIS_BUILD_NUMBER)
 # skip git parsing commands if no git
@@ -56,10 +51,7 @@ submodule: ## init git submodule
 	git submodule update
 
 .PHONY: lint
-lint: ci-lint ## alias for ci-lint
-
-.PHONY: ci-lint
-ci-lint: ## CI lint
+lint: ## CI lint
 	golangci-lint run
 
 .PHONY: metalint
@@ -119,24 +111,27 @@ $(REQUESTER):
 
 .PHONY: test_unit
 test_unit: ## run all unit tests
-	CGO_ENABLED=1 $(GOTEST) -count=1 $(TEST_ARGS) $(ALL_PACKAGES)
+	GOMAXPROCS=$(GOMAXPROCS) CGO_ENABLED=1 \
+		$(GOTEST) -count=$(TEST_COUNT) -p=$(TEST_PARALLEL) $(TEST_ARGS) -json ./...| tee unit-test.log
 
 .PHONY: functest
 functest: ## run functest FUNCTEST_COUNT times
-	CGO_ENABLED=1 $(GOTEST) -test.v $(TEST_ARGS) -tags "functest" ./application/functest -count=$(FUNCTEST_COUNT)
+	GOMAXPROCS=$(GOMAXPROCS) CGO_ENABLED=1 \
+		$(GOTEST) -test.v -p=$(TEST_PARALLEL) $(TEST_ARGS) -tags "functest" ./application/functest -count=$(TEST_COUNT)
 
 .PNONY: functest_race
 functest_race: ## run functest 10 times with -race flag
 	make clean
 	GOBUILD='go build -race' make build
-	FUNCTEST_COUNT=10 make functest
+	TEST_COUNT=10 make functest
 
 .PHONY: test_func
 test_func: functest ## alias for functest
 
 .PHONY: test_slow
 test_slow: ## run tests with slowtest tag
-	CGO_ENABLED=1 $(GOTEST) -count=1 $(TEST_ARGS) -tags slowtest ./...
+	CGO_ENABLED=1 GOMAXPROCS=$(GOMAXPROCS) \
+		$(GOTEST) $(TEST_ARGS) -p=$(TEST_PARALLEL) -tags slowtest $(ALL_PACKAGES) -count=$(TEST_COUNT)
 
 .PHONY: test
 test: test_unit ## alias for test_unit
@@ -151,82 +146,15 @@ test_with_coverage: $(ARTIFACTS_DIR) ## run unit tests with generation of covera
 
 .PHONY: test_with_coverage_fast
 test_with_coverage_fast: ## ???
-	CGO_ENABLED=1 $(GOTEST) $(TEST_ARGS) -tags coverage -count 1 --coverprofile=$(COVERPROFILE) --covermode=count $(ALL_PACKAGES)
+	CGO_ENABLED=1 $(GOTEST) $(TEST_ARGS) -tags coverage -count $(TEST_COUNT) --coverprofile=$(COVERPROFILE) --covermode=count $(ALL_PACKAGES)
 
 $(ARTIFACTS_DIR):
 	mkdir -p $(ARTIFACTS_DIR)
 
-.PHONY: ci-test-with-coverage
-ci-test-with-coverage: ## run unit tests with coverage, outputs json to stdout (CI)
-	GOMAXPROCS=$(CI_GOMAXPROCS) CGO_ENABLED=1 \
-		$(GOTEST) $(CI_TEST_ARGS) $(TEST_ARGS) -json -v -count 1 --coverprofile=$(COVERPROFILE) --covermode=count -tags 'coverage' $(ALL_PACKAGES)
-
-.PHONY: ci-test-unit
-ci-test-unit: ## run unit tests 10 times and -race flag, redirects json output to file (CI)
-	GOMAXPROCS=$(CI_GOMAXPROCS) CGO_ENABLED=1 \
-		$(GOTEST) $(CI_TEST_ARGS) $(TEST_ARGS) -json -v $(ALL_PACKAGES) -race -count 10
-
-.PHONY: ci-test-unit-long
-ci-test-unit-long: ## run unit tests 10 times and -race flag, repeat until temeout exceeded
-	TESTS_TIMEOUT=$(CI_LONG_TIMEOUT) $(LONG_TESTS_SCRIPT_PATH) "ci-test-unit"
-
-.PHONY: ci-test-unit-nightly
-ci-test-unit-nightly: ## run unit tests 800 times and -race flag, repeat until temeout exceeded
-	GOMAXPROCS=$(CI_GOMAXPROCS) CGO_ENABLED=1 \
-         $(GOTEST) $(TEST_ARGS_NIGHTLY) -json -v -failfast -p 10 -race -count 800 $(ALL_PACKAGES)
-
-.PHONY: ci-test-slow
-ci-test-slow: ## run slow tests just once, redirects json output to file (CI)
-	GOMAXPROCS=$(CI_GOMAXPROCS) CGO_ENABLED=1 \
-		$(GOTEST) $(CI_TEST_ARGS) $(TEST_ARGS) -json -v -failfast -tags slowtest ./... -count=$(TEST_COUNT)
-
-.PHONY: ci-test-slow-long
-ci-test-slow-long: ## run slow tests with race and count
-	CI_TEST_ARGS=" -race " \
-	TEST_ARGS=" -timeout 180m  " \
-	TEST_COUNT=50 \
-		$(MAKE) ci-test-slow
-
-.PHONY: ci-test-slow-long-repeat
-ci-test-slow-long-repeat: ## run slow tests with race and count, repeat until temeout exceeded
-	TESTS_TIMEOUT=$(CI_LONG_TIMEOUT) $(LONG_TESTS_SCRIPT_PATH) "ci-test-slow-long"
-
-.PHONY: ci-test-slow-nightly
-ci-test-slow-nightly: ## run slow tests with race and count (nightly run)
-	CI_TEST_ARGS=" -race " \
-	TEST_ARGS=" -timeout 480m  " \
-	TEST_COUNT=80 \
-		$(MAKE) ci-test-slow
-
-.PHONY: ci-test-slow-nightly-repeat
-ci-test-slow-nightly-repeat: ## run slow tests with race and count (nightly run), repeat until temeout exceeded
-	TESTS_TIMEOUT=$(CI_LONG_TIMEOUT) $(LONG_TESTS_SCRIPT_PATH) "ci-test-slow-nightly"
-
-.PHONY: ci-test-func-base
-ci-test-func-base: ## run functest, redirects json output to file (CI)
-	# GOMAXPROCS=2, because we launch at least 5 insolard nodes in functest + 1 pulsar,
-	# so try to be more honest with processors allocation.
-	GOMAXPROCS=$(CI_GOMAXPROCS) CGO_ENABLED=1  \
-		$(GOTEST) $(CI_TEST_ARGS) $(TEST_ARGS) -json -tags "functest bloattest" -v ./application/functest -count=$(FUNCTEST_COUNT) -failfast
-
-.PHONY: ci-test-func
-ci-test-func:  ## run functest 3 times
-	FUNCTEST_COUNT=3 \
-		$(MAKE) ci-test-func-base
-
-.PHONY: ci-test-func-long
-ci-test-func-long: ## run functest with race and a little count
-	CI_TEST_ARGS=" -p 10 -race " \
-	TEST_ARGS=" -timeout 300m " \
-	FUNCTEST_COUNT=10 \
-		$(MAKE) ci-test-func-base
-
-.PHONY: ci-test-func-nightly
-ci-test-func-nightly: ## run functest with large count and race
-	CI_TEST_ARGS=" -p 10 -race " \
-	TEST_ARGS=" -timeout 1200m " \
-	FUNCTEST_COUNT=200 \
-		$(MAKE) ci-test-func-base
+.PHONY: test-with-coverage
+test-with-coverage: ## run unit tests with coverage, outputs json to stdout (CI)
+	GOMAXPROCS=$(GOMAXPROCS) CGO_ENABLED=1 \
+		$(GOTEST) $(TEST_ARGS) -json -v -count=$(TEST_COUNT) -p=$(TEST_PARALLEL) --coverprofile=$(COVERPROFILE) --covermode=count -tags 'coverage' $(ALL_PACKAGES)
 
 .PHONY: regen-builtin
 regen-builtin: ## regenerate builtin contracts code

diff --git a/scripts/dev/run-tests-with-repeat.sh b/scripts/dev/run-tests-with-repeat.sh