From ea39074e171ca200fcb8f7f7bbcdf2e547f7e90f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 22 Sep 2023 13:31:49 -0400 Subject: [PATCH] feat: Better lint and test for devs and Makefile cleanup. This change does several things all towards the end of making it easier for developers to run tests and lint on go code. Now a developer can run either 'make go-test' or 'make lint' without having lots of C dependencies or running in a specific container or stacker. Things changed: * Add a build tag 'skipembed' that builds without needing cmd/stacker/lxc-wrapper/lxc-wrapper . This allows you to build (or test) without that file. In order to ensure that you do not get very far trying to run a 'stacker' binary, I've added a 'panic' if stacker binary is built with skipembed. * Add download-tools target to makefile to ease in downloading of regctl and zot. * Move the downloading of golangci-lint from the github workflow to Makefile. This makes it easier for developer to get it. It is also added to the 'download-tools' target. * be specific about the version of golangci-lint that is used. 1.54.2. golangci-lint's docs specifically say to do this: > IMPORTANT: It's highly recommended installing a specific version > of golangci-lint available on the releases page. changing it is simply a matter of changing the value in the makefile. * Move running of 'go test' out of the lint target to its own 'go-test' target. go-test target will now write a coverage.html so you can view coverage easily. * add 'dlbin' makefile "function" for downloading binaries, and use that for regctl and zot from their rules. * rename TOOLS_DIR to TOOLS_D (consistency with BUILD_D, and just shorter) * move coverage.txt output from test to hack/coverage.txt for easier git-ignoring. Signed-off-by: Scott Moser --- .github/workflows/build.yaml | 4 ++-- .gitignore | 2 ++ Makefile | 44 +++++++++++++++++++++++++----------- cmd/stacker/main.go | 7 +++--- cmd/stacker/main_embed.go | 10 ++++++++ cmd/stacker/main_noembed.go | 9 ++++++++ 6 files changed, 57 insertions(+), 19 deletions(-) create mode 100644 cmd/stacker/main_embed.go create mode 100644 cmd/stacker/main_noembed.go diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1168f3f2..6c0f7555 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -65,9 +65,9 @@ jobs: sudo add-apt-repository -y ppa:project-machine/squashfuse sudo apt-get update sudo apt-get install -yy lxc-utils lxc-dev libacl1-dev jq libcap-dev libseccomp-dev libpam-dev bats parallel libzstd-dev - GO111MODULE=off go get github.com/opencontainers/umoci/cmd/umoci - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin sudo apt-get install -yy autoconf automake make autogen autoconf libtool binutils git squashfs-tools libcryptsetup-dev libdevmapper-dev cryptsetup-bin squashfuse + GO111MODULE=off go get github.com/opencontainers/umoci/cmd/umoci + make download-tools echo "running kernel is: $(uname -a)" - name: Go-download run: | diff --git a/.gitignore b/.gitignore index f380af57..2c89f7fd 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ /stacker-dynamic .build +hack/ + # IDEs .vscode .idea diff --git a/Makefile b/Makefile index 35fbbf9e..926ebb13 100644 --- a/Makefile +++ b/Makefile @@ -24,14 +24,18 @@ STACKER_BUILD_UBUNTU_IMAGE?=$(STACKER_DOCKER_BASE)ubuntu:latest LXC_CLONE_URL?=https://github.com/lxc/lxc LXC_BRANCH?=stable-5.0 +HACK_D := $(TOP_LEVEL)/hack # helper tools -TOOLSDIR := $(shell pwd)/hack/tools -REGCLIENT := $(TOOLSDIR)/bin/regctl +TOOLS_D := $(HACK_D)/tools +REGCLIENT := $(TOOLS_D)/bin/regctl REGCLIENT_VERSION := v0.5.1 # OCI registry -ZOT := $(TOOLSDIR)/bin/zot +ZOT := $(TOOLS_D)/bin/zot ZOT_VERSION := 2.0.0-rc6 +GOLANGCI_LINT_VERSION = v1.54.2 +GOLANGCI_LINT = $(TOOLS_D)/golangci-lint/$(GOLANGCI_LINT_VERSION)/golangci-lint + STAGE1_STACKER ?= ./stacker-dynamic STACKER_DEPS = $(GO_SRC) go.mod go.sum @@ -61,22 +65,36 @@ go-download: go mod download .PHONY: lint -lint: cmd/stacker/lxc-wrapper/lxc-wrapper $(GO_SRC) +lint: $(GO_SRC) $(GOLANGCI_LINT) go mod tidy go fmt ./... && ([ -z $(CI) ] || git diff --exit-code) bash test/static-analysis.sh - go test -v -trimpath -cover -coverpkg stackerbuild.io/stacker/./... -coverprofile=coverage.txt -covermode=atomic -tags "$(BUILD_TAGS)" stackerbuild.io/stacker/./... - $(shell go env GOPATH)/bin/golangci-lint run --build-tags "$(BUILD_TAGS)" + $(GOLANGCI_LINT) run --build-tags "$(BUILD_TAGS) skipembed" + +.PHONY: go-test +go-test: + go test -v -trimpath -cover -coverprofile=$(HACK_D)/coverage.txt -covermode=atomic -tags "exclude_graphdriver_btrfs exclude_graphdriver_devicemapper containers_image_openpgp osusergo netgo skipembed" ./... + go tool cover -html $(HACK_D)/coverage.txt -o $(HACK_D)/coverage.html + +.PHONY: download-tools +download-tools: $(GOLANGCI_LINT) $(REGCLIENT) $(ZOT) + +$(GOLANGCI_LINT): + @mkdir -p $(dir $@) + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(dir $@)" + @mkdir -p "$(TOOLS_D)/bin" + ln -sf "$@" "$(TOOLS_D)/bin/" + +# dlbin is used with $(call dlbin,path,url) +# it downloads a url to path and makes it executable. +# it creates dest dir and atomically moves into place. t gets .pid +dlbin = set -x; mkdir -p $(dir $1) && t=$1.$$$$ && curl -Lo "$$t" "$2" && chmod +x "$$t" && mv "$$t" "$1" $(REGCLIENT): - mkdir -p $(TOOLSDIR)/bin - curl -Lo $(REGCLIENT) https://github.com/regclient/regclient/releases/download/$(REGCLIENT_VERSION)/regctl-linux-amd64 - chmod +x $(REGCLIENT) + $(call dlbin,$@,https://github.com/regclient/regclient/releases/download/$(REGCLIENT_VERSION)/regctl-linux-amd64) $(ZOT): - mkdir -p $(TOOLSDIR)/bin - curl -Lo $(ZOT) https://github.com/project-zot/zot/releases/download/v$(ZOT_VERSION)/zot-linux-amd64-minimal - chmod +x $(ZOT) + $(call dlbin,$@,https://github.com/regclient/regclient/releases/download/$(REGCLIENT_VERSION)/regctl-linux-amd64) TEST?=$(patsubst test/%.bats,%,$(wildcard test/*.bats)) PRIVILEGE_LEVEL?= @@ -84,7 +102,7 @@ PRIVILEGE_LEVEL?= # make check TEST=basic will run only the basic test # make check PRIVILEGE_LEVEL=unpriv will run only unprivileged tests .PHONY: check -check: lint test +check: lint test go-test .PHONY: test test: stacker $(REGCLIENT) $(ZOT) diff --git a/cmd/stacker/main.go b/cmd/stacker/main.go index ae4bf4b5..5ab577a0 100644 --- a/cmd/stacker/main.go +++ b/cmd/stacker/main.go @@ -1,7 +1,6 @@ package main import ( - "embed" "fmt" "os" "os/exec" @@ -28,9 +27,6 @@ var ( lxc_version = "" ) -//go:embed lxc-wrapper/lxc-wrapper -var embeddedFS embed.FS - func shouldShowProgress(ctx *cli.Context) bool { /* if the user provided explicit recommendations, follow those */ if ctx.Bool("no-progress") { @@ -84,6 +80,9 @@ func shouldSkipInternalUserns(ctx *cli.Context) bool { } func main() { + if !hasEmbedded { + panic("stacker was built without embedded binaries.") + } sigquits := make(chan os.Signal, 1) go func() { for range sigquits { diff --git a/cmd/stacker/main_embed.go b/cmd/stacker/main_embed.go new file mode 100644 index 00000000..8108c423 --- /dev/null +++ b/cmd/stacker/main_embed.go @@ -0,0 +1,10 @@ +//go:build !skipembed + +package main + +import "embed" + +//go:embed lxc-wrapper/lxc-wrapper +var embeddedFS embed.FS + +const hasEmbedded = true diff --git a/cmd/stacker/main_noembed.go b/cmd/stacker/main_noembed.go new file mode 100644 index 00000000..2790c3ea --- /dev/null +++ b/cmd/stacker/main_noembed.go @@ -0,0 +1,9 @@ +//go:build skipembed + +package main + +import "embed" + +var embeddedFS embed.FS + +const hasEmbedded = true