From f06b5015adc8794a88bb77c51413d040e225f3d8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 22 Sep 2023 13:31:49 -0400 Subject: [PATCH] feat: Better lint and test for devs and Makefile cleanup. This change does several things all towards the end of making it easier for developers to run tests and lint on go code. Now a developer can run either 'make go-test' or 'make lint' without having lots of C dependencies or running in a specific container or stacker. Things changed: * Add a build tag 'skipembed' that builds without needing cmd/stacker/lxc-wrapper/lxc-wrapper . This allows you to build (or test) without that file. In order to ensure that you do not get very far trying to run a 'stacker' binary, I've added a 'panic' if stacker binary is built with skipembed. * Add download-tools target to makefile to ease in downloading of regctl and zot. * Move the downloading of golangci-lint from the github workflow to Makefile. This makes it easier for developer to get it. It is also added to the 'download-tools' target. * be specific about the version of golangci-lint that is used. 1.54.2. golangci-lint's docs specifically say to do this: > IMPORTANT: It's highly recommended installing a specific version > of golangci-lint available on the releases page. changing it is simply a matter of changing the value in the makefile. * Move running of 'go test' out of the lint target to its own 'go-test' target. go-test target will now write a coverage.html so you can view coverage easily. * add 'dlbin' makefile "function" for downloading binaries, and use that for regctl and zot from their rules. Signed-off-by: Scott Moser --- .github/workflows/build.yaml | 4 ++-- .gitignore | 2 ++ Makefile | 43 +++++++++++++++++++++++++----------- cmd/stacker/main.go | 7 +++--- cmd/stacker/main_embed.go | 10 +++++++++ cmd/stacker/main_noembed.go | 9 ++++++++ 6 files changed, 56 insertions(+), 19 deletions(-) create mode 100644 cmd/stacker/main_embed.go create mode 100644 cmd/stacker/main_noembed.go diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1168f3f2..6c0f7555 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -65,9 +65,9 @@ jobs: sudo add-apt-repository -y ppa:project-machine/squashfuse sudo apt-get update sudo apt-get install -yy lxc-utils lxc-dev libacl1-dev jq libcap-dev libseccomp-dev libpam-dev bats parallel libzstd-dev - GO111MODULE=off go get github.com/opencontainers/umoci/cmd/umoci - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin sudo apt-get install -yy autoconf automake make autogen autoconf libtool binutils git squashfs-tools libcryptsetup-dev libdevmapper-dev cryptsetup-bin squashfuse + GO111MODULE=off go get github.com/opencontainers/umoci/cmd/umoci + make download-tools echo "running kernel is: $(uname -a)" - name: Go-download run: | diff --git a/.gitignore b/.gitignore index f380af57..5e99f24f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ /stacker /stacker-dynamic .build +coverage.html +coverage.txt # IDEs .vscode diff --git a/Makefile b/Makefile index 35fbbf9e..4c97adee 100644 --- a/Makefile +++ b/Makefile @@ -25,13 +25,16 @@ LXC_CLONE_URL?=https://github.com/lxc/lxc LXC_BRANCH?=stable-5.0 # helper tools -TOOLSDIR := $(shell pwd)/hack/tools -REGCLIENT := $(TOOLSDIR)/bin/regctl +TOOLS_D := $(TOP_LEVEL)/hack/tools +REGCLIENT := $(TOOLS_D)/bin/regctl REGCLIENT_VERSION := v0.5.1 # OCI registry -ZOT := $(TOOLSDIR)/bin/zot +ZOT := $(TOOLS_D)/bin/zot ZOT_VERSION := 2.0.0-rc6 +GOLANGCI_LINT_VERSION = v1.54.2 +GOLANGCI_LINT = $(TOOLS_D)/golangci-lint/$(GOLANGCI_LINT_VERSION)/golangci-lint + STAGE1_STACKER ?= ./stacker-dynamic STACKER_DEPS = $(GO_SRC) go.mod go.sum @@ -61,22 +64,36 @@ go-download: go mod download .PHONY: lint -lint: cmd/stacker/lxc-wrapper/lxc-wrapper $(GO_SRC) +lint: $(GO_SRC) $(GOLANGCI_LINT) go mod tidy go fmt ./... && ([ -z $(CI) ] || git diff --exit-code) bash test/static-analysis.sh - go test -v -trimpath -cover -coverpkg stackerbuild.io/stacker/./... -coverprofile=coverage.txt -covermode=atomic -tags "$(BUILD_TAGS)" stackerbuild.io/stacker/./... - $(shell go env GOPATH)/bin/golangci-lint run --build-tags "$(BUILD_TAGS)" + $(GOLANGCI_LINT) run --build-tags "$(BUILD_TAGS) skipembed" + +.PHONY: go-test +go-test: + go test -v -trimpath -cover -coverprofile=coverage.txt -covermode=atomic -tags "exclude_graphdriver_btrfs exclude_graphdriver_devicemapper containers_image_openpgp osusergo netgo skipembed" ./... + go tool cover -html coverage.txt -o coverage.html + +.PHONY: download-tools +download-tools: $(GOLANGCI_LINT) $(REGCLIENT) $(ZOT) + +$(GOLANGCI_LINT): + @mkdir -p $(dir $@) + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(dir $@)" + @mkdir -p "$(TOOLS_D)/bin" + ln -sf "$@" "$(TOOLS_D)/bin/" + +# dlbin is used with $(call dlbin,path,url) +# it downloads a url to path and makes it executable. +# it creates dest dir and atomically moves into place. t gets .pid +dlbin = set -x; mkdir -p $(dir $1) && t=$1.$$$$ && curl -Lo "$$t" "$2" && chmod +x "$$t" && mv "$$t" "$1" $(REGCLIENT): - mkdir -p $(TOOLSDIR)/bin - curl -Lo $(REGCLIENT) https://github.com/regclient/regclient/releases/download/$(REGCLIENT_VERSION)/regctl-linux-amd64 - chmod +x $(REGCLIENT) + $(call dlbin,$@,https://github.com/regclient/regclient/releases/download/$(REGCLIENT_VERSION)/regctl-linux-amd64) $(ZOT): - mkdir -p $(TOOLSDIR)/bin - curl -Lo $(ZOT) https://github.com/project-zot/zot/releases/download/v$(ZOT_VERSION)/zot-linux-amd64-minimal - chmod +x $(ZOT) + $(call dlbin,$@,https://github.com/regclient/regclient/releases/download/$(REGCLIENT_VERSION)/regctl-linux-amd64) TEST?=$(patsubst test/%.bats,%,$(wildcard test/*.bats)) PRIVILEGE_LEVEL?= @@ -84,7 +101,7 @@ PRIVILEGE_LEVEL?= # make check TEST=basic will run only the basic test # make check PRIVILEGE_LEVEL=unpriv will run only unprivileged tests .PHONY: check -check: lint test +check: lint test go-test .PHONY: test test: stacker $(REGCLIENT) $(ZOT) diff --git a/cmd/stacker/main.go b/cmd/stacker/main.go index ae4bf4b5..5ab577a0 100644 --- a/cmd/stacker/main.go +++ b/cmd/stacker/main.go @@ -1,7 +1,6 @@ package main import ( - "embed" "fmt" "os" "os/exec" @@ -28,9 +27,6 @@ var ( lxc_version = "" ) -//go:embed lxc-wrapper/lxc-wrapper -var embeddedFS embed.FS - func shouldShowProgress(ctx *cli.Context) bool { /* if the user provided explicit recommendations, follow those */ if ctx.Bool("no-progress") { @@ -84,6 +80,9 @@ func shouldSkipInternalUserns(ctx *cli.Context) bool { } func main() { + if !hasEmbedded { + panic("stacker was built without embedded binaries.") + } sigquits := make(chan os.Signal, 1) go func() { for range sigquits { diff --git a/cmd/stacker/main_embed.go b/cmd/stacker/main_embed.go new file mode 100644 index 00000000..8108c423 --- /dev/null +++ b/cmd/stacker/main_embed.go @@ -0,0 +1,10 @@ +//go:build !skipembed + +package main + +import "embed" + +//go:embed lxc-wrapper/lxc-wrapper +var embeddedFS embed.FS + +const hasEmbedded = true diff --git a/cmd/stacker/main_noembed.go b/cmd/stacker/main_noembed.go new file mode 100644 index 00000000..2790c3ea --- /dev/null +++ b/cmd/stacker/main_noembed.go @@ -0,0 +1,9 @@ +//go:build skipembed + +package main + +import "embed" + +var embeddedFS embed.FS + +const hasEmbedded = true