From 4bc26af6ca6bf771fd00e563b56ee20253c35e80 Mon Sep 17 00:00:00 2001 From: Dominik Richter Date: Wed, 20 Sep 2023 00:51:29 -0700 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20make=20providers/dist=20(#1796)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a build mode which (1) uses dist ldflags stripping all extra information from binaries and (2) statically links them, which matches the desired default distribution model. I had initially thought static linking would cause our binary size to increase, but curiously the opposite happened. The total size decreased by 4% on average across all providers. Additionally I tried the approach described by https://mt165.co.uk/blog/static-link-go/ with `-ldflags "-linkmode 'external' -extldflags '-static'"`. However, adding these two flags made things go haywire for me. Here are all 3 modes for all providers, `static` is `CGO_ENABLED=0` and `ldstatic` is with the above described flags in addition to cgo (tried with and without the mode, enabling it made things worse). Metadata info: ```bash > file ./dynamic/os ./dynamic/os: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, Go BuildID=fCJrahsXjKkjQs5MpD-W/zLViu-ustsl7O4003jUg/ekClRLJ1jK6oA7YygTfs/rXCvVpZNTzHOBj492AaP, stripped > file ./static/os ./static/os: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, Go BuildID=b3TlsmVkNeIgjt3Cu27P/muUGnVrC8e8TweOltimv/OvnHty3keb6BZpfGHBlf/RXgIHwoO7uUrRUXU0Dx8, stripped > file ./ldstatic/os ./ldstatic/os: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, BuildID[sha1]=e842577213cbdc978bcfa0a5b30da9b5b30af007, for GNU/Linux 4.4.0, stripped ``` Execution attempts: ```bash > ./dynamic/os This binary is a plugin. These are not meant to be executed directly. Please execute the program that consumes these plugins, which will load any plugins automatically # ie: works > ./static/os This binary is a plugin. These are not meant to be executed directly. Please execute the program that consumes these plugins, which will load any plugins automatically # ie: works > ./ldstatic/os [1] 847039 trace trap (core dumped) ./ldstatic/os # ie: crashes ``` Also trying the `ldstatic` approach with cnquery: ``` > cnquery shell → no provider specified, defaulting to local. Use --help to see all providers. FTL failed to start provider os error="failed to initialize plugin client: Unrecognized remote plugin message: \nFailed to read any lines from plugin's stdout\nThis usually means\n the plugin was not compiled for this architecture,\n the plugin is missing dynamic-link libraries necessary to run,\n the plugin is not executable by this process due to file permissions, or\n the plugin failed to negotiate the initial go-plugin protocol handshake\n\nAdditional notes about plugin:\n Path: /home/zero/.config/mondoo/providers/os/os\n Mode: -rwxr-xr-x\n Owner: 1000 [zero] (current: 1000 [zero])\n Group: 1000 [zero] (current: 1000 [zero])\n ELF architecture: EM_X86_64 (current architecture: amd64)\n" ``` It works with the regular static. Signed-off-by: Dominik Richter --- Makefile | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index fdbb99ce99..445711dcbe 100644 --- a/Makefile +++ b/Makefile @@ -85,6 +85,22 @@ define buildProvider cd ${$@_HOME} && GOOS=${TARGETOS} go build -o ${$@_DIST_BIN}${BIN_SUFFIX} ./main.go endef +define buildProviderDist + $(eval $@_HOME = $(1)) + $(eval $@_NAME = $(shell basename ${$@_HOME})) + $(eval $@_DIST = "${$@_HOME}"/dist) + $(eval $@_DIST_BIN = "./dist/${$@_NAME}") + $(eval $@_BIN = "${$@_DIST}"/"${$@_NAME}") + echo "--> [${$@_NAME}] process resources" + ./lr go ${$@_HOME}/resources/${$@_NAME}.lr --dist ${$@_DIST} + ./lr docs yaml ${$@_HOME}/resources/${$@_NAME}.lr --docs-file ${$@_HOME}/resources/${$@_NAME}.lr.manifest.yaml + ./lr docs json ${$@_HOME}/resources/${$@_NAME}.lr.manifest.yaml + echo "--> [${$@_NAME}] generate CLI json" + cd ${$@_HOME} && go run ./gen/main.go . + echo "--> [${$@_NAME}] creating ${$@_BIN}" + cd ${$@_HOME} && CGO_ENABLED=0 GOOS=${TARGETOS} go build ${LDFLAGSDIST} -o ${$@_DIST_BIN}${BIN_SUFFIX} ./main.go +endef + define installProvider $(eval $@_HOME = $(1)) $(eval $@_NAME = $(shell basename ${$@_HOME})) @@ -244,6 +260,28 @@ providers/build/aws: providers/lr providers/build/ms365: providers/lr @$(call buildProvider, providers/ms365) +providers/dist: + @$(call buildProviderDist, providers/network) + @$(call buildProviderDist, providers/os) + @$(call buildProviderDist, providers/ipmi) + @$(call buildProviderDist, providers/oci) + @$(call buildProviderDist, providers/slack) + @$(call buildProviderDist, providers/github) + @$(call buildProviderDist, providers/gitlab) + @$(call buildProviderDist, providers/terraform) + @$(call buildProviderDist, providers/vsphere) + @$(call buildProviderDist, providers/opcua) + @$(call buildProviderDist, providers/okta) + @$(call buildProviderDist, providers/google-workspace) + @$(call buildProviderDist, providers/arista) + @$(call buildProviderDist, providers/equinix) + @$(call buildProviderDist, providers/vcd) + @$(call buildProviderDist, providers/gcp) + @$(call buildProviderDist, providers/k8s) + @$(call buildProviderDist, providers/azure) + @$(call buildProviderDist, providers/ms365) + @$(call buildProviderDist, providers/aws) + providers/install: # @$(call installProvider, providers/core) @$(call installProvider, providers/network) @@ -267,7 +305,6 @@ providers/install: @$(call installProvider, providers/ms365) @$(call installProvider, providers/aws) - providers/bundle: @$(call bundleProvider, providers/network) @$(call bundleProvider, providers/os) @@ -290,7 +327,6 @@ providers/bundle: @$(call bundleProvider, providers/ms365) @$(call bundleProvider, providers/aws) - providers/test: @$(call testProvider, providers/core) @$(call testProvider, providers/network)