Split top-level go.mod from collector-specific module, multimod suppo…

…rt (#10)

Fixes f5/otel-arrow-adapter#113 belatedly.
Part of #3.

This uses the OTel-Golang toolchain for maintaining multi-module
repositories that is used by the OTel Collector. This at last splits
this repository into several modules, so that the primary Arrow-adapter
library can be incorporated without depending on everything used in all
of the utilities in the `collector/` subdirectory.

The top-level `go.mod` is split into two modules, the primary adapter
(top level) and the collector components.

The `otelarrowcol` package is now generated using the builder (making a
3rd module in the repo).

---------

Co-authored-by: Alex Boten <[email protected]>

Makefile

@@ -10,22 +10,111 @@
 MODULES := $(shell find . -name go.mod)
 
 GODIRS := $(foreach d,$(MODULES),$(shell dirname $d))
+GOCMD?= go
+GOOS := $(shell $(GOCMD) env GOOS)
+GOARCH := $(shell $(GOCMD) env GOARCH)
+BUILD_INFO=-ldflags "-X $(BUILD_INFO_IMPORT_PATH).Version=$(VERSION)"
+VERSION=$(shell git describe --always --match "v[0-9]*" HEAD)
+BUILD_INFO_IMPORT_PATH=go.opentelemetry.io/collector/internal/version
 
 .PHONY: all gotidy test build fmt
 
 all: gotidy test build
 
 test:
-	for dir in $(GODIRS); do (cd $${dir} && go test ./...); done
+	for dir in $(GODIRS); do (cd $${dir} && $(GOCMD) test ./...); done
 
 fmt:
-	for dir in $(GODIRS); do (cd $${dir} && go fmt ./...); done
+	for dir in $(GODIRS); do (cd $${dir} && $(GOCMD) fmt ./...); done
 
 build:
-	for dir in $(GODIRS); do (cd $${dir} && go build ./...); done
+	for dir in $(GODIRS); do (cd $${dir} && $(GOCMD) build ./...); done
 
 gotidy:
-	for dir in $(GODIRS); do (cd $${dir} && go mod tidy); done
+	for dir in $(GODIRS); do (cd $${dir} && $(GOCMD) mod tidy); done
 
 doc:
-	go run tools/data_model_gen/main.go
+	$(GOCMD) run tools/data_model_gen/main.go
+
+# Multimod can be installed using:
+#
+#   $(GOCMD) install github.com/open-telemetry/opentelemetry-go-build-tools/multimod@latest
+#
+# TODO install this locally
+MULTIMOD := multimod
+.PHONY: $(MULTIMOD)
+
+.PHONY: multimod-verify
+multimod-verify:
+	@echo "Validating versions.yaml"
+	$(MULTIMOD) verify
+
+MODSET?=beta
+.PHONY: multimod-prerelease
+multimod-prerelease: $(MULTIMOD)
+	$(MULTIMOD) prerelease -s=true -b=false -v ./versions.yaml -m ${MODSET}
+	$(MAKE) gotidy
+
+COMMIT?=HEAD
+REMOTE?[email protected]:open-telemetry/otel-arrow.git
+.PHONY: push-release
+push-release: $(MULTIMOD)
+	$(MULTIMOD) verify
+	set -e; for tag in `$(MULTIMOD) tag -m ${MODSET} -c ${COMMIT} --print-tags | grep -v "Using" `; do \
+		echo "pushing tag $${tag}"; \
+		git push ${REMOTE} $${tag}; \
+	done;
+
+.PHONY: prepare-release
+prepare-release:
+ifndef MODSET
+	@echo "MODSET not defined"
+	@echo "usage: make prepare-release RELEASE_CANDIDATE=<version eg 0.53.0> PREVIOUS_VERSION=<version eg 0.52.0> MODSET=beta"
+	exit 1
+endif
+ifdef PREVIOUS_VERSION
+	@echo "Previous version $(PREVIOUS_VERSION)"
+else
+	@echo "PREVIOUS_VERSION not defined"
+	@echo "usage: make prepare-release RELEASE_CANDIDATE=<version eg 0.53.0> PREVIOUS_VERSION=<version eg 0.52.0> MODSET=beta"
+	exit 1
+endif
+ifdef RELEASE_CANDIDATE
+	@echo "Preparing ${MODSET} release $(RELEASE_CANDIDATE)"
+else
+	@echo "RELEASE_CANDIDATE not defined"
+	@echo "usage: make prepare-release RELEASE_CANDIDATE=<version eg 0.53.0> PREVIOUS_VERSION=<version eg 0.52.0> MODSET=beta"
+	exit 1
+endif
+	# ensure a clean branch
+	git diff -s --exit-code || (echo "local repository not clean"; exit 1)
+	# update files with new version
+	sed -i.bak 's/$(PREVIOUS_VERSION)/$(RELEASE_CANDIDATE)/g' versions.yaml
+	sed -i.bak 's/$(PREVIOUS_VERSION)/$(RELEASE_CANDIDATE)/g' collector/cmd/otelarrowcol/build.yaml
+	find . -name "*.bak" -type f -delete
+	# commit changes before running multimod
+	git add .
+	git commit -m "prepare release $(RELEASE_CANDIDATE)"
+	$(MAKE) multimod-prerelease
+	# regenerate files
+	$(MAKE) genotelarrowcol
+	git add .
+	git commit -m "add multimod changes $(RELEASE_CANDIDATE)" || (echo "no multimod changes to commit")
+
+# OTC's builder can be installed using:
+#
+#   $(GOCMD) install go.opentelemetry.io/collector/cmd/builder@latest
+#
+# TODO install this locally
+BUILDER := builder
+.PHONY: $(BUILDER)
+
+.PHONY: genotelarrowcol
+genotelarrowcol:
+	$(BUILDER) --skip-compilation --config collector/cmd/otelarrowcol/build.yaml --output-path collector/cmd/otelarrowcol
+
+.PHONY: otelarrowcol
+otelarrowcol:
+	(cd collector/cmd/otelarrowcol && \
+		GO111MODULE=on CGO_ENABLED=0 \
+		$(GOCMD) build -trimpath -o ../../../bin/otelarrowcol_$(GOOS)_$(GOARCH) $(BUILD_INFO) .)

RELEASING.md

@@ -0,0 +1,38 @@
+To make a release of one or more Golang modules in the OTel Arrow
+repository, follow these steps.
+
+1. Using Git, checkout the version of the repo that will be released
+   and create a new branch for the release, for example,
+
+```
+git checkout main
+git pull upstream main
+git checkout -b release_xx_yy_zz
+```
+
+2. Using Make, prepare the release means updating Go modules and
+   checking in the changes, for example,
+
+```
+make prepare-release
+```
+
+3. Push the branch and open a PR to submit these changes to the
+   upstream repository's main branch.
+
+4. After merging the PR, pull the upstream commit, for example,
+
+```
+git checkout main
+git pull upstream main
+```
+
+5. Push the release, for example,
+
+```
+make push-release
+```
+
+The release has been published.  Note that these instructions do not
+cover the use of multiple module sets, since this repository uses a
+single module set named "beta" at this time.

collector/cmd/otelarrowcol/build.yaml

@@ -0,0 +1,52 @@
+dist:
+  module: github.com/open-telemetry/otel-arrow/collector/cmd/otelarrowcol
+  name: otelarrowcol
+  description: Development OTel-Arrow Collector binary, testing only.
+
+  # Note: this is replaced to match the current release using `sed`
+  version: 0.1.0
+
+  # Note: This should match the version of the core collector components used below
+  otelcol_version: 0.83.0
+
+exporters:
+  - import: go.opentelemetry.io/collector/exporter/loggingexporter
+    gomod: go.opentelemetry.io/collector/exporter/loggingexporter v0.83.0
+  - import: go.opentelemetry.io/collector/exporter/otlphttpexporter
+    gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.83.0
+  - import: github.com/open-telemetry/otel-arrow/collector/exporter/otelarrowexporter
+    gomod: github.com/open-telemetry/otel-arrow/collector v0.1.0
+  - import: github.com/open-telemetry/otel-arrow/collector/exporter/fileexporter
+    gomod: github.com/open-telemetry/otel-arrow/collector v0.1.0
+
+receivers:
+  - import: github.com/open-telemetry/otel-arrow/collector/receiver/otelarrowreceiver
+    gomod: github.com/open-telemetry/otel-arrow/collector v0.1.0
+  - import: github.com/open-telemetry/otel-arrow/collector/receiver/filereceiver
+    gomod: github.com/open-telemetry/otel-arrow/collector v0.1.0
+  - import: github.com/lightstep/telemetry-generator/generatorreceiver
+    gomod: github.com/lightstep/telemetry-generator/generatorreceiver v0.13.0
+
+processors:
+  - import: go.opentelemetry.io/collector/processor/batchprocessor
+    gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.83.0
+  - import: go.opentelemetry.io/collector/processor/memorylimiterprocessor
+    gomod: go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.83.0
+  - import: github.com/open-telemetry/otel-arrow/collector/processor/experimentprocessor
+    gomod: github.com/open-telemetry/otel-arrow/collector v0.1.0
+  - import: github.com/open-telemetry/otel-arrow/collector/processor/obfuscationprocessor
+    gomod: github.com/open-telemetry/otel-arrow/collector v0.1.0
+
+connectors:
+  - import: github.com/open-telemetry/otel-arrow/collector/connector/validationconnector
+    gomod: github.com/open-telemetry/otel-arrow/collector v0.1.0
+
+extensions:
+  - import: github.com/open-telemetry/opentelemetry-collector-contrib/extension/headerssetterextension
+    gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/headerssetterextension v0.83.0
+  - import: github.com/open-telemetry/opentelemetry-collector-contrib/extension/basicauthextension
+    gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/basicauthextension v0.83.0
+
+replaces:
+  - github.com/open-telemetry/otel-arrow => ../../..
+  - github.com/open-telemetry/otel-arrow/collector => ../../