Backport PR #2705 to release/v1.7 for add example-client docker image

.github/workflows/dockers-example-client-image.yaml

@@ -0,0 +1,76 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go
+
+name: "Build docker image: example-client"
+on:
+  push:
+    branches:
+      - main
+      - release/v*.*
+      - "!release/v*.*.*"
+    tags:
+      - "*.*.*"
+      - v*.*.*
+      - "*.*.*-*"
+      - v*.*.*-*
+  pull_request:
+    paths:
+      - hack/docker/gen/main.go
+      - dockers/example/client/Dockerfile
+      - hack/actions/gen/main.go
+      - .github/workflows/dockers-example-client-image.yaml
+      - .github/actions/docker-build/action.yaml
+      - .github/workflows/_docker-image.yaml
+      - example/client/**
+      - apis/grpc/**
+      - apis/proto/**
+      - go.mod
+      - go.sum
+      - versions/GO_VERSION
+      - internal/**
+      - "!internal/**/*_test.go"
+      - "!internal/**/*_mock.go"
+      - "!internal/db/**"
+      - Makefile
+      - Makefile.d/**
+  pull_request_target:
+    paths:
+      - hack/docker/gen/main.go
+      - dockers/example/client/Dockerfile
+      - hack/actions/gen/main.go
+      - .github/workflows/dockers-example-client-image.yaml
+      - .github/actions/docker-build/action.yaml
+      - .github/workflows/_docker-image.yaml
+      - example/client/**
+      - apis/grpc/**
+      - apis/proto/**
+      - go.mod
+      - go.sum
+      - versions/GO_VERSION
+      - internal/**
+      - "!internal/**/*_test.go"
+      - "!internal/**/*_mock.go"
+      - "!internal/db/**"
+      - Makefile
+      - Makefile.d/**
+jobs:
+  build:
+    uses: ./.github/workflows/_docker-image.yaml
+    with:
+      target: example-client
+    secrets: inherit

Makefile

@@ -24,8 +24,8 @@ TAG                            ?= latest
 CRORG                          ?= $(ORG)
 GHCRORG                         = ghcr.io/$(REPO)
 AGENT_IMAGE                     = $(NAME)-agent
-AGENT_NGT_IMAGE                 = $(AGENT_IMAGE)-ngt
 AGENT_FAISS_IMAGE               = $(AGENT_IMAGE)-faiss
+AGENT_NGT_IMAGE                 = $(AGENT_IMAGE)-ngt
 AGENT_SIDECAR_IMAGE             = $(AGENT_IMAGE)-sidecar
 BENCHMARK_JOB_IMAGE             = $(NAME)-benchmark-job
 BENCHMARK_OPERATOR_IMAGE        = $(NAME)-benchmark-operator
@@ -36,6 +36,7 @@ BUILDKIT_SYFT_SCANNER_IMAGE     = $(BUILDKIT_IMAGE)-syft-scanner
 CI_CONTAINER_IMAGE              = $(NAME)-ci-container
 DEV_CONTAINER_IMAGE             = $(NAME)-dev-container
 DISCOVERER_IMAGE                = $(NAME)-discoverer-k8s
+EXAMPLE_CLIENT_IMAGE            = $(NAME)-example-client
 FILTER_GATEWAY_IMAGE            = $(NAME)-filter-gateway
 HELM_OPERATOR_IMAGE             = $(NAME)-helm-operator
 INDEX_CORRECTION_IMAGE          = $(NAME)-index-correction

Makefile.d/build.mk

@@ -31,6 +31,7 @@ binary/build: \
 	cmd/tools/benchmark/job/job \
 	cmd/tools/benchmark/operator/operator \
 	cmd/tools/cli/loadtest/loadtest \
+	example/client/client \
 	cmd/agent/core/ngt/ngt \
 	cmd/agent/core/faiss/faiss \
 	rust/target/debug/agent \
@@ -103,6 +104,10 @@ cmd/tools/cli/loadtest/loadtest:
 	$(eval CGO_ENABLED = 1)
 	$(call go-build,tools/cli/loadtest,-linkmode 'external',$(LDFLAGS) $(HDF5_LDFLAGS), cgo,$(HDF5_VERSION),$@)
 
+example/client/client:
+	$(eval CGO_ENABLED = 1)
+	$(call go-example-build,example/client,-linkmode 'external',$(LDFLAGS) $(HDF5_LDFLAGS), cgo,$(HDF5_VERSION),$@)
+
 rust/target/release/agent:
 	pushd rust && cargo build -p agent --release && popd
 
@@ -119,6 +124,7 @@ binary/build/zip: \
 	artifacts/vald-benchmark-operator-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-cli-loadtest-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-discoverer-k8s-$(GOOS)-$(GOARCH).zip \
+	artifacts/vald-example-client-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-filter-gateway-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-index-correction-$(GOOS)-$(GOARCH).zip \
 	artifacts/vald-index-creation-$(GOOS)-$(GOARCH).zip \
@@ -192,3 +198,7 @@ artifacts/vald-readreplica-rotate-$(GOOS)-$(GOARCH).zip: cmd/index/job/readrepli
 artifacts/vald-index-operator-$(GOOS)-$(GOARCH).zip: cmd/index/operator/index-operator
 	$(call mkdir, $(dir $@))
 	zip --junk-paths $@ $<
+
+artifacts/vald-example-client-$(GOOS)-$(GOARCH).zip: example/client/client
+	$(call mkdir, $(dir $@))
+	zip --junk-paths $@ $<

Makefile.d/docker.mk

@@ -29,16 +29,17 @@ docker/build: \
 	docker/build/ci-container \
 	docker/build/dev-container \
 	docker/build/discoverer-k8s \
+	docker/build/example-client \
 	docker/build/gateway-filter \
 	docker/build/gateway-lb \
 	docker/build/gateway-mirror \
+	docker/build/helm-operator \
 	docker/build/index-correction \
 	docker/build/index-creation \
 	docker/build/index-operator \
 	docker/build/index-save \
 	docker/build/loadtest \
 	docker/build/manager-index \
-	docker/build/helm-operator \
 	docker/build/readreplica-rotate
 
 docker/xpanes/build:
@@ -56,6 +57,7 @@ docker/xpanes/build:
 		docker/build/ci-container \
 		docker/build/dev-container \
 		docker/build/discoverer-k8s \
+		docker/build/example-client \
 		docker/build/gateway-filter \
 		docker/build/gateway-lb \
 		docker/build/gateway-mirror \
@@ -383,3 +385,15 @@ docker/build/benchmark-operator:
 	@make DOCKERFILE="$(ROOTDIR)/dockers/tools/benchmark/operator/Dockerfile" \
 		IMAGE=$(BENCHMARK_OPERATOR_IMAGE) \
 		docker/build/image
+
+.PHONY: docker/name/example-client
+docker/name/example-client:
+	@echo "$(ORG)/$(EXAMPLE_CLIENT_IMAGE)"
+
+.PHONY: docker/build/example-client
+## build example client docker image
+docker/build/example-client:
+	@make DOCKERFILE="$(ROOTDIR)/dockers/example/client/Dockerfile" \
+		IMAGE=$(EXAMPLE_CLIENT_IMAGE) \
+		DOCKER_OPTS="--build-arg ZLIB_VERSION=$(ZLIB_VERSION) --build-arg HDF5_VERSION=$(HDF5_VERSION)" \
+		docker/build/image

Makefile.d/functions.mk

@@ -91,6 +91,35 @@ define go-build
 	$6 -version
 endef
 
+define go-example-build
+	echo $(GO_SOURCES_INTERNAL)
+	echo $(PBGOS)
+	echo $(shell find $(ROOTDIR)/$1 -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go')
+	cd $(ROOTDIR)/$1 && \
+	CFLAGS="$(CFLAGS)" \
+	CXXFLAGS="$(CXXFLAGS)" \
+	CGO_ENABLED=$(CGO_ENABLED) \
+	CGO_CXXFLAGS="$3" \
+	CGO_FFLAGS="$3" \
+	CGO_LDFLAGS="$3" \
+	GO111MODULE=on \
+	GOARCH=$(GOARCH) \
+	GOOS=$(GOOS) \
+	GOPRIVATE=$(GOPRIVATE) \
+	GO_VERSION=$(GO_VERSION) \
+	go build \
+		--ldflags "-w $2 \
+		-extldflags '$3' \
+		-buildid=" \
+		-modcacherw \
+		-mod=readonly \
+		-a \
+		-tags "osusergo netgo static_build$4" \
+		-trimpath \
+		-o $(ROOTDIR)/$6 \
+		main.go
+endef
+
 define telepresence
 	[ -z $(SWAP_IMAGE) ] && IMAGE=$2 || IMAGE=$(SWAP_IMAGE) \
 	&& echo "telepresence replaces $(SWAP_DEPLOYMENT_TYPE)/$1 with $${IMAGE}:$(SWAP_TAG)" \

dockers/example/client/Dockerfile

@@ -0,0 +1,95 @@
+# syntax = docker/dockerfile:latest
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
+ARG UPX_OPTIONS=-9
+# skipcq: DOK-DL3026,DOK-DL3007
+FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder
+LABEL maintainer="vdaas.org vald team <[email protected]>"
+# skipcq: DOK-DL3002
+USER root:root
+ARG TARGETARCH
+ARG TARGETOS
+ARG GO_VERSION
+ARG RUST_VERSION
+ENV APP_NAME=client
+ENV DEBIAN_FRONTEND=noninteractive
+ENV GO111MODULE=on
+ENV GOPATH=/go
+ENV GOROOT=/opt/go
+ENV HOME=/root
+ENV INITRD=No
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US.UTF-8
+ENV LC_ALL=en_US.UTF-8
+ENV ORG=vdaas
+ENV PKG=example/client
+ENV REPO=vald
+ENV TZ=Etc/UTC
+ENV USER=root
+ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH}
+WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008
+RUN --mount=type=bind,target=.,rw \
+    --mount=type=tmpfs,target=/tmp \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked,id=${APP_NAME} \
+    --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${APP_NAME} \
+    --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \
+    --mount=type=tmpfs,target="${GOPATH}/src" \
+    set -ex \
+    && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \
+    && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \
+    && apt-get clean \
+    && apt-get update -y \
+    && apt-get upgrade -y \
+    && apt-get install -y --no-install-recommends --fix-missing \
+    build-essential \
+    ca-certificates \
+    curl \
+    tzdata \
+    locales \
+    git \
+    cmake \
+    g++ \
+    gcc \
+    libssl-dev \
+    unzip \
+    libhdf5-dev \
+    libaec-dev \
+    && ldconfig \
+    && echo "${LANG} UTF-8" > /etc/locale.gen \
+    && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \
+    && locale-gen ${LANGUAGE} \
+    && update-locale LANG=${LANGUAGE} \
+    && dpkg-reconfigure -f noninteractive tzdata \
+    && apt-get clean \
+    && apt-get autoclean -y \
+    && apt-get autoremove -y \
+    && make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/install \
+    && make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/download \
+    && make hdf5/install \
+    && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" ${PKG}/${APP_NAME} \
+    && mv "${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}"
+# skipcq: DOK-DL3026,DOK-DL3007
+FROM gcr.io/distroless/static:nonroot
+LABEL maintainer="vdaas.org vald team <[email protected]>"
+COPY --from=builder /usr/bin/client /usr/bin/client
+# skipcq: DOK-DL3002
+USER nonroot:nonroot
+ENTRYPOINT ["/usr/bin/client"]

example/helm/values.yaml

@@ -62,7 +62,7 @@ agent:
     # The number of dimensions for feature vector of fashion-mnist dataset.
     dimension: 784
     # We use L2-Norm for distance_type.
-    distance_type: cos
+    distance_type: l2
     # The type of fashion-mnist's feature vectors.
     object_type: float
     # Check duration of automatic indexing.

hack/docker/gen/main.go

@@ -317,6 +317,10 @@ var (
 		"make GOARCH=\"${TARGETARCH}\" GOOS=\"${TARGETOS}\" REPO=\"${ORG}\" NAME=\"${REPO}\" cmd/${PKG}/${APP_NAME}",
 		"mv \"cmd/${PKG}/${APP_NAME}\" \"{{$.BinDir}}/${APP_NAME}\"",
 	}
+	goExampleBuildCommands = []string{
+		"make GOARCH=\"${TARGETARCH}\" GOOS=\"${TARGETOS}\" REPO=\"${ORG}\" NAME=\"${REPO}\" ${PKG}/${APP_NAME}",
+		"mv \"${PKG}/${APP_NAME}\" \"{{$.BinDir}}/${APP_NAME}\"",
+	}
 	rustBuildCommands = []string{
 		"make rust/target/release/${APP_NAME}",
 		"mv \"rust/target/release/${APP_NAME}\" \"{{$.BinDir}}/${APP_NAME}\"",
@@ -707,6 +711,14 @@ func main() {
 					ngtPreprocess,
 					faissPreprocess)...),
 		},
+		"vald-example-client": {
+			AppName:       "client",
+			PackageDir:    "example/client",
+			ExtraPackages: append(clangBuildDeps, "libhdf5-dev", "libaec-dev"),
+			Preprocess: []string{
+				"make hdf5/install",
+			},
+		},
 		"vald-buildbase": {
 			AppName:      "buildbase",
 			AliasImage:   true,
@@ -784,6 +796,8 @@ func main() {
 				}
 				if file.Exists(file.Join(os.Args[1], "cmd", data.PackageDir)) {
 					commands = append(commands, goBuildCommands...)
+				} else if strings.HasPrefix(data.PackageDir, "example") && file.Exists(file.Join(os.Args[1], data.PackageDir)) {
+					commands = append(commands, goExampleBuildCommands...)
 				}
 				data.RunCommands = commands
 				mounts := make([]string, 0, len(defaultMounts)+len(goDefaultMounts))