From 8473441349c806a7018c7bb1863239cacdf6bcbb Mon Sep 17 00:00:00 2001 From: HighPon Date: Thu, 14 Nov 2024 15:04:47 +0000 Subject: [PATCH 01/18] fix: create templates for export and import --- Makefile | 2 + Makefile.d/build.mk | 20 + Makefile.d/docker.mk | 26 + cmd/index/job/exportation/main.go | 26 + cmd/index/job/importation/main.go | 26 + dockers/index/job/exportation/Dockerfile | 87 +++ dockers/index/job/importation/Dockerfile | 87 +++ hack/actions/gen/main.go | 751 +++++++++++++++++++++++ hack/docker/gen/main.go | 8 + 9 files changed, 1033 insertions(+) create mode 100644 cmd/index/job/exportation/main.go create mode 100644 cmd/index/job/importation/main.go create mode 100644 dockers/index/job/exportation/Dockerfile create mode 100644 dockers/index/job/importation/Dockerfile create mode 100644 hack/actions/gen/main.go diff --git a/Makefile b/Makefile index 73645e3120..7678c07046 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,8 @@ HELM_OPERATOR_IMAGE = $(NAME)-helm-operator INDEX_CORRECTION_IMAGE = $(NAME)-index-correction INDEX_CREATION_IMAGE = $(NAME)-index-creation INDEX_DELETION_IMAGE = $(NAME)-index-deletion +INDEX_EXPORTATION_IMAGE = $(NAME)-index-exportion +INDEX_IMPORTATION_IMAGE = $(NAME)-index-importation INDEX_OPERATOR_IMAGE = $(NAME)-index-operator INDEX_SAVE_IMAGE = $(NAME)-index-save LB_GATEWAY_IMAGE = $(NAME)-lb-gateway diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk index e5e175c63c..76996f0edf 100644 --- a/Makefile.d/build.mk +++ b/Makefile.d/build.mk @@ -25,6 +25,8 @@ binary/build: \ cmd/index/job/correction/index-correction \ cmd/index/job/creation/index-creation \ cmd/index/job/deletion/index-deletion \ + cmd/index/job/exportation/index-exportation \ + cmd/index/job/importation/index-importation \ cmd/index/job/readreplica/rotate/readreplica-rotate \ cmd/index/job/save/index-save \ cmd/index/operator/index-operator \ @@ -85,6 +87,14 @@ cmd/index/job/deletion/index-deletion: $(eval CGO_ENABLED = 0) $(call go-build,index/job/deletion,,-static,,,$@) +cmd/index/job/exportation/index-exportation: + $(eval CGO_ENABLED = 0) + $(call go-build,index/job/exportation,,-static,,,$@) + +cmd/index/job/importation/index-importation: + $(eval CGO_ENABLED = 0) + $(call go-build,index/job/importation,,-static,,,$@) + cmd/index/job/save/index-save: $(eval CGO_ENABLED = 0) $(call go-build,index/job/save,,-static,,,$@) @@ -134,6 +144,8 @@ binary/build/zip: \ artifacts/vald-index-correction-$(GOOS)-$(GOARCH).zip \ artifacts/vald-index-creation-$(GOOS)-$(GOARCH).zip \ artifacts/vald-index-deletion-$(GOOS)-$(GOARCH).zip \ + artifacts/vald-index-exportation-$(GOOS)-$(GOARCH).zip \ + artifacts/vald-index-importation-$(GOOS)-$(GOARCH).zip \ artifacts/vald-index-operator-$(GOOS)-$(GOARCH).zip \ artifacts/vald-index-save-$(GOOS)-$(GOARCH).zip \ artifacts/vald-lb-gateway-$(GOOS)-$(GOARCH).zip \ @@ -197,6 +209,14 @@ artifacts/vald-index-deletion-$(GOOS)-$(GOARCH).zip: cmd/index/job/deletion/inde $(call mkdir, $(dir $@)) zip --junk-paths $@ $< +artifacts/vald-index-exporation-$(GOOS)-$(GOARCH).zip: cmd/index/job/exportation/index-exportation + $(call mkdir, $(dir $@)) + zip --junk-paths $@ $< + +artifacts/vald-index-importation-$(GOOS)-$(GOARCH).zip: cmd/index/job/importation/index-importation + $(call mkdir, $(dir $@)) + zip --junk-paths $@ $< + artifacts/vald-index-save-$(GOOS)-$(GOARCH).zip: cmd/index/job/save/index-save $(call mkdir, $(dir $@)) zip --junk-paths $@ $< diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index e4bb5b385d..524fc2050e 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -37,6 +37,8 @@ docker/build: \ docker/build/index-correction \ docker/build/index-creation \ docker/build/index-deletion \ + docker/build/index-exportation \ + docker/build/index-importation \ docker/build/index-operator \ docker/build/index-save \ docker/build/loadtest \ @@ -65,6 +67,8 @@ docker/xpanes/build: docker/build/index-correction \ docker/build/index-creation \ docker/build/index-deletion \ + docker/build/index-exportation \ + docker/build/index-importation \ docker/build/index-operator \ docker/build/index-save \ docker/build/loadtest \ @@ -354,6 +358,28 @@ docker/build/index-deletion: IMAGE=$(INDEX_DELETION_IMAGE) \ docker/build/image +.PHONY: docker/name/index-exportation +docker/name/index-exporation: + @echo "$(ORG)/$(INDEX_EXPORTATION_IMAGE)" + +.PHONY: docker/build/index-exportation +## build index-exportation image +docker/build/index-exportation: + @make DOCKERFILE="$(ROOTDIR)/dockers/index/job/exportation/Dockerfile" \ + IMAGE=$(INDEX_EXPORTATION_IMAGE) \ + docker/build/image + +.PHONY: docker/name/index-importation +docker/name/index-importation: + @echo "$(ORG)/$(INDEX_IMPORTATION_IMAGE)" + +.PHONY: docker/build/index-importation +## build index-importation image +docker/build/index-importation: + @make DOCKERFILE="$(ROOTDIR)/dockers/index/job/importation/Dockerfile" \ + IMAGE=$(INDEX_IMPORTATION_IMAGE) \ + docker/build/image + .PHONY: docker/name/index-operator docker/name/index-operator: @echo "$(ORG)/$(INDEX_OPERATOR_IMAGE)" diff --git a/cmd/index/job/exportation/main.go b/cmd/index/job/exportation/main.go new file mode 100644 index 0000000000..c5553f225e --- /dev/null +++ b/cmd/index/job/exportation/main.go @@ -0,0 +1,26 @@ +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import "fmt" + +const ( + maxVersion = "v0.0.10" + minVersion = "v0.0.0" + name = "index exportation job" +) + +func main() { + fmt.Println("hello world") +} diff --git a/cmd/index/job/importation/main.go b/cmd/index/job/importation/main.go new file mode 100644 index 0000000000..ab8fb8146e --- /dev/null +++ b/cmd/index/job/importation/main.go @@ -0,0 +1,26 @@ +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package main + +import "fmt" + +const ( + maxVersion = "v0.0.10" + minVersion = "v0.0.0" + name = "index importation job" +) + +func main() { + fmt.Println("hello world") +} diff --git a/dockers/index/job/exportation/Dockerfile b/dockers/index/job/exportation/Dockerfile new file mode 100644 index 0000000000..e31d6c41d9 --- /dev/null +++ b/dockers/index/job/exportation/Dockerfile @@ -0,0 +1,87 @@ +# syntax = docker/dockerfile:latest +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go +ARG UPX_OPTIONS=-9 +# skipcq: DOK-DL3026,DOK-DL3007 +FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder +LABEL maintainer="vdaas.org vald team " +# skipcq: DOK-DL3002 +USER root:root +ARG TARGETARCH +ARG TARGETOS +ARG GO_VERSION +ARG RUST_VERSION +ENV APP_NAME=index-exportation +ENV DEBIAN_FRONTEND=noninteractive +ENV GO111MODULE=on +ENV GOPATH=/go +ENV GOROOT=/opt/go +ENV HOME=/root +ENV INITRD=No +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 +ENV ORG=vdaas +ENV PKG=index/job/exportation +ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root +ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN --mount=type=bind,target=.,rw \ + --mount=type=tmpfs,target=/tmp \ + --mount=type=cache,target=/var/lib/apt,sharing=locked,id=${APP_NAME} \ + --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${APP_NAME} \ + --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ + --mount=type=tmpfs,target="${GOPATH}/src" \ + set -ex \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ + && apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends --fix-missing \ + build-essential \ + ca-certificates \ + curl \ + tzdata \ + locales \ + git \ + && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ + && apt-get clean \ + && apt-get autoclean -y \ + && apt-get autoremove -y \ + && make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/install \ + && make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/download \ + && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ + && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" +# skipcq: DOK-DL3026,DOK-DL3007 +FROM gcr.io/distroless/static:nonroot +LABEL maintainer="vdaas.org vald team " +COPY --from=builder /usr/bin/index-exportation /usr/bin/index-exportation +# skipcq: DOK-DL3002 +USER nonroot:nonroot +ENTRYPOINT ["/usr/bin/index-exportation"] \ No newline at end of file diff --git a/dockers/index/job/importation/Dockerfile b/dockers/index/job/importation/Dockerfile new file mode 100644 index 0000000000..c7eb529861 --- /dev/null +++ b/dockers/index/job/importation/Dockerfile @@ -0,0 +1,87 @@ +# syntax = docker/dockerfile:latest +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go +ARG UPX_OPTIONS=-9 +# skipcq: DOK-DL3026,DOK-DL3007 +FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder +LABEL maintainer="vdaas.org vald team " +# skipcq: DOK-DL3002 +USER root:root +ARG TARGETARCH +ARG TARGETOS +ARG GO_VERSION +ARG RUST_VERSION +ENV APP_NAME=index-importation +ENV DEBIAN_FRONTEND=noninteractive +ENV GO111MODULE=on +ENV GOPATH=/go +ENV GOROOT=/opt/go +ENV HOME=/root +ENV INITRD=No +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 +ENV ORG=vdaas +ENV PKG=index/job/importation +ENV REPO=vald +ENV TZ=Etc/UTC +ENV USER=root +ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} +WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 +RUN --mount=type=bind,target=.,rw \ + --mount=type=tmpfs,target=/tmp \ + --mount=type=cache,target=/var/lib/apt,sharing=locked,id=${APP_NAME} \ + --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${APP_NAME} \ + --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ + --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ + --mount=type=tmpfs,target="${GOPATH}/src" \ + set -ex \ + && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ + && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ + && apt-get clean \ + && apt-get update -y \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends --fix-missing \ + build-essential \ + ca-certificates \ + curl \ + tzdata \ + locales \ + git \ + && ldconfig \ + && echo "${LANG} UTF-8" > /etc/locale.gen \ + && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ + && locale-gen ${LANGUAGE} \ + && update-locale LANG=${LANGUAGE} \ + && dpkg-reconfigure -f noninteractive tzdata \ + && apt-get clean \ + && apt-get autoclean -y \ + && apt-get autoremove -y \ + && make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/install \ + && make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/download \ + && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ + && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" +# skipcq: DOK-DL3026,DOK-DL3007 +FROM gcr.io/distroless/static:nonroot +LABEL maintainer="vdaas.org vald team " +COPY --from=builder /usr/bin/index-importation /usr/bin/index-importation +# skipcq: DOK-DL3002 +USER nonroot:nonroot +ENTRYPOINT ["/usr/bin/index-importation"] \ No newline at end of file diff --git a/hack/actions/gen/main.go b/hack/actions/gen/main.go new file mode 100644 index 0000000000..b528c736aa --- /dev/null +++ b/hack/actions/gen/main.go @@ -0,0 +1,751 @@ +// +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "bytes" + "context" + "fmt" + "io/fs" + "os" + "os/signal" + "syscall" + "text/template" + "time" + + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/file" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/strings" + "github.com/vdaas/vald/internal/sync/errgroup" + "gopkg.in/yaml.v2" +) + +var license string = `# +# Copyright (C) 2019-{{.Year}} {{.Maintainer}} +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +` + +var licenseTmpl *template.Template = template.Must(template.New("license").Parse(license)) + +type ( + Workflow struct { + Name string `yaml:"name"` + On On `yaml:"on"` + Jobs Jobs `yaml:"jobs"` + } + + On struct { + Schedule Schedule `yaml:"schedule,omitempty"` + Push Push `yaml:"push"` + PullRequest PullRequest `yaml:"pull_request"` + PullRequestTarget PullRequest `yaml:"pull_request_target"` + } + + Schedule []struct { + Cron string `yaml:"cron,omitempty"` + } + + Push struct { + Branches []string `yaml:"branches"` + Tags []string `yaml:"tags"` + } + + PullRequest struct { + Paths Paths `yaml:"paths"` + } + + Jobs struct { + Build Build `yaml:"build"` + } + + Build struct { + Uses string `yaml:"uses"` + With With `yaml:"with"` + Secrets string `yaml:"secrets"` + } + + With struct { + Target string `yaml:"target"` + Platforms string `yaml:"platforms,omitempty"` + } + + Paths []string +) + +type Data struct { + AliasImage bool + ConfigExists bool + Year int + ContainerType ContainerType + AppName string + BinDir string + BuildUser string + BuilderImage string + BuilderTag string + BuildStageName string + Maintainer string + PackageDir string + RootDir string + RuntimeImage string + RuntimeTag string + RuntimeUser string + Name string + BuildPlatforms string + Arguments map[string]string + Environments map[string]string + Entrypoints []string + EnvironmentsSlice []string + ExtraCopies []string + ExtraImages []string + ExtraPackages []string + Preprocess []string + RunCommands []string + RunMounts []string + StageFiles []string + PullRequestPaths []string +} + +type ContainerType int + +const ( + Go ContainerType = iota + Rust + DevContainer + HelmOperator + CIContainer + Other +) + +const ( + organization = "vdaas" + repository = "vald" + defaultMaintainer = organization + ".org " + repository + " team <" + repository + "@" + organization + ".org>" + defaultBuildUser = "root:root" + maintainerKey = "MAINTAINER" + minimumArgumentLength = 2 + ubuntuVersion = "22.04" + + ngtPreprocess = "make ngt/install" + faissPreprocess = "make faiss/install" + + helmOperatorRootdir = "/opt/helm" + helmOperatorWatchFile = helmOperatorRootdir + "/watches.yaml" + helmOperatorChartsDir = helmOperatorRootdir + "/charts" +) + +var ( + clangBuildDeps = []string{ + "cmake", + "gcc", + "g++", + "unzip", + "libssl-dev", + } + ngtBuildDeps = []string{ + "liblapack-dev", + "libomp-dev", + "libopenblas-dev", + } + faissBuildDeps = []string{ + "gfortran", + } + devContainerDeps = []string{ + "gawk", + "gnupg2", + "graphviz", + "jq", + "libhdf5-dev", + "libaec-dev", + "sed", + "zip", + } + + ciContainerPreprocess = []string{ + "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} deps GO_CLEAN_DEPS=false", + "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} golangci-lint/install", + "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} gotestfmt/install", + "make cmake/install", + "make buf/install", + "make hdf5/install", + "make helm-docs/install", + "make helm/install", + "make k3d/install", + "make k9s/install", + "make kind/install", + "make kubectl/install", + "make kubelinter/install", + "make reviewdog/install", + "make tparse/install", + "make valdcli/install", + "make yq/install", + "make minikube/install", + "make stern/install", + "make telepresence/install", + } + + devContainerPreprocess = []string{ + "curl -fsSL https://deb.nodesource.com/setup_current.x | bash -", + "apt-get clean", + "apt-get update -y", + "apt-get upgrade -y", + "apt-get install -y --no-install-recommends --fix-missing nodejs", + "npm install -g npm@latest", + "apt-get clean", + "apt-get autoclean -y", + "apt-get autoremove -y", + "make delve/install", + "make gomodifytags/install", + "make gopls/install", + "make gotests/install", + "make impl/install", + "make staticcheck/install", + } +) + +const baseWorkflowTmpl string = `name: "Build docker image: %s" +on: + schedule: + - cron: "0 * * * *" + push: + branches: + - "main" + - "release/v*.*" + - "!release/v*.*.*" + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + pull_request: + paths: + - "hack/docker/gen/main.go" + - "dockers/%s/Dockerfile" + - "hack/actions/gen/main.go" + - ".github/workflows/dockers-%s-image.yaml" + - ".github/actions/docker-build/action.yaml" + - ".github/workflows/_docker-image.yaml" + - "cmd/%s/**" + - "pkg/%s/**" + pull_request_target: + paths: [] + +jobs: + build: + uses: "./.github/workflows/_docker-image.yaml" + with: + target: "%s" + platforms: "" + secrets: "inherit" +` + +const ( + cmdBenchOperatorsPath = "cmd/tools/benchmark/operators/**" + pkgBenchOperatorsPath = "pkg/tools/benchmark/operators/**" + cmdBenchJobsPath = "cmd/tools/benchmark/jobs/**" + pkgBenchJobsPath = "pkg/tools/benchmark/jobs/**" + + agentInternalPath = "pkg/agent/internal/**" + gatewayInternalPath = "pkg/gateway/internal/**" + + apisGrpcPath = "apis/grpc/**" + apisProtoPath = "apis/proto/**" + + hackPath = "hack/**" + + chartsValdPath = "charts/vald" + helmOperatorPath = "charts/vald-helm-operator" + chartPath = chartsValdPath + "/Chart.yaml" + valuesPath = chartsValdPath + "/values.yaml" + templatesPath = chartsValdPath + "/templates/**" + helmOperatorChartPath = helmOperatorPath + "/Chart.yaml" + helmOperatorValuesPath = helmOperatorPath + "/values.yaml" + helmOperatorTemplatesPath = helmOperatorPath + "/templates/**" + + goModPath = "go.mod" + goSumPath = "go.sum" + + cargoLockPath = "rust/Cargo.lock" + cargoTomlPath = "rust/Cargo.toml" + rustBinAgentDirPath = "rust/bin/agent" + rustNgtRsPath = "rust/libs/ngt-rs/**" + rustNgtPath = "rust/libs/ngt/**" + rustProtoPath = "rust/libs/proto/**" + + internalPath = "internal/**" + internalStoragePath = "internal/db/storage/blob/**" + excludeTestFilesPath = "!internal/**/*_test.go" + excludeMockFilesPath = "!internal/**/*_mock.go" + excludeDbPath = "!internal/db/**" + excludeK8sPath = "!internal/k8s/**" + + versionsPath = "versions" + operatorSDKVersionPath = versionsPath + "/OPERATOR_SDK_VERSION" + goVersionPath = versionsPath + "/GO_VERSION" + rustVersionPath = versionsPath + "/RUST_VERSION" + faissVersionPath = versionsPath + "/FAISS_VERSION" + ngtVersionPath = versionsPath + "/NGT_VERSION" + usearchVersionPath = versionsPath + "/USEARCH_VERSION" + + makefilePath = "Makefile" + makefileDirPath = "Makefile.d/**" +) + +const ( + agentNgt = "agent-ngt" + agentFaiss = "agent-faiss" + agentSidecar = "agent-sidecar" + agent = "agent" + + discovererK8s = "discoverer-k8s" + + gateway = "gateway" + gatewayLb = "gateway-lb" + gatewayFilter = "gateway-filter" + gatewayMirror = "gateway-mirror" + + managerIndex = "manager-index" + + indexCorrection = "index-correction" + indexCreation = "index-creation" + indexDeletion = "index-deletion" + indexExportation = "index-exportation" + indexImportation = "index-importation" + indexSave = "index-save" + indexOperator = "index-operator" + + readreplicaRotate = "readreplica-rotate" + + benchJob = "benchmark-job" + benchOperator = "benchmark-operator" + + helmOperator = "helm-operator" + + loadtest = "loadtest" + + ciContainer = "ci-container" + devContainer = "dev-container" + + buildbase = "buildbase" + buildkit = "buildkit" + binfmt = "binfmt" + buildkitSyftScanner = "buildkit-syft-scanner" +) + +const ( + multiPlatforms = amd64Platform + "," + arm64Platform + amd64Platform = "linux/amd64" + arm64Platform = "linux/arm64" +) + +func (data *Data) initPullRequestPaths() { + switch data.Name { + // the benchmark components trigger each other, not just themselves + case benchJob: + data.PullRequestPaths = append(data.PullRequestPaths, + cmdBenchOperatorsPath, + pkgBenchOperatorsPath, + ) + case benchOperator: + data.PullRequestPaths = append(data.PullRequestPaths, + cmdBenchJobsPath, + pkgBenchJobsPath, + ) + case agentFaiss, agentNgt: + data.PullRequestPaths = append(data.PullRequestPaths, agentInternalPath) + default: + if strings.Contains(strings.ToLower(data.Name), gateway) { + data.PullRequestPaths = append(data.PullRequestPaths, gatewayInternalPath) + } + } + + switch data.ContainerType { + case HelmOperator: + data.PullRequestPaths = append(data.PullRequestPaths, + chartPath, + valuesPath, + templatesPath, + helmOperatorChartPath, + helmOperatorValuesPath, + helmOperatorTemplatesPath, + operatorSDKVersionPath, + ) + case DevContainer, CIContainer: + data.PullRequestPaths = append(data.PullRequestPaths, + apisGrpcPath, + apisProtoPath, + hackPath, + ) + case Go: + data.PullRequestPaths = append(data.PullRequestPaths, + apisGrpcPath, + apisProtoPath, + goModPath, + goSumPath, + goVersionPath, + internalPath, + excludeTestFilesPath, + excludeMockFilesPath, + excludeDbPath, + ) + switch data.Name { + case discovererK8s, indexOperator, gatewayMirror, readreplicaRotate, agentNgt, benchJob, benchOperator: + default: + data.PullRequestPaths = append(data.PullRequestPaths, excludeK8sPath) + } + case Rust: + data.PullRequestPaths = append(data.PullRequestPaths, + apisGrpcPath, + apisProtoPath, + cargoLockPath, + cargoTomlPath, + rustBinAgentDirPath, + rustNgtRsPath, + rustNgtPath, + rustProtoPath, + rustVersionPath, + faissVersionPath, + ngtVersionPath, + ) + } + if strings.EqualFold(data.Name, agentFaiss) || data.ContainerType == Rust { + data.PullRequestPaths = append(data.PullRequestPaths, faissVersionPath) + } + if strings.EqualFold(data.Name, agentNgt) || data.ContainerType == Rust { + data.PullRequestPaths = append(data.PullRequestPaths, ngtVersionPath) + } + + if data.Name == agentSidecar { + data.PullRequestPaths = append(data.PullRequestPaths, internalStoragePath) + } + if !data.AliasImage { + data.PullRequestPaths = append(data.PullRequestPaths, makefilePath, makefileDirPath) + } +} + +func (data *Data) initData() { + data.initPullRequestPaths() + + if data.AliasImage { + data.BuildPlatforms = multiPlatforms + } + if data.ContainerType == CIContainer || data.Name == loadtest { + data.BuildPlatforms = amd64Platform + } + + data.Year = time.Now().Year() + if maintainer := os.Getenv(maintainerKey); maintainer != "" { + data.Maintainer = maintainer + } else { + data.Maintainer = defaultMaintainer + } +} + +func (data *Data) generateWorkflowStruct() (*Workflow, error) { + workflow := &Workflow{} + baseWorkflow := fmt.Sprintf(baseWorkflowTmpl, + data.Name, + data.PackageDir, + data.Name, + data.PackageDir, + data.PackageDir, + data.Name, + ) + err := yaml.NewDecoder(strings.NewReader(baseWorkflow)).Decode(workflow) + if err != nil { + return nil, fmt.Errorf("Error decoding YAML: %v", err) + } + + if !data.AliasImage { + workflow.On.Schedule = nil + } + workflow.On.PullRequest.Paths = append(workflow.On.PullRequest.Paths, data.PullRequestPaths...) + workflow.On.PullRequestTarget.Paths = workflow.On.PullRequest.Paths + workflow.Jobs.Build.With.Platforms = data.BuildPlatforms + + return workflow, nil +} + +func main() { + log.Init() + if len(os.Args) < minimumArgumentLength { + // skipcq: RVV-A0003 + log.Fatal(errors.New("invalid argument")) + } + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, + syscall.SIGQUIT, + syscall.SIGHUP, + syscall.SIGALRM, + syscall.SIGKILL, + syscall.SIGTERM) + defer cancel() + + eg, egctx := errgroup.New(ctx) + for n, d := range map[string]Data{ + "vald-agent-ngt": { + AppName: "ngt", + PackageDir: "agent/core/ngt", + ExtraPackages: append(clangBuildDeps, ngtBuildDeps...), + Preprocess: []string{ngtPreprocess}, + }, + "vald-agent-faiss": { + AppName: "faiss", + PackageDir: "agent/core/faiss", + ExtraPackages: append(clangBuildDeps, + append(ngtBuildDeps, + faissBuildDeps...)...), + Preprocess: []string{faissPreprocess}, + }, + "vald-agent": { + AppName: "agent", + PackageDir: "agent/core/agent", + ContainerType: Rust, + RuntimeImage: "gcr.io/distroless/cc-debian12", + ExtraPackages: append(clangBuildDeps, + append(ngtBuildDeps, + faissBuildDeps...)...), + Preprocess: []string{ + ngtPreprocess, + faissPreprocess, + }, + }, + "vald-agent-sidecar": { + AppName: "sidecar", + PackageDir: "agent/sidecar", + }, + "vald-discoverer-k8s": { + AppName: "discoverer", + PackageDir: "discoverer/k8s", + }, + "vald-gateway-lb": { + AppName: "lb", + PackageDir: "gateway/lb", + }, + "vald-gateway-filter": { + AppName: "filter", + PackageDir: "gateway/filter", + }, + "vald-gateway-mirror": { + AppName: "mirror", + PackageDir: "gateway/mirror", + }, + "vald-manager-index": { + AppName: "index", + PackageDir: "manager/index", + }, + "vald-index-correction": { + AppName: "index-correction", + PackageDir: "index/job/correction", + }, + "vald-index-creation": { + AppName: "index-creation", + PackageDir: "index/job/creation", + }, + "vald-index-deletion": { + AppName: "index-deletion", + PackageDir: "index/job/deletion", + }, + "vald-index-exportation": { + AppName: "index-exportation", + PackageDir: "index/job/exportation", + }, + "vald-index-importation": { + AppName: "index-importation", + PackageDir: "index/job/importation", + }, + "vald-index-save": { + AppName: "index-save", + PackageDir: "index/job/save", + }, + "vald-readreplica-rotate": { + AppName: "readreplica-rotate", + PackageDir: "index/job/readreplica/rotate", + }, + "vald-index-operator": { + AppName: "index-operator", + PackageDir: "index/operator", + }, + "vald-benchmark-job": { + AppName: "job", + PackageDir: "tools/benchmark/job", + ExtraPackages: append(clangBuildDeps, "libhdf5-dev", "libaec-dev"), + Preprocess: []string{ + "make hdf5/install", + }, + }, + "vald-benchmark-operator": { + AppName: "operator", + PackageDir: "tools/benchmark/operator", + }, + "vald-helm-operator": { + AppName: "helm-operator", + PackageDir: "operator/helm", + ContainerType: HelmOperator, + Arguments: map[string]string{ + "OPERATOR_SDK_VERSION": "latest", + }, + ExtraCopies: []string{ + "--from=operator /usr/local/bin/${APP_NAME} {{$.BinDir}}/${APP_NAME}", + }, + ExtraImages: []string{ + "quay.io/operator-framework/helm-operator:${OPERATOR_SDK_VERSION} AS operator", + }, + ExtraPackages: []string{"upx"}, + Preprocess: []string{ + "mkdir -p " + helmOperatorChartsDir, + `{ \ + echo "---"; \ + echo "- version: v1"; \ + echo " group: vald.vdaas.org"; \ + echo " kind: ValdRelease"; \ + echo " chart: ` + helmOperatorChartsDir + `/vald"; \ + echo "- version: v1"; \ + echo " group: vald.vdaas.org"; \ + echo " kind: ValdHelmOperatorRelease"; \ + echo " chart: ` + helmOperatorChartsDir + `/vald-helm-operator"; \ + } > ` + helmOperatorWatchFile, + "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} helm/schema/vald", + "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} helm/schema/vald-helm-operator", + "cp -r charts/* " + helmOperatorChartsDir + "/", + "upx \"{{$.BinDir}}/${APP_NAME}\"", + }, + StageFiles: []string{ + helmOperatorWatchFile, + helmOperatorChartsDir + "/vald", + helmOperatorChartsDir + "/vald-helm-operator", + }, + Entrypoints: []string{"{{$.BinDir}}/{{.AppName}}", "run", "--watches-file=" + helmOperatorWatchFile}, + }, + "vald-loadtest": { // note: this name is a little different from that of docker/gen/main.go + AppName: "loadtest", + PackageDir: "tools/cli/loadtest", + ExtraPackages: append(clangBuildDeps, "libhdf5-dev", "libaec-dev"), + Preprocess: []string{ + "make hdf5/install", + }, + }, + "vald-ci-container": { + AppName: "ci-container", + ContainerType: CIContainer, + PackageDir: "ci/base", + RuntimeUser: defaultBuildUser, + ExtraPackages: append([]string{"npm"}, append(clangBuildDeps, + append(ngtBuildDeps, + append(faissBuildDeps, + devContainerDeps...)...)...)...), + Preprocess: append(ciContainerPreprocess, ngtPreprocess, faissPreprocess), + Entrypoints: []string{"/bin/bash"}, + }, + "vald-dev-container": { + AppName: "dev-container", + BuilderImage: "mcr.microsoft.com/devcontainers/base", + BuilderTag: "ubuntu" + ubuntuVersion, + BuildUser: defaultBuildUser, + RuntimeUser: defaultBuildUser, + ContainerType: DevContainer, + PackageDir: "dev", + ExtraPackages: append(clangBuildDeps, + append(ngtBuildDeps, + append(faissBuildDeps, + devContainerDeps...)...)...), + Preprocess: append(devContainerPreprocess, + append(ciContainerPreprocess, + ngtPreprocess, + faissPreprocess)...), + }, + "vald-buildbase": { + AppName: "buildbase", + AliasImage: true, + PackageDir: "buildbase", + BuilderImage: "ubuntu", + BuilderTag: "devel", + }, + "vald-buildkit": { + AppName: "buildkit", + AliasImage: true, + PackageDir: "buildkit", + BuilderImage: "moby/buildkit", + BuilderTag: "master", + }, + "vald-binfmt": { + AppName: "binfmt", + AliasImage: true, + PackageDir: "binfmt", + BuilderImage: "tonistiigi/binfmt", + BuilderTag: "master", + }, + "vald-buildkit-syft-scanner": { + AppName: "scanner", + AliasImage: true, + PackageDir: "buildkit/syft/scanner", + BuilderImage: "docker/buildkit-syft-scanner", + BuilderTag: "edge", + BuildStageName: "scanner", + }, + } { + name := n + data := d + + eg.Go(safety.RecoverFunc(func() error { + data.Name = strings.TrimPrefix(name, "vald-") + data.initData() + + log.Infof("Generating %s's workflow", data.Name) + workflow, err := data.generateWorkflowStruct() + if err != nil { + return fmt.Errorf("Error generating workflowStruct: %w", err) + } + workflowYamlTmp, err := yaml.Marshal(workflow) + if err != nil { + return fmt.Errorf("error marshaling workflowStruct to YAML: %w", err) + } + + // remove the double quotation marks from the generated key "on": (note that the word "on" is a reserved word in gopkg.in/yaml.v2) + workflowYaml := strings.Replace(string(workflowYamlTmp), "\"on\":", "on:", 1) + + buf := bytes.NewBuffer(make([]byte, 0, len(license)+len(workflowYaml))) + err = licenseTmpl.Execute(buf, data) + if err != nil { + return fmt.Errorf("error executing template: %w", err) + } + buf.WriteString("\r\n") + buf.WriteString(workflowYaml) + _, err = file.OverWriteFile(egctx, file.Join(os.Args[1], ".github/workflows", "dockers-"+data.Name+"-image.yaml"), buf, fs.ModePerm) + if err != nil { + return fmt.Errorf("error writing file: %w", err) + } + return nil + })) + } + eg.Wait() +} diff --git a/hack/docker/gen/main.go b/hack/docker/gen/main.go index 811f7d12ff..0afbca5d65 100644 --- a/hack/docker/gen/main.go +++ b/hack/docker/gen/main.go @@ -735,6 +735,14 @@ func main() { AppName: "index-deletion", PackageDir: "index/job/deletion", }, + "vald-index-exportation": { + AppName: "index-exportation", + PackageDir: "index/job/exportation", + }, + "vald-index-importation": { + AppName: "index-importation", + PackageDir: "index/job/importation", + }, "vald-readreplica-rotate": { AppName: "readreplica-rotate", PackageDir: "index/job/readreplica/rotate", From de8ec0dfa2fe482f735798a31ae1a07d7b4e3fbb Mon Sep 17 00:00:00 2001 From: HighPon Date: Mon, 18 Nov 2024 04:17:07 +0000 Subject: [PATCH 02/18] [WIP] feat: Implenet vald index export --- Makefile | 2 +- cmd/index/job/exportation/main.go | 37 ++- cmd/index/job/exportation/sample.yaml | 193 +++++++++++++++ dockers/index/job/exportation/Dockerfile | 1 + internal/config/index_exporter.go | 43 ++++ k8s/index/job/exportation/configmap.yaml | 211 +++++++++++++++++ k8s/index/job/exportation/cronjob.yaml | 151 ++++++++++++ k8s/index/job/exportation/pvc.yaml | 12 + pkg/index/job/exportation/config/config.go | 66 ++++++ pkg/index/job/exportation/service/exporter.go | 222 ++++++++++++++++++ pkg/index/job/exportation/service/options.go | 104 ++++++++ .../job/exportation/usecase/exportation.go | 183 +++++++++++++++ 12 files changed, 1222 insertions(+), 3 deletions(-) create mode 100644 cmd/index/job/exportation/sample.yaml create mode 100644 internal/config/index_exporter.go create mode 100644 k8s/index/job/exportation/configmap.yaml create mode 100644 k8s/index/job/exportation/cronjob.yaml create mode 100644 k8s/index/job/exportation/pvc.yaml create mode 100644 pkg/index/job/exportation/config/config.go create mode 100644 pkg/index/job/exportation/service/exporter.go create mode 100644 pkg/index/job/exportation/service/options.go create mode 100644 pkg/index/job/exportation/usecase/exportation.go diff --git a/Makefile b/Makefile index 7678c07046..799713d206 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ HELM_OPERATOR_IMAGE = $(NAME)-helm-operator INDEX_CORRECTION_IMAGE = $(NAME)-index-correction INDEX_CREATION_IMAGE = $(NAME)-index-creation INDEX_DELETION_IMAGE = $(NAME)-index-deletion -INDEX_EXPORTATION_IMAGE = $(NAME)-index-exportion +INDEX_EXPORTATION_IMAGE = $(NAME)-index-exportation INDEX_IMPORTATION_IMAGE = $(NAME)-index-importation INDEX_OPERATOR_IMAGE = $(NAME)-index-operator INDEX_SAVE_IMAGE = $(NAME)-index-save diff --git a/cmd/index/job/exportation/main.go b/cmd/index/job/exportation/main.go index c5553f225e..8c16dd224b 100644 --- a/cmd/index/job/exportation/main.go +++ b/cmd/index/job/exportation/main.go @@ -13,7 +13,17 @@ // limitations under the License. package main -import "fmt" +import ( + "context" + "log" + + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/info" + "github.com/vdaas/vald/internal/runner" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/pkg/index/job/exportation/config" + "github.com/vdaas/vald/pkg/index/job/exportation/usecase" +) const ( maxVersion = "v0.0.10" @@ -22,5 +32,28 @@ const ( ) func main() { - fmt.Println("hello world") + if err := safety.RecoverFunc(func() error { + return runner.Do( + context.Background(), + runner.WithName(name), + runner.WithVersion(info.Version, maxVersion, minVersion), + runner.WithConfigLoader(func(path string) (any, *config.GlobalConfig, error) { + cfg, err := config.NewConfig(path) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to load "+name+"'s configuration") + } + return cfg, &cfg.GlobalConfig, nil + }), + runner.WithDaemonInitializer(func(cfg any) (runner.Runner, error) { + c, ok := cfg.(*config.Data) + if !ok { + return nil, errors.ErrInvalidConfig + } + return usecase.New(c) + }), + ) + })(); err != nil { + log.Fatal(err, info.Get()) + return + } } diff --git a/cmd/index/job/exportation/sample.yaml b/cmd/index/job/exportation/sample.yaml new file mode 100644 index 0000000000..4ed586d02f --- /dev/null +++ b/cmd/index/job/exportation/sample.yaml @@ -0,0 +1,193 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: v0.0.0 +time_zone: JST +logging: + format: raw + level: info + logger: glg +server_config: + servers: + - name: grpc + host: 0.0.0.0 + port: 8081 + grpc: + bidirectional_stream_concurrency: 20 + connection_timeout: "" + header_table_size: 0 + initial_conn_window_size: 0 + initial_window_size: 0 + interceptors: [] + keepalive: + max_conn_age: "" + max_conn_age_grace: "" + max_conn_idle: "" + time: "" + timeout: "" + max_header_list_size: 0 + max_receive_message_size: 0 + max_send_message_size: 0 + read_buffer_size: 0 + write_buffer_size: 0 + mode: GRPC + probe_wait_time: 3s + restart: true + health_check_servers: + - name: readiness + host: 0.0.0.0 + port: 3001 + http: + handler_timeout: "" + idle_timeout: "" + read_header_timeout: "" + read_timeout: "" + shutdown_duration: 0s + write_timeout: "" + mode: "" + probe_wait_time: 3s + metrics_servers: + startup_strategy: + - grpc + - readiness + full_shutdown_duration: 600s + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key +exporter: + concurrency: 1 + kvs_background_sync_interval: 5s + kvs_background_compaction_interval: 5s + index_path: "/var/export/index" + gateway: + addrs: + - vald-lb-gateway.default.svc.cluster.local:8081 + health_check_duration: "1s" + connection_pool: + enable_dns_resolver: true + enable_rebalance: true + old_conn_close_duration: 2m + rebalance_duration: 30m + size: 3 + backoff: + backoff_factor: 1.1 + backoff_time_limit: 5s + enable_error_log: true + initial_duration: 5ms + jitter_limit: 100ms + maximum_duration: 5s + retry_count: 100 + circuit_breaker: + closed_error_rate: 0.7 + closed_refresh_timeout: 10s + half_open_error_rate: 0.5 + min_samples: 1000 + open_timeout: 1s + call_option: + content_subtype: "" + max_recv_msg_size: 0 + max_retry_rpc_buffer_size: 0 + max_send_msg_size: 0 + wait_for_ready: true + dial_option: + authority: "" + backoff_base_delay: 1s + backoff_jitter: 0.2 + backoff_max_delay: 120s + backoff_multiplier: 1.6 + disable_retry: false + enable_backoff: false + idle_timeout: 1h + initial_connection_window_size: 2097152 + initial_window_size: 1048576 + insecure: true + interceptors: [] + keepalive: + permit_without_stream: false + time: "" + timeout: 30s + max_call_attempts: 0 + max_header_list_size: 0 + max_msg_size: 0 + min_connection_timeout: 20s + net: + dialer: + dual_stack_enabled: true + keepalive: "" + timeout: "" + dns: + cache_enabled: true + cache_expiration: 1h + refresh_duration: 30m + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + insecure_skip_verify: false + key: /path/to/key + read_buffer_size: 0 + shared_write_buffer: false + timeout: "" + user_agent: Vald-gRPC + write_buffer_size: 0 + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + insecure_skip_verify: false + key: /path/to/key +observability: + enabled: false + otlp: + collector_endpoint: "otel-collector.monitoring.svc.cluster.local:4317" + trace_batch_timeout: "1s" + trace_export_timeout: "1m" + trace_max_export_batch_size: 1024 + trace_max_queue_size: 256 + metrics_export_interval: "1s" + metrics_export_timeout: "1m" + attribute: + namespace: "_MY_POD_NAMESPACE_" + pod_name: "_MY_POD_NAME_" + node_name: "_MY_NODE_NAME_" + service_name: "vald-index-deletion" + metrics: + enable_cgo: true + enable_goroutine: true + enable_memory: true + enable_version_info: true + version_info_labels: + - vald_version + - server_name + - git_commit + - build_time + - go_version + - go_os + - go_arch + - algorithm_info + trace: + enabled: true diff --git a/dockers/index/job/exportation/Dockerfile b/dockers/index/job/exportation/Dockerfile index e31d6c41d9..495f985dab 100644 --- a/dockers/index/job/exportation/Dockerfile +++ b/dockers/index/job/exportation/Dockerfile @@ -82,6 +82,7 @@ RUN --mount=type=bind,target=.,rw \ FROM gcr.io/distroless/static:nonroot LABEL maintainer="vdaas.org vald team " COPY --from=builder /usr/bin/index-exportation /usr/bin/index-exportation +COPY cmd/index/job/exportation/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot ENTRYPOINT ["/usr/bin/index-exportation"] \ No newline at end of file diff --git a/internal/config/index_exporter.go b/internal/config/index_exporter.go new file mode 100644 index 0000000000..f2f9e3303f --- /dev/null +++ b/internal/config/index_exporter.go @@ -0,0 +1,43 @@ +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +// IndexExporter represents the configurations for index exportation. +type IndexExporter struct { + // Concurrency represents indexing concurrency. + Concurrency int `json:"concurrency" yaml:"concurrency"` + + // KVSBackgroundSyncInterval represents interval for checked id list kvs sync duration + KVSBackgroundSyncInterval string `json:"kvs_background_sync_interval" yaml:"kvs_background_sync_interval"` + + // KVSBackgroundCompactionInterval represents interval for checked id list kvs compaction duration + KVSBackgroundCompactionInterval string `json:"kvs_background_compaction_interval" yaml:"kvs_background_compaction_interval"` + + // IndexPath represents the export index file path + IndexPath string `json:"index_path,omitempty" yaml:"index_path"` + + // Gateway represent gateway service configuration + Gateway *GRPCClient `json:"gateway" yaml:"gateway"` +} + +func (e *IndexExporter) Bind() *IndexExporter { + e.KVSBackgroundCompactionInterval = GetActualValue(e.KVSBackgroundCompactionInterval) + e.KVSBackgroundSyncInterval = GetActualValue(e.KVSBackgroundSyncInterval) + e.IndexPath = GetActualValue(e.IndexPath) + + if e.Gateway != nil { + e.Gateway = e.Gateway.Bind() + } + return e +} diff --git a/k8s/index/job/exportation/configmap.yaml b/k8s/index/job/exportation/configmap.yaml new file mode 100644 index 0000000000..4f781a838d --- /dev/null +++ b/k8s/index/job/exportation/configmap.yaml @@ -0,0 +1,211 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: vald-index-exportation-config + labels: + app.kubernetes.io/name: vald + helm.sh/chart: vald-v1.7.14 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: v1.7.14 + app.kubernetes.io/component: vald-index-exportation +data: + config.yaml: | + --- + version: v0.0.0 + time_zone: JST + logging: + format: raw + level: info + logger: glg + server_config: + servers: + - name: grpc + host: 0.0.0.0 + port: 8081 + grpc: + bidirectional_stream_concurrency: 20 + connection_timeout: "" + enable_admin: true + enable_channelz: true + enable_reflection: true + header_table_size: 0 + initial_conn_window_size: 2097152 + initial_window_size: 1048576 + interceptors: + - RecoverInterceptor + keepalive: + max_conn_age: "" + max_conn_age_grace: "" + max_conn_idle: "" + min_time: 10m + permit_without_stream: false + time: 3h + timeout: 60s + max_concurrent_streams: 0 + max_header_list_size: 0 + max_receive_message_size: 0 + max_send_message_size: 0 + num_stream_workers: 0 + read_buffer_size: 0 + shared_write_buffer: false + wait_for_handlers: true + write_buffer_size: 0 + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false + socket_path: "" + mode: GRPC + health_check_servers: + - name: readiness + host: 0.0.0.0 + port: 3001 + http: + handler_timeout: "" + idle_timeout: "" + read_header_timeout: "" + read_timeout: "" + shutdown_duration: 0s + write_timeout: "" + mode: "" + probe_wait_time: 3s + metrics_servers: + startup_strategy: + - grpc + - readiness + full_shutdown_duration: 600s + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + key: /path/to/key + exporter: + concurrency: 20 + kvs_background_sync_interval: 5s + kvs_background_compaction_interval: 5s + index_path: "/data/export/index" + gateway: + addrs: + - vald-lb-gateway.default.svc.cluster.local:8081 + health_check_duration: "1s" + connection_pool: + enable_dns_resolver: true + enable_rebalance: true + old_conn_close_duration: 2m + rebalance_duration: 30m + size: 3 + backoff: + backoff_factor: 1.1 + backoff_time_limit: 5s + enable_error_log: true + initial_duration: 5ms + jitter_limit: 100ms + maximum_duration: 5s + retry_count: 100 + circuit_breaker: + closed_error_rate: 0.7 + closed_refresh_timeout: 10s + half_open_error_rate: 0.5 + min_samples: 1000 + open_timeout: 1s + call_option: + content_subtype: "" + max_recv_msg_size: 0 + max_retry_rpc_buffer_size: 0 + max_send_msg_size: 0 + wait_for_ready: true + dial_option: + authority: "" + backoff_base_delay: 1s + backoff_jitter: 0.2 + backoff_max_delay: 120s + backoff_multiplier: 1.6 + disable_retry: false + enable_backoff: false + idle_timeout: 1h + initial_connection_window_size: 2097152 + initial_window_size: 1048576 + insecure: true + interceptors: [] + keepalive: + permit_without_stream: false + time: "" + timeout: 30s + max_call_attempts: 0 + max_header_list_size: 0 + max_msg_size: 0 + min_connection_timeout: 20s + net: + dialer: + dual_stack_enabled: true + keepalive: "" + timeout: "" + dns: + cache_enabled: true + cache_expiration: 1h + refresh_duration: 30m + socket_option: + ip_recover_destination_addr: false + ip_transparent: false + reuse_addr: true + reuse_port: true + tcp_cork: false + tcp_defer_accept: false + tcp_fast_open: false + tcp_no_delay: false + tcp_quick_ack: false + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + insecure_skip_verify: false + key: /path/to/key + read_buffer_size: 0 + shared_write_buffer: false + timeout: "" + user_agent: Vald-gRPC + write_buffer_size: 0 + tls: + ca: /path/to/ca + cert: /path/to/cert + enabled: false + insecure_skip_verify: false + key: /path/to/key + observability: + enabled: false + otlp: + collector_endpoint: "otel-collector.monitoring.svc.cluster.local:4317" + trace_batch_timeout: "1s" + trace_export_timeout: "1m" + trace_max_export_batch_size: 1024 + trace_max_queue_size: 256 + metrics_export_interval: "1s" + metrics_export_timeout: "1m" + attribute: + namespace: "_MY_POD_NAMESPACE_" + pod_name: "_MY_POD_NAME_" + node_name: "_MY_NODE_NAME_" + service_name: "vald-index-deletion" + metrics: + enable_cgo: true + enable_goroutine: true + enable_memory: true + enable_version_info: true + version_info_labels: + - vald_version + - server_name + - git_commit + - build_time + - go_version + - go_os + - go_arch + - algorithm_info + trace: + enabled: true diff --git a/k8s/index/job/exportation/cronjob.yaml b/k8s/index/job/exportation/cronjob.yaml new file mode 100644 index 0000000000..60768bbf77 --- /dev/null +++ b/k8s/index/job/exportation/cronjob.yaml @@ -0,0 +1,151 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: batch/v1 +kind: CronJob +metadata: + name: vald-index-exportation + labels: + app: vald-index-exportation + app.kubernetes.io/name: vald + helm.sh/chart: vald-v1.7.14 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: release-name + app.kubernetes.io/component: vald-index-exportation + app.kubernetes.io/version: v1.7.14 +spec: + schedule: "* * * * *" + concurrencyPolicy: Forbid + suspend: true + startingDeadlineSeconds: 43200 + jobTemplate: + spec: + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + app: vald-index-exportation + app.kubernetes.io/name: vald + helm.sh/chart: vald-v1.7.14 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: release-name + app.kubernetes.io/component: vald-index-exportation + app.kubernetes.io/version: v1.7.14 + annotations: + pyroscope.io/scrape: "true" + pyroscope.io/application-name: vald-index-exportation + pyroscope.io/profile-cpu-enabled: "true" + pyroscope.io/profile-mem-enabled: "true" + pyroscope.io/port: "6060" + spec: + initContainers: + - name: wait-for-agent + image: busybox:stable + imagePullPolicy: Always + command: + - /bin/sh + - -e + - -c + - | + until [ "$(wget --server-response --spider --quiet http://vald-agent.default.svc.cluster.local:3001/readiness 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do + echo "waiting for agent to be ready..." + sleep 2; + done + - name: wait-for-discoverer + image: busybox:stable + imagePullPolicy: Always + command: + - /bin/sh + - -e + - -c + - | + until [ "$(wget --server-response --spider --quiet http://vald-discoverer.default.svc.cluster.local:3001/readiness 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do + echo "waiting for discoverer to be ready..." + sleep 2; + done + containers: + - name: vald-index-exportation + image: "vdaas/vald-index-exportation:latest" + imagePullPolicy: Never + volumeMounts: + - name: vald-index-exportation-config + mountPath: /etc/server/ + - name: export-pvc + mountPath: /data + livenessProbe: + failureThreshold: 2 + httpGet: + path: /liveness + port: liveness + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 2 + readinessProbe: + failureThreshold: 2 + httpGet: + path: /readiness + port: readiness + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 2 + startupProbe: + failureThreshold: 30 + httpGet: + path: /liveness + port: liveness + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 2 + ports: + - name: liveness + protocol: TCP + containerPort: 3000 + - name: readiness + protocol: TCP + containerPort: 3001 + - name: grpc + protocol: TCP + containerPort: 8081 + - name: pprof + protocol: TCP + containerPort: 6060 + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + volumes: + - name: vald-index-exportation-config + configMap: + defaultMode: 420 + name: vald-index-exportation-config + - name: export-pvc + persistentVolumeClaim: + claimName: export-pvc diff --git a/k8s/index/job/exportation/pvc.yaml b/k8s/index/job/exportation/pvc.yaml new file mode 100644 index 0000000000..474aebc6de --- /dev/null +++ b/k8s/index/job/exportation/pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: export-pvc + namespace: default +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: local-path diff --git a/pkg/index/job/exportation/config/config.go b/pkg/index/job/exportation/config/config.go new file mode 100644 index 0000000000..00c864822e --- /dev/null +++ b/pkg/index/job/exportation/config/config.go @@ -0,0 +1,66 @@ +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +import ( + "github.com/vdaas/vald/internal/config" + "github.com/vdaas/vald/internal/errors" +) + +// GlobalConfig is a type alias of config.GlobalConfig representing application base configurations. +type GlobalConfig = config.GlobalConfig + +// Data represents the application configurations. +type Data struct { + // GlobalConfig represents application base configurations. + config.GlobalConfig `json:",inline" yaml:",inline"` + + // Server represent all server configurations + Server *config.Servers `json:"server_config" yaml:"server_config"` + + // Observability represents observability configurations. + Observability *config.Observability `json:"observability" yaml:"observability"` + + // Exporter represents auto indexing service configurations. + Exporter *config.IndexExporter `json:"exporter" yaml:"exporter"` +} + +// NewConfig load configurations from file path. +func NewConfig(path string) (cfg *Data, err error) { + cfg = new(Data) + + if err = config.Read(path, &cfg); err != nil { + return nil, err + } + + if cfg != nil { + cfg.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + + if cfg.Observability != nil { + _ = cfg.Observability.Bind() + } else { + cfg.Observability = new(config.Observability).Bind() + } + + if cfg.Exporter != nil { + cfg.Exporter = cfg.Exporter.Bind() + } else { + return nil, errors.ErrInvalidConfig + } + + return cfg, nil +} diff --git a/pkg/index/job/exportation/service/exporter.go b/pkg/index/job/exportation/service/exporter.go new file mode 100644 index 0000000000..dcd919d03e --- /dev/null +++ b/pkg/index/job/exportation/service/exporter.go @@ -0,0 +1,222 @@ +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +import ( + "context" + "fmt" + "io" + "os" + "reflect" + "strconv" + "time" + + "github.com/vdaas/vald/apis/grpc/v1/payload" + "github.com/vdaas/vald/apis/grpc/v1/vald" + vc "github.com/vdaas/vald/internal/client/v1/client/vald" + "github.com/vdaas/vald/internal/db/kvs/pogreb" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/file" + "github.com/vdaas/vald/internal/log" + igrpc "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/observability/trace" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/sync/errgroup" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +const ( + apiName = "vald/index/job/export" + grpcMethodName = "vald.v1.StreamListObject/" + vald.StreamListObjectRPCName +) + +// Exporter represents an interface for exporting. +type Exporter interface { + StartClient(ctx context.Context) (<-chan error, error) + Start(ctx context.Context) error +} + +type export struct { + eg errgroup.Group + gateway vc.Client + storedVector pogreb.DB + + streamListConcurrency int + backgroundSyncInterval time.Duration + backgroundCompactionInterval time.Duration + indexPath string +} + +// New returns Exporter object if no error occurs. +func New(opts ...Option) (Exporter, error) { + e := new(export) + for _, opt := range append(defaultOpts, opts...) { + if err := opt(e); err != nil { + oerr := errors.ErrOptionFailed(err, reflect.ValueOf(opt)) + e := &errors.ErrCriticalOption{} + if errors.As(oerr, &e) { + log.Error(err) + return nil, oerr + } + log.Warn(oerr) + } + } + + if err := file.MkdirAll(e.indexPath, os.ModePerm); err != nil { + log.Errorf("failed to create dir %s", e.indexPath) + return nil, err + } + // Todo: Determine file name + path := file.Join(e.indexPath, fmt.Sprintf("%s.db", strconv.FormatInt(time.Now().Unix(), 10))) + db, err := pogreb.New(pogreb.WithPath(path), + pogreb.WithBackgroundCompactionInterval(e.backgroundCompactionInterval), + pogreb.WithBackgroundSyncInterval(e.backgroundSyncInterval)) + if err != nil { + log.Errorf("failed to open checked List kvs DB %s", path) + return nil, err + } + e.storedVector = db + return e, nil +} + +// StartClient starts the gRPC client. +func (e *export) StartClient(ctx context.Context) (<-chan error, error) { + ech := make(chan error, 1) + gch, err := e.gateway.Start(ctx) + if err != nil { + return nil, err + } + e.eg.Go(safety.RecoverFunc(func() (err error) { + defer close(ech) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err = <-gch: + } + if err != nil { + select { + case <-ctx.Done(): + return ctx.Err() + case ech <- err: + } + } + } + })) + return ech, nil +} + +func (e *export) Start(ctx context.Context) error { + err := e.doExportIndex(ctx, + func(ctx context.Context, rc vald.ObjectClient, copts ...grpc.CallOption) (vald.Object_StreamListObjectClient, error) { + return rc.StreamListObject(ctx, &payload.Object_List_Request{}, copts...) + }, + ) + return err +} + +func (e *export) doExportIndex( + ctx context.Context, + fn func(ctx context.Context, rc vald.ObjectClient, copts ...grpc.CallOption) (vald.Object_StreamListObjectClient, error), +) (errs error) { + ctx, span := trace.StartSpan(igrpc.WrapGRPCMethod(ctx, grpcMethodName), apiName+"/service/index.doExportIndex") + defer func() { + if span != nil { + span.End() + } + }() + + emptyReq := new(payload.Object_List_Request) + + eg, egctx := errgroup.WithContext(ctx) + eg.SetLimit(e.streamListConcurrency) + ctx, cancel := context.WithCancelCause(egctx) + gatewayAddrs := e.gateway.GRPCClient().ConnectedAddrs() + if len(gatewayAddrs) < 0 { + log.Errorf("Active gateway is not found.: %v ", ctx.Err()) + } + + conn, err := grpc.NewClient(gatewayAddrs[0], grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return err + } + + vcClient := vc.NewValdClient(conn) + grpcCallOpts := []grpc.CallOption{ + grpc.WaitForReady(true), + } + // stream, err := fn(ctx, vc.NewValdClient(conn), grpcCallOpts...) + stream, err := vcClient.StreamListObject(ctx, emptyReq, grpcCallOpts...) + if err != nil || stream == nil { + return err + } + + for { + select { + case <-ctx.Done(): + if !errors.Is(ctx.Err(), context.Canceled) { + log.Errorf("context done unexpectedly: %v", ctx.Err()) + } + if !errors.Is(context.Cause(ctx), io.EOF) { + log.Errorf("context canceled due to %v", ctx.Err()) + } + err = eg.Wait() + if err != nil { + log.Errorf("exporter returned error status errgroup returned error: %v", ctx.Err()) + } else { + log.Infof("exporter finished") + } + return nil + default: + res, err := stream.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + cancel(io.EOF) + } else { + cancel(errors.ErrStreamListObjectStreamFinishedUnexpectedly(err)) + } + } else if res != nil && res.GetVector() != nil && res.GetVector().GetId() != "" { + eg.Go(safety.RecoverFunc(func() (err error) { + objVec := res.GetVector() + log.Infof("received object vector id: %s, timestamp: %d", objVec.GetId(), objVec.GetTimestamp()) + + storedBinary, ok, err := e.storedVector.Get(objVec.GetId()) + if err != nil { + log.Errorf("failed to perform Get from check list but still try to finish processing without cache: %v", err) + return err + } + + var storedObjVec payload.Object_Vector + if ok { + if err := storedObjVec.UnmarshalVT(storedBinary); err != nil { + log.Errorf("failed to Unmarshal proto to payload.Object_Vector: %v", err) + return err + } + } + + isUpsertVector := !ok || storedObjVec.GetTimestamp() < objVec.GetTimestamp() + if isUpsertVector { + dAtA, err := objVec.MarshalVT() + if err != nil { + return err + } + e.storedVector.Set(objVec.GetId(), dAtA) + } + return nil + })) + } + } + } +} diff --git a/pkg/index/job/exportation/service/options.go b/pkg/index/job/exportation/service/options.go new file mode 100644 index 0000000000..6d4a5889af --- /dev/null +++ b/pkg/index/job/exportation/service/options.go @@ -0,0 +1,104 @@ +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package service + +import ( + "github.com/vdaas/vald/internal/client/v1/client/vald" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/sync/errgroup" + "github.com/vdaas/vald/internal/timeutil" +) + +type Option func(_ *export) error + +var defaultOpts = []Option{ + WithStreamListConcurrency(200), + WithKVSSyncInterval("5s"), + WithKVSCompactionInterval("5s"), + WithIndexPath("/var/export/index"), + WithErrGroup(errgroup.Get()), +} + +// WithStreamListConcurrency returns Option that sets streamListConcurrency. +func WithStreamListConcurrency(num int) Option { + return func(e *export) error { + if num <= 0 { + return errors.NewErrInvalidOption("streamListConcurrency", num) + } + e.streamListConcurrency = num + return nil + } +} + +// WithKVSSyncInterval returns Option that sets interval for background file sync. +func WithKVSSyncInterval(dur string) Option { + return func(e *export) error { + if dur == "" { + return nil + } + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + e.backgroundSyncInterval = d + return nil + } +} + +// WithKVSCompactionInterval returns Option that sets interval for background file compaction. +func WithKVSCompactionInterval(dur string) Option { + return func(e *export) error { + if dur == "" { + return nil + } + d, err := timeutil.Parse(dur) + if err != nil { + return err + } + e.backgroundCompactionInterval = d + return nil + } +} + +// WithIndexPath returns Option that sets indexPath. +func WithIndexPath(path string) Option { + return func(e *export) error { + if path == "" { + return errors.NewErrInvalidOption("indexPath", path) + } + e.indexPath = path + return nil + } +} + +// WithGateway returns Option that sets gateway client. +func WithGateway(client vald.Client) Option { + return func(e *export) error { + if client == nil { + return errors.NewErrCriticalOption("gateway", client) + } + e.gateway = client + return nil + } +} + +// WithErrGroup returns Option that set errgroup. +func WithErrGroup(eg errgroup.Group) Option { + return func(e *export) error { + if eg != nil { + e.eg = eg + } + return nil + } +} diff --git a/pkg/index/job/exportation/usecase/exportation.go b/pkg/index/job/exportation/usecase/exportation.go new file mode 100644 index 0000000000..c244bea159 --- /dev/null +++ b/pkg/index/job/exportation/usecase/exportation.go @@ -0,0 +1,183 @@ +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package usecase + +import ( + "context" + "os" + "syscall" + + "github.com/vdaas/vald/internal/client/v1/client/vald" + "github.com/vdaas/vald/internal/errors" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/observability" + "github.com/vdaas/vald/internal/runner" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/servers/starter" + "github.com/vdaas/vald/internal/sync/errgroup" + "github.com/vdaas/vald/pkg/index/job/exportation/config" + "github.com/vdaas/vald/pkg/index/job/exportation/service" +) + +type run struct { + eg errgroup.Group + cfg *config.Data + observability observability.Observability + server starter.Server + exporter service.Exporter +} + +// New returns Runner instance. +func New(cfg *config.Data) (_ runner.Runner, err error) { + eg := errgroup.Get() + + gOpts, err := cfg.Exporter.Gateway.Opts() + if err != nil { + return nil, err + } + // skipcq: CRT-D0001 + gOpts = append(gOpts, grpc.WithErrGroup(eg)) + + gateway, err := vald.New(vald.WithClient(grpc.New(gOpts...))) + if err != nil { + return nil, err + } + + exporter, err := service.New( + service.WithStreamListConcurrency(cfg.Exporter.Concurrency), + service.WithIndexPath(cfg.Exporter.IndexPath), + service.WithGateway(gateway), + ) + if err != nil { + return nil, err + } + + var obs observability.Observability + if cfg.Observability.Enabled { + obs, err = observability.NewWithConfig( + cfg.Observability, + ) + if err != nil { + return nil, err + } + } + + // grpcServerOptions := []server.Option{ + // server.WithGRPCOption( + // grpc.ChainUnaryInterceptor(recover.RecoverInterceptor()), + // grpc.ChainStreamInterceptor(recover.RecoverStreamInterceptor()), + // ), + // } + + // // For health check and metrics + // srv, err := starter.New(starter.WithConfig(cfg.Server), + // starter.WithGRPC(func(_ *iconf.Server) []server.Option { + // return grpcServerOptions + // }), + // ) + // if err != nil { + // return nil, err + // } + + return &run{ + eg: eg, + cfg: cfg, + observability: obs, + // server: srv, + exporter: exporter, + }, nil +} + +// PreStart is a method called before execution of Start, and it invokes the PreStart method of observability. +func (r *run) PreStart(ctx context.Context) error { + if r.observability != nil { + return r.observability.PreStart(ctx) + } + return nil +} + +// Start is a method used to initiate an operation in the run, and it returns a channel for receiving errors +// during the operation and an error representing any initialization errors. +func (r *run) Start(ctx context.Context) (<-chan error, error) { + ech := make(chan error, 3) + var sech, oech, cech <-chan error + if r.observability != nil { + oech = r.observability.Start(ctx) + } + // sech = r.server.ListenAndServe(ctx) + cech, err := r.exporter.StartClient(ctx) + if err != nil { + close(ech) + return nil, err + } + + r.eg.Go(safety.RecoverFunc(func() (err error) { + defer func() { + p, err := os.FindProcess(os.Getpid()) + if err != nil { + // using Fatal to avoid this process to be zombie + // skipcq: RVV-A0003 + log.Fatalf("failed to find my pid to kill %v", err) + return + } + log.Info("sending SIGTERM to myself to stop this job") + if err := p.Signal(syscall.SIGTERM); err != nil { + log.Error(err) + } + }() + return r.exporter.Start(ctx) + })) + + r.eg.Go(safety.RecoverFunc(func() (err error) { + defer close(ech) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err = <-oech: + case err = <-sech: + case err = <-cech: + } + if err != nil { + select { + case <-ctx.Done(): + return errors.Join(ctx.Err(), err) + case ech <- err: + } + } + } + })) + return ech, nil +} + +// PreStop is a method called before execution of Stop. +func (*run) PreStop(_ context.Context) error { + return nil +} + +// Stop is a method used to stop an operation in the run. +func (r *run) Stop(ctx context.Context) (errs error) { + if r.observability != nil { + if err := r.observability.Stop(ctx); err != nil { + errs = errors.Join(errs, err) + } + } + return errs +} + +// PostStop is a method called after execution of Stop. +func (*run) PostStop(_ context.Context) error { + return nil +} From 03e339fc467989cfd3af3b2fd7a3e22b493fbc37 Mon Sep 17 00:00:00 2001 From: HighPon Date: Thu, 12 Dec 2024 01:47:38 +0000 Subject: [PATCH 03/18] tmp commit --- cmd/index/job/exportation/main.go | 3 +- cmd/index/job/exportation/sample.yaml | 2 +- .../job/exportation/usecase/exportation.go | 40 ++++++++++--------- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/cmd/index/job/exportation/main.go b/cmd/index/job/exportation/main.go index 8c16dd224b..25eaaadda7 100644 --- a/cmd/index/job/exportation/main.go +++ b/cmd/index/job/exportation/main.go @@ -38,7 +38,8 @@ func main() { runner.WithName(name), runner.WithVersion(info.Version, maxVersion, minVersion), runner.WithConfigLoader(func(path string) (any, *config.GlobalConfig, error) { - cfg, err := config.NewConfig(path) + // cfg, err := config.NewConfig(path) + cfg, err := config.NewConfig("cmd/index/job/exportation/sample.yaml") if err != nil { return nil, nil, errors.Wrap(err, "failed to load "+name+"'s configuration") } diff --git a/cmd/index/job/exportation/sample.yaml b/cmd/index/job/exportation/sample.yaml index 4ed586d02f..4cc6ce98fd 100644 --- a/cmd/index/job/exportation/sample.yaml +++ b/cmd/index/job/exportation/sample.yaml @@ -75,7 +75,7 @@ exporter: index_path: "/var/export/index" gateway: addrs: - - vald-lb-gateway.default.svc.cluster.local:8081 + - localhost:20000 health_check_duration: "1s" connection_pool: enable_dns_resolver: true diff --git a/pkg/index/job/exportation/usecase/exportation.go b/pkg/index/job/exportation/usecase/exportation.go index c244bea159..f21bf6e6ac 100644 --- a/pkg/index/job/exportation/usecase/exportation.go +++ b/pkg/index/job/exportation/usecase/exportation.go @@ -19,12 +19,16 @@ import ( "syscall" "github.com/vdaas/vald/internal/client/v1/client/vald" + iconf "github.com/vdaas/vald/internal/config" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc" + "github.com/vdaas/vald/internal/net/grpc/interceptor/server/recover" "github.com/vdaas/vald/internal/observability" "github.com/vdaas/vald/internal/runner" "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/servers/server" "github.com/vdaas/vald/internal/servers/starter" "github.com/vdaas/vald/internal/sync/errgroup" "github.com/vdaas/vald/pkg/index/job/exportation/config" @@ -74,29 +78,29 @@ func New(cfg *config.Data) (_ runner.Runner, err error) { } } - // grpcServerOptions := []server.Option{ - // server.WithGRPCOption( - // grpc.ChainUnaryInterceptor(recover.RecoverInterceptor()), - // grpc.ChainStreamInterceptor(recover.RecoverStreamInterceptor()), - // ), - // } - - // // For health check and metrics - // srv, err := starter.New(starter.WithConfig(cfg.Server), - // starter.WithGRPC(func(_ *iconf.Server) []server.Option { - // return grpcServerOptions - // }), - // ) - // if err != nil { - // return nil, err - // } + grpcServerOptions := []server.Option{ + server.WithGRPCOption( + grpc.ChainUnaryInterceptor(recover.RecoverInterceptor()), + grpc.ChainStreamInterceptor(recover.RecoverStreamInterceptor()), + ), + } + + // For health check and metrics + srv, err := starter.New(starter.WithConfig(cfg.Server), + starter.WithGRPC(func(_ *iconf.Server) []server.Option { + return grpcServerOptions + }), + ) + if err != nil { + return nil, err + } return &run{ eg: eg, cfg: cfg, observability: obs, - // server: srv, - exporter: exporter, + server: srv, + exporter: exporter, }, nil } From b9168277a93e4bbcc8acd12d88818460b1ebef34 Mon Sep 17 00:00:00 2001 From: HighPon Date: Sun, 15 Dec 2024 17:23:32 +0000 Subject: [PATCH 04/18] fix: Delete index importation --- .../dockers-index-exportation-image.yaml | 247 ++++++++++++++++++ Makefile | 1 - Makefile.d/build.mk | 10 - Makefile.d/docker.mk | 13 - cmd/index/job/importation/main.go | 26 -- dockers/index/job/exportation/Dockerfile | 1 + dockers/index/job/importation/Dockerfile | 87 ------ hack/docker/gen/main.go | 4 - 8 files changed, 248 insertions(+), 141 deletions(-) create mode 100644 .github/workflows/dockers-index-exportation-image.yaml delete mode 100644 cmd/index/job/importation/main.go delete mode 100644 dockers/index/job/importation/Dockerfile diff --git a/.github/workflows/dockers-index-exportation-image.yaml b/.github/workflows/dockers-index-exportation-image.yaml new file mode 100644 index 0000000000..e1562edd70 --- /dev/null +++ b/.github/workflows/dockers-index-exportation-image.yaml @@ -0,0 +1,247 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go + + +name: 'Build docker image: index-exportation' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - '*.*.*-*' + - v*.*.* + - v*.*.*-* + pull_request: + paths: + - '!**/*_mock.go' + - '!**/*_test.go' + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - .github/workflows/dockers-index-exportation-image.yaml + - Makefile + - Makefile.d/** + - apis/grpc/v1/payload/*.go + - apis/grpc/v1/rpc/errdetails/*.go + - apis/grpc/v1/vald/*.go + - apis/proto/** + - cmd/index/job/exportation/*.go + - dockers/index/job/exportation/Dockerfile + - go.mod + - go.sum + - hack/docker/gen/main.go + - internal/backoff/*.go + - internal/cache/*.go + - internal/cache/cacher/*.go + - internal/cache/gache/*.go + - internal/circuitbreaker/*.go + - internal/client/v1/client/vald/*.go + - internal/config/*.go + - internal/conv/*.go + - internal/db/kvs/pogreb/*.go + - internal/db/kvs/redis/*.go + - internal/db/nosql/cassandra/*.go + - internal/db/rdb/mysql/*.go + - internal/db/rdb/mysql/dbr/*.go + - internal/encoding/json/*.go + - internal/errors/*.go + - internal/file/*.go + - internal/info/*.go + - internal/io/*.go + - internal/k8s/*.go + - internal/log/*.go + - internal/log/format/*.go + - internal/log/glg/*.go + - internal/log/level/*.go + - internal/log/logger/*.go + - internal/log/nop/*.go + - internal/log/retry/*.go + - internal/log/zap/*.go + - internal/net/*.go + - internal/net/control/*.go + - internal/net/grpc/*.go + - internal/net/grpc/admin/*.go + - internal/net/grpc/codes/*.go + - internal/net/grpc/credentials/*.go + - internal/net/grpc/errdetails/*.go + - internal/net/grpc/health/*.go + - internal/net/grpc/interceptor/client/metric/*.go + - internal/net/grpc/interceptor/client/trace/*.go + - internal/net/grpc/interceptor/server/logging/*.go + - internal/net/grpc/interceptor/server/metric/*.go + - internal/net/grpc/interceptor/server/recover/*.go + - internal/net/grpc/interceptor/server/trace/*.go + - internal/net/grpc/keepalive/*.go + - internal/net/grpc/logger/*.go + - internal/net/grpc/pool/*.go + - internal/net/grpc/proto/*.go + - internal/net/grpc/reflection/*.go + - internal/net/grpc/status/*.go + - internal/net/grpc/types/*.go + - internal/net/http/dump/*.go + - internal/net/http/json/*.go + - internal/net/http/metrics/*.go + - internal/net/http/middleware/*.go + - internal/net/http/rest/*.go + - internal/net/http/routing/*.go + - internal/net/quic/*.go + - internal/observability/*.go + - internal/observability/attribute/*.go + - internal/observability/exporter/*.go + - internal/observability/exporter/otlp/*.go + - internal/observability/metrics/*.go + - internal/observability/metrics/grpc/*.go + - internal/observability/metrics/mem/*.go + - internal/observability/metrics/runtime/cgo/*.go + - internal/observability/metrics/runtime/goroutine/*.go + - internal/observability/metrics/version/*.go + - internal/observability/trace/*.go + - internal/os/*.go + - internal/params/*.go + - internal/rand/*.go + - internal/runner/*.go + - internal/safety/*.go + - internal/servers/*.go + - internal/servers/server/*.go + - internal/servers/starter/*.go + - internal/strings/*.go + - internal/sync/*.go + - internal/sync/errgroup/*.go + - internal/sync/semaphore/*.go + - internal/sync/singleflight/*.go + - internal/timeutil/*.go + - internal/timeutil/location/*.go + - internal/tls/*.go + - internal/version/*.go + - pkg/index/job/exportation/config/*.go + - pkg/index/job/exportation/service/*.go + - pkg/index/job/exportation/usecase/*.go + - versions/GO_VERSION + pull_request_target: + paths: + - '!**/*_mock.go' + - '!**/*_test.go' + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - .github/workflows/dockers-index-exportation-image.yaml + - Makefile + - Makefile.d/** + - apis/grpc/v1/payload/*.go + - apis/grpc/v1/rpc/errdetails/*.go + - apis/grpc/v1/vald/*.go + - apis/proto/** + - cmd/index/job/exportation/*.go + - dockers/index/job/exportation/Dockerfile + - go.mod + - go.sum + - hack/docker/gen/main.go + - internal/backoff/*.go + - internal/cache/*.go + - internal/cache/cacher/*.go + - internal/cache/gache/*.go + - internal/circuitbreaker/*.go + - internal/client/v1/client/vald/*.go + - internal/config/*.go + - internal/conv/*.go + - internal/db/kvs/pogreb/*.go + - internal/db/kvs/redis/*.go + - internal/db/nosql/cassandra/*.go + - internal/db/rdb/mysql/*.go + - internal/db/rdb/mysql/dbr/*.go + - internal/encoding/json/*.go + - internal/errors/*.go + - internal/file/*.go + - internal/info/*.go + - internal/io/*.go + - internal/k8s/*.go + - internal/log/*.go + - internal/log/format/*.go + - internal/log/glg/*.go + - internal/log/level/*.go + - internal/log/logger/*.go + - internal/log/nop/*.go + - internal/log/retry/*.go + - internal/log/zap/*.go + - internal/net/*.go + - internal/net/control/*.go + - internal/net/grpc/*.go + - internal/net/grpc/admin/*.go + - internal/net/grpc/codes/*.go + - internal/net/grpc/credentials/*.go + - internal/net/grpc/errdetails/*.go + - internal/net/grpc/health/*.go + - internal/net/grpc/interceptor/client/metric/*.go + - internal/net/grpc/interceptor/client/trace/*.go + - internal/net/grpc/interceptor/server/logging/*.go + - internal/net/grpc/interceptor/server/metric/*.go + - internal/net/grpc/interceptor/server/recover/*.go + - internal/net/grpc/interceptor/server/trace/*.go + - internal/net/grpc/keepalive/*.go + - internal/net/grpc/logger/*.go + - internal/net/grpc/pool/*.go + - internal/net/grpc/proto/*.go + - internal/net/grpc/reflection/*.go + - internal/net/grpc/status/*.go + - internal/net/grpc/types/*.go + - internal/net/http/dump/*.go + - internal/net/http/json/*.go + - internal/net/http/metrics/*.go + - internal/net/http/middleware/*.go + - internal/net/http/rest/*.go + - internal/net/http/routing/*.go + - internal/net/quic/*.go + - internal/observability/*.go + - internal/observability/attribute/*.go + - internal/observability/exporter/*.go + - internal/observability/exporter/otlp/*.go + - internal/observability/metrics/*.go + - internal/observability/metrics/grpc/*.go + - internal/observability/metrics/mem/*.go + - internal/observability/metrics/runtime/cgo/*.go + - internal/observability/metrics/runtime/goroutine/*.go + - internal/observability/metrics/version/*.go + - internal/observability/trace/*.go + - internal/os/*.go + - internal/params/*.go + - internal/rand/*.go + - internal/runner/*.go + - internal/safety/*.go + - internal/servers/*.go + - internal/servers/server/*.go + - internal/servers/starter/*.go + - internal/strings/*.go + - internal/sync/*.go + - internal/sync/errgroup/*.go + - internal/sync/semaphore/*.go + - internal/sync/singleflight/*.go + - internal/timeutil/*.go + - internal/timeutil/location/*.go + - internal/tls/*.go + - internal/version/*.go + - pkg/index/job/exportation/config/*.go + - pkg/index/job/exportation/service/*.go + - pkg/index/job/exportation/usecase/*.go + - versions/GO_VERSION +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: index-exportation + secrets: inherit diff --git a/Makefile b/Makefile index 799713d206..f0e6083aea 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,6 @@ INDEX_CORRECTION_IMAGE = $(NAME)-index-correction INDEX_CREATION_IMAGE = $(NAME)-index-creation INDEX_DELETION_IMAGE = $(NAME)-index-deletion INDEX_EXPORTATION_IMAGE = $(NAME)-index-exportation -INDEX_IMPORTATION_IMAGE = $(NAME)-index-importation INDEX_OPERATOR_IMAGE = $(NAME)-index-operator INDEX_SAVE_IMAGE = $(NAME)-index-save LB_GATEWAY_IMAGE = $(NAME)-lb-gateway diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk index 76996f0edf..a5d62b78b5 100644 --- a/Makefile.d/build.mk +++ b/Makefile.d/build.mk @@ -26,7 +26,6 @@ binary/build: \ cmd/index/job/creation/index-creation \ cmd/index/job/deletion/index-deletion \ cmd/index/job/exportation/index-exportation \ - cmd/index/job/importation/index-importation \ cmd/index/job/readreplica/rotate/readreplica-rotate \ cmd/index/job/save/index-save \ cmd/index/operator/index-operator \ @@ -91,10 +90,6 @@ cmd/index/job/exportation/index-exportation: $(eval CGO_ENABLED = 0) $(call go-build,index/job/exportation,,-static,,,$@) -cmd/index/job/importation/index-importation: - $(eval CGO_ENABLED = 0) - $(call go-build,index/job/importation,,-static,,,$@) - cmd/index/job/save/index-save: $(eval CGO_ENABLED = 0) $(call go-build,index/job/save,,-static,,,$@) @@ -145,7 +140,6 @@ binary/build/zip: \ artifacts/vald-index-creation-$(GOOS)-$(GOARCH).zip \ artifacts/vald-index-deletion-$(GOOS)-$(GOARCH).zip \ artifacts/vald-index-exportation-$(GOOS)-$(GOARCH).zip \ - artifacts/vald-index-importation-$(GOOS)-$(GOARCH).zip \ artifacts/vald-index-operator-$(GOOS)-$(GOARCH).zip \ artifacts/vald-index-save-$(GOOS)-$(GOARCH).zip \ artifacts/vald-lb-gateway-$(GOOS)-$(GOARCH).zip \ @@ -213,10 +207,6 @@ artifacts/vald-index-exporation-$(GOOS)-$(GOARCH).zip: cmd/index/job/exportation $(call mkdir, $(dir $@)) zip --junk-paths $@ $< -artifacts/vald-index-importation-$(GOOS)-$(GOARCH).zip: cmd/index/job/importation/index-importation - $(call mkdir, $(dir $@)) - zip --junk-paths $@ $< - artifacts/vald-index-save-$(GOOS)-$(GOARCH).zip: cmd/index/job/save/index-save $(call mkdir, $(dir $@)) zip --junk-paths $@ $< diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index 524fc2050e..a76d931932 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -38,7 +38,6 @@ docker/build: \ docker/build/index-creation \ docker/build/index-deletion \ docker/build/index-exportation \ - docker/build/index-importation \ docker/build/index-operator \ docker/build/index-save \ docker/build/loadtest \ @@ -68,7 +67,6 @@ docker/xpanes/build: docker/build/index-creation \ docker/build/index-deletion \ docker/build/index-exportation \ - docker/build/index-importation \ docker/build/index-operator \ docker/build/index-save \ docker/build/loadtest \ @@ -369,17 +367,6 @@ docker/build/index-exportation: IMAGE=$(INDEX_EXPORTATION_IMAGE) \ docker/build/image -.PHONY: docker/name/index-importation -docker/name/index-importation: - @echo "$(ORG)/$(INDEX_IMPORTATION_IMAGE)" - -.PHONY: docker/build/index-importation -## build index-importation image -docker/build/index-importation: - @make DOCKERFILE="$(ROOTDIR)/dockers/index/job/importation/Dockerfile" \ - IMAGE=$(INDEX_IMPORTATION_IMAGE) \ - docker/build/image - .PHONY: docker/name/index-operator docker/name/index-operator: @echo "$(ORG)/$(INDEX_OPERATOR_IMAGE)" diff --git a/cmd/index/job/importation/main.go b/cmd/index/job/importation/main.go deleted file mode 100644 index ab8fb8146e..0000000000 --- a/cmd/index/job/importation/main.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (C) 2019-2024 vdaas.org vald team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package main - -import "fmt" - -const ( - maxVersion = "v0.0.10" - minVersion = "v0.0.0" - name = "index importation job" -) - -func main() { - fmt.Println("hello world") -} diff --git a/dockers/index/job/exportation/Dockerfile b/dockers/index/job/exportation/Dockerfile index 495f985dab..d329680172 100644 --- a/dockers/index/job/exportation/Dockerfile +++ b/dockers/index/job/exportation/Dockerfile @@ -1,4 +1,5 @@ # syntax = docker/dockerfile:latest +# check=error=true # # Copyright (C) 2019-2024 vdaas.org vald team # diff --git a/dockers/index/job/importation/Dockerfile b/dockers/index/job/importation/Dockerfile deleted file mode 100644 index c7eb529861..0000000000 --- a/dockers/index/job/importation/Dockerfile +++ /dev/null @@ -1,87 +0,0 @@ -# syntax = docker/dockerfile:latest -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go -ARG UPX_OPTIONS=-9 -# skipcq: DOK-DL3026,DOK-DL3007 -FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder -LABEL maintainer="vdaas.org vald team " -# skipcq: DOK-DL3002 -USER root:root -ARG TARGETARCH -ARG TARGETOS -ARG GO_VERSION -ARG RUST_VERSION -ENV APP_NAME=index-importation -ENV DEBIAN_FRONTEND=noninteractive -ENV GO111MODULE=on -ENV GOPATH=/go -ENV GOROOT=/opt/go -ENV HOME=/root -ENV INITRD=No -ENV LANG=en_US.UTF-8 -ENV LANGUAGE=en_US.UTF-8 -ENV LC_ALL=en_US.UTF-8 -ENV ORG=vdaas -ENV PKG=index/job/importation -ENV REPO=vald -ENV TZ=Etc/UTC -ENV USER=root -ENV PATH=${GOPATH}/bin:${GOROOT}/bin:/usr/local/bin:${PATH} -WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} -SHELL ["/bin/bash", "-o", "pipefail", "-c"] -#skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008 -RUN --mount=type=bind,target=.,rw \ - --mount=type=tmpfs,target=/tmp \ - --mount=type=cache,target=/var/lib/apt,sharing=locked,id=${APP_NAME} \ - --mount=type=cache,target=/var/cache/apt,sharing=locked,id=${APP_NAME} \ - --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${TARGETARCH}" \ - --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${TARGETARCH}" \ - --mount=type=tmpfs,target="${GOPATH}/src" \ - set -ex \ - && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \ - && echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf.d/no-install-recommends \ - && apt-get clean \ - && apt-get update -y \ - && apt-get upgrade -y \ - && apt-get install -y --no-install-recommends --fix-missing \ - build-essential \ - ca-certificates \ - curl \ - tzdata \ - locales \ - git \ - && ldconfig \ - && echo "${LANG} UTF-8" > /etc/locale.gen \ - && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \ - && locale-gen ${LANGUAGE} \ - && update-locale LANG=${LANGUAGE} \ - && dpkg-reconfigure -f noninteractive tzdata \ - && apt-get clean \ - && apt-get autoclean -y \ - && apt-get autoremove -y \ - && make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/install \ - && make GOPATH="${GOPATH}" GOROOT="${GOROOT}" GO_VERSION="${GO_VERSION}" go/download \ - && make GOARCH="${TARGETARCH}" GOOS="${TARGETOS}" REPO="${ORG}" NAME="${REPO}" cmd/${PKG}/${APP_NAME} \ - && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}" -# skipcq: DOK-DL3026,DOK-DL3007 -FROM gcr.io/distroless/static:nonroot -LABEL maintainer="vdaas.org vald team " -COPY --from=builder /usr/bin/index-importation /usr/bin/index-importation -# skipcq: DOK-DL3002 -USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index-importation"] \ No newline at end of file diff --git a/hack/docker/gen/main.go b/hack/docker/gen/main.go index 0afbca5d65..d90242ba7e 100644 --- a/hack/docker/gen/main.go +++ b/hack/docker/gen/main.go @@ -739,10 +739,6 @@ func main() { AppName: "index-exportation", PackageDir: "index/job/exportation", }, - "vald-index-importation": { - AppName: "index-importation", - PackageDir: "index/job/importation", - }, "vald-readreplica-rotate": { AppName: "readreplica-rotate", PackageDir: "index/job/readreplica/rotate", From 2e4662ed51004b6489e2f2d40ddba5875f8c2ee9 Mon Sep 17 00:00:00 2001 From: HighPon Date: Mon, 16 Dec 2024 03:59:38 +0000 Subject: [PATCH 05/18] fix: k8s manifest for exportation --- cmd/index/job/exportation/sample.yaml | 2 +- k8s/index/job/exportation/cronjob.yaml | 19 ++++--------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/cmd/index/job/exportation/sample.yaml b/cmd/index/job/exportation/sample.yaml index 4cc6ce98fd..4ed586d02f 100644 --- a/cmd/index/job/exportation/sample.yaml +++ b/cmd/index/job/exportation/sample.yaml @@ -75,7 +75,7 @@ exporter: index_path: "/var/export/index" gateway: addrs: - - localhost:20000 + - vald-lb-gateway.default.svc.cluster.local:8081 health_check_duration: "1s" connection_pool: enable_dns_resolver: true diff --git a/k8s/index/job/exportation/cronjob.yaml b/k8s/index/job/exportation/cronjob.yaml index 60768bbf77..4c3ceda683 100644 --- a/k8s/index/job/exportation/cronjob.yaml +++ b/k8s/index/job/exportation/cronjob.yaml @@ -51,7 +51,8 @@ spec: pyroscope.io/port: "6060" spec: initContainers: - - name: wait-for-agent + initContainers: + - name: wait-for-gateway-lb image: busybox:stable imagePullPolicy: Always command: @@ -59,20 +60,8 @@ spec: - -e - -c - | - until [ "$(wget --server-response --spider --quiet http://vald-agent.default.svc.cluster.local:3001/readiness 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do - echo "waiting for agent to be ready..." - sleep 2; - done - - name: wait-for-discoverer - image: busybox:stable - imagePullPolicy: Always - command: - - /bin/sh - - -e - - -c - - | - until [ "$(wget --server-response --spider --quiet http://vald-discoverer.default.svc.cluster.local:3001/readiness 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do - echo "waiting for discoverer to be ready..." + until [ "$(wget --server-response --spider --quiet http://vald-lb-gateway.default.svc.cluster.local:3001/readiness 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do + echo "waiting for gateway-lb to be ready..." sleep 2; done containers: From 633df960f930de716fc04668ecc5bb49dbb42450 Mon Sep 17 00:00:00 2001 From: HighPon Date: Mon, 16 Dec 2024 04:03:01 +0000 Subject: [PATCH 06/18] fix: refactor for exporter --- cmd/index/job/exportation/main.go | 3 +-- pkg/index/job/exportation/service/exporter.go | 12 +++--------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/cmd/index/job/exportation/main.go b/cmd/index/job/exportation/main.go index 25eaaadda7..8c16dd224b 100644 --- a/cmd/index/job/exportation/main.go +++ b/cmd/index/job/exportation/main.go @@ -38,8 +38,7 @@ func main() { runner.WithName(name), runner.WithVersion(info.Version, maxVersion, minVersion), runner.WithConfigLoader(func(path string) (any, *config.GlobalConfig, error) { - // cfg, err := config.NewConfig(path) - cfg, err := config.NewConfig("cmd/index/job/exportation/sample.yaml") + cfg, err := config.NewConfig(path) if err != nil { return nil, nil, errors.Wrap(err, "failed to load "+name+"'s configuration") } diff --git a/pkg/index/job/exportation/service/exporter.go b/pkg/index/job/exportation/service/exporter.go index dcd919d03e..32f5a662ff 100644 --- a/pkg/index/job/exportation/service/exporter.go +++ b/pkg/index/job/exportation/service/exporter.go @@ -78,7 +78,7 @@ func New(opts ...Option) (Exporter, error) { log.Errorf("failed to create dir %s", e.indexPath) return nil, err } - // Todo: Determine file name + path := file.Join(e.indexPath, fmt.Sprintf("%s.db", strconv.FormatInt(time.Now().Unix(), 10))) db, err := pogreb.New(pogreb.WithPath(path), pogreb.WithBackgroundCompactionInterval(e.backgroundCompactionInterval), @@ -119,17 +119,12 @@ func (e *export) StartClient(ctx context.Context) (<-chan error, error) { } func (e *export) Start(ctx context.Context) error { - err := e.doExportIndex(ctx, - func(ctx context.Context, rc vald.ObjectClient, copts ...grpc.CallOption) (vald.Object_StreamListObjectClient, error) { - return rc.StreamListObject(ctx, &payload.Object_List_Request{}, copts...) - }, - ) + err := e.doExportIndex(ctx) return err } func (e *export) doExportIndex( ctx context.Context, - fn func(ctx context.Context, rc vald.ObjectClient, copts ...grpc.CallOption) (vald.Object_StreamListObjectClient, error), ) (errs error) { ctx, span := trace.StartSpan(igrpc.WrapGRPCMethod(ctx, grpcMethodName), apiName+"/service/index.doExportIndex") defer func() { @@ -144,7 +139,7 @@ func (e *export) doExportIndex( eg.SetLimit(e.streamListConcurrency) ctx, cancel := context.WithCancelCause(egctx) gatewayAddrs := e.gateway.GRPCClient().ConnectedAddrs() - if len(gatewayAddrs) < 0 { + if len(gatewayAddrs) == 0 { log.Errorf("Active gateway is not found.: %v ", ctx.Err()) } @@ -157,7 +152,6 @@ func (e *export) doExportIndex( grpcCallOpts := []grpc.CallOption{ grpc.WaitForReady(true), } - // stream, err := fn(ctx, vc.NewValdClient(conn), grpcCallOpts...) stream, err := vcClient.StreamListObject(ctx, emptyReq, grpcCallOpts...) if err != nil || stream == nil { return err From befd06bbdd6cef0091c6f8a84e36f916f62d513a Mon Sep 17 00:00:00 2001 From: s-shiraki Date: Wed, 18 Dec 2024 00:07:29 +0900 Subject: [PATCH 07/18] Update pkg/index/job/exportation/service/exporter.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: s-shiraki --- pkg/index/job/exportation/service/exporter.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/index/job/exportation/service/exporter.go b/pkg/index/job/exportation/service/exporter.go index 32f5a662ff..80d830c3a9 100644 --- a/pkg/index/job/exportation/service/exporter.go +++ b/pkg/index/job/exportation/service/exporter.go @@ -140,7 +140,8 @@ func (e *export) doExportIndex( ctx, cancel := context.WithCancelCause(egctx) gatewayAddrs := e.gateway.GRPCClient().ConnectedAddrs() if len(gatewayAddrs) == 0 { - log.Errorf("Active gateway is not found.: %v ", ctx.Err()) + log.Errorf("Active gateway is not found: %v", ctx.Err()) + return errors.New("no active gateways available") } conn, err := grpc.NewClient(gatewayAddrs[0], grpc.WithTransportCredentials(insecure.NewCredentials())) From 156c66526948b3bd4684d45f28c5ed62ae5781a1 Mon Sep 17 00:00:00 2001 From: s-shiraki Date: Wed, 18 Dec 2024 00:09:44 +0900 Subject: [PATCH 08/18] Update pkg/index/job/exportation/service/exporter.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: s-shiraki --- pkg/index/job/exportation/service/exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/index/job/exportation/service/exporter.go b/pkg/index/job/exportation/service/exporter.go index 80d830c3a9..754796e4be 100644 --- a/pkg/index/job/exportation/service/exporter.go +++ b/pkg/index/job/exportation/service/exporter.go @@ -76,7 +76,7 @@ func New(opts ...Option) (Exporter, error) { if err := file.MkdirAll(e.indexPath, os.ModePerm); err != nil { log.Errorf("failed to create dir %s", e.indexPath) - return nil, err + return nil, errors.Wrap(err, "failed to create index path directory") } path := file.Join(e.indexPath, fmt.Sprintf("%s.db", strconv.FormatInt(time.Now().Unix(), 10))) From 18710d66ee8bcd4d187596554b886a24aa80226f Mon Sep 17 00:00:00 2001 From: s-shiraki Date: Wed, 18 Dec 2024 00:13:26 +0900 Subject: [PATCH 09/18] Update k8s/index/job/exportation/cronjob.yaml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: s-shiraki --- k8s/index/job/exportation/cronjob.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/index/job/exportation/cronjob.yaml b/k8s/index/job/exportation/cronjob.yaml index 4c3ceda683..b0c3f79fd4 100644 --- a/k8s/index/job/exportation/cronjob.yaml +++ b/k8s/index/job/exportation/cronjob.yaml @@ -26,7 +26,7 @@ metadata: app.kubernetes.io/component: vald-index-exportation app.kubernetes.io/version: v1.7.14 spec: - schedule: "* * * * *" + schedule: "0 * * * *" concurrencyPolicy: Forbid suspend: true startingDeadlineSeconds: 43200 From ad9736b302fab49c2378b5b2adfbc6cd0226ff2e Mon Sep 17 00:00:00 2001 From: s-shiraki Date: Wed, 18 Dec 2024 00:21:42 +0900 Subject: [PATCH 10/18] Update Makefile.d/build.mk Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: s-shiraki --- Makefile.d/build.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk index a5d62b78b5..15d9c6e9bd 100644 --- a/Makefile.d/build.mk +++ b/Makefile.d/build.mk @@ -203,7 +203,7 @@ artifacts/vald-index-deletion-$(GOOS)-$(GOARCH).zip: cmd/index/job/deletion/inde $(call mkdir, $(dir $@)) zip --junk-paths $@ $< -artifacts/vald-index-exporation-$(GOOS)-$(GOARCH).zip: cmd/index/job/exportation/index-exportation +artifacts/vald-index-exportation-$(GOOS)-$(GOARCH).zip: cmd/index/job/exportation/index-exportation $(call mkdir, $(dir $@)) zip --junk-paths $@ $< From d54817b9feddf7a53a8103f33533cff913fb7870 Mon Sep 17 00:00:00 2001 From: s-shiraki Date: Wed, 18 Dec 2024 00:22:43 +0900 Subject: [PATCH 11/18] Update k8s/index/job/exportation/configmap.yaml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: s-shiraki --- k8s/index/job/exportation/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/index/job/exportation/configmap.yaml b/k8s/index/job/exportation/configmap.yaml index 4f781a838d..4305880e7c 100644 --- a/k8s/index/job/exportation/configmap.yaml +++ b/k8s/index/job/exportation/configmap.yaml @@ -192,7 +192,7 @@ data: namespace: "_MY_POD_NAMESPACE_" pod_name: "_MY_POD_NAME_" node_name: "_MY_NODE_NAME_" - service_name: "vald-index-deletion" + service_name: "vald-index-exportation" metrics: enable_cgo: true enable_goroutine: true From f1087b0d096dd744af57c47f7925b50351e6d532 Mon Sep 17 00:00:00 2001 From: s-shiraki Date: Wed, 18 Dec 2024 00:31:34 +0900 Subject: [PATCH 12/18] Update Makefile.d/docker.mk Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Signed-off-by: s-shiraki --- Makefile.d/docker.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index a76d931932..04ed277034 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -357,7 +357,7 @@ docker/build/index-deletion: docker/build/image .PHONY: docker/name/index-exportation -docker/name/index-exporation: +docker/name/index-exportation: @echo "$(ORG)/$(INDEX_EXPORTATION_IMAGE)" .PHONY: docker/build/index-exportation From 9ce46735674fc52666d732bdb17ba688adbfec4d Mon Sep 17 00:00:00 2001 From: HighPon Date: Tue, 17 Dec 2024 15:33:59 +0000 Subject: [PATCH 13/18] fix: invalid yaml --- k8s/index/job/exportation/cronjob.yaml | 1 - pkg/index/job/exportation/usecase/exportation.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/k8s/index/job/exportation/cronjob.yaml b/k8s/index/job/exportation/cronjob.yaml index b0c3f79fd4..257a5fbfe7 100644 --- a/k8s/index/job/exportation/cronjob.yaml +++ b/k8s/index/job/exportation/cronjob.yaml @@ -51,7 +51,6 @@ spec: pyroscope.io/port: "6060" spec: initContainers: - initContainers: - name: wait-for-gateway-lb image: busybox:stable imagePullPolicy: Always diff --git a/pkg/index/job/exportation/usecase/exportation.go b/pkg/index/job/exportation/usecase/exportation.go index f21bf6e6ac..90e39c1693 100644 --- a/pkg/index/job/exportation/usecase/exportation.go +++ b/pkg/index/job/exportation/usecase/exportation.go @@ -120,7 +120,7 @@ func (r *run) Start(ctx context.Context) (<-chan error, error) { if r.observability != nil { oech = r.observability.Start(ctx) } - // sech = r.server.ListenAndServe(ctx) + sech = r.server.ListenAndServe(ctx) cech, err := r.exporter.StartClient(ctx) if err != nil { close(ech) From b6bcff3423a528212a8908947ddff7d02e20c79e Mon Sep 17 00:00:00 2001 From: HighPon Date: Tue, 17 Dec 2024 16:30:58 +0000 Subject: [PATCH 14/18] fix: Add shutdown for server --- pkg/index/job/exportation/service/exporter.go | 5 +++++ pkg/index/job/exportation/usecase/exportation.go | 11 ++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/index/job/exportation/service/exporter.go b/pkg/index/job/exportation/service/exporter.go index 754796e4be..749bd35367 100644 --- a/pkg/index/job/exportation/service/exporter.go +++ b/pkg/index/job/exportation/service/exporter.go @@ -46,6 +46,7 @@ const ( type Exporter interface { StartClient(ctx context.Context) (<-chan error, error) Start(ctx context.Context) error + PreStop(ctx context.Context) error } type export struct { @@ -215,3 +216,7 @@ func (e *export) doExportIndex( } } } + +func (e *export) PreStop(ctx context.Context) error { + return e.storedVector.Close(false) +} diff --git a/pkg/index/job/exportation/usecase/exportation.go b/pkg/index/job/exportation/usecase/exportation.go index 90e39c1693..488244ca5d 100644 --- a/pkg/index/job/exportation/usecase/exportation.go +++ b/pkg/index/job/exportation/usecase/exportation.go @@ -116,7 +116,7 @@ func (r *run) PreStart(ctx context.Context) error { // during the operation and an error representing any initialization errors. func (r *run) Start(ctx context.Context) (<-chan error, error) { ech := make(chan error, 3) - var sech, oech, cech <-chan error + var sech, oech <-chan error if r.observability != nil { oech = r.observability.Start(ctx) } @@ -167,8 +167,8 @@ func (r *run) Start(ctx context.Context) (<-chan error, error) { } // PreStop is a method called before execution of Stop. -func (*run) PreStop(_ context.Context) error { - return nil +func (r *run) PreStop(ctx context.Context) error { + return r.exporter.PreStop(ctx) } // Stop is a method used to stop an operation in the run. @@ -178,6 +178,11 @@ func (r *run) Stop(ctx context.Context) (errs error) { errs = errors.Join(errs, err) } } + if r.server != nil { + if err := r.server.Shutdown(ctx); err != nil { + errs = errors.Join(errs, err) + } + } return errs } From 192aaf92277a2b2e9a68bef9f4c5e803633321cc Mon Sep 17 00:00:00 2001 From: HighPon Date: Thu, 19 Dec 2024 04:28:32 +0000 Subject: [PATCH 15/18] fix: delete unused file --- hack/actions/gen/main.go | 751 --------------------------------------- 1 file changed, 751 deletions(-) delete mode 100644 hack/actions/gen/main.go diff --git a/hack/actions/gen/main.go b/hack/actions/gen/main.go deleted file mode 100644 index b528c736aa..0000000000 --- a/hack/actions/gen/main.go +++ /dev/null @@ -1,751 +0,0 @@ -// -// Copyright (C) 2019-2024 vdaas.org vald team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package main - -import ( - "bytes" - "context" - "fmt" - "io/fs" - "os" - "os/signal" - "syscall" - "text/template" - "time" - - "github.com/vdaas/vald/internal/errors" - "github.com/vdaas/vald/internal/file" - "github.com/vdaas/vald/internal/log" - "github.com/vdaas/vald/internal/safety" - "github.com/vdaas/vald/internal/strings" - "github.com/vdaas/vald/internal/sync/errgroup" - "gopkg.in/yaml.v2" -) - -var license string = `# -# Copyright (C) 2019-{{.Year}} {{.Maintainer}} -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go -` - -var licenseTmpl *template.Template = template.Must(template.New("license").Parse(license)) - -type ( - Workflow struct { - Name string `yaml:"name"` - On On `yaml:"on"` - Jobs Jobs `yaml:"jobs"` - } - - On struct { - Schedule Schedule `yaml:"schedule,omitempty"` - Push Push `yaml:"push"` - PullRequest PullRequest `yaml:"pull_request"` - PullRequestTarget PullRequest `yaml:"pull_request_target"` - } - - Schedule []struct { - Cron string `yaml:"cron,omitempty"` - } - - Push struct { - Branches []string `yaml:"branches"` - Tags []string `yaml:"tags"` - } - - PullRequest struct { - Paths Paths `yaml:"paths"` - } - - Jobs struct { - Build Build `yaml:"build"` - } - - Build struct { - Uses string `yaml:"uses"` - With With `yaml:"with"` - Secrets string `yaml:"secrets"` - } - - With struct { - Target string `yaml:"target"` - Platforms string `yaml:"platforms,omitempty"` - } - - Paths []string -) - -type Data struct { - AliasImage bool - ConfigExists bool - Year int - ContainerType ContainerType - AppName string - BinDir string - BuildUser string - BuilderImage string - BuilderTag string - BuildStageName string - Maintainer string - PackageDir string - RootDir string - RuntimeImage string - RuntimeTag string - RuntimeUser string - Name string - BuildPlatforms string - Arguments map[string]string - Environments map[string]string - Entrypoints []string - EnvironmentsSlice []string - ExtraCopies []string - ExtraImages []string - ExtraPackages []string - Preprocess []string - RunCommands []string - RunMounts []string - StageFiles []string - PullRequestPaths []string -} - -type ContainerType int - -const ( - Go ContainerType = iota - Rust - DevContainer - HelmOperator - CIContainer - Other -) - -const ( - organization = "vdaas" - repository = "vald" - defaultMaintainer = organization + ".org " + repository + " team <" + repository + "@" + organization + ".org>" - defaultBuildUser = "root:root" - maintainerKey = "MAINTAINER" - minimumArgumentLength = 2 - ubuntuVersion = "22.04" - - ngtPreprocess = "make ngt/install" - faissPreprocess = "make faiss/install" - - helmOperatorRootdir = "/opt/helm" - helmOperatorWatchFile = helmOperatorRootdir + "/watches.yaml" - helmOperatorChartsDir = helmOperatorRootdir + "/charts" -) - -var ( - clangBuildDeps = []string{ - "cmake", - "gcc", - "g++", - "unzip", - "libssl-dev", - } - ngtBuildDeps = []string{ - "liblapack-dev", - "libomp-dev", - "libopenblas-dev", - } - faissBuildDeps = []string{ - "gfortran", - } - devContainerDeps = []string{ - "gawk", - "gnupg2", - "graphviz", - "jq", - "libhdf5-dev", - "libaec-dev", - "sed", - "zip", - } - - ciContainerPreprocess = []string{ - "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} deps GO_CLEAN_DEPS=false", - "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} golangci-lint/install", - "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} gotestfmt/install", - "make cmake/install", - "make buf/install", - "make hdf5/install", - "make helm-docs/install", - "make helm/install", - "make k3d/install", - "make k9s/install", - "make kind/install", - "make kubectl/install", - "make kubelinter/install", - "make reviewdog/install", - "make tparse/install", - "make valdcli/install", - "make yq/install", - "make minikube/install", - "make stern/install", - "make telepresence/install", - } - - devContainerPreprocess = []string{ - "curl -fsSL https://deb.nodesource.com/setup_current.x | bash -", - "apt-get clean", - "apt-get update -y", - "apt-get upgrade -y", - "apt-get install -y --no-install-recommends --fix-missing nodejs", - "npm install -g npm@latest", - "apt-get clean", - "apt-get autoclean -y", - "apt-get autoremove -y", - "make delve/install", - "make gomodifytags/install", - "make gopls/install", - "make gotests/install", - "make impl/install", - "make staticcheck/install", - } -) - -const baseWorkflowTmpl string = `name: "Build docker image: %s" -on: - schedule: - - cron: "0 * * * *" - push: - branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - "hack/docker/gen/main.go" - - "dockers/%s/Dockerfile" - - "hack/actions/gen/main.go" - - ".github/workflows/dockers-%s-image.yaml" - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - "cmd/%s/**" - - "pkg/%s/**" - pull_request_target: - paths: [] - -jobs: - build: - uses: "./.github/workflows/_docker-image.yaml" - with: - target: "%s" - platforms: "" - secrets: "inherit" -` - -const ( - cmdBenchOperatorsPath = "cmd/tools/benchmark/operators/**" - pkgBenchOperatorsPath = "pkg/tools/benchmark/operators/**" - cmdBenchJobsPath = "cmd/tools/benchmark/jobs/**" - pkgBenchJobsPath = "pkg/tools/benchmark/jobs/**" - - agentInternalPath = "pkg/agent/internal/**" - gatewayInternalPath = "pkg/gateway/internal/**" - - apisGrpcPath = "apis/grpc/**" - apisProtoPath = "apis/proto/**" - - hackPath = "hack/**" - - chartsValdPath = "charts/vald" - helmOperatorPath = "charts/vald-helm-operator" - chartPath = chartsValdPath + "/Chart.yaml" - valuesPath = chartsValdPath + "/values.yaml" - templatesPath = chartsValdPath + "/templates/**" - helmOperatorChartPath = helmOperatorPath + "/Chart.yaml" - helmOperatorValuesPath = helmOperatorPath + "/values.yaml" - helmOperatorTemplatesPath = helmOperatorPath + "/templates/**" - - goModPath = "go.mod" - goSumPath = "go.sum" - - cargoLockPath = "rust/Cargo.lock" - cargoTomlPath = "rust/Cargo.toml" - rustBinAgentDirPath = "rust/bin/agent" - rustNgtRsPath = "rust/libs/ngt-rs/**" - rustNgtPath = "rust/libs/ngt/**" - rustProtoPath = "rust/libs/proto/**" - - internalPath = "internal/**" - internalStoragePath = "internal/db/storage/blob/**" - excludeTestFilesPath = "!internal/**/*_test.go" - excludeMockFilesPath = "!internal/**/*_mock.go" - excludeDbPath = "!internal/db/**" - excludeK8sPath = "!internal/k8s/**" - - versionsPath = "versions" - operatorSDKVersionPath = versionsPath + "/OPERATOR_SDK_VERSION" - goVersionPath = versionsPath + "/GO_VERSION" - rustVersionPath = versionsPath + "/RUST_VERSION" - faissVersionPath = versionsPath + "/FAISS_VERSION" - ngtVersionPath = versionsPath + "/NGT_VERSION" - usearchVersionPath = versionsPath + "/USEARCH_VERSION" - - makefilePath = "Makefile" - makefileDirPath = "Makefile.d/**" -) - -const ( - agentNgt = "agent-ngt" - agentFaiss = "agent-faiss" - agentSidecar = "agent-sidecar" - agent = "agent" - - discovererK8s = "discoverer-k8s" - - gateway = "gateway" - gatewayLb = "gateway-lb" - gatewayFilter = "gateway-filter" - gatewayMirror = "gateway-mirror" - - managerIndex = "manager-index" - - indexCorrection = "index-correction" - indexCreation = "index-creation" - indexDeletion = "index-deletion" - indexExportation = "index-exportation" - indexImportation = "index-importation" - indexSave = "index-save" - indexOperator = "index-operator" - - readreplicaRotate = "readreplica-rotate" - - benchJob = "benchmark-job" - benchOperator = "benchmark-operator" - - helmOperator = "helm-operator" - - loadtest = "loadtest" - - ciContainer = "ci-container" - devContainer = "dev-container" - - buildbase = "buildbase" - buildkit = "buildkit" - binfmt = "binfmt" - buildkitSyftScanner = "buildkit-syft-scanner" -) - -const ( - multiPlatforms = amd64Platform + "," + arm64Platform - amd64Platform = "linux/amd64" - arm64Platform = "linux/arm64" -) - -func (data *Data) initPullRequestPaths() { - switch data.Name { - // the benchmark components trigger each other, not just themselves - case benchJob: - data.PullRequestPaths = append(data.PullRequestPaths, - cmdBenchOperatorsPath, - pkgBenchOperatorsPath, - ) - case benchOperator: - data.PullRequestPaths = append(data.PullRequestPaths, - cmdBenchJobsPath, - pkgBenchJobsPath, - ) - case agentFaiss, agentNgt: - data.PullRequestPaths = append(data.PullRequestPaths, agentInternalPath) - default: - if strings.Contains(strings.ToLower(data.Name), gateway) { - data.PullRequestPaths = append(data.PullRequestPaths, gatewayInternalPath) - } - } - - switch data.ContainerType { - case HelmOperator: - data.PullRequestPaths = append(data.PullRequestPaths, - chartPath, - valuesPath, - templatesPath, - helmOperatorChartPath, - helmOperatorValuesPath, - helmOperatorTemplatesPath, - operatorSDKVersionPath, - ) - case DevContainer, CIContainer: - data.PullRequestPaths = append(data.PullRequestPaths, - apisGrpcPath, - apisProtoPath, - hackPath, - ) - case Go: - data.PullRequestPaths = append(data.PullRequestPaths, - apisGrpcPath, - apisProtoPath, - goModPath, - goSumPath, - goVersionPath, - internalPath, - excludeTestFilesPath, - excludeMockFilesPath, - excludeDbPath, - ) - switch data.Name { - case discovererK8s, indexOperator, gatewayMirror, readreplicaRotate, agentNgt, benchJob, benchOperator: - default: - data.PullRequestPaths = append(data.PullRequestPaths, excludeK8sPath) - } - case Rust: - data.PullRequestPaths = append(data.PullRequestPaths, - apisGrpcPath, - apisProtoPath, - cargoLockPath, - cargoTomlPath, - rustBinAgentDirPath, - rustNgtRsPath, - rustNgtPath, - rustProtoPath, - rustVersionPath, - faissVersionPath, - ngtVersionPath, - ) - } - if strings.EqualFold(data.Name, agentFaiss) || data.ContainerType == Rust { - data.PullRequestPaths = append(data.PullRequestPaths, faissVersionPath) - } - if strings.EqualFold(data.Name, agentNgt) || data.ContainerType == Rust { - data.PullRequestPaths = append(data.PullRequestPaths, ngtVersionPath) - } - - if data.Name == agentSidecar { - data.PullRequestPaths = append(data.PullRequestPaths, internalStoragePath) - } - if !data.AliasImage { - data.PullRequestPaths = append(data.PullRequestPaths, makefilePath, makefileDirPath) - } -} - -func (data *Data) initData() { - data.initPullRequestPaths() - - if data.AliasImage { - data.BuildPlatforms = multiPlatforms - } - if data.ContainerType == CIContainer || data.Name == loadtest { - data.BuildPlatforms = amd64Platform - } - - data.Year = time.Now().Year() - if maintainer := os.Getenv(maintainerKey); maintainer != "" { - data.Maintainer = maintainer - } else { - data.Maintainer = defaultMaintainer - } -} - -func (data *Data) generateWorkflowStruct() (*Workflow, error) { - workflow := &Workflow{} - baseWorkflow := fmt.Sprintf(baseWorkflowTmpl, - data.Name, - data.PackageDir, - data.Name, - data.PackageDir, - data.PackageDir, - data.Name, - ) - err := yaml.NewDecoder(strings.NewReader(baseWorkflow)).Decode(workflow) - if err != nil { - return nil, fmt.Errorf("Error decoding YAML: %v", err) - } - - if !data.AliasImage { - workflow.On.Schedule = nil - } - workflow.On.PullRequest.Paths = append(workflow.On.PullRequest.Paths, data.PullRequestPaths...) - workflow.On.PullRequestTarget.Paths = workflow.On.PullRequest.Paths - workflow.Jobs.Build.With.Platforms = data.BuildPlatforms - - return workflow, nil -} - -func main() { - log.Init() - if len(os.Args) < minimumArgumentLength { - // skipcq: RVV-A0003 - log.Fatal(errors.New("invalid argument")) - } - - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, - syscall.SIGQUIT, - syscall.SIGHUP, - syscall.SIGALRM, - syscall.SIGKILL, - syscall.SIGTERM) - defer cancel() - - eg, egctx := errgroup.New(ctx) - for n, d := range map[string]Data{ - "vald-agent-ngt": { - AppName: "ngt", - PackageDir: "agent/core/ngt", - ExtraPackages: append(clangBuildDeps, ngtBuildDeps...), - Preprocess: []string{ngtPreprocess}, - }, - "vald-agent-faiss": { - AppName: "faiss", - PackageDir: "agent/core/faiss", - ExtraPackages: append(clangBuildDeps, - append(ngtBuildDeps, - faissBuildDeps...)...), - Preprocess: []string{faissPreprocess}, - }, - "vald-agent": { - AppName: "agent", - PackageDir: "agent/core/agent", - ContainerType: Rust, - RuntimeImage: "gcr.io/distroless/cc-debian12", - ExtraPackages: append(clangBuildDeps, - append(ngtBuildDeps, - faissBuildDeps...)...), - Preprocess: []string{ - ngtPreprocess, - faissPreprocess, - }, - }, - "vald-agent-sidecar": { - AppName: "sidecar", - PackageDir: "agent/sidecar", - }, - "vald-discoverer-k8s": { - AppName: "discoverer", - PackageDir: "discoverer/k8s", - }, - "vald-gateway-lb": { - AppName: "lb", - PackageDir: "gateway/lb", - }, - "vald-gateway-filter": { - AppName: "filter", - PackageDir: "gateway/filter", - }, - "vald-gateway-mirror": { - AppName: "mirror", - PackageDir: "gateway/mirror", - }, - "vald-manager-index": { - AppName: "index", - PackageDir: "manager/index", - }, - "vald-index-correction": { - AppName: "index-correction", - PackageDir: "index/job/correction", - }, - "vald-index-creation": { - AppName: "index-creation", - PackageDir: "index/job/creation", - }, - "vald-index-deletion": { - AppName: "index-deletion", - PackageDir: "index/job/deletion", - }, - "vald-index-exportation": { - AppName: "index-exportation", - PackageDir: "index/job/exportation", - }, - "vald-index-importation": { - AppName: "index-importation", - PackageDir: "index/job/importation", - }, - "vald-index-save": { - AppName: "index-save", - PackageDir: "index/job/save", - }, - "vald-readreplica-rotate": { - AppName: "readreplica-rotate", - PackageDir: "index/job/readreplica/rotate", - }, - "vald-index-operator": { - AppName: "index-operator", - PackageDir: "index/operator", - }, - "vald-benchmark-job": { - AppName: "job", - PackageDir: "tools/benchmark/job", - ExtraPackages: append(clangBuildDeps, "libhdf5-dev", "libaec-dev"), - Preprocess: []string{ - "make hdf5/install", - }, - }, - "vald-benchmark-operator": { - AppName: "operator", - PackageDir: "tools/benchmark/operator", - }, - "vald-helm-operator": { - AppName: "helm-operator", - PackageDir: "operator/helm", - ContainerType: HelmOperator, - Arguments: map[string]string{ - "OPERATOR_SDK_VERSION": "latest", - }, - ExtraCopies: []string{ - "--from=operator /usr/local/bin/${APP_NAME} {{$.BinDir}}/${APP_NAME}", - }, - ExtraImages: []string{ - "quay.io/operator-framework/helm-operator:${OPERATOR_SDK_VERSION} AS operator", - }, - ExtraPackages: []string{"upx"}, - Preprocess: []string{ - "mkdir -p " + helmOperatorChartsDir, - `{ \ - echo "---"; \ - echo "- version: v1"; \ - echo " group: vald.vdaas.org"; \ - echo " kind: ValdRelease"; \ - echo " chart: ` + helmOperatorChartsDir + `/vald"; \ - echo "- version: v1"; \ - echo " group: vald.vdaas.org"; \ - echo " kind: ValdHelmOperatorRelease"; \ - echo " chart: ` + helmOperatorChartsDir + `/vald-helm-operator"; \ - } > ` + helmOperatorWatchFile, - "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} helm/schema/vald", - "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} helm/schema/vald-helm-operator", - "cp -r charts/* " + helmOperatorChartsDir + "/", - "upx \"{{$.BinDir}}/${APP_NAME}\"", - }, - StageFiles: []string{ - helmOperatorWatchFile, - helmOperatorChartsDir + "/vald", - helmOperatorChartsDir + "/vald-helm-operator", - }, - Entrypoints: []string{"{{$.BinDir}}/{{.AppName}}", "run", "--watches-file=" + helmOperatorWatchFile}, - }, - "vald-loadtest": { // note: this name is a little different from that of docker/gen/main.go - AppName: "loadtest", - PackageDir: "tools/cli/loadtest", - ExtraPackages: append(clangBuildDeps, "libhdf5-dev", "libaec-dev"), - Preprocess: []string{ - "make hdf5/install", - }, - }, - "vald-ci-container": { - AppName: "ci-container", - ContainerType: CIContainer, - PackageDir: "ci/base", - RuntimeUser: defaultBuildUser, - ExtraPackages: append([]string{"npm"}, append(clangBuildDeps, - append(ngtBuildDeps, - append(faissBuildDeps, - devContainerDeps...)...)...)...), - Preprocess: append(ciContainerPreprocess, ngtPreprocess, faissPreprocess), - Entrypoints: []string{"/bin/bash"}, - }, - "vald-dev-container": { - AppName: "dev-container", - BuilderImage: "mcr.microsoft.com/devcontainers/base", - BuilderTag: "ubuntu" + ubuntuVersion, - BuildUser: defaultBuildUser, - RuntimeUser: defaultBuildUser, - ContainerType: DevContainer, - PackageDir: "dev", - ExtraPackages: append(clangBuildDeps, - append(ngtBuildDeps, - append(faissBuildDeps, - devContainerDeps...)...)...), - Preprocess: append(devContainerPreprocess, - append(ciContainerPreprocess, - ngtPreprocess, - faissPreprocess)...), - }, - "vald-buildbase": { - AppName: "buildbase", - AliasImage: true, - PackageDir: "buildbase", - BuilderImage: "ubuntu", - BuilderTag: "devel", - }, - "vald-buildkit": { - AppName: "buildkit", - AliasImage: true, - PackageDir: "buildkit", - BuilderImage: "moby/buildkit", - BuilderTag: "master", - }, - "vald-binfmt": { - AppName: "binfmt", - AliasImage: true, - PackageDir: "binfmt", - BuilderImage: "tonistiigi/binfmt", - BuilderTag: "master", - }, - "vald-buildkit-syft-scanner": { - AppName: "scanner", - AliasImage: true, - PackageDir: "buildkit/syft/scanner", - BuilderImage: "docker/buildkit-syft-scanner", - BuilderTag: "edge", - BuildStageName: "scanner", - }, - } { - name := n - data := d - - eg.Go(safety.RecoverFunc(func() error { - data.Name = strings.TrimPrefix(name, "vald-") - data.initData() - - log.Infof("Generating %s's workflow", data.Name) - workflow, err := data.generateWorkflowStruct() - if err != nil { - return fmt.Errorf("Error generating workflowStruct: %w", err) - } - workflowYamlTmp, err := yaml.Marshal(workflow) - if err != nil { - return fmt.Errorf("error marshaling workflowStruct to YAML: %w", err) - } - - // remove the double quotation marks from the generated key "on": (note that the word "on" is a reserved word in gopkg.in/yaml.v2) - workflowYaml := strings.Replace(string(workflowYamlTmp), "\"on\":", "on:", 1) - - buf := bytes.NewBuffer(make([]byte, 0, len(license)+len(workflowYaml))) - err = licenseTmpl.Execute(buf, data) - if err != nil { - return fmt.Errorf("error executing template: %w", err) - } - buf.WriteString("\r\n") - buf.WriteString(workflowYaml) - _, err = file.OverWriteFile(egctx, file.Join(os.Args[1], ".github/workflows", "dockers-"+data.Name+"-image.yaml"), buf, fs.ModePerm) - if err != nil { - return fmt.Errorf("error writing file: %w", err) - } - return nil - })) - } - eg.Wait() -} From 4863f2eca086944e54363a97cc0a95b9b32c1665 Mon Sep 17 00:00:00 2001 From: HighPon Date: Mon, 6 Jan 2025 15:22:33 +0000 Subject: [PATCH 16/18] fix: update version in metadata --- k8s/index/job/exportation/configmap.yaml | 4 ++-- k8s/index/job/exportation/cronjob.yaml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/index/job/exportation/configmap.yaml b/k8s/index/job/exportation/configmap.yaml index 4305880e7c..6e9d81a0f1 100644 --- a/k8s/index/job/exportation/configmap.yaml +++ b/k8s/index/job/exportation/configmap.yaml @@ -4,10 +4,10 @@ metadata: name: vald-index-exportation-config labels: app.kubernetes.io/name: vald - helm.sh/chart: vald-v1.7.14 + helm.sh/chart: vald-v1.7.16 app.kubernetes.io/managed-by: Helm app.kubernetes.io/instance: release-name - app.kubernetes.io/version: v1.7.14 + app.kubernetes.io/version: v1.7.16 app.kubernetes.io/component: vald-index-exportation data: config.yaml: | diff --git a/k8s/index/job/exportation/cronjob.yaml b/k8s/index/job/exportation/cronjob.yaml index 257a5fbfe7..1f71df8183 100644 --- a/k8s/index/job/exportation/cronjob.yaml +++ b/k8s/index/job/exportation/cronjob.yaml @@ -20,11 +20,11 @@ metadata: labels: app: vald-index-exportation app.kubernetes.io/name: vald - helm.sh/chart: vald-v1.7.14 + helm.sh/chart: vald-v1.7.16 app.kubernetes.io/managed-by: Helm app.kubernetes.io/instance: release-name app.kubernetes.io/component: vald-index-exportation - app.kubernetes.io/version: v1.7.14 + app.kubernetes.io/version: v1.7.16 spec: schedule: "0 * * * *" concurrencyPolicy: Forbid @@ -38,11 +38,11 @@ spec: labels: app: vald-index-exportation app.kubernetes.io/name: vald - helm.sh/chart: vald-v1.7.14 + helm.sh/chart: vald-v1.7.16 app.kubernetes.io/managed-by: Helm app.kubernetes.io/instance: release-name app.kubernetes.io/component: vald-index-exportation - app.kubernetes.io/version: v1.7.14 + app.kubernetes.io/version: v1.7.16 annotations: pyroscope.io/scrape: "true" pyroscope.io/application-name: vald-index-exportation From ca18a84ebc3987fa424c3b2305486e5ca1b3e475 Mon Sep 17 00:00:00 2001 From: HighPon Date: Mon, 6 Jan 2025 15:26:48 +0000 Subject: [PATCH 17/18] fix: image tag for index exportation --- k8s/index/job/exportation/cronjob.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/index/job/exportation/cronjob.yaml b/k8s/index/job/exportation/cronjob.yaml index 1f71df8183..72415bcfdf 100644 --- a/k8s/index/job/exportation/cronjob.yaml +++ b/k8s/index/job/exportation/cronjob.yaml @@ -65,7 +65,7 @@ spec: done containers: - name: vald-index-exportation - image: "vdaas/vald-index-exportation:latest" + image: "vdaas/vald-index-exportation:nightly" imagePullPolicy: Never volumeMounts: - name: vald-index-exportation-config From f14a6619a783433aa9badc69ff1588f1b2a6ab61 Mon Sep 17 00:00:00 2001 From: HighPon Date: Tue, 7 Jan 2025 13:49:07 +0000 Subject: [PATCH 18/18] fix: update license --- .../dockers-index-exportation-image.yaml | 2 +- cmd/index/job/exportation/main.go | 2 +- cmd/index/job/exportation/sample.yaml | 2 +- dockers/index/job/exportation/Dockerfile | 4 ++-- internal/config/index_exporter.go | 2 +- k8s/index/job/exportation/cronjob.yaml | 2 +- k8s/index/job/exportation/pvc.yaml | 15 +++++++++++++++ pkg/index/job/exportation/config/config.go | 2 +- pkg/index/job/exportation/service/exporter.go | 2 +- pkg/index/job/exportation/service/options.go | 2 +- pkg/index/job/exportation/usecase/exportation.go | 2 +- 11 files changed, 26 insertions(+), 11 deletions(-) diff --git a/.github/workflows/dockers-index-exportation-image.yaml b/.github/workflows/dockers-index-exportation-image.yaml index e1562edd70..5e85d2bae8 100644 --- a/.github/workflows/dockers-index-exportation-image.yaml +++ b/.github/workflows/dockers-index-exportation-image.yaml @@ -1,5 +1,5 @@ # -# Copyright (C) 2019-2024 vdaas.org vald team +# Copyright (C) 2019-2025 vdaas.org vald team # # Licensed under the Apache License, Version 2.0 (the "License"); # You may not use this file except in compliance with the License. diff --git a/cmd/index/job/exportation/main.go b/cmd/index/job/exportation/main.go index 8c16dd224b..c7493321ca 100644 --- a/cmd/index/job/exportation/main.go +++ b/cmd/index/job/exportation/main.go @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2024 vdaas.org vald team +// Copyright (C) 2019-2025 vdaas.org vald team // // Licensed under the Apache License, Version 2.0 (the "License"); // You may not use this file except in compliance with the License. diff --git a/cmd/index/job/exportation/sample.yaml b/cmd/index/job/exportation/sample.yaml index 4ed586d02f..c397e4b710 100644 --- a/cmd/index/job/exportation/sample.yaml +++ b/cmd/index/job/exportation/sample.yaml @@ -1,5 +1,5 @@ # -# Copyright (C) 2019-2024 vdaas.org vald team +# Copyright (C) 2019-2025 vdaas.org vald team # # Licensed under the Apache License, Version 2.0 (the "License"); # You may not use this file except in compliance with the License. diff --git a/dockers/index/job/exportation/Dockerfile b/dockers/index/job/exportation/Dockerfile index d329680172..2ff5f582a6 100644 --- a/dockers/index/job/exportation/Dockerfile +++ b/dockers/index/job/exportation/Dockerfile @@ -1,7 +1,7 @@ # syntax = docker/dockerfile:latest # check=error=true # -# Copyright (C) 2019-2024 vdaas.org vald team +# Copyright (C) 2019-2025 vdaas.org vald team # # Licensed under the Apache License, Version 2.0 (the "License"); # You may not use this file except in compliance with the License. @@ -86,4 +86,4 @@ COPY --from=builder /usr/bin/index-exportation /usr/bin/index-exportation COPY cmd/index/job/exportation/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index-exportation"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/index-exportation"] diff --git a/internal/config/index_exporter.go b/internal/config/index_exporter.go index f2f9e3303f..5065392df1 100644 --- a/internal/config/index_exporter.go +++ b/internal/config/index_exporter.go @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2024 vdaas.org vald team +// Copyright (C) 2019-2025 vdaas.org vald team // // Licensed under the Apache License, Version 2.0 (the "License"); // You may not use this file except in compliance with the License. diff --git a/k8s/index/job/exportation/cronjob.yaml b/k8s/index/job/exportation/cronjob.yaml index 72415bcfdf..ec93b16074 100644 --- a/k8s/index/job/exportation/cronjob.yaml +++ b/k8s/index/job/exportation/cronjob.yaml @@ -1,5 +1,5 @@ # -# Copyright (C) 2019-2024 vdaas.org vald team +# Copyright (C) 2019-2025 vdaas.org vald team # # Licensed under the Apache License, Version 2.0 (the "License"); # You may not use this file except in compliance with the License. diff --git a/k8s/index/job/exportation/pvc.yaml b/k8s/index/job/exportation/pvc.yaml index 474aebc6de..36d0c0918e 100644 --- a/k8s/index/job/exportation/pvc.yaml +++ b/k8s/index/job/exportation/pvc.yaml @@ -1,3 +1,18 @@ +# +# Copyright (C) 2019-2025 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/pkg/index/job/exportation/config/config.go b/pkg/index/job/exportation/config/config.go index 00c864822e..062b3592ee 100644 --- a/pkg/index/job/exportation/config/config.go +++ b/pkg/index/job/exportation/config/config.go @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2024 vdaas.org vald team +// Copyright (C) 2019-2025 vdaas.org vald team // // Licensed under the Apache License, Version 2.0 (the "License"); // You may not use this file except in compliance with the License. diff --git a/pkg/index/job/exportation/service/exporter.go b/pkg/index/job/exportation/service/exporter.go index 749bd35367..b442c60723 100644 --- a/pkg/index/job/exportation/service/exporter.go +++ b/pkg/index/job/exportation/service/exporter.go @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2024 vdaas.org vald team +// Copyright (C) 2019-2025 vdaas.org vald team // // Licensed under the Apache License, Version 2.0 (the "License"); // You may not use this file except in compliance with the License. diff --git a/pkg/index/job/exportation/service/options.go b/pkg/index/job/exportation/service/options.go index 6d4a5889af..7ac8b08b02 100644 --- a/pkg/index/job/exportation/service/options.go +++ b/pkg/index/job/exportation/service/options.go @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2024 vdaas.org vald team +// Copyright (C) 2019-2025 vdaas.org vald team // // Licensed under the Apache License, Version 2.0 (the "License"); // You may not use this file except in compliance with the License. diff --git a/pkg/index/job/exportation/usecase/exportation.go b/pkg/index/job/exportation/usecase/exportation.go index 488244ca5d..ee2b764c00 100644 --- a/pkg/index/job/exportation/usecase/exportation.go +++ b/pkg/index/job/exportation/usecase/exportation.go @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2024 vdaas.org vald team +// Copyright (C) 2019-2025 vdaas.org vald team // // Licensed under the Apache License, Version 2.0 (the "License"); // You may not use this file except in compliance with the License.