Skip to content

Commit

Permalink
Migrate to buf schema v2 and move api related tasks to dedicated Make…
Browse files Browse the repository at this point in the history
…file for PMM v3 (#3178)

* Migrate to buf schema v2 and move api related tasks to dedicated Makefile

* Hit CI

---------

Co-authored-by: Jiří Čtvrtka <[email protected]>
  • Loading branch information
artemgavrilov and JiriCtvrtka authored Sep 9, 2024
1 parent 11cba1b commit 4242f0b
Show file tree
Hide file tree
Showing 9 changed files with 198 additions and 204 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ jobs:
- name: Update API compatibility descriptors
run: |
# log if descriptors changed, useful for "update descriptors" PRs
make descriptors
make -C api descriptors
git diff --text
- name: Run check-license
Expand Down
135 changes: 7 additions & 128 deletions Makefile.include
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
# vim: ts=8:sw=8:ft=make:noai:noet
SWAGGER_UI_VERSION?=latest


.PHONY: default init release gen test clean all
default: help
Expand All @@ -25,8 +24,8 @@ release: ## Build release versions of all components
make -C qan-api2 release

gen: clean ## Generate files
make gen-api
make clean_swagger
make -C api gen
make -C api clean-swagger

make -C agent gen
make -C admin gen
Expand All @@ -38,129 +37,13 @@ gen: clean ## Generate files
make format ## TODO: One formatting run is not enough, figure out why.
go install -v ./...

clean: ## Remove generated files
make -C api clean

gen-mocks:
find . -name mock_*.go -delete
./bin/mockery --config .mockery.yaml

gen-api: ## Generate PMM API
# generated by descriptors target
bin/buf breaking --against descriptor.bin api

bin/buf generate -v api

SPECS="\
api/agentlocal/v1 \
api/server/v1 \
api/user/v1 \
api/inventory/v1 \
api/management/v1 \
api/management/v1/service \
api/actions/v1 \
api/advisors/v1 \
api/alerting/v1 \
api/backup/v1 \
api/dump/v1beta1 \
api/accesscontrol/v1beta1 \
api/qan/v1 \
api/platform/v1"; \
for API in $$SPECS; do \
set -x ; \
bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \
bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \
bin/swagger validate $$API/json/$$(basename $$API).json ; \
bin/swagger generate client --with-flatten=expand --with-flatten=remove-unused --spec=$$API/json/$$(basename $$API).json --target=$$API/json \
--additional-initialism=aws \
--additional-initialism=db \
--additional-initialism=ok \
--additional-initialism=pmm \
--additional-initialism=psmdb \
--additional-initialism=pxc \
--additional-initialism=pt \
--additional-initialism=qan \
--additional-initialism=rds \
--additional-initialism=sql \
--additional-initialism=ha ; \
done

# generate public API spec, omit agentlocal and inventory (always private),
# as well as a number of protos that are in beta (not v1 yet, they all go to a similar call below)
bin/swagger mixin --output=api/swagger/swagger.json \
api/swagger/header.json \
api/server/v1/json/v1.json \
api/user/v1/json/v1.json \
api/inventory/v1/json/v1.json \
api/management/v1/json/v1.json \
api/actions/v1/json/v1.json \
api/alerting/v1/json/v1.json \
api/advisors/v1/json/v1.json \
api/backup/v1/json/v1.json \
api/qan/v1/json/v1.json \
api/platform/v1/json/v1.json
bin/swagger validate api/swagger/swagger.json

bin/swagger-order --output=api/swagger/swagger.json api/swagger/swagger.json

# generate API spec with all PMM Server APIs (omit agentlocal)
bin/swagger mixin --output=api/swagger/swagger-dev.json \
api/swagger/header-dev.json \
api/server/v1/json/v1.json \
api/user/v1/json/v1.json \
api/inventory/v1/json/v1.json \
api/management/v1/json/v1.json \
api/actions/v1/json/v1.json \
api/alerting/v1/json/v1.json \
api/advisors/v1/json/v1.json \
api/backup/v1/json/v1.json \
api/dump/v1beta1/json/v1beta1.json \
api/accesscontrol/v1beta1/json/v1beta1.json \
api/qan/v1/json/v1.json \
api/platform/v1/json/v1.json

bin/swagger validate api/swagger/swagger-dev.json

bin/swagger-order --output=api/swagger/swagger-dev.json api/swagger/swagger-dev.json

clean_swagger:
find api -name '*.swagger.json' -print -delete


ifeq ($(shell test "${SWAGGER_UI_VERSION}" = "latest" && echo 1 || echo 0), 1)
get_swagger_version:
override SWAGGER_UI_VERSION = $(shell curl --silent --head https://github.com/swagger-api/swagger-ui/releases/latest | grep -E '^[lL]ocation' | sed 's;^.*/;;')
else
get_swagger_version:
endif

update-swagger: get_swagger_version ## Update Swagger UI in api/swagger, use `SWAGGER_UI_VERSION=xxx make update-swagger` to choose a version other than latest
curl --output /tmp/swagger.tar.gz -sSfL "https://github.com/swagger-api/swagger-ui/archive/refs/tags/${SWAGGER_UI_VERSION}.tar.gz"
tar xf /tmp/swagger.tar.gz --transform 's;swagger-ui-.*/dist/;api/swagger/;g'
rm -f api/swagger/.npmrc
sed -i 's/url: ".*"/url: "\/swagger.json",\n validatorUrl: "none"/' api/swagger/swagger-initializer.js

clean: clean_swagger ## Remove generated files
find api -name '*.pb.go' -print -delete
find api -name '*.pb.gw.go' -print -delete
find api -name '*.validate.go' -print -delete

SPECS="\
api/agentlocal/v1 \
api/server/v1 \
api/user/v1 \
api/inventory/v1 \
api/management/v1 \
api/actions/v1 \
api/alerting/v1 \
api/advisors/v1 \
api/backup/v1 \
api/dump/v1beta1 \
api/accesscontrol/v1beta1 \
api/qan/v1 \
api/platform/v1"; \
for API in $$SPECS; do \
rm -fr $$API/json/client $$API/json/models $$API/json/$$(basename $$API).json ; \
done
rm -f api/swagger/swagger.json api/swagger/swagger-dev.json

test-common: ## Run tests from API (and other shared) packages only (i.e it ignores directories that are explicitly listed)
go test $(shell go list ./... | grep -v -e admin -e agent -e managed -e api-tests -e qan-api2 -e update)

Expand All @@ -182,14 +65,10 @@ check-all: check-license check ## Run golangci linter to check for changes ag
FILES = $(shell find . -type f -name '*.go')

format: ## Format source code
make -C api format
bin/gofumpt -l -w $(FILES)
bin/goimports -local github.com/percona/pmm -l -w $(FILES)
bin/gci write --section Standard --section Default --section "Prefix(github.com/percona/pmm)" $(FILES)
bin/buf format api -w

serve: ## Serve API documentation with nginx
nginx -p . -c api/nginx/nginx.conf

descriptors: ## Update API compatibility descriptors
#./prototool break descriptor-set . -o api/api.descriptor
bin/buf build -o descriptor.bin --as-file-descriptor-set api
127 changes: 127 additions & 0 deletions api/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
# vim: ts=8:sw=8:ft=make:noai:noet
SWAGGER_UI_VERSION?=latest

gen: ## Generate PMM API
# generated by descriptors target
../bin/buf breaking --against descriptor.bin .

../bin/buf generate -v .

SPECS="\
agentlocal/v1 \
server/v1 \
user/v1 \
inventory/v1 \
management/v1 \
management/v1/service \
actions/v1 \
advisors/v1 \
alerting/v1 \
backup/v1 \
dump/v1beta1 \
accesscontrol/v1beta1 \
qan/v1 \
platform/v1"; \
for API in $$SPECS; do \
set -x ; \
../bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \
../bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \
../bin/swagger validate $$API/json/$$(basename $$API).json ; \
../bin/swagger generate client --with-flatten=expand --with-flatten=remove-unused --spec=$$API/json/$$(basename $$API).json --target=$$API/json \
--additional-initialism=aws \
--additional-initialism=db \
--additional-initialism=ok \
--additional-initialism=pmm \
--additional-initialism=psmdb \
--additional-initialism=pxc \
--additional-initialism=pt \
--additional-initialism=qan \
--additional-initialism=rds \
--additional-initialism=sql \
--additional-initialism=ha ; \
done

# generate public API spec, omit agentlocal and inventory (always private),
# as well as a number of protos that are in beta (not v1 yet, they all go to a similar call below)
../bin/swagger mixin --output=swagger/swagger.json \
swagger/header.json \
server/v1/json/v1.json \
user/v1/json/v1.json \
inventory/v1/json/v1.json \
management/v1/json/v1.json \
actions/v1/json/v1.json \
alerting/v1/json/v1.json \
advisors/v1/json/v1.json \
backup/v1/json/v1.json \
qan/v1/json/v1.json \
platform/v1/json/v1.json
../bin/swagger validate swagger/swagger.json

../bin/swagger-order --output=swagger/swagger.json swagger/swagger.json

# generate API spec with all PMM Server APIs (omit agentlocal)
../bin/swagger mixin --output=swagger/swagger-dev.json \
swagger/header-dev.json \
server/v1/json/v1.json \
user/v1/json/v1.json \
inventory/v1/json/v1.json \
management/v1/json/v1.json \
actions/v1/json/v1.json \
alerting/v1/json/v1.json \
advisors/v1/json/v1.json \
backup/v1/json/v1.json \
dump/v1beta1/json/v1beta1.json \
accesscontrol/v1beta1/json/v1beta1.json \
qan/v1/json/v1.json \
platform/v1/json/v1.json

../bin/swagger validate swagger/swagger-dev.json

../bin/swagger-order --output=swagger/swagger-dev.json swagger/swagger-dev.json

format: ## Format API definitions
../bin/buf format . -w

clean-swagger:
find . -name '*.swagger.json' -print -delete


ifeq ($(shell test "${SWAGGER_UI_VERSION}" = "latest" && echo 1 || echo 0), 1)
get_swagger_version:
override SWAGGER_UI_VERSION = $(shell curl --silent --head https://github.com/swagger-api/swagger-ui/releases/latest | grep -E '^[lL]ocation' | sed 's;^.*/;;')
else
get_swagger_version:
endif

update-swagger: get_swagger_version ## Update Swagger UI in swagger, use `SWAGGER_UI_VERSION=xxx make update-swagger` to choose a version other than latest
curl --output /tmp/swagger.tar.gz -sSfL "https://github.com/swagger-api/swagger-ui/archive/refs/tags/${SWAGGER_UI_VERSION}.tar.gz"
tar xf /tmp/swagger.tar.gz --transform 's;swagger-ui-.*/dist/;swagger/;g'
rm -f swagger/.npmrc
sed -i 's/url: ".*"/url: "\/swagger.json",\n validatorUrl: "none"/' swagger/swagger-initializer.js

clean: clean-swagger ## Remove generated files
find . -name '*.pb.go' -print -delete
find . -name '*.pb.gw.go' -print -delete
find . -name '*.validate.go' -print -delete

SPECS="\
agentlocal/v1 \
server/v1 \
user/v1 \
inventory/v1 \
management/v1 \
actions/v1 \
alerting/v1 \
advisors/v1 \
backup/v1 \
dump/v1beta1 \
accesscontrol/v1beta1 \
qan/v1 \
platform/v1"; \
for API in $$SPECS; do \
rm -fr $$API/json/client $$API/json/models $$API/json/$$(basename $$API).json ; \
done
rm -f swagger/swagger.json swagger/swagger-dev.json

descriptors: ## Update API compatibility descriptors
../bin/buf build -o descriptor.bin --as-file-descriptor-set .
34 changes: 34 additions & 0 deletions api/buf.gen.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
version: v2
managed:
enabled: true
disable:
- file_option: go_package
module: buf.build/googleapis/googleapis
- file_option: go_package
module: buf.build/grpc-ecosystem/grpc-gateway
- file_option: go_package
module: buf.build/envoyproxy/protoc-gen-validate
override:
- file_option: go_package_prefix
value: github.com/percona/pmm/api
plugins:
- local: ../bin/protoc-gen-go
out: .
opt: paths=source_relative
- local: ../bin/protoc-gen-go-grpc
out: .
opt: paths=source_relative
- local: ../bin/protoc-gen-grpc-gateway
out: .
opt: paths=source_relative
- local: ../bin/protoc-gen-validate
out: .
opt:
- paths=source_relative
- lang=go
- local: ../bin/protoc-gen-openapiv2
out: .
opt:
- simple_operation_ids=true
- json_names_for_fields=false
- proto3_optional_nullable=true
20 changes: 7 additions & 13 deletions api/buf.lock
Original file line number Diff line number Diff line change
@@ -1,18 +1,12 @@
# Generated by buf. DO NOT EDIT.
version: v1
version: v2
deps:
- remote: buf.build
owner: envoyproxy
repository: protoc-gen-validate
- name: buf.build/envoyproxy/protoc-gen-validate
commit: eac44469a7af47e7839a7f1f3d7ac004
digest: shake256:0feabcde01b6b11e3c75a5e3f807968d5995626546f39c37e5d4205892b3a59cced0ed83b35a2eb9e6dddd3309660ad46b737c9dcd224b425de0a6654ce04417
- remote: buf.build
owner: googleapis
repository: googleapis
digest: b5:2ef7b25d9671fde82d58278e94d209788fb6a42936cc2d60e42303b67dcdb72b2d0e121238ba44caf1870a34edab55518df5d3f1ee1e62b8d9b6cf98817eae6e
- name: buf.build/googleapis/googleapis
commit: 28151c0d0a1641bf938a7672c500e01d
digest: shake256:49215edf8ef57f7863004539deff8834cfb2195113f0b890dd1f67815d9353e28e668019165b9d872395871eeafcbab3ccfdb2b5f11734d3cca95be9e8d139de
- remote: buf.build
owner: grpc-ecosystem
repository: grpc-gateway
digest: b5:93b70089baa4fc05a92d3e52db91a4b7812db3b57b9664f6cb301733938cb630e377a938e8a56779388171c749c1d42a2e9a6c6230f2ff45f127a8102a6a27d0
- name: buf.build/grpc-ecosystem/grpc-gateway
commit: 3f42134f4c564983838425bc43c7a65f
digest: shake256:3d11d4c0fe5e05fda0131afefbce233940e27f0c31c5d4e385686aea58ccd30f72053f61af432fa83f1fc11cda57f5f18ca3da26a29064f73c5a0d076bba8d92
digest: b5:291b947d8ac09492517557e4e72e294788cb8201afc7d0df7bda80fa10931adb60d4d669208a7696bf24f1ecb2a33a16d4c1e766e6f31809248b00343119569b
Loading

0 comments on commit 4242f0b

Please sign in to comment.