Skip to content

Commit

Permalink
Merge branch 'main' into port_conflict
Browse files Browse the repository at this point in the history
  • Loading branch information
rnovikovP authored Nov 30, 2023
2 parents 90b15ec + 9db1746 commit d2698e2
Show file tree
Hide file tree
Showing 280 changed files with 18,541 additions and 4,400 deletions.
156 changes: 156 additions & 0 deletions .github/workflows/api-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
name: 'API'

on:
# NOTE: no action on `push` and `pull_request` since the API tests need to be run
# against a container that has the latest changes from your code. Therefore, the
# PMM_SERVER_IMAGE should either be the one from your feature build or the local
# devcontainer. One more scenario is when we want to run the API tests on dev-latest
# to see if the tests are in a good shape.
# That said, this workflow is mostly a convenience if you prefer Github to Jenkins.
# https://github.com/Percona-Lab/jenkins-pipelines/blob/master/pmm/pmm2-api-tests.groovy

workflow_dispatch:
inputs:
MYSQL_IMAGE:
description: "MYSQL image version"
default: "percona:5.7"
required: true
type: string
POSTGRESQL_IMAGE:
description: "POSTGRESQL image version"
default: "postgres:12"
required: true
type: string
MONGODB_IMAGE:
description: "MONGODB image version"
default: "percona/percona-server-mongodb:4.4"
required: true
type: string
PMM_SERVER_IMAGE:
description: "PMM Server image version"
default: "perconalab/pmm-server:dev-latest"
required: true
type: string

workflow_call:
inputs:
BRANCH:
description: "The branch to source API tests from"
default: "main"
required: true
type: string
MYSQL_IMAGE:
description: "MYSQL image version"
default: "percona:5.7"
required: true
type: string
POSTGRESQL_IMAGE:
description: "POSTGRESQL image version"
default: "postgres:12"
required: true
type: string
MONGODB_IMAGE:
description: "MONGODB image version"
default: "percona/percona-server-mongodb:4.4"
required: true
type: string
PMM_SERVER_IMAGE:
description: "PMM Server image version"
default: "perconalab/pmm-server:dev-latest"
required: true
type: string

jobs:
test:
name: Tests
runs-on: ubuntu-22.04
env:
PMM_URL: http://admin:[email protected]
BRANCH: ${{ github.event.inputs.BRANCH || 'main' }}
MYSQL_IMAGE: ${{ github.event.inputs.MYSQL_IMAGE || 'percona:5.7' }}
POSTGRESQL_IMAGE: ${{ github.event.inputs.POSTGRESQL_IMAGE || 'postgres:12' }}
MONGODB_IMAGE: ${{ github.event.inputs.MONGODB_IMAGE || 'percona/percona-server-mongodb:4.4' }}
PMM_SERVER_IMAGE: ${{ github.event.inputs.PMM_SERVER_IMAGE || 'perconalab/pmm-server:dev-latest' }}

steps:
- name: Check out code
uses: actions/checkout@v4
with:
ref: ${{ env.BRANCH }}

- name: Login to docker.io registry
uses: docker/login-action@v3
with:
registry: docker.io
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}

- name: Run PMM Server
run: |
cat <<-EOF > compose.pmm-server-test.yml
services:
pmm-server:
image: ${{ env.PMM_SERVER_IMAGE }}
container_name: pmm-agent_pmm-server
environment:
- PMM_DEBUG=1
- PERCONA_TEST_CHECKS_INTERVAL=10s
- PERCONA_TEST_PLATFORM_ADDRESS=https://check-dev.percona.com
- PERCONA_TEST_PLATFORM_PUBLIC_KEY=RWTg+ZmCCjt7O8eWeAmTLAqW+1ozUbpRSKSwNTmO+exlS5KEIPYWuYdX
ports:
- 80:80
- 443:443
volumes:
- ./managed/testdata/checks:/srv/checks
EOF
# Run it and wait until it's healthy
docker compose -f compose.pmm-server-test.yml up -d --wait --wait-timeout=100
- name: Build the test image
shell: bash
run: |
docker build -t percona/pmm-api-tests .
- name: Run DB containers
shell: bash
run: |
pushd api-tests
docker compose up test_db # no daemon mode
# TODO: review the provisions below (copied from the Jenkins pipeline)
# MYSQL_IMAGE=${{env.MYSQL_IMAGE}} docker compose up -d mysql
# MONGO_IMAGE=${{env.MONGODB_IMAGE}} docker compose up -d mongo
# POSTGRES_IMAGE=${{env.POSTGRESQL_IMAGE}} docker compose up -d postgres
# docker compose up -d sysbench
popd
- name: Check connectivity to PMM Server
shell: bash
run: curl -f ${{env.PMM_URL}}/ping

- name: Run API tests
shell: bash
run: |
docker run -e PMM_SERVER_URL=${{env.PMM_URL}} \
-e PMM_RUN_UPDATE_TEST=0 \
-e PMM_RUN_STT_TESTS=0 \
--name pmm-api-tests \
--network host \
percona/pmm-api-tests
- name: Get PMM logs
if: ${{ failure() }}
run: curl --insecure ${{env.PMM_URL}}/logs.zip --output ${{ github.workspace }}/logs.zip || true

- name: Upload the logs on failure
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: "logs.zip"
path: ${{ github.workspace }}/logs.zip

- name: Run debug commands on failure
if: ${{ failure() }}
run: |
echo "----- ENVIRONMENT VARIABLES -----"
env | sort
2 changes: 1 addition & 1 deletion .github/workflows/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,4 @@ jobs:
run: gh pr merge --auto --squash "$PR_URL"
env:
PR_URL: ${{github.event.pull_request.html_url}}
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,6 @@ compose.yml

cli-tests/node_modules/
cli-tests/playwright-report/

api-tests/pmm-api-tests-output.txt
api-tests/pmm-api-tests-junit-report.xml
5 changes: 3 additions & 2 deletions Makefile.include
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ gen-api: ## Generate PMM API

bin/buf generate -v api

for API in api/agentlocalpb api/serverpb api/inventorypb api/managementpb api/managementpb/dbaas api/managementpb/ia api/managementpb/alerting api/managementpb/backup api/managementpb/azure api/managementpb/role api/qanpb api/managementpb/agent api/managementpb/node api/managementpb/service api/platformpb api/userpb; do \
for API in api/agentlocalpb api/serverpb api/inventorypb api/managementpb api/managementpb/dbaas api/managementpb/ia api/managementpb/alerting api/managementpb/backup api/managementpb/dump api/managementpb/azure api/managementpb/role api/qanpb api/managementpb/agent api/managementpb/node api/managementpb/service api/platformpb api/userpb; do \
set -x ; \
bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \
bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \
Expand Down Expand Up @@ -91,6 +91,7 @@ gen-api: ## Generate PMM API
api/managementpb/ia/json/ia.json \
api/managementpb/alerting/json/alerting.json \
api/managementpb/backup/json/backup.json \
api/managementpb/dump/json/dump.json \
api/managementpb/azure/json/azure.json \
api/managementpb/role/json/role.json \
api/managementpb/agent/json/agent.json \
Expand Down Expand Up @@ -127,7 +128,7 @@ clean: clean_swagger ## Remove generated files
find api -name '*.pb.gw.go' -print -delete
find api -name '*.validate.go' -print -delete

for API in api/agentlocalpb api/serverpb api/inventorypb api/managementpb api/managementpb/dbaas api/managementpb/ia api/managementpb/alerting api/managementpb/backup api/managementpb/role api/managementpb/agent api/managementpb/node api/managementpb/service api/qanpb api/platformpb ; do \
for API in api/agentlocalpb api/serverpb api/inventorypb api/managementpb api/managementpb/dbaas api/managementpb/ia api/managementpb/alerting api/managementpb/backup api/management/dump api/managementpb/role api/managementpb/agent api/managementpb/node api/managementpb/service api/qanpb api/platformpb ; do \
rm -fr $$API/json/client $$API/json/models $$API/json/$$(basename $$API).json ; \
done
rm -f api/swagger/swagger.json api/swagger/swagger-dev.json
Expand Down
6 changes: 4 additions & 2 deletions admin/commands/inventory/add_agent_mysqld_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ func (res *addAgentMysqldExporterResult) TablestatStatus() string {
}

switch {
case res.Agent.TablestatsGroupTableLimit == 0: // no limit
case res.Agent.TablestatsGroupTableLimit == 0: // server defined
s += " (the table count limit is not set)."
case res.Agent.TablestatsGroupTableLimit < 0: // always disabled
s += " (always)."
Expand Down Expand Up @@ -94,8 +94,9 @@ type AddAgentMysqldExporterCommand struct {
TLSCAFile string `name:"tls-ca" help:"Path to certificate authority certificate file"`
TLSCertFile string `name:"tls-cert" help:"Path to client certificate file"`
TLSKeyFile string `name:"tls-key" help:"Path to client key file"`
TablestatsGroupTableLimit int32 `placeholder:"number" help:"Tablestats group collectors will be disabled if there are more than that number of tables (default: 0 - always enabled; negative value - always disabled)"`
TablestatsGroupTableLimit int32 `placeholder:"number" help:"Tablestats group collectors will be disabled if there are more than that number of tables (default: server-defined, -1: always disabled)"`
PushMetrics bool `help:"Enables push metrics model flow, it will be sent to the server by an agent"`
ExposeExporter bool `help:"Expose the address of the exporter publicly on 0.0.0.0"`
DisableCollectors []string `help:"Comma-separated list of collector names to exclude from exporter"`
LogLevel string `enum:"debug,info,warn,error" default:"warn" help:"Service logging level. One of: [debug, info, warn, error]"`
}
Expand Down Expand Up @@ -140,6 +141,7 @@ func (cmd *AddAgentMysqldExporterCommand) RunCmd() (commands.Result, error) {
TLSKey: tlsKey,
TablestatsGroupTableLimit: cmd.TablestatsGroupTableLimit,
PushMetrics: cmd.PushMetrics,
ExposeExporter: cmd.ExposeExporter,
DisableCollectors: commands.ParseDisableCollectors(cmd.DisableCollectors),
LogLevel: &cmd.LogLevel,
},
Expand Down
2 changes: 2 additions & 0 deletions admin/commands/inventory/add_agent_node_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ type AddAgentNodeExporterCommand struct {
PMMAgentID string `arg:"" help:"The pmm-agent identifier which runs this instance"`
CustomLabels map[string]string `mapsep:"," help:"Custom user-assigned labels"`
PushMetrics bool `help:"Enables push metrics model flow, it will be sent to the server by an agent"`
ExposeExporter bool `help:"Expose the address of the exporter publicly on 0.0.0.0"`
DisableCollectors []string `help:"Comma-separated list of collector names to exclude from exporter"`
LogLevel string `enum:"debug,info,warn,error" default:"warn" help:"Service logging level. One of: [debug, info, warn, error]"`
}
Expand All @@ -57,6 +58,7 @@ func (cmd *AddAgentNodeExporterCommand) RunCmd() (commands.Result, error) {
PMMAgentID: cmd.PMMAgentID,
CustomLabels: customLabels,
PushMetrics: cmd.PushMetrics,
ExposeExporter: cmd.ExposeExporter,
DisableCollectors: commands.ParseDisableCollectors(cmd.DisableCollectors),
LogLevel: &cmd.LogLevel,
},
Expand Down
6 changes: 6 additions & 0 deletions admin/commands/inventory/add_agent_postgres_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ func (res *addAgentPostgresExporterResult) String() string {
}

// AddAgentPostgresExporterCommand is used by Kong for CLI flags and commands.
//
//nolint:lll
type AddAgentPostgresExporterCommand struct {
PMMAgentID string `arg:"" help:"The pmm-agent identifier which runs this instance"`
ServiceID string `arg:"" help:"Service identifier"`
Expand All @@ -55,13 +57,15 @@ type AddAgentPostgresExporterCommand struct {
CustomLabels map[string]string `mapsep:"," help:"Custom user-assigned labels"`
SkipConnectionCheck bool `help:"Skip connection check"`
PushMetrics bool `help:"Enables push metrics model flow, it will be sent to the server by an agent"`
ExposeExporter bool `help:"Expose the address of the exporter publicly on 0.0.0.0"`
DisableCollectors []string `help:"Comma-separated list of collector names to exclude from exporter"`
TLS bool `help:"Use TLS to connect to the database"`
TLSSkipVerify bool `help:"Skip TLS certificates validation"`
TLSCAFile string `help:"TLS CA certificate file"`
TLSCertFile string `help:"TLS certificate file"`
TLSKeyFile string `help:"TLS certificate key file"`
LogLevel string `enum:"debug,info,warn,error" default:"warn" help:"Service logging level. One of: [debug, info, warn, error]"`
AutoDiscoveryLimit int32 `default:"0" placeholder:"NUMBER" help:"Auto-discovery will be disabled if there are more than that number of databases (default: server-defined, -1: always disabled)"`
}

func (cmd *AddAgentPostgresExporterCommand) RunCmd() (commands.Result, error) {
Expand Down Expand Up @@ -98,7 +102,9 @@ func (cmd *AddAgentPostgresExporterCommand) RunCmd() (commands.Result, error) {
CustomLabels: customLabels,
SkipConnectionCheck: cmd.SkipConnectionCheck,
PushMetrics: cmd.PushMetrics,
ExposeExporter: cmd.ExposeExporter,
DisableCollectors: commands.ParseDisableCollectors(cmd.DisableCollectors),
AutoDiscoveryLimit: cmd.AutoDiscoveryLimit,

TLS: cmd.TLS,
TLSSkipVerify: cmd.TLSSkipVerify,
Expand Down
2 changes: 2 additions & 0 deletions admin/commands/inventory/add_agent_proxysql_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ type AddAgentProxysqlExporterCommand struct {
TLS bool `help:"Use TLS to connect to the database"`
TLSSkipVerify bool `help:"Skip TLS certificates validation"`
PushMetrics bool `help:"Enables push metrics model flow, it will be sent to the server by an agent"`
ExposeExporter bool `help:"Expose the address of the exporter publicly on 0.0.0.0"`
DisableCollectors []string `help:"Comma-separated list of collector names to exclude from exporter"`
LogLevel string `enum:"debug,info,warn,error,fatal" default:"warn" help:"Service logging level. One of: [debug, info, warn, error, fatal]"`
}
Expand All @@ -75,6 +76,7 @@ func (cmd *AddAgentProxysqlExporterCommand) RunCmd() (commands.Result, error) {
TLS: cmd.TLS,
TLSSkipVerify: cmd.TLSSkipVerify,
PushMetrics: cmd.PushMetrics,
ExposeExporter: cmd.ExposeExporter,
DisableCollectors: commands.ParseDisableCollectors(cmd.DisableCollectors),
LogLevel: &cmd.LogLevel,
},
Expand Down
4 changes: 3 additions & 1 deletion admin/commands/management/add_mongodb.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ type AddMongoDBCommand struct {
DisableCollectors []string `help:"Comma-separated list of collector names to exclude from exporter"`
StatsCollections []string `help:"Collections for collstats & indexstats"`
CollectionsLimit int32 `name:"max-collections-limit" default:"-1" help:"Disable collstats, dbstats, topmetrics and indexstats if there are more than <n> collections. 0: No limit. Default is -1, which let PMM automatically set this value"`
ExposeExporter bool `name:"expose-exporter" help:"Optionally expose the address of the exporter publicly on 0.0.0.0"`

AddCommonFlags
AddLogLevelFatalFlags
Expand Down Expand Up @@ -157,8 +158,9 @@ func (cmd *AddMongoDBCommand) RunCmd() (commands.Result, error) {
NodeID: cmd.NodeID,
ServiceName: serviceName,
Address: host,
Port: int64(port),
Socket: socket,
Port: int64(port),
ExposeExporter: cmd.ExposeExporter,
PMMAgentID: cmd.PMMAgentID,
Environment: cmd.Environment,
Cluster: cmd.Cluster,
Expand Down
4 changes: 3 additions & 1 deletion admin/commands/management/add_mysql.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func (res *addMySQLResult) TablestatStatus() string {
s := "Table statistics collection " + status

switch {
case res.MysqldExporter.TablestatsGroupTableLimit == 0: // no limit
case res.MysqldExporter.TablestatsGroupTableLimit == 0: // server defined
s += " (the table count limit is not set)."
case res.MysqldExporter.TablestatsGroupTableLimit < 0: // always disabled
s += " (always)."
Expand Down Expand Up @@ -120,6 +120,7 @@ type AddMySQLCommand struct {
CreateUser bool `hidden:"" help:"Create pmm user"`
MetricsMode string `enum:"${metricsModesEnum}" default:"auto" help:"Metrics flow mode, can be push - agent will push metrics, pull - server scrape metrics from agent or auto - chosen by server"`
DisableCollectors []string `help:"Comma-separated list of collector names to exclude from exporter"`
ExposeExporter bool `name:"expose-exporter" help:"Optionally expose the address of the exporter publicly on 0.0.0.0"`

AddCommonFlags
AddLogLevelNoFatalFlags
Expand Down Expand Up @@ -209,6 +210,7 @@ func (cmd *AddMySQLCommand) RunCmd() (commands.Result, error) {
Address: host,
Socket: socket,
Port: int64(port),
ExposeExporter: cmd.ExposeExporter,
PMMAgentID: cmd.PMMAgentID,
Environment: cmd.Environment,
Cluster: cmd.Cluster,
Expand Down
11 changes: 7 additions & 4 deletions admin/commands/management/add_postgresql.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ type AddPostgreSQLCommand struct {
DisableQueryExamples bool `name:"disable-queryexamples" help:"Disable collection of query examples"`
MetricsMode string `enum:"${metricsModesEnum}" default:"auto" help:"Metrics flow mode, can be push - agent will push metrics, pull - server scrape metrics from agent or auto - chosen by server"`
DisableCollectors []string `help:"Comma-separated list of collector names to exclude from exporter"`
ExposeExporter bool `name:"expose-exporter" help:"Optionally expose the address of the exporter publicly on 0.0.0.0"`
AutoDiscoveryLimit int32 `default:"0" placeholder:"NUMBER" help:"Auto-discovery will be disabled if there are more than that number of databases (default: server-defined, -1: always disabled)"`

AddCommonFlags
AddLogLevelNoFatalFlags
Expand Down Expand Up @@ -171,16 +173,16 @@ func (cmd *AddPostgreSQLCommand) RunCmd() (commands.Result, error) {

params := &postgresql.AddPostgreSQLParams{
Body: postgresql.AddPostgreSQLBody{
NodeID: cmd.NodeID,
ServiceName: serviceName,

NodeID: cmd.NodeID,
ServiceName: serviceName,
Address: host,
Socket: socket,
Port: int64(port),
ExposeExporter: cmd.ExposeExporter,
Username: cmd.Username,
Password: cmd.Password,
Database: cmd.Database,
AgentPassword: cmd.AgentPassword,
Socket: socket,
SkipConnectionCheck: cmd.SkipConnectionCheck,
DisableCommentsParsing: disableCommentsParsing,

Expand All @@ -203,6 +205,7 @@ func (cmd *AddPostgreSQLCommand) RunCmd() (commands.Result, error) {
DisableQueryExamples: cmd.DisableQueryExamples,
MetricsMode: pointer.ToString(strings.ToUpper(cmd.MetricsMode)),
DisableCollectors: commands.ParseDisableCollectors(cmd.DisableCollectors),
AutoDiscoveryLimit: cmd.AutoDiscoveryLimit,
LogLevel: &cmd.AddLogLevel,
},
Context: commands.Ctx,
Expand Down
Loading

0 comments on commit d2698e2

Please sign in to comment.