diff --git a/README.md b/README.md index 4f25fa0..292259e 100644 --- a/README.md +++ b/README.md @@ -78,12 +78,13 @@ uses dynamically passed workspaces via HTTP query parameter uses Azure service discovery to find all workspaces in one or multiple subscriptions -| GET parameter | Default | Required | Multiple | Description | -|------------------------|---------------------------|----------|----------|----------------------------------------------------------------------| -| `module` | | no | no | Filter queries by module name | -| `subscription` | | **yes** | yes | Uses all workspaces inside subscription | -| `cache` | | no | no | Use of internal metrics caching (time.Duration) | -| `parallel` | `$LOGANALYTICS_PARALLEL` | no | no | Number (int) of how many workspaces can be queried at the same time | +| GET parameter | Default | Required | Multiple | Description | +|----------------|--------------------------|----------|----------|------------------------------------------------------------------------------------------------------------------------------------------| +| `module` | | no | no | Filter queries by module name | +| `subscription` | | **yes** | yes | Uses all workspaces inside subscription | +| `filter` | | no | no | Advanced filter for `resource \| {filter} \| project id, customerId=properties.customerId` ResoruceGraph query (available with `23.6.0`) | +| `cache` | | no | no | Use of internal metrics caching (time.Duration) | +| `parallel` | `$LOGANALYTICS_PARALLEL` | no | no | Number (int) of how many workspaces can be queried at the same time | ## Global metrics diff --git a/go.mod b/go.mod index 7088972..91bebdb 100644 --- a/go.mod +++ b/go.mod @@ -8,18 +8,19 @@ require ( github.com/google/uuid v1.3.0 github.com/jessevdk/go-flags v1.5.0 github.com/patrickmn/go-cache v2.1.0+incompatible - github.com/prometheus/client_golang v1.15.1 + github.com/prometheus/client_golang v1.16.0 github.com/remeh/sizedwaitgroup v1.0.0 - github.com/webdevops/go-common v0.0.0-20230514184624-3c9d2cde95f8 + github.com/webdevops/go-common v0.0.0-20230612205735-2ee45347be15 go.uber.org/zap v1.24.0 ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0-beta.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.1.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.2.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -30,14 +31,14 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.43.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/crypto v0.10.0 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 6d853fa..ed0f583 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0-beta.2 h1:C3zKsGguxcLd8a2uEytB8+TFtBGd75bXRxEs0QBwsv0= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0-beta.2/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= @@ -10,10 +10,12 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNL github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/operationalinsights/armoperationalinsights v1.1.1 h1:2fNJSfRbULkn3BzwWcA5WO1wHDr6E14LkCiWHdkpfR4= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/operationalinsights/armoperationalinsights v1.1.1/go.mod h1:A5GfDiLHjdNwpQ2DLX6QNKt/wJJfOoLz4cB7JyUapxw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.0 h1:+fxpya1I3WJEvl+qzET7NO2MkjtYkiZJxIcEN690W64= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.0/go.mod h1:1wZ2sD5NUZGEk/RVNxjBFPqVHC39ekUDtmNnWhKhrGY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.1.1 h1:A+a54F7ygu4ANdV9hYsLMfiHFgjuwIUCG+6opLAvxJE= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.1.1/go.mod h1:ThfyMjs6auYrWPnYJjI3H4H++oVPrz01pizpu8lfl3A= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.2.0 h1:Pmy0+3ox1IC3sp6musv87BFPIdQbqyPFjn7I8I0o2Js= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.2.0/go.mod h1:ThfyMjs6auYrWPnYJjI3H4H++oVPrz01pizpu8lfl3A= github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -49,20 +51,20 @@ github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= -github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.43.0 h1:iq+BVjvYLei5f27wiuNiB1DN6DYQkp1c8Bx0Vykh5us= -github.com/prometheus/common v0.43.0/go.mod h1:NCvr5cQIh3Y/gy73/RdVtC9r8xxrxwJnB+2lB3BxrFc= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk= +github.com/prometheus/procfs v0.11.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/webdevops/go-common v0.0.0-20230514184624-3c9d2cde95f8 h1:PV3JDjPJ+CaUrra3omUx6pwo7ALacgb7VUNmLUpORgE= -github.com/webdevops/go-common v0.0.0-20230514184624-3c9d2cde95f8/go.mod h1:Jyf1rCdjOxgJuwTuZy4l0xOm30FTEa3QV3fQZdySKdw= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/webdevops/go-common v0.0.0-20230612205735-2ee45347be15 h1:3+zCVejFHGmjWyEIbrid1d483SJOvYgLW7q3xEYchjE= +github.com/webdevops/go-common v0.0.0-20230612205735-2ee45347be15/go.mod h1:KUQ7bNZHIHSU4BryRq2E77ob0hrXWZ6CtSbcnqKlUyw= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= @@ -70,17 +72,17 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= diff --git a/loganalytics/servicediscovery.go b/loganalytics/servicediscovery.go index c35130a..91810d6 100644 --- a/loganalytics/servicediscovery.go +++ b/loganalytics/servicediscovery.go @@ -10,7 +10,6 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/operationalinsights/armoperationalinsights" "github.com/webdevops/go-common/azuresdk/armclient" - "github.com/webdevops/go-common/utils/to" "go.uber.org/zap" ) @@ -100,7 +99,7 @@ func (sd *LogAnalyticsServiceDiscovery) ServiceDiscovery() { serviceDiscoveryCacheDuration = prober.Conf.Azure.ServiceDiscovery.CacheDuration cacheKey = fmt.Sprintf( "sd:%x", - string(sha1.New().Sum([]byte(fmt.Sprintf("%v", subscriptionList)))), + string(sha1.New().Sum([]byte(fmt.Sprintf("%v:%v", subscriptionList, prober.request.URL.String())))), ) // #nosec // try cache @@ -118,7 +117,7 @@ func (sd *LogAnalyticsServiceDiscovery) ServiceDiscovery() { } contextLogger.Debug("requesting list for workspaces via Azure API") - sd.requestWorkspacesFromAzure(contextLogger, subscriptionList) + sd.findWorkspaces(contextLogger, subscriptionList) // store to cache (if enabeld) if serviceDiscoveryCacheDuration != nil { @@ -131,28 +130,33 @@ func (sd *LogAnalyticsServiceDiscovery) ServiceDiscovery() { } } -func (sd *LogAnalyticsServiceDiscovery) requestWorkspacesFromAzure(logger *zap.SugaredLogger, subscriptionList []string) { +func (sd *LogAnalyticsServiceDiscovery) findWorkspaces(logger *zap.SugaredLogger, subscriptionList []string) { prober := sd.prober - for _, subscriptionId := range subscriptionList { - subscriptionLogger := logger.With(zap.String("subscription", subscriptionId)) - - pager := sd.ResourcesClient(subscriptionId).NewListPager(nil) - for pager.More() { - result, err := pager.NextPage(prober.ctx) - if err != nil { - subscriptionLogger.Panic(err) - } - - for _, workspace := range result.Value { - if workspace.Properties.CustomerID != nil { - prober.workspaceList = append( - prober.workspaceList, - to.String(workspace.Properties.CustomerID), - ) - } - } + query := "resources \n" + query += "| where type =~ \"Microsoft.OperationalInsights/workspaces\" \n" + if filter := prober.request.URL.Query().Get("filter"); len(filter) > 0 { + filter = strings.TrimSpace(filter) + filter = strings.TrimLeft(filter, "|") + if len(filter) >= 1 { + query += fmt.Sprintf("| %s \n", filter) } } + query += "| project id, customerId=properties.customerId" + result, err := prober.Azure.Client.ExecuteResourceGraphQuery( + prober.ctx, + subscriptionList, + query, + ) + if err != nil { + logger.Panic(err) + } + + for _, row := range result { + prober.workspaceList = append( + prober.workspaceList, + row["customerId"].(string), + ) + } } diff --git a/templates/query.html b/templates/query.html index a700da3..400b192 100644 --- a/templates/query.html +++ b/templates/query.html @@ -157,7 +157,7 @@

General

-
+
@@ -185,6 +185,14 @@

Service Discovery

+
+ +
+ +
ResourceGraph filter
+
+
+
@@ -277,8 +285,8 @@

Prometheus scrape_config

let formSetVisibility = () => { let endpoint = $("#endpoint:input").val().trim(); $("form.query div.row").removeClass("hidden"); - $("form.query div.row[query-endpoint]:not([query-endpoint*=\"" + endpoint + "\"])").addClass("hidden"); - $("form.query div.row[query-endpoint-exclude][query-endpoint-exclude*=\"" + endpoint + "\"]").addClass("hidden"); + $("form.query div.row[query-endpoint]:not([query-endpoint~=\"" + endpoint + "\"])").addClass("hidden"); + $("form.query div.row[query-endpoint-exclude][query-endpoint-exclude~=\"" + endpoint + "\"]").addClass("hidden"); }; let buildPrometheusScrapeConfig = (queryEndpoint, queryParams) => {