use resourcegraph for servicediscovery (add filter parameter for adva…

…nced filtering)

Signed-off-by: Markus Blaschke <[email protected]>

README.md

@@ -78,12 +78,13 @@ uses dynamically passed workspaces via HTTP query parameter
 
 uses Azure service discovery to find all workspaces in one or multiple subscriptions
 
-| GET parameter          | Default                   | Required | Multiple | Description                                                          |
-|------------------------|---------------------------|----------|----------|----------------------------------------------------------------------|
-| `module`               |                           | no       | no       | Filter queries by module name                                        |
-| `subscription`         |                           | **yes**  | yes      | Uses all workspaces inside subscription                              |
-| `cache`                |                           | no       | no       | Use of internal metrics caching (time.Duration)                      |
-| `parallel`             | `$LOGANALYTICS_PARALLEL`  | no       | no       | Number (int) of how many workspaces can be queried at the same time  |
+| GET parameter  | Default                  | Required | Multiple | Description                                                                                                                              |
+|----------------|--------------------------|----------|----------|------------------------------------------------------------------------------------------------------------------------------------------|
+| `module`       |                          | no       | no       | Filter queries by module name                                                                                                            |
+| `subscription` |                          | **yes**  | yes      | Uses all workspaces inside subscription                                                                                                  |
+| `filter`       |                          | no       | no       | Advanced filter for `resource \| {filter} \| project id, customerId=properties.customerId` ResoruceGraph query (available with `23.6.0`) |
+| `cache`        |                          | no       | no       | Use of internal metrics caching (time.Duration)                                                                                          |
+| `parallel`     | `$LOGANALYTICS_PARALLEL` | no       | no       | Number (int) of how many workspaces can be queried at the same time                                                                      |
 
 ## Global metrics
 

go.mod

@@ -8,18 +8,19 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/jessevdk/go-flags v1.5.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/prometheus/client_golang v1.15.1
+	github.com/prometheus/client_golang v1.16.0
 	github.com/remeh/sizedwaitgroup v1.0.0
-	github.com/webdevops/go-common v0.0.0-20230514184624-3c9d2cde95f8
+	github.com/webdevops/go-common v0.0.0-20230612205735-2ee45347be15
 	go.uber.org/zap v1.24.0
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0-beta.2 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.1.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.2.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -30,14 +31,14 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
-	github.com/prometheus/common v0.43.0 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.9.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
+	golang.org/x/crypto v0.10.0 // indirect
+	golang.org/x/net v0.11.0 // indirect
+	golang.org/x/sys v0.9.0 // indirect
+	golang.org/x/text v0.10.0 // indirect
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,5 +1,5 @@
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0-beta.2 h1:C3zKsGguxcLd8a2uEytB8+TFtBGd75bXRxEs0QBwsv0=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0-beta.2/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
@@ -10,10 +10,12 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNL
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/operationalinsights/armoperationalinsights v1.1.1 h1:2fNJSfRbULkn3BzwWcA5WO1wHDr6E14LkCiWHdkpfR4=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/operationalinsights/armoperationalinsights v1.1.1/go.mod h1:A5GfDiLHjdNwpQ2DLX6QNKt/wJJfOoLz4cB7JyUapxw=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.0 h1:+fxpya1I3WJEvl+qzET7NO2MkjtYkiZJxIcEN690W64=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.0/go.mod h1:1wZ2sD5NUZGEk/RVNxjBFPqVHC39ekUDtmNnWhKhrGY=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.1.1 h1:A+a54F7ygu4ANdV9hYsLMfiHFgjuwIUCG+6opLAvxJE=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.1.1/go.mod h1:ThfyMjs6auYrWPnYJjI3H4H++oVPrz01pizpu8lfl3A=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.2.0 h1:Pmy0+3ox1IC3sp6musv87BFPIdQbqyPFjn7I8I0o2Js=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.2.0/go.mod h1:ThfyMjs6auYrWPnYJjI3H4H++oVPrz01pizpu8lfl3A=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
@@ -49,38 +51,38 @@ github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI=
-github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
+github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
+github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
 github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
 github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
-github.com/prometheus/common v0.43.0 h1:iq+BVjvYLei5f27wiuNiB1DN6DYQkp1c8Bx0Vykh5us=
-github.com/prometheus/common v0.43.0/go.mod h1:NCvr5cQIh3Y/gy73/RdVtC9r8xxrxwJnB+2lB3BxrFc=
-github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
-github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/procfs v0.11.0 h1:5EAgkfkMl659uZPbe9AS2N68a7Cc1TJbPEuGzFuRbyk=
+github.com/prometheus/procfs v0.11.0/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/webdevops/go-common v0.0.0-20230514184624-3c9d2cde95f8 h1:PV3JDjPJ+CaUrra3omUx6pwo7ALacgb7VUNmLUpORgE=
-github.com/webdevops/go-common v0.0.0-20230514184624-3c9d2cde95f8/go.mod h1:Jyf1rCdjOxgJuwTuZy4l0xOm30FTEa3QV3fQZdySKdw=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/webdevops/go-common v0.0.0-20230612205735-2ee45347be15 h1:3+zCVejFHGmjWyEIbrid1d483SJOvYgLW7q3xEYchjE=
+github.com/webdevops/go-common v0.0.0-20230612205735-2ee45347be15/go.mod h1:KUQ7bNZHIHSU4BryRq2E77ob0hrXWZ6CtSbcnqKlUyw=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
 go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
-golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
-golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
+golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
+golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
+golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
+golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
+golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=

loganalytics/servicediscovery.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/operationalinsights/armoperationalinsights"
 	"github.com/webdevops/go-common/azuresdk/armclient"
-	"github.com/webdevops/go-common/utils/to"
 	"go.uber.org/zap"
 )
 
@@ -100,7 +99,7 @@ func (sd *LogAnalyticsServiceDiscovery) ServiceDiscovery() {
 		serviceDiscoveryCacheDuration = prober.Conf.Azure.ServiceDiscovery.CacheDuration
 		cacheKey = fmt.Sprintf(
 			"sd:%x",
-			string(sha1.New().Sum([]byte(fmt.Sprintf("%v", subscriptionList)))),
+			string(sha1.New().Sum([]byte(fmt.Sprintf("%v:%v", subscriptionList, prober.request.URL.String())))),
 		) // #nosec
 
 		// try cache
@@ -118,7 +117,7 @@ func (sd *LogAnalyticsServiceDiscovery) ServiceDiscovery() {
 	}
 
 	contextLogger.Debug("requesting list for workspaces via Azure API")
-	sd.requestWorkspacesFromAzure(contextLogger, subscriptionList)
+	sd.findWorkspaces(contextLogger, subscriptionList)
 
 	// store to cache (if enabeld)
 	if serviceDiscoveryCacheDuration != nil {
@@ -131,28 +130,33 @@ func (sd *LogAnalyticsServiceDiscovery) ServiceDiscovery() {
 	}
 }
 
-func (sd *LogAnalyticsServiceDiscovery) requestWorkspacesFromAzure(logger *zap.SugaredLogger, subscriptionList []string) {
+func (sd *LogAnalyticsServiceDiscovery) findWorkspaces(logger *zap.SugaredLogger, subscriptionList []string) {
 	prober := sd.prober
 
-	for _, subscriptionId := range subscriptionList {
-		subscriptionLogger := logger.With(zap.String("subscription", subscriptionId))
-
-		pager := sd.ResourcesClient(subscriptionId).NewListPager(nil)
-		for pager.More() {
-			result, err := pager.NextPage(prober.ctx)
-			if err != nil {
-				subscriptionLogger.Panic(err)
-			}
-
-			for _, workspace := range result.Value {
-				if workspace.Properties.CustomerID != nil {
-					prober.workspaceList = append(
-						prober.workspaceList,
-						to.String(workspace.Properties.CustomerID),
-					)
-				}
-			}
+	query := "resources \n"
+	query += "| where type =~ \"Microsoft.OperationalInsights/workspaces\" \n"
+	if filter := prober.request.URL.Query().Get("filter"); len(filter) > 0 {
+		filter = strings.TrimSpace(filter)
+		filter = strings.TrimLeft(filter, "|")
+		if len(filter) >= 1 {
+			query += fmt.Sprintf("| %s \n", filter)
 		}
 	}
+	query += "| project id, customerId=properties.customerId"
 
+	result, err := prober.Azure.Client.ExecuteResourceGraphQuery(
+		prober.ctx,
+		subscriptionList,
+		query,
+	)
+	if err != nil {
+		logger.Panic(err)
+	}
+
+	for _, row := range result {
+		prober.workspaceList = append(
+			prober.workspaceList,
+			row["customerId"].(string),
+		)
+	}
 }

templates/query.html

@@ -157,7 +157,7 @@ <h3>General</h3>
                 </div>
             </div>
 
-            <div class="mb-3 row">
+            <div class="mb-3 row" query-endpoint="/probe/workspace">
                 <label for="workspace" class="col-sm-2 col-form-label">workspace</label>
                 <div class="col-sm-10">
                     <textarea class="form-control" id="workspace" rows="3"></textarea>
@@ -185,6 +185,14 @@ <h3>Service Discovery</h3>
                 </div>
             </div>
 
+            <div class="mb-3 row" query-endpoint="/probe/subscription">
+                <label for="filter" class="col-sm-2 col-form-label">filter</label>
+                <div class="col-sm-10">
+                    <input type="text" class="form-control" id="filter" value="">
+                    <div class="form-text">ResourceGraph filter</div>
+                </div>
+            </div>
+
             <div class="mb-3 row">
                 <div class="offset-sm-2 col-sm-10">
                     <button type="button" class="btn btn-primary mb-3" id="sendQuery">Execute query</button>
@@ -277,8 +285,8 @@ <h2>Prometheus scrape_config</h2>
         let formSetVisibility = () => {
             let endpoint = $("#endpoint:input").val().trim();
             $("form.query div.row").removeClass("hidden");
-            $("form.query div.row[query-endpoint]:not([query-endpoint*=\"" + endpoint + "\"])").addClass("hidden");
-            $("form.query div.row[query-endpoint-exclude][query-endpoint-exclude*=\"" + endpoint + "\"]").addClass("hidden");
+            $("form.query div.row[query-endpoint]:not([query-endpoint~=\"" + endpoint + "\"])").addClass("hidden");
+            $("form.query div.row[query-endpoint-exclude][query-endpoint-exclude~=\"" + endpoint + "\"]").addClass("hidden");
         };
 
         let buildPrometheusScrapeConfig = (queryEndpoint, queryParams) => {