Get entity by attribute (#4311)

Adds a utility database function to search entities by ID. Because we
store entity properties as JSON, we add a GIN index.

Testing with 50.000 entities we get the following performance:
```
Nested Loop  (cost=30.39..109.45 rows=5 width=87) (actual time=0.290..0.292 rows=1 loops=1)
  ->  Bitmap Heap Scan on properties p  (cost=30.10..67.90 rows=5 width=16) (actual time=0.216..0.218 rows=1 loops=1)
"        Recheck Cond: (value @> '{""value"": ""MBymWMNbcv"", ""version"": ""v1""}'::jsonb)"
        Filter: (key = 'upstream_id'::text)
        Heap Blocks: exact=1
        ->  Bitmap Index Scan on idx_properties_value_gin  (cost=0.00..30.10 rows=10 width=0) (actual time=0.174..0.175 rows=1 loops=1)
"              Index Cond: (value @> '{""value"": ""MBymWMNbcv"", ""version"": ""v1""}'::jsonb)"
  ->  Index Scan using entity_instances_pkey on entity_instances ei  (cost=0.29..8.31 rows=1 width=87) (actual time=0.069..0.069 rows=1 loops=1)
        Index Cond: (id = p.entity_id)
        Filter: (entity_type = 'repository'::entities)
Planning Time: 2.365 ms
Execution Time: 0.590 ms
```

Compared to not using the index:
```
Nested Loop  (cost=0.29..3165.63 rows=5 width=87) (actual time=0.117..42.194 rows=1 loops=1)
  ->  Seq Scan on properties p  (cost=0.00..3124.07 rows=5 width=16) (actual time=0.032..42.108 rows=1 loops=1)
"        Filter: ((value @> '{""value"": ""MBymWMNbcv"", ""version"": ""v1""}'::jsonb) AND (key = 'upstream_id'::text))"
        Rows Removed by Filter: 100004
  ->  Index Scan using entity_instances_pkey on entity_instances ei  (cost=0.29..8.31 rows=1 width=87) (actual time=0.082..0.082 rows=1 loops=1)
        Index Cond: (id = p.entity_id)
        Filter: (entity_type = 'repository'::entities)
Planning Time: 1.104 ms
Execution Time: 42.300 ms
```

Related: #4179

database/migrations/000102_properties_upstream_id_index.down.sql

@@ -0,0 +1,19 @@
+-- Copyright 2023 Stacklok, Inc
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+BEGIN;
+
+DROP INDEX IF EXISTS idx_properties_value_gin;
+
+COMMIT;

database/migrations/000102_properties_upstream_id_index.up.sql

@@ -0,0 +1,20 @@
+-- Copyright 2023 Stacklok, Inc
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+BEGIN;
+
+-- Create index on properties for upstream_id
+CREATE INDEX idx_properties_value_gin ON properties USING GIN (value jsonb_path_ops);
+
+COMMIT;

database/query/entities.sql

@@ -99,4 +99,13 @@ WHERE entity_id = $1;
 
 -- name: DeleteAllPropertiesForEntity :exec
 DELETE FROM properties
-WHERE entity_id = $1;
+WHERE entity_id = $1;
+
+-- name: GetTypedEntitiesByProperty :many
+SELECT ei.*
+FROM entity_instances ei
+         JOIN properties p ON ei.id = p.entity_id
+WHERE ei.entity_type = sqlc.arg(entity_type)
+  AND (sqlc.arg(project_id)::uuid = '00000000-0000-0000-0000-000000000000'::uuid OR ei.project_id = sqlc.arg(project_id))
+  AND p.key = sqlc.arg(key)
+  AND p.value @> sqlc.arg(value)::jsonb;

internal/db/entities_test.go

@@ -202,6 +202,76 @@ func Test_PropertyCrud(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, anotherKeyVal.Value, "anothervalue")
 	})
+
+	t.Run("GetTypedEntitiesByPropertyV1", func(t *testing.T) {
+		t.Parallel()
+
+		const testRepoName = "testorg/testrepo_getbyprops"
+		const testArtifactName = "testorg/testartifact_getbyprops"
+
+		for i := 0; i < 50000; i++ {
+			createRandomEntity(t, proj.ID, prov.ID, EntitiesRepository)
+		}
+
+		repo, err := testQueries.CreateEntity(context.Background(), CreateEntityParams{
+			EntityType:     EntitiesRepository,
+			Name:           testRepoName,
+			ProjectID:      proj.ID,
+			ProviderID:     prov.ID,
+			OriginatedFrom: uuid.NullUUID{},
+		})
+		require.NoError(t, err)
+		require.NotEmpty(t, repo)
+
+		_, err = testQueries.UpsertPropertyValueV1(context.Background(), UpsertPropertyValueV1Params{
+			EntityID: repo.ID,
+			Key:      "sharedkey",
+			Value:    "sharedvalue",
+		})
+		require.NoError(t, err)
+
+		_, err = testQueries.UpsertPropertyValueV1(context.Background(), UpsertPropertyValueV1Params{
+			EntityID: repo.ID,
+			Key:      "repokey",
+			Value:    "repovalue",
+		})
+		require.NoError(t, err)
+
+		art, err := testQueries.CreateEntity(context.Background(), CreateEntityParams{
+			EntityType:     EntitiesArtifact,
+			Name:           testArtifactName,
+			ProjectID:      proj.ID,
+			ProviderID:     prov.ID,
+			OriginatedFrom: uuid.NullUUID{},
+		})
+		require.NoError(t, err)
+		require.NotEmpty(t, art)
+
+		_, err = testQueries.UpsertPropertyValueV1(context.Background(), UpsertPropertyValueV1Params{
+			EntityID: art.ID,
+			Key:      "sharedkey",
+			Value:    "sharedvalue",
+		})
+		require.NoError(t, err)
+
+		getEnt, err := testQueries.GetTypedEntitiesByPropertyV1(
+			context.Background(), proj.ID, EntitiesRepository, "sharedkey", "sharedvalue")
+		require.NoError(t, err)
+		require.Len(t, getEnt, 1)
+		require.Equal(t, getEnt[0].ID, repo.ID)
+
+		getEnt, err = testQueries.GetTypedEntitiesByPropertyV1(
+			context.Background(), proj.ID, EntitiesArtifact, "sharedkey", "sharedvalue")
+		require.NoError(t, err)
+		require.Len(t, getEnt, 1)
+		require.Equal(t, getEnt[0].ID, art.ID)
+
+		getEnt, err = testQueries.GetTypedEntitiesByPropertyV1(
+			context.Background(), proj.ID, EntitiesRepository, "repokey", "repovalue")
+		require.NoError(t, err)
+		require.Len(t, getEnt, 1)
+		require.Equal(t, getEnt[0].ID, repo.ID)
+	})
 }
 
 func propertyByKey(t *testing.T, props []PropertyValueV1, key string) PropertyValueV1 {

internal/db/entity_helpers.go

@@ -152,3 +152,20 @@ func (q *Queries) GetAllPropertyValuesV1(ctx context.Context, entityID uuid.UUID
 
 	return props, nil
 }
+
+// GetTypedEntitiesByPropertyV1 retrieves all entities with a property value
+func (q *Queries) GetTypedEntitiesByPropertyV1(
+	ctx context.Context, project uuid.UUID, entType Entities, key string, value any,
+) ([]EntityInstance, error) {
+	jsonVal, err := PropValueToDbV1(value)
+	if err != nil {
+		return nil, err
+	}
+
+	return q.GetTypedEntitiesByProperty(ctx, GetTypedEntitiesByPropertyParams{
+		EntityType: entType,
+		ProjectID:  project,
+		Key:        key,
+		Value:      jsonVal,
+	})
+}

internal/db/projects_test.go

@@ -30,6 +30,45 @@ import (
 	"github.com/stacklok/minder/internal/util/rand"
 )
 
+func createRandomEntity(t *testing.T, project uuid.UUID, provider uuid.UUID, entType Entities) {
+	t.Helper()
+
+	seed := time.Now().UnixNano()
+
+	ent, err := testQueries.CreateEntity(context.Background(), CreateEntityParams{
+		EntityType:     entType,
+		Name:           rand.RandomName(seed),
+		ProjectID:      project,
+		ProviderID:     provider,
+		OriginatedFrom: uuid.NullUUID{},
+	})
+	require.NoError(t, err)
+
+	prop, err := testQueries.UpsertPropertyValueV1(context.Background(), UpsertPropertyValueV1Params{
+		EntityID: ent.ID,
+		Key:      "testkey1",
+		Value:    rand.RandomName(seed),
+	})
+	require.NoError(t, err)
+	require.NotEmpty(t, prop)
+
+	prop, err = testQueries.UpsertPropertyValueV1(context.Background(), UpsertPropertyValueV1Params{
+		EntityID: ent.ID,
+		Key:      "testkey1",
+		Value:    rand.RandomName(seed),
+	})
+	require.NoError(t, err)
+	require.NotEmpty(t, prop)
+
+	prop, err = testQueries.UpsertPropertyValueV1(context.Background(), UpsertPropertyValueV1Params{
+		EntityID: ent.ID,
+		Key:      "upstream_id",
+		Value:    rand.RandomName(seed),
+	})
+	require.NoError(t, err)
+	require.NotEmpty(t, prop)
+}
+
 func createRandomProject(t *testing.T, orgID uuid.UUID) Project {
 	t.Helper()
 

internal/db/store.go

@@ -31,6 +31,9 @@ type ExtendQuerier interface {
 	UpsertPropertyValueV1(ctx context.Context, params UpsertPropertyValueV1Params) (Property, error)
 	GetPropertyValueV1(ctx context.Context, entityID uuid.UUID, key string) (PropertyValueV1, error)
 	GetAllPropertyValuesV1(ctx context.Context, entityID uuid.UUID) ([]PropertyValueV1, error)
+	GetTypedEntitiesByPropertyV1(
+		ctx context.Context, project uuid.UUID, entType Entities, key string, value any,
+	) ([]EntityInstance, error)
 }
 
 // Store provides all functions to execute db queries and transactions