Add central entities table with properties (#4123)

* Add central entities table with properties

Signed-off-by: Juan Antonio Osorio <[email protected]>

* Add entity population to migration command

This way it runs every time minder updates

Signed-off-by: Juan Antonio Osorio <[email protected]>

* Create repositories on entities table as well

Signed-off-by: Juan Antonio Osorio <[email protected]>

---------

Signed-off-by: Juan Antonio Osorio <[email protected]>

cmd/server/app/migrate_up.go

@@ -32,6 +32,7 @@ import (
 	"github.com/stacklok/minder/internal/authz"
 	"github.com/stacklok/minder/internal/config"
 	serverconfig "github.com/stacklok/minder/internal/config/server"
+	"github.com/stacklok/minder/internal/db"
 	"github.com/stacklok/minder/internal/logger"
 )
 
@@ -111,6 +112,21 @@ var upCmd = &cobra.Command{
 			return fmt.Errorf("error preparing authz client: %w", err)
 		}
 
+		cmd.Println("Performing entity migrations...")
+		store := db.NewStore(dbConn)
+
+		if err := store.TemporaryPopulateRepositories(ctx); err != nil {
+			cmd.Printf("Error while populating entities table with repos: %v\n", err)
+		}
+
+		if err := store.TemporaryPopulateArtifacts(ctx); err != nil {
+			cmd.Printf("Error while populating entities table with artifacts: %v\n", err)
+		}
+
+		if err := store.TemporaryPopulatePullRequests(ctx); err != nil {
+			cmd.Printf("Error while populating entities table with pull requests: %v\n", err)
+		}
+
 		return nil
 	},
 }

database/migrations/000090_entity_properties.down.sql

@@ -0,0 +1,20 @@
+-- Copyright 2024 Stacklok, Inc
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+BEGIN;
+
+DROP TABLE IF EXISTS properties;
+DROP TABLE IF EXISTS entities;
+
+COMMIT;

database/migrations/000090_entity_properties.up.sql

@@ -0,0 +1,37 @@
+-- Copyright 2024 Stacklok, Inc
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS entity_instances (
+	id          UUID NOT NULL DEFAULT gen_random_uuid() PRIMARY KEY,
+	entity_type entities NOT NULL, 
+	name        TEXT NOT NULL,
+	project_id  UUID NOT NULL REFERENCES projects(id),
+	provider_id UUID NOT NULL REFERENCES providers(id),
+	created_at  TIMESTAMP NOT NULL DEFAULT NOW(),
+	originated_from UUID REFERENCES entity_instances(id) ON DELETE CASCADE, -- this is for entities that originate from other entities
+	UNIQUE(project_id, provider_id, entity_type, name)
+);
+
+CREATE TABLE IF NOT EXISTS properties(
+	id          UUID PRIMARY KEY, -- surrogate ID
+	entity_id   UUID NOT NULL REFERENCES entity_instances(id) ON DELETE CASCADE,
+	key         TEXT NOT NULL, -- we need to validate and ensure there are no dots
+	value       JSONB NOT NULL,
+	updated_at  TIMESTAMP NOT NULL DEFAULT NOW(),
+	UNIQUE (entity_id, key)
+);
+
+COMMIT;

database/query/entities.sql

@@ -0,0 +1,66 @@
+-- CreateEntity adds an entry to the entity_instances table so it can be tracked by Minder.
+
+-- name: CreateEntity :one
+INSERT INTO entity_instances (
+    entity_type,
+    name,
+    project_id,
+    provider_id,
+    originated_from
+) VALUES ($1, $2, sqlc.arg(project_id), sqlc.arg(provider_id), sqlc.narg(originated_from))
+RETURNING *;
+
+-- CreateEntityWithID adds an entry to the entities table with a specific ID so it can be tracked by Minder.
+
+-- name: CreateEntityWithID :one
+INSERT INTO entity_instances (
+    id,
+    entity_type,
+    name,
+    project_id,
+    provider_id,
+    originated_from
+) VALUES ($1, $2, $3, sqlc.arg(project_id), sqlc.arg(provider_id), sqlc.narg(originated_from))
+RETURNING *;
+
+-- DeleteEntity removes an entity from the entity_instances table for a project.
+
+-- name: DeleteEntity :exec
+DELETE FROM entity_instances
+WHERE id = $1 AND project_id = $2;
+
+-- GetEntityByID retrieves an entity by its ID for a project or hierarchy of projects.
+
+-- name: GetEntityByID :one
+SELECT * FROM entity_instances
+WHERE entity_instances.id = $1 AND entity_instances.project_id = ANY(sqlc.arg(projects)::uuid[])
+LIMIT 1;
+
+-- GetEntityByName retrieves an entity by its name for a project or hierarchy of projects.
+SELECT * FROM entity_instances
+WHERE lower(entity_instances.name) = lower(sqlc.arg(name)) AND entity_instances.project_id = $1
+LIMIT 1;
+
+-- GetEntitiesByType retrieves all entities of a given type for a project or hierarchy of projects.
+-- this is how one would get all repositories, artifacts, etc.
+
+-- name: GetEntitiesByType :many
+SELECT * FROM entity_instances
+WHERE entity_instances.entity_type = $1 AND entity_instances.project_id = ANY(sqlc.arg(projects)::uuid[]);
+
+-- name: TemporaryPopulateRepositories :exec
+INSERT INTO entity_instances (id, entity_type, name, project_id, provider_id, created_at)
+SELECT id, 'repository', repo_owner || '/' || repo_name, project_id, provider_id, created_at FROM repositories
+WHERE NOT EXISTS (SELECT 1 FROM entity_instances WHERE entity_instances.id = repositories.id AND entity_instances.entity_type = 'repository');
+
+-- name: TemporaryPopulateArtifacts :exec
+INSERT INTO entity_instances (id, entity_type, name, project_id, provider_id, created_at, originated_from)
+SELECT artifacts.id, 'artifact', LOWER(repositories.repo_owner) || '/' || artifacts.artifact_name, repositories.project_id, repositories.provider_id, artifacts.created_at, artifacts.repository_id FROM artifacts
+JOIN repositories ON repositories.id = artifacts.repository_id
+WHERE NOT EXISTS (SELECT 1 FROM entity_instances WHERE entity_instances.id = artifacts.id AND entity_instances.entity_type = 'artifact');
+
+-- name: TemporaryPopulatePullRequests :exec
+INSERT INTO entity_instances (id, entity_type, name, project_id, provider_id, created_at, originated_from)
+SELECT pull_requests.id, 'pull_request', repositories.repo_owner || '/' || repositories.repo_name || '/' || pull_requests.pr_number::TEXT, repositories.project_id, repositories.provider_id, pull_requests.created_at, pull_requests.repository_id FROM pull_requests
+JOIN repositories ON repositories.id = pull_requests.repository_id
+WHERE NOT EXISTS (SELECT 1 FROM entity_instances WHERE entity_instances.id = pull_requests.id AND entity_instances.entity_type = 'pull_request');