From 53da19b6cf42e8bec1c1c0f5e87ba8ea30d40708 Mon Sep 17 00:00:00 2001 From: Artem Poltorzhitskiy Date: Mon, 13 Jan 2025 17:26:01 +0100 Subject: [PATCH] Feature: add rollup verification (#329) * Feature: add rollup verification * Fix: change verification status * Add admin middleware * Add some useful enpoints --- cmd/api/admin_middleware.go | 34 +++++++ cmd/api/handler/error.go | 1 + cmd/api/handler/rollup_auth.go | 94 ++++++++++++++++--- cmd/api/handler/validators.go | 25 +++++ cmd/api/handler/validators_test.go | 94 +++++++++++++++++++ cmd/api/init.go | 10 +- cmd/api/routes_test.go | 2 + database/views/22_leaderboard.sql | 2 + internal/storage/api_key.go | 27 ++++++ internal/storage/generic.go | 1 + internal/storage/mock/api_key.go | 83 ++++++++++++++++ internal/storage/mock/rollup.go | 39 ++++++++ internal/storage/postgres/apikey.go | 29 ++++++ internal/storage/postgres/apikey_test.go | 27 ++++++ internal/storage/postgres/core.go | 2 + .../20250113_rollup_verified.up.sql | 17 ++++ internal/storage/postgres/rollup.go | 17 +++- internal/storage/postgres/rollup_test.go | 20 +++- internal/storage/postgres/transaction.go | 4 +- internal/storage/rollup.go | 2 + test/data/apikey.yml | 2 + test/data/rollup.yml | 17 ++++ test/data/rollup_provider.yml | 3 + 23 files changed, 530 insertions(+), 22 deletions(-) create mode 100644 cmd/api/admin_middleware.go create mode 100644 internal/storage/api_key.go create mode 100644 internal/storage/mock/api_key.go create mode 100644 internal/storage/postgres/apikey.go create mode 100644 internal/storage/postgres/apikey_test.go create mode 100644 internal/storage/postgres/migrations/20250113_rollup_verified.up.sql create mode 100644 test/data/apikey.yml diff --git a/cmd/api/admin_middleware.go b/cmd/api/admin_middleware.go new file mode 100644 index 00000000..1aa74860 --- /dev/null +++ b/cmd/api/admin_middleware.go @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: 2024 PK Lab AG +// SPDX-License-Identifier: MIT + +package main + +import ( + "net/http" + + "github.com/celenium-io/celestia-indexer/cmd/api/handler" + "github.com/celenium-io/celestia-indexer/internal/storage" + "github.com/labstack/echo/v4" +) + +var accessDeniedErr = echo.Map{ + "error": "access denied", +} + +func AdminMiddleware() echo.MiddlewareFunc { + return checkOnAdminPermission +} + +func checkOnAdminPermission(next echo.HandlerFunc) echo.HandlerFunc { + return func(ctx echo.Context) error { + val := ctx.Get(handler.ApiKeyName) + apiKey, ok := val.(storage.ApiKey) + if !ok { + return ctx.JSON(http.StatusForbidden, accessDeniedErr) + } + if !apiKey.Admin { + return ctx.JSON(http.StatusForbidden, accessDeniedErr) + } + return next(ctx) + } +} diff --git a/cmd/api/handler/error.go b/cmd/api/handler/error.go index 6973ef10..b25e6527 100644 --- a/cmd/api/handler/error.go +++ b/cmd/api/handler/error.go @@ -16,6 +16,7 @@ var ( errInvalidHashLength = errors.New("invalid hash: should be 32 bytes length") errInvalidAddress = errors.New("invalid address") errUnknownAddress = errors.New("unknown address") + errInvalidApiKey = errors.New("invalid api key") errCancelRequest = "pq: canceling statement due to user request" ) diff --git a/cmd/api/handler/rollup_auth.go b/cmd/api/handler/rollup_auth.go index 4a04fa72..8682c4b2 100644 --- a/cmd/api/handler/rollup_auth.go +++ b/cmd/api/handler/rollup_auth.go @@ -6,7 +6,9 @@ package handler import ( "context" "encoding/base64" + "net/http" + "github.com/celenium-io/celestia-indexer/cmd/api/handler/responses" "github.com/celenium-io/celestia-indexer/internal/storage" "github.com/celenium-io/celestia-indexer/internal/storage/postgres" enums "github.com/celenium-io/celestia-indexer/internal/storage/types" @@ -43,7 +45,7 @@ type createRollupRequest struct { Website string `json:"website" validate:"omitempty,url"` GitHub string `json:"github" validate:"omitempty,url"` Twitter string `json:"twitter" validate:"omitempty,url"` - Logo string `json:"logo" validate:"omitempty,url"` + Logo string `json:"logo" validate:"required,url"` L2Beat string `json:"l2_beat" validate:"omitempty,url"` DeFiLama string `json:"defi_lama" validate:"omitempty"` Bridge string `json:"bridge" validate:"omitempty,eth_addr"` @@ -65,22 +67,31 @@ type rollupProvider struct { } func (handler RollupAuthHandler) Create(c echo.Context) error { + val := c.Get(ApiKeyName) + apiKey, ok := val.(storage.ApiKey) + if !ok { + return handleError(c, errInvalidApiKey, handler.address) + } + req, err := bindAndValidate[createRollupRequest](c) if err != nil { return badRequestError(c, err) } - if err := handler.createRollup(c.Request().Context(), req); err != nil { + rollupId, err := handler.createRollup(c.Request().Context(), req, apiKey.Admin) + if err != nil { return handleError(c, err, handler.rollups) } - return success(c) + return c.JSON(http.StatusOK, echo.Map{ + "id": rollupId, + }) } -func (handler RollupAuthHandler) createRollup(ctx context.Context, req *createRollupRequest) error { +func (handler RollupAuthHandler) createRollup(ctx context.Context, req *createRollupRequest, isAdmin bool) (uint64, error) { tx, err := postgres.BeginTransaction(ctx, handler.tx) if err != nil { - return err + return 0, err } rollup := storage.Rollup{ @@ -103,26 +114,30 @@ func (handler RollupAuthHandler) createRollup(ctx context.Context, req *createRo Category: enums.RollupCategory(req.Category), Slug: slug.Make(req.Name), Tags: req.Tags, + Verified: isAdmin, } if err := tx.SaveRollup(ctx, &rollup); err != nil { - return tx.HandleError(ctx, err) + return 0, tx.HandleError(ctx, err) } providers, err := handler.createProviders(ctx, rollup.Id, req.Providers...) if err != nil { - return tx.HandleError(ctx, err) + return 0, tx.HandleError(ctx, err) } if err := tx.SaveProviders(ctx, providers...); err != nil { - return tx.HandleError(ctx, err) + return 0, tx.HandleError(ctx, err) } if err := tx.RefreshLeaderboard(ctx); err != nil { - return tx.HandleError(ctx, err) + return 0, tx.HandleError(ctx, err) } - return tx.Flush(ctx) + if err := tx.Flush(ctx); err != nil { + return 0, err + } + return rollup.Id, nil } func (handler RollupAuthHandler) createProviders(ctx context.Context, rollupId uint64, data ...rollupProvider) ([]storage.RollupProvider, error) { @@ -178,19 +193,25 @@ type updateRollupRequest struct { } func (handler RollupAuthHandler) Update(c echo.Context) error { + val := c.Get(ApiKeyName) + apiKey, ok := val.(storage.ApiKey) + if !ok { + return handleError(c, errInvalidApiKey, handler.address) + } + req, err := bindAndValidate[updateRollupRequest](c) if err != nil { return badRequestError(c, err) } - if err := handler.updateRollup(c.Request().Context(), req); err != nil { + if err := handler.updateRollup(c.Request().Context(), req, apiKey.Admin); err != nil { return handleError(c, err, handler.rollups) } return success(c) } -func (handler RollupAuthHandler) updateRollup(ctx context.Context, req *updateRollupRequest) error { +func (handler RollupAuthHandler) updateRollup(ctx context.Context, req *updateRollupRequest, isAdmin bool) error { tx, err := postgres.BeginTransaction(ctx, handler.tx) if err != nil { return err @@ -221,6 +242,7 @@ func (handler RollupAuthHandler) updateRollup(ctx context.Context, req *updateRo Category: enums.RollupCategory(req.Category), Links: req.Links, Tags: req.Tags, + Verified: isAdmin, } if err := tx.UpdateRollup(ctx, &rollup); err != nil { @@ -282,3 +304,51 @@ func (handler RollupAuthHandler) deleteRollup(ctx context.Context, id uint64) er return tx.Flush(ctx) } + +func (handler RollupAuthHandler) Unverified(c echo.Context) error { + rollups, err := handler.rollups.Unverified(c.Request().Context()) + if err != nil { + return handleError(c, err, handler.rollups) + } + + response := make([]responses.Rollup, len(rollups)) + for i := range rollups { + response[i] = responses.NewRollup(&rollups[i]) + } + + return returnArray(c, response) +} + +type verifyRollupRequest struct { + Id uint64 `param:"id" validate:"required,min=1"` +} + +func (handler RollupAuthHandler) Verify(c echo.Context) error { + req, err := bindAndValidate[verifyRollupRequest](c) + if err != nil { + return badRequestError(c, err) + } + + if err := handler.verify(c.Request().Context(), req.Id); err != nil { + return handleError(c, err, handler.address) + } + + return success(c) +} + +func (handler RollupAuthHandler) verify(ctx context.Context, id uint64) error { + tx, err := postgres.BeginTransaction(ctx, handler.tx) + if err != nil { + return err + } + + err = tx.UpdateRollup(ctx, &storage.Rollup{ + Id: id, + Verified: true, + }) + if err != nil { + return tx.HandleError(ctx, err) + } + + return tx.Flush(ctx) +} diff --git a/cmd/api/handler/validators.go b/cmd/api/handler/validators.go index 5c592292..d7b9e4fd 100644 --- a/cmd/api/handler/validators.go +++ b/cmd/api/handler/validators.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "net/http" + "github.com/celenium-io/celestia-indexer/internal/storage" "github.com/celenium-io/celestia-indexer/internal/storage/types" pkgTypes "github.com/celenium-io/celestia-indexer/pkg/types" "github.com/cosmos/cosmos-sdk/types/bech32" @@ -119,3 +120,27 @@ func typeValidator() validator.Func { return err == nil } } + +type KeyValidator struct { + apiKeys storage.IApiKey + errChecker NoRows +} + +func NewKeyValidator(apiKeys storage.IApiKey, errChecker NoRows) KeyValidator { + return KeyValidator{apiKeys: apiKeys, errChecker: errChecker} +} + +const ApiKeyName = "api_key" + +func (kv KeyValidator) Validate(key string, c echo.Context) (bool, error) { + apiKey, err := kv.apiKeys.Get(c.Request().Context(), key) + if err != nil { + if kv.errChecker.IsNoRows(err) { + return false, nil + } + return false, err + } + c.Logger().Infof("using apikey: %s", apiKey.Description) + c.Set(ApiKeyName, apiKey) + return true, nil +} diff --git a/cmd/api/handler/validators_test.go b/cmd/api/handler/validators_test.go index b3f08be4..9f05fb1e 100644 --- a/cmd/api/handler/validators_test.go +++ b/cmd/api/handler/validators_test.go @@ -4,9 +4,17 @@ package handler import ( + "database/sql" + "net/http" + "net/http/httptest" "testing" + "github.com/celenium-io/celestia-indexer/internal/storage" + "github.com/celenium-io/celestia-indexer/internal/storage/mock" + "github.com/labstack/echo/v4" + "github.com/pkg/errors" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" ) func Test_isAddress(t *testing.T) { @@ -72,3 +80,89 @@ func Test_isValoperAddress(t *testing.T) { }) } } + +func TestKeyValidator_Validate(t *testing.T) { + t.Run("valid key", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + e := echo.New() + ctx := e.NewContext(req, rec) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + errChecker := mock.NewMockIDelegation(ctrl) + apiKeys := mock.NewMockIApiKey(ctrl) + kv := NewKeyValidator(apiKeys, errChecker) + + apiKeys.EXPECT(). + Get(gomock.Any(), "valid"). + Return(storage.ApiKey{ + Key: "valid", + Description: "descr", + }, nil). + Times(1) + + ok, err := kv.Validate("valid", ctx) + require.NoError(t, err) + require.True(t, ok) + }) + + t.Run("invalid key", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + e := echo.New() + ctx := e.NewContext(req, rec) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + errChecker := mock.NewMockIDelegation(ctrl) + apiKeys := mock.NewMockIApiKey(ctrl) + kv := NewKeyValidator(apiKeys, errChecker) + + apiKeys.EXPECT(). + Get(gomock.Any(), "invalid"). + Return(storage.ApiKey{}, sql.ErrNoRows). + Times(1) + + errChecker.EXPECT(). + IsNoRows(sql.ErrNoRows). + Return(true). + Times(1) + + ok, err := kv.Validate("invalid", ctx) + require.NoError(t, err) + require.False(t, ok) + }) + + t.Run("unexpected error", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + e := echo.New() + ctx := e.NewContext(req, rec) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + errChecker := mock.NewMockIDelegation(ctrl) + apiKeys := mock.NewMockIApiKey(ctrl) + kv := NewKeyValidator(apiKeys, errChecker) + + unexpectedErr := errors.New("unexpected") + + apiKeys.EXPECT(). + Get(gomock.Any(), "invalid"). + Return(storage.ApiKey{}, unexpectedErr). + Times(1) + + errChecker.EXPECT(). + IsNoRows(unexpectedErr). + Return(false). + Times(1) + + ok, err := kv.Validate("invalid", ctx) + require.Error(t, err) + require.False(t, ok) + }) +} diff --git a/cmd/api/init.go b/cmd/api/init.go index 7e0a7f5d..d710f7a7 100644 --- a/cmd/api/init.go +++ b/cmd/api/init.go @@ -497,19 +497,21 @@ func initHandlers(ctx context.Context, e *echo.Echo, cfg Config, db postgres.Sto auth := v1.Group("/auth") { + keyValidator := handler.NewKeyValidator(db.ApiKeys, db.BlobLogs) keyMiddleware := middleware.KeyAuthWithConfig(middleware.KeyAuthConfig{ KeyLookup: "header:Authorization", - Validator: func(key string, c echo.Context) (bool, error) { - return key == os.Getenv("API_AUTH_KEY"), nil - }, + Validator: keyValidator.Validate, }) + adminMiddleware := AdminMiddleware() rollupAuthHandler := handler.NewRollupAuthHandler(db.Rollup, db.Address, db.Namespace, db.Transactable) rollup := auth.Group("/rollup") { rollup.POST("/new", rollupAuthHandler.Create, keyMiddleware) rollup.PATCH("/:id", rollupAuthHandler.Update, keyMiddleware) - rollup.DELETE("/:id", rollupAuthHandler.Delete, keyMiddleware) + rollup.DELETE("/:id", rollupAuthHandler.Delete, keyMiddleware, adminMiddleware) + rollup.PATCH("/:id/verify", rollupAuthHandler.Verify, keyMiddleware, adminMiddleware) + rollup.GET("/unverified", rollupAuthHandler.Unverified, keyMiddleware, adminMiddleware) } } diff --git a/cmd/api/routes_test.go b/cmd/api/routes_test.go index 617d95e2..52ed74f8 100644 --- a/cmd/api/routes_test.go +++ b/cmd/api/routes_test.go @@ -53,6 +53,8 @@ func TestRoutes(t *testing.T) { "/v1/stats/changes_24h GET": {}, "/v1/rollup/count GET": {}, "/v1/auth/rollup/:id PATCH": {}, + "/v1/auth/rollup/:id/verify PATCH": {}, + "/v1/auth/rollup/unverified GET": {}, "/v1/address/:hash/undelegations GET": {}, "/v1/block/:height/messages GET": {}, "/v1/namespace/active GET": {}, diff --git a/database/views/22_leaderboard.sql b/database/views/22_leaderboard.sql index f2396030..daa7c0ef 100644 --- a/database/views/22_leaderboard.sql +++ b/database/views/22_leaderboard.sql @@ -20,6 +20,8 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS leaderboard AS group by 1, 2 ) as agg inner join rollup_provider as rp on rp.address_id = agg.signer_id AND (rp.namespace_id = agg.namespace_id OR rp.namespace_id = 0) + inner join rollup on rollup.id = rp.rollup_id + where rollup.verified = TRUE group by 1 ) select diff --git a/internal/storage/api_key.go b/internal/storage/api_key.go new file mode 100644 index 00000000..a5424a1b --- /dev/null +++ b/internal/storage/api_key.go @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2024 PK Lab AG +// SPDX-License-Identifier: MIT + +package storage + +import ( + "context" + + "github.com/uptrace/bun" +) + +//go:generate mockgen -source=$GOFILE -destination=mock/$GOFILE -package=mock -typed +type IApiKey interface { + Get(ctx context.Context, key string) (ApiKey, error) +} + +type ApiKey struct { + bun.BaseModel `bun:"apikey" comment:"Table with private api keys"` + + Key string `bun:"key,pk,notnull" comment:"Key"` + Description string `bun:"description" comment:"Additional info about issuer and user"` + Admin bool `bun:"admin" comment:"Verified user"` +} + +func (ApiKey) TableName() string { + return "apikey" +} diff --git a/internal/storage/generic.go b/internal/storage/generic.go index a8f826de..c8f63ac9 100644 --- a/internal/storage/generic.go +++ b/internal/storage/generic.go @@ -43,6 +43,7 @@ var Models = []any{ &Rollup{}, &RollupProvider{}, &Grant{}, + &ApiKey{}, } //go:generate mockgen -source=$GOFILE -destination=mock/$GOFILE -package=mock -typed diff --git a/internal/storage/mock/api_key.go b/internal/storage/mock/api_key.go new file mode 100644 index 00000000..c33ba3b7 --- /dev/null +++ b/internal/storage/mock/api_key.go @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: 2024 PK Lab AG +// SPDX-License-Identifier: MIT + +// Code generated by MockGen. DO NOT EDIT. +// Source: api_key.go +// +// Generated by this command: +// +// mockgen -source=api_key.go -destination=mock/api_key.go -package=mock -typed +// + +// Package mock is a generated GoMock package. +package mock + +import ( + context "context" + reflect "reflect" + + storage "github.com/celenium-io/celestia-indexer/internal/storage" + gomock "go.uber.org/mock/gomock" +) + +// MockIApiKey is a mock of IApiKey interface. +type MockIApiKey struct { + ctrl *gomock.Controller + recorder *MockIApiKeyMockRecorder +} + +// MockIApiKeyMockRecorder is the mock recorder for MockIApiKey. +type MockIApiKeyMockRecorder struct { + mock *MockIApiKey +} + +// NewMockIApiKey creates a new mock instance. +func NewMockIApiKey(ctrl *gomock.Controller) *MockIApiKey { + mock := &MockIApiKey{ctrl: ctrl} + mock.recorder = &MockIApiKeyMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockIApiKey) EXPECT() *MockIApiKeyMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockIApiKey) Get(ctx context.Context, key string) (storage.ApiKey, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", ctx, key) + ret0, _ := ret[0].(storage.ApiKey) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockIApiKeyMockRecorder) Get(ctx, key any) *MockIApiKeyGetCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockIApiKey)(nil).Get), ctx, key) + return &MockIApiKeyGetCall{Call: call} +} + +// MockIApiKeyGetCall wrap *gomock.Call +type MockIApiKeyGetCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockIApiKeyGetCall) Return(arg0 storage.ApiKey, arg1 error) *MockIApiKeyGetCall { + c.Call = c.Call.Return(arg0, arg1) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockIApiKeyGetCall) Do(f func(context.Context, string) (storage.ApiKey, error)) *MockIApiKeyGetCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockIApiKeyGetCall) DoAndReturn(f func(context.Context, string) (storage.ApiKey, error)) *MockIApiKeyGetCall { + c.Call = c.Call.DoAndReturn(f) + return c +} diff --git a/internal/storage/mock/rollup.go b/internal/storage/mock/rollup.go index f2db2567..5e4c6f65 100644 --- a/internal/storage/mock/rollup.go +++ b/internal/storage/mock/rollup.go @@ -783,6 +783,45 @@ func (c *MockIRollupTagsCall) DoAndReturn(f func(context.Context) ([]string, err return c } +// Unverified mocks base method. +func (m *MockIRollup) Unverified(ctx context.Context) ([]storage.Rollup, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Unverified", ctx) + ret0, _ := ret[0].([]storage.Rollup) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Unverified indicates an expected call of Unverified. +func (mr *MockIRollupMockRecorder) Unverified(ctx any) *MockIRollupUnverifiedCall { + mr.mock.ctrl.T.Helper() + call := mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unverified", reflect.TypeOf((*MockIRollup)(nil).Unverified), ctx) + return &MockIRollupUnverifiedCall{Call: call} +} + +// MockIRollupUnverifiedCall wrap *gomock.Call +type MockIRollupUnverifiedCall struct { + *gomock.Call +} + +// Return rewrite *gomock.Call.Return +func (c *MockIRollupUnverifiedCall) Return(rollups []storage.Rollup, err error) *MockIRollupUnverifiedCall { + c.Call = c.Call.Return(rollups, err) + return c +} + +// Do rewrite *gomock.Call.Do +func (c *MockIRollupUnverifiedCall) Do(f func(context.Context) ([]storage.Rollup, error)) *MockIRollupUnverifiedCall { + c.Call = c.Call.Do(f) + return c +} + +// DoAndReturn rewrite *gomock.Call.DoAndReturn +func (c *MockIRollupUnverifiedCall) DoAndReturn(f func(context.Context) ([]storage.Rollup, error)) *MockIRollupUnverifiedCall { + c.Call = c.Call.DoAndReturn(f) + return c +} + // Update mocks base method. func (m_2 *MockIRollup) Update(ctx context.Context, m *storage.Rollup) error { m_2.ctrl.T.Helper() diff --git a/internal/storage/postgres/apikey.go b/internal/storage/postgres/apikey.go new file mode 100644 index 00000000..166093d3 --- /dev/null +++ b/internal/storage/postgres/apikey.go @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: 2024 PK Lab AG +// SPDX-License-Identifier: MIT + +package postgres + +import ( + "context" + + "github.com/celenium-io/celestia-indexer/internal/storage" + "github.com/dipdup-net/go-lib/database" +) + +// ApiKey - +type ApiKey struct { + db *database.Bun +} + +// NewApiKey - +func NewApiKey(db *database.Bun) *ApiKey { + return &ApiKey{ + db: db, + } +} + +func (ak *ApiKey) Get(ctx context.Context, key string) (apikey storage.ApiKey, err error) { + apikey.Key = key + err = ak.db.DB().NewSelect().Model(&apikey).WherePK().Scan(ctx) + return +} diff --git a/internal/storage/postgres/apikey_test.go b/internal/storage/postgres/apikey_test.go new file mode 100644 index 00000000..d06f5bdd --- /dev/null +++ b/internal/storage/postgres/apikey_test.go @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2024 PK Lab AG +// SPDX-License-Identifier: MIT + +package postgres + +import ( + "context" + "time" +) + +func (s *StorageTestSuite) TestApiKeyValid() { + ctx, ctxCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer ctxCancel() + + key, err := s.storage.ApiKeys.Get(ctx, "test_key") + s.Require().NoError(err) + s.Require().EqualValues("test_key", key.Key) + s.Require().EqualValues("valid key", key.Description) +} + +func (s *StorageTestSuite) TestApiKeyInvalid() { + ctx, ctxCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer ctxCancel() + + _, err := s.storage.ApiKeys.Get(ctx, "invalid") + s.Require().Error(err) +} diff --git a/internal/storage/postgres/core.go b/internal/storage/postgres/core.go index 53beae1d..87d9f47a 100644 --- a/internal/storage/postgres/core.go +++ b/internal/storage/postgres/core.go @@ -50,6 +50,7 @@ type Storage struct { Jails models.IJail Rollup models.IRollup Grants models.IGrant + ApiKeys models.IApiKey Notificator *Notificator export models.Export @@ -97,6 +98,7 @@ func Create(ctx context.Context, cfg config.Database, scriptsDir string, withMig Jails: NewJail(strg.Connection()), Rollup: NewRollup(strg.Connection()), Grants: NewGrant(strg.Connection()), + ApiKeys: NewApiKey(strg.Connection()), Notificator: NewNotificator(cfg, strg.Connection().DB()), export: export, diff --git a/internal/storage/postgres/migrations/20250113_rollup_verified.up.sql b/internal/storage/postgres/migrations/20250113_rollup_verified.up.sql new file mode 100644 index 00000000..ebc4c44a --- /dev/null +++ b/internal/storage/postgres/migrations/20250113_rollup_verified.up.sql @@ -0,0 +1,17 @@ +ALTER TABLE public."rollup" ADD COLUMN IF NOT EXISTS verified bool NULL; + +--bun:split + +COMMENT ON COLUMN public."rollup".verified IS 'Flag is set when rollup was approved'; + +--bun:split + +UPDATE rollup set verified = TRUE where id > 0; + +--bun:split + +REFRESH MATERIALIZED VIEW leaderboard; + +--bun:split + +REFRESH MATERIALIZED VIEW leaderboard_day; \ No newline at end of file diff --git a/internal/storage/postgres/rollup.go b/internal/storage/postgres/rollup.go index b446abb8..2d25a541 100644 --- a/internal/storage/postgres/rollup.go +++ b/internal/storage/postgres/rollup.go @@ -81,7 +81,7 @@ func (r *Rollup) LeaderboardDay(ctx context.Context, fltrs storage.LeaderboardFi Column("avg_size", blobsCountColumn, "total_size", "total_fee", "throughput", "namespace_count", "pfb_count", "mb_price"). ColumnExpr("rollup.*"). Offset(fltrs.Offset). - Join("left join rollup on rollup.id = rollup_id") + Join("left join rollup on rollup.id = rollup_id AND rollup.verified = true") if len(fltrs.Category) > 0 { query = query.Where("category IN (?)", bun.In(fltrs.Category)) @@ -143,7 +143,7 @@ func (r *Rollup) RollupsByNamespace(ctx context.Context, namespaceId uint64, lim With("rollups", subQuery). Table("rollups"). ColumnExpr("rollup.*"). - Join("left join rollup on rollup.id = rollups.rollup_id"). + Join("left join rollup on rollup.id = rollups.rollup_id and rollup.verified = true"). Scan(ctx, &rollups) return } @@ -213,7 +213,7 @@ func (r *Rollup) Series(ctx context.Context, rollupId uint64, timeframe, column } func (r *Rollup) Count(ctx context.Context) (int64, error) { - count, err := r.DB().NewSelect().Model((*storage.Rollup)(nil)).Count(ctx) + count, err := r.DB().NewSelect().Model((*storage.Rollup)(nil)).Where("verified = TRUE").Count(ctx) return int64(count), err } @@ -313,7 +313,7 @@ func (r *Rollup) AllSeries(ctx context.Context) (items []storage.RollupHistogram err = r.DB().NewSelect(). TableExpr("(?) as series", subQuery). ColumnExpr("series.time as time, series.size as size, series.blobs_count as blobs_count, series.fee as fee, rollup.name as name, rollup.logo as logo"). - Join("left join rollup on rollup.id = series.rollup_id"). + Join("left join rollup on rollup.id = series.rollup_id and rollup.verified = true"). Scan(ctx, &items) return @@ -353,6 +353,15 @@ func (r *Rollup) Tags(ctx context.Context) (arr []string, err error) { Model((*storage.Rollup)(nil)). Distinct(). ColumnExpr("unnest(tags)"). + Where("verified = true"). Scan(ctx, &arr) return } + +func (r *Rollup) Unverified(ctx context.Context) (rollups []storage.Rollup, err error) { + err = r.DB().NewSelect(). + Model(&rollups). + Where("verified = false"). + Scan(ctx) + return +} diff --git a/internal/storage/postgres/rollup_test.go b/internal/storage/postgres/rollup_test.go index add46a0b..c6698217 100644 --- a/internal/storage/postgres/rollup_test.go +++ b/internal/storage/postgres/rollup_test.go @@ -299,7 +299,7 @@ func (s *StorageTestSuite) TestRollupAllSeries() { items, err := s.storage.Rollup.AllSeries(ctx) s.Require().NoError(err) - s.Require().Len(items, 5) + s.Require().Len(items, 7) } func (s *StorageTestSuite) TestRollupStatsGrouping() { @@ -334,3 +334,21 @@ func (s *StorageTestSuite) TestRollupTags() { s.Require().Contains(tags, "zk") s.Require().Contains(tags, "ai") } + +func (s *StorageTestSuite) TestCount() { + ctx, ctxCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer ctxCancel() + + count, err := s.storage.Rollup.Count(ctx) + s.Require().NoError(err) + s.Require().EqualValues(3, count) +} + +func (s *StorageTestSuite) TestUnverified() { + ctx, ctxCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer ctxCancel() + + rollups, err := s.storage.Rollup.Unverified(ctx) + s.Require().NoError(err) + s.Require().EqualValues(1, len(rollups)) +} diff --git a/internal/storage/postgres/transaction.go b/internal/storage/postgres/transaction.go index 7f1fd79e..f58e3a66 100644 --- a/internal/storage/postgres/transaction.go +++ b/internal/storage/postgres/transaction.go @@ -602,7 +602,7 @@ func (tx Transaction) SaveRollup(ctx context.Context, rollup *models.Rollup) err } func (tx Transaction) UpdateRollup(ctx context.Context, rollup *models.Rollup) error { - if rollup == nil || rollup.IsEmpty() { + if rollup == nil || (rollup.IsEmpty() && !rollup.Verified) { return nil } @@ -666,6 +666,8 @@ func (tx Transaction) UpdateRollup(ctx context.Context, rollup *models.Rollup) e query = query.Set("defi_lama = ?", rollup.DeFiLama) } + query = query.Set("verified = ?", rollup.Verified) + _, err := query.Exec(ctx) return err } diff --git a/internal/storage/rollup.go b/internal/storage/rollup.go index 8e70e9c2..cb7b3c58 100644 --- a/internal/storage/rollup.go +++ b/internal/storage/rollup.go @@ -45,6 +45,7 @@ type IRollup interface { BySlug(ctx context.Context, slug string) (RollupWithStats, error) RollupStatsGrouping(ctx context.Context, fltrs RollupGroupStatsFilters) ([]RollupGroupedStats, error) Tags(ctx context.Context) ([]string, error) + Unverified(ctx context.Context) (rollups []Rollup, err error) } // Rollup - @@ -71,6 +72,7 @@ type Rollup struct { Tags []string `bun:"tags,array"` VM string `bun:"vm" comment:"Virtual machine"` Links []string `bun:"links,array" comment:"Other links to rollup related sites"` + Verified bool `bun:"verified"` Providers []*RollupProvider `bun:"rel:has-many,join:id=rollup_id"` } diff --git a/test/data/apikey.yml b/test/data/apikey.yml new file mode 100644 index 00000000..1fa5b9d6 --- /dev/null +++ b/test/data/apikey.yml @@ -0,0 +1,2 @@ +- key: test_key + description: valid key \ No newline at end of file diff --git a/test/data/rollup.yml b/test/data/rollup.yml index 926423be..a3c4afca 100644 --- a/test/data/rollup.yml +++ b/test/data/rollup.yml @@ -12,6 +12,7 @@ defi_lama: defi links: RAW='{https://rollup1.com}' category: finance + verified: true - id: 2 name: Rollup 2 description: The second @@ -25,6 +26,7 @@ type: settled defi_lama: lama category: gaming + verified: true - id: 3 name: Rollup 3 description: The third @@ -38,3 +40,18 @@ type: sovereign defi_lama: name category: nft + verified: true +- id: 4 + name: Hidden Rollup + description: The fourth + website: https://rollup4.com + twitter: https://x.com/rollup4 + github: https://github.com/rollup4 + logo: https://rollup4.com/image.png + slug: rollup_4 + stack: Custom Stack + type: sovereign + defi_lama: name + category: nft + verified: false + tags: RAW='{hidden}' diff --git a/test/data/rollup_provider.yml b/test/data/rollup_provider.yml index 2e232080..d3754128 100644 --- a/test/data/rollup_provider.yml +++ b/test/data/rollup_provider.yml @@ -8,5 +8,8 @@ namespace_id: 2 address_id: 2 - rollup_id: 3 + namespace_id: 0 + address_id: 2 +- rollup_id: 4 namespace_id: 0 address_id: 2 \ No newline at end of file