chore(ci): improve redis usage in tests, add docker-test-redis command (

#2745)

* chore(ci): improve redis usage in tests, add docker-test-redis command
* Add TEST_DIRS argument
* add ratios cache test

Makefile

@@ -16,6 +16,13 @@ ifdef TEST_RUN
 	TEST_FLAGS = --tags=$(TEST_TAGS) $(TEST_PKG) --run=$(TEST_RUN)
 endif
 
+# Define default test directories if not specified
+TEST_DIRS?= libs services tools cmd
+# Allow specifying single directory via TEST_DIR=<dirname>
+ifdef TEST_DIR
+	TEST_DIRS = $(TEST_DIR)
+endif
+
 .PHONY: all buildcmd docker test create-json-schema lint clean download-mod
 all: test create-json-schema buildcmd
 
@@ -116,7 +123,14 @@ docker-test:
 	COMMIT=$(GIT_COMMIT) VERSION=$(GIT_VERSION) BUILD_TIME=$(BUILD_TIME) docker compose \
 		-f docker-compose.yml -f docker-compose.dev.yml up -d vault
 	$(eval VAULT_TOKEN = $(shell docker logs grant-vault 2>&1 | grep "Root Token" | tail -1 | cut -d ' ' -f 3 ))
-	VAULT_TOKEN=$(VAULT_TOKEN) PKG=$(TEST_PKG) RUN=$(TEST_RUN) docker compose -f docker-compose.yml -f docker-compose.dev.yml run --rm dev make test && cd main && go run main.go generate json-schema
+	VAULT_TOKEN=$(VAULT_TOKEN) TEST_DIRS="$(TEST_DIRS)" TEST_PKG=$(TEST_PKG) TEST_RUN=$(TEST_RUN) docker compose -f docker-compose.yml -f docker-compose.dev.yml run -T --rm dev make test
+
+docker-test-redis:
+	COMMIT=$(GIT_COMMIT) VERSION=$(GIT_VERSION) BUILD_TIME=$(BUILD_TIME) docker compose \
+		--profile redis-log -f docker-compose.yml -f docker-compose.dev.yml up -d redis --wait
+	docker exec grant-redis redis-cli MONITOR &
+	-make docker-test
+	docker compose -f docker-compose.yml -f docker-compose.dev.yml stop
 
 docker-dev:
 	$(eval VAULT_TOKEN = $(shell docker logs grant-vault 2>&1 | grep "Root Token" | tail -1 | cut -d ' ' -f 3 ))
@@ -170,10 +184,12 @@ create-json-schema:
 	cd main && go run main.go generate json-schema
 
 test:
-	cd libs && go test -count 1 -v -p 1 $(TEST_FLAGS) ./...
-	cd services && go test -count 1 -v -p 1 $(TEST_FLAGS) ./...
-	cd tools && go test -count 1 -v -p 1 $(TEST_FLAGS) ./...
-	cd cmd && go test -count 1 -v -p 1 $(TEST_FLAGS) ./...
+	@for dir in $(TEST_DIRS); do \
+		if [ -d "$$dir" ]; then \
+			echo "Testing $$dir..."; \
+			cd $$dir && go test -count 1 -v -p 1 $(TEST_FLAGS) && cd .. || exit 1; \
+		fi \
+	done
 
 format:
 	gofmt -s -w ./

docker-compose.dev.yml

@@ -1,6 +1,7 @@
 version: "3.4"
 
 volumes:
+  gomod:
   out:
     driver_opts:
       type: tmpfs
@@ -11,6 +12,7 @@ services:
     volumes:
       - ".:/src"
       - "out:/out"
+      - "gomod:/go/pkg/mod"
     security_opt:
       - no-new-privileges:true
     environment:
@@ -21,8 +23,20 @@ services:
       - VAULT_TOKEN
       - TEST_RUN
       - TEST_PKG
+      - TEST_DIRS
   vault:
     container_name: grant-vault
     image: vault:0.10.2
     networks:
       - grant
+  redislog:
+    container_name: grant-redis-log
+    image: redis
+    networks:
+      - grant
+    command: ["redis-cli", "-h", "grant-redis", "MONITOR"]
+    profiles:
+      - redis-log
+    depends_on:
+      redis:
+        condition: service_healthy

docker-compose.yml

@@ -162,11 +162,10 @@ services:
       - KAFKA_SSL_CERTIFICATE_LOCATION=/etc/kafka/secrets/consumer-ca1-signed.pem
       - KAFKA_SSL_KEY_LOCATION=/etc/kafka/secrets/consumer.client.key
       - OUTPUT_DIR="/out"
-      - PKG
       - PPROF_ENABLED=true
-      - RUN
       - TEST_PKG
       - TEST_RUN
+      - TEST_DIRS
       - TOKEN_LIST
       - UPHOLD_ACCESS_TOKEN
       - "DAPP_ALLOWED_CORS_ORIGINS=https://my-dapp.com"

libs/clients/coingecko/client_test.go

@@ -9,6 +9,7 @@ import (
 	"net/url"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/brave-intl/bat-go/libs/clients/coingecko"
 	appctx "github.com/brave-intl/bat-go/libs/context"
@@ -54,7 +55,7 @@ func (suite *CoingeckoTestSuite) SetupTest() {
 	// vs-currency limit
 	suite.ctx = context.WithValue(suite.ctx, appctx.CoingeckoVsCurrencyLimitCTXKey, coingeckoCurrencyLimit)
 
-	redisAddr := "redis://grant-redis:6379"
+	redisAddr := "redis://grant-redis:6379/1"
 	if len(os.Getenv("REDIS_ADDR")) > 0 {
 		redisAddr = os.Getenv("REDIS_ADDR")
 	}
@@ -71,6 +72,13 @@ func (suite *CoingeckoTestSuite) SetupTest() {
 	suite.Require().NoError(err, "Must be able to correctly initialize the client")
 }
 
+func (suite *CoingeckoTestSuite) TearDownTest() {
+	// flush all keys from the test Redis database
+	suite.Assert().NoError(suite.redis.FlushDB(suite.ctx).Err(), "Must be able to flush Redis database")
+	// work around Coingecko rate limit
+	time.Sleep(200 * time.Millisecond)
+}
+
 func (suite *CoingeckoTestSuite) TestFetchSimplePrice() {
 	resp, err := suite.client.FetchSimplePrice(suite.ctx, "basic-attention-token", "usd", false)
 	suite.Require().NoError(err, "should be able to fetch the simple price")

services/ratios/controllers_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/brave-intl/bat-go/libs/clients/stripe"
 	mockstripe "github.com/brave-intl/bat-go/libs/clients/stripe/mock"
 	appctx "github.com/brave-intl/bat-go/libs/context"
+	"github.com/brave-intl/bat-go/libs/inputs"
 	logutils "github.com/brave-intl/bat-go/libs/logging"
 	"github.com/brave-intl/bat-go/services/ratios"
 	"github.com/go-chi/chi"
@@ -36,6 +37,7 @@ type ControllersTestSuite struct {
 
 	ctx                 context.Context
 	service             *ratios.Service
+	redis               *redis.Client
 	mockCtrl            *gomock.Controller
 	mockCoingeckoClient *mockcoingecko.MockClient
 	mockStripeClient    *mockstripe.MockClient
@@ -66,12 +68,21 @@ func (suite *ControllersTestSuite) SetupSuite() {
 	// vs-currency limit
 	suite.ctx = context.WithValue(suite.ctx, appctx.CoingeckoVsCurrencyLimitCTXKey, 2)
 	// all this is setup in init service
-	suite.ctx = context.WithValue(suite.ctx, appctx.CoingeckoSymbolToIDCTXKey, map[string]string{"bat": "basic-attention-token"})
+	suite.ctx = context.WithValue(suite.ctx, appctx.CoingeckoSymbolToIDCTXKey, map[string]string{
+		"bat": "basic-attention-token",
+		"eth": "ethereum",
+	})
 	suite.ctx = context.WithValue(suite.ctx, appctx.CoingeckoContractToIDCTXKey, map[string]string{})
-	suite.ctx = context.WithValue(suite.ctx, appctx.CoingeckoIDToSymbolCTXKey, map[string]string{"basic-attention-token": "bat"})
-	suite.ctx = context.WithValue(suite.ctx, appctx.CoingeckoSupportedVsCurrenciesCTXKey, map[string]bool{"usd": true})
+	suite.ctx = context.WithValue(suite.ctx, appctx.CoingeckoIDToSymbolCTXKey, map[string]string{
+		"basic-attention-token": "bat",
+		"ethereum":              "eth",
+	})
+	suite.ctx = context.WithValue(suite.ctx, appctx.CoingeckoSupportedVsCurrenciesCTXKey, map[string]bool{
+		"usd": true,
+		"eur": true,
+	})
 
-	var redisAddr string = "redis://grant-redis:6379"
+	var redisAddr string = "redis://grant-redis:6379/2"
 	if len(os.Getenv("REDIS_ADDR")) > 0 {
 		redisAddr = os.Getenv("REDIS_ADDR")
 	}
@@ -90,9 +101,9 @@ func (suite *ControllersTestSuite) BeforeTest(sn, tn string) {
 	opts, err := redis.ParseURL(redisAddr)
 	suite.Require().NoError(err, "Must be able to parse redis URL")
 
-	redis := redis.NewClient(opts)
+	suite.redis = redis.NewClient(opts)
 
-	if err := redis.Ping(suite.ctx).Err(); err != nil {
+	if err := suite.redis.Ping(suite.ctx).Err(); err != nil {
 		suite.Require().NoError(err, "Must be able to ping redis")
 	}
 
@@ -102,14 +113,19 @@ func (suite *ControllersTestSuite) BeforeTest(sn, tn string) {
 	stripe := mockstripe.NewMockClient(suite.mockCtrl)
 	suite.mockStripeClient = stripe
 
-	suite.service = ratios.NewService(suite.ctx, coingecko, stripe, redis)
+	suite.service = ratios.NewService(suite.ctx, coingecko, stripe, suite.redis)
 	suite.Require().NoError(err, "failed to setup ratios service")
 }
 
 func (suite *ControllersTestSuite) AfterTest(sn, tn string) {
 	suite.mockCtrl.Finish()
 }
 
+func (suite *ControllersTestSuite) TearDownTest() {
+	// flush all keys from the test Redis database
+	suite.Assert().NoError(suite.redis.FlushDB(suite.ctx).Err(), "Must be able to flush Redis database")
+}
+
 func (suite *ControllersTestSuite) TestGetHistoryHandler() {
 	handler := ratios.GetHistoryHandler(suite.service)
 	req, err := http.NewRequest("GET", "/v2/history/coingecko/{coinID}/{vsCurrency}/{duration}", nil)
@@ -618,3 +634,78 @@ func (suite *ControllersTestSuite) TestCreateStripeOnrampSessionsHandler() {
 		suite.Require().Equal(ratiosResp.URL, "https://example.com")
 	}
 }
+
+func (suite *ControllersTestSuite) TestCacheOperations() {
+	// Initialize coins
+	var coinList ratios.CoingeckoCoinList
+	err := inputs.DecodeAndValidate(suite.ctx, &coinList, []byte("bat,eth"))
+	suite.Require().NoError(err)
+
+	// Test RecordCoinsAndCurrencies
+	err = suite.service.RecordCoinsAndCurrencies(
+		suite.ctx,
+		[]ratios.CoingeckoCoin(coinList),
+		[]ratios.CoingeckoVsCurrency{"usd", "eur"},
+	)
+	suite.Require().NoError(err, "Should record coins and currencies without error")
+
+	// Test GetTopCoins
+	topCoins, err := suite.service.GetTopCoins(suite.ctx, 10)
+	suite.Require().NoError(err, "Should get top coins without error")
+	suite.Require().Len(topCoins, 2, "Should have exactly 2 top coins (BAT, ETH)")
+	suite.Require().Contains(topCoins.String(), "basic-attention-token", "Should contain BAT in top coins")
+	suite.Require().Contains(topCoins.String(), "ethereum", "Should contain ETH in top coins")
+
+	// Test GetTopCurrencies
+	topCurrencies, err := suite.service.GetTopCurrencies(suite.ctx, 10)
+	suite.Require().NoError(err, "Should get top currencies without error")
+	suite.Require().Len(topCurrencies, 2, "Should have exactly 2 top currencies (USD, EUR)")
+	suite.Require().Contains(topCurrencies.String(), "usd", "Should contain USD in top currencies")
+	suite.Require().Contains(topCurrencies.String(), "eur", "Should contain EUR in top currencies")
+
+	// Test RunNextRelativeCachePrepopulationJob
+	// Setup mock response for FetchSimplePrice
+	mockResp := coingecko.SimplePriceResponse(map[string]map[string]decimal.Decimal{
+		"basic-attention-token": {
+			"usd":            decimal.NewFromFloat(0.25),
+			"usd_24h_change": decimal.NewFromFloat(5.25),
+		},
+		"ethereum": {
+			"usd":            decimal.NewFromFloat(2000.50),
+			"usd_24h_change": decimal.NewFromFloat(2.75),
+		},
+	})
+	suite.mockCoingeckoClient.EXPECT().
+		FetchSimplePrice(gomock.Any(), gomock.Any(), gomock.Any(), true).
+		Return(&mockResp, nil).
+		Times(1) // Expect exactly one call
+
+	// Run the job
+	ran, err := suite.service.RunNextRelativeCachePrepopulationJob(suite.ctx)
+	suite.Require().NoError(err, "Should run cache prepopulation job without error")
+	suite.Require().True(ran, "Should indicate job was run")
+
+	// Verify the data was cached by trying to retrieve it - this should NOT call Coingecko
+	rates, updated, err := suite.service.GetRelativeFromCache(
+		suite.ctx,
+		ratios.CoingeckoVsCurrencyList{"usd"},
+		[]ratios.CoingeckoCoin(coinList)...,
+	)
+	suite.Require().NoError(err, "Should get cached rates without error")
+	suite.Require().NotNil(rates, "Should have cached rates")
+	suite.Require().NotZero(updated, "Should have last updated timestamp")
+
+	// Verify the cached data matches what we expect
+	suite.Require().Contains((*rates)["basic-attention-token"], "usd")
+	suite.Require().Contains((*rates)["ethereum"], "usd")
+	suite.Require().Equal(
+		decimal.NewFromFloat(0.25),
+		(*rates)["basic-attention-token"]["usd"],
+		"BAT/USD rate should match",
+	)
+	suite.Require().Equal(
+		decimal.NewFromFloat(2000.50),
+		(*rates)["ethereum"]["usd"],
+		"ETH/USD rate should match",
+	)
+}