Skip to content

Commit

Permalink
cache with redis
Browse files Browse the repository at this point in the history
  • Loading branch information
mstgnz committed Dec 29, 2024
1 parent 49843fc commit d57b194
Show file tree
Hide file tree
Showing 7 changed files with 146 additions and 13 deletions.
3 changes: 3 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,6 @@ AWS_BUCKET=
DISABLE_DELETE=false
DISABLE_UPLOAD=false
DISABLE_GET=false

# Redis Configuration
REDIS_URL=redis://redis:6379
15 changes: 15 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,17 @@
version: '3.8'

services:
redis:
image: redis:7.2-alpine
container_name: cdn_redis
ports:
- "6379:6379"
volumes:
- redis_data:/data
command: redis-server --appendonly yes
networks:
- cdn

golang:
build:
context: .
Expand Down Expand Up @@ -38,3 +51,5 @@ networks:

volumes:
minio:
redis_data:
driver: local
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/glacier v1.19.6
github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1
github.com/fsnotify/fsnotify v1.7.0
github.com/go-redis/redis/v8 v8.11.5
github.com/gofiber/fiber/v2 v2.52.0
github.com/google/uuid v1.6.0
github.com/joho/godotenv v1.5.1
Expand Down Expand Up @@ -43,6 +44,7 @@ require (
github.com/aws/smithy-go v1.19.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
Expand Down
12 changes: 12 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
Expand All @@ -57,6 +59,8 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofiber/fiber/v2 v2.52.0 h1:S+qXi7y+/Pgvqq4DrSmREGiFwtB7Bu6+QFLuIHYw/UE=
github.com/gofiber/fiber/v2 v2.52.0/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
Expand Down Expand Up @@ -99,6 +103,12 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE=
github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down Expand Up @@ -198,6 +208,8 @@ gopkg.in/gographics/imagick.v3 v3.5.1 h1:58JqK0UCx5RfvbRggF5FKuK6jHwAtTQopUxK8mz
gopkg.in/gographics/imagick.v3 v3.5.1/go.mod h1:+Q9nyA2xRZXrDyTtJ/eko+8V/5E7bWYs08ndkZp8UmA=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
Expand Down
39 changes: 26 additions & 13 deletions handler/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"context"
"fmt"
"io"
"log"
"mime/multipart"
"net/http"
"strconv"
Expand Down Expand Up @@ -39,6 +40,7 @@ type image struct {
awsService service.AwsService
workerPool *worker.Pool
batchProc *batch.BatchProcessor
cache service.CacheService
}

// ImageProcessRequest represents an image processing request
Expand All @@ -59,11 +61,18 @@ func NewImage(minioClient *minio.Client, awsService service.AwsService) Image {
bp := batch.NewBatchProcessor(10, 5*time.Second, processBatch)
bp.Start()

// Initialize cache service
cacheService, err := service.NewCacheService("redis://localhost:6379")
if err != nil {
log.Printf("Failed to initialize cache service: %v", err)
}

return &image{
minioClient: minioClient,
awsService: awsService,
workerPool: wp,
batchProc: bp,
cache: cacheService,
}
}

Expand Down Expand Up @@ -91,53 +100,57 @@ func processBatch(items []batch.BatchItem) []batch.BatchItem {
}

func (i image) GetImage(c *fiber.Ctx) error {
c.Status(http.StatusNotFound)
ctx := context.Background()
bucket := c.Params("bucket")
objectName := c.Params("*")

var width uint
var height uint
var resize bool

bucket := c.Params("bucket")
objectName := c.Params("*")

if service.IsImageFile(objectName) {
resize, width, height = service.GetWidthAndHeight(c, service.ParamsType)
}

// Bucket exists
// Try to get from cache if resize is requested
if resize {
if cachedImage, err := i.cache.GetResizedImage(bucket, objectName, width, height); err == nil {
c.Set("Content-Type", http.DetectContentType(cachedImage))
return c.Send(cachedImage)
}
}

// Continue with normal flow if not in cache
if found, err := i.minioClient.BucketExists(ctx, bucket); !found || err != nil {
return c.SendFile("./public/notfound.png")
}

// Get Object
object, err := i.minioClient.GetObject(ctx, bucket, objectName, minio.GetObjectOptions{})

if err != nil {
return c.SendFile("./public/notfound.png")
}

// Convert Byte
getByte := service.StreamToByte(object)
if len(getByte) == 0 {
return c.SendFile("./public/notfound.png")
}

// get size
if err, orjWidth, orjHeight := service.ImagickGetWidthHeight(getByte); err == nil {
c.Set("Width", strconv.Itoa(int(orjWidth)))
c.Set("Height", strconv.Itoa(int(orjHeight)))
}

// Set Content Type
c.Set("Content-Type", http.DetectContentType(getByte))

// Send Resized Image
if resize {
return c.Send(service.ImagickResize(getByte, width, height))
resizedImage := service.ImagickResize(getByte, width, height)
// Cache the resized image
if err := i.cache.SetResizedImage(bucket, objectName, width, height, resizedImage); err != nil {
log.Printf("Failed to cache resized image: %v", err)
}
return c.Send(resizedImage)
}

// Send Original Image
c.Status(http.StatusFound)
return c.Send(getByte)
}
Expand Down
12 changes: 12 additions & 0 deletions makefile
Original file line number Diff line number Diff line change
Expand Up @@ -57,3 +57,15 @@ create_volume:
else \
echo "Volume '$(APP_NAME)' already exists, skipping creation."; \
fi

redis-cli:
docker exec -it cdn_redis redis-cli

redis-logs:
docker logs -f cdn_redis

redis-restart:
docker restart cdn_redis

redis-clear:
docker exec cdn_redis redis-cli FLUSHALL
76 changes: 76 additions & 0 deletions service/cache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package service

import (
"context"
"fmt"
"time"

"github.com/go-redis/redis/v8"
"github.com/mstgnz/cdn/pkg/observability"
"github.com/rs/zerolog"
)

type CacheService interface {
Get(key string) ([]byte, error)
Set(key string, value []byte, expiration time.Duration) error
Delete(key string) error
GetResizedImage(bucket, path string, width, height uint) ([]byte, error)
SetResizedImage(bucket, path string, width, height uint, data []byte) error
}

type redisCache struct {
client *redis.Client
logger zerolog.Logger
}

func NewCacheService(redisURL string) (CacheService, error) {
opt, err := redis.ParseURL(redisURL)
if err != nil {
return nil, err
}

client := redis.NewClient(opt)
if err := client.Ping(context.Background()).Err(); err != nil {
return nil, err
}

return &redisCache{
client: client,
logger: observability.Logger(),
}, nil
}

func (c *redisCache) Get(key string) ([]byte, error) {
start := time.Now()
defer func() {
duration := time.Since(start).Seconds()
observability.StorageOperationDuration.WithLabelValues("cache_get", "redis").Observe(duration)
}()

return c.client.Get(context.Background(), key).Bytes()
}

func (c *redisCache) Set(key string, value []byte, expiration time.Duration) error {
start := time.Now()
defer func() {
duration := time.Since(start).Seconds()
observability.StorageOperationDuration.WithLabelValues("cache_set", "redis").Observe(duration)
}()

return c.client.Set(context.Background(), key, value, expiration).Err()
}

func (c *redisCache) Delete(key string) error {
return c.client.Del(context.Background(), key).Err()
}

func (c *redisCache) GetResizedImage(bucket, path string, width, height uint) ([]byte, error) {
key := fmt.Sprintf("resize:%s:%s:%d:%d", bucket, path, width, height)
return c.Get(key)
}

func (c *redisCache) SetResizedImage(bucket, path string, width, height uint, data []byte) error {
key := fmt.Sprintf("resize:%s:%s:%d:%d", bucket, path, width, height)
// Cache for 24 hours
return c.Set(key, data, 24*time.Hour)
}

0 comments on commit d57b194

Please sign in to comment.