diff --git a/go.mod b/go.mod index 0bac047..64ca0bb 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,10 @@ go 1.19 require ( github.com/aliyun/alibaba-cloud-sdk-go v1.62.26 github.com/aliyun/aliyun-oss-go-sdk v2.2.4+incompatible + github.com/aws/aws-sdk-go-v2 v1.24.0 + github.com/aws/aws-sdk-go-v2/config v1.26.1 + github.com/aws/aws-sdk-go-v2/credentials v1.16.12 + github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 github.com/flamego/cache v1.1.0 github.com/flamego/csrf v1.0.1 github.com/flamego/flamego v1.7.0 @@ -42,6 +46,20 @@ require ( github.com/alecthomas/participle/v2 v2.0.0-beta.5 // indirect github.com/andybalholm/brotli v1.0.6 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 // indirect + github.com/aws/smithy-go v1.19.0 // indirect github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -66,7 +84,7 @@ require ( github.com/imroc/req/v3 v3.42.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.8 // indirect github.com/klauspost/compress v1.17.4 // indirect github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/go.sum b/go.sum index af32a26..1c88ccd 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,42 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aws/aws-sdk-go-v2 v1.24.0 h1:890+mqQ+hTpNuw0gGP6/4akolQkSToDJgHfQE7AwGuk= +github.com/aws/aws-sdk-go-v2 v1.24.0/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= +github.com/aws/aws-sdk-go-v2/config v1.26.1 h1:z6DqMxclFGL3Zfo+4Q0rLnAZ6yVkzCRxhRMsiRQnD1o= +github.com/aws/aws-sdk-go-v2/config v1.26.1/go.mod h1:ZB+CuKHRbb5v5F0oJtGdhFTelmrxd4iWO1lf0rQwSAg= +github.com/aws/aws-sdk-go-v2/credentials v1.16.12 h1:v/WgB8NxprNvr5inKIiVVrXPuuTegM+K8nncFkr1usU= +github.com/aws/aws-sdk-go-v2/credentials v1.16.12/go.mod h1:X21k0FjEJe+/pauud82HYiQbEr9jRKY3kXEIQ4hXeTQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 h1:w98BT5w+ao1/r5sUuiH6JkVzjowOKeOJRHERyy1vh58= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10/go.mod h1:K2WGI7vUvkIv1HoNbfBA1bvIZ+9kL3YVmWxeKuLQsiw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 h1:v+HbZaCGmOwnTTVS86Fleq0vPzOd7tnJGbFhP0stNLs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9/go.mod h1:Xjqy+Nyj7VDLBtCMkQYOw1QYfAEZCVLrfI0ezve8wd4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 h1:N94sVhRACtXyVcjXxrwK1SKFIJrA9pOJ5yu2eSHnmls= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9/go.mod h1:hqamLz7g1/4EJP+GH5NBhcUMLjW+gKLQabgyz6/7WAU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 h1:ugD6qzjYtB7zM5PN/ZIeaAIyefPaD82G8+SJopgvUpw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9/go.mod h1:YD0aYBWCrPENpHolhKw2XDlTIWae2GKXT1T4o6N6hiM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 h1:/90OR2XbSYfXucBMJ4U14wrjlfleq/0SB6dZDPncgmo= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9/go.mod h1:dN/Of9/fNZet7UrQQ6kTDo/VSwKPIq94vjlU16bRARc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 h1:Nf2sHxjMJR8CSImIVCONRi4g0Su3J+TSTbS7G0pUeMU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9/go.mod h1:idky4TER38YIjr2cADF1/ugFMKvZV7p//pVeV5LZbF0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 h1:iEAeF6YC3l4FzlJPP9H3Ko1TXpdjdqWffxXjp8SY6uk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9/go.mod h1:kjsXoK23q9Z/tLBrckZLLyvjhZoS+AGrzqzUfEClvMM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5 h1:Keso8lIOS+IzI2MkPZyK6G0LYcK3My2LQ+T5bxghEAY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5/go.mod h1:vADO6Jn+Rq4nDtfwNjhgR84qkZwiC6FqCaXdw/kYwjA= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 h1:ldSFWz9tEHAwHNmjx2Cvy1MjP5/L9kNoR0skc6wyOOM= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.5/go.mod h1:CaFfXLYL376jgbP7VKC96uFcU8Rlavak0UlAwk1Dlhc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 h1:2k9KmFawS63euAkY4/ixVNsYYwrwnd5fIvgEKkfZFNM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5/go.mod h1:W+nd4wWDVkSUIox9bacmkBP5NMFQeTJ/xqNabpzSR38= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.5 h1:5UYvv8JUvllZsRnfrcMQ+hJ9jNICmcgKPAO1CER25Wg= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.5/go.mod h1:XX5gh4CB7wAs4KhcF46G6C8a2i7eupU19dcAAE+EydU= +github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= +github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= @@ -421,8 +457,11 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -703,10 +742,6 @@ github.com/wuhan005/gadget v0.0.0-20221206194113-7619e407f1a0 h1:zOXiOJRG/FOohTl github.com/wuhan005/gadget v0.0.0-20221206194113-7619e407f1a0/go.mod h1:vmC2IdgzTpIRwn1ZpuV/I3k9AIbRJ7oqTHFenq/qwkE= github.com/wuhan005/govalid v0.0.0-20230216091828-820aa255fd21 h1:EHaQ4hLfjckhbI+AEleDHeb0cEN1bIAfvWJEhCe7e2Y= github.com/wuhan005/govalid v0.0.0-20230216091828-820aa255fd21/go.mod h1:zRrIdMbJM3Xe4lmXyrUi2xF9CE0+D4Y0OpQIMpjC0Vo= -github.com/wuhan005/share v0.0.0-20231209184419-eca1a7d3ce0f h1:lB+ah9nGM8KMQQ4+SHTVacv0SVqfvJ/CDQRJBnULs8Q= -github.com/wuhan005/share v0.0.0-20231209184419-eca1a7d3ce0f/go.mod h1:G8p8PUSyA5uKU1tMaY5eijYs0HIliTVYRL3Qits67f0= -github.com/wuhan005/share v0.0.0-20231209185053-39c5ef8e730f h1:bNBFKSZei2GP7/LUlWpqgD9NJqg531rudRPrDb9qtVg= -github.com/wuhan005/share v0.0.0-20231209185053-39c5ef8e730f/go.mod h1:G8p8PUSyA5uKU1tMaY5eijYs0HIliTVYRL3Qits67f0= github.com/wuhan005/share v0.0.0-20231209185408-058e3c51073d h1:3JLW8+VrQLQYGm4L3E1wLVQkjayz06UzorwWD3mH83E= github.com/wuhan005/share v0.0.0-20231209185408-058e3c51073d/go.mod h1:G8p8PUSyA5uKU1tMaY5eijYs0HIliTVYRL3Qits67f0= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -1170,6 +1205,7 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/conf/static.go b/internal/conf/static.go index 79e1c1f..6189063 100644 --- a/internal/conf/static.go +++ b/internal/conf/static.go @@ -55,13 +55,17 @@ var ( } Upload struct { - DefaultAvatarURL string `ini:"default_avatar"` - DefaultBackground string `ini:"default_background"` - AliyunEndpoint string `ini:"aliyun_endpoint"` - AliyunAccessID string `ini:"aliyun_access_id"` - AliyunAccessSecret string `ini:"aliyun_access_secret"` - AliyunBucket string `ini:"aliyun_bucket"` - AliyunBucketCDNHost string `ini:"aliyun_bucket_cdn_host"` + DefaultAvatarURL string `ini:"default_avatar"` + DefaultBackground string `ini:"default_background"` + AliyunEndpoint string `ini:"aliyun_endpoint"` + AliyunAccessID string `ini:"aliyun_access_id"` + AliyunAccessSecret string `ini:"aliyun_access_secret"` + AliyunBucket string `ini:"aliyun_bucket"` + AliyunBucketCDNHost string `ini:"aliyun_bucket_cdn_host"` + ImageBackupEndpoint string `ini:"image_backup_endpoint"` + ImageBackupAccessID string `ini:"image_backup_access_id"` + ImageBackupAccessSecret string `ini:"image_backup_access_secret"` + ImageBackupBucket string `ini:"image_backup_bucket"` } Mail struct { diff --git a/route/question/page.go b/route/question/page.go index 41dadbc..28216ff 100644 --- a/route/question/page.go +++ b/route/question/page.go @@ -5,6 +5,8 @@ package question import ( + "bytes" + gocontext "context" "crypto/md5" "fmt" "io" @@ -12,12 +14,17 @@ import ( "path/filepath" "time" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/flamego/recaptcha" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/wuhan005/govalid" "github.com/wuhan005/share/pkg/share" + "github.com/NekoWheel/NekoBox/internal/conf" "github.com/NekoWheel/NekoBox/internal/context" "github.com/NekoWheel/NekoBox/internal/db" "github.com/NekoWheel/NekoBox/internal/dbutil" @@ -249,7 +256,7 @@ func uploadImage(ctx context.Context, opts uploadImageOptions) error { } now := time.Now() - fileKey := fmt.Sprintf("%d/%d/%d/%d%s", now.Year(), now.Month(), opts.QuestionID, now.Unix(), fileExt) + fileKey := fmt.Sprintf("%d/%d/%d%s", now.Year(), now.Month(), now.Unix(), fileExt) uploadImageFile, err := image.Open() if err != nil { @@ -261,17 +268,49 @@ func uploadImage(ctx context.Context, opts uploadImageOptions) error { reader := io.TeeReader(uploadImageFile, hasher) fileMd5 := fmt.Sprintf("%x", hasher.Sum(nil)) + backupImageBuffer := bytes.Buffer{} + uploadReader := io.TeeReader(reader, &backupImageBuffer) + // Upload file with share. shareServerName := share.RandomServer() - publicURL, err := share.Reader(shareServerName, reader) + publicURL, err := share.Reader(shareServerName, uploadReader) if err != nil { return errors.Wrap(err, "upload image with share") - } publicURLs := map[string]string{ shareServerName: publicURL, } + // Backup file. + go func() { + if conf.Upload.ImageBackupEndpoint != "" { + ctx := gocontext.Background() + r2Resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: conf.Upload.ImageBackupEndpoint, + }, nil + }) + + cfg, err := config.LoadDefaultConfig(ctx, + config.WithEndpointResolverWithOptions(r2Resolver), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(conf.Upload.ImageBackupAccessID, conf.Upload.ImageBackupAccessSecret, "")), + ) + if err != nil { + logrus.WithContext(ctx).WithError(err).Error("Failed to load config") + return + } + + client := s3.NewFromConfig(cfg) + if _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(conf.Upload.ImageBackupBucket), + Key: aws.String(fileKey), + Body: &backupImageBuffer, + }); err != nil { + logrus.WithContext(ctx).WithError(err).Error("Failed to upload backup image") + } + } + }() + _, err = db.UploadImgaes.Create(ctx.Request().Context(), db.CreateUploadImageOptions{ Type: opts.Type, QuestionID: opts.QuestionID,