feat(bcs-bscp): feedserver support sync download file api

bcs-services/bcs-bscp/Makefile

@@ -146,6 +146,10 @@ openapi:
 build_bscp:
 	@cd ${PRO_DIR}/cmd && make all
 
+.PHONY: build_feed
+build_feed:
+	@cd ${PRO_DIR}/cmd && make feed
+
 .PHONY: build_frontend
 build_frontend:
 	cd ui; npm install --legacy-peer-deps; npm run build

bcs-services/bcs-bscp/cmd/Makefile

@@ -7,6 +7,10 @@ all:
 	@cd cache-service && make
 	@cd vault-server && make
 
+feed:
+	@cd feed-server && make
+	@cd cache-service && make
+
 server:
 	@cd api-server && make package
 	@cd config-server && make package

bcs-services/bcs-bscp/cmd/cache-service/service/cache/client/client.go

@@ -27,6 +27,7 @@ import (
 	pbds "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/protocol/data-service"
 	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/runtime/jsoni"
 	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/runtime/lock"
+	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/types"
 )
 
 // Interface defines all the supported operations to get resource cache.
@@ -42,6 +43,8 @@ type Interface interface {
 	GetReleasedKvValue(kt *kit.Kit, bizID, appID, releaseID uint32, key string) (string, error)
 	SetClientMetric(kt *kit.Kit, bizID, appID uint32, payload []byte) error
 	BatchUpsertClientMetrics(kt *kit.Kit, clientData []*pbclient.Client, clientEventData []*pbce.ClientEvent) error
+	GetAsyncDownloadTask(kt *kit.Kit, bizID uint32, taskID string) (string, error)
+	SetAsyncDownloadTask(kt *kit.Kit, bizID uint32, taskID string, task *types.AsyncDownloadTaskCache) error
 }
 
 // New initialize a cache client.
@@ -130,3 +133,22 @@ func (c *client) BatchUpsertClientMetrics(kt *kit.Kit, clientData []*pbclient.Cl
 	}
 	return nil
 }
+
+// GetAsyncDownloadTask get async download task from cache
+func (c *client) GetAsyncDownloadTask(kt *kit.Kit, bizID uint32, taskID string) (string, error) {
+	return c.bds.Get(kt.Ctx, keys.Key.AsyncDownloadTaskKey(bizID, taskID))
+}
+
+// SetAsyncDownloadTask set async download task to cache
+func (c *client) SetAsyncDownloadTask(kt *kit.Kit, bizID uint32, taskID string,
+	task *types.AsyncDownloadTaskCache) error {
+	js, err := jsoni.Marshal(task)
+	if err != nil {
+		return err
+	}
+	if err := c.bds.Set(kt.Ctx, keys.Key.AsyncDownloadTaskKey(bizID, taskID), string(js),
+		keys.Key.AppMetaTtlSec(false)); err != nil {
+		return err
+	}
+	return nil
+}

bcs-services/bcs-bscp/cmd/cache-service/service/cache/keys/cache.go

@@ -33,6 +33,7 @@ var Key = &keyGenerator{
 	releasedHookTTLRange:        [2]int{6 * oneDaySeconds, 7 * oneDaySeconds},
 	appMetaTTLRange:             [2]int{6 * oneDaySeconds, 7 * oneDaySeconds},
 	appHasRITTLRange:            [2]int{5 * 60, 10 * 60},
+	asyncDownloadTaskTTLRange:   [2]int{30 * 60, 60 * 60},
 }
 
 type namespace string
@@ -49,6 +50,7 @@ const (
 	appID               namespace = "app-id"
 	releasedKv          namespace = "released-kv"
 	clientMetric        namespace = "client-metric"
+	asyncDownloadTask   namespace = "async-download-task"
 )
 
 type keyGenerator struct {
@@ -61,6 +63,7 @@ type keyGenerator struct {
 	releasedHookTTLRange        [2]int
 	appMetaTTLRange             [2]int
 	appHasRITTLRange            [2]int
+	asyncDownloadTaskTTLRange   [2]int
 }
 
 // ClientMetricKey generate the client metric cache key.
@@ -72,6 +75,27 @@ func (k keyGenerator) ClientMetricKey(bizID uint32, appID uint32) string {
 	}.String()
 }
 
+// AsyncDownloadTaskKey generate the async download task cache key.
+func (k keyGenerator) AsyncDownloadTaskKey(bizID uint32, taskID string) string {
+	return element{
+		biz: bizID,
+		ns:  asyncDownloadTask,
+		key: taskID,
+	}.String()
+}
+
+// AsyncDownloadTaskTtlSec generate the async download task's TTL seconds
+func (k keyGenerator) AsyncDownloadTaskTtlSec(withRange bool) int {
+	if withRange {
+		//nolint:gosec
+		r := rand.New(rand.NewSource(time.Now().UnixNano()))
+		seconds := r.Intn(k.asyncDownloadTaskTTLRange[1]-k.asyncDownloadTaskTTLRange[0]) +
+			k.asyncDownloadTaskTTLRange[0]
+		return seconds
+	}
+	return k.asyncDownloadTaskTTLRange[0]
+}
+
 // ReleasedGroup generate a release's released group cache key to save all the released groups under this release
 func (k keyGenerator) ReleasedGroup(bizID uint32, appID uint32) string {
 	return element{

bcs-services/bcs-bscp/cmd/cache-service/service/feed.go

@@ -237,3 +237,34 @@ func (s *Service) SetClientMetric(ctx context.Context, req *pbcs.SetClientMetric
 	}
 	return &pbcs.SetClientMetricResp{}, nil
 }
+
+// SetAsyncDownloadTask set async download task
+func (s *Service) SetAsyncDownloadTask(ctx context.Context, req *pbcs.SetAsyncDownloadTaskReq) (
+	*pbcs.SetAsyncDownloadTaskResp, error) {
+	kt := kit.FromGrpcContext(ctx)
+	task := &types.AsyncDownloadTaskCache{
+		BizID:    req.BizId,
+		AppID:    req.AppId,
+		TaskID:   req.TaskId,
+		FilePath: req.FilePath,
+		FileName: req.FileName,
+	}
+	err := s.op.SetAsyncDownloadTask(kt, req.BizId, req.TaskId, task)
+	if err != nil {
+		return nil, err
+	}
+	return &pbcs.SetAsyncDownloadTaskResp{}, nil
+}
+
+// GetAsyncDownloadTask get async download task
+func (s *Service) GetAsyncDownloadTask(ctx context.Context, req *pbcs.GetAsyncDownloadTaskReq) (
+	*pbcs.JsonRawResp, error) {
+	kt := kit.FromGrpcContext(ctx)
+	task, err := s.op.GetAsyncDownloadTask(kt, req.BizId, req.TaskId)
+	if err != nil {
+		return nil, err
+	}
+	return &pbcs.JsonRawResp{
+		JsonRaw: task,
+	}, nil
+}

bcs-services/bcs-bscp/cmd/feed-server/app/app.go

@@ -25,6 +25,7 @@ import (
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/reflection"
 
+	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/cmd/feed-server/crontab"
 	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/cmd/feed-server/options"
 	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/cmd/feed-server/service"
 	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/cc"
@@ -68,6 +69,7 @@ type feedServer struct {
 	gwServe *http.Server //nolint:unused
 	sd      serviced.ServiceDiscover
 	service *service.Service
+	cleaner *crontab.SourceFileCacheCleaner
 }
 
 // prepare do prepare jobs before run feed server.
@@ -115,6 +117,10 @@ func (fs *feedServer) prepare(opt *options.Option) error {
 	}
 	fs.service = svc
 
+	cleaner := crontab.NewSourceFileCacheCleaner()
+	fs.cleaner = cleaner
+	fs.cleaner.Run()
+
 	return nil
 }
 
@@ -197,6 +203,9 @@ func (fs *feedServer) listenAndServe() error {
 }
 
 func (fs *feedServer) finalizer() {
+
+	fs.cleaner.Stop()
+
 	if err := fs.sd.Deregister(); err != nil {
 		logs.Errorf("process service shutdown, but deregister failed, err: %v", err)
 		return

bcs-services/bcs-bscp/cmd/feed-server/bll/async-download/asyncdownload.go

@@ -0,0 +1,67 @@
+/*
+ * Tencent is pleased to support the open source community by making Blueking Container Service available.
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ * http://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package asyncdownload NOTES
+package asyncdownload
+
+import (
+	clientset "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/cmd/feed-server/bll/client-set"
+	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/cmd/feed-server/bll/lcache"
+	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/cmd/feed-server/bll/types"
+	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/kit"
+	pkgtypes "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/pkg/types"
+)
+
+// New initialize the release service instance.
+func New(cs *clientset.ClientSet, cache *lcache.Cache) (*AsyncDownload, error) {
+	return &AsyncDownload{
+		cs:    cs,
+		cache: cache,
+	}, nil
+}
+
+// AsyncDownload defines async download related operations.
+type AsyncDownload struct {
+	cs    *clientset.ClientSet
+	cache *lcache.Cache
+}
+
+// CreateAsyncDownloadTask create sync download task record.
+func (ad *AsyncDownload) CreateAsyncDownloadTask(kt *kit.Kit, opts *types.AsyncDownloadTask) error {
+	return ad.cache.AsyncDownload.SetAsyncDownloadTask(kt, &pkgtypes.AsyncDownloadTaskCache{
+		BizID:    opts.BizID,
+		AppID:    opts.AppID,
+		TaskID:   opts.TaskID,
+		FilePath: opts.FilePath,
+		FileName: opts.FileName,
+	})
+}
+
+// GetAsyncDownloadTask get sync download task record.
+func (ad *AsyncDownload) GetAsyncDownloadTask(kt *kit.Kit, bizID uint32, taskID string) (
+	*types.AsyncDownloadTask, error) {
+
+	data, err := ad.cache.AsyncDownload.GetAsyncDownloadTask(kt, bizID, taskID)
+	if err != nil {
+		return nil, err
+	}
+
+	task := &types.AsyncDownloadTask{
+		BizID:    data.BizID,
+		AppID:    data.AppID,
+		TaskID:   data.TaskID,
+		FilePath: data.FilePath,
+		FileName: data.FileName,
+	}
+
+	return task, nil
+}

bcs-services/bcs-bscp/cmd/feed-server/bll/bll.go

@@ -17,6 +17,7 @@ package bll
 import (
 	"fmt"
 
+	asyncdownload "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/cmd/feed-server/bll/async-download"
 	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/cmd/feed-server/bll/auth"
 	clientset "github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/cmd/feed-server/bll/client-set"
 	"github.com/TencentBlueKing/bk-bcs/bcs-services/bcs-bscp/cmd/feed-server/bll/eventc"
@@ -64,28 +65,35 @@ func New(sd serviced.Discover, authorizer iamauth.Authorizer, name string) (*BLL
 	handler := &eventc.Handler{
 		GetMatchedRelease: rs.GetMatchedRelease,
 	}
-	if err := sch.Run(handler); err != nil {
-		return nil, fmt.Errorf("run scheduler faield, err: %v", err)
+	if e := sch.Run(handler); e != nil {
+		return nil, fmt.Errorf("run scheduler faield, err: %v", e)
+	}
+
+	asyncDownloader, err := asyncdownload.New(client, localCache)
+	if err != nil {
+		return nil, err
 	}
 
 	return &BLL{
-		client:  client,
-		release: rs,
-		auth:    auth.New(localCache),
-		cache:   localCache,
-		ob:      ob,
-		sch:     sch,
+		client:        client,
+		release:       rs,
+		auth:          auth.New(localCache),
+		cache:         localCache,
+		ob:            ob,
+		sch:           sch,
+		asyncdownload: asyncDownloader,
 	}, nil
 }
 
 // BLL defines business logical layer instance.
 type BLL struct {
-	client  *clientset.ClientSet
-	release *release.ReleasedService
-	auth    *auth.AuthService
-	cache   *lcache.Cache
-	ob      observer.Interface
-	sch     *eventc.Scheduler
+	client        *clientset.ClientSet
+	release       *release.ReleasedService
+	auth          *auth.AuthService
+	asyncdownload *asyncdownload.AsyncDownload
+	cache         *lcache.Cache
+	ob            observer.Interface
+	sch           *eventc.Scheduler
 }
 
 // Release return the release service instance.
@@ -112,3 +120,8 @@ func (b *BLL) RKvCache() *lcache.ReleasedKv {
 func (b *BLL) ClientMetric() *lcache.ClientMetric {
 	return b.cache.ClientMetric
 }
+
+// AsyncDownload return the async download instance.
+func (b *BLL) AsyncDownload() *asyncdownload.AsyncDownload {
+	return b.asyncdownload
+}