Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into l_main
Browse files Browse the repository at this point in the history 
# Conflicts:
#	.github/workflows/beta_release.yml
#	.github/workflows/build_docker.yml
#	go.mod
#	go.sum
  • Loading branch information
@Three-taile-dragon
Three-taile-dragon committed Sep 26, 2024
2 parents f775b8d + c16fb45 commit b23e0fc
Show file tree
Hide file tree
Showing 4 changed files with 40 additions and 287 deletions.
166 changes: 35 additions & 131 deletions drivers/halalcloud/driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,24 +4,25 @@ import (
"context"
"crypto/sha1"
"fmt"
"github.com/alist-org/alist/v3/drivers/base"
"github.com/alist-org/alist/v3/internal/driver"
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/internal/op"
"github.com/alist-org/alist/v3/pkg/http_range"
"github.com/alist-org/alist/v3/pkg/utils"
bauth "github.com/baidubce/bce-sdk-go/auth"
"github.com/baidubce/bce-sdk-go/bce"
"github.com/baidubce/bce-sdk-go/services/bos"
"github.com/baidubce/bce-sdk-go/services/bos/api"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/city404/v6-public-rpc-proto/go/v6/common"
pbPublicUser "github.com/city404/v6-public-rpc-proto/go/v6/user"
pubUserFile "github.com/city404/v6-public-rpc-proto/go/v6/userfile"
"github.com/jinzhu/copier"
"github.com/rclone/rclone/lib/readers"
"github.com/zzzhr1990/go-common-entity/userfile"
"io"
"net/url"
"path"
"strconv"
"strings"
"time"
)

Expand All @@ -31,7 +32,6 @@ type HalalCloud struct {
Addition

uploadThread int
fileStatus int // 文件状态 类型，0小文件(1M)、1中型文件(16M)、2大型文件(32M)
}

func (d *HalalCloud) Config() driver.Config {
Expand All @@ -48,10 +48,6 @@ func (d *HalalCloud) Init(ctx context.Context) error {
d.uploadThread, d.UploadThread = 3, "3"
}

if d.CustomUploadPartSize == "" {
d.CustomUploadPartSize = "0"
}

if d.HalalCommon == nil {
d.HalalCommon = &HalalCommon{
Common: &Common{},
Expand Down Expand Up @@ -186,10 +182,6 @@ func (d *HalalCloud) getFiles(ctx context.Context, dir model.Obj) ([]model.Obj,
client := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection())

opDir := d.GetCurrentDir(dir)
//if len(args) > 0 {
// opDir = d.GetCurrentOpDir(dir, args, 0)
//}
filesList := FilesList{}
for {
result, err := client.List(ctx, &pubUserFile.FileListRequest{
Parent: &pubUserFile.File{Path: opDir},
Expand All @@ -201,14 +193,9 @@ func (d *HalalCloud) getFiles(ctx context.Context, dir model.Obj) ([]model.Obj,
if err != nil {
return nil, err
}
err = copier.Copy(&filesList, result)
if err != nil {
return nil, err
}
if filesList.Files != nil && len(filesList.Files) > 0 {
for i := 0; i < len(filesList.Files); i++ {
files = append(files, filesList.Files[i])
}

for i := 0; len(result.Files) > i; i++ {
files = append(files, (*Files)(result.Files[i]))
}

if result.ListInfo == nil || result.ListInfo.Token == "" {
Expand All @@ -222,28 +209,11 @@ func (d *HalalCloud) getFiles(ctx context.Context, dir model.Obj) ([]model.Obj,

func (d *HalalCloud) getLink(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {

id := file.GetID()

newPath := userfile.NewFormattedPath(d.GetCurrentDir(file)).GetPath()
client := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection())
//if len(id) > 0 {
// newPath = ""
//}
ctx1, cancelFunc := context.WithCancel(context.Background())
defer cancelFunc()
result, err := client.ParseFileSlice(ctx1, &pubUserFile.File{
Identity: id,
Parent: file.(*Files).Parent,
Name: file.GetName(),
Path: newPath,
MimeType: file.(*Files).MimeType,
Size: file.(*Files).Size,
Type: file.(*Files).Type,
CreateTs: file.(*Files).CreateTs,
UpdateTs: file.(*Files).UpdateTs,
DeleteTs: file.(*Files).DeleteTs,
ContentIdentity: file.(*Files).ContentIdentity,
})

result, err := client.ParseFileSlice(ctx1, (*pubUserFile.File)(file.(*Files)))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -320,7 +290,6 @@ func (d *HalalCloud) getLink(ctx context.Context, file model.Obj, args model.Lin
func (d *HalalCloud) makeDir(ctx context.Context, dir model.Obj, name string) (model.Obj, error) {
newDir := userfile.NewFormattedPath(d.GetCurrentOpDir(dir, []string{name}, 0)).GetPath()
_, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Create(ctx, &pubUserFile.File{
// Parent: &pubUserFile.File{Path: currentDir},
Path: newDir,
})
return nil, err
Expand All @@ -347,9 +316,6 @@ func (d *HalalCloud) move(ctx context.Context, obj model.Obj, dir model.Obj) (mo
func (d *HalalCloud) rename(ctx context.Context, obj model.Obj, name string) (model.Obj, error) {
id := obj.GetID()
newPath := userfile.NewFormattedPath(d.GetCurrentOpDir(obj, []string{name}, 0)).GetPath()
//if len(id) > 0 {
// newPath = ""
//}
_, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Rename(ctx, &pubUserFile.File{
Path: newPath,
Identity: id,
Expand Down Expand Up @@ -399,101 +365,39 @@ func (d *HalalCloud) remove(ctx context.Context, obj model.Obj) error {

func (d *HalalCloud) put(ctx context.Context, dstDir model.Obj, fileStream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {

tempFile, err := fileStream.CacheFullInTempFile()
if err != nil {
return nil, err
}

newDir := userfile.NewFormattedPath(d.GetCurrentDir(dstDir)).GetPath()
newDir = strings.TrimSuffix(newDir, "/") + "/" + fileStream.GetName()
newDir := path.Join(dstDir.GetPath(), fileStream.GetName())
result, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).CreateUploadToken(ctx, &pubUserFile.File{
Path: newDir,
})
if err != nil {
return nil, err
}
clientConfig := bos.BosClientConfiguration{
Ak: result.AccessKey,
Sk: result.SecretKey,
Endpoint: result.Endpoint,
RedirectDisabled: false,
//SessionToken: result.SessionToken,
}

// 初始化一个BosClient
bosClient, err := bos.NewClientWithConfig(&clientConfig)
u, _ := url.Parse(result.Endpoint)
u.Host = "s3." + u.Host
result.Endpoint = u.String()
s, err := session.NewSession(&aws.Config{
HTTPClient: base.HttpClient,
Credentials: credentials.NewStaticCredentials(result.AccessKey, result.SecretKey, result.Token),
Region: aws.String(result.Region),
Endpoint: aws.String(result.Endpoint),
S3ForcePathStyle: aws.Bool(true),
})
if err != nil {
return nil, fmt.Errorf("failed to create bos client: %w", err)
return nil, err
}
stsCredential, err := bauth.NewSessionBceCredentials(
result.AccessKey,
result.SecretKey,
result.Token)
if err != nil {
return nil, fmt.Errorf("failed to create sts credential: %w", err)
uploader := s3manager.NewUploader(s, func(u *s3manager.Uploader) {
u.Concurrency = d.uploadThread
})
if fileStream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize {
uploader.PartSize = fileStream.GetSize() / (s3manager.MaxUploadParts - 1)
}
bosClient.Config.Credentials = stsCredential
bosClient.MaxParallel = int64(d.uploadThread)

d.setFileStatus(fileStream.GetSize()) // 设置文件状态

bosClient.MultipartSize = d.getSliceSize()

if fileStream.GetSize() < 1*utils.MB {
partBody, _ := bce.NewBodyFromSizedReader(tempFile, fileStream.GetSize())
_, err := bosClient.PutObject(result.Bucket, result.Key, partBody, nil)
//_, err = bosClient.PutObjectFromStream(result.GetBucket(), fileStream.GetName(), tempFile, nil)
if err != nil {
return nil, fmt.Errorf("failed to upload file: %v ===> %s/%s", err, clientConfig.Ak, clientConfig.Sk)
}
up(100)
} else {
res, err := bosClient.BasicInitiateMultipartUpload(result.Bucket, result.Key)
// 分块大小按MULTIPART_ALIGN=1MB对齐
partSize := (bosClient.MultipartSize +
bos.MULTIPART_ALIGN - 1) / bos.MULTIPART_ALIGN * bos.MULTIPART_ALIGN

// 获取文件大小，并计算分块数目，最大分块数MAX_PART_NUMBER=10000
fileSize := fileStream.GetSize()
partNum := (fileSize + partSize - 1) / partSize
if partNum > bos.MAX_PART_NUMBER { // 超过最大分块数，需调整分块大小
partSize = (fileSize + bos.MAX_PART_NUMBER + 1) / bos.MAX_PART_NUMBER
partSize = (partSize + bos.MULTIPART_ALIGN - 1) / bos.MULTIPART_ALIGN * bos.MULTIPART_ALIGN
partNum = (fileSize + partSize - 1) / partSize
}
// 创建保存每个分块上传后的ETag和PartNumber信息的列表
partEtags := make([]api.UploadInfoType, 0)

// 逐个分块上传
for i := int64(1); i <= partNum; i++ {
// 计算偏移offset和本次上传的大小uploadSize
uploadSize := partSize
offset := partSize * (i - 1)
left := fileSize - offset
if left < partSize {
uploadSize = left
}

// 创建指定大小的文件流
partBody, _ := bce.NewBodyFromSizedReader(tempFile, uploadSize)

// 上传当前分块
etag, _ := bosClient.BasicUploadPart(result.Bucket, result.Key, res.UploadId, int(i), partBody)

// 保存当前分块上传成功后返回的序号和ETag
partEtags = append(partEtags, api.UploadInfoType{int(i), etag})

up(float64(i) / float64(partNum) * 100)
}
_, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{
Bucket: aws.String(result.Bucket),
Key: aws.String(result.Key),
Body: io.TeeReader(fileStream, driver.NewProgress(fileStream.GetSize(), up)),
})
return nil, err

completeArgs := &api.CompleteMultipartUploadArgs{Parts: partEtags}
_, err = bosClient.CompleteMultipartUploadFromStruct(
result.Bucket, result.Key, res.UploadId, completeArgs)
if err != nil {
return nil, err
}
}
return nil, nil
}

var _ driver.Driver = (*HalalCloud)(nil)
5 changes: 2 additions & 3 deletions drivers/halalcloud/meta.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,8 @@ type Addition struct {
// Usually one of two
driver.RootPath
// define other
RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"`
UploadThread string `json:"upload_thread" default:"3" help:"1 <= thread <= 32"`
CustomUploadPartSize string `json:"custom_upload_part_size" default:"0" help:"0 for auto"`
RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"`
UploadThread string `json:"upload_thread" default:"3" help:"1 <= thread <= 32"`

AppID string `json:"app_id" required:"true" default:"alist/10001"`
AppVersion string `json:"app_version" required:"true" default:"1.0.0"`
Expand Down
44 changes: 2 additions & 42 deletions drivers/halalcloud/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"github.com/alist-org/alist/v3/internal/model"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/city404/v6-public-rpc-proto/go/v6/common"
pubUserFile "github.com/city404/v6-public-rpc-proto/go/v6/userfile"
"google.golang.org/grpc"
"time"
)
Expand Down Expand Up @@ -50,34 +51,7 @@ type FilesList struct {

var _ model.Obj = (*Files)(nil)

type Files struct {
Identity string `json:"identity,omitempty"`
Parent string `json:"parent,omitempty"`
Name string `json:"name,omitempty"`
Path string `json:"path,omitempty"`
MimeType string `json:"mime_type,omitempty"`
Size int64 `json:"size,omitempty"`
Type int64 `json:"type,omitempty"`
CreateTs int64 `json:"create_ts,omitempty"`
UpdateTs int64 `json:"update_ts,omitempty"`
DeleteTs int64 `json:"delete_ts,omitempty"`
Deleted bool `json:"deleted,omitempty"`
Dir bool `json:"dir,omitempty"`
Hidden bool `json:"hidden,omitempty"`
Locked bool `json:"locked,omitempty"`
Shared bool `json:"shared,omitempty"`
Starred bool `json:"starred,omitempty"`
Trashed bool `json:"trashed,omitempty"`
LockedAt int64 `json:"locked_at,omitempty"`
LockedBy string `json:"locked_by,omitempty"`
SharedAt int64 `json:"shared_at,omitempty"`
Flag int64 `json:"flag,omitempty"`
Unique string `json:"unique,omitempty"`
ContentIdentity string `json:"content_identity,omitempty"`
Label int64 `json:"label,omitempty"`
StoreType int64 `json:"store_type,omitempty"`
Version int64 `json:"version,omitempty"`
}
type Files pubUserFile.File

func (f *Files) GetSize() int64 {
return f.Size
Expand Down Expand Up @@ -114,20 +88,6 @@ func (f *Files) GetPath() string {
return f.Path
}

func fileToObj(f Files) *model.ObjThumb {
return &model.ObjThumb{
Object: model.Object{
ID: f.Identity,
Path: f.Path,
Name: f.Name,
Size: f.Size,
Modified: f.ModTime(),
Ctime: f.CreateTime(),
IsFolder: f.IsDir(),
},
}
}

type SteamFile struct {
file model.File
}
Expand Down
Loading

0 comments on commit b23e0fc

Please sign in to comment.