From 0c4dbdce442518a8d1bb74341a53f3051f24455a Mon Sep 17 00:00:00 2001 From: WoozyMasta Date: Sun, 12 Jan 2025 07:07:27 +0300 Subject: [PATCH] feat: filedetails add chunking and concurrency in request --- filedetails/filedetails.go | 6 +- filedetails/filedetails_test.go | 58 +++++++++++++ filedetails/get.go | 148 +++++++++++++++++++++++++++++--- 3 files changed, 200 insertions(+), 12 deletions(-) diff --git a/filedetails/filedetails.go b/filedetails/filedetails.go index 855d0e3..a13d8a2 100644 --- a/filedetails/filedetails.go +++ b/filedetails/filedetails.go @@ -58,6 +58,8 @@ To obtain an API key, visit: [Steam Dev API Key] package filedetails const ( - baseURL string = "https://api.steampowered.com/IPublishedFileService/GetDetails/v1/" - baseFileURL string = "https://steamcommunity.com/sharedfiles/filedetails/?id=" + baseURL string = "https://api.steampowered.com/IPublishedFileService/GetDetails/v1/" + baseFileURL string = "https://steamcommunity.com/sharedfiles/filedetails/?id=" + defaultChunkMax = 220 + defaultConns = 10 ) diff --git a/filedetails/filedetails_test.go b/filedetails/filedetails_test.go index a927015..4131c9c 100644 --- a/filedetails/filedetails_test.go +++ b/filedetails/filedetails_test.go @@ -2,8 +2,10 @@ package filedetails import ( "fmt" + "math/rand" "os" "testing" + "time" ) func TestGetMods(t *testing.T) { @@ -34,3 +36,59 @@ func TestGetMods(t *testing.T) { } } } + +func TestGetManyFiles(t *testing.T) { + key, ok := os.LookupEnv("STEAM_API_KEY") + if !ok { + t.Error("Steam API key must be pass in variable 'STEAM_API_KEY'") + } + + count := 700 + ids := randomIDs(count, 1500000000, 3500000000) + query := New(ids, key) + + files, err := query.Get() + if err != nil { + t.Errorf("Cant get files %v", err) + } + + if len(files) < count { + t.Errorf("Return %d files, but expected %d", len(files), count) + } +} + +func TestGetManyFilesConcurrent(t *testing.T) { + key, ok := os.LookupEnv("STEAM_API_KEY") + if !ok { + t.Error("Steam API key must be pass in variable 'STEAM_API_KEY'") + } + + count := 5000 + ids := randomIDs(count, 1500000000, 3500000000) + query := New(ids, key) + query.SetConcurrency(100) + + files, err := query.GetConcurrent() + if err != nil { + t.Errorf("Cant get files %v", err) + } + + if len(files) < count { + t.Errorf("Return %d files, but expected %d", len(files), count) + } +} + +func randomIDs(count int, min, max uint64) []uint64 { + if min > max { + return []uint64{min, max} + } + + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + numbers := make([]uint64, count) + + for i := 0; i < count; i++ { + numbers[i] = rnd.Uint64()%(max-min+1) + min + } + + return numbers +} diff --git a/filedetails/get.go b/filedetails/get.go index 1d1aedc..6cf6b98 100644 --- a/filedetails/get.go +++ b/filedetails/get.go @@ -8,13 +8,16 @@ import ( "reflect" "strconv" "strings" + "sync" json "github.com/json-iterator/go" ) // Structure describing the parameters of a request to IPublishedFileService/GetDetails/v1/ type Query struct { - key string // Access API key + key string // Access API key + concurrent int // Max items per chunk + chunkMax int // Concurrent requests Language string `json:"language,omitempty"` // Specifies the localized text to return. Defaults to English. //* ELanguage DesiredRevision string `json:"desired_revision,omitempty"` // Return the data for the specified revision. //* EPublishedFileRevision @@ -54,6 +57,8 @@ func New(fileIDs []uint64, key string) *Query { return &Query{ key: key, + concurrent: defaultConns, + chunkMax: defaultChunkMax, PublishedFileIDs: fileIDs, ShortDescription: true, StripDescriptionBBCode: true, @@ -71,6 +76,27 @@ func (q *Query) SetKey(key string) { q.key = key } +/* +SetConcurrency sets the count of concurrent jobs. + +Parameters: + - count: Count of concurrent jobs. +*/ +func (q *Query) SetConcurrency(count int) { + q.concurrent = count +} + +/* +SetChunkMax sets the maximum number of file IDs requested for a single chunk. +Experimentally calculated limit of 220 identifiers per request, after which we get error 414 URI Too Long + +Parameters: + - count: Count of concurrent jobs. +*/ +func (q *Query) SetChunkMax(count int) { + q.concurrent = count +} + /* SetFileIDs sets the list of published file IDs for the GetDetails request. @@ -112,6 +138,7 @@ Example: } // use details */ +// GetAll - sequential requests (chunks of 220) func (q *Query) Get() ([]FileDetail, error) { if q == nil { return nil, fmt.Errorf("Query request parameters not set") @@ -120,7 +147,89 @@ func (q *Query) Get() ([]FileDetail, error) { return nil, fmt.Errorf("Steam API key is empty or does not match") } - // build URL with query + // Split IDs into chunks + chunks := splitIntoChunks(q.PublishedFileIDs, q.chunkMax) + var allDetails []FileDetail + + // Make requests sequentially + for _, c := range chunks { + qq := &Query{ + key: q.key, + PublishedFileIDs: c, + AppID: q.AppID, + ShortDescription: q.ShortDescription, + StripDescriptionBBCode: q.StripDescriptionBBCode, + IncludeKVTags: q.IncludeKVTags, + } + + details, err := qq.getChunk() + if err != nil { + return allDetails, err + } + allDetails = append(allDetails, details...) + } + + return allDetails, nil +} + +// GetConcurrent - same as Get() but requests in parallel with a concurrency limit +func (q *Query) GetConcurrent() ([]FileDetail, error) { + if q == nil { + return nil, fmt.Errorf("Query request parameters not set") + } + if len(q.key) != 32 { + return nil, fmt.Errorf("Steam API key is empty or does not match") + } + + chunks := splitIntoChunks(q.PublishedFileIDs, q.chunkMax) + var allDetails []FileDetail + var mu sync.Mutex + wg := sync.WaitGroup{} + + // Buffered channel limits the number of concurrent requests + sem := make(chan struct{}, q.concurrent) + + for _, c := range chunks { + c := c // local copy for goroutine + wg.Add(1) + go func() { + defer wg.Done() + + // Acquire slot + sem <- struct{}{} + defer func() { <-sem }() + + qq := &Query{ + key: q.key, + PublishedFileIDs: c, + AppID: q.AppID, + ShortDescription: q.ShortDescription, + StripDescriptionBBCode: q.StripDescriptionBBCode, + IncludeKVTags: q.IncludeKVTags, + } + + details, err := qq.getChunk() + if err != nil { + // For real usage consider passing error via channel to handle them properly + fmt.Printf("Error in parallel chunk: %v\n", err) + return + } + + // Merge results with lock + mu.Lock() + allDetails = append(allDetails, details...) + mu.Unlock() + }() + } + + // Wait until all goroutines are done + wg.Wait() + + return allDetails, nil +} + +// getChunk - handles one chunk request +func (q *Query) getChunk() ([]FileDetail, error) { query := url.Values{} query.Set("key", q.key) for i, id := range q.PublishedFileIDs { @@ -130,6 +239,7 @@ func (q *Query) Get() ([]FileDetail, error) { v := reflect.ValueOf(*q) t := reflect.TypeOf(*q) + // Set additional params from struct tags for i := 0; i < v.NumField(); i++ { field := t.Field(i) value := v.Field(i) @@ -155,10 +265,10 @@ func (q *Query) Get() ([]FileDetail, error) { } } } - url := baseURL + "?" + query.Encode() - // get and decode response - req, err := http.NewRequest(http.MethodGet, url, nil) + u := baseURL + "?" + query.Encode() + + req, err := http.NewRequest(http.MethodGet, u, nil) if err != nil { return nil, err } @@ -168,8 +278,8 @@ func (q *Query) Get() ([]FileDetail, error) { return nil, err } defer func() { - if err := resp.Body.Close(); err != nil { - fmt.Printf("Error close response body: %v", err) + if cerr := resp.Body.Close(); cerr != nil { + fmt.Printf("Error close response body: %v\n", cerr) } }() @@ -182,6 +292,7 @@ func (q *Query) Get() ([]FileDetail, error) { Details []FileDetail `json:"publishedfiledetails"` } `json:"response"` } + body, err := io.ReadAll(resp.Body) if err != nil { return nil, err @@ -190,19 +301,19 @@ func (q *Query) Get() ([]FileDetail, error) { return nil, err } - // Set file details URL + // Set file details URL if not set for i, f := range result.Response.Details { if f.URL == "" { result.Response.Details[i].URL = fmt.Sprintf("%s%d", baseFileURL, f.PublishedFileID) } } - // Return struct if AppID not set + // Return struct if AppID is 0 if q.AppID == 0 { return result.Response.Details, nil } - // Validate AppID == ConsumerAppid + // Validate AppID == ConsumerAppID for _, f := range result.Response.Details { if q.AppID != f.ConsumerAppID { return result.Response.Details, fmt.Errorf( @@ -214,3 +325,20 @@ func (q *Query) Get() ([]FileDetail, error) { return result.Response.Details, nil } + +// splitIntoChunks - helper to split slice into sub-slices +func splitIntoChunks(ids []uint64, size int) [][]uint64 { + if len(ids) == 0 || size <= 0 { + return nil + } + + var chunks [][]uint64 + for i := 0; i < len(ids); i += size { + end := i + size + if end > len(ids) { + end = len(ids) + } + chunks = append(chunks, ids[i:end]) + } + return chunks +}