diff --git a/internal/sync/main.go b/internal/sync/main.go index 9cf9701..dcd5a3b 100644 --- a/internal/sync/main.go +++ b/internal/sync/main.go @@ -30,7 +30,7 @@ func init() { SyncCmd.MarkFlagRequired("destination") SyncCmd.Flags().StringP("from-file", "f", "", "Sync CID's from file") SyncCmd.Flags().IntP("batch", "b", 100, "Batch files to sync in paralel") - SyncCmd.Flags().IntP("cooldown", "c", 0, "Cooldown in seconds between the batches, by default not used. Only used with baches options, ignored otherwise") + SyncCmd.Flags().IntP("cooldown", "c", 0, "Cooldown in seconds between the batches, by default not used") } func Sync(cmd *cobra.Command) { @@ -43,7 +43,7 @@ func Sync(cmd *cobra.Command) { // check if syncing only the CIDS specified in the file fromFile, err := cmd.Flags().GetString("from-file") if err != nil { - fmt.Println(err) + log.Println(err) } // get source to sync from @@ -63,17 +63,17 @@ func Sync(cmd *cobra.Command) { } if cooldown < 0 { - fmt.Printf("The specified cooldown is not valid, it must be greater or equal to 0. Specified %d", cooldown) + log.Printf("The specified cooldown is not valid, it must be greater or equal to 0. Specified %d", cooldown) os.Exit(1) } batch, err := cmd.Flags().GetInt("batch") if err != nil { - fmt.Println(err) + log.Println(err) } + if batch <= 0 { - fmt.Printf("The specified batch is not valid, it must be greater than 0. Specified %d", batch) - cooldown = 0 + log.Printf("The specified batch is not valid, it must be greater than 0. Specified %d", batch) os.Exit(1) } @@ -82,14 +82,14 @@ func Sync(cmd *cobra.Command) { log.Printf("Syncing from <%s> to <%s> using as input the file <%s>\n", src, dst, fromFile) c, err := utils.ReadCIDFromFile(fromFile) if err != nil { - fmt.Println(err) + log.Println(err) os.Exit(1) } // Create our structure with the CID's cids, err = utils.SliceToCIDSStruct(c) if err != nil { - fmt.Println(err) + log.Println(err) } } else { log.Printf("Syncing from %s to %s\n", src, dst) @@ -101,7 +101,7 @@ func Sync(cmd *cobra.Command) { // TODO: implement retry backoff with pester resL, err := utils.PostCID(listURL, nil, "") if err != nil { - fmt.Println(err) + log.Println(err) } defer resL.Body.Close() @@ -111,7 +111,7 @@ func Sync(cmd *cobra.Command) { var j utils.IPFSCIDResponse err := json.Unmarshal(scanner.Bytes(), &j) if err != nil { - fmt.Printf("Error unmarshaling the response: %s", err) + log.Printf("Error unmarshaling the response: %s", err) } cids = append(cids, j) } diff --git a/internal/utils/types.go b/internal/utils/types.go index c00fd6a..5d3146a 100644 --- a/internal/utils/types.go +++ b/internal/utils/types.go @@ -2,6 +2,8 @@ package utils import "net/http" +var GATEWAY_TIMEOUT_HEADERS int8 = 60 + var DIR_LIST_ENDPOINT string = "/ipfs/api/v0/ls?arg=" var PIN_LIST_ENDPOINT string = "/ipfs/api/v0/pin/ls?stream=true" var CAT_ENDPOINT string = "/ipfs/api/v0/cat?arg=" @@ -34,8 +36,7 @@ type HTTPResult struct { } type IPFSCIDResponse struct { - Cid string `json:"cid"` - Type string `json:"type"` + Cid string `json:"cid"` } type IPFSResponse struct { diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 6126e89..dc42a8f 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -18,6 +18,7 @@ import ( func createTempDirWithFile(f []string) (*os.File, error) { dir := f[:len(f)-1] + // Create the directories for the CID structure err := os.MkdirAll(strings.Join(dir, "/"), 0755) if err != nil { @@ -35,7 +36,7 @@ func createTempDirWithFile(f []string) (*os.File, error) { func GetCID(url string, payload io.Reader) (*http.Response, error) { req, err := http.NewRequest(http.MethodPost, url, payload) if err != nil { - return nil, fmt.Errorf("Error creating HTTP request: %s", err) + return nil, fmt.Errorf("error creating HTTP request: %s", err) } // Set custom User-Agent for cloudflare WAF policies @@ -46,7 +47,7 @@ func GetCID(url string, payload io.Reader) (*http.Response, error) { res, err := client.Do(req) if err != nil { - return nil, fmt.Errorf("Error making API request: %s", err) + return nil, fmt.Errorf("error making API request: %s", err) } if s := res.Status; strings.HasPrefix(s, "5") || strings.HasPrefix(s, "4") { @@ -54,9 +55,9 @@ func GetCID(url string, payload io.Reader) (*http.Response, error) { var dirIPFS IPFSErrorResponse _ = UnmarshalToStruct[IPFSErrorResponse](res.Body, &dirIPFS) if dirIPFS.Message == DIR_ERROR { - return nil, fmt.Errorf("Cannot get this IPFS CID. Error message: %s", dirIPFS.Message) + return nil, fmt.Errorf("cannot get this IPFS CID. Error message: %s", dirIPFS.Message) } else { - return nil, fmt.Errorf("There was an error with the request. Error code: HTTP %s", s) + return nil, fmt.Errorf("there was an error with the request. Error code: HTTP %s", s) } } @@ -66,6 +67,7 @@ func GetCID(url string, payload io.Reader) (*http.Response, error) { func PostCID(dst string, payload []byte, fPath string) (*http.Response, error) { var tempFileName []string var base string + if len(fPath) != 0 { tempFileName = strings.Split(fPath, "/") base = tempFileName[0] @@ -83,14 +85,14 @@ func PostCID(dst string, payload []byte, fPath string) (*http.Response, error) { // Create a temporary file to store the IPFS object data tempFile, err := createTempDirWithFile(tempFileName) if err != nil { - return nil, fmt.Errorf("Error creating temporary file: %s", err) + return nil, fmt.Errorf("error creating temporary file: %s", err) } defer tempFile.Close() // Write the IPFS object data to the temporary file _, err = tempFile.Write(payload) if err != nil { - return nil, fmt.Errorf("Error writing data to temporary file: %s", err) + return nil, fmt.Errorf("error writing data to temporary file: %s", err) } // Create a new HTTP POST request to add the file to the destination @@ -98,7 +100,7 @@ func PostCID(dst string, payload []byte, fPath string) (*http.Response, error) { writer := multipart.NewWriter(body) filePart, err := writer.CreateFormFile("file", strings.Join(tempFileName, "/")) if err != nil { - return nil, fmt.Errorf("Error creating form file: %s", err) + return nil, fmt.Errorf("error creating form file: %s", err) } // Reset the temporary file pointer to the beginning @@ -107,14 +109,14 @@ func PostCID(dst string, payload []byte, fPath string) (*http.Response, error) { // Copy the temporary file data into the form file _, err = io.Copy(filePart, tempFile) if err != nil { - return nil, fmt.Errorf("Error copying file data: %s", err) + return nil, fmt.Errorf("error copying file data: %s", err) } writer.Close() // Close the multipart writer req, err := http.NewRequest(http.MethodPost, dst, body) if err != nil { - return nil, fmt.Errorf("There was an error creating the HTTP request: %s", err) + return nil, fmt.Errorf("there was an error creating the HTTP request: %s", err) } // Set custom User-Agent for cloudflare WAF policies @@ -133,16 +135,21 @@ func PostCID(dst string, payload []byte, fPath string) (*http.Response, error) { } defer os.RemoveAll(base) + // configure http transport + tr := &http.Transport{ + DisableCompression: false, + ResponseHeaderTimeout: time.Duration(time.Second * time.Duration(GATEWAY_TIMEOUT_HEADERS)), + } // Create an HTTP client - client := &http.Client{} + client := &http.Client{Transport: tr} res, err := client.Do(req) if err != nil { - return nil, fmt.Errorf("Error making API request: %s", err) + return nil, fmt.Errorf("error making API request: %s", err) } if s := res.Status; strings.HasPrefix(s, "5") || strings.HasPrefix(s, "4") { - return nil, fmt.Errorf("The endpoint responded with: HTTP %s", s) + return nil, fmt.Errorf("the endpoint responded with: HTTP %s", s) } return res, nil @@ -152,7 +159,7 @@ func ParseHTTPBody(h *http.Response) ([]byte, error) { // Read the body response body, err := ioutil.ReadAll(h.Body) if err != nil { - return nil, fmt.Errorf("Error reading response body: %s", err) + return nil, fmt.Errorf("error reading response body: %s", err) } return body, nil @@ -168,7 +175,7 @@ func GetCIDVersion(cid string) string { func TestIPFSHash(s string, d string) error { if s != d { - return fmt.Errorf("The source IPFS Hash is different from the destination Hash%s", "") + return fmt.Errorf("the source IPFS Hash is different from the destination Hash%s", "") } return nil @@ -187,7 +194,7 @@ func SliceToCIDSStruct(s []string) ([]IPFSCIDResponse, error) { a := fmt.Sprintf(`{"cid":"%s"}`, k) err := json.Unmarshal([]byte(a), &cid) if err != nil { - return nil, fmt.Errorf("Error unmarshaling from slice to IPFS Struct: %s", err) + return nil, fmt.Errorf("error unmarshaling from slice to IPFS Struct: %s", err) } cids = append(cids, cid) } @@ -200,7 +207,7 @@ func UnmarshalToStruct[V Data | IPFSResponse | IPFSCIDResponse | IPFSErrorRespon for scanner.Scan() { err := json.Unmarshal(scanner.Bytes(), &m) if err != nil { - return fmt.Errorf("Error Unmarshaling the structure: %s", err) + return fmt.Errorf("error Unmarshaling the structure: %s", err) } } @@ -226,18 +233,22 @@ func UnmarshalIPFSResponse(h io.ReadCloser, m *[]IPFSResponse) error { func ReadCIDFromFile(f string) ([]string, error) { file, err := os.Open(f) if err != nil { - return nil, fmt.Errorf("Error opening the file <%s>", f) + return nil, fmt.Errorf("error opening the file <%s>", f) } defer file.Close() var s []string scanner := bufio.NewScanner(file) for scanner.Scan() { - txt := scanner.Text() - if len(txt) > 0 { + line := scanner.Text() + if len(line) > 0 { s = append(s, scanner.Text()) } } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("there was an error reading from file <%s>: %s", f, err) + } + return s, nil }