increased gateway timeout

internal/sync/main.go

@@ -30,7 +30,7 @@ func init() {
 	SyncCmd.MarkFlagRequired("destination")
 	SyncCmd.Flags().StringP("from-file", "f", "", "Sync CID's from file")
 	SyncCmd.Flags().IntP("batch", "b", 100, "Batch files to sync in paralel")
-	SyncCmd.Flags().IntP("cooldown", "c", 0, "Cooldown in seconds between the batches, by default not used. Only used with baches options, ignored otherwise")
+	SyncCmd.Flags().IntP("cooldown", "c", 0, "Cooldown in seconds between the batches, by default not used")
 }
 
 func Sync(cmd *cobra.Command) {
@@ -43,7 +43,7 @@ func Sync(cmd *cobra.Command) {
 	// check if syncing only the CIDS specified in the file
 	fromFile, err := cmd.Flags().GetString("from-file")
 	if err != nil {
-		fmt.Println(err)
+		log.Println(err)
 	}
 
 	// get source to sync from
@@ -63,17 +63,17 @@ func Sync(cmd *cobra.Command) {
 	}
 
 	if cooldown < 0 {
-		fmt.Printf("The specified cooldown is not valid, it must be greater or equal to 0. Specified %d", cooldown)
+		log.Printf("The specified cooldown is not valid, it must be greater or equal to 0. Specified %d", cooldown)
 		os.Exit(1)
 	}
 
 	batch, err := cmd.Flags().GetInt("batch")
 	if err != nil {
-		fmt.Println(err)
+		log.Println(err)
 	}
+
 	if batch <= 0 {
-		fmt.Printf("The specified batch is not valid, it must be greater than 0. Specified %d", batch)
-		cooldown = 0
+		log.Printf("The specified batch is not valid, it must be greater than 0. Specified %d", batch)
 		os.Exit(1)
 	}
 
@@ -82,14 +82,14 @@ func Sync(cmd *cobra.Command) {
 		log.Printf("Syncing from <%s> to <%s> using as input the file <%s>\n", src, dst, fromFile)
 		c, err := utils.ReadCIDFromFile(fromFile)
 		if err != nil {
-			fmt.Println(err)
+			log.Println(err)
 			os.Exit(1)
 		}
 
 		// Create our structure with the CID's
 		cids, err = utils.SliceToCIDSStruct(c)
 		if err != nil {
-			fmt.Println(err)
+			log.Println(err)
 		}
 	} else {
 		log.Printf("Syncing from %s to %s\n", src, dst)
@@ -101,7 +101,7 @@ func Sync(cmd *cobra.Command) {
 		// TODO: implement retry backoff with pester
 		resL, err := utils.PostCID(listURL, nil, "")
 		if err != nil {
-			fmt.Println(err)
+			log.Println(err)
 		}
 		defer resL.Body.Close()
 
@@ -111,7 +111,7 @@ func Sync(cmd *cobra.Command) {
 			var j utils.IPFSCIDResponse
 			err := json.Unmarshal(scanner.Bytes(), &j)
 			if err != nil {
-				fmt.Printf("Error unmarshaling the response: %s", err)
+				log.Printf("Error unmarshaling the response: %s", err)
 			}
 			cids = append(cids, j)
 		}

internal/utils/types.go

@@ -2,6 +2,8 @@ package utils
 
 import "net/http"
 
+var GATEWAY_TIMEOUT_HEADERS int8 = 60
+
 var DIR_LIST_ENDPOINT string = "/ipfs/api/v0/ls?arg="
 var PIN_LIST_ENDPOINT string = "/ipfs/api/v0/pin/ls?stream=true"
 var CAT_ENDPOINT string = "/ipfs/api/v0/cat?arg="
@@ -34,8 +36,7 @@ type HTTPResult struct {
 }
 
 type IPFSCIDResponse struct {
-	Cid  string `json:"cid"`
-	Type string `json:"type"`
+	Cid string `json:"cid"`
 }
 
 type IPFSResponse struct {

internal/utils/utils.go

@@ -18,6 +18,7 @@ import (
 
 func createTempDirWithFile(f []string) (*os.File, error) {
 	dir := f[:len(f)-1]
+
 	// Create the directories for the CID structure
 	err := os.MkdirAll(strings.Join(dir, "/"), 0755)
 	if err != nil {
@@ -35,7 +36,7 @@ func createTempDirWithFile(f []string) (*os.File, error) {
 func GetCID(url string, payload io.Reader) (*http.Response, error) {
 	req, err := http.NewRequest(http.MethodPost, url, payload)
 	if err != nil {
-		return nil, fmt.Errorf("Error creating HTTP request: %s", err)
+		return nil, fmt.Errorf("error creating HTTP request: %s", err)
 	}
 
 	// Set custom User-Agent for cloudflare WAF policies
@@ -46,17 +47,17 @@ func GetCID(url string, payload io.Reader) (*http.Response, error) {
 
 	res, err := client.Do(req)
 	if err != nil {
-		return nil, fmt.Errorf("Error making API request: %s", err)
+		return nil, fmt.Errorf("error making API request: %s", err)
 	}
 
 	if s := res.Status; strings.HasPrefix(s, "5") || strings.HasPrefix(s, "4") {
 		// Check if the error is due to the CID being a directory
 		var dirIPFS IPFSErrorResponse
 		_ = UnmarshalToStruct[IPFSErrorResponse](res.Body, &dirIPFS)
 		if dirIPFS.Message == DIR_ERROR {
-			return nil, fmt.Errorf("Cannot get this IPFS CID. Error message: %s", dirIPFS.Message)
+			return nil, fmt.Errorf("cannot get this IPFS CID. Error message: %s", dirIPFS.Message)
 		} else {
-			return nil, fmt.Errorf("There was an error with the request. Error code: HTTP %s", s)
+			return nil, fmt.Errorf("there was an error with the request. Error code: HTTP %s", s)
 		}
 	}
 
@@ -66,6 +67,7 @@ func GetCID(url string, payload io.Reader) (*http.Response, error) {
 func PostCID(dst string, payload []byte, fPath string) (*http.Response, error) {
 	var tempFileName []string
 	var base string
+
 	if len(fPath) != 0 {
 		tempFileName = strings.Split(fPath, "/")
 		base = tempFileName[0]
@@ -83,22 +85,22 @@ func PostCID(dst string, payload []byte, fPath string) (*http.Response, error) {
 	// Create a temporary file to store the IPFS object data
 	tempFile, err := createTempDirWithFile(tempFileName)
 	if err != nil {
-		return nil, fmt.Errorf("Error creating temporary file: %s", err)
+		return nil, fmt.Errorf("error creating temporary file: %s", err)
 	}
 	defer tempFile.Close()
 
 	// Write the IPFS object data to the temporary file
 	_, err = tempFile.Write(payload)
 	if err != nil {
-		return nil, fmt.Errorf("Error writing data to temporary file: %s", err)
+		return nil, fmt.Errorf("error writing data to temporary file: %s", err)
 	}
 
 	// Create a new HTTP POST request to add the file to the destination
 	body := &bytes.Buffer{}
 	writer := multipart.NewWriter(body)
 	filePart, err := writer.CreateFormFile("file", strings.Join(tempFileName, "/"))
 	if err != nil {
-		return nil, fmt.Errorf("Error creating form file: %s", err)
+		return nil, fmt.Errorf("error creating form file: %s", err)
 	}
 
 	// Reset the temporary file pointer to the beginning
@@ -107,14 +109,14 @@ func PostCID(dst string, payload []byte, fPath string) (*http.Response, error) {
 	// Copy the temporary file data into the form file
 	_, err = io.Copy(filePart, tempFile)
 	if err != nil {
-		return nil, fmt.Errorf("Error copying file data: %s", err)
+		return nil, fmt.Errorf("error copying file data: %s", err)
 	}
 
 	writer.Close() // Close the multipart writer
 
 	req, err := http.NewRequest(http.MethodPost, dst, body)
 	if err != nil {
-		return nil, fmt.Errorf("There was an error creating the HTTP request: %s", err)
+		return nil, fmt.Errorf("there was an error creating the HTTP request: %s", err)
 	}
 
 	// Set custom User-Agent for cloudflare WAF policies
@@ -133,16 +135,21 @@ func PostCID(dst string, payload []byte, fPath string) (*http.Response, error) {
 	}
 	defer os.RemoveAll(base)
 
+	// configure http transport
+	tr := &http.Transport{
+		DisableCompression:    false,
+		ResponseHeaderTimeout: time.Duration(time.Second * time.Duration(GATEWAY_TIMEOUT_HEADERS)),
+	}
 	// Create an HTTP client
-	client := &http.Client{}
+	client := &http.Client{Transport: tr}
 
 	res, err := client.Do(req)
 	if err != nil {
-		return nil, fmt.Errorf("Error making API request: %s", err)
+		return nil, fmt.Errorf("error making API request: %s", err)
 	}
 
 	if s := res.Status; strings.HasPrefix(s, "5") || strings.HasPrefix(s, "4") {
-		return nil, fmt.Errorf("The endpoint responded with: HTTP %s", s)
+		return nil, fmt.Errorf("the endpoint responded with: HTTP %s", s)
 	}
 
 	return res, nil
@@ -152,7 +159,7 @@ func ParseHTTPBody(h *http.Response) ([]byte, error) {
 	// Read the body response
 	body, err := ioutil.ReadAll(h.Body)
 	if err != nil {
-		return nil, fmt.Errorf("Error reading response body: %s", err)
+		return nil, fmt.Errorf("error reading response body: %s", err)
 	}
 
 	return body, nil
@@ -168,7 +175,7 @@ func GetCIDVersion(cid string) string {
 
 func TestIPFSHash(s string, d string) error {
 	if s != d {
-		return fmt.Errorf("The source IPFS Hash is different from the destination Hash%s", "")
+		return fmt.Errorf("the source IPFS Hash is different from the destination Hash%s", "")
 	}
 
 	return nil
@@ -187,7 +194,7 @@ func SliceToCIDSStruct(s []string) ([]IPFSCIDResponse, error) {
 		a := fmt.Sprintf(`{"cid":"%s"}`, k)
 		err := json.Unmarshal([]byte(a), &cid)
 		if err != nil {
-			return nil, fmt.Errorf("Error unmarshaling from slice to IPFS Struct: %s", err)
+			return nil, fmt.Errorf("error unmarshaling from slice to IPFS Struct: %s", err)
 		}
 		cids = append(cids, cid)
 	}
@@ -200,7 +207,7 @@ func UnmarshalToStruct[V Data | IPFSResponse | IPFSCIDResponse | IPFSErrorRespon
 	for scanner.Scan() {
 		err := json.Unmarshal(scanner.Bytes(), &m)
 		if err != nil {
-			return fmt.Errorf("Error Unmarshaling the structure: %s", err)
+			return fmt.Errorf("error Unmarshaling the structure: %s", err)
 		}
 	}
 
@@ -226,18 +233,22 @@ func UnmarshalIPFSResponse(h io.ReadCloser, m *[]IPFSResponse) error {
 func ReadCIDFromFile(f string) ([]string, error) {
 	file, err := os.Open(f)
 	if err != nil {
-		return nil, fmt.Errorf("Error opening the file <%s>", f)
+		return nil, fmt.Errorf("error opening the file <%s>", f)
 	}
 	defer file.Close()
 
 	var s []string
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
-		txt := scanner.Text()
-		if len(txt) > 0 {
+		line := scanner.Text()
+		if len(line) > 0 {
 			s = append(s, scanner.Text())
 		}
 	}
 
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("there was an error reading from file <%s>: %s", f, err)
+	}
+
 	return s, nil
 }