From 002adcd22485a6e3560f66a6129a0a7ffb7ecd91 Mon Sep 17 00:00:00 2001 From: Pete Emerson Date: Wed, 21 Oct 2020 16:41:29 -0700 Subject: [PATCH 1/5] Show file count --- file.go | 18 +++++++++++------- file_test.go | 4 ++-- main.go | 12 ++++++++++-- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/file.go b/file.go index 4619e05..3374c91 100644 --- a/file.go +++ b/file.go @@ -82,12 +82,12 @@ func shouldExtractPathInZip(pathPrefix string, zipPath *zip.File) bool { } // Decompress the file at zipFileAbsPath and move only those files under filesToExtractFromZipPath to localPath -func extractFiles(zipFilePath, filesToExtractFromZipPath, localPath string) error { +func extractFiles(zipFilePath, filesToExtractFromZipPath, localPath string) (int, error) { // Open the zip file for reading. r, err := zip.OpenReader(zipFilePath) if err != nil { - return err + return 0, err } defer r.Close() @@ -102,6 +102,9 @@ func extractFiles(zipFilePath, filesToExtractFromZipPath, localPath string) erro // Add the path from which we will extract files to the path prefix so we can exclude the appropriate files pathPrefix = filepath.Join(pathPrefix, filesToExtractFromZipPath) + // Count the number of files (not directories) unpacked + fileCount := 0 + // Iterate through the files in the archive, // printing some of their contents. for _, f := range r.File { @@ -114,30 +117,31 @@ func extractFiles(zipFilePath, filesToExtractFromZipPath, localPath string) erro path := filepath.Join(localPath, strings.TrimPrefix(f.Name, pathPrefix)) err = os.MkdirAll(path, 0777) if err != nil { - return fmt.Errorf("Failed to create local directory %s: %s", path, err) + return 0, fmt.Errorf("Failed to create local directory %s: %s", path, err) } } else { // Read the file into a byte array readCloser, err := f.Open() if err != nil { - return fmt.Errorf("Failed to open file %s: %s", f.Name, err) + return 0, fmt.Errorf("Failed to open file %s: %s", f.Name, err) } byteArray, err := ioutil.ReadAll(readCloser) if err != nil { - return fmt.Errorf("Failed to read file %s: %s", f.Name, err) + return 0, fmt.Errorf("Failed to read file %s: %s", f.Name, err) } // Write the file err = ioutil.WriteFile(filepath.Join(localPath, strings.TrimPrefix(f.Name, pathPrefix)), byteArray, 0644) if err != nil { - return fmt.Errorf("Failed to write file: %s", err) + return 0, fmt.Errorf("Failed to write file: %s", err) } + fileCount++ } } } - return nil + return fileCount, nil } // Return an HTTP request that will fetch the given GitHub repo's zip file for the given tag, possibly with the gitHubOAuthToken in the header diff --git a/file_test.go b/file_test.go index f78a264..c36e297 100644 --- a/file_test.go +++ b/file_test.go @@ -307,7 +307,7 @@ func TestExtractFiles(t *testing.T) { } defer os.RemoveAll(tempDir) - err = extractFiles(tc.localFilePath, tc.filePathToExtract, tempDir) + _, err = extractFiles(tc.localFilePath, tc.filePathToExtract, tempDir) if err != nil { t.Fatalf("Failed to extract files: %s", err) } @@ -352,7 +352,7 @@ func TestExtractFilesExtractFile(t *testing.T) { filePathToExtract := "zzz.txt" localFileName := "/localzzz.txt" localPathName := filepath.Join(tempDir, localFileName) - err = extractFiles(zipFilePath, filePathToExtract, localPathName) + _, err = extractFiles(zipFilePath, filePathToExtract, localPathName) if err != nil { t.Fatalf("Failed to extract files: %s", err) } diff --git a/main.go b/main.go index f7a2e58..0932dc5 100644 --- a/main.go +++ b/main.go @@ -288,10 +288,18 @@ func downloadSourcePaths(sourcePaths []string, destPath string, githubRepo GitHu // Unzip and move the files we need to our destination for _, sourcePath := range sourcePaths { - fmt.Printf("Extracting files from %s to %s ...\n", sourcePath, destPath) - if err := extractFiles(localZipFilePath, sourcePath, destPath); err != nil { + fmt.Printf("Extracting files from %s to %s ... ", sourcePath, destPath) + if fileCount, err := extractFiles(localZipFilePath, sourcePath, destPath); err != nil { + fmt.Println() return fmt.Errorf("Error occurred while extracting files from GitHub zip file: %s", err.Error()) + } else { + plural := "" + if fileCount != 1 { + plural = "s" + } + fmt.Printf("%d file%s extracted\n", fileCount, plural) } + } fmt.Println("Download and file extraction complete.") From 35745bedd2666046af1993d54de178c9c1fef927 Mon Sep 17 00:00:00 2001 From: Pete Emerson Date: Mon, 26 Oct 2020 12:14:45 -0700 Subject: [PATCH 2/5] Return number of files in all cases --- file.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/file.go b/file.go index 3374c91..fce4072 100644 --- a/file.go +++ b/file.go @@ -117,24 +117,24 @@ func extractFiles(zipFilePath, filesToExtractFromZipPath, localPath string) (int path := filepath.Join(localPath, strings.TrimPrefix(f.Name, pathPrefix)) err = os.MkdirAll(path, 0777) if err != nil { - return 0, fmt.Errorf("Failed to create local directory %s: %s", path, err) + return fileCount, fmt.Errorf("Failed to create local directory %s: %s", path, err) } } else { // Read the file into a byte array readCloser, err := f.Open() if err != nil { - return 0, fmt.Errorf("Failed to open file %s: %s", f.Name, err) + return fileCount, fmt.Errorf("Failed to open file %s: %s", f.Name, err) } byteArray, err := ioutil.ReadAll(readCloser) if err != nil { - return 0, fmt.Errorf("Failed to read file %s: %s", f.Name, err) + return fileCount, fmt.Errorf("Failed to read file %s: %s", f.Name, err) } // Write the file err = ioutil.WriteFile(filepath.Join(localPath, strings.TrimPrefix(f.Name, pathPrefix)), byteArray, 0644) if err != nil { - return 0, fmt.Errorf("Failed to write file: %s", err) + return fileCount, fmt.Errorf("Failed to write file: %s", err) } fileCount++ } From eaf98f5c6fb1705a6b21cbf24f5c29408962da61 Mon Sep 17 00:00:00 2001 From: Pete Emerson Date: Mon, 26 Oct 2020 12:18:15 -0700 Subject: [PATCH 3/5] Print number of files extracted regardless of error --- main.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/main.go b/main.go index 0932dc5..bf399a8 100644 --- a/main.go +++ b/main.go @@ -289,15 +289,14 @@ func downloadSourcePaths(sourcePaths []string, destPath string, githubRepo GitHu // Unzip and move the files we need to our destination for _, sourcePath := range sourcePaths { fmt.Printf("Extracting files from %s to %s ... ", sourcePath, destPath) - if fileCount, err := extractFiles(localZipFilePath, sourcePath, destPath); err != nil { - fmt.Println() + fileCount, err := extractFiles(localZipFilePath, sourcePath, destPath) + plural := "" + if fileCount != 1 { + plural = "s" + } + fmt.Printf("%d file%s extracted\n", fileCount, plural) + if err != nil { return fmt.Errorf("Error occurred while extracting files from GitHub zip file: %s", err.Error()) - } else { - plural := "" - if fileCount != 1 { - plural = "s" - } - fmt.Printf("%d file%s extracted\n", fileCount, plural) } } From 4b41500f2d9a6d6b15d95077578ebcd7a4655247 Mon Sep 17 00:00:00 2001 From: Pete Emerson Date: Mon, 26 Oct 2020 12:53:19 -0700 Subject: [PATCH 4/5] Add file count test when extracting files --- file_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/file_test.go b/file_test.go index c36e297..114ed8d 100644 --- a/file_test.go +++ b/file_test.go @@ -307,11 +307,15 @@ func TestExtractFiles(t *testing.T) { } defer os.RemoveAll(tempDir) - _, err = extractFiles(tc.localFilePath, tc.filePathToExtract, tempDir) + fileCount, err := extractFiles(tc.localFilePath, tc.filePathToExtract, tempDir) if err != nil { t.Fatalf("Failed to extract files: %s", err) } + if fileCount != tc.expectedNumFiles { + t.Fatalf("Expected to extract %d files, extracted %d instead", tc.expectedNumFiles, fileCount) + } + // Count the number of files in the directory var numFiles int filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error { From 3923a2c48fc61489b4456f93cac3e439653b1fcb Mon Sep 17 00:00:00 2001 From: Pete Emerson Date: Mon, 26 Oct 2020 13:09:22 -0700 Subject: [PATCH 5/5] Check file count on file extraction --- file_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/file_test.go b/file_test.go index 114ed8d..40e5f78 100644 --- a/file_test.go +++ b/file_test.go @@ -355,12 +355,18 @@ func TestExtractFilesExtractFile(t *testing.T) { zipFilePath := "test-fixtures/fetch-test-public-0.0.4.zip" filePathToExtract := "zzz.txt" localFileName := "/localzzz.txt" + expectedFileCount := 1 localPathName := filepath.Join(tempDir, localFileName) - _, err = extractFiles(zipFilePath, filePathToExtract, localPathName) + fileCount, err := extractFiles(zipFilePath, filePathToExtract, localPathName) + if err != nil { t.Fatalf("Failed to extract files: %s", err) } + if fileCount != expectedFileCount { + t.Fatalf("Expected to extract %d files, extracted %d instead", expectedFileCount, fileCount) + } + filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error { relativeFilename := strings.TrimPrefix(path, tempDir)