✨ Read files content lazily [joelanford edits]

.github/workflows/go.yml

@@ -2,28 +2,26 @@ name: Go
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
 
 jobs:
-
   build:
     strategy:
       matrix:
-        go: ["1.16", "1.17", "1.18", "1.19"]
+        go: ["1.17", "1.18", "1.19", "1.20"]
         os: ["ubuntu-latest", "windows-latest", "macos-latest"]
     runs-on: ${{matrix.os}}
     steps:
-    - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-          stable: false
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
           go-version: ${{matrix.go}}
 
-    - name: Build
-      run: go build -v ./...
+      - name: Build
+        run: go build -v ./...
 
-    - name: Test
-      run: go test -v ./...
+      - name: Test
+        run: go test -v ./...

README.md

@@ -9,7 +9,7 @@
 
 ## Usage
 
-⚠️ go-tarfs needs go>=1.16
+⚠️ go-tarfs needs go>=1.17
 
 Install:
 ```sh

benchmarks/.gitignore

@@ -0,0 +1 @@
+*.tar

benchmarks/benchmarks_test.go

@@ -0,0 +1,270 @@
+package tarfs
+
+import (
+	"archive/tar"
+	"io"
+	"io/fs"
+	"math/rand"
+	"os"
+	"testing"
+
+	"github.com/nlepage/go-tarfs"
+)
+
+const chars = "abcdefghijklmnopqrstuvxyzABCDEFGHIJKLMNOPQRSTUVXYZ0123456789"
+
+var (
+	randomFileName = make(map[string]string)
+)
+
+func TestMain(m *testing.M) {
+	rand.Seed(3827653748965)
+	generateTarFile("many-small-files.tar", 10000, 1, 10000)
+	generateTarFile("few-large-files.tar", 10, 10000000, 100000000)
+	os.Exit(m.Run())
+}
+
+func BenchmarkOpenTarThenReadFile_ManySmallFiles(b *testing.B) {
+	fileName := randomFileName["many-small-files.tar"]
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		openTarThenReadFile("many-small-files.tar", fileName)
+	}
+}
+
+func BenchmarkOpenTarThenReadFile_ManySmallFiles_DisableSeek(b *testing.B) {
+	fileName := randomFileName["many-small-files.tar"]
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		openTarThenReadFile("many-small-files.tar", fileName, tarfs.DisableSeek(true))
+	}
+}
+
+func BenchmarkOpenTarThenReadFile_FewLargeFiles(b *testing.B) {
+	fileName := randomFileName["few-large-files.tar"]
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		openTarThenReadFile("few-large-files.tar", fileName)
+	}
+}
+
+func BenchmarkOpenTarThenReadFile_FewLargeFiles_DisableSeek(b *testing.B) {
+	fileName := randomFileName["few-large-files.tar"]
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		openTarThenReadFile("few-large-files.tar", fileName, tarfs.DisableSeek(true))
+	}
+}
+
+func BenchmarkReadFile_ManySmallFiles(b *testing.B) {
+	benchmarkReadFile(b, "many-small-files.tar")
+}
+
+func BenchmarkReadFile_ManySmallFiles_DisableSeek(b *testing.B) {
+	benchmarkReadFile(b, "many-small-files.tar", tarfs.DisableSeek(true))
+}
+
+func BenchmarkReadFile_FewLargeFiles(b *testing.B) {
+	benchmarkReadFile(b, "few-large-files.tar")
+}
+
+func BenchmarkReadFile_FewLargeFiles_DisableSeek(b *testing.B) {
+	benchmarkReadFile(b, "few-large-files.tar", tarfs.DisableSeek(true))
+}
+
+func BenchmarkOpenReadAndCloseFile_ManySmallFiles(b *testing.B) {
+	benchmarkOpenReadAndCloseFile(b, "many-small-files.tar")
+}
+
+func BenchmarkOpenReadAndCloseFile_ManySmallFiles_DisableSeek(b *testing.B) {
+	benchmarkOpenReadAndCloseFile(b, "many-small-files.tar", tarfs.DisableSeek(true))
+}
+
+func BenchmarkOpenReadAndCloseFile_FewLargeFiles(b *testing.B) {
+	benchmarkOpenReadAndCloseFile(b, "few-large-files.tar")
+}
+
+func BenchmarkOpenReadAndCloseFile_FewLargeFiles_DisableSeek(b *testing.B) {
+	benchmarkOpenReadAndCloseFile(b, "few-large-files.tar", tarfs.DisableSeek(true))
+}
+
+func BenchmarkOpenAndCloseFile_ManySmallFiles(b *testing.B) {
+	benchmarkOpenAndCloseFile(b, "many-small-files.tar")
+}
+
+func BenchmarkOpenAndCloseFile_ManySmallFiles_DisableSeek(b *testing.B) {
+	benchmarkOpenAndCloseFile(b, "many-small-files.tar", tarfs.DisableSeek(true))
+}
+
+func BenchmarkOpenAndCloseFile_FewLargeFiles(b *testing.B) {
+	benchmarkOpenAndCloseFile(b, "few-large-files.tar")
+}
+
+func BenchmarkOpenAndCloseFile_FewLargeFiles_DisableSeek(b *testing.B) {
+	benchmarkOpenAndCloseFile(b, "few-large-files.tar", tarfs.DisableSeek(true))
+}
+
+func benchmarkReadFile(b *testing.B, tarFileName string, options ...tarfs.Option) {
+	tf, err := os.Open(tarFileName)
+	if err != nil {
+		panic(err)
+	}
+	defer tf.Close()
+
+	tfs, err := tarfs.New(tf, options...)
+	if err != nil {
+		panic(err)
+	}
+
+	fileName := randomFileName[tarFileName]
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		if _, err := fs.ReadFile(tfs, fileName); err != nil {
+			panic(err)
+		}
+	}
+}
+
+func benchmarkOpenAndCloseFile(b *testing.B, tarFileName string, options ...tarfs.Option) {
+	tf, err := os.Open(tarFileName)
+	if err != nil {
+		panic(err)
+	}
+	defer tf.Close()
+
+	tfs, err := tarfs.New(tf, options...)
+	if err != nil {
+		panic(err)
+	}
+
+	fileName := randomFileName[tarFileName]
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		f, err := tfs.Open(fileName)
+		if err != nil {
+			panic(err)
+		}
+		if err := f.Close(); err != nil {
+			panic(err)
+		}
+	}
+}
+
+func benchmarkOpenReadAndCloseFile(b *testing.B, tarFileName string, options ...tarfs.Option) {
+	tf, err := os.Open(tarFileName)
+	if err != nil {
+		panic(err)
+	}
+	defer tf.Close()
+
+	tfs, err := tarfs.New(tf, options...)
+	if err != nil {
+		panic(err)
+	}
+
+	fileName := randomFileName[tarFileName]
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		f, err := tfs.Open(fileName)
+		if err != nil {
+			panic(err)
+		}
+		st, err := f.Stat()
+		if err != nil {
+			panic(err)
+		}
+		buf := make([]byte, st.Size())
+		if _, err := io.ReadFull(f, buf); err != nil {
+			panic(err)
+		}
+		if err := f.Close(); err != nil {
+			panic(err)
+		}
+	}
+}
+
+func openTarThenReadFile(tarName, fileName string, options ...tarfs.Option) {
+	tf, err := os.Open(tarName)
+	if err != nil {
+		panic(err)
+	}
+	defer tf.Close()
+
+	var tfs fs.FS
+
+	tfs, err = tarfs.New(tf, options...)
+	if err != nil {
+		panic(err)
+	}
+
+	if _, err := fs.ReadFile(tfs, fileName); err != nil {
+		panic(err)
+	}
+}
+
+func generateTarFile(tarName string, numFiles int, minSize, maxSize int) {
+	f, err := os.Create(tarName)
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+
+	w := tar.NewWriter(f)
+	buf := make([]byte, 1024)
+	randomFileIndex := rand.Intn(numFiles)
+	defer w.Close()
+
+	for i := 0; i < numFiles; i++ {
+		nameLength := rand.Intn(100) + 10
+		fileName := ""
+		for j := 0; j < nameLength; j++ {
+			fileName += string(chars[rand.Intn(len(chars))])
+		}
+
+		if i == randomFileIndex {
+			randomFileName[tarName] = fileName
+		}
+
+		bytesToWrite := rand.Intn(maxSize-minSize) + minSize
+
+		if err := w.WriteHeader(&tar.Header{
+			Name:     fileName,
+			Typeflag: tar.TypeReg,
+			Size:     int64(bytesToWrite),
+		}); err != nil {
+			panic(err)
+		}
+
+		for bytesToWrite != 0 {
+			if _, err := rand.Read(buf); err != nil {
+				panic(err)
+			}
+
+			if bytesToWrite < 1024 {
+				if _, err := w.Write(buf[:bytesToWrite]); err != nil {
+					panic(err)
+				}
+				bytesToWrite = 0
+			} else {
+				if _, err := w.Write(buf); err != nil {
+					panic(err)
+				}
+				bytesToWrite -= 1024
+			}
+		}
+	}
+}

errors.go

@@ -11,10 +11,22 @@ var (
 	ErrDir    = errors.New("is a directory")
 )
 
-func newErrNotDir(op, name string) error {
-	return &fs.PathError{Op: op, Path: name, Err: ErrNotDir}
+func newErrNotDir(op, path string) error {
+	return newErr(op, path, ErrNotDir)
 }
 
-func newErrDir(op, name string) error {
-	return &fs.PathError{Op: op, Path: name, Err: ErrDir}
+func newErrDir(op, path string) error {
+	return newErr(op, path, ErrDir)
+}
+
+func newErrClosed(op, path string) error {
+	return newErr(op, path, fs.ErrClosed)
+}
+
+func newErrNotExist(op, path string) error {
+	return newErr(op, path, fs.ErrNotExist)
+}
+
+func newErr(op, path string, err error) error {
+	return &fs.PathError{Op: op, Path: path, Err: err}
 }

example_stat_test.go

@@ -16,7 +16,7 @@ func Example_stat() {
 	}
 	defer tf.Close()
 
-	tfs, err := tarfs.New(tf)
+	tfs, err := tarfs.New(tf, tarfs.DisableSeek(true))
 	if err != nil {
 		panic(err)
 	}