perf: update fs utils to use fastwalk

analyzer.go

@@ -61,17 +61,17 @@ func AnalyzeProject(projectDir string, path string) []*analyzerapi.ProjectModule
 
 func initAnalyzers() {
 	analyzerapi.Analyzers = append(analyzerapi.Analyzers,
+		cargo.Analyzer{},
+		composer.Analyzer{},
 		container.Analyzer{},
+		dotnet.Analyzer{},
 		gomod.Analyzer{},
 		gradle.Analyzer{},
-		maven.Analyzer{},
 		helm.Analyzer{},
 		hugo.Analyzer{},
+		maven.Analyzer{},
+		mkdocs.Analyzer{},
 		node.Analyzer{},
 		python.Analyzer{},
-		mkdocs.Analyzer{},
-		dotnet.Analyzer{},
-		composer.Analyzer{},
-		cargo.Analyzer{},
 	)
 }

analyzer/cargo/cargo_test.go

@@ -4,12 +4,11 @@ import (
 	"testing"
 
 	"github.com/cidverse/repoanalyzer/analyzerapi"
-	"github.com/cidverse/repoanalyzer/util"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestAnalyzer_AnalyzeCargo(t *testing.T) {
-	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "cargo"))
+	ctx := analyzerapi.GetAnalyzerContext(analyzerapi.GetTestDataDir(t, "cargo"))
 
 	analyzer := Analyzer{}
 	result := analyzer.Analyze(ctx)

analyzer/composer/composer_test.go

@@ -4,12 +4,11 @@ import (
 	"testing"
 
 	"github.com/cidverse/repoanalyzer/analyzerapi"
-	"github.com/cidverse/repoanalyzer/util"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestAnalyzer_AnalyzeComposer(t *testing.T) {
-	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "composer"))
+	ctx := analyzerapi.GetAnalyzerContext(analyzerapi.GetTestDataDir(t, "composer"))
 
 	analyzer := Analyzer{}
 	result := analyzer.Analyze(ctx)

analyzer/container/container.go

@@ -4,7 +4,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/cidverse/repoanalyzer/util"
+	"github.com/cidverse/cidverseutils/filesystem"
 	"github.com/rs/zerolog/log"
 	"golang.org/x/exp/slices"
 
@@ -57,7 +57,7 @@ func (a Analyzer) Analyze(ctx analyzerapi.AnalyzerContext) []*analyzerapi.Projec
 		filename := filepath.Base(file)
 
 		if strings.HasSuffix(filename, ".sh") {
-			content, contentErr := util.GetFileContent(file)
+			content, contentErr := filesystem.GetFileContent(file)
 			if contentErr != nil {
 				log.Warn().Str("file", file).Msg("failed to read file content")
 			} else if strings.Contains(content, "buildah from") {

analyzer/dotnet/dotnet_test.go

@@ -4,12 +4,11 @@ import (
 	"testing"
 
 	"github.com/cidverse/repoanalyzer/analyzerapi"
-	"github.com/cidverse/repoanalyzer/util"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestAnalyzer_AnalyzeVisualStudioSolution(t *testing.T) {
-	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "dotnet"))
+	ctx := analyzerapi.GetAnalyzerContext(analyzerapi.GetTestDataDir(t, "dotnet"))
 
 	analyzer := Analyzer{}
 	result := analyzer.Analyze(ctx)

analyzer/gomod/gomod.go

@@ -3,8 +3,7 @@ package gomod
 import (
 	"path/filepath"
 
-	"github.com/cidverse/repoanalyzer/util"
-
+	"github.com/cidverse/cidverseutils/filesystem"
 	"github.com/cidverse/repoanalyzer/analyzerapi"
 	"github.com/gosimple/slug"
 	"golang.org/x/mod/modfile"
@@ -25,7 +24,7 @@ func (a Analyzer) Analyze(ctx analyzerapi.AnalyzerContext) []*analyzerapi.Projec
 		// detect build system syntax
 		if filename == "go.mod" {
 			// parse go.mod
-			contentBytes, contentReadErr := util.GetFileBytes(file)
+			contentBytes, contentReadErr := filesystem.GetFileBytes(file)
 			if contentReadErr != nil {
 				continue
 			}

analyzer/gradle/gradle_test.go

@@ -5,13 +5,12 @@ import (
 	"testing"
 
 	"github.com/cidverse/repoanalyzer/analyzerapi"
-	"github.com/cidverse/repoanalyzer/util"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestGradleAnalyzer_AnalyzeGroovy(t *testing.T) {
 	_ = os.Setenv("REPOANAYLZER_DEBUG", "true")
-	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "gradle-groovy"))
+	ctx := analyzerapi.GetAnalyzerContext(analyzerapi.GetTestDataDir(t, "gradle-groovy"))
 	analyzer := Analyzer{}
 	result := analyzer.Analyze(ctx)
 
@@ -40,7 +39,7 @@ func TestGradleAnalyzer_AnalyzeGroovy(t *testing.T) {
 
 func TestGradleAnalyzer_AnalyzeKotlin(t *testing.T) {
 	_ = os.Setenv("REPOANAYLZER_DEBUG", "true")
-	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "gradle-kotlin"))
+	ctx := analyzerapi.GetAnalyzerContext(analyzerapi.GetTestDataDir(t, "gradle-kotlin"))
 	analyzer := Analyzer{}
 	result := analyzer.Analyze(ctx)
 

analyzer/helm/helm_test.go

@@ -3,14 +3,12 @@ package helm
 import (
 	"testing"
 
-	"github.com/cidverse/repoanalyzer/util"
-
 	"github.com/cidverse/repoanalyzer/analyzerapi"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestAnalyzer_AnalyzeHugo(t *testing.T) {
-	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "helm"))
+	ctx := analyzerapi.GetAnalyzerContext(analyzerapi.GetTestDataDir(t, "helm"))
 	analyzer := Analyzer{}
 	result := analyzer.Analyze(ctx)
 

analyzer/hugo/hugo.go

@@ -3,8 +3,7 @@ package hugo
 import (
 	"path/filepath"
 
-	"github.com/cidverse/repoanalyzer/util"
-
+	"github.com/cidverse/cidverseutils/filesystem"
 	"github.com/cidverse/repoanalyzer/analyzerapi"
 	"github.com/gosimple/slug"
 )
@@ -23,7 +22,7 @@ func (a Analyzer) Analyze(ctx analyzerapi.AnalyzerContext) []*analyzerapi.Projec
 		filename := filepath.Base(file)
 		if filename == "config.toml" || filename == "config.yaml" {
 			hugoDir := filepath.Dir(file)
-			if util.DirectoryExists(filepath.Join(hugoDir, "content")) {
+			if filesystem.DirectoryExists(filepath.Join(hugoDir, "content")) {
 				// module
 				module := analyzerapi.ProjectModule{
 					RootDirectory:     ctx.ProjectDir,

analyzer/hugo/hugo_test.go

@@ -3,14 +3,12 @@ package hugo
 import (
 	"testing"
 
-	"github.com/cidverse/repoanalyzer/util"
-
 	"github.com/cidverse/repoanalyzer/analyzerapi"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestAnalyzer_AnalyzeHugo(t *testing.T) {
-	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "hugo"))
+	ctx := analyzerapi.GetAnalyzerContext(analyzerapi.GetTestDataDir(t, "hugo"))
 
 	analyzer := Analyzer{}
 	result := analyzer.Analyze(ctx)

analyzer/maven/maven_test.go

@@ -4,12 +4,11 @@ import (
 	"testing"
 
 	"github.com/cidverse/repoanalyzer/analyzerapi"
-	"github.com/cidverse/repoanalyzer/util"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestAnalyzer_AnalyzeMaven(t *testing.T) {
-	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "maven"))
+	ctx := analyzerapi.GetAnalyzerContext(analyzerapi.GetTestDataDir(t, "maven"))
 
 	analyzer := Analyzer{}
 	result := analyzer.Analyze(ctx)

analyzer/mkdocs/mkdocs_test.go

@@ -4,12 +4,11 @@ import (
 	"testing"
 
 	"github.com/cidverse/repoanalyzer/analyzerapi"
-	"github.com/cidverse/repoanalyzer/util"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestAnalyzer_AnalyzeMkdocs(t *testing.T) {
-	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "mkdocs"))
+	ctx := analyzerapi.GetAnalyzerContext(analyzerapi.GetTestDataDir(t, "mkdocs"))
 
 	analyzer := Analyzer{}
 	result := analyzer.Analyze(ctx)
@@ -27,7 +26,7 @@ func TestAnalyzer_AnalyzeMkdocs(t *testing.T) {
 }
 
 func TestAnalyzer_AnalyzeTechdocs(t *testing.T) {
-	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "techdocs"))
+	ctx := analyzerapi.GetAnalyzerContext(analyzerapi.GetTestDataDir(t, "techdocs"))
 
 	analyzer := Analyzer{}
 	result := analyzer.Analyze(ctx)

analyzer/node/node_test.go

@@ -3,13 +3,11 @@ package node
 import (
 	"testing"
 
-	"github.com/cidverse/repoanalyzer/util"
-
 	"github.com/cidverse/repoanalyzer/analyzerapi"
 )
 
 func TestAnalyzer_AnalyzeReact(t *testing.T) {
-	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "react"))
+	ctx := analyzerapi.GetAnalyzerContext(analyzerapi.GetTestDataDir(t, "react"))
 	analyzer := Analyzer{}
 	result := analyzer.Analyze(ctx)
 

analyzer/python/python_test.go

@@ -3,14 +3,13 @@ package python
 import (
 	"testing"
 
-	"github.com/cidverse/repoanalyzer/util"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/cidverse/repoanalyzer/analyzerapi"
 )
 
 func TestAnalyzer_AnalyzePythonPipfile(t *testing.T) {
-	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "python-pipfile"))
+	ctx := analyzerapi.GetAnalyzerContext(analyzerapi.GetTestDataDir(t, "python-pipfile"))
 	analyzer := Analyzer{}
 	result := analyzer.Analyze(ctx)
 
@@ -27,7 +26,7 @@ func TestAnalyzer_AnalyzePythonPipfile(t *testing.T) {
 }
 
 func TestAnalyzer_AnalyzePythonPeotry(t *testing.T) {
-	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "python-poetry"))
+	ctx := analyzerapi.GetAnalyzerContext(analyzerapi.GetTestDataDir(t, "python-poetry"))
 	analyzer := Analyzer{}
 	result := analyzer.Analyze(ctx)
 
@@ -44,7 +43,7 @@ func TestAnalyzer_AnalyzePythonPeotry(t *testing.T) {
 }
 
 func TestAnalyzer_AnalyzePythonRequirementsTXT(t *testing.T) {
-	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "python-requirementstxt"))
+	ctx := analyzerapi.GetAnalyzerContext(analyzerapi.GetTestDataDir(t, "python-requirementstxt"))
 	analyzer := Analyzer{}
 	result := analyzer.Analyze(ctx)
 
@@ -61,7 +60,7 @@ func TestAnalyzer_AnalyzePythonRequirementsTXT(t *testing.T) {
 }
 
 func TestAnalyzer_AnalyzePythonSetuppy(t *testing.T) {
-	ctx := analyzerapi.GetAnalyzerContext(util.GetTestDataDir(t, "python-setuppy"))
+	ctx := analyzerapi.GetAnalyzerContext(analyzerapi.GetTestDataDir(t, "python-setuppy"))
 	analyzer := Analyzer{}
 	result := analyzer.Analyze(ctx)
 

analyzerapi/test.go

@@ -0,0 +1,16 @@
+package analyzerapi
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func GetTestDataDir(t *testing.T, dir string) string {
+	cwd, err := os.Getwd()
+	assert.NoError(t, err)
+
+	return filepath.Join(filepath.Dir(cwd), "..", "testdata", dir)
+}

analyzerapi/types.go

@@ -121,7 +121,9 @@ type AnalyzerContext struct {
 
 	// FilesByExtension contains all files by extension
 	FilesByExtension map[string][]string `json:"files_by_extension"`
+}
 
-	// FilesWithoutExtension contains all files without an extension
-	FilesWithoutExtension []string `json:"files_without_extension"`
+func (ctx *AnalyzerContext) ContainsFile(extension string) bool {
+	_, ok := ctx.FilesByExtension[extension]
+	return ok
 }

analyzerapi/util.go

@@ -1,55 +1,29 @@
 package analyzerapi
 
 import (
-	"io/fs"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
 
-	"github.com/rs/zerolog/log"
+	"github.com/cidverse/cidverseutils/filesystem"
 	ignore "github.com/sabhiram/go-gitignore"
+	"golang.org/x/exp/slog"
 )
 
 func GetAnalyzerContext(projectDir string) AnalyzerContext {
 	// respect gitignore
 	ignoreMatcher := ProcessIgnoreFiles([]string{filepath.Join(projectDir, ".gitignore"), filepath.Join(projectDir, ".cidignore")})
 
-	// files
-	var files []string
-	filesByExtension := make(map[string][]string)
-	var filesWithoutExtension []string
-
-	err := filepath.WalkDir(projectDir, func(path string, d fs.DirEntry, err error) error {
-		// check for directory skip
-		if err != nil {
-			log.Warn().Err(err).Str("path", projectDir).Msg("output")
-			return nil
-		}
-
-		if d.IsDir() {
-			if d.Name() == ".git" || ignoreMatcher.MatchesPath(path) {
-				return filepath.SkipDir
-			}
-		} else {
-			if ignoreMatcher.MatchesPath(path) {
-				return nil
-			}
-
-			files = append(files, path)
-			splitByExt := strings.SplitN(d.Name(), ".", 2)
-			if len(splitByExt) == 2 {
-				filesByExtension[splitByExt[1]] = append(filesByExtension[splitByExt[1]], path)
-			} else {
-				filesWithoutExtension = append(filesWithoutExtension, path)
-			}
-		}
-
-		return nil
+	files, err := filesystem.FindFiles(projectDir, func(absPath string, name string) bool {
+		return ignoreMatcher.MatchesPath(absPath)
+	}, func(absPath string, name string) bool {
+		return true
 	})
 	if err != nil {
-		log.Fatal().Err(err).Str("path", projectDir).Msg("failed to get directory list")
+		slog.Error("failed to get directory list", err, slog.String("path", projectDir))
 	}
+	filesByExtension := filesystem.GenerateFileMapByDeepExtension(files)
 
 	// sorting
 	sort.Slice(files, func(i, j int) bool {
@@ -58,10 +32,9 @@ func GetAnalyzerContext(projectDir string) AnalyzerContext {
 
 	// result
 	return AnalyzerContext{
-		ProjectDir:            projectDir,
-		Files:                 files,
-		FilesByExtension:      filesByExtension,
-		FilesWithoutExtension: filesWithoutExtension,
+		ProjectDir:       projectDir,
+		Files:            files,
+		FilesByExtension: filesByExtension,
 	}
 }