⭐️ npm.packages resource (#3333)

* ⭐️ support loading direct dependencies for npm package lock
* ⭐️ npm package resource
* 🧹 handle package.json version ranges
* 🧹 simplify tests
* 🧹 ensure that the root package is included when we gather package json dependencies
* 🧹 address feedback

providers/os/resources/npm/npm.go

@@ -13,17 +13,24 @@ import (
 )
 
 type Parser interface {
-	Parse(r io.Reader) (*Package, []*Package, error)
+	Parse(r io.Reader, filename string) (NpmPackageInfo, error)
+}
+
+type NpmPackageInfo interface {
+	Root() *Package
+	Direct() []*Package
+	Transitive() []*Package
 }
 
 type Package struct {
-	Name        string
-	File        string
-	License     string
-	Description string
-	Version     string
-	Purl        string
-	Cpes        []string
+	Name              string
+	File              string
+	License           string
+	Description       string
+	Version           string
+	Purl              string
+	Cpes              []string
+	EvidenceLocations []string
 }
 
 // NewPackageUrl creates a npm package url for a given package name and version
@@ -43,14 +50,14 @@ func NewPackageUrl(name string, version string) string {
 		packageurl.TypeNPM,
 		namespace,
 		name,
-		version,
+		cleanVersion(version),
 		nil,
 		"").String()
 }
 
 func NewCpes(name string, version string) []string {
 	cpes := []string{}
-	cpeEntry, err := cpe.NewPackage2Cpe(name, name, version, "", "")
+	cpeEntry, err := cpe.NewPackage2Cpe(name, name, cleanVersion(version), "", "")
 	// we only add the cpe if it could be created
 	// if the cpe could not be created, we log the error and continue to ensure the package is still added to the list
 	if err != nil {
@@ -60,3 +67,13 @@ func NewCpes(name string, version string) []string {
 	}
 	return cpes
 }
+
+func cleanVersion(version string) string {
+	v := strings.ReplaceAll(version, "^", "")
+	v = strings.ReplaceAll(v, "~", "")
+	v = strings.ReplaceAll(v, ">", "")
+	v = strings.ReplaceAll(v, "<", "")
+	v = strings.ReplaceAll(v, "=", "")
+	v = strings.ReplaceAll(v, " ", "")
+	return v
+}

providers/os/resources/npm/package_json.go

@@ -32,6 +32,9 @@ type packageJson struct {
 	Engines         map[string]string     `jsonn:"engines"`
 	CPU             []string              `json:"cpu"`
 	OS              []string              `json:"os"`
+
+	// evidence is a list of file paths where the package.json was found
+	evidence []string `json:"-"`
 }
 
 // packageJsonPeople represents the author of the package
@@ -160,36 +163,61 @@ func (a *packageJsonLicense) UnmarshalJSON(b []byte) error {
 
 type PackageJsonParser struct{}
 
-func (p *PackageJsonParser) Parse(r io.Reader) (*Package, []*Package, error) {
+func (p *PackageJsonParser) Parse(r io.Reader, filename string) (NpmPackageInfo, error) {
 	data, err := io.ReadAll(r)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	var packageJson packageJson
 	err = json.Unmarshal(data, &packageJson)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
+	}
+
+	if filename != "" {
+		packageJson.evidence = append(packageJson.evidence, filename)
 	}
 
-	// add own package
+	return &packageJson, nil
+}
+
+func (p *packageJson) Root() *Package {
+
+	// root package
 	root := &Package{
-		Name:    packageJson.Name,
-		Version: packageJson.Version,
-		Purl:    NewPackageUrl(packageJson.Name, packageJson.Version),
-		Cpes:    NewCpes(packageJson.Name, packageJson.Version),
+		Name:              p.Name,
+		Version:           p.Version,
+		Purl:              NewPackageUrl(p.Name, p.Version),
+		Cpes:              NewCpes(p.Name, p.Version),
+		EvidenceLocations: p.evidence,
+	}
+
+	return root
+}
+
+func (p *packageJson) Direct() []*Package {
+	return nil
+}
+
+func (p *packageJson) Transitive() []*Package {
+	// transitive dependencies, includes the root package
+	transitive := []*Package{}
+
+	r := p.Root()
+	if r != nil {
+		transitive = append(transitive, r)
 	}
 
-	// add all dependencies
-	entries := []*Package{}
-	for k, v := range packageJson.Dependencies {
-		entries = append(entries, &Package{
-			Name:    k,
-			Version: v,
-			Purl:    NewPackageUrl(k, v),
-			Cpes:    NewCpes(k, v),
+	for k, v := range p.Dependencies {
+		transitive = append(transitive, &Package{
+			Name:              k,
+			Version:           v,
+			Purl:              NewPackageUrl(k, v),
+			Cpes:              NewCpes(k, v),
+			EvidenceLocations: p.evidence,
 		})
 	}
 
-	return root, entries, nil
+	return transitive
 }

providers/os/resources/npm/package_json_test.go

@@ -274,31 +274,46 @@ func TestPackageJsonParser(t *testing.T) {
 	require.NoError(t, err)
 	defer f.Close()
 
-	root, pkgs, err := (&PackageJsonParser{}).Parse(f)
+	info, err := (&PackageJsonParser{}).Parse(f, "path/package.json")
 	assert.Nil(t, err)
-	assert.Equal(t, 30, len(pkgs))
 
+	root := info.Root()
 	assert.Equal(t, &Package{
-		Name:    "express",
-		Version: "4.16.4",
-		Purl:    "pkg:npm/[email protected]",
-		Cpes:    []string{"cpe:2.3:a:express:express:4.16.4:*:*:*:*:*:*:*"},
+		Name:              "express",
+		Version:           "4.16.4",
+		Purl:              "pkg:npm/[email protected]",
+		Cpes:              []string{"cpe:2.3:a:express:express:4.16.4:*:*:*:*:*:*:*"},
+		EvidenceLocations: []string{"path/package.json"},
 	}, root)
 
-	p := findPkg(pkgs, "path-to-regexp")
+	transitive := info.Transitive()
+	assert.Equal(t, 31, len(transitive))
+
+	// ensure the package is in the transitive list
+	p := findPkg(transitive, "express")
+	assert.Equal(t, &Package{
+		Name:              "express",
+		Version:           "4.16.4",
+		Purl:              "pkg:npm/[email protected]",
+		Cpes:              []string{"cpe:2.3:a:express:express:4.16.4:*:*:*:*:*:*:*"},
+		EvidenceLocations: []string{"path/package.json"},
+	}, p)
+
+	p = findPkg(transitive, "path-to-regexp")
 	assert.Equal(t, &Package{
-		Name:    "path-to-regexp",
-		Version: "0.1.7",
-		Purl:    "pkg:npm/[email protected]",
-		Cpes:    []string{"cpe:2.3:a:path-to-regexp:path-to-regexp:0.1.7:*:*:*:*:*:*:*"},
+		Name:              "path-to-regexp",
+		Version:           "0.1.7",
+		Purl:              "pkg:npm/[email protected]",
+		Cpes:              []string{"cpe:2.3:a:path-to-regexp:path-to-regexp:0.1.7:*:*:*:*:*:*:*"},
+		EvidenceLocations: []string{"path/package.json"},
 	}, p)
 
-	p = findPkg(pkgs, "range-parser")
+	p = findPkg(transitive, "range-parser")
 	assert.Equal(t, &Package{
-		Name:    "range-parser",
-		Version: "~1.2.0",
-		// TODO: we need to handle the range properly
-		Purl: "pkg:npm/range-parser@~1.2.0",
-		Cpes: []string{"cpe:2.3:a:range-parser:range-parser:\\~1.2.0:*:*:*:*:*:*:*"},
+		Name:              "range-parser",
+		Version:           "~1.2.0",
+		Purl:              "pkg:npm/[email protected]",
+		Cpes:              []string{"cpe:2.3:a:range-parser:range-parser:1.2.0:*:*:*:*:*:*:*"},
+		EvidenceLocations: []string{"path/package.json"},
 	}, p)
 }

providers/os/resources/npm/package_lock.go

@@ -5,6 +5,7 @@ package npm
 
 import (
 	"io"
+	"strings"
 
 	"encoding/json"
 )
@@ -26,6 +27,9 @@ type packageLock struct {
 	// Dependencies contains legacy data for supporting versions of npm that use lockfileVersion: 1 or lower.
 	// We can ignore that for lockfileVersion: 2+
 	Dependencies map[string]packageLockDependency `jsonn:"dependencies"`
+
+	// evidence is a list of file paths where the package-lock was found
+	evidence []string `json:"-"`
 }
 
 type packageLockDependency struct {
@@ -36,11 +40,12 @@ type packageLockDependency struct {
 }
 
 type packageLockPackage struct {
-	Name      string             `json:"name"`
-	Version   string             `json:"version"`
-	Resolved  string             `json:"resolved"`
-	Integrity string             `json:"integrity"`
-	License   packageLockLicense `json:"license"`
+	Name         string             `json:"name"`
+	Version      string             `json:"version"`
+	Resolved     string             `json:"resolved"`
+	Integrity    string             `json:"integrity"`
+	License      packageLockLicense `json:"license"`
+	Dependencies map[string]string  `json:"dependencies"`
 }
 
 type packageLockLicense []string
@@ -67,48 +72,98 @@ func (l *packageLockLicense) UnmarshalJSON(data []byte) (err error) {
 
 // PackageLockParser is the parser for the package.lock file npm format.
 // see https://docs.npmjs.com/cli/v10/configuring-npm/package-lock-json
-type PackageLockParser struct{}
+type PackageLockParser struct {
+}
 
-func (p *PackageLockParser) Parse(r io.Reader) (*Package, []*Package, error) {
+func (p *PackageLockParser) Parse(r io.Reader, filename string) (NpmPackageInfo, error) {
 	var packageJsonLock packageLock
 	err := json.NewDecoder(r).Decode(&packageJsonLock)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
+	}
+
+	if filename != "" {
+		packageJsonLock.evidence = append(packageJsonLock.evidence, filename)
 	}
 
-	// add own package
+	return &packageJsonLock, nil
+}
+
+func (p *packageLock) Root() *Package {
 	root := &Package{
-		Name:    packageJsonLock.Name,
-		Version: packageJsonLock.Version,
-		Purl:    NewPackageUrl(packageJsonLock.Name, packageJsonLock.Version),
-		Cpes:    NewCpes(packageJsonLock.Name, packageJsonLock.Version),
+		Name:              p.Name,
+		Version:           p.Version,
+		Purl:              NewPackageUrl(p.Name, p.Version),
+		Cpes:              NewCpes(p.Name, p.Version),
+		EvidenceLocations: p.evidence,
+	}
+	return root
+}
+
+func (p *packageLock) Direct() []*Package {
+	// search for root package, read the packages field
+
+	// at this point we only support lockfileVersion: 2 with direct dependencies
+	if p.Packages == nil {
+		return nil
 	}
 
-	// add all dependencies
-	entries := []*Package{}
-	if packageJsonLock.Packages != nil {
-		for k, v := range packageJsonLock.Packages {
+	rootPkg, ok := p.Packages[""]
+	if !ok {
+		return nil
+	}
+
+	filteredList := []*Package{}
+	for k := range rootPkg.Dependencies {
+		pkg, ok := p.Packages[k]
+		if !ok {
+			continue
+		}
+
+		filteredList = append(filteredList, &Package{
+			Name:              packageLockPackageName(k),
+			Version:           pkg.Version,
+			Purl:              NewPackageUrl(k, pkg.Version),
+			Cpes:              NewCpes(k, pkg.Version),
+			EvidenceLocations: p.evidence,
+		})
+	}
+
+	return filteredList
+}
+
+func (p *packageLock) Transitive() []*Package {
+	transitive := []*Package{}
+	if p.Packages != nil {
+		for k, v := range p.Packages {
 			name := k
+			// skip root package since we have that already
 			if name == "" {
 				name = v.Name
 			}
-			entries = append(entries, &Package{
-				Name:    name,
-				Version: v.Version,
-				Purl:    NewPackageUrl(name, v.Version),
-				Cpes:    NewCpes(name, v.Version),
+
+			transitive = append(transitive, &Package{
+				Name:              packageLockPackageName(name),
+				Version:           v.Version,
+				Purl:              NewPackageUrl(k, v.Version),
+				Cpes:              NewCpes(k, v.Version),
+				EvidenceLocations: p.evidence,
 			})
 		}
-	} else if packageJsonLock.Dependencies != nil {
-		for k, v := range packageJsonLock.Dependencies {
-			entries = append(entries, &Package{
-				Name:    k,
-				Version: v.Version,
-				Purl:    NewPackageUrl(k, v.Version),
-				Cpes:    NewCpes(k, v.Version),
+	} else if p.Dependencies != nil {
+		for k, v := range p.Dependencies {
+			transitive = append(transitive, &Package{
+				Name:              k,
+				Version:           v.Version,
+				Purl:              NewPackageUrl(k, v.Version),
+				Cpes:              NewCpes(k, v.Version),
+				EvidenceLocations: p.evidence,
 			})
 		}
 	}
+	return transitive
+}
 
-	return root, entries, nil
+func packageLockPackageName(path string) string {
+	return strings.TrimPrefix(path, "node_modules/")
 }