Merge pull request google#314 from ackama:extractor/support-uv

PiperOrigin-RevId: 717862567

extractor/filesystem/language/python/uvlock/testdata/no-dependencies.lock

@@ -0,0 +1,7 @@
+version = 1
+requires-python = ">=3.10"
+
+[[package]]
+name = "uv-lockfiles"
+version = "0.1.0"
+source = { virtual = "." }

extractor/filesystem/language/python/uvlock/testdata/no-packages.lock

@@ -0,0 +1,2 @@
+version = 1
+requires-python = ">=3.10"

extractor/filesystem/language/python/uvlock/testdata/not-toml.txt

@@ -0,0 +1 @@
+this is not valid toml! (I think)

extractor/filesystem/language/python/uvlock/testdata/one-package.lock

@@ -0,0 +1,22 @@
+version = 1
+requires-python = ">=3.10"
+
+[[package]]
+name = "emoji"
+version = "2.14.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/13/64/812d7e2ae0ac2ade0d6583f911f99240c80f700afbe8391df10e547f564d/emoji-2.14.0.tar.gz", hash = "sha256:f68ac28915a2221667cddb3e6c589303c3c6954c6c5af6fefaec7f9bdf72fdca", size = 593932 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ef/56/4ddf8b36aa4b52077045b17ffb8958f3360b250df4143d1482d9d5bb54d5/emoji-2.14.0-py3-none-any.whl", hash = "sha256:fcc936bf374b1aec67dda5303ae99710ba88cc9cdce2d1a71c5f2204e6d78799", size = 586897 },
+]
+
+[[package]]
+name = "uv-lockfiles"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+    { name = "emoji" },
+]
+
+[package.metadata]
+requires-dist = [{ name = "emoji" }]

extractor/filesystem/language/python/uvlock/testdata/source-git.lock

@@ -0,0 +1,18 @@
+version = 1
+requires-python = ">=3.10"
+
+[[package]]
+name = "ruff"
+version = "0.8.1"
+source = { git = "https://github.com/astral-sh/ruff#84748be16341b76e073d117329f7f5f4ee2941ad" }
+
+[[package]]
+name = "uv-lockfiles"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+    { name = "ruff" },
+]
+
+[package.metadata]
+requires-dist = [{ name = "ruff", git = "https://github.com/astral-sh/ruff" }]

extractor/filesystem/language/python/uvlock/testdata/two-packages.lock

@@ -0,0 +1,40 @@
+version = 1
+requires-python = ">=3.10"
+
+[[package]]
+name = "emoji"
+version = "2.14.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/13/64/812d7e2ae0ac2ade0d6583f911f99240c80f700afbe8391df10e547f564d/emoji-2.14.0.tar.gz", hash = "sha256:f68ac28915a2221667cddb3e6c589303c3c6954c6c5af6fefaec7f9bdf72fdca", size = 593932 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ef/56/4ddf8b36aa4b52077045b17ffb8958f3360b250df4143d1482d9d5bb54d5/emoji-2.14.0-py3-none-any.whl", hash = "sha256:fcc936bf374b1aec67dda5303ae99710ba88cc9cdce2d1a71c5f2204e6d78799", size = 586897 },
+]
+
+[[package]]
+name = "protobuf"
+version = "4.25.5"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/67/dd/48d5fdb68ec74d70fabcc252e434492e56f70944d9f17b6a15e3746d2295/protobuf-4.25.5.tar.gz", hash = "sha256:7f8249476b4a9473645db7f8ab42b02fe1488cbe5fb72fddd445e0665afd8584", size = 380315 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/00/35/1b3c5a5e6107859c4ca902f4fbb762e48599b78129a05d20684fef4a4d04/protobuf-4.25.5-cp310-abi3-win32.whl", hash = "sha256:5e61fd921603f58d2f5acb2806a929b4675f8874ff5f330b7d6f7e2e784bbcd8", size = 392457 },
+    { url = "https://files.pythonhosted.org/packages/a7/ad/bf3f358e90b7e70bf7fb520702cb15307ef268262292d3bdb16ad8ebc815/protobuf-4.25.5-cp310-abi3-win_amd64.whl", hash = "sha256:4be0571adcbe712b282a330c6e89eae24281344429ae95c6d85e79e84780f5ea", size = 413449 },
+    { url = "https://files.pythonhosted.org/packages/51/49/d110f0a43beb365758a252203c43eaaad169fe7749da918869a8c991f726/protobuf-4.25.5-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:b2fde3d805354df675ea4c7c6338c1aecd254dfc9925e88c6d31a2bcb97eb173", size = 394248 },
+    { url = "https://files.pythonhosted.org/packages/c6/ab/0f384ca0bc6054b1a7b6009000ab75d28a5506e4459378b81280ae7fd358/protobuf-4.25.5-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:919ad92d9b0310070f8356c24b855c98df2b8bd207ebc1c0c6fcc9ab1e007f3d", size = 293717 },
+    { url = "https://files.pythonhosted.org/packages/05/a6/094a2640be576d760baa34c902dcb8199d89bce9ed7dd7a6af74dcbbd62d/protobuf-4.25.5-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fe14e16c22be926d3abfcb500e60cab068baf10b542b8c858fa27e098123e331", size = 294635 },
+    { url = "https://files.pythonhosted.org/packages/33/90/f198a61df8381fb43ae0fe81b3d2718e8dcc51ae8502c7657ab9381fbc4f/protobuf-4.25.5-py3-none-any.whl", hash = "sha256:0aebecb809cae990f8129ada5ca273d9d670b76d9bfc9b1809f0a9c02b7dbf41", size = 156467 },
+]
+
+[[package]]
+name = "uv-lockfiles"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+    { name = "emoji" },
+    { name = "protobuf" },
+]
+
+[package.metadata]
+requires-dist = [
+    { name = "emoji" },
+    { name = "protobuf", specifier = ">=3.19.0,<5.0.0.dev0" },
+]

extractor/filesystem/language/python/uvlock/uvlock.go

@@ -0,0 +1,149 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package uvlock extracts uv.lock files.
+package uvlock
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/BurntSushi/toml"
+	"github.com/google/osv-scalibr/extractor"
+	"github.com/google/osv-scalibr/extractor/filesystem"
+	"github.com/google/osv-scalibr/extractor/filesystem/language/python/internal/pypipurl"
+	"github.com/google/osv-scalibr/extractor/filesystem/osv"
+	"github.com/google/osv-scalibr/plugin"
+	"github.com/google/osv-scalibr/purl"
+)
+
+type uvLockPackageSource struct {
+	Virtual string `toml:"virtual"`
+	Git     string `toml:"git"`
+}
+
+type uvLockPackage struct {
+	Name    string              `toml:"name"`
+	Version string              `toml:"version"`
+	Source  uvLockPackageSource `toml:"source"`
+
+	// uv stores "groups" as a table under "package" after all the packages, which due
+	// to how TOML works means it ends up being a property on the last package, even
+	// through in this context it's a global property rather than being per-package
+	Groups map[string][]uvOptionalDependency `toml:"optional-dependencies"`
+}
+
+type uvOptionalDependency struct {
+	Name string `toml:"name"`
+}
+type uvLockFile struct {
+	Version  int             `toml:"version"`
+	Packages []uvLockPackage `toml:"package"`
+}
+
+// Extractor extracts python packages from uv.lock files.
+type Extractor struct{}
+
+// Name of the extractor
+func (e Extractor) Name() string { return "python/uvlock" }
+
+// Version of the extractor
+func (e Extractor) Version() int { return 0 }
+
+// Requirements of the extractor
+func (e Extractor) Requirements() *plugin.Capabilities {
+	return &plugin.Capabilities{}
+}
+
+// FileRequired returns true if the specified file matches uv lockfile patterns
+func (e Extractor) FileRequired(api filesystem.FileAPI) bool {
+	return filepath.Base(api.Path()) == "uv.lock"
+}
+
+// Extract extracts packages from uv.lock files passed through the scan input.
+func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]*extractor.Inventory, error) {
+	var parsedLockfile *uvLockFile
+
+	_, err := toml.NewDecoder(input.Reader).Decode(&parsedLockfile)
+
+	if err != nil {
+		return []*extractor.Inventory{}, fmt.Errorf("could not extract from %s: %w", input.Path, err)
+	}
+
+	packages := make([]*extractor.Inventory, 0, len(parsedLockfile.Packages))
+
+	var groups map[string][]uvOptionalDependency
+
+	// uv stores "groups" as a table under "package" after all the packages, which due
+	// to how TOML works means it ends up being a property on the last package, even
+	// through in this context it's a global property rather than being per-package
+	if len(parsedLockfile.Packages) > 0 {
+		groups = parsedLockfile.Packages[len(parsedLockfile.Packages)-1].Groups
+	}
+
+	for _, lockPackage := range parsedLockfile.Packages {
+		// skip including the root "package", since its name and version are most likely arbitrary
+		if lockPackage.Source.Virtual == "." {
+			continue
+		}
+
+		_, commit, _ := strings.Cut(lockPackage.Source.Git, "#")
+
+		pkgDetails := &extractor.Inventory{
+			Name:      lockPackage.Name,
+			Version:   lockPackage.Version,
+			Locations: []string{input.Path},
+		}
+
+		if commit != "" {
+			pkgDetails.SourceCode = &extractor.SourceCodeIdentifier{
+				Commit: commit,
+			}
+		}
+
+		depGroupVals := []string{}
+
+		for group, deps := range groups {
+			for _, dep := range deps {
+				if dep.Name == lockPackage.Name {
+					depGroupVals = append(depGroupVals, group)
+				}
+			}
+		}
+
+		sort.Strings(depGroupVals)
+
+		pkgDetails.Metadata = osv.DepGroupMetadata{
+			DepGroupVals: depGroupVals,
+		}
+		packages = append(packages, pkgDetails)
+	}
+
+	return packages, nil
+}
+
+// ToPURL converts an inventory created by this extractor into a PURL.
+func (e Extractor) ToPURL(i *extractor.Inventory) *purl.PackageURL {
+	return pypipurl.MakePackageURL(i)
+}
+
+// Ecosystem returns the OSV ecosystem ('PyPI') of the software extracted by this extractor.
+func (e Extractor) Ecosystem(i *extractor.Inventory) string {
+	return "PyPI"
+}
+
+var _ filesystem.Extractor = Extractor{}