Support for git repo as an upstream source

In addition to tarball and srpms, eext now supports git repo as an
upstream source.

impl/common.go

@@ -175,6 +175,50 @@ func checkRepo(repo string, pkg string, isPkgSubdirInRepo bool,
 	return nil
 }
 
+// Download the git repo, and create a tarball at the provided commit/tag.
+func archiveGitRepo(srcURL string, targetDir string, version string, revision string, pkg string,
+	errPrefix util.ErrPrefix) (string, string, error) {
+	rpmName := pkg + "-" + version
+	gitArchiveFile := rpmName + ".tar.gz"
+	gitArchiveFilePath := filepath.Join(targetDir, gitArchiveFile)
+
+	// Cloning the git repo to a temporary directory
+	// We won't delete the tmpDir here, since we need it to verify the git repo.
+	cloneDir, err := os.MkdirTemp("", rpmName)
+	if err != nil {
+		return "", "", fmt.Errorf("%serror while creating tempDir for %s, %s", errPrefix, pkg, err)
+	}
+	err = util.RunSystemCmdInDir(cloneDir, "git", "init")
+	if err != nil {
+		return "", "", fmt.Errorf("%sgit init at %s failed: %s", errPrefix, cloneDir, err)
+	}
+	err = util.RunSystemCmdInDir(cloneDir, "git", "remote", "add", "origin", srcURL)
+	if err != nil {
+		return "", "", fmt.Errorf("%sadding %s as git remote failed: %s", errPrefix, srcURL, err)
+	}
+	err = util.RunSystemCmdInDir(cloneDir, "git", "fetch", "origin", revision)
+	if err != nil {
+		return "", "", fmt.Errorf("%sfetching revision %s failed for %s: %s", errPrefix, revision, pkg, err)
+	}
+	err = util.RunSystemCmdInDir(cloneDir, "git", "reset", "--hard", "FETCH_HEAD")
+	if err != nil {
+		return "", "", fmt.Errorf("%sfetching HEAD at %s failed: %s", errPrefix, revision, err)
+	}
+
+	// Create the tarball from the specified commit/tag revision
+	archiveCmd := []string{"archive",
+		"--prefix", rpmName + "/",
+		"-o", gitArchiveFilePath,
+		revision,
+	}
+	err = util.RunSystemCmdInDir(cloneDir, "git", archiveCmd...)
+	if err != nil {
+		return "", "", fmt.Errorf("%sgit archive of %s failed: %s", errPrefix, pkg, err)
+	}
+
+	return gitArchiveFile, cloneDir, nil
+}
+
 // Download the resource srcURL to targetDir
 // srcURL could be URL or file path
 // If it is a file:// path, root directory is the

impl/create_srpm.go

@@ -8,11 +8,10 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 
-	"golang.org/x/exp/slices"
-
 	"code.arista.io/eos/tools/eext/manifest"
 	"code.arista.io/eos/tools/eext/srcconfig"
 	"code.arista.io/eos/tools/eext/util"
@@ -23,6 +22,7 @@ type upstreamSrcSpec struct {
 	sigFile      string
 	pubKeyPath   string
 	skipSigCheck bool
+	gitSpec      util.GitSpec
 }
 
 type srpmBuilder struct {
@@ -106,16 +106,38 @@ func (bldr *srpmBuilder) fetchUpstream() error {
 		var downloadErr error
 		upstreamSrc := upstreamSrcSpec{}
 
-		bldr.log("downloading %s", srcParams.SrcURL)
-		// Download source
-		if upstreamSrc.sourceFile, downloadErr = download(
-			srcParams.SrcURL,
-			downloadDir,
-			repo, pkg, isPkgSubdirInRepo,
-			bldr.errPrefix); downloadErr != nil {
-			return downloadErr
+		if bldr.pkgSpec.Type == "git" {
+			bldr.log("creating tarball for %s from repo %s", pkg, srcParams.SrcURL)
+			spec := util.GitSpec{
+				Revision: upstreamSrcFromManifest.GitSpec.Revision,
+			}
+			version := spec.GetVersionFromRevision()
+			revision := spec.Revision
+			var clonedDir string
+			upstreamSrc.sourceFile, clonedDir, downloadErr = archiveGitRepo(
+				srcParams.SrcURL,
+				downloadDir,
+				version, revision, pkg,
+				bldr.errPrefix)
+			if downloadErr != nil {
+				return downloadErr
+			}
+
+			spec.ClonedDir = clonedDir
+			upstreamSrc.gitSpec = spec
+			bldr.log("tarball created")
+		} else {
+			bldr.log("downloading %s", srcParams.SrcURL)
+			// Download source
+			if upstreamSrc.sourceFile, downloadErr = download(
+				srcParams.SrcURL,
+				downloadDir,
+				repo, pkg, isPkgSubdirInRepo,
+				bldr.errPrefix); downloadErr != nil {
+				return downloadErr
+			}
+			bldr.log("downloaded")
 		}
-		bldr.log("downloaded")
 
 		upstreamSrc.skipSigCheck = upstreamSrcFromManifest.Signature.SkipCheck
 		pubKey := upstreamSrcFromManifest.Signature.DetachedSignature.PubKey
@@ -150,6 +172,17 @@ func (bldr *srpmBuilder) fetchUpstream() error {
 				return fmt.Errorf("%sUnexpected public-key specified for SRPM",
 					bldr.errPrefix)
 			}
+		} else if bldr.pkgSpec.Type == "git" && !upstreamSrc.skipSigCheck {
+			if pubKey == "" {
+				return fmt.Errorf("%sexpected public-key for %s to verify git repo",
+					bldr.errPrefix, pkg)
+			}
+			pubKeyPath := filepath.Join(getDetachedSigDir(), pubKey)
+			if pathErr := util.CheckPath(pubKeyPath, false, false); pathErr != nil {
+				return fmt.Errorf("%sCannot find public-key at path %s",
+					bldr.errPrefix, pubKeyPath)
+			}
+			upstreamSrc.pubKeyPath = pubKeyPath
 		}
 
 		bldr.upstreamSrc = append(bldr.upstreamSrc, upstreamSrc)
@@ -216,6 +249,18 @@ func (bldr *srpmBuilder) verifyUpstream() error {
 		if err := bldr.verifyUpstreamSrpm(); err != nil {
 			return err
 		}
+	} else if bldr.pkgSpec.Type == "git" {
+		// Need to have cloned repo for this too, either use a special path or delete here after use
+		for _, upstreamSrc := range bldr.upstreamSrc {
+			if !upstreamSrc.skipSigCheck {
+				err := util.VerifyGitSignature(upstreamSrc.pubKeyPath, upstreamSrc.gitSpec, bldr.errPrefix)
+				if err != nil {
+					return err
+				}
+			}
+			// Deleting cloned git repo since we no longer require it.
+			os.RemoveAll(upstreamSrc.gitSpec.ClonedDir)
+		}
 	} else {
 		downloadDir := getDownloadDir(bldr.pkgSpec.Name)
 		for _, upstreamSrc := range bldr.upstreamSrc {
@@ -271,7 +316,7 @@ func (bldr *srpmBuilder) setupRpmbuildTreeSrpm() error {
 // also checks tarball signature
 func (bldr *srpmBuilder) setupRpmbuildTreeNonSrpm() error {
 
-	supportedTypes := []string{"tarball", "standalone"}
+	supportedTypes := []string{"tarball", "standalone", "git"}
 	if !slices.Contains(supportedTypes, bldr.pkgSpec.Type) {
 		panic(fmt.Sprintf("%ssetupRpmbuildTreeNonSrpm called for unsupported type %s",
 			bldr.errPrefix, bldr.pkgSpec.Type))
@@ -284,7 +329,7 @@ func (bldr *srpmBuilder) setupRpmbuildTreeNonSrpm() error {
 		return err
 	}
 
-	if bldr.pkgSpec.Type == "tarball" {
+	if bldr.pkgSpec.Type == "tarball" || bldr.pkgSpec.Type == "git" {
 		downloadDir := getDownloadDir(bldr.pkgSpec.Name)
 		for _, upstreamSrc := range bldr.upstreamSrc {
 			upstreamSourceFilePath := filepath.Join(downloadDir, upstreamSrc.sourceFile)
@@ -379,7 +424,7 @@ func (bldr *srpmBuilder) setupRpmbuildTree() error {
 		if err := bldr.setupRpmbuildTreeSrpm(); err != nil {
 			return err
 		}
-	} else if bldr.pkgSpec.Type == "tarball" || bldr.pkgSpec.Type == "standalone" {
+	} else if bldr.pkgSpec.Type == "tarball" || bldr.pkgSpec.Type == "standalone" || bldr.pkgSpec.Type == "git" {
 		if err := bldr.setupRpmbuildTreeNonSrpm(); err != nil {
 			return err
 		}

manifest/manifest.go

@@ -5,7 +5,7 @@ package manifest
 
 import (
 	"fmt"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 
 	"golang.org/x/exp/slices"
@@ -118,12 +118,18 @@ type SourceBundle struct {
 	SrcRepoParamsOverride srcconfig.SrcRepoParamsOverride `yaml:"override"`
 }
 
+type GitSpec struct {
+	Url      string `yaml:"url"`
+	Revision string `yaml:"revision"`
+}
+
 // UpstreamSrc spec
 // Lists each source bundle(tarball/srpm) and
 // detached signature file for tarball.
 type UpstreamSrc struct {
 	SourceBundle SourceBundle `yaml:"source-bundle"`
 	FullURL      string       `yaml:"full-url"`
+	GitSpec      GitSpec      `yaml:"git"`
 	Signature    Signature    `yaml:"signature"`
 }
 
@@ -147,7 +153,7 @@ type Manifest struct {
 }
 
 func (m Manifest) sanityCheck() error {
-	allowedPkgTypes := []string{"srpm", "unmodified-srpm", "tarball", "standalone"}
+	allowedPkgTypes := []string{"srpm", "unmodified-srpm", "tarball", "standalone", "git"}
 
 	for _, pkgSpec := range m.Package {
 		if pkgSpec.Name == "" {
@@ -184,23 +190,45 @@ func (m Manifest) sanityCheck() error {
 		}
 
 		for _, upStreamSrc := range pkgSpec.UpstreamSrc {
-			specifiedFullSrcURL := (upStreamSrc.FullURL != "")
-			specifiedSrcBundle := (upStreamSrc.SourceBundle != SourceBundle{})
-			if !specifiedFullSrcURL && !specifiedSrcBundle {
-				return fmt.Errorf("Specify source for Build in package %s, provide either full-url or source-bundle",
-					pkgSpec.Name)
-			}
+			if pkgSpec.Type == "git" {
+				specifiedUrl := (upStreamSrc.GitSpec.Url != "")
+				specifiedRevision := (upStreamSrc.GitSpec.Revision != "")
+				if !specifiedUrl {
+					return fmt.Errorf("please provide the url for git repo of package %s", pkgSpec.Name)
+				}
+				if !specifiedRevision {
+					return fmt.Errorf("please provide a commit/tag to define revision of package %s", pkgSpec.Name)
+				}
 
-			if specifiedFullSrcURL && specifiedSrcBundle {
-				return fmt.Errorf(
-					"Conflicting sources for Build in package %s, provide either full-url or source-bundle",
-					pkgSpec.Name)
-			}
+				specifiedSignature := (upStreamSrc.Signature != Signature{})
+				if specifiedSignature {
+					skipSigCheck := (upStreamSrc.Signature.SkipCheck)
+					specifiedPubKey := (upStreamSrc.Signature.DetachedSignature.PubKey != "")
+					if !skipSigCheck && !specifiedPubKey {
+						return fmt.Errorf(
+							"please provide the public key to verify git repo for package %s, or skip signature check",
+							pkgSpec.Name)
+					}
+				}
+			} else {
+				specifiedFullSrcURL := (upStreamSrc.FullURL != "")
+				specifiedSrcBundle := (upStreamSrc.SourceBundle != SourceBundle{})
+				if !specifiedFullSrcURL && !specifiedSrcBundle {
+					return fmt.Errorf("Specify source for Build in package %s, provide either full-url or source-bundle",
+						pkgSpec.Name)
+				}
 
-			specifiedFullSigURL := upStreamSrc.Signature.DetachedSignature.FullURL != ""
-			if specifiedFullSigURL && specifiedSrcBundle {
-				return fmt.Errorf("Conflicting signatures for Build in package %s, provide full-url or source-bundle",
-					pkgSpec.Name)
+				if specifiedFullSrcURL && specifiedSrcBundle {
+					return fmt.Errorf(
+						"Conflicting sources for Build in package %s, provide either full-url or source-bundle",
+						pkgSpec.Name)
+				}
+
+				specifiedFullSigURL := upStreamSrc.Signature.DetachedSignature.FullURL != ""
+				if specifiedFullSigURL && specifiedSrcBundle {
+					return fmt.Errorf("Conflicting signatures for Build in package %s, provide full-url or source-bundle",
+						pkgSpec.Name)
+				}
 			}
 		}
 	}
@@ -213,9 +241,9 @@ func LoadManifest(repo string) (*Manifest, error) {
 	repoDir := util.GetRepoDir(repo)
 
 	yamlPath := filepath.Join(repoDir, "eext.yaml")
-	yamlContents, readErr := ioutil.ReadFile(yamlPath)
+	yamlContents, readErr := os.ReadFile(yamlPath)
 	if readErr != nil {
-		return nil, fmt.Errorf("manifest.LoadManifest: ioutil.ReadFile on %s returned %s", yamlPath, readErr)
+		return nil, fmt.Errorf("manifest.LoadManifest: os.ReadFile on %s returned %s", yamlPath, readErr)
 	}
 
 	var manifest Manifest

manifest/manifest_test.go

@@ -31,12 +31,15 @@ func TestManifest(t *testing.T) {
 	viper.Set("SrcDir", dir)
 	defer viper.Reset()
 
-	t.Log("Copy sample manifest to test directory")
-	testutil.SetupManifest(t, dir, "pkg1", "sampleManifest1.yaml")
+	testFiles := []string{"sampleManifest1.yaml", "sampleManifest4.yaml"}
+	for _, testFile := range testFiles {
+		t.Logf("Copy sample manifest %s to test directory", testFile)
+		testutil.SetupManifest(t, dir, "pkg1", testFile)
 
-	t.Log("Testing Load")
-	testLoad(t, "pkg1")
-	t.Log("Load test passed")
+		t.Log("Testing Load")
+		testLoad(t, "pkg1")
+		t.Log("Load test passed")
+	}
 }
 
 type manifestTestVariant struct {

manifest/testData/sampleManifest4.yaml

@@ -0,0 +1,44 @@
+---
+package:
+  - name: libpcap1
+    upstream-sources:
+      - git:
+          url: https://github.com/the-tcpdump-group/libpcap
+          revision: 104271ba4a14de6743e43bcf87536786d8fddea4
+        signature:
+          detached-sig:
+            public-key: mrtparse/mrtparsePubKey.pem
+    type: git
+    build:
+      repo-bundle:
+        - name: foo
+          version: v1
+        - name: bar
+
+  - name: libpcap2
+    upstream-sources:
+      - git:
+          url: https://github.com/the-tcpdump-group/libpcap
+          revision: libpcap-1.10.1
+    type: git
+    build:
+      repo-bundle:
+        - name: foo
+          version: v1
+        - name: bar
+#
+#  - name: libpcap3
+#    upstream-sources:
+#      - snapshot:
+#          rel-version: 1.10.4
+#          commit: 104271ba4a14de6743e43bcf87536786d8fddea4
+#        signature:
+#          skip-check: true
+#          detached-sig:
+#            public-key: mrtparse/mrtparsePubKey.pem
+#    type: local-eextsubdir
+#    build:
+#      repo-bundle:
+#        - name: foo
+#          version: v1
+#        - name: bar

srcconfig/srcconfig.go

@@ -6,7 +6,6 @@ package srcconfig
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -222,9 +221,9 @@ func LoadSrcConfig() (*SrcConfig, error) {
 			cfgPath, statErr)
 	}
 
-	yamlContents, readErr := ioutil.ReadFile(cfgPath)
+	yamlContents, readErr := os.ReadFile(cfgPath)
 	if readErr != nil {
-		return nil, fmt.Errorf("srcconfig.LoadSrcConfig: ioutil.ReadFile on %s returned %s",
+		return nil, fmt.Errorf("srcconfig.LoadSrcConfig: os.ReadFile on %s returned %s",
 			cfgPath, readErr)
 	}