From 55502dd98dde46df19477bf170f1d792e3e693bc Mon Sep 17 00:00:00 2001 From: Manith Date: Tue, 25 Jun 2024 01:04:38 -0700 Subject: [PATCH] Support for git repo as an upstream source In addition to tarball and srpms, eext now supports git repo as an upstream source. --- impl/common.go | 44 ++++++++++++++ impl/create_srpm.go | 73 +++++++++++++++++----- manifest/manifest.go | 66 ++++++++++++++------ manifest/manifest_test.go | 13 ++-- manifest/testData/sampleManifest4.yaml | 44 ++++++++++++++ srcconfig/srcconfig.go | 5 +- util/util.go | 84 ++++++++++++++++++++++++++ 7 files changed, 288 insertions(+), 41 deletions(-) create mode 100644 manifest/testData/sampleManifest4.yaml diff --git a/impl/common.go b/impl/common.go index b6d391f..9255a80 100644 --- a/impl/common.go +++ b/impl/common.go @@ -175,6 +175,50 @@ func checkRepo(repo string, pkg string, isPkgSubdirInRepo bool, return nil } +// Download the git repo, and create a tarball at the provided commit/tag. +func archiveGitRepo(srcURL string, targetDir string, version string, revision string, pkg string, + errPrefix util.ErrPrefix) (string, string, error) { + rpmName := pkg + "-" + version + gitArchiveFile := rpmName + ".tar.gz" + gitArchiveFilePath := filepath.Join(targetDir, gitArchiveFile) + + // Cloning the git repo to a temporary directory + // We won't delete the tmpDir here, since we need it to verify the git repo. + cloneDir, err := os.MkdirTemp("", rpmName) + if err != nil { + return "", "", fmt.Errorf("%serror while creating tempDir for %s, %s", errPrefix, pkg, err) + } + err = util.RunSystemCmdInDir(cloneDir, "git", "init") + if err != nil { + return "", "", fmt.Errorf("%sgit init at %s failed: %s", errPrefix, cloneDir, err) + } + err = util.RunSystemCmdInDir(cloneDir, "git", "remote", "add", "origin", srcURL) + if err != nil { + return "", "", fmt.Errorf("%sadding %s as git remote failed: %s", errPrefix, srcURL, err) + } + err = util.RunSystemCmdInDir(cloneDir, "git", "fetch", "origin", revision) + if err != nil { + return "", "", fmt.Errorf("%sfetching revision %s failed for %s: %s", errPrefix, revision, pkg, err) + } + err = util.RunSystemCmdInDir(cloneDir, "git", "reset", "--hard", "FETCH_HEAD") + if err != nil { + return "", "", fmt.Errorf("%sfetching HEAD at %s failed: %s", errPrefix, revision, err) + } + + // Create the tarball from the specified commit/tag revision + archiveCmd := []string{"archive", + "--prefix", rpmName + "/", + "-o", gitArchiveFilePath, + revision, + } + err = util.RunSystemCmdInDir(cloneDir, "git", archiveCmd...) + if err != nil { + return "", "", fmt.Errorf("%sgit archive of %s failed: %s", errPrefix, pkg, err) + } + + return gitArchiveFile, cloneDir, nil +} + // Download the resource srcURL to targetDir // srcURL could be URL or file path // If it is a file:// path, root directory is the diff --git a/impl/create_srpm.go b/impl/create_srpm.go index bdd2f55..3a41c76 100644 --- a/impl/create_srpm.go +++ b/impl/create_srpm.go @@ -8,11 +8,10 @@ import ( "log" "os" "path/filepath" + "slices" "strconv" "strings" - "golang.org/x/exp/slices" - "code.arista.io/eos/tools/eext/manifest" "code.arista.io/eos/tools/eext/srcconfig" "code.arista.io/eos/tools/eext/util" @@ -23,6 +22,7 @@ type upstreamSrcSpec struct { sigFile string pubKeyPath string skipSigCheck bool + gitSpec util.GitSpec } type srpmBuilder struct { @@ -106,16 +106,38 @@ func (bldr *srpmBuilder) fetchUpstream() error { var downloadErr error upstreamSrc := upstreamSrcSpec{} - bldr.log("downloading %s", srcParams.SrcURL) - // Download source - if upstreamSrc.sourceFile, downloadErr = download( - srcParams.SrcURL, - downloadDir, - repo, pkg, isPkgSubdirInRepo, - bldr.errPrefix); downloadErr != nil { - return downloadErr + if bldr.pkgSpec.Type == "git" { + bldr.log("creating tarball for %s from repo %s", pkg, srcParams.SrcURL) + spec := util.GitSpec{ + Revision: upstreamSrcFromManifest.GitSpec.Revision, + } + version := spec.GetVersionFromRevision() + revision := spec.Revision + var clonedDir string + upstreamSrc.sourceFile, clonedDir, downloadErr = archiveGitRepo( + srcParams.SrcURL, + downloadDir, + version, revision, pkg, + bldr.errPrefix) + if downloadErr != nil { + return downloadErr + } + + spec.ClonedDir = clonedDir + upstreamSrc.gitSpec = spec + bldr.log("tarball created") + } else { + bldr.log("downloading %s", srcParams.SrcURL) + // Download source + if upstreamSrc.sourceFile, downloadErr = download( + srcParams.SrcURL, + downloadDir, + repo, pkg, isPkgSubdirInRepo, + bldr.errPrefix); downloadErr != nil { + return downloadErr + } + bldr.log("downloaded") } - bldr.log("downloaded") upstreamSrc.skipSigCheck = upstreamSrcFromManifest.Signature.SkipCheck pubKey := upstreamSrcFromManifest.Signature.DetachedSignature.PubKey @@ -150,6 +172,17 @@ func (bldr *srpmBuilder) fetchUpstream() error { return fmt.Errorf("%sUnexpected public-key specified for SRPM", bldr.errPrefix) } + } else if bldr.pkgSpec.Type == "git" && !upstreamSrc.skipSigCheck { + if pubKey == "" { + return fmt.Errorf("%sexpected public-key for %s to verify git repo", + bldr.errPrefix, pkg) + } + pubKeyPath := filepath.Join(getDetachedSigDir(), pubKey) + if pathErr := util.CheckPath(pubKeyPath, false, false); pathErr != nil { + return fmt.Errorf("%sCannot find public-key at path %s", + bldr.errPrefix, pubKeyPath) + } + upstreamSrc.pubKeyPath = pubKeyPath } bldr.upstreamSrc = append(bldr.upstreamSrc, upstreamSrc) @@ -216,6 +249,18 @@ func (bldr *srpmBuilder) verifyUpstream() error { if err := bldr.verifyUpstreamSrpm(); err != nil { return err } + } else if bldr.pkgSpec.Type == "git" { + // Need to have cloned repo for this too, either use a special path or delete here after use + for _, upstreamSrc := range bldr.upstreamSrc { + if !upstreamSrc.skipSigCheck { + err := util.VerifyGitSignature(upstreamSrc.pubKeyPath, upstreamSrc.gitSpec, bldr.errPrefix) + if err != nil { + return err + } + } + // Deleting cloned git repo since we no longer require it. + os.RemoveAll(upstreamSrc.gitSpec.ClonedDir) + } } else { downloadDir := getDownloadDir(bldr.pkgSpec.Name) for _, upstreamSrc := range bldr.upstreamSrc { @@ -271,7 +316,7 @@ func (bldr *srpmBuilder) setupRpmbuildTreeSrpm() error { // also checks tarball signature func (bldr *srpmBuilder) setupRpmbuildTreeNonSrpm() error { - supportedTypes := []string{"tarball", "standalone"} + supportedTypes := []string{"tarball", "standalone", "git"} if !slices.Contains(supportedTypes, bldr.pkgSpec.Type) { panic(fmt.Sprintf("%ssetupRpmbuildTreeNonSrpm called for unsupported type %s", bldr.errPrefix, bldr.pkgSpec.Type)) @@ -284,7 +329,7 @@ func (bldr *srpmBuilder) setupRpmbuildTreeNonSrpm() error { return err } - if bldr.pkgSpec.Type == "tarball" { + if bldr.pkgSpec.Type == "tarball" || bldr.pkgSpec.Type == "git" { downloadDir := getDownloadDir(bldr.pkgSpec.Name) for _, upstreamSrc := range bldr.upstreamSrc { upstreamSourceFilePath := filepath.Join(downloadDir, upstreamSrc.sourceFile) @@ -379,7 +424,7 @@ func (bldr *srpmBuilder) setupRpmbuildTree() error { if err := bldr.setupRpmbuildTreeSrpm(); err != nil { return err } - } else if bldr.pkgSpec.Type == "tarball" || bldr.pkgSpec.Type == "standalone" { + } else if bldr.pkgSpec.Type == "tarball" || bldr.pkgSpec.Type == "standalone" || bldr.pkgSpec.Type == "git" { if err := bldr.setupRpmbuildTreeNonSrpm(); err != nil { return err } diff --git a/manifest/manifest.go b/manifest/manifest.go index afabb5a..fa677b6 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -5,7 +5,7 @@ package manifest import ( "fmt" - "io/ioutil" + "os" "path/filepath" "golang.org/x/exp/slices" @@ -118,12 +118,18 @@ type SourceBundle struct { SrcRepoParamsOverride srcconfig.SrcRepoParamsOverride `yaml:"override"` } +type GitSpec struct { + Url string `yaml:"url"` + Revision string `yaml:"revision"` +} + // UpstreamSrc spec // Lists each source bundle(tarball/srpm) and // detached signature file for tarball. type UpstreamSrc struct { SourceBundle SourceBundle `yaml:"source-bundle"` FullURL string `yaml:"full-url"` + GitSpec GitSpec `yaml:"git"` Signature Signature `yaml:"signature"` } @@ -147,7 +153,7 @@ type Manifest struct { } func (m Manifest) sanityCheck() error { - allowedPkgTypes := []string{"srpm", "unmodified-srpm", "tarball", "standalone"} + allowedPkgTypes := []string{"srpm", "unmodified-srpm", "tarball", "standalone", "git"} for _, pkgSpec := range m.Package { if pkgSpec.Name == "" { @@ -184,23 +190,45 @@ func (m Manifest) sanityCheck() error { } for _, upStreamSrc := range pkgSpec.UpstreamSrc { - specifiedFullSrcURL := (upStreamSrc.FullURL != "") - specifiedSrcBundle := (upStreamSrc.SourceBundle != SourceBundle{}) - if !specifiedFullSrcURL && !specifiedSrcBundle { - return fmt.Errorf("Specify source for Build in package %s, provide either full-url or source-bundle", - pkgSpec.Name) - } + if pkgSpec.Type == "git" { + specifiedUrl := (upStreamSrc.GitSpec.Url != "") + specifiedRevision := (upStreamSrc.GitSpec.Revision != "") + if !specifiedUrl { + return fmt.Errorf("please provide the url for git repo of package %s", pkgSpec.Name) + } + if !specifiedRevision { + return fmt.Errorf("please provide a commit/tag to define revision of package %s", pkgSpec.Name) + } - if specifiedFullSrcURL && specifiedSrcBundle { - return fmt.Errorf( - "Conflicting sources for Build in package %s, provide either full-url or source-bundle", - pkgSpec.Name) - } + specifiedSignature := (upStreamSrc.Signature != Signature{}) + if specifiedSignature { + skipSigCheck := (upStreamSrc.Signature.SkipCheck) + specifiedPubKey := (upStreamSrc.Signature.DetachedSignature.PubKey != "") + if !skipSigCheck && !specifiedPubKey { + return fmt.Errorf( + "please provide the public key to verify git repo for package %s, or skip signature check", + pkgSpec.Name) + } + } + } else { + specifiedFullSrcURL := (upStreamSrc.FullURL != "") + specifiedSrcBundle := (upStreamSrc.SourceBundle != SourceBundle{}) + if !specifiedFullSrcURL && !specifiedSrcBundle { + return fmt.Errorf("Specify source for Build in package %s, provide either full-url or source-bundle", + pkgSpec.Name) + } - specifiedFullSigURL := upStreamSrc.Signature.DetachedSignature.FullURL != "" - if specifiedFullSigURL && specifiedSrcBundle { - return fmt.Errorf("Conflicting signatures for Build in package %s, provide full-url or source-bundle", - pkgSpec.Name) + if specifiedFullSrcURL && specifiedSrcBundle { + return fmt.Errorf( + "Conflicting sources for Build in package %s, provide either full-url or source-bundle", + pkgSpec.Name) + } + + specifiedFullSigURL := upStreamSrc.Signature.DetachedSignature.FullURL != "" + if specifiedFullSigURL && specifiedSrcBundle { + return fmt.Errorf("Conflicting signatures for Build in package %s, provide full-url or source-bundle", + pkgSpec.Name) + } } } } @@ -213,9 +241,9 @@ func LoadManifest(repo string) (*Manifest, error) { repoDir := util.GetRepoDir(repo) yamlPath := filepath.Join(repoDir, "eext.yaml") - yamlContents, readErr := ioutil.ReadFile(yamlPath) + yamlContents, readErr := os.ReadFile(yamlPath) if readErr != nil { - return nil, fmt.Errorf("manifest.LoadManifest: ioutil.ReadFile on %s returned %s", yamlPath, readErr) + return nil, fmt.Errorf("manifest.LoadManifest: os.ReadFile on %s returned %s", yamlPath, readErr) } var manifest Manifest diff --git a/manifest/manifest_test.go b/manifest/manifest_test.go index 45bb33c..cb27ecd 100644 --- a/manifest/manifest_test.go +++ b/manifest/manifest_test.go @@ -31,12 +31,15 @@ func TestManifest(t *testing.T) { viper.Set("SrcDir", dir) defer viper.Reset() - t.Log("Copy sample manifest to test directory") - testutil.SetupManifest(t, dir, "pkg1", "sampleManifest1.yaml") + testFiles := []string{"sampleManifest1.yaml", "sampleManifest4.yaml"} + for _, testFile := range testFiles { + t.Logf("Copy sample manifest %s to test directory", testFile) + testutil.SetupManifest(t, dir, "pkg1", testFile) - t.Log("Testing Load") - testLoad(t, "pkg1") - t.Log("Load test passed") + t.Log("Testing Load") + testLoad(t, "pkg1") + t.Log("Load test passed") + } } type manifestTestVariant struct { diff --git a/manifest/testData/sampleManifest4.yaml b/manifest/testData/sampleManifest4.yaml new file mode 100644 index 0000000..4f6da5c --- /dev/null +++ b/manifest/testData/sampleManifest4.yaml @@ -0,0 +1,44 @@ +--- +package: + - name: libpcap1 + upstream-sources: + - git: + url: https://github.com/the-tcpdump-group/libpcap + revision: 104271ba4a14de6743e43bcf87536786d8fddea4 + signature: + detached-sig: + public-key: mrtparse/mrtparsePubKey.pem + type: git + build: + repo-bundle: + - name: foo + version: v1 + - name: bar + + - name: libpcap2 + upstream-sources: + - git: + url: https://github.com/the-tcpdump-group/libpcap + revision: libpcap-1.10.1 + type: git + build: + repo-bundle: + - name: foo + version: v1 + - name: bar +# +# - name: libpcap3 +# upstream-sources: +# - snapshot: +# rel-version: 1.10.4 +# commit: 104271ba4a14de6743e43bcf87536786d8fddea4 +# signature: +# skip-check: true +# detached-sig: +# public-key: mrtparse/mrtparsePubKey.pem +# type: local-eextsubdir +# build: +# repo-bundle: +# - name: foo +# version: v1 +# - name: bar diff --git a/srcconfig/srcconfig.go b/srcconfig/srcconfig.go index f63d3b1..d3d5eb0 100644 --- a/srcconfig/srcconfig.go +++ b/srcconfig/srcconfig.go @@ -6,7 +6,6 @@ package srcconfig import ( "bytes" "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -222,9 +221,9 @@ func LoadSrcConfig() (*SrcConfig, error) { cfgPath, statErr) } - yamlContents, readErr := ioutil.ReadFile(cfgPath) + yamlContents, readErr := os.ReadFile(cfgPath) if readErr != nil { - return nil, fmt.Errorf("srcconfig.LoadSrcConfig: ioutil.ReadFile on %s returned %s", + return nil, fmt.Errorf("srcconfig.LoadSrcConfig: os.ReadFile on %s returned %s", cfgPath, readErr) } diff --git a/util/util.go b/util/util.go index b4012d0..dab9bd2 100644 --- a/util/util.go +++ b/util/util.go @@ -26,6 +26,28 @@ var GlobalVar Globals // ErrPrefix is a container type for error prefix strings. type ErrPrefix string +type GitSpec struct { + Revision string + ClonedDir string +} + +// Returns if the provided revision is a "COMMIT" or a "TAG" +func (spec *GitSpec) typeOfGitRevision() string { + // Check 1st line of git show + return "" +} + +// Returns a unique version number based on the commit/tag +func (spec *GitSpec) GetVersionFromRevision() string { + // If type is TAG, return as is + + // If type is commit + // If short commit, return as is + + // If long commit, reduce size + return "" +} + // RunSystemCmd runs a command on the shell and pipes to stdout and stderr func RunSystemCmd(name string, arg ...string) error { cmd := exec.Command(name, arg...) @@ -39,6 +61,20 @@ func RunSystemCmd(name string, arg ...string) error { return err } +// Runs the system command from a specified directory +func RunSystemCmdInDir(dir string, name string, arg ...string) error { + cmd := exec.Command(name, arg...) + cmd.Dir = dir + cmd.Stderr = os.Stderr + if !GlobalVar.Quiet { + cmd.Stdout = os.Stdout + } else { + cmd.Stdout = io.Discard + } + err := cmd.Run() + return err +} + // CheckOutput runs a command on the shell and returns stdout if it is successful // else it return the error func CheckOutput(name string, arg ...string) ( @@ -204,3 +240,51 @@ func VerifyTarballSignature( return nil } + +// VerifyGitSignature verifies that the git repo commit/tag is signed. +func VerifyGitSignature(pubKeyPath string, gitSpec GitSpec, errPrefix ErrPrefix) error { + tmpDir, mkdtErr := os.MkdirTemp("", "eext-keyring") + if mkdtErr != nil { + return fmt.Errorf("%sError '%s'creating temp dir for keyring", + errPrefix, mkdtErr) + } + defer os.RemoveAll(tmpDir) + + keyRingPath := filepath.Join(tmpDir, "eext.gpg") + baseArgs := []string{ + "--homedir", tmpDir, + "--no-default-keyring", "--keyring", keyRingPath} + gpgCmd := "gpg" + + // Create keyring + createKeyRingCmdArgs := append(baseArgs, "--fingerprint") + if err := RunSystemCmd(gpgCmd, createKeyRingCmdArgs...); err != nil { + return fmt.Errorf("%sError '%s'creating keyring", + errPrefix, err) + } + + // Import public key + importKeyCmdArgs := append(baseArgs, "--import", pubKeyPath) + if err := RunSystemCmd(gpgCmd, importKeyCmdArgs...); err != nil { + return fmt.Errorf("%sError '%s' importing public-key %s", + errPrefix, err, pubKeyPath) + } + + var verifyRepoCmd []string + revision := gitSpec.Revision + revisionType := gitSpec.typeOfGitRevision() + if revisionType == "COMMIT" { + verifyRepoCmd = []string{"verify-commit", "-v", revision} + } else if revisionType == "TAG" { + verifyRepoCmd = []string{"verify-tag", "-v", revision} + } else { + return fmt.Errorf("%sinvalid revision %s provided: %s, provide either a COMMIT or TAG", errPrefix, revision) + } + clonedDir := gitSpec.ClonedDir + err := RunSystemCmdInDir(clonedDir, "git", verifyRepoCmd...) + if err != nil { + return fmt.Errorf("%serror during verifying git repo at %s: %s", errPrefix, clonedDir, err) + } + + return nil +}