Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

partial response to #163 to add retries for dls #389

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

partial response to #163 to add retries for dls #389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
84a835e
partial response to #163 to add retries for dls
Oct 13, 2021
d682a88
test: add whitespace to test so doesn't also accidentally target new …
Oct 13, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 36 additions & 4 deletions cran/download-package.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@ import (
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"

"github.com/dpastoor/goutils"
"github.com/hashicorp/go-retryablehttp"
log "github.com/sirupsen/logrus"
"github.com/spf13/afero"
)
Expand Down Expand Up @@ -67,14 +69,30 @@ func getRepos(ds []PkgDl) map[string]RepoURL {
return rpm
}

func getIntEnv(val string) int {
ret, err := strconv.Atoi(val)
if err != nil {
log.Error("could not parse PKGR_DL_CONCURRENCY to int - reverting to 10")
return 10
}
return ret
}

// DownloadPackages downloads a set of packages concurrently
// noSecure will allow https fetching without validating the certificate chain.
// This occasionally is needed for repos that have self signed or certs not fully verifiable
// which will return errors such as x509: certificate signed by unknown authority
func DownloadPackages(fs afero.Fs, ds []PkgDl, baseDir string, rv RVersion, noSecure bool) (*PkgMap, error) {
startTime := time.Now()
result := NewPkgMap()
sem := make(chan struct{}, 10)

dlConcurrencyStr, set := os.LookupEnv("PKGR_DL_CONCURRENCY")
if !set {
dlConcurrencyStr = "10"
}
dlConcurrency := getIntEnv(dlConcurrencyStr)
log.Infof("downloading packages with concurrency: %v\n", dlConcurrency)
sem := make(chan struct{}, dlConcurrency)
wg := sync.WaitGroup{}
rpm := getRepos(ds)
for _, r := range rpm {
Expand Down Expand Up @@ -190,13 +208,27 @@ func DownloadPackage(fs afero.Fs, d PkgDl, dest string, rv RVersion, noSecure bo

log.WithField("package", d.Package.Package).Info("downloading package ")
var from io.ReadCloser
client := retryablehttp.NewClient()
client.RetryMax = 3
switch log.GetLevel() {
case log.DebugLevel, log.TraceLevel:
// for some reason client.Logger will not take the default log anyway
// this will also make it such that the log will only write at debug/trace levels
// it currently writes out messages like:
// [DEBUG] GET https://cran.r-project.org/bin/macosx/contrib/4.1/scales_1.1.1.tgz
// and get fed to Printf
lg := log.New()
lg.Level = log.GetLevel()
client.Logger = lg
default:
client.Logger = nil
}

client := &http.Client{}
if noSecure {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify : true},
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client = &http.Client{Transport: tr}
client.HTTPClient.Transport = tr
}

if strings.HasPrefix(pkgdl, "http") {
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ require (
github.com/dpastoor/goutils v1.2.0
github.com/fatih/structs v1.1.0
github.com/fatih/structtag v1.2.0
github.com/hashicorp/go-retryablehttp v0.7.0
github.com/metrumresearchgroup/command v0.1.0
github.com/mholt/archiver/v3 v3.5.0
github.com/mitchellh/go-homedir v1.1.0
Expand Down
4 changes: 4 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,10 +158,14 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-retryablehttp v0.7.0 h1:eu1EI/mbirUgP5C8hVsTNaGZreBDlYiwC1FZWkvQPQ4=
github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
Expand Down
2 changes: 1 addition & 1 deletion integration_tests/baseline/cache_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ func TestClean(t *testing.T) {
if err != nil {
t.Fatalf("error running pkgr install: %s", err)
}
downloadLogs := CollectGenericLogs(t, capture, "downloading package")
downloadLogs := CollectGenericLogs(t, capture, "downloading package ")
assert.Len(t, downloadLogs, 1, "expected exactly one package to be downloaded")
assert.Equal(t, "pillar", downloadLogs[0].Package)
toInstallLogs := CollectGenericLogs(t, capture, "package installation plan")
Expand Down