diff --git a/cmd/proxy/actions/app.go b/cmd/proxy/actions/app.go index af423a978..c5aec12ed 100644 --- a/cmd/proxy/actions/app.go +++ b/cmd/proxy/actions/app.go @@ -23,6 +23,8 @@ const Service = "proxy" // should be defined. This is the nerve center of your // application. func App(conf *config.Config) (http.Handler, error) { + fmt.Printf("Athens is running in %s mode\n", conf.Mode) + // ENV is used to help switch settings based on where the // application is being run. Default is "development". ENV := conf.GoEnv diff --git a/cmd/proxy/actions/app_proxy.go b/cmd/proxy/actions/app_proxy.go index b3dd3450d..b2fb10172 100644 --- a/cmd/proxy/actions/app_proxy.go +++ b/cmd/proxy/actions/app_proxy.go @@ -86,45 +86,50 @@ func addProxyRoutes( // 4. The plain stash.New just takes a request from upstream and saves it into storage. fs := afero.NewOsFs() - // TODO: remove before we release v0.7.0 - if c.GoProxy != "direct" && c.GoProxy != "" { - l.Error("GoProxy is deprecated, please use GoBinaryEnvVars") - } - if !c.GoBinaryEnvVars.HasKey("GONOSUMDB") { - c.GoBinaryEnvVars.Add("GONOSUMDB", strings.Join(c.NoSumPatterns, ",")) - } - if err := c.GoBinaryEnvVars.Validate(); err != nil { - return err - } - mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoGetDir, c.GoBinaryEnvVars, fs, c.PropagateAuthHost) - if err != nil { - return err - } + if c.Mode == config.ModeOnline { + // TODO: remove before we release v0.7.0 + if c.GoProxy != "direct" && c.GoProxy != "" { + l.Error("GoProxy is deprecated, please use GoBinaryEnvVars") + } + if !c.GoBinaryEnvVars.HasKey("GONOSUMDB") { + c.GoBinaryEnvVars.Add("GONOSUMDB", strings.Join(c.NoSumPatterns, ",")) + } + if err := c.GoBinaryEnvVars.Validate(); err != nil { + return err + } + mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoGetDir, c.GoBinaryEnvVars, fs, c.PropagateAuthHost) + if err != nil { + return err + } - lister := module.NewVCSLister(c.GoBinary, c.GoBinaryEnvVars, fs, c.PropagateAuthHost) - checker := storage.WithChecker(s) - withSingleFlight, err := getSingleFlight(c, checker) - if err != nil { - return err - } - st := stash.New(mf, s, indexer, stash.WithPool(c.GoGetWorkers), withSingleFlight) + lister := module.NewVCSLister(c.GoBinary, c.GoBinaryEnvVars, fs, c.PropagateAuthHost) + checker := storage.WithChecker(s) + withSingleFlight, err := getSingleFlight(c, checker) + if err != nil { + return err + } + st := stash.New(mf, s, indexer, stash.WithPool(c.GoGetWorkers), withSingleFlight) - df, err := mode.NewFile(c.DownloadMode, c.DownloadURL) - if err != nil { - return err - } + df, err := mode.NewFile(c.DownloadMode, c.DownloadURL) + if err != nil { + return err + } - dpOpts := &download.Opts{ - Storage: s, - Stasher: st, - Lister: lister, - DownloadFile: df, - } + dpOpts := &download.Opts{ + Storage: s, + Stasher: st, + Lister: lister, + DownloadFile: df, + } - dp := download.New(dpOpts, addons.WithPool(c.ProtocolWorkers)) + dp := download.New(dpOpts, addons.WithPool(c.ProtocolWorkers)) - handlerOpts := &download.HandlerOpts{Protocol: dp, Logger: l, DownloadFile: df} - download.RegisterHandlers(r, handlerOpts) + handlerOpts := &download.HandlerOpts{Protocol: dp, Logger: l, DownloadFile: df} + download.RegisterHandlers(r, handlerOpts) + } else { + handlerOpts := &download.OfflineHandlerOpts{Logger: l, Storage: s} + download.RegisterOfflineHandlers(r, handlerOpts) + } return nil } diff --git a/pkg/config/config.go b/pkg/config/config.go index 7f05cb58e..cada7d423 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -21,6 +21,7 @@ const defaultConfigFile = "athens.toml" // Config provides configuration values for all components type Config struct { TimeoutConf + Mode Mode `validate:"required" envconfig:"MODE"` GoEnv string `validate:"required" envconfig:"GO_ENV"` GoBinary string `validate:"required" envconfig:"GO_BINARY_PATH"` GoProxy string `envconfig:"GOPROXY"` @@ -62,6 +63,13 @@ type Config struct { Index *Index } +type Mode string + +const ( + ModeOffline Mode = "offline" + ModeOnline Mode = "online" +) + // EnvList is a list of key-value environment // variables that are passed to the Go command type EnvList []string diff --git a/pkg/download/handler.go b/pkg/download/handler.go index 13106e620..ee3d382fa 100644 --- a/pkg/download/handler.go +++ b/pkg/download/handler.go @@ -8,6 +8,7 @@ import ( "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/log" "github.com/gomods/athens/pkg/middleware" + "github.com/gomods/athens/pkg/storage" "github.com/gorilla/mux" ) @@ -15,6 +16,11 @@ import ( // a ready-to-go http handler that serves up cmd/go's download protocol. type ProtocolHandler func(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler +// OfflineProtocolHandler is a function that takes all it needs to return +// a ready-to-go http handler that serves up module information directly +// from storage, not using any sources on the internet +type OfflineProtocolHandler func(lggr log.Entry, s storage.Backend) http.Handler + // HandlerOpts are the generic options // for a ProtocolHandler type HandlerOpts struct { @@ -23,6 +29,12 @@ type HandlerOpts struct { DownloadFile *mode.DownloadFile } +type OfflineHandlerOpts struct { + Storage storage.Backend + Logger *log.Logger + // DownloadFile *mode.DownloadFile +} + // LogEntryHandler pulls a log entry from the request context. Thanks to the // LogEntryMiddleware, we should have a log entry stored in the context for each // request with request-specific fields. This will grab the entry and pass it to @@ -36,6 +48,15 @@ func LogEntryHandler(ph ProtocolHandler, opts *HandlerOpts) http.Handler { return http.HandlerFunc(f) } +func OfflineLogEntryHandler(ph OfflineProtocolHandler, opts *OfflineHandlerOpts) http.Handler { + f := func(w http.ResponseWriter, r *http.Request) { + ent := log.EntryFromContext(r.Context()) + handler := ph(ent, opts.Storage) + handler.ServeHTTP(w, r) + } + return http.HandlerFunc(f) +} + // RegisterHandlers is a convenience method that registers // all the download protocol paths for you. func RegisterHandlers(r *mux.Router, opts *HandlerOpts) { @@ -64,3 +85,21 @@ func getRedirectURL(base, downloadPath string) (string, error) { url.Path = path.Join(url.Path, downloadPath) return url.String(), nil } + +func RegisterOfflineHandlers(r *mux.Router, opts *OfflineHandlerOpts) { + // If true, this would only panic at boot time, static nil checks anyone? + if opts == nil || opts.Logger == nil { + panic("absolutely unacceptable handler opts") + } + noCacheMw := middleware.CacheControl("no-cache, no-store, must-revalidate") + + listHandler := OfflineLogEntryHandler(OfflineListHandler, opts) + r.Handle(PathList, noCacheMw(listHandler)) + + latestHandler := OfflineLogEntryHandler(OfflineLatestHandler, opts) + r.Handle(PathLatest, noCacheMw(latestHandler)).Methods(http.MethodGet) + + r.Handle(PathVersionInfo, OfflineLogEntryHandler(OfflineInfoHandler, opts)).Methods(http.MethodGet) + r.Handle(PathVersionModule, OfflineLogEntryHandler(OfflineModuleHandler, opts)).Methods(http.MethodGet) + r.Handle(PathVersionZip, OfflineLogEntryHandler(OfflineZipHandler, opts)).Methods(http.MethodGet) +} diff --git a/pkg/download/latest.go b/pkg/download/latest.go index e288141f4..7721a32d6 100644 --- a/pkg/download/latest.go +++ b/pkg/download/latest.go @@ -8,6 +8,8 @@ import ( "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/log" "github.com/gomods/athens/pkg/paths" + "github.com/gomods/athens/pkg/storage" + "github.com/sirupsen/logrus" ) // PathLatest URL. @@ -39,3 +41,42 @@ func LatestHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Hand } return http.HandlerFunc(f) } + +func OfflineLatestHandler(lggr log.Entry, s storage.Backend) http.Handler { + const op errors.Op = "download.LatestHandler" + f := func(w http.ResponseWriter, r *http.Request) { + mod, err := paths.GetModule(r) + if err != nil { + lggr.SystemErr(errors.E(op, err)) + w.WriteHeader(http.StatusInternalServerError) + return + } + + versions, err := s.List(r.Context(), mod) + if err != nil { + err = errors.E(op, err, logrus.ErrorLevel) + lggr.SystemErr(err) + w.WriteHeader(errors.Kind(err)) + return + } + if len(versions) < 1 { + err := errors.E(op, err, logrus.ErrorLevel) + lggr.SystemErr(err) + w.WriteHeader(errors.Kind(err)) + return + } + + latestVersion := versions[0] + info, err := s.Info(r.Context(), mod, latestVersion) + if err != nil { + err := errors.E(op, err, logrus.ErrorLevel) + lggr.SystemErr(err) + w.WriteHeader(errors.Kind(err)) + return + } + + w.Write(info) + } + return http.HandlerFunc(f) + +} diff --git a/pkg/download/list.go b/pkg/download/list.go index 3de919a4d..ed5d5ffc9 100644 --- a/pkg/download/list.go +++ b/pkg/download/list.go @@ -9,6 +9,8 @@ import ( "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/log" "github.com/gomods/athens/pkg/paths" + "github.com/gomods/athens/pkg/storage" + "github.com/sirupsen/logrus" ) // PathList URL. @@ -38,3 +40,29 @@ func ListHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handle } return http.HandlerFunc(f) } + +// OfflineListHandler returns an http.Handler capable of serving /@v/list endpoints +// directly from storage. No network traffic to anywhere other than the storage +// backend will be attempted +func OfflineListHandler(lggr log.Entry, s storage.Backend) http.Handler { + const op errors.Op = "download.OfflineListHandler" + f := func(w http.ResponseWriter, r *http.Request) { + mod, err := paths.GetModule(r) + if err != nil { + lggr.SystemErr(errors.E(op, err)) + w.WriteHeader(http.StatusInternalServerError) + return + } + versions, err := s.List(r.Context(), mod) + if err != nil { + err = errors.E(op, err, logrus.ErrorLevel) + lggr.SystemErr(err) + w.WriteHeader(errors.Kind(err)) + return + } + fmt.Fprint(w, strings.Join(versions, "\n")) + } + + return http.HandlerFunc(f) + +} diff --git a/pkg/download/version_info.go b/pkg/download/version_info.go index b0dd802e5..64c467fda 100644 --- a/pkg/download/version_info.go +++ b/pkg/download/version_info.go @@ -6,6 +6,7 @@ import ( "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/log" + "github.com/gomods/athens/pkg/storage" ) // PathVersionInfo URL. @@ -42,3 +43,25 @@ func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handle } return http.HandlerFunc(f) } + +func OfflineInfoHandler(lggr log.Entry, s storage.Backend) http.Handler { + const op errors.Op = "download.InfoHandler" + f := func(w http.ResponseWriter, r *http.Request) { + mod, ver, err := getModuleParams(r, op) + if err != nil { + lggr.SystemErr(err) + w.WriteHeader(errors.Kind(err)) + return + } + + info, err := s.Info(r.Context(), mod, ver) + if err != nil { + lggr.SystemErr(err) + w.WriteHeader(errors.Kind(err)) + return + } + w.Write(info) + } + return http.HandlerFunc(f) + +} diff --git a/pkg/download/version_module.go b/pkg/download/version_module.go index 46806894c..5bb154f74 100644 --- a/pkg/download/version_module.go +++ b/pkg/download/version_module.go @@ -6,6 +6,7 @@ import ( "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/log" + "github.com/gomods/athens/pkg/storage" ) // PathVersionModule URL. @@ -46,3 +47,20 @@ func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Hand } return http.HandlerFunc(f) } + +// OfflineModuleHandler implements GET baseURL/module/@v/version.mod +func OfflineModuleHandler(lggr log.Entry, s storage.Backend) http.Handler { + const op errors.Op = "download.VersionModuleHandler" + f := func(w http.ResponseWriter, r *http.Request) { + mod, ver, err := getModuleParams(r, op) + if err != nil { + err = errors.E(op, errors.M(mod), errors.V(ver), err) + lggr.SystemErr(err) + w.WriteHeader(errors.Kind(err)) + return + } + modBytes, err := s.GoMod(r.Context(), mod, ver) + w.Write(modBytes) + } + return http.HandlerFunc(f) +} diff --git a/pkg/download/version_zip.go b/pkg/download/version_zip.go index fe36673f3..a825a9e67 100644 --- a/pkg/download/version_zip.go +++ b/pkg/download/version_zip.go @@ -8,6 +8,7 @@ import ( "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/log" + "github.com/gomods/athens/pkg/storage" ) // PathVersionZip URL. @@ -55,3 +56,33 @@ func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler } return http.HandlerFunc(f) } + +// ZipHandler implements GET baseURL/module/@v/version.zip +func OfflineZipHandler(lggr log.Entry, s storage.Backend) http.Handler { + const op errors.Op = "download.ZipHandler" + f := func(w http.ResponseWriter, r *http.Request) { + mod, ver, err := getModuleParams(r, op) + if err != nil { + lggr.SystemErr(err) + w.WriteHeader(errors.Kind(err)) + return + } + + zip, err := s.Zip(r.Context(), mod, ver) + if err != nil { + severityLevel := errors.Expect(err, errors.KindNotFound, errors.KindRedirect) + err = errors.E(op, err, severityLevel) + lggr.SystemErr(err) + w.WriteHeader(errors.Kind(err)) + return + } + defer zip.Close() + + w.Header().Set("Content-Type", "application/zip") + _, err = io.Copy(w, zip) + if err != nil { + lggr.SystemErr(errors.E(op, errors.M(mod), errors.V(ver), err)) + } + } + return http.HandlerFunc(f) +}