Offline mode

cmd/proxy/actions/app.go

@@ -23,6 +23,8 @@ const Service = "proxy"
 // should be defined. This is the nerve center of your
 // application.
 func App(conf *config.Config) (http.Handler, error) {
+	fmt.Printf("Athens is running in %s mode\n", conf.Mode)
+
 	// ENV is used to help switch settings based on where the
 	// application is being run. Default is "development".
 	ENV := conf.GoEnv

cmd/proxy/actions/app_proxy.go

@@ -86,45 +86,50 @@ func addProxyRoutes(
 	// 4. The plain stash.New just takes a request from upstream and saves it into storage.
 	fs := afero.NewOsFs()
 
-	// TODO: remove before we release v0.7.0
-	if c.GoProxy != "direct" && c.GoProxy != "" {
-		l.Error("GoProxy is deprecated, please use GoBinaryEnvVars")
-	}
-	if !c.GoBinaryEnvVars.HasKey("GONOSUMDB") {
-		c.GoBinaryEnvVars.Add("GONOSUMDB", strings.Join(c.NoSumPatterns, ","))
-	}
-	if err := c.GoBinaryEnvVars.Validate(); err != nil {
-		return err
-	}
-	mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoGetDir, c.GoBinaryEnvVars, fs, c.PropagateAuthHost)
-	if err != nil {
-		return err
-	}
+	if c.Mode == config.ModeOnline {
+		// TODO: remove before we release v0.7.0
+		if c.GoProxy != "direct" && c.GoProxy != "" {
+			l.Error("GoProxy is deprecated, please use GoBinaryEnvVars")
+		}
+		if !c.GoBinaryEnvVars.HasKey("GONOSUMDB") {
+			c.GoBinaryEnvVars.Add("GONOSUMDB", strings.Join(c.NoSumPatterns, ","))
+		}
+		if err := c.GoBinaryEnvVars.Validate(); err != nil {
+			return err
+		}
+		mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoGetDir, c.GoBinaryEnvVars, fs, c.PropagateAuthHost)
+		if err != nil {
+			return err
+		}
 
-	lister := module.NewVCSLister(c.GoBinary, c.GoBinaryEnvVars, fs, c.PropagateAuthHost)
-	checker := storage.WithChecker(s)
-	withSingleFlight, err := getSingleFlight(c, checker)
-	if err != nil {
-		return err
-	}
-	st := stash.New(mf, s, indexer, stash.WithPool(c.GoGetWorkers), withSingleFlight)
+		lister := module.NewVCSLister(c.GoBinary, c.GoBinaryEnvVars, fs, c.PropagateAuthHost)
+		checker := storage.WithChecker(s)
+		withSingleFlight, err := getSingleFlight(c, checker)
+		if err != nil {
+			return err
+		}
+		st := stash.New(mf, s, indexer, stash.WithPool(c.GoGetWorkers), withSingleFlight)
 
-	df, err := mode.NewFile(c.DownloadMode, c.DownloadURL)
-	if err != nil {
-		return err
-	}
+		df, err := mode.NewFile(c.DownloadMode, c.DownloadURL)
+		if err != nil {
+			return err
+		}
 
-	dpOpts := &download.Opts{
-		Storage:      s,
-		Stasher:      st,
-		Lister:       lister,
-		DownloadFile: df,
-	}
+		dpOpts := &download.Opts{
+			Storage:      s,
+			Stasher:      st,
+			Lister:       lister,
+			DownloadFile: df,
+		}
 
-	dp := download.New(dpOpts, addons.WithPool(c.ProtocolWorkers))
+		dp := download.New(dpOpts, addons.WithPool(c.ProtocolWorkers))
 
-	handlerOpts := &download.HandlerOpts{Protocol: dp, Logger: l, DownloadFile: df}
-	download.RegisterHandlers(r, handlerOpts)
+		handlerOpts := &download.HandlerOpts{Protocol: dp, Logger: l, DownloadFile: df}
+		download.RegisterHandlers(r, handlerOpts)
+	} else {
+		handlerOpts := &download.OfflineHandlerOpts{Logger: l, Storage: s}
+		download.RegisterOfflineHandlers(r, handlerOpts)
+	}
 
 	return nil
 }

pkg/config/config.go

@@ -21,6 +21,7 @@ const defaultConfigFile = "athens.toml"
 // Config provides configuration values for all components
 type Config struct {
 	TimeoutConf
+	Mode              Mode      `validate:"required" envconfig:"MODE"`
 	GoEnv             string    `validate:"required" envconfig:"GO_ENV"`
 	GoBinary          string    `validate:"required" envconfig:"GO_BINARY_PATH"`
 	GoProxy           string    `envconfig:"GOPROXY"`
@@ -62,6 +63,13 @@ type Config struct {
 	Index             *Index
 }
 
+type Mode string
+
+const (
+	ModeOffline Mode = "offline"
+	ModeOnline  Mode = "online"
+)
+
 // EnvList is a list of key-value environment
 // variables that are passed to the Go command
 type EnvList []string

pkg/download/handler.go

@@ -8,13 +8,19 @@ import (
 	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/log"
 	"github.com/gomods/athens/pkg/middleware"
+	"github.com/gomods/athens/pkg/storage"
 	"github.com/gorilla/mux"
 )
 
 // ProtocolHandler is a function that takes all that it needs to return
 // a ready-to-go http handler that serves up cmd/go's download protocol.
 type ProtocolHandler func(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
 
+// OfflineProtocolHandler is a function that takes all it needs to return
+// a ready-to-go http handler that serves up module information directly
+// from storage, not using any sources on the internet
+type OfflineProtocolHandler func(lggr log.Entry, s storage.Backend) http.Handler
+
 // HandlerOpts are the generic options
 // for a ProtocolHandler
 type HandlerOpts struct {
@@ -23,6 +29,12 @@ type HandlerOpts struct {
 	DownloadFile *mode.DownloadFile
 }
 
+type OfflineHandlerOpts struct {
+	Storage storage.Backend
+	Logger  *log.Logger
+	// DownloadFile *mode.DownloadFile
+}
+
 // LogEntryHandler pulls a log entry from the request context. Thanks to the
 // LogEntryMiddleware, we should have a log entry stored in the context for each
 // request with request-specific fields. This will grab the entry and pass it to
@@ -36,6 +48,15 @@ func LogEntryHandler(ph ProtocolHandler, opts *HandlerOpts) http.Handler {
 	return http.HandlerFunc(f)
 }
 
+func OfflineLogEntryHandler(ph OfflineProtocolHandler, opts *OfflineHandlerOpts) http.Handler {
+	f := func(w http.ResponseWriter, r *http.Request) {
+		ent := log.EntryFromContext(r.Context())
+		handler := ph(ent, opts.Storage)
+		handler.ServeHTTP(w, r)
+	}
+	return http.HandlerFunc(f)
+}
+
 // RegisterHandlers is a convenience method that registers
 // all the download protocol paths for you.
 func RegisterHandlers(r *mux.Router, opts *HandlerOpts) {
@@ -64,3 +85,21 @@ func getRedirectURL(base, downloadPath string) (string, error) {
 	url.Path = path.Join(url.Path, downloadPath)
 	return url.String(), nil
 }
+
+func RegisterOfflineHandlers(r *mux.Router, opts *OfflineHandlerOpts) {
+	// If true, this would only panic at boot time, static nil checks anyone?
+	if opts == nil || opts.Logger == nil {
+		panic("absolutely unacceptable handler opts")
+	}
+	noCacheMw := middleware.CacheControl("no-cache, no-store, must-revalidate")
+
+	listHandler := OfflineLogEntryHandler(OfflineListHandler, opts)
+	r.Handle(PathList, noCacheMw(listHandler))
+
+	latestHandler := OfflineLogEntryHandler(OfflineLatestHandler, opts)
+	r.Handle(PathLatest, noCacheMw(latestHandler)).Methods(http.MethodGet)
+
+	r.Handle(PathVersionInfo, OfflineLogEntryHandler(OfflineInfoHandler, opts)).Methods(http.MethodGet)
+	r.Handle(PathVersionModule, OfflineLogEntryHandler(OfflineModuleHandler, opts)).Methods(http.MethodGet)
+	r.Handle(PathVersionZip, OfflineLogEntryHandler(OfflineZipHandler, opts)).Methods(http.MethodGet)
+}

pkg/download/latest.go

@@ -8,6 +8,8 @@ import (
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/log"
 	"github.com/gomods/athens/pkg/paths"
+	"github.com/gomods/athens/pkg/storage"
+	"github.com/sirupsen/logrus"
 )
 
 // PathLatest URL.
@@ -39,3 +41,42 @@ func LatestHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Hand
 	}
 	return http.HandlerFunc(f)
 }
+
+func OfflineLatestHandler(lggr log.Entry, s storage.Backend) http.Handler {
+	const op errors.Op = "download.LatestHandler"
+	f := func(w http.ResponseWriter, r *http.Request) {
+		mod, err := paths.GetModule(r)
+		if err != nil {
+			lggr.SystemErr(errors.E(op, err))
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+
+		versions, err := s.List(r.Context(), mod)
+		if err != nil {
+			err = errors.E(op, err, logrus.ErrorLevel)
+			lggr.SystemErr(err)
+			w.WriteHeader(errors.Kind(err))
+			return
+		}
+		if len(versions) < 1 {
+			err := errors.E(op, err, logrus.ErrorLevel)
+			lggr.SystemErr(err)
+			w.WriteHeader(errors.Kind(err))
+			return
+		}
+
+		latestVersion := versions[0]
+		info, err := s.Info(r.Context(), mod, latestVersion)
+		if err != nil {
+			err := errors.E(op, err, logrus.ErrorLevel)
+			lggr.SystemErr(err)
+			w.WriteHeader(errors.Kind(err))
+			return
+		}
+
+		w.Write(info)
+	}
+	return http.HandlerFunc(f)
+
+}

pkg/download/list.go

@@ -9,6 +9,8 @@ import (
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/log"
 	"github.com/gomods/athens/pkg/paths"
+	"github.com/gomods/athens/pkg/storage"
+	"github.com/sirupsen/logrus"
 )
 
 // PathList URL.
@@ -38,3 +40,29 @@ func ListHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handle
 	}
 	return http.HandlerFunc(f)
 }
+
+// OfflineListHandler returns an http.Handler capable of serving /@v/list endpoints
+// directly from storage. No network traffic to anywhere other than the storage
+// backend will be attempted
+func OfflineListHandler(lggr log.Entry, s storage.Backend) http.Handler {
+	const op errors.Op = "download.OfflineListHandler"
+	f := func(w http.ResponseWriter, r *http.Request) {
+		mod, err := paths.GetModule(r)
+		if err != nil {
+			lggr.SystemErr(errors.E(op, err))
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		versions, err := s.List(r.Context(), mod)
+		if err != nil {
+			err = errors.E(op, err, logrus.ErrorLevel)
+			lggr.SystemErr(err)
+			w.WriteHeader(errors.Kind(err))
+			return
+		}
+		fmt.Fprint(w, strings.Join(versions, "\n"))
+	}
+
+	return http.HandlerFunc(f)
+
+}

pkg/download/version_info.go

@@ -6,6 +6,7 @@ import (
 	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/log"
+	"github.com/gomods/athens/pkg/storage"
 )
 
 // PathVersionInfo URL.
@@ -42,3 +43,25 @@ func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handle
 	}
 	return http.HandlerFunc(f)
 }
+
+func OfflineInfoHandler(lggr log.Entry, s storage.Backend) http.Handler {
+	const op errors.Op = "download.InfoHandler"
+	f := func(w http.ResponseWriter, r *http.Request) {
+		mod, ver, err := getModuleParams(r, op)
+		if err != nil {
+			lggr.SystemErr(err)
+			w.WriteHeader(errors.Kind(err))
+			return
+		}
+
+		info, err := s.Info(r.Context(), mod, ver)
+		if err != nil {
+			lggr.SystemErr(err)
+			w.WriteHeader(errors.Kind(err))
+			return
+		}
+		w.Write(info)
+	}
+	return http.HandlerFunc(f)
+
+}

pkg/download/version_module.go

@@ -6,6 +6,7 @@ import (
 	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/log"
+	"github.com/gomods/athens/pkg/storage"
 )
 
 // PathVersionModule URL.
@@ -46,3 +47,20 @@ func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Hand
 	}
 	return http.HandlerFunc(f)
 }
+
+// OfflineModuleHandler implements GET baseURL/module/@v/version.mod
+func OfflineModuleHandler(lggr log.Entry, s storage.Backend) http.Handler {
+	const op errors.Op = "download.VersionModuleHandler"
+	f := func(w http.ResponseWriter, r *http.Request) {
+		mod, ver, err := getModuleParams(r, op)
+		if err != nil {
+			err = errors.E(op, errors.M(mod), errors.V(ver), err)
+			lggr.SystemErr(err)
+			w.WriteHeader(errors.Kind(err))
+			return
+		}
+		modBytes, err := s.GoMod(r.Context(), mod, ver)
+		w.Write(modBytes)
+	}
+	return http.HandlerFunc(f)
+}

pkg/download/version_zip.go

@@ -8,6 +8,7 @@ import (
 	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/log"
+	"github.com/gomods/athens/pkg/storage"
 )
 
 // PathVersionZip URL.
@@ -55,3 +56,33 @@ func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
 	}
 	return http.HandlerFunc(f)
 }
+
+// ZipHandler implements GET baseURL/module/@v/version.zip
+func OfflineZipHandler(lggr log.Entry, s storage.Backend) http.Handler {
+	const op errors.Op = "download.ZipHandler"
+	f := func(w http.ResponseWriter, r *http.Request) {
+		mod, ver, err := getModuleParams(r, op)
+		if err != nil {
+			lggr.SystemErr(err)
+			w.WriteHeader(errors.Kind(err))
+			return
+		}
+
+		zip, err := s.Zip(r.Context(), mod, ver)
+		if err != nil {
+			severityLevel := errors.Expect(err, errors.KindNotFound, errors.KindRedirect)
+			err = errors.E(op, err, severityLevel)
+			lggr.SystemErr(err)
+			w.WriteHeader(errors.Kind(err))
+			return
+		}
+		defer zip.Close()
+
+		w.Header().Set("Content-Type", "application/zip")
+		_, err = io.Copy(w, zip)
+		if err != nil {
+			lggr.SystemErr(errors.E(op, errors.M(mod), errors.V(ver), err))
+		}
+	}
+	return http.HandlerFunc(f)
+}