diff --git a/Dockerfile b/Dockerfile index e0c2b16fa54..97cea752d54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM alpine MAINTAINER Brian O'Kelley ADD prebid-server prebid-server COPY static static/ +COPY stored_requests/data stored_requests/data EXPOSE 8000 ENTRYPOINT ["/prebid-server"] CMD ["-v", "1", "-logtostderr"] diff --git a/cache/cache.go b/cache/legacy.go similarity index 91% rename from cache/cache.go rename to cache/legacy.go index c762e5906a6..9f053796caa 100644 --- a/cache/cache.go +++ b/cache/legacy.go @@ -1,6 +1,8 @@ package cache -import "github.com/prebid/prebid-server/pbs/buckets" +import ( + "github.com/prebid/prebid-server/pbs/buckets" +) type Domain struct { Domain string `json:"domain"` diff --git a/config/config.go b/config/config.go index 079ac4feeca..0e3042e79b1 100644 --- a/config/config.go +++ b/config/config.go @@ -1,8 +1,11 @@ package config import ( + "bytes" + "errors" "fmt" "github.com/spf13/viper" + "strconv" "strings" ) @@ -18,7 +21,13 @@ type Configuration struct { HostCookie HostCookie `mapstructure:"host_cookie"` Metrics Metrics `mapstructure:"metrics"` DataCache DataCache `mapstructure:"datacache"` + StoredRequests StoredRequests `mapstructure:"stored_requests"` Adapters map[string]Adapter `mapstructure:"adapters"` + MaxRequestSize int64 `mapstructure:"max_request_size"` +} + +func (cfg *Configuration) validate() error { + return cfg.StoredRequests.validate() } type HostCookie struct { @@ -59,6 +68,61 @@ type DataCache struct { TTLSeconds int `mapstructure:"ttl_seconds"` } +// StoredRequests configures the backend used to store requests on the server. +type StoredRequests struct { + // Files should be true if Stored Requests should be loaded from the filesystem. + Files bool `mapstructure:"filesystem"` + // Postgres should be non-nil if Stored Requests should be loaded from a Postgres database. + Postgres *PostgresConfig `mapstructure:"postgres"` +} + +func (cfg *StoredRequests) validate() error { + if cfg.Files && cfg.Postgres != nil { + return errors.New("Only one backend from {filesystem, postgres} can be used at the same time.") + } + + return nil +} + +// PostgresConfig configures the Postgres connection for Stored Requests +type PostgresConfig struct { + Database string `mapstructure:"dbname"` + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Username string `mapstructure:"user"` + Password string `mapstructure:"password"` + + // QueryTemplate is the Postgres Query which can be used to fetch configs from the database. + // It is a Template, rather than a full Query, because a single HTTP request may reference multiple Stored Requests. + // + // In the simplest case, this could be something like: + // SELECT id, requestData FROM stored_requests WHERE id in %ID_LIST% + // + // The MakeQuery function will transform this query into: + // SELECT id, requestData FROM stored_requests WHERE id in ($1, $2, $3, ...) + // + // ... where the number of "$x" args depends on how many IDs are nested within the HTTP request. + QueryTemplate string `mapstructure:"query"` +} + +// MakeQuery gets a stored-request-fetching query which can be used to fetch numRequests requests at once. +func (cfg *PostgresConfig) MakeQuery(numRequests int) (string, error) { + if numRequests < 1 { + return "", fmt.Errorf("can't generate query to fetch %d stored requests", numRequests) + } + final := bytes.NewBuffer(make([]byte, 0, 2+4*numRequests)) + final.WriteString("(") + for i := 1; i < numRequests; i++ { + final.WriteString("$") + final.WriteString(strconv.Itoa(i)) + final.WriteString(", ") + } + final.WriteString("$") + final.WriteString(strconv.Itoa(numRequests)) + final.WriteString(")") + return strings.Replace(cfg.QueryTemplate, "%ID_LIST%", final.String(), 1), nil +} + type Cache struct { Scheme string `mapstructure:"scheme"` Host string `mapstructure:"host"` @@ -76,7 +140,7 @@ func New() (*Configuration, error) { if err := viper.Unmarshal(&c); err != nil { return nil, err } - return &c, nil + return &c, c.validate() } //Allows for protocol relative URL if scheme is empty diff --git a/config/config_test.go b/config/config_test.go index b9d8a6afda3..6ff3404f734 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,10 +1,9 @@ -package config_test +package config import ( "bytes" "testing" - "github.com/prebid/prebid-server/config" "github.com/spf13/viper" ) @@ -27,7 +26,7 @@ func init() { func TestDefaults(t *testing.T) { - cfg, err := config.New() + cfg, err := New() if err != nil { t.Error(err.Error()) } @@ -115,7 +114,7 @@ func cmpInts(t *testing.T, key string, a int, b int) { func TestFullConfig(t *testing.T) { viper.SetConfigType("yaml") viper.ReadConfig(bytes.NewBuffer(fullConfig)) - cfg, err := config.New() + cfg, err := New() if err != nil { t.Fatal(err.Error()) } @@ -158,3 +157,58 @@ func TestFullConfig(t *testing.T) { cmpStrings(t, "adapters.facebook.usersync_url", cfg.Adapters["facebook"].UserSyncURL, "http://facebook.com/ortb/prebid-s2s") cmpStrings(t, "adapters.facebook.platform_id", cfg.Adapters["facebook"].PlatformID, "abcdefgh1234") } + +func TestValidConfig(t *testing.T) { + cfg := Configuration{ + StoredRequests: StoredRequests{ + Files: true, + }, + } + + if err := cfg.validate(); err != nil { + t.Errorf("OpenRTB filesystem config should work. %v", err) + } +} + +func TestInvalidStoredRequestsConfig(t *testing.T) { + cfg := Configuration{ + StoredRequests: StoredRequests{ + Files: true, + Postgres: &PostgresConfig{}, + }, + } + + if err := cfg.validate(); err == nil { + t.Error("OpenRTB Configs should not be allowed from both files and postgres.") + } +} + +func TestQueryMaker(t *testing.T) { + cfg := PostgresConfig{ + QueryTemplate: "SELECT id, config FROM table WHERE id in %ID_LIST%", + } + madeQuery, err := cfg.MakeQuery(3) + if err != nil { + t.Errorf("Unexpected error making query: %v", err) + } + if madeQuery != "SELECT id, config FROM table WHERE id in ($1, $2, $3)" { + t.Errorf(`Final query was not as expeted. Got "%s"`, madeQuery) + } + + madeQuery, err = cfg.MakeQuery(11) + if err != nil { + t.Errorf("Unexpected error making query: %v", err) + } + if madeQuery != "SELECT id, config FROM table WHERE id in ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)" { + t.Errorf(`Final query was not as expeted. Got "%s"`, madeQuery) + } +} + +func TestQueryMakerInvalid(t *testing.T) { + cfg := PostgresConfig{ + QueryTemplate: "SELECT id, config FROM table WHERE id in %ID_LIST%", + } + if _, err := cfg.MakeQuery(0); err == nil { + t.Errorf("MakeQuery function should return an error if given no IDs.") + } +} diff --git a/docs/developers/configuration.md b/docs/developers/configuration.md new file mode 100644 index 00000000000..5dc9af6fc3c --- /dev/null +++ b/docs/developers/configuration.md @@ -0,0 +1,11 @@ +# Configuration + +Configuration is handled by [Viper](https://github.com/spf13/viper), which supports [many ways](https://github.com/spf13/viper#why-viper) of setting config values. + +As a general rule, Prebid Server will log its resolved config values on startup and exit immediately if they're not valid. + +For development, it's easiest to define your config inside a `pbs.yaml` file in the project root. + +## Available options + +For now, see [the contract classes](../../config/config.go) in the code. diff --git a/docs/developers/stored-requests.md b/docs/developers/stored-requests.md new file mode 100644 index 00000000000..de7a6a6101f --- /dev/null +++ b/docs/developers/stored-requests.md @@ -0,0 +1,138 @@ +# Stored Requests + +This document gives a technical overview of the Stored Requests feature. + +Docs outlining the motivation and uses will be added sometime in the future. + +## Quickstart + +Configure your server to read stored requests from the filesystem: + +```yaml +stored_requests: + filesystem: true +``` + +Choose an ID to reference your stored request data. Throughout this doc, replace {id} with the ID you've chosen. + +Add the file `stored_requests/data/by_id/{id}.json` and populate it with some [Imp](https://www.iab.com/wp-content/uploads/2016/03/OpenRTB-API-Specification-Version-2-5-FINAL.pdf#page=17) data. + +```json +{ + "id": "test-imp-id", + "banner": { + "format": [ + { + "w": 300, + "h": 250 + }, + { + "w": 300, + "h": 600 + } + ] + }, + "ext": { + "appnexus": { + "placementId": 10433394 + } + } +} +``` + +Start your server. + +```bash +go build . +./prebid-server +``` + +And then `POST` to [`/openrtb2/auction`](../endpoints/openrtb2/auction.md) with your chosen ID. + +```json +{ + "id": "test-request-id", + "imp": [ + { + "ext": { + "prebid": { + "storedrequest": { + "id": "{id}" + } + } + } + } + ] +} +``` + +The auction will occur as if the HTTP request had included the content from `stored_requests/data/by_id/{id}.json` instead. + +## Partially Stored Requests + +You can also store _part_ of the Imp on the server. For example: + +```json +{ + "banner": { + "format": [ + { + "w": 300, + "h": 250 + }, + { + "w": 300, + "h": 600 + } + ] + }, + "ext": { + "appnexus": { + "placementId": 10433394 + } + } +} +``` + +This is not _fully_ legal OpenRTB `imp` data, since it lacks an `id`. + +However, incoming HTTP requests can fill in the missing data to complete the OpenRTB request: + +```json +{ + "id": "test-request-id", + "imp": [ + { + "id": "test-imp-id", + "ext": { + "prebid": { + "storedrequest": { + "id": "{id}" + } + } + } + } + ] + } +``` + +If the Stored Request and the HTTP request have conflicting properties, +they will be resolved with a [JSON Merge Patch](https://tools.ietf.org/html/rfc7386). +HTTP request properties will overwrite the Stored Request ones. + +## Alternate backends + +Stored Requests do not need to be saved to files. [Other backends](../../openrtb2_config/) are supported +with different [configuration options](configuration.yaml). For example: + +```yaml +stored_requests: + postgres: + host: localhost + port: 5432 + user: db-username + dbname: database-name + query: SELECT id, requestData FROM stored_requests WHERE id IN %ID_LIST%; +``` + +If you need support for a backend that you don't see, please [contribute it](contributing.md). diff --git a/docs/endpoints/openrtb2/auction.md b/docs/endpoints/openrtb2/auction.md index 52dfc377cc3..cfe1978eb0b 100644 --- a/docs/endpoints/openrtb2/auction.md +++ b/docs/endpoints/openrtb2/auction.md @@ -191,6 +191,20 @@ PBS requests new syncs by returning the `response.ext.usersync.{bidderName}.sync This contains info about every request and response sent by the bidder to its server. It is only returned on `test` bids for performance reasons, but may be useful during debugging. +#### Stored Requests + +`request.imp[i].ext.prebid.storedrequest` incorporates a [Stored Request](../../developers/stored-requests.md) from the server. + +A typical `storedrequest` value looks like this: + +``` +{ + "id": "some-id" +} +``` + +For more information, see the docs for [Stored Requests](../../developers/stored-requests.md). + ### OpenRTB Differences This section describes the ways in which Prebid Server **breaks** the OpenRTB spec. diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index 2346c0a45c2..f8d633075e2 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -11,34 +11,40 @@ import ( "errors" "github.com/prebid/prebid-server/openrtb_ext" "time" + "github.com/evanphx/json-patch" + "github.com/prebid/prebid-server/stored_requests" + "github.com/prebid/prebid-server/config" + "io" + "io/ioutil" + "github.com/buger/jsonparser" "github.com/golang/glog" ) -func NewEndpoint(ex exchange.Exchange, validator openrtb_ext.BidderParamValidator) (httprouter.Handle, error) { - if ex == nil || validator == nil { +func NewEndpoint(ex exchange.Exchange, validator openrtb_ext.BidderParamValidator, requestsById stored_requests.Fetcher, cfg *config.Configuration) (httprouter.Handle, error) { + if ex == nil || validator == nil || requestsById == nil || cfg == nil { return nil, errors.New("NewEndpoint requires non-nil arguments.") } - return httprouter.Handle((&endpointDeps{ex, validator}).Auction), nil + + return httprouter.Handle((&endpointDeps{ex, validator, requestsById, cfg}).Auction), nil } type endpointDeps struct { - ex exchange.Exchange - paramsValidator openrtb_ext.BidderParamValidator + ex exchange.Exchange + paramsValidator openrtb_ext.BidderParamValidator + storedReqFetcher stored_requests.Fetcher + cfg *config.Configuration } func (deps *endpointDeps) Auction(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - req, err := deps.parseRequest(r) - if err != nil { + req, ctx, cancel, errL := deps.parseRequest(r) + defer cancel() // Safe because parseRequest returns a no-op even if errors are present. + if len(errL) > 0 { w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(fmt.Sprintf("Invalid request format: %s", err.Error()))) + for _, err := range errL { + w.Write([]byte(fmt.Sprintf("Invalid request format: %s\n", err.Error()))) + } return } - ctx := context.Background() - cancel := func() { } - if req.TMax > 0 { - ctx, cancel = context.WithTimeout(ctx, time.Duration(req.TMax) * time.Millisecond) - defer cancel() - } response, err := deps.ex.HoldAuction(ctx, req) if err != nil { @@ -59,24 +65,57 @@ func (deps *endpointDeps) Auction(w http.ResponseWriter, r *http.Request, _ http } } -// parseRequest turns the HTTP request into an OpenRTB request. +// parseRequest turns the HTTP request into an OpenRTB request. This is guaranteed to return: // -// This will return an error if the request couldn't be parsed, or if the request isn't valid according -// to the OpenRTB 2.5 spec. +// - A context which times out appropriately, given the request. +// - A cancellation function which should be called if the auction finishes early. // -// It will also return errors for some of the "strong recommendations" in the spec, as long as -// the same request can be sent in a better way which agrees with the recommendations. -func (deps *endpointDeps) parseRequest(httpRequest *http.Request) (*openrtb.BidRequest, error) { - var ortbRequest openrtb.BidRequest - if err := json.NewDecoder(httpRequest.Body).Decode(&ortbRequest); err != nil { - return nil, err +// If the errors list is empty, then the returned request will be valid according to the OpenRTB 2.5 spec. +// In case of "strong recommendations" in the spec, it tends to be restrictive. If a better workaround is +// possible, it will return errors with messages that suggest improvements. +// +// If the errors list has at least one element, then no guarantees are made about the returned request. +func (deps *endpointDeps) parseRequest(httpRequest *http.Request) (req *openrtb.BidRequest, ctx context.Context, cancel func(), errs []error) { + req = &openrtb.BidRequest{} + ctx = context.Background() + cancel = func() { } + errs = nil + + // Pull the request body into a buffer, so we have it for later usage. + lr := &io.LimitedReader{ httpRequest.Body, deps.cfg.MaxRequestSize } + rawRequest, err := ioutil.ReadAll(lr) + if err != nil { + errs = []error{err} + return + } + // If the request size was too large, read through the rest of the request body so that the connection can be reused. + if lr.N <= 0 { + if written, err := io.Copy(ioutil.Discard, httpRequest.Body); written > 0 || err != nil { + errs = []error{fmt.Errorf("Request size exceeded max size of %d bytes.", deps.cfg.MaxRequestSize)} + return + } } - if err := deps.validateRequest(&ortbRequest); err != nil { - return nil, err + if err := json.Unmarshal(rawRequest, req); err != nil { + errs = []error{err} + return } - return &ortbRequest, nil + if req.TMax > 0 { + ctx, cancel = context.WithTimeout(ctx, time.Duration(req.TMax) * time.Millisecond) + } + + // Process any stored request directives in the impression objects. + if errL := deps.processStoredRequests(ctx, req, rawRequest); len(errL)>0 { + errs = errL + return + } + + if err := deps.validateRequest(req); err != nil { + errs = []error{err} + return + } + return } func (deps *endpointDeps) validateRequest(req *openrtb.BidRequest) error { @@ -221,10 +260,94 @@ func (deps *endpointDeps) validateImpExt(ext openrtb.RawJSON, impIndex int) erro if err := deps.paramsValidator.Validate(bidderName, ext); err != nil { return fmt.Errorf("request.imp[%d].ext.%s failed validation.\n%v", impIndex, bidder, err) } - } else { + } else if bidder != "prebid" { return fmt.Errorf("request.imp[%d].ext contains unknown bidder: %s", impIndex, bidder) } } return nil } + +// processStoredRequests merges any data referenced by request.imp[i].ext.prebid.storedrequest.id into the request, if necessary. +func (deps *endpointDeps) processStoredRequests(ctx context.Context, request *openrtb.BidRequest, rawRequest []byte) []error { + // Pull all the Stored Request IDs from the Imps. + storedReqIds, shortIds, errList := deps.findStoredRequestIds(request.Imp) + if len(shortIds) == 0 { + return nil + } + + storedReqs, errL := deps.storedReqFetcher.FetchRequests(ctx, shortIds) + if len(errL) > 0 { + return append(errList, errL...) + } + + // Get the raw JSON for Imps, so we don't have to worry about the effects of an UnMarshal/Marshal round. + rawImpsRaw, dt, _, err := jsonparser.Get(rawRequest, "imp") + if err != nil { + return append(errList, err) + } + if dt != jsonparser.Array { + return append(errList, fmt.Errorf("ERROR: could not parse Imp[] as an array, got %s", string(dt))) + } + rawImps := getArrayElements(rawImpsRaw) + // Process Imp level configs. + for i := 0; i < len(request.Imp); i++ { + // Check if a config was requested + if len(storedReqIds[i]) > 0 { + conf, ok := storedReqs[storedReqIds[i]] + if ok && len(conf) > 0 { + err := deps.mergeStoredData(&request.Imp[i], rawImps[i], conf) + if err != nil { + errList = append(errList, err) + } + } + } + } + return errList +} + +// Pull the Stored Request IDs from the Imps. Return both ID indexed by Imp array index, and a simple list of existing IDs. +func (deps *endpointDeps) findStoredRequestIds(imps []openrtb.Imp) ([]string, []string, []error) { + errList := make([]error, 0, len(imps)) + storedReqIds := make([]string, len(imps)) + shortIds := make([]string, 0, len(imps)) + for i := 0; i < len(imps); i++ { + if imps[i].Ext != nil && len(imps[i].Ext) > 0 { + // These keys should be kept in sync with openrtb_ext.ExtStoredRequest. + // The jsonparser is much faster than doing a full unmarshal to select a single value + storedReqId, _, _, err := jsonparser.Get(imps[i].Ext, "prebid", "storedrequest", "id") + storedReqString := string(storedReqId) + if err == nil && len(storedReqString) > 0 { + storedReqIds[i] = storedReqString + shortIds = append(shortIds, storedReqString) + } else if len(storedReqString) > 0 { + errList = append(errList, err) + storedReqIds[i] = "" + } + } else{ + storedReqIds[i] = "" + } + } + return storedReqIds, shortIds, errList +} + + +// Process the stored request data for an Imp. +// Need to modify the Imp object in place as we cannot simply assign one Imp to another (deep copy) +func (deps *endpointDeps) mergeStoredData(imp *openrtb.Imp, impJson []byte, storedReqData json.RawMessage) error { + newImp, err := jsonpatch.MergePatch(storedReqData, impJson) + if err != nil { + return err + } + err = json.Unmarshal(newImp, imp) + return err +} + +// Copied from jsonparser +func getArrayElements(data []byte) (result [][]byte) { + jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { + result = append(result, value) + }) + + return +} diff --git a/endpoints/openrtb2/auction_benchmark_test.go b/endpoints/openrtb2/auction_benchmark_test.go index a3fd296f8c9..79e9d21e914 100644 --- a/endpoints/openrtb2/auction_benchmark_test.go +++ b/endpoints/openrtb2/auction_benchmark_test.go @@ -6,6 +6,8 @@ import ( "net/http/httptest" "net/http" "github.com/prebid/prebid-server/exchange" + "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" + "github.com/prebid/prebid-server/config" ) // dummyServer returns the header bidding test ad. This response was scraped from a real appnexus server response. @@ -49,7 +51,7 @@ func newDummyRequest() *http.Request { func BenchmarkOpenrtbEndpoint(b *testing.B) { server := httptest.NewServer(http.HandlerFunc(dummyServer)) defer server.Close() - endpoint, _ := NewEndpoint(exchange.NewExchange(server.Client()), &bidderParamValidator{}) + endpoint, _ := NewEndpoint(exchange.NewExchange(server.Client()), &bidderParamValidator{}, empty_fetcher.EmptyFetcher(), &config.Configuration{ MaxRequestSize: maxSize }) b.ResetTimer() for n := 0; n < b.N; n++ { diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index a2dae855d38..8a80a2631d9 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -11,11 +11,17 @@ import ( "github.com/prebid/prebid-server/openrtb_ext" "bytes" "errors" + "github.com/evanphx/json-patch" + "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" + "github.com/prebid/prebid-server/config" + "io" ) +const maxSize = 1024 * 256 + // TestGoodRequests makes sure that the auction runs properly-formatted bids correctly. func TestGoodRequests(t *testing.T) { - endpoint, _ := NewEndpoint(&nobidExchange{}, &bidderParamValidator{}) + endpoint, _ := NewEndpoint(&nobidExchange{}, &bidderParamValidator{}, empty_fetcher.EmptyFetcher(), &config.Configuration{ MaxRequestSize: maxSize }) for _, requestData := range validRequests { request := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(requestData)) @@ -24,6 +30,7 @@ func TestGoodRequests(t *testing.T) { if recorder.Code != http.StatusOK { t.Errorf("Expected status %d. Got %d. Request data was %s", http.StatusOK, recorder.Code, requestData) + //t.Errorf("Response body was: %s", recorder.Body) } var response openrtb.BidResponse @@ -45,7 +52,7 @@ func TestGoodRequests(t *testing.T) { // TestBadRequests makes sure we return 400's on bad requests. func TestBadRequests(t *testing.T) { - endpoint, _ := NewEndpoint(&nobidExchange{}, &bidderParamValidator{}) + endpoint, _ := NewEndpoint(&nobidExchange{}, &bidderParamValidator{}, empty_fetcher.EmptyFetcher(), &config.Configuration{ MaxRequestSize: maxSize }) for _, badRequest := range invalidRequests { request := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(badRequest)) recorder := httptest.NewRecorder() @@ -60,7 +67,7 @@ func TestBadRequests(t *testing.T) { // TestNilExchange makes sure we fail when given nil for the Exchange. func TestNilExchange(t *testing.T) { - _, err := NewEndpoint(nil, &bidderParamValidator{}) + _, err := NewEndpoint(nil, &bidderParamValidator{}, empty_fetcher.EmptyFetcher(), &config.Configuration{ MaxRequestSize: maxSize }) if err == nil { t.Errorf("NewEndpoint should return an error when given a nil Exchange.") } @@ -68,7 +75,7 @@ func TestNilExchange(t *testing.T) { // TestNilValidator makes sure we fail when given nil for the BidderParamValidator. func TestNilValidator(t *testing.T) { - _, err := NewEndpoint(&nobidExchange{}, nil) + _, err := NewEndpoint(&nobidExchange{}, nil, empty_fetcher.EmptyFetcher(), &config.Configuration{ MaxRequestSize: maxSize }) if err == nil { t.Errorf("NewEndpoint should return an error when given a nil BidderParamValidator.") } @@ -76,7 +83,7 @@ func TestNilValidator(t *testing.T) { // TestExchangeError makes sure we return a 500 if the exchange auction fails. func TestExchangeError(t *testing.T) { - endpoint, _ := NewEndpoint(&brokenExchange{}, &bidderParamValidator{}) + endpoint, _ := NewEndpoint(&brokenExchange{}, &bidderParamValidator{}, empty_fetcher.EmptyFetcher(), &config.Configuration{ MaxRequestSize: maxSize }) request := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(validRequests[0])) recorder := httptest.NewRecorder() endpoint(recorder, request, nil) @@ -86,10 +93,94 @@ func TestExchangeError(t *testing.T) { } } +// Test the stored request functionality +func TestStoredRequests(t *testing.T) { + edep := &endpointDeps{&nobidExchange{}, &bidderParamValidator{}, &mockStoredReqFetcher{}, &config.Configuration{ MaxRequestSize: maxSize }} + + for i, requestData := range testStoredRequests { + Request := openrtb.BidRequest{} + err := json.Unmarshal(json.RawMessage(requestData), &Request) + if err != nil { + t.Errorf("Error unmashalling bid request: %s", err.Error()) + } + + errList := edep.processStoredRequests(context.Background(), &Request, json.RawMessage(requestData)) + if len(errList) != 0 { + for _, err := range errList { + if err != nil { + t.Errorf("processStoredRequests Error: %s", err.Error()) + } else { + t.Error("processStoredRequests Error: recieved nil error") + } + } + } + expectJson := json.RawMessage(testFinalRequests[i]) + requestJson, err := json.Marshal(Request) + if err != nil { + t.Errorf("Error mashalling bid request: %s", err.Error()) + } + if ! jsonpatch.Equal(requestJson, expectJson) { + t.Errorf("Error in processStoredRequests, test %d failed on compare\nFound:\n%s\nExpected:\n%s", i, string(requestJson), string(expectJson)) + } + + } +} + +// TestOversizedRequest makes sure we behave properly when the request size exceeds the configured max. +func TestOversizedRequest(t *testing.T) { + reqBody := `{"id":"request-id"}` + deps := &endpointDeps{ + &nobidExchange{}, + &bidderParamValidator{}, + &mockStoredReqFetcher{}, + &config.Configuration{ MaxRequestSize: int64(len(reqBody) - 1) }, + } + + req := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(reqBody)) + recorder := httptest.NewRecorder() + + deps.Auction(recorder, req, nil) + + if recorder.Code != http.StatusBadRequest { + t.Errorf("Endpoint should return a 400 if the request exceeds the size max.") + } + + if bytesRead, err := req.Body.Read(make([]byte, 1)); bytesRead != 0 || err != io.EOF { + t.Errorf("The request body should still be fully read.") + } +} + +// TestRequestSizeEdgeCase makes sure we behave properly when the request size *equals* the configured max. +func TestRequestSizeEdgeCase(t *testing.T) { + reqBody := validRequests[0] + deps := &endpointDeps{ + &nobidExchange{}, + &bidderParamValidator{}, + &mockStoredReqFetcher{}, + &config.Configuration{MaxRequestSize: int64(len(reqBody))}, + } + + req := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(reqBody)) + recorder := httptest.NewRecorder() + + deps.Auction(recorder, req, nil) + + if recorder.Code != http.StatusOK { + t.Errorf("Endpoint should return a 200 if the request equals the size max.") + } + + if bytesRead, err := req.Body.Read(make([]byte, 1)); bytesRead != 0 || err != io.EOF { + t.Errorf("The request body should have been read to completion.") + } +} // TestNoEncoding prevents #231. func TestNoEncoding(t *testing.T) { - endpoint, _ := NewEndpoint(&mockExchange{}, &bidderParamValidator{}) + endpoint, _ := NewEndpoint( + &mockExchange{}, + &bidderParamValidator{}, + &mockStoredReqFetcher{}, + &config.Configuration{ MaxRequestSize: maxSize }) request := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(validRequests[0])) recorder := httptest.NewRecorder() endpoint(recorder, request, nil) @@ -297,6 +388,159 @@ var invalidRequests = []string{ }]}`, } +// StoredRequest testing + +// Test stored request data +var testStoredRequestData = map[string]json.RawMessage{ + "1": json.RawMessage(`{ + "id": "adUnit1", + "ext": { + "appnexus": { + "placementId": "abc", + "position": "above", + "reserve": 0.35 + }, + "rubicon": { + "accountId": "abc" + } + } + }`), + "": json.RawMessage(""), +} + +// Incoming requests with stored request IDs +var testStoredRequests = []string{ + `{ + "id": "ThisID", + "imp": [ + { + "ext": { + "prebid": { + "storedrequest": { + "id": "1" + } + } + } + } + ], + "ext": { + "prebid": { + "cache": { + "markup": 1 + }, + "targeting": { + "lengthmax": 20 + } + } + } + }`, + `{ + "id": "ThisID", + "imp": [ + { + "id": "adUnit2", + "ext": { + "prebid": { + "storedrequest": { + "id": "1" + } + }, + "appnexus": { + "placementId": "def", + "trafficSourceCode": "mysite.com", + "reserve": null + }, + "rubicon": null + } + } + ], + "ext": { + "prebid": { + "cache": { + "markup": 1 + }, + "targeting": { + "lengthmax": 20 + } + } + } + }`, +} + +// The expected requests after stored request processing +var testFinalRequests = []string { + `{ + "id": "ThisID", + "imp": [ + { + "id": "adUnit1", + "ext": { + "appnexus": { + "placementId": "abc", + "position": "above", + "reserve": 0.35 + }, + "rubicon": { + "accountId": "abc" + }, + "prebid": { + "storedrequest": { + "id": "1" + } + } + } + } + ], + "ext": { + "prebid": { + "cache": { + "markup": 1 + }, + "targeting": { + "lengthmax": 20 + } + } + } + }`, + `{ + "id": "ThisID", + "imp": [ + { + "id": "adUnit2", + "ext": { + "prebid": { + "storedrequest": { + "id": "1" + } + }, + "appnexus": { + "placementId": "def", + "position": "above", + "trafficSourceCode": "mysite.com" + } + } + } + ], + "ext": { + "prebid": { + "cache": { + "markup": 1 + }, + "targeting": { + "lengthmax": 20 + } + } + } + }`, +} + +type mockStoredReqFetcher struct { +} + +func (cf mockStoredReqFetcher) FetchRequests(ctx context.Context, ids []string) (map[string]json.RawMessage, []error) { + return testStoredRequestData, nil +} + type mockExchange struct {} func (*mockExchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest) (*openrtb.BidResponse, error) { diff --git a/glide.lock b/glide.lock index 6d836d70e4c..9600cd25ef4 100644 --- a/glide.lock +++ b/glide.lock @@ -1,12 +1,18 @@ -hash: a2b11928aedf45e80e896b5a445c779619583ba44cdc0c5ffce0b167ea2e0e43 -updated: 2017-11-07T14:18:38.797738917-05:00 +hash: e580034b35dc3f0c4d82c9f390a643ea2a2c9cb879ea277a473685cf55985050 +updated: 2017-11-22T10:03:12.838157881-05:00 imports: - name: github.com/blang/semver version: 2ee87856327ba09384cabd113bc6b5d174e9ec0f +- name: github.com/buger/jsonparser + version: 5096fddd2cca678b0801ece8513a06e580981fd2 - name: github.com/cloudfoundry/gosigar version: 14c0a766933d517675369dae6f551f5c1a10f0d8 - name: github.com/coocood/freecache version: a47e26eb67ac2657e4b5a62b1975bb2b65e0b8b3 +- name: github.com/DATA-DOG/go-sqlmock + version: d76b18b42f285b792bf985118980ce9eacea9d10 +- name: github.com/evanphx/json-patch + version: 944e07253867aacae43c04b2e6a239005443f33a - name: github.com/fsnotify/fsnotify version: 4da3e2cfbabc9f751898f250b49f2439785783a1 - name: github.com/golang/glog diff --git a/glide.yaml b/glide.yaml index d466d2c2acf..932b1d0751a 100644 --- a/glide.yaml +++ b/glide.yaml @@ -26,6 +26,10 @@ import: - package: github.com/spaolacci/murmur3 - package: github.com/cloudfoundry/gosigar - package: github.com/mssola/user_agent +- package: github.com/evanphx/json-patch +- package: github.com/DATA-DOG/go-sqlmock + version: ^1.3.0 +- package: github.com/buger/jsonparser testImport: - package: github.com/erikstmartin/go-testdb - package: github.com/stretchr/testify diff --git a/openrtb_ext/imp.go b/openrtb_ext/imp.go index 4486645361b..946fa8c1626 100644 --- a/openrtb_ext/imp.go +++ b/openrtb_ext/imp.go @@ -8,10 +8,10 @@ type ExtImp struct { // ExtImpPrebid defines the contract for bidrequest.imp[i].ext.prebid type ExtImpPrebid struct { - Config *ExtConfig `json:"managedconfig"` + StoredRequest *ExtStoredRequest `json:"storedrequest"` } -// ExtConfig defines the contract for bidrequest.imp[i].ext.prebid.managedconfig -type ExtConfig struct { +// ExtStoredRequest defines the contract for bidrequest.imp[i].ext.prebid.storedrequest +type ExtStoredRequest struct { ID string `json:"id"` } diff --git a/pbs_light.go b/pbs_light.go index f37c3602fdb..48e2de19931 100644 --- a/pbs_light.go +++ b/pbs_light.go @@ -52,6 +52,10 @@ import ( "github.com/prebid/prebid-server/prebid" pbc "github.com/prebid/prebid-server/prebid_cache_client" "github.com/prebid/prebid-server/ssl" + "github.com/prebid/prebid-server/stored_requests" + "github.com/prebid/prebid-server/stored_requests/backends/db_fetcher" + "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" + "github.com/prebid/prebid-server/stored_requests/backends/file_fetcher" "strings" ) @@ -706,11 +710,13 @@ func init() { viper.SetDefault("datacache.type", "dummy") // no metrics configured by default (metrics{host|database|username|password}) + viper.SetDefault("stored_requests.filesystem", "true") viper.SetDefault("adapters.pubmatic.endpoint", "http://openbid.pubmatic.com/translator?source=prebid-server") viper.SetDefault("adapters.rubicon.endpoint", "http://staged-by.rubiconproject.com/a/api/exchange.json") viper.SetDefault("adapters.rubicon.usersync_url", "https://pixel.rubiconproject.com/exchange/sync.php?p=prebid") viper.SetDefault("adapters.pulsepoint.endpoint", "http://bid.contextweb.com/header/s/ortb/prebid-s2s") viper.SetDefault("adapters.index.usersync_url", "//ssum-sec.casalemedia.com/usermatchredir?s=184932&cb=https%3A%2F%2Fprebid.adnxs.com%2Fpbs%2Fv1%2Fsetuid%3Fbidder%3DindexExchange%26uid%3D") + viper.SetDefault("max_request_size", 1024*256) viper.SetDefault("adapters.conversant.endpoint", "http://media.msg.dotomi.com/s2s/header/24") viper.SetDefault("adapters.conversant.usersync_url", "http://prebid-match.dotomi.com/prebid/match?rurl=") viper.ReadInConfig() @@ -721,7 +727,7 @@ func init() { func main() { cfg, err := config.New() if err != nil { - glog.Errorf("Viper was unable to read configurations: %v", err) + glog.Fatalf("Viper was unable to read configurations: %v", err) } if err := serve(cfg); err != nil { @@ -836,7 +842,12 @@ func serve(cfg *config.Configuration) error { }, }) - openrtbEndpoint, err := openrtb2.NewEndpoint(theExchange, paramsValidator) + byId, err := NewFetcher(&(cfg.StoredRequests)) + if err != nil { + glog.Fatalf("Failed to initialize config backends. %v", err) + } + + openrtbEndpoint, err := openrtb2.NewEndpoint(theExchange, paramsValidator, byId, cfg) if err != nil { glog.Fatalf("Failed to create the openrtb endpoint handler. %v", err) } @@ -909,3 +920,23 @@ func serve(cfg *config.Configuration) error { return nil } + +const requestConfigPath = "./stored_requests/data/by_id" + +// NewFetchers returns an Account-based config fetcher and a Request-based config fetcher, in that order. +// If it can't generate both of those from the given config, then an error will be returned. +// +// This function assumes that the argument config has been validated. +func NewFetcher(cfg *config.StoredRequests) (byId stored_requests.Fetcher, err error) { + if cfg.Files { + glog.Infof("Loading Stored Requests from filesystem at path %s", requestConfigPath) + byId, err = file_fetcher.NewFileFetcher(requestConfigPath) + } else if cfg.Postgres != nil { + glog.Infof("Loading Stored Requests from Postgres with config: %#v", cfg.Postgres) + byId, err = db_fetcher.NewPostgres(cfg.Postgres) + } else { + glog.Warning("No Stored Request support configured. request.imp[i].ext.prebid.storedrequest will be ignored. If you need this, check your app config") + byId = empty_fetcher.EmptyFetcher() + } + return +} diff --git a/pbs_light_test.go b/pbs_light_test.go index 7757b479209..4316b7236f8 100644 --- a/pbs_light_test.go +++ b/pbs_light_test.go @@ -9,6 +9,7 @@ import ( "github.com/mxmCherry/openrtb" + "context" "github.com/julienschmidt/httprouter" "github.com/prebid/prebid-server/cache/dummycache" "github.com/prebid/prebid-server/config" @@ -486,6 +487,34 @@ func ensureHasKey(t *testing.T, data map[string]json.RawMessage, key string) { } } +func TestNewFilesFetcher(t *testing.T) { + fetcher, err := NewFetcher(&config.StoredRequests{ + Files: true, + }) + if err != nil { + t.Errorf("Error constructing file backends. %v", err) + } + if fetcher == nil { + t.Errorf("The file-backed fetcher should be non-nil.") + } +} + +func TestNewEmptyFetcher(t *testing.T) { + fetcher, err := NewFetcher(&config.StoredRequests{}) + if err != nil { + t.Errorf("Error constructing backends. %v", err) + } + if fetcher == nil { + t.Errorf("The fetcher should be non-nil, even with an empty config.") + } + if _, errs := fetcher.FetchRequests(context.Background(), []string{"some-id"}); len(errs) != 1 { + t.Errorf("The returned accountFetcher should fail on any ID.") + } + if _, errs := fetcher.FetchRequests(context.Background(), []string{"some-id"}); len(errs) != 1 { + t.Errorf("The returned requestFetcher should fail on any ID.") + } +} + type testValidator struct{} func (validator *testValidator) Validate(name openrtb_ext.BidderName, ext openrtb.RawJSON) error { diff --git a/stored_requests/backends/db_fetcher/fetcher.go b/stored_requests/backends/db_fetcher/fetcher.go new file mode 100644 index 00000000000..ff16c9cf017 --- /dev/null +++ b/stored_requests/backends/db_fetcher/fetcher.go @@ -0,0 +1,64 @@ +package db_fetcher + +import ( + "database/sql" + "encoding/json" + "fmt" + "context" + "github.com/golang/glog" +) + +// dbFetcher fetches Stored Requests from a database. This should be instantiated through the NewPostgres() function. +type dbFetcher struct { + db *sql.DB + queryMaker func(int) (string, error) +} + +func (fetcher *dbFetcher) FetchRequests(ctx context.Context, ids []string) (map[string]json.RawMessage, []error) { + if len(ids) < 1 { + return nil, nil + } + + query, err := fetcher.queryMaker(len(ids)) + if err != nil { + return nil, []error{err} + } + + idInterfaces := make([]interface{}, len(ids)) + for i := 0; i < len(ids); i++ { + idInterfaces[i] = ids[i] + } + + rows, err := fetcher.db.QueryContext(ctx, query, idInterfaces...) + if err != nil { + ctxErr := ctx.Err() + // This query might fail if the user chose an extremely short timeout. + // We don't care about these... but there may also be legit connection issues. + // Log any other errors so we have some idea what's going on. + if ctxErr == nil || ctxErr != context.DeadlineExceeded { + glog.Errorf("Error reading from Stored Request DB: %s", err.Error()) + } + return nil, []error{err} + } + defer rows.Close() + + reqData := make(map[string]json.RawMessage, len(ids)) + var errs []error = nil + for rows.Next() { + var id string + var thisReqData []byte + if err := rows.Scan(&id, &thisReqData); err != nil { + errs = append(errs, err) + } + + reqData[id] = thisReqData + } + + for _, id := range ids { + if _, ok := reqData[id]; !ok { + errs = append(errs, fmt.Errorf(`Stored Request with ID="%s" not found.`, id)) + } + } + + return reqData, errs +} diff --git a/stored_requests/backends/db_fetcher/fetcher_test.go b/stored_requests/backends/db_fetcher/fetcher_test.go new file mode 100644 index 00000000000..a82c7daa0fc --- /dev/null +++ b/stored_requests/backends/db_fetcher/fetcher_test.go @@ -0,0 +1,223 @@ +package db_fetcher + +import ( + "testing" + "github.com/DATA-DOG/go-sqlmock" + "regexp" + "fmt" + "encoding/json" + "database/sql/driver" + "errors" + "context" + "time" +) + +func TestEmptyQuery(t *testing.T) { + db, _, err := sqlmock.New() + if err != nil { + t.Fatalf("Unexpected error stubbing DB: %v", err) + } + defer db.Close() + + fetcher := dbFetcher{ + db: db, + queryMaker: successfulQueryMaker(""), + } + storedReqs, errs := fetcher.FetchRequests(context.Background(), nil) + if len(errs) != 0 { + t.Errorf("Unexpected errors: %v", errs) + } + if len(storedReqs) != 0 { + t.Errorf("Bad map size. Expected %d, got %d.", 0, len(storedReqs)) + } +} + +// TestGoodResponse makes sure we interpret DB responses properly when all the stored requests are there. +func TestGoodResponse(t *testing.T) { + mockQuery := "SELECT id, requestData FROM my_table WHERE id IN (?, ?)" + mockReturn := sqlmock.NewRows([]string{"id", "requestData"}). + AddRow("request-id", "{}") + + mock, fetcher, err := newFetcher(mockReturn, mockQuery, "request-id") + if err != nil { + t.Fatalf("Failed to create mock: %v", err) + } + defer fetcher.db.Close() + + storedReqs, errs := fetcher.FetchRequests(context.Background(), []string{"request-id"}) + + assertMockExpectations(t, mock) + assertErrorCount(t, 0, errs) + assertMapLength(t, 1, storedReqs) + assertHasData(t, storedReqs, "request-id", "{}") +} + +// TestPartialResponse makes sure we unpack things properly when the DB finds some of the stored requests. +func TestPartialResponse(t *testing.T) { + mockQuery := "SELECT id, requestData FROM my_table WHERE id IN (?, ?)" + mockReturn := sqlmock.NewRows([]string{"id", "requestData"}). + AddRow("stored-req-id", "{}") + + mock, fetcher, err := newFetcher(mockReturn, mockQuery, "stored-req-id", "stored-req-id-2") + if err != nil { + t.Fatalf("Failed to create mock: %v", err) + } + defer fetcher.db.Close() + + storedReqs, errs := fetcher.FetchRequests(context.Background(), []string{"stored-req-id", "stored-req-id-2"}) + + assertMockExpectations(t, mock) + assertErrorCount(t, 1, errs) + assertMapLength(t, 1, storedReqs) + assertHasData(t, storedReqs, "stored-req-id", "{}") +} + +// TestEmptyResponse makes sure we handle empty DB responses properly. +func TestEmptyResponse(t *testing.T) { + mockQuery := "SELECT id, requestData FROM my_table WHERE id IN (?, ?)" + mockReturn := sqlmock.NewRows([]string{"id", "requestData"}) + + mock, fetcher, err := newFetcher(mockReturn, mockQuery, "stored-req-id", "stored-req-id-2") + if err != nil { + t.Fatalf("Failed to create mock: %v", err) + } + defer fetcher.db.Close() + + storedReqs, errs := fetcher.FetchRequests(context.Background(), []string{"stored-req-id", "stored-req-id-2"}) + + assertMockExpectations(t, mock) + assertErrorCount(t, 2, errs) + assertMapLength(t, 0, storedReqs) +} + +// TestQueryMakerError makes sure we exit with an error if the queryMaker function fails. +func TestQueryMakerError(t *testing.T) { + fetcher := &dbFetcher{ + db: nil, + queryMaker: failedQueryMaker, + } + + storedReqs, errs := fetcher.FetchRequests(context.Background(), []string{"stored-req-id"}) + assertErrorCount(t, 1, errs) + assertMapLength(t, 0, storedReqs) +} + +// TestDatabaseError makes sure we exit with an error if the DB query fails. +func TestDatabaseError(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Failed to create mock: %v", err) + } + + mock.ExpectQuery(".*").WillReturnError(errors.New("Invalid query.")) + + fetcher := &dbFetcher{ + db: db, + queryMaker: successfulQueryMaker("SELECT id, requestData FROM my_table WHERE id IN (?, ?)"), + } + + cfgs, errs := fetcher.FetchRequests(context.Background(), []string{"stored-req-id"}) + assertErrorCount(t, 1, errs) + assertMapLength(t, 0, cfgs) +} + +// TestContextDeadlines makes sure a hung query returns when the timeout expires. +func TestContextDeadlines(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Failed to create mock: %v", err) + } + + mock.ExpectQuery(".*").WillDelayFor(2 * time.Minute) + + fetcher := &dbFetcher{ + db: db, + queryMaker: successfulQueryMaker("SELECT id, requestData FROM my_table WHERE id IN (?, ?)"), + } + + ctx, _ := context.WithTimeout(context.Background(), 1 * time.Nanosecond) + _, errs := fetcher.FetchRequests(ctx, []string{"id"}) + if len(errs) < 1 { + t.Errorf("dbFetcher should return an error when the context times out.") + } +} + +// TestContextCancelled makes sure a hung query returns when the context is cancelled. +func TestContextCancelled(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Failed to create mock: %v", err) + } + + mock.ExpectQuery(".*").WillDelayFor(2 * time.Minute) + + fetcher := &dbFetcher{ + db: db, + queryMaker: successfulQueryMaker("SELECT id, requestData FROM my_table WHERE id IN (?, ?)"), + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, errs := fetcher.FetchRequests(ctx, []string{"id"}) + if len(errs) < 1 { + t.Errorf("dbFetcher should return an error when the context is cancelled.") + } +} + +func newFetcher(rows *sqlmock.Rows, query string, args ...driver.Value) (sqlmock.Sqlmock, *dbFetcher, error) { + db, mock, err := sqlmock.New() + if err != nil { + return nil, nil, err + } + + queryRegex := fmt.Sprintf("^%s$", regexp.QuoteMeta(query)) + mock.ExpectQuery(queryRegex).WithArgs(args...).WillReturnRows(rows) + fetcher := &dbFetcher{ + db: db, + queryMaker: successfulQueryMaker(query), + } + + return mock, fetcher, nil +} + +func assertMapLength(t *testing.T, numExpected int, configs map[string]json.RawMessage) { + t.Helper() + if len(configs) != numExpected { + t.Errorf("Wrong num configs. Expected %d, Got %d.", numExpected, len(configs)) + } +} + +func assertMockExpectations(t *testing.T, mock sqlmock.Sqlmock) { + t.Helper() + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("Mock expectations not met: %v", err) + } +} + +func assertHasData(t *testing.T, data map[string]json.RawMessage, key string, value string) { + t.Helper() + cfg, ok := data[key] + if !ok { + t.Errorf("Missing expected stored request data: %s", key) + } + if string(cfg) != value { + t.Errorf("Bad data[%s] value. Expected %s, Got %s", key, value, cfg) + } +} + +func assertErrorCount(t *testing.T, num int, errs []error) { + t.Helper() + if len(errs) != num { + t.Errorf("Wrong number of errors. Expected %d. Got %d", num, len(errs)) + } +} + +func successfulQueryMaker(response string) func(int) (string, error) { + return func(numIds int) (string, error) { + return response, nil + } +} + +func failedQueryMaker(_ int)(string, error) { + return "", errors.New("The query maker failed.") +} diff --git a/stored_requests/backends/db_fetcher/postgres.go b/stored_requests/backends/db_fetcher/postgres.go new file mode 100644 index 00000000000..57d7b10888a --- /dev/null +++ b/stored_requests/backends/db_fetcher/postgres.go @@ -0,0 +1,60 @@ +package db_fetcher + +import ( + "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/stored_requests" + "database/sql" + "bytes" + "strconv" +) + +func NewPostgres(cfg *config.PostgresConfig) (stored_requests.Fetcher, error) { + db, err := sql.Open("postgres", confToPostgresDSN(cfg)) + if err != nil { + return nil, err + } + + return &dbFetcher{ + db: db, + queryMaker: cfg.MakeQuery, + }, nil +} + +// confToPostgresDSN converts our app config into a string for the pq driver. +// For their docs, and the intended behavior of this function, see: https://godoc.org/github.com/lib/pq +func confToPostgresDSN(cfg *config.PostgresConfig) string { + buffer := bytes.NewBuffer(nil) + + if cfg.Host != "" { + buffer.WriteString("host=") + buffer.WriteString(cfg.Host) + buffer.WriteString(" ") + } + + if cfg.Port > 0 { + buffer.WriteString("port=") + buffer.WriteString(strconv.Itoa(cfg.Port)) + buffer.WriteString(" ") + } + + if cfg.Username != "" { + buffer.WriteString("user=") + buffer.WriteString(cfg.Username) + buffer.WriteString(" ") + } + + if cfg.Password != "" { + buffer.WriteString("password=") + buffer.WriteString(cfg.Password) + buffer.WriteString(" ") + } + + if cfg.Database != "" { + buffer.WriteString("dbname=") + buffer.WriteString(cfg.Database) + buffer.WriteString(" ") + } + + buffer.WriteString("sslmode=disable") + return buffer.String() +} diff --git a/stored_requests/backends/db_fetcher/postgres_test.go b/stored_requests/backends/db_fetcher/postgres_test.go new file mode 100644 index 00000000000..086ef680a8f --- /dev/null +++ b/stored_requests/backends/db_fetcher/postgres_test.go @@ -0,0 +1,59 @@ +package db_fetcher + +import ( + "testing" + "github.com/prebid/prebid-server/config" + "strings" + "strconv" +) + +// TestDSNCreation makes sure we turn the config into a string expected by the Postgres driver library. +func TestDSNCreation(t *testing.T) { + db := "TestDB" + host := "somehost.com" + port := 20 + username := "someuser" + password := "somepassword" + query := "SELECT id, config FROM table WHERE id in %ID_LIST%" + + cfg := &config.PostgresConfig{ + Database: db, + Host: host, + Port: port, + Username: username, + Password: password, + QueryTemplate: query, + } + + dataSourceName := confToPostgresDSN(cfg) + paramList := strings.Split(dataSourceName, " ") + params := make(map[string]string, len(paramList)) + for _, param := range paramList { + keyVals := strings.Split(param, "=") + if len(keyVals) != 2 { + t.Fatalf(`param "%s" must only have one equals sign`, param) + } + if _, ok := params[keyVals[0]]; ok { + t.Fatalf("found duplicate param at key %s", keyVals[0]) + } + params[keyVals[0]] = keyVals[1] + } + + assertHasValue(t, params, "dbname", db) + assertHasValue(t, params, "host", host) + assertHasValue(t, params, "port", strconv.Itoa(port)) + assertHasValue(t, params, "user", username) + assertHasValue(t, params, "password", password) + assertHasValue(t, params, "sslmode", "disable") +} + +func assertHasValue(t *testing.T, m map[string]string, key string, val string) { + t.Helper() + realVal, ok := m[key] + if !ok { + t.Errorf("Map missing required key: %s", key) + } + if val != realVal { + t.Errorf("Unexpected value at key %s. Expected %s, Got %s", val, realVal) + } +} diff --git a/stored_requests/backends/empty_fetcher/fetcher.go b/stored_requests/backends/empty_fetcher/fetcher.go new file mode 100644 index 00000000000..de9d13d82a4 --- /dev/null +++ b/stored_requests/backends/empty_fetcher/fetcher.go @@ -0,0 +1,26 @@ +package empty_fetcher + +import ( + "encoding/json" + "fmt" + "github.com/prebid/prebid-server/stored_requests" + "context" +) + +// EmptyFetcher is a nil-object which has no Stored Requests. +// If PBS is configured to use this, then all the OpenRTB request data must be sent in the HTTP request. +func EmptyFetcher() stored_requests.Fetcher { + return &instance +} + +type emptyFetcher struct {} + +func (fetcher *emptyFetcher) FetchRequests(ctx context.Context, ids []string) (map[string]json.RawMessage, []error) { + errs := make([]error, 0, len(ids)) + for _, id := range ids { + errs = append(errs, fmt.Errorf(`Stored request with id="%s" not found.`, id)) + } + return nil, errs +} + +var instance = emptyFetcher{} diff --git a/stored_requests/backends/empty_fetcher/fetcher_test.go b/stored_requests/backends/empty_fetcher/fetcher_test.go new file mode 100644 index 00000000000..43103b97264 --- /dev/null +++ b/stored_requests/backends/empty_fetcher/fetcher_test.go @@ -0,0 +1,18 @@ +package empty_fetcher + +import ( + "testing" + "context" +) + +func TestErrorLength(t *testing.T) { + fetcher := EmptyFetcher() + + storedReqs, errs := fetcher.FetchRequests(context.Background(), []string{"a", "b"}) + if len(storedReqs) != 0 { + t.Errorf("The empty fetcher should never return stored requests. Got %d", len(storedReqs)) + } + if len(errs) != 2 { + t.Errorf("The empty fetcher should return 2 errors. Got %d", len(errs)) + } +} diff --git a/stored_requests/backends/file_fetcher/fetcher.go b/stored_requests/backends/file_fetcher/fetcher.go new file mode 100644 index 00000000000..f8359c1110e --- /dev/null +++ b/stored_requests/backends/file_fetcher/fetcher.go @@ -0,0 +1,50 @@ +package file_fetcher + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "strings" + "github.com/prebid/prebid-server/stored_requests" + "context" +) + +// NewFileFetcher _immediately_ loads stored request data from local files. +// These are stored in memory for low-latency reads. +// +// This expects each file in the directory to be named "{config_id}.json". +// For example, when asked to fetch the request with ID == "23", it will return the data from "directory/23.json". +func NewFileFetcher(directory string) (stored_requests.Fetcher, error) { + fileInfos, err := ioutil.ReadDir(directory) + if err != nil { + return nil, err + } + storedReqs := make(map[string]json.RawMessage, len(fileInfos)) + for _, fileInfo := range fileInfos { + fileData, err := ioutil.ReadFile(fmt.Sprintf("%s/%s", directory, fileInfo.Name())) + if err != nil { + return nil, err + } + if strings.HasSuffix(fileInfo.Name(), ".json") { // Skip the .gitignore + storedReqs[strings.TrimSuffix(fileInfo.Name(), ".json")] = json.RawMessage(fileData) + } + } + return &eagerFetcher{storedReqs}, nil +} + +type eagerFetcher struct { + storedReqs map[string]json.RawMessage +} + +func (fetcher *eagerFetcher) FetchRequests(ctx context.Context, ids []string) (map[string]json.RawMessage, []error) { + var errors []error = nil + for _, id := range ids { + if _, ok := fetcher.storedReqs[id]; !ok { + errors = append(errors, fmt.Errorf("No config found for id: %s", id)) + } + } + + // Even though there may be many other IDs here, the interface contract doesn't prohibit this. + // Returning the whole slice is much cheaper than making partial copies on each call. + return fetcher.storedReqs, errors +} diff --git a/stored_requests/backends/file_fetcher/fetcher_test.go b/stored_requests/backends/file_fetcher/fetcher_test.go new file mode 100644 index 00000000000..76075b269d9 --- /dev/null +++ b/stored_requests/backends/file_fetcher/fetcher_test.go @@ -0,0 +1,58 @@ +package file_fetcher + +import ( + "testing" + "encoding/json" + "context" +) + +func TestFileFetcher(t *testing.T) { + fetcher, err := NewFileFetcher("./test") + if err != nil { + t.Errorf("Failed to create a Fetcher: %v", err) + } + + storedReqs, errs := fetcher.FetchRequests(context.Background(), []string{"1", "2"}) + if len(errs) != 0 { + t.Errorf("There shouldn't be any errors when requesting known stored requests. Got %v", errs) + } + value, hasId := storedReqs["1"] + if !hasId { + t.Fatalf("Expected stored request data to have id: %d", 1) + } + + var req1Val map[string]string + if err := json.Unmarshal(value, &req1Val); err != nil { + t.Errorf("Failed to unmarshal 1: %v", err) + } + if len(req1Val) != 1 { + t.Errorf("Unexpected req1Val length. Expected %v, Got %s", 1, len(req1Val)) + } + data, hadKey := req1Val["test"] + if !hadKey { + t.Errorf("req1Val should have had a \"test\" key, but it didn't.") + } + if data != "foo" { + t.Errorf(`Bad data in "test" of stored request "1". Expected %s, Got %s`, "foo", data) + } + + value, hasId = storedReqs["2"] + if !hasId { + t.Fatalf("Expected stored request map to have id: %d", 2) + } + + var req2Val string + if err := json.Unmarshal(value, &req2Val); err != nil { + t.Errorf("Failed to unmarshal %d: %v", 2, err) + } + if req2Val != `esca"ped` { + t.Errorf(`Bad data in stored request "2". Expected %v, Got %s`, `esca"ped`, req2Val) + } +} + +func TestInvalidDirectory(t *testing.T) { + _, err := NewFileFetcher("./nonexistant-directory") + if err == nil { + t.Errorf("There should be an error if we use a directory which doesn't exist.") + } +} diff --git a/stored_requests/backends/file_fetcher/test/1.json b/stored_requests/backends/file_fetcher/test/1.json new file mode 100644 index 00000000000..0a740e74a76 --- /dev/null +++ b/stored_requests/backends/file_fetcher/test/1.json @@ -0,0 +1,3 @@ +{ + "test": "foo" +} diff --git a/stored_requests/backends/file_fetcher/test/2.json b/stored_requests/backends/file_fetcher/test/2.json new file mode 100644 index 00000000000..d302ea50709 --- /dev/null +++ b/stored_requests/backends/file_fetcher/test/2.json @@ -0,0 +1 @@ +"esca\"ped" diff --git a/stored_requests/data/by_id/.gitignore b/stored_requests/data/by_id/.gitignore new file mode 100644 index 00000000000..9a3be781f63 --- /dev/null +++ b/stored_requests/data/by_id/.gitignore @@ -0,0 +1,3 @@ +# Ignore everything in this directory, except for this file +* +!.gitignore diff --git a/stored_requests/fetcher.go b/stored_requests/fetcher.go new file mode 100644 index 00000000000..869ac70457b --- /dev/null +++ b/stored_requests/fetcher.go @@ -0,0 +1,18 @@ +package stored_requests + +import ( + "encoding/json" + "context" +) + +// Fetcher knows how to fetch Stored Request data by id. +// +// Implementations must be safe for concurrent access by multiple goroutines. +// Callers are expected to share a single instance as much as possible. +type Fetcher interface { + // FetchRequests fetches the stored requests for the given IDs. + // The returned map will have keys for every ID in the argument list, unless errors exist. + // + // The returned objects can only be read from. They may not be written to. + FetchRequests(ctx context.Context, ids []string) (map[string]json.RawMessage, []error) +}