From 929772a838766eaa2f0d633426d2f377fb480f03 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 20 Jun 2021 13:52:24 +0100 Subject: [PATCH] FEAT: Added an API in debug to return the nodes as JSON. Outputs the nodes to the logger at startup if in debug mode. Modified the alive service to set more parameters for the transport. Simplified the nodes debug handler. --- aliveService.go | 22 ++++++++++----- handlerNodes.go | 75 ++++++++++++++++++++++++++++++++++++++----------- handlers.go | 1 + local.go | 4 --- nodes_test.go | 1 + store.go | 29 ++++++++++++------- 6 files changed, 95 insertions(+), 37 deletions(-) diff --git a/aliveService.go b/aliveService.go index ae4bbbd..460c0fb 100644 --- a/aliveService.go +++ b/aliveService.go @@ -26,6 +26,9 @@ import ( "time" ) +// The size of the nounce used for the keep alive service. +const nounceSize = 32 + // aliveService type is service which polls known nodes to determine if they are // 'alive' and responding to requests. Only nodes that have not been accessed // for a period of time greater than the polling interval will be polled. On a @@ -69,20 +72,26 @@ func newAliveService(c Configuration, s storageManager) *aliveService { // quickly. func (a *aliveService) aliveLoop() { t := &http.Transport{ - DisableKeepAlives: true, - DisableCompression: true, - ForceAttemptHTTP2: false, - } + DisableKeepAlives: true, + DisableCompression: true, + ForceAttemptHTTP2: false, + MaxConnsPerHost: 1, + MaxIdleConnsPerHost: 1, + MaxIdleConns: len(a.store.nodes), + IdleConnTimeout: time.Second, + ResponseHeaderTimeout: time.Second, + ExpectContinueTimeout: time.Second} c := &http.Client{ Timeout: a.pollingInterval, Transport: t} - defer c.CloseIdleConnections() + a.ticker = time.NewTicker(a.pollingInterval) for _ = range a.ticker.C { a.ticker.Stop() for _, n := range a.store.nodes { a.pollNode(n, c) } + c.CloseIdleConnections() a.ticker.Reset(a.pollingInterval) } } @@ -183,11 +192,10 @@ func (a *aliveService) callAlive( // nonce returns a new nonce generated using crpyto/rand func nonce() ([]byte, error) { - b := make([]byte, 32) + b := make([]byte, nounceSize) _, err := rand.Read(b) if err != nil { return nil, err } - return b, nil } diff --git a/handlerNodes.go b/handlerNodes.go index 356dd99..5759015 100644 --- a/handlerNodes.go +++ b/handlerNodes.go @@ -17,6 +17,7 @@ package swift import ( + "encoding/json" "net/http" "time" ) @@ -50,28 +51,70 @@ func (nv *NodeViews) NodeViewItems() []NodeView { // template. func HandlerNodes(s *Services) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + nvs, err := getNodesView(s) + if err != nil { + returnAPIError(s, w, err, http.StatusInternalServerError) + } + sendHTMLTemplate(s, w, swiftNodesTemplate, &nvs) + } +} - var nvs NodeViews - - ns, err := s.store.getAllNodes() +// HandlerNodesJSON is a handler that returns a list of all the alive nodes +// which is then used to serialize to JSON. +func HandlerNodesJSON(s *Services) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + j, err := getJSON(s) if err != nil { returnAPIError(s, w, err, http.StatusInternalServerError) + return } + sendResponse(s, w, "application/json", j) + } +} + +func getJSON(s *Services) ([]byte, error) { - for _, n := range ns { - nv := NodeView{ - Network: n.network, - Domain: n.domain, - Created: n.created, - Starts: n.starts, - Expires: n.expires, - Role: n.role, - Accessed: n.accessed, - Alive: n.alive, - } - nvs.Nodes = append(nvs.Nodes, nv) + // Get all the nodes. + ns, err := s.store.getAllNodes() + if err != nil { + return nil, err + } + + // Turn them into a map. + nis := make(map[string]*node) + for _, n := range ns { + if n.alive { + nis[n.domain] = n } + } - sendHTMLTemplate(s, w, swiftNodesTemplate, &nvs) + // Turn the map into a JSON string. + j, err := json.Marshal(nis) + if err != nil { + return nil, err + } + + return j, nil +} + +func getNodesView(s *Services) (*NodeViews, error) { + var nvs NodeViews + ns, err := s.store.getAllNodes() + if err != nil { + return nil, err + } + for _, n := range ns { + nv := NodeView{ + Network: n.network, + Domain: n.domain, + Created: n.created, + Starts: n.starts, + Expires: n.expires, + Role: n.role, + Accessed: n.accessed, + Alive: n.alive, + } + nvs.Nodes = append(nvs.Nodes, nv) } + return &nvs, nil } diff --git a/handlers.go b/handlers.go index cfb8f50..3dd6f6b 100644 --- a/handlers.go +++ b/handlers.go @@ -41,6 +41,7 @@ func AddHandlers( if services.config.Debug { http.HandleFunc("/swift/nodes", HandlerNodes(services)) + http.HandleFunc("/swift/api/v1/nodes", HandlerNodesJSON(services)) } } diff --git a/local.go b/local.go index 1f69d77..4d687e5 100644 --- a/local.go +++ b/local.go @@ -115,10 +115,6 @@ func (l *Local) iterateNodes( // SetNode inserts or updates the node. func (l *Local) setNode(n *node) error { - // err := l.setNodeSecrets(n) - // if err != nil { - // return err - // } nis := make(map[string]*node) // Fetch all the records from the nodes file. diff --git a/nodes_test.go b/nodes_test.go index 6d673c9..824b523 100644 --- a/nodes_test.go +++ b/nodes_test.go @@ -126,6 +126,7 @@ func createNodes() (*nodes, error) { "test", fmt.Sprintf("node%d", i), time.Now().UTC(), + time.Now().UTC(), time.Now().UTC().AddDate(1, 0, 0), roleStorage, s.key) diff --git a/store.go b/store.go index 434e4ed..6e1de09 100644 --- a/store.go +++ b/store.go @@ -67,10 +67,10 @@ func NewStore(swiftConfig Configuration) []Store { os.Getenv("AZURE_STORAGE_ACCOUNT"), os.Getenv("AZURE_STORAGE_ACCESS_KEY"), os.Getenv("GCP_PROJECT"), - os.Getenv("SWIFT_NODES_FILE"), + os.Getenv("SWIFT_FILE"), os.Getenv("AWS_ENABLED") if len(azureAccountName) > 0 || len(azureAccountKey) > 0 { - log.Printf("SWIFT: Using Azure Table Storage") + log.Printf("SWIFT:Using Azure Table Storage") if len(azureAccountName) == 0 || len(azureAccountKey) == 0 { panic(errors.New("Either the AZURE_STORAGE_ACCOUNT or " + "AZURE_STORAGE_ACCESS_KEY environment variable is not set")) @@ -82,7 +82,7 @@ func NewStore(swiftConfig Configuration) []Store { swiftStores = append(swiftStores, swiftStore) } if len(gcpProject) > 0 { - log.Printf("SWIFT: Using Google Firebase") + log.Printf("SWIFT:Using Google Firebase") swiftStore, err := NewFirebase(gcpProject) if err != nil { panic(err) @@ -90,7 +90,7 @@ func NewStore(swiftConfig Configuration) []Store { swiftStores = append(swiftStores, swiftStore) } if len(swiftNodes) > 0 { - log.Printf("SWIFT: Using local storage") + log.Printf("SWIFT:Using local storage") swiftStore, err := NewLocalStore(swiftNodes) if err != nil { panic(err) @@ -98,7 +98,7 @@ func NewStore(swiftConfig Configuration) []Store { swiftStores = append(swiftStores, swiftStore) } if len(awsEnabled) > 0 { - log.Printf("SWIFT: Using AWS DynamoDB") + log.Printf("SWIFT:Using AWS DynamoDB") swiftStore, err := NewAWS() if err != nil { panic(err) @@ -107,15 +107,24 @@ func NewStore(swiftConfig Configuration) []Store { } if len(swiftStores) == 0 { - panic(fmt.Errorf("SWIFT: no store has been configured. " + + panic(fmt.Errorf("SWIFT:no store has been configured.\r\n" + "Provide details for store by specifying one or more sets of " + - "environment variables\r\n: " + + "environment variables:\r\n" + "(1) Azure Storage account details 'AZURE_STORAGE_ACCOUNT' & 'AZURE_STORAGE_ACCESS_KEY'\r\n" + - "(2) GCP project in 'GCP_PROJECT' \r\n" + - "(3) Local storage file paths in 'SWIFT_SECRETS_FILE' & 'SWIFT_NODES_FILE'\r\n" + - "(4) AWS Dynamo DB by setting 'AWS_SDK_LOAD_CONFIG' to true\r\n" + + "(2) GCP project in 'GCP_PROJECT'\r\n" + + "(3) Local storage file paths in 'SWIFT_FILE'\r\n" + + "(4) AWS Dynamo DB by setting 'AWS_ENABLED' to true\r\n" + "Refer to https://github.com/SWAN-community/swift-go/blob/main/README.md " + "for specifics on setting up each storage solution")) + } else if swiftConfig.Debug { + + // If in debug more log the nodes at startup. + for _, s := range swiftStores { + s.iterateNodes(func(n *node, s interface{}) error { + log.Println(fmt.Sprintf("SWIFT:\t%s", n.domain)) + return nil + }, nil) + } } return swiftStores