diff --git a/application.go b/application.go index 8decff7..df2bfd1 100644 --- a/application.go +++ b/application.go @@ -76,7 +76,7 @@ func (b *BigIP) CreateIapp(p *Iapp) error { func (b *BigIP) UpdateIapp(name string, p *Iapp) error { values := []string{} - values = append(values, "~Common~") + values = append(values, fmt.Sprintf("~%s~", p.Partition)) values = append(values, name) values = append(values, ".app~") values = append(values, name) @@ -86,11 +86,11 @@ func (b *BigIP) UpdateIapp(name string, p *Iapp) error { return b.patch(p, uriSysa, uriApp, uriService, result) } -func (b *BigIP) Iapp(name string) (*Iapp, error) { +func (b *BigIP) Iapp(name, partition string) (*Iapp, error) { var iapp Iapp log.Println(" Value of iapp before read ", &iapp) values := []string{} - values = append(values, "~Common~") + values = append(values, fmt.Sprintf("~%s~", partition)) values = append(values, name) values = append(values, ".app~") values = append(values, name) @@ -105,9 +105,9 @@ func (b *BigIP) Iapp(name string) (*Iapp, error) { return &iapp, nil } -func (b *BigIP) DeleteIapp(name string) error { +func (b *BigIP) DeleteIapp(name, partition string) error { values := []string{} - values = append(values, "~Common~") + values = append(values, fmt.Sprintf("~%s~", partition)) values = append(values, name) values = append(values, ".app~") values = append(values, name) diff --git a/as3bigip.go b/as3bigip.go index 2ecb852..451a5da 100644 --- a/as3bigip.go +++ b/as3bigip.go @@ -128,7 +128,7 @@ func (b *BigIP) PostAs3Bigip(as3NewJson string, tenantFilter string) (error, str return b.PostAs3Bigip(as3NewJson, tenantFilter) } for _, id := range taskIds { - if b.pollingStatus(id) { + if b.pollingStatus(id, 5*time.Second) { return b.PostAs3Bigip(as3NewJson, tenantFilter) } } @@ -201,7 +201,7 @@ func (b *BigIP) DeleteAs3Bigip(tenantName string) (error, string) { return b.DeleteAs3Bigip(tenantName) } for _, id := range taskIds { - if b.pollingStatus(id) { + if b.pollingStatus(id, 5*time.Second) { return b.DeleteAs3Bigip(tenantName) } } @@ -239,7 +239,7 @@ func (b *BigIP) ModifyAs3(tenantFilter string, as3_json string) error { return err } for _, id := range taskIds { - if b.pollingStatus(id) { + if b.pollingStatus(id, 5*time.Second) { return b.ModifyAs3(tenantFilter, as3_json) } } @@ -363,21 +363,25 @@ func (b *BigIP) getas3Taskid() ([]string, error) { } return taskIDs, nil } -func (b *BigIP) pollingStatus(id string) bool { + +func (b *BigIP) pollingStatus(id string, backoff time.Duration) bool { + log.Printf("[INFO]pollingStatus DELAY -- %d ", int(backoff.Seconds())) var taskList As3TaskType err, _ := b.getForEntity(&taskList, uriMgmt, uriShared, uriAppsvcs, uriTask, id) if err != nil { return false } - if taskList.Results[0].Code != 200 && taskList.Results[0].Code != 503 { - time.Sleep(1 * time.Second) - return b.pollingStatus(id) - } - if taskList.Results[0].Code == 503 { - return false + if taskList.Results[0].Code != 200 { + if backoff > 30*time.Second { + backoff = 30 * time.Second // cap at 30 seconds + } + time.Sleep(backoff) + return b.pollingStatus(id, backoff*2) // recursive call with doubled delay } + return true } + func (b *BigIP) GetTenantList(body interface{}) (string, int, string) { tenantList := make([]string, 0) applicationList := make([]string, 0) diff --git a/bigip.go b/bigip.go index 16cd0c2..5b1d22c 100644 --- a/bigip.go +++ b/bigip.go @@ -19,8 +19,8 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" + "net/url" "os" "reflect" "strings" @@ -29,10 +29,15 @@ import ( var defaultConfigOptions = &ConfigOptions{ APICallTimeout: 60 * time.Second, + // Define new configuration options; are these user-override-able at the provider level or does that take more work? + TokenTimeout: 1200 * time.Second, + APICallRetries: 10, } type ConfigOptions struct { APICallTimeout time.Duration + TokenTimeout time.Duration + APICallRetries int } type Config struct { @@ -58,6 +63,7 @@ type BigIP struct { UserAgent string Teem bool ConfigOptions *ConfigOptions + Transaction string } // APIRequest builds our request before sending it to the server. @@ -98,20 +104,20 @@ func (r *RequestError) Error() error { // NewSession sets up our connection to the BIG-IP system. // func NewSession(host, port, user, passwd string, configOptions *ConfigOptions) *BigIP { func NewSession(bigipConfig *Config) *BigIP { - var url string + var urlString string if !strings.HasPrefix(bigipConfig.Address, "http") { - url = fmt.Sprintf("https://%s", bigipConfig.Address) + urlString = fmt.Sprintf("https://%s", bigipConfig.Address) } else { - url = bigipConfig.Address + urlString = bigipConfig.Address } if bigipConfig.Port != "" { - url = url + ":" + bigipConfig.Port + urlString = urlString + ":" + bigipConfig.Port } if bigipConfig.ConfigOptions == nil { bigipConfig.ConfigOptions = defaultConfigOptions } return &BigIP{ - Host: url, + Host: urlString, User: bigipConfig.Username, Password: bigipConfig.Password, Transport: &http.Transport{ @@ -219,49 +225,68 @@ func (client *BigIP) ValidateConnection() error { // APICall is used to query the BIG-IP web API. func (b *BigIP) APICall(options *APIRequest) ([]byte, error) { var req *http.Request - client := &http.Client{ - Transport: b.Transport, - Timeout: b.ConfigOptions.APICallTimeout, - } var format string if strings.Contains(options.URL, "mgmt/") { format = "%s/%s" } else { format = "%s/mgmt/tm/%s" } - url := fmt.Sprintf(format, b.Host, options.URL) - body := bytes.NewReader([]byte(options.Body)) - req, _ = http.NewRequest(strings.ToUpper(options.Method), url, body) - if b.Token != "" { - req.Header.Set("X-F5-Auth-Token", b.Token) - } else if options.URL != "mgmt/shared/authn/login" { - req.SetBasicAuth(b.User, b.Password) - } - - //fmt.Println("REQ -- ", options.Method, " ", url," -- ",options.Body) - - if len(options.ContentType) > 0 { - req.Header.Set("Content-Type", options.ContentType) - } - - res, err := client.Do(req) - if err != nil { - return nil, err - } - - defer res.Body.Close() - - data, _ := ioutil.ReadAll(res.Body) + urlString := fmt.Sprintf(format, b.Host, options.URL) + maxRetries := b.ConfigOptions.APICallRetries + for i := 0; i < maxRetries; i++ { + body := bytes.NewReader([]byte(options.Body)) + req, _ = http.NewRequest(strings.ToUpper(options.Method), urlString, body) + b.Transport.Proxy = func(reqNew *http.Request) (*url.URL, error) { + return http.ProxyFromEnvironment(reqNew) + } + client := &http.Client{ + Transport: b.Transport, + Timeout: b.ConfigOptions.APICallTimeout, + } + if b.Token != "" { + req.Header.Set("X-F5-Auth-Token", b.Token) + } else if options.URL != "mgmt/shared/authn/login" { + req.SetBasicAuth(b.User, b.Password) + } - if res.StatusCode >= 400 { - if res.Header["Content-Type"][0] == "application/json" { - return data, b.checkError(data) + if len(b.Transaction) > 0 { + req.Header.Set("X-F5-REST-Coordination-Id", b.Transaction) } - return data, errors.New(fmt.Sprintf("HTTP %d :: %s", res.StatusCode, string(data[:]))) + if len(options.ContentType) > 0 { + req.Header.Set("Content-Type", options.ContentType) + } + res, err := client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + data, _ := io.ReadAll(res.Body) + contentType := "" + if ctHeaders, ok := res.Header["Content-Type"]; ok && len(ctHeaders) > 0 { + contentType = ctHeaders[0] + } + if res.StatusCode >= 400 { + if strings.Contains(contentType, "application/json") { + var reqError RequestError + err = json.Unmarshal(data, &reqError) + if err != nil { + return nil, err + } + // With how some of the requests come back from AS3, we sometimes have a nested error, so check the entire message for the "active asynchronous task" error + if res.StatusCode == 503 || reqError.Code == 503 || strings.Contains(strings.ToLower(reqError.Message), strings.ToLower("there is an active asynchronous task executing")) { + time.Sleep(10 * time.Second) + continue + } + return data, b.checkError(data) + } else { + return data, fmt.Errorf("HTTP %d :: %s", res.StatusCode, string(data[:])) + } + //return data, errors.New(fmt.Sprintf("HTTP %d :: %s", res.StatusCode, string(data[:]))) + } + return data, nil } - - return data, nil + return nil, fmt.Errorf("service unavailable after %d attempts", maxRetries) } func (b *BigIP) iControlPath(parts []string) string { @@ -418,10 +443,6 @@ func (b *BigIP) fastPatch(body interface{}, path ...string) ([]byte, error) { // Upload a file read from a Reader func (b *BigIP) Upload(r io.Reader, size int64, path ...string) (*Upload, error) { - client := &http.Client{ - Transport: b.Transport, - Timeout: b.ConfigOptions.APICallTimeout, - } options := &APIRequest{ Method: "post", URL: b.iControlPath(path), @@ -433,7 +454,7 @@ func (b *BigIP) Upload(r io.Reader, size int64, path ...string) (*Upload, error) } else { format = "%s/mgmt/%s" } - url := fmt.Sprintf(format, b.Host, options.URL) + urlString := fmt.Sprintf(format, b.Host, options.URL) chunkSize := 512 * 1024 var start, end int64 for { @@ -449,7 +470,7 @@ func (b *BigIP) Upload(r io.Reader, size int64, path ...string) (*Upload, error) chunk = chunk[:n] } body := bytes.NewReader(chunk) - req, _ := http.NewRequest(strings.ToUpper(options.Method), url, body) + req, _ := http.NewRequest(strings.ToUpper(options.Method), urlString, body) if b.Token != "" { req.Header.Set("X-F5-Auth-Token", b.Token) } else { @@ -457,12 +478,19 @@ func (b *BigIP) Upload(r io.Reader, size int64, path ...string) (*Upload, error) } req.Header.Add("Content-Type", options.ContentType) req.Header.Add("Content-Range", fmt.Sprintf("%d-%d/%d", start, end-1, size)) + b.Transport.Proxy = func(reqNew *http.Request) (*url.URL, error) { + return http.ProxyFromEnvironment(reqNew) + } + client := &http.Client{ + Transport: b.Transport, + Timeout: b.ConfigOptions.APICallTimeout, + } // Try to upload chunk res, err := client.Do(req) if err != nil { return nil, err } - data, _ := ioutil.ReadAll(res.Body) + data, _ := io.ReadAll(res.Body) if res.StatusCode >= 400 { if res.Header.Get("Content-Type") == "application/json" { return nil, b.checkError(data) @@ -484,7 +512,7 @@ func (b *BigIP) Upload(r io.Reader, size int64, path ...string) (*Upload, error) } } -// Get a url and populate an entity. If the entity does not exist (404) then the +// Get a urlString and populate an entity. If the entity does not exist (404) then the // passed entity will be untouched and false will be returned as the second parameter. // You can use this to distinguish between a missing entity or an actual error. func (b *BigIP) getForEntity(e interface{}, path ...string) (error, bool) { diff --git a/ltm.go b/ltm.go index cf259c2..ab2b370 100644 --- a/ltm.go +++ b/ltm.go @@ -675,6 +675,7 @@ type Policy struct { Name string PublishCopy string Partition string + Description string FullPath string Controls []string Requires []string @@ -685,6 +686,7 @@ type policyDTO struct { Name string `json:"name"` PublishCopy string `json:"publishedCopy"` Partition string `json:"partition,omitempty"` + Description string `json:"description"` Controls []string `json:"controls,omitempty"` Requires []string `json:"requires,omitempty"` Strategy string `json:"strategy,omitempty"` @@ -700,6 +702,7 @@ func (p *Policy) MarshalJSON() ([]byte, error) { PublishCopy: p.PublishCopy, Partition: p.Partition, Controls: p.Controls, + Description: p.Description, Requires: p.Requires, Strategy: p.Strategy, FullPath: p.FullPath, @@ -715,13 +718,13 @@ func (p *Policy) UnmarshalJSON(b []byte) error { if err != nil { return err } - p.Name = dto.Name p.PublishCopy = dto.PublishCopy p.Partition = dto.Partition p.Controls = dto.Controls p.Requires = dto.Requires p.Strategy = dto.Strategy + p.Description = dto.Description p.Rules = dto.Rules.Items p.FullPath = dto.FullPath @@ -2814,7 +2817,7 @@ func (b *BigIP) CheckDraftPolicy(name string, partition string) (bool, error) { if p.FullPath == "" { return false, nil } - return true , nil + return true, nil } func normalizePolicy(p *Policy) { diff --git a/sys.go b/sys.go index 4e3687b..668b086 100644 --- a/sys.go +++ b/sys.go @@ -15,6 +15,7 @@ import ( "fmt" "log" "os" + //"strings" "time" ) @@ -285,6 +286,7 @@ const ( uriSslCert = "ssl-cert" uriSslKey = "ssl-key" uriDataGroup = "data-group" + uriTransaction = "transaction" REST_DOWNLOAD_PATH = "/var/config/rest/downloads" ) @@ -304,7 +306,7 @@ type Certificate struct { CreatedBy string `json:"createdBy,omitempty"` CreateTime string `json:"createTime,omitempty"` Email string `json:"email,omitempty"` - ExpirationDate int `json:"expirationDate,omitempty"` + ExpirationDate int64 `json:"expirationDate,omitempty"` ExpirationString string `json:"expirationString,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` FullPath string `json:"fullPath,omitempty"` @@ -360,6 +362,17 @@ type Key struct { UpdatedBy string `json:"updatedBy,omitempty"` } +type Transaction struct { + TransID int64 `json:"transId,omitempty"` + State string `json:"state,omitempty"` + TimeoutSeconds int64 `json:"timeoutSeconds,omitempty"` + AsyncExecution bool `json:"asyncExecution,omitempty"` + ValidateOnly bool `json:"validateOnly,omitempty"` + ExecutionTimeout int64 `json:"executionTimeout,omitempty"` + ExecutionTime int64 `json:"executionTime,omitempty"` + FailureReason string `json:"failureReason,omitempty"` +} + // Certificates returns a list of certificates. func (b *BigIP) Certificates() (*Certificates, error) { var certs Certificates @@ -779,6 +792,40 @@ func (b *BigIP) CreateTRAP(name string, authPasswordEncrypted string, authProtoc return b.post(config, uriSys, uriSnmp, uriTraps) } +func (b *BigIP) StartTransaction() (*Transaction, error) { + body := make(map[string]interface{}) + resp, err := b.postReq(body, uriMgmt, uriTm, uriTransaction) + + if err != nil { + return nil, fmt.Errorf("error encountered while starting transaction: %v", err) + } + transaction := &Transaction{} + err = json.Unmarshal(resp, transaction) + if err != nil { + return nil, err + } + log.Printf("[INFO] Transaction: %v", transaction) + b.Transaction = fmt.Sprint(transaction.TransID) + return transaction, nil +} + +func (b *BigIP) EndTransaction(tId int64) error { + commitTransaction := map[string]interface{}{ + "state": "VALIDATING", + "validateOnly": false, + } + payload, err := json.Marshal(commitTransaction) + if err != nil { + return fmt.Errorf("unable create commit transaction payload: %s", err) + } + err = b.patch(payload, uriMgmt, uriTm, uriTransaction, string(tId)) + if err != nil { + return fmt.Errorf("%s", err) + } + b.Transaction = "" + return nil +} + func (b *BigIP) ModifyTRAP(config *TRAP) error { return b.patch(config, uriSys, uriSnmp, uriTraps) }