diff --git a/README.md b/README.md index 23ea380..ed94977 100644 --- a/README.md +++ b/README.md @@ -375,8 +375,8 @@ make test ```bash gofmt -w `find . -name '*.go' | grep -v vendor` go vet -all $(go list ./... | grep -v datatypes) -go get -d -v -t ./... -go test $(go list ./... | grep -v '/vendor/') -timeout=30s -parallel=4 -coverprofile coverage.out +go mod vendor +go test $(go list ./... | grep -v '/vendor/') -timeout=30s -coverprofile coverage.out ``` ### Updating dependencies diff --git a/datatypes/billing.go b/datatypes/billing.go index 69f4af1..bd900a3 100644 --- a/datatypes/billing.go +++ b/datatypes/billing.go @@ -1368,6 +1368,14 @@ type Billing_Item_Gateway_Appliance_Cluster struct { Billing_Item } +// The SoftLayer_Billing_Item_Gateway_License data type contains general information relating to a single SoftLayer billing item for a bare_metal_gateway_license +type Billing_Item_Gateway_License struct { + Billing_Item + + // no documentation yet + Resource *Network_Gateway `json:"resource,omitempty" xmlrpc:"resource,omitempty"` +} + // The SoftLayer_Billing_Item_Hardware data type contains general information relating to a single SoftLayer billing item for hardware. type Billing_Item_Hardware struct { Billing_Item diff --git a/examples/cmd/virtual_iter.go b/examples/cmd/virtual_iter.go new file mode 100644 index 0000000..9cb018e --- /dev/null +++ b/examples/cmd/virtual_iter.go @@ -0,0 +1,65 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/softlayer/softlayer-go/filter" + "github.com/softlayer/softlayer-go/helpers/virtual" + "github.com/softlayer/softlayer-go/session" + "github.com/softlayer/softlayer-go/sl" +) + +func init() { + rootCmd.AddCommand(listVirtCmd) +} + +var listVirtCmd = &cobra.Command{ + Use: "virt-list", + Short: "Lists all VSI on the account", + Long: `Lists all VSI on the account using an iterative aproach.`, + RunE: func(cmd *cobra.Command, args []string) error { + return RunListVirtCmd(cmd, args) + }, +} + +func RunListVirtCmd(cmd *cobra.Command, args []string) error { + + objectMask := "mask[id,hostname,domain,primaryIpAddress,primaryBackendIpAddress]" + // When using a result Limit to break up your API request, its important to include an orderBy objectFilter + // to enforce an order on the query, as the database might not always return results in the same order between + // queries otherwise + filters := filter.New() + filters = append(filters, filter.Path("virtualGuests.id").OrderBy("ASC")) + objectFilter := filters.Build() + // Sets up the session with authentication headers. + sess := session.New() + // uncomment to output API calls as they are made. + sess.Debug = true + + // Sets the mask, filter, result limit, and then makes the API call SoftLayer_Account::getHardware() + limit := 5 + options := sl.Options{ + Mask: objectMask, + Filter: objectFilter, + Limit: &limit, + } + + servers, err := virtual.GetVirtualGuestsIter(sess, &options) + if err != nil { + return err + } + fmt.Printf("Id, Hostname, Domain, IP Address\n") + + for _, server := range servers { + ipAddress := "-" + // Servers with a private only connection will not have a primary IP + if server.PrimaryIpAddress != nil { + ipAddress = *server.PrimaryIpAddress + } + fmt.Printf("%v, %v, %v, %v\n", *server.Id, *server.Hostname, *server.Domain, ipAddress) + } + + return nil +} diff --git a/examples/go.mod b/examples/go.mod index 4cd8eb9..e62da07 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -2,6 +2,8 @@ module github.com/softlayer/softlayer-go/examples go 1.21 +replace github.com/softlayer/softlayer-go => ../ + require ( github.com/jarcoal/httpmock v1.0.5 github.com/jedib0t/go-pretty/v6 v6.5.4 @@ -26,6 +28,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.20.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/examples/go.sum b/examples/go.sum index a6225fd..f95d294 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -54,6 +54,8 @@ golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= diff --git a/generator/templates.go b/generator/templates.go index e32594f..5f93600 100644 --- a/generator/templates.go +++ b/generator/templates.go @@ -120,7 +120,8 @@ import ( {{end}}err = r.Session.DoRequest("{{$rawBase}}", "{{.Name}}", {{if len .Parameters | lt 0}}params{{else}}nil{{end}}, &r.Options, &resp) return } - {{end}} + + {{end }} {{end}} `, license, codegenWarning) @@ -187,3 +188,35 @@ var _ = Describe("{{(index . 0 ).ServiceGroup}} Tests", func() { {{ end }} }) ` + +// Not Used, but could be added to the Service template if you want ALL methods that accept a resultLimit to page through results +var IterTemplate = `{{if .TypeArray}} + func (r {{$base}}) {{.Name|titleCase}}Iter({{range .Parameters}}{{phraseMethodArg $methodName .Name .TypeArray .Type}}{{end}}) ({{if .Type|ne "void"}}resp {{if .TypeArray}}[]{{end}}{{convertType .Type "services"}}, {{end}}err error) { + {{if len .Parameters | lt 0}}params := []interface{}{ + {{range .Parameters}}{{.Name|removeReserved}}, + {{end}} + } + {{end}}limit := r.Options.ValidateLimit() + err = r.Session.DoRequest("{{$rawBase}}", "{{.Name}}", {{if len .Parameters | lt 0}}params{{else}}nil{{end}}, &r.Options, &resp) + if err != nil { + return + } + apicalls := r.Options.GetRemainingAPICalls() + var wg sync.WaitGroup + for x := 1; x <= apicalls; x++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + offset := i * limit + this_resp := []{{convertType .Type "services"}}{} + options := r.Options + options.Offset = &offset + err = r.Session.DoRequest("{{$rawBase}}", "{{.Name}}", {{if len .Parameters | lt 0}}params{{else}}nil{{end}}, &options, &this_resp) + resp = append(resp, this_resp...) + }(x) + } + wg.Wait() + return + } + {{end }} +` diff --git a/go.mod b/go.mod index c322f4d..5ec977a 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.20.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 25e65cb..3c3f033 100644 --- a/go.sum +++ b/go.sum @@ -51,6 +51,7 @@ golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/helpers/hardware/hardware.go b/helpers/hardware/hardware.go index 2f1f095..d3caa0e 100644 --- a/helpers/hardware/hardware.go +++ b/helpers/hardware/hardware.go @@ -18,12 +18,13 @@ package hardware import ( "fmt" - "github.com/softlayer/softlayer-go/datatypes" "github.com/softlayer/softlayer-go/helpers/location" "github.com/softlayer/softlayer-go/services" "github.com/softlayer/softlayer-go/session" + "github.com/softlayer/softlayer-go/sl" "regexp" + "sync" ) // GeRouterByName returns a Hardware that matches the provided hostname, @@ -59,3 +60,36 @@ func GetRouterByName(sess *session.Session, hostname string, args ...interface{} return datatypes.Hardware{}, fmt.Errorf("No routers found with hostname of %s", hostname) } + +// Use go-routines to iterate through all hardware results. +// options should be any Mask or Filter you need, and a Limit if the default is too large. +// Any error in the subsequent API calls will be logged, but largely ignored +func GetHardwareIter(session session.SLSession, options *sl.Options) (resp []datatypes.Hardware, err error) { + + options.SetOffset(0) + limit := options.ValidateLimit() + + // Can't call service.GetVirtualGuests because it passes a copy of options, not the address to options sadly. + err = session.DoRequest("SoftLayer_Account", "getHardware", nil, options, &resp) + if err != nil { + return + } + apicalls := options.GetRemainingAPICalls() + var wg sync.WaitGroup + for x := 1; x <= apicalls; x++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + offset := i * limit + this_resp := []datatypes.Hardware{} + options.Offset = &offset + err = session.DoRequest("SoftLayer_Account", "getHardware", nil, options, &this_resp) + if err != nil { + fmt.Printf("[ERROR] %v\n", err) + } + resp = append(resp, this_resp...) + }(x) + } + wg.Wait() + return resp, err +} diff --git a/helpers/hardware/hardware_test.go b/helpers/hardware/hardware_test.go new file mode 100644 index 0000000..8f1dd4b --- /dev/null +++ b/helpers/hardware/hardware_test.go @@ -0,0 +1,38 @@ +package hardware_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/softlayer/softlayer-go/helpers/hardware" + "github.com/softlayer/softlayer-go/session/sessionfakes" + "github.com/softlayer/softlayer-go/sl" + "testing" +) + +func TestServices(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Helper Hardware Tests") +} + +var _ = Describe("Helper Hardware Tests", func() { + var slsession *sessionfakes.FakeSLSession + var options *sl.Options + BeforeEach(func() { + limit := 10 + slsession = &sessionfakes.FakeSLSession{} + options = &sl.Options{ + Mask: "mask[id,hostname]", + Filter: "", + Limit: &limit, + } + }) + + Context("GetHardwareIter Tests", func() { + + It("API call made properly", func() { + _, err := hardware.GetHardwareIter(slsession, options) + Expect(err).ToNot(HaveOccurred()) + Expect(slsession.DoRequestCallCount()).To(Equal(1)) + }) + }) +}) diff --git a/helpers/virtual/virtual.go b/helpers/virtual/virtual.go index 7a5f15b..cc51e3c 100644 --- a/helpers/virtual/virtual.go +++ b/helpers/virtual/virtual.go @@ -17,13 +17,14 @@ package virtual import ( - "time" - + "fmt" "github.com/softlayer/softlayer-go/datatypes" "github.com/softlayer/softlayer-go/helpers/product" "github.com/softlayer/softlayer-go/services" "github.com/softlayer/softlayer-go/session" "github.com/softlayer/softlayer-go/sl" + "sync" + "time" ) // Upgrade a virtual guest to a specified set of features (e.g. cpu, ram). @@ -159,3 +160,36 @@ func UpgradeVirtualGuestWithPreset( orderService := services.GetProductOrderService(sess) return orderService.PlaceOrder(&order, sl.Bool(false)) } + +// Use go-routines to iterate through all virtual guest results. +// optoins should be any Mask or Filter you need, and a Limit if the default is too large. +// Any error in the subsequent API calls will be logged, but largely ignored +func GetVirtualGuestsIter(session session.SLSession, options *sl.Options) (resp []datatypes.Virtual_Guest, err error) { + + options.SetOffset(0) + limit := options.ValidateLimit() + + // Can't call service.GetVirtualGuests because it passes a copy of options, not the address to options sadly. + err = session.DoRequest("SoftLayer_Account", "getVirtualGuests", nil, options, &resp) + if err != nil { + return + } + apicalls := options.GetRemainingAPICalls() + var wg sync.WaitGroup + for x := 1; x <= apicalls; x++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + offset := i * limit + this_resp := []datatypes.Virtual_Guest{} + options.Offset = &offset + err = session.DoRequest("SoftLayer_Account", "getVirtualGuests", nil, options, &this_resp) + if err != nil { + fmt.Printf("[ERROR] %v\n", err) + } + resp = append(resp, this_resp...) + }(x) + } + wg.Wait() + return resp, err +} diff --git a/helpers/virtual/virtual_test.go b/helpers/virtual/virtual_test.go new file mode 100644 index 0000000..9f09dc5 --- /dev/null +++ b/helpers/virtual/virtual_test.go @@ -0,0 +1,40 @@ +package virtual_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + // "github.com/softlayer/softlayer-go/datatypes" + + "github.com/softlayer/softlayer-go/helpers/virtual" + "github.com/softlayer/softlayer-go/session/sessionfakes" + "github.com/softlayer/softlayer-go/sl" + "testing" +) + +func TestServices(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Helper Virtual Tests") +} + +var _ = Describe("Helper Virtual Tests", func() { + var slsession *sessionfakes.FakeSLSession + var options *sl.Options + BeforeEach(func() { + limit := 10 + slsession = &sessionfakes.FakeSLSession{} + options = &sl.Options{ + Mask: "mask[id,hostname]", + Filter: "", + Limit: &limit, + } + }) + + Context("GetVirtualGuestsIter Tests", func() { + + It("API call made properly", func() { + _, err := virtual.GetVirtualGuestsIter(slsession, options) + Expect(err).ToNot(HaveOccurred()) + Expect(slsession.DoRequestCallCount()).To(Equal(1)) + }) + }) +}) diff --git a/session/rest.go b/session/rest.go index 856a475..7556fdc 100644 --- a/session/rest.go +++ b/session/rest.go @@ -283,9 +283,19 @@ func makeHTTPRequest( defer resp.Body.Close() responseBody, err := ioutil.ReadAll(resp.Body) + if err != nil { return nil, resp.StatusCode, err } + if resp.Header["Softlayer-Total-Items"] != nil && len(resp.Header["Softlayer-Total-Items"]) == 1 { + var str_err error + var total_items int + total_items, str_err = strconv.Atoi(resp.Header["Softlayer-Total-Items"][0]) + if str_err != nil { + log.Println("[Error] Unable to convert Softlayer-Total-Items to int: ", str_err) + } + options.SetTotalItems(total_items) + } if session.Debug { log.Println("[DEBUG] Status Code: ", resp.StatusCode) diff --git a/session/session.go b/session/session.go index 6b3697c..49e43de 100644 --- a/session/session.go +++ b/session/session.go @@ -262,7 +262,11 @@ func (r *Session) DoRequest(service string, method string, args []interface{}, o r.TransportHandler = getDefaultTransport(r.Endpoint) } - return r.TransportHandler.DoRequest(r, service, method, args, options, pResult) + err := r.TransportHandler.DoRequest(r, service, method, args, options, pResult) + if err != nil { + return err + } + return err } // SetTimeout creates a copy of the session and sets the passed timeout into it diff --git a/sl/errors_test.go b/sl/errors_test.go new file mode 100644 index 0000000..a917b13 --- /dev/null +++ b/sl/errors_test.go @@ -0,0 +1,37 @@ +package sl_test + +import ( + "errors" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/softlayer/softlayer-go/sl" +) + +var _ = Describe("Error Tests", func() { + var sl_err sl.Error + var base_err error + Context("Error Tests", func() { + BeforeEach(func() { + base_err = errors.New("TTTTT") + sl_err = sl.Error{ + StatusCode: 0, + Exception: "", + Message: "", + Wrapped: base_err, + } + }) + It("Basic Error Tests", func() { + Expect(sl_err.Error()).To(Equal("TTTTT")) + sl_err.Wrapped = nil + Expect(sl_err.Error()).To(Equal("")) + sl_err.Exception = "AAA" + Expect(sl_err.Error()).To(Equal("AAA: ")) + sl_err.Message = "BBB" + Expect(sl_err.Error()).To(Equal("AAA: BBB ")) + sl_err.StatusCode = 99 + Expect(sl_err.Error()).To(Equal("AAA: BBB (HTTP 99)")) + }) + + }) +}) diff --git a/sl/options.go b/sl/options.go index 5d1dd0a..d0d2b17 100644 --- a/sl/options.go +++ b/sl/options.go @@ -16,12 +16,45 @@ package sl -// Options contains the individual query parameters that can be applied to -// a request. +import ( + "math" +) + +var DefaultLimit = 50 + +// Options contains the individual query parameters that can be applied to a request. type Options struct { - Id *int - Mask string - Filter string - Limit *int - Offset *int + Id *int + Mask string + Filter string + Limit *int + Offset *int + TotalItems int +} + +// returns Math.Ciel((TotalItems - Limit) / Limit) +func (opt *Options) GetRemainingAPICalls() int { + Total := float64(opt.TotalItems) + Limit := float64(*opt.Limit) + return int(math.Ceil((Total - Limit) / Limit)) +} + +// Makes sure the limit is set to something, not 0 or 1. Will set to default if no other limit is set. +func (opt *Options) ValidateLimit() int { + if opt.Limit == nil || *opt.Limit < DefaultLimit { + opt.Limit = &DefaultLimit + } + return *opt.Limit +} + +func (opt *Options) SetTotalItems(total int) { + opt.TotalItems = total +} + +func (opt *Options) SetOffset(offset int) { + opt.Offset = &offset +} + +func (opt *Options) SetLimit(limit int) { + opt.Limit = &limit } diff --git a/sl/options_test.go b/sl/options_test.go new file mode 100644 index 0000000..a19eb62 --- /dev/null +++ b/sl/options_test.go @@ -0,0 +1,64 @@ +package sl_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/softlayer/softlayer-go/sl" +) + +var _ = Describe("Options Tests", func() { + var sl_limit int + var sl_id int + var sl_mask string + var sl_filter string + var sl_offset int + var sl_totalItems int + Context("Option Setter Testing", func() { + BeforeEach(func() { + sl_limit = 10 + sl_id = 99999 + sl_mask = "mask[id, hostname]" + sl_filter = `{"test":{"operation":"ok"}}` + sl_offset = 0 + sl_totalItems = 0 + }) + It("Test GetRemainingAPICalls", func() { + options := sl.Options{ + TotalItems: 1000, + Limit: &sl_limit, + } + result := options.GetRemainingAPICalls() + Expect(result).To(Equal(99)) + }) + It("Test ValidateLimit", func() { + options := sl.Options{} + limit := options.ValidateLimit() + Expect(limit).To(Equal(sl.DefaultLimit)) + Expect(*options.Limit).To(Equal(sl.DefaultLimit)) + options.SetLimit(123) + Expect(*options.Limit).To(Equal(123)) + }) + It("Setter Tests", func() { + options := sl.Options{} + options.SetTotalItems(44) + Expect(options.TotalItems).To(Equal(44)) + options.SetOffset(33) + Expect(*options.Offset).To(Equal(33)) + options.SetLimit(22) + Expect(*options.Limit).To(Equal(22)) + }) + It("Basic Setting Tests", func() { + options := sl.Options{ + Id: &sl_id, + Mask: sl_mask, + Filter: sl_filter, + Limit: &sl_limit, + Offset: &sl_offset, + TotalItems: sl_totalItems, + } + Expect(options.Limit).To(Equal(&sl_limit)) + }) + + }) +}) diff --git a/sl/sl_test.go b/sl/sl_test.go new file mode 100644 index 0000000..a971cea --- /dev/null +++ b/sl/sl_test.go @@ -0,0 +1,13 @@ +package sl_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "testing" +) + +func TestServices(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "SL Package Tests") +} diff --git a/sl/version_test.go b/sl/version_test.go new file mode 100644 index 0000000..f1987f5 --- /dev/null +++ b/sl/version_test.go @@ -0,0 +1,29 @@ +package sl_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/softlayer/softlayer-go/sl" +) + +var _ = Describe("Version Tests", func() { + Context("Get Version Tests", func() { + It("Setting Version", func() { + version := sl.VersionInfo{ + Major: 1, Minor: 2, Patch: 3, Pre: "", + } + Expect(version.String()).To(Equal("v1.2.3")) + }) + It("Pre Version", func() { + version := sl.VersionInfo{ + Major: 1, Minor: 2, Patch: 3, Pre: "a", + } + Expect(version.String()).To(Equal("v1.2.3-a")) + }) + It("Base Version", func() { + version := sl.Version + Expect(version.String()).Should(HavePrefix("v")) + }) + }) +})