From 8245194df633c392aa0e1f56efeec73d87737165 Mon Sep 17 00:00:00 2001 From: Heng Hu Date: Thu, 7 Dec 2017 02:10:15 -0800 Subject: [PATCH 1/3] Pagination for /wallet/addresses and /wallet/transactions --- node/api/api.go | 22 +++++++++++++++++ node/api/wallet.go | 61 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/node/api/api.go b/node/api/api.go index a31a2ce994..5b39bd9f20 100644 --- a/node/api/api.go +++ b/node/api/api.go @@ -4,11 +4,18 @@ import ( "encoding/json" "net/http" "strings" + "reflect" + "math" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" ) +const ( + // size of each page for paginated API responses + PAGINATION_SIZE = 25 +) + // Error is a type that is encoded as JSON and returned in an API response in // the event of an error. Only the Message field is required. More fields may // be added to this struct in the future for better error reporting. @@ -162,3 +169,18 @@ func WriteJSON(w http.ResponseWriter, obj interface{}) { func WriteSuccess(w http.ResponseWriter) { w.WriteHeader(http.StatusNoContent) } + +// returns the starting index, end index, and total pages of a given slice for the page +func GetPaginatedSliceIndicesAndTotalPages(sliceInterface interface{}, page int) (int, int, int) { + slice := reflect.ValueOf(sliceInterface) + if slice.Kind() != reflect.Slice { + build.Critical("attempting to paginate on non-slice object type: ", slice.Kind()) + } + startingPageIndex := int(math.Min(float64(page * PAGINATION_SIZE), float64(slice.Len()))) + endingPageIndex := int(math.Min(float64(startingPageIndex + PAGINATION_SIZE), float64(slice.Len()))); + totalPages := slice.Len() / PAGINATION_SIZE + if math.Mod(float64(slice.Len()), float64(PAGINATION_SIZE)) != 0 { + totalPages += 1 + } + return startingPageIndex, endingPageIndex, totalPages +} diff --git a/node/api/wallet.go b/node/api/wallet.go index 3edeb92103..1637642043 100644 --- a/node/api/wallet.go +++ b/node/api/wallet.go @@ -45,6 +45,11 @@ type ( Addresses []types.UnlockHash `json:"addresses"` } + WalletAddressesPaginatedGET struct { + Addresses []types.UnlockHash `json:"addresses"` + TotalPages int `json:"total_pages"` + } + // WalletInitPOST contains the primary seed that gets generated during a // POST call to /wallet/init. WalletInitPOST struct { @@ -90,6 +95,13 @@ type ( UnconfirmedTransactions []modules.ProcessedTransaction `json:"unconfirmedtransactions"` } + WalletTransactionsPaginatedGET struct { + ConfirmedTransactions []modules.ProcessedTransaction `json:"confirmedtransactions"` + UnconfirmedTransactions []modules.ProcessedTransaction `json:"unconfirmedtransactions"` + TotalConfirmedPages int `json:"total_confirmed_pages"` + TotalUnConfirmedPages int `json:"total_unconfirmed_pages"` + } + // WalletTransactionsGETaddr contains the set of wallet transactions // relevant to the input address provided in the call to // /wallet/transaction/:addr @@ -179,9 +191,25 @@ func (api *API) walletAddressHandler(w http.ResponseWriter, req *http.Request, _ // walletAddressHandler handles API calls to /wallet/addresses. func (api *API) walletAddressesHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { - WriteJSON(w, WalletAddressesGET{ - Addresses: api.wallet.AllAddresses(), - }) + pageString := req.FormValue("page") + allAddresses := api.wallet.AllAddresses() + if pageString == "" || allAddresses == nil { + WriteJSON(w, WalletAddressesGET{ + Addresses: allAddresses, + }) + } else { + page, err := strconv.Atoi(pageString) + if err != nil || page < 0 { + WriteError(w, Error{"page query must be a nonnegative integer"}, http.StatusBadRequest) + return + } + startPageIdx, endPageIndex, totalPages := GetPaginatedSliceIndicesAndTotalPages(allAddresses, page) + addresses := allAddresses[startPageIdx:endPageIndex] + WriteJSON(w, WalletAddressesPaginatedGET{ + Addresses: addresses, + TotalPages: totalPages, + }) + } } // walletBackupHandler handles API calls to /wallet/backup. @@ -520,11 +548,28 @@ func (api *API) walletTransactionsHandler(w http.ResponseWriter, req *http.Reque return } unconfirmedTxns := api.wallet.UnconfirmedTransactions() - - WriteJSON(w, WalletTransactionsGET{ - ConfirmedTransactions: confirmedTxns, - UnconfirmedTransactions: unconfirmedTxns, - }) + confirmedPageString, unconfirmedPageString := req.FormValue("confirmPage"), req.FormValue("unconfirmedPage") + if confirmedPageString == "" || unconfirmedPageString == "" || confirmedTxns == nil || unconfirmedTxns == nil { + WriteJSON(w, WalletTransactionsGET{ + ConfirmedTransactions: confirmedTxns, + UnconfirmedTransactions: unconfirmedTxns, + }) + } else { + confirmedPage, confirmedErr := strconv.Atoi(confirmedPageString) + uncnfrmedPage, uncnfrmedErr := strconv.Atoi(unconfirmedPageString) + if confirmedErr != nil || uncnfrmedErr != nil || confirmedPage < 0 || uncnfrmedPage < 0 { + WriteError(w, Error{"page query must be an nonnegative integer"}, http.StatusBadRequest) + return + } + startConfirmPageIdx, endConfirmPageIdx, totalConfirmPages := GetPaginatedSliceIndicesAndTotalPages(confirmedTxns, confirmedPage) + startUncnfrmPageIdx, endUncnfrmPageIdx, totalUncnfrmPages := GetPaginatedSliceIndicesAndTotalPages(unconfirmedTxns, uncnfrmedPage) + WriteJSON(w, WalletTransactionsPaginatedGET{ + ConfirmedTransactions: confirmedTxns[startConfirmPageIdx:endConfirmPageIdx], + UnconfirmedTransactions: unconfirmedTxns[startUncnfrmPageIdx:endUncnfrmPageIdx], + TotalConfirmedPages: totalConfirmPages, + TotalUnConfirmedPages: totalUncnfrmPages, + }) + } } // walletTransactionsAddrHandler handles API calls to From 5db8549d4bcd8b20ab3e32df97c0f5c6334d53ac Mon Sep 17 00:00:00 2001 From: Heng Hu Date: Thu, 7 Dec 2017 23:18:35 -0800 Subject: [PATCH 2/3] start and limit queries --- node/api/api.go | 93 ++++++++++++++++++++++++++++++++++++++++++---- node/api/wallet.go | 46 +++++++++++------------ 2 files changed, 109 insertions(+), 30 deletions(-) diff --git a/node/api/api.go b/node/api/api.go index 5b39bd9f20..0e532567a0 100644 --- a/node/api/api.go +++ b/node/api/api.go @@ -6,6 +6,7 @@ import ( "strings" "reflect" "math" + "strconv" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" @@ -13,7 +14,7 @@ import ( const ( // size of each page for paginated API responses - PAGINATION_SIZE = 25 + DEFAULT_PAGINATION_SIZE = 25 ) // Error is a type that is encoded as JSON and returned in an API response in @@ -36,6 +37,25 @@ type Error struct { // be valid or invalid depending on the current state of a module. } +type PaginationRequest struct { + HasPaginationQuery bool + PaginationQueryIsValid bool + Start int + Limit int +} + +type PaginationResponse struct { + Start int `json:"start"` + Limit int `json:"limit"` + TotalPages int `json:"total_pages"` +} + +type PaginationWrapper struct { + Start int + End int + TotalPages int +} + // Error implements the error interface for the Error type. It returns only the // Message field. func (err Error) Error() string { @@ -170,17 +190,76 @@ func WriteSuccess(w http.ResponseWriter) { w.WriteHeader(http.StatusNoContent) } +func GetPaginationDefaultRequest(req *http.Request) PaginationRequest { + return GetPaginationRequest(req, "start", "limit") +} + +func GetPaginationRequest(req *http.Request, startQueryParam string, limitQueryParam string) PaginationRequest { + startString := req.FormValue(startQueryParam) + if startString == "" { + return PaginationRequest { + HasPaginationQuery: false, + PaginationQueryIsValid: false, + Start: 0, + Limit: 0, + } + } + startIndex, err := strconv.Atoi(startString) + if err != nil || startIndex < 0 { + return PaginationRequest { + HasPaginationQuery: true, + PaginationQueryIsValid: false, + Start: 0, + Limit: 0, + } + } + limit:= DEFAULT_PAGINATION_SIZE + limitString := req.FormValue(limitQueryParam) + if limitString != "" { + limit, err = strconv.Atoi(limitString) + if err != nil || limit <= 0 { + return PaginationRequest { + HasPaginationQuery: true, + PaginationQueryIsValid: false, + Start: 0, + Limit: 0, + } + } + } + return PaginationRequest { + HasPaginationQuery: true, + PaginationQueryIsValid: true, + Start: startIndex, + Limit: limit, + } +} + // returns the starting index, end index, and total pages of a given slice for the page -func GetPaginatedSliceIndicesAndTotalPages(sliceInterface interface{}, page int) (int, int, int) { +func GetPaginationIndicesAndResponse(sliceInterface interface{}, paginationRequest PaginationRequest) (PaginationWrapper, PaginationResponse) { + if paginationRequest.Limit <= 0 { + build.Critical("pagination request limit must be greater than 0") + } slice := reflect.ValueOf(sliceInterface) if slice.Kind() != reflect.Slice { build.Critical("attempting to paginate on non-slice object type: ", slice.Kind()) } - startingPageIndex := int(math.Min(float64(page * PAGINATION_SIZE), float64(slice.Len()))) - endingPageIndex := int(math.Min(float64(startingPageIndex + PAGINATION_SIZE), float64(slice.Len()))); - totalPages := slice.Len() / PAGINATION_SIZE - if math.Mod(float64(slice.Len()), float64(PAGINATION_SIZE)) != 0 { + startingPageIndex := int(math.Min(float64(paginationRequest.Start), float64(slice.Len()))) + endingPageIndex := int(math.Min(float64(paginationRequest.Start + paginationRequest.Limit), float64(slice.Len()))); + totalPages := slice.Len() / paginationRequest.Limit + if math.Mod(float64(slice.Len()), float64(paginationRequest.Limit)) != 0 { totalPages += 1 } - return startingPageIndex, endingPageIndex, totalPages + + pagination := PaginationWrapper { + Start: startingPageIndex, + End: endingPageIndex, + TotalPages: totalPages, + } + + paginationResponse := PaginationResponse { + Start: startingPageIndex, + Limit: paginationRequest.Limit, + TotalPages: totalPages, + } + return pagination, paginationResponse } diff --git a/node/api/wallet.go b/node/api/wallet.go index 1637642043..15a880152f 100644 --- a/node/api/wallet.go +++ b/node/api/wallet.go @@ -47,7 +47,7 @@ type ( WalletAddressesPaginatedGET struct { Addresses []types.UnlockHash `json:"addresses"` - TotalPages int `json:"total_pages"` + Pagination PaginationResponse `json:"pagination"` } // WalletInitPOST contains the primary seed that gets generated during a @@ -98,8 +98,8 @@ type ( WalletTransactionsPaginatedGET struct { ConfirmedTransactions []modules.ProcessedTransaction `json:"confirmedtransactions"` UnconfirmedTransactions []modules.ProcessedTransaction `json:"unconfirmedtransactions"` - TotalConfirmedPages int `json:"total_confirmed_pages"` - TotalUnConfirmedPages int `json:"total_unconfirmed_pages"` + ConfirmedTransactionsPagination PaginationResponse `json:"confirmed_transactions_pagination"` + UnconfirmedTransactionsPagination PaginationResponse `json:"unconfirmed_transactions_pagination"` } // WalletTransactionsGETaddr contains the set of wallet transactions @@ -191,23 +191,22 @@ func (api *API) walletAddressHandler(w http.ResponseWriter, req *http.Request, _ // walletAddressHandler handles API calls to /wallet/addresses. func (api *API) walletAddressesHandler(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { - pageString := req.FormValue("page") + paginationRequest := GetPaginationDefaultRequest(req) allAddresses := api.wallet.AllAddresses() - if pageString == "" || allAddresses == nil { + if !paginationRequest.HasPaginationQuery || allAddresses == nil { WriteJSON(w, WalletAddressesGET{ Addresses: allAddresses, }) } else { - page, err := strconv.Atoi(pageString) - if err != nil || page < 0 { - WriteError(w, Error{"page query must be a nonnegative integer"}, http.StatusBadRequest) + if !paginationRequest.PaginationQueryIsValid { + WriteError(w, Error{"start and limit queries must be nonnegative integers"}, http.StatusBadRequest) return } - startPageIdx, endPageIndex, totalPages := GetPaginatedSliceIndicesAndTotalPages(allAddresses, page) - addresses := allAddresses[startPageIdx:endPageIndex] + pagination, paginationResponse := GetPaginationIndicesAndResponse(allAddresses, paginationRequest) + addresses := allAddresses[pagination.Start:pagination.End] WriteJSON(w, WalletAddressesPaginatedGET{ Addresses: addresses, - TotalPages: totalPages, + Pagination: paginationResponse, }) } } @@ -548,26 +547,27 @@ func (api *API) walletTransactionsHandler(w http.ResponseWriter, req *http.Reque return } unconfirmedTxns := api.wallet.UnconfirmedTransactions() - confirmedPageString, unconfirmedPageString := req.FormValue("confirmPage"), req.FormValue("unconfirmedPage") - if confirmedPageString == "" || unconfirmedPageString == "" || confirmedTxns == nil || unconfirmedTxns == nil { + + confirmedPaginationRequest := GetPaginationRequest(req, "confirmedTransactionsStart", "confirmedTransactionsLimit") + unconfirmedPaginationRequest := GetPaginationRequest(req, "unconfirmedTransactionsStart", "unconfirmedTransactionsLimit") + + if !confirmedPaginationRequest.HasPaginationQuery || !unconfirmedPaginationRequest.HasPaginationQuery || confirmedTxns == nil || unconfirmedTxns == nil { WriteJSON(w, WalletTransactionsGET{ ConfirmedTransactions: confirmedTxns, UnconfirmedTransactions: unconfirmedTxns, }) } else { - confirmedPage, confirmedErr := strconv.Atoi(confirmedPageString) - uncnfrmedPage, uncnfrmedErr := strconv.Atoi(unconfirmedPageString) - if confirmedErr != nil || uncnfrmedErr != nil || confirmedPage < 0 || uncnfrmedPage < 0 { - WriteError(w, Error{"page query must be an nonnegative integer"}, http.StatusBadRequest) + if !confirmedPaginationRequest.PaginationQueryIsValid || !unconfirmedPaginationRequest.PaginationQueryIsValid { + WriteError(w, Error{"start and limit queries must be nonnegative integers"}, http.StatusBadRequest) return } - startConfirmPageIdx, endConfirmPageIdx, totalConfirmPages := GetPaginatedSliceIndicesAndTotalPages(confirmedTxns, confirmedPage) - startUncnfrmPageIdx, endUncnfrmPageIdx, totalUncnfrmPages := GetPaginatedSliceIndicesAndTotalPages(unconfirmedTxns, uncnfrmedPage) + confirmedPagination, confirmedPaginationResponse := GetPaginationIndicesAndResponse(confirmedTxns, confirmedPaginationRequest) + unconfirmedPagination, unconfirmedPaginationResponse := GetPaginationIndicesAndResponse(unconfirmedTxns, unconfirmedPaginationRequest) WriteJSON(w, WalletTransactionsPaginatedGET{ - ConfirmedTransactions: confirmedTxns[startConfirmPageIdx:endConfirmPageIdx], - UnconfirmedTransactions: unconfirmedTxns[startUncnfrmPageIdx:endUncnfrmPageIdx], - TotalConfirmedPages: totalConfirmPages, - TotalUnConfirmedPages: totalUncnfrmPages, + ConfirmedTransactions: confirmedTxns[confirmedPagination.Start:confirmedPagination.End], + UnconfirmedTransactions: unconfirmedTxns[unconfirmedPagination.Start:unconfirmedPagination.End], + ConfirmedTransactionsPagination: confirmedPaginationResponse, + UnconfirmedTransactionsPagination: unconfirmedPaginationResponse, }) } } From 804e2e5385f66e68ff7b3fb47468246cbcec71b4 Mon Sep 17 00:00:00 2001 From: Heng Hu Date: Sat, 24 Feb 2018 17:03:15 -0800 Subject: [PATCH 3/3] go fmt --- node/api/api.go | 61 +++++++++++++++++++++------------------------- node/api/wallet.go | 20 +++++++-------- types/constants.go | 3 +++ 3 files changed, 41 insertions(+), 43 deletions(-) diff --git a/node/api/api.go b/node/api/api.go index 0e532567a0..51303684df 100644 --- a/node/api/api.go +++ b/node/api/api.go @@ -2,21 +2,16 @@ package api import ( "encoding/json" + "math" "net/http" - "strings" "reflect" - "math" "strconv" + "strings" "github.com/NebulousLabs/Sia/build" "github.com/NebulousLabs/Sia/modules" ) -const ( - // size of each page for paginated API responses - DEFAULT_PAGINATION_SIZE = 25 -) - // Error is a type that is encoded as JSON and returned in an API response in // the event of an error. Only the Message field is required. More fields may // be added to this struct in the future for better error reporting. @@ -38,22 +33,22 @@ type Error struct { } type PaginationRequest struct { - HasPaginationQuery bool - PaginationQueryIsValid bool - Start int - Limit int + HasPaginationQuery bool + PaginationQueryIsValid bool + Start int + Limit int } type PaginationResponse struct { - Start int `json:"start"` - Limit int `json:"limit"` - TotalPages int `json:"total_pages"` + Start int `json:"start"` + Limit int `json:"limit"` + TotalPages int `json:"total_pages"` } type PaginationWrapper struct { - Start int - End int - TotalPages int + Start int + End int + TotalPages int } // Error implements the error interface for the Error type. It returns only the @@ -197,8 +192,8 @@ func GetPaginationDefaultRequest(req *http.Request) PaginationRequest { func GetPaginationRequest(req *http.Request, startQueryParam string, limitQueryParam string) PaginationRequest { startString := req.FormValue(startQueryParam) if startString == "" { - return PaginationRequest { - HasPaginationQuery: false, + return PaginationRequest{ + HasPaginationQuery: false, PaginationQueryIsValid: false, Start: 0, Limit: 0, @@ -206,28 +201,28 @@ func GetPaginationRequest(req *http.Request, startQueryParam string, limitQueryP } startIndex, err := strconv.Atoi(startString) if err != nil || startIndex < 0 { - return PaginationRequest { - HasPaginationQuery: true, + return PaginationRequest{ + HasPaginationQuery: true, PaginationQueryIsValid: false, Start: 0, Limit: 0, } } - limit:= DEFAULT_PAGINATION_SIZE + limit := DefaultPaginationSize limitString := req.FormValue(limitQueryParam) if limitString != "" { limit, err = strconv.Atoi(limitString) if err != nil || limit <= 0 { - return PaginationRequest { - HasPaginationQuery: true, + return PaginationRequest{ + HasPaginationQuery: true, PaginationQueryIsValid: false, Start: 0, Limit: 0, } } } - return PaginationRequest { - HasPaginationQuery: true, + return PaginationRequest{ + HasPaginationQuery: true, PaginationQueryIsValid: true, Start: startIndex, Limit: limit, @@ -244,21 +239,21 @@ func GetPaginationIndicesAndResponse(sliceInterface interface{}, paginationReque build.Critical("attempting to paginate on non-slice object type: ", slice.Kind()) } startingPageIndex := int(math.Min(float64(paginationRequest.Start), float64(slice.Len()))) - endingPageIndex := int(math.Min(float64(paginationRequest.Start + paginationRequest.Limit), float64(slice.Len()))); + endingPageIndex := int(math.Min(float64(paginationRequest.Start+paginationRequest.Limit), float64(slice.Len()))) totalPages := slice.Len() / paginationRequest.Limit if math.Mod(float64(slice.Len()), float64(paginationRequest.Limit)) != 0 { totalPages += 1 } - pagination := PaginationWrapper { - Start: startingPageIndex, - End: endingPageIndex, + pagination := PaginationWrapper{ + Start: startingPageIndex, + End: endingPageIndex, TotalPages: totalPages, } - paginationResponse := PaginationResponse { - Start: startingPageIndex, - Limit: paginationRequest.Limit, + paginationResponse := PaginationResponse{ + Start: startingPageIndex, + Limit: paginationRequest.Limit, TotalPages: totalPages, } return pagination, paginationResponse diff --git a/node/api/wallet.go b/node/api/wallet.go index 15a880152f..540764a13b 100644 --- a/node/api/wallet.go +++ b/node/api/wallet.go @@ -46,8 +46,8 @@ type ( } WalletAddressesPaginatedGET struct { - Addresses []types.UnlockHash `json:"addresses"` - Pagination PaginationResponse `json:"pagination"` + Addresses []types.UnlockHash `json:"addresses"` + Pagination PaginationResponse `json:"pagination"` } // WalletInitPOST contains the primary seed that gets generated during a @@ -96,10 +96,10 @@ type ( } WalletTransactionsPaginatedGET struct { - ConfirmedTransactions []modules.ProcessedTransaction `json:"confirmedtransactions"` - UnconfirmedTransactions []modules.ProcessedTransaction `json:"unconfirmedtransactions"` - ConfirmedTransactionsPagination PaginationResponse `json:"confirmed_transactions_pagination"` - UnconfirmedTransactionsPagination PaginationResponse `json:"unconfirmed_transactions_pagination"` + ConfirmedTransactions []modules.ProcessedTransaction `json:"confirmedtransactions"` + UnconfirmedTransactions []modules.ProcessedTransaction `json:"unconfirmedtransactions"` + ConfirmedTransactionsPagination PaginationResponse `json:"confirmed_transactions_pagination"` + UnconfirmedTransactionsPagination PaginationResponse `json:"unconfirmed_transactions_pagination"` } // WalletTransactionsGETaddr contains the set of wallet transactions @@ -205,7 +205,7 @@ func (api *API) walletAddressesHandler(w http.ResponseWriter, req *http.Request, pagination, paginationResponse := GetPaginationIndicesAndResponse(allAddresses, paginationRequest) addresses := allAddresses[pagination.Start:pagination.End] WriteJSON(w, WalletAddressesPaginatedGET{ - Addresses: addresses, + Addresses: addresses, Pagination: paginationResponse, }) } @@ -564,9 +564,9 @@ func (api *API) walletTransactionsHandler(w http.ResponseWriter, req *http.Reque confirmedPagination, confirmedPaginationResponse := GetPaginationIndicesAndResponse(confirmedTxns, confirmedPaginationRequest) unconfirmedPagination, unconfirmedPaginationResponse := GetPaginationIndicesAndResponse(unconfirmedTxns, unconfirmedPaginationRequest) WriteJSON(w, WalletTransactionsPaginatedGET{ - ConfirmedTransactions: confirmedTxns[confirmedPagination.Start:confirmedPagination.End], - UnconfirmedTransactions: unconfirmedTxns[unconfirmedPagination.Start:unconfirmedPagination.End], - ConfirmedTransactionsPagination: confirmedPaginationResponse, + ConfirmedTransactions: confirmedTxns[confirmedPagination.Start:confirmedPagination.End], + UnconfirmedTransactions: unconfirmedTxns[unconfirmedPagination.Start:unconfirmedPagination.End], + ConfirmedTransactionsPagination: confirmedPaginationResponse, UnconfirmedTransactionsPagination: unconfirmedPaginationResponse, }) } diff --git a/types/constants.go b/types/constants.go index a00f7d5004..9814da0228 100644 --- a/types/constants.go +++ b/types/constants.go @@ -49,6 +49,9 @@ var ( SiafundCount = NewCurrency64(10000) SiafundPortion = big.NewRat(39, 1000) TargetWindow BlockHeight + + // size of each page for paginated API responses + DefaultPaginationSize = 50 ) // init checks which build constant is in place and initializes the variables