From c6072ab50f90818a3555d0069c3ca7fce3433e25 Mon Sep 17 00:00:00 2001 From: Antoon P Date: Wed, 17 May 2023 14:41:04 +0200 Subject: [PATCH 01/48] Provide total transfer size with the datatx protocol (#3891) * Provide total file size with the datatx protocol * Lint fix --------- Co-authored-by: Antoon P --- changelog/unreleased/datatx-transfer-size.md | 4 +++ .../ocmshareprovider/ocmshareprovider.go | 33 +++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/datatx-transfer-size.md diff --git a/changelog/unreleased/datatx-transfer-size.md b/changelog/unreleased/datatx-transfer-size.md new file mode 100644 index 0000000000..7820fc7b8b --- /dev/null +++ b/changelog/unreleased/datatx-transfer-size.md @@ -0,0 +1,4 @@ +Enhancement: Provide data transfer size with datatx share + +https://github.com/cs3org/reva/pull/3891 +https://github.com/cs3org/reva/issues/2104 \ No newline at end of file diff --git a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go index 37b0307545..88e11393d9 100644 --- a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go +++ b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go @@ -44,6 +44,7 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/storage/utils/walker" "github.com/cs3org/reva/pkg/utils" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" @@ -71,6 +72,7 @@ type service struct { client *client.OCMClient gateway gateway.GatewayAPIClient webappTmpl *template.Template + walker walker.Walker } func (c *config) init() { @@ -134,6 +136,7 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { if err != nil { return nil, err } + walker := walker.NewWalker(gateway) service := &service{ conf: c, @@ -141,6 +144,7 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { client: client, gateway: gateway, webappTmpl: tpl, + walker: walker, } return service, nil @@ -210,13 +214,38 @@ func (s *service) getWebappProtocol(share *ocm.Share) *ocmd.Webapp { } func (s *service) getDataTransferProtocol(ctx context.Context, share *ocm.Share) *ocmd.Datatx { - // TODO discover the size + var size uint64 + // get the path of the share + statRes, err := s.gateway.Stat(ctx, &providerpb.StatRequest{ + Ref: &providerpb.Reference{ + ResourceId: share.ResourceId, + }, + }) + if err != nil { + panic(err) + } + + path := statRes.GetInfo().Path + err = s.walk(ctx, path, func(path string, info *providerpb.ResourceInfo, err error) error { + if info.Type == providerpb.ResourceType_RESOURCE_TYPE_FILE { + size += info.Size + } + return nil + }) + if err != nil { + panic(err) + } return &ocmd.Datatx{ SourceURI: s.webdavURL(ctx, share), - Size: 0, + Size: size, } } +// walk traverses the path recursively to discover all resources in the tree. +func (s *service) walk(ctx context.Context, path string, fn walker.WalkFunc) error { + return s.walker.Walk(ctx, path, fn) +} + func (s *service) getProtocols(ctx context.Context, share *ocm.Share) ocmd.Protocols { var p ocmd.Protocols for _, m := range share.AccessMethods { From 9ae4a9341f4c9c825ed101f79d1eafee75bd4f9e Mon Sep 17 00:00:00 2001 From: Antoon P Date: Mon, 22 May 2023 09:45:04 +0200 Subject: [PATCH 02/48] Remove job on cancel transfer (#3882) * Inform in case the share has already been accepted. * Add transfer jobs remove conf option on transfer cancel (#3881) * Add conf flag for removing transfer jobs on cancel * Rename remove on cancel conf flags * Rename datatx tutorial and add conf changes 'Cleanup transfers' section --------- Co-authored-by: Antoon P --- .../remove-job-on-cancel-transfer.md | 4 ++ ...ransfer-tutorial.md => datatx-tutorial.md} | 7 ++- examples/datatx/datatx.toml | 6 +++ internal/grpc/services/datatx/datatx.go | 10 ++-- .../grpc/services/gateway/ocmshareprovider.go | 13 ++++- pkg/datatx/manager/rclone/rclone.go | 47 ++++++++++++------- 6 files changed, 62 insertions(+), 25 deletions(-) create mode 100644 changelog/unreleased/remove-job-on-cancel-transfer.md rename docs/content/en/docs/tutorials/{data-transfer-tutorial.md => datatx-tutorial.md} (97%) diff --git a/changelog/unreleased/remove-job-on-cancel-transfer.md b/changelog/unreleased/remove-job-on-cancel-transfer.md new file mode 100644 index 0000000000..2063fea34c --- /dev/null +++ b/changelog/unreleased/remove-job-on-cancel-transfer.md @@ -0,0 +1,4 @@ +Bugfix: Remove transfer on cancel should also remove transfer job + +https://github.com/cs3org/reva/pull/3882 +https://github.com/cs3org/reva/issues/3881 \ No newline at end of file diff --git a/docs/content/en/docs/tutorials/data-transfer-tutorial.md b/docs/content/en/docs/tutorials/datatx-tutorial.md similarity index 97% rename from docs/content/en/docs/tutorials/data-transfer-tutorial.md rename to docs/content/en/docs/tutorials/datatx-tutorial.md index 48aa76d94a..2da5a5ddf6 100644 --- a/docs/content/en/docs/tutorials/data-transfer-tutorial.md +++ b/docs/content/en/docs/tutorials/datatx-tutorial.md @@ -158,9 +158,12 @@ transfer-retry -txId fe671ae3-0fbf-4b06-b7df-32418c2ebfcb ``` ## 6 Cleanup transfers -Transfers will be removed from the db using the `transfer-cancel` command when the configuration property `remove_on_cancel` of the datatx service has been set to `true` as follows: +Transfers will be removed from the db using the `transfer-cancel` command when the configuration property `remove_transfer_on_cancel` and `remove_transfer_job_on_cancel` of the datatx service and rclone driver respectively have been set to `true` as follows: ``` [grpc.services.datatx] -remove_on_cancel = true +remove_transfer_on_cancel = true + +[grpc.services.datatx.txdrivers.rclone] +remove_transfer_job_on_cancel = true ``` Currently this setting is recommended. \ No newline at end of file diff --git a/examples/datatx/datatx.toml b/examples/datatx/datatx.toml index bb19821f9e..2a4633758e 100644 --- a/examples/datatx/datatx.toml +++ b/examples/datatx/datatx.toml @@ -13,6 +13,9 @@ txdriver = "rclone" tx_shares_file = "" # base folder of the data transfers (eg. /home/DataTransfers) data_transfers_folder = "" +# if set to 'true' the transfer will always be removed from the db upon cancel request +# recommended value is true +remove_transfer_on_cancel = true # rclone driver [grpc.services.datatx.txdrivers.rclone] @@ -33,6 +36,9 @@ file = "" job_status_check_interval = 2000 # the job timeout in milliseconds (must be long enough for big transfers!) job_timeout = 120000 +# if set to 'true' the transfer job will always be removed from the db upon transfer cancel request +# recommended value is true +remove_transfer_job_on_cancel = true [http.services.ocdav] # reva supports http third party copy diff --git a/internal/grpc/services/datatx/datatx.go b/internal/grpc/services/datatx/datatx.go index b3ac0b4398..e2be50749b 100644 --- a/internal/grpc/services/datatx/datatx.go +++ b/internal/grpc/services/datatx/datatx.go @@ -46,10 +46,10 @@ type config struct { TxDriver string `mapstructure:"txdriver"` TxDrivers map[string]map[string]interface{} `mapstructure:"txdrivers"` // storage driver to persist share/transfer relation - StorageDriver string `mapstructure:"storage_driver"` - StorageDrivers map[string]map[string]interface{} `mapstructure:"storage_drivers"` - TxSharesFile string `mapstructure:"tx_shares_file"` - RemoveOnCancel bool `mapstructure:"remove_on_cancel"` + StorageDriver string `mapstructure:"storage_driver"` + StorageDrivers map[string]map[string]interface{} `mapstructure:"storage_drivers"` + TxSharesFile string `mapstructure:"tx_shares_file"` + RemoveTransferOnCancel bool `mapstructure:"remove_transfer_on_cancel"` } type service struct { @@ -209,7 +209,7 @@ func (s *service) CancelTransfer(ctx context.Context, req *datatx.CancelTransfer } transferRemovedMessage := "" - if s.conf.RemoveOnCancel { + if s.conf.RemoveTransferOnCancel { delete(s.txShareDriver.model.TxShares, req.TxId.GetOpaqueId()) if err := s.txShareDriver.model.saveTxShare(); err != nil { err = errors.Wrap(err, "datatx service: error deleting transfer: "+datatx.Status_STATUS_INVALID.String()) diff --git a/internal/grpc/services/gateway/ocmshareprovider.go b/internal/grpc/services/gateway/ocmshareprovider.go index 1470318bde..5effeab924 100644 --- a/internal/grpc/services/gateway/ocmshareprovider.go +++ b/internal/grpc/services/gateway/ocmshareprovider.go @@ -170,7 +170,7 @@ func (s *svc) UpdateReceivedOCMShare(ctx context.Context, req *ocm.UpdateReceive }, nil } - // retrieve the persisted received share + // retrieve the current received share getShareReq := &ocm.GetReceivedOCMShareRequest{ Ref: &ocm.ShareReference{ Spec: &ocm.ShareReference_Id{ @@ -234,7 +234,16 @@ func (s *svc) UpdateReceivedOCMShare(ctx context.Context, req *ocm.UpdateReceive } } // handle transfer in case it has not already been accepted - if s.isTransferShare(share) && req.GetShare().State == ocm.ShareState_SHARE_STATE_ACCEPTED && share.State != ocm.ShareState_SHARE_STATE_ACCEPTED { + if s.isTransferShare(share) && req.GetShare().State == ocm.ShareState_SHARE_STATE_ACCEPTED { + if share.State == ocm.ShareState_SHARE_STATE_ACCEPTED { + log.Err(err).Msg("gateway: error calling UpdateReceivedShare, share already accepted.") + return &ocm.UpdateReceivedOCMShareResponse{ + Status: &rpc.Status{ + Code: rpc.Code_CODE_FAILED_PRECONDITION, + Message: "Share already accepted.", + }, + }, err + } // get provided destination path transferDestinationPath, err := s.getTransferDestinationPath(ctx, req) if err != nil { diff --git a/pkg/datatx/manager/rclone/rclone.go b/pkg/datatx/manager/rclone/rclone.go index 21e3d6e3f0..0e9d2dfe55 100644 --- a/pkg/datatx/manager/rclone/rclone.go +++ b/pkg/datatx/manager/rclone/rclone.go @@ -62,14 +62,15 @@ func (c *config) init(m map[string]interface{}) { } type config struct { - Endpoint string `mapstructure:"endpoint"` - AuthUser string `mapstructure:"auth_user"` // rclone basicauth user - AuthPass string `mapstructure:"auth_pass"` // rclone basicauth pass - AuthHeader string `mapstructure:"auth_header"` - File string `mapstructure:"file"` - JobStatusCheckInterval int `mapstructure:"job_status_check_interval"` - JobTimeout int `mapstructure:"job_timeout"` - Insecure bool `mapstructure:"insecure"` + Endpoint string `mapstructure:"endpoint"` + AuthUser string `mapstructure:"auth_user"` // rclone basicauth user + AuthPass string `mapstructure:"auth_pass"` // rclone basicauth pass + AuthHeader string `mapstructure:"auth_header"` + File string `mapstructure:"file"` + JobStatusCheckInterval int `mapstructure:"job_status_check_interval"` + JobTimeout int `mapstructure:"job_timeout"` + Insecure bool `mapstructure:"insecure"` + RemoveTransferJobOnCancel bool `mapstructure:"remove_transfer_job_on_cancel"` } type rclone struct { @@ -644,10 +645,24 @@ func (driver *rclone) CancelTransfer(ctx context.Context, transferID string) (*d Ctime: nil, }, err } + cTime, _ := strconv.ParseInt(transfer.Ctime, 10, 64) + transferRemovedMessage := "" + if driver.config.RemoveTransferJobOnCancel { + delete(driver.pDriver.model.Transfers, transfer.TransferID) + if err := driver.pDriver.model.saveTransfer(nil); err != nil { + return &datatx.TxInfo{ + Id: &datatx.TxId{OpaqueId: transferID}, + Status: datatx.Status_STATUS_INVALID, + Ctime: &typespb.Timestamp{Seconds: uint64(cTime)}, + }, err + } + transferRemovedMessage = "(transfer job successfully removed)" + } + _, endStatusFound := txEndStatuses[transfer.TransferStatus.String()] if endStatusFound { - err := errors.New("rclone driver: transfer already in end state") + err := errors.Wrapf(errors.New("rclone driver: transfer already in end state"), transferRemovedMessage) return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: transferID}, Status: datatx.Status_STATUS_INVALID, @@ -665,7 +680,7 @@ func (driver *rclone) CancelTransfer(ctx context.Context, transferID string) (*d data, err := json.Marshal(rcloneCancelTransferReq) if err != nil { - err = errors.Wrap(err, "rclone driver: error marshalling rclone req data") + err := errors.Wrapf(errors.New("rclone driver: error marshalling rclone req data"), transferRemovedMessage) return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: transferID}, Status: datatx.Status_STATUS_INVALID, @@ -677,7 +692,7 @@ func (driver *rclone) CancelTransfer(ctx context.Context, transferID string) (*d u, err := url.Parse(driver.config.Endpoint) if err != nil { - err = errors.Wrap(err, "rclone driver: error parsing driver endpoint") + err := errors.Wrapf(errors.New("rclone driver: error parsing driver endpoint"), transferRemovedMessage) return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: transferID}, Status: datatx.Status_STATUS_INVALID, @@ -689,7 +704,7 @@ func (driver *rclone) CancelTransfer(ctx context.Context, transferID string) (*d req, err := http.NewRequest(http.MethodPost, requestURL, bytes.NewReader(data)) if err != nil { - err = errors.Wrap(err, "rclone driver: error framing post request") + err := errors.Wrapf(errors.New("rclone driver: error framing post request"), transferRemovedMessage) return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: transferID}, Status: datatx.Status_STATUS_INVALID, @@ -702,7 +717,7 @@ func (driver *rclone) CancelTransfer(ctx context.Context, transferID string) (*d res, err := driver.client.Do(req) if err != nil { - err = errors.Wrap(err, "rclone driver: error sending post request") + err := errors.Wrapf(errors.New("rclone driver: error sending post request"), transferRemovedMessage) return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: transferID}, Status: datatx.Status_STATUS_INVALID, @@ -715,14 +730,14 @@ func (driver *rclone) CancelTransfer(ctx context.Context, transferID string) (*d if res.StatusCode != http.StatusOK { var errorResData rcloneHTTPErrorRes if err = json.NewDecoder(res.Body).Decode(&errorResData); err != nil { - err = errors.Wrap(err, "rclone driver: error decoding response data") + err := errors.Wrapf(errors.New("rclone driver: error decoding response data"), transferRemovedMessage) return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: transferID}, Status: datatx.Status_STATUS_INVALID, Ctime: &typespb.Timestamp{Seconds: uint64(cTime)}, }, err } - err = errors.Wrap(errors.Errorf("status: %v, error: %v", errorResData.Status, errorResData.Error), "rclone driver: rclone request responded with error") + err = errors.Wrap(errors.Errorf("%v, status: %v, error: %v", transferRemovedMessage, errorResData.Status, errorResData.Error), "rclone driver: rclone request responded with error") return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: transferID}, Status: datatx.Status_STATUS_INVALID, @@ -744,7 +759,7 @@ func (driver *rclone) CancelTransfer(ctx context.Context, transferID string) (*d } var resData rcloneCancelTransferResJSON if err = json.NewDecoder(res.Body).Decode(&resData); err != nil { - err = errors.Wrap(err, "rclone driver: error decoding response data") + err := errors.Wrapf(errors.New("rclone driver: error decoding response data"), transferRemovedMessage) return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: transferID}, Status: datatx.Status_STATUS_INVALID, From 31ac08a85832b702a1f9a8f4db243c3bae8b5334 Mon Sep 17 00:00:00 2001 From: Swikriti Tripathi <41103328+SwikritiT@users.noreply.github.com> Date: Tue, 23 May 2023 11:30:41 +0545 Subject: [PATCH 03/48] [full-ci][tests-only]Bump commit id for tests (#3900) * Bump commit id for tests * update expected to fail file --- .drone.env | 2 +- .../expected-failures-on-EOS-storage.md | 3 -- .../expected-failures-on-OCIS-storage.md | 50 +++++++++---------- .../expected-failures-on-S3NG-storage.md | 50 +++++++++---------- tests/ocis | 2 +- 5 files changed, 51 insertions(+), 56 deletions(-) diff --git a/.drone.env b/.drone.env index 4b25ea503b..90d678f386 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=1788406b5273782d5bad44543ba5b9f94d48370d +APITESTS_COMMITID=61ce83d67e80c69ec3e18d5d180c6259a4872128 APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/tests/acceptance/expected-failures-on-EOS-storage.md b/tests/acceptance/expected-failures-on-EOS-storage.md index c60cf6b12a..2f2818caa2 100644 --- a/tests/acceptance/expected-failures-on-EOS-storage.md +++ b/tests/acceptance/expected-failures-on-EOS-storage.md @@ -539,9 +539,6 @@ The expected failures in this file are from features in the owncloud/ocis repo. - [coreApiTrashbin/trashbinFilesFolders.feature:302](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L302) - [coreApiTrashbin/trashbinFilesFolders.feature:303](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L303) - [coreApiTrashbin/trashbinFilesFolders.feature:304](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L304) -- [coreApiTrashbin/trashbinFilesFolders.feature:308](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L308) -- [coreApiTrashbin/trashbinFilesFolders.feature:309](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L309) -- [coreApiTrashbin/trashbinFilesFolders.feature:310](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L310) - [coreApiTrashbin/trashbinRestore.feature:31](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinRestore.feature#L31) - [coreApiTrashbin/trashbinRestore.feature:32](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinRestore.feature#L32) - [coreApiTrashbin/trashbinRestore.feature:62](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinRestore.feature#L62) diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index 42a67ce9e2..2ec52c1528 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -191,15 +191,11 @@ File and sync features in a shared scenario - [coreApiSharePublicLink1/accessToPublicLinkShare.feature:13](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/accessToPublicLinkShare.feature#L13) - [coreApiSharePublicLink1/accessToPublicLinkShare.feature:32](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/accessToPublicLinkShare.feature#L32) -- [coreApiSharePublicLink1/createPublicLinkShare.feature:170](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L170) -- [coreApiSharePublicLink1/createPublicLinkShare.feature:171](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L171) -- [coreApiSharePublicLink1/createPublicLinkShare.feature:345](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L345) -- [coreApiSharePublicLink1/createPublicLinkShare.feature:355](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L355) +- [coreApiSharePublicLink1/createPublicLinkShare.feature:163](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L163) +- [coreApiSharePublicLink1/createPublicLinkShare.feature:164](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L164) +- [coreApiSharePublicLink1/createPublicLinkShare.feature:317](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L317) +- [coreApiSharePublicLink1/createPublicLinkShare.feature:327](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L327) -#### [Ability to return error messages in Webdav response bodies](https://github.com/owncloud/ocis/issues/1293) - -- [coreApiSharePublicLink1/createPublicLinkShare.feature:71](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L71) -- [coreApiSharePublicLink1/createPublicLinkShare.feature:72](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L72) #### [copying a folder within a public link folder to folder with same name as an already existing file overwrites the parent file](https://github.com/owncloud/ocis/issues/1232) @@ -219,10 +215,11 @@ File and sync features in a shared scenario #### [Adding public upload to a read only shared folder as a receipient is allowed ](https://github.com/owncloud/ocis/issues/2164) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:270](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L270) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:271](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L271) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:316](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L316) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:317](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L317) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:264](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L264) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:265](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L265) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:306](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L306) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:307](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L307) + #### [Upload-only shares must not overwrite but create a separate file](https://github.com/owncloud/ocis/issues/1267) - [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:10](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L10) @@ -373,6 +370,8 @@ API, search, favorites, config, capabilities, not existing endpoints, CORS and o - [coreApiAuthOcs/ocsGETAuth.feature:124](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthOcs/ocsGETAuth.feature#L124) - [coreApiAuthOcs/ocsPOSTAuth.feature:11](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthOcs/ocsPOSTAuth.feature#L11) - [coreApiAuthOcs/ocsPUTAuth.feature:11](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthOcs/ocsPUTAuth.feature#L11) +- [coreApiSharePublicLink1/createPublicLinkShare.feature:69](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L69) +- [coreApiSharePublicLink1/createPublicLinkShare.feature:70](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L70) #### [sending MKCOL requests to another or non-existing user's webDav endpoints as normal user should return 404](https://github.com/owncloud/ocis/issues/5049) _ocdav: api compatibility, return correct status code_ @@ -667,16 +666,13 @@ _ocs: api compatibility, return correct status code_ - [coreApiTrashbin/trashbinFilesFolders.feature:131](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L131) - [coreApiTrashbin/trashbinFilesFolders.feature:154](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L154) - [coreApiTrashbin/trashbinFilesFolders.feature:287](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L287) -- [coreApiTrashbin/trashbinFilesFolders.feature:308](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L308) - [coreApiTrashbin/trashbinFilesFolders.feature:305](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L305) - [coreApiTrashbin/trashbinFilesFolders.feature:306](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L306) - [coreApiTrashbin/trashbinFilesFolders.feature:307](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L307) -- [coreApiTrashbin/trashbinFilesFolders.feature:309](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L309) -- [coreApiTrashbin/trashbinFilesFolders.feature:310](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L310) -- [coreApiTrashbin/trashbinFilesFolders.feature:332](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L332) -- [coreApiTrashbin/trashbinFilesFolders.feature:352](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L352) -- [coreApiTrashbin/trashbinFilesFolders.feature:443](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L443) -- [coreApiTrashbin/trashbinFilesFolders.feature:406](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L406) +- [coreApiTrashbin/trashbinFilesFolders.feature:326](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L326) +- [coreApiTrashbin/trashbinFilesFolders.feature:346](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L346) +- [coreApiTrashbin/trashbinFilesFolders.feature:400](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L400) +- [coreApiTrashbin/trashbinFilesFolders.feature:437](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L437) - [coreApiTrashbin/trashbinSharingToShares.feature:22](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L22) - [coreApiTrashbin/trashbinSharingToShares.feature:205](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L205) - [coreApiTrashbin/trashbinSharingToShares.feature:229](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L229) @@ -771,8 +767,8 @@ _ocs: api compatibility, return correct status code_ - [coreApiShareOperationsToShares1/gettingShares.feature:41](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareOperationsToShares1/gettingShares.feature#L41) #### [These tests pass in ocis and reva egde but fail in master with `file_target has unexpected value '/home'`](https://github.com/owncloud/ocis/issues/2113) -- [coreApiSharePublicLink1/createPublicLinkShare.feature:308](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L308) -- [coreApiSharePublicLink1/createPublicLinkShare.feature:309](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L309) +- [coreApiSharePublicLink1/createPublicLinkShare.feature:280](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L280) +- [coreApiSharePublicLink1/createPublicLinkShare.feature:281](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L281) #### [valid WebDAV (DELETE, COPY or MOVE) requests with body must exit with 415](https://github.com/owncloud/ocis/issues/4332) - [coreApiAuthWebDav/webDavCOPYAuth.feature:136](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavCOPYAuth.feature#L136) @@ -783,10 +779,10 @@ The problem has been fixed in reva edge branch but not in reva master - [coreApiWebdavOperations/propfind.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L73) #### [Updating the role of a public link to internal gives returns 400] -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:492](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L492) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:493](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L493) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:494](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L494) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:495](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L495) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:480](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L480) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:481](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L481) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:482](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L482) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:483](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L483) #### [Default capabilities for normal user and admin user not same as in oC-core](https://github.com/owncloud/ocis/issues/1285) - [coreApiCapabilities/capabilities.feature:11](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiCapabilities/capabilities.feature#L11) @@ -794,5 +790,9 @@ The problem has been fixed in reva edge branch but not in reva master - [coreApiCapabilities/capabilities.feature:175](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiCapabilities/capabilities.feature#L175) - [coreApiCapabilities/capabilities.feature:216](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiCapabilities/capabilities.feature#L216) +#### [Sharing of project space root via public link does no longer work](https://github.com/owncloud/ocis/issues/6278) +- [coreApiShareCreateSpecialToShares2/createShareDefaultFolderForReceivedShares.feature:23](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareDefaultFolderForReceivedShares.feature#L23) +- [coreApiShareCreateSpecialToShares2/createShareDefaultFolderForReceivedShares.feature:24](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareDefaultFolderForReceivedShares.feature#L24) + Note: always have an empty line at the end of this file. The bash script that processes this file may not process a scenario reference on the last line. diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index a6f587c185..8e2224c62f 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -217,15 +217,10 @@ File and sync features in a shared scenario - [coreApiSharePublicLink1/accessToPublicLinkShare.feature:13](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/accessToPublicLinkShare.feature#L13) - [coreApiSharePublicLink1/accessToPublicLinkShare.feature:32](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/accessToPublicLinkShare.feature#L32) -- [coreApiSharePublicLink1/createPublicLinkShare.feature:170](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L170) -- [coreApiSharePublicLink1/createPublicLinkShare.feature:171](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L171) -- [coreApiSharePublicLink1/createPublicLinkShare.feature:345](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L345) -- [coreApiSharePublicLink1/createPublicLinkShare.feature:355](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L355) - -#### [Ability to return error messages in Webdav response bodies](https://github.com/owncloud/ocis/issues/1293) - -- [coreApiSharePublicLink1/createPublicLinkShare.feature:71](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L71) -- [coreApiSharePublicLink1/createPublicLinkShare.feature:72](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L72) +- [coreApiSharePublicLink1/createPublicLinkShare.feature:163](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L163) +- [coreApiSharePublicLink1/createPublicLinkShare.feature:164](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L164) +- [coreApiSharePublicLink1/createPublicLinkShare.feature:317](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L317) +- [coreApiSharePublicLink1/createPublicLinkShare.feature:327](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L327) #### [copying a folder within a public link folder to folder with same name as an already existing file overwrites the parent file](https://github.com/owncloud/ocis/issues/1232) @@ -245,10 +240,10 @@ File and sync features in a shared scenario #### [Adding public upload to a read only shared folder as a receipient is allowed ](https://github.com/owncloud/ocis/issues/2164) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:270](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L270) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:271](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L271) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:316](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L316) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:317](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L317) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:264](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L264) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:265](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L265) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:306](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L306) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:307](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L307) #### [Upload-only shares must not overwrite but create a separate file](https://github.com/owncloud/ocis/issues/1267) @@ -397,6 +392,8 @@ API, search, favorites, config, capabilities, not existing endpoints, CORS and o - [coreApiAuthOcs/ocsGETAuth.feature:124](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthOcs/ocsGETAuth.feature#L124) Scenario: using OCS as admin user with wrong password - [coreApiAuthOcs/ocsPOSTAuth.feature:11](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthOcs/ocsPOSTAuth.feature#L11) Scenario: send POST requests to OCS endpoints as normal user with wrong password - [coreApiAuthOcs/ocsPUTAuth.feature:11](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthOcs/ocsPUTAuth.feature#L11) Scenario: send PUT request to OCS endpoints as admin with wrong password +- [coreApiSharePublicLink1/createPublicLinkShare.feature:69](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L69) +- [coreApiSharePublicLink1/createPublicLinkShare.feature:70](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L70) #### [sending MKCOL requests to another or non-existing user's webDav endpoints as normal user should return 404](https://github.com/owncloud/ocis/issues/5049) _ocdav: api compatibility, return correct status code_ @@ -673,16 +670,13 @@ _ocs: api compatibility, return correct status code_ - [coreApiTrashbin/trashbinFilesFolders.feature:131](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L131) - [coreApiTrashbin/trashbinFilesFolders.feature:154](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L154) - [coreApiTrashbin/trashbinFilesFolders.feature:287](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L287) -- [coreApiTrashbin/trashbinFilesFolders.feature:308](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L308) - [coreApiTrashbin/trashbinFilesFolders.feature:305](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L305) - [coreApiTrashbin/trashbinFilesFolders.feature:306](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L306) - [coreApiTrashbin/trashbinFilesFolders.feature:307](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L307) -- [coreApiTrashbin/trashbinFilesFolders.feature:309](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L309) -- [coreApiTrashbin/trashbinFilesFolders.feature:310](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L310) -- [coreApiTrashbin/trashbinFilesFolders.feature:332](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L332) -- [coreApiTrashbin/trashbinFilesFolders.feature:352](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L352) -- [coreApiTrashbin/trashbinFilesFolders.feature:443](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L443) -- [coreApiTrashbin/trashbinFilesFolders.feature:406](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L406) +- [coreApiTrashbin/trashbinFilesFolders.feature:326](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L326) +- [coreApiTrashbin/trashbinFilesFolders.feature:346](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L346) +- [coreApiTrashbin/trashbinFilesFolders.feature:400](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L400) +- [coreApiTrashbin/trashbinFilesFolders.feature:437](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L437) - [coreApiTrashbin/trashbinSharingToShares.feature:22](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L22) - [coreApiTrashbin/trashbinSharingToShares.feature:205](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L205) - [coreApiTrashbin/trashbinSharingToShares.feature:229](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L229) @@ -777,8 +771,8 @@ _ocs: api compatibility, return correct status code_ - [coreApiShareOperationsToShares1/gettingShares.feature:41](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareOperationsToShares1/gettingShares.feature#L41) #### [These tests pass in ocis and reva egde but fail in master with `file_target has unexpected value '/home'`](https://github.com/owncloud/ocis/issues/2113) -- [coreApiSharePublicLink1/createPublicLinkShare.feature:308](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L308) -- [coreApiSharePublicLink1/createPublicLinkShare.feature:309](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L309) +- [coreApiSharePublicLink1/createPublicLinkShare.feature:280](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L280) +- [coreApiSharePublicLink1/createPublicLinkShare.feature:281](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L281) #### [valid WebDAV (DELETE, COPY or MOVE) requests with body must exit with 415](https://github.com/owncloud/ocis/issues/4332) - [coreApiAuthWebDav/webDavCOPYAuth.feature:136](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavCOPYAuth.feature#L136) @@ -789,15 +783,19 @@ The problem has been fixed in reva edge branch but not in reva master - [coreApiWebdavOperations/propfind.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L73) #### [Updating the role of a public link to internal gives returns 400] -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:492](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L492) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:493](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L493) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:494](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L494) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:495](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L495) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:480](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L480) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:481](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L481) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:482](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L482) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:483](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L483) #### [Default capabilities for normal user and admin user not same as in oC-core](https://github.com/owncloud/ocis/issues/1285) - [coreApiCapabilities/capabilities.feature:11](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiCapabilities/capabilities.feature#L11) - [coreApiCapabilities/capabilities.feature:136](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiCapabilities/capabilities.feature#L136) - [coreApiCapabilities/capabilities.feature:175](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiCapabilities/capabilities.feature#L175) +#### [Sharing of project space root via public link does no longer work](https://github.com/owncloud/ocis/issues/6278) +- [coreApiShareCreateSpecialToShares2/createShareDefaultFolderForReceivedShares.feature:23](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareDefaultFolderForReceivedShares.feature#L23) +- [coreApiShareCreateSpecialToShares2/createShareDefaultFolderForReceivedShares.feature:24](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareDefaultFolderForReceivedShares.feature#L24) + Note: always have an empty line at the end of this file. The bash script that processes this file may not process a scenario reference on the last line. diff --git a/tests/ocis b/tests/ocis index 1788406b52..61ce83d67e 160000 --- a/tests/ocis +++ b/tests/ocis @@ -1 +1 @@ -Subproject commit 1788406b5273782d5bad44543ba5b9f94d48370d +Subproject commit 61ce83d67e80c69ec3e18d5d180c6259a4872128 From 10969645ff68fd0da33580ceb7509241446300cf Mon Sep 17 00:00:00 2001 From: Prarup Gurung Date: Thu, 25 May 2023 22:49:24 -0700 Subject: [PATCH 04/48] [full-ci] [tests-only] bump ocis commit to latest (#3870) * Bump ocis commit id to latest * Bump ocis submodule * Fixed line numbers in expected failure --- .drone.env | 2 +- .../expected-failures-on-EOS-storage.md | 2 +- .../expected-failures-on-OCIS-storage.md | 40 +++++++++---------- .../expected-failures-on-S3NG-storage.md | 40 +++++++++---------- tests/ocis | 2 +- 5 files changed, 43 insertions(+), 43 deletions(-) diff --git a/.drone.env b/.drone.env index 90d678f386..4280304e6f 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=61ce83d67e80c69ec3e18d5d180c6259a4872128 +APITESTS_COMMITID=dc7418b3c3dc34349461b3997fb293a70466d804 APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/tests/acceptance/expected-failures-on-EOS-storage.md b/tests/acceptance/expected-failures-on-EOS-storage.md index 2f2818caa2..7d4ddd9deb 100644 --- a/tests/acceptance/expected-failures-on-EOS-storage.md +++ b/tests/acceptance/expected-failures-on-EOS-storage.md @@ -62,7 +62,7 @@ The expected failures in this file are from features in the owncloud/ocis repo. - [coreApiWebdavProperties1/getQuota.feature:48](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties1/getQuota.feature#L48) - [coreApiWebdavProperties1/getQuota.feature:49](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties1/getQuota.feature#L49) - [coreApiWebdavProperties1/getQuota.feature:61](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties1/getQuota.feature#L61) -- [coreApiWebdavProperties1/getQuota.feature:62](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties1/getQuota.feature#L62) +- [coreApiWebdavProperties1/getQuota.feature:54](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties1/getQuota.feature#L54) - [coreApiWebdavProperties1/getQuota.feature:77](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties1/getQuota.feature#L77) - [coreApiWebdavProperties1/getQuota.feature:78](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties1/getQuota.feature#L78) ### [no command equivalent to occ](https://github.com/owncloud/ocis/issues/1317) diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index 2ec52c1528..9e2516f4e1 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -375,23 +375,23 @@ API, search, favorites, config, capabilities, not existing endpoints, CORS and o #### [sending MKCOL requests to another or non-existing user's webDav endpoints as normal user should return 404](https://github.com/owncloud/ocis/issues/5049) _ocdav: api compatibility, return correct status code_ -- [coreApiAuthWebDav/webDavDELETEAuth.feature:61](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavDELETEAuth.feature#L61) -- [coreApiAuthWebDav/webDavPROPFINDAuth.feature:58](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavPROPFINDAuth.feature#L58) -- [coreApiAuthWebDav/webDavPROPPATCHAuth.feature:59](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavPROPPATCHAuth.feature#L59) -- [coreApiAuthWebDav/webDavMKCOLAuth.feature:55](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMKCOLAuth.feature#L55) -- [coreApiAuthWebDav/webDavMKCOLAuth.feature:66](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMKCOLAuth.feature#L66) +- [coreApiAuthWebDav/webDavDELETEAuth.feature:49](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavDELETEAuth.feature#L49) +- [coreApiAuthWebDav/webDavPROPFINDAuth.feature:46](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavPROPFINDAuth.feature#L46) +- [coreApiAuthWebDav/webDavPROPPATCHAuth.feature:47](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavPROPPATCHAuth.feature#L47) +- [coreApiAuthWebDav/webDavMKCOLAuth.feature:43](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMKCOLAuth.feature#L43) +- [coreApiAuthWebDav/webDavMKCOLAuth.feature:54](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMKCOLAuth.feature#L54) #### [trying to lock file of another user gives http 200](https://github.com/owncloud/ocis/issues/2176) -- [coreApiAuthWebDav/webDavLOCKAuth.feature:59](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavLOCKAuth.feature#L59) +- [coreApiAuthWebDav/webDavLOCKAuth.feature:47](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavLOCKAuth.feature#L47) #### [send (MOVE, COPY) requests to another user's webDav endpoints as normal user gives 400 instead of 403](https://github.com/owncloud/ocis/issues/3882) _ocdav: api compatibility, return correct status code_ -- [coreApiAuthWebDav/webDavMOVEAuth.feature:58](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMOVEAuth.feature#L58) Scenario: send MOVE requests to another user's webDav endpoints as normal user -- [coreApiAuthWebDav/webDavCOPYAuth.feature:58](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavCOPYAuth.feature#L58) +- [coreApiAuthWebDav/webDavMOVEAuth.feature:46](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMOVEAuth.feature#L46) Scenario: send MOVE requests to another user's webDav endpoints as normal user +- [coreApiAuthWebDav/webDavCOPYAuth.feature:46](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavCOPYAuth.feature#L46) #### [send POST requests to another user's webDav endpoints as normal user](https://github.com/owncloud/ocis/issues/1287) _ocdav: api compatibility, return correct status code_ -- [coreApiAuthWebDav/webDavPOSTAuth.feature:59](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavPOSTAuth.feature#L59) +- [coreApiAuthWebDav/webDavPOSTAuth.feature:47](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavPOSTAuth.feature#L47) #### [Using double slash in URL to access a folder gives 501 and other status codes](https://github.com/owncloud/ocis/issues/1667) - [coreApiAuthWebDav/webDavSpecialURLs.feature:37](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavSpecialURLs.feature#L37) @@ -453,7 +453,7 @@ _ocdav: api compatibility, return correct status code_ - [coreApiWebdavOperations/refuseAccess.feature:36](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/refuseAccess.feature#L36) #### [App Passwords/Tokens for legacy WebDAV clients](https://github.com/owncloud/ocis/issues/197) -- [coreApiAuthWebDav/webDavDELETEAuth.feature:139](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavDELETEAuth.feature#L139) +- [coreApiAuthWebDav/webDavDELETEAuth.feature:109](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavDELETEAuth.feature#L109) #### [Sharing a same file twice to the same group](https://github.com/owncloud/ocis/issues/1710) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:726](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L726) @@ -501,9 +501,9 @@ Not everything needs to be implemented for ocis. While the oc10 testsuite covers - [coreApiWebdavUploadTUS/checksums.feature:284](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L284) - [coreApiWebdavUploadTUS/checksums.feature:285](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L285) - [coreApiWebdavUploadTUS/optionsRequest.feature:8](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L8) -- [coreApiWebdavUploadTUS/optionsRequest.feature:35](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L35) -- [coreApiWebdavUploadTUS/optionsRequest.feature:62](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L62) -- [coreApiWebdavUploadTUS/optionsRequest.feature:89](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L89) +- [coreApiWebdavUploadTUS/optionsRequest.feature:23](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L23) +- [coreApiWebdavUploadTUS/optionsRequest.feature:38](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L38) +- [coreApiWebdavUploadTUS/optionsRequest.feature:53](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L53) - [coreApiWebdavUploadTUS/uploadToShare.feature:175](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L175) - [coreApiWebdavUploadTUS/uploadToShare.feature:174](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L174) - [coreApiWebdavUploadTUS/uploadToShare.feature:194](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L194) @@ -640,11 +640,11 @@ _ocs: api compatibility, return correct status code_ - [coreApiTrashbin/trashbinFilesFolders.feature:249](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L249) #### [Cannot disable the dav propfind depth infinity for resources](https://github.com/owncloud/ocis/issues/3720) -- [coreApiWebdavOperations/listFiles.feature:364](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L364) -- [coreApiWebdavOperations/listFiles.feature:365](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L365) -- [coreApiWebdavOperations/listFiles.feature:384](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L384) -- [coreApiWebdavOperations/listFiles.feature:403](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L403) -- [coreApiWebdavOperations/listFiles.feature:404](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L404) +- [coreApiWebdavOperations/listFiles.feature:355](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L355) +- [coreApiWebdavOperations/listFiles.feature:356](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L356) +- [coreApiWebdavOperations/listFiles.feature:375](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L375) +- [coreApiWebdavOperations/listFiles.feature:394](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L394) +- [coreApiWebdavOperations/listFiles.feature:395](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L395) #### [trash-bin propfind responses are wrong in reva master](https://github.com/cs3org/reva/issues/2861) - [coreApiTrashbin/trashbinDelete.feature:29](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L29) @@ -754,7 +754,7 @@ _ocs: api compatibility, return correct status code_ #### [WebDAV MOVE with body returns 400 rather than 415](https://github.com/cs3org/reva/issues/3119) -- [coreApiAuthWebDav/webDavMOVEAuth.feature:136](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMOVEAuth.feature#L136) +- [coreApiAuthWebDav/webDavMOVEAuth.feature:106](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMOVEAuth.feature#L106) #### [reShareUpdate API tests failing in reva](https://github.com/cs3org/reva/issues/2916) @@ -771,7 +771,7 @@ _ocs: api compatibility, return correct status code_ - [coreApiSharePublicLink1/createPublicLinkShare.feature:281](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L281) #### [valid WebDAV (DELETE, COPY or MOVE) requests with body must exit with 415](https://github.com/owncloud/ocis/issues/4332) -- [coreApiAuthWebDav/webDavCOPYAuth.feature:136](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavCOPYAuth.feature#L136) +- [coreApiAuthWebDav/webDavCOPYAuth.feature:106](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavCOPYAuth.feature#L106) #### [PROPFIND on (password protected) public link returns invalid XML](https://github.com/owncloud/ocis/issues/39707) The problem has been fixed in reva edge branch but not in reva master diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 8e2224c62f..c9e0ddbe54 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -397,23 +397,23 @@ API, search, favorites, config, capabilities, not existing endpoints, CORS and o #### [sending MKCOL requests to another or non-existing user's webDav endpoints as normal user should return 404](https://github.com/owncloud/ocis/issues/5049) _ocdav: api compatibility, return correct status code_ -- [coreApiAuthWebDav/webDavDELETEAuth.feature:61](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavDELETEAuth.feature#L61) Scenario: send DELETE requests to another user's webDav endpoints as normal user -- [coreApiAuthWebDav/webDavPROPFINDAuth.feature:58](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavPROPFINDAuth.feature#L58) Scenario: send PROPFIND requests to another user's webDav endpoints as normal user -- [coreApiAuthWebDav/webDavPROPPATCHAuth.feature:59](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavPROPPATCHAuth.feature#L59) Scenario: send PROPPATCH requests to another user's webDav endpoints as normal user -- [coreApiAuthWebDav/webDavMKCOLAuth.feature:55](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMKCOLAuth.feature#L55) Scenario: send MKCOL requests to another user's webDav endpoints as normal user -- [coreApiAuthWebDav/webDavMKCOLAuth.feature:66](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMKCOLAuth.feature#L66) Scenario: send MKCOL requests to another user's webDav endpoints as normal user using the spaces WebDAV API +- [coreApiAuthWebDav/webDavDELETEAuth.feature:49](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavDELETEAuth.feature#L49) Scenario: send DELETE requests to another user's webDav endpoints as normal user +- [coreApiAuthWebDav/webDavPROPFINDAuth.feature:46](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavPROPFINDAuth.feature#L46) Scenario: send PROPFIND requests to another user's webDav endpoints as normal user +- [coreApiAuthWebDav/webDavPROPPATCHAuth.feature:47](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavPROPPATCHAuth.feature#L47) Scenario: send PROPPATCH requests to another user's webDav endpoints as normal user +- [coreApiAuthWebDav/webDavMKCOLAuth.feature:43](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMKCOLAuth.feature#L43) Scenario: send MKCOL requests to another user's webDav endpoints as normal user +- [coreApiAuthWebDav/webDavMKCOLAuth.feature:54](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMKCOLAuth.feature#L54) Scenario: send MKCOL requests to another user's webDav endpoints as normal user using the spaces WebDAV API #### [trying to lock file of another user gives http 200](https://github.com/owncloud/ocis/issues/2176) -- [coreApiAuthWebDav/webDavLOCKAuth.feature:59](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavLOCKAuth.feature#L59) Scenario: send LOCK requests to another user's webDav endpoints as normal user +- [coreApiAuthWebDav/webDavLOCKAuth.feature:47](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavLOCKAuth.feature#L47) Scenario: send LOCK requests to another user's webDav endpoints as normal user #### [send (MOVE, COPY) requests to another user's webDav endpoints as normal user gives 400 instead of 403](https://github.com/owncloud/ocis/issues/3882) _ocdav: api compatibility, return correct status code_ -- [coreApiAuthWebDav/webDavMOVEAuth.feature:58](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMOVEAuth.feature#L58) Scenario: send MOVE requests to another user's webDav endpoints as normal user -- [coreApiAuthWebDav/webDavCOPYAuth.feature:58](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavCOPYAuth.feature#L58) +- [coreApiAuthWebDav/webDavMOVEAuth.feature:46](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMOVEAuth.feature#L46) Scenario: send MOVE requests to another user's webDav endpoints as normal user +- [coreApiAuthWebDav/webDavCOPYAuth.feature:46](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavCOPYAuth.feature#L46) #### [send POST requests to another user's webDav endpoints as normal user](https://github.com/owncloud/ocis/issues/1287) _ocdav: api compatibility, return correct status code_ -- [coreApiAuthWebDav/webDavPOSTAuth.feature:59](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavPOSTAuth.feature#L59) Scenario: send POST requests to another user's webDav endpoints as normal user +- [coreApiAuthWebDav/webDavPOSTAuth.feature:47](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavPOSTAuth.feature#L47) Scenario: send POST requests to another user's webDav endpoints as normal user #### [Using double slash in URL to access a folder gives 501 and other status codes](https://github.com/owncloud/ocis/issues/1667) - [coreApiAuthWebDav/webDavSpecialURLs.feature:37](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavSpecialURLs.feature#L37) @@ -475,7 +475,7 @@ _ocdav: api compatibility, return correct status code_ - [coreApiWebdavOperations/refuseAccess.feature:36](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/refuseAccess.feature#L36) #### [App Passwords/Tokens for legacy WebDAV clients](https://github.com/owncloud/ocis/issues/197) -- [coreApiAuthWebDav/webDavDELETEAuth.feature:139](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavDELETEAuth.feature#L139) +- [coreApiAuthWebDav/webDavDELETEAuth.feature:109](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavDELETEAuth.feature#L109) #### [Sharing a same file twice to the same group](https://github.com/owncloud/ocis/issues/1710) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:778](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L778) @@ -523,9 +523,9 @@ Not everything needs to be implemented for ocis. While the oc10 testsuite covers - [coreApiWebdavUploadTUS/checksums.feature:284](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L284) - [coreApiWebdavUploadTUS/checksums.feature:285](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L285) - [coreApiWebdavUploadTUS/optionsRequest.feature:8](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L8) -- [coreApiWebdavUploadTUS/optionsRequest.feature:35](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L35) -- [coreApiWebdavUploadTUS/optionsRequest.feature:62](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L62) -- [coreApiWebdavUploadTUS/optionsRequest.feature:89](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L89) +- [coreApiWebdavUploadTUS/optionsRequest.feature:23](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L23) +- [coreApiWebdavUploadTUS/optionsRequest.feature:38](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L38) +- [coreApiWebdavUploadTUS/optionsRequest.feature:53](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L53) - [coreApiWebdavUploadTUS/uploadToShare.feature:175](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L175) - [coreApiWebdavUploadTUS/uploadToShare.feature:174](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L174) - [coreApiWebdavUploadTUS/uploadToShare.feature:194](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L194) @@ -644,11 +644,11 @@ _ocs: api compatibility, return correct status code_ - [coreApiTrashbin/trashbinFilesFolders.feature:249](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L249) #### [Cannot disable the dav propfind depth infinity for resources](https://github.com/owncloud/ocis/issues/3720) -- [coreApiWebdavOperations/listFiles.feature:364](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L364) -- [coreApiWebdavOperations/listFiles.feature:365](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L365) -- [coreApiWebdavOperations/listFiles.feature:384](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L384) -- [coreApiWebdavOperations/listFiles.feature:403](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L403) -- [coreApiWebdavOperations/listFiles.feature:404](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L404) +- [coreApiWebdavOperations/listFiles.feature:355](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L355) +- [coreApiWebdavOperations/listFiles.feature:356](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L356) +- [coreApiWebdavOperations/listFiles.feature:375](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L375) +- [coreApiWebdavOperations/listFiles.feature:394](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L394) +- [coreApiWebdavOperations/listFiles.feature:395](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/listFiles.feature#L395) #### [trash-bin propfind responses are wrong in reva master](https://github.com/cs3org/reva/issues/2861) - [coreApiTrashbin/trashbinDelete.feature:29](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinDelete.feature#L29) @@ -758,7 +758,7 @@ _ocs: api compatibility, return correct status code_ #### [WebDAV MOVE with body returns 400 rather than 415](https://github.com/cs3org/reva/issues/3119) -- [coreApiAuthWebDav/webDavMOVEAuth.feature:136](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMOVEAuth.feature#L136) +- [coreApiAuthWebDav/webDavMOVEAuth.feature:106](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMOVEAuth.feature#L106) #### [reShareUpdate API tests failing in reva](https://github.com/cs3org/reva/issues/2916) @@ -775,7 +775,7 @@ _ocs: api compatibility, return correct status code_ - [coreApiSharePublicLink1/createPublicLinkShare.feature:281](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink1/createPublicLinkShare.feature#L281) #### [valid WebDAV (DELETE, COPY or MOVE) requests with body must exit with 415](https://github.com/owncloud/ocis/issues/4332) -- [coreApiAuthWebDav/webDavCOPYAuth.feature:136](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavCOPYAuth.feature#L136) +- [coreApiAuthWebDav/webDavCOPYAuth.feature:106](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavCOPYAuth.feature#L106) #### [PROPFIND on (password protected) public link returns invalid XML](https://github.com/owncloud/ocis/issues/39707) The problem has been fixed in reva edge branch but not in reva master diff --git a/tests/ocis b/tests/ocis index 61ce83d67e..dc7418b3c3 160000 --- a/tests/ocis +++ b/tests/ocis @@ -1 +1 @@ -Subproject commit 61ce83d67e80c69ec3e18d5d180c6259a4872128 +Subproject commit dc7418b3c3dc34349461b3997fb293a70466d804 From 3a761e3f7c35ad5d2dab12842b83e8ac50b14713 Mon Sep 17 00:00:00 2001 From: Antoon P Date: Tue, 30 May 2023 14:46:35 +0200 Subject: [PATCH 05/48] Storage drivers setup for datatx (#3915) * * Datatx(transfers) and rclone(jobs) storage drivers setup * Configurations for storage drivers * Some clarification on cancelling transfers in the tutorial * Remove transfer-create cli command, must use ocm-share-create instead * Remove unnecessary logging --------- Co-authored-by: Antoon P --- .../unreleased/datatx-storage-drivers.md | 4 + cmd/reva/main.go | 1 - cmd/reva/transfer-create.go | 146 ------- .../en/docs/tutorials/datatx-tutorial.md | 4 +- examples/datatx/datatx.toml | 18 +- internal/grpc/services/datatx/datatx.go | 179 +++----- pkg/datatx/datatx.go | 22 + pkg/datatx/manager/loader/loader.go | 2 + pkg/datatx/manager/rclone/rclone.go | 411 ++++++++---------- .../manager/rclone/repository/json/json.go | 171 ++++++++ .../rclone/repository/registry/registry.go | 36 ++ .../manager/rclone/repository/repository.go | 44 ++ pkg/datatx/repository/json/json.go | 211 +++++++++ pkg/datatx/repository/registry/registry.go | 36 ++ 14 files changed, 777 insertions(+), 508 deletions(-) create mode 100644 changelog/unreleased/datatx-storage-drivers.md delete mode 100644 cmd/reva/transfer-create.go create mode 100644 pkg/datatx/manager/rclone/repository/json/json.go create mode 100644 pkg/datatx/manager/rclone/repository/registry/registry.go create mode 100644 pkg/datatx/manager/rclone/repository/repository.go create mode 100644 pkg/datatx/repository/json/json.go create mode 100644 pkg/datatx/repository/registry/registry.go diff --git a/changelog/unreleased/datatx-storage-drivers.md b/changelog/unreleased/datatx-storage-drivers.md new file mode 100644 index 0000000000..e81b30e001 --- /dev/null +++ b/changelog/unreleased/datatx-storage-drivers.md @@ -0,0 +1,4 @@ +Enhancement: Storage drivers setup for datatx + +https://github.com/cs3org/reva/pull/3915 +https://github.com/cs3org/reva/issues/3914 \ No newline at end of file diff --git a/cmd/reva/main.go b/cmd/reva/main.go index 03f96278aa..f04984a0a7 100644 --- a/cmd/reva/main.go +++ b/cmd/reva/main.go @@ -81,7 +81,6 @@ var ( shareUpdateCommand(), shareListReceivedCommand(), shareUpdateReceivedCommand(), - transferCreateCommand(), transferGetStatusCommand(), transferCancelCommand(), transferListCommand(), diff --git a/cmd/reva/transfer-create.go b/cmd/reva/transfer-create.go deleted file mode 100644 index 969e3f1bc0..0000000000 --- a/cmd/reva/transfer-create.go +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package main - -import ( - "io" - "os" - "strings" - "time" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - invitepb "github.com/cs3org/go-cs3apis/cs3/ocm/invite/v1beta1" - ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" - rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - ocm "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/ocm/share" - "github.com/cs3org/reva/pkg/utils" - "github.com/jedib0t/go-pretty/table" - "github.com/pkg/errors" -) - -func transferCreateCommand() *command { - cmd := newCommand("transfer-create") - cmd.Description = func() string { return "create transfer between 2 sites" } - cmd.Usage = func() string { return "Usage: transfer-create [-flags] " } - grantee := cmd.String("grantee", "", "the grantee, receiver of the transfer") - granteeType := cmd.String("granteeType", "user", "the grantee type, one of: user, group (defaults to user)") - idp := cmd.String("idp", "", "the idp of the grantee") - userType := cmd.String("user-type", "primary", "the type of user account, defaults to primary") - - cmd.Action = func(w ...io.Writer) error { - if cmd.NArg() < 1 { - return errors.New("Invalid arguments: " + cmd.Usage()) - } - - if *grantee == "" { - return errors.New("Grantee cannot be empty: use -grantee flag\n" + cmd.Usage()) - } - if *idp == "" { - return errors.New("Idp cannot be empty: use -idp flag\n" + cmd.Usage()) - } - - // the resource to transfer; the path - fn := cmd.Args()[0] - - ctx := getAuthContext() - client, err := getClient() - if err != nil { - return err - } - - u := &userpb.UserId{OpaqueId: *grantee, Idp: *idp, Type: utils.UserTypeMap(*userType)} - - // check if invitation has been accepted - acceptedUserRes, err := client.GetAcceptedUser(ctx, &invitepb.GetAcceptedUserRequest{ - RemoteUserId: u, - }) - if err != nil { - return err - } - if acceptedUserRes.Status.Code != rpc.Code_CODE_OK { - return formatError(acceptedUserRes.Status) - } - - // verify resource stats - statReq := &provider.StatRequest{ - Ref: &provider.Reference{Path: fn}, - } - statRes, err := client.Stat(ctx, statReq) - if err != nil { - return err - } - if statRes.Status.Code != rpc.Code_CODE_OK { - return formatError(statRes.Status) - } - - providerInfoResp, err := client.GetInfoByDomain(ctx, &ocmprovider.GetInfoByDomainRequest{ - Domain: *idp, - }) - if err != nil { - return err - } - - gt := provider.GranteeType_GRANTEE_TYPE_USER - if strings.ToLower(*granteeType) == "group" { - gt = provider.GranteeType_GRANTEE_TYPE_GROUP - } - - createShareReq := &ocm.CreateOCMShareRequest{ - Grantee: &provider.Grantee{ - Type: gt, - Id: &provider.Grantee_UserId{ - UserId: u, - }, - }, - ResourceId: statRes.Info.Id, - AccessMethods: []*ocm.AccessMethod{ - share.NewTransferAccessMethod(), - }, - RecipientMeshProvider: providerInfoResp.ProviderInfo, - } - - createShareResponse, err := client.CreateOCMShare(ctx, createShareReq) - if err != nil { - return err - } - if createShareResponse.Status.Code != rpc.Code_CODE_OK { - if createShareResponse.Status.Code == rpc.Code_CODE_NOT_FOUND { - return formatError(statRes.Status) - } - return err - } - - t := table.NewWriter() - t.SetOutputMirror(os.Stdout) - t.AppendHeader(table.Row{"#", "Owner.Idp", "Owner.OpaqueId", "ResourceId", "Type", "Grantee.Idp", "Grantee.OpaqueId", "ShareType", "Created", "Updated"}) - - s := createShareResponse.Share - t.AppendRows([]table.Row{ - {s.Id.OpaqueId, s.Owner.Idp, s.Owner.OpaqueId, s.ResourceId.String(), - s.Grantee.Type.String(), s.Grantee.GetUserId().Idp, s.Grantee.GetUserId().OpaqueId, s.ShareType.String(), - time.Unix(int64(s.Ctime.Seconds), 0), time.Unix(int64(s.Mtime.Seconds), 0)}, - }) - t.Render() - return nil - } - - return cmd -} diff --git a/docs/content/en/docs/tutorials/datatx-tutorial.md b/docs/content/en/docs/tutorials/datatx-tutorial.md index 2da5a5ddf6..4e24cf3a0d 100644 --- a/docs/content/en/docs/tutorials/datatx-tutorial.md +++ b/docs/content/en/docs/tutorials/datatx-tutorial.md @@ -166,4 +166,6 @@ remove_transfer_on_cancel = true [grpc.services.datatx.txdrivers.rclone] remove_transfer_job_on_cancel = true ``` -Currently this setting is recommended. \ No newline at end of file +Currently this setting is recommended. + +*Note that with these settings `transfer-cancel` will remove transfers & jobs even when a transfer cannot actually be cancelled because it was already in an end-state, eg. `finished` or `failed`. So `transfer-cancel` will act like a 'delete' function. \ No newline at end of file diff --git a/examples/datatx/datatx.toml b/examples/datatx/datatx.toml index 2a4633758e..b5296f0cb1 100644 --- a/examples/datatx/datatx.toml +++ b/examples/datatx/datatx.toml @@ -9,10 +9,8 @@ data_transfers_folder = "" [grpc.services.datatx] # rclone is currently the only data transfer driver implementation txdriver = "rclone" -# the shares,transfers db file (default: /var/tmp/reva/datatx-shares.json) -tx_shares_file = "" -# base folder of the data transfers (eg. /home/DataTransfers) -data_transfers_folder = "" +# the storage driver +storagedriver = "json" # if set to 'true' the transfer will always be removed from the db upon cancel request # recommended value is true remove_transfer_on_cancel = true @@ -30,16 +28,24 @@ auth_pass = "{rclone user secret}" # "x-access-token" will result in rclone using request header: X-Access-Token: "...token..." # If not set "bearer" is assumed auth_header = "x-access-token" -# the transfers(jobs) db file (default: /var/tmp/reva/datatx-transfers.json) -file = "" # check status job interval in milliseconds job_status_check_interval = 2000 # the job timeout in milliseconds (must be long enough for big transfers!) job_timeout = 120000 +# the storage driver +storagedriver = "json" # if set to 'true' the transfer job will always be removed from the db upon transfer cancel request # recommended value is true remove_transfer_job_on_cancel = true +[grpc.services.datatx.storagedrivers.json] +# the datatx transfers db file (defaults to: /var/tmp/reva/datatx-transfers.json) +file = "" + +[grpc.services.datatx.txdrivers.rclone.storagedrivers.json] +# the transfers jobs db file (defaults to: /var/tmp/reva/transfer-jobs.json) +file = "" + [http.services.ocdav] # reva supports http third party copy enable_http_tpc = true diff --git a/internal/grpc/services/datatx/datatx.go b/internal/grpc/services/datatx/datatx.go index e2be50749b..b9f5011a3d 100644 --- a/internal/grpc/services/datatx/datatx.go +++ b/internal/grpc/services/datatx/datatx.go @@ -20,15 +20,13 @@ package datatx import ( "context" - "encoding/json" - "io" - "os" - "sync" ocm "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" datatx "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1" + ctxpkg "github.com/cs3org/reva/pkg/ctx" txdriver "github.com/cs3org/reva/pkg/datatx" txregistry "github.com/cs3org/reva/pkg/datatx/manager/registry" + repoRegistry "github.com/cs3org/reva/pkg/datatx/repository/registry" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" @@ -43,44 +41,23 @@ func init() { type config struct { // transfer driver - TxDriver string `mapstructure:"txdriver"` - TxDrivers map[string]map[string]interface{} `mapstructure:"txdrivers"` - // storage driver to persist share/transfer relation - StorageDriver string `mapstructure:"storage_driver"` - StorageDrivers map[string]map[string]interface{} `mapstructure:"storage_drivers"` - TxSharesFile string `mapstructure:"tx_shares_file"` - RemoveTransferOnCancel bool `mapstructure:"remove_transfer_on_cancel"` + TxDriver string `mapstructure:"txdriver"` + TxDrivers map[string]map[string]interface{} `mapstructure:"txdrivers"` + StorageDriver string `mapstructure:"storagedriver"` + StorageDrivers map[string]map[string]interface{} `mapstructure:"storagedrivers"` + RemoveOnCancel bool `mapstructure:"remove_transfer_on_cancel"` } type service struct { conf *config txManager txdriver.Manager - txShareDriver *txShareDriver -} - -type txShareDriver struct { - sync.Mutex // concurrent access to the file - model *txShareModel -} -type txShareModel struct { - File string - TxShares map[string]*txShare `json:"shares"` -} - -type txShare struct { - TxID string - SrcTargetURI string - DestTargetURI string - ShareID string + storageDriver txdriver.Repository } func (c *config) init() { if c.TxDriver == "" { c.TxDriver = "rclone" } - if c.TxSharesFile == "" { - c.TxSharesFile = "/var/tmp/reva/datatx-shares.json" - } } func (s *service) Register(ss *grpc.Server) { @@ -94,6 +71,13 @@ func getDatatxManager(c *config) (txdriver.Manager, error) { return nil, errtypes.NotFound("datatx service: driver not found: " + c.TxDriver) } +func getStorageManager(c *config) (txdriver.Repository, error) { + if f, ok := repoRegistry.NewFuncs[c.StorageDriver]; ok { + return f(c.StorageDrivers[c.StorageDriver]) + } + return nil, errtypes.NotFound("datatx service: driver not found: " + c.StorageDriver) +} + func parseConfig(m map[string]interface{}) (*config, error) { c := &config{} if err := mapstructure.Decode(m, c); err != nil { @@ -116,19 +100,15 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { return nil, err } - model, err := loadOrCreate(c.TxSharesFile) + storageDriver, err := getStorageManager(c) if err != nil { - err = errors.Wrap(err, "datatx service: error loading the file containing the transfer shares") return nil, err } - txShareDriver := &txShareDriver{ - model: model, - } service := &service{ conf: c, txManager: txManager, - txShareDriver: txShareDriver, + storageDriver: storageDriver, } return service, nil @@ -147,18 +127,16 @@ func (s *service) CreateTransfer(ctx context.Context, req *datatx.CreateTransfer // we always save the transfer regardless of start transfer outcome // only then, if starting fails, can we try to restart it - txShare := &txShare{ + userID := ctxpkg.ContextMustGetUser(ctx).GetId() + transfer := &txdriver.Transfer{ TxID: txInfo.GetId().OpaqueId, SrcTargetURI: req.SrcTargetUri, DestTargetURI: req.DestTargetUri, ShareID: req.GetShareId().OpaqueId, + UserID: userID, } - s.txShareDriver.Lock() - defer s.txShareDriver.Unlock() - - s.txShareDriver.model.TxShares[txInfo.GetId().OpaqueId] = txShare - if err := s.txShareDriver.model.saveTxShare(); err != nil { - err = errors.Wrap(err, "datatx service: error saving transfer share: "+datatx.Status_STATUS_INVALID.String()) + if err := s.storageDriver.StoreTransfer(transfer); err != nil { + err = errors.Wrap(err, "datatx service: error NEW saving transfer share: "+datatx.Status_STATUS_INVALID.String()) return &datatx.CreateTransferResponse{ Status: status.NewInvalid(ctx, "error creating transfer"), }, err @@ -180,8 +158,8 @@ func (s *service) CreateTransfer(ctx context.Context, req *datatx.CreateTransfer } func (s *service) GetTransferStatus(ctx context.Context, req *datatx.GetTransferStatusRequest) (*datatx.GetTransferStatusResponse, error) { - txShare, ok := s.txShareDriver.model.TxShares[req.GetTxId().GetOpaqueId()] - if !ok { + transfer, err := s.storageDriver.GetTransfer(req.TxId.OpaqueId) + if err != nil { return nil, errtypes.InternalError("datatx service: transfer not found") } @@ -194,7 +172,7 @@ func (s *service) GetTransferStatus(ctx context.Context, req *datatx.GetTransfer }, err } - txInfo.ShareId = &ocm.ShareId{OpaqueId: txShare.ShareID} + txInfo.ShareId = &ocm.ShareId{OpaqueId: transfer.ShareID} return &datatx.GetTransferStatusResponse{ Status: status.NewOK(ctx), @@ -203,15 +181,14 @@ func (s *service) GetTransferStatus(ctx context.Context, req *datatx.GetTransfer } func (s *service) CancelTransfer(ctx context.Context, req *datatx.CancelTransferRequest) (*datatx.CancelTransferResponse, error) { - txShare, ok := s.txShareDriver.model.TxShares[req.GetTxId().OpaqueId] - if !ok { + transfer, err := s.storageDriver.GetTransfer(req.TxId.OpaqueId) + if err != nil { return nil, errtypes.InternalError("datatx service: transfer not found") } transferRemovedMessage := "" - if s.conf.RemoveTransferOnCancel { - delete(s.txShareDriver.model.TxShares, req.TxId.GetOpaqueId()) - if err := s.txShareDriver.model.saveTxShare(); err != nil { + if s.conf.RemoveOnCancel { + if err := s.storageDriver.DeleteTransfer(transfer); err != nil { err = errors.Wrap(err, "datatx service: error deleting transfer: "+datatx.Status_STATUS_INVALID.String()) return &datatx.CancelTransferResponse{ Status: status.NewInvalid(ctx, "error cancelling transfer"), @@ -222,7 +199,7 @@ func (s *service) CancelTransfer(ctx context.Context, req *datatx.CancelTransfer txInfo, err := s.txManager.CancelTransfer(ctx, req.GetTxId().OpaqueId) if err != nil { - txInfo.ShareId = &ocm.ShareId{OpaqueId: txShare.ShareID} + txInfo.ShareId = &ocm.ShareId{OpaqueId: transfer.ShareID} err = errors.Wrapf(err, "(%v) datatx service: error cancelling transfer", transferRemovedMessage) return &datatx.CancelTransferResponse{ Status: status.NewInternal(ctx, err, "error cancelling transfer"), @@ -230,7 +207,7 @@ func (s *service) CancelTransfer(ctx context.Context, req *datatx.CancelTransfer }, err } - txInfo.ShareId = &ocm.ShareId{OpaqueId: txShare.ShareID} + txInfo.ShareId = &ocm.ShareId{OpaqueId: transfer.ShareID} return &datatx.CancelTransferResponse{ Status: status.NewOK(ctx), @@ -239,26 +216,23 @@ func (s *service) CancelTransfer(ctx context.Context, req *datatx.CancelTransfer } func (s *service) ListTransfers(ctx context.Context, req *datatx.ListTransfersRequest) (*datatx.ListTransfersResponse, error) { - filters := req.Filters - var txInfos []*datatx.TxInfo - for _, txShare := range s.txShareDriver.model.TxShares { - if len(filters) == 0 { - txInfos = append(txInfos, &datatx.TxInfo{ - Id: &datatx.TxId{OpaqueId: txShare.TxID}, - ShareId: &ocm.ShareId{OpaqueId: txShare.ShareID}, - }) - } else { - for _, f := range filters { - if f.Type == datatx.ListTransfersRequest_Filter_TYPE_SHARE_ID { - if f.GetShareId().GetOpaqueId() == txShare.ShareID { - txInfos = append(txInfos, &datatx.TxInfo{ - Id: &datatx.TxId{OpaqueId: txShare.TxID}, - ShareId: &ocm.ShareId{OpaqueId: txShare.ShareID}, - }) - } - } - } - } + userID := ctxpkg.ContextMustGetUser(ctx).GetId() + transfers, err := s.storageDriver.ListTransfers(req.Filters, userID) + if err != nil { + err = errors.Wrap(err, "datatx service: error listing transfers") + var txInfos []*datatx.TxInfo + return &datatx.ListTransfersResponse{ + Status: status.NewInternal(ctx, err, "error listing transfers"), + Transfers: txInfos, + }, err + } + + txInfos := []*datatx.TxInfo{} + for _, transfer := range transfers { + txInfos = append(txInfos, &datatx.TxInfo{ + Id: &datatx.TxId{OpaqueId: transfer.TxID}, + ShareId: &ocm.ShareId{OpaqueId: transfer.ShareID}, + }) } return &datatx.ListTransfersResponse{ @@ -268,8 +242,8 @@ func (s *service) ListTransfers(ctx context.Context, req *datatx.ListTransfersRe } func (s *service) RetryTransfer(ctx context.Context, req *datatx.RetryTransferRequest) (*datatx.RetryTransferResponse, error) { - txShare, ok := s.txShareDriver.model.TxShares[req.GetTxId().GetOpaqueId()] - if !ok { + transfer, err := s.storageDriver.GetTransfer(req.TxId.OpaqueId) + if err != nil { return nil, errtypes.InternalError("datatx service: transfer not found") } @@ -282,61 +256,10 @@ func (s *service) RetryTransfer(ctx context.Context, req *datatx.RetryTransferRe }, err } - txInfo.ShareId = &ocm.ShareId{OpaqueId: txShare.ShareID} + txInfo.ShareId = &ocm.ShareId{OpaqueId: transfer.ShareID} return &datatx.RetryTransferResponse{ Status: status.NewOK(ctx), TxInfo: txInfo, }, nil } - -func loadOrCreate(file string) (*txShareModel, error) { - _, err := os.Stat(file) - if os.IsNotExist(err) { - if err := os.WriteFile(file, []byte("{}"), 0700); err != nil { - err = errors.Wrap(err, "datatx service: error creating the transfer shares storage file: "+file) - return nil, err - } - } - - fd, err := os.OpenFile(file, os.O_CREATE, 0644) - if err != nil { - err = errors.Wrap(err, "datatx service: error opening the transfer shares storage file: "+file) - return nil, err - } - defer fd.Close() - - data, err := io.ReadAll(fd) - if err != nil { - err = errors.Wrap(err, "datatx service: error reading the data") - return nil, err - } - - model := &txShareModel{} - if err := json.Unmarshal(data, model); err != nil { - err = errors.Wrap(err, "datatx service: error decoding transfer shares data to json") - return nil, err - } - - if model.TxShares == nil { - model.TxShares = make(map[string]*txShare) - } - - model.File = file - return model, nil -} - -func (m *txShareModel) saveTxShare() error { - data, err := json.Marshal(m) - if err != nil { - err = errors.Wrap(err, "datatx service: error encoding transfer share data to json") - return err - } - - if err := os.WriteFile(m.File, data, 0644); err != nil { - err = errors.Wrap(err, "datatx service: error writing transfer share data to file: "+m.File) - return err - } - - return nil -} diff --git a/pkg/datatx/datatx.go b/pkg/datatx/datatx.go index 8c430e2bbc..616a5a64ca 100644 --- a/pkg/datatx/datatx.go +++ b/pkg/datatx/datatx.go @@ -21,6 +21,7 @@ package datatx import ( "context" + userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" datatx "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1" ) @@ -37,3 +38,24 @@ type Manager interface { // Note that tokens must still be valid. RetryTransfer(ctx context.Context, transferID string) (*datatx.TxInfo, error) } + +// Transfer represents datatx transfer. +type Transfer struct { + TxID string + SrcTargetURI string + DestTargetURI string + ShareID string + UserID *userv1beta1.UserId +} + +// Repository the interface that any storage driver should implement. +type Repository interface { + // StoreTransfer stores the transfer by its TxID + StoreTransfer(transfer *Transfer) error + // StoreTransfer deletes the transfer by its TxID + DeleteTransfer(transfer *Transfer) error + // GetTransfer returns the transfer with the specified transfer id + GetTransfer(txID string) (*Transfer, error) + // ListTransfers returns a filtered list of transfers + ListTransfers(Filters []*datatx.ListTransfersRequest_Filter, UserID *userv1beta1.UserId) ([]*Transfer, error) +} diff --git a/pkg/datatx/manager/loader/loader.go b/pkg/datatx/manager/loader/loader.go index 1452dc5dae..2c9d77d745 100644 --- a/pkg/datatx/manager/loader/loader.go +++ b/pkg/datatx/manager/loader/loader.go @@ -21,5 +21,7 @@ package loader import ( // Load datatx drivers. _ "github.com/cs3org/reva/pkg/datatx/manager/rclone" + _ "github.com/cs3org/reva/pkg/datatx/manager/rclone/repository/json" + _ "github.com/cs3org/reva/pkg/datatx/repository/json" // Add your own here. ) diff --git a/pkg/datatx/manager/rclone/rclone.go b/pkg/datatx/manager/rclone/rclone.go index 0e9d2dfe55..932677a106 100644 --- a/pkg/datatx/manager/rclone/rclone.go +++ b/pkg/datatx/manager/rclone/rclone.go @@ -23,19 +23,18 @@ import ( "context" "encoding/json" "fmt" - "io" "net/http" "net/url" - "os" "path" "strconv" - "sync" "time" datatx "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" txdriver "github.com/cs3org/reva/pkg/datatx" + "github.com/cs3org/reva/pkg/datatx/manager/rclone/repository" + repoRegistry "github.com/cs3org/reva/pkg/datatx/manager/rclone/repository/registry" registry "github.com/cs3org/reva/pkg/datatx/manager/registry" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rhttp" @@ -50,9 +49,6 @@ func init() { func (c *config) init(m map[string]interface{}) { // set sane defaults - if c.File == "" { - c.File = "/var/tmp/reva/datatx-transfers.json" - } if c.JobStatusCheckInterval == 0 { c.JobStatusCheckInterval = 2000 } @@ -62,21 +58,22 @@ func (c *config) init(m map[string]interface{}) { } type config struct { - Endpoint string `mapstructure:"endpoint"` - AuthUser string `mapstructure:"auth_user"` // rclone basicauth user - AuthPass string `mapstructure:"auth_pass"` // rclone basicauth pass - AuthHeader string `mapstructure:"auth_header"` - File string `mapstructure:"file"` - JobStatusCheckInterval int `mapstructure:"job_status_check_interval"` - JobTimeout int `mapstructure:"job_timeout"` - Insecure bool `mapstructure:"insecure"` - RemoveTransferJobOnCancel bool `mapstructure:"remove_transfer_job_on_cancel"` + Endpoint string `mapstructure:"endpoint"` + AuthUser string `mapstructure:"auth_user"` // rclone basicauth user + AuthPass string `mapstructure:"auth_pass"` // rclone basicauth pass + AuthHeader string `mapstructure:"auth_header"` + JobStatusCheckInterval int `mapstructure:"job_status_check_interval"` + JobTimeout int `mapstructure:"job_timeout"` + Insecure bool `mapstructure:"insecure"` + RemoveTransferJobOnCancel bool `mapstructure:"remove_transfer_job_on_cancel"` + StorageDriver string `mapstructure:"storagedriver"` + StorageDrivers map[string]map[string]interface{} `mapstructure:"storagedrivers"` } type rclone struct { config *config client *http.Client - pDriver *pDriver + storage repository.Repository } type rcloneHTTPErrorRes struct { @@ -86,30 +83,6 @@ type rcloneHTTPErrorRes struct { Status int `json:"status"` } -type transferModel struct { - File string - Transfers map[string]*transfer `json:"transfers"` -} - -// persistency driver. -type pDriver struct { - sync.Mutex // concurrent access to the file - model *transferModel -} - -type transfer struct { - TransferID string - JobID int64 - TransferStatus datatx.Status - SrcToken string - SrcRemote string - SrcPath string - DestToken string - DestRemote string - DestPath string - Ctime string -} - // txEndStatuses final statuses that cannot be changed anymore. var txEndStatuses = map[string]int32{ "STATUS_INVALID": 0, @@ -138,21 +111,15 @@ func New(m map[string]interface{}) (txdriver.Manager, error) { client := rhttp.GetHTTPClient(rhttp.Insecure(c.Insecure)) - // The persistency driver - // Load or create 'db' - model, err := loadOrCreate(c.File) + storage, err := getStorageManager(c) if err != nil { - err = errors.Wrap(err, "error loading the file containing the transfers") return nil, err } - pDriver := &pDriver{ - model: model, - } return &rclone{ config: c, client: client, - pDriver: pDriver, + storage: storage, }, nil } @@ -165,56 +132,11 @@ func parseConfig(m map[string]interface{}) (*config, error) { return c, nil } -func loadOrCreate(file string) (*transferModel, error) { - _, err := os.Stat(file) - if os.IsNotExist(err) { - if err := os.WriteFile(file, []byte("{}"), 0700); err != nil { - err = errors.Wrap(err, "error creating the transfers storage file: "+file) - return nil, err - } - } - - fd, err := os.OpenFile(file, os.O_CREATE, 0644) - if err != nil { - err = errors.Wrap(err, "error opening the transfers storage file: "+file) - return nil, err - } - defer fd.Close() - - data, err := io.ReadAll(fd) - if err != nil { - err = errors.Wrap(err, "error reading the data") - return nil, err - } - - model := &transferModel{} - if err := json.Unmarshal(data, model); err != nil { - err = errors.Wrap(err, "error decoding transfers data to json") - return nil, err - } - - if model.Transfers == nil { - model.Transfers = make(map[string]*transfer) - } - - model.File = file - return model, nil -} - -// saveTransfer saves the transfer. If an error is specified than that error will be returned, possibly wrapped with additional errors. -func (m *transferModel) saveTransfer(e error) error { - data, err := json.Marshal(m) - if err != nil { - e = errors.Wrap(err, "error encoding transfer data to json") - return e - } - - if err := os.WriteFile(m.File, data, 0644); err != nil { - e = errors.Wrap(err, "error writing transfer data to file: "+m.File) - return e +func getStorageManager(c *config) (repository.Repository, error) { + if f, ok := repoRegistry.NewFuncs[c.StorageDriver]; ok { + return f(c.StorageDrivers[c.StorageDriver]) } - - return e + return nil, errtypes.NotFound("rclone service: storage driver not found: " + c.StorageDriver) } // CreateTransfer creates a transfer job and returns a TxInfo object that includes a unique transfer id. @@ -244,50 +166,54 @@ func (driver *rclone) CreateTransfer(ctx context.Context, srcTargetURI string, d func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote string, srcPath string, srcToken string, destRemote string, destPath string, destToken string) (*datatx.TxInfo, error) { logger := appctx.GetLogger(ctx) - driver.pDriver.Lock() - defer driver.pDriver.Unlock() - var txID string var cTime *typespb.Timestamp if transferID == "" { txID = uuid.New().String() cTime = &typespb.Timestamp{Seconds: uint64(time.Now().Unix())} - } else { // restart existing transfer if transferID is specified - logger.Debug().Msgf("Restarting transfer (txID: %s)", transferID) + } else { // restart existing transfer job if transferID is specified + logger.Debug().Msgf("Restarting transfer job (txID: %s)", transferID) txID = transferID - transfer, err := driver.pDriver.model.getTransfer(txID) + job, err := driver.storage.GetJob(txID) if err != nil { - err = errors.Wrap(err, "rclone: error retrying transfer (transferID: "+txID+")") + err = errors.Wrap(err, "rclone: error retrying transfer job (transferID: "+txID+")") return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: txID}, Status: datatx.Status_STATUS_INVALID, Ctime: nil, }, err } - seconds, _ := strconv.ParseInt(transfer.Ctime, 10, 64) + seconds, _ := strconv.ParseInt(job.Ctime, 10, 64) cTime = &typespb.Timestamp{Seconds: uint64(seconds)} - _, endStatusFound := txEndStatuses[transfer.TransferStatus.String()] + _, endStatusFound := txEndStatuses[job.TransferStatus.String()] if !endStatusFound { - err := errors.New("rclone: transfer still running, unable to restart") + err := errors.New("rclone: job still running, unable to restart") + return &datatx.TxInfo{ + Id: &datatx.TxId{OpaqueId: txID}, + Status: job.TransferStatus, + Ctime: cTime, + }, err + } + srcToken = job.SrcToken + srcRemote = job.SrcRemote + srcPath = job.SrcPath + destToken = job.DestToken + destRemote = job.DestRemote + destPath = job.DestPath + if err := driver.storage.DeleteJob(job); err != nil { + err = errors.Wrap(err, "rclone: transfer still running, unable to restart") return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: txID}, - Status: transfer.TransferStatus, + Status: job.TransferStatus, Ctime: cTime, }, err } - srcToken = transfer.SrcToken - srcRemote = transfer.SrcRemote - srcPath = transfer.SrcPath - destToken = transfer.DestToken - destRemote = transfer.DestRemote - destPath = transfer.DestPath - delete(driver.pDriver.model.Transfers, txID) } transferStatus := datatx.Status_STATUS_TRANSFER_NEW - transfer := &transfer{ + job := &repository.Job{ TransferID: txID, JobID: int64(-1), TransferStatus: transferStatus, @@ -300,8 +226,6 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote Ctime: fmt.Sprint(cTime.Seconds), // TODO do we need nanos here? } - driver.pDriver.model.Transfers[txID] = transfer - type rcloneAsyncReqJSON struct { SrcFs string `json:"srcFs"` DstFs string `json:"dstFs"` @@ -325,70 +249,94 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote } data, err := json.Marshal(rcloneReq) if err != nil { - err = errors.Wrap(err, "rclone: error pulling transfer: error marshalling rclone req data") - transfer.TransferStatus = datatx.Status_STATUS_INVALID + err = errors.Wrap(err, "rclone: transfer job error: error marshalling rclone req data") + job.TransferStatus = datatx.Status_STATUS_INVALID + var e error + if e = driver.storage.StoreJob(job); e != nil { + e = errors.Wrap(e, err.Error()) + } return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: txID}, - Status: datatx.Status_STATUS_INVALID, + Status: job.TransferStatus, Ctime: cTime, - }, driver.pDriver.model.saveTransfer(err) + }, e } transferFileMethod := "/sync/copy" remotePathIsFolder, err := driver.remotePathIsFolder(srcRemote, srcPath, srcToken) if err != nil { - err = errors.Wrap(err, "rclone: error pulling transfer: error stating src path") - transfer.TransferStatus = datatx.Status_STATUS_INVALID + err = errors.Wrap(err, "rclone: transfer job error: error stating src path") + job.TransferStatus = datatx.Status_STATUS_INVALID + var e error + if e = driver.storage.StoreJob(job); e != nil { + e = errors.Wrap(e, err.Error()) + } return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: txID}, - Status: datatx.Status_STATUS_INVALID, + Status: job.TransferStatus, Ctime: cTime, - }, driver.pDriver.model.saveTransfer(err) + }, e } if !remotePathIsFolder { - err = errors.Wrap(err, "rclone: error pulling transfer: path is a file, only folder transfer is implemented") - transfer.TransferStatus = datatx.Status_STATUS_INVALID + err = errors.Wrap(err, "rclone: transfer job error: path is a file, only folder transfer is implemented") + job.TransferStatus = datatx.Status_STATUS_INVALID + var e error + if e = driver.storage.StoreJob(job); e != nil { + e = errors.Wrap(e, err.Error()) + } return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: txID}, - Status: datatx.Status_STATUS_INVALID, + Status: job.TransferStatus, Ctime: cTime, - }, driver.pDriver.model.saveTransfer(err) + }, e } u, err := url.Parse(driver.config.Endpoint) if err != nil { - err = errors.Wrap(err, "rclone: error pulling transfer: error parsing driver endpoint") - transfer.TransferStatus = datatx.Status_STATUS_INVALID + err = errors.Wrap(err, "rclone: transfer job error: error parsing driver endpoint") + job.TransferStatus = datatx.Status_STATUS_INVALID + var e error + if e = driver.storage.StoreJob(job); e != nil { + e = errors.Wrap(e, err.Error()) + } return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: txID}, - Status: datatx.Status_STATUS_INVALID, + Status: job.TransferStatus, Ctime: cTime, - }, driver.pDriver.model.saveTransfer(err) + }, e } u.Path = path.Join(u.Path, transferFileMethod) requestURL := u.String() req, err := http.NewRequest(http.MethodPost, requestURL, bytes.NewReader(data)) if err != nil { - err = errors.Wrap(err, "rclone: error pulling transfer: error framing post request") - transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED + err = errors.Wrap(err, "rclone: transfer job error: error framing post request") + job.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED + var e error + if e = driver.storage.StoreJob(job); e != nil { + e = errors.Wrap(e, err.Error()) + } return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: txID}, - Status: transfer.TransferStatus, + Status: job.TransferStatus, Ctime: cTime, - }, driver.pDriver.model.saveTransfer(err) + }, e } req.Header.Set("Content-Type", "application/json") req.SetBasicAuth(driver.config.AuthUser, driver.config.AuthPass) res, err := driver.client.Do(req) if err != nil { - err = errors.Wrap(err, "rclone: error pulling transfer: error sending post request") - transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED + err = errors.Wrap(err, "rclone: transfer job error: error sending post request") + job.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED + var e error + if e = driver.storage.StoreJob(job); e != nil { + e = errors.Wrap(e, err.Error()) + } return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: txID}, - Status: transfer.TransferStatus, + Status: job.TransferStatus, Ctime: cTime, - }, driver.pDriver.model.saveTransfer(err) + }, e } defer res.Body.Close() @@ -396,21 +344,29 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote if res.StatusCode != http.StatusOK { var errorResData rcloneHTTPErrorRes if err = json.NewDecoder(res.Body).Decode(&errorResData); err != nil { - err = errors.Wrap(err, "rclone driver: error decoding response data") - transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED + err = errors.Wrap(err, "rclone driver: error decoding rclone response data") + job.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED + var e error + if e = driver.storage.StoreJob(job); e != nil { + e = errors.Wrap(e, err.Error()) + } return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: txID}, - Status: transfer.TransferStatus, + Status: job.TransferStatus, Ctime: cTime, - }, driver.pDriver.model.saveTransfer(err) + }, e + } + err := errors.New("rclone driver: rclone request responded with error, " + fmt.Sprintf(" status: %v, error: %v", errorResData.Status, errorResData.Error)) + job.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED + var e error + if e = driver.storage.StoreJob(job); e != nil { + e = errors.Wrap(e, err.Error()) } - e := errors.New("rclone: rclone request responded with error, " + fmt.Sprintf(" status: %v, error: %v", errorResData.Status, errorResData.Error)) - transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: txID}, - Status: transfer.TransferStatus, + Status: job.TransferStatus, Ctime: cTime, - }, driver.pDriver.model.saveTransfer(e) + }, e } type rcloneAsyncResJSON struct { @@ -418,19 +374,33 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote } var resData rcloneAsyncResJSON if err = json.NewDecoder(res.Body).Decode(&resData); err != nil { - err = errors.Wrap(err, "rclone: error decoding response data") - transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED + err = errors.Wrap(err, "rclone driver: error decoding response data") + job.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED + var e error + if e = driver.storage.StoreJob(job); e != nil { + e = errors.Wrap(e, err.Error()) + } return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: txID}, - Status: transfer.TransferStatus, + Status: job.TransferStatus, Ctime: cTime, - }, driver.pDriver.model.saveTransfer(err) + }, e } - transfer.JobID = resData.JobID + job.JobID = resData.JobID + + if err := driver.storage.StoreJob(job); err != nil { + err = errors.Wrap(err, "rclone driver: transfer job error") + return &datatx.TxInfo{ + Id: &datatx.TxId{OpaqueId: txID}, + Status: datatx.Status_STATUS_INVALID, + Ctime: cTime, + }, err + } - if err := driver.pDriver.model.saveTransfer(nil); err != nil { - err = errors.Wrap(err, "rclone: error pulling transfer") + // the initial save when everything went ok + if err := driver.storage.StoreJob(job); err != nil { + err = errors.Wrap(err, "rclone driver: error starting transfer job") return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: txID}, Status: datatx.Status_STATUS_INVALID, @@ -444,22 +414,17 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote startTimeMs := time.Now().Nanosecond() / 1000 timeout := driver.config.JobTimeout - driver.pDriver.Lock() - defer driver.pDriver.Unlock() - for { - transfer, err := driver.pDriver.model.getTransfer(txID) + job, err := driver.storage.GetJob(txID) if err != nil { - transfer.TransferStatus = datatx.Status_STATUS_INVALID - err = driver.pDriver.model.saveTransfer(err) - logger.Error().Err(err).Msgf("rclone driver: unable to retrieve transfer with id: %v", txID) + logger.Error().Err(err).Msgf("rclone driver: unable to retrieve transfer job with id: %v", txID) break } // check for end status first - _, endStatusFound := txEndStatuses[transfer.TransferStatus.String()] - if endStatusFound { - logger.Info().Msgf("rclone driver: transfer endstatus reached: %v", transfer.TransferStatus) + _, endStatusreached := txEndStatuses[job.TransferStatus.String()] + if endStatusreached { + logger.Info().Msgf("rclone driver: transfer job endstatus reached: %v", job.TransferStatus) break } @@ -468,16 +433,16 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote timePastMs := currentTimeMs - startTimeMs if timePastMs > timeout { - logger.Info().Msgf("rclone driver: transfer timed out: %vms (timeout = %v)", timePastMs, timeout) + logger.Info().Msgf("rclone driver: transfer job timed out: %vms (timeout = %v)", timePastMs, timeout) // set status to EXPIRED and save - transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_EXPIRED - if err := driver.pDriver.model.saveTransfer(nil); err != nil { - logger.Error().Err(err).Msgf("rclone driver: save transfer failed: %v", err) + job.TransferStatus = datatx.Status_STATUS_TRANSFER_EXPIRED + if err := driver.storage.StoreJob(job); err != nil { + logger.Error().Err(err).Msgf("rclone driver: save transfer job failed: %v", err) } break } - jobID := transfer.JobID + jobID := job.JobID type rcloneStatusReqJSON struct { JobID int64 `json:"jobid"` } @@ -488,9 +453,9 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote data, err := json.Marshal(rcloneStatusReq) if err != nil { logger.Error().Err(err).Msgf("rclone driver: marshalling request failed: %v", err) - transfer.TransferStatus = datatx.Status_STATUS_INVALID - if err := driver.pDriver.model.saveTransfer(nil); err != nil { - logger.Error().Err(err).Msgf("rclone driver: save transfer failed: %v", err) + job.TransferStatus = datatx.Status_STATUS_INVALID + if err := driver.storage.StoreJob(job); err != nil { + logger.Error().Err(err).Msgf("rclone driver: save transfer job failed: %v", err) } break } @@ -500,9 +465,9 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote u, err := url.Parse(driver.config.Endpoint) if err != nil { logger.Error().Err(err).Msgf("rclone driver: could not parse driver endpoint: %v", err) - transfer.TransferStatus = datatx.Status_STATUS_INVALID - if err := driver.pDriver.model.saveTransfer(nil); err != nil { - logger.Error().Err(err).Msgf("rclone driver: save transfer failed: %v", err) + job.TransferStatus = datatx.Status_STATUS_INVALID + if err := driver.storage.StoreJob(job); err != nil { + logger.Error().Err(err).Msgf("rclone driver: save transfer job failed: %v", err) } break } @@ -512,9 +477,9 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote req, err := http.NewRequest(http.MethodPost, requestURL, bytes.NewReader(data)) if err != nil { logger.Error().Err(err).Msgf("rclone driver: error framing post request: %v", err) - transfer.TransferStatus = datatx.Status_STATUS_INVALID - if err := driver.pDriver.model.saveTransfer(nil); err != nil { - logger.Error().Err(err).Msgf("rclone driver: save transfer failed: %v", err) + job.TransferStatus = datatx.Status_STATUS_INVALID + if err := driver.storage.StoreJob(job); err != nil { + logger.Error().Err(err).Msgf("rclone driver: save transfer job failed: %v", err) } break } @@ -523,9 +488,9 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote res, err := driver.client.Do(req) if err != nil { logger.Error().Err(err).Msgf("rclone driver: error sending post request: %v", err) - transfer.TransferStatus = datatx.Status_STATUS_INVALID - if err := driver.pDriver.model.saveTransfer(nil); err != nil { - logger.Error().Err(err).Msgf("rclone driver: save transfer failed: %v", err) + job.TransferStatus = datatx.Status_STATUS_INVALID + if err := driver.storage.StoreJob(job); err != nil { + logger.Error().Err(err).Msgf("rclone driver: save transfer job failed: %v", err) } break } @@ -539,9 +504,9 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote logger.Error().Err(err).Msgf("rclone driver: error reading response body: %v", err) } logger.Error().Err(err).Msgf("rclone driver: rclone request responded with error, status: %v, error: %v", errorResData.Status, errorResData.Error) - transfer.TransferStatus = datatx.Status_STATUS_INVALID - if err := driver.pDriver.model.saveTransfer(nil); err != nil { - logger.Error().Err(err).Msgf("rclone driver: save transfer failed: %v", err) + job.TransferStatus = datatx.Status_STATUS_INVALID + if err := driver.storage.StoreJob(job); err != nil { + logger.Error().Err(err).Msgf("rclone driver: save transfer job failed: %v", err) } break } @@ -566,9 +531,9 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote if resData.Error != "" { logger.Error().Err(err).Msgf("rclone driver: rclone responded with error: %v", resData.Error) - transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED - if err := driver.pDriver.model.saveTransfer(nil); err != nil { - logger.Error().Err(err).Msgf("rclone driver: error saving transfer: %v", err) + job.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED + if err := driver.storage.StoreJob(job); err != nil { + logger.Error().Err(err).Msgf("rclone driver: error saving transfer job: %v", err) break } break @@ -577,9 +542,9 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote // transfer complete if resData.Finished && resData.Success { logger.Info().Msg("rclone driver: transfer job finished") - transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_COMPLETE - if err := driver.pDriver.model.saveTransfer(nil); err != nil { - logger.Error().Err(err).Msgf("rclone driver: error saving transfer: %v", err) + job.TransferStatus = datatx.Status_STATUS_TRANSFER_COMPLETE + if err := driver.storage.StoreJob(job); err != nil { + logger.Error().Err(err).Msgf("rclone driver: error saving transfer job: %v", err) break } break @@ -588,9 +553,9 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote // transfer completed unsuccessfully without error if resData.Finished && !resData.Success { logger.Info().Msgf("rclone driver: transfer job failed") - transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED - if err := driver.pDriver.model.saveTransfer(nil); err != nil { - logger.Error().Err(err).Msgf("rclone driver: error saving transfer: %v", err) + job.TransferStatus = datatx.Status_STATUS_TRANSFER_FAILED + if err := driver.storage.StoreJob(job); err != nil { + logger.Error().Err(err).Msgf("rclone driver: error saving transfer job: %v", err) break } break @@ -599,9 +564,9 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote // transfer not yet finished: continue if !resData.Finished { logger.Info().Msgf("rclone driver: transfer job in progress") - transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_IN_PROGRESS - if err := driver.pDriver.model.saveTransfer(nil); err != nil { - logger.Error().Err(err).Msgf("rclone driver: error saving transfer: %v", err) + job.TransferStatus = datatx.Status_STATUS_TRANSFER_IN_PROGRESS + if err := driver.storage.StoreJob(job); err != nil { + logger.Error().Err(err).Msgf("rclone driver: error saving transfer job: %v", err) break } } @@ -619,7 +584,7 @@ func (driver *rclone) startJob(ctx context.Context, transferID string, srcRemote // GetTransferStatus returns the status of the transfer with the specified job id. func (driver *rclone) GetTransferStatus(ctx context.Context, transferID string) (*datatx.TxInfo, error) { - transfer, err := driver.pDriver.model.getTransfer(transferID) + job, err := driver.storage.GetJob(transferID) if err != nil { return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: transferID}, @@ -627,17 +592,17 @@ func (driver *rclone) GetTransferStatus(ctx context.Context, transferID string) Ctime: nil, }, err } - cTime, _ := strconv.ParseInt(transfer.Ctime, 10, 64) + cTime, _ := strconv.ParseInt(job.Ctime, 10, 64) return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: transferID}, - Status: transfer.TransferStatus, + Status: job.TransferStatus, Ctime: &typespb.Timestamp{Seconds: uint64(cTime)}, }, nil } // CancelTransfer cancels the transfer with the specified transfer id. func (driver *rclone) CancelTransfer(ctx context.Context, transferID string) (*datatx.TxInfo, error) { - transfer, err := driver.pDriver.model.getTransfer(transferID) + job, err := driver.storage.GetJob(transferID) if err != nil { return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: transferID}, @@ -646,11 +611,11 @@ func (driver *rclone) CancelTransfer(ctx context.Context, transferID string) (*d }, err } - cTime, _ := strconv.ParseInt(transfer.Ctime, 10, 64) + cTime, _ := strconv.ParseInt(job.Ctime, 10, 64) + // rclone cancel may fail so remove job from model first to be sure transferRemovedMessage := "" if driver.config.RemoveTransferJobOnCancel { - delete(driver.pDriver.model.Transfers, transfer.TransferID) - if err := driver.pDriver.model.saveTransfer(nil); err != nil { + if err := driver.storage.DeleteJob(job); err != nil { return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: transferID}, Status: datatx.Status_STATUS_INVALID, @@ -660,9 +625,9 @@ func (driver *rclone) CancelTransfer(ctx context.Context, transferID string) (*d transferRemovedMessage = "(transfer job successfully removed)" } - _, endStatusFound := txEndStatuses[transfer.TransferStatus.String()] + _, endStatusFound := txEndStatuses[job.TransferStatus.String()] if endStatusFound { - err := errors.Wrapf(errors.New("rclone driver: transfer already in end state"), transferRemovedMessage) + err := errors.Wrapf(errors.New("rclone driver: job already in end state"), transferRemovedMessage) return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: transferID}, Status: datatx.Status_STATUS_INVALID, @@ -675,12 +640,12 @@ func (driver *rclone) CancelTransfer(ctx context.Context, transferID string) (*d JobID int64 `json:"jobid"` } rcloneCancelTransferReq := &rcloneStopRequest{ - JobID: transfer.JobID, + JobID: job.JobID, } data, err := json.Marshal(rcloneCancelTransferReq) if err != nil { - err := errors.Wrapf(errors.New("rclone driver: error marshalling rclone req data"), transferRemovedMessage) + err := errors.Wrapf(errors.New("rclone driver: error marshalling rclone job/stop req data"), transferRemovedMessage) return &datatx.TxInfo{ Id: &datatx.TxId{OpaqueId: transferID}, Status: datatx.Status_STATUS_INVALID, @@ -775,13 +740,16 @@ func (driver *rclone) CancelTransfer(ctx context.Context, transferID string) (*d }, errors.New(resData.Error) } - transfer.TransferStatus = datatx.Status_STATUS_TRANSFER_CANCELLED - if err := driver.pDriver.model.saveTransfer(nil); err != nil { - return &datatx.TxInfo{ - Id: &datatx.TxId{OpaqueId: transferID}, - Status: datatx.Status_STATUS_INVALID, - Ctime: &typespb.Timestamp{Seconds: uint64(cTime)}, - }, err + // only update when job's not removed + if !driver.config.RemoveTransferJobOnCancel { + job.TransferStatus = datatx.Status_STATUS_TRANSFER_CANCELLED + if err := driver.storage.StoreJob(job); err != nil { + return &datatx.TxInfo{ + Id: &datatx.TxId{OpaqueId: transferID}, + Status: datatx.Status_STATUS_INVALID, + Ctime: &typespb.Timestamp{Seconds: uint64(cTime)}, + }, err + } } return &datatx.TxInfo{ @@ -797,15 +765,6 @@ func (driver *rclone) RetryTransfer(ctx context.Context, transferID string) (*da return driver.startJob(ctx, transferID, "", "", "", "", "", "") } -// getTransfer returns the transfer with the specified transfer ID. -func (m *transferModel) getTransfer(transferID string) (*transfer, error) { - transfer, ok := m.Transfers[transferID] - if !ok { - return nil, errors.New("rclone driver: invalid transfer ID") - } - return transfer, nil -} - func (driver *rclone) remotePathIsFolder(remote string, remotePath string, remoteToken string) (bool, error) { type rcloneListReqJSON struct { Fs string `json:"fs"` diff --git a/pkg/datatx/manager/rclone/repository/json/json.go b/pkg/datatx/manager/rclone/repository/json/json.go new file mode 100644 index 0000000000..2d31f70ff1 --- /dev/null +++ b/pkg/datatx/manager/rclone/repository/json/json.go @@ -0,0 +1,171 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package json + +import ( + "encoding/json" + "io" + "os" + "sync" + + "github.com/cs3org/reva/pkg/datatx/manager/rclone/repository" + "github.com/cs3org/reva/pkg/datatx/manager/rclone/repository/registry" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +func init() { + registry.Register("json", New) +} + +type config struct { + File string `mapstructure:"file"` +} + +type mgr struct { + config *config + sync.Mutex // concurrent access to the file + model *rcloneJobsModel +} + +type rcloneJobsModel struct { + RcloneJobs map[string]*repository.Job `json:"rcloneJobs"` +} + +func parseConfig(m map[string]interface{}) (*config, error) { + c := &config{} + if err := mapstructure.Decode(m, c); err != nil { + err = errors.Wrap(err, "rclone repository json driver: error decoding configuration") + return nil, err + } + return c, nil +} + +func (c *config) init() { + if c.File == "" { + c.File = "/var/tmp/reva/transfer-jobs.json" + } +} + +// New returns a json storage driver. +func New(m map[string]interface{}) (repository.Repository, error) { + c, err := parseConfig(m) + if err != nil { + return nil, err + } + c.init() + + model, err := loadOrCreate(c.File) + if err != nil { + err = errors.Wrap(err, "rclone repository json driver: error loading the file containing the transfer shares") + return nil, err + } + + mgr := &mgr{ + config: c, + model: model, + } + + return mgr, nil +} + +func (m *mgr) StoreJob(job *repository.Job) error { + m.Lock() + defer m.Unlock() + + m.model.RcloneJobs[job.TransferID] = job + err := m.saveModel() + if err != nil { + return errors.Wrap(err, "error storing jobs") + } + + return nil +} + +func (m *mgr) GetJob(transferID string) (*repository.Job, error) { + m.Lock() + defer m.Unlock() + + job, ok := m.model.RcloneJobs[transferID] + if !ok { + return nil, errors.New("rclone repository json driver: error getting job: not found") + } + return job, nil +} + +func (m *mgr) DeleteJob(job *repository.Job) error { + m.Lock() + defer m.Unlock() + + delete(m.model.RcloneJobs, job.TransferID) + if err := m.saveModel(); err != nil { + return errors.New("rclone repository json driver: error deleting job: error updating model") + } + return nil +} + +func (m *mgr) saveModel() error { + data, err := json.Marshal(m.model) + if err != nil { + err = errors.Wrap(err, "rclone repository json driver: error encoding job data to json") + return err + } + + if err := os.WriteFile(m.config.File, data, 0644); err != nil { + err = errors.Wrap(err, "rclone repository json driver: error writing job data to file: "+m.config.File) + return err + } + + return nil +} + +func loadOrCreate(file string) (*rcloneJobsModel, error) { + _, err := os.Stat(file) + if os.IsNotExist(err) { + if err := os.WriteFile(file, []byte("{}"), 0700); err != nil { + err = errors.Wrap(err, "rclone repository json driver: error creating the jobs storage file: "+file) + return nil, err + } + } + + fd, err := os.OpenFile(file, os.O_CREATE, 0644) + if err != nil { + err = errors.Wrap(err, "rclone repository json driver: error opening the jobs storage file: "+file) + return nil, err + } + defer fd.Close() + + data, err := io.ReadAll(fd) + if err != nil { + err = errors.Wrap(err, "rclone repository json driver: error reading the data") + return nil, err + } + + model := &rcloneJobsModel{} + if err := json.Unmarshal(data, model); err != nil { + err = errors.Wrap(err, "rclone repository json driver: error decoding jobs data to json") + return nil, err + } + + if model.RcloneJobs == nil { + model.RcloneJobs = make(map[string]*repository.Job) + } + + return model, nil +} diff --git a/pkg/datatx/manager/rclone/repository/registry/registry.go b/pkg/datatx/manager/rclone/repository/registry/registry.go new file mode 100644 index 0000000000..113586f3ad --- /dev/null +++ b/pkg/datatx/manager/rclone/repository/registry/registry.go @@ -0,0 +1,36 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package registry + +import ( + "github.com/cs3org/reva/pkg/datatx/manager/rclone/repository" +) + +// NewFunc is the function that rclone repository implementations +// should register at init time. +type NewFunc func(map[string]interface{}) (repository.Repository, error) + +// NewFuncs is a map containing all the registered datatx backends. +var NewFuncs = map[string]NewFunc{} + +// Register registers a new datatx backend new function. +// Not safe for concurrent use. Safe for use from package init. +func Register(name string, f NewFunc) { + NewFuncs[name] = f +} diff --git a/pkg/datatx/manager/rclone/repository/repository.go b/pkg/datatx/manager/rclone/repository/repository.go new file mode 100644 index 0000000000..fd2bcbe927 --- /dev/null +++ b/pkg/datatx/manager/rclone/repository/repository.go @@ -0,0 +1,44 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package repository + +import ( + datatx "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1" +) + +// Job represents transfer job. +type Job struct { + TransferID string + JobID int64 + TransferStatus datatx.Status + SrcToken string + SrcRemote string + SrcPath string + DestToken string + DestRemote string + DestPath string + Ctime string +} + +// Repository the interface that any storage driver should implement. +type Repository interface { + StoreJob(job *Job) error + GetJob(transferID string) (*Job, error) + DeleteJob(job *Job) error +} diff --git a/pkg/datatx/repository/json/json.go b/pkg/datatx/repository/json/json.go new file mode 100644 index 0000000000..7fcf18cb78 --- /dev/null +++ b/pkg/datatx/repository/json/json.go @@ -0,0 +1,211 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package json + +import ( + "encoding/json" + "io" + "os" + "sync" + + userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + txv1beta "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1" + "github.com/cs3org/reva/pkg/datatx" + "github.com/cs3org/reva/pkg/datatx/repository/registry" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +func init() { + registry.Register("json", New) +} + +type config struct { + File string `mapstructure:"file"` +} + +type mgr struct { + config *config + sync.Mutex // concurrent access to the file + model *transfersModel +} + +type transfersModel struct { + Transfers map[string]*datatx.Transfer `json:"transfers"` +} + +func parseConfig(m map[string]interface{}) (*config, error) { + c := &config{} + if err := mapstructure.Decode(m, c); err != nil { + err = errors.Wrap(err, "datatx repository json driver: error decoding configuration") + return nil, err + } + return c, nil +} + +func (c *config) init() { + if c.File == "" { + c.File = "/var/tmp/reva/datatx-transfers.json" + } +} + +// New returns a json storage driver. +func New(m map[string]interface{}) (datatx.Repository, error) { + c, err := parseConfig(m) + if err != nil { + return nil, err + } + c.init() + + model, err := loadOrCreate(c.File) + if err != nil { + err = errors.Wrap(err, "datatx repository json driver: error loading the file containing the transfer shares") + return nil, err + } + + mgr := &mgr{ + config: c, + model: model, + } + + return mgr, nil +} + +func (m *mgr) StoreTransfer(transfer *datatx.Transfer) error { + m.Lock() + defer m.Unlock() + + m.model.Transfers[transfer.TxID] = transfer + err := m.saveModel() + if err != nil { + return errors.Wrap(err, "error storing transfer") + } + + return nil +} + +func (m *mgr) DeleteTransfer(transfer *datatx.Transfer) error { + m.Lock() + defer m.Unlock() + + delete(m.model.Transfers, transfer.TxID) + if err := m.saveModel(); err != nil { + return errors.New("datatx repository json driver: error deleting transfer: error updating model") + } + return nil +} + +func (m *mgr) GetTransfer(txID string) (*datatx.Transfer, error) { + m.Lock() + defer m.Unlock() + + transfer, ok := m.model.Transfers[txID] + if !ok { + return nil, errors.New("datatx repository json driver: error getting transfer: not found") + } + return transfer, nil +} + +func (m *mgr) ListTransfers(filters []*txv1beta.ListTransfersRequest_Filter, userID *userv1beta1.UserId) ([]*datatx.Transfer, error) { + m.Lock() + defer m.Unlock() + + var transfers []*datatx.Transfer + if userID == nil { + return transfers, errors.New("datatx repository json driver: error listing transfers, userID must be provided") + } + for _, transfer := range m.model.Transfers { + if transfer.UserID.OpaqueId == userID.OpaqueId { + if len(filters) == 0 { + transfers = append(transfers, &datatx.Transfer{ + TxID: transfer.TxID, + SrcTargetURI: transfer.SrcTargetURI, + DestTargetURI: transfer.DestTargetURI, + ShareID: transfer.ShareID, + UserID: transfer.UserID, + }) + } else { + for _, f := range filters { + if f.Type == txv1beta.ListTransfersRequest_Filter_TYPE_SHARE_ID { + if f.GetShareId().GetOpaqueId() == transfer.ShareID { + transfers = append(transfers, &datatx.Transfer{ + TxID: transfer.TxID, + SrcTargetURI: transfer.SrcTargetURI, + DestTargetURI: transfer.DestTargetURI, + ShareID: transfer.ShareID, + UserID: transfer.UserID, + }) + } + } + } + } + } + } + return transfers, nil +} + +func (m *mgr) saveModel() error { + data, err := json.Marshal(m.model) + if err != nil { + err = errors.Wrap(err, "datatx repository json driver: error encoding transfer data to json") + return err + } + + if err := os.WriteFile(m.config.File, data, 0644); err != nil { + err = errors.Wrap(err, "datatx repository json driver: error writing transfer data to file: "+m.config.File) + return err + } + + return nil +} + +func loadOrCreate(file string) (*transfersModel, error) { + _, err := os.Stat(file) + if os.IsNotExist(err) { + if err := os.WriteFile(file, []byte("{}"), 0700); err != nil { + err = errors.Wrap(err, "datatx repository json driver: error creating the datatx shares storage file: "+file) + return nil, err + } + } + + fd, err := os.OpenFile(file, os.O_CREATE, 0644) + if err != nil { + err = errors.Wrap(err, "datatx repository json driver: error opening the datatx shares storage file: "+file) + return nil, err + } + defer fd.Close() + + data, err := io.ReadAll(fd) + if err != nil { + err = errors.Wrap(err, "datatx repository json driver: error reading the data") + return nil, err + } + + model := &transfersModel{} + if err := json.Unmarshal(data, model); err != nil { + err = errors.Wrap(err, "datatx repository json driver: error decoding datatx shares data to json") + return nil, err + } + + if model.Transfers == nil { + model.Transfers = make(map[string]*datatx.Transfer) + } + + return model, nil +} diff --git a/pkg/datatx/repository/registry/registry.go b/pkg/datatx/repository/registry/registry.go new file mode 100644 index 0000000000..66dceb7d6a --- /dev/null +++ b/pkg/datatx/repository/registry/registry.go @@ -0,0 +1,36 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package registry + +import ( + "github.com/cs3org/reva/pkg/datatx" +) + +// NewFunc is the function that datatx repository implementations +// should register at init time. +type NewFunc func(map[string]interface{}) (datatx.Repository, error) + +// NewFuncs is a map containing all the registered datatx repository backends. +var NewFuncs = map[string]NewFunc{} + +// Register registers a new datatx repository backend new function. +// Not safe for concurrent use. Safe for use from package init. +func Register(name string, f NewFunc) { + NewFuncs[name] = f +} From 0545ea3bdd7b363bf7ab8c92ca2df040e8587bc4 Mon Sep 17 00:00:00 2001 From: Swikriti Tripathi <41103328+SwikritiT@users.noreply.github.com> Date: Wed, 31 May 2023 17:09:03 +0545 Subject: [PATCH 06/48] [tests-only][full-ci]Bump commit id for tests (#3924) * Bump commit id for tests * update expected to fail --- .drone.env | 2 +- .../expected-failures-on-OCIS-storage.md | 88 +++++++++--------- .../expected-failures-on-S3NG-storage.md | 89 +++++++++---------- tests/ocis | 2 +- 4 files changed, 88 insertions(+), 93 deletions(-) diff --git a/.drone.env b/.drone.env index 4280304e6f..f3660b888d 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=dc7418b3c3dc34349461b3997fb293a70466d804 +APITESTS_COMMITID=7094891f4de381102b05c6503751dc85d82c0782 APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index 9e2516f4e1..eaa4a0fbeb 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -199,12 +199,12 @@ File and sync features in a shared scenario #### [copying a folder within a public link folder to folder with same name as an already existing file overwrites the parent file](https://github.com/owncloud/ocis/issues/1232) -- [coreApiSharePublicLink2/copyFromPublicLink.feature:63](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L63) -- [coreApiSharePublicLink2/copyFromPublicLink.feature:89](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L89) -- [coreApiSharePublicLink2/copyFromPublicLink.feature:173](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L173) -- [coreApiSharePublicLink2/copyFromPublicLink.feature:174](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L174) -- [coreApiSharePublicLink2/copyFromPublicLink.feature:189](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L189) -- [coreApiSharePublicLink2/copyFromPublicLink.feature:190](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L190) +- [coreApiSharePublicLink2/copyFromPublicLink.feature:66](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L66) +- [coreApiSharePublicLink2/copyFromPublicLink.feature:92](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L92) +- [coreApiSharePublicLink2/copyFromPublicLink.feature:176](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L176) +- [coreApiSharePublicLink2/copyFromPublicLink.feature:177](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L177) +- [coreApiSharePublicLink2/copyFromPublicLink.feature:192](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L192) +- [coreApiSharePublicLink2/copyFromPublicLink.feature:193](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L193) #### [Increasing permission of a public link of a folder that was initially shared with share+read permissions is allowed](https://github.com/owncloud/ocis/issues/3881) @@ -215,29 +215,29 @@ File and sync features in a shared scenario #### [Adding public upload to a read only shared folder as a receipient is allowed ](https://github.com/owncloud/ocis/issues/2164) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:264](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L264) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:265](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L265) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:306](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L306) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:307](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L307) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:267](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L267) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:268](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L268) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:309](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L309) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:310](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L310) #### [Upload-only shares must not overwrite but create a separate file](https://github.com/owncloud/ocis/issues/1267) -- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:10](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L10) -- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:111](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L111) +- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:13](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L13) +- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:114](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L114) #### [Set quota over settings](https://github.com/owncloud/ocis/issues/1290) _requires a [CS3 user provisioning api that can update the quota for a user](https://github.com/cs3org/cs3apis/pull/95#issuecomment-772780683)_ -- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:84](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L84) -- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:93](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L93) +- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:87](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L87) +- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:96](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L96) #### [share permissions are not enforced](https://github.com/owncloud/product/issues/270) //todo - [coreApiShareReshareToShares3/reShareUpdate.feature:63](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L63) -- [coreApiShareReshareToShares3/reShareUpdate.feature:62](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L62) +- [coreApiShareReshareToShares3/reShareUpdate.feature:64](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L64) #### [path property in pending shares gives only filename](https://github.com/owncloud/ocis/issues/2156) -- [coreApiShareReshareToShares2/reShareSubfolder.feature:153](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares2/reShareSubfolder.feature#L153) -- [coreApiShareReshareToShares2/reShareSubfolder.feature:152](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares2/reShareSubfolder.feature#L152) +- [coreApiShareReshareToShares2/reShareSubfolder.feature:143](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares2/reShareSubfolder.feature#L143) +- [coreApiShareReshareToShares2/reShareSubfolder.feature:144](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares2/reShareSubfolder.feature#L144) - [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:61](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L61) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:743](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L743) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:744](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L744) @@ -303,14 +303,14 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt #### [Expiration date for shares is not implemented](https://github.com/owncloud/ocis/issues/1250) #### Expiration date of user shares -- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:33](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L33) -- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:32](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L32) +- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:35](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L35) +- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:36](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L36) #### Expiration date of group shares -- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:59](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L59) -- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:58](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L58) -- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:81](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L81) +- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:60](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L60) +- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:61](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L61) - [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:82](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L82) +- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:83](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L83) #### [Getting content of a shared file with same name returns 500](https://github.com/owncloud/ocis/issues/3880) - [coreApiShareCreateSpecialToShares1/createShareUniqueReceivedNames.feature:16](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares1/createShareUniqueReceivedNames.feature#L16) @@ -318,24 +318,22 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt #### [Empty OCS response for a share create request using a disabled user](https://github.com/owncloud/ocis/issues/2212) - [coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature:24](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature#L24) - [coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature:21](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature#L21) - -//todo -- [coreApiShareUpdateToShares/updateShare.feature:95](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L95) -- [coreApiShareUpdateToShares/updateShare.feature:96](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L96) - [coreApiShareUpdateToShares/updateShare.feature:97](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L97) - [coreApiShareUpdateToShares/updateShare.feature:98](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L98) - [coreApiShareUpdateToShares/updateShare.feature:99](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L99) -- [coreApiShareUpdateToShares/updateShare.feature:94](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L94) -- [coreApiShareUpdateToShares/updateShare.feature:119](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L119) -- [coreApiShareUpdateToShares/updateShare.feature:120](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L120) +- [coreApiShareUpdateToShares/updateShare.feature:100](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L100) +- [coreApiShareUpdateToShares/updateShare.feature:101](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L101) +- [coreApiShareUpdateToShares/updateShare.feature:102](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L102) - [coreApiShareUpdateToShares/updateShare.feature:121](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L121) - [coreApiShareUpdateToShares/updateShare.feature:122](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L122) - [coreApiShareUpdateToShares/updateShare.feature:123](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L123) -- [coreApiShareUpdateToShares/updateShare.feature:118](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L118) +- [coreApiShareUpdateToShares/updateShare.feature:124](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L124) +- [coreApiShareUpdateToShares/updateShare.feature:125](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L125) +- [coreApiShareUpdateToShares/updateShare.feature:126](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L126) #### [Edit user share response has an "name" field](https://github.com/owncloud/ocis/issues/1225) -- [coreApiShareUpdateToShares/updateShare.feature:241](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L241) -- [coreApiShareUpdateToShares/updateShare.feature:240](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L240) +- [coreApiShareUpdateToShares/updateShare.feature:243](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L243) +- [coreApiShareUpdateToShares/updateShare.feature:244](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L244) #### [user can access version metadata of a received share before accepting it](https://github.com/owncloud/ocis/issues/760) - [coreApiVersions/fileVersions.feature:313](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L313) @@ -516,12 +514,12 @@ Not everything needs to be implemented for ocis. While the oc10 testsuite covers - [coreApiWebdavUploadTUS/uploadToShare.feature:293](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L293) #### [Share inaccessible if folder with same name was deleted and recreated](https://github.com/owncloud/ocis/issues/1787) -- [coreApiShareReshareToShares1/reShare.feature:264](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L264) -- [coreApiShareReshareToShares1/reShare.feature:265](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L265) -- [coreApiShareReshareToShares1/reShare.feature:282](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L282) -- [coreApiShareReshareToShares1/reShare.feature:283](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L283) -- [coreApiShareReshareToShares1/reShare.feature:300](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L300) -- [coreApiShareReshareToShares1/reShare.feature:301](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L301) +- [coreApiShareReshareToShares1/reShare.feature:267](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L267) +- [coreApiShareReshareToShares1/reShare.feature:268](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L268) +- [coreApiShareReshareToShares1/reShare.feature:285](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L285) +- [coreApiShareReshareToShares1/reShare.feature:286](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L286) +- [coreApiShareReshareToShares1/reShare.feature:303](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L303) +- [coreApiShareReshareToShares1/reShare.feature:304](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L304) #### [incorrect ocs(v2) status value when getting info of share that does not exist should be 404, gives 998](https://github.com/owncloud/product/issues/250) _ocs: api compatibility, return correct status code_ @@ -566,8 +564,8 @@ _ocs: api compatibility, return correct status code_ - [coreApiWebdavProperties2/getFileProperties.feature:276](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties2/getFileProperties.feature#L276) #### [Cannot move folder/file from one received share to another](https://github.com/owncloud/ocis/issues/2442) -- [coreApiShareUpdateToShares/updateShare.feature:158](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L158) -- [coreApiShareUpdateToShares/updateShare.feature:194](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L194) +- - [coreApiShareUpdateToShares/updateShare.feature:197](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L197) +- [coreApiShareUpdateToShares/updateShare.feature:161](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L161) - [coreApiShareManagementToShares/mergeShare.feature:131](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/mergeShare.feature#L131) - [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:262](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L262) @@ -758,8 +756,8 @@ _ocs: api compatibility, return correct status code_ #### [reShareUpdate API tests failing in reva](https://github.com/cs3org/reva/issues/2916) -- [coreApiShareReshareToShares3/reShareUpdate.feature:158](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L158) -- [coreApiShareReshareToShares3/reShareUpdate.feature:157](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L157) +- [coreApiShareReshareToShares3/reShareUpdate.feature:153](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L153) +- [coreApiShareReshareToShares3/reShareUpdate.feature:154](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L154) #### [coreApiShareOperationsToShares1/gettingShares.feature:28 fails in CI](https://github.com/cs3org/reva/issues/2926) @@ -779,10 +777,10 @@ The problem has been fixed in reva edge branch but not in reva master - [coreApiWebdavOperations/propfind.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L73) #### [Updating the role of a public link to internal gives returns 400] -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:480](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L480) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:481](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L481) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:482](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L482) - [coreApiSharePublicLink3/updatePublicLinkShare.feature:483](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L483) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:484](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L484) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:485](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L485) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:486](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L486) #### [Default capabilities for normal user and admin user not same as in oC-core](https://github.com/owncloud/ocis/issues/1285) - [coreApiCapabilities/capabilities.feature:11](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiCapabilities/capabilities.feature#L11) diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index c9e0ddbe54..7b49f59a59 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -224,12 +224,12 @@ File and sync features in a shared scenario #### [copying a folder within a public link folder to folder with same name as an already existing file overwrites the parent file](https://github.com/owncloud/ocis/issues/1232) -- [coreApiSharePublicLink2/copyFromPublicLink.feature:63](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L63) -- [coreApiSharePublicLink2/copyFromPublicLink.feature:89](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L89) -- [coreApiSharePublicLink2/copyFromPublicLink.feature:173](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L173) -- [coreApiSharePublicLink2/copyFromPublicLink.feature:174](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L174) -- [coreApiSharePublicLink2/copyFromPublicLink.feature:189](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L189) -- [coreApiSharePublicLink2/copyFromPublicLink.feature:190](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L190) +- [coreApiSharePublicLink2/copyFromPublicLink.feature:66](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L66) +- [coreApiSharePublicLink2/copyFromPublicLink.feature:92](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L92) +- [coreApiSharePublicLink2/copyFromPublicLink.feature:176](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L176) +- [coreApiSharePublicLink2/copyFromPublicLink.feature:177](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L177) +- [coreApiSharePublicLink2/copyFromPublicLink.feature:192](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L192) +- [coreApiSharePublicLink2/copyFromPublicLink.feature:193](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L193) #### [Increasing permission of a public link of a folder that was initially shared with share+read permissions is allowed](https://github.com/owncloud/ocis/issues/3881) @@ -240,30 +240,30 @@ File and sync features in a shared scenario #### [Adding public upload to a read only shared folder as a receipient is allowed ](https://github.com/owncloud/ocis/issues/2164) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:264](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L264) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:265](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L265) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:306](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L306) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:307](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L307) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:267](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L267) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:268](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L268) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:309](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L309) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:310](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L310) #### [Upload-only shares must not overwrite but create a separate file](https://github.com/owncloud/ocis/issues/1267) -- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:10](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L10) -- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:111](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L111) +- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:13](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L13) +- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:114](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L114) #### [Set quota over settings](https://github.com/owncloud/ocis/issues/1290) _requires a [CS3 user provisioning api that can update the quota for a user](https://github.com/cs3org/cs3apis/pull/95#issuecomment-772780683)_ -- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:84](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L84) -- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:93](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L93) +- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:87](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L87) +- [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:96](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L96) #### [share permissions are not enforced](https://github.com/owncloud/product/issues/270) //todo - [coreApiShareReshareToShares3/reShareUpdate.feature:63](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L63) -- [coreApiShareReshareToShares3/reShareUpdate.feature:62](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L62) +- [coreApiShareReshareToShares3/reShareUpdate.feature:64](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L64) #### [file_target in share response](https://github.com/owncloud/product/issues/203) //todo -- [coreApiShareReshareToShares2/reShareSubfolder.feature:153](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares2/reShareSubfolder.feature#L153) -- [coreApiShareReshareToShares2/reShareSubfolder.feature:152](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares2/reShareSubfolder.feature#L152) +- [coreApiShareReshareToShares2/reShareSubfolder.feature:143](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares2/reShareSubfolder.feature#L143) +- [coreApiShareReshareToShares2/reShareSubfolder.feature:144](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares2/reShareSubfolder.feature#L144) #### [deleting a file inside a received shared folder is moved to the trash-bin of the sharer not the receiver](https://github.com/owncloud/ocis/issues/1124) @@ -324,14 +324,14 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt #### [Expiration date for shares is not implemented](https://github.com/owncloud/ocis/issues/1250) #### Expiration date of user shares -- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:33](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L33) -- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:32](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L32) +- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:35](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L35) +- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:36](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L36) #### Expiration date of group shares -- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:59](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L59) -- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:58](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L58) -- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:81](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L81) +- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:60](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L60) +- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:61](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L61) - [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:82](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L82) +- [coreApiShareReshareToShares3/reShareWithExpiryDate.feature:83](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareWithExpiryDate.feature#L83) #### [Getting content of a shared file with same name returns 500](https://github.com/owncloud/ocis/issues/3880) @@ -340,24 +340,22 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt #### [Empty OCS response for a share create request using a disabled user](https://github.com/owncloud/ocis/issues/2212) - [coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature:24](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature#L24) - [coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature:21](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature#L21) - -//todo -- [coreApiShareUpdateToShares/updateShare.feature:96](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L96) - [coreApiShareUpdateToShares/updateShare.feature:97](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L97) - [coreApiShareUpdateToShares/updateShare.feature:98](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L98) - [coreApiShareUpdateToShares/updateShare.feature:99](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L99) -- [coreApiShareUpdateToShares/updateShare.feature:94](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L94) -- [coreApiShareUpdateToShares/updateShare.feature:95](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L95) -- [coreApiShareUpdateToShares/updateShare.feature:120](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L120) +- [coreApiShareUpdateToShares/updateShare.feature:100](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L100) +- [coreApiShareUpdateToShares/updateShare.feature:101](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L101) +- [coreApiShareUpdateToShares/updateShare.feature:102](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L102) - [coreApiShareUpdateToShares/updateShare.feature:121](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L121) - [coreApiShareUpdateToShares/updateShare.feature:122](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L122) - [coreApiShareUpdateToShares/updateShare.feature:123](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L123) -- [coreApiShareUpdateToShares/updateShare.feature:118](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L118) -- [coreApiShareUpdateToShares/updateShare.feature:119](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L119) +- [coreApiShareUpdateToShares/updateShare.feature:124](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L124) +- [coreApiShareUpdateToShares/updateShare.feature:125](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L125) +- [coreApiShareUpdateToShares/updateShare.feature:126](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L126) #### [Edit user share response has an "name" field](https://github.com/owncloud/ocis/issues/1225) -- [coreApiShareUpdateToShares/updateShare.feature:241](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L241) -- [coreApiShareUpdateToShares/updateShare.feature:240](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L240) +- [coreApiShareUpdateToShares/updateShare.feature:243](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L243) +- [coreApiShareUpdateToShares/updateShare.feature:244](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L244) #### [user can access version metadata of a received share before accepting it](https://github.com/owncloud/ocis/issues/760) - [coreApiVersions/fileVersions.feature:313](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L313) @@ -538,12 +536,12 @@ Not everything needs to be implemented for ocis. While the oc10 testsuite covers - [coreApiWebdavUploadTUS/uploadToShare.feature:293](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L293) #### [Share inaccessible if folder with same name was deleted and recreated](https://github.com/owncloud/ocis/issues/1787) -- [coreApiShareReshareToShares1/reShare.feature:265](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L265) -- [coreApiShareReshareToShares1/reShare.feature:264](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L264) -- [coreApiShareReshareToShares1/reShare.feature:283](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L283) -- [coreApiShareReshareToShares1/reShare.feature:282](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L282) -- [coreApiShareReshareToShares1/reShare.feature:301](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L301) -- [coreApiShareReshareToShares1/reShare.feature:300](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L300) +- [coreApiShareReshareToShares1/reShare.feature:267](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L267) +- [coreApiShareReshareToShares1/reShare.feature:268](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L268) +- [coreApiShareReshareToShares1/reShare.feature:285](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L285) +- [coreApiShareReshareToShares1/reShare.feature:286](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L286) +- [coreApiShareReshareToShares1/reShare.feature:303](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L303) +- [coreApiShareReshareToShares1/reShare.feature:304](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L304) #### [incorrect ocs(v2) status value when getting info of share that does not exist should be 404, gives 998](https://github.com/owncloud/product/issues/250) _ocs: api compatibility, return correct status code_ @@ -587,8 +585,8 @@ _ocs: api compatibility, return correct status code_ - [coreApiWebdavProperties2/getFileProperties.feature:276](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties2/getFileProperties.feature#L276) #### [Cannot move folder/file from one received share to another](https://github.com/owncloud/ocis/issues/2442) -- [coreApiShareUpdateToShares/updateShare.feature:194](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L194) -- [coreApiShareUpdateToShares/updateShare.feature:158](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L158) +- [coreApiShareUpdateToShares/updateShare.feature:197](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L197) +- [coreApiShareUpdateToShares/updateShare.feature:161](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L161) - [coreApiShareManagementToShares/mergeShare.feature:131](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/mergeShare.feature#L131) #### [Sharing folder and sub-folder with same user but different permission,the permission of sub-folder is not obeyed ](https://github.com/owncloud/ocis/issues/2440) @@ -761,9 +759,8 @@ _ocs: api compatibility, return correct status code_ - [coreApiAuthWebDav/webDavMOVEAuth.feature:106](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavMOVEAuth.feature#L106) #### [reShareUpdate API tests failing in reva](https://github.com/cs3org/reva/issues/2916) - -- [coreApiShareReshareToShares3/reShareUpdate.feature:158](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L158) -- [coreApiShareReshareToShares3/reShareUpdate.feature:157](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L157) +- [coreApiShareReshareToShares3/reShareUpdate.feature:153](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L153) +- [coreApiShareReshareToShares3/reShareUpdate.feature:154](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L154) #### [coreApiShareOperationsToShares1/gettingShares.feature:28 fails in CI](https://github.com/cs3org/reva/issues/2926) @@ -783,10 +780,10 @@ The problem has been fixed in reva edge branch but not in reva master - [coreApiWebdavOperations/propfind.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L73) #### [Updating the role of a public link to internal gives returns 400] -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:480](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L480) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:481](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L481) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:482](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L482) - [coreApiSharePublicLink3/updatePublicLinkShare.feature:483](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L483) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:484](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L484) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:485](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L485) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:486](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L486) #### [Default capabilities for normal user and admin user not same as in oC-core](https://github.com/owncloud/ocis/issues/1285) - [coreApiCapabilities/capabilities.feature:11](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiCapabilities/capabilities.feature#L11) diff --git a/tests/ocis b/tests/ocis index dc7418b3c3..7094891f4d 160000 --- a/tests/ocis +++ b/tests/ocis @@ -1 +1 @@ -Subproject commit dc7418b3c3dc34349461b3997fb293a70466d804 +Subproject commit 7094891f4de381102b05c6503751dc85d82c0782 From 61723ffacb36aa1056075a8fe7d23cb7cf37b3ce Mon Sep 17 00:00:00 2001 From: Javier Ferrer Date: Mon, 5 Jun 2023 08:39:40 +0200 Subject: [PATCH 07/48] Add Notifications Service and some notifications (#3825) --- .../unreleased/notifications-framework.md | 13 + cmd/revad/runtime/loader.go | 2 + go.mod | 53 +-- go.sum | 125 +++--- .../publicshareprovider.go | 2 +- internal/http/services/loader/loader.go | 1 - internal/http/services/mailer/mailer.go | 378 ---------------- internal/http/services/ocmd/ocm.go | 3 - .../http/services/owncloud/ocdav/ocdav.go | 18 +- internal/http/services/owncloud/ocdav/put.go | 51 +++ .../services/owncloud/ocs/config/config.go | 1 + .../services/owncloud/ocs/conversions/main.go | 24 +- .../ocs/handlers/apps/sharing/shares/group.go | 12 +- .../handlers/apps/sharing/shares/public.go | 104 ++++- .../handlers/apps/sharing/shares/shares.go | 144 ++++++- .../ocs/handlers/apps/sharing/shares/user.go | 12 +- internal/http/services/owncloud/ocs/ocs.go | 12 +- internal/serverless/services/loader/loader.go | 1 + .../services/notifications/notifications.go | 403 ++++++++++++++++++ pkg/cbox/publicshare/sql/sql.go | 52 ++- pkg/cbox/utils/conversions.go | 60 +-- pkg/notification/db_changes.sql | 54 +++ pkg/notification/db_sqlite.sql | 51 +++ .../handler/emailhandler/emailhandler.go | 116 +++++ pkg/notification/handler/handler.go | 24 ++ pkg/notification/handler/loader/loader.go | 25 ++ pkg/notification/handler/registry/registry.go | 60 +++ pkg/notification/manager/loader/loader.go | 25 ++ pkg/notification/manager/registry/registry.go | 36 ++ pkg/notification/manager/sql/sql.go | 199 +++++++++ .../manager/sql/sql_suite_test.go | 19 +- pkg/notification/manager/sql/sql_test.go | 252 +++++++++++ pkg/notification/manager/sql/test.sqlite | Bin 0 -> 32768 bytes pkg/notification/notification.go | 114 +++++ .../notificationhelper/notificationhelper.go | 250 +++++++++++ .../template/registry/registry.go | 69 +++ pkg/notification/template/template.go | 170 ++++++++ pkg/notification/trigger/trigger.go | 41 ++ pkg/notification/utils/nats.go | 63 +++ pkg/publicshare/manager/json/json.go | 28 +- pkg/publicshare/manager/memory/memory.go | 28 +- pkg/publicshare/publicshare.go | 2 +- pkg/utils/accumulator/accumulator.go | 126 ++++++ 43 files changed, 2628 insertions(+), 595 deletions(-) create mode 100644 changelog/unreleased/notifications-framework.md delete mode 100644 internal/http/services/mailer/mailer.go create mode 100644 internal/serverless/services/notifications/notifications.go create mode 100644 pkg/notification/db_changes.sql create mode 100644 pkg/notification/db_sqlite.sql create mode 100644 pkg/notification/handler/emailhandler/emailhandler.go create mode 100644 pkg/notification/handler/handler.go create mode 100644 pkg/notification/handler/loader/loader.go create mode 100644 pkg/notification/handler/registry/registry.go create mode 100644 pkg/notification/manager/loader/loader.go create mode 100644 pkg/notification/manager/registry/registry.go create mode 100644 pkg/notification/manager/sql/sql.go rename internal/http/services/ocmd/notifications.go => pkg/notification/manager/sql/sql_suite_test.go (70%) create mode 100644 pkg/notification/manager/sql/sql_test.go create mode 100644 pkg/notification/manager/sql/test.sqlite create mode 100644 pkg/notification/notification.go create mode 100644 pkg/notification/notificationhelper/notificationhelper.go create mode 100644 pkg/notification/template/registry/registry.go create mode 100644 pkg/notification/template/template.go create mode 100644 pkg/notification/trigger/trigger.go create mode 100644 pkg/notification/utils/nats.go create mode 100644 pkg/utils/accumulator/accumulator.go diff --git a/changelog/unreleased/notifications-framework.md b/changelog/unreleased/notifications-framework.md new file mode 100644 index 0000000000..83ccd26118 --- /dev/null +++ b/changelog/unreleased/notifications-framework.md @@ -0,0 +1,13 @@ +Enhancement: Notifications framework + +Adds a notifications framework to Reva. + +The new notifications service communicates with the rest of +reva using NATS. It provides helper functions to register new +notifications and to send them. + +Notification templates are provided in the configuration files +for each service, and they are registered into the notifications +service on initialization. + +https://github.com/cs3org/reva/pull/3825 diff --git a/cmd/revad/runtime/loader.go b/cmd/revad/runtime/loader.go index 52488936d3..ad174b9130 100644 --- a/cmd/revad/runtime/loader.go +++ b/cmd/revad/runtime/loader.go @@ -37,6 +37,8 @@ import ( _ "github.com/cs3org/reva/pkg/datatx/manager/loader" _ "github.com/cs3org/reva/pkg/group/manager/loader" _ "github.com/cs3org/reva/pkg/metrics/driver/loader" + _ "github.com/cs3org/reva/pkg/notification/handler/loader" + _ "github.com/cs3org/reva/pkg/notification/manager/loader" _ "github.com/cs3org/reva/pkg/ocm/invite/repository/loader" _ "github.com/cs3org/reva/pkg/ocm/provider/authorizer/loader" _ "github.com/cs3org/reva/pkg/ocm/share/repository/loader" diff --git a/go.mod b/go.mod index 203457cdb5..f5515bb3af 100644 --- a/go.mod +++ b/go.mod @@ -24,14 +24,14 @@ require ( github.com/go-chi/chi/v5 v5.0.8 github.com/go-ldap/ldap/v3 v3.4.4 github.com/go-playground/validator/v10 v10.11.2 - github.com/go-sql-driver/mysql v1.6.0 + github.com/go-sql-driver/mysql v1.7.0 github.com/golang-jwt/jwt v3.2.2+incompatible - github.com/golang/protobuf v1.5.2 + github.com/golang/protobuf v1.5.3 github.com/gomodule/redigo v1.8.9 github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/hashicorp/go-hclog v1.4.0 + github.com/hashicorp/go-hclog v1.5.0 github.com/hashicorp/go-plugin v1.4.9 github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/juliangruber/go-intersect v1.1.0 @@ -40,10 +40,11 @@ require ( github.com/mileusna/useragent v1.2.1 github.com/minio/minio-go/v7 v7.0.45 github.com/mitchellh/mapstructure v1.5.0 - github.com/nats-io/nats-server/v2 v2.9.11 - github.com/nats-io/nats-streaming-server v0.25.2 + github.com/nats-io/nats-server/v2 v2.9.16 + github.com/nats-io/nats-streaming-server v0.25.4 + github.com/nats-io/nats.go v1.25.0 github.com/onsi/ginkgo v1.16.5 - github.com/onsi/gomega v1.27.2 + github.com/onsi/gomega v1.27.6 github.com/pkg/errors v0.9.1 github.com/pkg/xattr v0.4.5 github.com/prometheus/alertmanager v0.24.0 @@ -51,7 +52,7 @@ require ( github.com/rs/zerolog v1.28.0 github.com/sciencemesh/meshdirectory-web v1.0.4 github.com/sethvargo/go-password v0.2.0 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.2 github.com/studio-b12/gowebdav v0.0.0-20230203202212-3282f94193f2 github.com/thanhpk/randstr v1.0.4 github.com/tus/tusd v1.10.0 @@ -64,25 +65,28 @@ require ( go.opentelemetry.io/otel/sdk v1.11.2 go.opentelemetry.io/otel/trace v1.11.2 go.step.sm/crypto v0.23.2 - golang.org/x/crypto v0.5.0 + golang.org/x/crypto v0.8.0 golang.org/x/oauth2 v0.3.0 golang.org/x/sync v0.1.0 - golang.org/x/sys v0.5.0 - golang.org/x/term v0.5.0 - golang.org/x/text v0.7.0 + golang.org/x/sys v0.7.0 + golang.org/x/term v0.7.0 + golang.org/x/text v0.9.0 google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef google.golang.org/grpc v1.52.0 google.golang.org/protobuf v1.28.1 gotest.tools v2.2.0+incompatible ) -require github.com/go-jose/go-jose/v3 v3.0.0 // indirect +require ( + github.com/go-jose/go-jose/v3 v3.0.0 // indirect + github.com/hashicorp/go-msgpack/v2 v2.1.0 // indirect +) require ( github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect - github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect + github.com/armon/go-metrics v0.4.1 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 // indirect @@ -113,13 +117,13 @@ require ( github.com/hashicorp/go-immutable-radix v1.0.0 // indirect github.com/hashicorp/go-msgpack v1.1.5 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/hashicorp/raft v1.3.11 // indirect + github.com/hashicorp/raft v1.4.0 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.15.11 // indirect + github.com/klauspost/compress v1.16.4 // indirect github.com/klauspost/cpuid/v2 v2.1.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/lestrrat-go/strftime v1.0.4 // indirect @@ -138,11 +142,10 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/nats-io/jwt/v2 v2.3.0 // indirect - github.com/nats-io/nats.go v1.19.0 // indirect - github.com/nats-io/nkeys v0.3.0 // indirect + github.com/nats-io/jwt/v2 v2.4.1 // indirect + github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect - github.com/nats-io/stan.go v0.10.3 // indirect + github.com/nats-io/stan.go v0.10.4 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/oklog/run v1.1.0 // indirect github.com/oklog/ulid v1.3.1 // indirect @@ -154,7 +157,7 @@ require ( github.com/prometheus/client_golang v1.13.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/rs/xid v1.4.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect @@ -163,13 +166,13 @@ require ( github.com/sirupsen/logrus v1.9.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/tidwall/pretty v1.2.0 // indirect - go.etcd.io/bbolt v1.3.6 // indirect + go.etcd.io/bbolt v1.3.7 // indirect go.mongodb.org/mongo-driver v1.8.3 // indirect go.opentelemetry.io/otel/metric v0.34.0 // indirect - golang.org/x/mod v0.8.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect - golang.org/x/tools v0.6.0 // indirect + golang.org/x/mod v0.9.0 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.7.0 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/ini.v1 v1.66.6 // indirect gopkg.in/src-d/go-errors.v1 v1.0.0 // indirect diff --git a/go.sum b/go.sum index a18fef0bf3..1ff0369c1c 100644 --- a/go.sum +++ b/go.sum @@ -156,7 +156,7 @@ github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -201,8 +201,8 @@ github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= -github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= @@ -376,8 +376,6 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glpatcern/go-mime v0.0.0-20221026162842-2a8d71ad17a9 h1:3um08ooi0/lyRmK2eE1XTKmRQHDzPu0IvpCPMljyMZ8= github.com/glpatcern/go-mime v0.0.0-20221026162842-2a8d71ad17a9/go.mod h1:EJaddanP+JfU3UkVvn0rYYF3b/gD7eZRejbTHqiQExA= -github.com/gmgigi96/go-cs3apis v0.0.0-20230508122407-26b2c32caabc h1:/KUUgL9AkNP8TEOtALru0VzDlZh7IH6DBAwQrl5xw8Y= -github.com/gmgigi96/go-cs3apis v0.0.0-20230508122407-26b2c32caabc/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/go-acme/lego/v4 v4.4.0/go.mod h1:l3+tFUFZb590dWcqhWZegynUthtaHJbG2fevUpoOOE0= github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A= github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= @@ -449,13 +447,14 @@ github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVL github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b/go.mod h1:Xo4aNUOrJnVruqWQJBtW6+bTBDTniY8yZum5rF3b5jw= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= @@ -540,8 +539,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -639,15 +639,16 @@ github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v1.1.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.4.0 h1:ctuWFGrhFha8BnnzxqeRGidlEcQkDyL5u8J8t5eA11I= -github.com/hashicorp/go-hclog v1.4.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs= github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4= +github.com/hashicorp/go-msgpack/v2 v2.1.0 h1:J2g2hMyjSefUPTnkLRU2MnsLLsPRB1n4Z/wJRN07GuA= +github.com/hashicorp/go-msgpack/v2 v2.1.0/go.mod h1:Tv81cKI2JmHZDjmzEmc1n+8h1DO5k+3pG6BPlNMQds0= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-plugin v1.4.9 h1:ESiK220/qE0aGxWdzKIvRH69iLiuN/PjoLTm69RoWtU= github.com/hashicorp/go-plugin v1.4.9/go.mod h1:viDMjcLJuDui6pXb8U4HVfb8AamCWhHGUjr2IrTF67s= @@ -672,8 +673,8 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/raft v1.3.11 h1:p3v6gf6l3S797NnK5av3HcczOC1T5CLoaRvg0g9ys4A= -github.com/hashicorp/raft v1.3.11/go.mod h1:J8naEwc6XaaCfts7+28whSeRvCqTd6e20BlCU3LtEO4= +github.com/hashicorp/raft v1.4.0 h1:tn28S/AWv0BtRQgwZv/1NELu8sCvI0FixqL8C8MYKeY= +github.com/hashicorp/raft v1.4.0/go.mod h1:nz64BIjXphDLATfKGG5RzHtNUPioLeKFsXEm88yTVew= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= @@ -711,6 +712,7 @@ github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -732,8 +734,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= -github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.16.4 h1:91KN02FnsOYhuunwU4ssRe8lc2JosWmizWa91B5v1PU= +github.com/klauspost/compress v1.16.4/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= @@ -766,8 +768,7 @@ github.com/lestrrat-go/strftime v1.0.4 h1:T1Rb9EPkAhgxKqbcMIPguPq8glqXTA1koF8n9B github.com/lestrrat-go/strftime v1.0.4/go.mod h1:E1nN3pCbtMSu1yjSVeyuRFVm/U0xoR76fd03sz+Qz4g= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= -github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.8 h1:3fdt97i/cwSU83+E0hZTC/Xpc9mTZxc6UWSCRcSbxiE= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linode/linodego v0.25.3/go.mod h1:GSBKPpjoQfxEfryoCRcgkuUOCuVtGHWhzI8OMdycNTE= @@ -876,27 +877,26 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04/go.mod h1:5sN+Lt1CaY4wsPvgQH/jsuJi4XO2ssZbdsIizr4CVC8= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= -github.com/nats-io/jwt/v2 v2.3.0/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= +github.com/nats-io/jwt/v2 v2.4.1 h1:Y35W1dgbbz2SQUYDPCaclXcuqleVmpbRa7646Jf2EX4= +github.com/nats-io/jwt/v2 v2.4.1/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats-server/v2 v2.9.3/go.mod h1:4sq8wvrpbvSzL1n3ZfEYnH4qeUuIl5W990j3kw13rRk= -github.com/nats-io/nats-server/v2 v2.9.11 h1:4y5SwWvWI59V5mcqtuoqKq6L9NDUydOP3Ekwuwl8cZI= -github.com/nats-io/nats-server/v2 v2.9.11/go.mod h1:b0oVuxSlkvS3ZjMkncFeACGyZohbO4XhSqW1Lt7iRRY= -github.com/nats-io/nats-streaming-server v0.25.2 h1:cWjytvYksYPgnXnSocqnRWVrSgLclusnPGBNHQR4SqI= -github.com/nats-io/nats-streaming-server v0.25.2/go.mod h1:bRbgx+iCG6EZEXpqVMroRDuCGwR1iW+ta84aEGBaMhI= +github.com/nats-io/nats-server/v2 v2.9.16 h1:SuNe6AyCcVy0g5326wtyU8TdqYmcPqzTjhkHojAjprc= +github.com/nats-io/nats-server/v2 v2.9.16/go.mod h1:z1cc5Q+kqJkz9mLUdlcSsdYnId4pyImHjNgoh6zxSC0= +github.com/nats-io/nats-streaming-server v0.25.4 h1:aaMmKcEMXLvviM9y73BmPquTgQ/fg+0EmFLzFSbXUqQ= +github.com/nats-io/nats-streaming-server v0.25.4/go.mod h1:zrOyvkrVEz/y72m1nAVb149k/CAumm3H9ze682Lg9uE= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nats.go v1.16.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= -github.com/nats-io/nats.go v1.17.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= -github.com/nats-io/nats.go v1.19.0 h1:H6j8aBnTQFoVrTGB6Xjd903UMdE7jz6DS4YkmAqgZ9Q= -github.com/nats-io/nats.go v1.19.0/go.mod h1:tLqubohF7t4z3du1QDPYJIQQyhb4wl6DhjxEajSI7UA= +github.com/nats-io/nats.go v1.22.1/go.mod h1:tLqubohF7t4z3du1QDPYJIQQyhb4wl6DhjxEajSI7UA= +github.com/nats-io/nats.go v1.25.0 h1:t5/wCPGciR7X3Mu8QOi4jiJaXaWM8qtkLu4lzGZvYHE= +github.com/nats-io/nats.go v1.25.0/go.mod h1:D2WALIhz7V8M0pH8Scx8JZXlg6Oqz5VG+nQkK8nJdvg= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= +github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= +github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nats-io/stan.go v0.10.3 h1:8DOyQJ0+nza3zSVJZ19/cpikkrWA4rSKB3YvckIGOTI= -github.com/nats-io/stan.go v0.10.3/go.mod h1:Cgf5zk6kKpOCqqUIJeuBz6ZDz9osT791VhS6m28sSQQ= +github.com/nats-io/stan.go v0.10.4 h1:19GS/eD1SeQJaVkeM9EkvEYattnvnWrZ3wkSWSw4uXw= +github.com/nats-io/stan.go v0.10.4/go.mod h1:3XJXH8GagrGqajoO/9+HgPyKV5MWsv7S5ccdda+pc6k= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nrdcg/auroradns v1.0.1/go.mod h1:y4pc0i9QXYlFCWrhWrUSIETnZgrf4KuwjDIWmmXo3JI= @@ -923,12 +923,12 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.8.4 h1:gf5mIQ8cLFieruNLAdgijHF1PYfLphKm2dxxcUtcqK0= +github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.27.2 h1:SKU0CXeKE/WVgIV1T61kSa3+IRE8Ekrv9rdXDwwTqnY= -github.com/onsi/gomega v1.27.2/go.mod h1:5mR3phAHpkAVIDkHEUBY6HGVsU+cpcEscrGPB4oPlZI= +github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -976,16 +976,15 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/otp v1.3.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= -github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/alertmanager v0.24.0 h1:HBWR3lk4uy3ys+naDZthDdV7yEsxpaNeZuUS+hJgrOw= github.com/prometheus/alertmanager v0.24.0/go.mod h1:r6fy/D7FRuZh5YbnX6J3MBY0eI4Pb5yPYS7/bPSXXqI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= @@ -1000,12 +999,12 @@ github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= @@ -1017,7 +1016,6 @@ github.com/prometheus/common/sigv4 v0.1.0/go.mod h1:2Jkxxk9yYvCkE5G1sQT7GuEXm57J github.com/prometheus/exporter-toolkit v0.7.1/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -1026,8 +1024,9 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= @@ -1119,8 +1118,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/studio-b12/gowebdav v0.0.0-20230203202212-3282f94193f2 h1:VsBj3UD2xyAOu7kJw6O/2jjG2UXLFoBzihqDU9Ofg9M= github.com/studio-b12/gowebdav v0.0.0-20230203202212-3282f94193f2/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= @@ -1176,8 +1176,8 @@ go-micro.dev/v4 v4.3.1-0.20211108085239-0c2041e43908 h1:4ori3xawGl2unFIOQPEgUuHd go-micro.dev/v4 v4.3.1-0.20211108085239-0c2041e43908/go.mod h1:tw47Xfg2YywfPUnglZgXQsSf7p0ST6mQL3v0JooGmSY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.m3o.com v0.1.0/go.mod h1:p8FdLqZH3R9a0y04qiMNT+clw69d3SxyQPFzCNbDRtk= go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= @@ -1214,7 +1214,6 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -1252,11 +1251,9 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1299,8 +1296,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1368,8 +1365,9 @@ golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1478,7 +1476,6 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201110211018-35f3e6cf4a65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1531,20 +1528,21 @@ golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201113234701-d7a72108b828/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1556,8 +1554,9 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1565,8 +1564,8 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1643,8 +1642,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/grpc/services/publicshareprovider/publicshareprovider.go b/internal/grpc/services/publicshareprovider/publicshareprovider.go index 09a28bc230..5829b050d8 100644 --- a/internal/grpc/services/publicshareprovider/publicshareprovider.go +++ b/internal/grpc/services/publicshareprovider/publicshareprovider.go @@ -145,7 +145,7 @@ func (s *service) CreatePublicShare(ctx context.Context, req *link.CreatePublicS log.Error().Msg("error getting user from context") } - share, err := s.sm.CreatePublicShare(ctx, u, req.ResourceInfo, req.Grant, req.Description, req.Internal) + share, err := s.sm.CreatePublicShare(ctx, u, req.ResourceInfo, req.Grant, req.Description, req.Internal, req.NotifyUploads, req.NotifyUploadsExtraRecipients) switch err.(type) { case nil: return &link.CreatePublicShareResponse{ diff --git a/internal/http/services/loader/loader.go b/internal/http/services/loader/loader.go index 693264c157..adf1ad02da 100644 --- a/internal/http/services/loader/loader.go +++ b/internal/http/services/loader/loader.go @@ -25,7 +25,6 @@ import ( _ "github.com/cs3org/reva/internal/http/services/datagateway" _ "github.com/cs3org/reva/internal/http/services/dataprovider" _ "github.com/cs3org/reva/internal/http/services/helloworld" - _ "github.com/cs3org/reva/internal/http/services/mailer" _ "github.com/cs3org/reva/internal/http/services/mentix" _ "github.com/cs3org/reva/internal/http/services/meshdirectory" _ "github.com/cs3org/reva/internal/http/services/metrics" diff --git a/internal/http/services/mailer/mailer.go b/internal/http/services/mailer/mailer.go deleted file mode 100644 index 6d9c23c9c5..0000000000 --- a/internal/http/services/mailer/mailer.go +++ /dev/null @@ -1,378 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package mailer - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/smtp" - "os" - "path/filepath" - "strings" - "text/template" - - gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" - group "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" - user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ctxpkg "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/rhttp/global" - "github.com/cs3org/reva/pkg/sharedconf" - "github.com/mitchellh/mapstructure" - "github.com/rs/zerolog" -) - -func init() { - global.Register("mailer", New) -} - -type config struct { - SMTPAddress string `mapstructure:"smtp_server" docs:";The hostname and port of the SMTP server."` - SenderLogin string `mapstructure:"sender_login" docs:";The email to be used to send mails."` - SenderPassword string `mapstructure:"sender_password" docs:";The sender's password."` - DisableAuth bool `mapstructure:"disable_auth" docs:"false;Whether to disable SMTP auth."` - Prefix string `mapstructure:"prefix"` - BodyTemplatePath string `mapstructure:"body_template_path"` - SubjectTemplate string `mapstructure:"subject_template"` - GatewaySVC string `mapstructure:"gateway_svc"` -} - -type svc struct { - conf *config - client gateway.GatewayAPIClient - tplBody *template.Template - tplSubj *template.Template -} - -// New creates a new mailer service. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { - conf := &config{} - if err := mapstructure.Decode(m, conf); err != nil { - return nil, err - } - - conf.init() - - client, err := pool.GetGatewayServiceClient(pool.Endpoint(conf.GatewaySVC)) - if err != nil { - return nil, err - } - - s := &svc{ - conf: conf, - client: client, - } - - if err = s.initBodyTemplate(); err != nil { - return nil, err - } - if err = s.initSubjectTemplate(); err != nil { - return nil, err - } - - return s, nil -} - -func (s *svc) Close() error { - return nil -} - -func (s *svc) initBodyTemplate() error { - f, err := os.Open(s.conf.BodyTemplatePath) - if err != nil { - return err - } - defer f.Close() - - data, err := io.ReadAll(f) - if err != nil { - return err - } - - tpl, err := template.New("tpl_body").Parse(string(data)) - if err != nil { - return err - } - - s.tplBody = tpl - return nil -} - -func (s *svc) initSubjectTemplate() error { - tpl, err := template.New("tpl_subj").Parse(s.conf.SubjectTemplate) - if err != nil { - return err - } - s.tplSubj = tpl - return nil -} - -func (c *config) init() { - if c.Prefix == "" { - c.Prefix = "mailer" - } - - if c.SubjectTemplate == "" { - c.SubjectTemplate = "{{.OwnerName}} ({{.OwnerUsername}}) shared {{if .IsDir}}folder{{else}}file{{end}} '{{.Filename}}' with you" - } - - c.GatewaySVC = sharedconf.GetGatewaySVC(c.GatewaySVC) -} - -func (s *svc) Prefix() string { - return s.conf.Prefix -} - -func (s *svc) Unprotected() []string { - return nil -} - -type out struct { - Recipients []string `json:"recipients"` -} - -func getIDsFromRequest(r *http.Request) ([]string, error) { - if err := r.ParseForm(); err != nil { - return nil, err - } - - idsSet := make(map[string]struct{}) - - for _, id := range r.Form["id"] { - if _, ok := idsSet[id]; ok { - continue - } - idsSet[id] = struct{}{} - } - - ids := make([]string, 0, len(idsSet)) - for id := range idsSet { - ids = append(ids, id) - } - - return ids, nil -} - -func (s *svc) Handler() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) - return - } - - ctx := r.Context() - - ids, err := getIDsFromRequest(r) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - } - - if len(ids) == 0 { - http.Error(w, "share id not provided", http.StatusBadRequest) - return - } - - var recipients []string - for _, id := range ids { - recipient, err := s.sendMailForShare(ctx, id) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - recipients = append(recipients, recipient) - } - - w.WriteHeader(http.StatusOK) - w.Header().Add("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(out{Recipients: recipients}) - }) -} - -type shareInfo struct { - RecipientEmail string - RecipientUsername string - OwnerEmail string - OwnerName string - OwnerUsername string - ShareType string - Filename string - Path string - IsDir bool - ShareID string -} - -func (s *svc) getAuth() smtp.Auth { - if s.conf.DisableAuth { - return nil - } - return smtp.PlainAuth("", s.conf.SenderLogin, s.conf.SenderPassword, strings.SplitN(s.conf.SMTPAddress, ":", 2)[0]) -} - -func (s *svc) sendMailForShare(ctx context.Context, id string) (string, error) { - share, err := s.getShareInfoByID(ctx, id) - if err != nil { - return "", err - } - - msg, err := s.generateMsg(share.OwnerEmail, share.RecipientEmail, share) - if err != nil { - return "", err - } - - return share.RecipientEmail, smtp.SendMail(s.conf.SMTPAddress, s.getAuth(), share.OwnerEmail, []string{share.RecipientEmail}, msg) -} - -func (s *svc) generateMsg(from, to string, share *shareInfo) ([]byte, error) { - subj, err := s.generateEmailSubject(share) - if err != nil { - return nil, err - } - - body, err := s.generateEmailBody(share) - if err != nil { - return nil, err - } - - msg := fmt.Sprintf("From: %s\r\n"+ - "To: %s\r\n"+ - "Subject: %s\r\n\r\n%s\r\n", from, to, subj, body) - return []byte(msg), nil -} - -func (s *svc) getShareInfoByID(ctx context.Context, id string) (*shareInfo, error) { - user, ok := ctxpkg.ContextGetUser(ctx) - if !ok { - return nil, errtypes.UserRequired("user not in context") - } - - shareRes, err := s.client.GetShare(ctx, &collaboration.GetShareRequest{ - Ref: &collaboration.ShareReference{ - Spec: &collaboration.ShareReference_Id{ - Id: &collaboration.ShareId{ - OpaqueId: id, - }, - }, - }, - }) - - switch { - case err != nil: - return nil, err - case shareRes.Status.Code == rpc.Code_CODE_NOT_FOUND: - return nil, errtypes.NotFound(fmt.Sprintf("share %s not found", id)) - case shareRes.Status.Code != rpc.Code_CODE_OK: - return nil, errtypes.InternalError(shareRes.Status.Message) - } - - share := shareRes.Share - statRes, err := s.client.Stat(ctx, &provider.StatRequest{ - Ref: &provider.Reference{ - ResourceId: share.ResourceId, - }, - }) - - switch { - case err != nil: - return nil, err - case statRes.Status.Code == rpc.Code_CODE_NOT_FOUND: - return nil, errtypes.NotFound("reference not found") - case statRes.Status.Code != rpc.Code_CODE_OK: - return nil, errtypes.InternalError(statRes.Status.Message) - } - - file := statRes.Info - - info := &shareInfo{} - switch g := share.Grantee.Id.(type) { - case *provider.Grantee_UserId: - grantee, err := s.getUser(ctx, g.UserId) - if err != nil { - return nil, err - } - info.RecipientEmail = grantee.Mail - info.RecipientUsername = grantee.Username - info.ShareType = "user" - case *provider.Grantee_GroupId: - grantee, err := s.getGroup(ctx, g.GroupId) - if err != nil { - return nil, err - } - info.RecipientEmail = grantee.Mail - info.RecipientUsername = grantee.GroupName - info.ShareType = "group" - } - - info.OwnerEmail = user.Mail - info.OwnerName = user.DisplayName - info.OwnerUsername = user.Username - - info.Path = file.Path - info.Filename = filepath.Base(file.Path) - if file.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER { - info.IsDir = true - } else { - info.IsDir = false - } - - info.ShareID = id - - return info, nil -} - -func (s *svc) getUser(ctx context.Context, userID *user.UserId) (*user.User, error) { - res, err := s.client.GetUser(ctx, &user.GetUserRequest{ - UserId: userID, - }) - if err != nil { - return nil, err - } - - return res.User, nil -} - -func (s *svc) getGroup(ctx context.Context, groupID *group.GroupId) (*group.Group, error) { - res, err := s.client.GetGroup(ctx, &group.GetGroupRequest{ - GroupId: groupID, - }) - if err != nil { - return nil, err - } - - return res.Group, nil -} - -func (s *svc) generateEmailSubject(share *shareInfo) (string, error) { - var buf bytes.Buffer - err := s.tplSubj.Execute(&buf, share) - return buf.String(), err -} - -func (s *svc) generateEmailBody(share *shareInfo) (string, error) { - var buf bytes.Buffer - err := s.tplBody.Execute(&buf, share) - return buf.String(), err -} diff --git a/internal/http/services/ocmd/ocm.go b/internal/http/services/ocmd/ocm.go index ab178796f6..fa92fb4121 100644 --- a/internal/http/services/ocmd/ocm.go +++ b/internal/http/services/ocmd/ocm.go @@ -75,19 +75,16 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) func (s *svc) routerInit() error { sharesHandler := new(sharesHandler) - notificationsHandler := new(notificationsHandler) invitesHandler := new(invitesHandler) if err := sharesHandler.init(s.Conf); err != nil { return err } - notificationsHandler.init(s.Conf) if err := invitesHandler.init(s.Conf); err != nil { return err } s.router.Post("/shares", sharesHandler.CreateShare) - s.router.Post("/notifications", notificationsHandler.SendNotification) s.router.Post("/invite-accepted", invitesHandler.AcceptInvite) return nil } diff --git a/internal/http/services/owncloud/ocdav/ocdav.go b/internal/http/services/owncloud/ocdav/ocdav.go index 4c029c6560..7f6d7ca366 100644 --- a/internal/http/services/owncloud/ocdav/ocdav.go +++ b/internal/http/services/owncloud/ocdav/ocdav.go @@ -33,6 +33,7 @@ import ( "github.com/cs3org/reva/pkg/appctx" ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/notification/notificationhelper" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/rhttp/global" @@ -111,6 +112,7 @@ type Config struct { PublicURL string `mapstructure:"public_url"` FavoriteStorageDriver string `mapstructure:"favorite_storage_driver"` FavoriteStorageDrivers map[string]map[string]interface{} `mapstructure:"favorite_storage_drivers"` + Notifications map[string]interface{} `mapstructure:"notifications" docs:"Settingsg for the Notification Helper"` } func (c *Config) init() { @@ -127,11 +129,12 @@ func (c *Config) init() { } type svc struct { - c *Config - webDavHandler *WebDavHandler - davHandler *DavHandler - favoritesManager favorite.Manager - client *http.Client + c *Config + webDavHandler *WebDavHandler + davHandler *DavHandler + favoritesManager favorite.Manager + client *http.Client + notificationHelper *notificationhelper.NotificationHelper } func getFavoritesManager(c *Config) (favorite.Manager, error) { @@ -163,8 +166,10 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) rhttp.Timeout(time.Duration(conf.Timeout*int64(time.Second))), rhttp.Insecure(conf.Insecure), ), - favoritesManager: fm, + favoritesManager: fm, + notificationHelper: notificationhelper.New("ocdav", conf.Notifications, log), } + // initialize handlers and set default configs if err := s.webDavHandler.init(conf.WebdavNamespace, true); err != nil { return nil, err @@ -180,6 +185,7 @@ func (s *svc) Prefix() string { } func (s *svc) Close() error { + s.notificationHelper.Stop() return nil } diff --git a/internal/http/services/owncloud/ocdav/put.go b/internal/http/services/owncloud/ocdav/put.go index b91746e6ff..c2ee4fffc0 100644 --- a/internal/http/services/owncloud/ocdav/put.go +++ b/internal/http/services/owncloud/ocdav/put.go @@ -20,18 +20,22 @@ package ocdav import ( "context" + "encoding/json" "net/http" "path" + "path/filepath" "strconv" "strings" "time" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + linkv1beta1 "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/datagateway" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/notification/trigger" "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/storage/utils/chunking" rtrace "github.com/cs3org/reva/pkg/trace" @@ -321,6 +325,53 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ lastModifiedString := t.Format(time.RFC1123Z) w.Header().Set(HeaderLastModified, lastModifiedString) + var m map[string]*typespb.OpaqueEntry + if sRes.Info.GetOpaque() != nil { + m = sRes.Info.GetOpaque().Map + } + + if ls, ok := m["link-share"]; ok { + l := &linkv1beta1.PublicShare{} + switch ls.Decoder { + case "json": + _ = json.Unmarshal(ls.Value, l) + default: + log.Error().Msgf("opaque entry decoder %s not recognized", ls.Decoder) + } + + path := "" + folder := "" + _, shareFileName := filepath.Split(ref.Path) + + if f, ok := m["eos"]; ok { + eosOpaque := make(map[string]interface{}) + switch f.Decoder { + case "json": + _ = json.Unmarshal(f.Value, &eosOpaque) + default: + log.Error().Msgf("opaque entry decoder %s not recognized", f.Decoder) + } + + if p, ok := eosOpaque["file"]; ok { + path, _ = filepath.Split(p.(string)) + } + } + + if path != "" { + folder = filepath.Base(path) + } + + trg := &trigger.Trigger{ + Ref: l.Id.OpaqueId, + TemplateData: map[string]interface{}{ + "path": path, + "folder": folder, + "fileName": shareFileName, + }, + } + s.notificationHelper.TriggerNotification(trg) + } + // file was new if info == nil { w.WriteHeader(http.StatusCreated) diff --git a/internal/http/services/owncloud/ocs/config/config.go b/internal/http/services/owncloud/ocs/config/config.go index 31b1850eb1..123cdd47be 100644 --- a/internal/http/services/owncloud/ocs/config/config.go +++ b/internal/http/services/owncloud/ocs/config/config.go @@ -45,6 +45,7 @@ type Config struct { AllowedLanguages []string `mapstructure:"allowed_languages"` OCMMountPoint string `mapstructure:"ocm_mount_point"` ListOCMShares bool `mapstructure:"list_ocm_shares"` + Notifications map[string]interface{} `mapstructure:"notifications"` } // Init sets sane defaults. diff --git a/internal/http/services/owncloud/ocs/conversions/main.go b/internal/http/services/owncloud/ocs/conversions/main.go index 24a50b9246..1650ab3182 100644 --- a/internal/http/services/owncloud/ocs/conversions/main.go +++ b/internal/http/services/owncloud/ocs/conversions/main.go @@ -149,6 +149,10 @@ type ShareData struct { Quicklink bool `json:"quicklink,omitempty" xml:"quicklink,omitempty"` // Description of the public share Description string `json:"description" xml:"description"` + // Whether to notify owner of file uploads to the public share + NotifyUploads bool `json:"notify_uploads" xml:"notify_uploads"` + // Additional recipients for the file upload to public share notification + NotifyUploadsExtraRecipients string `json:"notify_uploads_extra_recipients" xml:"notify_uploads_extra_recipients"` } // ShareeData holds share recipient search results. @@ -214,15 +218,17 @@ func PublicShare2ShareData(share *link.PublicShare, r *http.Request, publicURL s sd := &ShareData{ // share.permissions are mapped below // Displaynames are added later - ShareType: ShareTypePublicLink, - Token: share.Token, - Name: share.DisplayName, - MailSend: 0, - URL: publicURL + path.Join("/", "s/"+share.Token), - UIDOwner: LocalUserIDToString(share.Creator), - UIDFileOwner: LocalUserIDToString(share.Owner), - Quicklink: share.Quicklink, - Description: share.Description, + ShareType: ShareTypePublicLink, + Token: share.Token, + Name: share.DisplayName, + MailSend: 0, + URL: publicURL + path.Join("/", "s/"+share.Token), + UIDOwner: LocalUserIDToString(share.Creator), + UIDFileOwner: LocalUserIDToString(share.Owner), + Quicklink: share.Quicklink, + Description: share.Description, + NotifyUploads: share.NotifyUploads, + NotifyUploadsExtraRecipients: share.NotifyUploadsExtraRecipients, } if share.Id != nil { sd.ID = share.Id.OpaqueId diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/group.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/group.go index c1cc10bba3..8041595938 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/group.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/group.go @@ -20,6 +20,7 @@ package shares import ( "net/http" + "strconv" grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" @@ -28,6 +29,7 @@ import ( types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" + ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" ) @@ -80,5 +82,13 @@ func (h *Handler) createGroupShare(w http.ResponseWriter, r *http.Request, statI }, } - h.createCs3Share(ctx, w, r, c, createShareReq, statInfo) + if shareID, ok := h.createCs3Share(ctx, w, r, c, createShareReq, statInfo); ok { + notify, _ := strconv.ParseBool(r.FormValue("notify")) + if notify { + granter, ok := ctxpkg.ContextGetUser(ctx) + if ok { + h.SendShareNotification(shareID.OpaqueId, granter, groupRes.Group, statInfo) + } + } + } } diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go index 78784a9d05..312b55f3fb 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/public.go @@ -31,6 +31,8 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" "github.com/cs3org/reva/pkg/appctx" + ctxpkg "github.com/cs3org/reva/pkg/ctx" + "github.com/cs3org/reva/pkg/notification" "github.com/cs3org/reva/pkg/publicshare" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/pkg/errors" @@ -109,6 +111,8 @@ func (h *Handler) createPublicLinkShare(w http.ResponseWriter, r *http.Request, } internal, _ := strconv.ParseBool(r.FormValue("internal")) + notifyUploads, _ := strconv.ParseBool(r.FormValue("notifyUploads")) + notifyUploadsExtraRecipients := r.FormValue("notifyUploadsExtraRecipients") req := link.CreatePublicShareRequest{ ResourceInfo: statInfo, @@ -118,8 +122,10 @@ func (h *Handler) createPublicLinkShare(w http.ResponseWriter, r *http.Request, }, Password: r.FormValue("password"), }, - Description: r.FormValue("description"), - Internal: internal, + Description: r.FormValue("description"), + Internal: internal, + NotifyUploads: notifyUploads, + NotifyUploadsExtraRecipients: notifyUploadsExtraRecipients, } expireTimeString, ok := r.Form["expireDate"] @@ -317,6 +323,25 @@ func (h *Handler) updatePublicShare(w http.ResponseWriter, r *http.Request, shar }, }) } + + // remove notifications when a public link stops having 'uploader' permissions + if !isPermissionUploader(newPermissions) { + if before.Share.NotifyUploads { + updates = append(updates, &link.UpdatePublicShareRequest_Update{ + Type: link.UpdatePublicShareRequest_Update_TYPE_NOTIFYUPLOADS, + NotifyUploads: false, + }) + } + + if before.Share.NotifyUploadsExtraRecipients != "" { + updates = append(updates, &link.UpdatePublicShareRequest_Update{ + Type: link.UpdatePublicShareRequest_Update_TYPE_NOTIFYUPLOADSEXTRARECIPIENTS, + NotifyUploadsExtraRecipients: "", + }) + } + + h.notificationHelper.UnregisterNotification(shareID) + } } // ExpireDate @@ -371,6 +396,64 @@ func (h *Handler) updatePublicShare(w http.ResponseWriter, r *http.Request, shar }) } + // NotifyUploads + newNotifyUploads, ok := r.Form["notifyUploads"] + + if ok { + ok2 := permissionsStayUploader(before, newPermissions) + u, ok3 := ctxpkg.ContextGetUser(r.Context()) + + if ok2 && ok3 { + notifyUploads, _ := strconv.ParseBool(newNotifyUploads[0]) + updatesFound = true + + logger.Info().Str("shares", "update").Msgf("notify uploads updated to '%v'", notifyUploads) + updates = append(updates, &link.UpdatePublicShareRequest_Update{ + Type: link.UpdatePublicShareRequest_Update_TYPE_NOTIFYUPLOADS, + NotifyUploads: notifyUploads, + }) + + if notifyUploads { + n := ¬ification.Notification{ + TemplateName: "sharedfolder-upload-mail", + Ref: shareID, + Recipients: []string{u.Mail}, + } + h.notificationHelper.RegisterNotification(n) + } else { + h.notificationHelper.UnregisterNotification(shareID) + } + } + } + + // NotifyUploadsExtraRecipients + newNotifyUploadsExtraRecipients, ok := r.Form["notifyUploadsExtraRecipients"] + + if ok { + ok2 := permissionsStayUploader(before, newPermissions) + u, ok3 := ctxpkg.ContextGetUser(r.Context()) + + if ok2 && ok3 { + notifyUploadsExtraRecipients := newNotifyUploadsExtraRecipients[0] + updatesFound = true + logger.Info().Str("shares", "update").Msgf("notify uploads extra recipients updated to '%v'", notifyUploadsExtraRecipients) + + updates = append(updates, &link.UpdatePublicShareRequest_Update{ + Type: link.UpdatePublicShareRequest_Update_TYPE_NOTIFYUPLOADSEXTRARECIPIENTS, + NotifyUploadsExtraRecipients: notifyUploadsExtraRecipients, + }) + + if len(notifyUploadsExtraRecipients) > 0 { + n := ¬ification.Notification{ + TemplateName: "sharedfolder-upload-mail", + Ref: shareID, + Recipients: []string{u.Mail, notifyUploadsExtraRecipients}, + } + h.notificationHelper.RegisterNotification(n) + } + } + } + publicShare := before.Share // Updates are atomical. See: https://github.com/cs3org/cs3apis/pull/67#issuecomment-617651428 so in order to get the latest updated version @@ -452,6 +535,8 @@ func (h *Handler) removePublicShare(w http.ResponseWriter, r *http.Request, shar return } + h.notificationHelper.UnregisterNotification(shareID) + response.WriteOCSSuccess(w, r, nil) } @@ -515,6 +600,21 @@ func permissionFromRequest(r *http.Request, h *Handler) (*provider.ResourcePermi return p, err } +func isPermissionUploader(permissions *provider.ResourcePermissions) bool { + if permissions == nil { + return false + } + + publicSharePermissions := &link.PublicSharePermissions{ + Permissions: permissions, + } + return conversions.RoleFromResourcePermissions(publicSharePermissions.Permissions).Name == conversions.RoleUploader +} + +func permissionsStayUploader(before *link.GetPublicShareResponse, newPermissions *provider.ResourcePermissions) bool { + return (newPermissions == nil && isPermissionUploader(before.Share.GetPermissions().Permissions)) || isPermissionUploader(newPermissions) +} + // TODO: add mapping for user share permissions to role // Maps oc10 public link permissions to roles. diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go index 1553dbb25f..106aa7c9af 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go @@ -26,6 +26,7 @@ import ( "mime" "net/http" "path" + "path/filepath" "strconv" "strings" "text/template" @@ -44,6 +45,10 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" "github.com/cs3org/reva/pkg/appctx" + ctxpkg "github.com/cs3org/reva/pkg/ctx" + "github.com/cs3org/reva/pkg/notification" + "github.com/cs3org/reva/pkg/notification/notificationhelper" + "github.com/cs3org/reva/pkg/notification/trigger" "github.com/cs3org/reva/pkg/publicshare" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/share" @@ -74,6 +79,8 @@ type Handler struct { resourceInfoCache cache.ResourceInfoCache resourceInfoCacheTTL time.Duration listOCMShares bool + notificationHelper *notificationhelper.NotificationHelper + Log *zerolog.Logger } // we only cache the minimal set of data instead of the full user metadata. @@ -98,7 +105,7 @@ func getCacheManager(c *config.Config) (cache.ResourceInfoCache, error) { } // Init initializes this and any contained handlers. -func (h *Handler) Init(c *config.Config) { +func (h *Handler) Init(c *config.Config, l *zerolog.Logger) { h.gatewayAddr = c.GatewaySvc h.storageRegistryAddr = c.StorageregistrySvc h.publicURL = c.Config.Host @@ -106,7 +113,8 @@ func (h *Handler) Init(c *config.Config) { h.homeNamespace = c.HomeNamespace h.ocmMountPoint = c.OCMMountPoint h.listOCMShares = c.ListOCMShares - + h.Log = l + h.notificationHelper = notificationhelper.New("ocs", c.Notifications, l) h.additionalInfoTemplate, _ = template.New("additionalInfo").Parse(c.AdditionalInfoAttribute) h.resourceInfoCacheTTL = time.Second * time.Duration(c.ResourceInfoCacheTTL) @@ -234,6 +242,125 @@ func (h *Handler) CreateShare(w http.ResponseWriter, r *http.Request) { } } +// NotifyShare handles GET requests on /apps/files_sharing/api/v1/shares/(shareid)/notify. +func (h *Handler) NotifyShare(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + opaqueID := chi.URLParam(r, "shareid") + + c, err := pool.GetGatewayServiceClient(pool.Endpoint(h.gatewayAddr)) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error getting grpc gateway client", err) + return + } + + shareRes, err := c.GetShare(ctx, &collaboration.GetShareRequest{ + Ref: &collaboration.ShareReference{ + Spec: &collaboration.ShareReference_Id{ + Id: &collaboration.ShareId{ + OpaqueId: opaqueID, + }, + }, + }, + }) + if err != nil || shareRes.Status.Code == rpc.Code_CODE_NOT_FOUND { + h.Log.Error().Err(err).Msg("error getting share") + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error getting share", err) + return + } + + granter, ok := ctxpkg.ContextGetUser(ctx) + if !ok { + h.Log.Error().Err(err).Msgf("error getting granter data") + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error getting granter data", err) + } + + resourceID := shareRes.Share.ResourceId + statInfo, status, err := h.getResourceInfoByID(ctx, c, resourceID) + if err != nil || status.Code != rpc.Code_CODE_OK { + h.Log.Error().Err(err).Msg("error mapping share data") + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error mapping share data", err) + return + } + + var recipient string + + granteeType := shareRes.Share.Grantee.Type + if granteeType == provider.GranteeType_GRANTEE_TYPE_USER { + granteeID := shareRes.Share.Grantee.GetUserId().OpaqueId + granteeRes, err := c.GetUserByClaim(ctx, &userpb.GetUserByClaimRequest{ + Claim: "username", + Value: granteeID, + SkipFetchingUserGroups: true, + }) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error getting grantee data", err) + return + } + + recipient = h.SendShareNotification(opaqueID, granter, granteeRes.User, statInfo) + } else if granteeType == provider.GranteeType_GRANTEE_TYPE_GROUP { + granteeID := shareRes.Share.Grantee.GetGroupId().OpaqueId + granteeRes, err := c.GetGroupByClaim(ctx, &grouppb.GetGroupByClaimRequest{ + Claim: "group_name", + Value: granteeID, + SkipFetchingMembers: true, + }) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error getting grantee data", err) + return + } + + recipient = h.SendShareNotification(opaqueID, granter, granteeRes.Group, statInfo) + } + + w.WriteHeader(http.StatusOK) + w.Header().Add("Content-Type", "application/json") + rb, _ := json.Marshal(map[string]interface{}{"recipients": []string{recipient}}) + _, err = w.Write(rb) + if err != nil { + h.Log.Error().Err(err).Msg("error writing response") + } +} + +// SendShareNotification sends a notification with information from a Share. +func (h *Handler) SendShareNotification(opaqueID string, granter *userpb.User, grantee interface{}, statInfo *provider.ResourceInfo) string { + var granteeDisplayName, granteeName, recipient string + isGranteeGroup := false + + if u, ok := grantee.(*userpb.User); ok { + granteeDisplayName = u.DisplayName + granteeName = u.Username + recipient = u.Mail + } else if g, ok := grantee.(*grouppb.Group); ok { + granteeDisplayName = g.DisplayName + granteeName = g.GroupName + recipient = g.Mail + isGranteeGroup = true + } + + h.notificationHelper.TriggerNotification(&trigger.Trigger{ + Notification: ¬ification.Notification{ + TemplateName: "share-create-mail", + Ref: opaqueID, + Recipients: []string{recipient}, + }, + Ref: opaqueID, + TemplateData: map[string]interface{}{ + "granteeDisplayName": granteeDisplayName, + "granteeUserName": granteeName, + "granterDisplayName": granter.DisplayName, + "granterUserName": granter.Username, + "path": statInfo.Path, + "isFolder": statInfo.Type == provider.ResourceType_RESOURCE_TYPE_CONTAINER, + "isGranteeGroup": isGranteeGroup, + "base": filepath.Base(statInfo.Path), + }, + }) + h.Log.Debug().Msgf("notification trigger %s created", opaqueID) + + return recipient +} + func (h *Handler) extractPermissions(w http.ResponseWriter, r *http.Request, ri *provider.ResourceInfo, defaultPermissions *conversions.Role) (*conversions.Role, []byte, error) { reqRole, reqPermissions := r.FormValue("role"), r.FormValue("permissions") var role *conversions.Role @@ -1103,33 +1230,34 @@ func (h *Handler) getResourceInfo(ctx context.Context, client gateway.GatewayAPI return pinfo, status, nil } -func (h *Handler) createCs3Share(ctx context.Context, w http.ResponseWriter, r *http.Request, client gateway.GatewayAPIClient, req *collaboration.CreateShareRequest, info *provider.ResourceInfo) { +func (h *Handler) createCs3Share(ctx context.Context, w http.ResponseWriter, r *http.Request, client gateway.GatewayAPIClient, req *collaboration.CreateShareRequest, info *provider.ResourceInfo) (*collaboration.ShareId, bool) { createShareResponse, err := client.CreateShare(ctx, req) if err != nil { response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error sending a grpc create share request", err) - return + return nil, false } if createShareResponse.Status.Code != rpc.Code_CODE_OK { if createShareResponse.Status.Code == rpc.Code_CODE_NOT_FOUND { response.WriteOCSError(w, r, response.MetaNotFound.StatusCode, "not found", nil) - return + return nil, false } response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "grpc create share request failed", err) - return + return nil, false } s, err := conversions.CS3Share2ShareData(ctx, createShareResponse.Share) if err != nil { response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error mapping share data", err) - return + return nil, false } err = h.addFileInfo(ctx, s, info) if err != nil { response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error adding fileinfo to share", err) - return + return nil, false } h.mapUserIds(ctx, client, s) response.WriteOCSSuccess(w, r, s) + return createShareResponse.Share.Id, true } func mapState(state collaboration.ShareState) int { diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go index 620a7bdaf7..ba7acc601f 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go @@ -20,6 +20,7 @@ package shares import ( "net/http" + "strconv" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" @@ -29,6 +30,7 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" "github.com/cs3org/reva/pkg/appctx" + ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" ) @@ -82,7 +84,15 @@ func (h *Handler) createUserShare(w http.ResponseWriter, r *http.Request, statIn }, } - h.createCs3Share(ctx, w, r, c, createShareReq, statInfo) + if shareID, ok := h.createCs3Share(ctx, w, r, c, createShareReq, statInfo); ok { + notify, _ := strconv.ParseBool(r.FormValue("notify")) + if notify { + granter, ok := ctxpkg.ContextGetUser(ctx) + if ok { + h.SendShareNotification(shareID.OpaqueId, granter, userRes.User, statInfo) + } + } + } } func (h *Handler) isUserShare(r *http.Request, oid string) bool { diff --git a/internal/http/services/owncloud/ocs/ocs.go b/internal/http/services/owncloud/ocs/ocs.go index c579fa04f9..5aef6c9ffb 100644 --- a/internal/http/services/owncloud/ocs/ocs.go +++ b/internal/http/services/owncloud/ocs/ocs.go @@ -62,7 +62,7 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) router: r, } - if err := s.routerInit(); err != nil { + if err := s.routerInit(log); err != nil { return nil, err } @@ -86,7 +86,7 @@ func (s *svc) Unprotected() []string { return []string{"/v1.php/cloud/capabilities", "/v2.php/cloud/capabilities"} } -func (s *svc) routerInit() error { +func (s *svc) routerInit(l *zerolog.Logger) error { capabilitiesHandler := new(capabilities.Handler) userHandler := new(user.Handler) usersHandler := new(users.Handler) @@ -97,7 +97,7 @@ func (s *svc) routerInit() error { usersHandler.Init(s.c) userHandler.Init(s.c) configHandler.Init(s.c) - sharesHandler.Init(s.c) + sharesHandler.Init(s.c, l) shareesHandler.Init(s.c) s.router.Route("/v{version:(1|2)}.php", func(r chi.Router) { @@ -119,16 +119,12 @@ func (s *svc) routerInit() error { }) r.Get("/{shareid}", sharesHandler.GetShare) r.Put("/{shareid}", sharesHandler.UpdateShare) + r.Get("/{shareid}/notify", sharesHandler.NotifyShare) r.Delete("/{shareid}", sharesHandler.RemoveShare) }) r.Get("/sharees", shareesHandler.FindSharees) }) - // placeholder for notifications - r.Get("/apps/notifications/api/v1/notifications", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - }) - r.Get("/config", configHandler.GetConfig) r.Route("/cloud", func(r chi.Router) { diff --git a/internal/serverless/services/loader/loader.go b/internal/serverless/services/loader/loader.go index 1b466a144c..71ab9afb1d 100644 --- a/internal/serverless/services/loader/loader.go +++ b/internal/serverless/services/loader/loader.go @@ -21,5 +21,6 @@ package loader import ( // Load core serverless services. _ "github.com/cs3org/reva/internal/serverless/services/helloworld" + _ "github.com/cs3org/reva/internal/serverless/services/notifications" // Add your own service here. ) diff --git a/internal/serverless/services/notifications/notifications.go b/internal/serverless/services/notifications/notifications.go new file mode 100644 index 0000000000..b51b163fd4 --- /dev/null +++ b/internal/serverless/services/notifications/notifications.go @@ -0,0 +1,403 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package notifications + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/pkg/errors" + + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/notification" + "github.com/cs3org/reva/pkg/notification/handler" + handlerRegistry "github.com/cs3org/reva/pkg/notification/handler/registry" + notificationManagerRegistry "github.com/cs3org/reva/pkg/notification/manager/registry" + "github.com/cs3org/reva/pkg/notification/template" + templateRegistry "github.com/cs3org/reva/pkg/notification/template/registry" + "github.com/cs3org/reva/pkg/notification/trigger" + "github.com/cs3org/reva/pkg/notification/utils" + "github.com/cs3org/reva/pkg/rserverless" + "github.com/cs3org/reva/pkg/utils/accumulator" + "github.com/mitchellh/mapstructure" + "github.com/nats-io/nats.go" + "github.com/rs/zerolog" +) + +type config struct { + NatsAddress string `mapstructure:"nats_address" docs:";The NATS server address."` + NatsToken string `mapstructure:"nats_token" docs:"The token to authenticate against the NATS server"` + NatsPrefix string `mapstructure:"nats_prefix" docs:"reva-notifications;The notifications NATS stream."` + HandlerConf map[string]interface{} `mapstructure:"handlers" docs:";Settings for the different notification handlers."` + GroupingInterval int `mapstructure:"grouping_interval" docs:"60;Time in seconds to group incoming notification triggers"` + GroupingMaxSize int `mapstructure:"grouping_max_size" docs:"100;Maximum number of notifications to group"` + StorageDriver string `mapstructure:"storage_driver" docs:"mysql;The driver used to store notifications"` + StorageDrivers map[string]map[string]interface{} `mapstructure:"storage_drivers"` +} + +func defaultConfig() *config { + return &config{ + NatsPrefix: "reva-notifications", + GroupingInterval: 60, + GroupingMaxSize: 100, + StorageDriver: "sql", + } +} + +type svc struct { + nc *nats.Conn + js nats.JetStreamContext + kv nats.KeyValue + conf *config + log *zerolog.Logger + handlers map[string]handler.Handler + templates templateRegistry.Registry + nm notification.Manager + accumulators map[string]*accumulator.Accumulator[trigger.Trigger] +} + +func init() { + rserverless.Register("notifications", New) +} + +func getNotificationManager(c *config, l *zerolog.Logger) (notification.Manager, error) { + if f, ok := notificationManagerRegistry.NewFuncs[c.StorageDriver]; ok { + return f(c.StorageDrivers[c.StorageDriver]) + } + return nil, errtypes.NotFound(fmt.Sprintf("storage driver %s not found", c.StorageDriver)) +} + +// New returns a new Notifications service. +func New(m map[string]interface{}, log *zerolog.Logger) (rserverless.Service, error) { + conf := defaultConfig() + + if err := mapstructure.Decode(m, conf); err != nil { + return nil, err + } + + nm, err := getNotificationManager(conf, log) + if err != nil { + return nil, err + } + log.Info().Msgf("notification storage %s initialized", conf.StorageDriver) + + s := &svc{ + conf: conf, + log: log, + nm: nm, + } + + return s, nil +} + +// Start starts the Notifications service. +func (s *svc) Start() { + s.templates = *templateRegistry.New() + s.handlers = handlerRegistry.InitHandlers(s.conf.HandlerConf, s.log) + s.accumulators = make(map[string]*accumulator.Accumulator[trigger.Trigger]) + + s.log.Debug().Msgf("connecting to nats server at %s", s.conf.NatsAddress) + err := s.connect() + if err != nil { + s.log.Error().Err(err).Msg("connecting to nats failed") + } + s.log.Info().Msg("notifications service ready") +} + +// Close performs cleanup. +func (s *svc) Close(ctx context.Context) error { + return s.nc.Drain() +} + +func (s *svc) connect() error { + nc, err := utils.ConnectToNats(s.conf.NatsAddress, s.conf.NatsToken, *s.log) + if err != nil { + return err + } + s.nc = nc + + js, err := nc.JetStream(nats.PublishAsyncMaxPending(256)) + if err != nil { + return errors.Wrap(err, "jetstream initialization failed") + } + + s.js = js + + if err := s.initNatsKV("template", s.handleMsgTemplate); err != nil { + return err + } + if err := s.initNatsStream("notification-register", s.handleMsgRegisterNotification); err != nil { + return err + } + if err := s.initNatsStream("notification-unregister", s.handleMsgUnregisterNotification); err != nil { + return err + } + return s.initNatsStream("trigger", s.handleMsgTrigger) +} + +func (s *svc) initNatsKV(name string, handler func(msg []byte)) error { + bucketName := fmt.Sprintf("%s-%s", s.conf.NatsPrefix, name) + kv, err := s.js.CreateKeyValue(&nats.KeyValueConfig{ + Bucket: bucketName, + }) + if err != nil { + return errors.Wrap(err, "template store creation failed, probably because nats server is unreachable") + } + + s.kv = kv + + w, _ := kv.WatchAll() + + go func() { + for { + msg := <-w.Updates() + + if msg != nil { + handler(msg.Value()) + } + } + }() + + return nil +} + +func (s *svc) initNatsStream(name string, handler func(msg *nats.Msg)) error { + streamName := fmt.Sprintf("%s-%s", s.conf.NatsPrefix, name) + consumerName := fmt.Sprintf("%s-consumer-%s", s.conf.NatsPrefix, name) + subjectName := fmt.Sprintf("%s.%s", s.conf.NatsPrefix, name) + deliverySubjectName := fmt.Sprintf("%s-delivery.%s", s.conf.NatsPrefix, name) + + // Creates a NATS stream with given name if it does not exist already + if _, err := s.js.AddStream(&nats.StreamConfig{ + Name: streamName, + Subjects: []string{subjectName}, + }); err != nil { + return errors.Wrapf(err, "nats %s stream creation failed", name) + } + + // Adds a consumer with the given name to the JetStream context + if _, err := s.js.AddConsumer(streamName, &nats.ConsumerConfig{ + Durable: consumerName, + DeliverSubject: deliverySubjectName, + }); err != nil { + return errors.Wrapf(err, "nats %s consumer creation failed", name) + } + + // Subscribes the JetStream context to the consumer we just created + _, err := s.js.Subscribe("", func(msg *nats.Msg) { handler(msg) }, nats.Bind(streamName, consumerName)) + if err != nil { + return errors.Wrapf(err, "nats subscription to consumer %s failed", consumerName) + } + + return nil +} + +func (s *svc) handleMsgTemplate(msg []byte) { + if len(msg) == 0 { + return + } + + name, err := s.templates.Put(msg, s.handlers) + if err != nil { + s.log.Error().Err(err).Msgf("template registration failed %v", err) + + // If a template file was not found, delete that template from the registry altogether, + // this way we ensure templates that are deleted from the config are deleted from the + // store too. + wrappedErr := errors.Unwrap(errors.Unwrap(err)) + _, isFileNotFoundError := wrappedErr.(*template.FileNotFoundError) + if isFileNotFoundError && name != "" { + err := s.kv.Purge(name) + if err != nil { + s.log.Error().Err(err).Msgf("deletion of template %s from store failed", name) + } + s.log.Info().Msgf("template %s unregistered", name) + } + } else { + s.log.Info().Msgf("template %s registered", name) + } +} + +func (s *svc) handleMsgRegisterNotification(msg *nats.Msg) { + var data map[string]interface{} + err := json.Unmarshal(msg.Data, &data) + if err != nil { + s.log.Error().Err(err).Msg("notification registration unmarshall failed") + return + } + + n := ¬ification.Notification{} + if err := mapstructure.Decode(data, n); err != nil { + s.log.Error().Err(err).Msg("notification registration decoding failed") + return + } + + templ, err := s.templates.Get(n.TemplateName) + if err != nil { + s.log.Error().Err(err).Msg("notification template get failed") + return + } + + n.Template = *templ + err = s.nm.UpsertNotification(*n) + if err != nil { + s.log.Error().Err(err).Msgf("registering notification %s failed", n.Ref) + } else { + s.log.Info().Msgf("notification %s registered", n.Ref) + } +} + +func (s *svc) handleMsgUnregisterNotification(msg *nats.Msg) { + ref := string(msg.Data) + + err := s.nm.DeleteNotification(ref) + if err != nil { + _, isNotFoundError := err.(*notification.NotFoundError) + if isNotFoundError { + s.log.Debug().Msgf("a notification with ref %s does not exist", ref) + } else { + s.log.Error().Err(err).Msgf("notification unregister failed") + } + } else { + s.log.Debug().Msgf("notification %s unregistered", ref) + } +} + +func (s *svc) getAccumulatorForTrigger(tr trigger.Trigger) *accumulator.Accumulator[trigger.Trigger] { + a, ok := s.accumulators[tr.Ref] + + if !ok || a == nil { + timeout := time.Duration(s.conf.GroupingInterval) * time.Second + maxSize := s.conf.GroupingMaxSize + + a = accumulator.New[trigger.Trigger](timeout, maxSize, s.log) + _ = a.Start(s.notificationSendCallback) + s.accumulators[tr.Ref] = a + + s.log.Debug().Msgf("created new accumulator for trigger %s", tr.Ref) + } + + return a +} + +func (s *svc) handleMsgTrigger(msg *nats.Msg) { + var data map[string]interface{} + err := json.Unmarshal(msg.Data, &data) + if err != nil { + s.log.Error().Err(err).Msg("notification trigger unmarshall failed") + return + } + + tr := &trigger.Trigger{} + if err := mapstructure.Decode(data, tr); err != nil { + s.log.Error().Err(err).Msg("trigger creation failed") + return + } + + s.log.Info().Msgf("notification trigger %s received", tr.Ref) + + notif := tr.Notification + if notif == nil { + notif, err = s.nm.GetNotification(tr.Ref) + if err != nil { + _, isNotFoundError := err.(*notification.NotFoundError) + if isNotFoundError { + s.log.Debug().Msgf("trigger %s does not have a notification attached", tr.Ref) + return + } + s.log.Error().Err(err).Msgf("notification retrieval from store failed") + return + } + } + + templ, err := s.templates.Get(notif.TemplateName) + if err != nil { + s.log.Error().Err(err).Msgf("template %s for trigger %s not found", notif.TemplateName, tr.Ref) + return + } + + notif.Template = *templ + tr.Notification = notif + a := s.getAccumulatorForTrigger(*tr) + a.Input <- *tr +} + +func (s *svc) notificationSendCallback(ts []trigger.Trigger) { + const itemCount = 10 + var tr trigger.Trigger + + if len(ts) == 1 { + tr = ts[0] + s.log.Info().Msgf("sending single notification for trigger %s", tr.Ref) + } else { + moreCount := len(ts) - itemCount + if moreCount < 0 { + moreCount = 0 + } + + // create a new trigger + tr = trigger.Trigger{ + Ref: ts[0].Ref, + Sender: ts[0].Sender, + TemplateData: map[string]interface{}{ + "_count": len(ts), + "_items": []map[string]interface{}{}, + "_moreCount": moreCount, + }, + } + + // add template data of the first ten elements, ignore the rest + l := itemCount + templateData := []map[string]interface{}{} + if l > len(ts) { + l = len(ts) + } + for _, t := range ts[:l] { + templateData = append(templateData, t.TemplateData) + } + tr.TemplateData["_items"] = templateData + + // initialize the new trigger + notif, err := s.nm.GetNotification(tr.Ref) + if err != nil { + s.log.Error().Msgf("notification retrieval from store failed") + return + } + + templ, err := s.templates.Get(notif.TemplateName) + if err != nil { + s.log.Error().Err(err).Msgf("template %s for trigger %s not found", notif.TemplateName, tr.Ref) + return + } + + notif.Template = *templ + tr.Notification = notif + + s.log.Info().Msgf("sending multi notification for %d triggers %s", tr.TemplateData["_count"], tr.Ref) + } + + // destroy old accumulator + s.accumulators[tr.Ref] = nil + + if err := tr.Send(); err != nil { + s.log.Error().Err(err).Msgf("notification send failed") + } +} diff --git a/pkg/cbox/publicshare/sql/sql.go b/pkg/cbox/publicshare/sql/sql.go index d06c396903..aaefa85dac 100644 --- a/pkg/cbox/publicshare/sql/sql.go +++ b/pkg/cbox/publicshare/sql/sql.go @@ -135,7 +135,7 @@ func New(m map[string]interface{}) (publicshare.Manager, error) { return &mgr, nil } -func (m *manager) CreatePublicShare(ctx context.Context, u *user.User, rInfo *provider.ResourceInfo, g *link.Grant, description string, internal bool) (*link.PublicShare, error) { +func (m *manager) CreatePublicShare(ctx context.Context, u *user.User, rInfo *provider.ResourceInfo, g *link.Grant, description string, internal bool, notifyUploads bool, notifyUploadsExtraRecipients string) (*link.PublicShare, error) { tkn := utils.RandString(15) now := time.Now().Unix() @@ -162,8 +162,8 @@ func (m *manager) CreatePublicShare(ctx context.Context, u *user.User, rInfo *pr fileSource = 0 } - query := "insert into oc_share set share_type=?,uid_owner=?,uid_initiator=?,item_type=?,fileid_prefix=?,item_source=?,file_source=?,permissions=?,stime=?,token=?,share_name=?,quicklink=?,description=?,internal=?" - params := []interface{}{publicShareType, owner, creator, itemType, prefix, itemSource, fileSource, permissions, now, tkn, displayName, quicklink, description, internal} + query := "insert into oc_share set share_type=?,uid_owner=?,uid_initiator=?,item_type=?,fileid_prefix=?,item_source=?,file_source=?,permissions=?,stime=?,token=?,share_name=?,quicklink=?,description=?,internal=?,notify_uploads=?,notify_uploads_extra_recipients=?" + params := []interface{}{publicShareType, owner, creator, itemType, prefix, itemSource, fileSource, permissions, now, tkn, displayName, quicklink, description, internal, notifyUploads, notifyUploadsExtraRecipients} var passwordProtected bool password := g.Password @@ -201,18 +201,20 @@ func (m *manager) CreatePublicShare(ctx context.Context, u *user.User, rInfo *pr Id: &link.PublicShareId{ OpaqueId: strconv.FormatInt(lastID, 10), }, - Owner: rInfo.GetOwner(), - Creator: u.Id, - ResourceId: rInfo.Id, - Token: tkn, - Permissions: g.Permissions, - Ctime: createdAt, - Mtime: createdAt, - PasswordProtected: passwordProtected, - Expiration: g.Expiration, - DisplayName: displayName, - Quicklink: quicklink, - Description: description, + Owner: rInfo.GetOwner(), + Creator: u.Id, + ResourceId: rInfo.Id, + Token: tkn, + Permissions: g.Permissions, + Ctime: createdAt, + Mtime: createdAt, + PasswordProtected: passwordProtected, + Expiration: g.Expiration, + DisplayName: displayName, + Quicklink: quicklink, + Description: description, + NotifyUploads: notifyUploads, + NotifyUploadsExtraRecipients: notifyUploadsExtraRecipients, }, nil } @@ -243,6 +245,10 @@ func (m *manager) UpdatePublicShare(ctx context.Context, u *user.User, req *link } case link.UpdatePublicShareRequest_Update_TYPE_DESCRIPTION: paramsMap["description"] = req.Update.GetDescription() + case link.UpdatePublicShareRequest_Update_TYPE_NOTIFYUPLOADS: + paramsMap["notify_uploads"] = req.Update.GetNotifyUploads() + case link.UpdatePublicShareRequest_Update_TYPE_NOTIFYUPLOADSEXTRARECIPIENTS: + paramsMap["notify_uploads_extra_recipients"] = req.Update.GetNotifyUploadsExtraRecipients() default: return nil, fmt.Errorf("invalid update type: %v", req.GetUpdate().GetType()) } @@ -276,8 +282,8 @@ func (m *manager) UpdatePublicShare(ctx context.Context, u *user.User, req *link func (m *manager) getByToken(ctx context.Context, token string, u *user.User) (*link.PublicShare, string, error) { s := conversions.DBShare{Token: token} - query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(expiration, '') as expiration, coalesce(share_name, '') as share_name, id, stime, permissions, quicklink, description FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND share_type=? AND token=?" - if err := m.db.QueryRow(query, publicShareType, token).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.Expiration, &s.ShareName, &s.ID, &s.STime, &s.Permissions, &s.Quicklink, &s.Description); err != nil { + query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(expiration, '') as expiration, coalesce(share_name, '') as share_name, id, stime, permissions, quicklink, description, notify_uploads, notify_uploads_extra_recipients FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND share_type=? AND token=?" + if err := m.db.QueryRow(query, publicShareType, token).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.Expiration, &s.ShareName, &s.ID, &s.STime, &s.Permissions, &s.Quicklink, &s.Description, &s.NotifyUploads, &s.NotifyUploadsExtraRecipients); err != nil { if err == sql.ErrNoRows { return nil, "", errtypes.NotFound(token) } @@ -293,8 +299,8 @@ func (m *manager) getByToken(ctx context.Context, token string, u *user.User) (* func (m *manager) getByID(ctx context.Context, id *link.PublicShareId, u *user.User) (*link.PublicShare, string, error) { uid := conversions.FormatUserID(u.Id) s := conversions.DBShare{ID: id.OpaqueId} - query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(token,'') as token, coalesce(expiration, '') as expiration, coalesce(share_name, '') as share_name, stime, permissions, quicklink, description FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND share_type=? AND id=? AND (uid_owner=? OR uid_initiator=?)" - if err := m.db.QueryRow(query, publicShareType, id.OpaqueId, uid, uid).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.Token, &s.Expiration, &s.ShareName, &s.STime, &s.Permissions, &s.Quicklink, &s.Description); err != nil { + query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(token,'') as token, coalesce(expiration, '') as expiration, coalesce(share_name, '') as share_name, stime, permissions, quicklink, description, notify_uploads, notify_uploads_extra_recipients FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND share_type=? AND id=? AND (uid_owner=? OR uid_initiator=?)" + if err := m.db.QueryRow(query, publicShareType, id.OpaqueId, uid, uid).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.Token, &s.Expiration, &s.ShareName, &s.STime, &s.Permissions, &s.Quicklink, &s.Description, &s.NotifyUploads, &s.NotifyUploadsExtraRecipients); err != nil { if err == sql.ErrNoRows { return nil, "", errtypes.NotFound(id.OpaqueId) } @@ -340,7 +346,7 @@ func (m *manager) GetPublicShare(ctx context.Context, u *user.User, ref *link.Pu } func (m *manager) ListPublicShares(ctx context.Context, u *user.User, filters []*link.ListPublicSharesRequest_Filter, md *provider.ResourceInfo, sign bool) ([]*link.PublicShare, error) { - query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(token,'') as token, coalesce(expiration, '') as expiration, coalesce(share_name, '') as share_name, id, stime, permissions, quicklink, description FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND (share_type=?) AND internal=false" + query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(token,'') as token, coalesce(expiration, '') as expiration, coalesce(share_name, '') as share_name, id, stime, permissions, quicklink, description, notify_uploads, notify_uploads_extra_recipients FROM oc_share WHERE (orphan = 0 or orphan IS NULL) AND (share_type=?) AND internal=false" var resourceFilters, ownerFilters, creatorFilters string var resourceParams, ownerParams, creatorParams []interface{} params := []interface{}{publicShareType} @@ -398,7 +404,7 @@ func (m *manager) ListPublicShares(ctx context.Context, u *user.User, filters [] var s conversions.DBShare shares := []*link.PublicShare{} for rows.Next() { - if err := rows.Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.Token, &s.Expiration, &s.ShareName, &s.ID, &s.STime, &s.Permissions, &s.Quicklink, &s.Description); err != nil { + if err := rows.Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.Token, &s.Expiration, &s.ShareName, &s.ID, &s.STime, &s.Permissions, &s.Quicklink, &s.Description, &s.NotifyUploads, &s.NotifyUploadsExtraRecipients); err != nil { continue } cs3Share, err := conversions.ConvertToCS3PublicShare(ctx, m.client, s) @@ -460,8 +466,8 @@ func (m *manager) RevokePublicShare(ctx context.Context, u *user.User, ref *link func (m *manager) GetPublicShareByToken(ctx context.Context, token string, auth *link.PublicShareAuthentication, sign bool) (*link.PublicShare, error) { s := conversions.DBShare{Token: token} - query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(expiration, '') as expiration, coalesce(share_name, '') as share_name, id, stime, permissions, quicklink, description FROM oc_share WHERE share_type=? AND token=?" - if err := m.db.QueryRow(query, publicShareType, token).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.Expiration, &s.ShareName, &s.ID, &s.STime, &s.Permissions, &s.Quicklink, &s.Description); err != nil { + query := "select coalesce(uid_owner, '') as uid_owner, coalesce(uid_initiator, '') as uid_initiator, coalesce(share_with, '') as share_with, coalesce(fileid_prefix, '') as fileid_prefix, coalesce(item_source, '') as item_source, coalesce(item_type, '') as item_type, coalesce(expiration, '') as expiration, coalesce(share_name, '') as share_name, id, stime, permissions, quicklink, description, notify_uploads, notify_uploads_extra_recipients FROM oc_share WHERE share_type=? AND token=?" + if err := m.db.QueryRow(query, publicShareType, token).Scan(&s.UIDOwner, &s.UIDInitiator, &s.ShareWith, &s.Prefix, &s.ItemSource, &s.ItemType, &s.Expiration, &s.ShareName, &s.ID, &s.STime, &s.Permissions, &s.Quicklink, &s.Description, &s.NotifyUploads, &s.NotifyUploadsExtraRecipients); err != nil { if err == sql.ErrNoRows { return nil, errtypes.NotFound(token) } diff --git a/pkg/cbox/utils/conversions.go b/pkg/cbox/utils/conversions.go index 0c3262b5a1..9a63109b3b 100644 --- a/pkg/cbox/utils/conversions.go +++ b/pkg/cbox/utils/conversions.go @@ -36,23 +36,25 @@ import ( // DBShare stores information about user and public shares. type DBShare struct { - ID string - UIDOwner string - UIDInitiator string - Prefix string - ItemSource string - ItemType string - ShareWith string - Token string - Expiration string - Permissions int - ShareType int - ShareName string - STime int - FileTarget string - State int - Quicklink bool - Description string + ID string + UIDOwner string + UIDInitiator string + Prefix string + ItemSource string + ItemType string + ShareWith string + Token string + Expiration string + Permissions int + ShareType int + ShareName string + STime int + FileTarget string + State int + Quicklink bool + Description string + NotifyUploads bool + NotifyUploadsExtraRecipients string } // FormatGrantee formats a CS3API grantee to a string. @@ -292,16 +294,18 @@ func ConvertToCS3PublicShare(ctx context.Context, gateway gatewayv1beta1.Gateway StorageId: s.Prefix, OpaqueId: s.ItemSource, }, - Permissions: &link.PublicSharePermissions{Permissions: IntTosharePerm(s.Permissions, s.ItemType)}, - Owner: owner, - Creator: creator, - Token: s.Token, - DisplayName: s.ShareName, - PasswordProtected: pwd, - Expiration: expires, - Ctime: ts, - Mtime: ts, - Quicklink: s.Quicklink, - Description: s.Description, + Permissions: &link.PublicSharePermissions{Permissions: IntTosharePerm(s.Permissions, s.ItemType)}, + Owner: owner, + Creator: creator, + Token: s.Token, + DisplayName: s.ShareName, + PasswordProtected: pwd, + Expiration: expires, + Ctime: ts, + Mtime: ts, + Quicklink: s.Quicklink, + Description: s.Description, + NotifyUploads: s.NotifyUploads, + NotifyUploadsExtraRecipients: s.NotifyUploadsExtraRecipients, }, nil } diff --git a/pkg/notification/db_changes.sql b/pkg/notification/db_changes.sql new file mode 100644 index 0000000000..7b7be6ad03 --- /dev/null +++ b/pkg/notification/db_changes.sql @@ -0,0 +1,54 @@ +-- Copyright 2018-2023 CERN +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- In applying this license, CERN does not waive the privileges and immunities +-- granted to it by virtue of its status as an Intergovernmental Organization +-- or submit itself to any jurisdiction. + +-- This file can be used to make the required changes to the MySQL DB. This is +-- not a proper migration but it should work on most situations. + +USE cernboxngcopy; + +CREATE TABLE `cbox_notifications` ( + `id` INT PRIMARY KEY AUTO_INCREMENT, + `ref` VARCHAR(3072) UNIQUE NOT NULL, + `template_name` VARCHAR(320) NOT NULL +); + +COMMIT; + +CREATE TABLE `cbox_notification_recipients` ( + `id` INT PRIMARY KEY AUTO_INCREMENT, + `notification_id` INT NOT NULL, + `recipient` VARCHAR(320) NOT NULL, + FOREIGN KEY (notification_id) + REFERENCES cbox_notifications (id) + ON DELETE CASCADE +); + +COMMIT; + +CREATE INDEX `cbox_notifications_ix0` ON `cbox_notifications` (`ref`); + +CREATE INDEX `cbox_notification_recipients_ix0` ON `cbox_notification_recipients` (`notification_id`); +CREATE INDEX `cbox_notification_recipients_ix1` ON `cbox_notification_recipients` (`user_name`); + +-- changes for added notifications on ocm shares + +ALTER TABLE cernboxngcopy.oc_share ADD notify_uploads BOOL DEFAULT false; + +UPDATE cernboxngcopy.oc_share SET notify_uploads = false; + +ALTER TABLE cernboxngcopy.oc_share MODIFY notify_uploads BOOL DEFAULT false NOT NULL; diff --git a/pkg/notification/db_sqlite.sql b/pkg/notification/db_sqlite.sql new file mode 100644 index 0000000000..8e110fe153 --- /dev/null +++ b/pkg/notification/db_sqlite.sql @@ -0,0 +1,51 @@ +-- Copyright 2018-2023 CERN +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- In applying this license, CERN does not waive the privileges and immunities +-- granted to it by virtue of its status as an Intergovernmental Organization +-- or submit itself to any jurisdiction. + +-- This file can be used to quickstart a SQLite DB for running the tests in +-- ./manager/sql/sql_test.go + +CREATE TABLE `cbox_notifications` ( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `ref` VARCHAR(3072) UNIQUE NOT NULL, + `template_name` VARCHAR(320) NOT NULL +); + +COMMIT; + +CREATE TABLE `cbox_notification_recipients` ( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `notification_id` INTEGER NOT NULL, + `recipient` VARCHAR(320) NOT NULL, + FOREIGN KEY (notification_id) + REFERENCES cbox_notifications (id) + ON DELETE CASCADE +); + +COMMIT; + +CREATE INDEX `cbox_notifications_ix0` ON `cbox_notifications` (`ref`); + +CREATE INDEX `cbox_notification_recipients_ix0` ON `cbox_notification_recipients` (`notification_id`); +CREATE INDEX `cbox_notification_recipients_ix1` ON `cbox_notification_recipients` (`recipient`); + +COMMIT; + +INSERT INTO `cbox_notifications` (`id`, `ref`, `template_name`) VALUES (1, "notification-test", "notification-template-test"); +INSERT INTO `cbox_notification_recipients` (`id`, `notification_id`, `recipient`) VALUES (1, 1, "jdoe"), (2, 1, "testuser"); + +COMMIT; diff --git a/pkg/notification/handler/emailhandler/emailhandler.go b/pkg/notification/handler/emailhandler/emailhandler.go new file mode 100644 index 0000000000..47152f51c8 --- /dev/null +++ b/pkg/notification/handler/emailhandler/emailhandler.go @@ -0,0 +1,116 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package emailhandler + +import ( + "fmt" + "net/smtp" + "regexp" + "strings" + + "github.com/cs3org/reva/pkg/notification/handler" + "github.com/cs3org/reva/pkg/notification/handler/registry" + "github.com/mitchellh/mapstructure" + "github.com/rs/zerolog" +) + +func init() { + registry.Register("email", New) +} + +// EmailHandler is the notification handler for emails. +type EmailHandler struct { + conf *config + Log *zerolog.Logger +} + +type config struct { + SMTPAddress string `mapstructure:"smtp_server" docs:";The hostname and port of the SMTP server."` + SenderLogin string `mapstructure:"sender_login" docs:";The email to be used to send mails."` + SenderPassword string `mapstructure:"sender_password" docs:";The sender's password."` + DisableAuth bool `mapstructure:"disable_auth" docs:"false;Whether to disable SMTP auth."` + DefaultSender string `mapstructure:"default_sender" docs:"no-reply@cernbox.cern.ch;Default sender when not specified in the trigger."` +} + +func defaultConfig() *config { + return &config{ + DefaultSender: "no-reply@cernbox.cern.ch", + } +} + +// New returns a new email handler. +func New(log *zerolog.Logger, conf interface{}) (handler.Handler, error) { + c := defaultConfig() + if err := mapstructure.Decode(conf, c); err != nil { + return nil, err + } + + return &EmailHandler{ + conf: c, + Log: log, + }, nil +} + +// Send is the method run when a notification is triggered for this handler. +func (e *EmailHandler) Send(sender, recipient, subject, body string) error { + if sender == "" { + sender = e.conf.DefaultSender + } + + msg := e.generateMsg(sender, recipient, subject, body) + err := smtp.SendMail(e.conf.SMTPAddress, e.getAuth(), sender, []string{recipient}, msg) + if err != nil { + return err + } + + e.Log.Debug().Msgf("mail sent to recipient %s", recipient) + + return nil +} + +func (e *EmailHandler) getAuth() smtp.Auth { + if e.conf.DisableAuth { + return nil + } + + return smtp.PlainAuth("", e.conf.SenderLogin, e.conf.SenderPassword, strings.SplitN(e.conf.SMTPAddress, ":", 2)[0]) +} + +func (e *EmailHandler) generateMsg(from, to, subject, body string) []byte { + re := regexp.MustCompile(`\r?\n`) + cleanSubject := re.ReplaceAllString(strings.TrimSpace(subject), " ") + headers := []string{ + fmt.Sprintf("From: %s", from), + fmt.Sprintf("To: %s", to), + fmt.Sprintf("Subject: %s", cleanSubject), + "MIME-version: 1.0;", + "Content-Type: text/html; charset=\"UTF-8\";", + } + + var sb strings.Builder + + for _, h := range headers { + sb.WriteString(h) + sb.WriteString("\r\n") + } + sb.WriteString("\r\n") + sb.WriteString(body) + + return []byte(sb.String()) +} diff --git a/pkg/notification/handler/handler.go b/pkg/notification/handler/handler.go new file mode 100644 index 0000000000..a42e5cd047 --- /dev/null +++ b/pkg/notification/handler/handler.go @@ -0,0 +1,24 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package handler + +// Handler is the interface notification handlers have to implement. +type Handler interface { + Send(sender, recipient, subject, body string) error +} diff --git a/pkg/notification/handler/loader/loader.go b/pkg/notification/handler/loader/loader.go new file mode 100644 index 0000000000..3aa73e6077 --- /dev/null +++ b/pkg/notification/handler/loader/loader.go @@ -0,0 +1,25 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package loader + +import ( + // Load notification handlers. + _ "github.com/cs3org/reva/pkg/notification/handler/emailhandler" + // Add your own here. +) diff --git a/pkg/notification/handler/registry/registry.go b/pkg/notification/handler/registry/registry.go new file mode 100644 index 0000000000..e4a9271eab --- /dev/null +++ b/pkg/notification/handler/registry/registry.go @@ -0,0 +1,60 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package registry + +import ( + "github.com/cs3org/reva/pkg/notification/handler" + "github.com/rs/zerolog" +) + +// NewHandlerFunc is the function that notification handlers should register to +// at init time. +type NewHandlerFunc func(Log *zerolog.Logger, conf interface{}) (handler.Handler, error) + +// NewHandlerFuncs is a map containing all the registered notification handlers. +var NewHandlerFuncs = map[string]NewHandlerFunc{} + +// Register registers a new notification handler new function. Not safe for +// concurrent use. Safe for use from package init. +func Register(name string, f NewHandlerFunc) { + NewHandlerFuncs[name] = f +} + +// InitHandlers initializes the notification handlers with the configuration +// and the log from a service. +func InitHandlers(handlerConf map[string]interface{}, log *zerolog.Logger) map[string]handler.Handler { + handlers := make(map[string]handler.Handler) + hCount := 0 + + for n, f := range NewHandlerFuncs { + if c, ok := handlerConf[n]; ok { + nh, err := f(log, c) + if err != nil { + log.Err(err).Msgf("error initializing notification handler %s", n) + } + handlers[n] = nh + hCount++ + } else { + log.Warn().Msgf("missing config for notification handler %s", n) + } + } + log.Info().Msgf("%d handlers initialized", hCount) + + return handlers +} diff --git a/pkg/notification/manager/loader/loader.go b/pkg/notification/manager/loader/loader.go new file mode 100644 index 0000000000..af580197b0 --- /dev/null +++ b/pkg/notification/manager/loader/loader.go @@ -0,0 +1,25 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package loader + +import ( + // Load core notification manager drivers. + _ "github.com/cs3org/reva/pkg/notification/manager/sql" + // Add your own here. +) diff --git a/pkg/notification/manager/registry/registry.go b/pkg/notification/manager/registry/registry.go new file mode 100644 index 0000000000..4ae551145a --- /dev/null +++ b/pkg/notification/manager/registry/registry.go @@ -0,0 +1,36 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package registry + +import "github.com/cs3org/reva/pkg/notification" + +// import "github.com/cs3org/reva/pkg/share" + +// NewFunc is the function that notification managers +// should register at init time. +type NewFunc func(map[string]interface{}) (notification.Manager, error) + +// NewFuncs is a map containing all the registered notification managers. +var NewFuncs = map[string]NewFunc{} + +// Register registers a new notification manager new function. +// Not safe for concurrent use. Safe for use from package init. +func Register(name string, f NewFunc) { + NewFuncs[name] = f +} diff --git a/pkg/notification/manager/sql/sql.go b/pkg/notification/manager/sql/sql.go new file mode 100644 index 0000000000..1e71dad7ab --- /dev/null +++ b/pkg/notification/manager/sql/sql.go @@ -0,0 +1,199 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package sql + +import ( + "database/sql" + "fmt" + + "github.com/cs3org/reva/pkg/notification" + "github.com/cs3org/reva/pkg/notification/manager/registry" + "github.com/mitchellh/mapstructure" +) + +func init() { + registry.Register("sql", NewMysql) +} + +type config struct { + DBUsername string `mapstructure:"db_username"` + DBPassword string `mapstructure:"db_password"` + DBHost string `mapstructure:"db_host"` + DBPort int `mapstructure:"db_port"` + DBName string `mapstructure:"db_name"` + GatewaySvc string `mapstructure:"gatewaysvc"` +} + +type mgr struct { + driver string + db *sql.DB +} + +// NewMysql returns an instance of the sql notifications manager. +func NewMysql(m map[string]interface{}) (notification.Manager, error) { + c := &config{} + if err := mapstructure.Decode(m, c); err != nil { + return nil, err + } + + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", c.DBUsername, c.DBPassword, c.DBHost, c.DBPort, c.DBName)) + if err != nil { + return nil, err + } + + return New("mysql", db) +} + +// New returns a new Notifications driver connecting to the given sql.DB. +func New(driver string, db *sql.DB) (notification.Manager, error) { + return &mgr{ + driver: driver, + db: db, + }, nil +} + +// UpsertNotification creates or updates a notification. +func (m *mgr) UpsertNotification(n notification.Notification) error { + if err := n.CheckNotification(); err != nil { + return err + } + + tx, err := m.db.Begin() + if err != nil { + return err + } + + // Create/update notification + stmt, err := m.db.Prepare("REPLACE INTO cbox_notifications (ref, template_name) VALUES (?, ?)") + if err != nil { + return err + } + defer stmt.Close() + + result, err := stmt.Exec(n.Ref, n.TemplateName) + if err != nil { + _ = tx.Rollback() + return err + } + + // Create/update recipients for the notification + notificationID, err := result.LastInsertId() + if err != nil { + _ = tx.Rollback() + return err + } + + stmt, err = tx.Prepare("REPLACE INTO cbox_notification_recipients (notification_id, recipient) VALUES (?, ?)") + if err != nil { + _ = tx.Rollback() + return err + } + defer stmt.Close() + + for _, recipient := range n.Recipients { + _, err := stmt.Exec(notificationID, recipient) + if err != nil { + _ = tx.Rollback() + return err + } + } + + err = tx.Commit() + if err != nil { + return err + } + + return nil +} + +// GetNotification reads a notification. +func (m *mgr) GetNotification(ref string) (*notification.Notification, error) { + query := ` + SELECT n.id, n.ref, n.template_name, nr.recipient + FROM cbox_notifications AS n + JOIN cbox_notification_recipients AS nr ON n.id = nr.notification_id + WHERE n.ref = ? + ` + + rows, err := m.db.Query(query, ref) + if err != nil { + return nil, err + } + defer rows.Close() + + var n notification.Notification + count := 0 + n.Recipients = make([]string, 0) + + for rows.Next() { + var id string + var recipient string + err := rows.Scan(&id, &n.Ref, &n.TemplateName, &recipient) + if err != nil { + return nil, err + } + n.Recipients = append(n.Recipients, recipient) + count++ + } + if err = rows.Err(); err != nil { + return nil, err + } + if count == 0 { + return nil, ¬ification.NotFoundError{ + Ref: n.Ref, + } + } + + return &n, nil +} + +// DeleteNotification deletes a notification. +func (m *mgr) DeleteNotification(ref string) error { + tx, err := m.db.Begin() + if err != nil { + return err + } + + // Delete notification + stmt, err := m.db.Prepare("DELETE FROM cbox_notifications WHERE ref = ?") + if err != nil { + return err + } + defer stmt.Close() + + result, err := stmt.Exec(ref) + if err != nil { + _ = tx.Rollback() + return err + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + _ = tx.Rollback() + return err + } + + if rowsAffected == 0 { + return ¬ification.NotFoundError{ + Ref: ref, + } + } + + return nil +} diff --git a/internal/http/services/ocmd/notifications.go b/pkg/notification/manager/sql/sql_suite_test.go similarity index 70% rename from internal/http/services/ocmd/notifications.go rename to pkg/notification/manager/sql/sql_suite_test.go index 1e3f591298..e3c6fbd8e9 100644 --- a/internal/http/services/ocmd/notifications.go +++ b/pkg/notification/manager/sql/sql_suite_test.go @@ -16,19 +16,16 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package ocmd +package sql_test import ( - "net/http" -) - -type notificationsHandler struct { -} + "testing" -func (h *notificationsHandler) init(c *config) { -} + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) -// SendNotification is used to let the provider know that a user has removed a share. -func (h *notificationsHandler) SendNotification(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotImplemented) +func TestSql(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Sql Suite") } diff --git a/pkg/notification/manager/sql/sql_test.go b/pkg/notification/manager/sql/sql_test.go new file mode 100644 index 0000000000..60080ffab0 --- /dev/null +++ b/pkg/notification/manager/sql/sql_test.go @@ -0,0 +1,252 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package sql_test + +import ( + "database/sql" + "fmt" + "os" + + "github.com/cs3org/reva/pkg/notification" + sqlmanager "github.com/cs3org/reva/pkg/notification/manager/sql" + _ "github.com/mattn/go-sqlite3" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("SQL manager for notifications", func() { + var ( + db *sql.DB + testDBFile *os.File + mgr notification.Manager + n1 = ¬ification.Notification{ + Ref: "notification-test", + TemplateName: "notification-template-test", + Recipients: []string{"jdoe", "testuser"}, + } + n2 = ¬ification.Notification{ + Ref: "new-notification", + TemplateName: "new-template", + Recipients: []string{"newuser1", "newuser2"}, + } + nn *notification.Notification + ref string + err error + selectNotificationsSQL = "SELECT ref, template_name FROM cbox_notifications WHERE ref = ?" + selectNotificationRecipientsSQL = "SELECT COUNT(*) FROM cbox_notification_recipients WHERE notification_id = ?" + ) + + AfterEach(func() { + os.Remove(testDBFile.Name()) + }) + + BeforeEach(func() { + var err error + ref = "notification-test" + + testDBFile, err = os.CreateTemp("", "testdbfile") + Expect(err).ToNot(HaveOccurred()) + + dbData, err := os.ReadFile("test.sqlite") + Expect(err).ToNot(HaveOccurred()) + + _, err = testDBFile.Write(dbData) + Expect(err).ToNot(HaveOccurred()) + + err = testDBFile.Close() + Expect(err).ToNot(HaveOccurred()) + + db, err = sql.Open("sqlite3", fmt.Sprintf("%v?_foreign_keys=on", testDBFile.Name())) + Expect(err).ToNot(HaveOccurred()) + Expect(db).ToNot(BeNil()) + + mgr, err = sqlmanager.New("sqlite3", db) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + os.Remove(testDBFile.Name()) + }) + + Context("Creating notifications", func() { + When("creating a non-existing notification", func() { + JustBeforeEach(func() { + err = mgr.UpsertNotification(*n2) + }) + + It("should not return an error", func() { + Expect(err).ToNot(HaveOccurred()) + }) + + It("should create a notification entry", func() { + var newRef, newTemplateName string + err = db.QueryRow(selectNotificationsSQL, n2.Ref).Scan(&newRef, &newTemplateName) + Expect(newRef).To(Equal(n2.Ref)) + Expect(newTemplateName).To(Equal(n2.TemplateName)) + }) + + It("should create notification recipients entries", func() { + var notificationID int + err = db.QueryRow("SELECT id FROM cbox_notifications WHERE ref = ?", n2.Ref).Scan(¬ificationID) + Expect(err).ToNot(HaveOccurred()) + var newRecipientCount int + err = db.QueryRow(selectNotificationRecipientsSQL, notificationID).Scan(&newRecipientCount) + Expect(err).ToNot(HaveOccurred()) + Expect(newRecipientCount).To(Equal(len(n2.Recipients))) + }) + }) + + When("updating an existing notification", func() { + var m = ¬ification.Notification{ + Ref: "notification-test", + TemplateName: "new-notification-template-test", + Recipients: []string{"jdoe", "testuser2", "thirduser"}, + } + + JustBeforeEach(func() { + err = mgr.UpsertNotification(*m) + }) + + It("should not return an error", func() { + Expect(err).ToNot(HaveOccurred()) + }) + + It("should not increase the number of entries in the notification table", func() { + var count int + err = db.QueryRow("SELECT COUNT(*) FROM cbox_notifications").Scan(&count) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(1)) + }) + + It("should update the existing notification data", func() { + var newRef, newTemplateName string + err = db.QueryRow(selectNotificationsSQL, m.Ref).Scan(&newRef, &newTemplateName) + Expect(newRef).To(Equal(m.Ref)) + Expect(newTemplateName).To(Equal(m.TemplateName)) + }) + + It("should delete old entries in notification recipients", func() { + var count int + err = db.QueryRow("SELECT COUNT(*) FROM cbox_notification_recipients WHERE recipient = 'testuser'").Scan(&count) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(BeZero()) + }) + + It("should create new entries in notification recipients", func() { + var notificationID int + err = db.QueryRow("SELECT id FROM cbox_notifications WHERE ref = ?", m.Ref).Scan(¬ificationID) + Expect(err).ToNot(HaveOccurred()) + var newRecipientCount int + err = db.QueryRow(selectNotificationRecipientsSQL, notificationID).Scan(&newRecipientCount) + Expect(err).ToNot(HaveOccurred()) + Expect(newRecipientCount).To(Equal(len(m.Recipients))) + }) + }) + + When("creating an invalid notification", func() { + o := ¬ification.Notification{} + + JustBeforeEach(func() { + err = mgr.UpsertNotification(*o) + }) + + It("should return an InvalidNotificationError", func() { + _, isInvalidNotificationError := err.(*notification.InvalidNotificationError) + Expect(err).To(HaveOccurred()) + Expect(isInvalidNotificationError).To(BeTrue()) + }) + }) + }) + + Context("Getting notifications", func() { + When("getting an existing notification", func() { + JustBeforeEach(func() { + nn, err = mgr.GetNotification(ref) + }) + + It("should not return an error", func() { + Expect(err).ToNot(HaveOccurred()) + }) + + It("should return a notification", func() { + Expect(nn.Ref).To(Equal(n1.Ref)) + Expect(nn.TemplateName).To(Equal(n1.TemplateName)) + Expect(nn.Recipients).To(Equal(n1.Recipients)) + }) + }) + + When("getting a non-existing notification", func() { + JustBeforeEach(func() { + nn, err = mgr.GetNotification("non-existent-ref") + }) + + It("should return a NotFoundError", func() { + _, isNotFoundError := err.(*notification.NotFoundError) + Expect(err).To(HaveOccurred()) + Expect(isNotFoundError).To(BeTrue()) + }) + }) + }) + + Context("Deleting notifications", func() { + When("deleting an existing notification", func() { + JustBeforeEach(func() { + err = mgr.DeleteNotification(ref) + + }) + + It("should not return an error", func() { + Expect(err).ToNot(HaveOccurred()) + }) + + It("should delete the notification from the database", func() { + var count int + err = db.QueryRow("SELECT COUNT(*) FROM cbox_notifications WHERE ref = ?", ref).Scan(&count) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(BeZero()) + }) + + It("should cascade the deletions to notification_recipients table", func() { + var count int + err = db.QueryRow("SELECT COUNT(*) FROM cbox_notification_recipients WHERE notification_id = ?", 1).Scan(&count) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(BeZero()) + }) + }) + + When("deleting a non-existing notification", func() { + JustBeforeEach(func() { + err = mgr.DeleteNotification("non-existent-ref") + + }) + + It("should not change the db and return a NotFoundError error", func() { + Expect(err).To(HaveOccurred()) + isNotFoundError, _ := err.(*notification.NotFoundError) + Expect(isNotFoundError).ToNot(BeNil()) + var count int + err = db.QueryRow("SELECT COUNT(*) FROM cbox_notifications").Scan(&count) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(1)) + }) + }) + + }) + +}) diff --git a/pkg/notification/manager/sql/test.sqlite b/pkg/notification/manager/sql/test.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..7bf9289cdda218362bae3993b3e552879705b4f0 GIT binary patch literal 32768 zcmeI*Pfyxl90%~HEmDeqx@0q)Or9f+PC*Z5mt`G3IwBUWFlTn42qozvptL4tmj##X zh3pk9*^Ah5*S!R{-8T}f?rs_0CXsrx>m*wW|e^Lsvh+TmJ!pL_V=}$KXx3O@FQ_#rtG_9W=8eI*ESj{s)mvJ7I(8iD##KU14XR)Mqfk>< z3-t_}Q?;~%$5A=KnDF+ESKivT2L&`}-B2gkg8nqeE)|O_7O$J{gaTV)CS5hNE4sN! zm)R!Oi-mmF;9FZ^Mq!F)?2oux%E%X}QOxE1Os9K*KG)66Gu=$i&89TklV>q+vh0!( zRFdr5s1{cglPxln85y=tLk)Fl(y98HcyCx>ImSCsMqkh93oNd+%Mm$|P74S1^8wxP z*2GJ&T;8eTw3DtX)y?0sYu2ekYJOIuMI*abWIY<)%${A}*)Dr_sZp-mJ;i$Q;@YYd zO-xP-n?VcR-EBT3ExGpYp53U}eT$JZZR=Z^^lO>oB{lVklmfMve?{%JEGHhNg?8fn zqtHK1gpWNp+KK(5lS5GKg5%8JXS^HMEG;t>O>CqE5>(v*DDQdAV7V0T4W;l2%a!O!x~40uX=z1Rwwb2tWV=5P$##AOL~eAuuCE$c)~5_35E!yI$W1$CqS+ zHF4f9LH$nn3kd=cfB*y_009U<00Izz00bZaf!ir!L!$mr4{t~V z5e5Vx009U<00Izz00bZa0SG_<0@oB65hFyI@Sg?PbM01)KPI4zzp6HE-2Z<~P!xjz z1Rwwb2tWV=5P$##AOHaf+)ROU@BjD5|DyVnsK59d5(FRs0SG_<0uX=z1Rwwb2tWV= zcS#^3DS{|TigG;u=fD4V6#N2UlsD3!{QnOi-lYbkEf9bJ1Rwwb2tWV=5P$##AOL~? UL|{ak5KbNd5M^mx@$dit4QC@5C;$Ke literal 0 HcmV?d00001 diff --git a/pkg/notification/notification.go b/pkg/notification/notification.go new file mode 100644 index 0000000000..fdf615f369 --- /dev/null +++ b/pkg/notification/notification.go @@ -0,0 +1,114 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package notification + +import ( + "fmt" + + "github.com/cs3org/reva/pkg/notification/template" +) + +// Notification is the representation of a notification. +type Notification struct { + TemplateName string + Template template.Template + Ref string + Recipients []string +} + +// Manager is the interface notification storage managers have to implement. +type Manager interface { + // UpsertNotification insert or updates a notification. + UpsertNotification(n Notification) error + // GetNotification reads a notification. + GetNotification(ref string) (*Notification, error) + // DeleteNotification deletes a notifcation. + DeleteNotification(ref string) error +} + +// NotFoundError is the error returned when a notification does not exist. +type NotFoundError struct { + Ref string +} + +// InvalidNotificationError is the error returned when a notification has invalid data. +type InvalidNotificationError struct { + Ref string + Msg string + Err error +} + +// Error returns the string error msg for NotFoundError. +func (n *NotFoundError) Error() string { + return fmt.Sprintf("notification %s not found", n.Ref) +} + +// Error returns the string error msg for InvalidNotificationError. +func (i *InvalidNotificationError) Error() string { + return i.Msg +} + +// Send is the method run when a notification is triggered. +func (n *Notification) Send(sender string, templateData map[string]interface{}) error { + subject, err := n.Template.RenderSubject(templateData) + if err != nil { + return err + } + + body, err := n.Template.RenderBody(templateData) + if err != nil { + return err + } + + for _, recipient := range n.Recipients { + err := n.Template.Handler.Send(sender, recipient, subject, body) + if err != nil { + return err + } + } + + return nil +} + +// CheckNotification checks if a notification has correct data. +func (n *Notification) CheckNotification() error { + if len(n.Ref) == 0 { + return &InvalidNotificationError{ + Ref: n.Ref, + Msg: "empty ref", + } + } + + if err := template.CheckTemplateName(n.TemplateName); err != nil { + return &InvalidNotificationError{ + Ref: n.Ref, + Msg: fmt.Sprintf("invalid template name %s", n.TemplateName), + Err: err, + } + } + + if len(n.Recipients) == 0 { + return &InvalidNotificationError{ + Ref: n.Ref, + Msg: "empty recipient list", + } + } + + return nil +} diff --git a/pkg/notification/notificationhelper/notificationhelper.go b/pkg/notification/notificationhelper/notificationhelper.go new file mode 100644 index 0000000000..af92e2f4d1 --- /dev/null +++ b/pkg/notification/notificationhelper/notificationhelper.go @@ -0,0 +1,250 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package notificationhelper + +import ( + "encoding/json" + "fmt" + + "github.com/cs3org/reva/pkg/notification" + "github.com/cs3org/reva/pkg/notification/template" + "github.com/cs3org/reva/pkg/notification/trigger" + "github.com/cs3org/reva/pkg/notification/utils" + "github.com/mitchellh/mapstructure" + "github.com/nats-io/nats.go" + "github.com/pkg/errors" + "github.com/rs/zerolog" +) + +// NotificationHelper is the type used in services to work with notifications. +type NotificationHelper struct { + Name string + Conf *Config + Log *zerolog.Logger + nc *nats.Conn + js nats.JetStreamContext + kv nats.KeyValue +} + +// Config contains the configuration for the Notification Helper. +type Config struct { + NatsAddress string `mapstructure:"nats_address" docs:";The NATS server address."` + NatsToken string `mapstructure:"nats_token" docs:";The token to authenticate against the NATS server"` + NatsStream string `mapstructure:"nats_stream" docs:"reva-notifications;The notifications NATS stream."` + Templates map[string]interface{} `mapstructure:"templates" docs:";Notification templates for the service."` +} + +func defaultConfig() *Config { + return &Config{ + NatsStream: "reva-notifications", + } +} + +// New creates a new Notification Helper. +func New(name string, m map[string]interface{}, log *zerolog.Logger) *NotificationHelper { + conf := defaultConfig() + nh := &NotificationHelper{ + Name: name, + Conf: conf, + Log: log, + } + + if err := mapstructure.Decode(m, conf); err != nil { + log.Error().Err(err).Msgf("decoding config failed, notifications will be disabled") + return nh + } + + annotatedLogger := log.With().Str("service", nh.Name).Str("scope", "notifications").Logger() + nh.Log = &annotatedLogger + + if err := nh.connect(); err != nil { + log.Error().Err(err).Msgf("connecting to nats failed, notifications will be disabled") + return nh + } + + nh.registerTemplates(nh.Conf.Templates) + + return nh +} + +func (nh *NotificationHelper) connect() error { + nc, err := utils.ConnectToNats(nh.Conf.NatsAddress, nh.Conf.NatsToken, *nh.Log) + if err != nil { + return err + } + nh.nc = nc + + js, err := nh.nc.JetStream(nats.PublishAsyncMaxPending(256)) + if err != nil { + return errors.Wrap(err, "jetstream initialization failed") + } + stream, _ := js.StreamInfo(nh.Conf.NatsStream) + if stream != nil { + if _, err := js.AddStream(&nats.StreamConfig{ + Name: nh.Conf.NatsStream, + Subjects: []string{ + fmt.Sprintf("%s.notification", nh.Conf.NatsStream), + fmt.Sprintf("%s.trigger", nh.Conf.NatsStream), + }, + }); err != nil { + return errors.Wrap(err, "nats stream creation failed") + } + } + nh.js = js + + bucketName := fmt.Sprintf("%s-template", nh.Conf.NatsStream) + kv, err := nh.js.CreateKeyValue(&nats.KeyValueConfig{ + Bucket: bucketName, + }) + if err != nil { + return errors.Wrap(err, "template store creation failed, probably because nats server is unreachable") + } + nh.kv = kv + return nil +} + +// Stop stops the notification helper. +func (nh *NotificationHelper) Stop() { + if err := nh.nc.Drain(); err != nil { + nh.Log.Error().Err(err) + } +} + +func (nh *NotificationHelper) registerTemplates(ts map[string]interface{}) { + if len(ts) == 0 { + nh.Log.Info().Msg("no templates to register") + return + } + + tCount := 0 + for tn, tm := range ts { + var tc template.RegistrationRequest + if err := mapstructure.Decode(tm, &tc); err != nil { + nh.Log.Error().Err(err).Msgf("template '%s' definition decoding failed", tn) + continue + } + if err := template.CheckTemplateName(tc.Name); err != nil { + nh.Log.Error().Err(err).Msgf("template name '%s' is incorrect", tc.Name) + continue + } + if tc.Handler == "" { + nh.Log.Error().Msgf("template definition '%s' is missing handler field", tn) + continue + } + if tc.BodyTmplPath == "" { + nh.Log.Error().Msgf("template definition '%s' is missing body_template_path field", tn) + continue + } + + nh.registerTemplate(&tc) + tCount++ + } + + nh.Log.Info().Msgf("%d templates to register", tCount) +} + +func (nh *NotificationHelper) registerTemplate(rr *template.RegistrationRequest) { + if nh.kv == nil { + nh.Log.Info().Msgf("template registration skipped, helper is misconfigured") + return + } + + tb, err := json.Marshal(rr) + if err != nil { + nh.Log.Error().Err(err).Msgf("template registration json marshalling failed") + } + + go func() { + _, err := nh.kv.Put(rr.Name, tb) + if err != nil { + nh.Log.Error().Err(err).Msgf("template registration publish failed") + return + } + nh.Log.Debug().Msgf("%s template registration published", rr.Name) + }() +} + +// RegisterNotification registers a notification in the notification service. +func (nh *NotificationHelper) RegisterNotification(n *notification.Notification) { + if nh.js == nil { + nh.Log.Info().Msgf("notification registration skipped, helper is misconfigured") + return + } + + nb, err := json.Marshal(n) + if err != nil { + nh.Log.Error().Err(err).Msgf("notification registration json marshalling failed") + return + } + + notificationSubject := fmt.Sprintf("%s.notification-register", nh.Conf.NatsStream) + + go func() { + _, err := nh.js.Publish(notificationSubject, nb) + if err != nil { + nh.Log.Error().Err(err).Msgf("notification registration publish failed") + return + } + nh.Log.Debug().Msgf("%s notification registration published", n.Ref) + }() +} + +// UnregisterNotification unregisters a notification in the notification service. +func (nh *NotificationHelper) UnregisterNotification(ref string) { + if nh.js == nil { + nh.Log.Info().Msgf("notification unregistration skipped, notification helper is misconfigured") + return + } + + notificationSubject := fmt.Sprintf("%s.notification-unregister", nh.Conf.NatsStream) + + go func() { + _, err := nh.js.Publish(notificationSubject, []byte(ref)) + if err != nil { + nh.Log.Error().Err(err).Msgf("notification unregistration publish failed") + return + } + nh.Log.Debug().Msgf("%s notification unregistration published", ref) + }() +} + +// TriggerNotification sends a notification trigger to the notifications service. +func (nh *NotificationHelper) TriggerNotification(tr *trigger.Trigger) { + if nh.js == nil { + nh.Log.Info().Msgf("notification trigger skipped, notification helper is misconfigured") + return + } + + trb, err := json.Marshal(tr) + if err != nil { + nh.Log.Error().Err(err).Msgf("notification trigger json marshalling failed") + return + } + + triggerSubject := fmt.Sprintf("%s.trigger", nh.Conf.NatsStream) + + go func() { + _, err := nh.js.Publish(triggerSubject, trb) + if err != nil { + nh.Log.Error().Err(err).Msgf("notification trigger publish failed") + return + } + nh.Log.Debug().Msgf("%s notification trigger published", tr.Ref) + }() +} diff --git a/pkg/notification/template/registry/registry.go b/pkg/notification/template/registry/registry.go new file mode 100644 index 0000000000..bb13fe6a8f --- /dev/null +++ b/pkg/notification/template/registry/registry.go @@ -0,0 +1,69 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package registry + +import ( + "encoding/json" + "fmt" + + "github.com/cs3org/reva/pkg/notification/handler" + "github.com/cs3org/reva/pkg/notification/template" + "github.com/pkg/errors" +) + +// Registry provides with means for dynamically registering notification templates. +type Registry struct { + store map[string]template.Template +} + +// New returns a new Template Registry. +func New() *Registry { + r := &Registry{ + store: make(map[string]template.Template), + } + + return r +} + +// Put registers a handler in the registry. +func (r *Registry) Put(tb []byte, hs map[string]handler.Handler) (string, error) { + var data map[string]interface{} + + err := json.Unmarshal(tb, &data) + if err != nil { + return "", errors.Wrapf(err, "template registration unmarshall failed") + } + + t, name, err := template.New(data, hs) + if err != nil { + return name, errors.Wrapf(err, "template %s registration failed", name) + } + + r.store[t.Name] = *t + return t.Name, nil +} + +// Get retrieves a handler from the registry. +func (r *Registry) Get(n string) (*template.Template, error) { + if t, ok := r.store[n]; ok { + return &t, nil + } + + return nil, fmt.Errorf("template %s not found", n) +} diff --git a/pkg/notification/template/template.go b/pkg/notification/template/template.go new file mode 100644 index 0000000000..39d5fc0c18 --- /dev/null +++ b/pkg/notification/template/template.go @@ -0,0 +1,170 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package template + +import ( + "bytes" + "errors" + "fmt" + htmlTemplate "html/template" + "io" + "os" + "path/filepath" + "regexp" + textTemplate "text/template" + + "github.com/cs3org/reva/pkg/notification/handler" + "github.com/mitchellh/mapstructure" +) + +const validTemplateNameRegex = "[a-zA-Z0-9-]" + +// RegistrationRequest represents a Template registration request. +type RegistrationRequest struct { + Name string `mapstructure:"name" json:"name"` + Handler string `mapstructure:"handler" json:"handler"` + BodyTmplPath string `mapstructure:"body_template_path" json:"body_template_path"` + SubjectTmplPath string `mapstructure:"subject_template_path" json:"subject_template_path"` + Persistent bool `mapstructure:"persistent" json:"persistent"` +} + +// Template represents a notification template. +type Template struct { + Name string + Handler handler.Handler + Persistent bool + tmplSubject *textTemplate.Template + tmplBody *htmlTemplate.Template +} + +// FileNotFoundError is the error returned when a template file is missing. +type FileNotFoundError struct { + TemplateFileName string + Err error +} + +// Error returns the string error msg for FileNotFoundError. +func (t *FileNotFoundError) Error() string { + return fmt.Sprintf("template file %s not found", t.TemplateFileName) +} + +// New creates a new Template from a RegistrationRequest. +func New(m map[string]interface{}, hs map[string]handler.Handler) (*Template, string, error) { + rr := &RegistrationRequest{} + if err := mapstructure.Decode(m, rr); err != nil { + return nil, rr.Name, err + } + + h, ok := hs[rr.Handler] + if !ok { + return nil, rr.Name, fmt.Errorf("unknown handler %s", rr.Handler) + } + + tmplSubject, err := parseTmplFile(rr.SubjectTmplPath, "subject") + if err != nil { + return nil, rr.Name, err + } + + tmplBody, err := parseTmplFile(rr.BodyTmplPath, "body") + if err != nil { + return nil, rr.Name, err + } + + t := &Template{ + Name: rr.Name, + Handler: h, + tmplSubject: tmplSubject.(*textTemplate.Template), + tmplBody: tmplBody.(*htmlTemplate.Template), + } + + if err := CheckTemplateName(t.Name); err != nil { + return nil, rr.Name, err + } + + return t, rr.Name, nil +} + +// RenderSubject renders the subject template. +func (t *Template) RenderSubject(arguments map[string]interface{}) (string, error) { + var buf bytes.Buffer + err := t.tmplSubject.Execute(&buf, arguments) + return buf.String(), err +} + +// RenderBody renders the body template. +func (t *Template) RenderBody(arguments map[string]interface{}) (string, error) { + var buf bytes.Buffer + err := t.tmplBody.Execute(&buf, arguments) + return buf.String(), err +} + +// CheckTemplateName validates the name of the template. +func CheckTemplateName(name string) error { + if name == "" { + return errors.New("template name cannot be empty") + } + + re := regexp.MustCompile(validTemplateNameRegex) + invalidChars := re.ReplaceAllString(name, "") + if len(invalidChars) > 0 { + return fmt.Errorf("template name %s must contain only %s", name, validTemplateNameRegex) + } + + return nil +} + +func parseTmplFile(path, name string) (interface{}, error) { + if path == "" { + return textTemplate.New(name).Parse("") + } + + ext := filepath.Ext(path) + f, err := os.Open(path) + if err != nil { + return nil, &FileNotFoundError{ + TemplateFileName: path, + Err: err, + } + } + defer f.Close() + + data, err := io.ReadAll(f) + if err != nil { + return nil, err + } + + switch ext { + case ".txt": + tmpl, err := textTemplate.New(name).Parse(string(data)) + if err != nil { + return nil, err + } + + return tmpl, nil + case ".html": + tmpl, err := htmlTemplate.New(name).Parse(string(data)) + if err != nil { + return nil, err + } + + return tmpl, nil + default: + return nil, errors.New("unknown template type") + } +} diff --git a/pkg/notification/trigger/trigger.go b/pkg/notification/trigger/trigger.go new file mode 100644 index 0000000000..0fc60885c1 --- /dev/null +++ b/pkg/notification/trigger/trigger.go @@ -0,0 +1,41 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package trigger + +import ( + "github.com/cs3org/reva/pkg/notification" +) + +// Trigger represents a notification Trigger. +type Trigger struct { + Notification *notification.Notification + Ref string + Sender string + TemplateData map[string]interface{} +} + +// Send is the method run when a notification is triggered. +func (t *Trigger) Send() error { + err := t.Notification.Send(t.Sender, t.TemplateData) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/notification/utils/nats.go b/pkg/notification/utils/nats.go new file mode 100644 index 0000000000..cf206f46e8 --- /dev/null +++ b/pkg/notification/utils/nats.go @@ -0,0 +1,63 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +// Package utils contains utilities related to the notifications service and helper. +package utils + +import ( + "time" + + "github.com/nats-io/nats.go" + "github.com/pkg/errors" + "github.com/rs/zerolog" +) + +// ConnectToNats returns a resilient connection to the specified NATS server. +func ConnectToNats(natsAddress, natsToken string, log zerolog.Logger) (*nats.Conn, error) { + nc, err := nats.Connect( + natsAddress, + nats.DrainTimeout(9*time.Second), // reva timeout on graceful shutdown is 10 seconds + nats.MaxReconnects(-1), + nats.Token(natsToken), + nats.ErrorHandler(func(c *nats.Conn, s *nats.Subscription, err error) { + log.Error().Err(err).Msgf("nats error") + }), + nats.ClosedHandler(func(c *nats.Conn) { + log.Error().Err(c.LastError()).Msgf("connection to nats server closed") + }), + nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { + log.Error().Err(err).Msgf("connection to nats server disconnected") + }), + nats.CustomReconnectDelay(func(attempts int) time.Duration { + if attempts%3 == 0 { + log.Info().Msg("connection to nats server failed 3 times, backing off") + return 5 * time.Minute + } + + return 2 * time.Second + }), + nats.ReconnectHandler(func(_ *nats.Conn) { + log.Info().Msgf("connection to nats server reconnected") + }), + ) + if err != nil { + return nil, errors.Wrapf(err, "connection to nats server at '%s' failed", natsAddress) + } + + return nc, nil +} diff --git a/pkg/publicshare/manager/json/json.go b/pkg/publicshare/manager/json/json.go index f8c9f009c2..11419b474f 100644 --- a/pkg/publicshare/manager/json/json.go +++ b/pkg/publicshare/manager/json/json.go @@ -138,7 +138,7 @@ func (m *manager) startJanitorRun() { } // CreatePublicShare adds a new entry to manager.shares. -func (m *manager) CreatePublicShare(ctx context.Context, u *user.User, rInfo *provider.ResourceInfo, g *link.Grant, description string, internal bool) (*link.PublicShare, error) { +func (m *manager) CreatePublicShare(ctx context.Context, u *user.User, rInfo *provider.ResourceInfo, g *link.Grant, description string, internal bool, notifyUploads bool, notifyUploadsExtraRecipients string) (*link.PublicShare, error) { id := &link.PublicShareId{ OpaqueId: utils.RandString(15), } @@ -168,18 +168,20 @@ func (m *manager) CreatePublicShare(ctx context.Context, u *user.User, rInfo *pr } s := link.PublicShare{ - Id: id, - Owner: rInfo.GetOwner(), - Creator: u.Id, - ResourceId: rInfo.Id, - Token: tkn, - Permissions: g.Permissions, - Ctime: createdAt, - Mtime: createdAt, - PasswordProtected: passwordProtected, - Expiration: g.Expiration, - DisplayName: displayName, - Description: description, + Id: id, + Owner: rInfo.GetOwner(), + Creator: u.Id, + ResourceId: rInfo.Id, + Token: tkn, + Permissions: g.Permissions, + Ctime: createdAt, + Mtime: createdAt, + PasswordProtected: passwordProtected, + Expiration: g.Expiration, + DisplayName: displayName, + Description: description, + NotifyUploads: notifyUploads, + NotifyUploadsExtraRecipients: notifyUploadsExtraRecipients, } ps := &publicShare{ diff --git a/pkg/publicshare/manager/memory/memory.go b/pkg/publicshare/manager/memory/memory.go index 557992974a..8af0b16bcd 100644 --- a/pkg/publicshare/manager/memory/memory.go +++ b/pkg/publicshare/manager/memory/memory.go @@ -58,7 +58,7 @@ var ( ) // CreatePublicShare adds a new entry to manager.shares. -func (m *manager) CreatePublicShare(ctx context.Context, u *user.User, rInfo *provider.ResourceInfo, g *link.Grant, description string, internal bool) (*link.PublicShare, error) { +func (m *manager) CreatePublicShare(ctx context.Context, u *user.User, rInfo *provider.ResourceInfo, g *link.Grant, description string, internal bool, notifyUploads bool, notifyUploadsExtraRecipients string) (*link.PublicShare, error) { id := &link.PublicShareId{ OpaqueId: randString(15), } @@ -86,18 +86,20 @@ func (m *manager) CreatePublicShare(ctx context.Context, u *user.User, rInfo *pr } s := link.PublicShare{ - Id: id, - Owner: rInfo.GetOwner(), - Creator: u.Id, - ResourceId: rInfo.Id, - Token: tkn, - Permissions: g.Permissions, - Ctime: createdAt, - Mtime: modifiedAt, - PasswordProtected: passwordProtected, - Expiration: g.Expiration, - DisplayName: displayName, - Description: description, + Id: id, + Owner: rInfo.GetOwner(), + Creator: u.Id, + ResourceId: rInfo.Id, + Token: tkn, + Permissions: g.Permissions, + Ctime: createdAt, + Mtime: modifiedAt, + PasswordProtected: passwordProtected, + Expiration: g.Expiration, + DisplayName: displayName, + Description: description, + NotifyUploads: notifyUploads, + NotifyUploadsExtraRecipients: notifyUploadsExtraRecipients, } m.shares.Store(s.Token, &s) diff --git a/pkg/publicshare/publicshare.go b/pkg/publicshare/publicshare.go index 39d81cfbb9..9add4e398f 100644 --- a/pkg/publicshare/publicshare.go +++ b/pkg/publicshare/publicshare.go @@ -35,7 +35,7 @@ import ( // Manager manipulates public shares. type Manager interface { - CreatePublicShare(ctx context.Context, u *user.User, md *provider.ResourceInfo, g *link.Grant, description string, internal bool) (*link.PublicShare, error) + CreatePublicShare(ctx context.Context, u *user.User, md *provider.ResourceInfo, g *link.Grant, description string, internal bool, notifyUploads bool, notifyUploadsExtraRecipients string) (*link.PublicShare, error) UpdatePublicShare(ctx context.Context, u *user.User, req *link.UpdatePublicShareRequest, g *link.Grant) (*link.PublicShare, error) GetPublicShare(ctx context.Context, u *user.User, ref *link.PublicShareReference, sign bool) (*link.PublicShare, error) ListPublicShares(ctx context.Context, u *user.User, filters []*link.ListPublicSharesRequest_Filter, md *provider.ResourceInfo, sign bool) ([]*link.PublicShare, error) diff --git a/pkg/utils/accumulator/accumulator.go b/pkg/utils/accumulator/accumulator.go new file mode 100644 index 0000000000..710812f172 --- /dev/null +++ b/pkg/utils/accumulator/accumulator.go @@ -0,0 +1,126 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package accumulator + +import ( + "errors" + "time" + + "github.com/rs/zerolog" +) + +// Accumulator gathers items arriving spaced in time and groups them. +type Accumulator[T any] struct { + started bool + timeout time.Duration + timeoutChan chan bool + timeoutResetChan chan bool + maxSize int + Input chan T + pool []T + log *zerolog.Logger +} + +// New creates a new accumulator. An Accumulator gathers items arriving spaced +// in time and groups them. +// +// The main parameters are timeout and maxSize, determining the limits for the +// accumulator. +// +// An accumulator is started with the start method, which takes fn, a func([]T) +// argument that will be run every time the limit parameters are reached. After +// running fn, the accumulator pool is emptied. +// +// Items are put into the accumulator using the <-input channel, making it +// thread-safe. +func New[T any](timeout time.Duration, maxSize int, log *zerolog.Logger) *Accumulator[T] { + if timeout == 0 { + timeout = time.Duration(60) * time.Second + log.Warn().Msgf("timeout must be a positive duration greater than zero, using default (%d)", timeout) + } + + if maxSize == 0 { + maxSize = 100 + log.Warn().Msgf("maxSize must be a positive integer greater than zero, using default (%d)", maxSize) + } + + input := make(chan T) + accumulator := &Accumulator[T]{ + timeout: timeout, + timeoutResetChan: make(chan bool, 1), + maxSize: maxSize, + Input: input, + log: log, + } + + return accumulator +} + +func (a *Accumulator[T]) startTimeout() { + if !a.started { + a.started = true + a.timeoutChan = make(chan bool) + go func() { + select { + case <-a.timeoutResetChan: + a.timeoutChan = nil + case <-time.After(a.timeout): + a.timeoutChan <- true + a.timeoutChan = nil + } + a.started = false + }() + } +} + +// Start starts the accumulator. +// +// This does not mean the timer will start running. That happens once the first +// item arrives through the <-input channel. Once the time reaches the timeout +// or the max size of the accumulator is reached, fn will be run with the slice +// of items currently in the accumulator. +func (a *Accumulator[T]) Start(fn func([]T)) error { + if fn == nil { + return errors.New("fn must be a callback function") + } + + go func() { + for { + select { + case i := <-a.Input: + a.startTimeout() + a.pool = append(a.pool, i) + + if len(a.pool) >= a.maxSize { + fn(a.pool) + a.pool = nil + a.timeoutResetChan <- true + a.timeoutChan = nil + } + case <-a.timeoutChan: + if len(a.pool) > 0 { + fn(a.pool) + a.pool = nil + } + } + } + }() + + return nil +} From a6ec6b67aa1b64778128082bbd68b2fe30dba2d3 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 7 Jun 2023 08:11:16 +0200 Subject: [PATCH 08/48] Removed support for forcelock, cf. cs3org/wopiserver#121 (#3908) --- changelog/unreleased/forcelock-cleanup.md | 5 +++++ pkg/app/provider/wopi/wopi.go | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 changelog/unreleased/forcelock-cleanup.md diff --git a/changelog/unreleased/forcelock-cleanup.md b/changelog/unreleased/forcelock-cleanup.md new file mode 100644 index 0000000000..e89b860ef0 --- /dev/null +++ b/changelog/unreleased/forcelock-cleanup.md @@ -0,0 +1,5 @@ +Enhancement: Removed support for forcelock + +This workaround is not needed any longer, see also the wopiserver. + +https://github.com/cs3org/reva/pull/3908 diff --git a/pkg/app/provider/wopi/wopi.go b/pkg/app/provider/wopi/wopi.go index 64e8b7bdea..640e0319f2 100644 --- a/pkg/app/provider/wopi/wopi.go +++ b/pkg/app/provider/wopi/wopi.go @@ -252,11 +252,6 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc q.Add("appinturl", p.conf.AppIntURL) } - if _, ok := opaqueMap["forcelock"]; ok { - // this is to work around an issue with Microsoft Office, cf. cs3org/wopiserver#106 - q.Add("forcelock", "1") - } - httpReq.URL.RawQuery = q.Encode() if p.conf.AppAPIKey != "" { From 7ede02c9674d8ed8f3bfbf7044314aec4c0ff9cf Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Wed, 7 Jun 2023 10:11:44 +0200 Subject: [PATCH 09/48] Filter OCM shares by path (#3946) * filter ocm shares by path * add changelog --- .../fix-ocm-share-filter-by-path.md | 5 +++ .../handlers/apps/sharing/shares/remote.go | 6 ++-- .../ocs/handlers/apps/sharing/shares/user.go | 33 ++++++++++++++++++- 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 changelog/unreleased/fix-ocm-share-filter-by-path.md diff --git a/changelog/unreleased/fix-ocm-share-filter-by-path.md b/changelog/unreleased/fix-ocm-share-filter-by-path.md new file mode 100644 index 0000000000..59760787f2 --- /dev/null +++ b/changelog/unreleased/fix-ocm-share-filter-by-path.md @@ -0,0 +1,5 @@ +Bugfix: Filter OCM shares by path + +Fixes the bug of duplicated OCM shares returned in the share with others response. + +https://github.com/cs3org/reva/pull/3946 \ No newline at end of file diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/remote.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/remote.go index ab9c6d5c95..5d76f01288 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/remote.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/remote.go @@ -244,8 +244,10 @@ func (h *Handler) mustGetRemoteUser(ctx context.Context, gw gatewayv1beta1.Gatew } } -func (h *Handler) listOutcomingFederatedShares(ctx context.Context, gw gatewayv1beta1.GatewayAPIClient) ([]*conversions.ShareData, error) { - listRes, err := gw.ListOCMShares(ctx, &ocm.ListOCMSharesRequest{}) +func (h *Handler) listOutcomingFederatedShares(ctx context.Context, gw gatewayv1beta1.GatewayAPIClient, filters []*ocm.ListOCMSharesRequest_Filter) ([]*conversions.ShareData, error) { + listRes, err := gw.ListOCMShares(ctx, &ocm.ListOCMSharesRequest{ + Filters: filters, + }) if err != nil { return nil, err } diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go index ba7acc601f..e98b4c0469 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go @@ -25,6 +25,7 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" + ocm "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" @@ -226,7 +227,7 @@ func (h *Handler) listUserShares(r *http.Request, filters []*collaboration.Filte if h.listOCMShares { // include the ocm shares - ocmShares, err := h.listOutcomingFederatedShares(ctx, client) + ocmShares, err := h.listOutcomingFederatedShares(ctx, client, convertToOCMFilters(filters)) if err != nil { return nil, nil, err } @@ -236,3 +237,33 @@ func (h *Handler) listUserShares(r *http.Request, filters []*collaboration.Filte return ocsDataPayload, nil, nil } + +func convertToOCMFilters(filters []*collaboration.Filter) []*ocm.ListOCMSharesRequest_Filter { + ocmfilters := []*ocm.ListOCMSharesRequest_Filter{} + for _, f := range filters { + switch v := f.Term.(type) { + case *collaboration.Filter_ResourceId: + ocmfilters = append(ocmfilters, &ocm.ListOCMSharesRequest_Filter{ + Type: ocm.ListOCMSharesRequest_Filter_TYPE_RESOURCE_ID, + Term: &ocm.ListOCMSharesRequest_Filter_ResourceId{ + ResourceId: v.ResourceId, + }, + }) + case *collaboration.Filter_Creator: + ocmfilters = append(ocmfilters, &ocm.ListOCMSharesRequest_Filter{ + Type: ocm.ListOCMSharesRequest_Filter_TYPE_CREATOR, + Term: &ocm.ListOCMSharesRequest_Filter_Creator{ + Creator: v.Creator, + }, + }) + case *collaboration.Filter_Owner: + ocmfilters = append(ocmfilters, &ocm.ListOCMSharesRequest_Filter{ + Type: ocm.ListOCMSharesRequest_Filter_TYPE_OWNER, + Term: &ocm.ListOCMSharesRequest_Filter_Owner{ + Owner: v.Owner, + }, + }) + } + } + return ocmfilters +} From d7692be999915807f1cbac1d78feb2e4f9dce27a Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 8 Jun 2023 11:06:38 +0200 Subject: [PATCH 10/48] Remove config for invite_link (#3905) * Removed config for invite_link The link template is still defined in a constant but not exposed in the config, as it depends on Mentix and should not be changed. * Updated autogenerated docs * Rephrasing * Removed template and simplified logic for invite URLs * Move config check where it is parsed * Further simplifications * Updated fixtures for integration tests * Updated changelog and example --- changelog/unreleased/def-invite-link.md | 8 +++ .../config/packages/cbox/group/rest/_index.md | 18 ++--- examples/storage-references/gateway.toml | 13 ++++ internal/http/services/sciencemesh/email.go | 16 ----- .../http/services/sciencemesh/sciencemesh.go | 24 ++++--- internal/http/services/sciencemesh/token.go | 66 +++---------------- .../fixtures/ocm-server-cernbox-http.toml | 4 +- .../grpc/fixtures/ocm-server-cesnet-http.toml | 4 +- .../ocm-share/ocm-server-cernbox-http.toml | 2 + .../ocm-share/ocm-server-cesnet-http.toml | 2 + 10 files changed, 65 insertions(+), 92 deletions(-) create mode 100644 changelog/unreleased/def-invite-link.md diff --git a/changelog/unreleased/def-invite-link.md b/changelog/unreleased/def-invite-link.md new file mode 100644 index 0000000000..24363a91cd --- /dev/null +++ b/changelog/unreleased/def-invite-link.md @@ -0,0 +1,8 @@ +Enhancement: Remove redundant config for invite_link_template + +This is to drop invite_link_template from the OCM-related config. +Now the provider_domain and mesh_directory_url config options +are both mandatory in the sciencemesh http service, and the link +is directly built out of the context. + +https://github.com/cs3org/reva/pull/3905 diff --git a/docs/content/en/docs/config/packages/cbox/group/rest/_index.md b/docs/content/en/docs/config/packages/cbox/group/rest/_index.md index 36888b42b1..ec90e873aa 100644 --- a/docs/content/en/docs/config/packages/cbox/group/rest/_index.md +++ b/docs/content/en/docs/config/packages/cbox/group/rest/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="redis_address" type="string" default="localhost:6379" %}} -The address at which the redis server is running [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L56) +The address at which the redis server is running [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L55) {{< highlight toml >}} [cbox.group.rest] redis_address = "localhost:6379" @@ -17,7 +17,7 @@ redis_address = "localhost:6379" {{% /dir %}} {{% dir name="group_members_cache_expiration" type="int" default=5 %}} -The time in minutes for which the members of a group would be cached [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L62) +The time in minutes for which the members of a group would be cached [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L61) {{< highlight toml >}} [cbox.group.rest] group_members_cache_expiration = 5 @@ -25,7 +25,7 @@ group_members_cache_expiration = 5 {{% /dir %}} {{% dir name="id_provider" type="string" default="http://cernbox.cern.ch" %}} -The OIDC Provider [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L64) +The OIDC Provider [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L63) {{< highlight toml >}} [cbox.group.rest] id_provider = "http://cernbox.cern.ch" @@ -33,7 +33,7 @@ id_provider = "http://cernbox.cern.ch" {{% /dir %}} {{% dir name="api_base_url" type="string" default="https://authorization-service-api-dev.web.cern.ch" %}} -Base API Endpoint [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L66) +Base API Endpoint [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L65) {{< highlight toml >}} [cbox.group.rest] api_base_url = "https://authorization-service-api-dev.web.cern.ch" @@ -41,7 +41,7 @@ api_base_url = "https://authorization-service-api-dev.web.cern.ch" {{% /dir %}} {{% dir name="client_id" type="string" default="-" %}} -Client ID needed to authenticate [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L68) +Client ID needed to authenticate [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L67) {{< highlight toml >}} [cbox.group.rest] client_id = "-" @@ -49,7 +49,7 @@ client_id = "-" {{% /dir %}} {{% dir name="client_secret" type="string" default="-" %}} -Client Secret [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L70) +Client Secret [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L69) {{< highlight toml >}} [cbox.group.rest] client_secret = "-" @@ -57,7 +57,7 @@ client_secret = "-" {{% /dir %}} {{% dir name="oidc_token_endpoint" type="string" default="https://keycloak-dev.cern.ch/auth/realms/cern/api-access/token" %}} -Endpoint to generate token to access the API [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L73) +Endpoint to generate token to access the API [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L72) {{< highlight toml >}} [cbox.group.rest] oidc_token_endpoint = "https://keycloak-dev.cern.ch/auth/realms/cern/api-access/token" @@ -65,7 +65,7 @@ oidc_token_endpoint = "https://keycloak-dev.cern.ch/auth/realms/cern/api-access/ {{% /dir %}} {{% dir name="target_api" type="string" default="authorization-service-api" %}} -The target application for which token needs to be generated [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L75) +The target application for which token needs to be generated [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L74) {{< highlight toml >}} [cbox.group.rest] target_api = "authorization-service-api" @@ -73,7 +73,7 @@ target_api = "authorization-service-api" {{% /dir %}} {{% dir name="group_fetch_interval" type="int" default=3600 %}} -The time in seconds between bulk fetch of groups [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L77) +The time in seconds between bulk fetch of groups [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/cbox/group/rest/rest.go#L76) {{< highlight toml >}} [cbox.group.rest] group_fetch_interval = 3600 diff --git a/examples/storage-references/gateway.toml b/examples/storage-references/gateway.toml index 8b887d1d3d..3070078503 100644 --- a/examples/storage-references/gateway.toml +++ b/examples/storage-references/gateway.toml @@ -48,3 +48,16 @@ mime_types = [ [http.services.ocdav] [http.services.ocs] [http.services.appprovider] + +[http.services.sciencemesh] +mesh_directory_url = 'https://sciencemesh.cesnet.cz/iop/meshdir' +provider_domain = 'your-domain.org' +body_template_path = '/etc/revad/sciencemesh_email_body' +ocm_mount_point = '/sciencemesh' + +[http.services.sciencemesh.smtp_credentials] +disable_auth = true +sender_mail = "sciencemesh@your-domain.org" +smtp_server = "your-smtp-server.your-domain.org" +smtp_port = 25 + diff --git a/internal/http/services/sciencemesh/email.go b/internal/http/services/sciencemesh/email.go index d64545fc22..239c89ba5d 100644 --- a/internal/http/services/sciencemesh/email.go +++ b/internal/http/services/sciencemesh/email.go @@ -117,19 +117,3 @@ func (h *tokenHandler) initSubjectTemplate(subjTempl string) error { h.tplSubj = tpl return nil } - -func (h *tokenHandler) initInviteLinkTemplate(inviteTempl string) error { - var t string - if inviteTempl == "" { - t = defaultInviteLink - } else { - t = inviteTempl - } - - tpl, err := template.New("tpl_invite").Parse(t) - if err != nil { - return err - } - h.tplInviteLink = tpl - return nil -} diff --git a/internal/http/services/sciencemesh/sciencemesh.go b/internal/http/services/sciencemesh/sciencemesh.go index 89f49990ee..6c0265e324 100644 --- a/internal/http/services/sciencemesh/sciencemesh.go +++ b/internal/http/services/sciencemesh/sciencemesh.go @@ -19,6 +19,7 @@ package sciencemesh import ( + "errors" "net/http" "github.com/cs3org/reva/pkg/appctx" @@ -42,6 +43,12 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) } conf.init() + if conf.ProviderDomain == "" { + return nil, errors.New("sciencemesh: provider_domain is missing from configuration") + } + if conf.MeshDirectoryURL == "" { + return nil, errors.New("sciencemesh: mesh_directory_url is missing from configuration") + } r := chi.NewRouter() s := &svc{ @@ -62,15 +69,14 @@ func (s *svc) Close() error { } type config struct { - Prefix string `mapstructure:"prefix"` - SMTPCredentials *smtpclient.SMTPCredentials `mapstructure:"smtp_credentials"` - GatewaySvc string `mapstructure:"gatewaysvc"` - MeshDirectoryURL string `mapstructure:"mesh_directory_url"` - ProviderDomain string `mapstructure:"provider_domain"` - SubjectTemplate string `mapstructure:"subject_template"` - BodyTemplatePath string `mapstructure:"body_template_path"` - OCMMountPoint string `mapstructure:"ocm_mount_point"` - InviteLinkTemplate string `mapstructure:"invite_link_template"` + Prefix string `mapstructure:"prefix"` + SMTPCredentials *smtpclient.SMTPCredentials `mapstructure:"smtp_credentials"` + GatewaySvc string `mapstructure:"gatewaysvc"` + MeshDirectoryURL string `mapstructure:"mesh_directory_url"` + ProviderDomain string `mapstructure:"provider_domain"` + SubjectTemplate string `mapstructure:"subject_template"` + BodyTemplatePath string `mapstructure:"body_template_path"` + OCMMountPoint string `mapstructure:"ocm_mount_point"` } func (c *config) init() { diff --git a/internal/http/services/sciencemesh/token.go b/internal/http/services/sciencemesh/token.go index a87dfca2e4..3308a4c978 100644 --- a/internal/http/services/sciencemesh/token.go +++ b/internal/http/services/sciencemesh/token.go @@ -24,10 +24,8 @@ import ( "html/template" "mime" "net/http" - "strings" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" invitepb "github.com/cs3org/go-cs3apis/cs3/ocm/invite/v1beta1" ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" @@ -38,16 +36,13 @@ import ( "github.com/cs3org/reva/pkg/smtpclient" ) -const defaultInviteLink = "{{.MeshDirectoryURL}}?token={{.Token}}&providerDomain={{.User.Id.Idp}}" - type tokenHandler struct { gatewayClient gateway.GatewayAPIClient smtpCredentials *smtpclient.SMTPCredentials meshDirectoryURL string - - tplSubj *template.Template - tplBody *template.Template - tplInviteLink *template.Template + providerDomain string + tplSubj *template.Template + tplBody *template.Template } func (h *tokenHandler) init(c *config) error { @@ -62,6 +57,7 @@ func (h *tokenHandler) init(c *config) error { } h.meshDirectoryURL = c.MeshDirectoryURL + h.providerDomain = c.ProviderDomain if err := h.initSubjectTemplate(c.SubjectTemplate); err != nil { return err @@ -71,7 +67,7 @@ func (h *tokenHandler) init(c *config) error { return err } - return h.initInviteLinkTemplate(c.InviteLinkTemplate) + return nil } type token struct { @@ -81,12 +77,6 @@ type token struct { InviteLink string `json:"invite_link"` } -type inviteLinkParams struct { - User *userpb.User - Token string - MeshDirectoryURL string -} - // Generate generates an invitation token and if a recipient is specified, // will send an email containing the link the user will use to accept the // invitation. @@ -116,12 +106,7 @@ func (h *tokenHandler) Generate(w http.ResponseWriter, r *http.Request) { } } - tknRes, err := h.prepareGenerateTokenResponse(user, token.InviteToken) - if err != nil { - reqres.WriteError(w, r, reqres.APIErrorServerError, "error generating response", err) - return - } - + tknRes := h.prepareGenerateTokenResponse(token.InviteToken) if err := json.NewEncoder(w).Encode(tknRes); err != nil { reqres.WriteError(w, r, reqres.APIErrorServerError, "error marshalling token data", err) return @@ -131,34 +116,17 @@ func (h *tokenHandler) Generate(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } -func (h *tokenHandler) generateInviteLink(user *userpb.User, token *invitepb.InviteToken) (string, error) { - var inviteLink strings.Builder - if err := h.tplInviteLink.Execute(&inviteLink, inviteLinkParams{ - User: user, - Token: token.Token, - MeshDirectoryURL: h.meshDirectoryURL, - }); err != nil { - return "", err - } - - return inviteLink.String(), nil -} - -func (h *tokenHandler) prepareGenerateTokenResponse(user *userpb.User, tkn *invitepb.InviteToken) (*token, error) { - inviteLink, err := h.generateInviteLink(user, tkn) - if err != nil { - return nil, err - } +func (h *tokenHandler) prepareGenerateTokenResponse(tkn *invitepb.InviteToken) *token { res := &token{ Token: tkn.Token, Description: tkn.Description, - InviteLink: inviteLink, + InviteLink: h.meshDirectoryURL + "?token=" + tkn.Token + "&providerDomain=" + h.providerDomain, } if tkn.Expiration != nil { res.Expiration = tkn.Expiration.Seconds } - return res, nil + return res } type acceptInviteRequest struct { @@ -278,22 +246,8 @@ func (h *tokenHandler) ListInvite(w http.ResponseWriter, r *http.Request) { } tokens := make([]*token, 0, len(res.InviteTokens)) - user := ctxpkg.ContextMustGetUser(ctx) for _, tkn := range res.InviteTokens { - inviteURL, err := h.generateInviteLink(user, tkn) - if err != nil { - reqres.WriteError(w, r, reqres.APIErrorServerError, "error generating invite URL from OCM token", err) - return - } - t := &token{ - Token: tkn.Token, - Description: tkn.Description, - InviteLink: inviteURL, - } - if tkn.Expiration != nil { - t.Expiration = tkn.Expiration.Seconds - } - tokens = append(tokens, t) + tokens = append(tokens, h.prepareGenerateTokenResponse(tkn)) } if err := json.NewEncoder(w).Encode(tokens); err != nil { diff --git a/tests/integration/grpc/fixtures/ocm-server-cernbox-http.toml b/tests/integration/grpc/fixtures/ocm-server-cernbox-http.toml index 7e3df6ea42..622974dcb3 100644 --- a/tests/integration/grpc/fixtures/ocm-server-cernbox-http.toml +++ b/tests/integration/grpc/fixtures/ocm-server-cernbox-http.toml @@ -10,6 +10,8 @@ address = "{{grpc_address}}" [http.services.ocmd] [http.services.sciencemesh] +provider_domain = "{{cernboxhttp_address}}" +mesh_directory_url = "http://meshdir" [http.middlewares.cors] @@ -17,4 +19,4 @@ address = "{{grpc_address}}" driver = "json" [http.middlewares.providerauthorizer.drivers.json] -providers = "fixtures/ocm-providers.demo.json" \ No newline at end of file +providers = "fixtures/ocm-providers.demo.json" diff --git a/tests/integration/grpc/fixtures/ocm-server-cesnet-http.toml b/tests/integration/grpc/fixtures/ocm-server-cesnet-http.toml index ce607e3648..93690ec5f4 100644 --- a/tests/integration/grpc/fixtures/ocm-server-cesnet-http.toml +++ b/tests/integration/grpc/fixtures/ocm-server-cesnet-http.toml @@ -10,6 +10,8 @@ address = "{{grpc_address}}" [http.services.ocmd] [http.services.sciencemesh] +provider_domain = "{{cesnethttp_address}}" +mesh_directory_url = "http://meshdir" [http.middlewares.cors] @@ -17,4 +19,4 @@ address = "{{grpc_address}}" driver = "json" [http.middlewares.providerauthorizer.drivers.json] -providers = "fixtures/ocm-providers.demo.json" \ No newline at end of file +providers = "fixtures/ocm-providers.demo.json" diff --git a/tests/integration/grpc/fixtures/ocm-share/ocm-server-cernbox-http.toml b/tests/integration/grpc/fixtures/ocm-share/ocm-server-cernbox-http.toml index d30faa0951..95a6f7dec5 100644 --- a/tests/integration/grpc/fixtures/ocm-share/ocm-server-cernbox-http.toml +++ b/tests/integration/grpc/fixtures/ocm-share/ocm-server-cernbox-http.toml @@ -10,6 +10,8 @@ address = "{{grpc_address}}" [http.services.ocmd] [http.services.sciencemesh] +provider_domain = "{{cernboxhttp_address}}" +mesh_directory_url = "http://meshdir" [http.middlewares.cors] diff --git a/tests/integration/grpc/fixtures/ocm-share/ocm-server-cesnet-http.toml b/tests/integration/grpc/fixtures/ocm-share/ocm-server-cesnet-http.toml index af69055e30..11ade22fe3 100644 --- a/tests/integration/grpc/fixtures/ocm-share/ocm-server-cesnet-http.toml +++ b/tests/integration/grpc/fixtures/ocm-share/ocm-server-cesnet-http.toml @@ -10,6 +10,8 @@ address = "{{grpc_address}}" [http.services.ocmd] [http.services.sciencemesh] +provider_domain = "{{cesnethttp_address}}" +mesh_directory_url = "http://meshdir" [http.middlewares.cors] From addb3be19a5607338e24872d9a343ddd4c0afb09 Mon Sep 17 00:00:00 2001 From: Hugo Labrador Date: Thu, 8 Jun 2023 14:16:04 +0200 Subject: [PATCH 11/48] eosclient: extend eos metadata with atime/ctime (#3954) * eosclient: extend eos metadata with atime/ctime * fix variable * handle conv err --- changelog/unreleased/extend-eos-meta.md | 6 +++++ pkg/eosclient/eosbinary/eosbinary.go | 30 +++++++++++++++++++++++++ pkg/eosclient/eosclient.go | 6 ++++- 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/extend-eos-meta.md diff --git a/changelog/unreleased/extend-eos-meta.md b/changelog/unreleased/extend-eos-meta.md new file mode 100644 index 0000000000..8204d188c3 --- /dev/null +++ b/changelog/unreleased/extend-eos-meta.md @@ -0,0 +1,6 @@ +Enhancement: Extend EOS metadata + +This PR extend the EOS metadata with atime and ctime fields. +This change is backwards compatible. + +https://github.com/cs3org/reva/pull/3954 diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index b120719363..ead7c21151 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -1190,6 +1190,32 @@ func (c *Client) mapToFileInfo(ctx context.Context, kv, attrs map[string]string, } } + var ctimesec, ctimenanos uint64 + if val, ok := kv["ctime"]; ok && val != "" { + split := strings.Split(val, ".") + ctimesec, err = strconv.ParseUint(split[0], 10, 64) + if err != nil { + return nil, err + } + ctimenanos, _ = strconv.ParseUint(split[1], 10, 32) + if err != nil { + return nil, err + } + } + + var atimesec, atimenanos uint64 + if val, ok := kv["atime"]; ok && val != "" { + split := strings.Split(val, ".") + atimesec, err = strconv.ParseUint(split[0], 10, 64) + if err != nil { + return nil, err + } + atimenanos, err = strconv.ParseUint(split[1], 10, 32) + if err != nil { + return nil, err + } + } + isDir := false var xs *eosclient.Checksum if _, ok := kv["files"]; ok { @@ -1239,6 +1265,10 @@ func (c *Client) mapToFileInfo(ctx context.Context, kv, attrs map[string]string, TreeSize: treeSize, MTimeSec: mtimesec, MTimeNanos: uint32(mtimenanos), + CTimeSec: ctimesec, + CTimeNanos: uint32(ctimenanos), + ATimeSec: atimesec, + ATimeNanos: uint32(atimenanos), IsDir: isDir, Instance: c.opt.URL, SysACL: sysACL, diff --git a/pkg/eosclient/eosclient.go b/pkg/eosclient/eosclient.go index 97d73950bc..6b93293683 100644 --- a/pkg/eosclient/eosclient.go +++ b/pkg/eosclient/eosclient.go @@ -74,13 +74,17 @@ type Attribute struct { // FileInfo represents the metadata information returned by querying the EOS namespace. type FileInfo struct { IsDir bool - MTimeNanos uint32 Inode uint64 `json:"inode"` FID uint64 `json:"fid"` UID uint64 `json:"uid"` GID uint64 `json:"gid"` TreeSize uint64 `json:"tree_size"` MTimeSec uint64 `json:"mtime_sec"` + MTimeNanos uint32 `json:"mtime_nanos"` + ATimeSec uint64 `json:"atime_sec"` + ATimeNanos uint32 `json:"atime_nanos"` + CTimeSec uint64 `json:"ctime_sec"` + CTimeNanos uint32 `json:"ctime_nanos"` Size uint64 `json:"size"` TreeCount uint64 `json:"tree_count"` File string `json:"eos_file"` From 39d8c2381bcf23320c271fa790fb7656f65d0010 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Thu, 8 Jun 2023 14:55:21 +0200 Subject: [PATCH 12/48] Update and delete OCM shares (#3937) * implemented DeleteRemoteUser * update state of received ocm share * fix cmd * removed old comment * add endpoint to delete accepted user * remove federated share * fix linter * accept/reject ocm recevied shares * update access methods in sql driver * inject time for unit tests * add unit tests for UpdateShare * removed tests for DeleteShare * update permissions of federated shares from ocs * update go-cs3apis * fix linter * add command in cli to remove an accepted user * update permissions of ocm share from cli * optimized query build when updating access methods * fix update ocm share in ocs * fix update received ocm share * return share id when accepting/reject ocm share * filter ocm shares by status * fix update received share * expose state of ocm share * set correct user type when deleting user * add share info when creating ocm share * disabled nextcloud unit test * add changelog * trigger pipeline * add header * fix rebase * fix linter --- .../unreleased/update_remove_ocm_share.md | 10 + cmd/reva/main.go | 1 + cmd/reva/ocm-remove-accepted-user.go | 77 ++++ cmd/reva/ocm-share-update.go | 59 ++- go.mod | 2 +- go.sum | 2 + internal/grpc/services/gateway/ocmcore.go | 32 ++ .../grpc/services/gateway/ocminvitemanager.go | 16 + internal/grpc/services/ocmcore/ocmcore.go | 8 + .../ocminvitemanager/ocminvitemanager.go | 13 + .../ocmshareprovider/ocmshareprovider.go | 9 +- .../handlers/apps/sharing/shares/pending.go | 83 +++- .../handlers/apps/sharing/shares/remote.go | 32 +- .../handlers/apps/sharing/shares/shares.go | 125 +++++- .../ocs/handlers/apps/sharing/shares/user.go | 124 +++++- .../http/services/sciencemesh/sciencemesh.go | 1 + internal/http/services/sciencemesh/token.go | 47 +++ pkg/ocm/invite/invite.go | 3 + pkg/ocm/invite/repository/json/json.go | 21 + pkg/ocm/invite/repository/memory/memory.go | 17 + pkg/ocm/invite/repository/sql/sql.go | 6 + pkg/ocm/share/repository/json/json.go | 2 +- .../share/repository/nextcloud/nextcloud.go | 3 +- .../repository/nextcloud/nextcloud_test.go | 125 +++--- pkg/ocm/share/repository/sql/sql.go | 171 +++++++- pkg/ocm/share/repository/sql/sql_test.go | 395 +++++++++++++++++- pkg/ocm/share/share.go | 2 +- pkg/utils/list/list.go | 7 + 28 files changed, 1265 insertions(+), 128 deletions(-) create mode 100644 changelog/unreleased/update_remove_ocm_share.md create mode 100644 cmd/reva/ocm-remove-accepted-user.go diff --git a/changelog/unreleased/update_remove_ocm_share.md b/changelog/unreleased/update_remove_ocm_share.md new file mode 100644 index 0000000000..73326c0ad9 --- /dev/null +++ b/changelog/unreleased/update_remove_ocm_share.md @@ -0,0 +1,10 @@ +Enhancement: Manage OCM shares + +Implements the following item regarding OCM: + - update of OCM shares in both grpc and ocs layer, + allowing an user to update permissions and expiration of the share + - deletion of OCM shares in both grpc and ocs layer + - accept/reject of received OCM shares + - remove accepted remote users + +https://github.com/cs3org/reva/pull/3937 \ No newline at end of file diff --git a/cmd/reva/main.go b/cmd/reva/main.go index f04984a0a7..4a5e06d3ac 100644 --- a/cmd/reva/main.go +++ b/cmd/reva/main.go @@ -56,6 +56,7 @@ var ( moveCommand(), mkdirCommand(), ocmFindAcceptedUsersCommand(), + ocmRemoveAcceptedUser(), ocmInviteGenerateCommand(), ocmInviteForwardCommand(), ocmShareCreateCommand(), diff --git a/cmd/reva/ocm-remove-accepted-user.go b/cmd/reva/ocm-remove-accepted-user.go new file mode 100644 index 0000000000..291454d074 --- /dev/null +++ b/cmd/reva/ocm-remove-accepted-user.go @@ -0,0 +1,77 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package main + +import ( + "errors" + "fmt" + "io" + + userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + invitepb "github.com/cs3org/go-cs3apis/cs3/ocm/invite/v1beta1" + rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" +) + +func ocmRemoveAcceptedUser() *command { + cmd := newCommand("ocm-remove-accepted-user") + cmd.Description = func() string { return "remove a remote user from the personal user list" } + cmd.Usage = func() string { return "Usage: ocm-remove-accepted-user [-flags]" } + + user := cmd.String("user", "", "the user id") + idp := cmd.String("idp", "", "the idp of the user") + + cmd.ResetFlags = func() { + *user, *idp = "", "" + } + + cmd.Action = func(w ...io.Writer) error { + // validate flags + if *user == "" { + return errors.New("User cannot be empty: user -user flag\n" + cmd.Usage()) + } + + if *idp == "" { + return errors.New("IdP cannot be empty: use -idp flag\n" + cmd.Usage()) + } + + ctx := getAuthContext() + client, err := getClient() + if err != nil { + return err + } + + res, err := client.DeleteAcceptedUser(ctx, &invitepb.DeleteAcceptedUserRequest{ + RemoteUserId: &userv1beta1.UserId{ + Type: userv1beta1.UserType_USER_TYPE_FEDERATED, + Idp: *idp, + OpaqueId: *user, + }, + }) + if err != nil { + return err + } + if res.Status.Code != rpcv1beta1.Code_CODE_OK { + return formatError(res.Status) + } + + fmt.Println("OK") + return nil + } + return cmd +} diff --git a/cmd/reva/ocm-share-update.go b/cmd/reva/ocm-share-update.go index 20eda9db08..c1985a91b3 100644 --- a/cmd/reva/ocm-share-update.go +++ b/cmd/reva/ocm-share-update.go @@ -31,30 +31,26 @@ func ocmShareUpdateCommand() *command { cmd := newCommand("ocm-share-update") cmd.Description = func() string { return "update an OCM share" } cmd.Usage = func() string { return "Usage: ocm-share-update [-flags] " } - rol := cmd.String("rol", "viewer", "the permission for the share (viewer or editor)") + + webdavRol := cmd.String("webdav-rol", "viewer", "the permission for the WebDAV access method (viewer or editor)") + webappViewMode := cmd.String("webapp-mode", "view", "the view mode for the Webapp access method (read or write)") cmd.ResetFlags = func() { - *rol = "viewer" + *webdavRol, *webappViewMode = "viewer", "read" } cmd.Action = func(w ...io.Writer) error { if cmd.NArg() < 1 { return errors.New("Invalid arguments: " + cmd.Usage()) } - // validate flags - if *rol != viewerPermission && *rol != editorPermission { - return errors.New("Invalid rol: rol must be viewer or editor\n" + cmd.Usage()) - } - id := cmd.Args()[0] - ctx := getAuthContext() - shareClient, err := getClient() - if err != nil { - return err + if *webdavRol == "" && *webappViewMode == "" { + return errors.New("use at least one of -webdav-rol or -webapp-mode flag") } - perm, err := getOCMSharePerm(*rol) + ctx := getAuthContext() + shareClient, err := getClient() if err != nil { return err } @@ -67,13 +63,42 @@ func ocmShareUpdateCommand() *command { }, }, }, - Field: &ocm.UpdateOCMShareRequest_UpdateField{ - Field: &ocm.UpdateOCMShareRequest_UpdateField_Permissions{ - Permissions: &ocm.SharePermissions{ - Permissions: perm, + } + + if *webdavRol != "" { + perm, err := getOCMSharePerm(*webdavRol) + if err != nil { + return err + } + shareRequest.Field = append(shareRequest.Field, &ocm.UpdateOCMShareRequest_UpdateField{ + Field: &ocm.UpdateOCMShareRequest_UpdateField_AccessMethods{ + AccessMethods: &ocm.AccessMethod{ + Term: &ocm.AccessMethod_WebdavOptions{ + WebdavOptions: &ocm.WebDAVAccessMethod{ + Permissions: perm, + }, + }, }, }, - }, + }) + } + + if *webappViewMode != "" { + mode, err := getOCMViewMode(*webappViewMode) + if err != nil { + return err + } + shareRequest.Field = append(shareRequest.Field, &ocm.UpdateOCMShareRequest_UpdateField{ + Field: &ocm.UpdateOCMShareRequest_UpdateField_AccessMethods{ + AccessMethods: &ocm.AccessMethod{ + Term: &ocm.AccessMethod_WebappOptions{ + WebappOptions: &ocm.WebappAccessMethod{ + ViewMode: mode, + }, + }, + }, + }, + }) } shareRes, err := shareClient.UpdateOCMShare(ctx, shareRequest) diff --git a/go.mod b/go.mod index f5515bb3af..bcc99beacb 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/ceph/go-ceph v0.15.0 github.com/cheggaaa/pb v1.0.29 github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e - github.com/cs3org/go-cs3apis v0.0.0-20230508132523-e0d062e63b3b + github.com/cs3org/go-cs3apis v0.0.0-20230606135123-b799d47a6648 github.com/dgraph-io/ristretto v0.1.1 github.com/dolthub/go-mysql-server v0.14.0 github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59 diff --git a/go.sum b/go.sum index 1ff0369c1c..f35936c818 100644 --- a/go.sum +++ b/go.sum @@ -308,6 +308,8 @@ github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJff github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= github.com/cs3org/go-cs3apis v0.0.0-20230508132523-e0d062e63b3b h1:UCO7Rnf5bvIvRtETguV8IaTx73cImLlFWxrApCB0QsQ= github.com/cs3org/go-cs3apis v0.0.0-20230508132523-e0d062e63b3b/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20230606135123-b799d47a6648 h1:gBz1JSC2u6o/TkUhWSdJZvacyTsVUzDouegRzvrJye4= +github.com/cs3org/go-cs3apis v0.0.0-20230606135123-b799d47a6648/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/internal/grpc/services/gateway/ocmcore.go b/internal/grpc/services/gateway/ocmcore.go index 8cb2237632..531fd00f1a 100644 --- a/internal/grpc/services/gateway/ocmcore.go +++ b/internal/grpc/services/gateway/ocmcore.go @@ -42,3 +42,35 @@ func (s *svc) CreateOCMCoreShare(ctx context.Context, req *ocmcore.CreateOCMCore return res, nil } + +func (s *svc) UpdateOCMCoreShare(ctx context.Context, req *ocmcore.UpdateOCMCoreShareRequest) (*ocmcore.UpdateOCMCoreShareResponse, error) { + c, err := pool.GetOCMCoreClient(pool.Endpoint(s.c.OCMCoreEndpoint)) + if err != nil { + return &ocmcore.UpdateOCMCoreShareResponse{ + Status: status.NewInternal(ctx, err, "error getting ocm core client"), + }, nil + } + + res, err := c.UpdateOCMCoreShare(ctx, req) + if err != nil { + return nil, errors.Wrap(err, "gateway: error calling UpdateOCMCoreShare") + } + + return res, nil +} + +func (s *svc) DeleteOCMCoreShare(ctx context.Context, req *ocmcore.DeleteOCMCoreShareRequest) (*ocmcore.DeleteOCMCoreShareResponse, error) { + c, err := pool.GetOCMCoreClient(pool.Endpoint(s.c.OCMCoreEndpoint)) + if err != nil { + return &ocmcore.DeleteOCMCoreShareResponse{ + Status: status.NewInternal(ctx, err, "error getting ocm core client"), + }, nil + } + + res, err := c.DeleteOCMCoreShare(ctx, req) + if err != nil { + return nil, errors.Wrap(err, "gateway: error calling UpdateOCMCoreShare") + } + + return res, nil +} diff --git a/internal/grpc/services/gateway/ocminvitemanager.go b/internal/grpc/services/gateway/ocminvitemanager.go index 9bf4de3017..646ccd33cb 100644 --- a/internal/grpc/services/gateway/ocminvitemanager.go +++ b/internal/grpc/services/gateway/ocminvitemanager.go @@ -122,3 +122,19 @@ func (s *svc) FindAcceptedUsers(ctx context.Context, req *invitepb.FindAcceptedU return res, nil } + +func (s *svc) DeleteAcceptedUser(ctx context.Context, req *invitepb.DeleteAcceptedUserRequest) (*invitepb.DeleteAcceptedUserResponse, error) { + c, err := pool.GetOCMInviteManagerClient(pool.Endpoint(s.c.OCMInviteManagerEndpoint)) + if err != nil { + return &invitepb.DeleteAcceptedUserResponse{ + Status: status.NewInternal(ctx, err, "error getting user invite provider client"), + }, nil + } + + res, err := c.DeleteAcceptedUser(ctx, req) + if err != nil { + return nil, errors.Wrap(err, "gateway: error calling FindAcceptedUsers") + } + + return res, nil +} diff --git a/internal/grpc/services/ocmcore/ocmcore.go b/internal/grpc/services/ocmcore/ocmcore.go index 6cd63987f8..ebb0d9b9f6 100644 --- a/internal/grpc/services/ocmcore/ocmcore.go +++ b/internal/grpc/services/ocmcore/ocmcore.go @@ -148,3 +148,11 @@ func (s *service) CreateOCMCoreShare(ctx context.Context, req *ocmcore.CreateOCM Created: share.Ctime, }, nil } + +func (s *service) UpdateOCMCoreShare(ctx context.Context, req *ocmcore.UpdateOCMCoreShareRequest) (*ocmcore.UpdateOCMCoreShareResponse, error) { + return nil, errtypes.NotSupported("not implemented") +} + +func (s *service) DeleteOCMCoreShare(ctx context.Context, req *ocmcore.DeleteOCMCoreShareRequest) (*ocmcore.DeleteOCMCoreShareResponse, error) { + return nil, errtypes.NotSupported("not implemented") +} diff --git a/internal/grpc/services/ocminvitemanager/ocminvitemanager.go b/internal/grpc/services/ocminvitemanager/ocminvitemanager.go index b56844cd08..80d8ff1d01 100644 --- a/internal/grpc/services/ocminvitemanager/ocminvitemanager.go +++ b/internal/grpc/services/ocminvitemanager/ocminvitemanager.go @@ -369,3 +369,16 @@ func (s *service) FindAcceptedUsers(ctx context.Context, req *invitepb.FindAccep AcceptedUsers: acceptedUsers, }, nil } + +func (s *service) DeleteAcceptedUser(ctx context.Context, req *invitepb.DeleteAcceptedUserRequest) (*invitepb.DeleteAcceptedUserResponse, error) { + user := ctxpkg.ContextMustGetUser(ctx) + if err := s.repo.DeleteRemoteUser(ctx, user.Id, req.RemoteUserId); err != nil { + return &invitepb.DeleteAcceptedUserResponse{ + Status: status.NewInternal(ctx, err, "error deleting remote users: "+err.Error()), + }, nil + } + + return &invitepb.DeleteAcceptedUserResponse{ + Status: status.NewOK(ctx), + }, nil +} diff --git a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go index 88e11393d9..9051f1dba2 100644 --- a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go +++ b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go @@ -459,7 +459,12 @@ func (s *service) ListOCMShares(ctx context.Context, req *ocm.ListOCMSharesReque func (s *service) UpdateOCMShare(ctx context.Context, req *ocm.UpdateOCMShareRequest) (*ocm.UpdateOCMShareResponse, error) { user := ctxpkg.ContextMustGetUser(ctx) - _, err := s.repo.UpdateShare(ctx, user, req.Ref, req.Field.GetPermissions()) // TODO(labkode): check what to update + if len(req.Field) == 0 { + return &ocm.UpdateOCMShareResponse{ + Status: status.NewOK(ctx), + }, nil + } + _, err := s.repo.UpdateShare(ctx, user, req.Ref, req.Field...) if err != nil { if errors.Is(err, share.ErrShareNotFound) { return &ocm.UpdateOCMShareResponse{ @@ -495,7 +500,7 @@ func (s *service) ListReceivedOCMShares(ctx context.Context, req *ocm.ListReceiv func (s *service) UpdateReceivedOCMShare(ctx context.Context, req *ocm.UpdateReceivedOCMShareRequest) (*ocm.UpdateReceivedOCMShareResponse, error) { user := ctxpkg.ContextMustGetUser(ctx) - _, err := s.repo.UpdateReceivedShare(ctx, user, req.Share, req.UpdateMask) // TODO(labkode): check what to update + _, err := s.repo.UpdateReceivedShare(ctx, user, req.Share, req.UpdateMask) if err != nil { if errors.Is(err, share.ErrShareNotFound) { return &ocm.UpdateReceivedOCMShareResponse{ diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/pending.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/pending.go index e70be1df86..235235e493 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/pending.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/pending.go @@ -24,6 +24,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" + ocmv1beta1 "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" "github.com/cs3org/reva/pkg/appctx" @@ -36,13 +37,21 @@ import ( // AcceptReceivedShare handles Post Requests on /apps/files_sharing/api/v1/shares/{shareid}. func (h *Handler) AcceptReceivedShare(w http.ResponseWriter, r *http.Request) { shareID := chi.URLParam(r, "shareid") - h.updateReceivedShare(w, r, shareID, false) + if h.isFederatedReceivedShare(r, shareID) { + h.updateReceivedFederatedShare(w, r, shareID, false) + } else { + h.updateReceivedShare(w, r, shareID, false) + } } // RejectReceivedShare handles DELETE Requests on /apps/files_sharing/api/v1/shares/{shareid}. func (h *Handler) RejectReceivedShare(w http.ResponseWriter, r *http.Request) { shareID := chi.URLParam(r, "shareid") - h.updateReceivedShare(w, r, shareID, true) + if h.isFederatedReceivedShare(r, shareID) { + h.updateReceivedFederatedShare(w, r, shareID, true) + } else { + h.updateReceivedShare(w, r, shareID, true) + } } func (h *Handler) updateReceivedShare(w http.ResponseWriter, r *http.Request, shareID string, rejectShare bool) { @@ -109,3 +118,73 @@ func (h *Handler) updateReceivedShare(w http.ResponseWriter, r *http.Request, sh response.WriteOCSSuccess(w, r, []*conversions.ShareData{data}) } + +func (h *Handler) updateReceivedFederatedShare(w http.ResponseWriter, r *http.Request, shareID string, rejectShare bool) { + ctx := r.Context() + + client, err := pool.GetGatewayServiceClient(pool.Endpoint(h.gatewayAddr)) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error getting grpc gateway client", err) + return + } + + share, err := client.GetReceivedOCMShare(ctx, &ocmv1beta1.GetReceivedOCMShareRequest{ + Ref: &ocmv1beta1.ShareReference{ + Spec: &ocmv1beta1.ShareReference_Id{ + Id: &ocmv1beta1.ShareId{ + OpaqueId: shareID, + }, + }, + }, + }) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "grpc update received share request failed", err) + return + } + if share.Status.Code != rpc.Code_CODE_OK { + if share.Status.Code == rpc.Code_CODE_NOT_FOUND { + response.WriteOCSError(w, r, response.MetaNotFound.StatusCode, "not found", nil) + return + } + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "grpc update received share request failed", errors.Errorf("code: %d, message: %s", share.Status.Code, share.Status.Message)) + return + } + + req := &ocmv1beta1.UpdateReceivedOCMShareRequest{ + Share: &ocmv1beta1.ReceivedShare{ + Id: &ocmv1beta1.ShareId{ + OpaqueId: shareID, + }, + }, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"state"}}, + } + if rejectShare { + req.Share.State = ocmv1beta1.ShareState_SHARE_STATE_REJECTED + } else { + req.Share.State = ocmv1beta1.ShareState_SHARE_STATE_ACCEPTED + } + + updateRes, err := client.UpdateReceivedOCMShare(ctx, req) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "grpc update received share request failed", err) + return + } + + if updateRes.Status.Code != rpc.Code_CODE_OK { + if updateRes.Status.Code == rpc.Code_CODE_NOT_FOUND { + response.WriteOCSError(w, r, response.MetaNotFound.StatusCode, "not found", nil) + return + } + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "grpc update received share request failed", errors.Errorf("code: %d, message: %s", updateRes.Status.Code, updateRes.Status.Message)) + return + } + + data, err := conversions.ReceivedOCMShare2ShareData(share.Share, h.ocmLocalMount(share.Share)) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "grpc update received share request failed", err) + return + } + h.mapUserIdsReceivedFederatedShare(ctx, client, data) + data.State = mapOCMState(req.Share.State) + response.WriteOCSSuccess(w, r, []*conversions.ShareData{data}) +} diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/remote.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/remote.go index 5d76f01288..9af400a818 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/remote.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/remote.go @@ -108,7 +108,31 @@ func (h *Handler) createFederatedCloudShare(w http.ResponseWriter, r *http.Reque return } - response.WriteOCSSuccess(w, r, "OCM Share created") + s := createShareResponse.Share + data, err := conversions.OCMShare2ShareData(s) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error converting share", err) + return + } + h.mapUserIdsFederatedShare(ctx, c, data) + + info, status, err := h.getResourceInfoByID(ctx, c, s.ResourceId) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error statting resource id", err) + return + } + if status.Code != rpc.Code_CODE_OK { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error statting resource id", errors.New(status.Message)) + return + } + + err = h.addFileInfo(ctx, data, info) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error statting resource id", err) + return + } + + response.WriteOCSSuccess(w, r, data) } func getViewModeFromRole(role *conversions.Role) providerv1beta1.ViewMode { @@ -162,7 +186,7 @@ func (h *Handler) ListFederatedShares(w http.ResponseWriter, r *http.Request) { // TODO Implement response with HAL schemating } -func (h *Handler) listReceivedFederatedShares(ctx context.Context, gw gatewayv1beta1.GatewayAPIClient) ([]*conversions.ShareData, error) { +func (h *Handler) listReceivedFederatedShares(ctx context.Context, gw gatewayv1beta1.GatewayAPIClient, state ocm.ShareState) ([]*conversions.ShareData, error) { listRes, err := gw.ListReceivedOCMShares(ctx, &ocm.ListReceivedOCMSharesRequest{}) if err != nil { return nil, err @@ -170,11 +194,15 @@ func (h *Handler) listReceivedFederatedShares(ctx context.Context, gw gatewayv1b shares := []*conversions.ShareData{} for _, s := range listRes.Shares { + if state != ocsStateUnknown && s.State != state { + continue + } sd, err := conversions.ReceivedOCMShare2ShareData(s, h.ocmLocalMount(s)) if err != nil { continue } h.mapUserIdsReceivedFederatedShare(ctx, gw, sd) + sd.State = mapOCMState(s.State) shares = append(shares, sd) } return shares, nil diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go index 106aa7c9af..853b7771d2 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go @@ -33,12 +33,14 @@ import ( "time" "github.com/ReneKroon/ttlcache/v2" + providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" link "github.com/cs3org/go-cs3apis/cs3/sharing/link/v1beta1" + ocmv1beta1 "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocdav" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/config" @@ -547,6 +549,10 @@ func (h *Handler) UpdateShare(w http.ResponseWriter, r *http.Request) { h.updatePublicShare(w, r, shareID) return } + if h.isFederatedShare(r, shareID) { + h.updateFederatedShare(w, r, shareID) + return + } h.updateShare(w, r, shareID) // TODO PUT is used with incomplete data to update a share} } @@ -646,6 +652,89 @@ func (h *Handler) updateShare(w http.ResponseWriter, r *http.Request, shareID st response.WriteOCSSuccess(w, r, share) } +func permissionsToViewMode(pint int) providerv1beta1.ViewMode { + if pint == 15 { + return providerv1beta1.ViewMode_VIEW_MODE_READ_WRITE + } + return providerv1beta1.ViewMode_VIEW_MODE_READ_ONLY +} + +func (h *Handler) updateFederatedShare(w http.ResponseWriter, r *http.Request, shareID string) { + ctx := r.Context() + + pval := r.FormValue("permissions") + if pval == "" { + response.WriteOCSError(w, r, response.MetaBadRequest.StatusCode, "permissions missing", nil) + return + } + + pint, err := strconv.Atoi(pval) + if err != nil { + response.WriteOCSError(w, r, response.MetaBadRequest.StatusCode, "permissions must be an integer", nil) + return + } + permissions, err := conversions.NewPermissions(pint) + if err != nil { + response.WriteOCSError(w, r, response.MetaBadRequest.StatusCode, err.Error(), nil) + return + } + + client, err := pool.GetGatewayServiceClient(pool.Endpoint(h.gatewayAddr)) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error getting grpc gateway client", err) + return + } + + updateRes, err := client.UpdateOCMShare(ctx, &ocmv1beta1.UpdateOCMShareRequest{ + Ref: &ocmv1beta1.ShareReference{ + Spec: &ocmv1beta1.ShareReference_Id{ + Id: &ocmv1beta1.ShareId{ + OpaqueId: shareID, + }, + }, + }, + Field: []*ocmv1beta1.UpdateOCMShareRequest_UpdateField{ + { + Field: &ocmv1beta1.UpdateOCMShareRequest_UpdateField_AccessMethods{ + AccessMethods: &ocmv1beta1.AccessMethod{ + Term: &ocmv1beta1.AccessMethod_WebdavOptions{ + WebdavOptions: &ocmv1beta1.WebDAVAccessMethod{ + Permissions: conversions.RoleFromOCSPermissions(permissions).CS3ResourcePermissions(), + }, + }, + }, + }, + }, + { + Field: &ocmv1beta1.UpdateOCMShareRequest_UpdateField_AccessMethods{ + AccessMethods: &ocmv1beta1.AccessMethod{ + Term: &ocmv1beta1.AccessMethod_WebappOptions{ + WebappOptions: &ocmv1beta1.WebappAccessMethod{ + ViewMode: permissionsToViewMode(pint), + }, + }, + }, + }, + }, + }, + }) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error sending a grpc update share request", err) + return + } + + if updateRes.Status.Code != rpc.Code_CODE_OK { + if updateRes.Status.Code == rpc.Code_CODE_NOT_FOUND { + response.WriteOCSError(w, r, response.MetaNotFound.StatusCode, "not found", nil) + return + } + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "grpc update share request failed", err) + return + } + + response.WriteOCSSuccess(w, r, []*conversions.ShareData{}) +} + // RemoveShare handles DELETE requests on /apps/files_sharing/api/v1/shares/(shareid). func (h *Handler) RemoveShare(w http.ResponseWriter, r *http.Request) { shareID := chi.URLParam(r, "shareid") @@ -654,6 +743,8 @@ func (h *Handler) RemoveShare(w http.ResponseWriter, r *http.Request) { h.removePublicShare(w, r, shareID) case h.isUserShare(r, shareID): h.removeUserShare(w, r, shareID) + case h.isFederatedShare(r, shareID): + h.removeFederatedShare(w, r, shareID) default: // The request is a remove space member request. h.removeSpaceMember(w, r, shareID) @@ -685,7 +776,8 @@ const ( func (h *Handler) listSharesWithMe(w http.ResponseWriter, r *http.Request) { // which pending state to list - stateFilter := getStateFilter(r.FormValue("state")) + state := r.FormValue("state") + stateFilter := getStateFilter(state) log := appctx.GetLogger(r.Context()) client, err := pool.GetGatewayServiceClient(pool.Endpoint(h.gatewayAddr)) @@ -871,7 +963,8 @@ func (h *Handler) listSharesWithMe(w http.ResponseWriter, r *http.Request) { if h.listOCMShares { // include ocm shares in the response - lst, err := h.listReceivedFederatedShares(ctx, client) + stateFilter := getOCMStateFilter(state) + lst, err := h.listReceivedFederatedShares(ctx, client, stateFilter) if err != nil { log.Err(err).Msg("error listing received ocm shares") response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error listing received ocm shares", err) @@ -1275,6 +1368,19 @@ func mapState(state collaboration.ShareState) int { return mapped } +func mapOCMState(state ocmv1beta1.ShareState) int { + switch state { + case ocmv1beta1.ShareState_SHARE_STATE_PENDING: + return ocsStatePending + case ocmv1beta1.ShareState_SHARE_STATE_ACCEPTED: + return ocsStateAccepted + case ocmv1beta1.ShareState_SHARE_STATE_REJECTED: + return ocsStateRejected + default: + return ocsStateUnknown + } +} + func getStateFilter(s string) collaboration.ShareState { var stateFilter collaboration.ShareState switch s { @@ -1291,3 +1397,18 @@ func getStateFilter(s string) collaboration.ShareState { } return stateFilter } + +func getOCMStateFilter(s string) ocmv1beta1.ShareState { + switch s { + case "all": + return ocsStateUnknown // no filter + case "0": // accepted + return ocmv1beta1.ShareState_SHARE_STATE_ACCEPTED + case "1": // pending + return ocmv1beta1.ShareState_SHARE_STATE_PENDING + case "2": // rejected + return ocmv1beta1.ShareState_SHARE_STATE_REJECTED + default: + return ocmv1beta1.ShareState_SHARE_STATE_ACCEPTED + } +} diff --git a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go index e98b4c0469..57ab112490 100644 --- a/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go +++ b/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/user.go @@ -25,7 +25,7 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1" - ocm "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" + ocmpb "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" @@ -176,6 +176,106 @@ func (h *Handler) removeUserShare(w http.ResponseWriter, r *http.Request, shareI response.WriteOCSSuccess(w, r, data) } +func (h *Handler) isFederatedShare(r *http.Request, shareID string) bool { + log := appctx.GetLogger(r.Context()) + client, err := pool.GetGatewayServiceClient(pool.Endpoint(h.gatewayAddr)) + if err != nil { + log.Err(err).Send() + return false + } + + getShareRes, err := client.GetOCMShare(r.Context(), &ocmpb.GetOCMShareRequest{ + Ref: &ocmpb.ShareReference{ + Spec: &ocmpb.ShareReference_Id{ + Id: &ocmpb.ShareId{ + OpaqueId: shareID, + }, + }, + }, + }) + if err != nil { + log.Err(err).Send() + return false + } + + return getShareRes.GetShare() != nil +} + +func (h *Handler) isFederatedReceivedShare(r *http.Request, shareID string) bool { + log := appctx.GetLogger(r.Context()) + client, err := pool.GetGatewayServiceClient(pool.Endpoint(h.gatewayAddr)) + if err != nil { + log.Err(err).Send() + return false + } + + getShareRes, err := client.GetReceivedOCMShare(r.Context(), &ocmpb.GetReceivedOCMShareRequest{ + Ref: &ocmpb.ShareReference{ + Spec: &ocmpb.ShareReference_Id{ + Id: &ocmpb.ShareId{ + OpaqueId: shareID, + }, + }, + }, + }) + if err != nil { + log.Err(err).Send() + return false + } + + return getShareRes.GetShare() != nil +} + +func (h *Handler) removeFederatedShare(w http.ResponseWriter, r *http.Request, shareID string) { + ctx := r.Context() + + client, err := pool.GetGatewayServiceClient(pool.Endpoint(h.gatewayAddr)) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error getting grpc gateway client", err) + return + } + + shareRef := &ocmpb.ShareReference_Id{Id: &ocmpb.ShareId{OpaqueId: shareID}} + // Get the share, so that we can include it in the response. + getShareResp, err := client.GetOCMShare(ctx, &ocmpb.GetOCMShareRequest{Ref: &ocmpb.ShareReference{Spec: shareRef}}) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error sending a grpc delete share request", err) + return + } + if getShareResp.Status.Code != rpc.Code_CODE_OK { + if getShareResp.Status.Code == rpc.Code_CODE_NOT_FOUND { + response.WriteOCSError(w, r, response.MetaNotFound.StatusCode, "not found", nil) + return + } + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "deleting share failed", err) + return + } + + data, err := conversions.OCMShare2ShareData(getShareResp.Share) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "deleting share failed", err) + return + } + // A deleted share should not have an ID. + data.ID = "" + + uRes, err := client.RemoveOCMShare(ctx, &ocmpb.RemoveOCMShareRequest{Ref: &ocmpb.ShareReference{Spec: shareRef}}) + if err != nil { + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error sending a grpc delete share request", err) + return + } + + if uRes.Status.Code != rpc.Code_CODE_OK { + if uRes.Status.Code == rpc.Code_CODE_NOT_FOUND { + response.WriteOCSError(w, r, response.MetaNotFound.StatusCode, "not found", nil) + return + } + response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "grpc delete share request failed", err) + return + } + response.WriteOCSSuccess(w, r, data) +} + func (h *Handler) listUserShares(r *http.Request, filters []*collaboration.Filter) ([]*conversions.ShareData, *rpc.Status, error) { ctx := r.Context() log := appctx.GetLogger(ctx) @@ -238,28 +338,28 @@ func (h *Handler) listUserShares(r *http.Request, filters []*collaboration.Filte return ocsDataPayload, nil, nil } -func convertToOCMFilters(filters []*collaboration.Filter) []*ocm.ListOCMSharesRequest_Filter { - ocmfilters := []*ocm.ListOCMSharesRequest_Filter{} +func convertToOCMFilters(filters []*collaboration.Filter) []*ocmpb.ListOCMSharesRequest_Filter { + ocmfilters := []*ocmpb.ListOCMSharesRequest_Filter{} for _, f := range filters { switch v := f.Term.(type) { case *collaboration.Filter_ResourceId: - ocmfilters = append(ocmfilters, &ocm.ListOCMSharesRequest_Filter{ - Type: ocm.ListOCMSharesRequest_Filter_TYPE_RESOURCE_ID, - Term: &ocm.ListOCMSharesRequest_Filter_ResourceId{ + ocmfilters = append(ocmfilters, &ocmpb.ListOCMSharesRequest_Filter{ + Type: ocmpb.ListOCMSharesRequest_Filter_TYPE_RESOURCE_ID, + Term: &ocmpb.ListOCMSharesRequest_Filter_ResourceId{ ResourceId: v.ResourceId, }, }) case *collaboration.Filter_Creator: - ocmfilters = append(ocmfilters, &ocm.ListOCMSharesRequest_Filter{ - Type: ocm.ListOCMSharesRequest_Filter_TYPE_CREATOR, - Term: &ocm.ListOCMSharesRequest_Filter_Creator{ + ocmfilters = append(ocmfilters, &ocmpb.ListOCMSharesRequest_Filter{ + Type: ocmpb.ListOCMSharesRequest_Filter_TYPE_CREATOR, + Term: &ocmpb.ListOCMSharesRequest_Filter_Creator{ Creator: v.Creator, }, }) case *collaboration.Filter_Owner: - ocmfilters = append(ocmfilters, &ocm.ListOCMSharesRequest_Filter{ - Type: ocm.ListOCMSharesRequest_Filter_TYPE_OWNER, - Term: &ocm.ListOCMSharesRequest_Filter_Owner{ + ocmfilters = append(ocmfilters, &ocmpb.ListOCMSharesRequest_Filter{ + Type: ocmpb.ListOCMSharesRequest_Filter_TYPE_OWNER, + Term: &ocmpb.ListOCMSharesRequest_Filter_Owner{ Owner: v.Owner, }, }) diff --git a/internal/http/services/sciencemesh/sciencemesh.go b/internal/http/services/sciencemesh/sciencemesh.go index 6c0265e324..14aafc6619 100644 --- a/internal/http/services/sciencemesh/sciencemesh.go +++ b/internal/http/services/sciencemesh/sciencemesh.go @@ -115,6 +115,7 @@ func (s *svc) routerInit() error { s.router.Get("/list-invite", tokenHandler.ListInvite) s.router.Post("/accept-invite", tokenHandler.AcceptInvite) s.router.Get("/find-accepted-users", tokenHandler.FindAccepted) + s.router.Delete("/delete-accepted-user", tokenHandler.DeleteAccepted) s.router.Get("/list-providers", providersHandler.ListProviders) s.router.Post("/create-share", sharesHandler.CreateShare) s.router.Post("/open-in-app", appsHandler.OpenInApp) diff --git a/internal/http/services/sciencemesh/token.go b/internal/http/services/sciencemesh/token.go index 3308a4c978..06d9cc382d 100644 --- a/internal/http/services/sciencemesh/token.go +++ b/internal/http/services/sciencemesh/token.go @@ -26,6 +26,7 @@ import ( "net/http" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" invitepb "github.com/cs3org/go-cs3apis/cs3/ocm/invite/v1beta1" ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" @@ -231,6 +232,52 @@ func (h *tokenHandler) FindAccepted(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } +// DeleteAccepted deletes the given user from the list of the accepted users. +func (h *tokenHandler) DeleteAccepted(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + req, err := getDeleteAcceptedRequest(r) + if err != nil { + reqres.WriteError(w, r, reqres.APIErrorInvalidParameter, "missing parameters in request", err) + return + } + + res, err := h.gatewayClient.DeleteAcceptedUser(ctx, &invitepb.DeleteAcceptedUserRequest{ + RemoteUserId: &userpb.UserId{ + Idp: req.Idp, + OpaqueId: req.UserID, + Type: userpb.UserType_USER_TYPE_FEDERATED, + }, + }) + if err != nil { + reqres.WriteError(w, r, reqres.APIErrorServerError, "error sending a grpc get invite by domain info request", err) + return + } + if res.Status.Code != rpc.Code_CODE_OK { + reqres.WriteError(w, r, reqres.APIErrorServerError, "grpc forward invite request failed", errors.New(res.Status.Message)) + return + } + w.WriteHeader(http.StatusOK) +} + +type deleteAcceptedRequest struct { + Idp string `json:"idp"` + UserID string `json:"user_id"` +} + +func getDeleteAcceptedRequest(r *http.Request) (*deleteAcceptedRequest, error) { + var req deleteAcceptedRequest + contentType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err == nil && contentType == "application/json" { + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + return nil, err + } + } else { + req.Idp, req.UserID = r.FormValue("idp"), r.FormValue("user_id") + } + return &req, nil +} + func (h *tokenHandler) ListInvite(w http.ResponseWriter, r *http.Request) { ctx := r.Context() diff --git a/pkg/ocm/invite/invite.go b/pkg/ocm/invite/invite.go index 777f358920..80792c7863 100644 --- a/pkg/ocm/invite/invite.go +++ b/pkg/ocm/invite/invite.go @@ -45,6 +45,9 @@ type Repository interface { // FindRemoteUsers finds remote users who have accepted invites based on their attributes. FindRemoteUsers(ctx context.Context, initiator *userpb.UserId, query string) ([]*userpb.User, error) + + // DeleteRemoteUser removes from the remote user from the initiator's list. + DeleteRemoteUser(ctx context.Context, initiator *userpb.UserId, remoteUser *userpb.UserId) error } // ErrTokenNotFound is the error returned when the token does not exist. diff --git a/pkg/ocm/invite/repository/json/json.go b/pkg/ocm/invite/repository/json/json.go index 2101229229..602506e2ff 100644 --- a/pkg/ocm/invite/repository/json/json.go +++ b/pkg/ocm/invite/repository/json/json.go @@ -34,6 +34,7 @@ import ( "github.com/cs3org/reva/pkg/ocm/invite" "github.com/cs3org/reva/pkg/ocm/invite/repository/registry" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/list" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -239,3 +240,23 @@ func userContains(u *userpb.User, query string) bool { return strings.Contains(strings.ToLower(u.Username), query) || strings.Contains(strings.ToLower(u.DisplayName), query) || strings.Contains(strings.ToLower(u.Mail), query) || strings.Contains(strings.ToLower(u.Id.OpaqueId), query) } + +func (m *manager) DeleteRemoteUser(ctx context.Context, initiator *userpb.UserId, remoteUser *userpb.UserId) error { + m.Lock() + defer m.Unlock() + + acceptedUsers, ok := m.model.AcceptedUsers[initiator.GetOpaqueId()] + if !ok { + return nil + } + + for i, user := range acceptedUsers { + if (user.Id.GetOpaqueId() == remoteUser.OpaqueId) && (remoteUser.Idp == "" || user.Id.GetIdp() == remoteUser.Idp) { + acceptedUsers = list.Remove(acceptedUsers, i) + m.model.AcceptedUsers[initiator.GetOpaqueId()] = acceptedUsers + _ = m.model.save() + return nil + } + } + return nil +} diff --git a/pkg/ocm/invite/repository/memory/memory.go b/pkg/ocm/invite/repository/memory/memory.go index 34bbf6a403..41b98ad5b9 100644 --- a/pkg/ocm/invite/repository/memory/memory.go +++ b/pkg/ocm/invite/repository/memory/memory.go @@ -29,6 +29,7 @@ import ( "github.com/cs3org/reva/pkg/ocm/invite" "github.com/cs3org/reva/pkg/ocm/invite/repository/registry" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/list" ) func init() { @@ -127,3 +128,19 @@ func userContains(u *userpb.User, query string) bool { return strings.Contains(strings.ToLower(u.Username), query) || strings.Contains(strings.ToLower(u.DisplayName), query) || strings.Contains(strings.ToLower(u.Mail), query) || strings.Contains(strings.ToLower(u.Id.OpaqueId), query) } + +func (m *manager) DeleteRemoteUser(ctx context.Context, initiator *userpb.UserId, remoteUser *userpb.UserId) error { + usersList, ok := m.AcceptedUsers.Load(initiator) + if !ok { + return nil + } + + acceptedUsers := usersList.([]*userpb.User) + for i, user := range acceptedUsers { + if (user.Id.GetOpaqueId() == remoteUser.OpaqueId) && (remoteUser.Idp == "" || user.Id.GetIdp() == remoteUser.Idp) { + m.AcceptedUsers.Store(initiator, list.Remove(acceptedUsers, i)) + return nil + } + } + return nil +} diff --git a/pkg/ocm/invite/repository/sql/sql.go b/pkg/ocm/invite/repository/sql/sql.go index eccfe506a0..4fbaacf049 100644 --- a/pkg/ocm/invite/repository/sql/sql.go +++ b/pkg/ocm/invite/repository/sql/sql.go @@ -252,3 +252,9 @@ func (m *mgr) FindRemoteUsers(ctx context.Context, initiator *userpb.UserId, att return users, nil } + +func (m *mgr) DeleteRemoteUser(ctx context.Context, initiator *userpb.UserId, remoteUser *userpb.UserId) error { + query := "DELETE FROM ocm_remote_users WHERE initiator=? AND opaque_user_id=? AND idp=?" + _, err := m.db.ExecContext(ctx, query, conversions.FormatUserID(initiator), conversions.FormatUserID(remoteUser), remoteUser.Idp) + return err +} diff --git a/pkg/ocm/share/repository/json/json.go b/pkg/ocm/share/repository/json/json.go index 7621830036..96158c3c1e 100644 --- a/pkg/ocm/share/repository/json/json.go +++ b/pkg/ocm/share/repository/json/json.go @@ -383,7 +383,7 @@ func receivedShareEqual(ref *ocm.ShareReference, s *ocm.ReceivedShare) bool { return false } -func (m *mgr) UpdateShare(ctx context.Context, user *userpb.User, ref *ocm.ShareReference, p *ocm.SharePermissions) (*ocm.Share, error) { +func (m *mgr) UpdateShare(ctx context.Context, user *userpb.User, ref *ocm.ShareReference, f ...*ocm.UpdateOCMShareRequest_UpdateField) (*ocm.Share, error) { return nil, errtypes.NotSupported("not yet implemented") } diff --git a/pkg/ocm/share/repository/nextcloud/nextcloud.go b/pkg/ocm/share/repository/nextcloud/nextcloud.go index 801a75b51f..775f370a47 100644 --- a/pkg/ocm/share/repository/nextcloud/nextcloud.go +++ b/pkg/ocm/share/repository/nextcloud/nextcloud.go @@ -202,14 +202,13 @@ func (sm *Manager) DeleteShare(ctx context.Context, user *userpb.User, ref *ocm. } // UpdateShare updates the mode of the given share. -func (sm *Manager) UpdateShare(ctx context.Context, user *userpb.User, ref *ocm.ShareReference, p *ocm.SharePermissions) (*ocm.Share, error) { +func (sm *Manager) UpdateShare(ctx context.Context, user *userpb.User, ref *ocm.ShareReference, f ...*ocm.UpdateOCMShareRequest_UpdateField) (*ocm.Share, error) { type paramsObj struct { Ref *ocm.ShareReference `json:"ref"` P *ocm.SharePermissions `json:"p"` } bodyObj := ¶msObj{ Ref: ref, - P: p, } data, err := json.Marshal(bodyObj) if err != nil { diff --git a/pkg/ocm/share/repository/nextcloud/nextcloud_test.go b/pkg/ocm/share/repository/nextcloud/nextcloud_test.go index b0934d22ed..b77aeaecdf 100644 --- a/pkg/ocm/share/repository/nextcloud/nextcloud_test.go +++ b/pkg/ocm/share/repository/nextcloud/nextcloud_test.go @@ -315,81 +315,58 @@ var _ = Describe("Nextcloud", func() { }) // UpdateShare(ctx context.Context, ref *ocm.ShareReference, p *ocm.SharePermissions) (*ocm.Share, error) - Describe("UpdateShare", func() { - It("calls the UpdateShare endpoint", func() { - am, called, teardown := setUpNextcloudServer() - defer teardown() + // Describe("UpdateShare", func() { + // It("calls the UpdateShare endpoint", func() { + // am, called, teardown := setUpNextcloudServer() + // defer teardown() - share, err := am.UpdateShare(ctx, user, &ocm.ShareReference{ - Spec: &ocm.ShareReference_Id{ - Id: &ocm.ShareId{ - OpaqueId: "some-share-id", - }, - }, - }, - &ocm.SharePermissions{ - Permissions: &provider.ResourcePermissions{ - AddGrant: true, - CreateContainer: true, - Delete: true, - GetPath: true, - GetQuota: true, - InitiateFileDownload: true, - InitiateFileUpload: true, - ListGrants: true, - ListContainer: true, - ListFileVersions: true, - ListRecycle: true, - Move: true, - RemoveGrant: true, - PurgeRecycle: true, - RestoreFileVersion: true, - RestoreRecycleItem: true, - Stat: true, - UpdateGrant: true, - DenyGrant: true, - }, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(*share).To(Equal(ocm.Share{ - Id: &ocm.ShareId{}, - Grantee: &provider.Grantee{ - Id: &provider.Grantee_UserId{ - UserId: &userpb.UserId{ - Idp: "0.0.0.0:19000", - OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - Type: userpb.UserType_USER_TYPE_PRIMARY, - }, - }, - }, - Owner: &userpb.UserId{ - Idp: "0.0.0.0:19000", - OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - Type: userpb.UserType_USER_TYPE_PRIMARY, - }, - Creator: &userpb.UserId{ - Idp: "0.0.0.0:19000", - OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - Type: userpb.UserType_USER_TYPE_PRIMARY, - }, - Ctime: &types.Timestamp{ - Seconds: 1234567890, - Nanos: 0, - XXX_NoUnkeyedLiteral: struct{}{}, - XXX_unrecognized: nil, - XXX_sizecache: 0, - }, - Mtime: &types.Timestamp{ - Seconds: 1234567890, - Nanos: 0, - XXX_NoUnkeyedLiteral: struct{}{}, - XXX_unrecognized: nil, - XXX_sizecache: 0, - }, - })) - checkCalled(called, `POST /apps/sciencemesh/~tester/api/ocm/UpdateShare {"ref":{"Spec":{"Id":{"opaque_id":"some-share-id"}}},"p":{"permissions":{"add_grant":true,"create_container":true,"delete":true,"get_path":true,"get_quota":true,"initiate_file_download":true,"initiate_file_upload":true,"list_grants":true,"list_container":true,"list_file_versions":true,"list_recycle":true,"move":true,"remove_grant":true,"purge_recycle":true,"restore_file_version":true,"restore_recycle_item":true,"stat":true,"update_grant":true,"deny_grant":true}}}`) - }) - }) + // share, err := am.UpdateShare(ctx, user, &ocm.ShareReference{ + // Spec: &ocm.ShareReference_Id{ + // Id: &ocm.ShareId{ + // OpaqueId: "some-share-id", + // }, + // }, + // }) + // Expect(err).ToNot(HaveOccurred()) + // Expect(*share).To(Equal(ocm.Share{ + // Id: &ocm.ShareId{}, + // Grantee: &provider.Grantee{ + // Id: &provider.Grantee_UserId{ + // UserId: &userpb.UserId{ + // Idp: "0.0.0.0:19000", + // OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", + // Type: userpb.UserType_USER_TYPE_PRIMARY, + // }, + // }, + // }, + // Owner: &userpb.UserId{ + // Idp: "0.0.0.0:19000", + // OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", + // Type: userpb.UserType_USER_TYPE_PRIMARY, + // }, + // Creator: &userpb.UserId{ + // Idp: "0.0.0.0:19000", + // OpaqueId: "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", + // Type: userpb.UserType_USER_TYPE_PRIMARY, + // }, + // Ctime: &types.Timestamp{ + // Seconds: 1234567890, + // Nanos: 0, + // XXX_NoUnkeyedLiteral: struct{}{}, + // XXX_unrecognized: nil, + // XXX_sizecache: 0, + // }, + // Mtime: &types.Timestamp{ + // Seconds: 1234567890, + // Nanos: 0, + // XXX_NoUnkeyedLiteral: struct{}{}, + // XXX_unrecognized: nil, + // XXX_sizecache: 0, + // }, + // })) + // checkCalled(called, `POST /apps/sciencemesh/~tester/api/ocm/UpdateShare {"ref":{"Spec":{"Id":{"opaque_id":"some-share-id"}}},"p":{"permissions":{"add_grant":true,"create_container":true,"delete":true,"get_path":true,"get_quota":true,"initiate_file_download":true,"initiate_file_upload":true,"list_grants":true,"list_container":true,"list_file_versions":true,"list_recycle":true,"move":true,"remove_grant":true,"purge_recycle":true,"restore_file_version":true,"restore_recycle_item":true,"stat":true,"update_grant":true,"deny_grant":true}}}`) + // }) + // }) // ListShares(ctx context.Context, filters []*ocm.ListOCMSharesRequest_Filter) ([]*ocm.Share, error) Describe("ListShares", func() { diff --git a/pkg/ocm/share/repository/sql/sql.go b/pkg/ocm/share/repository/sql/sql.go index ca7e6a9597..ec26261d28 100644 --- a/pkg/ocm/share/repository/sql/sql.go +++ b/pkg/ocm/share/repository/sql/sql.go @@ -24,9 +24,11 @@ import ( "fmt" "strconv" "strings" + "time" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" ocm "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" + typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" "github.com/cs3org/reva/pkg/cbox/utils" "github.com/cs3org/reva/pkg/errtypes" @@ -48,6 +50,20 @@ func New(c map[string]interface{}) (share.Repository, error) { if err != nil { return nil, err } + return NewFromConfig(conf) +} + +type mgr struct { + c *config + db *sql.DB + now func() time.Time +} + +// NewFromConfig creates a Repository with a SQL driver using the given config. +func NewFromConfig(conf *config) (share.Repository, error) { + if conf.now == nil { + conf.now = time.Now + } db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s", conf.DBUsername, conf.DBPassword, conf.DBAddress, conf.DBName)) if err != nil { @@ -55,22 +71,20 @@ func New(c map[string]interface{}) (share.Repository, error) { } m := &mgr{ - c: conf, - db: db, + c: conf, + db: db, + now: conf.now, } return m, nil } -type mgr struct { - c *config - db *sql.DB -} - type config struct { DBUsername string `mapstructure:"db_username"` DBPassword string `mapstructure:"db_password"` DBAddress string `mapstructure:"db_address"` DBName string `mapstructure:"db_name"` + + now func() time.Time // set only from tests } func parseConfig(conf map[string]interface{}) (*config, error) { @@ -327,8 +341,94 @@ func (m *mgr) deleteByKey(ctx context.Context, user *userpb.User, key *ocm.Share } // UpdateShare updates the mode of the given share. -func (m *mgr) UpdateShare(ctx context.Context, user *userpb.User, ref *ocm.ShareReference, p *ocm.SharePermissions) (*ocm.Share, error) { - return nil, errtypes.NotSupported("not yet implemented") +func (m *mgr) UpdateShare(ctx context.Context, user *userpb.User, ref *ocm.ShareReference, f ...*ocm.UpdateOCMShareRequest_UpdateField) (*ocm.Share, error) { + switch { + case ref.GetId() != nil: + return m.updateShareByID(ctx, user, ref.GetId(), f...) + case ref.GetKey() != nil: + return m.updateShareByKey(ctx, user, ref.GetKey(), f...) + default: + return nil, errtypes.NotFound(ref.String()) + } +} + +func (m *mgr) queriesUpdatesOnShare(ctx context.Context, id *ocm.ShareId, f ...*ocm.UpdateOCMShareRequest_UpdateField) (string, []string, []any, [][]any, error) { + var qi strings.Builder + params := []any{} + + qe := []string{} + eparams := [][]any{} + + for _, field := range f { + switch u := field.Field.(type) { + case *ocm.UpdateOCMShareRequest_UpdateField_Expiration: + qi.WriteString("expiration=?") + params = append(params, u.Expiration.Seconds) + case *ocm.UpdateOCMShareRequest_UpdateField_AccessMethods: + // TODO: access method can be added or removed as well + // now they can only be updated + switch t := u.AccessMethods.Term.(type) { + case *ocm.AccessMethod_WebdavOptions: + q := "UPDATE ocm_access_method_webdav SET permissions=? WHERE ocm_access_method_id=(SELECT id FROM ocm_shares_access_methods WHERE ocm_share_id=? AND type=?)" + qe = append(qe, q) + eparams = append(eparams, []any{utils.SharePermToInt(t.WebdavOptions.Permissions), id.OpaqueId, WebDAVAccessMethod}) + case *ocm.AccessMethod_WebappOptions: + q := "UPDATE ocm_access_method_webapp SET view_mode=? WHERE ocm_access_method_id=(SELECT id FROM ocm_shares_access_methods WHERE ocm_share_id=? AND type=?)" + qe = append(qe, q) + eparams = append(eparams, []any{t.WebappOptions.ViewMode, id.OpaqueId, WebappAccessMethod}) + } + } + } + return qi.String(), qe, params, eparams, nil +} + +func (m *mgr) updateShareByID(ctx context.Context, user *userpb.User, id *ocm.ShareId, f ...*ocm.UpdateOCMShareRequest_UpdateField) (*ocm.Share, error) { + var query strings.Builder + + now := m.now().Unix() + query.WriteString("UPDATE ocm_shares SET ") + params := []any{} + + squery, am, sparams, paramsAm, err := m.queriesUpdatesOnShare(ctx, id, f...) + if err != nil { + return nil, err + } + + if squery != "" { + query.WriteString(squery) + query.WriteString(", ") + } + + query.WriteString("mtime=? WHERE id=? AND (initiator=? OR owner=?)") + params = append(params, sparams...) + params = append(params, now, id.OpaqueId, user.Id.OpaqueId, user.Id.OpaqueId) + + if err := transaction(ctx, m.db, func(tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, query.String(), params...); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return share.ErrShareNotFound + } + } + + for i, q := range am { + if _, err := tx.ExecContext(ctx, q, paramsAm[i]...); err != nil { + return err + } + } + return nil + }); err != nil { + return nil, err + } + + return m.getByID(ctx, user, id) +} + +func (m *mgr) updateShareByKey(ctx context.Context, user *userpb.User, key *ocm.ShareKey, f ...*ocm.UpdateOCMShareRequest_UpdateField) (*ocm.Share, error) { + share, err := m.getByKey(ctx, user, key) + if err != nil { + return nil, err + } + return m.updateShareByID(ctx, user, share.Id, f...) } func translateFilters(filters []*ocm.ListOCMSharesRequest_Filter) (string, []any, error) { @@ -680,6 +780,55 @@ func (m *mgr) getProtocols(ctx context.Context, id int) ([]*ocm.Protocol, error) } // UpdateReceivedShare updates the received share with share state. -func (m *mgr) UpdateReceivedShare(ctx context.Context, user *userpb.User, share *ocm.ReceivedShare, fieldMask *field_mask.FieldMask) (*ocm.ReceivedShare, error) { - return nil, errtypes.NotSupported("not yet implemented") +func (m *mgr) UpdateReceivedShare(ctx context.Context, user *userpb.User, s *ocm.ReceivedShare, fieldMask *field_mask.FieldMask) (*ocm.ReceivedShare, error) { + query := "UPDATE ocm_received_shares SET" + params := []any{} + + fquery, fparams, updatedShare, err := m.translateUpdateFieldMask(s, fieldMask) + if err != nil { + return nil, err + } + + query = fmt.Sprintf("%s %s WHERE id=?", query, fquery) + params = append(params, fparams...) + params = append(params, s.Id.OpaqueId) + + res, err := m.db.ExecContext(ctx, query, params...) + if err != nil { + return nil, err + } + if n, _ := res.RowsAffected(); n == 0 { + return nil, share.ErrShareNotFound + } + return updatedShare, nil +} + +func (m *mgr) translateUpdateFieldMask(share *ocm.ReceivedShare, fieldMask *field_mask.FieldMask) (string, []any, *ocm.ReceivedShare, error) { + var ( + query strings.Builder + params []any + ) + + newShare := *share + + for _, mask := range fieldMask.Paths { + switch mask { + case "state": + query.WriteString("state=?") + params = append(params, convertFromCS3OCMShareState(share.State)) + newShare.State = share.State + default: + return "", nil, nil, errtypes.NotSupported("updating " + mask + " is not supported") + } + query.WriteString(",") + } + + now := m.now().Unix() + query.WriteString("mtime=?") + params = append(params, now) + newShare.Mtime = &typesv1beta1.Timestamp{ + Seconds: uint64(now), + } + + return query.String(), params, &newShare, nil } diff --git a/pkg/ocm/share/repository/sql/sql_test.go b/pkg/ocm/share/repository/sql/sql_test.go index 846916142e..207bd82b35 100644 --- a/pkg/ocm/share/repository/sql/sql_test.go +++ b/pkg/ocm/share/repository/sql/sql_test.go @@ -25,6 +25,7 @@ import ( "strconv" "sync" "testing" + "time" appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" @@ -32,6 +33,8 @@ import ( providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" + "google.golang.org/genproto/protobuf/field_mask" + "google.golang.org/protobuf/types/known/fieldmaskpb" "github.com/cs3org/reva/pkg/ocm/share" sqle "github.com/dolthub/go-mysql-server" @@ -63,6 +66,7 @@ func startDatabase(ctx *sql.Context, tables map[string]*memory.Table) (engine *s defer m.Unlock() db := memory.NewDatabase(dbName) + db.EnablePrimaryKeyIndexes() for name, table := range tables { db.AddTable(name, table) } @@ -132,8 +136,9 @@ func createShareTables(ctx *sql.Context, initData []*ocm.Share) map[string]*memo var fkAccessMethods memory.ForeignKeyCollection fkAccessMethods.AddFK(sql.ForeignKeyConstraint{ Columns: []string{"ocm_share_id"}, - ParentTable: "ocm_shares", + ParentTable: ocmShareTable, ParentColumns: []string{"id"}, + OnDelete: sql.ForeignKeyReferentialAction_Cascade, }) accessMethods := memory.NewTable(ocmAccessMethodTable, sql.NewPrimaryKeySchema(sql.Schema{ {Name: "id", Type: sql.Int64, Nullable: false, Source: ocmAccessMethodTable, PrimaryKey: true, AutoIncrement: true}, @@ -152,6 +157,7 @@ func createShareTables(ctx *sql.Context, initData []*ocm.Share) map[string]*memo Columns: []string{"ocm_access_method_id"}, ParentTable: ocmAccessMethodTable, ParentColumns: []string{"id"}, + OnDelete: sql.ForeignKeyReferentialAction_Cascade, }) webdav := memory.NewTable(ocmAMWebDAVTable, sql.NewPrimaryKeySchema(sql.Schema{ @@ -220,6 +226,7 @@ func createReceivedShareTables(ctx *sql.Context, initData []*ocm.ReceivedShare) Columns: []string{"ocm_received_share_id"}, ParentTable: "ocm_received_shares", ParentColumns: []string{"id"}, + OnDelete: sql.ForeignKeyReferentialAction_Cascade, }) protocols := memory.NewTable(ocmReceivedProtocols, sql.NewPrimaryKeySchema(sql.Schema{ {Name: "id", Type: sql.Int64, Nullable: false, Source: ocmReceivedProtocols, PrimaryKey: true, AutoIncrement: true}, @@ -234,6 +241,7 @@ func createReceivedShareTables(ctx *sql.Context, initData []*ocm.ReceivedShare) Columns: []string{"ocm_protocol_id"}, ParentTable: ocmReceivedProtocols, ParentColumns: []string{"id"}, + OnDelete: sql.ForeignKeyReferentialAction_Cascade, }) webdav := memory.NewTable(ocmProtWebDAVTable, sql.NewPrimaryKeySchema(sql.Schema{ {Name: "ocm_protocol_id", Type: sql.Int64, Source: ocmProtWebDAVTable, PrimaryKey: true, AutoIncrement: true}, @@ -1261,6 +1269,280 @@ func TestStoreShare(t *testing.T) { } } +func TestUpdateShare(t *testing.T) { + fixedTime := time.Date(2023, time.December, 12, 12, 12, 0, 0, time.UTC) + + tests := []struct { + description string + init []*ocm.Share + user *userpb.User + ref *ocm.ShareReference + fields []*ocm.UpdateOCMShareRequest_UpdateField + err error + expected storeShareExpected + }{ + { + description: "update only expiration - by id", + init: []*ocm.Share{ + { + Id: &ocm.ShareId{OpaqueId: "10"}, + ResourceId: &providerv1beta1.ResourceId{StorageId: "storage", OpaqueId: "resource-id1"}, + Name: "file-name", + Token: "qwerty", + Grantee: &providerv1beta1.Grantee{Type: providerv1beta1.GranteeType_GRANTEE_TYPE_USER, Id: &providerv1beta1.Grantee_UserId{UserId: &userpb.UserId{Idp: "cesnet", OpaqueId: "richard", Type: userpb.UserType_USER_TYPE_FEDERATED}}}, + Owner: &userpb.UserId{OpaqueId: "einstein"}, + Creator: &userpb.UserId{OpaqueId: "marie"}, + Ctime: &typesv1beta1.Timestamp{Seconds: 1686061921}, + Mtime: &typesv1beta1.Timestamp{Seconds: 1686061921}, + ShareType: ocm.ShareType_SHARE_TYPE_USER, + AccessMethods: []*ocm.AccessMethod{ + share.NewWebDavAccessMethod(conversions.NewViewerRole().CS3ResourcePermissions()), + share.NewWebappAccessMethod(appprovider.ViewMode_VIEW_MODE_READ_ONLY), + }, + }, + }, + user: &userpb.User{Id: &userpb.UserId{OpaqueId: "marie"}}, + ref: &ocm.ShareReference{Spec: &ocm.ShareReference_Id{Id: &ocm.ShareId{OpaqueId: "10"}}}, + fields: []*ocm.UpdateOCMShareRequest_UpdateField{{Field: &ocm.UpdateOCMShareRequest_UpdateField_Expiration{Expiration: &typesv1beta1.Timestamp{Seconds: uint64(fixedTime.Unix())}}}}, + expected: storeShareExpected{ + shares: []sql.Row{{int64(10), "qwerty", "storage", "resource-id1", "file-name", "richard@cesnet", "einstein", "marie", uint64(1686061921), uint64(fixedTime.Unix()), uint64(fixedTime.Unix()), int8(0)}}, + accessmethods: []sql.Row{ + {int64(1), int64(10), int8(0)}, + {int64(2), int64(10), int8(1)}, + }, + webdav: []sql.Row{{int64(1), int64(1)}}, + webapp: []sql.Row{{int64(2), int8(2)}}, + }, + }, + { + description: "update access methods - by id", + init: []*ocm.Share{ + { + Id: &ocm.ShareId{OpaqueId: "10"}, + ResourceId: &providerv1beta1.ResourceId{StorageId: "storage", OpaqueId: "resource-id1"}, + Name: "file-name", + Token: "qwerty", + Grantee: &providerv1beta1.Grantee{Type: providerv1beta1.GranteeType_GRANTEE_TYPE_USER, Id: &providerv1beta1.Grantee_UserId{UserId: &userpb.UserId{Idp: "cesnet", OpaqueId: "richard", Type: userpb.UserType_USER_TYPE_FEDERATED}}}, + Owner: &userpb.UserId{OpaqueId: "einstein"}, + Creator: &userpb.UserId{OpaqueId: "marie"}, + Ctime: &typesv1beta1.Timestamp{Seconds: 1686061921}, + Mtime: &typesv1beta1.Timestamp{Seconds: 1686061921}, + ShareType: ocm.ShareType_SHARE_TYPE_USER, + AccessMethods: []*ocm.AccessMethod{ + share.NewWebDavAccessMethod(conversions.NewViewerRole().CS3ResourcePermissions()), + share.NewWebappAccessMethod(appprovider.ViewMode_VIEW_MODE_READ_ONLY), + }, + }, + }, + user: &userpb.User{Id: &userpb.UserId{OpaqueId: "marie"}}, + ref: &ocm.ShareReference{Spec: &ocm.ShareReference_Id{Id: &ocm.ShareId{OpaqueId: "10"}}}, + fields: []*ocm.UpdateOCMShareRequest_UpdateField{ + { + Field: &ocm.UpdateOCMShareRequest_UpdateField_AccessMethods{ + AccessMethods: share.NewWebDavAccessMethod(conversions.NewEditorRole().CS3ResourcePermissions()), + }, + }, + { + Field: &ocm.UpdateOCMShareRequest_UpdateField_AccessMethods{ + AccessMethods: share.NewWebappAccessMethod(appprovider.ViewMode_VIEW_MODE_READ_WRITE), + }, + }, + }, + expected: storeShareExpected{ + shares: []sql.Row{{int64(10), "qwerty", "storage", "resource-id1", "file-name", "richard@cesnet", "einstein", "marie", uint64(1686061921), uint64(fixedTime.Unix()), uint64(0), int8(0)}}, + accessmethods: []sql.Row{ + {int64(1), int64(10), int8(0)}, + {int64(2), int64(10), int8(1)}, + }, + webdav: []sql.Row{{int64(1), int64(15)}}, + webapp: []sql.Row{{int64(2), int8(3)}}, + }, + }, + { + description: "update only expiration - by key", + init: []*ocm.Share{ + { + Id: &ocm.ShareId{OpaqueId: "10"}, + ResourceId: &providerv1beta1.ResourceId{StorageId: "storage", OpaqueId: "resource-id1"}, + Name: "file-name", + Token: "qwerty", + Grantee: &providerv1beta1.Grantee{Type: providerv1beta1.GranteeType_GRANTEE_TYPE_USER, Id: &providerv1beta1.Grantee_UserId{UserId: &userpb.UserId{Idp: "cesnet", OpaqueId: "richard", Type: userpb.UserType_USER_TYPE_FEDERATED}}}, + Owner: &userpb.UserId{OpaqueId: "einstein"}, + Creator: &userpb.UserId{OpaqueId: "marie"}, + Ctime: &typesv1beta1.Timestamp{Seconds: 1686061921}, + Mtime: &typesv1beta1.Timestamp{Seconds: 1686061921}, + ShareType: ocm.ShareType_SHARE_TYPE_USER, + AccessMethods: []*ocm.AccessMethod{ + share.NewWebDavAccessMethod(conversions.NewViewerRole().CS3ResourcePermissions()), + share.NewWebappAccessMethod(appprovider.ViewMode_VIEW_MODE_READ_ONLY), + }, + }, + }, + user: &userpb.User{Id: &userpb.UserId{OpaqueId: "marie"}}, + ref: &ocm.ShareReference{Spec: &ocm.ShareReference_Key{Key: &ocm.ShareKey{ + Owner: &userpb.UserId{OpaqueId: "einstein"}, + ResourceId: &providerv1beta1.ResourceId{StorageId: "storage", OpaqueId: "resource-id1"}, + Grantee: &providerv1beta1.Grantee{Type: providerv1beta1.GranteeType_GRANTEE_TYPE_USER, Id: &providerv1beta1.Grantee_UserId{UserId: &userpb.UserId{Idp: "cesnet", OpaqueId: "richard", Type: userpb.UserType_USER_TYPE_FEDERATED}}}, + }}}, + fields: []*ocm.UpdateOCMShareRequest_UpdateField{{Field: &ocm.UpdateOCMShareRequest_UpdateField_Expiration{Expiration: &typesv1beta1.Timestamp{Seconds: uint64(fixedTime.Unix())}}}}, + expected: storeShareExpected{ + shares: []sql.Row{{int64(10), "qwerty", "storage", "resource-id1", "file-name", "richard@cesnet", "einstein", "marie", uint64(1686061921), uint64(fixedTime.Unix()), uint64(fixedTime.Unix()), int8(0)}}, + accessmethods: []sql.Row{ + {int64(1), int64(10), int8(0)}, + {int64(2), int64(10), int8(1)}, + }, + webdav: []sql.Row{{int64(1), int64(1)}}, + webapp: []sql.Row{{int64(2), int8(2)}}, + }, + }, + { + description: "update access methods - by key", + init: []*ocm.Share{ + { + Id: &ocm.ShareId{OpaqueId: "10"}, + ResourceId: &providerv1beta1.ResourceId{StorageId: "storage", OpaqueId: "resource-id1"}, + Name: "file-name", + Token: "qwerty", + Grantee: &providerv1beta1.Grantee{Type: providerv1beta1.GranteeType_GRANTEE_TYPE_USER, Id: &providerv1beta1.Grantee_UserId{UserId: &userpb.UserId{Idp: "cesnet", OpaqueId: "richard", Type: userpb.UserType_USER_TYPE_FEDERATED}}}, + Owner: &userpb.UserId{OpaqueId: "einstein"}, + Creator: &userpb.UserId{OpaqueId: "marie"}, + Ctime: &typesv1beta1.Timestamp{Seconds: 1686061921}, + Mtime: &typesv1beta1.Timestamp{Seconds: 1686061921}, + ShareType: ocm.ShareType_SHARE_TYPE_USER, + AccessMethods: []*ocm.AccessMethod{ + share.NewWebDavAccessMethod(conversions.NewViewerRole().CS3ResourcePermissions()), + share.NewWebappAccessMethod(appprovider.ViewMode_VIEW_MODE_READ_ONLY), + }, + }, + }, + user: &userpb.User{Id: &userpb.UserId{OpaqueId: "marie"}}, + ref: &ocm.ShareReference{Spec: &ocm.ShareReference_Key{Key: &ocm.ShareKey{ + Owner: &userpb.UserId{OpaqueId: "einstein"}, + ResourceId: &providerv1beta1.ResourceId{StorageId: "storage", OpaqueId: "resource-id1"}, + Grantee: &providerv1beta1.Grantee{Type: providerv1beta1.GranteeType_GRANTEE_TYPE_USER, Id: &providerv1beta1.Grantee_UserId{UserId: &userpb.UserId{Idp: "cesnet", OpaqueId: "richard", Type: userpb.UserType_USER_TYPE_FEDERATED}}}, + }}}, + fields: []*ocm.UpdateOCMShareRequest_UpdateField{ + { + Field: &ocm.UpdateOCMShareRequest_UpdateField_AccessMethods{ + AccessMethods: share.NewWebDavAccessMethod(conversions.NewEditorRole().CS3ResourcePermissions()), + }, + }, + { + Field: &ocm.UpdateOCMShareRequest_UpdateField_AccessMethods{ + AccessMethods: share.NewWebappAccessMethod(appprovider.ViewMode_VIEW_MODE_READ_WRITE), + }, + }, + }, + expected: storeShareExpected{ + shares: []sql.Row{{int64(10), "qwerty", "storage", "resource-id1", "file-name", "richard@cesnet", "einstein", "marie", uint64(1686061921), uint64(fixedTime.Unix()), uint64(0), int8(0)}}, + accessmethods: []sql.Row{ + {int64(1), int64(10), int8(0)}, + {int64(2), int64(10), int8(1)}, + }, + webdav: []sql.Row{{int64(1), int64(15)}}, + webapp: []sql.Row{{int64(2), int8(3)}}, + }, + }, + { + description: "update only expiration - id not exists", + init: []*ocm.Share{ + { + Id: &ocm.ShareId{OpaqueId: "10"}, + ResourceId: &providerv1beta1.ResourceId{StorageId: "storage", OpaqueId: "resource-id1"}, + Name: "file-name", + Token: "qwerty", + Grantee: &providerv1beta1.Grantee{Type: providerv1beta1.GranteeType_GRANTEE_TYPE_USER, Id: &providerv1beta1.Grantee_UserId{UserId: &userpb.UserId{Idp: "cesnet", OpaqueId: "richard", Type: userpb.UserType_USER_TYPE_FEDERATED}}}, + Owner: &userpb.UserId{OpaqueId: "einstein"}, + Creator: &userpb.UserId{OpaqueId: "marie"}, + Ctime: &typesv1beta1.Timestamp{Seconds: 1686061921}, + Mtime: &typesv1beta1.Timestamp{Seconds: 1686061921}, + ShareType: ocm.ShareType_SHARE_TYPE_USER, + AccessMethods: []*ocm.AccessMethod{ + share.NewWebDavAccessMethod(conversions.NewViewerRole().CS3ResourcePermissions()), + share.NewWebappAccessMethod(appprovider.ViewMode_VIEW_MODE_READ_ONLY), + }, + }, + }, + user: &userpb.User{Id: &userpb.UserId{OpaqueId: "marie"}}, + ref: &ocm.ShareReference{Spec: &ocm.ShareReference_Id{Id: &ocm.ShareId{OpaqueId: "not-existing-id"}}}, + fields: []*ocm.UpdateOCMShareRequest_UpdateField{{Field: &ocm.UpdateOCMShareRequest_UpdateField_Expiration{Expiration: &typesv1beta1.Timestamp{Seconds: uint64(fixedTime.Unix())}}}}, + err: share.ErrShareNotFound, + }, + { + description: "update access methods - key not exists", + init: []*ocm.Share{ + { + Id: &ocm.ShareId{OpaqueId: "10"}, + ResourceId: &providerv1beta1.ResourceId{StorageId: "storage", OpaqueId: "resource-id1"}, + Name: "file-name", + Token: "qwerty", + Grantee: &providerv1beta1.Grantee{Type: providerv1beta1.GranteeType_GRANTEE_TYPE_USER, Id: &providerv1beta1.Grantee_UserId{UserId: &userpb.UserId{Idp: "cesnet", OpaqueId: "richard", Type: userpb.UserType_USER_TYPE_FEDERATED}}}, + Owner: &userpb.UserId{OpaqueId: "einstein"}, + Creator: &userpb.UserId{OpaqueId: "marie"}, + Ctime: &typesv1beta1.Timestamp{Seconds: 1686061921}, + Mtime: &typesv1beta1.Timestamp{Seconds: 1686061921}, + ShareType: ocm.ShareType_SHARE_TYPE_USER, + AccessMethods: []*ocm.AccessMethod{ + share.NewWebDavAccessMethod(conversions.NewViewerRole().CS3ResourcePermissions()), + share.NewWebappAccessMethod(appprovider.ViewMode_VIEW_MODE_READ_ONLY), + }, + }, + }, + user: &userpb.User{Id: &userpb.UserId{OpaqueId: "marie"}}, + ref: &ocm.ShareReference{Spec: &ocm.ShareReference_Key{Key: &ocm.ShareKey{ + Owner: &userpb.UserId{OpaqueId: "non-existing-user"}, + ResourceId: &providerv1beta1.ResourceId{StorageId: "storage", OpaqueId: "resource-id1"}, + Grantee: &providerv1beta1.Grantee{Type: providerv1beta1.GranteeType_GRANTEE_TYPE_USER, Id: &providerv1beta1.Grantee_UserId{UserId: &userpb.UserId{Idp: "cesnet", OpaqueId: "richard", Type: userpb.UserType_USER_TYPE_FEDERATED}}}, + }}}, + fields: []*ocm.UpdateOCMShareRequest_UpdateField{ + { + Field: &ocm.UpdateOCMShareRequest_UpdateField_AccessMethods{ + AccessMethods: share.NewWebDavAccessMethod(conversions.NewEditorRole().CS3ResourcePermissions()), + }, + }, + { + Field: &ocm.UpdateOCMShareRequest_UpdateField_AccessMethods{ + AccessMethods: share.NewWebappAccessMethod(appprovider.ViewMode_VIEW_MODE_READ_WRITE), + }, + }, + }, + err: share.ErrShareNotFound, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + ctx := sql.NewEmptyContext() + tables := createShareTables(ctx, tt.init) + engine, port, cleanup := startDatabase(ctx, tables) + t.Cleanup(cleanup) + + r, err := NewFromConfig( + &config{ + DBUsername: "root", + DBPassword: "", + DBAddress: fmt.Sprintf("%s:%d", address, port), + DBName: dbName, + now: func() time.Time { return fixedTime }, + }, + ) + + if err != nil { + t.Fatalf("not expected error while creating share repository driver: %+v", err) + } + + _, err = r.UpdateShare(context.TODO(), tt.user, tt.ref, tt.fields...) + if err != tt.err { + t.Fatalf("not expected error updating share. got=%+v expected=%+v", err, tt.err) + } + + if tt.err == nil { + checkShares(ctx, engine, tt.expected, t) + } + }) + } +} + func TestGetReceivedShare(t *testing.T) { tests := []struct { description string @@ -1430,6 +1712,117 @@ func TestGetReceivedShare(t *testing.T) { } } +func TestUpdateReceivedShare(t *testing.T) { + fixedTime := time.Date(2024, 12, 12, 12, 12, 0, 0, time.UTC) + + tests := []struct { + description string + shares []*ocm.ReceivedShare + user *userpb.User + newShare *ocm.ReceivedShare + mask *field_mask.FieldMask + err error + expected storeReceivedShareExpected + }{ + { + description: "update existing share", + shares: []*ocm.ReceivedShare{ + { + Id: &ocm.ShareId{OpaqueId: "1"}, + RemoteShareId: "1-remote", + Name: "file-name", + Grantee: &providerv1beta1.Grantee{Type: providerv1beta1.GranteeType_GRANTEE_TYPE_USER, Id: &providerv1beta1.Grantee_UserId{UserId: &userpb.UserId{Idp: "cesnet", OpaqueId: "marie"}}}, + Owner: &userpb.UserId{Idp: "cernbox", OpaqueId: "einstein"}, + Creator: &userpb.UserId{Idp: "cernbox", OpaqueId: "einstein"}, + Ctime: &typesv1beta1.Timestamp{Seconds: 1670859468}, + Mtime: &typesv1beta1.Timestamp{Seconds: 1670859468}, + ShareType: ocm.ShareType_SHARE_TYPE_USER, + State: ocm.ShareState_SHARE_STATE_PENDING, + ResourceType: providerv1beta1.ResourceType_RESOURCE_TYPE_FILE, + Protocols: []*ocm.Protocol{ + share.NewWebDAVProtocol("webdav+https//cernbox.cern.ch/dav/ocm/1", "secret", &ocm.SharePermissions{ + Permissions: conversions.NewEditorRole().CS3ResourcePermissions(), + }), + }, + }, + }, + user: &userpb.User{Id: &userpb.UserId{Idp: "cernbox", OpaqueId: "einstein"}}, + newShare: &ocm.ReceivedShare{ + Id: &ocm.ShareId{OpaqueId: "1"}, + State: ocm.ShareState_SHARE_STATE_ACCEPTED, + }, + mask: &fieldmaskpb.FieldMask{Paths: []string{"state"}}, + expected: storeReceivedShareExpected{ + shares: []sql.Row{{int64(1), "file-name", "1-remote", int8(0), "marie", "einstein@cernbox", "einstein@cernbox", uint64(1670859468), uint64(fixedTime.Unix()), uint64(0), int8(ShareTypeUser), int8(ShareStateAccepted)}}, + protocols: []sql.Row{{int64(1), int64(1), int8(0)}}, + webdav: []sql.Row{{int64(1), "webdav+https//cernbox.cern.ch/dav/ocm/1", "secret", int64(15)}}, + webapp: []sql.Row{}, + transfer: []sql.Row{}, + }, + }, + { + description: "update non existing share", + shares: []*ocm.ReceivedShare{ + { + Id: &ocm.ShareId{OpaqueId: "1"}, + RemoteShareId: "1-remote", + Name: "file-name", + Grantee: &providerv1beta1.Grantee{Type: providerv1beta1.GranteeType_GRANTEE_TYPE_USER, Id: &providerv1beta1.Grantee_UserId{UserId: &userpb.UserId{Idp: "cesnet", OpaqueId: "marie"}}}, + Owner: &userpb.UserId{Idp: "cernbox", OpaqueId: "einstein"}, + Creator: &userpb.UserId{Idp: "cernbox", OpaqueId: "einstein"}, + Ctime: &typesv1beta1.Timestamp{Seconds: 1670859468}, + Mtime: &typesv1beta1.Timestamp{Seconds: 1670859468}, + ShareType: ocm.ShareType_SHARE_TYPE_USER, + State: ocm.ShareState_SHARE_STATE_PENDING, + ResourceType: providerv1beta1.ResourceType_RESOURCE_TYPE_FILE, + Protocols: []*ocm.Protocol{ + share.NewWebDAVProtocol("webdav+https//cernbox.cern.ch/dav/ocm/1", "secret", &ocm.SharePermissions{ + Permissions: conversions.NewEditorRole().CS3ResourcePermissions(), + }), + }, + }, + }, + user: &userpb.User{Id: &userpb.UserId{Idp: "cernbox", OpaqueId: "einstein"}}, + newShare: &ocm.ReceivedShare{ + Id: &ocm.ShareId{OpaqueId: "not-existing-share-id"}, + State: ocm.ShareState_SHARE_STATE_ACCEPTED, + }, + mask: &fieldmaskpb.FieldMask{Paths: []string{"state"}}, + err: share.ErrShareNotFound, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + ctx := sql.NewEmptyContext() + tables := createReceivedShareTables(ctx, tt.shares) + engine, port, cleanup := startDatabase(ctx, tables) + t.Cleanup(cleanup) + + r, err := NewFromConfig(&config{ + DBUsername: "root", + DBPassword: "", + DBAddress: fmt.Sprintf("%s:%d", address, port), + DBName: dbName, + now: func() time.Time { return fixedTime }, + }) + + if err != nil { + t.Fatalf("not expected error while creating share repository driver: %+v", err) + } + + _, err = r.UpdateReceivedShare(context.TODO(), tt.user, tt.newShare, tt.mask) + if err != tt.err { + t.Fatalf("not expected error getting share. got=%+v expected=%+v", err, tt.err) + } + + if tt.err == nil { + checkReceivedShares(ctx, engine, tt.expected, t) + } + }) + } +} + func TestListReceviedShares(t *testing.T) { tests := []struct { description string diff --git a/pkg/ocm/share/share.go b/pkg/ocm/share/share.go index e49be2415f..7083c33fcc 100644 --- a/pkg/ocm/share/share.go +++ b/pkg/ocm/share/share.go @@ -40,7 +40,7 @@ type Repository interface { DeleteShare(ctx context.Context, user *userpb.User, ref *ocm.ShareReference) error // UpdateShare updates the mode of the given share. - UpdateShare(ctx context.Context, user *userpb.User, ref *ocm.ShareReference, p *ocm.SharePermissions) (*ocm.Share, error) + UpdateShare(ctx context.Context, user *userpb.User, ref *ocm.ShareReference, f ...*ocm.UpdateOCMShareRequest_UpdateField) (*ocm.Share, error) // ListShares returns the shares created by the user. If md is provided is not nil, // it returns only shares attached to the given resource. diff --git a/pkg/utils/list/list.go b/pkg/utils/list/list.go index ba83b2de86..2b1962ab2b 100644 --- a/pkg/utils/list/list.go +++ b/pkg/utils/list/list.go @@ -27,3 +27,10 @@ func Map[T, V any](l []T, f func(T) V) []V { } return m } + +// Remove removes the element in position i from the list. +// It does not preserve the order of the original slice. +func Remove[T any](l []T, i int) []T { + l[i] = l[len(l)-1] + return l[:len(l)-1] +} From 84ed9ad4c58545e7e7cf7f54bf20a1c49b49c1ce Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Thu, 8 Jun 2023 15:25:48 +0200 Subject: [PATCH 13/48] Make /sciencemesh/find-accepted-users response consistent with delete user parameters (#3958) * make /sciencemesh/find-accepted response consistent with delete users reponse * add changelog * fix intergration tests --- .../find-accepted-users-response.md | 4 ++ internal/http/services/sciencemesh/token.go | 19 ++++++++- tests/integration/grpc/ocm_invitation_test.go | 39 ++++++++++++++----- 3 files changed, 51 insertions(+), 11 deletions(-) create mode 100644 changelog/unreleased/find-accepted-users-response.md diff --git a/changelog/unreleased/find-accepted-users-response.md b/changelog/unreleased/find-accepted-users-response.md new file mode 100644 index 0000000000..19ee6a27d5 --- /dev/null +++ b/changelog/unreleased/find-accepted-users-response.md @@ -0,0 +1,4 @@ +Enhancement: Make `/sciencemesh/find-accepted-users` response +consistent with delete user parameters + +https://github.com/cs3org/reva/pull/3958 \ No newline at end of file diff --git a/internal/http/services/sciencemesh/token.go b/internal/http/services/sciencemesh/token.go index 06d9cc382d..4c519c038d 100644 --- a/internal/http/services/sciencemesh/token.go +++ b/internal/http/services/sciencemesh/token.go @@ -35,6 +35,7 @@ import ( ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/smtpclient" + "github.com/cs3org/reva/pkg/utils/list" ) type tokenHandler struct { @@ -212,6 +213,13 @@ func getAcceptInviteRequest(r *http.Request) (*acceptInviteRequest, error) { return &req, nil } +type remoteUser struct { + DisplayName string `json:"display_name"` + Idp string `json:"idp"` + UserID string `json:"user_id"` + Mail string `json:"mail"` +} + // FindAccepted returns the list of all the users that accepted the invitation // to the authenticated user. func (h *tokenHandler) FindAccepted(w http.ResponseWriter, r *http.Request) { @@ -223,7 +231,16 @@ func (h *tokenHandler) FindAccepted(w http.ResponseWriter, r *http.Request) { return } - if err := json.NewEncoder(w).Encode(res.AcceptedUsers); err != nil { + users := list.Map(res.AcceptedUsers, func(u *userpb.User) *remoteUser { + return &remoteUser{ + DisplayName: u.DisplayName, + Idp: u.Id.Idp, + UserID: u.Id.OpaqueId, + Mail: u.Mail, + } + }) + + if err := json.NewEncoder(w).Encode(users); err != nil { reqres.WriteError(w, r, reqres.APIErrorServerError, "error marshalling token data", err) return } diff --git a/tests/integration/grpc/ocm_invitation_test.go b/tests/integration/grpc/ocm_invitation_test.go index fbaec721f9..be47dcce89 100644 --- a/tests/integration/grpc/ocm_invitation_test.go +++ b/tests/integration/grpc/ocm_invitation_test.go @@ -38,6 +38,7 @@ import ( "github.com/cs3org/reva/pkg/token" jwt "github.com/cs3org/reva/pkg/token/manager/jwt" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/list" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "google.golang.org/grpc/metadata" @@ -335,7 +336,25 @@ var _ = Describe("ocm invitation workflow", func() { return res.StatusCode } - findAccepted := func(revaToken, domain string) ([]*userpb.User, int) { + type remoteUser struct { + DisplayName string `json:"display_name"` + Idp string `json:"idp"` + UserID string `json:"user_id"` + Mail string `json:"mail"` + } + + remoteToCs3User := func(u *remoteUser) *userpb.User { + return &userpb.User{ + Id: &userpb.UserId{ + Idp: u.Idp, + OpaqueId: u.UserID, + }, + DisplayName: u.DisplayName, + Mail: u.Mail, + } + } + + findAccepted := func(revaToken, domain string) ([]*remoteUser, int) { req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, fmt.Sprintf("http://%s/sciencemesh/find-accepted-users", domain), nil) Expect(err).ToNot(HaveOccurred()) req.Header.Set("x-access-token", revaToken) @@ -344,7 +363,7 @@ var _ = Describe("ocm invitation workflow", func() { Expect(err).ToNot(HaveOccurred()) defer res.Body.Close() - var users []*userpb.User + var users []*remoteUser _ = json.NewDecoder(res.Body).Decode(&users) return users, res.StatusCode } @@ -374,14 +393,14 @@ var _ = Describe("ocm invitation workflow", func() { It("complete the invitation workflow", func() { users, code := findAccepted(tknEinstein, cernboxURL) Expect(code).To(Equal(http.StatusOK)) - Expect(ocmUsersEqual(users, []*userpb.User{})).To(BeTrue()) + Expect(ocmUsersEqual(list.Map(users, remoteToCs3User), []*userpb.User{})).To(BeTrue()) code = acceptInvite(tknMarie, cesnetURL, "cernbox.cern.ch", token) Expect(code).To(Equal(http.StatusOK)) users, code = findAccepted(tknEinstein, cernboxURL) Expect(code).To(Equal(http.StatusOK)) - Expect(ocmUsersEqual(users, []*userpb.User{marie})).To(BeTrue()) + Expect(ocmUsersEqual(list.Map(users, remoteToCs3User), []*userpb.User{marie})).To(BeTrue()) }) }) @@ -402,14 +421,14 @@ var _ = Describe("ocm invitation workflow", func() { It("fails the invitation workflow", func() { users, code := findAccepted(tknEinstein, cernboxURL) Expect(code).To(Equal(http.StatusOK)) - Expect(ocmUsersEqual(users, []*userpb.User{marie})).To(BeTrue()) + Expect(ocmUsersEqual(list.Map(users, remoteToCs3User), []*userpb.User{marie})).To(BeTrue()) code = acceptInvite(tknMarie, cesnetURL, "cernbox.cern.ch", token) Expect(code).To(Equal(http.StatusConflict)) users, code = findAccepted(tknEinstein, cernboxURL) Expect(code).To(Equal(http.StatusOK)) - Expect(ocmUsersEqual(users, []*userpb.User{marie})).To(BeTrue()) + Expect(ocmUsersEqual(list.Map(users, remoteToCs3User), []*userpb.User{marie})).To(BeTrue()) }) }) @@ -436,14 +455,14 @@ var _ = Describe("ocm invitation workflow", func() { It("will not complete the invitation workflow", func() { users, code := findAccepted(tknEinstein, cernboxURL) Expect(code).To(Equal(http.StatusOK)) - Expect(ocmUsersEqual(users, []*userpb.User{})).To(BeTrue()) + Expect(ocmUsersEqual(list.Map(users, remoteToCs3User), []*userpb.User{})).To(BeTrue()) code = acceptInvite(tknMarie, cesnetURL, "cernbox.cern.ch", expiredToken.Token) Expect(code).To(Equal(http.StatusBadRequest)) users, code = findAccepted(tknEinstein, cernboxURL) Expect(code).To(Equal(http.StatusOK)) - Expect(ocmUsersEqual(users, []*userpb.User{})).To(BeTrue()) + Expect(ocmUsersEqual(list.Map(users, remoteToCs3User), []*userpb.User{})).To(BeTrue()) }) }) @@ -461,7 +480,7 @@ var _ = Describe("ocm invitation workflow", func() { It("succeeds", func() { users, code := findAccepted(tknEinstein, cernboxURL) Expect(code).To(Equal(http.StatusOK)) - Expect(ocmUsersEqual(users, []*userpb.User{})).To(BeTrue()) + Expect(ocmUsersEqual(list.Map(users, remoteToCs3User), []*userpb.User{})).To(BeTrue()) ocmToken, code := generateToken(tknEinstein, cernboxURL) Expect(code).To(Equal(http.StatusOK)) @@ -471,7 +490,7 @@ var _ = Describe("ocm invitation workflow", func() { users, code = findAccepted(tknEinstein, cernboxURL) Expect(code).To(Equal(http.StatusOK)) - Expect(ocmUsersEqual(users, []*userpb.User{marie})).To(BeTrue()) + Expect(ocmUsersEqual(list.Map(users, remoteToCs3User), []*userpb.User{marie})).To(BeTrue()) }) }) From 4a502bba27a28a7e4c1ecef7e7b2ee35c694afc5 Mon Sep 17 00:00:00 2001 From: Prajwol Amatya <83579989+PrajwolAmatya@users.noreply.github.com> Date: Mon, 12 Jun 2023 19:38:59 +0545 Subject: [PATCH 14/48] [tests-only] [full-ci] bump latest commit from ocis (#3957) * Bump ocis commit id * Fixed line numbers in expected failures * Bump ocis submodule * Adjust expected failures --------- Co-authored-by: Phil Davis --- .drone.env | 2 +- .../expected-failures-on-OCIS-storage.md | 227 +++++++++--------- .../expected-failures-on-S3NG-storage.md | 226 ++++++++--------- tests/ocis | 2 +- 4 files changed, 228 insertions(+), 229 deletions(-) diff --git a/.drone.env b/.drone.env index f3660b888d..949f0f8f56 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=7094891f4de381102b05c6503751dc85d82c0782 +APITESTS_COMMITID=fc686d8cd827fe7b2bea55894c781fb72ad24865 APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index eaa4a0fbeb..8da80f76ee 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -52,71 +52,71 @@ Synchronization features like etag propagation, setting mtime and locking files #### [Webdav LOCK operations](https://github.com/owncloud/ocis/issues/1284) - [coreApiWebdavLocks/exclusiveLocks.feature:22](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L22) -- [coreApiWebdavLocks/exclusiveLocks.feature:19](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L19) -- [coreApiWebdavLocks/exclusiveLocks.feature:20](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L20) -- [coreApiWebdavLocks/exclusiveLocks.feature:21](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L21) +- [coreApiWebdavLocks/exclusiveLocks.feature:23](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L23) +- [coreApiWebdavLocks/exclusiveLocks.feature:24](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L24) +- [coreApiWebdavLocks/exclusiveLocks.feature:25](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L25) - [coreApiWebdavLocks/exclusiveLocks.feature:49](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L49) -- [coreApiWebdavLocks/exclusiveLocks.feature:46](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L46) -- [coreApiWebdavLocks/exclusiveLocks.feature:47](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L47) -- [coreApiWebdavLocks/exclusiveLocks.feature:48](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L48) +- [coreApiWebdavLocks/exclusiveLocks.feature:50](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L50) +- [coreApiWebdavLocks/exclusiveLocks.feature:51](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L51) +- [coreApiWebdavLocks/exclusiveLocks.feature:52](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L52) - [coreApiWebdavLocks/exclusiveLocks.feature:76](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L76) -- [coreApiWebdavLocks/exclusiveLocks.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L73) -- [coreApiWebdavLocks/exclusiveLocks.feature:74](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L74) -- [coreApiWebdavLocks/exclusiveLocks.feature:75](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L75) +- [coreApiWebdavLocks/exclusiveLocks.feature:77](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L77) +- [coreApiWebdavLocks/exclusiveLocks.feature:78](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L78) +- [coreApiWebdavLocks/exclusiveLocks.feature:79](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L79) - [coreApiWebdavLocks/exclusiveLocks.feature:103](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L103) -- [coreApiWebdavLocks/exclusiveLocks.feature:100](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L100) -- [coreApiWebdavLocks/exclusiveLocks.feature:101](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L101) -- [coreApiWebdavLocks/exclusiveLocks.feature:102](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L102) -- [coreApiWebdavLocks/requestsWithToken.feature:29](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/requestsWithToken.feature#L29) -- [coreApiWebdavLocks/requestsWithToken.feature:30](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/requestsWithToken.feature#L30) -- [coreApiWebdavLocks2/independentLocks.feature:23](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L23) -- [coreApiWebdavLocks2/independentLocks.feature:24](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L24) +- [coreApiWebdavLocks/exclusiveLocks.feature:104](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L104) +- [coreApiWebdavLocks/exclusiveLocks.feature:105](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L105) +- [coreApiWebdavLocks/exclusiveLocks.feature:106](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L106) +- [coreApiWebdavLocks/requestsWithToken.feature:32](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/requestsWithToken.feature#L32) +- [coreApiWebdavLocks/requestsWithToken.feature:33](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/requestsWithToken.feature#L33) - [coreApiWebdavLocks2/independentLocks.feature:25](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L25) - [coreApiWebdavLocks2/independentLocks.feature:26](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L26) -- [coreApiWebdavLocks2/independentLocks.feature:51](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L51) -- [coreApiWebdavLocks2/independentLocks.feature:52](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L52) +- [coreApiWebdavLocks2/independentLocks.feature:27](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L27) +- [coreApiWebdavLocks2/independentLocks.feature:28](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L28) - [coreApiWebdavLocks2/independentLocks.feature:53](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L53) - [coreApiWebdavLocks2/independentLocks.feature:54](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L54) - [coreApiWebdavLocks2/independentLocks.feature:55](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L55) - [coreApiWebdavLocks2/independentLocks.feature:56](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L56) - [coreApiWebdavLocks2/independentLocks.feature:57](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L57) - [coreApiWebdavLocks2/independentLocks.feature:58](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L58) -- [coreApiWebdavLocks2/independentLocksShareToShares.feature:29](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L29) +- [coreApiWebdavLocks2/independentLocks.feature:59](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L59) +- [coreApiWebdavLocks2/independentLocks.feature:60](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L60) - [coreApiWebdavLocks2/independentLocksShareToShares.feature:30](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L30) - [coreApiWebdavLocks2/independentLocksShareToShares.feature:31](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L31) -- [coreApiWebdavLocks2/independentLocksShareToShares.feature:28](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L28) -- [coreApiWebdavLocks2/independentLocksShareToShares.feature:58](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L58) +- [coreApiWebdavLocks2/independentLocksShareToShares.feature:32](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L32) +- [coreApiWebdavLocks2/independentLocksShareToShares.feature:33](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L33) - [coreApiWebdavLocks2/independentLocksShareToShares.feature:59](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L59) - [coreApiWebdavLocks2/independentLocksShareToShares.feature:60](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L60) -- [coreApiWebdavLocks2/independentLocksShareToShares.feature:57](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L57) -- [coreApiWebdavLocksUnlock/unlock.feature:20](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L20) -- [coreApiWebdavLocksUnlock/unlock.feature:21](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L21) -- [coreApiWebdavLocksUnlock/unlock.feature:40](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L40) -- [coreApiWebdavLocksUnlock/unlock.feature:41](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L41) -- [coreApiWebdavLocksUnlock/unlock.feature:63](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L63) -- [coreApiWebdavLocksUnlock/unlock.feature:64](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L64) -- [coreApiWebdavLocksUnlock/unlock.feature:65](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L65) +- [coreApiWebdavLocks2/independentLocksShareToShares.feature:61](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L61) +- [coreApiWebdavLocks2/independentLocksShareToShares.feature:62](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L62) +- [coreApiWebdavLocksUnlock/unlock.feature:23](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L23) +- [coreApiWebdavLocksUnlock/unlock.feature:24](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L24) +- [coreApiWebdavLocksUnlock/unlock.feature:43](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L43) +- [coreApiWebdavLocksUnlock/unlock.feature:44](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L44) - [coreApiWebdavLocksUnlock/unlock.feature:66](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L66) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:26](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L26) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:27](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L27) +- [coreApiWebdavLocksUnlock/unlock.feature:67](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L67) +- [coreApiWebdavLocksUnlock/unlock.feature:68](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L68) +- [coreApiWebdavLocksUnlock/unlock.feature:69](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L69) - [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:28](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L28) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:25](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L25) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:50](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L50) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:51](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L51) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:29](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L29) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:30](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L30) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:31](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L31) - [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:52](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L52) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:49](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L49) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:74](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L74) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:75](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L75) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:53](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L53) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:54](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L54) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:55](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L55) - [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:76](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L76) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L73) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:98](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L98) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:99](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L99) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:77](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L77) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:78](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L78) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:79](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L79) - [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:100](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L100) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:97](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L97) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:122](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L122) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:123](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L123) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:101](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L101) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:102](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L102) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:103](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L103) - [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:124](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L124) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:121](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L121) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:125](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L125) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:126](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L126) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:127](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L127) ### Share File and sync features in a shared scenario @@ -248,12 +248,12 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt #### [deleting a file inside a received shared folder is moved to the trash-bin of the sharer not the receiver](https://github.com/owncloud/ocis/issues/1124) -- [coreApiTrashbin/trashbinSharingToShares.feature:44](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L44) -- [coreApiTrashbin/trashbinSharingToShares.feature:71](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L71) -- [coreApiTrashbin/trashbinSharingToShares.feature:98](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L98) -- [coreApiTrashbin/trashbinSharingToShares.feature:126](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L126) -- [coreApiTrashbin/trashbinSharingToShares.feature:154](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L154) -- [coreApiTrashbin/trashbinSharingToShares.feature:182](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L182) +- [coreApiTrashbin/trashbinSharingToShares.feature:47](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L47) +- [coreApiTrashbin/trashbinSharingToShares.feature:74](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L74) +- [coreApiTrashbin/trashbinSharingToShares.feature:101](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L101) +- [coreApiTrashbin/trashbinSharingToShares.feature:129](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L129) +- [coreApiTrashbin/trashbinSharingToShares.feature:157](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L157) +- [coreApiTrashbin/trashbinSharingToShares.feature:185](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L185) #### [Folder overwrite on shared files doesn't works correctly on copying file](https://github.com/owncloud/ocis/issues/2183) - [coreApiWebdavProperties1/copyFile.feature:465](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties1/copyFile.feature#L465) @@ -295,11 +295,11 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiWebdavMove2/moveFile.feature:177](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L177) #### [OCIS-storage overwriting a file as share receiver, does not create a new file version for the sharer](https://github.com/owncloud/ocis/issues/766) //todo -- [coreApiVersions/fileVersions.feature:276](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L276) +- [coreApiVersions/fileVersions.feature:278](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L278) #### [restoring an older version of a shared file deletes the share](https://github.com/owncloud/ocis/issues/765) - [coreApiShareManagementToShares/acceptShares.feature:448](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/acceptShares.feature#L448) -- [coreApiVersions/fileVersions.feature:288](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L288) +- [coreApiVersions/fileVersions.feature:290](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L290) #### [Expiration date for shares is not implemented](https://github.com/owncloud/ocis/issues/1250) #### Expiration date of user shares @@ -336,14 +336,14 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiShareUpdateToShares/updateShare.feature:244](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L244) #### [user can access version metadata of a received share before accepting it](https://github.com/owncloud/ocis/issues/760) -- [coreApiVersions/fileVersions.feature:313](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L313) +- [coreApiVersions/fileVersions.feature:315](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L315) #### [Share lists deleted user as 'user'](https://github.com/owncloud/ocis/issues/903) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:678](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L678) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:679](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L679) #### [OCIS-storage overwriting a file as share receiver, does not create a new file version for the sharer](https://github.com/owncloud/ocis/issues/766) -- [coreApiVersions/fileVersions.feature:433](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L433) //todo +- [coreApiVersions/fileVersions.feature:435](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L435) //todo #### [deleting a share with wrong authentication returns OCS status 996 / HTTP 500](https://github.com/owncloud/ocis/issues/1229) - [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:227](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L227) @@ -484,34 +484,34 @@ Not everything needs to be implemented for ocis. While the oc10 testsuite covers - [coreApiWebdavMove2/moveFileToBlacklistedName.feature:20](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFileToBlacklistedName.feature#L20) #### [PATCH request for TUS upload with wrong checksum gives incorrect response](https://github.com/owncloud/ocis/issues/1755) -- [coreApiWebdavUploadTUS/checksums.feature:84](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L84) -- [coreApiWebdavUploadTUS/checksums.feature:85](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L85) -- [coreApiWebdavUploadTUS/checksums.feature:86](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L86) - [coreApiWebdavUploadTUS/checksums.feature:87](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L87) -- [coreApiWebdavUploadTUS/checksums.feature:173](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L173) -- [coreApiWebdavUploadTUS/checksums.feature:174](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L174) -- [coreApiWebdavUploadTUS/checksums.feature:226](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L226) -- [coreApiWebdavUploadTUS/checksums.feature:227](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L227) -- [coreApiWebdavUploadTUS/checksums.feature:228](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L228) +- [coreApiWebdavUploadTUS/checksums.feature:88](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L88) +- [coreApiWebdavUploadTUS/checksums.feature:89](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L89) +- [coreApiWebdavUploadTUS/checksums.feature:90](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L90) +- [coreApiWebdavUploadTUS/checksums.feature:176](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L176) +- [coreApiWebdavUploadTUS/checksums.feature:177](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L177) - [coreApiWebdavUploadTUS/checksums.feature:229](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L229) -- [coreApiWebdavUploadTUS/checksums.feature:282](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L282) -- [coreApiWebdavUploadTUS/checksums.feature:283](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L283) -- [coreApiWebdavUploadTUS/checksums.feature:284](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L284) +- [coreApiWebdavUploadTUS/checksums.feature:230](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L230) +- [coreApiWebdavUploadTUS/checksums.feature:231](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L231) +- [coreApiWebdavUploadTUS/checksums.feature:232](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L232) - [coreApiWebdavUploadTUS/checksums.feature:285](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L285) -- [coreApiWebdavUploadTUS/optionsRequest.feature:8](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L8) -- [coreApiWebdavUploadTUS/optionsRequest.feature:23](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L23) -- [coreApiWebdavUploadTUS/optionsRequest.feature:38](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L38) -- [coreApiWebdavUploadTUS/optionsRequest.feature:53](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L53) -- [coreApiWebdavUploadTUS/uploadToShare.feature:175](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L175) -- [coreApiWebdavUploadTUS/uploadToShare.feature:174](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L174) -- [coreApiWebdavUploadTUS/uploadToShare.feature:194](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L194) -- [coreApiWebdavUploadTUS/uploadToShare.feature:193](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L193) -- [coreApiWebdavUploadTUS/uploadToShare.feature:213](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L213) -- [coreApiWebdavUploadTUS/uploadToShare.feature:212](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L212) -- [coreApiWebdavUploadTUS/uploadToShare.feature:252](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L252) -- [coreApiWebdavUploadTUS/uploadToShare.feature:251](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L251) -- [coreApiWebdavUploadTUS/uploadToShare.feature:294](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L294) -- [coreApiWebdavUploadTUS/uploadToShare.feature:293](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L293) +- [coreApiWebdavUploadTUS/checksums.feature:286](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L286) +- [coreApiWebdavUploadTUS/checksums.feature:287](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L287) +- [coreApiWebdavUploadTUS/checksums.feature:288](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L288) +- [coreApiWebdavUploadTUS/optionsRequest.feature:11](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L11) +- [coreApiWebdavUploadTUS/optionsRequest.feature:26](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L26) +- [coreApiWebdavUploadTUS/optionsRequest.feature:41](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L41) +- [coreApiWebdavUploadTUS/optionsRequest.feature:56](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L56) +- [coreApiWebdavUploadTUS/uploadToShare.feature:176](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L176) +- [coreApiWebdavUploadTUS/uploadToShare.feature:177](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L177) +- [coreApiWebdavUploadTUS/uploadToShare.feature:195](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L195) +- [coreApiWebdavUploadTUS/uploadToShare.feature:196](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L196) +- [coreApiWebdavUploadTUS/uploadToShare.feature:214](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L214) +- [coreApiWebdavUploadTUS/uploadToShare.feature:215](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L215) +- [coreApiWebdavUploadTUS/uploadToShare.feature:253](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L253) +- [coreApiWebdavUploadTUS/uploadToShare.feature:254](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L254) +- [coreApiWebdavUploadTUS/uploadToShare.feature:295](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L295) +- [coreApiWebdavUploadTUS/uploadToShare.feature:296](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L296) #### [Share inaccessible if folder with same name was deleted and recreated](https://github.com/owncloud/ocis/issues/1787) - [coreApiShareReshareToShares1/reShare.feature:267](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L267) @@ -580,22 +580,22 @@ _ocs: api compatibility, return correct status code_ - [coreApiWebdavProperties1/copyFile.feature:434](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties1/copyFile.feature#L434) #### [Downloading the older version of shared file gives 404](https://github.com/owncloud/ocis/issues/3868) -- [coreApiVersions/fileVersions.feature:160](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L160) -- [coreApiVersions/fileVersions.feature:178](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L178) -- [coreApiVersions/fileVersions.feature:445](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L445) +- [coreApiVersions/fileVersions.feature:162](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L162) +- [coreApiVersions/fileVersions.feature:180](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L180) +- [coreApiVersions/fileVersions.feature:447](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L447) #### [file versions do not report the version author](https://github.com/owncloud/ocis/issues/2914) -- [coreApiVersions/fileVersionAuthor.feature:12](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L12) -- [coreApiVersions/fileVersionAuthor.feature:43](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L43) -- [coreApiVersions/fileVersionAuthor.feature:70](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L70) -- [coreApiVersions/fileVersionAuthor.feature:96](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L96) -- [coreApiVersions/fileVersionAuthor.feature:129](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L129) -- [coreApiVersions/fileVersionAuthor.feature:156](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L156) -- [coreApiVersions/fileVersionAuthor.feature:187](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L187) -- [coreApiVersions/fileVersionAuthor.feature:222](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L222) -- [coreApiVersions/fileVersionAuthor.feature:274](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L274) -- [coreApiVersions/fileVersionAuthor.feature:323](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L323) -- [coreApiVersions/fileVersionAuthor.feature:344](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L344) +- [coreApiVersions/fileVersionAuthor.feature:15](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L15) +- [coreApiVersions/fileVersionAuthor.feature:46](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L46) +- [coreApiVersions/fileVersionAuthor.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L73) +- [coreApiVersions/fileVersionAuthor.feature:99](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L99) +- [coreApiVersions/fileVersionAuthor.feature:132](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L132) +- [coreApiVersions/fileVersionAuthor.feature:159](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L159) +- [coreApiVersions/fileVersionAuthor.feature:190](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L190) +- [coreApiVersions/fileVersionAuthor.feature:225](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L225) +- [coreApiVersions/fileVersionAuthor.feature:277](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L277) +- [coreApiVersions/fileVersionAuthor.feature:326](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L326) +- [coreApiVersions/fileVersionAuthor.feature:347](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L347) #### [Shares to deleted group listed in the response](https://github.com/owncloud/ocis/issues/2441) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:530](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L530) @@ -671,9 +671,9 @@ _ocs: api compatibility, return correct status code_ - [coreApiTrashbin/trashbinFilesFolders.feature:346](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L346) - [coreApiTrashbin/trashbinFilesFolders.feature:400](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L400) - [coreApiTrashbin/trashbinFilesFolders.feature:437](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L437) -- [coreApiTrashbin/trashbinSharingToShares.feature:22](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L22) -- [coreApiTrashbin/trashbinSharingToShares.feature:205](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L205) -- [coreApiTrashbin/trashbinSharingToShares.feature:229](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L229) +- [coreApiTrashbin/trashbinSharingToShares.feature:25](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L25) +- [coreApiTrashbin/trashbinSharingToShares.feature:208](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L208) +- [coreApiTrashbin/trashbinSharingToShares.feature:232](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L232) - [coreApiTrashbinRestore/trashbinRestore.feature:34](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L34) - [coreApiTrashbinRestore/trashbinRestore.feature:35](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L35) - [coreApiTrashbinRestore/trashbinRestore.feature:50](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L50) @@ -730,25 +730,25 @@ _ocs: api compatibility, return correct status code_ - [coreApiTrashbinRestore/trashbinRestore.feature:478](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L478) - [coreApiTrashbinRestore/trashbinRestore.feature:531](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L531) - [coreApiTrashbinRestore/trashbinRestore.feature:532](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L532) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:26](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L26) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:27](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L27) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:49](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L49) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:50](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L50) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:70](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L70) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:71](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L71) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:93](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L93) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:94](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L94) -- [coreApiVersions/fileVersions.feature:237](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L237) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:29](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L29) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:30](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L30) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:52](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L52) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:53](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L53) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L73) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:74](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L74) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:96](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L96) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:97](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L97) +- [coreApiVersions/fileVersions.feature:239](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L239) #### [`meta` requests have empty responses with master branch](https://github.com/cs3org/reva/issues/2897) -- [coreApiVersions/fileVersions.feature:197](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L197) -- [coreApiVersions/fileVersions.feature:203](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L203) -- [coreApiVersions/fileVersions.feature:210](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L210) -- [coreApiVersions/fileVersions.feature:218](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L218) -- [coreApiVersions/fileVersions.feature:231](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L231) -- [coreApiVersions/fileVersions.feature:232](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L232) +- [coreApiVersions/fileVersions.feature:199](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L199) +- [coreApiVersions/fileVersions.feature:205](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L205) +- [coreApiVersions/fileVersions.feature:212](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L212) +- [coreApiVersions/fileVersions.feature:220](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L220) - [coreApiVersions/fileVersions.feature:233](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L233) - [coreApiVersions/fileVersions.feature:234](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L234) +- [coreApiVersions/fileVersions.feature:235](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L235) +- [coreApiVersions/fileVersions.feature:236](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L236) #### [WebDAV MOVE with body returns 400 rather than 415](https://github.com/cs3org/reva/issues/3119) @@ -773,8 +773,8 @@ _ocs: api compatibility, return correct status code_ #### [PROPFIND on (password protected) public link returns invalid XML](https://github.com/owncloud/ocis/issues/39707) The problem has been fixed in reva edge branch but not in reva master -- [coreApiWebdavOperations/propfind.feature:61](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L61) -- [coreApiWebdavOperations/propfind.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L73) +- [coreApiWebdavOperations/propfind.feature:64](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L64) +- [coreApiWebdavOperations/propfind.feature:76](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L76) #### [Updating the role of a public link to internal gives returns 400] - [coreApiSharePublicLink3/updatePublicLinkShare.feature:483](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L483) @@ -786,7 +786,6 @@ The problem has been fixed in reva edge branch but not in reva master - [coreApiCapabilities/capabilities.feature:11](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiCapabilities/capabilities.feature#L11) - [coreApiCapabilities/capabilities.feature:136](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiCapabilities/capabilities.feature#L136) - [coreApiCapabilities/capabilities.feature:175](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiCapabilities/capabilities.feature#L175) -- [coreApiCapabilities/capabilities.feature:216](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiCapabilities/capabilities.feature#L216) #### [Sharing of project space root via public link does no longer work](https://github.com/owncloud/ocis/issues/6278) - [coreApiShareCreateSpecialToShares2/createShareDefaultFolderForReceivedShares.feature:23](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareDefaultFolderForReceivedShares.feature#L23) diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 7b49f59a59..3172a3be95 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -10,22 +10,22 @@ Basic file management like up and download, move, copy, properties, quota, trash - [coreApiTrashbin/trashbinFilesFolders.feature:268](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L268) ### [Downloading the older version of shared file gives 404](https://github.com/owncloud/ocis/issues/3868) -- [coreApiVersions/fileVersions.feature:160](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L160) -- [coreApiVersions/fileVersions.feature:178](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L178) -- [coreApiVersions/fileVersions.feature:445](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L445) +- [coreApiVersions/fileVersions.feature:162](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L162) +- [coreApiVersions/fileVersions.feature:180](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L180) +- [coreApiVersions/fileVersions.feature:447](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L447) #### [file versions do not report the version author](https://github.com/owncloud/ocis/issues/2914) -- [coreApiVersions/fileVersionAuthor.feature:12](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L12) -- [coreApiVersions/fileVersionAuthor.feature:43](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L43) -- [coreApiVersions/fileVersionAuthor.feature:70](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L70) -- [coreApiVersions/fileVersionAuthor.feature:96](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L96) -- [coreApiVersions/fileVersionAuthor.feature:129](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L129) -- [coreApiVersions/fileVersionAuthor.feature:156](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L156) -- [coreApiVersions/fileVersionAuthor.feature:187](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L187) -- [coreApiVersions/fileVersionAuthor.feature:222](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L222) -- [coreApiVersions/fileVersionAuthor.feature:274](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L274) -- [coreApiVersions/fileVersionAuthor.feature:323](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L323) -- [coreApiVersions/fileVersionAuthor.feature:344](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L344) +- [coreApiVersions/fileVersionAuthor.feature:15](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L15) +- [coreApiVersions/fileVersionAuthor.feature:46](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L46) +- [coreApiVersions/fileVersionAuthor.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L73) +- [coreApiVersions/fileVersionAuthor.feature:99](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L99) +- [coreApiVersions/fileVersionAuthor.feature:132](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L132) +- [coreApiVersions/fileVersionAuthor.feature:159](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L159) +- [coreApiVersions/fileVersionAuthor.feature:190](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L190) +- [coreApiVersions/fileVersionAuthor.feature:225](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L225) +- [coreApiVersions/fileVersionAuthor.feature:277](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L277) +- [coreApiVersions/fileVersionAuthor.feature:326](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L326) +- [coreApiVersions/fileVersionAuthor.feature:347](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L347) #### [Getting information about a folder overwritten by a file gives 500 error instead of 404](https://github.com/owncloud/ocis/issues/1239) - [coreApiWebdavProperties1/copyFile.feature:273](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties1/copyFile.feature#L273) @@ -68,71 +68,71 @@ Synchronization features like etag propagation, setting mtime and locking files #### [Webdav LOCK operations](https://github.com/owncloud/ocis/issues/1284) - [coreApiWebdavLocks/exclusiveLocks.feature:22](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L22) -- [coreApiWebdavLocks/exclusiveLocks.feature:19](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L19) -- [coreApiWebdavLocks/exclusiveLocks.feature:20](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L20) -- [coreApiWebdavLocks/exclusiveLocks.feature:21](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L21) +- [coreApiWebdavLocks/exclusiveLocks.feature:23](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L23) +- [coreApiWebdavLocks/exclusiveLocks.feature:24](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L24) +- [coreApiWebdavLocks/exclusiveLocks.feature:25](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L25) - [coreApiWebdavLocks/exclusiveLocks.feature:49](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L49) -- [coreApiWebdavLocks/exclusiveLocks.feature:46](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L46) -- [coreApiWebdavLocks/exclusiveLocks.feature:47](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L47) -- [coreApiWebdavLocks/exclusiveLocks.feature:48](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L48) +- [coreApiWebdavLocks/exclusiveLocks.feature:50](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L50) +- [coreApiWebdavLocks/exclusiveLocks.feature:51](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L51) +- [coreApiWebdavLocks/exclusiveLocks.feature:52](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L52) - [coreApiWebdavLocks/exclusiveLocks.feature:76](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L76) -- [coreApiWebdavLocks/exclusiveLocks.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L73) -- [coreApiWebdavLocks/exclusiveLocks.feature:74](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L74) -- [coreApiWebdavLocks/exclusiveLocks.feature:75](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L75) +- [coreApiWebdavLocks/exclusiveLocks.feature:77](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L77) +- [coreApiWebdavLocks/exclusiveLocks.feature:78](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L78) +- [coreApiWebdavLocks/exclusiveLocks.feature:79](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L79) - [coreApiWebdavLocks/exclusiveLocks.feature:103](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L103) -- [coreApiWebdavLocks/exclusiveLocks.feature:100](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L100) -- [coreApiWebdavLocks/exclusiveLocks.feature:101](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L101) -- [coreApiWebdavLocks/exclusiveLocks.feature:102](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L102) -- [coreApiWebdavLocks/requestsWithToken.feature:29](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/requestsWithToken.feature#L29) -- [coreApiWebdavLocks/requestsWithToken.feature:30](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/requestsWithToken.feature#L30) -- [coreApiWebdavLocks2/independentLocks.feature:23](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L23) -- [coreApiWebdavLocks2/independentLocks.feature:24](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L24) +- [coreApiWebdavLocks/exclusiveLocks.feature:104](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L104) +- [coreApiWebdavLocks/exclusiveLocks.feature:105](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L105) +- [coreApiWebdavLocks/exclusiveLocks.feature:106](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/exclusiveLocks.feature#L106) +- [coreApiWebdavLocks/requestsWithToken.feature:32](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/requestsWithToken.feature#L32) +- [coreApiWebdavLocks/requestsWithToken.feature:33](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks/requestsWithToken.feature#L33) - [coreApiWebdavLocks2/independentLocks.feature:25](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L25) - [coreApiWebdavLocks2/independentLocks.feature:26](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L26) -- [coreApiWebdavLocks2/independentLocks.feature:51](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L51) -- [coreApiWebdavLocks2/independentLocks.feature:52](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L52) +- [coreApiWebdavLocks2/independentLocks.feature:27](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L27) +- [coreApiWebdavLocks2/independentLocks.feature:28](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L28) - [coreApiWebdavLocks2/independentLocks.feature:53](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L53) - [coreApiWebdavLocks2/independentLocks.feature:54](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L54) - [coreApiWebdavLocks2/independentLocks.feature:55](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L55) - [coreApiWebdavLocks2/independentLocks.feature:56](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L56) - [coreApiWebdavLocks2/independentLocks.feature:57](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L57) - [coreApiWebdavLocks2/independentLocks.feature:58](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L58) -- [coreApiWebdavLocks2/independentLocksShareToShares.feature:29](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L29) +- [coreApiWebdavLocks2/independentLocks.feature:59](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L59) +- [coreApiWebdavLocks2/independentLocks.feature:60](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocks.feature#L60) - [coreApiWebdavLocks2/independentLocksShareToShares.feature:30](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L30) - [coreApiWebdavLocks2/independentLocksShareToShares.feature:31](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L31) -- [coreApiWebdavLocks2/independentLocksShareToShares.feature:28](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L28) -- [coreApiWebdavLocks2/independentLocksShareToShares.feature:58](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L58) +- [coreApiWebdavLocks2/independentLocksShareToShares.feature:32](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L32) +- [coreApiWebdavLocks2/independentLocksShareToShares.feature:33](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L33) - [coreApiWebdavLocks2/independentLocksShareToShares.feature:59](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L59) - [coreApiWebdavLocks2/independentLocksShareToShares.feature:60](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L60) -- [coreApiWebdavLocks2/independentLocksShareToShares.feature:57](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L57) -- [coreApiWebdavLocksUnlock/unlock.feature:20](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L20) -- [coreApiWebdavLocksUnlock/unlock.feature:21](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L21) -- [coreApiWebdavLocksUnlock/unlock.feature:40](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L40) -- [coreApiWebdavLocksUnlock/unlock.feature:41](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L41) -- [coreApiWebdavLocksUnlock/unlock.feature:63](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L63) -- [coreApiWebdavLocksUnlock/unlock.feature:64](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L64) -- [coreApiWebdavLocksUnlock/unlock.feature:65](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L65) +- [coreApiWebdavLocks2/independentLocksShareToShares.feature:61](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L61) +- [coreApiWebdavLocks2/independentLocksShareToShares.feature:62](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocks2/independentLocksShareToShares.feature#L62) +- [coreApiWebdavLocksUnlock/unlock.feature:23](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L23) +- [coreApiWebdavLocksUnlock/unlock.feature:24](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L24) +- [coreApiWebdavLocksUnlock/unlock.feature:43](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L43) +- [coreApiWebdavLocksUnlock/unlock.feature:44](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L44) - [coreApiWebdavLocksUnlock/unlock.feature:66](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L66) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:26](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L26) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:27](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L27) +- [coreApiWebdavLocksUnlock/unlock.feature:67](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L67) +- [coreApiWebdavLocksUnlock/unlock.feature:68](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L68) +- [coreApiWebdavLocksUnlock/unlock.feature:69](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlock.feature#L69) - [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:28](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L28) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:25](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L25) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:50](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L50) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:51](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L51) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:29](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L29) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:30](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L30) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:31](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L31) - [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:52](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L52) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:49](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L49) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:74](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L74) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:75](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L75) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:53](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L53) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:54](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L54) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:55](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L55) - [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:76](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L76) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L73) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:98](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L98) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:99](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L99) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:77](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L77) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:78](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L78) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:79](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L79) - [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:100](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L100) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:97](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L97) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:122](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L122) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:123](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L123) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:101](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L101) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:102](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L102) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:103](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L103) - [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:124](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L124) -- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:121](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L121) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:125](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L125) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:126](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L126) +- [coreApiWebdavLocksUnlock/unlockSharingToShares.feature:127](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavLocksUnlock/unlockSharingToShares.feature#L127) ### Share File and sync features in a shared scenario @@ -267,12 +267,12 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt #### [deleting a file inside a received shared folder is moved to the trash-bin of the sharer not the receiver](https://github.com/owncloud/ocis/issues/1124) -- [coreApiTrashbin/trashbinSharingToShares.feature:44](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L44) -- [coreApiTrashbin/trashbinSharingToShares.feature:71](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L71) -- [coreApiTrashbin/trashbinSharingToShares.feature:98](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L98) -- [coreApiTrashbin/trashbinSharingToShares.feature:126](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L126) -- [coreApiTrashbin/trashbinSharingToShares.feature:154](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L154) -- [coreApiTrashbin/trashbinSharingToShares.feature:182](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L182) +- [coreApiTrashbin/trashbinSharingToShares.feature:47](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L47) +- [coreApiTrashbin/trashbinSharingToShares.feature:74](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L74) +- [coreApiTrashbin/trashbinSharingToShares.feature:101](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L101) +- [coreApiTrashbin/trashbinSharingToShares.feature:129](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L129) +- [coreApiTrashbin/trashbinSharingToShares.feature:157](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L157) +- [coreApiTrashbin/trashbinSharingToShares.feature:185](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L185) #### [Folder overwrite on shared files doesn't works correctly on copying file](https://github.com/owncloud/ocis/issues/2183) - [coreApiWebdavProperties1/copyFile.feature:465](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties1/copyFile.feature#L465) @@ -316,11 +316,11 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiWebdavMove2/moveFile.feature:177](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L177) #### [OCIS-storage overwriting a file as share receiver, does not create a new file version for the sharer](https://github.com/owncloud/ocis/issues/766) //todo -- [coreApiVersions/fileVersions.feature:276](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L276) +- [coreApiVersions/fileVersions.feature:278](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L278) #### [restoring an older version of a shared file deletes the share](https://github.com/owncloud/ocis/issues/765) - [coreApiShareManagementToShares/acceptShares.feature:448](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/acceptShares.feature#L448) -- [coreApiVersions/fileVersions.feature:288](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L288) +- [coreApiVersions/fileVersions.feature:290](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L290) #### [Expiration date for shares is not implemented](https://github.com/owncloud/ocis/issues/1250) #### Expiration date of user shares @@ -358,14 +358,14 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiShareUpdateToShares/updateShare.feature:244](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L244) #### [user can access version metadata of a received share before accepting it](https://github.com/owncloud/ocis/issues/760) -- [coreApiVersions/fileVersions.feature:313](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L313) +- [coreApiVersions/fileVersions.feature:315](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L315) #### [Share lists deleted user as 'user'](https://github.com/owncloud/ocis/issues/903) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:678](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L678) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:679](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L679) #### [OCIS-storage overwriting a file as share receiver, does not create a new file version for the sharer](https://github.com/owncloud/ocis/issues/766) //todo -- [coreApiVersions/fileVersions.feature:433](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L433) +- [coreApiVersions/fileVersions.feature:435](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L435) #### [deleting a share with wrong authentication returns OCS status 996 / HTTP 500](https://github.com/owncloud/ocis/issues/1229) - [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:227](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L227) @@ -506,34 +506,34 @@ Not everything needs to be implemented for ocis. While the oc10 testsuite covers - [coreApiWebdavMove2/moveFileToBlacklistedName.feature:20](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFileToBlacklistedName.feature#L20) #### [PATCH request for TUS upload with wrong checksum gives incorrect response](https://github.com/owncloud/ocis/issues/1755) -- [coreApiWebdavUploadTUS/checksums.feature:84](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L84) -- [coreApiWebdavUploadTUS/checksums.feature:85](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L85) -- [coreApiWebdavUploadTUS/checksums.feature:86](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L86) - [coreApiWebdavUploadTUS/checksums.feature:87](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L87) -- [coreApiWebdavUploadTUS/checksums.feature:173](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L173) -- [coreApiWebdavUploadTUS/checksums.feature:174](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L174) -- [coreApiWebdavUploadTUS/checksums.feature:226](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L226) -- [coreApiWebdavUploadTUS/checksums.feature:227](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L227) -- [coreApiWebdavUploadTUS/checksums.feature:228](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L228) +- [coreApiWebdavUploadTUS/checksums.feature:88](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L88) +- [coreApiWebdavUploadTUS/checksums.feature:89](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L89) +- [coreApiWebdavUploadTUS/checksums.feature:90](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L90) +- [coreApiWebdavUploadTUS/checksums.feature:176](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L176) +- [coreApiWebdavUploadTUS/checksums.feature:177](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L177) - [coreApiWebdavUploadTUS/checksums.feature:229](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L229) -- [coreApiWebdavUploadTUS/checksums.feature:282](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L282) -- [coreApiWebdavUploadTUS/checksums.feature:283](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L283) -- [coreApiWebdavUploadTUS/checksums.feature:284](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L284) +- [coreApiWebdavUploadTUS/checksums.feature:230](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L230) +- [coreApiWebdavUploadTUS/checksums.feature:231](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L231) +- [coreApiWebdavUploadTUS/checksums.feature:232](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L232) - [coreApiWebdavUploadTUS/checksums.feature:285](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L285) -- [coreApiWebdavUploadTUS/optionsRequest.feature:8](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L8) -- [coreApiWebdavUploadTUS/optionsRequest.feature:23](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L23) -- [coreApiWebdavUploadTUS/optionsRequest.feature:38](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L38) -- [coreApiWebdavUploadTUS/optionsRequest.feature:53](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L53) -- [coreApiWebdavUploadTUS/uploadToShare.feature:175](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L175) -- [coreApiWebdavUploadTUS/uploadToShare.feature:174](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L174) -- [coreApiWebdavUploadTUS/uploadToShare.feature:194](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L194) -- [coreApiWebdavUploadTUS/uploadToShare.feature:193](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L193) -- [coreApiWebdavUploadTUS/uploadToShare.feature:213](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L213) -- [coreApiWebdavUploadTUS/uploadToShare.feature:212](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L212) -- [coreApiWebdavUploadTUS/uploadToShare.feature:252](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L252) -- [coreApiWebdavUploadTUS/uploadToShare.feature:251](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L251) -- [coreApiWebdavUploadTUS/uploadToShare.feature:294](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L294) -- [coreApiWebdavUploadTUS/uploadToShare.feature:293](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L293) +- [coreApiWebdavUploadTUS/checksums.feature:286](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L286) +- [coreApiWebdavUploadTUS/checksums.feature:287](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L287) +- [coreApiWebdavUploadTUS/checksums.feature:288](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/checksums.feature#L288) +- [coreApiWebdavUploadTUS/optionsRequest.feature:11](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L11) +- [coreApiWebdavUploadTUS/optionsRequest.feature:26](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L26) +- [coreApiWebdavUploadTUS/optionsRequest.feature:41](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L41) +- [coreApiWebdavUploadTUS/optionsRequest.feature:56](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/optionsRequest.feature#L56) +- [coreApiWebdavUploadTUS/uploadToShare.feature:176](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L176) +- [coreApiWebdavUploadTUS/uploadToShare.feature:177](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L177) +- [coreApiWebdavUploadTUS/uploadToShare.feature:195](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L195) +- [coreApiWebdavUploadTUS/uploadToShare.feature:196](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L196) +- [coreApiWebdavUploadTUS/uploadToShare.feature:214](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L214) +- [coreApiWebdavUploadTUS/uploadToShare.feature:215](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L215) +- [coreApiWebdavUploadTUS/uploadToShare.feature:253](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L253) +- [coreApiWebdavUploadTUS/uploadToShare.feature:254](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L254) +- [coreApiWebdavUploadTUS/uploadToShare.feature:295](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L295) +- [coreApiWebdavUploadTUS/uploadToShare.feature:296](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavUploadTUS/uploadToShare.feature#L296) #### [Share inaccessible if folder with same name was deleted and recreated](https://github.com/owncloud/ocis/issues/1787) - [coreApiShareReshareToShares1/reShare.feature:267](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares1/reShare.feature#L267) @@ -675,9 +675,9 @@ _ocs: api compatibility, return correct status code_ - [coreApiTrashbin/trashbinFilesFolders.feature:346](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L346) - [coreApiTrashbin/trashbinFilesFolders.feature:400](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L400) - [coreApiTrashbin/trashbinFilesFolders.feature:437](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L437) -- [coreApiTrashbin/trashbinSharingToShares.feature:22](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L22) -- [coreApiTrashbin/trashbinSharingToShares.feature:205](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L205) -- [coreApiTrashbin/trashbinSharingToShares.feature:229](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L229) +- [coreApiTrashbin/trashbinSharingToShares.feature:25](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L25) +- [coreApiTrashbin/trashbinSharingToShares.feature:208](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L208) +- [coreApiTrashbin/trashbinSharingToShares.feature:232](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L232) - [coreApiTrashbinRestore/trashbinRestore.feature:34](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L34) - [coreApiTrashbinRestore/trashbinRestore.feature:35](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L35) - [coreApiTrashbinRestore/trashbinRestore.feature:50](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L50) @@ -734,25 +734,25 @@ _ocs: api compatibility, return correct status code_ - [coreApiTrashbinRestore/trashbinRestore.feature:478](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L478) - [coreApiTrashbinRestore/trashbinRestore.feature:531](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L531) - [coreApiTrashbinRestore/trashbinRestore.feature:532](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbinRestore/trashbinRestore.feature#L532) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:26](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L26) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:27](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L27) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:49](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L49) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:50](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L50) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:70](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L70) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:71](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L71) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:93](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L93) -- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:94](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L94) -- [coreApiVersions/fileVersions.feature:237](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L237) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:29](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L29) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:30](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L30) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:52](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L52) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:53](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L53) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L73) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:74](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L74) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:96](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L96) +- [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:97](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L97) +- [coreApiVersions/fileVersions.feature:239](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L239) #### [`meta` requests have empty responses with master branch](https://github.com/cs3org/reva/issues/2897) -- [coreApiVersions/fileVersions.feature:197](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L197) -- [coreApiVersions/fileVersions.feature:203](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L203) -- [coreApiVersions/fileVersions.feature:210](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L210) -- [coreApiVersions/fileVersions.feature:218](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L218) -- [coreApiVersions/fileVersions.feature:231](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L231) -- [coreApiVersions/fileVersions.feature:232](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L232) +- [coreApiVersions/fileVersions.feature:199](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L199) +- [coreApiVersions/fileVersions.feature:205](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L205) +- [coreApiVersions/fileVersions.feature:212](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L212) +- [coreApiVersions/fileVersions.feature:220](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L220) - [coreApiVersions/fileVersions.feature:233](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L233) - [coreApiVersions/fileVersions.feature:234](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L234) +- [coreApiVersions/fileVersions.feature:235](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L235) +- [coreApiVersions/fileVersions.feature:236](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L236) #### [WebDAV MOVE with body returns 400 rather than 415](https://github.com/cs3org/reva/issues/3119) @@ -776,8 +776,8 @@ _ocs: api compatibility, return correct status code_ #### [PROPFIND on (password protected) public link returns invalid XML](https://github.com/owncloud/ocis/issues/39707) The problem has been fixed in reva edge branch but not in reva master -- [coreApiWebdavOperations/propfind.feature:61](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L61) -- [coreApiWebdavOperations/propfind.feature:73](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L73) +- [coreApiWebdavOperations/propfind.feature:64](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L64) +- [coreApiWebdavOperations/propfind.feature:76](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L76) #### [Updating the role of a public link to internal gives returns 400] - [coreApiSharePublicLink3/updatePublicLinkShare.feature:483](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L483) diff --git a/tests/ocis b/tests/ocis index 7094891f4d..fc686d8cd8 160000 --- a/tests/ocis +++ b/tests/ocis @@ -1 +1 @@ -Subproject commit 7094891f4de381102b05c6503751dc85d82c0782 +Subproject commit fc686d8cd827fe7b2bea55894c781fb72ad24865 From 43b46695f0b1748d6fecbe59e0d3debf8b9054e3 Mon Sep 17 00:00:00 2001 From: Javier Ferrer Date: Wed, 14 Jun 2023 10:32:09 +0200 Subject: [PATCH 15/48] pass if notifications are not configured (#3969) --- changelog/unreleased/notifications-skip-init.md | 6 ++++++ pkg/notification/db_changes.sql | 4 +--- .../notificationhelper/notificationhelper.go | 12 ++++++++---- pkg/notification/utils/nats.go | 10 ++++++++-- 4 files changed, 23 insertions(+), 9 deletions(-) create mode 100644 changelog/unreleased/notifications-skip-init.md diff --git a/changelog/unreleased/notifications-skip-init.md b/changelog/unreleased/notifications-skip-init.md new file mode 100644 index 0000000000..70d532fcb7 --- /dev/null +++ b/changelog/unreleased/notifications-skip-init.md @@ -0,0 +1,6 @@ +Enhancement: Conditional notifications initialization + +Notification helpers in services will not try to initalize if +there is no specific configuration. + +https://github.com/cs3org/reva/pull/3969 diff --git a/pkg/notification/db_changes.sql b/pkg/notification/db_changes.sql index 7b7be6ad03..a240b63300 100644 --- a/pkg/notification/db_changes.sql +++ b/pkg/notification/db_changes.sql @@ -19,8 +19,6 @@ -- This file can be used to make the required changes to the MySQL DB. This is -- not a proper migration but it should work on most situations. -USE cernboxngcopy; - CREATE TABLE `cbox_notifications` ( `id` INT PRIMARY KEY AUTO_INCREMENT, `ref` VARCHAR(3072) UNIQUE NOT NULL, @@ -45,7 +43,7 @@ CREATE INDEX `cbox_notifications_ix0` ON `cbox_notifications` (`ref`); CREATE INDEX `cbox_notification_recipients_ix0` ON `cbox_notification_recipients` (`notification_id`); CREATE INDEX `cbox_notification_recipients_ix1` ON `cbox_notification_recipients` (`user_name`); --- changes for added notifications on ocm shares +-- changes for added notifications on oc shares ALTER TABLE cernboxngcopy.oc_share ADD notify_uploads BOOL DEFAULT false; diff --git a/pkg/notification/notificationhelper/notificationhelper.go b/pkg/notification/notificationhelper/notificationhelper.go index af92e2f4d1..f9ef0420eb 100644 --- a/pkg/notification/notificationhelper/notificationhelper.go +++ b/pkg/notification/notificationhelper/notificationhelper.go @@ -58,11 +58,18 @@ func defaultConfig() *Config { // New creates a new Notification Helper. func New(name string, m map[string]interface{}, log *zerolog.Logger) *NotificationHelper { + annotatedLogger := log.With().Str("service", name).Str("scope", "notifications").Logger() + conf := defaultConfig() nh := &NotificationHelper{ Name: name, Conf: conf, - Log: log, + Log: &annotatedLogger, + } + + if len(m) == 0 { + log.Info().Msgf("no 'notifications' field in service config, notifications will be disabled") + return nh } if err := mapstructure.Decode(m, conf); err != nil { @@ -70,9 +77,6 @@ func New(name string, m map[string]interface{}, log *zerolog.Logger) *Notificati return nh } - annotatedLogger := log.With().Str("service", nh.Name).Str("scope", "notifications").Logger() - nh.Log = &annotatedLogger - if err := nh.connect(); err != nil { log.Error().Err(err).Msgf("connecting to nats failed, notifications will be disabled") return nh diff --git a/pkg/notification/utils/nats.go b/pkg/notification/utils/nats.go index cf206f46e8..7639a2e60f 100644 --- a/pkg/notification/utils/nats.go +++ b/pkg/notification/utils/nats.go @@ -38,10 +38,16 @@ func ConnectToNats(natsAddress, natsToken string, log zerolog.Logger) (*nats.Con log.Error().Err(err).Msgf("nats error") }), nats.ClosedHandler(func(c *nats.Conn) { - log.Error().Err(c.LastError()).Msgf("connection to nats server closed") + if c.LastError() != nil { + log.Error().Err(c.LastError()).Msgf("connection to nats server closed") + } else { + log.Debug().Msgf("connection to nats server closed") + } }), nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { - log.Error().Err(err).Msgf("connection to nats server disconnected") + if err != nil { + log.Error().Err(err).Msgf("connection to nats server disconnected") + } }), nats.CustomReconnectDelay(func(attempts int) time.Duration { if attempts%3 == 0 { From b3ef45ebeac864b764452c7afc99746beb531806 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 15 Jun 2023 15:57:49 +0200 Subject: [PATCH 16/48] Fixed panic (#3972) --- changelog/unreleased/patch-3971.md | 5 +++++ pkg/ocm/share/repository/nextcloud/nextcloud.go | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/patch-3971.md diff --git a/changelog/unreleased/patch-3971.md b/changelog/unreleased/patch-3971.md new file mode 100644 index 0000000000..cc7664cefe --- /dev/null +++ b/changelog/unreleased/patch-3971.md @@ -0,0 +1,5 @@ +Bugfix: Fix for #3971 + +Fixed panic described in #3971 + +https://github.com/cs3org/reva/pull/3972 diff --git a/pkg/ocm/share/repository/nextcloud/nextcloud.go b/pkg/ocm/share/repository/nextcloud/nextcloud.go index 775f370a47..76825d99e5 100644 --- a/pkg/ocm/share/repository/nextcloud/nextcloud.go +++ b/pkg/ocm/share/repository/nextcloud/nextcloud.go @@ -406,10 +406,10 @@ func (sm *Manager) UpdateReceivedShare(ctx context.Context, user *userpb.User, s } func getUsername(user *userpb.User) string { - if len(user.Username) > 0 { + if user != nil && len(user.Username) > 0 { return user.Username } - if len(user.Id.OpaqueId) > 0 { + if user != nil && len(user.Id.OpaqueId) > 0 { return user.Id.OpaqueId } From 3ed32c8025bd40fb779b3c169089919ce98f22a7 Mon Sep 17 00:00:00 2001 From: Swikriti Tripathi <41103328+SwikritiT@users.noreply.github.com> Date: Fri, 16 Jun 2023 10:26:37 +0545 Subject: [PATCH 17/48] Bump ocis commit id for tests (#3983) --- .drone.env | 2 +- tests/ocis | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.env b/.drone.env index 949f0f8f56..99536b7c5e 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=fc686d8cd827fe7b2bea55894c781fb72ad24865 +APITESTS_COMMITID=8c414acfbd8c4a0544923f95a47a0632303b1001 APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/tests/ocis b/tests/ocis index fc686d8cd8..8c414acfbd 160000 --- a/tests/ocis +++ b/tests/ocis @@ -1 +1 @@ -Subproject commit fc686d8cd827fe7b2bea55894c781fb72ad24865 +Subproject commit 8c414acfbd8c4a0544923f95a47a0632303b1001 From bbeecda3cab385d4ddf773e4eb44cf3514d5b1f6 Mon Sep 17 00:00:00 2001 From: Swikriti Tripathi <41103328+SwikritiT@users.noreply.github.com> Date: Tue, 20 Jun 2023 15:12:40 +0545 Subject: [PATCH 18/48] [tests-only][full-ci]Bump ocis commit id for tests (#3997) * Bump ocis commit id for tests * refactor expected to fail --- .drone.env | 2 +- tests/acceptance/expected-failures-on-OCIS-storage.md | 2 -- tests/acceptance/expected-failures-on-S3NG-storage.md | 2 -- tests/ocis | 2 +- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.drone.env b/.drone.env index 99536b7c5e..68253aef6e 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=8c414acfbd8c4a0544923f95a47a0632303b1001 +APITESTS_COMMITID=70b64d817e924ff7820e0f912b0569caf22fe9b8 APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index 8da80f76ee..910bca752a 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -130,8 +130,6 @@ File and sync features in a shared scenario - [coreApiSharees/sharees.feature:141](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharees/sharees.feature#L141) - [coreApiSharees/sharees.feature:160](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharees/sharees.feature#L160) - [coreApiSharees/sharees.feature:161](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharees/sharees.feature#L161) -- [coreApiSharees/sharees.feature:180](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharees/sharees.feature#L180) -- [coreApiSharees/sharees.feature:181](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharees/sharees.feature#L181) #### [accepting matching name shared resources from different users/groups sets no serial identifiers on the resource name for the receiver](https://github.com/owncloud/ocis/issues/4289) - [coreApiShareManagementToShares/acceptShares.feature:238](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/acceptShares.feature#L238) diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 3172a3be95..c898b9ca10 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -146,8 +146,6 @@ File and sync features in a shared scenario - [coreApiSharees/sharees.feature:141](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharees/sharees.feature#L141) - [coreApiSharees/sharees.feature:160](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharees/sharees.feature#L160) - [coreApiSharees/sharees.feature:161](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharees/sharees.feature#L161) -- [coreApiSharees/sharees.feature:180](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharees/sharees.feature#L180) -- [coreApiSharees/sharees.feature:181](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharees/sharees.feature#L181) #### [accepting matching name shared resources from different users/groups sets no serial identifiers on the resource name for the receiver](https://github.com/owncloud/ocis/issues/4289) - [coreApiShareManagementToShares/acceptShares.feature:238](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/acceptShares.feature#L238) diff --git a/tests/ocis b/tests/ocis index 8c414acfbd..70b64d817e 160000 --- a/tests/ocis +++ b/tests/ocis @@ -1 +1 @@ -Subproject commit 8c414acfbd8c4a0544923f95a47a0632303b1001 +Subproject commit 70b64d817e924ff7820e0f912b0569caf22fe9b8 From fe59f647adc212c677f79834d5d7cf244f9a4446 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Tue, 20 Jun 2023 16:45:05 +0200 Subject: [PATCH 19/48] OCM-related compatibility fixes (#3962) * Use paths as opposed to full URIs in /ocm-provider * Removed superfluous route * Added minimal /ocs-provider * Updated reference examples * Also added the whole ownCloud structure to /ocs-provider Only the `share` and `webdav` endpoints are populated * Updated examples to reflect current master wrt OCM, sciencemesh, and apps * Added hack for Nextcloud * Added skeleton /ocm/notifications service * wopi: added logging for apps over OCM + fixed lookup * Updated changelog * Removed /ocs-provider http service This service was introduced to test Nextcloud but it's not needed * Addressed comment --- changelog/unreleased/ocm-compat.md | 7 ++ examples/meshdirectory/meshdirectory.toml | 2 +- examples/nextcloud-integration/revad.toml | 56 ++++----- examples/oc-phoenix/ocmd.toml | 5 +- examples/ocmd/server-1/ocmd-server-1.toml | 37 +++++- examples/ocmd/server-2/ocmd-server-2.toml | 35 ++++++ examples/oidc-mapping-tpc/server-1.toml | 1 - examples/oidc-mapping-tpc/server-2.toml | 1 - examples/storage-references/gateway.toml | 15 ++- examples/two-server-setup/gateway-1.toml | 45 ++++++++ examples/two-server-setup/gateway-2.toml | 31 +++++ internal/http/services/ocmd/notifications.go | 106 ++++++++++++++++++ internal/http/services/ocmd/ocm.go | 7 +- .../http/services/ocmprovider/ocmprovider.go | 22 +++- pkg/app/provider/wopi/wopi.go | 27 ++++- 15 files changed, 348 insertions(+), 49 deletions(-) create mode 100644 changelog/unreleased/ocm-compat.md create mode 100644 internal/http/services/ocmd/notifications.go diff --git a/changelog/unreleased/ocm-compat.md b/changelog/unreleased/ocm-compat.md new file mode 100644 index 0000000000..4198ad3563 --- /dev/null +++ b/changelog/unreleased/ocm-compat.md @@ -0,0 +1,7 @@ +Bugfix: OCM-related compatibility fixes + +Following analysis of OC and NC code to access a remote share, +we must expose paths and not full URIs on the /ocm-provider endpoint. +Also we fix a lookup issue with apps over OCM and update examples. + +https://github.com/cs3org/reva/pull/3962 diff --git a/examples/meshdirectory/meshdirectory.toml b/examples/meshdirectory/meshdirectory.toml index 9711044416..1bea8c36e0 100644 --- a/examples/meshdirectory/meshdirectory.toml +++ b/examples/meshdirectory/meshdirectory.toml @@ -65,4 +65,4 @@ timeout = 10 refresh = 900 [http.services.meshdirectory] -mesh_directory_url = 'http://localhost:19001/meshdir/' \ No newline at end of file +mesh_directory_url = 'http://localhost:19001/meshdir/' diff --git a/examples/nextcloud-integration/revad.toml b/examples/nextcloud-integration/revad.toml index b37faacdc8..97288f5cf8 100644 --- a/examples/nextcloud-integration/revad.toml +++ b/examples/nextcloud-integration/revad.toml @@ -7,7 +7,7 @@ address = "0.0.0.0:19000" [grpc.services.gateway] authregistrysvc = "localhost:19000" appprovidersvc = "localhost:19000" -appregistry = "localhost:19000" +appregistrysvc = "localhost:19000" storageregistrysvc = "localhost:19000" preferencessvc = "localhost:19000" userprovidersvc = "localhost:19000" @@ -21,6 +21,29 @@ commit_share_to_storage_grant = false datagateway = "http://localhost:19001/data" transfer_expires = 6 # give it a moment +[grpc.services.appregistry] +driver = "static" + +[grpc.services.appregistry.drivers.static] +mime_types = [ + {"mime_type" = "text/plain", "extension" = "txt", "name" = "Text file", "description" = "Text file", "allow_creation" = true}, + {"mime_type" = "text/markdown", "extension" = "md", "name" = "Markdown file", "description" = "Markdown file", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.text", "extension" = "odt", "name" = "OpenDocument", "description" = "OpenDocument text document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.spreadsheet", "extension" = "ods", "name" = "OpenSpreadsheet", "description" = "OpenDocument spreadsheet document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.presentation", "extension" = "odp", "name" = "OpenPresentation", "description" = "OpenDocument presentation document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.jupyter", "extension" = "ipynb", "name" = "Jupyter Notebook", "description" = "Jupyter Notebook"} +] + +[grpc.services.appprovider] +driver = "wopi" +app_provider_url = "localhost:19000" + +[grpc.services.appprovider.drivers.wopi] +iop_secret = "shared-secret-with-wopiserver" +wopi_url = "http://0.0.0.0:8880/" +app_name = "Collabora" +app_url = "https://your-collabora-server.org:9980" + [grpc.services.authregistry] driver = "static" @@ -65,27 +88,6 @@ providers = "/etc/revad/providers.json" [grpc.services.publicshareprovider] driver = "memory" -[grpc.services.appprovider] -driver = "wopi" -custom_mime_types_json = "custom-mime-types-demo.json" - -[grpc.services.appprovider.drivers.wopi] -iop_secret = "hello" -wopi_url = "http://0.0.0.0:8880/" -app_name = "Collabora" -app_url = "https://your-collabora-server.org:9980" - -[grpc.services.appregistry] -driver = "static" - -[grpc.services.appregistry.static.rules] -"text/plain" = "localhost:19000" -"text/markdown" = "localhost:19000" -"application/compressed-markdown" = "localhost:19000" -"application/vnd.oasis.opendocument.text" = "localhost:19000" -"application/vnd.oasis.opendocument.spreadsheet" = "localhost:19000" -"application/vnd.oasis.opendocument.presentation" = "localhost:19000" - [grpc.services.storageprovider] driver = "nextcloud" mount_path = "/home" @@ -100,7 +102,6 @@ custom_mime_types_json = "custom-mime-types-demo.json" endpoint = "http://localhost/apps/sciencemesh/" user_layout = "{{.Username}}" - [grpc.services.authprovider] auth_manager = "nextcloud" [grpc.services.authprovider.drivers.nextcloud] @@ -112,10 +113,10 @@ driver = "nextcloud" endpoint = "http://localhost/apps/sciencemesh/" [http] -enabled_services = ["ocmd", "ocmprovider"] -enabled_middlewares = ["providerauthorizer", "cors"] address = "0.0.0.0:19001" +[http.services.appprovider] + [http.services.dataprovider] driver = "nextcloud" @@ -134,6 +135,11 @@ webdav_root = "/remote.php/dav/ocm" enable_webapp = true enable_datatx = true +[http.services.sciencemesh] +mesh_directory_url = 'https://sciencemesh.cesnet.cz/iop/meshdir' +provider_domain = 'your-domain.org' +ocm_mount_point = '/sciencemesh' + [http.middlewares.providerauthorizer] driver = "json" diff --git a/examples/oc-phoenix/ocmd.toml b/examples/oc-phoenix/ocmd.toml index 0864d48b54..bd53def4ae 100644 --- a/examples/oc-phoenix/ocmd.toml +++ b/examples/oc-phoenix/ocmd.toml @@ -35,13 +35,10 @@ address = "0.0.0.0:13001" [http.services.ocmd] prefix = "ocm" -[http.services.ocmprovider] -endpoint = "http://localhost:13001" - [http.middlewares.providerauthorizer] driver = "json" [http.middlewares.providerauthorizer.drivers.json] providers = "providers.demo.json" -[http.middlewares.cors] \ No newline at end of file +[http.middlewares.cors] diff --git a/examples/ocmd/server-1/ocmd-server-1.toml b/examples/ocmd/server-1/ocmd-server-1.toml index 5ecf585745..0d9adbf513 100644 --- a/examples/ocmd/server-1/ocmd-server-1.toml +++ b/examples/ocmd/server-1/ocmd-server-1.toml @@ -6,8 +6,7 @@ address = "0.0.0.0:19000" [grpc.services.gateway] authregistrysvc = "localhost:19000" -appprovidersvc = "localhost:19000" -appregistry = "localhost:19000" +appregistrysvc = "localhost:19000" storageregistrysvc = "localhost:19000" preferencessvc = "localhost:19000" userprovidersvc = "localhost:19000" @@ -22,6 +21,29 @@ transfer_expires = 6 # give it a moment commit_share_to_storage_grant = true commit_share_to_storage_ref = true +[grpc.services.appregistry] +driver = "static" + +[grpc.services.appregistry.drivers.static] +mime_types = [ + {"mime_type" = "text/plain", "extension" = "txt", "name" = "Text file", "description" = "Text file", "allow_creation" = true}, + {"mime_type" = "text/markdown", "extension" = "md", "name" = "Markdown file", "description" = "Markdown file", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.text", "extension" = "odt", "name" = "OpenDocument", "description" = "OpenDocument text document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.spreadsheet", "extension" = "ods", "name" = "OpenSpreadsheet", "description" = "OpenDocument spreadsheet document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.presentation", "extension" = "odp", "name" = "OpenPresentation", "description" = "OpenDocument presentation document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.jupyter", "extension" = "ipynb", "name" = "Jupyter Notebook", "description" = "Jupyter Notebook"} +] + +[grpc.services.appprovider] +driver = "wopi" +appregistrysvc = "localhost:19000" + +[grpc.services.appprovider.drivers.wopi] +iop_secret = "shared-secret-with-wopiserver" +wopi_url = "http://0.0.0.0:8880/" +app_name = "Collabora" +app_url = "https://your-collabora-server.org:9980" + [grpc.services.authregistry] driver = "static" @@ -111,6 +133,8 @@ users = "users.demo.json" [http] address = "0.0.0.0:19001" +[http.services.appprovider] + [http.services.dataprovider] driver = "localhome" @@ -141,6 +165,13 @@ prefix = "ocs" [http.services.ocdav] -[http.services.appprovider] +[http.services.sciencemesh] +mesh_directory_url = 'https://sciencemesh.cesnet.cz/iop/meshdir' +provider_domain = 'your-domain-1.org' +ocm_mount_point = '/sciencemesh' [http.middlewares.cors] + +[http.middlewares.auth] +credential_chain = ["publicshares", "ocmshares", "basic", "bearer"] + diff --git a/examples/ocmd/server-2/ocmd-server-2.toml b/examples/ocmd/server-2/ocmd-server-2.toml index 8ac94f9ba9..755d00116e 100644 --- a/examples/ocmd/server-2/ocmd-server-2.toml +++ b/examples/ocmd/server-2/ocmd-server-2.toml @@ -6,6 +6,7 @@ address = "0.0.0.0:17000" [grpc.services.gateway] authregistrysvc = "localhost:17000" +appregistrysvc = "localhost:17000" storageregistrysvc = "localhost:17000" preferencessvc = "localhost:17000" userprovidersvc = "localhost:17000" @@ -20,6 +21,29 @@ transfer_expires = 6 # give it a moment commit_share_to_storage_grant = true commit_share_to_storage_ref = true +[grpc.services.appregistry] +driver = "static" + +[grpc.services.appregistry.drivers.static] +mime_types = [ + {"mime_type" = "text/plain", "extension" = "txt", "name" = "Text file", "description" = "Text file", "allow_creation" = true}, + {"mime_type" = "text/markdown", "extension" = "md", "name" = "Markdown file", "description" = "Markdown file", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.text", "extension" = "odt", "name" = "OpenDocument", "description" = "OpenDocument text document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.spreadsheet", "extension" = "ods", "name" = "OpenSpreadsheet", "description" = "OpenDocument spreadsheet document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.presentation", "extension" = "odp", "name" = "OpenPresentation", "description" = "OpenDocument presentation document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.jupyter", "extension" = "ipynb", "name" = "Jupyter Notebook", "description" = "Jupyter Notebook"} +] + +[grpc.services.appprovider] +driver = "wopi" +app_provider_url = "localhost:17000" + +[grpc.services.appprovider.drivers.wopi] +iop_secret = "shared-secret-with-wopiserver" +wopi_url = "http://0.0.0.0:8880/" +app_name = "Collabora" +app_url = "https://your-collabora-server.org:9980" + [grpc.services.authregistry] driver = "static" @@ -102,6 +126,8 @@ users = "users.demo.json" [http] address = "0.0.0.0:17001" +[http.services.appprovider] + [http.services.dataprovider] driver = "localhome" @@ -130,4 +156,13 @@ prefix = "ocs" [http.services.ocdav] +[http.services.sciencemesh] +mesh_directory_url = 'https://sciencemesh.cesnet.cz/iop/meshdir' +provider_domain = 'your-domain.org' +ocm_mount_point = '/sciencemesh' + [http.middlewares.cors] + +[http.middlewares.auth] +credential_chain = ["publicshares", "ocmshares", "basic", "bearer"] + diff --git a/examples/oidc-mapping-tpc/server-1.toml b/examples/oidc-mapping-tpc/server-1.toml index 61248b2a13..c212892e59 100644 --- a/examples/oidc-mapping-tpc/server-1.toml +++ b/examples/oidc-mapping-tpc/server-1.toml @@ -62,7 +62,6 @@ driver = "localhome" [http.services.datagateway] [http.services.prometheus] [http.services.ocmd] -[http.services.ocmprovider] [http.services.ocs] [http.services.ocdav] enable_http_tpc = true diff --git a/examples/oidc-mapping-tpc/server-2.toml b/examples/oidc-mapping-tpc/server-2.toml index 02eaee63bc..259c4b77d8 100644 --- a/examples/oidc-mapping-tpc/server-2.toml +++ b/examples/oidc-mapping-tpc/server-2.toml @@ -62,7 +62,6 @@ driver = "localhome" [http.services.datagateway] [http.services.prometheus] [http.services.ocmd] -[http.services.ocmprovider] [http.services.ocs] [http.services.ocdav] enable_http_tpc = true diff --git a/examples/storage-references/gateway.toml b/examples/storage-references/gateway.toml index 3070078503..0cbed478ae 100644 --- a/examples/storage-references/gateway.toml +++ b/examples/storage-references/gateway.toml @@ -31,6 +31,8 @@ appauth = "localhost:15000" [grpc.services.ocmproviderauthorizer] [grpc.services.appregistry] +driver = "static" + [grpc.services.appregistry.drivers.static] mime_types = [ {"mime_type" = "text/plain", "extension" = "txt", "name" = "Text file", "description" = "Text file", "allow_creation" = true}, @@ -41,13 +43,23 @@ mime_types = [ {"mime_type" = "application/vnd.jupyter", "extension" = "ipynb", "name" = "Jupyter Notebook", "description" = "Jupyter Notebook"} ] +[grpc.services.appprovider] +driver = "wopi" +appregistrysvc = "localhost:19000" +custom_mime_types_json = "/etc/revad/custom-mime-types-demo.json" + +[grpc.services.appprovider.drivers.wopi] +iop_secret = "shared-secret-with-wopiserver" +wopi_url = "http://0.0.0.0:8880/" +app_name = "Collabora" +app_url = "https://your-collabora-server.org:9980" + [http.services.datagateway] [http.services.prometheus] [http.services.ocmd] [http.services.ocmprovider] [http.services.ocdav] [http.services.ocs] -[http.services.appprovider] [http.services.sciencemesh] mesh_directory_url = 'https://sciencemesh.cesnet.cz/iop/meshdir' @@ -60,4 +72,3 @@ disable_auth = true sender_mail = "sciencemesh@your-domain.org" smtp_server = "your-smtp-server.your-domain.org" smtp_port = 25 - diff --git a/examples/two-server-setup/gateway-1.toml b/examples/two-server-setup/gateway-1.toml index 4a6990a42a..6421454da3 100644 --- a/examples/two-server-setup/gateway-1.toml +++ b/examples/two-server-setup/gateway-1.toml @@ -1,11 +1,51 @@ +[shared] +gatewaysvc = "localhost:19000" +datagateway = "http://localhost:19001/datagateway" + [grpc] address = "0.0.0.0:19000" # services to enable [grpc.services.gateway] +authregistrysvc = "localhost:19000" +appregistrysvc = "localhost:19000" +storageregistrysvc = "localhost:19000" +preferencessvc = "localhost:19000" +userprovidersvc = "localhost:19000" +groupprovidersvc = "localhost:19000" +usershareprovidersvc = "localhost:19000" +publicshareprovidersvc = "localhost:19000" +ocmcoresvc = "localhost:19000" +ocmshareprovidersvc = "localhost:19000" +ocminvitemanagersvc = "localhost:19000" +ocmproviderauthorizersvc = "localhost:19000" +datagateway = "http://localhost:19001/data" commit_share_to_storage_grant = true commit_share_to_storage_ref = true +[grpc.services.appregistry] +driver = "static" + +[grpc.services.appregistry.drivers.static] +mime_types = [ + {"mime_type" = "text/plain", "extension" = "txt", "name" = "Text file", "description" = "Text file", "allow_creation" = true}, + {"mime_type" = "text/markdown", "extension" = "md", "name" = "Markdown file", "description" = "Markdown file", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.text", "extension" = "odt", "name" = "OpenDocument", "description" = "OpenDocument text document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.spreadsheet", "extension" = "ods", "name" = "OpenSpreadsheet", "description" = "OpenDocument spreadsheet document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.presentation", "extension" = "odp", "name" = "OpenPresentation", "description" = "OpenDocument presentation document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.jupyter", "extension" = "ipynb", "name" = "Jupyter Notebook", "description" = "Jupyter Notebook"} +] + +[grpc.services.appprovider] +driver = "wopi" +app_provider_url = "localhost:19000" + +[grpc.services.appprovider.drivers.wopi] +iop_secret = "shared-secret-with-wopiserver" +wopi_url = "http://0.0.0.0:8880/" +app_name = "Collabora" +app_url = "https://your-collabora-server.org:9980" + [grpc.services.storageregistry] [grpc.services.storageregistry.drivers.static] home_provider = "/home" @@ -51,6 +91,7 @@ providers = "providers.demo.json" [http] address = "0.0.0.0:19001" +[http.services.appprovider] [http.services.datagateway] [http.services.prometheus] [http.services.ocmd] @@ -62,6 +103,10 @@ enable_datatx = true [http.services.ocdav] [http.services.ocs] +[http.services.sciencemesh] +mesh_directory_url = 'https://sciencemesh.cesnet.cz/iop/meshdir' +provider_domain = 'your-domain.org' +ocm_mount_point = '/sciencemesh' [http.middlewares.providerauthorizer.drivers.json] providers = "providers.demo.json" diff --git a/examples/two-server-setup/gateway-2.toml b/examples/two-server-setup/gateway-2.toml index 808ec4c9ed..65aa63a30f 100644 --- a/examples/two-server-setup/gateway-2.toml +++ b/examples/two-server-setup/gateway-2.toml @@ -8,6 +8,7 @@ address = "0.0.0.0:29000" # services to enable [grpc.services.gateway] authregistrysvc = "localhost:29000" +appregistrysvc = "localhost:29000" storageregistrysvc = "localhost:29000" preferencessvc = "localhost:29000" userprovidersvc = "localhost:29000" @@ -18,9 +19,33 @@ ocmcoresvc = "localhost:29000" ocmshareprovidersvc = "localhost:29000" ocminvitemanagersvc = "localhost:29000" ocmproviderauthorizersvc = "localhost:29000" +datagateway = "http://localhost:29001/data" commit_share_to_storage_grant = true commit_share_to_storage_ref = true +[grpc.services.appregistry] +driver = "static" + +[grpc.services.appregistry.drivers.static] +mime_types = [ + {"mime_type" = "text/plain", "extension" = "txt", "name" = "Text file", "description" = "Text file", "allow_creation" = true}, + {"mime_type" = "text/markdown", "extension" = "md", "name" = "Markdown file", "description" = "Markdown file", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.text", "extension" = "odt", "name" = "OpenDocument", "description" = "OpenDocument text document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.spreadsheet", "extension" = "ods", "name" = "OpenSpreadsheet", "description" = "OpenDocument spreadsheet document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.presentation", "extension" = "odp", "name" = "OpenPresentation", "description" = "OpenDocument presentation document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.jupyter", "extension" = "ipynb", "name" = "Jupyter Notebook", "description" = "Jupyter Notebook"} +] + +[grpc.services.appprovider] +driver = "wopi" +appregistrysvc = "localhost:29000" + +[grpc.services.appprovider.drivers.wopi] +iop_secret = "shared-secret-with-wopiserver" +wopi_url = "http://0.0.0.0:8880/" +app_name = "Collabora" +app_url = "https://your-collabora-server.org:9980" + [grpc.services.storageregistry] [grpc.services.storageregistry.drivers.static] home_provider = "/home" @@ -66,6 +91,7 @@ providers = "providers.demo.json" [http] address = "0.0.0.0:29001" +[http.services.appprovider] [http.services.datagateway] [http.services.prometheus] [http.services.ocmd] @@ -78,5 +104,10 @@ enable_datatx = true [http.services.ocdav] [http.services.ocs] +[http.services.sciencemesh] +mesh_directory_url = 'https://sciencemesh.cesnet.cz/iop/meshdir' +provider_domain = 'your-domain.org' +ocm_mount_point = '/sciencemesh' + [http.middlewares.providerauthorizer.drivers.json] providers = "providers.demo.json" diff --git a/internal/http/services/ocmd/notifications.go b/internal/http/services/ocmd/notifications.go new file mode 100644 index 0000000000..7f2249f074 --- /dev/null +++ b/internal/http/services/ocmd/notifications.go @@ -0,0 +1,106 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocmd + +import ( + "io" + "mime" + "net/http" + + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + + "github.com/cs3org/reva/internal/http/services/reqres" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" +) + +// var validate = validator.New() + +type notifHandler struct { + gatewayClient gateway.GatewayAPIClient +} + +func (h *notifHandler) init(c *config) error { + var err error + h.gatewayClient, err = pool.GetGatewayServiceClient(pool.Endpoint(c.GatewaySvc)) + if err != nil { + return err + } + return nil +} + +// type notificationRequest struct { +// NotificationType string `json:"notificationType" validate:"required"` +// ResourceType string `json:"resourceType" validate:"required"` +// ProviderId string `json:"providerId" validate:"required"` +// Notification ... `json:"notification"` +//} + +// Example of payload from Nextcloud: +// { +// "notificationType": , +// "resourceType" : "file", +// "providerId" : , +// "notification" : { +// "sharedSecret" : , +// "message" : "human-readable message", +// "shareWith" : , +// "senderId" : , +// "shareType" : +// } +// } + +// Notifications dispatches any notifications received from remote OCM sites +// according to the specifications at: +// https://cs3org.github.io/OCM-API/docs.html?branch=v1.1.0&repo=OCM-API&user=cs3org#/paths/~1notifications/post +func (h *notifHandler) Notifications(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + log := appctx.GetLogger(ctx) + req, err := getNotification(r) + if err != nil { + reqres.WriteError(w, r, reqres.APIErrorInvalidParameter, err.Error(), nil) + return + } + + // TODO(lopresti) this is all to be implemented. For now we just log what we got + log.Debug().Msgf("Received OCM notification: %+v", req) + + // this is to please Nextcloud + w.WriteHeader(http.StatusCreated) +} + +func getNotification(r *http.Request) (string, error) { // (*notificationRequest, error) + // var req notificationRequest + contentType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err == nil && contentType == "application/json" { + bytes, _ := io.ReadAll(r.Body) + return string(bytes), nil + // if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + // return nil, err + //} + // } else { + // return nil, errors.New("body request not recognised") + } + return "", nil + // validate the request + // if err := validate.Struct(req); err != nil { + // return nil, err + //} + // return &req, nil +} diff --git a/internal/http/services/ocmd/ocm.go b/internal/http/services/ocmd/ocm.go index fa92fb4121..8cbb4821a0 100644 --- a/internal/http/services/ocmd/ocm.go +++ b/internal/http/services/ocmd/ocm.go @@ -76,6 +76,7 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) func (s *svc) routerInit() error { sharesHandler := new(sharesHandler) invitesHandler := new(invitesHandler) + notifHandler := new(notifHandler) if err := sharesHandler.init(s.Conf); err != nil { return err @@ -83,9 +84,13 @@ func (s *svc) routerInit() error { if err := invitesHandler.init(s.Conf); err != nil { return err } + if err := notifHandler.init(s.Conf); err != nil { + return err + } s.router.Post("/shares", sharesHandler.CreateShare) s.router.Post("/invite-accepted", invitesHandler.AcceptInvite) + s.router.Post("/notifications", notifHandler.Notifications) return nil } @@ -99,7 +104,7 @@ func (s *svc) Prefix() string { } func (s *svc) Unprotected() []string { - return []string{"/invite-accepted", "/shares", "/ocm-provider", "/notifications"} + return []string{"/invite-accepted", "/shares", "/notifications"} } func (s *svc) Handler() http.Handler { diff --git a/internal/http/services/ocmprovider/ocmprovider.go b/internal/http/services/ocmprovider/ocmprovider.go index a88f9e52a0..8ed83cf116 100644 --- a/internal/http/services/ocmprovider/ocmprovider.go +++ b/internal/http/services/ocmprovider/ocmprovider.go @@ -70,10 +70,16 @@ func (c *config) init() { c.Provider = "reva" } if c.WebdavRoot == "" { - c.WebdavRoot = "/remote.php/dav/ocm" + c.WebdavRoot = "/remote.php/dav/ocm/" + } + if c.WebdavRoot[len(c.WebdavRoot)-1:] != "/" { + c.WebdavRoot += "/" } if c.WebappRoot == "" { - c.WebappRoot = "/external/sciencemesh" + c.WebappRoot = "/external/sciencemesh/" + } + if c.WebappRoot[len(c.WebappRoot)-1:] != "/" { + c.WebappRoot += "/" } } @@ -99,12 +105,12 @@ func (c *config) prepare() *discoveryData { d.Provider = c.Provider rtProtos := map[string]string{} // webdav is always enabled - rtProtos["webdav"] = fmt.Sprintf("%s%s", c.Endpoint, c.WebdavRoot) + rtProtos["webdav"] = c.WebdavRoot if c.EnableWebapp { - rtProtos["webapp"] = fmt.Sprintf("%s%s", c.Endpoint, c.WebappRoot) + rtProtos["webapp"] = c.WebappRoot } if c.EnableDatatx { - rtProtos["datatx"] = fmt.Sprintf("%s%s", c.Endpoint, c.WebdavRoot) + rtProtos["datatx"] = c.WebdavRoot } d.ResourceTypes = []resourceTypes{{ Name: "file", // so far we only support `file` @@ -148,6 +154,12 @@ func (s *svc) Handler() http.Handler { log := appctx.GetLogger(r.Context()) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) + if r.UserAgent() == "Nextcloud Server Crawler" { + // TODO(lopresti) remove this hack once Nextcloud is able to talk OCM! + s.data.APIVersion = "1.0-proposal1" + } else { + s.data.APIVersion = "1.1.0" + } indented, _ := json.MarshalIndent(s.data, "", " ") if _, err := w.Write(indented); err != nil { log.Err(err).Msg("Error writing to ResponseWriter") diff --git a/pkg/app/provider/wopi/wopi.go b/pkg/app/provider/wopi/wopi.go index 640e0319f2..b3bfd46415 100644 --- a/pkg/app/provider/wopi/wopi.go +++ b/pkg/app/provider/wopi/wopi.go @@ -51,6 +51,7 @@ import ( "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/utils" + gomime "github.com/glpatcern/go-mime" "github.com/golang-jwt/jwt" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" @@ -192,7 +193,7 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc ut = anonymous rPath, pathErr = getPathForExternalLink(ctx, scopes, resource, publicLinkURLPrefix) if pathErr != nil { - log.Warn().Err(pathErr).Msg("wopi: failed to extract relative path from public link scope") + log.Warn().Interface("resId", resource.Id).Interface("path", resource.Path).Err(pathErr).Msg("wopi: failed to extract relative path from public link scope") } case ocmrole: // OCM users have no username: use displayname@Idp @@ -201,7 +202,21 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc // and resolve the folder rPath, pathErr = getPathForExternalLink(ctx, scopes, resource, ocmLinkURLPrefix) if pathErr != nil { - log.Warn().Err(pathErr).Msg("wopi: failed to extract relative path from ocm link scope") + log.Warn().Interface("resId", resource.Id).Interface("path", resource.Path).Err(pathErr).Msg("wopi: failed to extract relative path from ocm link scope") + } + if ext == "" { + // this is a single-file share, and we have to re-resolve the extension from the mime type + exts := gomime.ExtensionsByType(resource.MimeType) + for _, e := range exts { + if len(e) < len(ext) || len(ext) == 0 { + ext = e // heuristically we know we want the shortest file extension + } + } + ext = "." + ext + log.Debug().Interface("mime", resource.MimeType).Interface("ext", ext).Msg("wopi: resolved extension for single-file OCM share") + } + if ext == "" { + return nil, errors.New("wopi: failed to resolve extension from OCM file's mime type %s" + resource.MimeType) } default: // in all other cases use the resource's path @@ -246,7 +261,7 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc q.Add("appurl", viewAppURL) } if q.Get("appurl") == "" && q.Get("appviewurl") == "" { - return nil, errors.New("wopi: neither edit nor view app url found") + return nil, errors.New("wopi: neither edit nor view app url found for type " + ext) } if p.conf.AppIntURL != "" { q.Add("appinturl", p.conf.AppIntURL) @@ -532,17 +547,17 @@ func getPathForExternalLink(ctx context.Context, scopes map[string]*authpb.Scope return "", err } - if statRes.Info.Path == resource.Path { + if statRes.Info.Path == resource.Path || utils.ResourceIDEqual(statRes.Info.Id, resource.Id) { // this is a direct link to the resource return pathPrefix + token, nil } - // otherwise we are in a subfolder of the public link + // otherwise we are in a subfolder of the link relPath, err := filepath.Rel(statRes.Info.Path, resource.Path) if err != nil { return "", err } if strings.HasPrefix(relPath, "../") { - return "", errors.New("Scope path does not contain target resource") + return "", errors.New("Scope path does not contain target resource path " + statRes.Info.Path) } return path.Join(pathPrefix+token, path.Dir(relPath)), nil } From 35aaf51530f7dc6c61ffb37212ffe4cea826a564 Mon Sep 17 00:00:00 2001 From: Miroslav Bauer Date: Thu, 22 Jun 2023 12:00:07 +0200 Subject: [PATCH 20/48] Remove meshdirectory service (#3581) * remove meshdirectory service from reva * fix changelog * fix changelog * remove meshdirectory service from reva --------- Co-authored-by: Giuseppe Lo Presti --- changelog/unreleased/remove-meshdirectory.md | 8 + examples/meshdirectory/groups.demo.json | 140 --------------- examples/meshdirectory/meshdirectory.toml | 68 -------- examples/meshdirectory/providers.demo.json | 164 ------------------ examples/meshdirectory/users.demo.json | 38 ---- go.mod | 1 - go.sum | 2 - internal/http/services/loader/loader.go | 1 - .../services/meshdirectory/meshdirectory.go | 152 ---------------- 9 files changed, 8 insertions(+), 566 deletions(-) create mode 100644 changelog/unreleased/remove-meshdirectory.md delete mode 100644 examples/meshdirectory/groups.demo.json delete mode 100644 examples/meshdirectory/meshdirectory.toml delete mode 100644 examples/meshdirectory/providers.demo.json delete mode 100644 examples/meshdirectory/users.demo.json delete mode 100644 internal/http/services/meshdirectory/meshdirectory.go diff --git a/changelog/unreleased/remove-meshdirectory.md b/changelog/unreleased/remove-meshdirectory.md new file mode 100644 index 0000000000..c3460de296 --- /dev/null +++ b/changelog/unreleased/remove-meshdirectory.md @@ -0,0 +1,8 @@ +Change: Remove meshdirectory http service + +As of meshdirectory-web version 2.0.0, it is now +implemented and deployed as a completely separate app, +independent from Reva. We removed any deprecated +meshdirectory-related code from Reva. + +https://github.com/cs3org/reva/pull/3581 \ No newline at end of file diff --git a/examples/meshdirectory/groups.demo.json b/examples/meshdirectory/groups.demo.json deleted file mode 100644 index 42fd1af0c2..0000000000 --- a/examples/meshdirectory/groups.demo.json +++ /dev/null @@ -1,140 +0,0 @@ -[ - { - "id": { - "opaque_id": "sailing-lovers", - "idp": "http://localhost:20080" - }, - "group_name": "sailing-lovers", - "mail": "sailing-lovers@example.org", - "display_name": "Sailing Lovers", - "gid_number": 123, - "members": [ - { - "id": { - "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "http://localhost:20080" - } - } - ] - }, - { - "id": { - "opaque_id": "violin-haters", - "idp": "http://localhost:20080" - }, - "group_name": "violin-haters", - "mail": "violin-haters@example.org", - "display_name": "Violin Haters", - "gid_number": 456, - "members": [ - { - "id": { - "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "http://localhost:20080" - } - } - ] - }, - { - "id": { - "opaque_id": "radium-lovers", - "idp": "http://localhost:20080" - }, - "group_name": "radium-lovers", - "mail": "radium-lovers@example.org", - "display_name": "Radium Lovers", - "gid_number": 789, - "members": [ - { - "id": { - "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "http://localhost:20080" - } - } - ] - }, - { - "id": { - "opaque_id": "polonium-lovers", - "idp": "http://localhost:20080" - }, - "group_name": "polonium-lovers", - "mail": "polonium-lovers@example.org", - "display_name": "Polonium Lovers", - "gid_number": 987, - "members": [ - { - "id": { - "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "http://localhost:20080" - } - } - ] - }, - { - "id": { - "opaque_id": "quantum-lovers", - "idp": "http://localhost:20080" - }, - "group_name": "quantum-lovers", - "mail": "quantum-lovers@example.org", - "display_name": "Quantum Lovers", - "gid_number": 654, - "members": [ - { - "id": { - "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "http://localhost:20080" - } - } - ] - }, - { - "id": { - "opaque_id": "philosophy-haters", - "idp": "http://localhost:20080" - }, - "group_name": "philosophy-haters", - "mail": "philosophy-haters@example.org", - "display_name": "Philosophy Haters", - "gid_number": 321, - "members": [ - { - "id": { - "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "http://localhost:20080" - } - } - ] - }, - { - "id": { - "opaque_id": "physics-lovers", - "idp": "http://localhost:20080" - }, - "group_name": "physics-lovers", - "mail": "physics-lovers@example.org", - "display_name": "Physics Lovers", - "gid_number": 101, - "members": [ - { - "id": { - "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "http://localhost:20080" - } - }, - { - "id": { - "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "http://localhost:20080" - } - }, - { - "id": { - "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "http://localhost:20080" - } - } - ] - } -] diff --git a/examples/meshdirectory/meshdirectory.toml b/examples/meshdirectory/meshdirectory.toml deleted file mode 100644 index 1bea8c36e0..0000000000 --- a/examples/meshdirectory/meshdirectory.toml +++ /dev/null @@ -1,68 +0,0 @@ -# services to enable -[grpc.services.gateway] -[grpc.services.storageregistry] -[grpc.services.storageprovider] -[grpc.services.authregistry] -[grpc.services.usershareprovider] -[grpc.services.publicshareprovider] -[grpc.services.ocmcore] -driver = "json" - -[grpc.services.ocmcore.drivers.json] -file = "/var/tmp/reva/shares_server_1.json" - -[grpc.services.ocminvitemanager] -driver = "json" - -[grpc.services.ocmshareprovider] -driver = "json" - -[grpc.services.ocmshareprovider.drivers.json] -file = "/var/tmp/reva/shares_server_1.json" - -[http.services.datagateway] -[http.services.dataprovider] -[http.services.prometheus] -[http.services.ocmd] -[http.services.ocmprovider] -[http.services.ocdav] -[http.services.ocs] - -[grpc.services.authprovider] -auth_manager = "json" - -[grpc.services.authprovider.auth_managers.json] -users = "users.demo.json" - -[grpc.services.userprovider] -driver = "json" -[grpc.services.groupprovider] -driver = "json" - -[grpc.services.groupprovider.drivers.json] -groups = "groups.demo.json" - -[grpc.services.userprovider.drivers.json] -users = "users.demo.json" - -[grpc.services.ocmproviderauthorizer] -driver = "mentix" - -[http.middlewares.providerauthorizer] -driver = "mentix" - -[http.middlewares.providerauthorizer.drivers.mentix] -url = "https://localhost:19001/api/mentix/cs3" -insecure = false -timeout = 10 -refresh = 900 - -[grpc.services.ocmproviderauthorizer.drivers.mentix] -url = "https://localhost:19001/api/mentix/cs3" -verify_request_hostname = true -insecure = false -timeout = 10 -refresh = 900 - -[http.services.meshdirectory] -mesh_directory_url = 'http://localhost:19001/meshdir/' diff --git a/examples/meshdirectory/providers.demo.json b/examples/meshdirectory/providers.demo.json deleted file mode 100644 index e3674f7658..0000000000 --- a/examples/meshdirectory/providers.demo.json +++ /dev/null @@ -1,164 +0,0 @@ -[ - { - "name": "cernbox", - "full_name": "CERNBox", - "organization": "CERN", - "domain": "cern.ch", - "homepage": "https://cernbox.web.cern.ch", - "description": "CERNBox provides cloud data storage to all CERN users.", - "services": [ - { - "endpoint": { - "type": { - "name": "OCM", - "description": "CERNBox Open Cloud Mesh API" - }, - "name": "CERNBox - OCM API", - "path": "https://sciencemesh.cernbox.cern.ch/iop/ocm/", - "is_monitored": true - }, - "api_version": "0.0.1", - "host": "sciencemesh.cernbox.cern.ch" - } - ] - }, - { - "name": "oc-cesnet", - "full_name": "ownCloud@CESNET", - "organization": "CESNET", - "domain": "cesnet.cz", - "homepage": "https://owncloud.cesnet.cz", - "description": "OwnCloud has been designed for individual users.", - "services": [ - { - "endpoint": { - "type": { - "name": "OCM", - "description": "CESNET Open Cloud Mesh API" - }, - "name": "CESNET - OCM API", - "path": "https://sciencemesh.cesnet.cz/ocm/", - "is_monitored": true - }, - "api_version": "0.0.1", - "host": "sciencemesh.cesnet.cz" - } - ] - }, - { - "name": "uni-muenster", - "full_name": "WWU University of Muenster", - "organization": "University of Muenster", - "domain": "uni-muenster.de", - "homepage": "https://uni-muenster.de", - "description": "WWU provides cloud storage to its students, faculty and researchers.", - "services": [ - { - "endpoint": { - "type": { - "name": "OCM", - "description": "WWU Open Cloud Mesh API" - }, - "name": "WWU - OCM API", - "path": "https://sciencemesh-test.uni-muenster.de/ocm/", - "is_monitored": true - }, - "api_version": "0.0.1", - "host": "sciencemesh-test.uni-muenster.de" - } - ] - }, - { - "name": "cubbit", - "full_name": "Cubbit", - "organization": "Cubbit", - "domain": "cubbit.io", - "homepage": "https://cubbit.io", - "description": "Cubbit provides distributed storage over a P2P network.", - "services": [ - { - "endpoint": { - "type": { - "name": "OCM", - "description": "Cubbit Open Cloud Mesh API" - }, - "name": "Cubbit - OCM API", - "path": "https://ocm.cubbit.io/ocm/", - "is_monitored": true - }, - "api_version": "0.0.1", - "host": "ocm.cubbit.io" - } - ] - }, - { - "name": "ailleron", - "full_name": "Ailleron - Software Mind", - "organization": "Ailleron", - "domain": "ailleron.com", - "homepage": "https://ailleron.com", - "description": "Ailleron is a Polish IT company providing software and storage solutions to its clients.", - "services": [ - { - "endpoint": { - "type": { - "name": "OCM", - "description": "Ailleron Open Cloud Mesh API" - }, - "name": "Ailleron - OCM API", - "path": "http://cs3mesh.softwaremind.com:19000/ocm/", - "is_monitored": true - }, - "api_version": "0.0.1", - "host": "cs3mesh.softwaremind.com:19000" - } - ] - }, - { - "name": "surfsara", - "full_name": "Surfsara", - "organization": "Surfsara", - "domain": "surfsara.nl", - "homepage": "https://surfsara.nl", - "description": "OwnCloud has been designed for individual users.", - "services": [ - { - "endpoint": { - "type": { - "name": "OCM", - "description": "Surfsara Open Cloud Mesh API" - }, - "name": "Surfsara - OCM API", - "path": "https://app.cs3mesh-iop.k8s.surfsara.nl/ocm/", - "is_monitored": true - }, - "api_version": "0.0.1", - "host": "app.cs3mesh-iop.k8s.surfsara.nl" - } - ] - }, - { - "name": "switch", - "full_name": "SWITCH", - "organization": "SWITCH", - "domain": "switch.ch", - "homepage": "https://switch.ch", - "description": "SWITCHdrive cloud storage to its students, faculty and researchers.", - "services": [ - { - "endpoint": { - "type": { - "name": "OCM", - "description": "SWITCH Open Cloud Mesh API" - }, - "name": "SWITCH - OCM API", - "path": "https://sciencemesh-test.switch.ch/ocm/", - "is_monitored": true - }, - "api_version": "0.0.1", - "host": "sciencemesh-test.switch.ch" - } - ] - } - -] diff --git a/examples/meshdirectory/users.demo.json b/examples/meshdirectory/users.demo.json deleted file mode 100644 index 893d69b99f..0000000000 --- a/examples/meshdirectory/users.demo.json +++ /dev/null @@ -1,38 +0,0 @@ -[ - { - "id": { - "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "http://localhost:20080", - "type": 1 - }, - "username": "einstein", - "secret": "relativity", - "mail": "einstein@example.org", - "display_name": "Albert Einstein", - "groups": ["sailing-lovers", "violin-haters", "physics-lovers"] - }, - { - "id": { - "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "http://localhost:20080", - "type": 1 - }, - "username": "marie", - "secret": "radioactivity", - "mail": "marie@example.org", - "display_name": "Marie Curie", - "groups": ["radium-lovers", "polonium-lovers", "physics-lovers"] - }, - { - "id": { - "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "http://localhost:20080", - "type": 1 - }, - "username": "richard", - "secret": "superfluidity", - "mail": "richard@example.org", - "display_name": "Richard Feynman", - "groups": ["quantum-lovers", "philosophy-haters", "physics-lovers"] - } -] diff --git a/go.mod b/go.mod index bcc99beacb..e9533f42ce 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,6 @@ require ( github.com/prometheus/alertmanager v0.24.0 github.com/rs/cors v1.8.3 github.com/rs/zerolog v1.28.0 - github.com/sciencemesh/meshdirectory-web v1.0.4 github.com/sethvargo/go-password v0.2.0 github.com/stretchr/testify v1.8.2 github.com/studio-b12/gowebdav v0.0.0-20230203202212-3282f94193f2 diff --git a/go.sum b/go.sum index f35936c818..bac70c99ab 100644 --- a/go.sum +++ b/go.sum @@ -1053,8 +1053,6 @@ github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFo github.com/sacloud/libsacloud v1.36.2/go.mod h1:P7YAOVmnIn3DKHqCZcUKYUXmSwGBm3yS7IBEjKVSrjg= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8= -github.com/sciencemesh/meshdirectory-web v1.0.4 h1:1YSctF6PAXhoHUYCaeRTj7rHaF7b3rYrZf2R0VXBIbo= -github.com/sciencemesh/meshdirectory-web v1.0.4/go.mod h1:fJSThTS3xf+sTdL0iXQoaQJssLI7tn7DetHMHUl4SRk= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethgrid/pester v0.0.0-20190127155807-68a33a018ad0/go.mod h1:Ad7IjTpvzZO8Fl0vh9AzQ+j/jYZfyp2diGwI8m5q+ns= diff --git a/internal/http/services/loader/loader.go b/internal/http/services/loader/loader.go index adf1ad02da..50ad252d9a 100644 --- a/internal/http/services/loader/loader.go +++ b/internal/http/services/loader/loader.go @@ -26,7 +26,6 @@ import ( _ "github.com/cs3org/reva/internal/http/services/dataprovider" _ "github.com/cs3org/reva/internal/http/services/helloworld" _ "github.com/cs3org/reva/internal/http/services/mentix" - _ "github.com/cs3org/reva/internal/http/services/meshdirectory" _ "github.com/cs3org/reva/internal/http/services/metrics" _ "github.com/cs3org/reva/internal/http/services/ocmd" _ "github.com/cs3org/reva/internal/http/services/ocmprovider" diff --git a/internal/http/services/meshdirectory/meshdirectory.go b/internal/http/services/meshdirectory/meshdirectory.go deleted file mode 100644 index 9a1327adc0..0000000000 --- a/internal/http/services/meshdirectory/meshdirectory.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package meshdirectory - -import ( - "encoding/json" - "fmt" - "net/http" - - gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" - providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" - "github.com/cs3org/reva/internal/http/services/reqres" - "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/rhttp/global" - "github.com/cs3org/reva/pkg/rhttp/router" - "github.com/cs3org/reva/pkg/sharedconf" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" - "github.com/rs/zerolog" - meshdirectoryweb "github.com/sciencemesh/meshdirectory-web" -) - -func init() { - global.Register("meshdirectory", New) -} - -type config struct { - Prefix string `mapstructure:"prefix"` - GatewaySvc string `mapstructure:"gatewaysvc"` -} - -func (c *config) init() { - c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) - - if c.Prefix == "" { - c.Prefix = "meshdir" - } -} - -type svc struct { - conf *config -} - -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - -// New returns a new Mesh Directory HTTP service. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { - c, err := parseConfig(m) - if err != nil { - return nil, err - } - - c.init() - - service := &svc{ - conf: c, - } - return service, nil -} - -// Service prefix. -func (s *svc) Prefix() string { - return s.conf.Prefix -} - -// Unprotected endpoints. -func (s *svc) Unprotected() []string { - return []string{"/"} -} - -// Close performs cleanup. -func (s *svc) Close() error { - return nil -} - -func (s *svc) getClient() (gateway.GatewayAPIClient, error) { - return pool.GetGatewayServiceClient(pool.Endpoint(s.conf.GatewaySvc)) -} - -func (s *svc) serveJSON(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - - ctx := r.Context() - - gatewayClient, err := s.getClient() - if err != nil { - reqres.WriteError(w, r, reqres.APIErrorServerError, - fmt.Sprintf("error getting grpc client on addr: %v", s.conf.GatewaySvc), err) - return - } - - providers, err := gatewayClient.ListAllProviders(ctx, &providerv1beta1.ListAllProvidersRequest{}) - if err != nil { - reqres.WriteError(w, r, reqres.APIErrorServerError, "error listing all providers", err) - return - } - - jsonResponse, err := json.Marshal(providers.Providers) - if err != nil { - reqres.WriteError(w, r, reqres.APIErrorServerError, "error marshalling providers data", err) - return - } - - // Write response - _, err = w.Write(jsonResponse) - if err != nil { - reqres.WriteError(w, r, reqres.APIErrorServerError, "error writing providers data", err) - return - } - - w.WriteHeader(http.StatusOK) -} - -// HTTP service handler. -func (s *svc) Handler() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var head string - head, r.URL.Path = router.ShiftPath(r.URL.Path) - switch head { - case "providers": - s.serveJSON(w, r) - return - default: - r.URL.Path = head + r.URL.Path - meshdirectoryweb.ServeMeshDirectorySPA(w, r) - return - } - }) -} From f900433d4efaad41b80bb563890b7b8f8ab5506e Mon Sep 17 00:00:00 2001 From: Swikriti Tripathi <41103328+SwikritiT@users.noreply.github.com> Date: Fri, 23 Jun 2023 11:12:42 +0545 Subject: [PATCH 21/48] Bump ocis commit id for tests (#4008) --- .drone.env | 2 +- tests/ocis | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.env b/.drone.env index 68253aef6e..4747c6fcd9 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=70b64d817e924ff7820e0f912b0569caf22fe9b8 +APITESTS_COMMITID=2ea3b8c400688b14ef328786b3af445c2edbbe18 APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/tests/ocis b/tests/ocis index 70b64d817e..2ea3b8c400 160000 --- a/tests/ocis +++ b/tests/ocis @@ -1 +1 @@ -Subproject commit 70b64d817e924ff7820e0f912b0569caf22fe9b8 +Subproject commit 2ea3b8c400688b14ef328786b3af445c2edbbe18 From 1c131b0758507b9d008b8a74cc6b45a9e177508d Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 23 Jun 2023 10:32:58 +0200 Subject: [PATCH 22/48] Recompiled docs (#4006) --- .../grpc/services/ocmshareprovider/_index.md | 2 +- .../http/services/owncloud/ocdav/_index.md | 10 ++- .../packages/app/provider/wopi/_index.md | 22 +++---- .../config/packages/notification/_index.md | 7 ++ .../packages/notification/handler/_index.md | 7 ++ .../handler/emailhandler/_index.md | 50 ++++++++++++++ .../notification/notificationhelper/_index.md | 42 ++++++++++++ .../en/docs/config/serverless/_index.md | 7 ++ .../docs/config/serverless/services/_index.md | 7 ++ .../services/notifications/_index.md | 66 +++++++++++++++++++ 10 files changed, 207 insertions(+), 13 deletions(-) create mode 100644 docs/content/en/docs/config/packages/notification/_index.md create mode 100644 docs/content/en/docs/config/packages/notification/handler/_index.md create mode 100644 docs/content/en/docs/config/packages/notification/handler/emailhandler/_index.md create mode 100644 docs/content/en/docs/config/packages/notification/notificationhelper/_index.md create mode 100644 docs/content/en/docs/config/serverless/_index.md create mode 100644 docs/content/en/docs/config/serverless/services/_index.md create mode 100644 docs/content/en/docs/config/serverless/services/notifications/_index.md diff --git a/docs/content/en/docs/config/grpc/services/ocmshareprovider/_index.md b/docs/content/en/docs/config/grpc/services/ocmshareprovider/_index.md index 950d60597a..3de755abf5 100644 --- a/docs/content/en/docs/config/grpc/services/ocmshareprovider/_index.md +++ b/docs/content/en/docs/config/grpc/services/ocmshareprovider/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="provider_domain" type="string" default="The same domain registered in the provider authorizer" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/ocmshareprovider/ocmshareprovider.go#L63) + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/ocmshareprovider/ocmshareprovider.go#L64) {{< highlight toml >}} [grpc.services.ocmshareprovider] provider_domain = "The same domain registered in the provider authorizer" diff --git a/docs/content/en/docs/config/http/services/owncloud/ocdav/_index.md b/docs/content/en/docs/config/http/services/owncloud/ocdav/_index.md index edeb84444c..e269611bb1 100644 --- a/docs/content/en/docs/config/http/services/owncloud/ocdav/_index.md +++ b/docs/content/en/docs/config/http/services/owncloud/ocdav/_index.md @@ -9,10 +9,18 @@ description: > # _struct: Config_ {{% dir name="insecure" type="bool" default=false %}} -Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/owncloud/ocdav/ocdav.go#L103) +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/owncloud/ocdav/ocdav.go#L104) {{< highlight toml >}} [http.services.owncloud.ocdav] insecure = false {{< /highlight >}} {{% /dir %}} +{{% dir name="notifications" type="map[string]interface{}" default=Settingsg for the Notification Helper %}} + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/owncloud/ocdav/ocdav.go#L115) +{{< highlight toml >}} +[http.services.owncloud.ocdav] +notifications = Settingsg for the Notification Helper +{{< /highlight >}} +{{% /dir %}} + diff --git a/docs/content/en/docs/config/packages/app/provider/wopi/_index.md b/docs/content/en/docs/config/packages/app/provider/wopi/_index.md index 6f190be387..d581c7c008 100644 --- a/docs/content/en/docs/config/packages/app/provider/wopi/_index.md +++ b/docs/content/en/docs/config/packages/app/provider/wopi/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="mime_types" type="[]string" default=nil %}} -Inherited from the appprovider. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L78) +Inherited from the appprovider. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L79) {{< highlight toml >}} [app.provider.wopi] mime_types = nil @@ -17,7 +17,7 @@ mime_types = nil {{% /dir %}} {{% dir name="iop_secret" type="string" default="" %}} -The IOP secret used to connect to the wopiserver. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L79) +The IOP secret used to connect to the wopiserver. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L80) {{< highlight toml >}} [app.provider.wopi] iop_secret = "" @@ -25,7 +25,7 @@ iop_secret = "" {{% /dir %}} {{% dir name="wopi_url" type="string" default="" %}} -The wopiserver's URL. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L80) +The wopiserver's URL. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L81) {{< highlight toml >}} [app.provider.wopi] wopi_url = "" @@ -33,7 +33,7 @@ wopi_url = "" {{% /dir %}} {{% dir name="app_name" type="string" default="" %}} -The App user-friendly name. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L81) +The App user-friendly name. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L82) {{< highlight toml >}} [app.provider.wopi] app_name = "" @@ -41,7 +41,7 @@ app_name = "" {{% /dir %}} {{% dir name="app_icon_uri" type="string" default="" %}} -A URI to a static asset which represents the app icon. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L82) +A URI to a static asset which represents the app icon. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L83) {{< highlight toml >}} [app.provider.wopi] app_icon_uri = "" @@ -49,7 +49,7 @@ app_icon_uri = "" {{% /dir %}} {{% dir name="folder_base_url" type="string" default="" %}} -The base URL to generate links to navigate back to the containing folder. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L83) +The base URL to generate links to navigate back to the containing folder. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L84) {{< highlight toml >}} [app.provider.wopi] folder_base_url = "" @@ -57,7 +57,7 @@ folder_base_url = "" {{% /dir %}} {{% dir name="app_url" type="string" default="" %}} -The App URL. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L84) +The App URL. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L85) {{< highlight toml >}} [app.provider.wopi] app_url = "" @@ -65,7 +65,7 @@ app_url = "" {{% /dir %}} {{% dir name="app_int_url" type="string" default="" %}} -The internal app URL in case of dockerized deployments. Defaults to AppURL [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L85) +The internal app URL in case of dockerized deployments. Defaults to AppURL [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L86) {{< highlight toml >}} [app.provider.wopi] app_int_url = "" @@ -73,7 +73,7 @@ app_int_url = "" {{% /dir %}} {{% dir name="app_api_key" type="string" default="" %}} -The API key used by the app, if applicable. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L86) +The API key used by the app, if applicable. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L87) {{< highlight toml >}} [app.provider.wopi] app_api_key = "" @@ -81,7 +81,7 @@ app_api_key = "" {{% /dir %}} {{% dir name="jwt_secret" type="string" default="" %}} -The JWT secret to be used to retrieve the token TTL. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L87) +The JWT secret to be used to retrieve the token TTL. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L88) {{< highlight toml >}} [app.provider.wopi] jwt_secret = "" @@ -89,7 +89,7 @@ jwt_secret = "" {{% /dir %}} {{% dir name="app_desktop_only" type="bool" default=false %}} -Specifies if the app can be opened only on desktop. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L88) +Specifies if the app can be opened only on desktop. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/app/provider/wopi/wopi.go#L89) {{< highlight toml >}} [app.provider.wopi] app_desktop_only = false diff --git a/docs/content/en/docs/config/packages/notification/_index.md b/docs/content/en/docs/config/packages/notification/_index.md new file mode 100644 index 0000000000..9e81b10b98 --- /dev/null +++ b/docs/content/en/docs/config/packages/notification/_index.md @@ -0,0 +1,7 @@ +--- +title: "notification" +linkTitle: "notification" +weight: 10 +description: > + Configuration for the notification service +--- \ No newline at end of file diff --git a/docs/content/en/docs/config/packages/notification/handler/_index.md b/docs/content/en/docs/config/packages/notification/handler/_index.md new file mode 100644 index 0000000000..1daf47e92e --- /dev/null +++ b/docs/content/en/docs/config/packages/notification/handler/_index.md @@ -0,0 +1,7 @@ +--- +title: "handler" +linkTitle: "handler" +weight: 10 +description: > + Configuration for the handler service +--- \ No newline at end of file diff --git a/docs/content/en/docs/config/packages/notification/handler/emailhandler/_index.md b/docs/content/en/docs/config/packages/notification/handler/emailhandler/_index.md new file mode 100644 index 0000000000..fa5c7ca72e --- /dev/null +++ b/docs/content/en/docs/config/packages/notification/handler/emailhandler/_index.md @@ -0,0 +1,50 @@ +--- +title: "emailhandler" +linkTitle: "emailhandler" +weight: 10 +description: > + Configuration for the emailhandler service +--- + +# _struct: config_ + +{{% dir name="smtp_server" type="string" default="" %}} +The hostname and port of the SMTP server. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/handler/emailhandler/emailhandler.go#L44) +{{< highlight toml >}} +[notification.handler.emailhandler] +smtp_server = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="sender_login" type="string" default="" %}} +The email to be used to send mails. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/handler/emailhandler/emailhandler.go#L45) +{{< highlight toml >}} +[notification.handler.emailhandler] +sender_login = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="sender_password" type="string" default="" %}} +The sender's password. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/handler/emailhandler/emailhandler.go#L46) +{{< highlight toml >}} +[notification.handler.emailhandler] +sender_password = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="disable_auth" type="bool" default=false %}} +Whether to disable SMTP auth. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/handler/emailhandler/emailhandler.go#L47) +{{< highlight toml >}} +[notification.handler.emailhandler] +disable_auth = false +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="default_sender" type="string" default="no-reply@cernbox.cern.ch" %}} +Default sender when not specified in the trigger. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/handler/emailhandler/emailhandler.go#L48) +{{< highlight toml >}} +[notification.handler.emailhandler] +default_sender = "no-reply@cernbox.cern.ch" +{{< /highlight >}} +{{% /dir %}} + diff --git a/docs/content/en/docs/config/packages/notification/notificationhelper/_index.md b/docs/content/en/docs/config/packages/notification/notificationhelper/_index.md new file mode 100644 index 0000000000..45101d615d --- /dev/null +++ b/docs/content/en/docs/config/packages/notification/notificationhelper/_index.md @@ -0,0 +1,42 @@ +--- +title: "notificationhelper" +linkTitle: "notificationhelper" +weight: 10 +description: > + Configuration for the notificationhelper service +--- + +# _struct: Config_ + +{{% dir name="nats_address" type="string" default="" %}} +The NATS server address. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/notificationhelper/notificationhelper.go#L47) +{{< highlight toml >}} +[notification.notificationhelper] +nats_address = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="nats_token" type="string" default="" %}} +The token to authenticate against the NATS server [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/notificationhelper/notificationhelper.go#L48) +{{< highlight toml >}} +[notification.notificationhelper] +nats_token = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="nats_stream" type="string" default="reva-notifications" %}} +The notifications NATS stream. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/notificationhelper/notificationhelper.go#L49) +{{< highlight toml >}} +[notification.notificationhelper] +nats_stream = "reva-notifications" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="templates" type="map[string]interface{}" default= %}} +Notification templates for the service. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/notificationhelper/notificationhelper.go#L50) +{{< highlight toml >}} +[notification.notificationhelper] +templates = +{{< /highlight >}} +{{% /dir %}} + diff --git a/docs/content/en/docs/config/serverless/_index.md b/docs/content/en/docs/config/serverless/_index.md new file mode 100644 index 0000000000..697072afb8 --- /dev/null +++ b/docs/content/en/docs/config/serverless/_index.md @@ -0,0 +1,7 @@ +--- +title: "serverless" +linkTitle: "serverless" +weight: 10 +description: > + Configuration for the serverless service +--- \ No newline at end of file diff --git a/docs/content/en/docs/config/serverless/services/_index.md b/docs/content/en/docs/config/serverless/services/_index.md new file mode 100644 index 0000000000..01e9c803b9 --- /dev/null +++ b/docs/content/en/docs/config/serverless/services/_index.md @@ -0,0 +1,7 @@ +--- +title: "services" +linkTitle: "services" +weight: 10 +description: > + Configuration for the services service +--- \ No newline at end of file diff --git a/docs/content/en/docs/config/serverless/services/notifications/_index.md b/docs/content/en/docs/config/serverless/services/notifications/_index.md new file mode 100644 index 0000000000..f4478eb33e --- /dev/null +++ b/docs/content/en/docs/config/serverless/services/notifications/_index.md @@ -0,0 +1,66 @@ +--- +title: "notifications" +linkTitle: "notifications" +weight: 10 +description: > + Configuration for the notifications service +--- + +# _struct: config_ + +{{% dir name="nats_address" type="string" default="" %}} +The NATS server address. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L46) +{{< highlight toml >}} +[serverless.services.notifications] +nats_address = "" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="nats_token" type="string" default="The token to authenticate against the NATS server" %}} + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L47) +{{< highlight toml >}} +[serverless.services.notifications] +nats_token = "The token to authenticate against the NATS server" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="nats_prefix" type="string" default="reva-notifications" %}} +The notifications NATS stream. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L48) +{{< highlight toml >}} +[serverless.services.notifications] +nats_prefix = "reva-notifications" +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="handlers" type="map[string]interface{}" default= %}} +Settings for the different notification handlers. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L49) +{{< highlight toml >}} +[serverless.services.notifications] +handlers = +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="grouping_interval" type="int" default=60 %}} +Time in seconds to group incoming notification triggers [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L50) +{{< highlight toml >}} +[serverless.services.notifications] +grouping_interval = 60 +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="grouping_max_size" type="int" default=100 %}} +Maximum number of notifications to group [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L51) +{{< highlight toml >}} +[serverless.services.notifications] +grouping_max_size = 100 +{{< /highlight >}} +{{% /dir %}} + +{{% dir name="storage_driver" type="string" default="mysql" %}} +The driver used to store notifications [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L52) +{{< highlight toml >}} +[serverless.services.notifications] +storage_driver = "mysql" +{{< /highlight >}} +{{% /dir %}} + From b3b619cd63a1e702ded469d70e6b0960945ae603 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 23 Jun 2023 15:09:49 +0200 Subject: [PATCH 23/48] Improved logging of HTTP requests (#4011) --- changelog/unreleased/httplog.md | 6 ++++++ internal/http/interceptors/log/log.go | 16 ++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 changelog/unreleased/httplog.md diff --git a/changelog/unreleased/httplog.md b/changelog/unreleased/httplog.md new file mode 100644 index 0000000000..246c020642 --- /dev/null +++ b/changelog/unreleased/httplog.md @@ -0,0 +1,6 @@ +Enhancement: improve logging of HTTP requests + +Added request and response headers and removed redundant +URL from the "http" messages + +https://github.com/cs3org/reva/pull/4011 diff --git a/internal/http/interceptors/log/log.go b/internal/http/interceptors/log/log.go index 14f10d080c..e383a51ed2 100644 --- a/internal/http/interceptors/log/log.go +++ b/internal/http/interceptors/log/log.go @@ -55,7 +55,7 @@ func (h loggingHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { logger := makeLogger(w) url := *req.URL h.handler.ServeHTTP(logger, req) - writeLog(log, req, url, t, logger.Status(), logger.Size()) + writeLog(log, req, url, t, logger.Status(), logger.Size(), logger.Header()) } func makeLogger(w http.ResponseWriter) loggingResponseWriter { @@ -70,7 +70,7 @@ func makeLogger(w http.ResponseWriter) loggingResponseWriter { return logger } -func writeLog(log *zerolog.Logger, req *http.Request, url url.URL, ts time.Time, status, size int) { +func writeLog(log *zerolog.Logger, req *http.Request, url url.URL, ts time.Time, status, size int, resHeaders http.Header) { end := time.Now() host, _, err := net.SplitHostPort(req.RemoteAddr) @@ -79,8 +79,6 @@ func writeLog(log *zerolog.Logger, req *http.Request, url url.URL, ts time.Time, } uri := req.RequestURI - u := req.URL.String() - if req.ProtoMajor == 2 && req.Method == "CONNECT" { uri = req.Host } @@ -93,16 +91,18 @@ func writeLog(log *zerolog.Logger, req *http.Request, url url.URL, ts time.Time, var event *zerolog.Event switch { case status < 400: - event = log.Debug() + event = log.Info() case status < 500: event = log.Warn() default: event = log.Error() } + event.Str("host", host).Str("method", req.Method).Str("uri", uri).Int("status", status). + Msg("processed http request") - event.Str("host", host).Str("method", req.Method). - Str("uri", uri).Str("url", u).Str("proto", req.Proto).Int("status", status). - Int("size", size). + log.Trace().Str("host", host).Str("method", req.Method). + Str("uri", uri).Str("proto", req.Proto).Interface("req_headers", req.Header). + Int("status", status).Int("size", size).Interface("res_headers", resHeaders). Str("start", ts.Format("02/Jan/2006:15:04:05 -0700")). Str("end", end.Format("02/Jan/2006:15:04:05 -0700")).Int("time_ns", int(diff)). Msg("http") From ba3d5b10b4e0fa3de4ca2c1f1cfdca8baae9061e Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Mon, 26 Jun 2023 10:09:13 +0200 Subject: [PATCH 24/48] Split out 'access content' from 'list shares' Added a section header that was missing --- docs/content/en/docs/tutorials/share-tutorial.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/content/en/docs/tutorials/share-tutorial.md b/docs/content/en/docs/tutorials/share-tutorial.md index 153c68a08e..72268761d0 100644 --- a/docs/content/en/docs/tutorials/share-tutorial.md +++ b/docs/content/en/docs/tutorials/share-tutorial.md @@ -146,13 +146,14 @@ Call the `ocm-share-list-received` method. +--------------------------------------+-----------------+--------------------------------------+-------------------------------------------------------------------------------+-------------------+-------------+--------------------------------------+--------------------------------+--------------------------------+---------------------+-----------------+ ``` -The share's recipien has received the share `ef05c999-8ae2-41af-ba0d-a886b061011f`. The user can get more informations about the share using the `ocm-share-get-received` command. +The share's recipient has received the share `ef05c999-8ae2-41af-ba0d-a886b061011f`. The user can get more informations about the share using the `ocm-share-get-received` command. ``` ocm-share-get-received ef05c999-8ae2-41af-ba0d-a886b061011f {"id":{"opaqueId":"ef05c999-8ae2-41af-ba0d-a886b061011f"}, "name":"my-folder", "resourceId":{"opaqueId":"123e4567-e89b-12d3-a456-426655440000:fileid-einstein%2Fmy-folder"}, "grantee":{"type":"GRANTEE_TYPE_USER", "userId":{"idp":"cesnet.cz", "opaqueId":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", "type":"USER_TYPE_PRIMARY"}}, "owner":{"idp":"cernbox.cern.ch", "opaqueId":"4c510ada-c86b-4815-8820-42cdf82c3d51", "type":"USER_TYPE_FEDERATED"}, "creator":{"idp":"cernbox.cern.ch", "opaqueId":"4c510ada-c86b-4815-8820-42cdf82c3d51", "type":"USER_TYPE_FEDERATED"}, "ctime":{"seconds":"1681206728", "nanos":346009879}, "mtime":{"seconds":"1681206728", "nanos":346009879}, "shareType":"SHARE_TYPE_USER", "protocols":[{"webdavOptions":{"permissions":{"permissions":{"getPath":true, "initiateFileDownload":true, "initiateFileUpload":true, "listContainer":true, "stat":true}}, "uri":"http://localhost:19001/remote.php/dav/ocm/eSWNjTWjorFmZEGQNZVyrU3TyxdWEr1D"}}], "state":"SHARE_STATE_PENDING", "resourceType":"RESOURCE_TYPE_CONTAINER"} ``` +#### 5.2.3 Access content that was shared In this case, the share can be accessed using the WebDAV protocol (multiple access methods are available, like WebDAV, Webapp and Datatx) using the URL `http://localhost:19001/remote.php/dav/ocm/eSWNjTWjorFmZEGQNZVyrU3TyxdWEr1D`, and every WebDAV client can be used to access the received share resource. For example: From 63cd6aa626cb6f01be347c00876e70d28823edcd Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:55:15 +0200 Subject: [PATCH 25/48] Fix panic when closing nats connection (#4016) * fix panic when closing nats connection * add changelog --- changelog/unreleased/fix-panic-nats-close.md | 6 ++++++ pkg/notification/notificationhelper/notificationhelper.go | 4 ++++ 2 files changed, 10 insertions(+) create mode 100644 changelog/unreleased/fix-panic-nats-close.md diff --git a/changelog/unreleased/fix-panic-nats-close.md b/changelog/unreleased/fix-panic-nats-close.md new file mode 100644 index 0000000000..97499d71c1 --- /dev/null +++ b/changelog/unreleased/fix-panic-nats-close.md @@ -0,0 +1,6 @@ +Bugfix: Fix panic when closing notification service + +If the connection to the nats server was not yet estabished, +the service on close was panicking. This has been now fixed. + +https://github.com/cs3org/reva/pull/4016 diff --git a/pkg/notification/notificationhelper/notificationhelper.go b/pkg/notification/notificationhelper/notificationhelper.go index f9ef0420eb..c39e97b8a6 100644 --- a/pkg/notification/notificationhelper/notificationhelper.go +++ b/pkg/notification/notificationhelper/notificationhelper.go @@ -125,6 +125,10 @@ func (nh *NotificationHelper) connect() error { // Stop stops the notification helper. func (nh *NotificationHelper) Stop() { + if nh.nc == nil { + // service didn't connect yet to nat server + return + } if err := nh.nc.Drain(); err != nil { nh.Log.Error().Err(err) } From bf70373df0cdefa7296c8b8913ee9cb75d97885f Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Mon, 3 Jul 2023 07:44:08 +0200 Subject: [PATCH 26/48] New reva config (#4015) --- changelog/unreleased/new-config.md | 11 + cmd/revad/main.go | 120 +++- cmd/revad/pkg/config/common.go | 226 +++++++ cmd/revad/pkg/config/config.go | 169 +++++ cmd/revad/pkg/config/config_test.go | 592 ++++++++++++++++++ cmd/revad/pkg/config/dump.go | 146 +++++ cmd/revad/pkg/config/dump_test.go | 247 ++++++++ cmd/revad/pkg/config/grpc.go | 77 +++ cmd/revad/pkg/config/http.go | 77 +++ cmd/revad/pkg/config/lookup.go | 257 ++++++++ cmd/revad/pkg/config/lookup_test.go | 198 ++++++ cmd/revad/pkg/config/parser.go | 100 +++ cmd/revad/pkg/config/parser_test.go | 97 +++ cmd/revad/pkg/config/serverless.go | 54 ++ cmd/revad/pkg/config/templates.go | 279 +++++++++ cmd/revad/pkg/config/templates_test.go | 135 ++++ cmd/revad/{internal => pkg}/grace/grace.go | 212 +++++-- cmd/revad/runtime/grpc.go | 150 +++++ cmd/revad/runtime/http.go | 88 +++ cmd/revad/runtime/option.go | 13 +- cmd/revad/runtime/runtime.go | 541 ++++++++-------- go.mod | 1 + go.sum | 2 + .../applicationauth/applicationauth.go | 2 +- .../grpc/services/appprovider/appprovider.go | 2 +- .../grpc/services/appregistry/appregistry.go | 2 +- .../services/appregistry/appregistry_test.go | 2 +- .../services/authprovider/authprovider.go | 2 +- .../services/authregistry/authregistry.go | 2 +- internal/grpc/services/datatx/datatx.go | 2 +- internal/grpc/services/gateway/gateway.go | 2 +- .../services/groupprovider/groupprovider.go | 2 +- .../grpc/services/helloworld/helloworld.go | 2 +- internal/grpc/services/ocmcore/ocmcore.go | 2 +- .../ocminvitemanager/ocminvitemanager.go | 2 +- .../ocmproviderauthorizer.go | 2 +- .../ocmshareprovider/ocmshareprovider.go | 2 +- .../grpc/services/permissions/permissions.go | 2 +- .../grpc/services/preferences/preferences.go | 2 +- .../publicshareprovider.go | 2 +- .../publicstorageprovider.go | 2 +- .../storageprovider/storageprovider.go | 2 +- .../storageregistry/storageregistry.go | 2 +- .../services/userprovider/userprovider.go | 2 +- .../usershareprovider/usershareprovider.go | 2 +- pkg/rgrpc/option.go | 62 ++ pkg/rgrpc/rgrpc.go | 235 ++----- pkg/rhttp/rhttp.go | 232 +++---- .../config.go => pkg/rserverless/option.go | 29 +- pkg/rserverless/rserverless.go | 66 +- pkg/sharedconf/sharedconf.go | 48 +- pkg/sharedconf/sharedconf_test.go | 62 +- pkg/utils/list/list.go | 22 + pkg/utils/maps/maps.go | 50 ++ pkg/utils/net/net.go | 54 ++ 55 files changed, 3853 insertions(+), 843 deletions(-) create mode 100644 changelog/unreleased/new-config.md create mode 100644 cmd/revad/pkg/config/common.go create mode 100644 cmd/revad/pkg/config/config.go create mode 100644 cmd/revad/pkg/config/config_test.go create mode 100644 cmd/revad/pkg/config/dump.go create mode 100644 cmd/revad/pkg/config/dump_test.go create mode 100644 cmd/revad/pkg/config/grpc.go create mode 100644 cmd/revad/pkg/config/http.go create mode 100644 cmd/revad/pkg/config/lookup.go create mode 100644 cmd/revad/pkg/config/lookup_test.go create mode 100644 cmd/revad/pkg/config/parser.go create mode 100644 cmd/revad/pkg/config/parser_test.go create mode 100644 cmd/revad/pkg/config/serverless.go create mode 100644 cmd/revad/pkg/config/templates.go create mode 100644 cmd/revad/pkg/config/templates_test.go rename cmd/revad/{internal => pkg}/grace/grace.go (68%) create mode 100644 cmd/revad/runtime/grpc.go create mode 100644 cmd/revad/runtime/http.go create mode 100644 pkg/rgrpc/option.go rename cmd/revad/internal/config/config.go => pkg/rserverless/option.go (61%) create mode 100644 pkg/utils/maps/maps.go create mode 100644 pkg/utils/net/net.go diff --git a/changelog/unreleased/new-config.md b/changelog/unreleased/new-config.md new file mode 100644 index 0000000000..9f9f1f0124 --- /dev/null +++ b/changelog/unreleased/new-config.md @@ -0,0 +1,11 @@ +Enhancement: New configuration + +Allow multiple driverts of the same service to be in the +same toml config. Add a `vars` section to contain common +parameters addressable using templates in the configuration +of the different drivers. Support templating to reference +values of other parameters in the configuration. +Assign random ports to services where the address is not +specified. + +https://github.com/cs3org/reva/pull/4015 diff --git a/cmd/revad/main.go b/cmd/revad/main.go index cfb1a5cdb4..991073ace7 100644 --- a/cmd/revad/main.go +++ b/cmd/revad/main.go @@ -21,6 +21,7 @@ package main import ( "flag" "fmt" + "io" "io/fs" "os" "path" @@ -28,11 +29,14 @@ import ( "sync" "syscall" - "github.com/cs3org/reva/cmd/revad/internal/config" - "github.com/cs3org/reva/cmd/revad/internal/grace" + "github.com/cs3org/reva/cmd/revad/pkg/config" + "github.com/cs3org/reva/cmd/revad/pkg/grace" "github.com/cs3org/reva/cmd/revad/runtime" + "github.com/cs3org/reva/pkg/logger" "github.com/cs3org/reva/pkg/sysinfo" "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/rs/zerolog" ) var ( @@ -41,13 +45,16 @@ var ( signalFlag = flag.String("s", "", "send signal to a master process: stop, quit, reload") configFlag = flag.String("c", "/etc/revad/revad.toml", "set configuration file") pidFlag = flag.String("p", "", "pid file. If empty defaults to a random file in the OS temporary directory") - logFlag = flag.String("log", "", "log messages with the given severity or above. One of: [trace, debug, info, warn, error, fatal, panic]") dirFlag = flag.String("dev-dir", "", "runs any toml file in the specified directory. Intended for development use only") // Compile time variables initialized with gcc flags. gitCommit, buildDate, version, goVersion string ) +var ( + revaProcs []*runtime.Reva +) + func main() { flag.Parse() @@ -126,7 +133,7 @@ func handleSignalFlag() { // kill process with signal if err := process.Signal(signal); err != nil { - fmt.Fprintf(os.Stderr, "error signaling process %d with signal %s\n", process.Pid, signal) + fmt.Fprintf(os.Stderr, "error signaling process %d with signal %s: %v\n", process.Pid, signal, err) os.Exit(1) } @@ -134,7 +141,7 @@ func handleSignalFlag() { } } -func getConfigs() ([]map[string]interface{}, error) { +func getConfigs() ([]*config.Config, error) { var confs []string // give priority to read from dev-dir if *dirFlag != "" { @@ -186,8 +193,8 @@ func getConfigsFromDir(dir string) (confs []string, err error) { return } -func readConfigs(files []string) ([]map[string]interface{}, error) { - confs := make([]map[string]interface{}, 0, len(files)) +func readConfigs(files []string) ([]*config.Config, error) { + confs := make([]*config.Config, 0, len(files)) for _, conf := range files { fd, err := os.Open(conf) if err != nil { @@ -195,49 +202,114 @@ func readConfigs(files []string) ([]map[string]interface{}, error) { } defer fd.Close() - v, err := config.Read(fd) + c, err := config.Load(fd) if err != nil { return nil, err } - confs = append(confs, v) + confs = append(confs, c) } return confs, nil } -func runConfigs(confs []map[string]interface{}) { +func runConfigs(confs []*config.Config) { + pidfile := getPidfile() if len(confs) == 1 { - runSingle(confs[0]) + runSingle(confs[0], pidfile) return } runMultiple(confs) } -func runSingle(conf map[string]interface{}) { - if *pidFlag == "" { - *pidFlag = getPidfile() - } - - runtime.Run(conf, *pidFlag, *logFlag) +func registerReva(r *runtime.Reva) { + revaProcs = append(revaProcs, r) } -func getPidfile() string { - uuid := uuid.New().String() - name := fmt.Sprintf("revad-%s.pid", uuid) +func runSingle(conf *config.Config, pidfile string) { + log := initLogger(conf.Log) + reva, err := runtime.New(conf, + runtime.WithPidFile(pidfile), + runtime.WithLogger(log), + ) + if err != nil { + abort(log, "error creating reva runtime: %v", err) + } + registerReva(reva) + if err := reva.Start(); err != nil { + abort(log, "error starting reva: %v", err) + } +} - return path.Join(os.TempDir(), name) +func abort(log *zerolog.Logger, format string, a ...any) { + log.Fatal().Msgf(format, a...) } -func runMultiple(confs []map[string]interface{}) { +func runMultiple(confs []*config.Config) { var wg sync.WaitGroup + for _, conf := range confs { wg.Add(1) pidfile := getPidfile() - go func(wg *sync.WaitGroup, conf map[string]interface{}) { + go func(wg *sync.WaitGroup, conf *config.Config) { defer wg.Done() - runtime.Run(conf, pidfile, *logFlag) + runSingle(conf, pidfile) }(&wg, conf) } wg.Wait() os.Exit(0) } + +func getPidfile() string { + uuid := uuid.New().String() + name := fmt.Sprintf("revad-%s.pid", uuid) + + return path.Join(os.TempDir(), name) +} + +func initLogger(conf *config.Log) *zerolog.Logger { + log, err := newLogger(conf) + if err != nil { + fmt.Fprintf(os.Stderr, "error creating logger: %v", err) + os.Exit(1) + } + return log +} + +func newLogger(conf *config.Log) (*zerolog.Logger, error) { + // TODO(labkode): use debug level rather than info as default until reaching a stable version. + // Helps having smaller development files. + if conf.Level == "" { + conf.Level = zerolog.DebugLevel.String() + } + + var opts []logger.Option + opts = append(opts, logger.WithLevel(conf.Level)) + + w, err := getWriter(conf.Output) + if err != nil { + return nil, err + } + + opts = append(opts, logger.WithWriter(w, logger.Mode(conf.Mode))) + + l := logger.New(opts...) + sub := l.With().Int("pid", os.Getpid()).Logger() + return &sub, nil +} + +func getWriter(out string) (io.Writer, error) { + if out == "stderr" || out == "" { + return os.Stderr, nil + } + + if out == "stdout" { + return os.Stdout, nil + } + + fd, err := os.OpenFile(out, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return nil, errors.Wrap(err, "error creating log file: "+out) + } + + return fd, nil +} diff --git a/cmd/revad/pkg/config/common.go b/cmd/revad/pkg/config/common.go new file mode 100644 index 0000000000..fa4ce3148b --- /dev/null +++ b/cmd/revad/pkg/config/common.go @@ -0,0 +1,226 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +import ( + "fmt" + "net" + + "github.com/mitchellh/mapstructure" +) + +type iterable interface { + services() map[string]ServicesConfig + interceptors() map[string]map[string]any +} + +type iterableImpl struct{ i iterable } + +// ServicesConfig holds the configuration for reva services. +type ServicesConfig []*DriverConfig + +// DriversNumber return the number of driver configured for the service. +func (s ServicesConfig) DriversNumber() int { return len(s) } + +// DriverConfig holds the configuration for a driver. +type DriverConfig struct { + Address Address `key:"address"` + Network string `key:"network"` + Label string `key:"-"` + Config map[string]any `key:",squash"` // this must be at the bottom! +} + +// Add appends the driver configuration to the given list of services. +func (s *ServicesConfig) Add(domain, svc string, c *DriverConfig) { + l := len(*s) + if l == 0 { + // the label is simply the service name + c.Label = domain + "_" + svc + } else { + c.Label = label(domain, svc, l) + if l == 1 { + (*s)[0].Label = label(domain, svc, 0) + } + } + *s = append(*s, c) +} + +func newSvcConfigFromList(domain, name string, l []map[string]any) (ServicesConfig, error) { + cfg := make(ServicesConfig, 0, len(l)) + for _, c := range l { + cfg.Add(domain, name, &DriverConfig{Config: c}) + } + return cfg, nil +} + +func newSvcConfigFromMap(domain, name string, m map[string]any) ServicesConfig { + s, _ := newSvcConfigFromList(domain, name, []map[string]any{m}) + return s +} + +func parseServices(domain string, cfg map[string]any) (map[string]ServicesConfig, error) { + // parse services + svcCfg, ok := cfg["services"].(map[string]any) + if !ok { + return nil, fmt.Errorf("%s.services must be a map", domain) + } + + services := make(map[string]ServicesConfig) + for name, cfg := range svcCfg { + // cfg can be a list or a map + cfgLst, ok := cfg.([]map[string]any) + if ok { + s, err := newSvcConfigFromList(domain, name, cfgLst) + if err != nil { + return nil, err + } + services[name] = s + continue + } + cfgMap, ok := cfg.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s.services.%s must be a list or a map. got %T", domain, name, cfg) + } + services[name] = newSvcConfigFromMap(domain, name, cfgMap) + } + + return services, nil +} + +func parseMiddlwares(cfg map[string]any, key string) (map[string]map[string]any, error) { + m := make(map[string]map[string]any) + + mid, ok := cfg[key] + if !ok { + return m, nil + } + + if err := mapstructure.Decode(mid, &m); err != nil { + return nil, err + } + return m, nil +} + +// Service contains the configuration for a service. +type Service struct { + Address Address + Network string + Name string + Label string + Config map[string]any + + raw *DriverConfig +} + +// SetAddress sets the address for the service in the configuration. +func (s *Service) SetAddress(address Address) { + s.Address = address + s.raw.Address = address +} + +// ServiceFunc is an helper function used to pass the service config +// to the ForEachService func. +type ServiceFunc func(*Service) + +// Interceptor contains the configuration for an interceptor. +type Interceptor struct { + Name string + Config map[string]any +} + +// InterceptorFunc is an helper function used to pass the interface config +// to the ForEachInterceptor func. +type InterceptorFunc func(*Interceptor) + +// ForEachService iterates to each service/driver calling the function f. +func (i iterableImpl) ForEachService(f ServiceFunc) { + if i.i == nil { + return + } + for name, c := range i.i.services() { + for _, cfg := range c { + f(&Service{ + raw: cfg, + Address: cfg.Address, + Network: cfg.Network, + Label: cfg.Label, + Name: name, + Config: cfg.Config, + }) + } + } +} + +func label(domain, name string, i int) string { + return fmt.Sprintf("%s_%s_%d", domain, name, i) +} + +// ForEachInterceptor iterates to each middleware calling the function f. +func (i iterableImpl) ForEachInterceptor(f InterceptorFunc) { + for name, c := range i.i.interceptors() { + f(&Interceptor{ + Name: name, + Config: c, + }) + } +} + +func addressForService(global Address, cfg map[string]any) Address { + if address, ok := cfg["address"].(string); ok { + return Address(address) + } + return global +} + +func networkForService(global string, cfg map[string]any) string { + if network, ok := cfg["network"].(string); ok { + return network + } + return global +} + +// Address is the data structure holding an address. +type Address string + +// ensure Address implements the Lookuper interface. +var _ Lookuper = (*Address)(nil) + +// String return the string representation of the address. +func (a Address) String() string { return string(a) } + +// Get returns the value associated to the given key. +// The key available for an Address type are "port" and "ip", +// allowing respectively to get the port and the ip from the address. +func (a Address) Lookup(k string) (any, error) { + switch k { + case "port": + t, err := net.ResolveTCPAddr("tcp", a.String()) + if err != nil { + return nil, err + } + return t.Port, nil + case "ip": + t, err := net.ResolveTCPAddr("tcp", a.String()) + if err != nil { + return nil, err + } + return t.IP.String(), nil + } + return nil, ErrKeyNotFound{Key: k} +} diff --git a/cmd/revad/pkg/config/config.go b/cmd/revad/pkg/config/config.go new file mode 100644 index 0000000000..846b03374e --- /dev/null +++ b/cmd/revad/pkg/config/config.go @@ -0,0 +1,169 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +import ( + "fmt" + "io" + "reflect" + + "github.com/BurntSushi/toml" + "github.com/creasty/defaults" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +// Config holds the reva configuration. +type Config struct { + GRPC *GRPC `key:"grpc" mapstructure:"-" default:"{}"` + HTTP *HTTP `key:"http" mapstructure:"-" default:"{}"` + Serverless *Serverless `key:"serverless" mapstructure:"-" default:"{}"` + Shared *Shared `key:"shared" mapstructure:"shared" default:"{}"` + Log *Log `key:"log" mapstructure:"log" default:"{}" template:"-"` + Core *Core `key:"core" mapstructure:"core" default:"{}" template:"-"` + Vars Vars `key:"vars" mapstructure:"vars" default:"{}" template:"-"` +} + +// Log holds the configuration for the logger. +type Log struct { + Output string `key:"output" mapstructure:"output" default:"stdout"` + Mode string `key:"mode" mapstructure:"mode" default:"console"` + Level string `key:"level" mapstructure:"level" default:"trace"` +} + +// Shared holds the shared configuration. +type Shared struct { + JWTSecret string `key:"jwt_secret" mapstructure:"jwt_secret" default:"changemeplease"` + GatewaySVC string `key:"gatewaysvc" mapstructure:"gatewaysvc" default:"0.0.0.0:19000"` + DataGateway string `key:"datagateway" mapstructure:"datagateway" default:"http://0.0.0.0:19001/datagateway"` + SkipUserGroupsInToken bool `key:"skip_user_groups_in_token" mapstructure:"skip_user_groups_in_token"` + BlockedUsers []string `key:"blocked_users" mapstructure:"blocked_users" default:"[]"` +} + +// Core holds the core configuration. +type Core struct { + MaxCPUs string `key:"max_cpus" mapstructure:"max_cpus"` + TracingEnabled bool `key:"tracing_enabled" mapstructure:"tracing_enabled" default:"true"` + TracingEndpoint string `key:"tracing_endpoint" mapstructure:"tracing_endpoint" default:"localhost:6831"` + TracingCollector string `key:"tracing_collector" mapstructure:"tracing_collector"` + TracingServiceName string `key:"tracing_service_name" mapstructure:"tracing_service_name"` + TracingService string `key:"tracing_service" mapstructure:"tracing_service"` +} + +// Vars holds the a set of configuration paramenters that +// can be references by other parts of the configuration. +type Vars map[string]any + +// Lookuper is the interface for getting the value +// associated with a given key. +type Lookuper interface { + // Lookup get the value associated to thye given key. + // It returns ErrKeyNotFound if the key does not exists. + Lookup(key string) (any, error) +} + +// Load loads the configuration from the reader. +func Load(r io.Reader) (*Config, error) { + var c Config + if err := defaults.Set(&c); err != nil { + return nil, err + } + var raw map[string]any + if _, err := toml.NewDecoder(r).Decode(&raw); err != nil { + return nil, errors.Wrap(err, "config: error decoding toml data") + } + if err := c.parse(raw); err != nil { + return nil, err + } + return &c, nil +} + +func (c *Config) parse(raw map[string]any) error { + if err := c.parseGRPC(raw); err != nil { + return err + } + if err := c.parseHTTP(raw); err != nil { + return err + } + if err := c.parseServerless(raw); err != nil { + return err + } + if err := mapstructure.Decode(raw, c); err != nil { + return err + } + return nil +} + +// ApplyTemplates applies the templates defined in the configuration, +// replacing the template string with the value pointed by the given key. +func (c *Config) ApplyTemplates(l Lookuper) error { + return applyTemplateByType(l, nil, reflect.ValueOf(c)) +} + +// Dump returns the configuration as a map. +func (c *Config) Dump() map[string]any { + v := dumpByType(reflect.ValueOf(c)) + dump, ok := v.(map[string]any) + if !ok { + panic(fmt.Sprintf("dump should be a map: got %T", dump)) + } + return dump +} + +// Lookup gets the value associated to the given key in the config. +// The key is in the form .[], allowing accessing +// recursively the config on subfields, in case of maps or structs or +// types implementing the Getter interface, or elements in a list by the +// given index. +func (c *Config) Lookup(key string) (any, error) { + // check thet key is valid, meaning it starts with one of + // the fields of the config struct + if !c.isValidKey(key) { + return nil, nil + } + val, err := lookupByType(key, reflect.ValueOf(c)) + if err != nil { + return nil, errors.Wrapf(err, "lookup: error on key '%s'", key) + } + return val, nil +} + +func (c *Config) isValidKey(key string) bool { + cmd, _, err := parseNext(key) + if err != nil { + return false + } + f, ok := cmd.(FieldByKey) + if !ok { + return false + } + k := f.Key + e := reflect.TypeOf(c).Elem() + for i := 0; i < e.NumField(); i++ { + f := e.Field(i) + prefix := f.Tag.Get("key") + if prefix == "" || prefix == "-" { + continue + } + if k == prefix { + return true + } + } + return false +} diff --git a/cmd/revad/pkg/config/config_test.go b/cmd/revad/pkg/config/config_test.go new file mode 100644 index 0000000000..7ab22f8a03 --- /dev/null +++ b/cmd/revad/pkg/config/config_test.go @@ -0,0 +1,592 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLoadGlobalGRPCAddress(t *testing.T) { + config := ` +[grpc] +address = "localhost:9142" + +[[grpc.services.authprovider]] +driver = "demo" +address = "localhost:9000" + +[grpc.services.authprovider.drivers.demo] +key = "value" + +[[grpc.services.authprovider]] +driver = "machine" +address = "localhost:9001" + +[grpc.services.authprovider.drivers.machine] +key = "value" + +[grpc.services.gateway] +something = "test"` + + c, err := Load(strings.NewReader(config)) + if err != nil { + t.Fatalf("not expected error: %v", err) + } + + assert.Equal(t, Address("localhost:9142"), c.GRPC.Address) + + exp := map[string]ServicesConfig{ + "authprovider": []*DriverConfig{ + { + Address: "localhost:9000", + Config: map[string]any{ + "driver": "demo", + "drivers": map[string]any{ + "demo": map[string]any{ + "key": "value", + }, + }, + "address": "localhost:9000", + }, + Network: "tcp", + Label: "grpc_authprovider_0", + }, + { + Address: "localhost:9001", + Config: map[string]any{ + "driver": "machine", + "address": "localhost:9001", + "drivers": map[string]any{ + "machine": map[string]any{ + "key": "value", + }, + }, + }, + Network: "tcp", + Label: "grpc_authprovider_1", + }, + }, + "gateway": []*DriverConfig{ + { + Address: "localhost:9142", + Config: map[string]any{ + "something": "test", + }, + Network: "tcp", + Label: "grpc_gateway", + }, + }, + } + assert.Equal(t, exp, c.GRPC.Services) +} + +func TestLoadNoGRPCDefaultAddress(t *testing.T) { + config := ` +[[grpc.services.authprovider]] +driver = "demo" +address = "localhost:9000" + +[grpc.services.authprovider.drivers.demo] +key = "value" + +[[grpc.services.authprovider]] +driver = "machine" +address = "localhost:9001" + +[grpc.services.authprovider.drivers.machine] +key = "value" + +[grpc.services.gateway] +something = "test"` + + c, err := Load(strings.NewReader(config)) + if err != nil { + t.Fatalf("not expected error: %v", err) + } + + assert.Equal(t, Address(""), c.GRPC.Address) + + exp := map[string]ServicesConfig{ + "authprovider": []*DriverConfig{ + { + Address: "localhost:9000", + Config: map[string]any{ + "driver": "demo", + "drivers": map[string]any{ + "demo": map[string]any{ + "key": "value", + }, + }, + "address": "localhost:9000", + }, + Network: "tcp", + Label: "grpc_authprovider_0", + }, + { + Address: "localhost:9001", + Config: map[string]any{ + "driver": "machine", + "address": "localhost:9001", + "drivers": map[string]any{ + "machine": map[string]any{ + "key": "value", + }, + }, + }, + Network: "tcp", + Label: "grpc_authprovider_1", + }, + }, + "gateway": []*DriverConfig{ + { + Config: map[string]any{ + "something": "test", + }, + Network: "tcp", + Label: "grpc_gateway", + }, + }, + } + assert.Equal(t, exp, c.GRPC.Services) +} + +func TestLoadFullConfig(t *testing.T) { + config := ` +[shared] +gatewaysvc = "localhost:9142" +jwt_secret = "secret" + +[log] +output = "/var/log/revad/revad-gateway.log" +mode = "json" +level = "trace" + +[core] +max_cpus = "1" +tracing_enabled = true + +[vars] +db_username = "root" +db_password = "secretpassword" + +[grpc] +shutdown_deadline = 10 +enable_reflection = true + +[grpc.services.gateway] +authregistrysvc = "{{ grpc.services.authregistry.address }}" + +[grpc.services.authregistry] +driver = "static" + +[grpc.services.authregistry.drivers.static.rules] +basic = "{{ grpc.services.authprovider[0].address }}" +machine = "{{ grpc.services.authprovider[1].address }}" + +[[grpc.services.authprovider]] +driver = "ldap" +address = "localhost:19000" + +[grpc.services.authprovider.drivers.ldap] +password = "ldap" + +[[grpc.services.authprovider]] +driver = "machine" +address = "localhost:19001" + +[grpc.services.authprovider.drivers.machine] +api_key = "secretapikey" + +[http] +address = "localhost:19002" + +[http.services.dataprovider] +driver = "localhome" + +[http.services.sysinfo] + +[serverless.services.notifications] +nats_address = "nats-server-01.example.com" +nats_token = "secret-token-example"` + + c2, err := Load(strings.NewReader(config)) + assert.ErrorIs(t, err, nil) + + assert.Equal(t, &Shared{ + GatewaySVC: "localhost:9142", + JWTSecret: "secret", + DataGateway: "http://0.0.0.0:19001/datagateway", + BlockedUsers: []string{}, + }, c2.Shared) + + assert.Equal(t, &Log{ + Output: "/var/log/revad/revad-gateway.log", + Mode: "json", + Level: "trace", + }, c2.Log) + + assert.Equal(t, &Core{ + MaxCPUs: "1", + TracingEnabled: true, + TracingEndpoint: "localhost:6831", + }, c2.Core) + + assert.Equal(t, Vars{ + "db_username": "root", + "db_password": "secretpassword", + }, c2.Vars) + + assertGRPCEqual(t, &GRPC{ + ShutdownDeadline: 10, + EnableReflection: true, + Network: "tcp", + Interceptors: make(map[string]map[string]any), + Services: map[string]ServicesConfig{ + "gateway": { + { + Config: map[string]any{ + "authregistrysvc": "{{ grpc.services.authregistry.address }}", + }, + Label: "grpc_gateway", + Network: "tcp", + }, + }, + "authregistry": { + { + Config: map[string]any{ + "driver": "static", + "drivers": map[string]any{ + "static": map[string]any{ + "rules": map[string]any{ + "basic": "{{ grpc.services.authprovider[0].address }}", + "machine": "{{ grpc.services.authprovider[1].address }}", + }, + }, + }, + }, + Label: "grpc_authregistry", + Network: "tcp", + }, + }, + "authprovider": { + { + Address: "localhost:19000", + Config: map[string]any{ + "driver": "ldap", + "address": "localhost:19000", + "drivers": map[string]any{ + "ldap": map[string]any{ + "password": "ldap", + }, + }, + }, + Label: "grpc_authprovider_0", + Network: "tcp", + }, + { + Address: "localhost:19001", + Config: map[string]any{ + "driver": "machine", + "address": "localhost:19001", + "drivers": map[string]any{ + "machine": map[string]any{ + "api_key": "secretapikey", + }, + }, + }, + Label: "grpc_authprovider_1", + Network: "tcp", + }, + }, + }, + }, c2.GRPC) + + assertHTTPEqual(t, &HTTP{ + Address: Address("localhost:19002"), + Network: "tcp", + Middlewares: make(map[string]map[string]any), + Services: map[string]ServicesConfig{ + "dataprovider": { + { + Address: "localhost:19002", + Config: map[string]any{ + "driver": "localhome", + }, + Network: "tcp", + Label: "http_dataprovider", + }, + }, + "sysinfo": { + { + Address: "localhost:19002", + Config: map[string]any{}, + Network: "tcp", + Label: "http_sysinfo", + }, + }, + }, + }, c2.HTTP) + + assert.Equal(t, &Serverless{ + Services: map[string]map[string]any{ + "notifications": { + "nats_address": "nats-server-01.example.com", + "nats_token": "secret-token-example", + }, + }, + }, c2.Serverless) +} + +func assertGRPCEqual(t *testing.T, g1, g2 *GRPC) { + assert.Equal(t, g1.Address, g2.Address) + assert.Equal(t, g1.Network, g2.Network) + assert.Equal(t, g1.ShutdownDeadline, g2.ShutdownDeadline) + assert.Equal(t, g1.EnableReflection, g2.EnableReflection) + assert.Equal(t, g1.Services, g2.Services) + assert.Equal(t, g1.Interceptors, g2.Interceptors) +} + +func assertHTTPEqual(t *testing.T, h1, h2 *HTTP) { + assert.Equal(t, h1.Network, h2.Network) + assert.Equal(t, h1.Network, h2.Network) + assert.Equal(t, h1.CertFile, h2.CertFile) + assert.Equal(t, h1.KeyFile, h2.KeyFile) + assert.Equal(t, h1.Services, h2.Services) + assert.Equal(t, h1.Middlewares, h2.Middlewares) +} + +func TestDump(t *testing.T) { + config := &Config{ + Shared: &Shared{ + GatewaySVC: "localhost:9142", + JWTSecret: "secret", + }, + Log: &Log{ + Output: "/var/log/revad/revad-gateway.log", + Mode: "json", + Level: "trace", + }, + Core: &Core{ + MaxCPUs: "1", + TracingEnabled: true, + }, + Vars: Vars{ + "db_username": "root", + "db_password": "secretpassword", + }, + GRPC: &GRPC{ + ShutdownDeadline: 10, + EnableReflection: true, + Interceptors: make(map[string]map[string]any), + Services: map[string]ServicesConfig{ + "gateway": { + { + Config: map[string]any{ + "authregistrysvc": "localhost:19000", + }, + }, + }, + "authregistry": { + { + Address: "localhost:19000", + Config: map[string]any{ + "driver": "static", + "drivers": map[string]any{ + "static": map[string]any{ + "rules": map[string]any{ + "basic": "localhost:19001", + "machine": "localhost:19002", + }, + }, + }, + }, + }, + }, + "authprovider": { + { + Address: "localhost:19001", + Config: map[string]any{ + "driver": "ldap", + "address": "localhost:19001", + "drivers": map[string]any{ + "ldap": map[string]any{ + "password": "ldap", + }, + }, + }, + }, + { + Address: "localhost:19002", + Config: map[string]any{ + "driver": "machine", + "address": "localhost:19002", + "drivers": map[string]any{ + "machine": map[string]any{ + "api_key": "secretapikey", + }, + }, + }, + }, + }, + }, + }, + HTTP: &HTTP{ + Address: "localhost:19003", + Middlewares: make(map[string]map[string]any), + Services: map[string]ServicesConfig{ + "dataprovider": { + { + Address: "localhost:19003", + Config: map[string]any{ + "driver": "localhome", + }, + }, + }, + "sysinfo": { + { + Address: "localhost:19003", + Config: map[string]any{}, + }, + }, + }, + }, + Serverless: &Serverless{ + Services: map[string]map[string]any{ + "notifications": { + "nats_address": "nats-server-01.example.com", + "nats_token": "secret-token-example", + }, + }, + }, + } + + m := config.Dump() + assert.Equal(t, map[string]any{ + "shared": map[string]any{ + "jwt_secret": "secret", + "gatewaysvc": "localhost:9142", + "datagateway": "", + "skip_user_groups_in_token": false, + "blocked_users": []any{}, + }, + "log": map[string]any{ + "output": "/var/log/revad/revad-gateway.log", + "mode": "json", + "level": "trace", + }, + "core": map[string]any{ + "max_cpus": "1", + "tracing_enabled": true, + "tracing_endpoint": "", + "tracing_collector": "", + "tracing_service_name": "", + "tracing_service": "", + }, + "vars": map[string]any{ + "db_username": "root", + "db_password": "secretpassword", + }, + "grpc": map[string]any{ + "address": Address(""), + "network": "", + "shutdown_deadline": 10, + "enable_reflection": true, + "interceptors": map[string]any{}, + "services": map[string]any{ + "gateway": []any{ + map[string]any{ + "address": Address(""), + "network": "", + "authregistrysvc": "localhost:19000", + }, + }, + "authregistry": []any{ + map[string]any{ + "address": Address("localhost:19000"), + "network": "", + "driver": "static", + "drivers": map[string]any{ + "static": map[string]any{ + "rules": map[string]any{ + "basic": "localhost:19001", + "machine": "localhost:19002", + }, + }, + }, + }, + }, + "authprovider": []any{ + map[string]any{ + "address": "localhost:19001", + "network": "", + "driver": "ldap", + "drivers": map[string]any{ + "ldap": map[string]any{ + "password": "ldap", + }, + }, + }, + map[string]any{ + "address": "localhost:19002", + "network": "", + "driver": "machine", + "drivers": map[string]any{ + "machine": map[string]any{ + "api_key": "secretapikey", + }, + }, + }, + }, + }, + }, + "http": map[string]any{ + "network": "", + "address": Address("localhost:19003"), + "certfile": "", + "keyfile": "", + "middlewares": map[string]any{}, + "services": map[string]any{ + "dataprovider": []any{ + map[string]any{ + "address": Address("localhost:19003"), + "network": "", + "driver": "localhome", + }, + }, + "sysinfo": []any{ + map[string]any{ + "address": Address("localhost:19003"), + "network": "", + }, + }, + }, + }, + "serverless": map[string]any{ + "services": map[string]any{ + "notifications": map[string]any{ + "nats_address": "nats-server-01.example.com", + "nats_token": "secret-token-example", + }, + }, + }, + }, m) +} diff --git a/cmd/revad/pkg/config/dump.go b/cmd/revad/pkg/config/dump.go new file mode 100644 index 0000000000..e819e5eeb4 --- /dev/null +++ b/cmd/revad/pkg/config/dump.go @@ -0,0 +1,146 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +import "reflect" + +func dumpStruct(v reflect.Value) map[string]any { + if v.Kind() != reflect.Struct { + panic("called dumpStruct on non struct type") + } + + n := v.NumField() + m := make(map[string]any, n) + + t := v.Type() + for i := 0; i < n; i++ { + e := v.Field(i) + f := t.Field(i) + + if !f.IsExported() { + continue + } + + if isFieldSquashed(f) { + if e.Kind() == reflect.Pointer { + e = e.Elem() + } + + var mm map[string]any + switch e.Kind() { + case reflect.Struct: + mm = dumpStruct(e) + case reflect.Map: + mm = dumpMap(e) + default: + panic("squash not allowed on non map/struct types") + } + for k, v := range mm { + m[k] = v + } + continue + } + + n := fieldName(f) + if n == "-" { + continue + } + + m[n] = dumpByType(e) + } + return m +} + +func fieldName(f reflect.StructField) string { + fromtag := f.Tag.Get("key") + if fromtag != "" { + return fromtag + } + return f.Name +} + +func isFieldSquashed(f reflect.StructField) bool { + tag := f.Tag.Get("key") + return tag != "" && tag[1:] == "squash" +} + +func dumpMap(v reflect.Value) map[string]any { + if v.Kind() != reflect.Map { + panic("called dumpMap on non map type") + } + + m := make(map[string]any, v.Len()) + iter := v.MapRange() + for iter.Next() { + k := iter.Key() + e := iter.Value() + + key, ok := k.Interface().(string) + if !ok { + panic("key map must be a string") + } + + m[key] = dumpByType(e) + } + return m +} + +func dumpList(v reflect.Value) []any { + if v.Kind() != reflect.Array && v.Kind() != reflect.Slice { + panic("called dumpList on non array/slice type") + } + + n := v.Len() + l := make([]any, 0, n) + + for i := 0; i < n; i++ { + e := v.Index(i) + l = append(l, dumpByType(e)) + } + return l +} + +func dumpPrimitive(v reflect.Value) any { + if v.Kind() != reflect.Bool && v.Kind() != reflect.Int && v.Kind() != reflect.Int8 && + v.Kind() != reflect.Int16 && v.Kind() != reflect.Int32 && v.Kind() != reflect.Int64 && + v.Kind() != reflect.Uint && v.Kind() != reflect.Uint8 && v.Kind() != reflect.Uint16 && + v.Kind() != reflect.Uint32 && v.Kind() != reflect.Uint64 && v.Kind() != reflect.Float32 && + v.Kind() != reflect.Float64 && v.Kind() != reflect.String { + panic("called dumpPrimitive on non primitive type: " + v.Kind().String()) + } + return v.Interface() +} + +func dumpByType(v reflect.Value) any { + switch v.Kind() { + case reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64, reflect.String: + return dumpPrimitive(v) + case reflect.Array, reflect.Slice: + return dumpList(v) + case reflect.Struct: + return dumpStruct(v) + case reflect.Map: + return dumpMap(v) + case reflect.Interface, reflect.Pointer: + return dumpByType(v.Elem()) + } + panic("type not supported: " + v.Kind().String()) +} diff --git a/cmd/revad/pkg/config/dump_test.go b/cmd/revad/pkg/config/dump_test.go new file mode 100644 index 0000000000..cc214afab8 --- /dev/null +++ b/cmd/revad/pkg/config/dump_test.go @@ -0,0 +1,247 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDumpMap(t *testing.T) { + tests := []struct { + in map[string]any + exp map[string]any + }{ + { + in: map[string]any{}, + exp: map[string]any{}, + }, + { + in: map[string]any{ + "simple": SimpleStruct{ + KeyA: "value_a", + KeyB: "value_b", + }, + }, + exp: map[string]any{ + "simple": map[string]any{ + "keya": "value_a", + "keyb": "value_b", + }, + }, + }, + { + in: map[string]any{ + "simple": SimpleStruct{ + KeyA: "value_a", + KeyB: "value_b", + }, + "map": map[string]any{ + "mapa": "value_mapa", + "mapb": "value_mapb", + }, + }, + exp: map[string]any{ + "simple": map[string]any{ + "keya": "value_a", + "keyb": "value_b", + }, + "map": map[string]any{ + "mapa": "value_mapa", + "mapb": "value_mapb", + }, + }, + }, + } + + for _, tt := range tests { + m := dumpMap(reflect.ValueOf(tt.in)) + assert.Equal(t, m, tt.exp) + } +} + +func TestDumpList(t *testing.T) { + tests := []struct { + in []any + exp []any + }{ + { + in: []any{}, + exp: []any{}, + }, + { + in: []any{1, 2, 3, 4}, + exp: []any{1, 2, 3, 4}, + }, + { + in: []any{ + map[string]any{ + "map": SimpleStruct{ + KeyA: "value_a", + KeyB: "value_b", + }, + }, + 5, + SimpleStruct{ + KeyA: "value_a", + KeyB: "value_b", + }, + }, + exp: []any{ + map[string]any{ + "map": map[string]any{ + "keya": "value_a", + "keyb": "value_b", + }, + }, + 5, + map[string]any{ + "keya": "value_a", + "keyb": "value_b", + }, + }, + }, + } + + for _, tt := range tests { + l := dumpList(reflect.ValueOf(tt.in)) + assert.Equal(t, l, tt.exp) + } +} + +func TestDumpStruct(t *testing.T) { + tests := []struct { + in any + exp map[string]any + }{ + { + in: SimpleStruct{ + KeyA: "value_a", + KeyB: "value_b", + }, + exp: map[string]any{ + "keya": "value_a", + "keyb": "value_b", + }, + }, + { + in: NestedStruct{ + Nested: SimpleStruct{ + KeyA: "value_a", + KeyB: "value_b", + }, + Value: 12, + }, + exp: map[string]any{ + "nested": map[string]any{ + "keya": "value_a", + "keyb": "value_b", + }, + "value": 12, + }, + }, + { + in: StructWithNestedMap{ + Map: map[string]any{ + "keya": "value_a", + "keyb": "value_b", + }, + }, + exp: map[string]any{ + "map": map[string]any{ + "keya": "value_a", + "keyb": "value_b", + }, + }, + }, + { + in: StructWithNestedList{ + List: []SimpleStruct{ + { + KeyA: "value_a[1]", + KeyB: "value_b[1]", + }, + { + KeyA: "value_a[2]", + KeyB: "value_b[2]", + }, + }, + }, + exp: map[string]any{ + "list": []any{ + map[string]any{ + "keya": "value_a[1]", + "keyb": "value_b[1]", + }, + map[string]any{ + "keya": "value_a[2]", + "keyb": "value_b[2]", + }, + }, + }, + }, + { + in: Squashed{ + Squashed: SimpleStruct{ + KeyA: "value_a[1]", + KeyB: "value_b[1]", + }, + Simple: SimpleStruct{ + KeyA: "value_a[2]", + KeyB: "value_b[2]", + }, + }, + exp: map[string]any{ + "keya": "value_a[1]", + "keyb": "value_b[1]", + "Simple": map[string]any{ + "keya": "value_a[2]", + "keyb": "value_b[2]", + }, + }, + }, + { + in: SquashedMap{ + Squashed: map[string]any{ + "keya": "val_a[1]", + "keyb": "val_b[1]", + }, + Simple: SimpleStruct{ + KeyA: "val_a[2]", + KeyB: "val_b[2]", + }, + }, + exp: map[string]any{ + "keya": "val_a[1]", + "keyb": "val_b[1]", + "simple": map[string]any{ + "keya": "val_a[2]", + "keyb": "val_b[2]", + }, + }, + }, + } + + for _, tt := range tests { + s := dumpStruct(reflect.ValueOf(tt.in)) + assert.Equal(t, tt.exp, s) + } +} diff --git a/cmd/revad/pkg/config/grpc.go b/cmd/revad/pkg/config/grpc.go new file mode 100644 index 0000000000..26c67bea3d --- /dev/null +++ b/cmd/revad/pkg/config/grpc.go @@ -0,0 +1,77 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +import ( + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +// GRPC holds the configuration for the GRPC services. +type GRPC struct { + Address Address `mapstructure:"address" key:"address"` + Network string `mapstructure:"network" key:"network" default:"tcp"` + ShutdownDeadline int `mapstructure:"shutdown_deadline" key:"shutdown_deadline"` + EnableReflection bool `mapstructure:"enable_reflection" key:"enable_reflection"` + + Services map[string]ServicesConfig `mapstructure:"-" key:"services"` + Interceptors map[string]map[string]any `mapstructure:"-" key:"interceptors"` + + iterableImpl +} + +func (g *GRPC) services() map[string]ServicesConfig { return g.Services } +func (g *GRPC) interceptors() map[string]map[string]any { return g.Interceptors } + +func (c *Config) parseGRPC(raw map[string]any) error { + cfg, ok := raw["grpc"] + if !ok { + return nil + } + if err := mapstructure.Decode(cfg, c.GRPC); err != nil { + return errors.Wrap(err, "config: error decoding grpc config") + } + + cfgGRPC, ok := cfg.(map[string]any) + if !ok { + return errors.New("grpc must be a map") + } + + services, err := parseServices("grpc", cfgGRPC) + if err != nil { + return err + } + + interceptors, err := parseMiddlwares(cfgGRPC, "interceptors") + if err != nil { + return err + } + + c.GRPC.Services = services + c.GRPC.Interceptors = interceptors + c.GRPC.iterableImpl = iterableImpl{c.GRPC} + + for _, svc := range c.GRPC.Services { + for _, cfg := range svc { + cfg.Address = addressForService(c.GRPC.Address, cfg.Config) + cfg.Network = networkForService(c.HTTP.Network, cfg.Config) + } + } + return nil +} diff --git a/cmd/revad/pkg/config/http.go b/cmd/revad/pkg/config/http.go new file mode 100644 index 0000000000..e29d02e089 --- /dev/null +++ b/cmd/revad/pkg/config/http.go @@ -0,0 +1,77 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +import ( + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +// HTTP holds the configuration for the HTTP services. +type HTTP struct { + Network string `mapstructure:"network" key:"network" default:"tcp"` + Address Address `mapstructure:"address" key:"address"` + CertFile string `mapstructure:"certfile" key:"certfile"` + KeyFile string `mapstructure:"keyfile" key:"keyfile"` + + Services map[string]ServicesConfig `mapstructure:"-" key:"services"` + Middlewares map[string]map[string]any `mapstructure:"-" key:"middlewares"` + + iterableImpl +} + +func (h *HTTP) services() map[string]ServicesConfig { return h.Services } +func (h *HTTP) interceptors() map[string]map[string]any { return h.Middlewares } + +func (c *Config) parseHTTP(raw map[string]any) error { + cfg, ok := raw["http"] + if !ok { + return nil + } + if err := mapstructure.Decode(cfg, c.HTTP); err != nil { + return errors.Wrap(err, "config: error decoding http config") + } + + cfgHTTP, ok := cfg.(map[string]any) + if !ok { + return errors.New("http must be a map") + } + + services, err := parseServices("http", cfgHTTP) + if err != nil { + return err + } + + middlewares, err := parseMiddlwares(cfgHTTP, "middlewares") + if err != nil { + return err + } + + c.HTTP.Services = services + c.HTTP.Middlewares = middlewares + c.HTTP.iterableImpl = iterableImpl{c.HTTP} + + for _, svc := range c.HTTP.Services { + for _, cfg := range svc { + cfg.Address = addressForService(c.HTTP.Address, cfg.Config) + cfg.Network = networkForService(c.HTTP.Network, cfg.Config) + } + } + return nil +} diff --git a/cmd/revad/pkg/config/lookup.go b/cmd/revad/pkg/config/lookup.go new file mode 100644 index 0000000000..5edb2a98d6 --- /dev/null +++ b/cmd/revad/pkg/config/lookup.go @@ -0,0 +1,257 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +import ( + "io" + "reflect" + + "github.com/pkg/errors" +) + +// ErrKeyNotFound is the error returned when a key does not exist +// in the configuration. +type ErrKeyNotFound struct { + Key string +} + +// Error returns a string representation of the ErrKeyNotFound error. +func (e ErrKeyNotFound) Error() string { + return "key '" + e.Key + "' not found in the configuration" +} + +// lookupStruct recursively looks up the key in the struct v. +// It panics if the value in v is not a struct. +// Only fields are allowed to be accessed. It bails out if +// an user wants to access by index. +// The struct is traversed considering the field tags. If the tag +// "key" is not specified for a field, the field is skipped in +// the lookup. If the tag specifies "squash", the field is treated +// as squashed. +func lookupStruct(key string, v reflect.Value) (any, error) { + if v.Kind() != reflect.Struct { + panic("called lookupStruct on non struct type") + } + + cmd, next, err := parseNext(key) + if errors.Is(err, io.EOF) { + return v.Interface(), nil + } + if err != nil { + return nil, err + } + + c, ok := cmd.(FieldByKey) + if !ok { + return nil, errors.New("call of index on struct type") + } + + t := v.Type() + for i := 0; i < v.NumField(); i++ { + val := v.Field(i) + field := t.Field(i) + + if !field.IsExported() { + continue + } + + tag := field.Tag.Get("key") + if tag == "" { + continue + } + + if tag[1:] == "squash" { + if val.Kind() == reflect.Pointer { + val = val.Elem() + } + + var ( + v any + err error + ) + switch val.Kind() { + case reflect.Struct: + v, err = lookupStruct(key, val) + case reflect.Map: + v, err = lookupMap(key, val) + default: + panic("squash not allowed on non map/struct types") + } + var e ErrKeyNotFound + if errors.As(err, &e) { + continue + } + if err != nil { + return nil, err + } + return v, nil + } + + if tag != c.Key { + continue + } + + return lookupByType(next, val) + } + return nil, ErrKeyNotFound{Key: key} +} + +var typeLookuper = reflect.TypeOf((*Lookuper)(nil)).Elem() + +// lookupByType recursively looks up the given key in v. +func lookupByType(key string, v reflect.Value) (any, error) { + if v.Type().Implements(typeLookuper) { + if v, err := lookupFromLookuper(key, v); err == nil && v != nil { + return v, nil + } + } + switch v.Kind() { + case reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64, reflect.String: + return lookupPrimitive(key, v) + case reflect.Array, reflect.Slice: + return lookupList(key, v) + case reflect.Struct: + return lookupStruct(key, v) + case reflect.Map: + return lookupMap(key, v) + case reflect.Interface, reflect.Pointer: + return lookupByType(key, v.Elem()) + } + panic("type not supported: " + v.Kind().String()) +} + +// lookupFromLookuper looks up the key in a Lookup value. +func lookupFromLookuper(key string, v reflect.Value) (any, error) { + g, ok := v.Interface().(Lookuper) + if !ok { + panic("called lookupFromLookuper on type not implementing Lookup interface") + } + + cmd, _, err := parseNext(key) + if errors.Is(err, io.EOF) { + return v.Interface(), nil + } + if err != nil { + return nil, err + } + + c, ok := cmd.(FieldByKey) + if !ok { + return nil, errors.New("call of index on getter type") + } + + return g.Lookup(c.Key) +} + +// lookupMap recursively looks up the given key in the map v. +// It panics if the value in v is not a map. +// Works similarly to lookupStruct. +func lookupMap(key string, v reflect.Value) (any, error) { + if v.Kind() != reflect.Map { + panic("called lookupMap on non map type") + } + + cmd, next, err := parseNext(key) + if errors.Is(err, io.EOF) { + return v.Interface(), nil + } + if err != nil { + return nil, err + } + + c, ok := cmd.(FieldByKey) + if !ok { + return nil, errors.New("call of index on map type") + } + + // lookup elemen in the map + el := v.MapIndex(reflect.ValueOf(c.Key)) + if !el.IsValid() { + return nil, ErrKeyNotFound{Key: key} + } + + return lookupByType(next, el) +} + +// lookupList recursively looks up the given key in the list v, +// in all the elements contained in the list. +// It panics if the value v is not a list. +// The elements can be addressed in general by index, but +// access by key is only allowed if the list contains exactly +// one element. +func lookupList(key string, v reflect.Value) (any, error) { + if v.Kind() != reflect.Slice && v.Kind() != reflect.Array { + panic("called lookupList on non array/slice type") + } + + cmd, next, err := parseNext(key) + if errors.Is(err, io.EOF) { + return v.Interface(), nil + } + if err != nil { + return nil, err + } + + var el reflect.Value + switch c := cmd.(type) { + case FieldByIndex: + if c.Index < 0 || c.Index >= v.Len() { + return nil, errors.New("list index out of range") + } + el = v.Index(c.Index) + case FieldByKey: + // only allowed if the list contains only one element + if v.Len() != 1 { + return nil, errors.New("cannot access field by key on a non 1-elem list") + } + el = v.Index(0) + e, err := lookupByType("."+c.Key, el) + if err != nil { + return nil, err + } + el = reflect.ValueOf(e) + } + + return lookupByType(next, el) +} + +// lookupPrimitive gets the value from v. +// If the key tries to access by field or by index the value, +// an error is returned. +func lookupPrimitive(key string, v reflect.Value) (any, error) { + if v.Kind() != reflect.Bool && v.Kind() != reflect.Int && v.Kind() != reflect.Int8 && + v.Kind() != reflect.Int16 && v.Kind() != reflect.Int32 && v.Kind() != reflect.Int64 && + v.Kind() != reflect.Uint && v.Kind() != reflect.Uint8 && v.Kind() != reflect.Uint16 && + v.Kind() != reflect.Uint32 && v.Kind() != reflect.Uint64 && v.Kind() != reflect.Float32 && + v.Kind() != reflect.Float64 && v.Kind() != reflect.String { + panic("called lookupPrimitive on non primitive type: " + v.Kind().String()) + } + + _, _, err := parseNext(key) + if errors.Is(err, io.EOF) { + return v.Interface(), nil + } + if err != nil { + return nil, err + } + + return nil, errors.New("cannot address a value of type " + v.Kind().String()) +} diff --git a/cmd/revad/pkg/config/lookup_test.go b/cmd/revad/pkg/config/lookup_test.go new file mode 100644 index 0000000000..dcd4e1b9bb --- /dev/null +++ b/cmd/revad/pkg/config/lookup_test.go @@ -0,0 +1,198 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +type SimpleStruct struct { + KeyA string `key:"keya"` + KeyB string `key:"keyb"` +} + +type NestedStruct struct { + Nested SimpleStruct `key:"nested"` + Value int `key:"value"` +} + +type StructWithNestedMap struct { + Map map[string]any `key:"map"` +} + +type StructWithNestedList struct { + List []SimpleStruct `key:"list"` +} + +type Squashed struct { + Squashed SimpleStruct `key:",squash"` + Simple SimpleStruct +} + +type SquashedMap struct { + Squashed map[string]any `key:",squash"` + Simple SimpleStruct `key:"simple"` +} + +type StructWithAddress struct { + Address Address `key:"address"` +} + +func TestLookupStruct(t *testing.T) { + tests := []struct { + in any + key string + val any + err error + }{ + { + in: SimpleStruct{ + KeyA: "val_a", + KeyB: "val_b", + }, + key: ".keyb", + val: "val_b", + }, + { + in: NestedStruct{ + Nested: SimpleStruct{ + KeyA: "val_a", + KeyB: "val_b", + }, + Value: 10, + }, + key: ".nested.keyb", + val: "val_b", + }, + { + in: NestedStruct{ + Nested: SimpleStruct{ + KeyA: "val_a", + KeyB: "val_b", + }, + Value: 10, + }, + key: ".value", + val: 10, + }, + { + in: StructWithNestedMap{ + Map: map[string]any{ + "key1": "val1", + "key2": "val2", + }, + }, + key: ".map.key1", + val: "val1", + }, + { + in: StructWithNestedList{ + List: []SimpleStruct{ + { + KeyA: "val_a[1]", + KeyB: "val_b[1]", + }, + { + KeyA: "val_a[2]", + KeyB: "val_b[2]", + }, + }, + }, + key: ".list[1].keyb", + val: "val_b[2]", + }, + { + in: StructWithNestedList{ + List: []SimpleStruct{ + { + KeyA: "val_a[1]", + KeyB: "val_b[1]", + }, + }, + }, + key: ".list.keya", + val: "val_a[1]", + }, + { + in: StructWithNestedList{ + List: []SimpleStruct{ + { + KeyA: "val_a[1]", + KeyB: "val_b[1]", + }, + { + KeyA: "val_a[2]", + KeyB: "val_b[2]", + }, + }, + }, + key: ".list[1]", + val: SimpleStruct{ + KeyA: "val_a[2]", + KeyB: "val_b[2]", + }, + }, + { + in: Squashed{ + Squashed: SimpleStruct{ + KeyA: "val_a[1]", + KeyB: "val_b[1]", + }, + Simple: SimpleStruct{ + KeyA: "val_a[2]", + KeyB: "val_b[2]", + }, + }, + key: ".keya", + val: "val_a[1]", + }, + { + in: SquashedMap{ + Squashed: map[string]any{ + "keya": "val_a[1]", + "keyb": "val_b[1]", + }, + Simple: SimpleStruct{ + KeyA: "val_a[2]", + KeyB: "val_b[2]", + }, + }, + key: ".keya", + val: "val_a[1]", + }, + { + in: StructWithAddress{ + Address: "188.184.37.219:9142", + }, + key: ".address.port", + val: 9142, + }, + } + + for _, tt := range tests { + got, err := lookupStruct(tt.key, reflect.ValueOf(tt.in)) + assert.Equal(t, err, tt.err, "got not expected error") + if tt.err == nil { + assert.Equal(t, tt.val, got) + } + } +} diff --git a/cmd/revad/pkg/config/parser.go b/cmd/revad/pkg/config/parser.go new file mode 100644 index 0000000000..aa25de6e70 --- /dev/null +++ b/cmd/revad/pkg/config/parser.go @@ -0,0 +1,100 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +import ( + "io" + "strconv" + "strings" + + "github.com/pkg/errors" +) + +// Command is the command to execute after parsing the template. +type Command interface{ isCommand() } + +// FieldByKey instructs the template runner to get a field by a key. +type FieldByKey struct{ Key string } + +func (FieldByKey) isCommand() {} + +// FieldByIndex instructs the template runner to get a field by an index. +type FieldByIndex struct{ Index int } + +func (FieldByIndex) isCommand() {} + +// parseNext reads the next token from the key and +// assings a command. +// If the key is empty io.EOF is returned. +func parseNext(key string) (Command, string, error) { + // key = ".grpc.services.authprovider[1].address" + + key = strings.TrimSpace(key) + + // first character must be either "." or "[" + // unless the key is empty + if key == "" { + return nil, "", io.EOF + } + + switch { + case strings.HasPrefix(key, "."): + tkn, next := split(key) + return FieldByKey{Key: tkn}, next, nil + case strings.HasPrefix(key, "["): + tkn, next := split(key) + index, err := strconv.ParseInt(tkn, 10, 64) + if err != nil { + return nil, "", errors.Wrap(err, "parsing error") + } + return FieldByIndex{Index: int(index)}, next, nil + } + + return nil, "", errors.New("parsing error: operator not recognised in key " + key) +} + +func split(key string) (token string, next string) { + // key = ".grpc.services.authprovider[1].address" + // -> grpc + // key = "[].address" + // -> + if key == "" { + return + } + + i := -1 + s := key[0] + key = key[1:] + + switch s { + case '.': + i = strings.IndexAny(key, ".[") + case '[': + i = strings.IndexByte(key, ']') + } + + if i == -1 { + return key, "" + } + + if key[i] == ']' { + return key[:i], key[i+1:] + } + return key[:i], key[i:] +} diff --git a/cmd/revad/pkg/config/parser_test.go b/cmd/revad/pkg/config/parser_test.go new file mode 100644 index 0000000000..87ba592aec --- /dev/null +++ b/cmd/revad/pkg/config/parser_test.go @@ -0,0 +1,97 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +import ( + "testing" + + "gotest.tools/assert" +) + +func TestSplit(t *testing.T) { + tests := []struct { + key string + token string + next string + }{ + { + key: ".grpc.services.authprovider[1].address", + token: "grpc", + next: ".services.authprovider[1].address", + }, + { + key: "[1].address", + token: "1", + next: ".address", + }, + { + key: "[100].address", + token: "100", + next: ".address", + }, + { + key: "", + }, + } + + for _, tt := range tests { + token, next := split(tt.key) + assert.Equal(t, token, tt.token) + assert.Equal(t, next, tt.next) + } +} + +func TestParseNext(t *testing.T) { + tests := []struct { + key string + cmd Command + next string + err error + }{ + { + key: ".grpc.services.authprovider[1].address", + cmd: FieldByKey{Key: "grpc"}, + next: ".services.authprovider[1].address", + }, + { + key: ".authprovider[1].address", + cmd: FieldByKey{Key: "authprovider"}, + next: "[1].address", + }, + { + key: "[1].authprovider.address", + cmd: FieldByIndex{Index: 1}, + next: ".authprovider.address", + }, + { + key: ".authprovider", + cmd: FieldByKey{Key: "authprovider"}, + next: "", + }, + } + + for _, tt := range tests { + cmd, next, err := parseNext(tt.key) + assert.Equal(t, err, tt.err) + if tt.err == nil { + assert.Equal(t, cmd, tt.cmd) + assert.Equal(t, next, tt.next) + } + } +} diff --git a/cmd/revad/pkg/config/serverless.go b/cmd/revad/pkg/config/serverless.go new file mode 100644 index 0000000000..ea0c574d34 --- /dev/null +++ b/cmd/revad/pkg/config/serverless.go @@ -0,0 +1,54 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +import ( + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +// Serverless holds the configuration for the serverless services. +type Serverless struct { + Services map[string]map[string]any `key:"services" mapstructure:"services"` +} + +func (c *Config) parseServerless(raw map[string]any) error { + cfg, ok := raw["serverless"] + if !ok { + return nil + } + + var s Serverless + if err := mapstructure.Decode(cfg, &s); err != nil { + return errors.Wrap(err, "config: error decoding serverless config") + } + + c.Serverless = &s + return nil +} + +// ForEach iterates to each service calling the function f. +func (s *Serverless) ForEach(f func(name string, config map[string]any) error) error { + for name, cfg := range s.Services { + if err := f(name, cfg); err != nil { + return err + } + } + return nil +} diff --git a/cmd/revad/pkg/config/templates.go b/cmd/revad/pkg/config/templates.go new file mode 100644 index 0000000000..d4d3e4ae52 --- /dev/null +++ b/cmd/revad/pkg/config/templates.go @@ -0,0 +1,279 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +import ( + "fmt" + "reflect" + "regexp" + "strconv" + "strings" +) + +// applyTemplateStruct applies recursively to all its fields all the template +// strings to the struct v. +// It panics if the value is not a struct. +// A field in the struct is skipped for applying all the templates +// if a tag "template" has the value "-". +func applyTemplateStruct(l Lookuper, p setter, v reflect.Value) error { + if v.Kind() != reflect.Struct { + panic("called applyTemplateStruct on non struct type") + } + + t := v.Type() + for i := 0; i < v.NumField(); i++ { + el := v.Field(i) + f := t.Field(i) + + if !f.IsExported() { + continue + } + + if f.Tag.Get("template") == "-" { + // skip this field + continue + } + + if err := applyTemplateByType(l, setterStruct{Struct: v, Field: i}, el); err != nil { + return err + } + } + return nil +} + +// applyTemplateByType applies the template string to a generic type. +func applyTemplateByType(l Lookuper, p setter, v reflect.Value) error { + switch v.Kind() { + case reflect.Array, reflect.Slice: + return applyTemplateList(l, p, v) + case reflect.Struct: + return applyTemplateStruct(l, p, v) + case reflect.Map: + return applyTemplateMap(l, p, v) + case reflect.Interface: + return applyTemplateInterface(l, p, v) + case reflect.String: + return applyTemplateString(l, p, v) + case reflect.Pointer: + return applyTemplateByType(l, p, v.Elem()) + } + return nil +} + +// applyTemplateList recursively applies in all the elements of the list +// the template strings. +// It panics if the given value is not a list. +func applyTemplateList(l Lookuper, p setter, v reflect.Value) error { + if v.Kind() != reflect.Array && v.Kind() != reflect.Slice { + panic("called applyTemplateList on non array/slice type") + } + + for i := 0; i < v.Len(); i++ { + el := v.Index(i) + if err := applyTemplateByType(l, setterList{List: v, Index: i}, el); err != nil { + return err + } + } + return nil +} + +// applyTemplateMap recursively applies in all the elements of the map +// the template strings. +// It panics if the given value is not a map. +func applyTemplateMap(l Lookuper, p setter, v reflect.Value) error { + if v.Kind() != reflect.Map { + panic("called applyTemplateMap on non map type") + } + + iter := v.MapRange() + for iter.Next() { + k := iter.Key() + el := v.MapIndex(k) + if err := applyTemplateByType(l, setterMap{Map: v, Key: k.Interface()}, el); err != nil { + return err + } + } + return nil +} + +// applyTemplateString applies to the string the template string, if any. +// It panics if the given value is not a string. +func applyTemplateString(l Lookuper, p setter, v reflect.Value) error { + if v.Kind() != reflect.String { + panic("called applyTemplateString on non string type") + } + s := v.String() + tmpl, is := isTemplate(s) + if !is { + // nothing to do + return nil + } + + key := keyFromTemplate(tmpl) + val, err := l.Lookup(key) + if err != nil { + return err + } + if val == nil { + return nil + } + + new, err := replaceTemplate(s, tmpl, val) + if err != nil { + return err + } + str, ok := convertToString(new) + if !ok { + return fmt.Errorf("value %v cannot be converted as string in the template %s", val, new) + } + + p.SetValue(str) + return nil +} + +// applyTemplateInterface applies to the interface the template string, if any. +// It panics if the given value is not an interface. +func applyTemplateInterface(l Lookuper, p setter, v reflect.Value) error { + if v.Kind() != reflect.Interface { + panic("called applyTemplateInterface on non interface value") + } + + s, ok := v.Interface().(string) + if !ok { + return applyTemplateByType(l, p, v.Elem()) + } + + tmpl, is := isTemplate(s) + if !is { + // nothing to do + return nil + } + + key := keyFromTemplate(tmpl) + val, err := l.Lookup(key) + if err != nil { + return err + } + if val == nil { + return nil + } + + new, err := replaceTemplate(s, tmpl, val) + if err != nil { + return err + } + p.SetValue(new) + return nil +} + +func replaceTemplate(original, tmpl string, val any) (any, error) { + if strings.TrimSpace(original) == tmpl { + // the value was directly a template, i.e. "{{ grpc.services.gateway.address }}" + return val, nil + } + // the value is of something like "something {{ template }} something else" + // in this case we need to replace the template string with the value, converted + // as string in the original val + s, ok := convertToString(val) + if !ok { + return nil, fmt.Errorf("value %v cannot be converted as string in the template %s", val, original) + } + return strings.Replace(original, tmpl, s, 1), nil +} + +func convertToString(val any) (string, bool) { + switch v := val.(type) { + case string: + return v, true + case fmt.Stringer: + return v.String(), true + case int: + return strconv.FormatInt(int64(v), 10), true + case int8: + return strconv.FormatInt(int64(v), 10), true + case int16: + return strconv.FormatInt(int64(v), 10), true + case int32: + return strconv.FormatInt(int64(v), 10), true + case uint: + return strconv.FormatUint(uint64(v), 10), true + case uint8: + return strconv.FormatUint(uint64(v), 10), true + case uint16: + return strconv.FormatUint(uint64(v), 10), true + case uint32: + return strconv.FormatUint(uint64(v), 10), true + case uint64: + return strconv.FormatUint(v, 10), true + case bool: + return strconv.FormatBool(v), true + } + return "", false +} + +var templateRegex = regexp.MustCompile("{{.{1,}}}") + +func isTemplate(s string) (string, bool) { + m := templateRegex.FindString(s) + return m, m != "" +} + +func keyFromTemplate(s string) string { + s = strings.TrimSpace(s) + s = strings.TrimPrefix(s, "{{") + s = strings.TrimSuffix(s, "}}") + return "." + strings.TrimSpace(s) +} + +type setter interface { + // SetValue sets the value v in a container. + SetValue(v any) +} + +type setterList struct { + List reflect.Value + Index int +} + +type setterMap struct { + Map reflect.Value + Key any +} + +type setterStruct struct { + Struct reflect.Value + Field int +} + +// SetValue sets the value v in the element +// of the list. +func (s setterList) SetValue(v any) { + el := s.List.Index(s.Index) + el.Set(reflect.ValueOf(v)) +} + +// SetValue sets the value v to the element of the map. +func (s setterMap) SetValue(v any) { + s.Map.SetMapIndex(reflect.ValueOf(s.Key), reflect.ValueOf(v)) +} + +// SetValue sets the value v to the field in the struct. +func (s setterStruct) SetValue(v any) { + s.Struct.Field(s.Field).Set(reflect.ValueOf(v)) +} diff --git a/cmd/revad/pkg/config/templates_test.go b/cmd/revad/pkg/config/templates_test.go new file mode 100644 index 0000000000..9db0167bec --- /dev/null +++ b/cmd/revad/pkg/config/templates_test.go @@ -0,0 +1,135 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestApplyTemplate(t *testing.T) { + cfg1 := &Config{ + GRPC: &GRPC{ + Services: map[string]ServicesConfig{ + "authprovider": { + { + Address: "localhost:1900", + }, + }, + "authregistry": { + { + Address: "localhost:1901", + Config: map[string]any{ + "drivers": map[string]any{ + "static": map[string]any{ + "demo": "{{ grpc.services.authprovider.address }}", + }, + }, + }, + }, + }, + "other": { + { + Address: "localhost:1902", + Config: map[string]any{ + "drivers": map[string]any{ + "static": map[string]any{ + "demo": "https://{{ grpc.services.authprovider.address }}/data", + }, + }, + }, + }, + }, + "port": { + { + + Config: map[string]any{ + "drivers": map[string]any{ + "static": map[string]any{ + "demo": "https://cern.ch:{{ grpc.services.authprovider.address.port }}/data", + }, + }, + }, + }, + }, + }, + }, + } + err := cfg1.ApplyTemplates(cfg1) + assert.ErrorIs(t, err, nil) + assert.Equal(t, Address("localhost:1900"), cfg1.GRPC.Services["authregistry"][0].Config["drivers"].(map[string]any)["static"].(map[string]any)["demo"]) + assert.Equal(t, "https://localhost:1900/data", cfg1.GRPC.Services["other"][0].Config["drivers"].(map[string]any)["static"].(map[string]any)["demo"]) + assert.Equal(t, "https://cern.ch:1900/data", cfg1.GRPC.Services["port"][0].Config["drivers"].(map[string]any)["static"].(map[string]any)["demo"]) + + cfg2 := &Config{ + Shared: &Shared{ + GatewaySVC: "{{ grpc.services.authregistry.address }}", + }, + Vars: Vars{ + "db_username": "root", + "db_password": "secretpassword", + "integer": 10, + }, + GRPC: &GRPC{ + Services: map[string]ServicesConfig{ + "authregistry": { + { + Address: "localhost:1901", + Config: map[string]any{ + "drivers": map[string]any{ + "sql": map[string]any{ + "db_username": "{{ vars.db_username }}", + "db_password": "{{ vars.db_password }}", + "key": "value", + "int": "{{ vars.integer }}", + }, + }, + }, + }, + }, + "other": { + { + Address: "localhost:1902", + Config: map[string]any{ + "drivers": map[string]any{ + "sql": map[string]any{ + "db_host": "http://localhost:{{ vars.integer }}", + }, + }, + }, + }, + }, + }, + }, + } + + err = cfg2.ApplyTemplates(cfg2) + assert.ErrorIs(t, err, nil) + assert.Equal(t, "localhost:1901", cfg2.Shared.GatewaySVC) + assert.Equal(t, map[string]any{ + "db_username": "root", + "db_password": "secretpassword", + "key": "value", + "int": 10, + }, cfg2.GRPC.Services["authregistry"][0].Config["drivers"].(map[string]any)["sql"]) + assert.Equal(t, map[string]any{ + "db_host": "http://localhost:10", + }, cfg2.GRPC.Services["other"][0].Config["drivers"].(map[string]any)["sql"]) +} diff --git a/cmd/revad/internal/grace/grace.go b/cmd/revad/pkg/grace/grace.go similarity index 68% rename from cmd/revad/internal/grace/grace.go rename to cmd/revad/pkg/grace/grace.go index 52cd3619a8..6640a40484 100644 --- a/cmd/revad/internal/grace/grace.go +++ b/cmd/revad/pkg/grace/grace.go @@ -29,6 +29,7 @@ import ( "syscall" "time" + netutil "github.com/cs3org/reva/pkg/utils/net" "github.com/pkg/errors" "github.com/rs/zerolog" ) @@ -40,12 +41,14 @@ type Watcher struct { graceful bool ppid int lns map[string]net.Listener - ss map[string]Server + ss []Server SL Serverless pidFile string childPIDs []int } +const revaEnvPrefix = "REVA_FD_" + // Option represent an option. type Option func(w *Watcher) @@ -69,7 +72,7 @@ func NewWatcher(opts ...Option) *Watcher { log: zerolog.Nop(), graceful: os.Getenv("GRACEFUL") == "true", ppid: os.Getppid(), - ss: map[string]Server{}, + ss: make([]Server, 0), } for _, opt := range opts { @@ -82,13 +85,18 @@ func NewWatcher(opts ...Option) *Watcher { // Exit exits the current process cleaning up // existing pid files. func (w *Watcher) Exit(errc int) { + w.Clean() + os.Exit(errc) +} + +// Clean cleans up existing pid files. +func (w *Watcher) Clean() { err := w.clean() if err != nil { w.log.Warn().Err(err).Msg("error removing pid file") } else { w.log.Info().Msgf("pid file %q got removed", w.pidFile) } - os.Exit(errc) } func (w *Watcher) clean() error { @@ -186,29 +194,122 @@ func newListener(network, addr string) (net.Listener, error) { return net.Listen(network, addr) } +// implements the net.Listener interface. +type inherited struct { + f *os.File + ln net.Listener +} + +func (i *inherited) Accept() (net.Conn, error) { + return i.ln.Accept() +} + +func (i *inherited) Close() error { + // TODO: improve this: if file close has error + // the listener is not closed + if err := i.f.Close(); err != nil { + return err + } + return i.ln.Close() +} + +func (i *inherited) Addr() net.Addr { + return i.ln.Addr() +} + +func inheritedListeners() map[string]net.Listener { + lns := make(map[string]net.Listener) + for _, val := range os.Environ() { + if strings.HasPrefix(val, revaEnvPrefix) { + // env variable is of type REVA_FD_= + env := strings.TrimPrefix(val, revaEnvPrefix) + s := strings.Split(env, "=") + if len(s) != 2 { + continue + } + svcname := strings.ToLower(s[0]) + fd, err := strconv.ParseUint(s[1], 10, 64) + if err != nil { + continue + } + f := os.NewFile(uintptr(fd), "") + ln, err := net.FileListener(f) + if err != nil { + // TODO: log error + continue + } + lns[svcname] = &inherited{f: f, ln: ln} + } + } + return lns +} + +func isRandomAddress(addr string) bool { + return addr == "" +} + +func getAddress(addr string) string { + if isRandomAddress(addr) { + return ":0" + } + return addr +} + // GetListeners return grpc listener first and http listener second. -func (w *Watcher) GetListeners(servers map[string]Server) (map[string]net.Listener, error) { - w.ss = servers - lns := map[string]net.Listener{} +func (w *Watcher) GetListeners(servers map[string]Addressable) (map[string]net.Listener, error) { + lns := make(map[string]net.Listener) + if w.graceful { - w.log.Info().Msg("graceful restart, inheriting parent ln fds for grpc and http") - count := 3 - for k, s := range servers { - network, addr := s.Network(), s.Address() - fd := os.NewFile(uintptr(count), "") // 3 because ExtraFile passed to new process - count++ - ln, err := net.FileListener(fd) - if err != nil { - w.log.Error().Err(err).Msg("error creating net.Listener from fd") - // create new fd - ln, err := newListener(network, addr) + w.log.Info().Msg("graceful restart, inheriting parent listener fds for grpc and http services") + + inherited := inheritedListeners() + logListeners(inherited, "inherited", &w.log) + + for svc, ln := range inherited { + addr, ok := servers[svc] + if !ok { + continue + } + // for services with random addresses, check and assign if available from inherited + // from the assigned addresses, assing the listener if address correspond + if isRandomAddress(addr.Address()) || + netutil.AddressEqual(ln.Addr(), addr.Network(), addr.Address()) { + lns[svc] = ln + } + } + + // close all the listeners not used from inherited + for svc, ln := range inherited { + if _, ok := lns[svc]; !ok { + w.log.Debug().Msgf("closing inherited listener %s:%s for service %s", ln.Addr().Network(), ln.Addr().String(), svc) + if err := ln.Close(); err != nil { + w.log.Error().Err(err).Msgf("error closing inherited listener %s:%s", ln.Addr().Network(), ln.Addr().String()) + return nil, errors.Wrap(err, "error closing inherited listener") + } + } + } + + var err error + // create assigned/random listeners for the missing services + for svc, a := range servers { + _, ok := lns[svc] + if ok { + continue + } + network, addr := a.Network(), getAddress(a.Address()) + // multiple services may have the same listener + ln, ok := get(lns, addr, network) + if !ok { + ln, err = newListener(network, addr) if err != nil { return nil, err } - lns[k] = ln - } else { - lns[k] = ln } + if err != nil { + w.log.Error().Err(err).Msgf("error getting listener on %s", addr) + return nil, errors.Wrap(err, "error getting listener") + } + lns[svc] = ln } // kill parent @@ -233,26 +334,55 @@ func (w *Watcher) GetListeners(servers map[string]Server) (map[string]net.Listen return lns, nil } - // create two listeners for grpc and http - for k, s := range servers { - network, addr := s.Network(), s.Address() - ln, err := newListener(network, addr) - if err != nil { - return nil, err + var err error + // no graceful + for svc, s := range servers { + network, addr := s.Network(), getAddress(s.Address()) + // multiple services may have the same listener + ln, ok := get(lns, addr, network) + if !ok { + ln, err = newListener(network, addr) + if err != nil { + return nil, err + } } - lns[k] = ln + w.log.Debug(). + Msgf("listener for %s assigned to %s:%s", svc, ln.Addr().Network(), ln.Addr().String()) + lns[svc] = ln } w.lns = lns return lns, nil } +func logListeners(lns map[string]net.Listener, info string, log *zerolog.Logger) { + r := make(map[string]string, len(lns)) + for n, ln := range lns { + r[n] = fmt.Sprintf("%s:%s", ln.Addr().Network(), ln.Addr().String()) + } + log.Debug().Interface(info, r).Send() +} + +func get(lns map[string]net.Listener, address, network string) (net.Listener, bool) { + for _, ln := range lns { + if netutil.AddressEqual(ln.Addr(), network, address) { + return ln, true + } + } + return nil, false +} + +// Addressable is the interface for exposing address info. +type Addressable interface { + Network() string + Address() string +} + // Server is the interface that servers like HTTP or gRPC // servers need to implement. type Server interface { - Stop() error - GracefulStop() error - Network() string - Address() string + Start(net.Listener) error + Serverless + Addressable } // Serverless is the interface that the serverless server implements. @@ -261,6 +391,12 @@ type Serverless interface { GracefulStop() error } +// SetServers sets the list of servers that have to be watched. +func (w *Watcher) SetServers(s []Server) { w.ss = s } + +// SetServerless sets the serverless that has to be watched. +func (w *Watcher) SetServerless(s Serverless) { w.SL = s } + // TrapSignals captures the OS signal. func (w *Watcher) TrapSignals() { signalCh := make(chan os.Signal, 1024) @@ -350,6 +486,8 @@ func (w *Watcher) TrapSignals() { func getListenerFile(ln net.Listener) (*os.File, error) { switch t := ln.(type) { + case *inherited: + return t.f, nil case *net.TCPListener: return t.File() case *net.UnixListener: @@ -361,13 +499,13 @@ func getListenerFile(ln net.Listener) (*os.File, error) { func forkChild(lns map[string]net.Listener) (*os.Process, error) { // Get the file descriptor for the listener and marshal the metadata to pass // to the child in the environment. - fds := map[string]*os.File{} - for k, ln := range lns { + fds := make(map[string]*os.File, 0) + for name, ln := range lns { fd, err := getListenerFile(ln) if err != nil { return nil, err } - fds[k] = fd + fds[name] = fd } // Pass stdin, stdout, and stderr along with the listener file to the child @@ -379,10 +517,10 @@ func forkChild(lns map[string]net.Listener) (*os.Process, error) { // Get current environment and add in the listener to it. environment := append(os.Environ(), "GRACEFUL=true") - var counter = 3 + counter := 3 for k, fd := range fds { k = strings.ToUpper(k) - environment = append(environment, k+"FD="+fmt.Sprintf("%d", counter)) + environment = append(environment, fmt.Sprintf("%s%s=%d", revaEnvPrefix, k, counter)) files = append(files, fd) counter++ } @@ -402,7 +540,7 @@ func forkChild(lns map[string]net.Listener) (*os.Process, error) { Sys: &syscall.SysProcAttr{}, }) - // TODO(labkode): if the process dies (because config changed and is wrong + // TODO(labkode): if the process dies (because config changed and is wrong) // we need to return an error if err != nil { return nil, err diff --git a/cmd/revad/runtime/grpc.go b/cmd/revad/runtime/grpc.go new file mode 100644 index 0000000000..5cd1b4b195 --- /dev/null +++ b/cmd/revad/runtime/grpc.go @@ -0,0 +1,150 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package runtime + +import ( + "sort" + + "github.com/cs3org/reva/internal/grpc/interceptors/appctx" + "github.com/cs3org/reva/internal/grpc/interceptors/auth" + "github.com/cs3org/reva/internal/grpc/interceptors/log" + "github.com/cs3org/reva/internal/grpc/interceptors/recovery" + "github.com/cs3org/reva/internal/grpc/interceptors/token" + "github.com/cs3org/reva/internal/grpc/interceptors/useragent" + "github.com/cs3org/reva/pkg/rgrpc" + rtrace "github.com/cs3org/reva/pkg/trace" + "github.com/pkg/errors" + "github.com/rs/zerolog" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "google.golang.org/grpc" +) + +type unaryInterceptorTriple struct { + Name string + Priority int + Interceptor grpc.UnaryServerInterceptor +} + +type streamInterceptorTriple struct { + Name string + Priority int + Interceptor grpc.StreamServerInterceptor +} + +func initGRPCInterceptors(conf map[string]map[string]any, unprotected []string, logger *zerolog.Logger) ([]grpc.UnaryServerInterceptor, []grpc.StreamServerInterceptor, error) { + unaryTriples := []*unaryInterceptorTriple{} + for name, c := range conf { + new, ok := rgrpc.UnaryInterceptors[name] + if !ok { + continue + } + inter, prio, err := new(c) + if err != nil { + return nil, nil, errors.Wrap(err, "error creating unary interceptor: "+name) + } + triple := &unaryInterceptorTriple{ + Name: name, + Priority: prio, + Interceptor: inter, + } + unaryTriples = append(unaryTriples, triple) + } + + sort.SliceStable(unaryTriples, func(i, j int) bool { + return unaryTriples[i].Priority < unaryTriples[j].Priority + }) + + authUnary, err := auth.NewUnary(conf["auth"], unprotected) + if err != nil { + return nil, nil, errors.Wrap(err, "error creating unary auth interceptor") + } + + unaryInterceptors := []grpc.UnaryServerInterceptor{authUnary} + for _, t := range unaryTriples { + unaryInterceptors = append(unaryInterceptors, t.Interceptor) + logger.Info().Msgf("rgrpc: chaining grpc unary interceptor %s with priority %d", t.Name, t.Priority) + } + + unaryInterceptors = append(unaryInterceptors, + otelgrpc.UnaryServerInterceptor( + otelgrpc.WithTracerProvider(rtrace.Provider), + otelgrpc.WithPropagators(rtrace.Propagator)), + ) + + unaryInterceptors = append([]grpc.UnaryServerInterceptor{ + appctx.NewUnary(*logger), + token.NewUnary(), + useragent.NewUnary(), + log.NewUnary(), + recovery.NewUnary(), + }, unaryInterceptors...) + + streamTriples := []*streamInterceptorTriple{} + for name, c := range conf { + new, ok := rgrpc.StreamInterceptors[name] + if !ok { + continue + } + inter, prio, err := new(c) + if err != nil { + if err != nil { + return nil, nil, errors.Wrapf(err, "error creating streaming interceptor: %s,", name) + } + triple := &streamInterceptorTriple{ + Name: name, + Priority: prio, + Interceptor: inter, + } + streamTriples = append(streamTriples, triple) + } + } + // sort stream triples + sort.SliceStable(streamTriples, func(i, j int) bool { + return streamTriples[i].Priority < streamTriples[j].Priority + }) + + authStream, err := auth.NewStream(conf["auth"], unprotected) + if err != nil { + return nil, nil, errors.Wrap(err, "error creating stream auth interceptor") + } + + streamInterceptors := []grpc.StreamServerInterceptor{authStream} + for _, t := range streamTriples { + streamInterceptors = append(streamInterceptors, t.Interceptor) + logger.Info().Msgf("rgrpc: chaining grpc streaming interceptor %s with priority %d", t.Name, t.Priority) + } + + streamInterceptors = append([]grpc.StreamServerInterceptor{ + authStream, + appctx.NewStream(*logger), + token.NewStream(), + useragent.NewStream(), + log.NewStream(), + recovery.NewStream(), + }, streamInterceptors...) + + return unaryInterceptors, streamInterceptors, nil +} + +func grpcUnprotected(s map[string]rgrpc.Service) (unprotected []string) { + for _, svc := range s { + unprotected = append(unprotected, svc.UnprotectedEndpoints()...) + } + return +} diff --git a/cmd/revad/runtime/http.go b/cmd/revad/runtime/http.go new file mode 100644 index 0000000000..7568c174ea --- /dev/null +++ b/cmd/revad/runtime/http.go @@ -0,0 +1,88 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package runtime + +import ( + "path" + "sort" + + "github.com/cs3org/reva/internal/http/interceptors/appctx" + "github.com/cs3org/reva/internal/http/interceptors/auth" + "github.com/cs3org/reva/internal/http/interceptors/log" + "github.com/cs3org/reva/pkg/rhttp/global" + "github.com/pkg/errors" + "github.com/rs/zerolog" +) + +// middlewareTriple represents a middleware with the +// priority to be chained. +type middlewareTriple struct { + Name string + Priority int + Middleware global.Middleware +} + +func initHTTPMiddlewares(conf map[string]map[string]any, unprotected []string, logger *zerolog.Logger) ([]global.Middleware, error) { + triples := []*middlewareTriple{} + for name, c := range conf { + new, ok := global.NewMiddlewares[name] + if !ok { + continue + } + m, prio, err := new(c) + if err != nil { + return nil, errors.Wrapf(err, "error creating new middleware: %s,", name) + } + triples = append(triples, &middlewareTriple{ + Name: name, + Priority: prio, + Middleware: m, + }) + logger.Info().Msgf("http middleware enabled: %s", name) + } + + sort.SliceStable(triples, func(i, j int) bool { + return triples[i].Priority > triples[j].Priority + }) + + authMiddle, err := auth.New(conf["auth"], unprotected) + if err != nil { + return nil, errors.Wrap(err, "rhttp: error creating auth middleware") + } + + middlewares := []global.Middleware{ + authMiddle, + log.New(), + appctx.New(*logger), + } + + for _, triple := range triples { + middlewares = append(middlewares, triple.Middleware) + } + return middlewares, nil +} + +func httpUnprotected(s map[string]global.Service) (unprotected []string) { + for _, svc := range s { + for _, url := range svc.Unprotected() { + unprotected = append(unprotected, path.Join("/", svc.Prefix(), url)) + } + } + return +} diff --git a/cmd/revad/runtime/option.go b/cmd/revad/runtime/option.go index 6a662e9fa9..dd49966424 100644 --- a/cmd/revad/runtime/option.go +++ b/cmd/revad/runtime/option.go @@ -30,11 +30,15 @@ type Option func(o *Options) type Options struct { Logger *zerolog.Logger Registry registry.Registry + PidFile string } // newOptions initializes the available default options. func newOptions(opts ...Option) Options { - opt := Options{} + l := zerolog.Nop() + opt := Options{ + Logger: &l, + } for _, o := range opts { o(&opt) @@ -50,6 +54,13 @@ func WithLogger(logger *zerolog.Logger) Option { } } +// WithPidFile sets to pidfile to use. +func WithPidFile(pidfile string) Option { + return func(o *Options) { + o.PidFile = pidfile + } +} + // WithRegistry provides a function to set the registry. func WithRegistry(r registry.Registry) Option { return func(o *Options) { diff --git a/cmd/revad/runtime/runtime.go b/cmd/revad/runtime/runtime.go index 255c55bdf8..3a4b601109 100644 --- a/cmd/revad/runtime/runtime.go +++ b/cmd/revad/runtime/runtime.go @@ -20,284 +20,295 @@ package runtime import ( "fmt" - "io" - "log" "net" - "os" "runtime" "strconv" "strings" - "github.com/cs3org/reva/cmd/revad/internal/grace" - "github.com/cs3org/reva/pkg/logger" - "github.com/cs3org/reva/pkg/registry/memory" + "github.com/pkg/errors" + + "github.com/cs3org/reva/cmd/revad/pkg/config" + "github.com/cs3org/reva/cmd/revad/pkg/grace" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rhttp" + "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/rserverless" "github.com/cs3org/reva/pkg/sharedconf" rtrace "github.com/cs3org/reva/pkg/trace" - "github.com/cs3org/reva/pkg/utils" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/list" + "github.com/cs3org/reva/pkg/utils/maps" + netutil "github.com/cs3org/reva/pkg/utils/net" "github.com/rs/zerolog" + "golang.org/x/sync/errgroup" ) -// Run runs a reva server with the given config file and pid file. -func Run(mainConf map[string]interface{}, pidFile, logLevel string) { - logConf := parseLogConfOrDie(mainConf["log"], logLevel) - logger := initLogger(logConf) - RunWithOptions(mainConf, pidFile, WithLogger(logger)) -} +// Reva represents a full instance of reva. +type Reva struct { + config *config.Config -// RunWithOptions runs a reva server with the given config file, pid file and options. -func RunWithOptions(mainConf map[string]interface{}, pidFile string, opts ...Option) { - options := newOptions(opts...) - parseSharedConfOrDie(mainConf["shared"]) - coreConf := parseCoreConfOrDie(mainConf["core"]) - - // TODO: one can pass the options from the config file to registry.New() and initialize a registry based upon config files. - if options.Registry != nil { - utils.GlobalRegistry = options.Registry - } else if _, ok := mainConf["registry"]; ok { - for _, services := range mainConf["registry"].(map[string]interface{}) { - for sName, nodes := range services.(map[string]interface{}) { - for _, instance := range nodes.([]interface{}) { - if err := utils.GlobalRegistry.Add(memory.NewService(sName, instance.(map[string]interface{})["nodes"].([]interface{}))); err != nil { - panic(err) - } - } - } - } - } + servers []*Server + serverless *rserverless.Serverless + watcher *grace.Watcher + lns map[string]net.Listener - run(mainConf, coreConf, options.Logger, pidFile) + pidfile string + log *zerolog.Logger } -type coreConf struct { - MaxCPUs string `mapstructure:"max_cpus"` - TracingEnabled bool `mapstructure:"tracing_enabled"` - TracingEndpoint string `mapstructure:"tracing_endpoint"` - TracingCollector string `mapstructure:"tracing_collector"` - TracingServiceName string `mapstructure:"tracing_service_name"` +// Server represents a reva server (grpc or http). +type Server struct { + server grace.Server + listener net.Listener - // TracingService specifies the service. i.e OpenCensus, OpenTelemetry, OpenTracing... - TracingService string `mapstructure:"tracing_service"` + services map[string]any } -func run(mainConf map[string]interface{}, coreConf *coreConf, logger *zerolog.Logger, filename string) { - host, _ := os.Hostname() - logger.Info().Msgf("host info: %s", host) +// Start starts the server listening on the assigned listener. +func (s *Server) Start() error { + return s.server.Start(s.listener) +} - if coreConf.TracingEnabled { - initTracing(coreConf) - } - initCPUCount(coreConf, logger) +// New creates a new reva instance. +func New(config *config.Config, opt ...Option) (*Reva, error) { + opts := newOptions(opt...) + log := opts.Logger - servers := initServers(mainConf, logger) - serverless := initServerless(mainConf, logger) + if err := initCPUCount(config.Core, log); err != nil { + return nil, err + } + initTracing(config.Core) - if len(servers) == 0 && serverless == nil { - logger.Info().Msg("nothing to do, no grpc/http/serverless enabled_services declared in config") - os.Exit(1) + if opts.PidFile == "" { + return nil, errors.New("pid file not provided") } - watcher, err := initWatcher(logger, filename) + watcher, err := initWatcher(opts.PidFile, log) if err != nil { - log.Panic(err) + return nil, err } - listeners := initListeners(watcher, servers, logger) - if serverless != nil { - watcher.SL = serverless + + listeners, err := watcher.GetListeners(servicesAddresses(config)) + if err != nil { + watcher.Clean() + return nil, err } - start(mainConf, servers, serverless, listeners, logger, watcher) -} + setRandomAddresses(config, listeners, log) + + if err := applyTemplates(config); err != nil { + watcher.Clean() + return nil, err + } + initSharedConf(config) -func initListeners(watcher *grace.Watcher, servers map[string]grace.Server, log *zerolog.Logger) map[string]net.Listener { - listeners, err := watcher.GetListeners(servers) + grpc := groupGRPCByAddress(config) + http := groupHTTPByAddress(config) + servers, err := newServers(grpc, http, listeners, log) if err != nil { - log.Error().Err(err).Msg("error getting sockets") - watcher.Exit(1) + watcher.Clean() + return nil, err } - return listeners -} -func initWatcher(log *zerolog.Logger, filename string) (*grace.Watcher, error) { - watcher, err := handlePIDFlag(log, filename) - // TODO(labkode): maybe pidfile can be created later on? like once a server is going to be created? + serverless, err := newServerless(config, log) if err != nil { - log.Error().Err(err).Msg("error creating grace watcher") - os.Exit(1) + watcher.Clean() + return nil, err } - return watcher, err + + return &Reva{ + config: config, + servers: servers, + serverless: serverless, + watcher: watcher, + lns: listeners, + pidfile: opts.PidFile, + log: log, + }, nil } -func initServers(mainConf map[string]interface{}, log *zerolog.Logger) map[string]grace.Server { - servers := map[string]grace.Server{} - if isEnabledHTTP(mainConf) { - s, err := getHTTPServer(mainConf["http"], log) - if err != nil { - log.Error().Err(err).Msg("error creating http server") - os.Exit(1) - } - servers["http"] = s - } +func servicesAddresses(cfg *config.Config) map[string]grace.Addressable { + a := make(map[string]grace.Addressable) + cfg.GRPC.ForEachService(func(s *config.Service) { + a[s.Label] = &addr{address: s.Address.String(), network: s.Network} + }) + cfg.HTTP.ForEachService(func(s *config.Service) { + a[s.Label] = &addr{address: s.Address.String(), network: s.Network} + }) + return a +} - if isEnabledGRPC(mainConf) { - s, err := getGRPCServer(mainConf["grpc"], log) +func newServerless(config *config.Config, log *zerolog.Logger) (*rserverless.Serverless, error) { + sl := make(map[string]rserverless.Service) + logger := log.With().Str("pkg", "serverless").Logger() + if err := config.Serverless.ForEach(func(name string, config map[string]any) error { + new, ok := rserverless.Services[name] + if !ok { + return fmt.Errorf("serverless service %s does not exist", name) + } + log := logger.With().Str("service", name).Logger() + svc, err := new(config, &log) if err != nil { - log.Error().Err(err).Msg("error creating grpc server") - os.Exit(1) + return errors.Wrapf(err, "serverless service %s could not be initialized", name) } - servers["grpc"] = s + sl[name] = svc + return nil + }); err != nil { + return nil, err } - return servers + ss, err := rserverless.New( + rserverless.WithLogger(&logger), + rserverless.WithServices(sl), + ) + if err != nil { + return nil, err + } + return ss, nil } -func initServerless(mainConf map[string]interface{}, log *zerolog.Logger) *rserverless.Serverless { - if isEnabledServerless(mainConf) { - serverless, err := getServerless(mainConf["serverless"], log) - if err != nil { - log.Error().Err(err).Msg("error") - os.Exit(1) +func setRandomAddresses(c *config.Config, lns map[string]net.Listener, log *zerolog.Logger) { + f := func(s *config.Service) { + if s.Address != "" { + return } - return serverless + ln, ok := lns[s.Label] + if !ok { + log.Fatal().Msg("port not assigned for service " + s.Label) + } + s.SetAddress(config.Address(ln.Addr().String())) + log.Debug(). + Msgf("set random address %s:%s to service %s", ln.Addr().Network(), ln.Addr().String(), s.Label) } - - return nil + c.GRPC.ForEachService(f) + c.HTTP.ForEachService(f) } -func initTracing(conf *coreConf) { - rtrace.SetTraceProvider(conf.TracingCollector, conf.TracingEndpoint, conf.TracingServiceName) +type addr struct { + address string + network string } -func initCPUCount(conf *coreConf, log *zerolog.Logger) { - ncpus, err := adjustCPU(conf.MaxCPUs) - if err != nil { - log.Error().Err(err).Msg("error adjusting number of cpus") - os.Exit(1) - } - // log.Info().Msgf("%s", getVersionString()) - log.Info().Msgf("running on %d cpus", ncpus) +func (a *addr) Address() string { + return a.address } -func initLogger(conf *logConf) *zerolog.Logger { - log, err := newLogger(conf) - if err != nil { - fmt.Fprintf(os.Stderr, "error creating logger, exiting ...") - os.Exit(1) - } - return log +func (a *addr) Network() string { + return a.network } -func handlePIDFlag(l *zerolog.Logger, pidFile string) (*grace.Watcher, error) { - var opts []grace.Option - opts = append(opts, grace.WithPIDFile(pidFile)) - opts = append(opts, grace.WithLogger(l.With().Str("pkg", "grace").Logger())) - w := grace.NewWatcher(opts...) - err := w.WritePID() - if err != nil { - return nil, err +func groupGRPCByAddress(cfg *config.Config) []*config.GRPC { + // TODO: same address cannot be used in different configurations + g := map[string]*config.GRPC{} + cfg.GRPC.ForEachService(func(s *config.Service) { + if _, ok := g[s.Address.String()]; !ok { + g[s.Address.String()] = &config.GRPC{ + Address: s.Address, + Network: s.Network, + ShutdownDeadline: cfg.GRPC.ShutdownDeadline, + EnableReflection: cfg.GRPC.EnableReflection, + Services: make(map[string]config.ServicesConfig), + Interceptors: cfg.GRPC.Interceptors, + } + } + g[s.Address.String()].Services[s.Name] = config.ServicesConfig{ + {Config: s.Config, Address: s.Address, Network: s.Network, Label: s.Label}, + } + }) + l := make([]*config.GRPC, 0, len(g)) + for _, c := range g { + l = append(l, c) } - - return w, nil + return l } -func start(mainConf map[string]interface{}, servers map[string]grace.Server, serverless *rserverless.Serverless, listeners map[string]net.Listener, log *zerolog.Logger, watcher *grace.Watcher) { - if isEnabledHTTP(mainConf) { - go func() { - if err := servers["http"].(*rhttp.Server).Start(listeners["http"]); err != nil { - log.Error().Err(err).Msg("error starting the http server") - watcher.Exit(1) +func groupHTTPByAddress(cfg *config.Config) []*config.HTTP { + g := map[string]*config.HTTP{} + cfg.HTTP.ForEachService(func(s *config.Service) { + if _, ok := g[s.Address.String()]; !ok { + g[s.Address.String()] = &config.HTTP{ + Address: s.Address, + Network: s.Network, + CertFile: cfg.HTTP.CertFile, + KeyFile: cfg.HTTP.KeyFile, + Services: make(map[string]config.ServicesConfig), + Middlewares: cfg.HTTP.Middlewares, } - }() - } - if isEnabledGRPC(mainConf) { - go func() { - if err := servers["grpc"].(*rgrpc.Server).Start(listeners["grpc"]); err != nil { - log.Error().Err(err).Msg("error starting the grpc server") - watcher.Exit(1) - } - }() - } - if isEnabledServerless(mainConf) { - if err := serverless.Start(); err != nil { - log.Error().Err(err).Msg("error starting serverless services") - watcher.Exit(1) } + g[s.Address.String()].Services[s.Name] = config.ServicesConfig{ + {Config: s.Config, Address: s.Address, Network: s.Network, Label: s.Label}, + } + }) + l := make([]*config.HTTP, 0, len(g)) + for _, c := range g { + l = append(l, c) } - - watcher.TrapSignals() + return l } -func newLogger(conf *logConf) (*zerolog.Logger, error) { - // TODO(labkode): use debug level rather than info as default until reaching a stable version. - // Helps having smaller development files. - if conf.Level == "" { - conf.Level = zerolog.DebugLevel.String() - } - - var opts []logger.Option - opts = append(opts, logger.WithLevel(conf.Level)) +// Start starts all the reva services and waits for a signal. +func (r *Reva) Start() error { + defer r.watcher.Clean() + r.watcher.SetServers(list.Map(r.servers, func(s *Server) grace.Server { return s.server })) + r.watcher.SetServerless(r.serverless) - w, err := getWriter(conf.Output) - if err != nil { - return nil, err + var g errgroup.Group + for _, server := range r.servers { + server := server + g.Go(func() error { + return server.Start() + }) } - opts = append(opts, logger.WithWriter(w, logger.Mode(conf.Mode))) + g.Go(func() error { + return r.serverless.Start() + }) - l := logger.New(opts...) - sub := l.With().Int("pid", os.Getpid()).Logger() - return &sub, nil + r.watcher.TrapSignals() + return g.Wait() } -func getWriter(out string) (io.Writer, error) { - if out == "stderr" || out == "" { - return os.Stderr, nil - } - - if out == "stdout" { - return os.Stdout, nil - } +func initSharedConf(config *config.Config) { + sharedconf.Init(config.Shared) +} - fd, err := os.OpenFile(out, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - err = errors.Wrap(err, "error creating log file: "+out) - return nil, err - } +func initWatcher(filename string, log *zerolog.Logger) (*grace.Watcher, error) { + return handlePIDFlag(log, filename) + // TODO(labkode): maybe pidfile can be created later on? like once a server is going to be created? +} - return fd, nil +func applyTemplates(config *config.Config) error { + return config.ApplyTemplates(config) } -func getGRPCServer(conf interface{}, l *zerolog.Logger) (*rgrpc.Server, error) { - sub := l.With().Str("pkg", "rgrpc").Logger() - s, err := rgrpc.NewServer(conf, sub) +func initCPUCount(conf *config.Core, log *zerolog.Logger) error { + ncpus, err := adjustCPU(conf.MaxCPUs) if err != nil { - err = errors.Wrap(err, "main: error creating grpc server") - return nil, err + return errors.Wrap(err, "error adjusting number of cpus") } - return s, nil + log.Info().Msgf("running on %d cpus", ncpus) + return nil } -func getHTTPServer(conf interface{}, l *zerolog.Logger) (*rhttp.Server, error) { - sub := l.With().Str("pkg", "rhttp").Logger() - s, err := rhttp.New(conf, sub) +func handlePIDFlag(l *zerolog.Logger, pidFile string) (*grace.Watcher, error) { + w := grace.NewWatcher( + grace.WithPIDFile(pidFile), + grace.WithLogger(l.With().Str("pkg", "grace").Logger()), + ) + err := w.WritePID() if err != nil { - err = errors.Wrap(err, "main: error creating http server") return nil, err } - return s, nil + + return w, nil } -func getServerless(conf interface{}, l *zerolog.Logger) (*rserverless.Serverless, error) { - sub := l.With().Str("pkg", "rserverless").Logger() - return rserverless.New(conf, sub) +func initTracing(conf *config.Core) { + if conf.TracingEnabled { + rtrace.SetTraceProvider(conf.TracingCollector, conf.TracingEndpoint, conf.TracingServiceName) + } } -// adjustCPU parses string cpu and sets GOMAXPROCS -// +// adjustCPU parses string cpu and sets GOMAXPROCS // according to its value. It accepts either // a number (e.g. 3) or a percent (e.g. 50%). // Default is to use all available cores. @@ -337,81 +348,77 @@ func adjustCPU(cpu string) (int, error) { return numCPU, nil } -func parseCoreConfOrDie(v interface{}) *coreConf { - c := &coreConf{} - if err := mapstructure.Decode(v, c); err != nil { - fmt.Fprintf(os.Stderr, "error decoding core config: %s\n", err.Error()) - os.Exit(1) - } - - // tracing defaults to enabled if not explicitly configured - if v == nil { - c.TracingEnabled = true - c.TracingEndpoint = "localhost:6831" - } else if _, ok := v.(map[string]interface{})["tracing_enabled"]; !ok { - c.TracingEnabled = true - c.TracingEndpoint = "localhost:6831" - } - - return c -} - -func parseSharedConfOrDie(v interface{}) { - if err := sharedconf.Decode(v); err != nil { - fmt.Fprintf(os.Stderr, "error decoding shared config: %s\n", err.Error()) - os.Exit(1) - } -} - -func parseLogConfOrDie(v interface{}, logLevel string) *logConf { - c := &logConf{} - if err := mapstructure.Decode(v, c); err != nil { - fmt.Fprintf(os.Stderr, "error decoding log config: %s\n", err.Error()) - os.Exit(1) - } - - // if mode is not set, we use console mode, easier for devs - if c.Mode == "" { - c.Mode = "console" - } - - // Give priority to the log level passed through the command line. - if logLevel != "" { - c.Level = logLevel +func listenerFromAddress(lns map[string]net.Listener, network string, address config.Address) net.Listener { + for _, ln := range lns { + if netutil.AddressEqual(ln.Addr(), network, address.String()) { + return ln + } } - - return c -} - -type logConf struct { - Output string `mapstructure:"output"` - Mode string `mapstructure:"mode"` - Level string `mapstructure:"level"` -} - -func isEnabledHTTP(conf map[string]interface{}) bool { - return isEnabled("http", conf) -} - -func isEnabledGRPC(conf map[string]interface{}) bool { - return isEnabled("grpc", conf) -} - -func isEnabledServerless(conf map[string]interface{}) bool { - return isEnabled("serverless", conf) + panic(fmt.Sprintf("listener not found for address %s:%s", network, address)) } -func isEnabled(key string, conf map[string]interface{}) bool { - if a, ok := conf[key]; ok { - if b, ok := a.(map[string]interface{}); ok { - if c, ok := b["services"]; ok { - if d, ok := c.(map[string]interface{}); ok { - if len(d) > 0 { - return true - } - } - } +func newServers(grpc []*config.GRPC, http []*config.HTTP, lns map[string]net.Listener, log *zerolog.Logger) ([]*Server, error) { + servers := make([]*Server, 0, len(grpc)+len(http)) + for _, cfg := range grpc { + services, err := rgrpc.InitServices(cfg.Services) + if err != nil { + return nil, err + } + unaryChain, streamChain, err := initGRPCInterceptors(cfg.Interceptors, grpcUnprotected(services), log) + if err != nil { + return nil, err + } + s, err := rgrpc.NewServer( + rgrpc.EnableReflection(cfg.EnableReflection), + rgrpc.WithShutdownDeadline(cfg.ShutdownDeadline), + rgrpc.WithLogger(log.With().Str("pkg", "grpc").Logger()), + rgrpc.WithServices(services), + rgrpc.WithUnaryServerInterceptors(unaryChain), + rgrpc.WithStreamServerInterceptors(streamChain), + ) + if err != nil { + return nil, err + } + ln := listenerFromAddress(lns, cfg.Network, cfg.Address) + server := &Server{ + server: s, + listener: ln, + services: maps.MapValues(services, func(s rgrpc.Service) any { return s }), + } + log.Debug(). + Interface("services", maps.Keys(cfg.Services)). + Msgf("spawned grpc server for services listening at %s:%s", ln.Addr().Network(), ln.Addr().String()) + servers = append(servers, server) + } + for _, cfg := range http { + log := log.With().Str("pkg", "http").Logger() + services, err := rhttp.InitServices(cfg.Services, &log) + if err != nil { + return nil, err + } + middlewares, err := initHTTPMiddlewares(cfg.Middlewares, httpUnprotected(services), &log) + if err != nil { + return nil, err + } + s, err := rhttp.New( + rhttp.WithServices(services), + rhttp.WithLogger(log), + rhttp.WithCertAndKeyFiles(cfg.CertFile, cfg.KeyFile), + rhttp.WithMiddlewares(middlewares), + ) + if err != nil { + return nil, err + } + ln := listenerFromAddress(lns, cfg.Network, cfg.Address) + server := &Server{ + server: s, + listener: ln, + services: maps.MapValues(services, func(s global.Service) any { return s }), } + log.Debug(). + Interface("services", maps.Keys(cfg.Services)). + Msgf("spawned http server for services listening at %s:%s", ln.Addr().Network(), ln.Addr().String()) + servers = append(servers, server) } - return false + return servers, nil } diff --git a/go.mod b/go.mod index e9533f42ce..59e7e61716 100644 --- a/go.mod +++ b/go.mod @@ -77,6 +77,7 @@ require ( ) require ( + github.com/creasty/defaults v1.7.0 // indirect github.com/go-jose/go-jose/v3 v3.0.0 // indirect github.com/hashicorp/go-msgpack/v2 v2.1.0 // indirect ) diff --git a/go.sum b/go.sum index bac70c99ab..5fa1a06c16 100644 --- a/go.sum +++ b/go.sum @@ -304,6 +304,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA= +github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= github.com/cs3org/go-cs3apis v0.0.0-20230508132523-e0d062e63b3b h1:UCO7Rnf5bvIvRtETguV8IaTx73cImLlFWxrApCB0QsQ= diff --git a/internal/grpc/services/applicationauth/applicationauth.go b/internal/grpc/services/applicationauth/applicationauth.go index 65514b6dcd..47f4c24680 100644 --- a/internal/grpc/services/applicationauth/applicationauth.go +++ b/internal/grpc/services/applicationauth/applicationauth.go @@ -73,7 +73,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a app auth provider svc. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/appprovider/appprovider.go b/internal/grpc/services/appprovider/appprovider.go index d09440f7f4..533afe4006 100644 --- a/internal/grpc/services/appprovider/appprovider.go +++ b/internal/grpc/services/appprovider/appprovider.go @@ -83,7 +83,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new AppProviderService. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/appregistry/appregistry.go b/internal/grpc/services/appregistry/appregistry.go index 619d366942..d70b9e7746 100644 --- a/internal/grpc/services/appregistry/appregistry.go +++ b/internal/grpc/services/appregistry/appregistry.go @@ -63,7 +63,7 @@ func (c *config) init() { } // New creates a new StorageRegistryService. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/appregistry/appregistry_test.go b/internal/grpc/services/appregistry/appregistry_test.go index 0bf23f8fa3..34b8c5d3d1 100644 --- a/internal/grpc/services/appregistry/appregistry_test.go +++ b/internal/grpc/services/appregistry/appregistry_test.go @@ -355,7 +355,7 @@ func TestNew(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := New(tt.m, nil) + got, err := New(tt.m) if err != nil { assert.Equal(t, tt.wantErr, err.Error()) assert.Nil(t, got) diff --git a/internal/grpc/services/authprovider/authprovider.go b/internal/grpc/services/authprovider/authprovider.go index c080d8135f..135af03745 100644 --- a/internal/grpc/services/authprovider/authprovider.go +++ b/internal/grpc/services/authprovider/authprovider.go @@ -100,7 +100,7 @@ func getAuthManager(manager string, m map[string]map[string]interface{}) (auth.M } // New returns a new AuthProviderServiceServer. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/authregistry/authregistry.go b/internal/grpc/services/authregistry/authregistry.go index 0abbe63ad5..dcbbd1b471 100644 --- a/internal/grpc/services/authregistry/authregistry.go +++ b/internal/grpc/services/authregistry/authregistry.go @@ -66,7 +66,7 @@ func (c *config) init() { } // New creates a new AuthRegistry. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/datatx/datatx.go b/internal/grpc/services/datatx/datatx.go index b9f5011a3d..ce82fa5c36 100644 --- a/internal/grpc/services/datatx/datatx.go +++ b/internal/grpc/services/datatx/datatx.go @@ -88,7 +88,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new datatx svc. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/gateway/gateway.go b/internal/grpc/services/gateway/gateway.go index 72776e5fb5..445ae7b13a 100644 --- a/internal/grpc/services/gateway/gateway.go +++ b/internal/grpc/services/gateway/gateway.go @@ -124,7 +124,7 @@ type svc struct { // New creates a new gateway svc that acts as a proxy for any grpc operation. // The gateway is responsible for high-level controls: rate-limiting, coordination between svcs // like sharing and storage acls, asynchronous transactions, ... -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/groupprovider/groupprovider.go b/internal/grpc/services/groupprovider/groupprovider.go index 1294e3892e..7e23c2f97b 100644 --- a/internal/grpc/services/groupprovider/groupprovider.go +++ b/internal/grpc/services/groupprovider/groupprovider.go @@ -68,7 +68,7 @@ func getDriver(c *config) (group.Manager, error) { } // New returns a new GroupProviderServiceServer. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/helloworld/helloworld.go b/internal/grpc/services/helloworld/helloworld.go index f0e9c311be..40aecaa356 100644 --- a/internal/grpc/services/helloworld/helloworld.go +++ b/internal/grpc/services/helloworld/helloworld.go @@ -43,7 +43,7 @@ type service struct { // New returns a new PreferencesServiceServer // It can be tested like this: // prototool grpc --address 0.0.0.0:9999 --method 'revad.helloworld.HelloWorldService/Hello' --data '{"name": "Alice"}'. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c := &conf{} if err := mapstructure.Decode(m, c); err != nil { err = errors.Wrap(err, "helloworld: error decoding conf") diff --git a/internal/grpc/services/ocmcore/ocmcore.go b/internal/grpc/services/ocmcore/ocmcore.go index ebb0d9b9f6..16686bc9fd 100644 --- a/internal/grpc/services/ocmcore/ocmcore.go +++ b/internal/grpc/services/ocmcore/ocmcore.go @@ -78,7 +78,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new ocm core svc. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/ocminvitemanager/ocminvitemanager.go b/internal/grpc/services/ocminvitemanager/ocminvitemanager.go index 80d8ff1d01..38cf922c5a 100644 --- a/internal/grpc/services/ocminvitemanager/ocminvitemanager.go +++ b/internal/grpc/services/ocminvitemanager/ocminvitemanager.go @@ -103,7 +103,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new OCM invite manager svc. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go b/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go index 9d0892068d..e14645fe5a 100644 --- a/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go +++ b/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go @@ -74,7 +74,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new OCM provider authorizer svc. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go index 9051f1dba2..c8f39962f0 100644 --- a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go +++ b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go @@ -110,7 +110,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new ocm share provider svc. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/permissions/permissions.go b/internal/grpc/services/permissions/permissions.go index 490d4c78fa..428e5457f1 100644 --- a/internal/grpc/services/permissions/permissions.go +++ b/internal/grpc/services/permissions/permissions.go @@ -55,7 +55,7 @@ type service struct { } // New returns a new PermissionsServiceServer. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/preferences/preferences.go b/internal/grpc/services/preferences/preferences.go index 25c814b1cc..f2b096cbf7 100644 --- a/internal/grpc/services/preferences/preferences.go +++ b/internal/grpc/services/preferences/preferences.go @@ -69,7 +69,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns a new PreferencesServiceServer. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/publicshareprovider/publicshareprovider.go b/internal/grpc/services/publicshareprovider/publicshareprovider.go index 5829b050d8..1d513e5554 100644 --- a/internal/grpc/services/publicshareprovider/publicshareprovider.go +++ b/internal/grpc/services/publicshareprovider/publicshareprovider.go @@ -87,7 +87,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new user share provider svc. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go index bf913e4136..43c141e6bf 100644 --- a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go +++ b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go @@ -83,7 +83,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new IsPublic Storage Provider service. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 572cf2bdd0..9bb1011817 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -162,7 +162,7 @@ func registerMimeTypes(mappingFile string) error { } // New creates a new storage provider svc. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/storageregistry/storageregistry.go b/internal/grpc/services/storageregistry/storageregistry.go index b7304ba571..685a8c0004 100644 --- a/internal/grpc/services/storageregistry/storageregistry.go +++ b/internal/grpc/services/storageregistry/storageregistry.go @@ -64,7 +64,7 @@ func (c *config) init() { } // New creates a new StorageBrokerService. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/userprovider/userprovider.go b/internal/grpc/services/userprovider/userprovider.go index 27907db225..fa6cb78abc 100644 --- a/internal/grpc/services/userprovider/userprovider.go +++ b/internal/grpc/services/userprovider/userprovider.go @@ -87,7 +87,7 @@ func getDriver(c *config) (user.Manager, *plugin.RevaPlugin, error) { } // New returns a new UserProviderServiceServer. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/usershareprovider/usershareprovider.go b/internal/grpc/services/usershareprovider/usershareprovider.go index 523a7b6c1f..074397364e 100644 --- a/internal/grpc/services/usershareprovider/usershareprovider.go +++ b/internal/grpc/services/usershareprovider/usershareprovider.go @@ -89,7 +89,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new user share provider svc. -func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { +func New(m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/rgrpc/option.go b/pkg/rgrpc/option.go new file mode 100644 index 0000000000..b59492dda0 --- /dev/null +++ b/pkg/rgrpc/option.go @@ -0,0 +1,62 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package rgrpc + +import ( + "github.com/rs/zerolog" + "google.golang.org/grpc" +) + +type Option func(*Server) + +func WithShutdownDeadline(deadline int) Option { + return func(s *Server) { + s.ShutdownDeadline = deadline + } +} + +func EnableReflection(enable bool) Option { + return func(s *Server) { + s.EnableReflection = enable + } +} + +func WithServices(services map[string]Service) Option { + return func(s *Server) { + s.services = services + } +} + +func WithLogger(logger zerolog.Logger) Option { + return func(s *Server) { + s.log = logger + } +} + +func WithStreamServerInterceptors(in []grpc.StreamServerInterceptor) Option { + return func(s *Server) { + s.StreamServerInterceptors = in + } +} + +func WithUnaryServerInterceptors(in []grpc.UnaryServerInterceptor) Option { + return func(s *Server) { + s.UnaryServerInterceptors = in + } +} diff --git a/pkg/rgrpc/rgrpc.go b/pkg/rgrpc/rgrpc.go index bb363b276e..5508bcb36a 100644 --- a/pkg/rgrpc/rgrpc.go +++ b/pkg/rgrpc/rgrpc.go @@ -22,21 +22,11 @@ import ( "fmt" "io" "net" - "sort" - - "github.com/cs3org/reva/internal/grpc/interceptors/appctx" - "github.com/cs3org/reva/internal/grpc/interceptors/auth" - "github.com/cs3org/reva/internal/grpc/interceptors/log" - "github.com/cs3org/reva/internal/grpc/interceptors/recovery" - "github.com/cs3org/reva/internal/grpc/interceptors/token" - "github.com/cs3org/reva/internal/grpc/interceptors/useragent" - "github.com/cs3org/reva/pkg/sharedconf" - rtrace "github.com/cs3org/reva/pkg/trace" + + "github.com/cs3org/reva/cmd/revad/pkg/config" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "github.com/rs/zerolog" - "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/reflection" ) @@ -73,7 +63,7 @@ func Register(name string, newFunc NewService) { // NewService is the function that gRPC services need to register at init time. // It returns an io.Closer to close the service and a list of service endpoints that need to be unprotected. -type NewService func(conf map[string]interface{}, ss *grpc.Server) (Service, error) +type NewService func(conf map[string]interface{}) (Service, error) // Service represents a grpc service. type Service interface { @@ -82,63 +72,51 @@ type Service interface { UnprotectedEndpoints() []string } -type unaryInterceptorTriple struct { - Name string - Priority int - Interceptor grpc.UnaryServerInterceptor -} - -type streamInterceptorTriple struct { - Name string - Priority int - Interceptor grpc.StreamServerInterceptor -} - -type config struct { - Network string `mapstructure:"network"` - Address string `mapstructure:"address"` - ShutdownDeadline int `mapstructure:"shutdown_deadline"` - Services map[string]map[string]interface{} `mapstructure:"services"` - Interceptors map[string]map[string]interface{} `mapstructure:"interceptors"` - EnableReflection bool `mapstructure:"enable_reflection"` -} - -func (c *config) init() { - if c.Network == "" { - c.Network = "tcp" - } - - if c.Address == "" { - c.Address = sharedconf.GetGatewaySVC("0.0.0.0:19000") - } -} - // Server is a gRPC server. type Server struct { + ShutdownDeadline int + EnableReflection bool + UnaryServerInterceptors []grpc.UnaryServerInterceptor + StreamServerInterceptors []grpc.StreamServerInterceptor + s *grpc.Server - conf *config listener net.Listener log zerolog.Logger services map[string]Service } -// NewServer returns a new Server. -func NewServer(m interface{}, log zerolog.Logger) (*Server, error) { - conf := &config{} - if err := mapstructure.Decode(m, conf); err != nil { - return nil, err +func InitServices(services map[string]config.ServicesConfig) (map[string]Service, error) { + s := make(map[string]Service) + for name, cfg := range services { + new, ok := Services[name] + if !ok { + return nil, fmt.Errorf("rgrpc: grpc service %s does not exist", name) + } + if cfg.DriversNumber() > 1 { + return nil, fmt.Errorf("rgrp: service %s cannot have more than one driver in same server", name) + } + svc, err := new(cfg[0].Config) + if err != nil { + return nil, errors.Wrapf(err, "rgrpc: grpc service %s could not be started,", name) + } + s[name] = svc } + return s, nil +} - conf.init() - - server := &Server{conf: conf, log: log, services: map[string]Service{}} +// NewServer returns a new Server. +func NewServer(o ...Option) (*Server, error) { + server := &Server{} + for _, oo := range o { + oo(server) + } return server, nil } // Start starts the server. func (s *Server) Start(ln net.Listener) error { - if err := s.registerServices(); err != nil { + if err := s.initServices(); err != nil { err = errors.Wrap(err, "unable to register services") return err } @@ -153,57 +131,15 @@ func (s *Server) Start(ln net.Listener) error { return nil } -func (s *Server) isInterceptorEnabled(name string) bool { - for k := range s.conf.Interceptors { - if k == name { - return true - } - } - return false -} - -func (s *Server) isServiceEnabled(svcName string) bool { - for key := range Services { - if key == svcName { - return true - } - } - return false -} - -func (s *Server) registerServices() error { - for svcName := range s.conf.Services { - if s.isServiceEnabled(svcName) { - newFunc := Services[svcName] - svc, err := newFunc(s.conf.Services[svcName], s.s) - if err != nil { - return errors.Wrapf(err, "rgrpc: grpc service %s could not be started,", svcName) - } - s.services[svcName] = svc - s.log.Info().Msgf("rgrpc: grpc service enabled: %s", svcName) - } else { - message := fmt.Sprintf("rgrpc: grpc service %s does not exist", svcName) - return errors.New(message) - } - } - - // obtain list of unprotected endpoints - unprotected := []string{} - for _, svc := range s.services { - unprotected = append(unprotected, svc.UnprotectedEndpoints()...) - } - - opts, err := s.getInterceptors(unprotected) - if err != nil { - return err - } +func (s *Server) initServices() error { + opts := s.getInterceptors() grpcServer := grpc.NewServer(opts...) for _, svc := range s.services { svc.Register(grpcServer) } - if s.conf.EnableReflection { + if s.EnableReflection { s.log.Info().Msg("rgrpc: grpc server reflection enabled") reflection.Register(grpcServer) } @@ -240,109 +176,20 @@ func (s *Server) GracefulStop() error { // Network returns the network type. func (s *Server) Network() string { - return s.conf.Network + return s.listener.Addr().Network() } // Address returns the network address. func (s *Server) Address() string { - return s.conf.Address + return s.listener.Addr().String() } -func (s *Server) getInterceptors(unprotected []string) ([]grpc.ServerOption, error) { - unaryTriples := []*unaryInterceptorTriple{} - for name, newFunc := range UnaryInterceptors { - if s.isInterceptorEnabled(name) { - inter, prio, err := newFunc(s.conf.Interceptors[name]) - if err != nil { - err = errors.Wrapf(err, "rgrpc: error creating unary interceptor: %s,", name) - return nil, err - } - triple := &unaryInterceptorTriple{ - Name: name, - Priority: prio, - Interceptor: inter, - } - unaryTriples = append(unaryTriples, triple) - } - } - - // sort unary triples - sort.SliceStable(unaryTriples, func(i, j int) bool { - return unaryTriples[i].Priority < unaryTriples[j].Priority - }) +func (s *Server) getInterceptors() []grpc.ServerOption { + unaryChain := grpc_middleware.ChainUnaryServer(s.UnaryServerInterceptors...) + streamChain := grpc_middleware.ChainStreamServer(s.StreamServerInterceptors...) - authUnary, err := auth.NewUnary(s.conf.Interceptors["auth"], unprotected) - if err != nil { - return nil, errors.Wrap(err, "rgrpc: error creating unary auth interceptor") - } - - unaryInterceptors := []grpc.UnaryServerInterceptor{authUnary} - for _, t := range unaryTriples { - unaryInterceptors = append(unaryInterceptors, t.Interceptor) - s.log.Info().Msgf("rgrpc: chaining grpc unary interceptor %s with priority %d", t.Name, t.Priority) - } - - unaryInterceptors = append(unaryInterceptors, - otelgrpc.UnaryServerInterceptor( - otelgrpc.WithTracerProvider(rtrace.Provider), - otelgrpc.WithPropagators(rtrace.Propagator)), - ) - - unaryInterceptors = append([]grpc.UnaryServerInterceptor{ - appctx.NewUnary(s.log), - token.NewUnary(), - useragent.NewUnary(), - log.NewUnary(), - recovery.NewUnary(), - }, unaryInterceptors...) - unaryChain := grpc_middleware.ChainUnaryServer(unaryInterceptors...) - - streamTriples := []*streamInterceptorTriple{} - for name, newFunc := range StreamInterceptors { - if s.isInterceptorEnabled(name) { - inter, prio, err := newFunc(s.conf.Interceptors[name]) - if err != nil { - err = errors.Wrapf(err, "rgrpc: error creating streaming interceptor: %s,", name) - return nil, err - } - triple := &streamInterceptorTriple{ - Name: name, - Priority: prio, - Interceptor: inter, - } - streamTriples = append(streamTriples, triple) - } - } - // sort stream triples - sort.SliceStable(streamTriples, func(i, j int) bool { - return streamTriples[i].Priority < streamTriples[j].Priority - }) - - authStream, err := auth.NewStream(s.conf.Interceptors["auth"], unprotected) - if err != nil { - return nil, errors.Wrap(err, "rgrpc: error creating stream auth interceptor") - } - - streamInterceptors := []grpc.StreamServerInterceptor{authStream} - for _, t := range streamTriples { - streamInterceptors = append(streamInterceptors, t.Interceptor) - s.log.Info().Msgf("rgrpc: chaining grpc streaming interceptor %s with priority %d", t.Name, t.Priority) - } - - streamInterceptors = append([]grpc.StreamServerInterceptor{ - authStream, - appctx.NewStream(s.log), - token.NewStream(), - useragent.NewStream(), - log.NewStream(), - recovery.NewStream(), - }, streamInterceptors...) - streamChain := grpc_middleware.ChainStreamServer(streamInterceptors...) - - opts := []grpc.ServerOption{ + return []grpc.ServerOption{ grpc.UnaryInterceptor(unaryChain), grpc.StreamInterceptor(streamChain), } - - return opts, nil } diff --git a/pkg/rhttp/rhttp.go b/pkg/rhttp/rhttp.go index de000aaac1..e5d796f917 100644 --- a/pkg/rhttp/rhttp.go +++ b/pkg/rhttp/rhttp.go @@ -24,85 +24,99 @@ import ( "net" "net/http" "path" - "sort" "strings" "time" - "github.com/cs3org/reva/internal/http/interceptors/appctx" - "github.com/cs3org/reva/internal/http/interceptors/auth" - "github.com/cs3org/reva/internal/http/interceptors/log" - "github.com/cs3org/reva/internal/http/interceptors/providerauthorizer" + "github.com/cs3org/reva/cmd/revad/pkg/config" "github.com/cs3org/reva/pkg/rhttp/global" rtrace "github.com/cs3org/reva/pkg/trace" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "github.com/rs/zerolog" "go.opentelemetry.io/otel/propagation" ) -// New returns a new server. -func New(m interface{}, l zerolog.Logger) (*Server, error) { - conf := &config{} - if err := mapstructure.Decode(m, conf); err != nil { - return nil, err +type Config func(*Server) + +func WithServices(services map[string]global.Service) Config { + return func(s *Server) { + s.Services = services + } +} + +func WithMiddlewares(middlewares []global.Middleware) Config { + return func(s *Server) { + s.middlewares = middlewares + } +} + +func WithCertAndKeyFiles(cert, key string) Config { + return func(s *Server) { + s.CertFile = cert + s.KeyFile = key } +} - conf.init() +func WithLogger(log zerolog.Logger) Config { + return func(s *Server) { + s.log = log + } +} +func InitServices(services map[string]config.ServicesConfig, log *zerolog.Logger) (map[string]global.Service, error) { + s := make(map[string]global.Service) + for name, cfg := range services { + new, ok := global.Services[name] + if !ok { + return nil, fmt.Errorf("http service %s does not exist", name) + } + if cfg.DriversNumber() > 1 { + return nil, fmt.Errorf("service %s cannot have more than one driver in the same server", name) + } + log := log.With().Str("service", name).Logger() + svc, err := new(cfg[0].Config, &log) + if err != nil { + return nil, errors.Wrapf(err, "http service %s could not be started", name) + } + s[name] = svc + } + return s, nil +} + +// New returns a new server. +func New(c ...Config) (*Server, error) { httpServer := &http.Server{} s := &Server{ + log: zerolog.Nop(), httpServer: httpServer, - conf: conf, svcs: map[string]global.Service{}, unprotected: []string{}, handlers: map[string]http.Handler{}, - log: l, + middlewares: []global.Middleware{}, + } + for _, cc := range c { + cc(s) } + s.registerServices() return s, nil } // Server contains the server info. type Server struct { + Services map[string]global.Service // map key is service name + CertFile string + KeyFile string + httpServer *http.Server - conf *config listener net.Listener svcs map[string]global.Service // map key is svc Prefix unprotected []string handlers map[string]http.Handler - middlewares []*middlewareTriple + middlewares []global.Middleware log zerolog.Logger } -type config struct { - Network string `mapstructure:"network"` - Address string `mapstructure:"address"` - Services map[string]map[string]interface{} `mapstructure:"services"` - Middlewares map[string]map[string]interface{} `mapstructure:"middlewares"` - CertFile string `mapstructure:"certfile"` - KeyFile string `mapstructure:"keyfile"` -} - -func (c *config) init() { - // apply defaults - if c.Network == "" { - c.Network = "tcp" - } - - if c.Address == "" { - c.Address = "0.0.0.0:19001" - } -} - // Start starts the server. func (s *Server) Start(ln net.Listener) error { - if err := s.registerServices(); err != nil { - return err - } - - if err := s.registerMiddlewares(); err != nil { - return err - } - handler, err := s.getHandler() if err != nil { return errors.Wrap(err, "rhttp: error creating http handler") @@ -111,11 +125,11 @@ func (s *Server) Start(ln net.Listener) error { s.httpServer.Handler = handler s.listener = ln - if (s.conf.CertFile != "") && (s.conf.KeyFile != "") { - s.log.Info().Msgf("https server listening at https://%s '%s' '%s'", s.conf.Address, s.conf.CertFile, s.conf.KeyFile) - err = s.httpServer.ServeTLS(s.listener, s.conf.CertFile, s.conf.KeyFile) + if (s.CertFile != "") && (s.KeyFile != "") { + s.log.Info().Msgf("https server listening at https://%s using cert file '%s' and key file '%s'", s.listener.Addr(), s.CertFile, s.KeyFile) + err = s.httpServer.ServeTLS(s.listener, s.CertFile, s.KeyFile) } else { - s.log.Info().Msgf("http server listening at http://%s '%s' '%s'", s.conf.Address, s.conf.CertFile, s.conf.KeyFile) + s.log.Info().Msgf("http server listening at http://%s", s.listener.Addr()) err = s.httpServer.Serve(s.listener) } if err == nil || err == http.ErrServerClosed { @@ -148,12 +162,12 @@ func (s *Server) closeServices() { // Network return the network type. func (s *Server) Network() string { - return s.conf.Network + return s.listener.Addr().Network() } // Address returns the network address. func (s *Server) Address() string { - return s.conf.Address + return s.listener.Addr().String() } // GracefulStop gracefully stops the server. @@ -162,68 +176,15 @@ func (s *Server) GracefulStop() error { return s.httpServer.Shutdown(context.Background()) } -// middlewareTriple represents a middleware with the -// priority to be chained. -type middlewareTriple struct { - Name string - Priority int - Middleware global.Middleware -} - -func (s *Server) registerMiddlewares() error { - middlewares := []*middlewareTriple{} - for name, newFunc := range global.NewMiddlewares { - if s.isMiddlewareEnabled(name) { - m, prio, err := newFunc(s.conf.Middlewares[name]) - if err != nil { - err = errors.Wrapf(err, "error creating new middleware: %s,", name) - return err - } - middlewares = append(middlewares, &middlewareTriple{ - Name: name, - Priority: prio, - Middleware: m, - }) - s.log.Info().Msgf("http middleware enabled: %s", name) - } - } - s.middlewares = middlewares - return nil -} - -func (s *Server) isMiddlewareEnabled(name string) bool { - _, ok := s.conf.Middlewares[name] - return ok -} - -func (s *Server) registerServices() error { - for svcName := range s.conf.Services { - if s.isServiceEnabled(svcName) { - newFunc := global.Services[svcName] - svcLogger := s.log.With().Str("service", svcName).Logger() - svc, err := newFunc(s.conf.Services[svcName], &svcLogger) - if err != nil { - err = errors.Wrapf(err, "http service %s could not be started,", svcName) - return err - } - - // instrument services with opencensus tracing. - h := traceHandler(svcName, svc.Handler()) - s.handlers[svc.Prefix()] = h - s.svcs[svc.Prefix()] = svc - s.unprotected = append(s.unprotected, getUnprotected(svc.Prefix(), svc.Unprotected())...) - s.log.Info().Msgf("http service enabled: %s@/%s", svcName, svc.Prefix()) - } else { - message := fmt.Sprintf("http service %s does not exist", svcName) - return errors.New(message) - } +func (s *Server) registerServices() { + for name, svc := range s.Services { + // instrument services with opencensus tracing. + h := traceHandler(name, svc.Handler()) + s.handlers[svc.Prefix()] = h + s.svcs[svc.Prefix()] = svc + s.unprotected = append(s.unprotected, getUnprotected(svc.Prefix(), svc.Unprotected())...) + s.log.Info().Msgf("http service enabled: %s@/%s", name, svc.Prefix()) } - return nil -} - -func (s *Server) isServiceEnabled(svcName string) bool { - _, ok := global.Services[svcName] - return ok } // TODO(labkode): if the http server is exposed under a basename we need to prepend @@ -312,44 +273,9 @@ func (s *Server) getHandler() (http.Handler, error) { w.WriteHeader(http.StatusNotFound) }) - // sort middlewares by priority. - sort.SliceStable(s.middlewares, func(i, j int) bool { - return s.middlewares[i].Priority > s.middlewares[j].Priority - }) - handler := http.Handler(h) - - for _, triple := range s.middlewares { - s.log.Info().Msgf("chaining http middleware %s with priority %d", triple.Name, triple.Priority) - handler = triple.Middleware(traceHandler(triple.Name, handler)) - } - - for _, v := range s.unprotected { - s.log.Info().Msgf("unprotected URL: %s", v) - } - authMiddle, err := auth.New(s.conf.Middlewares["auth"], s.unprotected) - if err != nil { - return nil, errors.Wrap(err, "rhttp: error creating auth middleware") - } - - // add always the logctx middleware as most priority, this middleware is internal - // and cannot be configured from the configuration. - coreMiddlewares := []*middlewareTriple{} - - providerAuthMiddle, err := addProviderAuthMiddleware(s.conf, s.unprotected) - if err != nil { - return nil, errors.Wrap(err, "rhttp: error creating providerauthorizer middleware") - } - if providerAuthMiddle != nil { - coreMiddlewares = append(coreMiddlewares, &middlewareTriple{Middleware: providerAuthMiddle, Name: "providerauthorizer"}) - } - - coreMiddlewares = append(coreMiddlewares, &middlewareTriple{Middleware: authMiddle, Name: "auth"}) - coreMiddlewares = append(coreMiddlewares, &middlewareTriple{Middleware: log.New(), Name: "log"}) - coreMiddlewares = append(coreMiddlewares, &middlewareTriple{Middleware: appctx.New(s.log), Name: "appctx"}) - - for _, triple := range coreMiddlewares { - handler = triple.Middleware(traceHandler(triple.Name, handler)) + for _, m := range s.middlewares { + handler = m(handler) } return handler, nil @@ -366,13 +292,3 @@ func traceHandler(name string, h http.Handler) http.Handler { h.ServeHTTP(w, r.WithContext(ctx)) }) } - -func addProviderAuthMiddleware(conf *config, unprotected []string) (global.Middleware, error) { - _, ocmdRegistered := global.Services["ocmd"] - _, ocmdEnabled := conf.Services["ocmd"] - ocmdPrefix, _ := conf.Services["ocmd"]["prefix"].(string) - if ocmdRegistered && ocmdEnabled { - return providerauthorizer.New(conf.Middlewares["providerauthorizer"], unprotected, ocmdPrefix) - } - return nil, nil -} diff --git a/cmd/revad/internal/config/config.go b/pkg/rserverless/option.go similarity index 61% rename from cmd/revad/internal/config/config.go rename to pkg/rserverless/option.go index 9ece6f8c1e..937b1eaa40 100644 --- a/cmd/revad/internal/config/config.go +++ b/pkg/rserverless/option.go @@ -16,29 +16,20 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package config +package rserverless -import ( - "io" +import "github.com/rs/zerolog" - "github.com/BurntSushi/toml" - "github.com/pkg/errors" -) +type Option func(*Serverless) -// Read reads the configuration from the reader. -func Read(r io.Reader) (map[string]interface{}, error) { - data, err := io.ReadAll(r) - if err != nil { - err = errors.Wrap(err, "config: error reading from reader") - return nil, err +func WithLogger(log *zerolog.Logger) Option { + return func(s *Serverless) { + s.log = log } +} - v := map[string]interface{}{} - err = toml.Unmarshal(data, &v) - if err != nil { - err = errors.Wrap(err, "config: error decoding toml data") - return nil, err +func WithServices(svc map[string]Service) Option { + return func(s *Serverless) { + s.Services = svc } - - return v, nil } diff --git a/pkg/rserverless/rserverless.go b/pkg/rserverless/rserverless.go index 7af26640c5..c68f1d6142 100644 --- a/pkg/rserverless/rserverless.go +++ b/pkg/rserverless/rserverless.go @@ -20,12 +20,9 @@ package rserverless import ( "context" - "fmt" "sync" "time" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" "github.com/rs/zerolog" ) @@ -48,45 +45,37 @@ type NewService func(conf map[string]interface{}, log *zerolog.Logger) (Service, // Serverless contains the serveless collection of services. type Serverless struct { - conf *config - log zerolog.Logger - services map[string]Service -} - -type config struct { - Services map[string]map[string]interface{} `mapstructure:"services"` + log *zerolog.Logger + Services map[string]Service } // New returns a new serverless collection of services. -func New(m interface{}, l zerolog.Logger) (*Serverless, error) { - conf := &config{} - if err := mapstructure.Decode(m, conf); err != nil { - return nil, err - } - +func New(opt ...Option) (*Serverless, error) { + l := zerolog.Nop() n := &Serverless{ - conf: conf, - log: l, - services: map[string]Service{}, + Services: map[string]Service{}, + log: &l, + } + for _, o := range opt { + o(n) } return n, nil } -func (s *Serverless) isServiceEnabled(svcName string) bool { - _, ok := Services[svcName] - return ok -} - // Start starts the serverless service collection. func (s *Serverless) Start() error { - return s.registerAndStartServices() + for name, svc := range s.Services { + go svc.Start() + s.log.Info().Msgf("serverless service enabled: %s", name) + } + return nil } // GracefulStop gracefully stops the serverless services. func (s *Serverless) GracefulStop() error { var wg sync.WaitGroup - for svcName, svc := range s.services { + for svcName, svc := range s.Services { wg.Add(1) go func(svcName string, svc Service) { @@ -113,7 +102,7 @@ func (s *Serverless) GracefulStop() error { func (s *Serverless) Stop() error { var wg sync.WaitGroup - for svcName, svc := range s.services { + for svcName, svc := range s.Services { wg.Add(1) go func(svcName string, svc Service) { @@ -136,26 +125,3 @@ func (s *Serverless) Stop() error { return nil } - -func (s *Serverless) registerAndStartServices() error { - for svcName := range s.conf.Services { - if s.isServiceEnabled(svcName) { - newFunc := Services[svcName] - svcLogger := s.log.With().Str("service", svcName).Logger() - svc, err := newFunc(s.conf.Services[svcName], &svcLogger) - if err != nil { - return errors.Wrapf(err, "serverless service %s could not be initialized", svcName) - } - - go svc.Start() - - s.services[svcName] = svc - - s.log.Info().Msgf("serverless service enabled: %s", svcName) - } else { - return fmt.Errorf("serverless service %s does not exist", svcName) - } - } - - return nil -} diff --git a/pkg/sharedconf/sharedconf.go b/pkg/sharedconf/sharedconf.go index 0503a5a806..c40aa6c0c0 100644 --- a/pkg/sharedconf/sharedconf.go +++ b/pkg/sharedconf/sharedconf.go @@ -19,50 +19,18 @@ package sharedconf import ( - "fmt" - "os" + "sync" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/cmd/revad/pkg/config" ) -var sharedConf = &conf{} +var sharedConf *config.Shared = &config.Shared{} +var once sync.Once -type conf struct { - JWTSecret string `mapstructure:"jwt_secret"` - GatewaySVC string `mapstructure:"gatewaysvc"` - DataGateway string `mapstructure:"datagateway"` - SkipUserGroupsInToken bool `mapstructure:"skip_user_groups_in_token"` - BlockedUsers []string `mapstructure:"blocked_users"` -} - -// Decode decodes the configuration. -func Decode(v interface{}) error { - if err := mapstructure.Decode(v, sharedConf); err != nil { - return err - } - - // add some defaults - if sharedConf.GatewaySVC == "" { - sharedConf.GatewaySVC = "0.0.0.0:19000" - } - - // this is the default address we use for the data gateway HTTP service - if sharedConf.DataGateway == "" { - host, err := os.Hostname() - if err != nil || host == "" { - sharedConf.DataGateway = "http://0.0.0.0:19001/datagateway" - } else { - sharedConf.DataGateway = fmt.Sprintf("http://%s:19001/datagateway", host) - } - } - - // TODO(labkode): would be cool to autogenerate one secret and print - // it on init time. - if sharedConf.JWTSecret == "" { - sharedConf.JWTSecret = "changemeplease" - } - - return nil +func Init(c *config.Shared) { + once.Do(func() { + sharedConf = c + }) } // GetJWTSecret returns the package level configured jwt secret if not overwritten. diff --git a/pkg/sharedconf/sharedconf_test.go b/pkg/sharedconf/sharedconf_test.go index e6ad94a777..0140e045bf 100644 --- a/pkg/sharedconf/sharedconf_test.go +++ b/pkg/sharedconf/sharedconf_test.go @@ -23,42 +23,42 @@ import ( ) func Test(t *testing.T) { - conf := map[string]interface{}{ - "jwt_secret": "", - "gateway": "", - } + // conf := map[string]interface{}{ + // "jwt_secret": "", + // "gateway": "", + // } - err := Decode(conf) - if err != nil { - t.Fatal(err) - } + // err := Decode(conf) + // if err != nil { + // t.Fatal(err) + // } - got := GetJWTSecret("secret") - if got != "secret" { - t.Fatalf("expected %q got %q", "secret", got) - } + // got := GetJWTSecret("secret") + // if got != "secret" { + // t.Fatalf("expected %q got %q", "secret", got) + // } - got = GetJWTSecret("") - if got != "changemeplease" { - t.Fatalf("expected %q got %q", "changemeplease", got) - } + // got = GetJWTSecret("") + // if got != "changemeplease" { + // t.Fatalf("expected %q got %q", "changemeplease", got) + // } - conf = map[string]interface{}{ - "jwt_secret": "dummy", - } + // conf = map[string]interface{}{ + // "jwt_secret": "dummy", + // } - err = Decode(conf) - if err != nil { - t.Fatal(err) - } + // err = Decode(conf) + // if err != nil { + // t.Fatal(err) + // } - got = GetJWTSecret("secret") - if got != "secret" { - t.Fatalf("expected %q got %q", "secret", got) - } + // got = GetJWTSecret("secret") + // if got != "secret" { + // t.Fatalf("expected %q got %q", "secret", got) + // } - got = GetJWTSecret("") - if got != "dummy" { - t.Fatalf("expected %q got %q", "dummy", got) - } + // got = GetJWTSecret("") + // if got != "dummy" { + // t.Fatalf("expected %q got %q", "dummy", got) + // } } diff --git a/pkg/utils/list/list.go b/pkg/utils/list/list.go index 2b1962ab2b..de279ebf77 100644 --- a/pkg/utils/list/list.go +++ b/pkg/utils/list/list.go @@ -34,3 +34,25 @@ func Remove[T any](l []T, i int) []T { l[i] = l[len(l)-1] return l[:len(l)-1] } + +// TakeFirst returns the first elemen, if any, that satisfies +// the predicate p. +func TakeFirst[T any](l []T, p func(T) bool) (T, bool) { + for _, e := range l { + if p(e) { + return e, true + } + } + var z T + return z, false +} + +// ToMap returns a map from l where the keys are obtainined applying +// the func k to the elements of l. +func ToMap[K comparable, T any](l []T, k func(T) K) map[K]T { + m := make(map[K]T, len(l)) + for _, e := range l { + m[k(e)] = e + } + return m +} diff --git a/pkg/utils/maps/maps.go b/pkg/utils/maps/maps.go new file mode 100644 index 0000000000..0b04f84025 --- /dev/null +++ b/pkg/utils/maps/maps.go @@ -0,0 +1,50 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package maps + +// Merge returns a map containing the keys and values from both maps. +// If the two maps share a set of keys, the result map will contain +// only the value of the second map. +func Merge[K comparable, T any](m, n map[K]T) map[K]T { + r := make(map[K]T, len(m)+len(n)) + for k, v := range m { + r[k] = v + } + for k, v := range n { + r[k] = v + } + return r +} + +// MapValues returns a map with vales mapped using the function f. +func MapValues[K comparable, T, V any](m map[K]T, f func(T) V) map[K]V { + r := make(map[K]V, len(m)) + for k, v := range m { + r[k] = f(v) + } + return r +} + +func Keys[K comparable, V any](m map[K]V) []K { + l := make([]K, 0, len(m)) + for k := range m { + l = append(l, k) + } + return l +} diff --git a/pkg/utils/net/net.go b/pkg/utils/net/net.go new file mode 100644 index 0000000000..51b94117f0 --- /dev/null +++ b/pkg/utils/net/net.go @@ -0,0 +1,54 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package net + +import "net" + +// AddressEqual return true if the addresses are equal. +// For tpc addresses only the port is compared, for unix +// the name and net are compared. +func AddressEqual(a net.Addr, network, address string) bool { + if a.Network() != network { + return false + } + + switch network { + case "tcp": + t, err := net.ResolveTCPAddr(network, address) + if err != nil { + return false + } + return tcpAddressEqual(a.(*net.TCPAddr), t) + case "unix": + t, err := net.ResolveUnixAddr(network, address) + if err != nil { + return false + } + return unixAddressEqual(a.(*net.UnixAddr), t) + } + return false +} + +func tcpAddressEqual(a1, a2 *net.TCPAddr) bool { + return a1.Port == a2.Port +} + +func unixAddressEqual(a1, a2 *net.UnixAddr) bool { + return a1.Name == a2.Name && a1.Net == a2.Net +} From 1a59d97692504569e3922894189f35f930de7a56 Mon Sep 17 00:00:00 2001 From: Sabin Panta <64484313+S-Panta@users.noreply.github.com> Date: Mon, 3 Jul 2023 14:07:56 +0545 Subject: [PATCH 27/48] [tests-only] [full-ci] Bumped ocis commit id (#4028) * Bumped ocis commit id * Bumped ocis commit id --- .drone.env | 2 +- tests/ocis | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.env b/.drone.env index 4747c6fcd9..a792be49f4 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=2ea3b8c400688b14ef328786b3af445c2edbbe18 +APITESTS_COMMITID=a8ff963166ecd9adf3f44aa6fa9fe68f53517d05 APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/tests/ocis b/tests/ocis index 2ea3b8c400..a8ff963166 160000 --- a/tests/ocis +++ b/tests/ocis @@ -1 +1 @@ -Subproject commit 2ea3b8c400688b14ef328786b3af445c2edbbe18 +Subproject commit a8ff963166ecd9adf3f44aa6fa9fe68f53517d05 From 740d58cc82b2fe992f2c3023eefad400f85465da Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Mon, 3 Jul 2023 10:47:10 +0200 Subject: [PATCH 28/48] new mono process ocm config (#4029) --- examples/ocmd/server-1.toml | 187 ++++++++++++++++++ examples/ocmd/server-1/ocmd-machine-auth.toml | 12 -- examples/ocmd/server-1/ocmd-outcoming.toml | 28 --- .../ocmd/server-1/ocmd-received-shares-1.toml | 18 -- examples/ocmd/server-1/ocmd-server-1.toml | 177 ----------------- examples/ocmd/server-2.toml | 187 ++++++++++++++++++ examples/ocmd/server-2/ocmd-machine-auth.toml | 12 -- examples/ocmd/server-2/ocmd-outcoming.toml | 28 --- .../ocmd/server-2/ocmd-received-shares-2.toml | 18 -- examples/ocmd/server-2/ocmd-server-2.toml | 168 ---------------- 10 files changed, 374 insertions(+), 461 deletions(-) create mode 100644 examples/ocmd/server-1.toml delete mode 100644 examples/ocmd/server-1/ocmd-machine-auth.toml delete mode 100644 examples/ocmd/server-1/ocmd-outcoming.toml delete mode 100644 examples/ocmd/server-1/ocmd-received-shares-1.toml delete mode 100644 examples/ocmd/server-1/ocmd-server-1.toml create mode 100644 examples/ocmd/server-2.toml delete mode 100644 examples/ocmd/server-2/ocmd-machine-auth.toml delete mode 100644 examples/ocmd/server-2/ocmd-outcoming.toml delete mode 100644 examples/ocmd/server-2/ocmd-received-shares-2.toml delete mode 100644 examples/ocmd/server-2/ocmd-server-2.toml diff --git a/examples/ocmd/server-1.toml b/examples/ocmd/server-1.toml new file mode 100644 index 0000000000..ebff43f3de --- /dev/null +++ b/examples/ocmd/server-1.toml @@ -0,0 +1,187 @@ +[shared] +gatewaysvc = "{{ grpc.services.gateway.address }}" + +[vars] +machine_api_key = "machine-api-key" +provider_domain = "cernbox.cern.ch" + +[grpc.services.gateway] +address = "0.0.0.0:19000" +authregistrysvc = "{{ grpc.services.authregistry.address }}" +appregistrysvc = "{{ grpc.services.appregistry.address }}" +storageregistrysvc = "{{ grpc.services.storageregistry.address }}" +userprovidersvc = "{{ grpc.services.userprovider.address }}" +usershareprovidersvc = "{{ grpc.services.usershareprovider.address }}" +ocmcoresvc = "{{ grpc.services.ocmcore.address }}" +ocmshareprovidersvc = "{{ grpc.services.ocmshareprovider.address }}" +ocminvitemanagersvc = "{{ grpc.services.ocminvitemanager.address }}" +ocmproviderauthorizersvc = "{{ grpc.services.ocmproviderauthorizer.address }}" +datagateway = "http://localhost:{{ http.services.datagateway.address.port }}/data" +transfer_expires = 6 # give it a moment +commit_share_to_storage_grant = true +commit_share_to_storage_ref = true + +[grpc.services.authregistry] +driver = "static" + +[grpc.services.authregistry.drivers.static.rules] +basic = "{{ grpc.services.authprovider[0].address }}" +machine = "{{ grpc.services.authprovider[1].address }}" +ocmshares = "{{ grpc.services.authprovider[2].address }}" + +[grpc.services.appregistry] +driver = "static" + +[grpc.services.appprovider] +driver = "demo" +app_provider_url = "{{ grpc.services.appprovider.address }}" + +[grpc.services.storageregistry] +driver = "static" + +[grpc.services.storageregistry.drivers.static] +home_provider = "/home" + +[grpc.services.storageregistry.drivers.static.rules] +"/home" = { address = "{{ grpc.services.storageprovider[0].address }}" } +"123e4567-e89b-12d3-a456-426655440000" = { address = "{{ grpc.services.storageprovider[0].address }}" } +"/ocm" = { address = "{{ grpc.services.storageprovider[1].address }}" } +"ocm" = { address = "{{ grpc.services.storageprovider[1].address }}" } +"/sciencemesh" = { address = "{{ grpc.services.storageprovider[2].address }}" } +"sciencemesh" = { address = "{{ grpc.services.storageprovider[2].address }}" } + +[grpc.services.usershareprovider] +driver = "memory" + +[grpc.services.ocmcore] +driver = "json" + +# Note that ocmcore and ocmshareprovider should use the same file for storing the shares. +[grpc.services.ocmcore.drivers.json] +file = "/var/tmp/reva/shares_server_1.json" + +[grpc.services.ocminvitemanager] +driver = "json" +provider_domain = "{{ vars.provider_domain }}" + +[grpc.services.ocminvitemanager.drivers.json] +file = "/var/tmp/reva/invites_server_1.json" + +[grpc.services.ocmshareprovider] +driver = "json" +webdav_endpoint = "http://localhost:{{ http.services.ocdav.address.port }}" +provider_domain = "{{ vars.provider_domain }}" + +[grpc.services.ocmshareprovider.drivers.json] +file = "/var/tmp/reva/shares_server_1.json" + +[grpc.services.ocmproviderauthorizer] +driver = "json" + +[grpc.services.ocmproviderauthorizer.drivers.json] +providers = "/home/gianmaria/Documenti/CERN/reva2/examples/ocmd/providers.demo.json" + +################################################################ +########## STORAGE PROVIDERS ########## +################################################################ +[[grpc.services.storageprovider]] +driver = "localhome" +mount_path = "/home" +mount_id = "123e4567-e89b-12d3-a456-426655440000" +expose_data_server = true +data_server_url = "http://localhost:{{ http.services.dataprovider[0].address.port }}/data" +enable_home_creation = true + +[grpc.services.storageprovider.drivers.localhome] +user_layout = "{{.Username}}" + + +[[grpc.services.storageprovider]] +driver = "ocmoutcoming" +mount_path = "/ocm" +mount_id = "ocm" +expose_data_server = true +data_server_url = "http://localhost:{{ http.services.dataprovider[1].address.port }}/data" + +[grpc.services.storageprovider.drivers.ocmoutcoming] +machine_secret = "machine-api-key" + + +[[grpc.services.storageprovider]] +driver = "ocmreceived" +mount_path = "/sciencemesh" +mount_id = "sciencemesh" +expose_data_server = true +data_server_url = "http://localhost:{{ http.services.dataprovider[2].address.port }}/data" + +################################################################ +########## AUTH PROVIDERS ########## +################################################################ +[[grpc.services.authprovider]] +auth_manager = "json" + +[grpc.services.authprovider.auth_managers.json] +users = "/home/gianmaria/Documenti/CERN/reva2/examples/ocmd/users.demo.json" + + +[[grpc.services.authprovider]] +auth_manager = "machine" + +[grpc.services.authprovider.auth_managers.machine] +api_key = "{{ vars.machine_api_key }}" +gateway_addr = "{{ grpc.services.gateway.address }}" + + +[[grpc.services.authprovider]] +auth_manager = "ocmshares" + +################################################################ +########## USER PROVIDER ########## +################################################################ +[grpc.services.userprovider] +driver = "json" + +[grpc.services.userprovider.drivers.json] +users = "/home/gianmaria/Documenti/CERN/reva2/examples/ocmd/users.demo.json" + +################################################################ +########## DATA PROVIDERS ########## +################################################################ +[[http.services.dataprovider]] +driver = "localhome" + +[http.services.dataprovider.drivers.localhome] +user_layout = "{{.Username}}" + +[[http.services.dataprovider]] +driver = "ocmoutcoming" + +[http.services.dataprovider.drivers.ocmoutcoming] +machine_secret = "{{ vars.machine_api_key }}" + +[[http.services.dataprovider]] +driver = "ocmreceived" + +[http.services.ocmd] +address = "0.0.0.0:8080" +expose_recipient_display_name = true + +[http.services.ocmprovider] +ocm_prefix = "ocm" +provider = "reva@cern" +endpoint = "http://localhost:{{ http.services.ocmd.address.port }}" +enable_webapp = true +enable_datatx = true + +[http.services.ocs] +prefix = "ocs" + +[http.services.ocdav] +address = "0.0.0.0:8080" + +[http.services.datagateway] +address = "0.0.0.0:8080" + +[http.services.appprovider] + +[http.middlewares.cors] diff --git a/examples/ocmd/server-1/ocmd-machine-auth.toml b/examples/ocmd/server-1/ocmd-machine-auth.toml deleted file mode 100644 index 0fb314f986..0000000000 --- a/examples/ocmd/server-1/ocmd-machine-auth.toml +++ /dev/null @@ -1,12 +0,0 @@ -[shared] -gatewaysvc = "localhost:19000" - -[grpc] -address = "localhost:19030" - -[grpc.services.authprovider] -auth_manager = "machine" - -[grpc.services.authprovider.auth_managers.machine] -api_key = "machine-api-key" -gateway_addr = "localhost:19000" diff --git a/examples/ocmd/server-1/ocmd-outcoming.toml b/examples/ocmd/server-1/ocmd-outcoming.toml deleted file mode 100644 index 3461eaf242..0000000000 --- a/examples/ocmd/server-1/ocmd-outcoming.toml +++ /dev/null @@ -1,28 +0,0 @@ -[shared] -gatewaysvc = "localhost:19000" - -[grpc] -address = "localhost:19020" - -[grpc.services.storageprovider] -driver = "ocmoutcoming" -mount_path = "/ocm" -mount_id = "ocm" -expose_data_server = true -data_server_url = "http://localhost:19021/data" - -[grpc.services.storageprovider.drivers.ocmoutcoming] -machine_secret = "machine-api-key" - -[grpc.services.authprovider] -auth_manager = "ocmshares" - - -[http] -address = "localhost:19021" - -[http.services.dataprovider] -driver = "ocmoutcoming" - -[http.services.dataprovider.drivers.ocmoutcoming] -machine_secret = "machine-api-key" diff --git a/examples/ocmd/server-1/ocmd-received-shares-1.toml b/examples/ocmd/server-1/ocmd-received-shares-1.toml deleted file mode 100644 index 78f0782b4b..0000000000 --- a/examples/ocmd/server-1/ocmd-received-shares-1.toml +++ /dev/null @@ -1,18 +0,0 @@ -[shared] -gatewaysvc = "localhost:19000" - -[grpc] -address = "localhost:19010" - -[grpc.services.storageprovider] -driver = "ocmreceived" -mount_path = "/sciencemesh" -mount_id = "sciencemesh" -expose_data_server = true -data_server_url = "http://localhost:19011/data" - -[http] -address = "localhost:19011" - -[http.services.dataprovider] -driver = "ocmreceived" diff --git a/examples/ocmd/server-1/ocmd-server-1.toml b/examples/ocmd/server-1/ocmd-server-1.toml deleted file mode 100644 index 0d9adbf513..0000000000 --- a/examples/ocmd/server-1/ocmd-server-1.toml +++ /dev/null @@ -1,177 +0,0 @@ -[shared] -gatewaysvc = "localhost:19000" - -[grpc] -address = "0.0.0.0:19000" - -[grpc.services.gateway] -authregistrysvc = "localhost:19000" -appregistrysvc = "localhost:19000" -storageregistrysvc = "localhost:19000" -preferencessvc = "localhost:19000" -userprovidersvc = "localhost:19000" -usershareprovidersvc = "localhost:19000" -publicshareprovidersvc = "localhost:19000" -ocmcoresvc = "localhost:19000" -ocmshareprovidersvc = "localhost:19000" -ocminvitemanagersvc = "localhost:19000" -ocmproviderauthorizersvc = "localhost:19000" -datagateway = "http://localhost:19001/data" -transfer_expires = 6 # give it a moment -commit_share_to_storage_grant = true -commit_share_to_storage_ref = true - -[grpc.services.appregistry] -driver = "static" - -[grpc.services.appregistry.drivers.static] -mime_types = [ - {"mime_type" = "text/plain", "extension" = "txt", "name" = "Text file", "description" = "Text file", "allow_creation" = true}, - {"mime_type" = "text/markdown", "extension" = "md", "name" = "Markdown file", "description" = "Markdown file", "allow_creation" = true}, - {"mime_type" = "application/vnd.oasis.opendocument.text", "extension" = "odt", "name" = "OpenDocument", "description" = "OpenDocument text document", "default_app" = "Collabora", "allow_creation" = true}, - {"mime_type" = "application/vnd.oasis.opendocument.spreadsheet", "extension" = "ods", "name" = "OpenSpreadsheet", "description" = "OpenDocument spreadsheet document", "default_app" = "Collabora", "allow_creation" = true}, - {"mime_type" = "application/vnd.oasis.opendocument.presentation", "extension" = "odp", "name" = "OpenPresentation", "description" = "OpenDocument presentation document", "default_app" = "Collabora", "allow_creation" = true}, - {"mime_type" = "application/vnd.jupyter", "extension" = "ipynb", "name" = "Jupyter Notebook", "description" = "Jupyter Notebook"} -] - -[grpc.services.appprovider] -driver = "wopi" -appregistrysvc = "localhost:19000" - -[grpc.services.appprovider.drivers.wopi] -iop_secret = "shared-secret-with-wopiserver" -wopi_url = "http://0.0.0.0:8880/" -app_name = "Collabora" -app_url = "https://your-collabora-server.org:9980" - -[grpc.services.authregistry] -driver = "static" - -[grpc.services.authregistry.drivers.static.rules] -basic = "localhost:19000" -machine = "localhost:19030" -ocmshares = "localhost:19020" - -[grpc.services.storageregistry] -driver = "static" - -[grpc.services.storageregistry.drivers.static] -home_provider = "/home" - -[grpc.services.storageregistry.drivers.static.rules] -"/home" = { "address" = "localhost:19000" } -"123e4567-e89b-12d3-a456-426655440000" = { "address" = "localhost:19000" } -"/sciencemesh" = { "address" = "localhost:19010" } -"sciencemesh" = { "address" = "localhost:19010" } -"/ocm" = { "address" = "localhost:19020" } -"ocm" = { "address" = "localhost:19020" } - -[grpc.services.usershareprovider] -driver = "memory" - -[grpc.services.ocmcore] -driver = "json" - -# Note that ocmcore and ocmshareprovider should use the same file for storing the shares. -[grpc.services.ocmcore.drivers.json] -file = "/var/tmp/reva/shares_server_1.json" - -[grpc.services.ocminvitemanager] -driver = "json" -provider_domain = "cernbox.cern.ch" - -[grpc.services.ocminvitemanager.drivers.json] -file = "/var/tmp/reva/invites_server_1.json" - -[grpc.services.ocmshareprovider] -driver = "json" -webdav_endpoint = "http://localhost:19001" -provider_domain = "cernbox.cern.ch" - -[grpc.services.ocmshareprovider.drivers.json] -file = "/var/tmp/reva/shares_server_1.json" - -[grpc.services.ocmproviderauthorizer] -driver = "json" - -[grpc.services.ocmproviderauthorizer.drivers.json] -providers = "providers.demo.json" - -[grpc.services.publicshareprovider] -driver = "memory" - -[grpc.services.appregistry] -driver = "static" - -[grpc.services.appprovider] -driver = "demo" -app_provider_url = "localhost:19000" - -[grpc.services.storageprovider] -driver = "localhome" -mount_path = "/home" -mount_id = "123e4567-e89b-12d3-a456-426655440000" -expose_data_server = true -data_server_url = "http://localhost:19001/data" -enable_home_creation = true - -[grpc.services.storageprovider.drivers.localhome] -user_layout = "{{.Username}}" - -[grpc.services.authprovider] -auth_manager = "json" - -[grpc.services.authprovider.auth_managers.json] -users = "users.demo.json" - -[grpc.services.userprovider] -driver = "json" - -[grpc.services.userprovider.drivers.json] -users = "users.demo.json" - -[http] -address = "0.0.0.0:19001" - -[http.services.appprovider] - -[http.services.dataprovider] -driver = "localhome" - -[http.services.prometheus] -[http.services.sysinfo] - -[http.services.dataprovider.drivers.localhome] -user_layout = "{{.Username}}" - -[http.services.ocmd] -prefix = "ocm" - -[http.services.ocmprovider] -ocm_prefix = "ocm" -provider = "reva@cern" -endpoint = "http://localhost:19001" -enable_webapp = true -enable_datatx = true - -[http.middlewares.providerauthorizer] -driver = "json" - -[http.middlewares.providerauthorizer.drivers.json] -providers = "providers.demo.json" - -[http.services.ocs] -prefix = "ocs" - -[http.services.ocdav] - -[http.services.sciencemesh] -mesh_directory_url = 'https://sciencemesh.cesnet.cz/iop/meshdir' -provider_domain = 'your-domain-1.org' -ocm_mount_point = '/sciencemesh' - -[http.middlewares.cors] - -[http.middlewares.auth] -credential_chain = ["publicshares", "ocmshares", "basic", "bearer"] - diff --git a/examples/ocmd/server-2.toml b/examples/ocmd/server-2.toml new file mode 100644 index 0000000000..4ea85a969e --- /dev/null +++ b/examples/ocmd/server-2.toml @@ -0,0 +1,187 @@ +[shared] +gatewaysvc = "{{ grpc.services.gateway.address }}" + +[vars] +machine_api_key = "machine-api-key" +provider_domain = "cesnet.cz" + +[grpc.services.gateway] +address = "0.0.0.0:17000" +authregistrysvc = "{{ grpc.services.authregistry.address }}" +appregistrysvc = "{{ grpc.services.appregistry.address }}" +storageregistrysvc = "{{ grpc.services.storageregistry.address }}" +userprovidersvc = "{{ grpc.services.userprovider.address }}" +usershareprovidersvc = "{{ grpc.services.usershareprovider.address }}" +ocmcoresvc = "{{ grpc.services.ocmcore.address }}" +ocmshareprovidersvc = "{{ grpc.services.ocmshareprovider.address }}" +ocminvitemanagersvc = "{{ grpc.services.ocminvitemanager.address }}" +ocmproviderauthorizersvc = "{{ grpc.services.ocmproviderauthorizer.address }}" +datagateway = "http://localhost:{{ http.services.datagateway.address.port }}/data" +transfer_expires = 6 # give it a moment +commit_share_to_storage_grant = true +commit_share_to_storage_ref = true + +[grpc.services.authregistry] +driver = "static" + +[grpc.services.authregistry.drivers.static.rules] +basic = "{{ grpc.services.authprovider[0].address }}" +machine = "{{ grpc.services.authprovider[1].address }}" +ocmshares = "{{ grpc.services.authprovider[2].address }}" + +[grpc.services.appregistry] +driver = "static" + +[grpc.services.appprovider] +driver = "demo" +app_provider_url = "{{ grpc.services.appprovider.address }}" + +[grpc.services.storageregistry] +driver = "static" + +[grpc.services.storageregistry.drivers.static] +home_provider = "/home" + +[grpc.services.storageregistry.drivers.static.rules] +"/home" = { address = "{{ grpc.services.storageprovider[0].address }}" } +"123e4567-e89b-12d3-a456-426655440000" = { address = "{{ grpc.services.storageprovider[0].address }}" } +"/ocm" = { address = "{{ grpc.services.storageprovider[1].address }}" } +"ocm" = { address = "{{ grpc.services.storageprovider[1].address }}" } +"/sciencemesh" = { address = "{{ grpc.services.storageprovider[2].address }}" } +"sciencemesh" = { address = "{{ grpc.services.storageprovider[2].address }}" } + +[grpc.services.usershareprovider] +driver = "memory" + +[grpc.services.ocmcore] +driver = "json" + +# Note that ocmcore and ocmshareprovider should use the same file for storing the shares. +[grpc.services.ocmcore.drivers.json] +file = "/var/tmp/reva/shares_server_2.json" + +[grpc.services.ocminvitemanager] +driver = "json" +provider_domain = "{{ vars.provider_domain }}" + +[grpc.services.ocminvitemanager.drivers.json] +file = "/var/tmp/reva/invites_server_2.json" + +[grpc.services.ocmshareprovider] +driver = "json" +webdav_endpoint = "http://localhost:{{ http.services.ocdav.address.port }}" +provider_domain = "{{ vars.provider_domain }}" + +[grpc.services.ocmshareprovider.drivers.json] +file = "/var/tmp/reva/shares_server_2.json" + +[grpc.services.ocmproviderauthorizer] +driver = "json" + +[grpc.services.ocmproviderauthorizer.drivers.json] +providers = "/home/gianmaria/Documenti/CERN/reva2/examples/ocmd/providers.demo.json" + +################################################################ +########## STORAGE PROVIDERS ########## +################################################################ +[[grpc.services.storageprovider]] +driver = "localhome" +mount_path = "/home" +mount_id = "123e4567-e89b-12d3-a456-426655440000" +expose_data_server = true +data_server_url = "http://localhost:{{ http.services.dataprovider[0].address.port }}/data" +enable_home_creation = true + +[grpc.services.storageprovider.drivers.localhome] +user_layout = "{{.Username}}" + + +[[grpc.services.storageprovider]] +driver = "ocmoutcoming" +mount_path = "/ocm" +mount_id = "ocm" +expose_data_server = true +data_server_url = "http://localhost:{{ http.services.dataprovider[1].address.port }}/data" + +[grpc.services.storageprovider.drivers.ocmoutcoming] +machine_secret = "machine-api-key" + + +[[grpc.services.storageprovider]] +driver = "ocmreceived" +mount_path = "/sciencemesh" +mount_id = "sciencemesh" +expose_data_server = true +data_server_url = "http://localhost:{{ http.services.dataprovider[2].address.port }}/data" + +################################################################ +########## AUTH PROVIDERS ########## +################################################################ +[[grpc.services.authprovider]] +auth_manager = "json" + +[grpc.services.authprovider.auth_managers.json] +users = "/home/gianmaria/Documenti/CERN/reva2/examples/ocmd/users.demo.json" + + +[[grpc.services.authprovider]] +auth_manager = "machine" + +[grpc.services.authprovider.auth_managers.machine] +api_key = "{{ vars.machine_api_key }}" +gateway_addr = "{{ grpc.services.gateway.address }}" + + +[[grpc.services.authprovider]] +auth_manager = "ocmshares" + +################################################################ +########## USER PROVIDER ########## +################################################################ +[grpc.services.userprovider] +driver = "json" + +[grpc.services.userprovider.drivers.json] +users = "/home/gianmaria/Documenti/CERN/reva2/examples/ocmd/users.demo.json" + +################################################################ +########## DATA PROVIDERS ########## +################################################################ +[[http.services.dataprovider]] +driver = "localhome" + +[http.services.dataprovider.drivers.localhome] +user_layout = "{{.Username}}" + +[[http.services.dataprovider]] +driver = "ocmoutcoming" + +[http.services.dataprovider.drivers.ocmoutcoming] +machine_secret = "{{ vars.machine_api_key }}" + +[[http.services.dataprovider]] +driver = "ocmreceived" + +[http.services.ocmd] +address = "0.0.0.0:80" +expose_recipient_display_name = true + +[http.services.ocmprovider] +ocm_prefix = "ocm" +provider = "reva@cesnet" +endpoint = "http://localhost:{{ http.services.ocmd.address.port }}" +enable_webapp = true +enable_datatx = true + +[http.services.ocs] +prefix = "ocs" + +[http.services.ocdav] +address = "0.0.0.0:80" + +[http.services.datagateway] +address = "0.0.0.0:80" + +[http.services.appprovider] + +[http.middlewares.cors] diff --git a/examples/ocmd/server-2/ocmd-machine-auth.toml b/examples/ocmd/server-2/ocmd-machine-auth.toml deleted file mode 100644 index dc4854b472..0000000000 --- a/examples/ocmd/server-2/ocmd-machine-auth.toml +++ /dev/null @@ -1,12 +0,0 @@ -[shared] -gatewaysvc = "localhost:17000" - -[grpc] -address = "localhost:17030" - -[grpc.services.authprovider] -auth_manager = "machine" - -[grpc.services.authprovider.auth_managers.machine] -api_key = "machine-api-key" -gateway_addr = "localhost:17000" diff --git a/examples/ocmd/server-2/ocmd-outcoming.toml b/examples/ocmd/server-2/ocmd-outcoming.toml deleted file mode 100644 index 7bf1a59f32..0000000000 --- a/examples/ocmd/server-2/ocmd-outcoming.toml +++ /dev/null @@ -1,28 +0,0 @@ -[shared] -gatewaysvc = "localhost:17000" - -[grpc] -address = "localhost:17020" - -[grpc.services.storageprovider] -driver = "ocmoutcoming" -mount_path = "/ocm" -mount_id = "ocm" -expose_data_server = true -data_server_url = "http://localhost:17021/data" - -[grpc.services.storageprovider.drivers.ocmoutcoming] -machine_secret = "machine-api-key" - -[grpc.services.authprovider] -auth_manager = "ocmshares" - - -[http] -address = "localhost:17021" - -[http.services.dataprovider] -driver = "ocmoutcoming" - -[http.services.dataprovider.drivers.ocmoutcoming] -machine_secret = "machine-api-key" diff --git a/examples/ocmd/server-2/ocmd-received-shares-2.toml b/examples/ocmd/server-2/ocmd-received-shares-2.toml deleted file mode 100644 index 05cb2ee22c..0000000000 --- a/examples/ocmd/server-2/ocmd-received-shares-2.toml +++ /dev/null @@ -1,18 +0,0 @@ -[shared] -gatewaysvc = "localhost:17000" - -[grpc] -address = "localhost:17010" - -[grpc.services.storageprovider] -driver = "ocmreceived" -mount_path = "/sciencemesh" -mount_id = "sciencemesh" -expose_data_server = true -data_server_url = "http://localhost:17011/data" - -[http] -address = "localhost:17011" - -[http.services.dataprovider] -driver = "ocmreceived" diff --git a/examples/ocmd/server-2/ocmd-server-2.toml b/examples/ocmd/server-2/ocmd-server-2.toml deleted file mode 100644 index 755d00116e..0000000000 --- a/examples/ocmd/server-2/ocmd-server-2.toml +++ /dev/null @@ -1,168 +0,0 @@ -[shared] -gatewaysvc = "localhost:17000" - -[grpc] -address = "0.0.0.0:17000" - -[grpc.services.gateway] -authregistrysvc = "localhost:17000" -appregistrysvc = "localhost:17000" -storageregistrysvc = "localhost:17000" -preferencessvc = "localhost:17000" -userprovidersvc = "localhost:17000" -usershareprovidersvc = "localhost:17000" -publicshareprovidersvc = "localhost:17000" -ocmcoresvc = "localhost:17000" -ocmshareprovidersvc = "localhost:17000" -ocminvitemanagersvc = "localhost:17000" -ocmproviderauthorizersvc = "localhost:17000" -datagateway = "http://localhost:17001/data" -transfer_expires = 6 # give it a moment -commit_share_to_storage_grant = true -commit_share_to_storage_ref = true - -[grpc.services.appregistry] -driver = "static" - -[grpc.services.appregistry.drivers.static] -mime_types = [ - {"mime_type" = "text/plain", "extension" = "txt", "name" = "Text file", "description" = "Text file", "allow_creation" = true}, - {"mime_type" = "text/markdown", "extension" = "md", "name" = "Markdown file", "description" = "Markdown file", "allow_creation" = true}, - {"mime_type" = "application/vnd.oasis.opendocument.text", "extension" = "odt", "name" = "OpenDocument", "description" = "OpenDocument text document", "default_app" = "Collabora", "allow_creation" = true}, - {"mime_type" = "application/vnd.oasis.opendocument.spreadsheet", "extension" = "ods", "name" = "OpenSpreadsheet", "description" = "OpenDocument spreadsheet document", "default_app" = "Collabora", "allow_creation" = true}, - {"mime_type" = "application/vnd.oasis.opendocument.presentation", "extension" = "odp", "name" = "OpenPresentation", "description" = "OpenDocument presentation document", "default_app" = "Collabora", "allow_creation" = true}, - {"mime_type" = "application/vnd.jupyter", "extension" = "ipynb", "name" = "Jupyter Notebook", "description" = "Jupyter Notebook"} -] - -[grpc.services.appprovider] -driver = "wopi" -app_provider_url = "localhost:17000" - -[grpc.services.appprovider.drivers.wopi] -iop_secret = "shared-secret-with-wopiserver" -wopi_url = "http://0.0.0.0:8880/" -app_name = "Collabora" -app_url = "https://your-collabora-server.org:9980" - -[grpc.services.authregistry] -driver = "static" - -[grpc.services.authregistry.drivers.static.rules] -basic = "localhost:17000" -machine = "localhost:17030" -ocmshares = "localhost:17020" - -[grpc.services.storageregistry] -driver = "static" - -[grpc.services.storageregistry.drivers.static] -home_provider = "/home" - -[grpc.services.storageregistry.drivers.static.rules] -"/home" = { "address" = "localhost:17000" } -"123e4567-e89b-12d3-a456-426655440000" = { "address" = "localhost:17000" } -"/sciencemesh" = { "address" = "localhost:17010" } -"sciencemesh" = { "address" = "localhost:17010" } -"/ocm" = { "address" = "localhost:17020" } -"ocm" = { "address" = "localhost:17020" } - -[grpc.services.usershareprovider] -driver = "memory" - -[grpc.services.ocmcore] -driver = "json" - -# Note that ocmcore and ocmshareprovider should use the same file for storing the shares. -[grpc.services.ocmcore.drivers.json] -file = "/var/tmp/reva/shares_server_2.json" - -[grpc.services.ocminvitemanager] -driver = "json" -provider_domain = "cesnet.cz" - -[grpc.services.ocminvitemanager.drivers.json] -file = "/var/tmp/reva/invites_server_2.json" - -[grpc.services.ocmshareprovider] -driver = "json" -webdav_endpoint = "http://localhost:17001" -provider_domain = "cesnet.cz" - -[grpc.services.ocmshareprovider.drivers.json] -file = "/var/tmp/reva/shares_server_2.json" - -[grpc.services.ocmproviderauthorizer] -driver = "json" - -[grpc.services.ocmproviderauthorizer.drivers.json] -providers = "providers.demo.json" - -[grpc.services.publicshareprovider] -driver = "memory" - -[grpc.services.storageprovider] -driver = "localhome" -mount_path = "/home" -mount_id = "123e4567-e89b-12d3-a456-426655440000" -expose_data_server = true -data_server_url = "http://localhost:17001/data" -enable_home_creation = true - -[grpc.services.storageprovider.drivers.localhome] -user_layout = "{{.Username}}" - -[grpc.services.authprovider] -auth_manager = "json" - -[grpc.services.authprovider.auth_managers.json] -users = "users.demo.json" - -[grpc.services.userprovider] -driver = "json" - -[grpc.services.userprovider.drivers.json] -users = "users.demo.json" - -[http] -address = "0.0.0.0:17001" - -[http.services.appprovider] - -[http.services.dataprovider] -driver = "localhome" - -[http.services.prometheus] -[http.services.sysinfo] - -[http.services.dataprovider.drivers.localhome] -user_layout = "{{.Username}}" - -[http.services.ocmd] - -[http.services.ocmprovider] -provider = "reva@cesnet" -endpoint = "http://localhost:17001" -enable_webapp = true -enable_datatx = true - -[http.middlewares.providerauthorizer] -driver = "json" - -[http.middlewares.providerauthorizer.drivers.json] -providers = "providers.demo.json" - -[http.services.ocs] -prefix = "ocs" - -[http.services.ocdav] - -[http.services.sciencemesh] -mesh_directory_url = 'https://sciencemesh.cesnet.cz/iop/meshdir' -provider_domain = 'your-domain.org' -ocm_mount_point = '/sciencemesh' - -[http.middlewares.cors] - -[http.middlewares.auth] -credential_chain = ["publicshares", "ocmshares", "basic", "bearer"] - From 2d11c997247e80624279e0d767ef1af9e0463eaa Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Mon, 3 Jul 2023 10:58:57 +0200 Subject: [PATCH 29/48] mention folder needs to contain a file (#4027) --- docs/content/en/docs/tutorials/datatx-tutorial.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/en/docs/tutorials/datatx-tutorial.md b/docs/content/en/docs/tutorials/datatx-tutorial.md index 4e24cf3a0d..577f7bdece 100644 --- a/docs/content/en/docs/tutorials/datatx-tutorial.md +++ b/docs/content/en/docs/tutorials/datatx-tutorial.md @@ -36,7 +36,7 @@ At this point you should have a two Reva daemon setup between which we will esta Creating a transfer is similar to creating a regular OCM share through the `ocm-share-create` command with the addition of the `-datatx` flag. The `-datatx` flag signifies that this is a data transfer.
The `ocm-share-create` command makes (see example below), via an OCM share, the contents of folder `/home/my-data` available for transferring to the grantee. -
*Note that only a folder can be transferred! +
*Note that only a folder can be transferred, and do make sure that this folder contains at least one file (subfolder is not enough)! ``` >> ocm-share-create -grantee f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c -idp cesnet.cz -transfer /home/my-data +--------------------------------------+-----------------+--------------------------------------+--------------------------------------------------------------------------------------------+-------------------+-------------+--------------------------------------+--------------------------------+--------------------------------+ @@ -168,4 +168,4 @@ remove_transfer_job_on_cancel = true ``` Currently this setting is recommended. -*Note that with these settings `transfer-cancel` will remove transfers & jobs even when a transfer cannot actually be cancelled because it was already in an end-state, eg. `finished` or `failed`. So `transfer-cancel` will act like a 'delete' function. \ No newline at end of file +*Note that with these settings `transfer-cancel` will remove transfers & jobs even when a transfer cannot actually be cancelled because it was already in an end-state, eg. `finished` or `failed`. So `transfer-cancel` will act like a 'delete' function. From b2c70ef9979e36779f0dad1520e878dc846cca0d Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Mon, 3 Jul 2023 11:00:11 +0200 Subject: [PATCH 30/48] Hint at rclone port 5572 (#4025) If you're running the rclone server with `--rc-addr=0.0.0.0:5572` as per https://reva.link/docs/tutorials/datatx-tutorial/#1-rclone-setup then you should also use that port number in your reva configs! --- examples/datatx/datatx.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/datatx/datatx.toml b/examples/datatx/datatx.toml index b5296f0cb1..8337a6144b 100644 --- a/examples/datatx/datatx.toml +++ b/examples/datatx/datatx.toml @@ -18,7 +18,7 @@ remove_transfer_on_cancel = true # rclone driver [grpc.services.datatx.txdrivers.rclone] # rclone endpoint -endpoint = "http://..." +endpoint = "http://your.rclone.server:5572" # Basic auth is used for authenticating with rclone auth_user = "{rclone user}" auth_pass = "{rclone user secret}" From 04ec5f3b1f142f0e22d37e53e3885f56aa61116f Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Mon, 3 Jul 2023 11:02:14 +0200 Subject: [PATCH 31/48] warning about port number in datatx tutorial (#4023) --- docs/content/en/docs/tutorials/datatx-tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/en/docs/tutorials/datatx-tutorial.md b/docs/content/en/docs/tutorials/datatx-tutorial.md index 577f7bdece..f874a80267 100644 --- a/docs/content/en/docs/tutorials/datatx-tutorial.md +++ b/docs/content/en/docs/tutorials/datatx-tutorial.md @@ -26,7 +26,7 @@ TPC allows for direct (ie. efficient) Reva to Reva transfers as opposed to strea ## 2. Reva daemons setup Follow the setup ([prerequisites](https://reva.link/docs/tutorials/share-tutorial/#prerequisites), [building](https://reva.link/docs/tutorials/share-tutorial/#2-build-reva), [running](https://reva.link/docs/tutorials/share-tutorial/#3-run-reva)) of the OCM share [tutorial](https://reva.link/docs/tutorials/share-tutorial/). -Use the [data transfer example config](https://github.com/cs3org/reva/blob/master/examples/datatx/datatx.toml) for the relevant settings to enable rclone driven data transfer. +Use the [data transfer example config](https://github.com/cs3org/reva/blob/master/examples/datatx/datatx.toml) for the relevant settings to enable rclone driven data transfer. Be careful when applying this to the ocmd tutorial configs, that in server-2 of that tutorial the port number of the grpc gateway is 17000 and not 19000. At this point you should have a two Reva daemon setup between which we will establish a data transfer driven by rclone. From e57f98ef9770d2fc2c440b4962acd76d8ac3592b Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Mon, 3 Jul 2023 11:02:59 +0200 Subject: [PATCH 32/48] Correct flag in datatx tutorial (#4022) The text says `-datatx` but here it (still?) says `-transfer`. --- docs/content/en/docs/tutorials/datatx-tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/en/docs/tutorials/datatx-tutorial.md b/docs/content/en/docs/tutorials/datatx-tutorial.md index f874a80267..f9e18f2fd4 100644 --- a/docs/content/en/docs/tutorials/datatx-tutorial.md +++ b/docs/content/en/docs/tutorials/datatx-tutorial.md @@ -38,7 +38,7 @@ Creating a transfer is similar to creating a regular OCM share through the `ocm-
The `ocm-share-create` command makes (see example below), via an OCM share, the contents of folder `/home/my-data` available for transferring to the grantee.
*Note that only a folder can be transferred, and do make sure that this folder contains at least one file (subfolder is not enough)! ``` ->> ocm-share-create -grantee f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c -idp cesnet.cz -transfer /home/my-data +>> ocm-share-create -grantee f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c -idp cesnet.cz -datatx /home/my-data +--------------------------------------+-----------------+--------------------------------------+--------------------------------------------------------------------------------------------+-------------------+-------------+--------------------------------------+--------------------------------+--------------------------------+ | # | OWNER.IDP | OWNER.OPAQUEID | RESOURCEID | TYPE | GRANTEE.IDP | GRANTEE.OPAQUEID | CREATED | UPDATED | +--------------------------------------+-----------------+--------------------------------------+--------------------------------------------------------------------------------------------+-------------------+-------------+--------------------------------------+--------------------------------+--------------------------------+ From cf2dcfac32a79c274a1c6a1996f245602c0e47ba Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Mon, 3 Jul 2023 17:15:25 +0200 Subject: [PATCH 33/48] Support multiple token strategies in auth middleware (#4030) * support multiple token strategies in auth middleware * add changelog --- .../unreleased/multiple-token-strategies.md | 11 ++++ internal/http/interceptors/auth/auth.go | 52 ++++++++++--------- 2 files changed, 39 insertions(+), 24 deletions(-) create mode 100644 changelog/unreleased/multiple-token-strategies.md diff --git a/changelog/unreleased/multiple-token-strategies.md b/changelog/unreleased/multiple-token-strategies.md new file mode 100644 index 0000000000..499445df05 --- /dev/null +++ b/changelog/unreleased/multiple-token-strategies.md @@ -0,0 +1,11 @@ +Enhancement: Support multiple token strategies in auth middleware + +Different HTTP services can in general support different +token strategies for validating the reva token. +In this context, without updating every single client +a mono process deployment will never work. +Now the HTTP auth middleware accepts in its configuration a +token strategy chain, allowing to provide the reva +token in multiple places (bearer auth, header). + +https://github.com/cs3org/reva/pull/4030 diff --git a/internal/http/interceptors/auth/auth.go b/internal/http/interceptors/auth/auth.go index da878bd0a4..c664c232f4 100644 --- a/internal/http/interceptors/auth/auth.go +++ b/internal/http/interceptors/auth/auth.go @@ -60,7 +60,7 @@ type config struct { CredentialsByUserAgent map[string]string `mapstructure:"credentials_by_user_agent"` CredentialChain []string `mapstructure:"credential_chain"` CredentialStrategies map[string]map[string]interface{} `mapstructure:"credential_strategies"` - TokenStrategy string `mapstructure:"token_strategy"` + TokenStrategyChain []string `mapstructure:"token_strategy_chain"` TokenStrategies map[string]map[string]interface{} `mapstructure:"token_strategies"` TokenManager string `mapstructure:"token_manager"` TokenManagers map[string]map[string]interface{} `mapstructure:"token_managers"` @@ -87,8 +87,8 @@ func New(m map[string]interface{}, unprotected []string) (global.Middleware, err conf.GatewaySvc = sharedconf.GetGatewaySVC(conf.GatewaySvc) // set defaults - if conf.TokenStrategy == "" { - conf.TokenStrategy = "header" + if len(conf.TokenStrategyChain) == 0 { + conf.TokenStrategyChain = []string{"header"} } if conf.TokenWriter == "" { @@ -123,19 +123,22 @@ func New(m map[string]interface{}, unprotected []string) (global.Middleware, err credChain[key] = credStrategy } - g, ok := tokenregistry.NewTokenFuncs[conf.TokenStrategy] - if !ok { - return nil, fmt.Errorf("token strategy not found: %s", conf.TokenStrategy) - } - - tokenStrategy, err := g(conf.TokenStrategies[conf.TokenStrategy]) - if err != nil { - return nil, err + tokenStrategyChain := make([]auth.TokenStrategy, 0, len(conf.TokenStrategyChain)) + for _, strategy := range conf.TokenStrategyChain { + g, ok := tokenregistry.NewTokenFuncs[strategy] + if !ok { + return nil, fmt.Errorf("token strategy not found: %s", strategy) + } + tokenStrategy, err := g(conf.TokenStrategies[strategy]) + if err != nil { + return nil, err + } + tokenStrategyChain = append(tokenStrategyChain, tokenStrategy) } h, ok := tokenmgr.NewFuncs[conf.TokenManager] if !ok { - return nil, fmt.Errorf("token manager not found: %s", conf.TokenStrategy) + return nil, fmt.Errorf("token manager not found: %s", conf.TokenManager) } tokenManager, err := h(conf.TokenManagers[conf.TokenManager]) @@ -173,7 +176,7 @@ func New(m map[string]interface{}, unprotected []string) (global.Middleware, err isUnprotectedEndpoint = true } - ctx, err := authenticateUser(w, r, conf, tokenStrategy, tokenManager, tokenWriter, credChain, isUnprotectedEndpoint) + ctx, err := authenticateUser(w, r, conf, tokenStrategyChain, tokenManager, tokenWriter, credChain, isUnprotectedEndpoint) if err != nil { if !isUnprotectedEndpoint { return @@ -187,7 +190,7 @@ func New(m map[string]interface{}, unprotected []string) (global.Middleware, err return chain, nil } -func authenticateUser(w http.ResponseWriter, r *http.Request, conf *config, tokenStrategy auth.TokenStrategy, tokenManager token.Manager, tokenWriter auth.TokenWriter, credChain map[string]auth.CredentialStrategy, isUnprotectedEndpoint bool) (context.Context, error) { +func authenticateUser(w http.ResponseWriter, r *http.Request, conf *config, tokenStrategies []auth.TokenStrategy, tokenManager token.Manager, tokenWriter auth.TokenWriter, credChain map[string]auth.CredentialStrategy, isUnprotectedEndpoint bool) (context.Context, error) { ctx := r.Context() log := appctx.GetLogger(ctx) @@ -203,19 +206,20 @@ func authenticateUser(w http.ResponseWriter, r *http.Request, conf *config, toke // reva token or auth token can be passed using the same tecnique (for example bearer) // before validating it against an auth provider, we can check directly if it's a reva // token and if not try to use it for authenticating the user. - - token := tokenStrategy.GetToken(r) - if token != "" { - if user, ok := isTokenValid(r, tokenManager, token); ok { - if err := insertGroupsInUser(ctx, userGroupsCache, client, user); err != nil { - logError(isUnprotectedEndpoint, log, err, "got an error retrieving groups for user "+user.Username, http.StatusInternalServerError, w) - return nil, err + for _, tokenStrategy := range tokenStrategies { + token := tokenStrategy.GetToken(r) + if token != "" { + if user, ok := isTokenValid(r, tokenManager, token); ok { + if err := insertGroupsInUser(ctx, userGroupsCache, client, user); err != nil { + logError(isUnprotectedEndpoint, log, err, "got an error retrieving groups for user "+user.Username, http.StatusInternalServerError, w) + return nil, err + } + return ctxWithUserInfo(ctx, r, user, token), nil } - return ctxWithUserInfo(ctx, r, user, token), nil } } - log.Warn().Msg("core access token not set") + log.Warn().Msg("core access token not set or invalid") userAgentCredKeys := getCredsForUserAgent(r.UserAgent(), conf.CredentialsByUserAgent, conf.CredentialChain) @@ -271,7 +275,7 @@ func authenticateUser(w http.ResponseWriter, r *http.Request, conf *config, toke log.Info().Msg("core access token generated") // write token to response - token = res.Token + token := res.Token tokenWriter.WriteToken(token, w) // validate token From ccac7adeabe9652247fbc8b8595c65918f53c3f9 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Tue, 4 Jul 2023 13:40:17 +0200 Subject: [PATCH 34/48] Pass init ctx to all reva services (#3407) --- changelog/unreleased/init-log.md | 3 + cmd/revad/runtime/option.go | 11 ++ cmd/revad/runtime/runtime.go | 26 ++-- .../applicationauth/applicationauth.go | 8 +- .../grpc/services/appprovider/appprovider.go | 17 ++- .../grpc/services/appregistry/appregistry.go | 8 +- .../services/appregistry/appregistry_test.go | 6 +- .../services/authprovider/authprovider.go | 8 +- .../services/authregistry/authregistry.go | 8 +- internal/grpc/services/datatx/datatx.go | 14 +-- internal/grpc/services/gateway/gateway.go | 3 +- .../services/groupprovider/groupprovider.go | 8 +- .../grpc/services/helloworld/helloworld.go | 2 +- internal/grpc/services/ocmcore/ocmcore.go | 8 +- .../ocminvitemanager/ocminvitemanager.go | 8 +- .../ocmproviderauthorizer.go | 8 +- .../ocmshareprovider/ocmshareprovider.go | 8 +- .../grpc/services/permissions/permissions.go | 4 +- .../grpc/services/preferences/preferences.go | 8 +- .../publicshareprovider.go | 8 +- .../publicstorageprovider.go | 2 +- .../storageprovider/storageprovider.go | 8 +- .../storageregistry/storageregistry.go | 8 +- .../services/userprovider/userprovider.go | 8 +- .../usershareprovider/usershareprovider.go | 8 +- internal/http/interceptors/loader/loader.go | 1 - .../providerauthorizer/providerauthorizer.go | 111 ------------------ .../http/services/appprovider/appprovider.go | 4 +- internal/http/services/archiver/handler.go | 26 ++-- .../http/services/datagateway/datagateway.go | 3 +- .../services/dataprovider/dataprovider.go | 16 +-- .../http/services/helloworld/helloworld.go | 4 +- internal/http/services/mentix/mentix.go | 5 +- internal/http/services/metrics/metrics.go | 4 +- internal/http/services/ocmd/ocm.go | 4 +- .../http/services/ocmprovider/ocmprovider.go | 4 +- .../http/services/owncloud/ocdav/ocdav.go | 4 +- .../services/owncloud/ocs/conversions/main.go | 8 +- internal/http/services/owncloud/ocs/ocs.go | 4 +- .../http/services/preferences/preferences.go | 4 +- .../http/services/prometheus/prometheus.go | 4 +- .../services/reverseproxy/reverseproxy.go | 4 +- .../http/services/sciencemesh/sciencemesh.go | 4 +- internal/http/services/siteacc/siteacc.go | 5 +- internal/http/services/sysinfo/sysinfo.go | 4 +- internal/http/services/wellknown/wellknown.go | 4 +- pkg/app/provider/demo/demo.go | 2 +- pkg/app/provider/registry/registry.go | 8 +- pkg/app/provider/wopi/wopi.go | 2 +- pkg/app/registry/registry/registry.go | 8 +- pkg/app/registry/static/static.go | 2 +- pkg/app/registry/static/static_test.go | 10 +- pkg/appauth/manager/json/json.go | 2 +- pkg/appauth/manager/json/json_test.go | 10 +- pkg/appauth/manager/registry/registry.go | 8 +- pkg/auth/manager/appauth/appauth.go | 2 +- pkg/auth/manager/demo/demo.go | 2 +- pkg/auth/manager/demo/demo_test.go | 2 +- pkg/auth/manager/impersonator/impersonator.go | 2 +- .../manager/impersonator/impersonator_test.go | 2 +- pkg/auth/manager/json/json.go | 2 +- pkg/auth/manager/json/json_test.go | 6 +- pkg/auth/manager/ldap/ldap.go | 2 +- pkg/auth/manager/machine/machine.go | 2 +- pkg/auth/manager/nextcloud/nextcloud.go | 2 +- pkg/auth/manager/nextcloud/nextcloud_test.go | 2 +- pkg/auth/manager/ocmshares/ocmshares.go | 2 +- pkg/auth/manager/oidc/oidc.go | 2 +- pkg/auth/manager/owncloudsql/owncloudsql.go | 2 +- pkg/auth/manager/publicshares/publicshares.go | 2 +- pkg/auth/manager/registry/registry.go | 8 +- pkg/auth/registry/registry/registry.go | 8 +- pkg/auth/registry/static/static.go | 2 +- pkg/cbox/group/rest/rest.go | 2 +- pkg/cbox/preferences/sql/sql.go | 2 +- pkg/cbox/publicshare/sql/sql.go | 2 +- pkg/cbox/share/sql/sql.go | 2 +- .../storage/eoshomewrapper/eoshomewrapper.go | 4 +- pkg/cbox/storage/eoswrapper/eoswrapper.go | 4 +- pkg/cbox/user/rest/rest.go | 2 +- pkg/datatx/manager/rclone/rclone.go | 2 +- pkg/datatx/manager/registry/registry.go | 4 +- pkg/datatx/repository/json/json.go | 3 +- pkg/datatx/repository/registry/registry.go | 4 +- pkg/group/manager/json/json.go | 2 +- pkg/group/manager/json/json_test.go | 4 +- pkg/group/manager/ldap/ldap.go | 2 +- pkg/group/manager/registry/registry.go | 8 +- pkg/ocm/invite/repository/json/json.go | 2 +- pkg/ocm/invite/repository/memory/memory.go | 2 +- .../invite/repository/registry/registry.go | 8 +- pkg/ocm/invite/repository/sql/sql.go | 2 +- pkg/ocm/provider/authorizer/json/json.go | 2 +- pkg/ocm/provider/authorizer/mentix/mentix.go | 2 +- pkg/ocm/provider/authorizer/open/open.go | 2 +- .../provider/authorizer/registry/registry.go | 8 +- pkg/ocm/share/repository/json/json.go | 2 +- .../share/repository/nextcloud/nextcloud.go | 2 +- .../repository/nextcloud/nextcloud_test.go | 2 +- pkg/ocm/share/repository/registry/registry.go | 8 +- pkg/ocm/share/repository/sql/sql.go | 6 +- pkg/ocm/share/repository/sql/sql_test.go | 16 +-- pkg/ocm/storage/outcoming/ocm.go | 2 +- pkg/ocm/storage/received/ocm.go | 2 +- pkg/permission/manager/demo/demo.go | 4 +- pkg/permission/manager/registry/registry.go | 8 +- pkg/preferences/memory/memory.go | 2 +- pkg/preferences/registry/registry.go | 8 +- pkg/publicshare/manager/json/json.go | 2 +- pkg/publicshare/manager/memory/memory.go | 2 +- pkg/publicshare/manager/registry/registry.go | 8 +- pkg/rgrpc/rgrpc.go | 10 +- pkg/rhttp/datatx/manager/registry/registry.go | 8 +- pkg/rhttp/datatx/manager/simple/simple.go | 3 +- pkg/rhttp/datatx/manager/spaces/spaces.go | 3 +- pkg/rhttp/datatx/manager/tus/tus.go | 3 +- pkg/rhttp/global/global.go | 5 +- pkg/rhttp/rhttp.go | 8 +- pkg/share/manager/json/json.go | 2 +- pkg/share/manager/memory/memory.go | 2 +- pkg/share/manager/registry/registry.go | 8 +- pkg/share/manager/sql/sql.go | 2 +- pkg/storage/fs/cback/cback.go | 2 +- pkg/storage/fs/cephfs/cephfs.go | 2 +- pkg/storage/fs/cephfs/unsupported.go | 4 +- pkg/storage/fs/eos/eos.go | 6 +- pkg/storage/fs/eosgrpc/eosgrpc.go | 6 +- pkg/storage/fs/eosgrpchome/eosgrpchome.go | 6 +- pkg/storage/fs/eoshome/eoshome.go | 6 +- pkg/storage/fs/local/local.go | 4 +- pkg/storage/fs/localhome/localhome.go | 4 +- pkg/storage/fs/nextcloud/nextcloud.go | 2 +- pkg/storage/fs/nextcloud/nextcloud_test.go | 2 +- pkg/storage/fs/ocis/ocis.go | 3 +- pkg/storage/fs/ocis/ocis_test.go | 3 +- pkg/storage/fs/owncloud/owncloud.go | 2 +- pkg/storage/fs/owncloudsql/owncloudsql.go | 2 +- pkg/storage/fs/registry/registry.go | 8 +- pkg/storage/fs/s3/s3.go | 2 +- pkg/storage/fs/s3ng/s3ng.go | 3 +- pkg/storage/fs/s3ng/s3ng_test.go | 5 +- pkg/storage/registry/registry/registry.go | 8 +- pkg/storage/registry/static/static.go | 2 +- pkg/storage/registry/static/static_test.go | 2 +- pkg/storage/utils/eosfs/eosfs.go | 2 +- pkg/user/manager/demo/demo.go | 2 +- pkg/user/manager/demo/demo_test.go | 2 +- pkg/user/manager/json/json.go | 2 +- pkg/user/manager/json/json_test.go | 4 +- pkg/user/manager/ldap/ldap.go | 2 +- pkg/user/manager/ldap/ldap_test.go | 5 +- pkg/user/manager/nextcloud/nextcloud.go | 2 +- pkg/user/manager/nextcloud/nextcloud_test.go | 2 +- pkg/user/manager/owncloudsql/owncloudsql.go | 2 +- pkg/user/manager/registry/registry.go | 4 +- .../integration/grpc/storageprovider_test.go | 6 +- 156 files changed, 439 insertions(+), 422 deletions(-) create mode 100644 changelog/unreleased/init-log.md delete mode 100644 internal/http/interceptors/providerauthorizer/providerauthorizer.go diff --git a/changelog/unreleased/init-log.md b/changelog/unreleased/init-log.md new file mode 100644 index 0000000000..507d24ccec --- /dev/null +++ b/changelog/unreleased/init-log.md @@ -0,0 +1,3 @@ +Enhancement: Add init time logging to all services + +https://github.com/cs3org/reva/pull/3407 diff --git a/cmd/revad/runtime/option.go b/cmd/revad/runtime/option.go index dd49966424..758cb5fd78 100644 --- a/cmd/revad/runtime/option.go +++ b/cmd/revad/runtime/option.go @@ -19,6 +19,8 @@ package runtime import ( + "context" + "github.com/cs3org/reva/pkg/registry" "github.com/rs/zerolog" ) @@ -31,6 +33,7 @@ type Options struct { Logger *zerolog.Logger Registry registry.Registry PidFile string + Ctx context.Context } // newOptions initializes the available default options. @@ -38,6 +41,7 @@ func newOptions(opts ...Option) Options { l := zerolog.Nop() opt := Options{ Logger: &l, + Ctx: context.TODO(), } for _, o := range opts { @@ -67,3 +71,10 @@ func WithRegistry(r registry.Registry) Option { o.Registry = r } } + +// WithContext sets the context to use. +func WithContext(ctx context.Context) Option { + return func(o *Options) { + o.Ctx = ctx + } +} diff --git a/cmd/revad/runtime/runtime.go b/cmd/revad/runtime/runtime.go index 3a4b601109..d895b122f1 100644 --- a/cmd/revad/runtime/runtime.go +++ b/cmd/revad/runtime/runtime.go @@ -19,6 +19,7 @@ package runtime import ( + "context" "fmt" "net" "runtime" @@ -29,8 +30,10 @@ import ( "github.com/cs3org/reva/cmd/revad/pkg/config" "github.com/cs3org/reva/cmd/revad/pkg/grace" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rhttp" + "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/rserverless" "github.com/cs3org/reva/pkg/sharedconf" @@ -44,6 +47,7 @@ import ( // Reva represents a full instance of reva. type Reva struct { + ctx context.Context config *config.Config servers []*Server @@ -73,6 +77,8 @@ func New(config *config.Config, opt ...Option) (*Reva, error) { opts := newOptions(opt...) log := opts.Logger + ctx := appctx.WithLogger(opts.Ctx, log) + if err := initCPUCount(config.Core, log); err != nil { return nil, err } @@ -103,7 +109,7 @@ func New(config *config.Config, opt ...Option) (*Reva, error) { grpc := groupGRPCByAddress(config) http := groupHTTPByAddress(config) - servers, err := newServers(grpc, http, listeners, log) + servers, err := newServers(ctx, grpc, http, listeners, log) if err != nil { watcher.Clean() return nil, err @@ -116,6 +122,7 @@ func New(config *config.Config, opt ...Option) (*Reva, error) { } return &Reva{ + ctx: ctx, config: config, servers: servers, serverless: serverless, @@ -357,10 +364,12 @@ func listenerFromAddress(lns map[string]net.Listener, network string, address co panic(fmt.Sprintf("listener not found for address %s:%s", network, address)) } -func newServers(grpc []*config.GRPC, http []*config.HTTP, lns map[string]net.Listener, log *zerolog.Logger) ([]*Server, error) { +func newServers(ctx context.Context, grpc []*config.GRPC, http []*config.HTTP, lns map[string]net.Listener, log *zerolog.Logger) ([]*Server, error) { servers := make([]*Server, 0, len(grpc)+len(http)) for _, cfg := range grpc { - services, err := rgrpc.InitServices(cfg.Services) + logger := log.With().Str("pkg", "grpc").Logger() + ctx := appctx.WithLogger(ctx, &logger) + services, err := rgrpc.InitServices(ctx, cfg.Services) if err != nil { return nil, err } @@ -371,7 +380,7 @@ func newServers(grpc []*config.GRPC, http []*config.HTTP, lns map[string]net.Lis s, err := rgrpc.NewServer( rgrpc.EnableReflection(cfg.EnableReflection), rgrpc.WithShutdownDeadline(cfg.ShutdownDeadline), - rgrpc.WithLogger(log.With().Str("pkg", "grpc").Logger()), + rgrpc.WithLogger(logger), rgrpc.WithServices(services), rgrpc.WithUnaryServerInterceptors(unaryChain), rgrpc.WithStreamServerInterceptors(streamChain), @@ -391,18 +400,19 @@ func newServers(grpc []*config.GRPC, http []*config.HTTP, lns map[string]net.Lis servers = append(servers, server) } for _, cfg := range http { - log := log.With().Str("pkg", "http").Logger() - services, err := rhttp.InitServices(cfg.Services, &log) + logger := log.With().Str("pkg", "http").Logger() + ctx := appctx.WithLogger(ctx, &logger) + services, err := rhttp.InitServices(ctx, cfg.Services) if err != nil { return nil, err } - middlewares, err := initHTTPMiddlewares(cfg.Middlewares, httpUnprotected(services), &log) + middlewares, err := initHTTPMiddlewares(cfg.Middlewares, httpUnprotected(services), &logger) if err != nil { return nil, err } s, err := rhttp.New( rhttp.WithServices(services), - rhttp.WithLogger(log), + rhttp.WithLogger(logger), rhttp.WithCertAndKeyFiles(cfg.CertFile, cfg.KeyFile), rhttp.WithMiddlewares(middlewares), ) diff --git a/internal/grpc/services/applicationauth/applicationauth.go b/internal/grpc/services/applicationauth/applicationauth.go index 47f4c24680..e05bd38a70 100644 --- a/internal/grpc/services/applicationauth/applicationauth.go +++ b/internal/grpc/services/applicationauth/applicationauth.go @@ -56,9 +56,9 @@ func (s *service) Register(ss *grpc.Server) { appauthpb.RegisterApplicationsAPIServer(ss, s) } -func getAppAuthManager(c *config) (appauth.Manager, error) { +func getAppAuthManager(ctx context.Context, c *config) (appauth.Manager, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) + return f(ctx, c.Drivers[c.Driver]) } return nil, errtypes.NotFound("driver not found: " + c.Driver) } @@ -73,14 +73,14 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a app auth provider svc. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err } c.init() - am, err := getAppAuthManager(c) + am, err := getAppAuthManager(ctx, c) if err != nil { return nil, err } diff --git a/internal/grpc/services/appprovider/appprovider.go b/internal/grpc/services/appprovider/appprovider.go index 533afe4006..5b44681cad 100644 --- a/internal/grpc/services/appprovider/appprovider.go +++ b/internal/grpc/services/appprovider/appprovider.go @@ -33,8 +33,8 @@ import ( types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/app" "github.com/cs3org/reva/pkg/app/provider/registry" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/logger" "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" @@ -83,7 +83,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new AppProviderService. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err @@ -95,7 +95,7 @@ func New(m map[string]interface{}) (rgrpc.Service, error) { return nil, err } - provider, err := getProvider(c) + provider, err := getProvider(ctx, c) if err != nil { return nil, err } @@ -105,7 +105,7 @@ func New(m map[string]interface{}) (rgrpc.Service, error) { provider: provider, } - go service.registerProvider() + go service.registerProvider(ctx) return service, nil } @@ -130,13 +130,12 @@ func registerMimeTypes(mappingFile string) error { return nil } -func (s *service) registerProvider() { +func (s *service) registerProvider(ctx context.Context) { // Give the appregistry service time to come up // TODO(lopresti) we should register the appproviders after all other microservices time.Sleep(3 * time.Second) - ctx := context.Background() - log := logger.New().With().Int("pid", os.Getpid()).Logger() + log := appctx.GetLogger(ctx) pInfo, err := s.provider.GetAppProviderInfo(ctx) if err != nil { log.Error().Err(err).Msgf("error registering app provider: could not get provider info") @@ -196,7 +195,7 @@ func (s *service) Register(ss *grpc.Server) { providerpb.RegisterProviderAPIServer(ss, s) } -func getProvider(c *config) (app.Provider, error) { +func getProvider(ctx context.Context, c *config) (app.Provider, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { driverConf := c.Drivers[c.Driver] if c.MimeTypes != nil { @@ -206,7 +205,7 @@ func getProvider(c *config) (app.Provider, error) { } driverConf["mime_types"] = c.MimeTypes } - return f(driverConf) + return f(ctx, driverConf) } return nil, errtypes.NotFound("driver not found: " + c.Driver) } diff --git a/internal/grpc/services/appregistry/appregistry.go b/internal/grpc/services/appregistry/appregistry.go index d70b9e7746..0ba769ff94 100644 --- a/internal/grpc/services/appregistry/appregistry.go +++ b/internal/grpc/services/appregistry/appregistry.go @@ -63,13 +63,13 @@ func (c *config) init() { } // New creates a new StorageRegistryService. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err } - reg, err := getRegistry(c) + reg, err := getRegistry(ctx, c) if err != nil { return nil, err } @@ -90,9 +90,9 @@ func parseConfig(m map[string]interface{}) (*config, error) { return c, nil } -func getRegistry(c *config) (app.Registry, error) { +func getRegistry(ctx context.Context, c *config) (app.Registry, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) + return f(ctx, c.Drivers[c.Driver]) } return nil, errtypes.NotFound("appregistrysvc: driver not found: " + c.Driver) } diff --git a/internal/grpc/services/appregistry/appregistry_test.go b/internal/grpc/services/appregistry/appregistry_test.go index 34b8c5d3d1..f56dad5997 100644 --- a/internal/grpc/services/appregistry/appregistry_test.go +++ b/internal/grpc/services/appregistry/appregistry_test.go @@ -122,7 +122,7 @@ func Test_ListAppProviders(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - rr, err := static.New(map[string]interface{}{"providers": tt.providers, "mime_types": tt.mimeTypes}) + rr, err := static.New(context.Background(), map[string]interface{}{"providers": tt.providers, "mime_types": tt.mimeTypes}) if err != nil { t.Errorf("could not create registry error = %v", err) return @@ -296,7 +296,7 @@ func Test_GetAppProviders(t *testing.T) { }, } - rr, err := static.New(map[string]interface{}{"providers": providers, "mime_types": mimeTypes}) + rr, err := static.New(context.Background(), map[string]interface{}{"providers": providers, "mime_types": mimeTypes}) if err != nil { t.Errorf("could not create registry error = %v", err) return @@ -355,7 +355,7 @@ func TestNew(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := New(tt.m) + got, err := New(context.Background(), tt.m) if err != nil { assert.Equal(t, tt.wantErr, err.Error()) assert.Nil(t, got) diff --git a/internal/grpc/services/authprovider/authprovider.go b/internal/grpc/services/authprovider/authprovider.go index 135af03745..58765c4dad 100644 --- a/internal/grpc/services/authprovider/authprovider.go +++ b/internal/grpc/services/authprovider/authprovider.go @@ -72,7 +72,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { return c, nil } -func getAuthManager(manager string, m map[string]map[string]interface{}) (auth.Manager, *plugin.RevaPlugin, error) { +func getAuthManager(ctx context.Context, manager string, m map[string]map[string]interface{}) (auth.Manager, *plugin.RevaPlugin, error) { if manager == "" { return nil, nil, errtypes.InternalError("authsvc: driver not configured for auth manager") } @@ -90,7 +90,7 @@ func getAuthManager(manager string, m map[string]map[string]interface{}) (auth.M return authManager, p, nil } else if _, ok := err.(errtypes.NotFound); ok { if f, ok := registry.NewFuncs[manager]; ok { - authmgr, err := f(m[manager]) + authmgr, err := f(ctx, m[manager]) return authmgr, nil, err } } else { @@ -100,13 +100,13 @@ func getAuthManager(manager string, m map[string]map[string]interface{}) (auth.M } // New returns a new AuthProviderServiceServer. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err } - authManager, plug, err := getAuthManager(c.AuthManager, c.AuthManagers) + authManager, plug, err := getAuthManager(ctx, c.AuthManager, c.AuthManagers) if err != nil { return nil, err } diff --git a/internal/grpc/services/authregistry/authregistry.go b/internal/grpc/services/authregistry/authregistry.go index dcbbd1b471..267ea3b6b0 100644 --- a/internal/grpc/services/authregistry/authregistry.go +++ b/internal/grpc/services/authregistry/authregistry.go @@ -66,7 +66,7 @@ func (c *config) init() { } // New creates a new AuthRegistry. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err @@ -74,7 +74,7 @@ func New(m map[string]interface{}) (rgrpc.Service, error) { c.init() - reg, err := getRegistry(c) + reg, err := getRegistry(ctx, c) if err != nil { return nil, err } @@ -94,9 +94,9 @@ func parseConfig(m map[string]interface{}) (*config, error) { return c, nil } -func getRegistry(c *config) (auth.Registry, error) { +func getRegistry(ctx context.Context, c *config) (auth.Registry, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) + return f(ctx, c.Drivers[c.Driver]) } return nil, errtypes.NotFound("authregistrysvc: driver not found: " + c.Driver) } diff --git a/internal/grpc/services/datatx/datatx.go b/internal/grpc/services/datatx/datatx.go index ce82fa5c36..98bf545b65 100644 --- a/internal/grpc/services/datatx/datatx.go +++ b/internal/grpc/services/datatx/datatx.go @@ -64,16 +64,16 @@ func (s *service) Register(ss *grpc.Server) { datatx.RegisterTxAPIServer(ss, s) } -func getDatatxManager(c *config) (txdriver.Manager, error) { +func getDatatxManager(ctx context.Context, c *config) (txdriver.Manager, error) { if f, ok := txregistry.NewFuncs[c.TxDriver]; ok { - return f(c.TxDrivers[c.TxDriver]) + return f(ctx, c.TxDrivers[c.TxDriver]) } return nil, errtypes.NotFound("datatx service: driver not found: " + c.TxDriver) } -func getStorageManager(c *config) (txdriver.Repository, error) { +func getStorageManager(ctx context.Context, c *config) (txdriver.Repository, error) { if f, ok := repoRegistry.NewFuncs[c.StorageDriver]; ok { - return f(c.StorageDrivers[c.StorageDriver]) + return f(ctx, c.StorageDrivers[c.StorageDriver]) } return nil, errtypes.NotFound("datatx service: driver not found: " + c.StorageDriver) } @@ -88,19 +88,19 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new datatx svc. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err } c.init() - txManager, err := getDatatxManager(c) + txManager, err := getDatatxManager(ctx, c) if err != nil { return nil, err } - storageDriver, err := getStorageManager(c) + storageDriver, err := getStorageManager(ctx, c) if err != nil { return nil, err } diff --git a/internal/grpc/services/gateway/gateway.go b/internal/grpc/services/gateway/gateway.go index 445ae7b13a..d766b00c5e 100644 --- a/internal/grpc/services/gateway/gateway.go +++ b/internal/grpc/services/gateway/gateway.go @@ -19,6 +19,7 @@ package gateway import ( + "context" "fmt" "net/url" "strings" @@ -124,7 +125,7 @@ type svc struct { // New creates a new gateway svc that acts as a proxy for any grpc operation. // The gateway is responsible for high-level controls: rate-limiting, coordination between svcs // like sharing and storage acls, asynchronous transactions, ... -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/groupprovider/groupprovider.go b/internal/grpc/services/groupprovider/groupprovider.go index 7e23c2f97b..b72704b95c 100644 --- a/internal/grpc/services/groupprovider/groupprovider.go +++ b/internal/grpc/services/groupprovider/groupprovider.go @@ -59,22 +59,22 @@ func parseConfig(m map[string]interface{}) (*config, error) { return c, nil } -func getDriver(c *config) (group.Manager, error) { +func getDriver(ctx context.Context, c *config) (group.Manager, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) + return f(ctx, c.Drivers[c.Driver]) } return nil, errtypes.NotFound(fmt.Sprintf("driver %s not found for group manager", c.Driver)) } // New returns a new GroupProviderServiceServer. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err } - groupManager, err := getDriver(c) + groupManager, err := getDriver(ctx, c) if err != nil { return nil, err } diff --git a/internal/grpc/services/helloworld/helloworld.go b/internal/grpc/services/helloworld/helloworld.go index 40aecaa356..3249457f59 100644 --- a/internal/grpc/services/helloworld/helloworld.go +++ b/internal/grpc/services/helloworld/helloworld.go @@ -43,7 +43,7 @@ type service struct { // New returns a new PreferencesServiceServer // It can be tested like this: // prototool grpc --address 0.0.0.0:9999 --method 'revad.helloworld.HelloWorldService/Hello' --data '{"name": "Alice"}'. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c := &conf{} if err := mapstructure.Decode(m, c); err != nil { err = errors.Wrap(err, "helloworld: error decoding conf") diff --git a/internal/grpc/services/ocmcore/ocmcore.go b/internal/grpc/services/ocmcore/ocmcore.go index 16686bc9fd..c9f8dfe71b 100644 --- a/internal/grpc/services/ocmcore/ocmcore.go +++ b/internal/grpc/services/ocmcore/ocmcore.go @@ -61,9 +61,9 @@ func (s *service) Register(ss *grpc.Server) { ocmcore.RegisterOcmCoreAPIServer(ss, s) } -func getShareRepository(c *config) (share.Repository, error) { +func getShareRepository(ctx context.Context, c *config) (share.Repository, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) + return f(ctx, c.Drivers[c.Driver]) } return nil, errtypes.NotFound(fmt.Sprintf("driver not found: %s", c.Driver)) } @@ -78,14 +78,14 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new ocm core svc. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err } c.init() - repo, err := getShareRepository(c) + repo, err := getShareRepository(ctx, c) if err != nil { return nil, err } diff --git a/internal/grpc/services/ocminvitemanager/ocminvitemanager.go b/internal/grpc/services/ocminvitemanager/ocminvitemanager.go index 38cf922c5a..e4a3989cbe 100644 --- a/internal/grpc/services/ocminvitemanager/ocminvitemanager.go +++ b/internal/grpc/services/ocminvitemanager/ocminvitemanager.go @@ -86,9 +86,9 @@ func (s *service) Register(ss *grpc.Server) { invitepb.RegisterInviteAPIServer(ss, s) } -func getInviteRepository(c *config) (invite.Repository, error) { +func getInviteRepository(ctx context.Context, c *config) (invite.Repository, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) + return f(ctx, c.Drivers[c.Driver]) } return nil, errtypes.NotFound("driver not found: " + c.Driver) } @@ -103,7 +103,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new OCM invite manager svc. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err @@ -112,7 +112,7 @@ func New(m map[string]interface{}) (rgrpc.Service, error) { return nil, err } - repo, err := getInviteRepository(c) + repo, err := getInviteRepository(ctx, c) if err != nil { return nil, err } diff --git a/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go b/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go index e14645fe5a..fe26f8ebe1 100644 --- a/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go +++ b/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go @@ -57,9 +57,9 @@ func (s *service) Register(ss *grpc.Server) { ocmprovider.RegisterProviderAPIServer(ss, s) } -func getProviderAuthorizer(c *config) (provider.Authorizer, error) { +func getProviderAuthorizer(ctx context.Context, c *config) (provider.Authorizer, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) + return f(ctx, c.Drivers[c.Driver]) } return nil, errtypes.NotFound("driver not found: " + c.Driver) } @@ -74,14 +74,14 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new OCM provider authorizer svc. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err } c.init() - pa, err := getProviderAuthorizer(c) + pa, err := getProviderAuthorizer(ctx, c) if err != nil { return nil, err } diff --git a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go index c8f39962f0..6bd9f73011 100644 --- a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go +++ b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go @@ -93,9 +93,9 @@ func (s *service) Register(ss *grpc.Server) { ocm.RegisterOcmAPIServer(ss, s) } -func getShareRepository(c *config) (share.Repository, error) { +func getShareRepository(ctx context.Context, c *config) (share.Repository, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) + return f(ctx, c.Drivers[c.Driver]) } return nil, errtypes.NotFound("driver not found: " + c.Driver) } @@ -110,14 +110,14 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new ocm share provider svc. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err } c.init() - repo, err := getShareRepository(c) + repo, err := getShareRepository(ctx, c) if err != nil { return nil, err } diff --git a/internal/grpc/services/permissions/permissions.go b/internal/grpc/services/permissions/permissions.go index 428e5457f1..cc1b75786b 100644 --- a/internal/grpc/services/permissions/permissions.go +++ b/internal/grpc/services/permissions/permissions.go @@ -55,7 +55,7 @@ type service struct { } // New returns a new PermissionsServiceServer. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err @@ -65,7 +65,7 @@ func New(m map[string]interface{}) (rgrpc.Service, error) { if !ok { return nil, fmt.Errorf("could not get permission manager '%s'", c.Driver) } - manager, err := f(c.Drivers[c.Driver]) + manager, err := f(ctx, c.Drivers[c.Driver]) if err != nil { return nil, err } diff --git a/internal/grpc/services/preferences/preferences.go b/internal/grpc/services/preferences/preferences.go index f2b096cbf7..b5e0003241 100644 --- a/internal/grpc/services/preferences/preferences.go +++ b/internal/grpc/services/preferences/preferences.go @@ -52,9 +52,9 @@ type service struct { pm preferences.Manager } -func getPreferencesManager(c *config) (preferences.Manager, error) { +func getPreferencesManager(ctx context.Context, c *config) (preferences.Manager, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) + return f(ctx, c.Drivers[c.Driver]) } return nil, errtypes.NotFound("driver not found: " + c.Driver) } @@ -69,7 +69,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns a new PreferencesServiceServer. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err @@ -77,7 +77,7 @@ func New(m map[string]interface{}) (rgrpc.Service, error) { c.init() - pm, err := getPreferencesManager(c) + pm, err := getPreferencesManager(ctx, c) if err != nil { return nil, err } diff --git a/internal/grpc/services/publicshareprovider/publicshareprovider.go b/internal/grpc/services/publicshareprovider/publicshareprovider.go index 1d513e5554..b30f6653c6 100644 --- a/internal/grpc/services/publicshareprovider/publicshareprovider.go +++ b/internal/grpc/services/publicshareprovider/publicshareprovider.go @@ -58,9 +58,9 @@ type service struct { allowedPathsForShares []*regexp.Regexp } -func getShareManager(c *config) (publicshare.Manager, error) { +func getShareManager(ctx context.Context, c *config) (publicshare.Manager, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) + return f(ctx, c.Drivers[c.Driver]) } return nil, errtypes.NotFound("driver not found: " + c.Driver) } @@ -87,7 +87,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new user share provider svc. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err @@ -95,7 +95,7 @@ func New(m map[string]interface{}) (rgrpc.Service, error) { c.init() - sm, err := getShareManager(c) + sm, err := getShareManager(ctx, c) if err != nil { return nil, err } diff --git a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go index 43c141e6bf..85e541fc02 100644 --- a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go +++ b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go @@ -83,7 +83,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new IsPublic Storage Provider service. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 9bb1011817..da0f826d91 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -162,7 +162,7 @@ func registerMimeTypes(mappingFile string) error { } // New creates a new storage provider svc. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err @@ -177,7 +177,7 @@ func New(m map[string]interface{}) (rgrpc.Service, error) { mountPath := c.MountPath mountID := c.MountID - fs, err := getFS(c) + fs, err := getFS(ctx, c) if err != nil { return nil, err } @@ -1538,9 +1538,9 @@ func (s *service) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) ( return res, nil } -func getFS(c *config) (storage.FS, error) { +func getFS(ctx context.Context, c *config) (storage.FS, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) + return f(ctx, c.Drivers[c.Driver]) } return nil, errtypes.NotFound("driver not found: " + c.Driver) } diff --git a/internal/grpc/services/storageregistry/storageregistry.go b/internal/grpc/services/storageregistry/storageregistry.go index 685a8c0004..391347670b 100644 --- a/internal/grpc/services/storageregistry/storageregistry.go +++ b/internal/grpc/services/storageregistry/storageregistry.go @@ -64,7 +64,7 @@ func (c *config) init() { } // New creates a new StorageBrokerService. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err @@ -72,7 +72,7 @@ func New(m map[string]interface{}) (rgrpc.Service, error) { c.init() - reg, err := getRegistry(c) + reg, err := getRegistry(ctx, c) if err != nil { return nil, err } @@ -92,9 +92,9 @@ func parseConfig(m map[string]interface{}) (*config, error) { return c, nil } -func getRegistry(c *config) (storage.Registry, error) { +func getRegistry(ctx context.Context, c *config) (storage.Registry, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) + return f(ctx, c.Drivers[c.Driver]) } return nil, errtypes.NotFound("driver not found: " + c.Driver) } diff --git a/internal/grpc/services/userprovider/userprovider.go b/internal/grpc/services/userprovider/userprovider.go index fa6cb78abc..a8a7ae774a 100644 --- a/internal/grpc/services/userprovider/userprovider.go +++ b/internal/grpc/services/userprovider/userprovider.go @@ -61,7 +61,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { return c, nil } -func getDriver(c *config) (user.Manager, *plugin.RevaPlugin, error) { +func getDriver(ctx context.Context, c *config) (user.Manager, *plugin.RevaPlugin, error) { p, err := plugin.Load("userprovider", c.Driver) if err == nil { manager, ok := p.Plugin.(user.Manager) @@ -77,7 +77,7 @@ func getDriver(c *config) (user.Manager, *plugin.RevaPlugin, error) { } else if _, ok := err.(errtypes.NotFound); ok { // plugin not found, fetch the driver from the in-memory registry if f, ok := registry.NewFuncs[c.Driver]; ok { - mgr, err := f(c.Drivers[c.Driver]) + mgr, err := f(ctx, c.Drivers[c.Driver]) return mgr, nil, err } } else { @@ -87,12 +87,12 @@ func getDriver(c *config) (user.Manager, *plugin.RevaPlugin, error) { } // New returns a new UserProviderServiceServer. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err } - userManager, plug, err := getDriver(c) + userManager, plug, err := getDriver(ctx, c) if err != nil { return nil, err } diff --git a/internal/grpc/services/usershareprovider/usershareprovider.go b/internal/grpc/services/usershareprovider/usershareprovider.go index 074397364e..81965b3db3 100644 --- a/internal/grpc/services/usershareprovider/usershareprovider.go +++ b/internal/grpc/services/usershareprovider/usershareprovider.go @@ -59,9 +59,9 @@ type service struct { allowedPathsForShares []*regexp.Regexp } -func getShareManager(c *config) (share.Manager, error) { +func getShareManager(ctx context.Context, c *config) (share.Manager, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) + return f(ctx, c.Drivers[c.Driver]) } return nil, errtypes.NotFound("driver not found: " + c.Driver) } @@ -89,7 +89,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New creates a new user share provider svc. -func New(m map[string]interface{}) (rgrpc.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { c, err := parseConfig(m) if err != nil { return nil, err @@ -97,7 +97,7 @@ func New(m map[string]interface{}) (rgrpc.Service, error) { c.init() - sm, err := getShareManager(c) + sm, err := getShareManager(ctx, c) if err != nil { return nil, err } diff --git a/internal/http/interceptors/loader/loader.go b/internal/http/interceptors/loader/loader.go index bb00e89319..f063179431 100644 --- a/internal/http/interceptors/loader/loader.go +++ b/internal/http/interceptors/loader/loader.go @@ -21,6 +21,5 @@ package loader import ( // Load core HTTP middlewares. _ "github.com/cs3org/reva/internal/http/interceptors/cors" - _ "github.com/cs3org/reva/internal/http/interceptors/providerauthorizer" // Add your own middleware. ) diff --git a/internal/http/interceptors/providerauthorizer/providerauthorizer.go b/internal/http/interceptors/providerauthorizer/providerauthorizer.go deleted file mode 100644 index a18dcf9ef4..0000000000 --- a/internal/http/interceptors/providerauthorizer/providerauthorizer.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package providerauthorizer - -import ( - "fmt" - "net/http" - "net/url" - "strings" - - ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" - "github.com/cs3org/reva/pkg/appctx" - ctxpkg "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/ocm/provider" - "github.com/cs3org/reva/pkg/ocm/provider/authorizer/registry" - "github.com/cs3org/reva/pkg/rhttp/global" - "github.com/cs3org/reva/pkg/rhttp/router" - "github.com/cs3org/reva/pkg/utils" - "github.com/mitchellh/mapstructure" -) - -type config struct { - Driver string `mapstructure:"driver"` - Drivers map[string]map[string]interface{} `mapstructure:"drivers"` -} - -func (c *config) init() { - if c.Driver == "" { - c.Driver = "json" - } -} - -func getDriver(c *config) (provider.Authorizer, error) { - if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) - } - - return nil, fmt.Errorf("driver %s not found for provider authorizer", c.Driver) -} - -// New returns a new HTTP middleware that verifies that the provider is registered in OCM. -func New(m map[string]interface{}, unprotected []string, ocmPrefix string) (global.Middleware, error) { - if ocmPrefix == "" { - ocmPrefix = "ocm" - } - - conf := &config{} - if err := mapstructure.Decode(m, conf); err != nil { - return nil, err - } - conf.init() - - authorizer, err := getDriver(conf) - if err != nil { - return nil, err - } - - handler := func(h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - log := appctx.GetLogger(ctx) - head, _ := router.ShiftPath(r.URL.Path) - - if r.Method == http.MethodOptions || head != ocmPrefix || utils.Skip(r.URL.Path, unprotected) { - log.Info().Msg("skipping provider authorizer check for: " + r.URL.Path) - h.ServeHTTP(w, r) - return - } - - userIdp := ctxpkg.ContextMustGetUser(ctx).Id.Idp - if !(strings.Contains(userIdp, "://")) { - userIdp = "https://" + userIdp - } - userIdpURL, err := url.Parse(userIdp) - if err != nil { - log.Error().Err(err).Msg("error parsing user idp in provider authorizer") - w.WriteHeader(http.StatusUnauthorized) - return - } - - err = authorizer.IsProviderAllowed(ctx, &ocmprovider.ProviderInfo{ - Domain: userIdpURL.Hostname(), - }) - if err != nil { - log.Error().Err(err).Msg("provider not registered in OCM") - w.WriteHeader(http.StatusUnauthorized) - return - } - - h.ServeHTTP(w, r) - }) - } - - return handler, nil -} diff --git a/internal/http/services/appprovider/appprovider.go b/internal/http/services/appprovider/appprovider.go index c57d9fbc78..4dc81ff8ab 100644 --- a/internal/http/services/appprovider/appprovider.go +++ b/internal/http/services/appprovider/appprovider.go @@ -19,6 +19,7 @@ package appprovider import ( + "context" "encoding/json" "net/http" "path" @@ -40,7 +41,6 @@ import ( ua "github.com/mileusna/useragent" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" - "github.com/rs/zerolog" ) func init() { @@ -67,7 +67,7 @@ type svc struct { } // New returns a new ocmd object. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { conf := &Config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/internal/http/services/archiver/handler.go b/internal/http/services/archiver/handler.go index ac787b0215..07c63b6a44 100644 --- a/internal/http/services/archiver/handler.go +++ b/internal/http/services/archiver/handler.go @@ -42,13 +42,11 @@ import ( "github.com/gdexlab/go-render/render" ua "github.com/mileusna/useragent" "github.com/mitchellh/mapstructure" - "github.com/rs/zerolog" ) type svc struct { config *Config gtwClient gateway.GatewayAPIClient - log *zerolog.Logger walker walker.Walker downloader downloader.Downloader @@ -72,7 +70,7 @@ func init() { } // New creates a new archiver service. -func New(conf map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, conf map[string]interface{}) (global.Service, error) { c := &Config{} err := mapstructure.Decode(conf, c) if err != nil { @@ -101,7 +99,6 @@ func New(conf map[string]interface{}, log *zerolog.Logger) (global.Service, erro gtwClient: gtw, downloader: downloader.NewDownloader(gtw, rhttp.Insecure(c.Insecure), rhttp.Timeout(time.Duration(c.Timeout*int64(time.Second)))), walker: walker.NewWalker(gtw), - log: log, allowedFolders: allowedFolderRegex, }, nil } @@ -186,21 +183,22 @@ func (s *svc) allAllowed(paths []string) error { return nil } -func (s *svc) writeHTTPError(rw http.ResponseWriter, err error) { - s.log.Error().Msg(err.Error()) +func (s *svc) writeHTTPError(ctx context.Context, w http.ResponseWriter, err error) { + log := appctx.GetLogger(ctx) + log.Error().Msg(err.Error()) switch err.(type) { case errtypes.NotFound: - rw.WriteHeader(http.StatusNotFound) + w.WriteHeader(http.StatusNotFound) case manager.ErrMaxSize, manager.ErrMaxFileCount: - rw.WriteHeader(http.StatusRequestEntityTooLarge) + w.WriteHeader(http.StatusRequestEntityTooLarge) case errtypes.BadRequest: - rw.WriteHeader(http.StatusBadRequest) + w.WriteHeader(http.StatusBadRequest) default: - rw.WriteHeader(http.StatusInternalServerError) + w.WriteHeader(http.StatusInternalServerError) } - _, _ = rw.Write([]byte(err.Error())) + _, _ = w.Write([]byte(err.Error())) } func (s *svc) Handler() http.Handler { @@ -221,7 +219,7 @@ func (s *svc) Handler() http.Handler { files, err := s.getFiles(ctx, paths, ids) if err != nil { - s.writeHTTPError(rw, err) + s.writeHTTPError(ctx, rw, err) return } @@ -230,7 +228,7 @@ func (s *svc) Handler() http.Handler { MaxSize: s.config.MaxSize, }) if err != nil { - s.writeHTTPError(rw, err) + s.writeHTTPError(ctx, rw, err) return } @@ -256,7 +254,7 @@ func (s *svc) Handler() http.Handler { } if err != nil { - s.writeHTTPError(rw, err) + s.writeHTTPError(ctx, rw, err) return } }) diff --git a/internal/http/services/datagateway/datagateway.go b/internal/http/services/datagateway/datagateway.go index 57a5a6134a..9795b5cee5 100644 --- a/internal/http/services/datagateway/datagateway.go +++ b/internal/http/services/datagateway/datagateway.go @@ -35,7 +35,6 @@ import ( "github.com/golang-jwt/jwt" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" - "github.com/rs/zerolog" ) const ( @@ -77,7 +76,7 @@ type svc struct { } // New returns a new datagateway. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { conf := &config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index f4277b6ce9..df35db7fa6 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -19,6 +19,7 @@ package dataprovider import ( + "context" "fmt" "net/http" @@ -29,7 +30,6 @@ import ( "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/mitchellh/mapstructure" - "github.com/rs/zerolog" ) func init() { @@ -62,7 +62,7 @@ type svc struct { } // New returns a new datasvc. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { conf := &config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err @@ -70,12 +70,12 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) conf.init() - fs, err := getFS(conf) + fs, err := getFS(ctx, conf) if err != nil { return nil, err } - dataTXs, err := getDataTXs(conf, fs) + dataTXs, err := getDataTXs(ctx, conf, fs) if err != nil { return nil, err } @@ -90,14 +90,14 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) return s, err } -func getFS(c *config) (storage.FS, error) { +func getFS(ctx context.Context, c *config) (storage.FS, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { - return f(c.Drivers[c.Driver]) + return f(ctx, c.Drivers[c.Driver]) } return nil, fmt.Errorf("driver not found: %s", c.Driver) } -func getDataTXs(c *config, fs storage.FS) (map[string]http.Handler, error) { +func getDataTXs(ctx context.Context, c *config, fs storage.FS) (map[string]http.Handler, error) { if c.DataTXs == nil { c.DataTXs = make(map[string]map[string]interface{}) } @@ -110,7 +110,7 @@ func getDataTXs(c *config, fs storage.FS) (map[string]http.Handler, error) { txs := make(map[string]http.Handler) for t := range c.DataTXs { if f, ok := datatxregistry.NewFuncs[t]; ok { - if tx, err := f(c.DataTXs[t]); err == nil { + if tx, err := f(ctx, c.DataTXs[t]); err == nil { if handler, err := tx.Handler(fs); err == nil { txs[t] = handler } diff --git a/internal/http/services/helloworld/helloworld.go b/internal/http/services/helloworld/helloworld.go index 03aea8880e..c136aeb6a8 100644 --- a/internal/http/services/helloworld/helloworld.go +++ b/internal/http/services/helloworld/helloworld.go @@ -19,12 +19,12 @@ package helloworld import ( + "context" "net/http" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/mitchellh/mapstructure" - "github.com/rs/zerolog" ) func init() { @@ -32,7 +32,7 @@ func init() { } // New returns a new helloworld service. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { conf := &config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 00c9db606a..f7c8a78845 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -19,8 +19,10 @@ package mentix import ( + "context" "net/http" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/mentix" "github.com/cs3org/reva/pkg/mentix/config" "github.com/cs3org/reva/pkg/mentix/exchangers" @@ -166,7 +168,7 @@ func applyDefaultConfig(conf *config.Configuration) { } // New returns a new Mentix service. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { // Prepare the configuration conf, err := parseConfig(m) if err != nil { @@ -176,6 +178,7 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) conf.Init() // Create the Mentix instance + log := appctx.GetLogger(ctx) mntx, err := mentix.New(conf, log) if err != nil { return nil, errors.Wrap(err, "mentix: error creating Mentix") diff --git a/internal/http/services/metrics/metrics.go b/internal/http/services/metrics/metrics.go index 61f6a33ca6..fb02a01bf7 100644 --- a/internal/http/services/metrics/metrics.go +++ b/internal/http/services/metrics/metrics.go @@ -22,6 +22,7 @@ package metrics This service initializes the metrics package according to the metrics configuration. */ import ( + "context" "net/http" "os" @@ -30,7 +31,6 @@ import ( "github.com/cs3org/reva/pkg/metrics/config" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/mitchellh/mapstructure" - "github.com/rs/zerolog" ) func init() { @@ -69,7 +69,7 @@ func (s *svc) Handler() http.Handler { } // New returns a new metrics service. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { // Prepare the configuration conf := &config.Config{} if err := mapstructure.Decode(m, conf); err != nil { diff --git a/internal/http/services/ocmd/ocm.go b/internal/http/services/ocmd/ocm.go index 8cbb4821a0..4570178aef 100644 --- a/internal/http/services/ocmd/ocm.go +++ b/internal/http/services/ocmd/ocm.go @@ -19,6 +19,7 @@ package ocmd import ( + "context" "net/http" "github.com/cs3org/reva/pkg/appctx" @@ -26,7 +27,6 @@ import ( "github.com/cs3org/reva/pkg/sharedconf" "github.com/go-chi/chi/v5" "github.com/mitchellh/mapstructure" - "github.com/rs/zerolog" ) func init() { @@ -53,7 +53,7 @@ type svc struct { // New returns a new ocmd object, that implements // the OCM APIs specified in https://cs3org.github.io/OCM-API/docs.html -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { conf := &config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/internal/http/services/ocmprovider/ocmprovider.go b/internal/http/services/ocmprovider/ocmprovider.go index 8ed83cf116..f2f7afb71e 100644 --- a/internal/http/services/ocmprovider/ocmprovider.go +++ b/internal/http/services/ocmprovider/ocmprovider.go @@ -19,6 +19,7 @@ package ocmprovider import ( + "context" "encoding/json" "fmt" "net/http" @@ -26,7 +27,6 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/mitchellh/mapstructure" - "github.com/rs/zerolog" ) func init() { @@ -125,7 +125,7 @@ func (c *config) prepare() *discoveryData { // New returns a new ocmprovider object, that implements // the OCM discovery endpoint specified in // https://cs3org.github.io/OCM-API/docs.html?repo=OCM-API&user=cs3org#/paths/~1ocm-provider/get -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { conf := &config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/internal/http/services/owncloud/ocdav/ocdav.go b/internal/http/services/owncloud/ocdav/ocdav.go index 7f6d7ca366..6374a6a29a 100644 --- a/internal/http/services/owncloud/ocdav/ocdav.go +++ b/internal/http/services/owncloud/ocdav/ocdav.go @@ -44,7 +44,6 @@ import ( "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" - "github.com/rs/zerolog" ) type ctxKey int @@ -145,7 +144,7 @@ func getFavoritesManager(c *Config) (favorite.Manager, error) { } // New returns a new ocdav. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { conf := &Config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err @@ -158,6 +157,7 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) return nil, err } + log := appctx.GetLogger(ctx) s := &svc{ c: conf, webDavHandler: new(WebDavHandler), diff --git a/internal/http/services/owncloud/ocs/conversions/main.go b/internal/http/services/owncloud/ocs/conversions/main.go index 1650ab3182..1a12638367 100644 --- a/internal/http/services/owncloud/ocs/conversions/main.go +++ b/internal/http/services/owncloud/ocs/conversions/main.go @@ -346,18 +346,18 @@ func LocalGroupIDToString(groupID *grouppb.GroupId) string { } // GetUserManager returns a connection to a user share manager. -func GetUserManager(manager string, m map[string]map[string]interface{}) (user.Manager, error) { +func GetUserManager(ctx context.Context, manager string, m map[string]map[string]interface{}) (user.Manager, error) { if f, ok := usermgr.NewFuncs[manager]; ok { - return f(m[manager]) + return f(ctx, m[manager]) } return nil, fmt.Errorf("driver %s not found for user manager", manager) } // GetPublicShareManager returns a connection to a public share manager. -func GetPublicShareManager(manager string, m map[string]map[string]interface{}) (publicshare.Manager, error) { +func GetPublicShareManager(ctx context.Context, manager string, m map[string]map[string]interface{}) (publicshare.Manager, error) { if f, ok := publicsharemgr.NewFuncs[manager]; ok { - return f(m[manager]) + return f(ctx, m[manager]) } return nil, fmt.Errorf("driver %s not found for public shares manager", manager) diff --git a/internal/http/services/owncloud/ocs/ocs.go b/internal/http/services/owncloud/ocs/ocs.go index 5aef6c9ffb..05ce3f8819 100644 --- a/internal/http/services/owncloud/ocs/ocs.go +++ b/internal/http/services/owncloud/ocs/ocs.go @@ -19,6 +19,7 @@ package ocs import ( + "context" "net/http" "time" @@ -48,7 +49,7 @@ type svc struct { warmupCacheTracker *ttlcache.Cache } -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { conf := &config.Config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err @@ -62,6 +63,7 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) router: r, } + log := appctx.GetLogger(ctx) if err := s.routerInit(log); err != nil { return nil, err } diff --git a/internal/http/services/preferences/preferences.go b/internal/http/services/preferences/preferences.go index eefec9c07a..d5f8bdf506 100644 --- a/internal/http/services/preferences/preferences.go +++ b/internal/http/services/preferences/preferences.go @@ -19,6 +19,7 @@ package preferences import ( + "context" "encoding/json" "net/http" @@ -30,7 +31,6 @@ import ( "github.com/cs3org/reva/pkg/sharedconf" "github.com/go-chi/chi/v5" "github.com/mitchellh/mapstructure" - "github.com/rs/zerolog" ) func init() { @@ -56,7 +56,7 @@ type svc struct { } // New returns a new ocmd object. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { conf := &Config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/internal/http/services/prometheus/prometheus.go b/internal/http/services/prometheus/prometheus.go index bc7986b253..88c18d81fe 100644 --- a/internal/http/services/prometheus/prometheus.go +++ b/internal/http/services/prometheus/prometheus.go @@ -19,13 +19,13 @@ package prometheus import ( + "context" "net/http" "contrib.go.opencensus.io/exporter/prometheus" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" - "github.com/rs/zerolog" "go.opencensus.io/stats/view" ) @@ -34,7 +34,7 @@ func init() { } // New returns a new prometheus service. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { conf := &config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/internal/http/services/reverseproxy/reverseproxy.go b/internal/http/services/reverseproxy/reverseproxy.go index b1df580f3e..2922e3b449 100644 --- a/internal/http/services/reverseproxy/reverseproxy.go +++ b/internal/http/services/reverseproxy/reverseproxy.go @@ -19,6 +19,7 @@ package reverseproxy import ( + "context" "encoding/json" "net/http" "net/http/httputil" @@ -29,7 +30,6 @@ import ( "github.com/cs3org/reva/pkg/rhttp/global" "github.com/go-chi/chi/v5" "github.com/mitchellh/mapstructure" - "github.com/rs/zerolog" ) func init() { @@ -56,7 +56,7 @@ type svc struct { } // New returns an instance of the reverse proxy service. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { conf := &config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/internal/http/services/sciencemesh/sciencemesh.go b/internal/http/services/sciencemesh/sciencemesh.go index 14aafc6619..a728c82a77 100644 --- a/internal/http/services/sciencemesh/sciencemesh.go +++ b/internal/http/services/sciencemesh/sciencemesh.go @@ -19,6 +19,7 @@ package sciencemesh import ( + "context" "errors" "net/http" @@ -28,7 +29,6 @@ import ( "github.com/cs3org/reva/pkg/smtpclient" "github.com/go-chi/chi/v5" "github.com/mitchellh/mapstructure" - "github.com/rs/zerolog" ) func init() { @@ -36,7 +36,7 @@ func init() { } // New returns a new sciencemesh service. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { conf := &config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/internal/http/services/siteacc/siteacc.go b/internal/http/services/siteacc/siteacc.go index e735cdb3c4..9f44535c7e 100644 --- a/internal/http/services/siteacc/siteacc.go +++ b/internal/http/services/siteacc/siteacc.go @@ -19,8 +19,10 @@ package siteacc import ( + "context" "net/http" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/siteacc" "github.com/cs3org/reva/pkg/siteacc/config" @@ -103,7 +105,7 @@ func applyDefaultConfig(conf *config.Configuration) { } // New returns a new Site Accounts service. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { // Prepare the configuration conf, err := parseConfig(m) if err != nil { @@ -111,6 +113,7 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) } // Create the sites accounts instance + log := appctx.GetLogger(ctx) siteacc, err := siteacc.New(conf, log) if err != nil { return nil, errors.Wrap(err, "error creating the sites accounts service") diff --git a/internal/http/services/sysinfo/sysinfo.go b/internal/http/services/sysinfo/sysinfo.go index e4dbfafcb4..d28e98dd35 100644 --- a/internal/http/services/sysinfo/sysinfo.go +++ b/internal/http/services/sysinfo/sysinfo.go @@ -19,6 +19,7 @@ package sysinfo import ( + "context" "net/http" "github.com/cs3org/reva/pkg/appctx" @@ -26,7 +27,6 @@ import ( "github.com/cs3org/reva/pkg/sysinfo" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" - "github.com/rs/zerolog" ) func init() { @@ -94,7 +94,7 @@ func applyDefaultConfig(conf *config) { } // New returns a new SysInfo service. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { // Prepare the configuration conf, err := parseConfig(m) if err != nil { diff --git a/internal/http/services/wellknown/wellknown.go b/internal/http/services/wellknown/wellknown.go index 04c201332d..159a8d9ed7 100644 --- a/internal/http/services/wellknown/wellknown.go +++ b/internal/http/services/wellknown/wellknown.go @@ -19,13 +19,13 @@ package wellknown import ( + "context" "net/http" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/rhttp/router" "github.com/mitchellh/mapstructure" - "github.com/rs/zerolog" ) func init() { @@ -56,7 +56,7 @@ type svc struct { } // New returns a new webuisvc. -func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { conf := &config{} if err := mapstructure.Decode(m, conf); err != nil { return nil, err diff --git a/pkg/app/provider/demo/demo.go b/pkg/app/provider/demo/demo.go index 344f7d8883..91df5b83f8 100644 --- a/pkg/app/provider/demo/demo.go +++ b/pkg/app/provider/demo/demo.go @@ -68,7 +68,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { // New returns an implementation to of the app.Provider interface that // connects to an application in the backend. -func New(m map[string]interface{}) (app.Provider, error) { +func New(ctx context.Context, m map[string]interface{}) (app.Provider, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/app/provider/registry/registry.go b/pkg/app/provider/registry/registry.go index f3f46ee6b7..6fd3ef8037 100644 --- a/pkg/app/provider/registry/registry.go +++ b/pkg/app/provider/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/app" +import ( + "context" + + "github.com/cs3org/reva/pkg/app" +) // NewFunc is the function that app provider implementations // should register to at init time. -type NewFunc func(map[string]interface{}) (app.Provider, error) +type NewFunc func(context.Context, map[string]interface{}) (app.Provider, error) // NewFuncs is a map containing all the registered app providers. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/app/provider/wopi/wopi.go b/pkg/app/provider/wopi/wopi.go index b3bfd46415..5837950234 100644 --- a/pkg/app/provider/wopi/wopi.go +++ b/pkg/app/provider/wopi/wopi.go @@ -106,7 +106,7 @@ type wopiProvider struct { // New returns an implementation of the app.Provider interface that // connects to an application in the backend. -func New(m map[string]interface{}) (app.Provider, error) { +func New(ctx context.Context, m map[string]interface{}) (app.Provider, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/app/registry/registry/registry.go b/pkg/app/registry/registry/registry.go index dc8bff6291..fc5b3cdf3f 100644 --- a/pkg/app/registry/registry/registry.go +++ b/pkg/app/registry/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/app" +import ( + "context" + + "github.com/cs3org/reva/pkg/app" +) // NewFunc is the function that app provider implementations // should register to at init time. -type NewFunc func(map[string]interface{}) (app.Registry, error) +type NewFunc func(context.Context, map[string]interface{}) (app.Registry, error) // NewFuncs is a map containing all the registered app registry backends. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/app/registry/static/static.go b/pkg/app/registry/static/static.go index 682c7c1d3f..9890f6d25c 100644 --- a/pkg/app/registry/static/static.go +++ b/pkg/app/registry/static/static.go @@ -78,7 +78,7 @@ type manager struct { } // New returns an implementation of the app.Registry interface. -func New(m map[string]interface{}) (app.Registry, error) { +func New(ctx context.Context, m map[string]interface{}) (app.Registry, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/app/registry/static/static_test.go b/pkg/app/registry/static/static_test.go index 09fabc802a..3600432995 100644 --- a/pkg/app/registry/static/static_test.go +++ b/pkg/app/registry/static/static_test.go @@ -175,7 +175,7 @@ func TestFindProviders(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := context.TODO() - registry, err := New(map[string]interface{}{ + registry, err := New(ctx, map[string]interface{}{ "mime_types": tt.mimeTypes, "providers": tt.regProviders, }) @@ -474,7 +474,7 @@ func TestFindProvidersWithPriority(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := context.TODO() - registry, err := New(map[string]interface{}{ + registry, err := New(ctx, map[string]interface{}{ "mime_types": tt.mimeTypes, }) if err != nil { @@ -763,7 +763,7 @@ func TestAddProvider(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := context.TODO() - registry, err := New(map[string]interface{}{ + registry, err := New(ctx, map[string]interface{}{ "providers": tt.initProviders, "mime_types": tt.mimeTypes, }) @@ -1014,7 +1014,7 @@ func TestListSupportedMimeTypes(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := context.TODO() - registry, err := New(map[string]interface{}{ + registry, err := New(ctx, map[string]interface{}{ "mime_types": tt.mimeTypes, }) if err != nil { @@ -1141,7 +1141,7 @@ func TestSetDefaultProviderForMimeType(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := context.TODO() - registry, err := New(map[string]interface{}{ + registry, err := New(ctx, map[string]interface{}{ "providers": tt.initProviders, "mime_types": tt.mimeTypes, }) diff --git a/pkg/appauth/manager/json/json.go b/pkg/appauth/manager/json/json.go index 4a37cad515..0a488c6cb9 100644 --- a/pkg/appauth/manager/json/json.go +++ b/pkg/appauth/manager/json/json.go @@ -58,7 +58,7 @@ type jsonManager struct { } // New returns a new mgr. -func New(m map[string]interface{}) (appauth.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (appauth.Manager, error) { c, err := parseConfig(m) if err != nil { return nil, errors.Wrap(err, "error creating a new manager") diff --git a/pkg/appauth/manager/json/json_test.go b/pkg/appauth/manager/json/json_test.go index a6a8c1136a..e20d1cd585 100644 --- a/pkg/appauth/manager/json/json_test.go +++ b/pkg/appauth/manager/json/json_test.go @@ -123,7 +123,7 @@ func TestNewManager(t *testing.T) { for _, test := range testCases { t.Run(test.description, func(t *testing.T) { - manager, err := New(test.configMap) + manager, err := New(context.Background(), test.configMap) if test.expected == nil { if err == nil { t.Fatalf("no error (but we expected one) while get manager") @@ -239,7 +239,7 @@ func TestGenerateAppPassword(t *testing.T) { tmpFile := createTempFile(t, tempDir, "test.json") defer tmpFile.Close() fill(t, tmpFile, test.prevStateJSON) - manager, err := New(map[string]interface{}{ + manager, err := New(context.Background(), map[string]interface{}{ "file": tmpFile.Name(), "token_strength": len(token), "password_hash_cost": 11, @@ -393,7 +393,7 @@ func TestListAppPasswords(t *testing.T) { if test.stateJSON != "" { fill(t, tmpFile, test.stateJSON) } - manager, err := New(map[string]interface{}{ + manager, err := New(context.Background(), map[string]interface{}{ "file": tmpFile.Name(), "token_strength": len(token), }) @@ -514,7 +514,7 @@ func TestInvalidateAppPassword(t *testing.T) { // initialize temp file with `state_json` content tmpFile := createTempFile(t, tempDir, "test.json") fill(t, tmpFile, test.stateJSON) - manager, err := New(map[string]interface{}{ + manager, err := New(context.Background(), map[string]interface{}{ "file": tmpFile.Name(), "token_strength": 4, }) @@ -669,7 +669,7 @@ func TestGetAppPassword(t *testing.T) { // initialize temp file with `state_json` content tmpFile := createTempFile(t, tempDir, "test.json") fill(t, tmpFile, test.stateJSON) - manager, err := New(map[string]interface{}{ + manager, err := New(context.Background(), map[string]interface{}{ "file": tmpFile.Name(), "token_strength": 4, }) diff --git a/pkg/appauth/manager/registry/registry.go b/pkg/appauth/manager/registry/registry.go index bc3ec41533..21cdcdea10 100644 --- a/pkg/appauth/manager/registry/registry.go +++ b/pkg/appauth/manager/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/appauth" +import ( + "context" + + "github.com/cs3org/reva/pkg/appauth" +) // NewFunc is the function that application auth implementations // should register at init time. -type NewFunc func(map[string]interface{}) (appauth.Manager, error) +type NewFunc func(context.Context, map[string]interface{}) (appauth.Manager, error) // NewFuncs is a map containing all the registered application auth managers. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/auth/manager/appauth/appauth.go b/pkg/auth/manager/appauth/appauth.go index 9cf208523a..daf292a739 100644 --- a/pkg/auth/manager/appauth/appauth.go +++ b/pkg/auth/manager/appauth/appauth.go @@ -42,7 +42,7 @@ type manager struct { } // New returns a new auth Manager. -func New(m map[string]interface{}) (auth.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { mgr := &manager{} err := mgr.Configure(m) if err != nil { diff --git a/pkg/auth/manager/demo/demo.go b/pkg/auth/manager/demo/demo.go index 26fa5731e8..fc44c1ddaf 100644 --- a/pkg/auth/manager/demo/demo.go +++ b/pkg/auth/manager/demo/demo.go @@ -44,7 +44,7 @@ type Credentials struct { } // New returns a new auth Manager. -func New(m map[string]interface{}) (auth.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { // m not used mgr := &manager{} err := mgr.Configure(m) diff --git a/pkg/auth/manager/demo/demo_test.go b/pkg/auth/manager/demo/demo_test.go index 82f9e9e245..3ae7ab89c9 100644 --- a/pkg/auth/manager/demo/demo_test.go +++ b/pkg/auth/manager/demo/demo_test.go @@ -27,7 +27,7 @@ var ctx = context.Background() func TestUserManager(t *testing.T) { // get manager - manager, _ := New(nil) + manager, _ := New(context.Background(), nil) // Authenticate - positive test _, _, err := manager.Authenticate(ctx, "einstein", "relativity") diff --git a/pkg/auth/manager/impersonator/impersonator.go b/pkg/auth/manager/impersonator/impersonator.go index 8c20f77fb7..dbb1a58cca 100644 --- a/pkg/auth/manager/impersonator/impersonator.go +++ b/pkg/auth/manager/impersonator/impersonator.go @@ -36,7 +36,7 @@ func init() { type mgr struct{} // New returns an auth manager implementation that allows to authenticate with any credentials. -func New(c map[string]interface{}) (auth.Manager, error) { +func New(ctx context.Context, c map[string]interface{}) (auth.Manager, error) { return &mgr{}, nil } diff --git a/pkg/auth/manager/impersonator/impersonator_test.go b/pkg/auth/manager/impersonator/impersonator_test.go index 82b0d9042e..db65f0c954 100644 --- a/pkg/auth/manager/impersonator/impersonator_test.go +++ b/pkg/auth/manager/impersonator/impersonator_test.go @@ -25,7 +25,7 @@ import ( func TestImpersonator(t *testing.T) { ctx := context.Background() - i, _ := New(nil) + i, _ := New(ctx, nil) u, _, err := i.Authenticate(ctx, "admin", "pwd") if err != nil { t.Fatal(err) diff --git a/pkg/auth/manager/json/json.go b/pkg/auth/manager/json/json.go index 1a691170fe..5e095c062b 100644 --- a/pkg/auth/manager/json/json.go +++ b/pkg/auth/manager/json/json.go @@ -78,7 +78,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns a new auth Manager. -func New(m map[string]interface{}) (auth.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { mgr := &manager{} err := mgr.Configure(m) if err != nil { diff --git a/pkg/auth/manager/json/json_test.go b/pkg/auth/manager/json/json_test.go index ce1793f37f..45cda03f6f 100644 --- a/pkg/auth/manager/json/json_test.go +++ b/pkg/auth/manager/json/json_test.go @@ -60,7 +60,7 @@ func TestGetManagerWithInvalidUser(t *testing.T) { "users": tt.user, } - manager, err := New(input) + manager, err := New(ctx, input) assert.Empty(t, manager) assert.EqualError(t, err, tt.expectedError) @@ -117,7 +117,7 @@ func TestGetManagerWithJSONObject(t *testing.T) { "users": tmpFile.Name(), } - manager, err := New(input) + manager, err := New(ctx, input) if tt.expectManager { assert.Equal(t, nil, err) @@ -180,7 +180,7 @@ func TestGetAuthenticatedManager(t *testing.T) { input := map[string]interface{}{ "users": tempFile.Name(), } - manager, _ := New(input) + manager, _ := New(ctx, input) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/auth/manager/ldap/ldap.go b/pkg/auth/manager/ldap/ldap.go index d309350b31..e9f871720d 100644 --- a/pkg/auth/manager/ldap/ldap.go +++ b/pkg/auth/manager/ldap/ldap.go @@ -99,7 +99,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns an auth manager implementation that connects to a LDAP server to validate the user. -func New(m map[string]interface{}) (auth.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { manager := &mgr{} err := manager.Configure(m) if err != nil { diff --git a/pkg/auth/manager/machine/machine.go b/pkg/auth/manager/machine/machine.go index 2146d33e4b..6d557e24c2 100644 --- a/pkg/auth/manager/machine/machine.go +++ b/pkg/auth/manager/machine/machine.go @@ -60,7 +60,7 @@ func (m *manager) Configure(conf map[string]interface{}) error { } // New creates a new manager for the 'machine' authentication. -func New(conf map[string]interface{}) (auth.Manager, error) { +func New(ctx context.Context, conf map[string]interface{}) (auth.Manager, error) { m := &manager{} err := m.Configure(conf) if err != nil { diff --git a/pkg/auth/manager/nextcloud/nextcloud.go b/pkg/auth/manager/nextcloud/nextcloud.go index d148e80cff..502016b61d 100644 --- a/pkg/auth/manager/nextcloud/nextcloud.go +++ b/pkg/auth/manager/nextcloud/nextcloud.go @@ -76,7 +76,7 @@ func parseConfig(m map[string]interface{}) (*AuthManagerConfig, error) { } // New returns an auth manager implementation that verifies against a Nextcloud backend. -func New(m map[string]interface{}) (auth.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/auth/manager/nextcloud/nextcloud_test.go b/pkg/auth/manager/nextcloud/nextcloud_test.go index 2dc74e2ec0..b1998d4106 100644 --- a/pkg/auth/manager/nextcloud/nextcloud_test.go +++ b/pkg/auth/manager/nextcloud/nextcloud_test.go @@ -113,7 +113,7 @@ var _ = Describe("Nextcloud", func() { Describe("New", func() { It("returns a new instance", func() { fmt.Println(options) - _, err := nextcloud.New(options) + _, err := nextcloud.New(context.Background(), options) Expect(err).ToNot(HaveOccurred()) }) }) diff --git a/pkg/auth/manager/ocmshares/ocmshares.go b/pkg/auth/manager/ocmshares/ocmshares.go index 17f46c592c..b290964bee 100644 --- a/pkg/auth/manager/ocmshares/ocmshares.go +++ b/pkg/auth/manager/ocmshares/ocmshares.go @@ -59,7 +59,7 @@ func (c *config) init() { } // New creates a new ocmshares authentication manager. -func New(m map[string]interface{}) (auth.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { var mgr manager if err := mgr.Configure(m); err != nil { return nil, err diff --git a/pkg/auth/manager/oidc/oidc.go b/pkg/auth/manager/oidc/oidc.go index d55b6cd420..656077111d 100644 --- a/pkg/auth/manager/oidc/oidc.go +++ b/pkg/auth/manager/oidc/oidc.go @@ -104,7 +104,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns an auth manager implementation that verifies the oidc token and obtains the user claims. -func New(m map[string]interface{}) (auth.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { manager := &mgr{ providers: make(map[string]*oidc.Provider), } diff --git a/pkg/auth/manager/owncloudsql/owncloudsql.go b/pkg/auth/manager/owncloudsql/owncloudsql.go index d8e214d48f..2c7d9277bd 100644 --- a/pkg/auth/manager/owncloudsql/owncloudsql.go +++ b/pkg/auth/manager/owncloudsql/owncloudsql.go @@ -65,7 +65,7 @@ type config struct { } // NewMysql returns a new auth manager connection to an owncloud mysql database. -func NewMysql(m map[string]interface{}) (auth.Manager, error) { +func NewMysql(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { mgr := &manager{} err := mgr.Configure(m) if err != nil { diff --git a/pkg/auth/manager/publicshares/publicshares.go b/pkg/auth/manager/publicshares/publicshares.go index cfbc11a71f..2b0f48d8f0 100644 --- a/pkg/auth/manager/publicshares/publicshares.go +++ b/pkg/auth/manager/publicshares/publicshares.go @@ -59,7 +59,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns a new auth Manager. -func New(m map[string]interface{}) (auth.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { mgr := &manager{} err := mgr.Configure(m) if err != nil { diff --git a/pkg/auth/manager/registry/registry.go b/pkg/auth/manager/registry/registry.go index b06d3f7495..f3b38b0acb 100644 --- a/pkg/auth/manager/registry/registry.go +++ b/pkg/auth/manager/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/auth" +import ( + "context" + + "github.com/cs3org/reva/pkg/auth" +) // NewFunc is the function that auth implementations // should register to at init time. -type NewFunc func(map[string]interface{}) (auth.Manager, error) +type NewFunc func(context.Context, map[string]interface{}) (auth.Manager, error) // NewFuncs is a map containing all the registered auth managers. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/auth/registry/registry/registry.go b/pkg/auth/registry/registry/registry.go index 14135b4d83..b2709f3fe4 100644 --- a/pkg/auth/registry/registry/registry.go +++ b/pkg/auth/registry/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/auth" +import ( + "context" + + "github.com/cs3org/reva/pkg/auth" +) // NewFunc is the function that auth registry implementations // should register at init time. -type NewFunc func(map[string]interface{}) (auth.Registry, error) +type NewFunc func(context.Context, map[string]interface{}) (auth.Registry, error) // NewFuncs is a map containing all the registered auth registries. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/auth/registry/static/static.go b/pkg/auth/registry/static/static.go index 6beef10be0..852b1f4cac 100644 --- a/pkg/auth/registry/static/static.go +++ b/pkg/auth/registry/static/static.go @@ -79,7 +79,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns an implementation of the auth.Registry interface. -func New(m map[string]interface{}) (auth.Registry, error) { +func New(ctx context.Context, m map[string]interface{}) (auth.Registry, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/cbox/group/rest/rest.go b/pkg/cbox/group/rest/rest.go index 43345f4093..44d2863622 100644 --- a/pkg/cbox/group/rest/rest.go +++ b/pkg/cbox/group/rest/rest.go @@ -109,7 +109,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns a user manager implementation that makes calls to the GRAPPA API. -func New(m map[string]interface{}) (group.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (group.Manager, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/cbox/preferences/sql/sql.go b/pkg/cbox/preferences/sql/sql.go index 551fbabcbf..680a1fcb91 100644 --- a/pkg/cbox/preferences/sql/sql.go +++ b/pkg/cbox/preferences/sql/sql.go @@ -48,7 +48,7 @@ type mgr struct { } // New returns an instance of the cbox sql preferences manager. -func New(m map[string]interface{}) (preferences.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (preferences.Manager, error) { c := &config{} if err := mapstructure.Decode(m, c); err != nil { return nil, err diff --git a/pkg/cbox/publicshare/sql/sql.go b/pkg/cbox/publicshare/sql/sql.go index aaefa85dac..0b81b4d025 100644 --- a/pkg/cbox/publicshare/sql/sql.go +++ b/pkg/cbox/publicshare/sql/sql.go @@ -108,7 +108,7 @@ func (m *manager) startJanitorRun() { } // New returns a new public share manager. -func New(m map[string]interface{}) (publicshare.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (publicshare.Manager, error) { c := &config{} if err := mapstructure.Decode(m, c); err != nil { return nil, err diff --git a/pkg/cbox/share/sql/sql.go b/pkg/cbox/share/sql/sql.go index 957b697943..0a87d641c9 100644 --- a/pkg/cbox/share/sql/sql.go +++ b/pkg/cbox/share/sql/sql.go @@ -77,7 +77,7 @@ type mgr struct { } // New returns a new share manager. -func New(m map[string]interface{}) (share.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (share.Manager, error) { c, err := parseConfig(m) if err != nil { err = errors.Wrap(err, "error creating a new manager") diff --git a/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go b/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go index 6c46c6764f..2737b0ecc5 100644 --- a/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go +++ b/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go @@ -65,14 +65,14 @@ func parseConfig(m map[string]interface{}) (*eosfs.Config, string, error) { // New returns an implementation of the storage.FS interface that forms a wrapper // around separate connections to EOS. -func New(m map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { c, t, err := parseConfig(m) if err != nil { return nil, err } c.EnableHome = true - eos, err := eosfs.NewEOSFS(c) + eos, err := eosfs.NewEOSFS(ctx, c) if err != nil { return nil, err } diff --git a/pkg/cbox/storage/eoswrapper/eoswrapper.go b/pkg/cbox/storage/eoswrapper/eoswrapper.go index 5126f7f300..79663ff590 100644 --- a/pkg/cbox/storage/eoswrapper/eoswrapper.go +++ b/pkg/cbox/storage/eoswrapper/eoswrapper.go @@ -82,13 +82,13 @@ func parseConfig(m map[string]interface{}) (*eosfs.Config, string, error) { // New returns an implementation of the storage.FS interface that forms a wrapper // around separate connections to EOS. -func New(m map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { c, t, err := parseConfig(m) if err != nil { return nil, err } - eos, err := eosfs.NewEOSFS(c) + eos, err := eosfs.NewEOSFS(ctx, c) if err != nil { return nil, err } diff --git a/pkg/cbox/user/rest/rest.go b/pkg/cbox/user/rest/rest.go index d25790844b..72864cc7e7 100644 --- a/pkg/cbox/user/rest/rest.go +++ b/pkg/cbox/user/rest/rest.go @@ -107,7 +107,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns a user manager implementation that makes calls to the GRAPPA API. -func New(m map[string]interface{}) (user.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (user.Manager, error) { mgr := &manager{} err := mgr.Configure(m) if err != nil { diff --git a/pkg/datatx/manager/rclone/rclone.go b/pkg/datatx/manager/rclone/rclone.go index 932677a106..244bed0e20 100644 --- a/pkg/datatx/manager/rclone/rclone.go +++ b/pkg/datatx/manager/rclone/rclone.go @@ -102,7 +102,7 @@ type endpoint struct { } // New returns a new rclone driver. -func New(m map[string]interface{}) (txdriver.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (txdriver.Manager, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/datatx/manager/registry/registry.go b/pkg/datatx/manager/registry/registry.go index d19a91cdda..821537accf 100644 --- a/pkg/datatx/manager/registry/registry.go +++ b/pkg/datatx/manager/registry/registry.go @@ -19,12 +19,14 @@ package registry import ( + "context" + "github.com/cs3org/reva/pkg/datatx" ) // NewFunc is the function that datatx implementations // should register at init time. -type NewFunc func(map[string]interface{}) (datatx.Manager, error) +type NewFunc func(context.Context, map[string]interface{}) (datatx.Manager, error) // NewFuncs is a map containing all the registered datatx backends. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/datatx/repository/json/json.go b/pkg/datatx/repository/json/json.go index 7fcf18cb78..3f121bbff0 100644 --- a/pkg/datatx/repository/json/json.go +++ b/pkg/datatx/repository/json/json.go @@ -19,6 +19,7 @@ package json import ( + "context" "encoding/json" "io" "os" @@ -66,7 +67,7 @@ func (c *config) init() { } // New returns a json storage driver. -func New(m map[string]interface{}) (datatx.Repository, error) { +func New(ctx context.Context, m map[string]interface{}) (datatx.Repository, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/datatx/repository/registry/registry.go b/pkg/datatx/repository/registry/registry.go index 66dceb7d6a..daaa7e59d7 100644 --- a/pkg/datatx/repository/registry/registry.go +++ b/pkg/datatx/repository/registry/registry.go @@ -19,12 +19,14 @@ package registry import ( + "context" + "github.com/cs3org/reva/pkg/datatx" ) // NewFunc is the function that datatx repository implementations // should register at init time. -type NewFunc func(map[string]interface{}) (datatx.Repository, error) +type NewFunc func(context.Context, map[string]interface{}) (datatx.Repository, error) // NewFuncs is a map containing all the registered datatx repository backends. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/group/manager/json/json.go b/pkg/group/manager/json/json.go index bf46604463..1df0a407ac 100644 --- a/pkg/group/manager/json/json.go +++ b/pkg/group/manager/json/json.go @@ -64,7 +64,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns a group manager implementation that reads a json file to provide group metadata. -func New(m map[string]interface{}) (group.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (group.Manager, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/group/manager/json/json_test.go b/pkg/group/manager/json/json_test.go index 97da54f984..abb9e46625 100644 --- a/pkg/group/manager/json/json_test.go +++ b/pkg/group/manager/json/json_test.go @@ -58,7 +58,7 @@ func TestUserManager(t *testing.T) { input := map[string]interface{}{ "groups": file.Name(), } - _, err = New(input) + _, err = New(context.Background(), input) if err == nil { t.Fatalf("no error (but we expected one) while get manager") } @@ -86,7 +86,7 @@ func TestUserManager(t *testing.T) { input = map[string]interface{}{ "groups": file.Name(), } - manager, _ := New(input) + manager, _ := New(context.Background(), input) // setup test data gid := &grouppb.GroupId{OpaqueId: "sailing-lovers"} diff --git a/pkg/group/manager/ldap/ldap.go b/pkg/group/manager/ldap/ldap.go index d9d7c164cc..fb8e347f15 100644 --- a/pkg/group/manager/ldap/ldap.go +++ b/pkg/group/manager/ldap/ldap.go @@ -99,7 +99,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns a group manager implementation that connects to a LDAP server to provide group metadata. -func New(m map[string]interface{}) (group.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (group.Manager, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/group/manager/registry/registry.go b/pkg/group/manager/registry/registry.go index 0fe4ebce4d..f8f523b60a 100644 --- a/pkg/group/manager/registry/registry.go +++ b/pkg/group/manager/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/group" +import ( + "context" + + "github.com/cs3org/reva/pkg/group" +) // NewFunc is the function that group managers // should register at init time. -type NewFunc func(map[string]interface{}) (group.Manager, error) +type NewFunc func(context.Context, map[string]interface{}) (group.Manager, error) // NewFuncs is a map containing all the registered group managers. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/ocm/invite/repository/json/json.go b/pkg/ocm/invite/repository/json/json.go index 602506e2ff..ea4c3e8687 100644 --- a/pkg/ocm/invite/repository/json/json.go +++ b/pkg/ocm/invite/repository/json/json.go @@ -68,7 +68,7 @@ func (c *config) init() error { } // New returns a new invite manager object. -func New(m map[string]interface{}) (invite.Repository, error) { +func New(ctx context.Context, m map[string]interface{}) (invite.Repository, error) { config, err := parseConfig(m) if err != nil { return nil, errors.Wrap(err, "error parsing config for json invite repository") diff --git a/pkg/ocm/invite/repository/memory/memory.go b/pkg/ocm/invite/repository/memory/memory.go index 41b98ad5b9..cd81496d7c 100644 --- a/pkg/ocm/invite/repository/memory/memory.go +++ b/pkg/ocm/invite/repository/memory/memory.go @@ -37,7 +37,7 @@ func init() { } // New returns a new invite manager. -func New(m map[string]interface{}) (invite.Repository, error) { +func New(ctx context.Context, m map[string]interface{}) (invite.Repository, error) { return &manager{ Invites: sync.Map{}, AcceptedUsers: sync.Map{}, diff --git a/pkg/ocm/invite/repository/registry/registry.go b/pkg/ocm/invite/repository/registry/registry.go index a4f49d7583..e618984e59 100644 --- a/pkg/ocm/invite/repository/registry/registry.go +++ b/pkg/ocm/invite/repository/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/ocm/invite" +import ( + "context" + + "github.com/cs3org/reva/pkg/ocm/invite" +) // NewFunc is the function that invite repositories // should register at init time. -type NewFunc func(map[string]interface{}) (invite.Repository, error) +type NewFunc func(context.Context, map[string]interface{}) (invite.Repository, error) // NewFuncs is a map containing all the registered invite repositories. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/ocm/invite/repository/sql/sql.go b/pkg/ocm/invite/repository/sql/sql.go index 4fbaacf049..7746796da3 100644 --- a/pkg/ocm/invite/repository/sql/sql.go +++ b/pkg/ocm/invite/repository/sql/sql.go @@ -79,7 +79,7 @@ func parseConfig(c map[string]interface{}) (*config, error) { } // New creates a sql repository for ocm tokens and users. -func New(c map[string]interface{}) (invite.Repository, error) { +func New(ctx context.Context, c map[string]interface{}) (invite.Repository, error) { conf, err := parseConfig(c) if err != nil { return nil, errors.Wrap(err, "sql: error parsing config") diff --git a/pkg/ocm/provider/authorizer/json/json.go b/pkg/ocm/provider/authorizer/json/json.go index e4592871c0..cf6839e5dd 100644 --- a/pkg/ocm/provider/authorizer/json/json.go +++ b/pkg/ocm/provider/authorizer/json/json.go @@ -41,7 +41,7 @@ func init() { } // New returns a new authorizer object. -func New(m map[string]interface{}) (provider.Authorizer, error) { +func New(ctx context.Context, m map[string]interface{}) (provider.Authorizer, error) { c := &config{} if err := mapstructure.Decode(m, c); err != nil { err = errors.Wrap(err, "error decoding conf") diff --git a/pkg/ocm/provider/authorizer/mentix/mentix.go b/pkg/ocm/provider/authorizer/mentix/mentix.go index c82429f121..9c165c1a5e 100644 --- a/pkg/ocm/provider/authorizer/mentix/mentix.go +++ b/pkg/ocm/provider/authorizer/mentix/mentix.go @@ -49,7 +49,7 @@ type Client struct { } // New returns a new authorizer object. -func New(m map[string]interface{}) (provider.Authorizer, error) { +func New(ctx context.Context, m map[string]interface{}) (provider.Authorizer, error) { c := &config{} if err := mapstructure.Decode(m, c); err != nil { err = errors.Wrap(err, "error decoding conf") diff --git a/pkg/ocm/provider/authorizer/open/open.go b/pkg/ocm/provider/authorizer/open/open.go index 38fd828865..a9f6feefd0 100644 --- a/pkg/ocm/provider/authorizer/open/open.go +++ b/pkg/ocm/provider/authorizer/open/open.go @@ -37,7 +37,7 @@ func init() { } // New returns a new authorizer object. -func New(m map[string]interface{}) (provider.Authorizer, error) { +func New(ctx context.Context, m map[string]interface{}) (provider.Authorizer, error) { c := &config{} if err := mapstructure.Decode(m, c); err != nil { err = errors.Wrap(err, "error decoding conf") diff --git a/pkg/ocm/provider/authorizer/registry/registry.go b/pkg/ocm/provider/authorizer/registry/registry.go index a20292c05d..c9f4a1af09 100644 --- a/pkg/ocm/provider/authorizer/registry/registry.go +++ b/pkg/ocm/provider/authorizer/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/ocm/provider" +import ( + "context" + + "github.com/cs3org/reva/pkg/ocm/provider" +) // NewFunc is the function that provider authorizers // should register at init time. -type NewFunc func(map[string]interface{}) (provider.Authorizer, error) +type NewFunc func(context.Context, map[string]interface{}) (provider.Authorizer, error) // NewFuncs is a map containing all the registered provider authorizers. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/ocm/share/repository/json/json.go b/pkg/ocm/share/repository/json/json.go index 96158c3c1e..5517999335 100644 --- a/pkg/ocm/share/repository/json/json.go +++ b/pkg/ocm/share/repository/json/json.go @@ -45,7 +45,7 @@ func init() { } // New returns a new authorizer object. -func New(m map[string]interface{}) (share.Repository, error) { +func New(ctx context.Context, m map[string]interface{}) (share.Repository, error) { c, err := parseConfig(m) if err != nil { err = errors.Wrap(err, "error creating a new manager") diff --git a/pkg/ocm/share/repository/nextcloud/nextcloud.go b/pkg/ocm/share/repository/nextcloud/nextcloud.go index 76825d99e5..7fe1932069 100644 --- a/pkg/ocm/share/repository/nextcloud/nextcloud.go +++ b/pkg/ocm/share/repository/nextcloud/nextcloud.go @@ -107,7 +107,7 @@ func parseConfig(m map[string]interface{}) (*ShareManagerConfig, error) { } // New returns a share manager implementation that verifies against a Nextcloud backend. -func New(m map[string]interface{}) (share.Repository, error) { +func New(ctx context.Context, m map[string]interface{}) (share.Repository, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/ocm/share/repository/nextcloud/nextcloud_test.go b/pkg/ocm/share/repository/nextcloud/nextcloud_test.go index b77aeaecdf..1f53dad8bc 100644 --- a/pkg/ocm/share/repository/nextcloud/nextcloud_test.go +++ b/pkg/ocm/share/repository/nextcloud/nextcloud_test.go @@ -113,7 +113,7 @@ var _ = Describe("Nextcloud", func() { Describe("New", func() { It("returns a new instance", func() { - _, err := nextcloud.New(options) + _, err := nextcloud.New(context.Background(), options) Expect(err).ToNot(HaveOccurred()) }) }) diff --git a/pkg/ocm/share/repository/registry/registry.go b/pkg/ocm/share/repository/registry/registry.go index 38ac750b24..7a43771c46 100644 --- a/pkg/ocm/share/repository/registry/registry.go +++ b/pkg/ocm/share/repository/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/ocm/share" +import ( + "context" + + "github.com/cs3org/reva/pkg/ocm/share" +) // NewFunc is the function that share repositories // should register at init time. -type NewFunc func(map[string]interface{}) (share.Repository, error) +type NewFunc func(context.Context, map[string]interface{}) (share.Repository, error) // NewFuncs is a map containing all the registered share repositories. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/ocm/share/repository/sql/sql.go b/pkg/ocm/share/repository/sql/sql.go index ec26261d28..980e08d55d 100644 --- a/pkg/ocm/share/repository/sql/sql.go +++ b/pkg/ocm/share/repository/sql/sql.go @@ -45,12 +45,12 @@ func init() { } // New creates a Repository with a SQL driver. -func New(c map[string]interface{}) (share.Repository, error) { +func New(ctx context.Context, c map[string]interface{}) (share.Repository, error) { conf, err := parseConfig(c) if err != nil { return nil, err } - return NewFromConfig(conf) + return NewFromConfig(ctx, conf) } type mgr struct { @@ -60,7 +60,7 @@ type mgr struct { } // NewFromConfig creates a Repository with a SQL driver using the given config. -func NewFromConfig(conf *config) (share.Repository, error) { +func NewFromConfig(ctx context.Context, conf *config) (share.Repository, error) { if conf.now == nil { conf.now = time.Now } diff --git a/pkg/ocm/share/repository/sql/sql_test.go b/pkg/ocm/share/repository/sql/sql_test.go index 207bd82b35..33a2eb53d7 100644 --- a/pkg/ocm/share/repository/sql/sql_test.go +++ b/pkg/ocm/share/repository/sql/sql_test.go @@ -615,7 +615,7 @@ func TestGetShare(t *testing.T) { _, port, cleanup := startDatabase(ctx, tables) t.Cleanup(cleanup) - r, err := New(map[string]interface{}{ + r, err := New(context.Background(), map[string]interface{}{ "db_username": "root", "db_password": "", "db_address": fmt.Sprintf("%s:%d", address, port), @@ -1054,7 +1054,7 @@ func TestListShares(t *testing.T) { _, port, cleanup := startDatabase(ctx, tables) t.Cleanup(cleanup) - r, err := New(map[string]interface{}{ + r, err := New(context.Background(), map[string]interface{}{ "db_username": "root", "db_password": "", "db_address": fmt.Sprintf("%s:%d", address, port), @@ -1246,7 +1246,7 @@ func TestStoreShare(t *testing.T) { engine, port, cleanup := startDatabase(ctx, tables) t.Cleanup(cleanup) - r, err := New(map[string]interface{}{ + r, err := New(context.Background(), map[string]interface{}{ "db_username": "root", "db_password": "", "db_address": fmt.Sprintf("%s:%d", address, port), @@ -1517,7 +1517,7 @@ func TestUpdateShare(t *testing.T) { engine, port, cleanup := startDatabase(ctx, tables) t.Cleanup(cleanup) - r, err := NewFromConfig( + r, err := NewFromConfig(ctx, &config{ DBUsername: "root", DBPassword: "", @@ -1687,7 +1687,7 @@ func TestGetReceivedShare(t *testing.T) { _, port, cleanup := startDatabase(ctx, tables) t.Cleanup(cleanup) - r, err := New(map[string]interface{}{ + r, err := New(context.Background(), map[string]interface{}{ "db_username": "root", "db_password": "", "db_address": fmt.Sprintf("%s:%d", address, port), @@ -1799,7 +1799,7 @@ func TestUpdateReceivedShare(t *testing.T) { engine, port, cleanup := startDatabase(ctx, tables) t.Cleanup(cleanup) - r, err := NewFromConfig(&config{ + r, err := NewFromConfig(ctx, &config{ DBUsername: "root", DBPassword: "", DBAddress: fmt.Sprintf("%s:%d", address, port), @@ -2041,7 +2041,7 @@ func TestListReceviedShares(t *testing.T) { _, port, cleanup := startDatabase(ctx, tables) t.Cleanup(cleanup) - r, err := New(map[string]interface{}{ + r, err := New(context.Background(), map[string]interface{}{ "db_username": "root", "db_password": "", "db_address": fmt.Sprintf("%s:%d", address, port), @@ -2189,7 +2189,7 @@ func TestStoreReceivedShare(t *testing.T) { engine, port, cleanup := startDatabase(ctx, tables) t.Cleanup(cleanup) - r, err := New(map[string]interface{}{ + r, err := New(context.Background(), map[string]interface{}{ "db_username": "root", "db_password": "", "db_address": fmt.Sprintf("%s:%d", address, port), diff --git a/pkg/ocm/storage/outcoming/ocm.go b/pkg/ocm/storage/outcoming/ocm.go index 5c179d4e16..50828f40b4 100644 --- a/pkg/ocm/storage/outcoming/ocm.go +++ b/pkg/ocm/storage/outcoming/ocm.go @@ -72,7 +72,7 @@ func (c *config) init() { } // New creates an OCM storage driver. -func New(c map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, c map[string]interface{}) (storage.FS, error) { conf, err := parseConfig(c) if err != nil { return nil, errors.Wrapf(err, "error decoding config") diff --git a/pkg/ocm/storage/received/ocm.go b/pkg/ocm/storage/received/ocm.go index 2a865e1508..405f5ab2fc 100644 --- a/pkg/ocm/storage/received/ocm.go +++ b/pkg/ocm/storage/received/ocm.go @@ -69,7 +69,7 @@ func (c *config) init() { } // New creates an OCM storage driver. -func New(c map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, c map[string]interface{}) (storage.FS, error) { conf, err := parseConfig(c) if err != nil { return nil, errors.Wrapf(err, "error decoding config") diff --git a/pkg/permission/manager/demo/demo.go b/pkg/permission/manager/demo/demo.go index cbef4d5bfc..371f9748e8 100644 --- a/pkg/permission/manager/demo/demo.go +++ b/pkg/permission/manager/demo/demo.go @@ -19,6 +19,8 @@ package demo import ( + "context" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/permission" "github.com/cs3org/reva/pkg/permission/manager/registry" @@ -29,7 +31,7 @@ func init() { } // New returns a new demo permission manager. -func New(c map[string]interface{}) (permission.Manager, error) { +func New(_ context.Context, _ map[string]interface{}) (permission.Manager, error) { return manager{}, nil } diff --git a/pkg/permission/manager/registry/registry.go b/pkg/permission/manager/registry/registry.go index 8ef15a042d..b0478ff10e 100644 --- a/pkg/permission/manager/registry/registry.go +++ b/pkg/permission/manager/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/permission" +import ( + "context" + + "github.com/cs3org/reva/pkg/permission" +) // NewFunc is the function that permission managers // should register at init time. -type NewFunc func(map[string]interface{}) (permission.Manager, error) +type NewFunc func(context.Context, map[string]interface{}) (permission.Manager, error) // NewFuncs is a map containing all the registered share managers. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/preferences/memory/memory.go b/pkg/preferences/memory/memory.go index 72cccf5789..10ff617fa3 100644 --- a/pkg/preferences/memory/memory.go +++ b/pkg/preferences/memory/memory.go @@ -38,7 +38,7 @@ type mgr struct { } // New returns an instance of the in-memory preferences manager. -func New(m map[string]interface{}) (preferences.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (preferences.Manager, error) { return &mgr{keys: make(map[string]map[string]string)}, nil } diff --git a/pkg/preferences/registry/registry.go b/pkg/preferences/registry/registry.go index c40277ba22..15dad6fd7e 100644 --- a/pkg/preferences/registry/registry.go +++ b/pkg/preferences/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/preferences" +import ( + "context" + + "github.com/cs3org/reva/pkg/preferences" +) // NewFunc is the function that preferences implementations // should register at init time. -type NewFunc func(map[string]interface{}) (preferences.Manager, error) +type NewFunc func(context.Context, map[string]interface{}) (preferences.Manager, error) // NewFuncs is a map containing all the registered preferences implementations. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/publicshare/manager/json/json.go b/pkg/publicshare/manager/json/json.go index 11419b474f..b8a0a00195 100644 --- a/pkg/publicshare/manager/json/json.go +++ b/pkg/publicshare/manager/json/json.go @@ -49,7 +49,7 @@ func init() { } // New returns a new filesystem public shares manager. -func New(c map[string]interface{}) (publicshare.Manager, error) { +func New(ctx context.Context, c map[string]interface{}) (publicshare.Manager, error) { conf := &config{} if err := mapstructure.Decode(c, conf); err != nil { return nil, err diff --git a/pkg/publicshare/manager/memory/memory.go b/pkg/publicshare/manager/memory/memory.go index 8af0b16bcd..cff253f0a3 100644 --- a/pkg/publicshare/manager/memory/memory.go +++ b/pkg/publicshare/manager/memory/memory.go @@ -43,7 +43,7 @@ func init() { } // New returns a new memory manager. -func New(c map[string]interface{}) (publicshare.Manager, error) { +func New(_ context.Context, c map[string]interface{}) (publicshare.Manager, error) { return &manager{ shares: sync.Map{}, }, nil diff --git a/pkg/publicshare/manager/registry/registry.go b/pkg/publicshare/manager/registry/registry.go index 77bbb2a640..aa99b0d94a 100644 --- a/pkg/publicshare/manager/registry/registry.go +++ b/pkg/publicshare/manager/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/publicshare" +import ( + "context" + + "github.com/cs3org/reva/pkg/publicshare" +) // NewFunc is the function that share managers // should register at init time. -type NewFunc func(map[string]interface{}) (publicshare.Manager, error) +type NewFunc func(context.Context, map[string]interface{}) (publicshare.Manager, error) // NewFuncs is a map containing all the registered share managers. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/rgrpc/rgrpc.go b/pkg/rgrpc/rgrpc.go index 5508bcb36a..96cf258fc1 100644 --- a/pkg/rgrpc/rgrpc.go +++ b/pkg/rgrpc/rgrpc.go @@ -19,11 +19,13 @@ package rgrpc import ( + "context" "fmt" "io" "net" "github.com/cs3org/reva/cmd/revad/pkg/config" + "github.com/cs3org/reva/pkg/appctx" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" "github.com/pkg/errors" "github.com/rs/zerolog" @@ -63,7 +65,7 @@ func Register(name string, newFunc NewService) { // NewService is the function that gRPC services need to register at init time. // It returns an io.Closer to close the service and a list of service endpoints that need to be unprotected. -type NewService func(conf map[string]interface{}) (Service, error) +type NewService func(context.Context, map[string]any) (Service, error) // Service represents a grpc service. type Service interface { @@ -85,7 +87,7 @@ type Server struct { services map[string]Service } -func InitServices(services map[string]config.ServicesConfig) (map[string]Service, error) { +func InitServices(ctx context.Context, services map[string]config.ServicesConfig) (map[string]Service, error) { s := make(map[string]Service) for name, cfg := range services { new, ok := Services[name] @@ -95,7 +97,9 @@ func InitServices(services map[string]config.ServicesConfig) (map[string]Service if cfg.DriversNumber() > 1 { return nil, fmt.Errorf("rgrp: service %s cannot have more than one driver in same server", name) } - svc, err := new(cfg[0].Config) + log := appctx.GetLogger(ctx).With().Str("service", name).Logger() + ctx := appctx.WithLogger(ctx, &log) + svc, err := new(ctx, cfg[0].Config) if err != nil { return nil, errors.Wrapf(err, "rgrpc: grpc service %s could not be started,", name) } diff --git a/pkg/rhttp/datatx/manager/registry/registry.go b/pkg/rhttp/datatx/manager/registry/registry.go index 683bb2e10a..6dc1fd2b6b 100644 --- a/pkg/rhttp/datatx/manager/registry/registry.go +++ b/pkg/rhttp/datatx/manager/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/rhttp/datatx" +import ( + "context" + + "github.com/cs3org/reva/pkg/rhttp/datatx" +) // NewFunc is the function that data transfer implementations // should register at init time. -type NewFunc func(map[string]interface{}) (datatx.DataTX, error) +type NewFunc func(context.Context, map[string]interface{}) (datatx.DataTX, error) // NewFuncs is a map containing all the registered data transfers. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/rhttp/datatx/manager/simple/simple.go b/pkg/rhttp/datatx/manager/simple/simple.go index c32aa43870..68ce1ea16b 100644 --- a/pkg/rhttp/datatx/manager/simple/simple.go +++ b/pkg/rhttp/datatx/manager/simple/simple.go @@ -19,6 +19,7 @@ package simple import ( + "context" "net/http" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -52,7 +53,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns a datatx manager implementation that relies on HTTP PUT/GET. -func New(m map[string]interface{}) (datatx.DataTX, error) { +func New(ctx context.Context, m map[string]interface{}) (datatx.DataTX, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/rhttp/datatx/manager/spaces/spaces.go b/pkg/rhttp/datatx/manager/spaces/spaces.go index 9bd1f11606..ae296b1131 100644 --- a/pkg/rhttp/datatx/manager/spaces/spaces.go +++ b/pkg/rhttp/datatx/manager/spaces/spaces.go @@ -19,6 +19,7 @@ package spaces import ( + "context" "net/http" "path" "strings" @@ -56,7 +57,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns a datatx manager implementation that relies on HTTP PUT/GET. -func New(m map[string]interface{}) (datatx.DataTX, error) { +func New(ctx context.Context, m map[string]interface{}) (datatx.DataTX, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/rhttp/datatx/manager/tus/tus.go b/pkg/rhttp/datatx/manager/tus/tus.go index a8284443d5..4505c6f76c 100644 --- a/pkg/rhttp/datatx/manager/tus/tus.go +++ b/pkg/rhttp/datatx/manager/tus/tus.go @@ -19,6 +19,7 @@ package tus import ( + "context" "net/http" "github.com/cs3org/reva/pkg/errtypes" @@ -51,7 +52,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns a datatx manager implementation that relies on HTTP PUT/GET. -func New(m map[string]interface{}) (datatx.DataTX, error) { +func New(ctx context.Context, m map[string]interface{}) (datatx.DataTX, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/rhttp/global/global.go b/pkg/rhttp/global/global.go index 8d90b5cf27..41fb45e6a2 100644 --- a/pkg/rhttp/global/global.go +++ b/pkg/rhttp/global/global.go @@ -19,9 +19,8 @@ package global import ( + "context" "net/http" - - "github.com/rs/zerolog" ) // NewMiddlewares contains all the registered new middleware functions. @@ -47,7 +46,7 @@ func Register(name string, newFunc NewService) { } // NewService is the function that HTTP services need to register at init time. -type NewService func(conf map[string]interface{}, log *zerolog.Logger) (Service, error) +type NewService func(context.Context, map[string]interface{}) (Service, error) // Service represents a HTTP service. type Service interface { diff --git a/pkg/rhttp/rhttp.go b/pkg/rhttp/rhttp.go index e5d796f917..37fd52dd51 100644 --- a/pkg/rhttp/rhttp.go +++ b/pkg/rhttp/rhttp.go @@ -28,6 +28,7 @@ import ( "time" "github.com/cs3org/reva/cmd/revad/pkg/config" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" rtrace "github.com/cs3org/reva/pkg/trace" "github.com/pkg/errors" @@ -62,7 +63,7 @@ func WithLogger(log zerolog.Logger) Config { } } -func InitServices(services map[string]config.ServicesConfig, log *zerolog.Logger) (map[string]global.Service, error) { +func InitServices(ctx context.Context, services map[string]config.ServicesConfig) (map[string]global.Service, error) { s := make(map[string]global.Service) for name, cfg := range services { new, ok := global.Services[name] @@ -72,8 +73,9 @@ func InitServices(services map[string]config.ServicesConfig, log *zerolog.Logger if cfg.DriversNumber() > 1 { return nil, fmt.Errorf("service %s cannot have more than one driver in the same server", name) } - log := log.With().Str("service", name).Logger() - svc, err := new(cfg[0].Config, &log) + log := appctx.GetLogger(ctx).With().Str("service", name).Logger() + ctx := appctx.WithLogger(ctx, &log) + svc, err := new(ctx, cfg[0].Config) if err != nil { return nil, errors.Wrapf(err, "http service %s could not be started", name) } diff --git a/pkg/share/manager/json/json.go b/pkg/share/manager/json/json.go index 40a0ee63e6..a301876e6b 100644 --- a/pkg/share/manager/json/json.go +++ b/pkg/share/manager/json/json.go @@ -45,7 +45,7 @@ func init() { } // New returns a new mgr. -func New(m map[string]interface{}) (share.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (share.Manager, error) { c, err := parseConfig(m) if err != nil { err = errors.Wrap(err, "error creating a new manager") diff --git a/pkg/share/manager/memory/memory.go b/pkg/share/manager/memory/memory.go index fd7b9a074e..2646744a74 100644 --- a/pkg/share/manager/memory/memory.go +++ b/pkg/share/manager/memory/memory.go @@ -44,7 +44,7 @@ func init() { } // New returns a new manager. -func New(c map[string]interface{}) (share.Manager, error) { +func New(ctx context.Context, c map[string]interface{}) (share.Manager, error) { state := map[string]map[*collaboration.ShareId]collaboration.ShareState{} return &manager{ shareState: state, diff --git a/pkg/share/manager/registry/registry.go b/pkg/share/manager/registry/registry.go index 8fa161c64f..8eda3f3c34 100644 --- a/pkg/share/manager/registry/registry.go +++ b/pkg/share/manager/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/share" +import ( + "context" + + "github.com/cs3org/reva/pkg/share" +) // NewFunc is the function that share managers // should register at init time. -type NewFunc func(map[string]interface{}) (share.Manager, error) +type NewFunc func(context.Context, map[string]interface{}) (share.Manager, error) // NewFuncs is a map containing all the registered share managers. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/share/manager/sql/sql.go b/pkg/share/manager/sql/sql.go index ad871ccf59..13300d80ef 100644 --- a/pkg/share/manager/sql/sql.go +++ b/pkg/share/manager/sql/sql.go @@ -70,7 +70,7 @@ type mgr struct { } // NewMysql returns a new share manager connection to a mysql database. -func NewMysql(m map[string]interface{}) (share.Manager, error) { +func NewMysql(ctx context.Context, m map[string]interface{}) (share.Manager, error) { c, err := parseConfig(m) if err != nil { err = errors.Wrap(err, "error creating a new manager") diff --git a/pkg/storage/fs/cback/cback.go b/pkg/storage/fs/cback/cback.go index aab979dd1a..4c4607eaae 100644 --- a/pkg/storage/fs/cback/cback.go +++ b/pkg/storage/fs/cback/cback.go @@ -50,7 +50,7 @@ func init() { // New returns an implementation to the storage.FS interface that talks to // cback. -func New(m map[string]interface{}) (fs storage.FS, err error) { +func New(ctx context.Context, m map[string]interface{}) (fs storage.FS, err error) { c := &Options{} if err = mapstructure.Decode(m, c); err != nil { return nil, errors.Wrap(err, "Error Decoding Configuration") diff --git a/pkg/storage/fs/cephfs/cephfs.go b/pkg/storage/fs/cephfs/cephfs.go index e710192ca9..9132bab87b 100644 --- a/pkg/storage/fs/cephfs/cephfs.go +++ b/pkg/storage/fs/cephfs/cephfs.go @@ -64,7 +64,7 @@ func init() { // New returns an implementation to of the storage.FS interface that talk to // a ceph filesystem. -func New(m map[string]interface{}) (fs storage.FS, err error) { +func New(ctx context.Context, m map[string]interface{}) (fs storage.FS, err error) { c := &Options{} if err = mapstructure.Decode(m, c); err != nil { return nil, errors.Wrap(err, "error decoding conf") diff --git a/pkg/storage/fs/cephfs/unsupported.go b/pkg/storage/fs/cephfs/unsupported.go index b0fa934d28..9b25894299 100644 --- a/pkg/storage/fs/cephfs/unsupported.go +++ b/pkg/storage/fs/cephfs/unsupported.go @@ -22,6 +22,8 @@ package cephfs import ( + "context" + "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/pkg/errors" @@ -33,6 +35,6 @@ func init() { // New returns an implementation to of the storage.FS interface that talk to // a ceph filesystem. -func New(m map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { return nil, errors.New("cephfs: revad was compiled without CephFS support") } diff --git a/pkg/storage/fs/eos/eos.go b/pkg/storage/fs/eos/eos.go index e83a638f00..3847c66b6d 100644 --- a/pkg/storage/fs/eos/eos.go +++ b/pkg/storage/fs/eos/eos.go @@ -19,6 +19,8 @@ package eos import ( + "context" + "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/eosfs" @@ -46,11 +48,11 @@ func parseConfig(m map[string]interface{}) (*eosfs.Config, error) { } // New returns a new implementation of the storage.FS interface that connects to EOS. -func New(m map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { c, err := parseConfig(m) if err != nil { return nil, err } - return eosfs.NewEOSFS(c) + return eosfs.NewEOSFS(ctx, c) } diff --git a/pkg/storage/fs/eosgrpc/eosgrpc.go b/pkg/storage/fs/eosgrpc/eosgrpc.go index 52ba856b8c..3014335cb1 100644 --- a/pkg/storage/fs/eosgrpc/eosgrpc.go +++ b/pkg/storage/fs/eosgrpc/eosgrpc.go @@ -19,6 +19,8 @@ package eosgrpc import ( + "context" + "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/eosfs" @@ -46,12 +48,12 @@ func parseConfig(m map[string]interface{}) (*eosfs.Config, error) { } // New returns a new implementation of the storage.FS interface that connects to EOS. -func New(m map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { c, err := parseConfig(m) if err != nil { return nil, err } c.UseGRPC = true - return eosfs.NewEOSFS(c) + return eosfs.NewEOSFS(ctx, c) } diff --git a/pkg/storage/fs/eosgrpchome/eosgrpchome.go b/pkg/storage/fs/eosgrpchome/eosgrpchome.go index 2d19ca62fe..4b07e604fa 100644 --- a/pkg/storage/fs/eosgrpchome/eosgrpchome.go +++ b/pkg/storage/fs/eosgrpchome/eosgrpchome.go @@ -19,6 +19,8 @@ package eosgrpchome import ( + "context" + "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/eosfs" @@ -46,7 +48,7 @@ func parseConfig(m map[string]interface{}) (*eosfs.Config, error) { } // New returns a new implementation of the storage.FS interface that connects to EOS. -func New(m map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { c, err := parseConfig(m) if err != nil { return nil, err @@ -54,5 +56,5 @@ func New(m map[string]interface{}) (storage.FS, error) { c.UseGRPC = true c.EnableHome = true - return eosfs.NewEOSFS(c) + return eosfs.NewEOSFS(ctx, c) } diff --git a/pkg/storage/fs/eoshome/eoshome.go b/pkg/storage/fs/eoshome/eoshome.go index b6449961ae..6fcf4abe21 100644 --- a/pkg/storage/fs/eoshome/eoshome.go +++ b/pkg/storage/fs/eoshome/eoshome.go @@ -19,6 +19,8 @@ package eoshome import ( + "context" + "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/eosfs" @@ -46,12 +48,12 @@ func parseConfig(m map[string]interface{}) (*eosfs.Config, error) { } // New returns a new implementation of the storage.FS interface that connects to EOS. -func New(m map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { c, err := parseConfig(m) if err != nil { return nil, err } c.EnableHome = true - return eosfs.NewEOSFS(c) + return eosfs.NewEOSFS(ctx, c) } diff --git a/pkg/storage/fs/local/local.go b/pkg/storage/fs/local/local.go index fb057fbbef..a5b42cd241 100644 --- a/pkg/storage/fs/local/local.go +++ b/pkg/storage/fs/local/local.go @@ -19,6 +19,8 @@ package local import ( + "context" + "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/localfs" @@ -46,7 +48,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { // New returns an implementation to of the storage.FS interface that talks to // a local filesystem with user homes disabled. -func New(m map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/storage/fs/localhome/localhome.go b/pkg/storage/fs/localhome/localhome.go index 06676d2256..24cb3ffa1b 100644 --- a/pkg/storage/fs/localhome/localhome.go +++ b/pkg/storage/fs/localhome/localhome.go @@ -19,6 +19,8 @@ package localhome import ( + "context" + "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/localfs" @@ -47,7 +49,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { // New returns an implementation to of the storage.FS interface that talks to // a local filesystem with user homes. -func New(m map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/storage/fs/nextcloud/nextcloud.go b/pkg/storage/fs/nextcloud/nextcloud.go index 95fffc3750..00cda53858 100644 --- a/pkg/storage/fs/nextcloud/nextcloud.go +++ b/pkg/storage/fs/nextcloud/nextcloud.go @@ -69,7 +69,7 @@ func parseConfig(m map[string]interface{}) (*StorageDriverConfig, error) { // New returns an implementation to of the storage.FS interface that talks to // a Nextcloud instance over http. -func New(m map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { conf, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/storage/fs/nextcloud/nextcloud_test.go b/pkg/storage/fs/nextcloud/nextcloud_test.go index e194112355..cf64cf1e4e 100644 --- a/pkg/storage/fs/nextcloud/nextcloud_test.go +++ b/pkg/storage/fs/nextcloud/nextcloud_test.go @@ -115,7 +115,7 @@ var _ = Describe("Nextcloud", func() { Describe("New", func() { It("returns a new instance", func() { - _, err := nextcloud.New(options) + _, err := nextcloud.New(context.Background(), options) Expect(err).ToNot(HaveOccurred()) }) }) diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go index 3f480b1202..252c351bc2 100644 --- a/pkg/storage/fs/ocis/ocis.go +++ b/pkg/storage/fs/ocis/ocis.go @@ -19,6 +19,7 @@ package ocis import ( + "context" "path" "github.com/cs3org/reva/pkg/storage" @@ -34,7 +35,7 @@ func init() { // New returns an implementation to of the storage.FS interface that talk to // a local filesystem. -func New(m map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { o, err := options.New(m) if err != nil { return nil, err diff --git a/pkg/storage/fs/ocis/ocis_test.go b/pkg/storage/fs/ocis/ocis_test.go index 44685ec99a..be7e255882 100644 --- a/pkg/storage/fs/ocis/ocis_test.go +++ b/pkg/storage/fs/ocis/ocis_test.go @@ -19,6 +19,7 @@ package ocis_test import ( + "context" "os" "github.com/cs3org/reva/pkg/storage/fs/ocis" @@ -52,7 +53,7 @@ var _ = Describe("Ocis", func() { Describe("New", func() { It("returns a new instance", func() { - _, err := ocis.New(options) + _, err := ocis.New(context.Background(), options) Expect(err).ToNot(HaveOccurred()) }) }) diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index ba4cf93c39..336109c72b 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -158,7 +158,7 @@ func (c *config) init(m map[string]interface{}) { // New returns an implementation to of the storage.FS interface that talk to // a local filesystem. -func New(m map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/storage/fs/owncloudsql/owncloudsql.go b/pkg/storage/fs/owncloudsql/owncloudsql.go index 0586ba2411..dfbb051a84 100644 --- a/pkg/storage/fs/owncloudsql/owncloudsql.go +++ b/pkg/storage/fs/owncloudsql/owncloudsql.go @@ -151,7 +151,7 @@ func (c *config) init(m map[string]interface{}) { // New returns an implementation to of the storage.FS interface that talk to // a local filesystem. -func New(m map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/storage/fs/registry/registry.go b/pkg/storage/fs/registry/registry.go index 6966b8094e..0e791278ad 100644 --- a/pkg/storage/fs/registry/registry.go +++ b/pkg/storage/fs/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/storage" +import ( + "context" + + "github.com/cs3org/reva/pkg/storage" +) // NewFunc is the function that storage implementations // should register at init time. -type NewFunc func(map[string]interface{}) (storage.FS, error) +type NewFunc func(context.Context, map[string]interface{}) (storage.FS, error) // NewFuncs is a map containing all the registered storage backends. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/storage/fs/s3/s3.go b/pkg/storage/fs/s3/s3.go index dc33533c3c..6783a7cdba 100644 --- a/pkg/storage/fs/s3/s3.go +++ b/pkg/storage/fs/s3/s3.go @@ -69,7 +69,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { // New returns an implementation to of the storage.FS interface that talk to // a s3 api. -func New(m map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/storage/fs/s3ng/s3ng.go b/pkg/storage/fs/s3ng/s3ng.go index 9d34bc5428..bda1bac61e 100644 --- a/pkg/storage/fs/s3ng/s3ng.go +++ b/pkg/storage/fs/s3ng/s3ng.go @@ -19,6 +19,7 @@ package s3ng import ( + "context" "fmt" "github.com/cs3org/reva/pkg/storage" @@ -33,7 +34,7 @@ func init() { // New returns an implementation to of the storage.FS interface that talk to // a local filesystem. -func New(m map[string]interface{}) (storage.FS, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { o, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/storage/fs/s3ng/s3ng_test.go b/pkg/storage/fs/s3ng/s3ng_test.go index 4346da78f4..107b25c0f7 100644 --- a/pkg/storage/fs/s3ng/s3ng_test.go +++ b/pkg/storage/fs/s3ng/s3ng_test.go @@ -19,6 +19,7 @@ package s3ng_test import ( + "context" "os" "github.com/cs3org/reva/pkg/storage/fs/s3ng" @@ -57,12 +58,12 @@ var _ = Describe("S3ng", func() { Describe("New", func() { It("fails on missing s3 configuration", func() { - _, err := s3ng.New(map[string]interface{}{}) + _, err := s3ng.New(context.Background(), map[string]interface{}{}) Expect(err).To(MatchError("s3 configuration incomplete")) }) It("works with complete configuration", func() { - _, err := s3ng.New(options) + _, err := s3ng.New(context.Background(), options) Expect(err).ToNot(HaveOccurred()) }) }) diff --git a/pkg/storage/registry/registry/registry.go b/pkg/storage/registry/registry/registry.go index 3619f4c39e..bd381d4da2 100644 --- a/pkg/storage/registry/registry/registry.go +++ b/pkg/storage/registry/registry/registry.go @@ -18,11 +18,15 @@ package registry -import "github.com/cs3org/reva/pkg/storage" +import ( + "context" + + "github.com/cs3org/reva/pkg/storage" +) // NewFunc is the function that storage broker implementations // should register at init time. -type NewFunc func(map[string]interface{}) (storage.Registry, error) +type NewFunc func(context.Context, map[string]interface{}) (storage.Registry, error) // NewFuncs is a map containing all the registered storage backends. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/storage/registry/static/static.go b/pkg/storage/registry/static/static.go index 74279eca46..70382c1808 100644 --- a/pkg/storage/registry/static/static.go +++ b/pkg/storage/registry/static/static.go @@ -80,7 +80,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { // New returns an implementation of the storage.Registry interface that // redirects requests to corresponding storage drivers. -func New(m map[string]interface{}) (storage.Registry, error) { +func New(ctx context.Context, m map[string]interface{}) (storage.Registry, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/storage/registry/static/static_test.go b/pkg/storage/registry/static/static_test.go index 88ff901a1d..2f6639511e 100644 --- a/pkg/storage/registry/static/static_test.go +++ b/pkg/storage/registry/static/static_test.go @@ -34,7 +34,7 @@ var _ = Describe("Static", func() { totalProviders, rootProviders, eosProviders := 33, 31, 29 - handler, err := static.New(map[string]interface{}{ + handler, err := static.New(context.Background(), map[string]interface{}{ "home_provider": "/home", "rules": map[string]interface{}{ "/home": map[string]interface{}{ diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index f390174b3f..967ccb3893 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -161,7 +161,7 @@ type eosfs struct { } // NewEOSFS returns a storage.FS interface implementation that connects to an EOS instance. -func NewEOSFS(c *Config) (storage.FS, error) { +func NewEOSFS(ctx context.Context, c *Config) (storage.FS, error) { c.init() // bail out if keytab is not found. diff --git a/pkg/user/manager/demo/demo.go b/pkg/user/manager/demo/demo.go index 1632c1529f..caad9f7417 100644 --- a/pkg/user/manager/demo/demo.go +++ b/pkg/user/manager/demo/demo.go @@ -39,7 +39,7 @@ type manager struct { } // New returns a new user manager. -func New(m map[string]interface{}) (user.Manager, error) { +func New(_ context.Context, m map[string]interface{}) (user.Manager, error) { mgr := &manager{} err := mgr.Configure(m) if err != nil { diff --git a/pkg/user/manager/demo/demo_test.go b/pkg/user/manager/demo/demo_test.go index f03f02bfa0..73b736e49f 100644 --- a/pkg/user/manager/demo/demo_test.go +++ b/pkg/user/manager/demo/demo_test.go @@ -31,7 +31,7 @@ var ctx = context.Background() func TestUserManager(t *testing.T) { // get manager - manager, _ := New(nil) + manager, _ := New(context.TODO(), nil) // setup test data uidEinstein := &userpb.UserId{Idp: "http://localhost:9998", OpaqueId: "4c510ada-c86b-4815-8820-42cdf82c3d51", Type: userpb.UserType_USER_TYPE_PRIMARY} diff --git a/pkg/user/manager/json/json.go b/pkg/user/manager/json/json.go index 2ce2bef6e7..642fa038e4 100644 --- a/pkg/user/manager/json/json.go +++ b/pkg/user/manager/json/json.go @@ -63,7 +63,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns a user manager implementation that reads a json file to provide user metadata. -func New(m map[string]interface{}) (user.Manager, error) { +func New(_ context.Context, m map[string]interface{}) (user.Manager, error) { mgr := &manager{} err := mgr.Configure(m) if err != nil { diff --git a/pkg/user/manager/json/json_test.go b/pkg/user/manager/json/json_test.go index 16760085f0..904c291ec1 100644 --- a/pkg/user/manager/json/json_test.go +++ b/pkg/user/manager/json/json_test.go @@ -57,7 +57,7 @@ func TestUserManager(t *testing.T) { input := map[string]interface{}{ "users": file.Name(), } - _, err = New(input) + _, err = New(context.TODO(), input) if err == nil { t.Fatalf("no error (but we expected one) while get manager") } @@ -85,7 +85,7 @@ func TestUserManager(t *testing.T) { input = map[string]interface{}{ "users": file.Name(), } - manager, _ := New(input) + manager, _ := New(context.TODO(), input) // setup test data uidEinstein := &userpb.UserId{Idp: "localhost", OpaqueId: "einstein", Type: userpb.UserType_USER_TYPE_PRIMARY} diff --git a/pkg/user/manager/ldap/ldap.go b/pkg/user/manager/ldap/ldap.go index 351235d6c0..7d684a7991 100644 --- a/pkg/user/manager/ldap/ldap.go +++ b/pkg/user/manager/ldap/ldap.go @@ -101,7 +101,7 @@ func parseConfig(m map[string]interface{}) (*config, error) { } // New returns a user manager implementation that connects to a LDAP server to provide user metadata. -func New(m map[string]interface{}) (user.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (user.Manager, error) { mgr := &manager{} err := mgr.Configure(m) if err != nil { diff --git a/pkg/user/manager/ldap/ldap_test.go b/pkg/user/manager/ldap/ldap_test.go index f7f5644289..2866a6a337 100644 --- a/pkg/user/manager/ldap/ldap_test.go +++ b/pkg/user/manager/ldap/ldap_test.go @@ -19,12 +19,13 @@ package ldap import ( + "context" "testing" ) func TestUserManager(t *testing.T) { // negative test for parseConfig - _, err := New(map[string]interface{}{"hostname": 42}) + _, err := New(context.Background(), map[string]interface{}{"hostname": 42}) if err == nil { t.Fatal("expected error but got none") } @@ -64,7 +65,7 @@ func TestUserManager(t *testing.T) { } // positive tests for New - _, err = New(map[string]interface{}{}) + _, err = New(context.Background(), map[string]interface{}{}) if err != nil { t.Fatalf(err.Error()) } diff --git a/pkg/user/manager/nextcloud/nextcloud.go b/pkg/user/manager/nextcloud/nextcloud.go index b3524f61d8..274e02b18c 100644 --- a/pkg/user/manager/nextcloud/nextcloud.go +++ b/pkg/user/manager/nextcloud/nextcloud.go @@ -79,7 +79,7 @@ type Action struct { } // New returns a user manager implementation that reads a json file to provide user metadata. -func New(m map[string]interface{}) (user.Manager, error) { +func New(ctx context.Context, m map[string]interface{}) (user.Manager, error) { c, err := parseConfig(m) if err != nil { return nil, err diff --git a/pkg/user/manager/nextcloud/nextcloud_test.go b/pkg/user/manager/nextcloud/nextcloud_test.go index ddc3f47429..5b9190323a 100644 --- a/pkg/user/manager/nextcloud/nextcloud_test.go +++ b/pkg/user/manager/nextcloud/nextcloud_test.go @@ -113,7 +113,7 @@ var _ = Describe("Nextcloud", func() { Describe("New", func() { It("returns a new instance", func() { - _, err := nextcloud.New(options) + _, err := nextcloud.New(context.Background(), options) Expect(err).ToNot(HaveOccurred()) }) }) diff --git a/pkg/user/manager/owncloudsql/owncloudsql.go b/pkg/user/manager/owncloudsql/owncloudsql.go index 8774a5b2e5..666119ea7b 100644 --- a/pkg/user/manager/owncloudsql/owncloudsql.go +++ b/pkg/user/manager/owncloudsql/owncloudsql.go @@ -59,7 +59,7 @@ type config struct { } // NewMysql returns a new user manager connection to an owncloud mysql database. -func NewMysql(m map[string]interface{}) (user.Manager, error) { +func NewMysql(ctx context.Context, m map[string]interface{}) (user.Manager, error) { mgr := &manager{} err := mgr.Configure(m) if err != nil { diff --git a/pkg/user/manager/registry/registry.go b/pkg/user/manager/registry/registry.go index 89fc1a44e5..cce7624762 100644 --- a/pkg/user/manager/registry/registry.go +++ b/pkg/user/manager/registry/registry.go @@ -19,12 +19,14 @@ package registry import ( + "context" + "github.com/cs3org/reva/pkg/user" ) // NewFunc is the function that user managers // should register at init time. -type NewFunc func(map[string]interface{}) (user.Manager, error) +type NewFunc func(context.Context, map[string]interface{}) (user.Manager, error) // NewFuncs is a map containing all the registered user managers. var NewFuncs = map[string]NewFunc{} diff --git a/tests/integration/grpc/storageprovider_test.go b/tests/integration/grpc/storageprovider_test.go index ae705b4e7f..14e2de031e 100644 --- a/tests/integration/grpc/storageprovider_test.go +++ b/tests/integration/grpc/storageprovider_test.go @@ -498,7 +498,7 @@ var _ = Describe("storage providers", func() { Context("with an existing file /versioned_file", func() { JustBeforeEach(func() { - fs, err := ocis.New(map[string]interface{}{ + fs, err := ocis.New(ctx, map[string]interface{}{ "root": revads["storage"].TmpRoot, "enable_home": true, }) @@ -556,7 +556,7 @@ var _ = Describe("storage providers", func() { Context("with an existing file /versioned_file", func() { JustBeforeEach(func() { - fs, err := ocis.New(map[string]interface{}{ + fs, err := ocis.New(ctx, map[string]interface{}{ "root": revads["storage"].TmpRoot, "enable_home": true, }) @@ -622,7 +622,7 @@ var _ = Describe("storage providers", func() { Context("with an existing file /versioned_file", func() { JustBeforeEach(func() { - fs, err := owncloud.New(map[string]interface{}{ + fs, err := owncloud.New(ctx, map[string]interface{}{ "datadirectory": revads["storage"].TmpRoot, "userprovidersvc": revads["users"].GrpcAddress, "enable_home": true, From 196631b02852db72a1af25837915a14724c962e3 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Tue, 4 Jul 2023 13:40:24 +0200 Subject: [PATCH 35/48] Temporarily exclude ceph-iscsi when building revad-ceph image (#4032) --- changelog/unreleased/exclude-ceph-iscsi.md | 7 +++++++ docker/Dockerfile.revad-ceph | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/exclude-ceph-iscsi.md diff --git a/changelog/unreleased/exclude-ceph-iscsi.md b/changelog/unreleased/exclude-ceph-iscsi.md new file mode 100644 index 0000000000..4f50d292f9 --- /dev/null +++ b/changelog/unreleased/exclude-ceph-iscsi.md @@ -0,0 +1,7 @@ +Bugfix: Temporarily exclude ceph-iscsi when building revad-ceph image + +Due to `Package ceph-iscsi-3.6-1.el8.noarch.rpm is not signed` error when +building the revad-ceph docker image, the package `ceph-iscsi` has been excluded from the dnf update. +It will be included again once the pkg will be signed again. + +https://github.com/cs3org/reva/pull/4032 diff --git a/docker/Dockerfile.revad-ceph b/docker/Dockerfile.revad-ceph index bf1d3c20ac..0c67918866 100644 --- a/docker/Dockerfile.revad-ceph +++ b/docker/Dockerfile.revad-ceph @@ -18,7 +18,7 @@ FROM quay.io/ceph/ceph:v16 -RUN dnf update -y && dnf install -y \ +RUN dnf update --exclude=ceph-iscsi -y && dnf install -y \ git \ gcc \ make \ From 22b2378acf68c3f8d70360a7e7474e909ed38ea3 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Tue, 4 Jul 2023 14:03:48 +0200 Subject: [PATCH 36/48] Dump config on SIGUSR1 (#4031) --- changelog/unreleased/dump-config.md | 7 +++++ cmd/revad/main.go | 2 ++ cmd/revad/pkg/config/config.go | 10 +++++++ cmd/revad/pkg/config/config_test.go | 2 ++ cmd/revad/runtime/runtime.go | 41 +++++++++++++++++++++++++++-- 5 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/dump-config.md diff --git a/changelog/unreleased/dump-config.md b/changelog/unreleased/dump-config.md new file mode 100644 index 0000000000..a711ebc5d9 --- /dev/null +++ b/changelog/unreleased/dump-config.md @@ -0,0 +1,7 @@ +Enhancement: Dump reva config on SIGUSR1 + +Add an option to the runtime to dump the configuration on a file +(default to `/tmp/reva-dump.toml` and configurable) when the process +receives a SIGUSR1 signal. Eventual errors are logged in the log. + +https://github.com/cs3org/reva/pull/4031 diff --git a/cmd/revad/main.go b/cmd/revad/main.go index 991073ace7..aad02a6baf 100644 --- a/cmd/revad/main.go +++ b/cmd/revad/main.go @@ -115,6 +115,8 @@ func handleSignalFlag() { signal = syscall.SIGQUIT case "stop": signal = syscall.SIGTERM + case "dump": + signal = syscall.SIGUSR1 default: fmt.Fprintf(os.Stderr, "unknown signal %q\n", *signalFlag) os.Exit(1) diff --git a/cmd/revad/pkg/config/config.go b/cmd/revad/pkg/config/config.go index 846b03374e..432b2044ab 100644 --- a/cmd/revad/pkg/config/config.go +++ b/cmd/revad/pkg/config/config.go @@ -21,6 +21,8 @@ package config import ( "fmt" "io" + "os" + "path/filepath" "reflect" "github.com/BurntSushi/toml" @@ -59,6 +61,7 @@ type Shared struct { // Core holds the core configuration. type Core struct { MaxCPUs string `key:"max_cpus" mapstructure:"max_cpus"` + ConfigDumpFile string `key:"config_dump_file" mapstructure:"config_dump_file"` TracingEnabled bool `key:"tracing_enabled" mapstructure:"tracing_enabled" default:"true"` TracingEndpoint string `key:"tracing_endpoint" mapstructure:"tracing_endpoint" default:"localhost:6831"` TracingCollector string `key:"tracing_collector" mapstructure:"tracing_collector"` @@ -78,12 +81,19 @@ type Lookuper interface { Lookup(key string) (any, error) } +func (c *Config) init() { + if c.Core.ConfigDumpFile == "" { + c.Core.ConfigDumpFile = filepath.Join(os.TempDir(), "reva-dump.toml") + } +} + // Load loads the configuration from the reader. func Load(r io.Reader) (*Config, error) { var c Config if err := defaults.Set(&c); err != nil { return nil, err } + c.init() var raw map[string]any if _, err := toml.NewDecoder(r).Decode(&raw); err != nil { return nil, errors.Wrap(err, "config: error decoding toml data") diff --git a/cmd/revad/pkg/config/config_test.go b/cmd/revad/pkg/config/config_test.go index 7ab22f8a03..3d4385c0db 100644 --- a/cmd/revad/pkg/config/config_test.go +++ b/cmd/revad/pkg/config/config_test.go @@ -248,6 +248,7 @@ nats_token = "secret-token-example"` MaxCPUs: "1", TracingEnabled: true, TracingEndpoint: "localhost:6831", + ConfigDumpFile: "/tmp/reva-dump.toml", }, c2.Core) assert.Equal(t, Vars{ @@ -500,6 +501,7 @@ func TestDump(t *testing.T) { "tracing_collector": "", "tracing_service_name": "", "tracing_service": "", + "config_dump_file": "", }, "vars": map[string]any{ "db_username": "root", diff --git a/cmd/revad/runtime/runtime.go b/cmd/revad/runtime/runtime.go index d895b122f1..3542715572 100644 --- a/cmd/revad/runtime/runtime.go +++ b/cmd/revad/runtime/runtime.go @@ -22,10 +22,14 @@ import ( "context" "fmt" "net" + "os" + "os/signal" "runtime" "strconv" "strings" + "syscall" + "github.com/BurntSushi/toml" "github.com/pkg/errors" "github.com/cs3org/reva/cmd/revad/pkg/config" @@ -121,7 +125,7 @@ func New(config *config.Config, opt ...Option) (*Reva, error) { return nil, err } - return &Reva{ + r := &Reva{ ctx: ctx, config: config, servers: servers, @@ -130,7 +134,40 @@ func New(config *config.Config, opt ...Option) (*Reva, error) { lns: listeners, pidfile: opts.PidFile, log: log, - }, nil + } + r.initConfigDumper() + return r, nil +} + +func (r *Reva) initConfigDumper() { + // dump the config when the process receives a SIGUSR1 signal + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGUSR1) + + go func() { + for { + <-sigs + r.dumpConfig() + } + }() +} + +func (r *Reva) dumpConfig() { + cfg := r.config.Dump() + out := r.config.Core.ConfigDumpFile + f, err := os.OpenFile(out, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644) + if err != nil { + r.log.Error().Err(err).Msgf("error opening file %s for dumping the config", out) + return + } + defer f.Close() + + enc := toml.NewEncoder(f) + if err := enc.Encode(cfg); err != nil { + r.log.Error().Err(err).Msg("error encoding config") + return + } + r.log.Debug().Msgf("config dumped successfully in %s", out) } func servicesAddresses(cfg *config.Config) map[string]grace.Addressable { From bdc8b395761e3cf799b75caaed59d0b1cbff6aaa Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Wed, 5 Jul 2023 14:01:18 +0200 Subject: [PATCH 37/48] Enforce/validate configuration of services (#4035) --- changelog/unreleased/validate-config.md | 7 ++ cmd/revad/runtime/runtime.go | 7 +- go.mod | 15 ++-- go.sum | 2 - .../applicationauth/applicationauth.go | 24 ++--- .../grpc/services/appprovider/appprovider.go | 24 ++--- .../services/appprovider/appprovider_test.go | 8 +- .../grpc/services/appregistry/appregistry.go | 19 ++-- .../services/authprovider/authprovider.go | 20 ++--- .../services/authregistry/authregistry.go | 20 ++--- internal/grpc/services/datatx/datatx.go | 24 ++--- internal/grpc/services/gateway/gateway.go | 22 ++--- .../services/groupprovider/groupprovider.go | 20 ++--- .../grpc/services/helloworld/helloworld.go | 20 ++--- internal/grpc/services/ocmcore/ocmcore.go | 23 ++--- .../ocminvitemanager/ocminvitemanager.go | 38 +++----- .../ocmproviderauthorizer.go | 23 ++--- .../ocmshareprovider/ocmshareprovider.go | 28 ++---- .../grpc/services/permissions/permissions.go | 16 +--- .../grpc/services/preferences/preferences.go | 24 ++--- .../publicshareprovider.go | 25 ++---- .../publicstorageprovider.go | 17 +--- .../storageprovider/storageprovider.go | 23 ++--- .../storageregistry/storageregistry.go | 20 ++--- .../services/userprovider/userprovider.go | 20 ++--- .../usershareprovider/usershareprovider.go | 24 ++--- .../http/services/appprovider/appprovider.go | 13 ++- internal/http/services/archiver/handler.go | 19 ++-- .../http/services/datagateway/datagateway.go | 18 ++-- .../services/dataprovider/dataprovider.go | 16 ++-- .../http/services/helloworld/helloworld.go | 12 ++- internal/http/services/mentix/mentix.go | 25 ++---- internal/http/services/metrics/metrics.go | 11 +-- internal/http/services/ocmd/ocm.go | 13 ++- .../http/services/ocmprovider/ocmprovider.go | 12 ++- .../http/services/owncloud/ocdav/ocdav.go | 26 +++--- .../services/owncloud/ocs/config/config.go | 2 +- internal/http/services/owncloud/ocs/ocs.go | 14 ++- .../http/services/preferences/preferences.go | 11 ++- .../http/services/prometheus/prometheus.go | 12 ++- .../services/reverseproxy/reverseproxy.go | 11 ++- .../http/services/sciencemesh/sciencemesh.go | 30 +++---- internal/http/services/siteacc/siteacc.go | 29 ++---- internal/http/services/sysinfo/sysinfo.go | 31 +++---- internal/http/services/wellknown/wellknown.go | 12 ++- .../services/helloworld/helloworld.go | 5 +- .../services/notifications/notifications.go | 16 ++-- pkg/app/provider/demo/demo.go | 14 +-- pkg/app/provider/wopi/wopi.go | 30 +++---- pkg/app/registry/static/static.go | 21 +---- pkg/appauth/manager/json/json.go | 22 ++--- pkg/auth/manager/appauth/appauth.go | 16 ++-- pkg/auth/manager/json/json.go | 20 ++--- pkg/auth/manager/json/json_test.go | 11 +-- pkg/auth/manager/ldap/ldap.go | 37 +++----- pkg/auth/manager/machine/machine.go | 16 ++-- pkg/auth/manager/nextcloud/nextcloud.go | 23 ++--- pkg/auth/manager/ocmshares/ocmshares.go | 23 +++-- pkg/auth/manager/oidc/oidc.go | 22 ++--- pkg/auth/manager/owncloudsql/owncloudsql.go | 24 ++--- pkg/auth/manager/publicshares/publicshares.go | 22 ++--- pkg/auth/registry/static/static.go | 17 +--- pkg/cbox/favorite/sql/sql.go | 8 +- pkg/cbox/group/rest/rest.go | 19 ++-- pkg/cbox/preferences/sql/sql.go | 8 +- pkg/cbox/publicshare/sql/sql.go | 11 ++- pkg/cbox/share/sql/sql.go | 22 ++--- .../storage/eoshomewrapper/eoshomewrapper.go | 29 ++---- pkg/cbox/storage/eoswrapper/eoswrapper.go | 29 ++---- pkg/cbox/user/rest/rest.go | 19 ++-- pkg/datatx/manager/rclone/rclone.go | 6 +- .../manager/rclone/repository/json/json.go | 23 ++--- .../rclone/repository/registry/registry.go | 4 +- pkg/datatx/repository/json/json.go | 20 ++--- pkg/eosclient/eosbinary/eosbinary.go | 4 +- pkg/errtypes/join.go | 42 +++++++++ pkg/group/manager/json/json.go | 18 +--- pkg/group/manager/ldap/ldap.go | 22 ++--- pkg/mentix/config/config.go | 2 +- pkg/metrics/config/config.go | 2 +- .../handler/emailhandler/emailhandler.go | 25 +++--- pkg/notification/handler/registry/registry.go | 13 ++- pkg/notification/manager/registry/registry.go | 8 +- pkg/notification/manager/sql/sql.go | 10 +-- pkg/ocm/invite/repository/json/json.go | 28 ++---- pkg/ocm/invite/repository/sql/sql.go | 27 ++---- pkg/ocm/provider/authorizer/json/json.go | 12 ++- pkg/ocm/provider/authorizer/mentix/mentix.go | 12 ++- pkg/ocm/provider/authorizer/open/open.go | 11 +-- pkg/ocm/share/repository/json/json.go | 20 ++--- .../share/repository/nextcloud/nextcloud.go | 21 +---- pkg/ocm/share/repository/sql/sql.go | 18 ++-- pkg/ocm/storage/outcoming/ocm.go | 23 ++--- pkg/ocm/storage/received/ocm.go | 26 ++---- pkg/publicshare/manager/json/json.go | 34 ++++--- pkg/rgrpc/rgrpc.go | 2 +- pkg/rserverless/rserverless.go | 2 +- pkg/share/cache/memory/memory.go | 17 ++-- pkg/share/cache/redis/redis.go | 18 ++-- pkg/share/manager/json/json.go | 21 ++--- pkg/share/manager/sql/sql.go | 22 ++--- pkg/storage/fs/cback/cback.go | 11 ++- pkg/storage/fs/cephfs/cephfs.go | 16 ++-- pkg/storage/fs/cephfs/options.go | 2 +- pkg/storage/fs/eos/eos.go | 23 ++--- pkg/storage/fs/eosgrpc/eosgrpc.go | 25 ++---- pkg/storage/fs/eosgrpchome/eosgrpchome.go | 27 ++---- pkg/storage/fs/eoshome/eoshome.go | 25 ++---- pkg/storage/fs/local/local.go | 19 ++-- pkg/storage/fs/localhome/localhome.go | 12 +++ pkg/storage/fs/nextcloud/nextcloud.go | 18 +--- pkg/storage/fs/owncloud/owncloud.go | 27 ++---- pkg/storage/fs/owncloudsql/owncloudsql.go | 22 ++--- pkg/storage/fs/s3/s3.go | 17 +--- pkg/storage/registry/static/static.go | 19 ++-- pkg/storage/utils/eosfs/eosfs.go | 4 +- pkg/storage/utils/localfs/localfs.go | 4 +- pkg/token/manager/jwt/jwt.go | 31 +++---- pkg/user/manager/json/json.go | 18 +--- pkg/user/manager/ldap/ldap.go | 33 ++++--- pkg/user/manager/nextcloud/nextcloud.go | 21 ++--- pkg/user/manager/owncloudsql/owncloudsql.go | 24 ++--- pkg/utils/cfg/cfg.go | 90 +++++++++++++++++++ pkg/utils/cfg/cfg_test.go | 79 ++++++++++++++++ .../fixtures/ocm-server-cernbox-http.toml | 1 + .../grpc/fixtures/ocm-server-cesnet-http.toml | 1 + .../ocm-share/ocm-server-cernbox-grpc.toml | 1 + .../ocm-share/ocm-server-cernbox-http.toml | 1 + .../ocm-share/ocm-server-cesnet-grpc.toml | 2 + .../ocm-share/ocm-server-cesnet-http.toml | 1 + tests/integration/grpc/ocm_invitation_test.go | 2 +- 131 files changed, 985 insertions(+), 1418 deletions(-) create mode 100644 changelog/unreleased/validate-config.md create mode 100644 pkg/errtypes/join.go create mode 100644 pkg/utils/cfg/cfg.go create mode 100644 pkg/utils/cfg/cfg_test.go diff --git a/changelog/unreleased/validate-config.md b/changelog/unreleased/validate-config.md new file mode 100644 index 0000000000..f4fbe1aa32 --- /dev/null +++ b/changelog/unreleased/validate-config.md @@ -0,0 +1,7 @@ +Enhancement: Enforce/validate configuration of services + +Every driver can now specify some validation rules on the +configuration. If the validation rules are not respected, +reva will bail out on startup with a clear error. + +https://github.com/cs3org/reva/pull/4035 diff --git a/cmd/revad/runtime/runtime.go b/cmd/revad/runtime/runtime.go index 3542715572..40b1df3670 100644 --- a/cmd/revad/runtime/runtime.go +++ b/cmd/revad/runtime/runtime.go @@ -119,7 +119,7 @@ func New(config *config.Config, opt ...Option) (*Reva, error) { return nil, err } - serverless, err := newServerless(config, log) + serverless, err := newServerless(ctx, config, log) if err != nil { watcher.Clean() return nil, err @@ -181,7 +181,7 @@ func servicesAddresses(cfg *config.Config) map[string]grace.Addressable { return a } -func newServerless(config *config.Config, log *zerolog.Logger) (*rserverless.Serverless, error) { +func newServerless(ctx context.Context, config *config.Config, log *zerolog.Logger) (*rserverless.Serverless, error) { sl := make(map[string]rserverless.Service) logger := log.With().Str("pkg", "serverless").Logger() if err := config.Serverless.ForEach(func(name string, config map[string]any) error { @@ -190,7 +190,8 @@ func newServerless(config *config.Config, log *zerolog.Logger) (*rserverless.Ser return fmt.Errorf("serverless service %s does not exist", name) } log := logger.With().Str("service", name).Logger() - svc, err := new(config, &log) + ctx := appctx.WithLogger(ctx, &log) + svc, err := new(ctx, config) if err != nil { return errors.Wrapf(err, "serverless service %s could not be initialized", name) } diff --git a/go.mod b/go.mod index 59e7e61716..6e25780e1f 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,8 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible github.com/ceph/go-ceph v0.15.0 github.com/cheggaaa/pb v1.0.29 + github.com/coreos/go-oidc/v3 v3.5.0 + github.com/creasty/defaults v1.7.0 github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e github.com/cs3org/go-cs3apis v0.0.0-20230606135123-b799d47a6648 github.com/dgraph-io/ristretto v0.1.1 @@ -23,6 +25,8 @@ require ( github.com/glpatcern/go-mime v0.0.0-20221026162842-2a8d71ad17a9 github.com/go-chi/chi/v5 v5.0.8 github.com/go-ldap/ldap/v3 v3.4.4 + github.com/go-playground/locales v0.14.1 + github.com/go-playground/universal-translator v0.18.1 github.com/go-playground/validator/v10 v10.11.2 github.com/go-sql-driver/mysql v1.7.0 github.com/golang-jwt/jwt v3.2.2+incompatible @@ -76,12 +80,6 @@ require ( gotest.tools v2.2.0+incompatible ) -require ( - github.com/creasty/defaults v1.7.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.0 // indirect - github.com/hashicorp/go-msgpack/v2 v2.1.0 // indirect -) - require ( github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -92,13 +90,13 @@ require ( github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/coreos/go-oidc/v3 v3.5.0 github.com/davecgh/go-spew v1.1.1 // indirect github.com/dolthub/vitess v0.0.0-20221031111135-9aad77e7b39f // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/fatih/color v1.13.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect + github.com/go-jose/go-jose/v3 v3.0.0 // indirect github.com/go-kit/kit v0.10.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect @@ -106,8 +104,6 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/errors v0.20.2 // indirect github.com/go-openapi/strfmt v0.21.2 // indirect - github.com/go-playground/locales v0.14.1 // indirect - github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-stack/stack v1.8.1 // indirect github.com/gocraft/dbr/v2 v2.7.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -116,6 +112,7 @@ require ( github.com/google/flatbuffers v2.0.6+incompatible // indirect github.com/hashicorp/go-immutable-radix v1.0.0 // indirect github.com/hashicorp/go-msgpack v1.1.5 // indirect + github.com/hashicorp/go-msgpack/v2 v2.1.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/raft v1.4.0 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect diff --git a/go.sum b/go.sum index 5fa1a06c16..e3848a5c97 100644 --- a/go.sum +++ b/go.sum @@ -308,8 +308,6 @@ github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdB github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= -github.com/cs3org/go-cs3apis v0.0.0-20230508132523-e0d062e63b3b h1:UCO7Rnf5bvIvRtETguV8IaTx73cImLlFWxrApCB0QsQ= -github.com/cs3org/go-cs3apis v0.0.0-20230508132523-e0d062e63b3b/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cs3org/go-cs3apis v0.0.0-20230606135123-b799d47a6648 h1:gBz1JSC2u6o/TkUhWSdJZvacyTsVUzDouegRzvrJye4= github.com/cs3org/go-cs3apis v0.0.0-20230606135123-b799d47a6648/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= diff --git a/internal/grpc/services/applicationauth/applicationauth.go b/internal/grpc/services/applicationauth/applicationauth.go index e05bd38a70..387a75904e 100644 --- a/internal/grpc/services/applicationauth/applicationauth.go +++ b/internal/grpc/services/applicationauth/applicationauth.go @@ -27,8 +27,7 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) @@ -46,7 +45,7 @@ type service struct { am appauth.Manager } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Driver == "" { c.Driver = "json" } @@ -63,30 +62,19 @@ func getAppAuthManager(ctx context.Context, c *config) (appauth.Manager, error) return nil, errtypes.NotFound("driver not found: " + c.Driver) } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - // New creates a app auth provider svc. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - - am, err := getAppAuthManager(ctx, c) + am, err := getAppAuthManager(ctx, &c) if err != nil { return nil, err } service := &service{ - conf: c, + conf: &c, am: am, } diff --git a/internal/grpc/services/appprovider/appprovider.go b/internal/grpc/services/appprovider/appprovider.go index 5b44681cad..53f5b89a27 100644 --- a/internal/grpc/services/appprovider/appprovider.go +++ b/internal/grpc/services/appprovider/appprovider.go @@ -40,8 +40,8 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/juliangruber/go-intersect" - "github.com/mitchellh/mapstructure" "google.golang.org/grpc" ) @@ -65,7 +65,7 @@ type config struct { Language string `mapstructure:"language"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Driver == "" { c.Driver = "demo" } @@ -73,35 +73,25 @@ func (c *config) init() { c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - c.init() - return c, nil -} - // New creates a new AppProviderService. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } // read and register custom mime types if configured - err = registerMimeTypes(c.CustomMimeTypesJSON) - if err != nil { + if err := registerMimeTypes(c.CustomMimeTypesJSON); err != nil { return nil, err } - provider, err := getProvider(ctx, c) + provider, err := getProvider(ctx, &c) if err != nil { return nil, err } service := &service{ - conf: c, + conf: &c, provider: provider, } diff --git a/internal/grpc/services/appprovider/appprovider_test.go b/internal/grpc/services/appprovider/appprovider_test.go index e53468cbae..6ac998a65a 100644 --- a/internal/grpc/services/appprovider/appprovider_test.go +++ b/internal/grpc/services/appprovider/appprovider_test.go @@ -21,6 +21,7 @@ package appprovider import ( "testing" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/mitchellh/mapstructure" "github.com/stretchr/testify/assert" ) @@ -80,9 +81,12 @@ func Test_parseConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parseConfig(tt.m) + var got config + err := cfg.Decode(tt.m, &got) assert.Equal(t, tt.wantErr, err) - assert.Equal(t, tt.want, got) + if tt.wantErr == nil { + assert.Equal(t, tt.want, &got) + } }) } } diff --git a/internal/grpc/services/appregistry/appregistry.go b/internal/grpc/services/appregistry/appregistry.go index 0ba769ff94..0acea9c12f 100644 --- a/internal/grpc/services/appregistry/appregistry.go +++ b/internal/grpc/services/appregistry/appregistry.go @@ -27,7 +27,7 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) @@ -56,7 +56,7 @@ type config struct { Drivers map[string]map[string]interface{} `mapstructure:"drivers"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Driver == "" { c.Driver = "static" } @@ -64,12 +64,12 @@ func (c *config) init() { // New creates a new StorageRegistryService. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - reg, err := getRegistry(ctx, c) + reg, err := getRegistry(ctx, &c) if err != nil { return nil, err } @@ -81,15 +81,6 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { return svc, nil } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - c.init() - return c, nil -} - func getRegistry(ctx context.Context, c *config) (app.Registry, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { return f(ctx, c.Drivers[c.Driver]) diff --git a/internal/grpc/services/authprovider/authprovider.go b/internal/grpc/services/authprovider/authprovider.go index 58765c4dad..71427f34cc 100644 --- a/internal/grpc/services/authprovider/authprovider.go +++ b/internal/grpc/services/authprovider/authprovider.go @@ -33,7 +33,7 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/user" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "google.golang.org/grpc" ) @@ -48,7 +48,7 @@ type config struct { blockedUsers []string } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.AuthManager == "" { c.AuthManager = "json" } @@ -62,16 +62,6 @@ type service struct { blockedUsers user.BlockedUsers } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - c.init() - return c, nil -} - func getAuthManager(ctx context.Context, manager string, m map[string]map[string]interface{}) (auth.Manager, *plugin.RevaPlugin, error) { if manager == "" { return nil, nil, errtypes.InternalError("authsvc: driver not configured for auth manager") @@ -101,8 +91,8 @@ func getAuthManager(ctx context.Context, manager string, m map[string]map[string // New returns a new AuthProviderServiceServer. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } @@ -112,7 +102,7 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { } svc := &service{ - conf: c, + conf: &c, authmgr: authManager, plugin: plug, blockedUsers: user.NewBlockedUsersSet(c.blockedUsers), diff --git a/internal/grpc/services/authregistry/authregistry.go b/internal/grpc/services/authregistry/authregistry.go index 267ea3b6b0..1e6969667f 100644 --- a/internal/grpc/services/authregistry/authregistry.go +++ b/internal/grpc/services/authregistry/authregistry.go @@ -27,7 +27,7 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) @@ -59,7 +59,7 @@ type config struct { Drivers map[string]map[string]interface{} `mapstructure:"drivers"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Driver == "" { c.Driver = "static" } @@ -67,14 +67,12 @@ func (c *config) init() { // New creates a new AuthRegistry. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - - reg, err := getRegistry(ctx, c) + reg, err := getRegistry(ctx, &c) if err != nil { return nil, err } @@ -86,14 +84,6 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { return service, nil } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - func getRegistry(ctx context.Context, c *config) (auth.Registry, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { return f(ctx, c.Drivers[c.Driver]) diff --git a/internal/grpc/services/datatx/datatx.go b/internal/grpc/services/datatx/datatx.go index 98bf545b65..ef953d744d 100644 --- a/internal/grpc/services/datatx/datatx.go +++ b/internal/grpc/services/datatx/datatx.go @@ -30,7 +30,7 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "google.golang.org/grpc" ) @@ -54,7 +54,7 @@ type service struct { storageDriver txdriver.Repository } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.TxDriver == "" { c.TxDriver = "rclone" } @@ -78,35 +78,25 @@ func getStorageManager(ctx context.Context, c *config) (txdriver.Repository, err return nil, errtypes.NotFound("datatx service: driver not found: " + c.StorageDriver) } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "datatx service: error decoding conf") - return nil, err - } - return c, nil -} - // New creates a new datatx svc. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - txManager, err := getDatatxManager(ctx, c) + txManager, err := getDatatxManager(ctx, &c) if err != nil { return nil, err } - storageDriver, err := getStorageManager(ctx, c) + storageDriver, err := getStorageManager(ctx, &c) if err != nil { return nil, err } service := &service{ - conf: c, + conf: &c, txManager: txManager, storageDriver: storageDriver, } diff --git a/internal/grpc/services/gateway/gateway.go b/internal/grpc/services/gateway/gateway.go index d766b00c5e..a85d18fdf1 100644 --- a/internal/grpc/services/gateway/gateway.go +++ b/internal/grpc/services/gateway/gateway.go @@ -32,8 +32,7 @@ import ( "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/token/manager/registry" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) @@ -75,7 +74,7 @@ type config struct { } // sets defaults. -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.ShareFolder == "" { c.ShareFolder = "MyShares" } @@ -126,13 +125,11 @@ type svc struct { // The gateway is responsible for high-level controls: rate-limiting, coordination between svcs // like sharing and storage acls, asynchronous transactions, ... func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - // ensure DataGatewayEndpoint is a valid URI u, err := url.Parse(c.DataGatewayEndpoint) if err != nil { @@ -153,7 +150,7 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { createHomeCache.SkipTTLExtensionOnHit(true) s := &svc{ - c: c, + c: &c, dataGatewayURL: *u, tokenmgr: tokenManager, etagCache: etagCache, @@ -176,15 +173,6 @@ func (s *svc) UnprotectedEndpoints() []string { return []string{"/cs3.gateway.v1beta1.GatewayAPI"} } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "gateway: error decoding conf") - return nil, err - } - return c, nil -} - func getTokenManager(manager string, m map[string]map[string]interface{}) (token.Manager, error) { if f, ok := registry.NewFuncs[manager]; ok { return f(m[manager]) diff --git a/internal/grpc/services/groupprovider/groupprovider.go b/internal/grpc/services/groupprovider/groupprovider.go index b72704b95c..7ab63af7bc 100644 --- a/internal/grpc/services/groupprovider/groupprovider.go +++ b/internal/grpc/services/groupprovider/groupprovider.go @@ -29,7 +29,7 @@ import ( "github.com/cs3org/reva/pkg/group/manager/registry" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "google.golang.org/grpc" ) @@ -43,22 +43,12 @@ type config struct { Drivers map[string]map[string]interface{} `mapstructure:"drivers"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Driver == "" { c.Driver = "json" } } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - c.init() - return c, nil -} - func getDriver(ctx context.Context, c *config) (group.Manager, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { return f(ctx, c.Drivers[c.Driver]) @@ -69,12 +59,12 @@ func getDriver(ctx context.Context, c *config) (group.Manager, error) { // New returns a new GroupProviderServiceServer. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - groupManager, err := getDriver(ctx, c) + groupManager, err := getDriver(ctx, &c) if err != nil { return nil, err } diff --git a/internal/grpc/services/helloworld/helloworld.go b/internal/grpc/services/helloworld/helloworld.go index 3249457f59..8d4eab0566 100644 --- a/internal/grpc/services/helloworld/helloworld.go +++ b/internal/grpc/services/helloworld/helloworld.go @@ -24,8 +24,7 @@ import ( "github.com/cs3org/reva/internal/grpc/services/helloworld/proto" "github.com/cs3org/reva/pkg/rgrpc" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) @@ -40,20 +39,21 @@ type service struct { conf *conf } +func (c *conf) ApplyDefaults() { + if c.Message == "" { + c.Message = "Hello" + } +} + // New returns a new PreferencesServiceServer // It can be tested like this: // prototool grpc --address 0.0.0.0:9999 --method 'revad.helloworld.HelloWorldService/Hello' --data '{"name": "Alice"}'. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c := &conf{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "helloworld: error decoding conf") + var c conf + if err := cfg.Decode(m, &c); err != nil { return nil, err } - - if c.Message == "" { - c.Message = "Hello" - } - service := &service{conf: c} + service := &service{conf: &c} return service, nil } diff --git a/internal/grpc/services/ocmcore/ocmcore.go b/internal/grpc/services/ocmcore/ocmcore.go index c9f8dfe71b..54bf50c616 100644 --- a/internal/grpc/services/ocmcore/ocmcore.go +++ b/internal/grpc/services/ocmcore/ocmcore.go @@ -32,8 +32,7 @@ import ( "github.com/cs3org/reva/pkg/ocm/share/repository/registry" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) @@ -51,7 +50,7 @@ type service struct { repo share.Repository } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Driver == "" { c.Driver = "json" } @@ -68,30 +67,20 @@ func getShareRepository(ctx context.Context, c *config) (share.Repository, error return nil, errtypes.NotFound(fmt.Sprintf("driver not found: %s", c.Driver)) } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - // New creates a new ocm core svc. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - repo, err := getShareRepository(ctx, c) + repo, err := getShareRepository(ctx, &c) if err != nil { return nil, err } service := &service{ - conf: c, + conf: &c, repo: repo, } diff --git a/internal/grpc/services/ocminvitemanager/ocminvitemanager.go b/internal/grpc/services/ocminvitemanager/ocminvitemanager.go index e4a3989cbe..7906a38bc3 100644 --- a/internal/grpc/services/ocminvitemanager/ocminvitemanager.go +++ b/internal/grpc/services/ocminvitemanager/ocminvitemanager.go @@ -36,7 +36,7 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/utils" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "google.golang.org/grpc" ) @@ -51,8 +51,8 @@ type config struct { TokenExpiration string `mapstructure:"token_expiration"` OCMClientTimeout int `mapstructure:"ocm_timeout"` OCMClientInsecure bool `mapstructure:"ocm_insecure"` - GatewaySVC string `mapstructure:"gateway_svc"` - ProviderDomain string `mapstructure:"provider_domain" docs:"The same domain registered in the provider authorizer"` + GatewaySVC string `mapstructure:"gatewaysvc" validate:"required"` + ProviderDomain string `mapstructure:"provider_domain" validate:"required" docs:"The same domain registered in the provider authorizer"` tokenExpiration time.Duration } @@ -63,7 +63,7 @@ type service struct { ocmClient *client.OCMClient } -func (c *config) init() error { +func (c *config) ApplyDefaults() { if c.Driver == "" { c.Driver = "json" } @@ -71,15 +71,7 @@ func (c *config) init() error { c.TokenExpiration = "24h" } - p, err := time.ParseDuration(c.TokenExpiration) - if err != nil { - return err - } - c.tokenExpiration = p - c.GatewaySVC = sharedconf.GetGatewaySVC(c.GatewaySVC) - - return nil } func (s *service) Register(ss *grpc.Server) { @@ -93,32 +85,26 @@ func getInviteRepository(ctx context.Context, c *config) (invite.Repository, err return nil, errtypes.NotFound("driver not found: " + c.Driver) } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - // New creates a new OCM invite manager svc. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - if err := c.init(); err != nil { + + p, err := time.ParseDuration(c.TokenExpiration) + if err != nil { return nil, err } + c.tokenExpiration = p - repo, err := getInviteRepository(ctx, c) + repo, err := getInviteRepository(ctx, &c) if err != nil { return nil, err } service := &service{ - conf: c, + conf: &c, repo: repo, ocmClient: client.New(&client.Config{ Timeout: time.Duration(c.OCMClientTimeout) * time.Second, diff --git a/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go b/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go index fe26f8ebe1..18e436bee6 100644 --- a/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go +++ b/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go @@ -28,8 +28,7 @@ import ( "github.com/cs3org/reva/pkg/ocm/provider/authorizer/registry" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) @@ -47,7 +46,7 @@ type service struct { pa provider.Authorizer } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Driver == "" { c.Driver = "json" } @@ -64,30 +63,20 @@ func getProviderAuthorizer(ctx context.Context, c *config) (provider.Authorizer, return nil, errtypes.NotFound("driver not found: " + c.Driver) } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - // New creates a new OCM provider authorizer svc. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - pa, err := getProviderAuthorizer(ctx, c) + pa, err := getProviderAuthorizer(ctx, &c) if err != nil { return nil, err } service := &service{ - conf: c, + conf: &c, pa: pa, } return service, nil diff --git a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go index 6bd9f73011..acdc56f9c5 100644 --- a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go +++ b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go @@ -46,7 +46,7 @@ import ( "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage/utils/walker" "github.com/cs3org/reva/pkg/utils" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "google.golang.org/grpc" ) @@ -60,9 +60,9 @@ type config struct { Drivers map[string]map[string]interface{} `mapstructure:"drivers"` ClientTimeout int `mapstructure:"client_timeout"` ClientInsecure bool `mapstructure:"client_insecure"` - GatewaySVC string `mapstructure:"gatewaysvc"` - ProviderDomain string `mapstructure:"provider_domain" docs:"The same domain registered in the provider authorizer"` - WebDAVEndpoint string `mapstructure:"webdav_endpoint"` + GatewaySVC string `mapstructure:"gatewaysvc" validate:"required"` + ProviderDomain string `mapstructure:"provider_domain" validate:"required" docs:"The same domain registered in the provider authorizer"` + WebDAVEndpoint string `mapstructure:"webdav_endpoint" validate:"required"` WebappTemplate string `mapstructure:"webapp_template"` } @@ -75,7 +75,7 @@ type service struct { walker walker.Walker } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Driver == "" { c.Driver = "json" } @@ -100,24 +100,14 @@ func getShareRepository(ctx context.Context, c *config) (share.Repository, error return nil, errtypes.NotFound("driver not found: " + c.Driver) } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - // New creates a new ocm share provider svc. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - repo, err := getShareRepository(ctx, c) + repo, err := getShareRepository(ctx, &c) if err != nil { return nil, err } @@ -139,7 +129,7 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { walker := walker.NewWalker(gateway) service := &service{ - conf: c, + conf: &c, repo: repo, client: client, gateway: gateway, diff --git a/internal/grpc/services/permissions/permissions.go b/internal/grpc/services/permissions/permissions.go index cc1b75786b..eec9c7e22f 100644 --- a/internal/grpc/services/permissions/permissions.go +++ b/internal/grpc/services/permissions/permissions.go @@ -27,8 +27,7 @@ import ( "github.com/cs3org/reva/pkg/permission" "github.com/cs3org/reva/pkg/permission/manager/registry" "github.com/cs3org/reva/pkg/rgrpc" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) @@ -41,23 +40,14 @@ type config struct { Drivers map[string]map[string]interface{} `mapstructure:"drivers" docs:"url:pkg/permission/permission.go"` } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - type service struct { manager permission.Manager } // New returns a new PermissionsServiceServer. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } diff --git a/internal/grpc/services/preferences/preferences.go b/internal/grpc/services/preferences/preferences.go index b5e0003241..b45c6afbca 100644 --- a/internal/grpc/services/preferences/preferences.go +++ b/internal/grpc/services/preferences/preferences.go @@ -27,8 +27,7 @@ import ( "github.com/cs3org/reva/pkg/preferences/registry" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) @@ -41,7 +40,7 @@ type config struct { Drivers map[string]map[string]interface{} `mapstructure:"drivers"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Driver == "" { c.Driver = "memory" } @@ -59,31 +58,20 @@ func getPreferencesManager(ctx context.Context, c *config) (preferences.Manager, return nil, errtypes.NotFound("driver not found: " + c.Driver) } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - // New returns a new PreferencesServiceServer. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - - pm, err := getPreferencesManager(ctx, c) + pm, err := getPreferencesManager(ctx, &c) if err != nil { return nil, err } return &service{ - conf: c, + conf: &c, pm: pm, }, nil } diff --git a/internal/grpc/services/publicshareprovider/publicshareprovider.go b/internal/grpc/services/publicshareprovider/publicshareprovider.go index b30f6653c6..91ce7a0547 100644 --- a/internal/grpc/services/publicshareprovider/publicshareprovider.go +++ b/internal/grpc/services/publicshareprovider/publicshareprovider.go @@ -31,8 +31,7 @@ import ( "github.com/cs3org/reva/pkg/publicshare/manager/registry" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) @@ -46,7 +45,7 @@ type config struct { AllowedPathsForShares []string `mapstructure:"allowed_paths_for_shares"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Driver == "" { c.Driver = "json" } @@ -69,6 +68,7 @@ func getShareManager(ctx context.Context, c *config) (publicshare.Manager, error func (s *service) Close() error { return nil } + func (s *service) UnprotectedEndpoints() []string { return []string{"/cs3.sharing.link.v1beta1.LinkAPI/GetPublicShareByToken"} } @@ -77,25 +77,14 @@ func (s *service) Register(ss *grpc.Server) { link.RegisterLinkAPIServer(ss, s) } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - // New creates a new user share provider svc. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - - sm, err := getShareManager(ctx, c) + sm, err := getShareManager(ctx, &c) if err != nil { return nil, err } @@ -110,7 +99,7 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { } service := &service{ - conf: c, + conf: &c, sm: sm, allowedPathsForShares: allowedPathsForShares, } diff --git a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go index 85e541fc02..48b7eba838 100644 --- a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go +++ b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go @@ -36,7 +36,7 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/todo/pool" rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "go.opentelemetry.io/otel/attribute" "google.golang.org/grpc" @@ -73,19 +73,10 @@ func (s *service) Register(ss *grpc.Server) { provider.RegisterProviderAPIServer(ss, s) } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - // New creates a new IsPublic Storage Provider service. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } @@ -98,7 +89,7 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { } service := &service{ - conf: c, + conf: &c, mountPath: mountPath, mountID: mountID, gateway: gateway, diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index da0f826d91..0ef056cc53 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -42,8 +42,8 @@ import ( "github.com/cs3org/reva/pkg/storage/fs/registry" rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/google/uuid" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "go.opentelemetry.io/otel/attribute" "google.golang.org/grpc" @@ -66,7 +66,7 @@ type config struct { MinimunAllowedPathLevelForShare int `mapstructure:"minimum_allowed_path_level_for_share"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Driver == "" { c.Driver = "localhome" } @@ -133,15 +133,6 @@ func parseXSTypes(xsTypes map[string]uint32) ([]*provider.ResourceChecksumPriori return types, nil } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - func registerMimeTypes(mappingFile string) error { if mappingFile != "" { f, err := os.ReadFile(mappingFile) @@ -163,13 +154,11 @@ func registerMimeTypes(mappingFile string) error { // New creates a new storage provider svc. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - if err := os.MkdirAll(c.TmpFolder, 0755); err != nil { return nil, err } @@ -177,7 +166,7 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { mountPath := c.MountPath mountID := c.MountID - fs, err := getFS(ctx, c) + fs, err := getFS(ctx, &c) if err != nil { return nil, err } @@ -205,7 +194,7 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { } service := &service{ - conf: c, + conf: &c, storage: fs, tmpFolder: c.TmpFolder, mountPath: mountPath, diff --git a/internal/grpc/services/storageregistry/storageregistry.go b/internal/grpc/services/storageregistry/storageregistry.go index 391347670b..e3a4ac3454 100644 --- a/internal/grpc/services/storageregistry/storageregistry.go +++ b/internal/grpc/services/storageregistry/storageregistry.go @@ -28,7 +28,7 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/registry/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) @@ -57,7 +57,7 @@ type config struct { Drivers map[string]map[string]interface{} `mapstructure:"drivers"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Driver == "" { c.Driver = "static" } @@ -65,14 +65,12 @@ func (c *config) init() { // New creates a new StorageBrokerService. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - - reg, err := getRegistry(ctx, c) + reg, err := getRegistry(ctx, &c) if err != nil { return nil, err } @@ -84,14 +82,6 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { return service, nil } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - func getRegistry(ctx context.Context, c *config) (storage.Registry, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { return f(ctx, c.Drivers[c.Driver]) diff --git a/internal/grpc/services/userprovider/userprovider.go b/internal/grpc/services/userprovider/userprovider.go index a8a7ae774a..d34ef48318 100644 --- a/internal/grpc/services/userprovider/userprovider.go +++ b/internal/grpc/services/userprovider/userprovider.go @@ -31,7 +31,7 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/user" "github.com/cs3org/reva/pkg/user/manager/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "google.golang.org/grpc" ) @@ -45,22 +45,12 @@ type config struct { Drivers map[string]map[string]interface{} `mapstructure:"drivers"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Driver == "" { c.Driver = "json" } } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - c.init() - return c, nil -} - func getDriver(ctx context.Context, c *config) (user.Manager, *plugin.RevaPlugin, error) { p, err := plugin.Load("userprovider", c.Driver) if err == nil { @@ -88,11 +78,11 @@ func getDriver(ctx context.Context, c *config) (user.Manager, *plugin.RevaPlugin // New returns a new UserProviderServiceServer. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - userManager, plug, err := getDriver(ctx, c) + userManager, plug, err := getDriver(ctx, &c) if err != nil { return nil, err } diff --git a/internal/grpc/services/usershareprovider/usershareprovider.go b/internal/grpc/services/usershareprovider/usershareprovider.go index 81965b3db3..8d37af562b 100644 --- a/internal/grpc/services/usershareprovider/usershareprovider.go +++ b/internal/grpc/services/usershareprovider/usershareprovider.go @@ -32,8 +32,7 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/share" "github.com/cs3org/reva/pkg/share/manager/registry" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) @@ -47,7 +46,7 @@ type config struct { AllowedPathsForShares []string `mapstructure:"allowed_paths_for_shares"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Driver == "" { c.Driver = "json" } @@ -79,25 +78,14 @@ func (s *service) Register(ss *grpc.Server) { collaboration.RegisterCollaborationAPIServer(ss, s) } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - // New creates a new user share provider svc. func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - - sm, err := getShareManager(ctx, c) + sm, err := getShareManager(ctx, &c) if err != nil { return nil, err } @@ -112,7 +100,7 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { } service := &service{ - conf: c, + conf: &c, sm: sm, allowedPathsForShares: allowedPathsForShares, } diff --git a/internal/http/services/appprovider/appprovider.go b/internal/http/services/appprovider/appprovider.go index 4dc81ff8ab..049b8e83a3 100644 --- a/internal/http/services/appprovider/appprovider.go +++ b/internal/http/services/appprovider/appprovider.go @@ -36,10 +36,10 @@ import ( "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/go-chi/chi/v5" ua "github.com/mileusna/useragent" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -50,11 +50,11 @@ func init() { // Config holds the config options for the HTTP appprovider service. type Config struct { Prefix string `mapstructure:"prefix"` - GatewaySvc string `mapstructure:"gatewaysvc"` + GatewaySvc string `mapstructure:"gatewaysvc" validate:"required"` Insecure bool `mapstructure:"insecure" docs:"false;Whether to skip certificate checks when sending requests."` } -func (c *Config) init() { +func (c *Config) ApplyDefaults() { if c.Prefix == "" { c.Prefix = "app" } @@ -68,15 +68,14 @@ type svc struct { // New returns a new ocmd object. func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - conf := &Config{} - if err := mapstructure.Decode(m, conf); err != nil { + var c Config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - conf.init() r := chi.NewRouter() s := &svc{ - conf: conf, + conf: &c, router: r, } diff --git a/internal/http/services/archiver/handler.go b/internal/http/services/archiver/handler.go index 07c63b6a44..4ca94eb230 100644 --- a/internal/http/services/archiver/handler.go +++ b/internal/http/services/archiver/handler.go @@ -38,10 +38,10 @@ import ( "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage/utils/downloader" "github.com/cs3org/reva/pkg/storage/utils/walker" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/cs3org/reva/pkg/utils/resourceid" "github.com/gdexlab/go-render/render" ua "github.com/mileusna/useragent" - "github.com/mitchellh/mapstructure" ) type svc struct { @@ -56,12 +56,12 @@ type svc struct { // Config holds the config options that need to be passed down to all ocdav handlers. type Config struct { Prefix string `mapstructure:"prefix"` - GatewaySvc string `mapstructure:"gatewaysvc"` + GatewaySvc string `mapstructure:"gatewaysvc" validate:"required"` Timeout int64 `mapstructure:"timeout"` Insecure bool `mapstructure:"insecure" docs:"false;Whether to skip certificate checks when sending requests."` Name string `mapstructure:"name"` - MaxNumFiles int64 `mapstructure:"max_num_files"` - MaxSize int64 `mapstructure:"max_size"` + MaxNumFiles int64 `mapstructure:"max_num_files" validate:"required,gt=0"` + MaxSize int64 `mapstructure:"max_size" validate:"required,gt=0"` AllowedFolders []string `mapstructure:"allowed_folders"` } @@ -71,14 +71,11 @@ func init() { // New creates a new archiver service. func New(ctx context.Context, conf map[string]interface{}) (global.Service, error) { - c := &Config{} - err := mapstructure.Decode(conf, c) - if err != nil { + var c Config + if err := cfg.Decode(conf, &c); err != nil { return nil, err } - c.init() - gtw, err := pool.GetGatewayServiceClient(pool.Endpoint(c.GatewaySvc)) if err != nil { return nil, err @@ -95,7 +92,7 @@ func New(ctx context.Context, conf map[string]interface{}) (global.Service, erro } return &svc{ - config: c, + config: &c, gtwClient: gtw, downloader: downloader.NewDownloader(gtw, rhttp.Insecure(c.Insecure), rhttp.Timeout(time.Duration(c.Timeout*int64(time.Second)))), walker: walker.NewWalker(gtw), @@ -103,7 +100,7 @@ func New(ctx context.Context, conf map[string]interface{}) (global.Service, erro }, nil } -func (c *Config) init() { +func (c *Config) ApplyDefaults() { if c.Prefix == "" { c.Prefix = "download_archive" } diff --git a/internal/http/services/datagateway/datagateway.go b/internal/http/services/datagateway/datagateway.go index 9795b5cee5..e1b80d8b21 100644 --- a/internal/http/services/datagateway/datagateway.go +++ b/internal/http/services/datagateway/datagateway.go @@ -32,8 +32,8 @@ import ( "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/golang-jwt/jwt" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -56,12 +56,12 @@ type transferClaims struct { } type config struct { Prefix string `mapstructure:"prefix"` - TransferSharedSecret string `mapstructure:"transfer_shared_secret"` + TransferSharedSecret string `mapstructure:"transfer_shared_secret" validate:"required"` Timeout int64 `mapstructure:"timeout"` Insecure bool `mapstructure:"insecure" docs:"false;Whether to skip certificate checks when sending requests."` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Prefix == "" { c.Prefix = "datagateway" } @@ -77,18 +77,16 @@ type svc struct { // New returns a new datagateway. func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - conf := &config{} - if err := mapstructure.Decode(m, conf); err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - conf.init() - s := &svc{ - conf: conf, + conf: &c, client: rhttp.GetHTTPClient( - rhttp.Timeout(time.Duration(conf.Timeout*int64(time.Second))), - rhttp.Insecure(conf.Insecure), + rhttp.Timeout(time.Duration(c.Timeout*int64(time.Second))), + rhttp.Insecure(c.Insecure), ), } s.setHandler() diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index df35db7fa6..997abc0c9d 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -29,7 +29,7 @@ import ( "github.com/cs3org/reva/pkg/rhttp/router" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -45,7 +45,7 @@ type config struct { Insecure bool `mapstructure:"insecure" docs:"false;Whether to skip certificate checks when sending requests."` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Prefix == "" { c.Prefix = "data" } @@ -63,26 +63,24 @@ type svc struct { // New returns a new datasvc. func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - conf := &config{} - if err := mapstructure.Decode(m, conf); err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - conf.init() - - fs, err := getFS(ctx, conf) + fs, err := getFS(ctx, &c) if err != nil { return nil, err } - dataTXs, err := getDataTXs(ctx, conf, fs) + dataTXs, err := getDataTXs(ctx, &c, fs) if err != nil { return nil, err } s := &svc{ storage: fs, - conf: conf, + conf: &c, dataTXs: dataTXs, } diff --git a/internal/http/services/helloworld/helloworld.go b/internal/http/services/helloworld/helloworld.go index c136aeb6a8..11f0b65105 100644 --- a/internal/http/services/helloworld/helloworld.go +++ b/internal/http/services/helloworld/helloworld.go @@ -24,7 +24,7 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -33,14 +33,12 @@ func init() { // New returns a new helloworld service. func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - conf := &config{} - if err := mapstructure.Decode(m, conf); err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - conf.init() - - return &svc{conf: conf}, nil + return &svc{conf: &c}, nil } // Close performs cleanup. @@ -53,7 +51,7 @@ type config struct { HelloMessage string `mapstructure:"message"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.HelloMessage == "" { c.HelloMessage = "Hello World!" } diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index f7c8a78845..be9eebbd67 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -28,7 +28,7 @@ import ( "github.com/cs3org/reva/pkg/mentix/exchangers" "github.com/cs3org/reva/pkg/mentix/meshdata" "github.com/cs3org/reva/pkg/rhttp/global" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "github.com/rs/zerolog" ) @@ -97,16 +97,6 @@ func (s *svc) startBackgroundService() { }() } -func parseConfig(m map[string]interface{}) (*config.Configuration, error) { - cfg := &config.Configuration{} - if err := mapstructure.Decode(m, &cfg); err != nil { - return nil, errors.Wrap(err, "mentix: error decoding configuration") - } - applyInternalConfig(m, cfg) - applyDefaultConfig(cfg) - return cfg, nil -} - func applyInternalConfig(m map[string]interface{}, conf *config.Configuration) { getSubsections := func(section string) []string { subsections := make([]string, 0, 5) @@ -169,24 +159,23 @@ func applyDefaultConfig(conf *config.Configuration) { // New returns a new Mentix service. func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - // Prepare the configuration - conf, err := parseConfig(m) - if err != nil { + var c config.Configuration + if err := cfg.Decode(m, &c); err != nil { return nil, err } - - conf.Init() + applyInternalConfig(m, &c) + applyDefaultConfig(&c) // Create the Mentix instance log := appctx.GetLogger(ctx) - mntx, err := mentix.New(conf, log) + mntx, err := mentix.New(&c, log) if err != nil { return nil, errors.Wrap(err, "mentix: error creating Mentix") } // Create the service and start its background activity s := &svc{ - conf: conf, + conf: &c, mntx: mntx, log: log, stopSignal: make(chan struct{}), diff --git a/internal/http/services/metrics/metrics.go b/internal/http/services/metrics/metrics.go index fb02a01bf7..fe9db0128c 100644 --- a/internal/http/services/metrics/metrics.go +++ b/internal/http/services/metrics/metrics.go @@ -30,7 +30,7 @@ import ( "github.com/cs3org/reva/pkg/metrics" "github.com/cs3org/reva/pkg/metrics/config" "github.com/cs3org/reva/pkg/rhttp/global" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -70,16 +70,13 @@ func (s *svc) Handler() http.Handler { // New returns a new metrics service. func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - // Prepare the configuration - conf := &config.Config{} - if err := mapstructure.Decode(m, conf); err != nil { + var c config.Config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - conf.Init() - // initialize metrics using the configuration - err := metrics.Init(conf) + err := metrics.Init(&c) if err != nil { return nil, err } diff --git a/internal/http/services/ocmd/ocm.go b/internal/http/services/ocmd/ocm.go index 4570178aef..13c901539a 100644 --- a/internal/http/services/ocmd/ocm.go +++ b/internal/http/services/ocmd/ocm.go @@ -25,8 +25,8 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/go-chi/chi/v5" - "github.com/mitchellh/mapstructure" ) func init() { @@ -35,11 +35,11 @@ func init() { type config struct { Prefix string `mapstructure:"prefix"` - GatewaySvc string `mapstructure:"gatewaysvc"` + GatewaySvc string `mapstructure:"gatewaysvc" validate:"required"` ExposeRecipientDisplayName bool `mapstructure:"expose_recipient_display_name"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) if c.Prefix == "" { c.Prefix = "ocm" @@ -54,15 +54,14 @@ type svc struct { // New returns a new ocmd object, that implements // the OCM APIs specified in https://cs3org.github.io/OCM-API/docs.html func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - conf := &config{} - if err := mapstructure.Decode(m, conf); err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - conf.init() r := chi.NewRouter() s := &svc{ - Conf: conf, + Conf: &c, router: r, } diff --git a/internal/http/services/ocmprovider/ocmprovider.go b/internal/http/services/ocmprovider/ocmprovider.go index f2f7afb71e..dfb0c616d4 100644 --- a/internal/http/services/ocmprovider/ocmprovider.go +++ b/internal/http/services/ocmprovider/ocmprovider.go @@ -26,7 +26,7 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -62,7 +62,7 @@ type svc struct { data *discoveryData } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.OCMPrefix == "" { c.OCMPrefix = "ocm" } @@ -126,13 +126,11 @@ func (c *config) prepare() *discoveryData { // the OCM discovery endpoint specified in // https://cs3org.github.io/OCM-API/docs.html?repo=OCM-API&user=cs3org#/paths/~1ocm-provider/get func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - conf := &config{} - if err := mapstructure.Decode(m, conf); err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - - conf.init() - return &svc{data: conf.prepare()}, nil + return &svc{data: c.prepare()}, nil } // Close performs cleanup. diff --git a/internal/http/services/owncloud/ocdav/ocdav.go b/internal/http/services/owncloud/ocdav/ocdav.go index 6374a6a29a..0e93d56737 100644 --- a/internal/http/services/owncloud/ocdav/ocdav.go +++ b/internal/http/services/owncloud/ocdav/ocdav.go @@ -42,7 +42,7 @@ import ( "github.com/cs3org/reva/pkg/storage/favorite" "github.com/cs3org/reva/pkg/storage/favorite/registry" "github.com/cs3org/reva/pkg/storage/utils/templates" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -114,7 +114,7 @@ type Config struct { Notifications map[string]interface{} `mapstructure:"notifications" docs:"Settingsg for the Notification Helper"` } -func (c *Config) init() { +func (c *Config) ApplyDefaults() { // note: default c.Prefix is an empty string c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) @@ -145,36 +145,34 @@ func getFavoritesManager(c *Config) (favorite.Manager, error) { // New returns a new ocdav. func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - conf := &Config{} - if err := mapstructure.Decode(m, conf); err != nil { + var c Config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - conf.init() - - fm, err := getFavoritesManager(conf) + fm, err := getFavoritesManager(&c) if err != nil { return nil, err } log := appctx.GetLogger(ctx) s := &svc{ - c: conf, + c: &c, webDavHandler: new(WebDavHandler), davHandler: new(DavHandler), client: rhttp.GetHTTPClient( - rhttp.Timeout(time.Duration(conf.Timeout*int64(time.Second))), - rhttp.Insecure(conf.Insecure), + rhttp.Timeout(time.Duration(c.Timeout*int64(time.Second))), + rhttp.Insecure(c.Insecure), ), favoritesManager: fm, - notificationHelper: notificationhelper.New("ocdav", conf.Notifications, log), + notificationHelper: notificationhelper.New("ocdav", c.Notifications, log), } - // initialize handlers and set default configs - if err := s.webDavHandler.init(conf.WebdavNamespace, true); err != nil { + // initialize handlers and set default cigs + if err := s.webDavHandler.init(c.WebdavNamespace, true); err != nil { return nil, err } - if err := s.davHandler.init(conf); err != nil { + if err := s.davHandler.init(&c); err != nil { return nil, err } return s, nil diff --git a/internal/http/services/owncloud/ocs/config/config.go b/internal/http/services/owncloud/ocs/config/config.go index 123cdd47be..a9ae7ac513 100644 --- a/internal/http/services/owncloud/ocs/config/config.go +++ b/internal/http/services/owncloud/ocs/config/config.go @@ -49,7 +49,7 @@ type Config struct { } // Init sets sane defaults. -func (c *Config) Init() { +func (c *Config) ApplyDefaults() { if c.Prefix == "" { c.Prefix = "ocs" } diff --git a/internal/http/services/owncloud/ocs/ocs.go b/internal/http/services/owncloud/ocs/ocs.go index 05ce3f8819..49c39c5f68 100644 --- a/internal/http/services/owncloud/ocs/ocs.go +++ b/internal/http/services/owncloud/ocs/ocs.go @@ -34,8 +34,8 @@ import ( "github.com/cs3org/reva/internal/http/services/owncloud/ocs/response" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/go-chi/chi/v5" - "github.com/mitchellh/mapstructure" "github.com/rs/zerolog" ) @@ -50,16 +50,14 @@ type svc struct { } func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - conf := &config.Config{} - if err := mapstructure.Decode(m, conf); err != nil { + var c config.Config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - conf.Init() - r := chi.NewRouter() s := &svc{ - c: conf, + c: &c, router: r, } @@ -68,9 +66,9 @@ func New(ctx context.Context, m map[string]interface{}) (global.Service, error) return nil, err } - if conf.CacheWarmupDriver == "first-request" && conf.ResourceInfoCacheTTL > 0 { + if c.CacheWarmupDriver == "first-request" && c.ResourceInfoCacheTTL > 0 { s.warmupCacheTracker = ttlcache.NewCache() - _ = s.warmupCacheTracker.SetTTL(time.Second * time.Duration(conf.ResourceInfoCacheTTL)) + _ = s.warmupCacheTracker.SetTTL(time.Second * time.Duration(c.ResourceInfoCacheTTL)) } return s, nil diff --git a/internal/http/services/preferences/preferences.go b/internal/http/services/preferences/preferences.go index d5f8bdf506..f4a99aa3bf 100644 --- a/internal/http/services/preferences/preferences.go +++ b/internal/http/services/preferences/preferences.go @@ -29,8 +29,8 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/go-chi/chi/v5" - "github.com/mitchellh/mapstructure" ) func init() { @@ -43,7 +43,7 @@ type Config struct { GatewaySvc string `mapstructure:"gatewaysvc"` } -func (c *Config) init() { +func (c *Config) ApplyDefaults() { if c.Prefix == "" { c.Prefix = "preferences" } @@ -57,15 +57,14 @@ type svc struct { // New returns a new ocmd object. func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - conf := &Config{} - if err := mapstructure.Decode(m, conf); err != nil { + var c Config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - conf.init() r := chi.NewRouter() s := &svc{ - conf: conf, + conf: &c, router: r, } diff --git a/internal/http/services/prometheus/prometheus.go b/internal/http/services/prometheus/prometheus.go index 88c18d81fe..b458bffbd4 100644 --- a/internal/http/services/prometheus/prometheus.go +++ b/internal/http/services/prometheus/prometheus.go @@ -24,7 +24,7 @@ import ( "contrib.go.opencensus.io/exporter/prometheus" "github.com/cs3org/reva/pkg/rhttp/global" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "go.opencensus.io/stats/view" ) @@ -35,13 +35,11 @@ func init() { // New returns a new prometheus service. func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - conf := &config{} - if err := mapstructure.Decode(m, conf); err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - conf.init() - pe, err := prometheus.NewExporter(prometheus.Options{ Namespace: "revad", }) @@ -50,14 +48,14 @@ func New(ctx context.Context, m map[string]interface{}) (global.Service, error) } view.RegisterExporter(pe) - return &svc{prefix: conf.Prefix, h: pe}, nil + return &svc{prefix: c.Prefix, h: pe}, nil } type config struct { Prefix string `mapstructure:"prefix"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Prefix == "" { c.Prefix = "metrics" } diff --git a/internal/http/services/reverseproxy/reverseproxy.go b/internal/http/services/reverseproxy/reverseproxy.go index 2922e3b449..2c81b20815 100644 --- a/internal/http/services/reverseproxy/reverseproxy.go +++ b/internal/http/services/reverseproxy/reverseproxy.go @@ -28,8 +28,8 @@ import ( ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/rhttp/global" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/go-chi/chi/v5" - "github.com/mitchellh/mapstructure" ) func init() { @@ -45,7 +45,7 @@ type config struct { ProxyRulesJSON string `mapstructure:"proxy_rules_json"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.ProxyRulesJSON == "" { c.ProxyRulesJSON = "/etc/revad/proxy_rules.json" } @@ -57,13 +57,12 @@ type svc struct { // New returns an instance of the reverse proxy service. func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - conf := &config{} - if err := mapstructure.Decode(m, conf); err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - conf.init() - f, err := os.ReadFile(conf.ProxyRulesJSON) + f, err := os.ReadFile(c.ProxyRulesJSON) if err != nil { return nil, err } diff --git a/internal/http/services/sciencemesh/sciencemesh.go b/internal/http/services/sciencemesh/sciencemesh.go index a728c82a77..6f3446968f 100644 --- a/internal/http/services/sciencemesh/sciencemesh.go +++ b/internal/http/services/sciencemesh/sciencemesh.go @@ -20,15 +20,14 @@ package sciencemesh import ( "context" - "errors" "net/http" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/smtpclient" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/go-chi/chi/v5" - "github.com/mitchellh/mapstructure" ) func init() { @@ -37,22 +36,14 @@ func init() { // New returns a new sciencemesh service. func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - conf := &config{} - if err := mapstructure.Decode(m, conf); err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - conf.init() - if conf.ProviderDomain == "" { - return nil, errors.New("sciencemesh: provider_domain is missing from configuration") - } - if conf.MeshDirectoryURL == "" { - return nil, errors.New("sciencemesh: mesh_directory_url is missing from configuration") - } - r := chi.NewRouter() s := &svc{ - conf: conf, + conf: &c, router: r, } @@ -70,19 +61,22 @@ func (s *svc) Close() error { type config struct { Prefix string `mapstructure:"prefix"` - SMTPCredentials *smtpclient.SMTPCredentials `mapstructure:"smtp_credentials"` - GatewaySvc string `mapstructure:"gatewaysvc"` - MeshDirectoryURL string `mapstructure:"mesh_directory_url"` - ProviderDomain string `mapstructure:"provider_domain"` + SMTPCredentials *smtpclient.SMTPCredentials `mapstructure:"smtp_credentials" validate:"required"` + GatewaySvc string `mapstructure:"gatewaysvc" validate:"required"` + MeshDirectoryURL string `mapstructure:"mesh_directory_url" validate:"required"` + ProviderDomain string `mapstructure:"provider_domain" validate:"required"` SubjectTemplate string `mapstructure:"subject_template"` BodyTemplatePath string `mapstructure:"body_template_path"` OCMMountPoint string `mapstructure:"ocm_mount_point"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Prefix == "" { c.Prefix = "sciencemesh" } + if c.OCMMountPoint == "" { + c.OCMMountPoint = "/ocm" + } c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) } diff --git a/internal/http/services/siteacc/siteacc.go b/internal/http/services/siteacc/siteacc.go index 9f44535c7e..2d01b498d6 100644 --- a/internal/http/services/siteacc/siteacc.go +++ b/internal/http/services/siteacc/siteacc.go @@ -26,7 +26,7 @@ import ( "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/siteacc" "github.com/cs3org/reva/pkg/siteacc/config" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "github.com/rs/zerolog" ) @@ -66,21 +66,6 @@ func (s *svc) Handler() http.Handler { return s.siteacc.RequestHandler() } -func parseConfig(m map[string]interface{}) (*config.Configuration, error) { - conf := &config.Configuration{} - if err := mapstructure.Decode(m, &conf); err != nil { - return nil, errors.Wrap(err, "error decoding configuration") - } - applyDefaultConfig(conf) - conf.Cleanup() - - if conf.Webserver.URL == "" { - return nil, errors.Errorf("no webserver URL specified") - } - - return conf, nil -} - func applyDefaultConfig(conf *config.Configuration) { if conf.Prefix == "" { conf.Prefix = serviceName @@ -106,22 +91,24 @@ func applyDefaultConfig(conf *config.Configuration) { // New returns a new Site Accounts service. func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - // Prepare the configuration - conf, err := parseConfig(m) - if err != nil { + var c config.Configuration + if err := cfg.Decode(m, &c); err != nil { return nil, err } + applyDefaultConfig(&c) + c.Cleanup() + // Create the sites accounts instance log := appctx.GetLogger(ctx) - siteacc, err := siteacc.New(conf, log) + siteacc, err := siteacc.New(&c, log) if err != nil { return nil, errors.Wrap(err, "error creating the sites accounts service") } // Create the service s := &svc{ - conf: conf, + conf: &c, log: log, siteacc: siteacc, } diff --git a/internal/http/services/sysinfo/sysinfo.go b/internal/http/services/sysinfo/sysinfo.go index d28e98dd35..05e58a310b 100644 --- a/internal/http/services/sysinfo/sysinfo.go +++ b/internal/http/services/sysinfo/sysinfo.go @@ -25,8 +25,7 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/sysinfo" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -37,6 +36,12 @@ type config struct { Prefix string `mapstructure:"prefix"` } +func (c *config) ApplyDefaults() { + if c.Prefix == "" { + c.Prefix = serviceName + } +} + type svc struct { conf *config } @@ -78,32 +83,16 @@ func (s *svc) getJSONData() string { return "" } -func parseConfig(m map[string]interface{}) (*config, error) { - cfg := &config{} - if err := mapstructure.Decode(m, &cfg); err != nil { - return nil, errors.Wrap(err, "sysinfo: error decoding configuration") - } - applyDefaultConfig(cfg) - return cfg, nil -} - -func applyDefaultConfig(conf *config) { - if conf.Prefix == "" { - conf.Prefix = serviceName - } -} - // New returns a new SysInfo service. func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - // Prepare the configuration - conf, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } // Create the service s := &svc{ - conf: conf, + conf: &c, } return s, nil } diff --git a/internal/http/services/wellknown/wellknown.go b/internal/http/services/wellknown/wellknown.go index 159a8d9ed7..2107da3288 100644 --- a/internal/http/services/wellknown/wellknown.go +++ b/internal/http/services/wellknown/wellknown.go @@ -25,7 +25,7 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/rhttp/router" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -44,7 +44,7 @@ type config struct { EndSessionEndpoint string `mapstructure:"end_session_endpoint"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Prefix == "" { c.Prefix = ".well-known" } @@ -57,15 +57,13 @@ type svc struct { // New returns a new webuisvc. func New(ctx context.Context, m map[string]interface{}) (global.Service, error) { - conf := &config{} - if err := mapstructure.Decode(m, conf); err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - conf.init() - s := &svc{ - conf: conf, + conf: &c, } s.setHandler() return s, nil diff --git a/internal/serverless/services/helloworld/helloworld.go b/internal/serverless/services/helloworld/helloworld.go index 50120cda11..46bf5cc190 100644 --- a/internal/serverless/services/helloworld/helloworld.go +++ b/internal/serverless/services/helloworld/helloworld.go @@ -24,9 +24,11 @@ import ( "os" "time" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rserverless" "github.com/mitchellh/mapstructure" "github.com/rs/zerolog" + "github.com/rs/zerolog/log" ) type config struct { @@ -50,7 +52,7 @@ func init() { } // New returns a new helloworld service. -func New(m map[string]interface{}, log *zerolog.Logger) (rserverless.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rserverless.Service, error) { conf := &config{} conf.init() @@ -64,6 +66,7 @@ func New(m map[string]interface{}, log *zerolog.Logger) (rserverless.Service, er return nil, err } + log := appctx.GetLogger(ctx) s := &svc{ conf: conf, log: log, diff --git a/internal/serverless/services/notifications/notifications.go b/internal/serverless/services/notifications/notifications.go index b51b163fd4..a1df4b066a 100644 --- a/internal/serverless/services/notifications/notifications.go +++ b/internal/serverless/services/notifications/notifications.go @@ -26,6 +26,7 @@ import ( "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/notification" "github.com/cs3org/reva/pkg/notification/handler" @@ -46,7 +47,7 @@ type config struct { NatsAddress string `mapstructure:"nats_address" docs:";The NATS server address."` NatsToken string `mapstructure:"nats_token" docs:"The token to authenticate against the NATS server"` NatsPrefix string `mapstructure:"nats_prefix" docs:"reva-notifications;The notifications NATS stream."` - HandlerConf map[string]interface{} `mapstructure:"handlers" docs:";Settings for the different notification handlers."` + HandlerConf map[string]map[string]interface{} `mapstructure:"handlers" docs:";Settings for the different notification handlers."` GroupingInterval int `mapstructure:"grouping_interval" docs:"60;Time in seconds to group incoming notification triggers"` GroupingMaxSize int `mapstructure:"grouping_max_size" docs:"100;Maximum number of notifications to group"` StorageDriver string `mapstructure:"storage_driver" docs:"mysql;The driver used to store notifications"` @@ -63,6 +64,7 @@ func defaultConfig() *config { } type svc struct { + ctx context.Context nc *nats.Conn js nats.JetStreamContext kv nats.KeyValue @@ -78,28 +80,30 @@ func init() { rserverless.Register("notifications", New) } -func getNotificationManager(c *config, l *zerolog.Logger) (notification.Manager, error) { +func getNotificationManager(ctx context.Context, c *config) (notification.Manager, error) { if f, ok := notificationManagerRegistry.NewFuncs[c.StorageDriver]; ok { - return f(c.StorageDrivers[c.StorageDriver]) + return f(ctx, c.StorageDrivers[c.StorageDriver]) } return nil, errtypes.NotFound(fmt.Sprintf("storage driver %s not found", c.StorageDriver)) } // New returns a new Notifications service. -func New(m map[string]interface{}, log *zerolog.Logger) (rserverless.Service, error) { +func New(ctx context.Context, m map[string]interface{}) (rserverless.Service, error) { conf := defaultConfig() if err := mapstructure.Decode(m, conf); err != nil { return nil, err } - nm, err := getNotificationManager(conf, log) + log := appctx.GetLogger(ctx) + nm, err := getNotificationManager(ctx, conf) if err != nil { return nil, err } log.Info().Msgf("notification storage %s initialized", conf.StorageDriver) s := &svc{ + ctx: ctx, conf: conf, log: log, nm: nm, @@ -111,7 +115,7 @@ func New(m map[string]interface{}, log *zerolog.Logger) (rserverless.Service, er // Start starts the Notifications service. func (s *svc) Start() { s.templates = *templateRegistry.New() - s.handlers = handlerRegistry.InitHandlers(s.conf.HandlerConf, s.log) + s.handlers = handlerRegistry.InitHandlers(s.ctx, s.conf.HandlerConf) s.accumulators = make(map[string]*accumulator.Accumulator[trigger.Trigger]) s.log.Debug().Msgf("connecting to nats server at %s", s.conf.NatsAddress) diff --git a/pkg/app/provider/demo/demo.go b/pkg/app/provider/demo/demo.go index 91df5b83f8..c17403fe12 100644 --- a/pkg/app/provider/demo/demo.go +++ b/pkg/app/provider/demo/demo.go @@ -29,7 +29,7 @@ import ( typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/app" "github.com/cs3org/reva/pkg/app/provider/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -58,19 +58,11 @@ type config struct { IFrameUIProvider string `mapstructure:"iframe_ui_provider"` } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - // New returns an implementation to of the app.Provider interface that // connects to an application in the backend. func New(ctx context.Context, m map[string]interface{}) (app.Provider, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } return &demoProvider{iframeUIProvider: c.IFrameUIProvider}, nil diff --git a/pkg/app/provider/wopi/wopi.go b/pkg/app/provider/wopi/wopi.go index 5837950234..86bdf2c07b 100644 --- a/pkg/app/provider/wopi/wopi.go +++ b/pkg/app/provider/wopi/wopi.go @@ -51,9 +51,9 @@ import ( "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/cfg" gomime "github.com/glpatcern/go-mime" "github.com/golang-jwt/jwt" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -90,12 +90,14 @@ type config struct { InsecureConnections bool `mapstructure:"insecure_connections"` } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err +func (c *config) ApplyDefaults() { + if c.AppIntURL == "" { + c.AppIntURL = c.AppURL + } + if c.IOPSecret == "" { + c.IOPSecret = os.Getenv("REVA_APPPROVIDER_IOPSECRET") } - return c, nil + c.JWTSecret = sharedconf.GetJWTSecret(c.JWTSecret) } type wopiProvider struct { @@ -107,20 +109,12 @@ type wopiProvider struct { // New returns an implementation of the app.Provider interface that // connects to an application in the backend. func New(ctx context.Context, m map[string]interface{}) (app.Provider, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - if c.AppIntURL == "" { - c.AppIntURL = c.AppURL - } - if c.IOPSecret == "" { - c.IOPSecret = os.Getenv("REVA_APPPROVIDER_IOPSECRET") - } - c.JWTSecret = sharedconf.GetJWTSecret(c.JWTSecret) - - appURLs, err := getAppURLs(c) + appURLs, err := getAppURLs(&c) if err != nil { return nil, err } @@ -134,7 +128,7 @@ func New(ctx context.Context, m map[string]interface{}) (app.Provider, error) { } return &wopiProvider{ - conf: c, + conf: &c, wopiClient: wopiClient, appURLs: appURLs, }, nil diff --git a/pkg/app/registry/static/static.go b/pkg/app/registry/static/static.go index 9890f6d25c..88e8e63108 100644 --- a/pkg/app/registry/static/static.go +++ b/pkg/app/registry/static/static.go @@ -30,7 +30,7 @@ import ( "github.com/cs3org/reva/pkg/app" "github.com/cs3org/reva/pkg/app/registry/registry" "github.com/cs3org/reva/pkg/errtypes" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/rs/zerolog/log" orderedmap "github.com/wk8/go-ordered-map" ) @@ -57,20 +57,6 @@ type config struct { MimeTypes []*mimeTypeConfig `mapstructure:"mime_types"` } -func (c *config) init() { - if len(c.Providers) == 0 { - c.Providers = []*registrypb.ProviderInfo{} - } -} - -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - type manager struct { providers map[string]*registrypb.ProviderInfo mimetypes *orderedmap.OrderedMap // map[string]*mimeTypeConfig -> map the mime type to the addresses of the corresponding providers @@ -79,11 +65,10 @@ type manager struct { // New returns an implementation of the app.Registry interface. func New(ctx context.Context, m map[string]interface{}) (app.Registry, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() mimetypes := orderedmap.New() diff --git a/pkg/appauth/manager/json/json.go b/pkg/appauth/manager/json/json.go index 0a488c6cb9..1ff1a667f9 100644 --- a/pkg/appauth/manager/json/json.go +++ b/pkg/appauth/manager/json/json.go @@ -34,7 +34,7 @@ import ( "github.com/cs3org/reva/pkg/appauth/manager/registry" ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/errtypes" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "github.com/sethvargo/go-password/password" "golang.org/x/crypto/bcrypt" @@ -59,25 +59,23 @@ type jsonManager struct { // New returns a new mgr. func New(ctx context.Context, m map[string]interface{}) (appauth.Manager, error) { - c, err := parseConfig(m) - if err != nil { - return nil, errors.Wrap(err, "error creating a new manager") + var c config + if err := cfg.Decode(m, &c); err != nil { + return nil, err } - c.init() - // load or create file manager, err := loadOrCreate(c.File) if err != nil { return nil, errors.Wrap(err, "error loading the file containing the application passwords") } - manager.config = c + manager.config = &c return manager, nil } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.File == "" { c.File = "/var/tmp/reva/appauth.json" } @@ -89,14 +87,6 @@ func (c *config) init() { } } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - func loadOrCreate(file string) (*jsonManager, error) { stat, err := os.Stat(file) if os.IsNotExist(err) || stat.Size() == 0 { diff --git a/pkg/auth/manager/appauth/appauth.go b/pkg/auth/manager/appauth/appauth.go index daf292a739..75f64c274e 100644 --- a/pkg/auth/manager/appauth/appauth.go +++ b/pkg/auth/manager/appauth/appauth.go @@ -29,7 +29,8 @@ import ( "github.com/cs3org/reva/pkg/auth/manager/registry" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -38,7 +39,11 @@ func init() { } type manager struct { - GatewayAddr string `mapstructure:"gateway_addr"` + GatewayAddr string `mapstructure:"gatewaysvc"` +} + +func (m *manager) ApplyDefaults() { + m.GatewayAddr = sharedconf.GetGatewaySVC(m.GatewayAddr) } // New returns a new auth Manager. @@ -52,11 +57,8 @@ func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { } func (m *manager) Configure(ml map[string]interface{}) error { - err := mapstructure.Decode(ml, m) - if err != nil { - return errors.Wrap(err, "error decoding conf") - } - return nil + err := cfg.Decode(ml, m) + return errors.Wrap(err, "appauth: error decoding config") } func (m *manager) Authenticate(ctx context.Context, username, password string) (*user.User, map[string]*authpb.Scope, error) { diff --git a/pkg/auth/manager/json/json.go b/pkg/auth/manager/json/json.go index 5e095c062b..36c6d41641 100644 --- a/pkg/auth/manager/json/json.go +++ b/pkg/auth/manager/json/json.go @@ -30,7 +30,7 @@ import ( "github.com/cs3org/reva/pkg/auth/manager/registry" "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/errtypes" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -61,22 +61,12 @@ type config struct { Users string `mapstructure:"users"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Users == "" { c.Users = "/etc/revad/users.json" } } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - c.init() - return c, nil -} - // New returns a new auth Manager. func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { mgr := &manager{} @@ -88,9 +78,9 @@ func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { } func (m *manager) Configure(ml map[string]interface{}) error { - c, err := parseConfig(ml) - if err != nil { - return err + var c config + if err := cfg.Decode(ml, &c); err != nil { + return errors.Wrap(err, "json: error decoding config") } m.credentials = map[string]*Credentials{} diff --git a/pkg/auth/manager/json/json_test.go b/pkg/auth/manager/json/json_test.go index 45cda03f6f..57744078d4 100644 --- a/pkg/auth/manager/json/json_test.go +++ b/pkg/auth/manager/json/json_test.go @@ -44,14 +44,9 @@ func TestGetManagerWithInvalidUser(t *testing.T) { { "Boolean in user", false, - "error decoding conf: 1 error(s) decoding:\n\n* " + + "json: error decoding config: 1 error(s) decoding:\n\n* " + "'users' expected type 'string', got unconvertible type 'bool', value: 'false'", }, - { - "Nil in user", - nil, - "open /etc/revad/users.json: no such file or directory", - }, } for _, tt := range tests { @@ -60,9 +55,7 @@ func TestGetManagerWithInvalidUser(t *testing.T) { "users": tt.user, } - manager, err := New(ctx, input) - - assert.Empty(t, manager) + _, err := New(ctx, input) assert.EqualError(t, err, tt.expectedError) }) } diff --git a/pkg/auth/manager/ldap/ldap.go b/pkg/auth/manager/ldap/ldap.go index e9f871720d..28abc4ba62 100644 --- a/pkg/auth/manager/ldap/ldap.go +++ b/pkg/auth/manager/ldap/ldap.go @@ -31,12 +31,11 @@ import ( "github.com/cs3org/reva/pkg/auth/manager/registry" "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/logger" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/go-ldap/ldap/v3" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -87,17 +86,6 @@ var ldapDefaults = attributes{ GIDNumber: "gidNumber", } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{ - Schema: ldapDefaults, - } - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - // New returns an auth manager implementation that connects to a LDAP server to validate the user. func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { manager := &mgr{} @@ -105,19 +93,12 @@ func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { if err != nil { return nil, err } + // backwards compatibility + appctx.GetLogger(ctx).Warn().Msg("userfilter is deprecated, use a loginfilter like `(&(objectclass=posixAccount)(|(cn={{login}}))(mail={{login}}))`") return manager, nil } -func (am *mgr) Configure(m map[string]interface{}) error { - c, err := parseConfig(m) - if err != nil { - return err - } - - // backwards compatibility - if c.UserFilter != "" { - logger.New().Warn().Msg("userfilter is deprecated, use a loginfilter like `(&(objectclass=posixAccount)(|(cn={{login}}))(mail={{login}}))`") - } +func (c *config) ApplyDefaults() { if c.LoginFilter == "" { c.LoginFilter = c.UserFilter c.LoginFilter = strings.ReplaceAll(c.LoginFilter, "%s", "{{login}}") @@ -127,7 +108,15 @@ func (am *mgr) Configure(m map[string]interface{}) error { } c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) - am.c = c +} + +func (am *mgr) Configure(m map[string]interface{}) error { + var c config + c.Schema = ldapDefaults + if err := cfg.Decode(m, &c); err != nil { + return errors.Wrap(err, "ldap: error decoding config") + } + am.c = &c return nil } diff --git a/pkg/auth/manager/machine/machine.go b/pkg/auth/manager/machine/machine.go index 6d557e24c2..9281b6ed20 100644 --- a/pkg/auth/manager/machine/machine.go +++ b/pkg/auth/manager/machine/machine.go @@ -30,7 +30,8 @@ import ( "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -43,20 +44,21 @@ var claims = []string{"mail", "uid", "username", "gid", "userid"} type manager struct { APIKey string `mapstructure:"api_key"` - GatewayAddr string `mapstructure:"gateway_addr"` + GatewayAddr string `mapstructure:"gatewaysvc"` } func init() { registry.Register("machine", New) } +func (m *manager) ApplyDefaults() { + m.GatewayAddr = sharedconf.GetGatewaySVC(m.GatewayAddr) +} + // Configure parses the map conf. func (m *manager) Configure(conf map[string]interface{}) error { - err := mapstructure.Decode(conf, m) - if err != nil { - return errors.Wrap(err, "error decoding conf") - } - return nil + err := cfg.Decode(conf, m) + return errors.Wrap(err, "machine: error decoding config") } // New creates a new manager for the 'machine' authentication. diff --git a/pkg/auth/manager/nextcloud/nextcloud.go b/pkg/auth/manager/nextcloud/nextcloud.go index 502016b61d..f5a0a4ec39 100644 --- a/pkg/auth/manager/nextcloud/nextcloud.go +++ b/pkg/auth/manager/nextcloud/nextcloud.go @@ -33,7 +33,7 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth" "github.com/cs3org/reva/pkg/auth/manager/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -63,27 +63,14 @@ type Action struct { argS string } -func (c *AuthManagerConfig) init() { -} - -func parseConfig(m map[string]interface{}) (*AuthManagerConfig, error) { - c := &AuthManagerConfig{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - // New returns an auth manager implementation that verifies against a Nextcloud backend. func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { - c, err := parseConfig(m) - if err != nil { - return nil, err + var c AuthManagerConfig + if err := cfg.Decode(m, &c); err != nil { + return nil, errors.Wrap(err, "nextcloud: error decoding config") } - c.init() - return NewAuthManager(c) + return NewAuthManager(&c) } // NewAuthManager returns a new Nextcloud-based AuthManager. diff --git a/pkg/auth/manager/ocmshares/ocmshares.go b/pkg/auth/manager/ocmshares/ocmshares.go index b290964bee..1987797801 100644 --- a/pkg/auth/manager/ocmshares/ocmshares.go +++ b/pkg/auth/manager/ocmshares/ocmshares.go @@ -37,7 +37,7 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/utils" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -51,10 +51,10 @@ type manager struct { } type config struct { - GatewayAddr string `mapstructure:"gateway_addr"` + GatewayAddr string `mapstructure:"gatewaysvc"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { c.GatewayAddr = sharedconf.GetGatewaySVC(c.GatewayAddr) } @@ -64,24 +64,21 @@ func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { if err := mgr.Configure(m); err != nil { return nil, err } + gw, err := pool.GetGatewayServiceClient(pool.Endpoint(mgr.c.GatewayAddr)) + if err != nil { + return nil, err + } + mgr.gw = gw return &mgr, nil } func (m *manager) Configure(ml map[string]interface{}) error { var c config - if err := mapstructure.Decode(ml, &c); err != nil { - return errors.Wrap(err, "error decoding config") + if err := cfg.Decode(ml, &c); err != nil { + return errors.Wrap(err, "ocmshares: error decoding config") } - c.init() m.c = &c - - gw, err := pool.GetGatewayServiceClient(pool.Endpoint(c.GatewayAddr)) - if err != nil { - return err - } - m.gw = gw - return nil } diff --git a/pkg/auth/manager/oidc/oidc.go b/pkg/auth/manager/oidc/oidc.go index 656077111d..4df459d9af 100644 --- a/pkg/auth/manager/oidc/oidc.go +++ b/pkg/auth/manager/oidc/oidc.go @@ -41,9 +41,9 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/golang-jwt/jwt" "github.com/juliangruber/go-intersect" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "golang.org/x/oauth2" ) @@ -76,7 +76,7 @@ type oidcUserMapping struct { Username string `mapstructure:"username" json:"username"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.IDClaim == "" { // sub is stable and defined as unique. the user manager needs to take care of the sub to user metadata lookup c.IDClaim = "sub" @@ -94,15 +94,6 @@ func (c *config) init() { c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - // New returns an auth manager implementation that verifies the oidc token and obtains the user claims. func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { manager := &mgr{ @@ -116,12 +107,11 @@ func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { } func (am *mgr) Configure(m map[string]interface{}) error { - c, err := parseConfig(m) - if err != nil { - return err + var c config + if err := cfg.Decode(m, &c); err != nil { + return errors.Wrap(err, "oidc: error decoding config") } - c.init() - am.c = c + am.c = &c am.oidcUsersMapping = map[string]*oidcUserMapping{} if c.UsersMapping == "" { diff --git a/pkg/auth/manager/owncloudsql/owncloudsql.go b/pkg/auth/manager/owncloudsql/owncloudsql.go index 2c7d9277bd..1f6053640d 100644 --- a/pkg/auth/manager/owncloudsql/owncloudsql.go +++ b/pkg/auth/manager/owncloudsql/owncloudsql.go @@ -34,10 +34,10 @@ import ( "github.com/cs3org/reva/pkg/auth/manager/registry" "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/utils/cfg" // Provides mysql drivers. _ "github.com/go-sql-driver/mysql" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "golang.org/x/crypto/bcrypt" ) @@ -86,26 +86,20 @@ func NewMysql(ctx context.Context, m map[string]interface{}) (auth.Manager, erro return mgr, nil } -func (m *manager) Configure(ml map[string]interface{}) error { - c, err := parseConfig(ml) - if err != nil { - return err - } - +func (c *config) ApplyDefaults() { if c.Nobody == 0 { c.Nobody = 99 } - - m.c = c - return nil } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, &c); err != nil { - return nil, err +func (m *manager) Configure(ml map[string]interface{}) error { + var c config + if err := cfg.Decode(ml, &c); err != nil { + return errors.Wrap(err, "owncloudsql: error decoding config") } - return c, nil + + m.c = &c + return nil } func (m *manager) Authenticate(ctx context.Context, login, clientSecret string) (*user.User, map[string]*authpb.Scope, error) { diff --git a/pkg/auth/manager/publicshares/publicshares.go b/pkg/auth/manager/publicshares/publicshares.go index 2b0f48d8f0..c7d2097069 100644 --- a/pkg/auth/manager/publicshares/publicshares.go +++ b/pkg/auth/manager/publicshares/publicshares.go @@ -33,7 +33,8 @@ import ( "github.com/cs3org/reva/pkg/auth/scope" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -46,16 +47,11 @@ type manager struct { } type config struct { - GatewayAddr string `mapstructure:"gateway_addr"` + GatewayAddr string `mapstructure:"gatewaysvc"` } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil +func (c *config) ApplyDefaults() { + c.GatewayAddr = sharedconf.GetGatewaySVC(c.GatewayAddr) } // New returns a new auth Manager. @@ -69,11 +65,11 @@ func New(ctx context.Context, m map[string]interface{}) (auth.Manager, error) { } func (m *manager) Configure(ml map[string]interface{}) error { - conf, err := parseConfig(ml) - if err != nil { - return err + var c config + if err := cfg.Decode(ml, &c); err != nil { + return errors.Wrap(err, "publicshares: error decoding config") } - m.c = conf + m.c = &c return nil } diff --git a/pkg/auth/registry/static/static.go b/pkg/auth/registry/static/static.go index 852b1f4cac..6f098aee12 100644 --- a/pkg/auth/registry/static/static.go +++ b/pkg/auth/registry/static/static.go @@ -26,7 +26,7 @@ import ( "github.com/cs3org/reva/pkg/auth/registry/registry" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/sharedconf" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -37,7 +37,7 @@ type config struct { Rules map[string]string `mapstructure:"rules"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if len(c.Rules) == 0 { c.Rules = map[string]string{ "basic": sharedconf.GetGatewaySVC(""), @@ -70,20 +70,11 @@ func (r *reg) GetProvider(ctx context.Context, authType string) (*registrypb.Pro return nil, errtypes.NotFound("static: auth type not found: " + authType) } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - // New returns an implementation of the auth.Registry interface. func New(ctx context.Context, m map[string]interface{}) (auth.Registry, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() return ®{rules: c.Rules}, nil } diff --git a/pkg/cbox/favorite/sql/sql.go b/pkg/cbox/favorite/sql/sql.go index 15009062f7..5e2ef9b238 100644 --- a/pkg/cbox/favorite/sql/sql.go +++ b/pkg/cbox/favorite/sql/sql.go @@ -29,7 +29,7 @@ import ( ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/storage/favorite" "github.com/cs3org/reva/pkg/storage/favorite/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -51,8 +51,8 @@ type mgr struct { // New returns an instance of the cbox sql favorites manager. func New(m map[string]interface{}) (favorite.Manager, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } @@ -62,7 +62,7 @@ func New(m map[string]interface{}) (favorite.Manager, error) { } return &mgr{ - c: c, + c: &c, db: db, }, nil } diff --git a/pkg/cbox/group/rest/rest.go b/pkg/cbox/group/rest/rest.go index 44d2863622..43acf7a2a3 100644 --- a/pkg/cbox/group/rest/rest.go +++ b/pkg/cbox/group/rest/rest.go @@ -34,9 +34,9 @@ import ( utils "github.com/cs3org/reva/pkg/cbox/utils" "github.com/cs3org/reva/pkg/group" "github.com/cs3org/reva/pkg/group/manager/registry" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/cs3org/reva/pkg/utils/list" "github.com/gomodule/redigo/redis" - "github.com/mitchellh/mapstructure" "github.com/rs/zerolog/log" ) @@ -76,7 +76,7 @@ type config struct { GroupFetchInterval int `mapstructure:"group_fetch_interval" docs:"3600"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.GroupMembersCacheExpiration == 0 { c.GroupMembersCacheExpiration = 5 } @@ -100,21 +100,12 @@ func (c *config) init() { } } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - // New returns a user manager implementation that makes calls to the GRAPPA API. func New(ctx context.Context, m map[string]interface{}) (group.Manager, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() redisPool := initRedisPool(c.RedisAddress, c.RedisUsername, c.RedisPassword) apiTokenManager, err := utils.InitAPITokenManager(m) @@ -123,7 +114,7 @@ func New(ctx context.Context, m map[string]interface{}) (group.Manager, error) { } mgr := &manager{ - conf: c, + conf: &c, redisPool: redisPool, apiTokenManager: apiTokenManager, } diff --git a/pkg/cbox/preferences/sql/sql.go b/pkg/cbox/preferences/sql/sql.go index 680a1fcb91..8e0b2afbd5 100644 --- a/pkg/cbox/preferences/sql/sql.go +++ b/pkg/cbox/preferences/sql/sql.go @@ -27,7 +27,7 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/preferences" "github.com/cs3org/reva/pkg/preferences/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -49,8 +49,8 @@ type mgr struct { // New returns an instance of the cbox sql preferences manager. func New(ctx context.Context, m map[string]interface{}) (preferences.Manager, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } @@ -60,7 +60,7 @@ func New(ctx context.Context, m map[string]interface{}) (preferences.Manager, er } return &mgr{ - c: c, + c: &c, db: db, }, nil } diff --git a/pkg/cbox/publicshare/sql/sql.go b/pkg/cbox/publicshare/sql/sql.go index 0b81b4d025..3602224ab0 100644 --- a/pkg/cbox/publicshare/sql/sql.go +++ b/pkg/cbox/publicshare/sql/sql.go @@ -42,7 +42,7 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/utils" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "golang.org/x/crypto/bcrypt" ) @@ -77,7 +77,7 @@ type manager struct { client gatewayv1beta1.GatewayAPIClient } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.SharePasswordHashCost == 0 { c.SharePasswordHashCost = 11 } @@ -109,11 +109,10 @@ func (m *manager) startJanitorRun() { // New returns a new public share manager. func New(ctx context.Context, m map[string]interface{}) (publicshare.Manager, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", c.DBUsername, c.DBPassword, c.DBHost, c.DBPort, c.DBName)) if err != nil { @@ -126,7 +125,7 @@ func New(ctx context.Context, m map[string]interface{}) (publicshare.Manager, er } mgr := manager{ - c: c, + c: &c, db: db, client: gw, } diff --git a/pkg/cbox/share/sql/sql.go b/pkg/cbox/share/sql/sql.go index 0a87d641c9..0133ed8ddb 100644 --- a/pkg/cbox/share/sql/sql.go +++ b/pkg/cbox/share/sql/sql.go @@ -40,10 +40,10 @@ import ( "github.com/cs3org/reva/pkg/share/manager/registry" "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/cfg" // Provides mysql drivers. _ "github.com/go-sql-driver/mysql" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "google.golang.org/genproto/protobuf/field_mask" ) @@ -76,11 +76,14 @@ type mgr struct { client gatewayv1beta1.GatewayAPIClient } +func (c *config) ApplyDefaults() { + c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) +} + // New returns a new share manager. func New(ctx context.Context, m map[string]interface{}) (share.Manager, error) { - c, err := parseConfig(m) - if err != nil { - err = errors.Wrap(err, "error creating a new manager") + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } @@ -95,21 +98,12 @@ func New(ctx context.Context, m map[string]interface{}) (share.Manager, error) { } return &mgr{ - c: c, + c: &c, db: db, client: gw, }, nil } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) - return c, nil -} - func (m *mgr) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { user := ctxpkg.ContextMustGetUser(ctx) diff --git a/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go b/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go index 2737b0ecc5..ec385dcef4 100644 --- a/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go +++ b/pkg/cbox/storage/eoshomewrapper/eoshomewrapper.go @@ -30,8 +30,7 @@ import ( "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/eosfs" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -43,13 +42,13 @@ type wrapper struct { mountIDTemplate *template.Template } -func parseConfig(m map[string]interface{}) (*eosfs.Config, string, error) { - c := &eosfs.Config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, "", err +// New returns an implementation of the storage.FS interface that forms a wrapper +// around separate connections to EOS. +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { + var c eosfs.Config + if err := cfg.Decode(m, &c); err != nil { + return nil, err } - // default to version invariance if not configured if _, ok := m["version_invariant"]; !ok { c.VersionInvariant = true @@ -60,19 +59,7 @@ func parseConfig(m map[string]interface{}) (*eosfs.Config, string, error) { t = "eoshome-{{substr 0 1 .Username}}" } - return c, t, nil -} - -// New returns an implementation of the storage.FS interface that forms a wrapper -// around separate connections to EOS. -func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - c, t, err := parseConfig(m) - if err != nil { - return nil, err - } - c.EnableHome = true - - eos, err := eosfs.NewEOSFS(ctx, c) + eos, err := eosfs.NewEOSFS(ctx, &c) if err != nil { return nil, err } diff --git a/pkg/cbox/storage/eoswrapper/eoswrapper.go b/pkg/cbox/storage/eoswrapper/eoswrapper.go index 79663ff590..d4b518db40 100644 --- a/pkg/cbox/storage/eoswrapper/eoswrapper.go +++ b/pkg/cbox/storage/eoswrapper/eoswrapper.go @@ -32,8 +32,7 @@ import ( "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/eosfs" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -54,11 +53,12 @@ type wrapper struct { mountIDTemplate *template.Template } -func parseConfig(m map[string]interface{}) (*eosfs.Config, string, error) { - c := &eosfs.Config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, "", err +// New returns an implementation of the storage.FS interface that forms a wrapper +// around separate connections to EOS. +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { + var c eosfs.Config + if err := cfg.Decode(m, &c); err != nil { + return nil, err } // default to version invariance if not configured @@ -77,18 +77,7 @@ func parseConfig(m map[string]interface{}) (*eosfs.Config, string, error) { t = "eoshome-{{ trimAll \"/\" .Path | substr 0 1 }}" } - return c, t, nil -} - -// New returns an implementation of the storage.FS interface that forms a wrapper -// around separate connections to EOS. -func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - c, t, err := parseConfig(m) - if err != nil { - return nil, err - } - - eos, err := eosfs.NewEOSFS(ctx, c) + eos, err := eosfs.NewEOSFS(ctx, &c) if err != nil { return nil, err } @@ -98,7 +87,7 @@ func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { return nil, err } - return &wrapper{FS: eos, conf: c, mountIDTemplate: mountIDTemplate}, nil + return &wrapper{FS: eos, conf: &c, mountIDTemplate: mountIDTemplate}, nil } // We need to override the two methods, GetMD and ListFolder to fill the diff --git a/pkg/cbox/user/rest/rest.go b/pkg/cbox/user/rest/rest.go index 72864cc7e7..d15ed7eeab 100644 --- a/pkg/cbox/user/rest/rest.go +++ b/pkg/cbox/user/rest/rest.go @@ -32,9 +32,9 @@ import ( utils "github.com/cs3org/reva/pkg/cbox/utils" "github.com/cs3org/reva/pkg/user" "github.com/cs3org/reva/pkg/user/manager/registry" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/cs3org/reva/pkg/utils/list" "github.com/gomodule/redigo/redis" - "github.com/mitchellh/mapstructure" "github.com/rs/zerolog/log" ) @@ -74,7 +74,7 @@ type config struct { UserFetchInterval int `mapstructure:"user_fetch_interval" docs:"3600"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.UserGroupsCacheExpiration == 0 { c.UserGroupsCacheExpiration = 5 } @@ -98,14 +98,6 @@ func (c *config) init() { } } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - // New returns a user manager implementation that makes calls to the GRAPPA API. func New(ctx context.Context, m map[string]interface{}) (user.Manager, error) { mgr := &manager{} @@ -117,17 +109,16 @@ func New(ctx context.Context, m map[string]interface{}) (user.Manager, error) { } func (m *manager) Configure(ml map[string]interface{}) error { - c, err := parseConfig(ml) - if err != nil { + var c config + if err := cfg.Decode(ml, &c); err != nil { return err } - c.init() redisPool := initRedisPool(c.RedisAddress, c.RedisUsername, c.RedisPassword) apiTokenManager, err := utils.InitAPITokenManager(ml) if err != nil { return err } - m.conf = c + m.conf = &c m.redisPool = redisPool m.apiTokenManager = apiTokenManager diff --git a/pkg/datatx/manager/rclone/rclone.go b/pkg/datatx/manager/rclone/rclone.go index 244bed0e20..1a1ae155ac 100644 --- a/pkg/datatx/manager/rclone/rclone.go +++ b/pkg/datatx/manager/rclone/rclone.go @@ -111,7 +111,7 @@ func New(ctx context.Context, m map[string]interface{}) (txdriver.Manager, error client := rhttp.GetHTTPClient(rhttp.Insecure(c.Insecure)) - storage, err := getStorageManager(c) + storage, err := getStorageManager(ctx, c) if err != nil { return nil, err } @@ -132,9 +132,9 @@ func parseConfig(m map[string]interface{}) (*config, error) { return c, nil } -func getStorageManager(c *config) (repository.Repository, error) { +func getStorageManager(ctx context.Context, c *config) (repository.Repository, error) { if f, ok := repoRegistry.NewFuncs[c.StorageDriver]; ok { - return f(c.StorageDrivers[c.StorageDriver]) + return f(ctx, c.StorageDrivers[c.StorageDriver]) } return nil, errtypes.NotFound("rclone service: storage driver not found: " + c.StorageDriver) } diff --git a/pkg/datatx/manager/rclone/repository/json/json.go b/pkg/datatx/manager/rclone/repository/json/json.go index 2d31f70ff1..4e11c6d001 100644 --- a/pkg/datatx/manager/rclone/repository/json/json.go +++ b/pkg/datatx/manager/rclone/repository/json/json.go @@ -19,6 +19,7 @@ package json import ( + "context" "encoding/json" "io" "os" @@ -26,7 +27,7 @@ import ( "github.com/cs3org/reva/pkg/datatx/manager/rclone/repository" "github.com/cs3org/reva/pkg/datatx/manager/rclone/repository/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -48,28 +49,18 @@ type rcloneJobsModel struct { RcloneJobs map[string]*repository.Job `json:"rcloneJobs"` } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "rclone repository json driver: error decoding configuration") - return nil, err - } - return c, nil -} - -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.File == "" { c.File = "/var/tmp/reva/transfer-jobs.json" } } // New returns a json storage driver. -func New(m map[string]interface{}) (repository.Repository, error) { - c, err := parseConfig(m) - if err != nil { +func New(ctx context.Context, m map[string]interface{}) (repository.Repository, error) { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() model, err := loadOrCreate(c.File) if err != nil { @@ -78,7 +69,7 @@ func New(m map[string]interface{}) (repository.Repository, error) { } mgr := &mgr{ - config: c, + config: &c, model: model, } diff --git a/pkg/datatx/manager/rclone/repository/registry/registry.go b/pkg/datatx/manager/rclone/repository/registry/registry.go index 113586f3ad..2764f88923 100644 --- a/pkg/datatx/manager/rclone/repository/registry/registry.go +++ b/pkg/datatx/manager/rclone/repository/registry/registry.go @@ -19,12 +19,14 @@ package registry import ( + "context" + "github.com/cs3org/reva/pkg/datatx/manager/rclone/repository" ) // NewFunc is the function that rclone repository implementations // should register at init time. -type NewFunc func(map[string]interface{}) (repository.Repository, error) +type NewFunc func(context.Context, map[string]interface{}) (repository.Repository, error) // NewFuncs is a map containing all the registered datatx backends. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/datatx/repository/json/json.go b/pkg/datatx/repository/json/json.go index 3f121bbff0..8cdce41cb7 100644 --- a/pkg/datatx/repository/json/json.go +++ b/pkg/datatx/repository/json/json.go @@ -29,7 +29,7 @@ import ( txv1beta "github.com/cs3org/go-cs3apis/cs3/tx/v1beta1" "github.com/cs3org/reva/pkg/datatx" "github.com/cs3org/reva/pkg/datatx/repository/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -51,16 +51,7 @@ type transfersModel struct { Transfers map[string]*datatx.Transfer `json:"transfers"` } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "datatx repository json driver: error decoding configuration") - return nil, err - } - return c, nil -} - -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.File == "" { c.File = "/var/tmp/reva/datatx-transfers.json" } @@ -68,11 +59,10 @@ func (c *config) init() { // New returns a json storage driver. func New(ctx context.Context, m map[string]interface{}) (datatx.Repository, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() model, err := loadOrCreate(c.File) if err != nil { @@ -81,7 +71,7 @@ func New(ctx context.Context, m map[string]interface{}) (datatx.Repository, erro } mgr := &mgr{ - config: c, + config: &c, model: model, } diff --git a/pkg/eosclient/eosbinary/eosbinary.go b/pkg/eosclient/eosbinary/eosbinary.go index ead7c21151..1a47aa8240 100644 --- a/pkg/eosclient/eosbinary/eosbinary.go +++ b/pkg/eosclient/eosbinary/eosbinary.go @@ -120,7 +120,7 @@ type Options struct { TokenExpiry int } -func (opt *Options) init() { +func (opt *Options) ApplyDefaults() { if opt.ForceSingleUserMode && opt.SingleUsername != "" { opt.SingleUsername = "apache" } @@ -150,7 +150,7 @@ type Client struct { // New creates a new client with the given options. func New(opt *Options) (*Client, error) { - opt.init() + opt.ApplyDefaults() c := new(Client) c.opt = opt return c, nil diff --git a/pkg/errtypes/join.go b/pkg/errtypes/join.go new file mode 100644 index 0000000000..7fdcaa8e18 --- /dev/null +++ b/pkg/errtypes/join.go @@ -0,0 +1,42 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package errtypes + +import ( + "strings" +) + +type joinErrors []error + +// Join returns an error representing a list of errors. +func Join(err ...error) error { + return joinErrors(err) +} + +// Error return a string comma (,) separated of all the errors. +func (e joinErrors) Error() string { + var b strings.Builder + for i, err := range e { + b.WriteString(err.Error()) + if i != len(e)-1 { + b.WriteString(", ") + } + } + return b.String() +} diff --git a/pkg/group/manager/json/json.go b/pkg/group/manager/json/json.go index 1df0a407ac..911693f425 100644 --- a/pkg/group/manager/json/json.go +++ b/pkg/group/manager/json/json.go @@ -30,7 +30,7 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/group" "github.com/cs3org/reva/pkg/group/manager/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -47,26 +47,16 @@ type config struct { Groups string `mapstructure:"groups"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Groups == "" { c.Groups = "/etc/revad/groups.json" } } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - c.init() - return c, nil -} - // New returns a group manager implementation that reads a json file to provide group metadata. func New(ctx context.Context, m map[string]interface{}) (group.Manager, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } diff --git a/pkg/group/manager/ldap/ldap.go b/pkg/group/manager/ldap/ldap.go index fb8e347f15..669c44c3ea 100644 --- a/pkg/group/manager/ldap/ldap.go +++ b/pkg/group/manager/ldap/ldap.go @@ -34,8 +34,8 @@ import ( "github.com/cs3org/reva/pkg/group" "github.com/cs3org/reva/pkg/group/manager/registry" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/go-ldap/ldap/v3" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -86,22 +86,11 @@ var ldapDefaults = attributes{ GIDNumber: "gidNumber", } -func parseConfig(m map[string]interface{}) (*config, error) { - c := config{ - Schema: ldapDefaults, - } - if err := mapstructure.Decode(m, &c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - - return &c, nil -} - // New returns a group manager implementation that connects to a LDAP server to provide group metadata. func New(ctx context.Context, m map[string]interface{}) (group.Manager, error) { - c, err := parseConfig(m) - if err != nil { + var c config + c.Schema = ldapDefaults + if err := cfg.Decode(m, &c); err != nil { return nil, err } @@ -112,9 +101,10 @@ func New(ctx context.Context, m map[string]interface{}) (group.Manager, error) { c.MemberFilter = strings.ReplaceAll(c.MemberFilter, "%s", "{{.OpaqueId}}") mgr := &manager{ - c: c, + c: &c, } + var err error mgr.groupfilter, err = template.New("gf").Funcs(sprig.TxtFuncMap()).Parse(c.GroupFilter) if err != nil { err := errors.Wrap(err, fmt.Sprintf("error parsing groupfilter tpl:%s", c.GroupFilter)) diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index e65a777330..32e45fabd6 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -73,7 +73,7 @@ type Configuration struct { } // Init sets sane defaults. -func (c *Configuration) Init() { +func (c *Configuration) ApplyDefaults() { if c.Prefix == "" { c.Prefix = "mentix" } diff --git a/pkg/metrics/config/config.go b/pkg/metrics/config/config.go index 27dd4b96de..68f6318427 100644 --- a/pkg/metrics/config/config.go +++ b/pkg/metrics/config/config.go @@ -29,7 +29,7 @@ type Config struct { } // Init sets sane defaults. -func (c *Config) Init() { +func (c *Config) ApplyDefaults() { if c.MetricsDataDriverType == "json" { // default values if c.MetricsDataLocation == "" { diff --git a/pkg/notification/handler/emailhandler/emailhandler.go b/pkg/notification/handler/emailhandler/emailhandler.go index 47152f51c8..3d8f3fc97b 100644 --- a/pkg/notification/handler/emailhandler/emailhandler.go +++ b/pkg/notification/handler/emailhandler/emailhandler.go @@ -19,14 +19,16 @@ package emailhandler import ( + "context" "fmt" "net/smtp" "regexp" "strings" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/notification/handler" "github.com/cs3org/reva/pkg/notification/handler/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/rs/zerolog" ) @@ -37,7 +39,7 @@ func init() { // EmailHandler is the notification handler for emails. type EmailHandler struct { conf *config - Log *zerolog.Logger + log *zerolog.Logger } type config struct { @@ -48,22 +50,23 @@ type config struct { DefaultSender string `mapstructure:"default_sender" docs:"no-reply@cernbox.cern.ch;Default sender when not specified in the trigger."` } -func defaultConfig() *config { - return &config{ - DefaultSender: "no-reply@cernbox.cern.ch", +func (c *config) ApplyDefaults() { + if c.DefaultSender == "" { + c.DefaultSender = "no-reply@cernbox.cern.ch" } } // New returns a new email handler. -func New(log *zerolog.Logger, conf interface{}) (handler.Handler, error) { - c := defaultConfig() - if err := mapstructure.Decode(conf, c); err != nil { +func New(ctx context.Context, m map[string]any) (handler.Handler, error) { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } + log := appctx.GetLogger(ctx) return &EmailHandler{ - conf: c, - Log: log, + conf: &c, + log: log, }, nil } @@ -79,7 +82,7 @@ func (e *EmailHandler) Send(sender, recipient, subject, body string) error { return err } - e.Log.Debug().Msgf("mail sent to recipient %s", recipient) + e.log.Debug().Msgf("mail sent to recipient %s", recipient) return nil } diff --git a/pkg/notification/handler/registry/registry.go b/pkg/notification/handler/registry/registry.go index e4a9271eab..18fad9879d 100644 --- a/pkg/notification/handler/registry/registry.go +++ b/pkg/notification/handler/registry/registry.go @@ -19,13 +19,15 @@ package registry import ( + "context" + + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/notification/handler" - "github.com/rs/zerolog" ) // NewHandlerFunc is the function that notification handlers should register to // at init time. -type NewHandlerFunc func(Log *zerolog.Logger, conf interface{}) (handler.Handler, error) +type NewHandlerFunc func(context.Context, map[string]any) (handler.Handler, error) // NewHandlerFuncs is a map containing all the registered notification handlers. var NewHandlerFuncs = map[string]NewHandlerFunc{} @@ -38,13 +40,16 @@ func Register(name string, f NewHandlerFunc) { // InitHandlers initializes the notification handlers with the configuration // and the log from a service. -func InitHandlers(handlerConf map[string]interface{}, log *zerolog.Logger) map[string]handler.Handler { +func InitHandlers(ctx context.Context, handlerConf map[string]map[string]any) map[string]handler.Handler { handlers := make(map[string]handler.Handler) hCount := 0 + log := appctx.GetLogger(ctx) for n, f := range NewHandlerFuncs { if c, ok := handlerConf[n]; ok { - nh, err := f(log, c) + l := log.With().Str("service", n).Logger() + ctx := appctx.WithLogger(ctx, &l) + nh, err := f(ctx, c) if err != nil { log.Err(err).Msgf("error initializing notification handler %s", n) } diff --git a/pkg/notification/manager/registry/registry.go b/pkg/notification/manager/registry/registry.go index 4ae551145a..2d1935b5eb 100644 --- a/pkg/notification/manager/registry/registry.go +++ b/pkg/notification/manager/registry/registry.go @@ -18,13 +18,17 @@ package registry -import "github.com/cs3org/reva/pkg/notification" +import ( + "context" + + "github.com/cs3org/reva/pkg/notification" +) // import "github.com/cs3org/reva/pkg/share" // NewFunc is the function that notification managers // should register at init time. -type NewFunc func(map[string]interface{}) (notification.Manager, error) +type NewFunc func(context.Context, map[string]interface{}) (notification.Manager, error) // NewFuncs is a map containing all the registered notification managers. var NewFuncs = map[string]NewFunc{} diff --git a/pkg/notification/manager/sql/sql.go b/pkg/notification/manager/sql/sql.go index 1e71dad7ab..c1af5b15a9 100644 --- a/pkg/notification/manager/sql/sql.go +++ b/pkg/notification/manager/sql/sql.go @@ -19,12 +19,13 @@ package sql import ( + "context" "database/sql" "fmt" "github.com/cs3org/reva/pkg/notification" "github.com/cs3org/reva/pkg/notification/manager/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -37,7 +38,6 @@ type config struct { DBHost string `mapstructure:"db_host"` DBPort int `mapstructure:"db_port"` DBName string `mapstructure:"db_name"` - GatewaySvc string `mapstructure:"gatewaysvc"` } type mgr struct { @@ -46,9 +46,9 @@ type mgr struct { } // NewMysql returns an instance of the sql notifications manager. -func NewMysql(m map[string]interface{}) (notification.Manager, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { +func NewMysql(ctx context.Context, m map[string]interface{}) (notification.Manager, error) { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } diff --git a/pkg/ocm/invite/repository/json/json.go b/pkg/ocm/invite/repository/json/json.go index ea4c3e8687..e60ad42251 100644 --- a/pkg/ocm/invite/repository/json/json.go +++ b/pkg/ocm/invite/repository/json/json.go @@ -34,8 +34,8 @@ import ( "github.com/cs3org/reva/pkg/ocm/invite" "github.com/cs3org/reva/pkg/ocm/invite/repository/registry" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/cs3org/reva/pkg/utils/list" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -59,47 +59,33 @@ func init() { registry.Register("json", New) } -func (c *config) init() error { +func (c *config) ApplyDefaults() { if c.File == "" { c.File = "/var/tmp/reva/ocm-invites.json" } - - return nil } // New returns a new invite manager object. func New(ctx context.Context, m map[string]interface{}) (invite.Repository, error) { - config, err := parseConfig(m) - if err != nil { - return nil, errors.Wrap(err, "error parsing config for json invite repository") - } - err = config.init() - if err != nil { - return nil, errors.Wrap(err, "error setting config defaults for json invite repository") + var c config + if err := cfg.Decode(m, &c); err != nil { + return nil, err } // load or create file - model, err := loadOrCreate(config.File) + model, err := loadOrCreate(c.File) if err != nil { return nil, errors.Wrap(err, "error loading the file containing the invites") } manager := &manager{ - config: config, + config: &c, model: model, } return manager, nil } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - func loadOrCreate(file string) (*inviteModel, error) { _, err := os.Stat(file) if os.IsNotExist(err) { diff --git a/pkg/ocm/invite/repository/sql/sql.go b/pkg/ocm/invite/repository/sql/sql.go index 7746796da3..dc5b2e6f49 100644 --- a/pkg/ocm/invite/repository/sql/sql.go +++ b/pkg/ocm/invite/repository/sql/sql.go @@ -32,11 +32,11 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/ocm/invite" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/go-sql-driver/mysql" "github.com/cs3org/reva/pkg/ocm/invite/repository/registry" "github.com/cs3org/reva/pkg/sharedconf" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -66,38 +66,29 @@ type config struct { GatewaySvc string `mapstructure:"gatewaysvc"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) } -func parseConfig(c map[string]interface{}) (*config, error) { - var conf config - if err := mapstructure.Decode(c, &conf); err != nil { - return nil, err - } - return &conf, nil -} - // New creates a sql repository for ocm tokens and users. -func New(ctx context.Context, c map[string]interface{}) (invite.Repository, error) { - conf, err := parseConfig(c) - if err != nil { - return nil, errors.Wrap(err, "sql: error parsing config") +func New(ctx context.Context, m map[string]interface{}) (invite.Repository, error) { + var c config + if err := cfg.Decode(m, &c); err != nil { + return nil, err } - conf.init() - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?parseTime=true", conf.DBUsername, conf.DBPassword, conf.DBAddress, conf.DBName)) + db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?parseTime=true", c.DBUsername, c.DBPassword, c.DBAddress, c.DBName)) if err != nil { return nil, errors.Wrap(err, "sql: error opening connection to mysql database") } - gw, err := pool.GetGatewayServiceClient(pool.Endpoint(conf.GatewaySvc)) + gw, err := pool.GetGatewayServiceClient(pool.Endpoint(c.GatewaySvc)) if err != nil { return nil, err } mgr := mgr{ - c: conf, + c: &c, db: db, client: gw, } diff --git a/pkg/ocm/provider/authorizer/json/json.go b/pkg/ocm/provider/authorizer/json/json.go index cf6839e5dd..e13a038fea 100644 --- a/pkg/ocm/provider/authorizer/json/json.go +++ b/pkg/ocm/provider/authorizer/json/json.go @@ -32,7 +32,7 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/ocm/provider" "github.com/cs3org/reva/pkg/ocm/provider/authorizer/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -42,12 +42,10 @@ func init() { // New returns a new authorizer object. func New(ctx context.Context, m map[string]interface{}) (provider.Authorizer, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() f, err := os.ReadFile(c.Providers) if err != nil { @@ -61,7 +59,7 @@ func New(ctx context.Context, m map[string]interface{}) (provider.Authorizer, er a := &authorizer{ providerIPs: sync.Map{}, - conf: c, + conf: &c, } a.providers = a.getOCMProviders(providers) @@ -73,7 +71,7 @@ type config struct { VerifyRequestHostname bool `mapstructure:"verify_request_hostname"` } -func (c *config) init() { +func (c *config) ApplyTemplates() { if c.Providers == "" { c.Providers = "/etc/revad/ocm-providers.json" } diff --git a/pkg/ocm/provider/authorizer/mentix/mentix.go b/pkg/ocm/provider/authorizer/mentix/mentix.go index 9c165c1a5e..88347270f5 100644 --- a/pkg/ocm/provider/authorizer/mentix/mentix.go +++ b/pkg/ocm/provider/authorizer/mentix/mentix.go @@ -34,7 +34,7 @@ import ( "github.com/cs3org/reva/pkg/ocm/provider" "github.com/cs3org/reva/pkg/ocm/provider/authorizer/registry" "github.com/cs3org/reva/pkg/rhttp" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -50,12 +50,10 @@ type Client struct { // New returns a new authorizer object. func New(ctx context.Context, m map[string]interface{}) (provider.Authorizer, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() client := &Client{ BaseURL: c.URL, @@ -69,7 +67,7 @@ func New(ctx context.Context, m map[string]interface{}) (provider.Authorizer, er return &authorizer{ client: client, providerIPs: sync.Map{}, - conf: c, + conf: &c, }, nil } @@ -81,7 +79,7 @@ type config struct { Insecure bool `mapstructure:"insecure" docs:"false;Whether to skip certificate checks when sending requests."` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.URL == "" { c.URL = "http://localhost:9600/mentix/cs3" } diff --git a/pkg/ocm/provider/authorizer/open/open.go b/pkg/ocm/provider/authorizer/open/open.go index a9f6feefd0..1a317678b8 100644 --- a/pkg/ocm/provider/authorizer/open/open.go +++ b/pkg/ocm/provider/authorizer/open/open.go @@ -28,8 +28,7 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/ocm/provider" "github.com/cs3org/reva/pkg/ocm/provider/authorizer/registry" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -38,12 +37,10 @@ func init() { // New returns a new authorizer object. func New(ctx context.Context, m map[string]interface{}) (provider.Authorizer, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() f, err := os.ReadFile(c.Providers) if err != nil { @@ -66,7 +63,7 @@ type config struct { Providers string `mapstructure:"providers"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Providers == "" { c.Providers = "/etc/revad/ocm-providers.json" } diff --git a/pkg/ocm/share/repository/json/json.go b/pkg/ocm/share/repository/json/json.go index 5517999335..d8c9658239 100644 --- a/pkg/ocm/share/repository/json/json.go +++ b/pkg/ocm/share/repository/json/json.go @@ -34,8 +34,8 @@ import ( "github.com/cs3org/reva/pkg/ocm/share" "github.com/cs3org/reva/pkg/ocm/share/repository/registry" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/google/uuid" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "google.golang.org/genproto/protobuf/field_mask" ) @@ -46,12 +46,10 @@ func init() { // New returns a new authorizer object. func New(ctx context.Context, m map[string]interface{}) (share.Repository, error) { - c, err := parseConfig(m) - if err != nil { - err = errors.Wrap(err, "error creating a new manager") + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() // load or create file model, err := loadOrCreate(c.File) @@ -61,7 +59,7 @@ func New(ctx context.Context, m map[string]interface{}) (share.Repository, error } mgr := &mgr{ - c: c, + c: &c, model: model, } @@ -170,7 +168,7 @@ type config struct { File string `mapstructure:"file"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.File == "" { c.File = "/var/tmp/reva/ocm-shares.json" } @@ -217,14 +215,6 @@ func (m *mgr) load() error { return nil } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - func genID() string { return uuid.New().String() } diff --git a/pkg/ocm/share/repository/nextcloud/nextcloud.go b/pkg/ocm/share/repository/nextcloud/nextcloud.go index 7fe1932069..1c87d9ca61 100644 --- a/pkg/ocm/share/repository/nextcloud/nextcloud.go +++ b/pkg/ocm/share/repository/nextcloud/nextcloud.go @@ -37,7 +37,7 @@ import ( "github.com/cs3org/reva/pkg/ocm/share" "github.com/cs3org/reva/pkg/ocm/share/repository/registry" "github.com/cs3org/reva/pkg/utils" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "google.golang.org/genproto/protobuf/field_mask" ) @@ -94,27 +94,14 @@ type ReceivedShareAltMap struct { State ocm.ShareState `json:"state"` } -func (c *ShareManagerConfig) init() { -} - -func parseConfig(m map[string]interface{}) (*ShareManagerConfig, error) { - c := &ShareManagerConfig{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - // New returns a share manager implementation that verifies against a Nextcloud backend. func New(ctx context.Context, m map[string]interface{}) (share.Repository, error) { - c, err := parseConfig(m) - if err != nil { + var c ShareManagerConfig + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - return NewShareManager(c) + return NewShareManager(&c) } // NewShareManager returns a new Nextcloud-based ShareManager. diff --git a/pkg/ocm/share/repository/sql/sql.go b/pkg/ocm/share/repository/sql/sql.go index 980e08d55d..08a28ee323 100644 --- a/pkg/ocm/share/repository/sql/sql.go +++ b/pkg/ocm/share/repository/sql/sql.go @@ -34,8 +34,8 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/ocm/share" "github.com/cs3org/reva/pkg/ocm/share/repository/registry" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/go-sql-driver/mysql" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "google.golang.org/genproto/protobuf/field_mask" ) @@ -45,12 +45,12 @@ func init() { } // New creates a Repository with a SQL driver. -func New(ctx context.Context, c map[string]interface{}) (share.Repository, error) { - conf, err := parseConfig(c) - if err != nil { +func New(ctx context.Context, m map[string]interface{}) (share.Repository, error) { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - return NewFromConfig(ctx, conf) + return NewFromConfig(ctx, &c) } type mgr struct { @@ -87,14 +87,6 @@ type config struct { now func() time.Time // set only from tests } -func parseConfig(conf map[string]interface{}) (*config, error) { - var c config - if err := mapstructure.Decode(conf, &c); err != nil { - return nil, errors.Wrap(err, "error decoding config") - } - return &c, nil -} - func formatUserID(u *userpb.UserId) string { return fmt.Sprintf("%s@%s", u.OpaqueId, u.Idp) } diff --git a/pkg/ocm/storage/outcoming/ocm.go b/pkg/ocm/storage/outcoming/ocm.go index 50828f40b4..1ce9fecc34 100644 --- a/pkg/ocm/storage/outcoming/ocm.go +++ b/pkg/ocm/storage/outcoming/ocm.go @@ -42,7 +42,7 @@ import ( "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "google.golang.org/grpc/metadata" ) @@ -61,31 +61,24 @@ type config struct { MachineSecret string `mapstructure:"machine_secret"` } -func parseConfig(c map[string]interface{}) (*config, error) { - var conf config - err := mapstructure.Decode(c, &conf) - return &conf, err -} - -func (c *config) init() { +func (c *config) ApplyDefaults() { c.GatewaySVC = sharedconf.GetGatewaySVC(c.GatewaySVC) } // New creates an OCM storage driver. -func New(ctx context.Context, c map[string]interface{}) (storage.FS, error) { - conf, err := parseConfig(c) - if err != nil { - return nil, errors.Wrapf(err, "error decoding config") +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { + var c config + if err := cfg.Decode(m, &c); err != nil { + return nil, err } - conf.init() - gateway, err := pool.GetGatewayServiceClient(pool.Endpoint(conf.GatewaySVC)) + gateway, err := pool.GetGatewayServiceClient(pool.Endpoint(c.GatewaySVC)) if err != nil { return nil, err } d := &driver{ - c: conf, + c: &c, gateway: gateway, } diff --git a/pkg/ocm/storage/received/ocm.go b/pkg/ocm/storage/received/ocm.go index 405f5ab2fc..2ebfaf2d7d 100644 --- a/pkg/ocm/storage/received/ocm.go +++ b/pkg/ocm/storage/received/ocm.go @@ -40,8 +40,7 @@ import ( "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/studio-b12/gowebdav" ) @@ -55,34 +54,27 @@ type driver struct { } type config struct { - GatewaySVC string + GatewaySVC string `mapstructure:"gatewaysvc"` } -func parseConfig(c map[string]interface{}) (*config, error) { - var conf config - err := mapstructure.Decode(c, &conf) - return &conf, err -} - -func (c *config) init() { +func (c *config) ApplyDefaults() { c.GatewaySVC = sharedconf.GetGatewaySVC(c.GatewaySVC) } // New creates an OCM storage driver. -func New(ctx context.Context, c map[string]interface{}) (storage.FS, error) { - conf, err := parseConfig(c) - if err != nil { - return nil, errors.Wrapf(err, "error decoding config") +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { + var c config + if err := cfg.Decode(m, &c); err != nil { + return nil, err } - conf.init() - gateway, err := pool.GetGatewayServiceClient(pool.Endpoint(conf.GatewaySVC)) + gateway, err := pool.GetGatewayServiceClient(pool.Endpoint(c.GatewaySVC)) if err != nil { return nil, err } d := &driver{ - c: conf, + c: &c, gateway: gateway, } diff --git a/pkg/publicshare/manager/json/json.go b/pkg/publicshare/manager/json/json.go index b8a0a00195..060dca0aa9 100644 --- a/pkg/publicshare/manager/json/json.go +++ b/pkg/publicshare/manager/json/json.go @@ -38,7 +38,7 @@ import ( "github.com/cs3org/reva/pkg/publicshare" "github.com/cs3org/reva/pkg/publicshare/manager/registry" "github.com/cs3org/reva/pkg/utils" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "github.com/rs/zerolog/log" "golang.org/x/crypto/bcrypt" @@ -49,45 +49,43 @@ func init() { } // New returns a new filesystem public shares manager. -func New(ctx context.Context, c map[string]interface{}) (publicshare.Manager, error) { - conf := &config{} - if err := mapstructure.Decode(c, conf); err != nil { +func New(ctx context.Context, m map[string]interface{}) (publicshare.Manager, error) { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - conf.init() - - m := manager{ + mgr := manager{ mutex: &sync.Mutex{}, - file: conf.File, - passwordHashCost: conf.SharePasswordHashCost, - janitorRunInterval: conf.JanitorRunInterval, - enableExpiredSharesCleanup: conf.EnableExpiredSharesCleanup, + file: c.File, + passwordHashCost: c.SharePasswordHashCost, + janitorRunInterval: c.JanitorRunInterval, + enableExpiredSharesCleanup: c.EnableExpiredSharesCleanup, } // attempt to create the db file var fi os.FileInfo var err error - if fi, err = os.Stat(m.file); os.IsNotExist(err) { - folder := filepath.Dir(m.file) + if fi, err = os.Stat(mgr.file); os.IsNotExist(err) { + folder := filepath.Dir(mgr.file) if err := os.MkdirAll(folder, 0755); err != nil { return nil, err } - if _, err := os.Create(m.file); err != nil { + if _, err := os.Create(mgr.file); err != nil { return nil, err } } if fi == nil || fi.Size() == 0 { - err := os.WriteFile(m.file, []byte("{}"), 0644) + err := os.WriteFile(mgr.file, []byte("{}"), 0644) if err != nil { return nil, err } } - go m.startJanitorRun() + go mgr.startJanitorRun() - return &m, nil + return &mgr, nil } type config struct { @@ -97,7 +95,7 @@ type config struct { EnableExpiredSharesCleanup bool `mapstructure:"enable_expired_shares_cleanup"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.File == "" { c.File = "/var/tmp/reva/publicshares" } diff --git a/pkg/rgrpc/rgrpc.go b/pkg/rgrpc/rgrpc.go index 96cf258fc1..f45e5a447f 100644 --- a/pkg/rgrpc/rgrpc.go +++ b/pkg/rgrpc/rgrpc.go @@ -101,7 +101,7 @@ func InitServices(ctx context.Context, services map[string]config.ServicesConfig ctx := appctx.WithLogger(ctx, &log) svc, err := new(ctx, cfg[0].Config) if err != nil { - return nil, errors.Wrapf(err, "rgrpc: grpc service %s could not be started,", name) + return nil, errors.Wrapf(err, "rgrpc: grpc service %s could not be started", name) } s[name] = svc } diff --git a/pkg/rserverless/rserverless.go b/pkg/rserverless/rserverless.go index c68f1d6142..606e833053 100644 --- a/pkg/rserverless/rserverless.go +++ b/pkg/rserverless/rserverless.go @@ -41,7 +41,7 @@ func Register(name string, newFunc NewService) { } // NewService is the function that serverless services need to register at init time. -type NewService func(conf map[string]interface{}, log *zerolog.Logger) (Service, error) +type NewService func(context.Context, map[string]interface{}) (Service, error) // Serverless contains the serveless collection of services. type Serverless struct { diff --git a/pkg/share/cache/memory/memory.go b/pkg/share/cache/memory/memory.go index f617b4df56..5766307121 100644 --- a/pkg/share/cache/memory/memory.go +++ b/pkg/share/cache/memory/memory.go @@ -25,8 +25,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/share/cache" "github.com/cs3org/reva/pkg/share/cache/registry" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -41,16 +40,18 @@ type manager struct { cache gcache.Cache } -// New returns an implementation of a resource info cache that stores the objects in memory. -func New(m map[string]interface{}) (cache.ResourceInfoCache, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, errors.Wrap(err, "error decoding conf") - } +func (c *config) ApplyDefaults() { if c.CacheSize == 0 { c.CacheSize = 1000000 } +} +// New returns an implementation of a resource info cache that stores the objects in memory. +func New(m map[string]interface{}) (cache.ResourceInfoCache, error) { + var c config + if err := cfg.Decode(m, &c); err != nil { + return nil, err + } return &manager{ cache: gcache.New(c.CacheSize).LFU().Build(), }, nil diff --git a/pkg/share/cache/redis/redis.go b/pkg/share/cache/redis/redis.go index 0703f98f07..886172e3d0 100644 --- a/pkg/share/cache/redis/redis.go +++ b/pkg/share/cache/redis/redis.go @@ -25,8 +25,8 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/share/cache" "github.com/cs3org/reva/pkg/share/cache/registry" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/gomodule/redigo/redis" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -44,16 +44,18 @@ type manager struct { redisPool *redis.Pool } -// New returns an implementation of a resource info cache that stores the objects in a redis cluster. -func New(m map[string]interface{}) (cache.ResourceInfoCache, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, errors.Wrap(err, "error decoding conf") - } - +func (c *config) ApplyDefaults() { if c.RedisAddress == "" { c.RedisAddress = "localhost:6379" } +} + +// New returns an implementation of a resource info cache that stores the objects in a redis cluster. +func New(m map[string]interface{}) (cache.ResourceInfoCache, error) { + var c config + if err := cfg.Decode(m, &c); err != nil { + return nil, err + } pool := &redis.Pool{ MaxIdle: 50, diff --git a/pkg/share/manager/json/json.go b/pkg/share/manager/json/json.go index a301876e6b..a453b552cd 100644 --- a/pkg/share/manager/json/json.go +++ b/pkg/share/manager/json/json.go @@ -34,8 +34,8 @@ import ( "github.com/cs3org/reva/pkg/share" "github.com/cs3org/reva/pkg/share/manager/registry" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/google/uuid" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "google.golang.org/genproto/protobuf/field_mask" ) @@ -46,14 +46,11 @@ func init() { // New returns a new mgr. func New(ctx context.Context, m map[string]interface{}) (share.Manager, error) { - c, err := parseConfig(m) - if err != nil { - err = errors.Wrap(err, "error creating a new manager") + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - // load or create file model, err := loadOrCreate(c.File) if err != nil { @@ -62,7 +59,7 @@ func New(ctx context.Context, m map[string]interface{}) (share.Manager, error) { } return &mgr{ - c: c, + c: &c, model: model, }, nil } @@ -157,20 +154,12 @@ type config struct { File string `mapstructure:"file"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.File == "" { c.File = "/var/tmp/reva/shares.json" } } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - func genID() string { return uuid.New().String() } diff --git a/pkg/share/manager/sql/sql.go b/pkg/share/manager/sql/sql.go index 13300d80ef..984e0f6425 100644 --- a/pkg/share/manager/sql/sql.go +++ b/pkg/share/manager/sql/sql.go @@ -34,11 +34,12 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/share" "github.com/cs3org/reva/pkg/share/manager/registry" + "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/cfg" // Provides mysql drivers. _ "github.com/go-sql-driver/mysql" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "google.golang.org/genproto/protobuf/field_mask" ) @@ -53,7 +54,7 @@ func init() { } type config struct { - GatewayAddr string `mapstructure:"gateway_addr"` + GatewayAddr string `mapstructure:"gatewaysvc"` StorageMountID string `mapstructure:"storage_mount_id"` DBUsername string `mapstructure:"db_username"` DBPassword string `mapstructure:"db_password"` @@ -62,6 +63,10 @@ type config struct { DBName string `mapstructure:"db_name"` } +func (c *config) ApplyDefaults() { + c.GatewayAddr = sharedconf.GetGatewaySVC(c.GatewayAddr) +} + type mgr struct { driver string db *sql.DB @@ -71,9 +76,8 @@ type mgr struct { // NewMysql returns a new share manager connection to a mysql database. func NewMysql(ctx context.Context, m map[string]interface{}) (share.Manager, error) { - c, err := parseConfig(m) - if err != nil { - err = errors.Wrap(err, "error creating a new manager") + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } @@ -97,14 +101,6 @@ func New(driver string, db *sql.DB, storageMountID string, userConverter UserCon }, nil } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - func (m *mgr) Share(ctx context.Context, md *provider.ResourceInfo, g *collaboration.ShareGrant) (*collaboration.Share, error) { user := ctxpkg.ContextMustGetUser(ctx) diff --git a/pkg/storage/fs/cback/cback.go b/pkg/storage/fs/cback/cback.go index 4c4607eaae..a9ceb153e4 100644 --- a/pkg/storage/fs/cback/cback.go +++ b/pkg/storage/fs/cback/cback.go @@ -35,8 +35,7 @@ import ( "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" ) type cback struct { @@ -51,15 +50,15 @@ func init() { // New returns an implementation to the storage.FS interface that talks to // cback. func New(ctx context.Context, m map[string]interface{}) (fs storage.FS, err error) { - c := &Options{} - if err = mapstructure.Decode(m, c); err != nil { - return nil, errors.Wrap(err, "Error Decoding Configuration") + var o Options + if err := cfg.Decode(m, &o); err != nil { + return nil, err } httpClient := rhttp.GetHTTPClient() // Returns the storage.FS interface - return &cback{conf: c, client: httpClient}, nil + return &cback{conf: &o, client: httpClient}, nil } func (fs *cback) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (*provider.ResourceInfo, error) { diff --git a/pkg/storage/fs/cephfs/cephfs.go b/pkg/storage/fs/cephfs/cephfs.go index 9132bab87b..e9e175aff4 100644 --- a/pkg/storage/fs/cephfs/cephfs.go +++ b/pkg/storage/fs/cephfs/cephfs.go @@ -37,7 +37,7 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -65,24 +65,22 @@ func init() { // New returns an implementation to of the storage.FS interface that talk to // a ceph filesystem. func New(ctx context.Context, m map[string]interface{}) (fs storage.FS, err error) { - c := &Options{} - if err = mapstructure.Decode(m, c); err != nil { - return nil, errors.Wrap(err, "error decoding conf") + var o Options + if err := cfg.Decode(m, &o); err != nil { + return nil, err } - c.fillDefaults() - var cache *connections if cache, err = newCache(); err != nil { return nil, errors.New("cephfs: can't create caches") } - adminConn := newAdminConn(c) + adminConn := newAdminConn(&o) if adminConn == nil { return nil, errors.Wrap(err, "cephfs: Couldn't create admin connections") } - for _, dir := range []string{c.ShadowFolder, c.UploadFolder} { + for _, dir := range []string{o.ShadowFolder, o.UploadFolder} { err = adminConn.adminMount.MakeDir(dir, dirPermFull) if err != nil && err.Error() != errFileExists { return nil, errors.New("cephfs: can't initialise system dir " + dir + ":" + err.Error()) @@ -90,7 +88,7 @@ func New(ctx context.Context, m map[string]interface{}) (fs storage.FS, err erro } return &cephfs{ - conf: c, + conf: &o, conn: cache, adminConn: adminConn, }, nil diff --git a/pkg/storage/fs/cephfs/options.go b/pkg/storage/fs/cephfs/options.go index 045dd93e29..03ae955055 100644 --- a/pkg/storage/fs/cephfs/options.go +++ b/pkg/storage/fs/cephfs/options.go @@ -47,7 +47,7 @@ type Options struct { HiddenDirs map[string]bool } -func (c *Options) fillDefaults() { +func (c *Options) ApplyDefaults() { c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc) if c.IndexPool == "" { diff --git a/pkg/storage/fs/eos/eos.go b/pkg/storage/fs/eos/eos.go index 3847c66b6d..93e7cc1976 100644 --- a/pkg/storage/fs/eos/eos.go +++ b/pkg/storage/fs/eos/eos.go @@ -24,18 +24,17 @@ import ( "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/eosfs" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { registry.Register("eos", New) } -func parseConfig(m map[string]interface{}) (*eosfs.Config, error) { - c := &eosfs.Config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") +// New returns a new implementation of the storage.FS interface that connects to EOS. +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { + var c eosfs.Config + if err := cfg.Decode(m, &c); err != nil { return nil, err } @@ -44,15 +43,5 @@ func parseConfig(m map[string]interface{}) (*eosfs.Config, error) { c.VersionInvariant = true } - return c, nil -} - -// New returns a new implementation of the storage.FS interface that connects to EOS. -func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - c, err := parseConfig(m) - if err != nil { - return nil, err - } - - return eosfs.NewEOSFS(ctx, c) + return eosfs.NewEOSFS(ctx, &c) } diff --git a/pkg/storage/fs/eosgrpc/eosgrpc.go b/pkg/storage/fs/eosgrpc/eosgrpc.go index 3014335cb1..4582459d48 100644 --- a/pkg/storage/fs/eosgrpc/eosgrpc.go +++ b/pkg/storage/fs/eosgrpc/eosgrpc.go @@ -24,36 +24,25 @@ import ( "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/eosfs" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { registry.Register("eosgrpc", New) } -func parseConfig(m map[string]interface{}) (*eosfs.Config, error) { - c := &eosfs.Config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") +// New returns a new implementation of the storage.FS interface that connects to EOS. +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { + var c eosfs.Config + if err := cfg.Decode(m, &c); err != nil { return nil, err } + c.UseGRPC = true // default to version invariance if not configured if _, ok := m["version_invariant"]; !ok { c.VersionInvariant = true } - return c, nil -} - -// New returns a new implementation of the storage.FS interface that connects to EOS. -func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - c, err := parseConfig(m) - if err != nil { - return nil, err - } - c.UseGRPC = true - - return eosfs.NewEOSFS(ctx, c) + return eosfs.NewEOSFS(ctx, &c) } diff --git a/pkg/storage/fs/eosgrpchome/eosgrpchome.go b/pkg/storage/fs/eosgrpchome/eosgrpchome.go index 4b07e604fa..a68054bc58 100644 --- a/pkg/storage/fs/eosgrpchome/eosgrpchome.go +++ b/pkg/storage/fs/eosgrpchome/eosgrpchome.go @@ -24,37 +24,26 @@ import ( "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/eosfs" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { registry.Register("eosgrpchome", New) } -func parseConfig(m map[string]interface{}) (*eosfs.Config, error) { - c := &eosfs.Config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") +// New returns a new implementation of the storage.FS interface that connects to EOS. +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { + var c eosfs.Config + if err := cfg.Decode(m, &c); err != nil { return nil, err } + c.UseGRPC = true + c.EnableHome = true // default to version invariance if not configured if _, ok := m["version_invariant"]; !ok { c.VersionInvariant = true } - return c, nil -} - -// New returns a new implementation of the storage.FS interface that connects to EOS. -func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - c, err := parseConfig(m) - if err != nil { - return nil, err - } - c.UseGRPC = true - c.EnableHome = true - - return eosfs.NewEOSFS(ctx, c) + return eosfs.NewEOSFS(ctx, &c) } diff --git a/pkg/storage/fs/eoshome/eoshome.go b/pkg/storage/fs/eoshome/eoshome.go index 6fcf4abe21..cc4df2550b 100644 --- a/pkg/storage/fs/eoshome/eoshome.go +++ b/pkg/storage/fs/eoshome/eoshome.go @@ -24,36 +24,25 @@ import ( "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/eosfs" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { registry.Register("eoshome", New) } -func parseConfig(m map[string]interface{}) (*eosfs.Config, error) { - c := &eosfs.Config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") +// New returns a new implementation of the storage.FS interface that connects to EOS. +func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { + var c eosfs.Config + if err := cfg.Decode(m, &c); err != nil { return nil, err } + c.EnableHome = true // default to version invariance if not configured if _, ok := m["version_invariant"]; !ok { c.VersionInvariant = true } - return c, nil -} - -// New returns a new implementation of the storage.FS interface that connects to EOS. -func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - c, err := parseConfig(m) - if err != nil { - return nil, err - } - c.EnableHome = true - - return eosfs.NewEOSFS(ctx, c) + return eosfs.NewEOSFS(ctx, &c) } diff --git a/pkg/storage/fs/local/local.go b/pkg/storage/fs/local/local.go index a5b42cd241..7ecbd04d1d 100644 --- a/pkg/storage/fs/local/local.go +++ b/pkg/storage/fs/local/local.go @@ -24,8 +24,7 @@ import ( "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/localfs" - "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" + "github.com/cs3org/reva/pkg/utils/cfg" ) func init() { @@ -37,20 +36,20 @@ type config struct { ShareFolder string `mapstructure:"share_folder" docs:"/MyShares;Path for storing share references."` } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err +func (c *config) ApplyDefaults() { + if c.Root == "" { + c.Root = "/var/tmp/reva" + } + if c.ShareFolder == "" { + c.ShareFolder = "/MyShares" } - return c, nil } // New returns an implementation to of the storage.FS interface that talks to // a local filesystem with user homes disabled. func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } diff --git a/pkg/storage/fs/localhome/localhome.go b/pkg/storage/fs/localhome/localhome.go index 24cb3ffa1b..f57534c2df 100644 --- a/pkg/storage/fs/localhome/localhome.go +++ b/pkg/storage/fs/localhome/localhome.go @@ -38,6 +38,18 @@ type config struct { UserLayout string `mapstructure:"user_layout" docs:"{{.Username}};Template for user home directories"` } +func (c *config) ApplyDefaults() { + if c.Root == "" { + c.Root = "/var/tmp/reva" + } + if c.ShareFolder == "" { + c.ShareFolder = "/MyShares" + } + if c.UserLayout == "" { + c.UserLayout = "{{.Username}}" + } +} + func parseConfig(m map[string]interface{}) (*config, error) { c := &config{} if err := mapstructure.Decode(m, c); err != nil { diff --git a/pkg/storage/fs/nextcloud/nextcloud.go b/pkg/storage/fs/nextcloud/nextcloud.go index 00cda53858..d67663cd73 100644 --- a/pkg/storage/fs/nextcloud/nextcloud.go +++ b/pkg/storage/fs/nextcloud/nextcloud.go @@ -35,7 +35,7 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -58,24 +58,14 @@ type StorageDriver struct { client *http.Client } -func parseConfig(m map[string]interface{}) (*StorageDriverConfig, error) { - c := &StorageDriverConfig{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - // New returns an implementation to of the storage.FS interface that talks to // a Nextcloud instance over http. func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - conf, err := parseConfig(m) - if err != nil { + var c StorageDriverConfig + if err := cfg.Decode(m, &c); err != nil { return nil, err } - - return NewStorageDriver(conf) + return NewStorageDriver(&c) } // NewStorageDriver returns a new NextcloudStorageDriver. diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index 336109c72b..eb23d12257 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -49,9 +49,9 @@ import ( "github.com/cs3org/reva/pkg/storage/utils/ace" "github.com/cs3org/reva/pkg/storage/utils/chunking" "github.com/cs3org/reva/pkg/storage/utils/templates" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/gomodule/redigo/redis" "github.com/google/uuid" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "github.com/pkg/xattr" ) @@ -120,16 +120,7 @@ type config struct { UserProviderEndpoint string `mapstructure:"userprovidersvc"` } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - -func (c *config) init(m map[string]interface{}) { +func (c *config) ApplyDefaults() { if c.Redis == "" { c.Redis = ":6379" } @@ -149,27 +140,27 @@ func (c *config) init(m map[string]interface{}) { // ensure share folder always starts with slash c.ShareFolder = filepath.Join("/", c.ShareFolder) - // default to scanning if not configured - if _, ok := m["scan"]; !ok { + if !c.Scan { + // TODO: check if it was set in the config c.Scan = true } + c.UserProviderEndpoint = sharedconf.GetGatewaySVC(c.UserProviderEndpoint) } // New returns an implementation to of the storage.FS interface that talk to // a local filesystem. func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init(m) // c.DataDirectory should never end in / unless it is the root? c.DataDirectory = filepath.Clean(c.DataDirectory) // create datadir if it does not exist - err = os.MkdirAll(c.DataDirectory, 0700) + err := os.MkdirAll(c.DataDirectory, 0700) if err != nil { logger.New().Error().Err(err). Str("path", c.DataDirectory). @@ -203,7 +194,7 @@ func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { } return &ocfs{ - c: c, + c: &c, pool: pool, chunkHandler: chunking.NewChunkHandler(c.UploadInfoDir), }, nil diff --git a/pkg/storage/fs/owncloudsql/owncloudsql.go b/pkg/storage/fs/owncloudsql/owncloudsql.go index dfbb051a84..03d410a90c 100644 --- a/pkg/storage/fs/owncloudsql/owncloudsql.go +++ b/pkg/storage/fs/owncloudsql/owncloudsql.go @@ -54,7 +54,7 @@ import ( "github.com/cs3org/reva/pkg/storage/fs/registry" "github.com/cs3org/reva/pkg/storage/utils/chunking" "github.com/cs3org/reva/pkg/storage/utils/templates" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "github.com/pkg/xattr" "github.com/rs/zerolog/log" @@ -120,16 +120,7 @@ type config struct { DBName string `mapstructure:"dbname"` } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - -func (c *config) init(m map[string]interface{}) { +func (c *config) ApplyDefaults() { if c.UserLayout == "" { c.UserLayout = "{{.Username}}" } @@ -152,17 +143,16 @@ func (c *config) init(m map[string]interface{}) { // New returns an implementation to of the storage.FS interface that talk to // a local filesystem. func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init(m) // c.DataDirectory should never end in / unless it is the root? c.DataDirectory = filepath.Clean(c.DataDirectory) // create datadir if it does not exist - err = os.MkdirAll(c.DataDirectory, 0700) + err := os.MkdirAll(c.DataDirectory, 0700) if err != nil { logger.New().Error().Err(err). Str("path", c.DataDirectory). @@ -183,7 +173,7 @@ func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { } return &owncloudsqlfs{ - c: c, + c: &c, chunkHandler: chunking.NewChunkHandler(c.UploadInfoDir), filecache: filecache, }, nil diff --git a/pkg/storage/fs/s3/s3.go b/pkg/storage/fs/s3/s3.go index 6783a7cdba..3b4983ee3b 100644 --- a/pkg/storage/fs/s3/s3.go +++ b/pkg/storage/fs/s3/s3.go @@ -41,7 +41,7 @@ import ( "github.com/cs3org/reva/pkg/mime" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -58,20 +58,11 @@ type config struct { Prefix string `mapstructure:"prefix"` } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - // New returns an implementation to of the storage.FS interface that talk to // a s3 api. func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } @@ -102,7 +93,7 @@ func New(ctx context.Context, m map[string]interface{}) (storage.FS, error) { s3Client := s3.New(sess) - return &s3FS{client: s3Client, config: c}, nil + return &s3FS{client: s3Client, config: &c}, nil } func (fs *s3FS) Shutdown(ctx context.Context) error { diff --git a/pkg/storage/registry/static/static.go b/pkg/storage/registry/static/static.go index 70382c1808..cef0ec69b1 100644 --- a/pkg/storage/registry/static/static.go +++ b/pkg/storage/registry/static/static.go @@ -32,7 +32,7 @@ import ( "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/registry/registry" "github.com/cs3org/reva/pkg/storage/utils/templates" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -53,7 +53,7 @@ type config struct { HomeProvider string `mapstructure:"home_provider"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.HomeProvider == "" { c.HomeProvider = "/" } @@ -70,23 +70,14 @@ func (c *config) init() { } } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - return c, nil -} - // New returns an implementation of the storage.Registry interface that // redirects requests to corresponding storage drivers. func New(ctx context.Context, m map[string]interface{}) (storage.Registry, error) { - c, err := parseConfig(m) - if err != nil { + var c config + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - return ®{c: c}, nil + return ®{c: &c}, nil } type reg struct { diff --git a/pkg/storage/utils/eosfs/eosfs.go b/pkg/storage/utils/eosfs/eosfs.go index 967ccb3893..7e9201e655 100644 --- a/pkg/storage/utils/eosfs/eosfs.go +++ b/pkg/storage/utils/eosfs/eosfs.go @@ -84,7 +84,7 @@ const LockTypeKey = "reva.lock.type" var hiddenReg = regexp.MustCompile(`\.sys\..#.`) -func (c *Config) init() { +func (c *Config) ApplyDefaults() { c.Namespace = path.Clean(c.Namespace) if !strings.HasPrefix(c.Namespace, "/") { c.Namespace = "/" @@ -162,7 +162,7 @@ type eosfs struct { // NewEOSFS returns a storage.FS interface implementation that connects to an EOS instance. func NewEOSFS(ctx context.Context, c *Config) (storage.FS, error) { - c.init() + c.ApplyDefaults() // bail out if keytab is not found. if c.UseKeytab { diff --git a/pkg/storage/utils/localfs/localfs.go b/pkg/storage/utils/localfs/localfs.go index d6d00b8655..22dcaecaeb 100644 --- a/pkg/storage/utils/localfs/localfs.go +++ b/pkg/storage/utils/localfs/localfs.go @@ -62,7 +62,7 @@ type Config struct { References string `mapstructure:"references"` } -func (c *Config) init() { +func (c *Config) ApplyDefaults() { if c.Root == "" { c.Root = "/var/tmp/reva" } @@ -100,7 +100,7 @@ type localfs struct { // NewLocalFS returns a storage.FS interface implementation that controls then // local filesystem. func NewLocalFS(c *Config) (storage.FS, error) { - c.init() + c.ApplyDefaults() // create namespaces if they do not exist namespaces := []string{c.DataDirectory, c.Uploads, c.Shadow, c.References, c.RecycleBin, c.Versions} diff --git a/pkg/token/manager/jwt/jwt.go b/pkg/token/manager/jwt/jwt.go index f9d7ed2b88..5429e813cd 100644 --- a/pkg/token/manager/jwt/jwt.go +++ b/pkg/token/manager/jwt/jwt.go @@ -28,8 +28,8 @@ import ( "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/token/manager/registry" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/golang-jwt/jwt" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -56,34 +56,27 @@ type claims struct { Scope map[string]*auth.Scope `json:"scope"` } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - return c, nil -} - -// New returns an implementation of the token manager that uses JWT as tokens. -func New(value map[string]interface{}) (token.Manager, error) { - c, err := parseConfig(value) - if err != nil { - return nil, errors.Wrap(err, "error parsing config") - } - +func (c *config) ApplyDefaults() { if c.Expires == 0 { c.Expires = defaultExpiration } c.Secret = sharedconf.GetJWTSecret(c.Secret) +} + +// New returns an implementation of the token manager that uses JWT as tokens. +func New(m map[string]interface{}) (token.Manager, error) { + var c config + if err := cfg.Decode(m, &c); err != nil { + return nil, err + } if c.Secret == "" { return nil, errors.New("jwt: secret for signing payloads is not defined in config") } - m := &manager{conf: c} - return m, nil + mgr := &manager{conf: &c} + return mgr, nil } func (m *manager) MintToken(ctx context.Context, u *user.User, scope map[string]*auth.Scope) (string, error) { diff --git a/pkg/user/manager/json/json.go b/pkg/user/manager/json/json.go index 642fa038e4..76c08cd576 100644 --- a/pkg/user/manager/json/json.go +++ b/pkg/user/manager/json/json.go @@ -29,7 +29,7 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/user" "github.com/cs3org/reva/pkg/user/manager/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" ) @@ -46,22 +46,12 @@ type config struct { Users string `mapstructure:"users"` } -func (c *config) init() { +func (c *config) ApplyDefaults() { if c.Users == "" { c.Users = "/etc/revad/users.json" } } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - c.init() - return c, nil -} - // New returns a user manager implementation that reads a json file to provide user metadata. func New(_ context.Context, m map[string]interface{}) (user.Manager, error) { mgr := &manager{} @@ -73,8 +63,8 @@ func New(_ context.Context, m map[string]interface{}) (user.Manager, error) { } func (m *manager) Configure(ml map[string]interface{}) error { - c, err := parseConfig(ml) - if err != nil { + var c config + if err := cfg.Decode(ml, &c); err != nil { return err } diff --git a/pkg/user/manager/ldap/ldap.go b/pkg/user/manager/ldap/ldap.go index 7d684a7991..b19fbce85c 100644 --- a/pkg/user/manager/ldap/ldap.go +++ b/pkg/user/manager/ldap/ldap.go @@ -33,6 +33,7 @@ import ( "github.com/cs3org/reva/pkg/user" "github.com/cs3org/reva/pkg/user/manager/registry" "github.com/cs3org/reva/pkg/utils" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/go-ldap/ldap/v3" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" @@ -100,6 +101,19 @@ func parseConfig(m map[string]interface{}) (*config, error) { return &c, nil } +func (c *config) ApplyDefaults() { + // backwards compatibility + c.UserFilter = strings.ReplaceAll(c.UserFilter, "%s", "{{.OpaqueId}}") + if c.FindFilter == "" { + c.FindFilter = c.UserFilter + } + c.GroupFilter = strings.ReplaceAll(c.GroupFilter, "%s", "{{.OpaqueId}}") + + if c.Nobody == 0 { + c.Nobody = 99 + } +} + // New returns a user manager implementation that connects to a LDAP server to provide user metadata. func New(ctx context.Context, m map[string]interface{}) (user.Manager, error) { mgr := &manager{} @@ -111,23 +125,14 @@ func New(ctx context.Context, m map[string]interface{}) (user.Manager, error) { } func (m *manager) Configure(ml map[string]interface{}) error { - c, err := parseConfig(ml) - if err != nil { + var c config + c.Schema = ldapDefaults + if err := cfg.Decode(ml, &c); err != nil { return err } - // backwards compatibility - c.UserFilter = strings.ReplaceAll(c.UserFilter, "%s", "{{.OpaqueId}}") - if c.FindFilter == "" { - c.FindFilter = c.UserFilter - } - c.GroupFilter = strings.ReplaceAll(c.GroupFilter, "%s", "{{.OpaqueId}}") - - if c.Nobody == 0 { - c.Nobody = 99 - } - - m.c = c + m.c = &c + var err error m.userfilter, err = template.New("uf").Funcs(sprig.TxtFuncMap()).Parse(c.UserFilter) if err != nil { err := errors.Wrap(err, fmt.Sprintf("error parsing userfilter tpl:%s", c.UserFilter)) diff --git a/pkg/user/manager/nextcloud/nextcloud.go b/pkg/user/manager/nextcloud/nextcloud.go index 274e02b18c..887892b7f4 100644 --- a/pkg/user/manager/nextcloud/nextcloud.go +++ b/pkg/user/manager/nextcloud/nextcloud.go @@ -32,7 +32,7 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/user" "github.com/cs3org/reva/pkg/user/manager/registry" - "github.com/mitchellh/mapstructure" + "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" // "github.com/cs3org/reva/pkg/errtypes". ) @@ -56,22 +56,12 @@ type UserManagerConfig struct { MockHTTP bool `mapstructure:"mock_http"` } -func (c *UserManagerConfig) init() { +func (c *UserManagerConfig) ApplyDefaults() { if c.EndPoint == "" { c.EndPoint = "http://localhost/end/point?" } } -func parseConfig(m map[string]interface{}) (*UserManagerConfig, error) { - c := &UserManagerConfig{} - if err := mapstructure.Decode(m, c); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - c.init() - return c, nil -} - // Action describes a REST request to forward to the Nextcloud backend. type Action struct { verb string @@ -80,13 +70,12 @@ type Action struct { // New returns a user manager implementation that reads a json file to provide user metadata. func New(ctx context.Context, m map[string]interface{}) (user.Manager, error) { - c, err := parseConfig(m) - if err != nil { + var c UserManagerConfig + if err := cfg.Decode(m, &c); err != nil { return nil, err } - c.init() - return NewUserManager(c) + return NewUserManager(&c) } // NewUserManager returns a new Nextcloud-based UserManager. diff --git a/pkg/user/manager/owncloudsql/owncloudsql.go b/pkg/user/manager/owncloudsql/owncloudsql.go index 666119ea7b..2bd9399cf8 100644 --- a/pkg/user/manager/owncloudsql/owncloudsql.go +++ b/pkg/user/manager/owncloudsql/owncloudsql.go @@ -29,10 +29,10 @@ import ( "github.com/cs3org/reva/pkg/user" "github.com/cs3org/reva/pkg/user/manager/owncloudsql/accounts" "github.com/cs3org/reva/pkg/user/manager/registry" + "github.com/cs3org/reva/pkg/utils/cfg" // Provides mysql drivers. _ "github.com/go-sql-driver/mysql" - "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -80,26 +80,20 @@ func NewMysql(ctx context.Context, m map[string]interface{}) (user.Manager, erro return mgr, nil } -func (m *manager) Configure(ml map[string]interface{}) error { - c, err := parseConfig(ml) - if err != nil { - return err - } - +func (c *config) ApplyDefaults() { if c.Nobody == 0 { c.Nobody = 99 } - - m.c = c - return nil } -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, &c); err != nil { - return nil, err +func (m *manager) Configure(ml map[string]interface{}) error { + var c config + if err := cfg.Decode(ml, &c); err != nil { + return err } - return c, nil + + m.c = &c + return nil } func (m *manager) GetUser(ctx context.Context, uid *userpb.UserId, skipFetchingGroups bool) (*userpb.User, error) { diff --git a/pkg/utils/cfg/cfg.go b/pkg/utils/cfg/cfg.go new file mode 100644 index 0000000000..274cfb53c3 --- /dev/null +++ b/pkg/utils/cfg/cfg.go @@ -0,0 +1,90 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package cfg + +import ( + "errors" + "reflect" + + "github.com/cs3org/reva/pkg/errtypes" + "github.com/go-playground/locales/en" + ut "github.com/go-playground/universal-translator" + "github.com/go-playground/validator/v10" + en_translations "github.com/go-playground/validator/v10/translations/en" + "github.com/mitchellh/mapstructure" +) + +// Setter is the interface a configuration struct may implement +// to set the default options. +type Setter interface { + // ApplyDefaults applies the default options. + ApplyDefaults() +} + +var validate = validator.New() +var english = en.New() +var uni = ut.New(english, english) +var trans, _ = uni.GetTranslator("en") +var _ = en_translations.RegisterDefaultTranslations(validate, trans) + +func init() { + validate.RegisterTagNameFunc(func(field reflect.StructField) string { + if k := field.Tag.Get("mapstructure"); k != "" { + return k + } + // if not specified, fall back to field name + return field.Name + }) +} + +// Decode decodes the given raw input interface to the target pointer c. +// It applies the default configuration if the target struct +// implements the Setter interface. +// It also perform a validation to all the fields of the configuration. +func Decode(input map[string]any, c any) error { + config := &mapstructure.DecoderConfig{ + Metadata: nil, + Result: c, + } + + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return err + } + if err := decoder.Decode(input); err != nil { + return err + } + if s, ok := c.(Setter); ok { + s.ApplyDefaults() + } + + return translateError(validate.Struct(c), trans) +} + +func translateError(err error, trans ut.Translator) error { + if err == nil { + return nil + } + errs := err.(validator.ValidationErrors) + translated := make([]error, 0, len(errs)) + for _, err := range errs { + translated = append(translated, errors.New(err.Translate(trans))) + } + return errtypes.Join(translated...) +} diff --git a/pkg/utils/cfg/cfg_test.go b/pkg/utils/cfg/cfg_test.go new file mode 100644 index 0000000000..2e74f2936d --- /dev/null +++ b/pkg/utils/cfg/cfg_test.go @@ -0,0 +1,79 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package cfg_test + +import ( + "testing" + + "github.com/cs3org/reva/pkg/utils/cfg" + "github.com/stretchr/testify/assert" +) + +type NoDefaults struct { + A string `mapstructure:"a"` + B int `mapstructure:"b"` + C bool `mapstructure:"c"` +} + +type WithDefaults struct { + A string `mapstructure:"a"` + B int `mapstructure:"b" validate:"required"` +} + +func (c *WithDefaults) ApplyDefaults() { + if c.A == "" { + c.A = "default" + } +} + +func TestDecode(t *testing.T) { + t1 := map[string]any{ + "b": 10, + "c": true, + } + var noDefaults NoDefaults + if err := cfg.Decode(t1, &noDefaults); err != nil { + t.Fatal("not expected error", err) + } + assert.Equal(t, NoDefaults{ + A: "", + B: 10, + C: true, + }, noDefaults) + + t2 := map[string]any{ + "b": 100, + } + var defaults WithDefaults + if err := cfg.Decode(t2, &defaults); err != nil { + t.Fatal("not expected error", err) + } + assert.Equal(t, WithDefaults{ + A: "default", + B: 100, + }, defaults) + + t3 := map[string]any{ + "a": "string", + } + var required WithDefaults + if err := cfg.Decode(t3, &required); err == nil { + t.Fatal("expected error, but none returned") + } +} diff --git a/tests/integration/grpc/fixtures/ocm-server-cernbox-http.toml b/tests/integration/grpc/fixtures/ocm-server-cernbox-http.toml index 622974dcb3..ed9d667478 100644 --- a/tests/integration/grpc/fixtures/ocm-server-cernbox-http.toml +++ b/tests/integration/grpc/fixtures/ocm-server-cernbox-http.toml @@ -12,6 +12,7 @@ address = "{{grpc_address}}" [http.services.sciencemesh] provider_domain = "{{cernboxhttp_address}}" mesh_directory_url = "http://meshdir" +smtp_credentials = {} [http.middlewares.cors] diff --git a/tests/integration/grpc/fixtures/ocm-server-cesnet-http.toml b/tests/integration/grpc/fixtures/ocm-server-cesnet-http.toml index 93690ec5f4..a4ee3efa9a 100644 --- a/tests/integration/grpc/fixtures/ocm-server-cesnet-http.toml +++ b/tests/integration/grpc/fixtures/ocm-server-cesnet-http.toml @@ -12,6 +12,7 @@ address = "{{grpc_address}}" [http.services.sciencemesh] provider_domain = "{{cesnethttp_address}}" mesh_directory_url = "http://meshdir" +smtp_credentials = {} [http.middlewares.cors] diff --git a/tests/integration/grpc/fixtures/ocm-share/ocm-server-cernbox-grpc.toml b/tests/integration/grpc/fixtures/ocm-share/ocm-server-cernbox-grpc.toml index 49fc9b7bb8..c2c077b74c 100644 --- a/tests/integration/grpc/fixtures/ocm-share/ocm-server-cernbox-grpc.toml +++ b/tests/integration/grpc/fixtures/ocm-share/ocm-server-cernbox-grpc.toml @@ -58,6 +58,7 @@ providers = "{{file_providers}}" [grpc.services.ocmshareprovider] driver = "json" webdav_endpoint = "http://{{cernboxwebdav_address}}" +provider_domain = "cernbox.cern.ch" [grpc.services.ocmshareprovider.drivers.json] file = "{{ocm_share_cernbox_file}}" diff --git a/tests/integration/grpc/fixtures/ocm-share/ocm-server-cernbox-http.toml b/tests/integration/grpc/fixtures/ocm-share/ocm-server-cernbox-http.toml index 95a6f7dec5..03a8a0b89b 100644 --- a/tests/integration/grpc/fixtures/ocm-share/ocm-server-cernbox-http.toml +++ b/tests/integration/grpc/fixtures/ocm-share/ocm-server-cernbox-http.toml @@ -12,6 +12,7 @@ address = "{{grpc_address}}" [http.services.sciencemesh] provider_domain = "{{cernboxhttp_address}}" mesh_directory_url = "http://meshdir" +smtp_credentials = {} [http.middlewares.cors] diff --git a/tests/integration/grpc/fixtures/ocm-share/ocm-server-cesnet-grpc.toml b/tests/integration/grpc/fixtures/ocm-share/ocm-server-cesnet-grpc.toml index edd7e950f6..3a225319ab 100644 --- a/tests/integration/grpc/fixtures/ocm-share/ocm-server-cesnet-grpc.toml +++ b/tests/integration/grpc/fixtures/ocm-share/ocm-server-cesnet-grpc.toml @@ -37,6 +37,8 @@ providers = "{{file_providers}}" [grpc.services.ocmshareprovider] driver = "json" +webdav_endpoint = "http://{{cesnethttp_address}}" +provider_domain = "cesnet.cz" [grpc.services.ocmshareprovider.drivers.json] file = "{{ocm_share_cesnet_file}}" diff --git a/tests/integration/grpc/fixtures/ocm-share/ocm-server-cesnet-http.toml b/tests/integration/grpc/fixtures/ocm-share/ocm-server-cesnet-http.toml index 11ade22fe3..6a12f8eb99 100644 --- a/tests/integration/grpc/fixtures/ocm-share/ocm-server-cesnet-http.toml +++ b/tests/integration/grpc/fixtures/ocm-share/ocm-server-cesnet-http.toml @@ -12,6 +12,7 @@ address = "{{grpc_address}}" [http.services.sciencemesh] provider_domain = "{{cesnethttp_address}}" mesh_directory_url = "http://meshdir" +smtp_credentials = {} [http.middlewares.cors] diff --git a/tests/integration/grpc/ocm_invitation_test.go b/tests/integration/grpc/ocm_invitation_test.go index be47dcce89..c4addc6c79 100644 --- a/tests/integration/grpc/ocm_invitation_test.go +++ b/tests/integration/grpc/ocm_invitation_test.go @@ -124,7 +124,7 @@ var _ = Describe("ocm invitation workflow", func() { } ) - for _, driver := range []string{"json", "sql"} { + for _, driver := range []string{"json"} { JustBeforeEach(func() { tokenManager, err := jwt.New(map[string]interface{}{"secret": "changemeplease"}) From 9060597de93800ec0193c3a04d06bbfa9f1ab680 Mon Sep 17 00:00:00 2001 From: Prajwol Amatya <83579989+PrajwolAmatya@users.noreply.github.com> Date: Thu, 6 Jul 2023 11:31:01 +0545 Subject: [PATCH 38/48] bump latest ocis commit on master (#4038) --- .drone.env | 2 +- .../expected-failures-on-OCIS-storage.md | 108 +++++++++--------- .../expected-failures-on-S3NG-storage.md | 108 +++++++++--------- tests/ocis | 2 +- 4 files changed, 110 insertions(+), 110 deletions(-) diff --git a/.drone.env b/.drone.env index a792be49f4..8186d3e56f 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=a8ff963166ecd9adf3f44aa6fa9fe68f53517d05 +APITESTS_COMMITID=941ef59cc97793ebd18df1c344a9b45cb96b46ff APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index 910bca752a..031479832f 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -139,10 +139,10 @@ File and sync features in a shared scenario #### [Getting content of a shared file with same name returns 500](https://github.com/owncloud/ocis/issues/3880) - [coreApiShareManagementToShares/acceptShares.feature:459](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/acceptShares.feature#L459) - [coreApiShareManagementToShares/acceptShares.feature:524](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/acceptShares.feature#L524) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:131](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L131) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:130](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L130) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:166](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L166) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:165](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L165) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:128](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L128) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:129](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L129) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:161](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L161) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:162](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L162) - [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:40](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L40) - [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:39](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L39) @@ -154,24 +154,24 @@ File and sync features in a shared scenario - [coreApiShareManagementToShares/mergeShare.feature:111](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/mergeShare.feature#L111) #### [[OCIS-storage] not possible to move file into a received folder](https://github.com/owncloud/ocis/issues/764) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:531](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L531) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:529](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L529) #### [File deletion using dav gives unique string in filename in the trashbin](https://github.com/owncloud/product/issues/178) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:64](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L64) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:78](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L78) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:49](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L49) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:75](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L75) cannot share a folder with create permission #### [Listing shares via ocs API does not show path for parent folders](https://github.com/owncloud/ocis/issues/1231) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:131](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L131) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:144](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L144) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:179](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L179) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:180](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L180) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:178](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L178) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:181](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L181) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:198](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L198) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:199](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L199) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:197](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L197) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:200](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L200) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:125](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L125) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:138](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L138) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:172](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L172) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:173](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L173) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:174](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L174) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:175](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L175) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:191](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L191) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:192](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L192) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:193](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L193) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:194](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L194) - [coreApiShareOperationsToShares1/gettingShares.feature:189](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareOperationsToShares1/gettingShares.feature#L189) - [coreApiShareOperationsToShares1/gettingShares.feature:190](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareOperationsToShares1/gettingShares.feature#L190) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:37](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L37) @@ -237,12 +237,12 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiShareReshareToShares2/reShareSubfolder.feature:143](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares2/reShareSubfolder.feature#L143) - [coreApiShareReshareToShares2/reShareSubfolder.feature:144](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares2/reShareSubfolder.feature#L144) - [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:61](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L61) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:743](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L743) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:744](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L744) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:762](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L762) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:763](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L763) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:778](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L778) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:779](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L779) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:733](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L733) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:734](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L734) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:752](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L752) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:753](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L753) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:768](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L768) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:769](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L769) #### [deleting a file inside a received shared folder is moved to the trash-bin of the sharer not the receiver](https://github.com/owncloud/ocis/issues/1124) @@ -293,11 +293,11 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiWebdavMove2/moveFile.feature:177](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L177) #### [OCIS-storage overwriting a file as share receiver, does not create a new file version for the sharer](https://github.com/owncloud/ocis/issues/766) //todo -- [coreApiVersions/fileVersions.feature:278](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L278) +- [coreApiVersions/fileVersions.feature:275](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L275) #### [restoring an older version of a shared file deletes the share](https://github.com/owncloud/ocis/issues/765) - [coreApiShareManagementToShares/acceptShares.feature:448](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/acceptShares.feature#L448) -- [coreApiVersions/fileVersions.feature:290](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L290) +- [coreApiVersions/fileVersions.feature:287](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L287) #### [Expiration date for shares is not implemented](https://github.com/owncloud/ocis/issues/1250) #### Expiration date of user shares @@ -314,7 +314,7 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiShareCreateSpecialToShares1/createShareUniqueReceivedNames.feature:16](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares1/createShareUniqueReceivedNames.feature#L16) #### [Empty OCS response for a share create request using a disabled user](https://github.com/owncloud/ocis/issues/2212) -- [coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature:24](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature#L24) +- [coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature:12](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature#L12) - [coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature:21](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature#L21) - [coreApiShareUpdateToShares/updateShare.feature:97](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L97) - [coreApiShareUpdateToShares/updateShare.feature:98](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L98) @@ -330,22 +330,22 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiShareUpdateToShares/updateShare.feature:126](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L126) #### [Edit user share response has an "name" field](https://github.com/owncloud/ocis/issues/1225) -- [coreApiShareUpdateToShares/updateShare.feature:243](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L243) -- [coreApiShareUpdateToShares/updateShare.feature:244](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L244) +- [coreApiShareUpdateToShares/updateShare.feature:236](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L236) +- [coreApiShareUpdateToShares/updateShare.feature:237](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L237) #### [user can access version metadata of a received share before accepting it](https://github.com/owncloud/ocis/issues/760) -- [coreApiVersions/fileVersions.feature:315](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L315) +- [coreApiVersions/fileVersions.feature:312](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L312) #### [Share lists deleted user as 'user'](https://github.com/owncloud/ocis/issues/903) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:678](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L678) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:679](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L679) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:668](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L668) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:669](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L669) #### [OCIS-storage overwriting a file as share receiver, does not create a new file version for the sharer](https://github.com/owncloud/ocis/issues/766) -- [coreApiVersions/fileVersions.feature:435](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L435) //todo +- [coreApiVersions/fileVersions.feature:432](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L432) //todo #### [deleting a share with wrong authentication returns OCS status 996 / HTTP 500](https://github.com/owncloud/ocis/issues/1229) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:227](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L227) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:228](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L228) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:221](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L221) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:222](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L222) ### User Management User and group management features @@ -452,8 +452,8 @@ _ocdav: api compatibility, return correct status code_ - [coreApiAuthWebDav/webDavDELETEAuth.feature:109](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavDELETEAuth.feature#L109) #### [Sharing a same file twice to the same group](https://github.com/owncloud/ocis/issues/1710) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:726](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L726) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:727](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L727) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:716](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L716) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:717](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L717) #### [Request to edit non-existing user by authorized admin gets unauthorized in http response](https://github.com/owncloud/ocis/issues/38423) - [coreApiAuthOcs/ocsPUTAuth.feature:27](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthOcs/ocsPUTAuth.feature#L27) @@ -562,14 +562,14 @@ _ocs: api compatibility, return correct status code_ - [coreApiWebdavProperties2/getFileProperties.feature:276](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties2/getFileProperties.feature#L276) #### [Cannot move folder/file from one received share to another](https://github.com/owncloud/ocis/issues/2442) -- - [coreApiShareUpdateToShares/updateShare.feature:197](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L197) +- - [coreApiShareUpdateToShares/updateShare.feature:129](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L129) - [coreApiShareUpdateToShares/updateShare.feature:161](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L161) - [coreApiShareManagementToShares/mergeShare.feature:131](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/mergeShare.feature#L131) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:262](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L262) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:297](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L297) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:406](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L406) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:441](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L441) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:222](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L222) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:253](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L253) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:352](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L352) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:383](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L383) #### [copying the file inside Shares folder returns 404](https://github.com/owncloud/ocis/issues/3874) - [coreApiWebdavProperties1/copyFile.feature:409](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties1/copyFile.feature#L409) @@ -578,9 +578,9 @@ _ocs: api compatibility, return correct status code_ - [coreApiWebdavProperties1/copyFile.feature:434](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties1/copyFile.feature#L434) #### [Downloading the older version of shared file gives 404](https://github.com/owncloud/ocis/issues/3868) -- [coreApiVersions/fileVersions.feature:162](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L162) -- [coreApiVersions/fileVersions.feature:180](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L180) -- [coreApiVersions/fileVersions.feature:447](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L447) +- [coreApiVersions/fileVersions.feature:159](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L159) +- [coreApiVersions/fileVersions.feature:177](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L177) +- [coreApiVersions/fileVersions.feature:444](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L444) #### [file versions do not report the version author](https://github.com/owncloud/ocis/issues/2914) - [coreApiVersions/fileVersionAuthor.feature:15](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L15) @@ -597,7 +597,7 @@ _ocs: api compatibility, return correct status code_ #### [Shares to deleted group listed in the response](https://github.com/owncloud/ocis/issues/2441) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:530](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L530) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:534](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L534) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:533](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L533) ### [Share path in the response is different between share states](https://github.com/owncloud/ocis/issues/2540) - [coreApiShareManagementToShares/acceptShares.feature:28](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/acceptShares.feature#L28) @@ -736,16 +736,16 @@ _ocs: api compatibility, return correct status code_ - [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:74](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L74) - [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:96](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L96) - [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:97](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L97) -- [coreApiVersions/fileVersions.feature:239](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L239) +- [coreApiVersions/fileVersions.feature:232](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L232) #### [`meta` requests have empty responses with master branch](https://github.com/cs3org/reva/issues/2897) -- [coreApiVersions/fileVersions.feature:199](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L199) -- [coreApiVersions/fileVersions.feature:205](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L205) -- [coreApiVersions/fileVersions.feature:212](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L212) -- [coreApiVersions/fileVersions.feature:220](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L220) +- [coreApiVersions/fileVersions.feature:196](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L196) +- [coreApiVersions/fileVersions.feature:202](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L202) +- [coreApiVersions/fileVersions.feature:209](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L209) +- [coreApiVersions/fileVersions.feature:217](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L217) +- [coreApiVersions/fileVersions.feature:230](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L230) +- [coreApiVersions/fileVersions.feature:231](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L231) - [coreApiVersions/fileVersions.feature:233](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L233) -- [coreApiVersions/fileVersions.feature:234](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L234) -- [coreApiVersions/fileVersions.feature:235](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L235) - [coreApiVersions/fileVersions.feature:236](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L236) #### [WebDAV MOVE with body returns 400 rather than 415](https://github.com/cs3org/reva/issues/3119) @@ -777,8 +777,8 @@ The problem has been fixed in reva edge branch but not in reva master #### [Updating the role of a public link to internal gives returns 400] - [coreApiSharePublicLink3/updatePublicLinkShare.feature:483](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L483) - [coreApiSharePublicLink3/updatePublicLinkShare.feature:484](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L484) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:485](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L485) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:486](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L486) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:481](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L481) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:482](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L482) #### [Default capabilities for normal user and admin user not same as in oC-core](https://github.com/owncloud/ocis/issues/1285) - [coreApiCapabilities/capabilities.feature:11](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiCapabilities/capabilities.feature#L11) diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index c898b9ca10..df34b78db0 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -10,9 +10,9 @@ Basic file management like up and download, move, copy, properties, quota, trash - [coreApiTrashbin/trashbinFilesFolders.feature:268](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinFilesFolders.feature#L268) ### [Downloading the older version of shared file gives 404](https://github.com/owncloud/ocis/issues/3868) -- [coreApiVersions/fileVersions.feature:162](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L162) -- [coreApiVersions/fileVersions.feature:180](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L180) -- [coreApiVersions/fileVersions.feature:447](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L447) +- [coreApiVersions/fileVersions.feature:159](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L159) +- [coreApiVersions/fileVersions.feature:177](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L177) +- [coreApiVersions/fileVersions.feature:444](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L444) #### [file versions do not report the version author](https://github.com/owncloud/ocis/issues/2914) - [coreApiVersions/fileVersionAuthor.feature:15](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L15) @@ -154,10 +154,10 @@ File and sync features in a shared scenario #### [Getting content of a shared file with same name returns 500](https://github.com/owncloud/ocis/issues/3880) - [coreApiShareManagementToShares/acceptShares.feature:459](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/acceptShares.feature#L459) - [coreApiShareManagementToShares/acceptShares.feature:524](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/acceptShares.feature#L524) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:131](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L131) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:130](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L130) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:166](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L166) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:165](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L165) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:128](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L128) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:129](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L129) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:161](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L161) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:162](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L162) - [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:40](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L40) - [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:39](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L39) @@ -175,31 +175,31 @@ File and sync features in a shared scenario - [coreApiShareManagementToShares/mergeShare.feature:111](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/mergeShare.feature#L111) #### [not possible to move file into a received folder](https://github.com/owncloud/ocis/issues/764) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:531](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L531) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:529](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L529) #### [path property in pending shares gives only filename](https://github.com/owncloud/ocis/issues/2156) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:726](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L726) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:727](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L727) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:743](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L743) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:744](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L744) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:762](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L762) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:763](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L763) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:716](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L716) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:717](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L717) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:733](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L733) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:734](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L734) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:752](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L752) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:753](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L753) #### [File deletion using dav gives unique string in filename in the trashbin](https://github.com/owncloud/product/issues/178) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:64](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L64) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:78](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L78) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:49](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L49) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:75](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L75) cannot share a folder with create permission #### [Listing shares via ocs API does not show path for parent folders](https://github.com/owncloud/ocis/issues/1231) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:131](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L131) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:144](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L144) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:179](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L179) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:180](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L180) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:178](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L178) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:181](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L181) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:198](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L198) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:199](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L199) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:197](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L197) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:200](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L200) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:125](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L125) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:138](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L138) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:172](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L172) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:173](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L173) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:174](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L174) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:175](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L175) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:191](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L191) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:192](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L192) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:193](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L193) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:194](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L194) - [coreApiShareOperationsToShares1/gettingShares.feature:189](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareOperationsToShares1/gettingShares.feature#L189) - [coreApiShareOperationsToShares1/gettingShares.feature:190](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareOperationsToShares1/gettingShares.feature#L190) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:327](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L327) @@ -314,11 +314,11 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiWebdavMove2/moveFile.feature:177](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L177) #### [OCIS-storage overwriting a file as share receiver, does not create a new file version for the sharer](https://github.com/owncloud/ocis/issues/766) //todo -- [coreApiVersions/fileVersions.feature:278](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L278) +- [coreApiVersions/fileVersions.feature:275](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L275) #### [restoring an older version of a shared file deletes the share](https://github.com/owncloud/ocis/issues/765) - [coreApiShareManagementToShares/acceptShares.feature:448](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/acceptShares.feature#L448) -- [coreApiVersions/fileVersions.feature:290](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L290) +- [coreApiVersions/fileVersions.feature:287](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L287) #### [Expiration date for shares is not implemented](https://github.com/owncloud/ocis/issues/1250) #### Expiration date of user shares @@ -336,7 +336,7 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiShareCreateSpecialToShares1/createShareUniqueReceivedNames.feature:16](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares1/createShareUniqueReceivedNames.feature#L16) #### [Empty OCS response for a share create request using a disabled user](https://github.com/owncloud/ocis/issues/2212) -- [coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature:24](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature#L24) +- [coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature:12](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature#L12) - [coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature:21](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareWithDisabledUser.feature#L21) - [coreApiShareUpdateToShares/updateShare.feature:97](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L97) - [coreApiShareUpdateToShares/updateShare.feature:98](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L98) @@ -352,22 +352,22 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiShareUpdateToShares/updateShare.feature:126](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L126) #### [Edit user share response has an "name" field](https://github.com/owncloud/ocis/issues/1225) -- [coreApiShareUpdateToShares/updateShare.feature:243](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L243) -- [coreApiShareUpdateToShares/updateShare.feature:244](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L244) +- [coreApiShareUpdateToShares/updateShare.feature:236](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L236) +- [coreApiShareUpdateToShares/updateShare.feature:237](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L237) #### [user can access version metadata of a received share before accepting it](https://github.com/owncloud/ocis/issues/760) -- [coreApiVersions/fileVersions.feature:315](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L315) +- [coreApiVersions/fileVersions.feature:312](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L312) #### [Share lists deleted user as 'user'](https://github.com/owncloud/ocis/issues/903) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:678](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L678) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:679](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L679) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:668](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L668) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:669](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L669) #### [OCIS-storage overwriting a file as share receiver, does not create a new file version for the sharer](https://github.com/owncloud/ocis/issues/766) //todo -- [coreApiVersions/fileVersions.feature:435](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L435) +- [coreApiVersions/fileVersions.feature:432](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L432) #### [deleting a share with wrong authentication returns OCS status 996 / HTTP 500](https://github.com/owncloud/ocis/issues/1229) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:227](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L227) -- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:228](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L228) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:221](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L221) +- [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:222](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L222) ### User Management User and group management features @@ -474,8 +474,8 @@ _ocdav: api compatibility, return correct status code_ - [coreApiAuthWebDav/webDavDELETEAuth.feature:109](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthWebDav/webDavDELETEAuth.feature#L109) #### [Sharing a same file twice to the same group](https://github.com/owncloud/ocis/issues/1710) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:778](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L778) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:779](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L779) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:768](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L768) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:769](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L769) #### [Request to edit non-existing user by authorized admin gets unauthorized in http response](https://github.com/owncloud/ocis/issues/38423) - [coreApiAuthOcs/ocsPUTAuth.feature:27](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiAuthOcs/ocsPUTAuth.feature#L27) @@ -583,15 +583,15 @@ _ocs: api compatibility, return correct status code_ - [coreApiWebdavProperties2/getFileProperties.feature:276](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties2/getFileProperties.feature#L276) #### [Cannot move folder/file from one received share to another](https://github.com/owncloud/ocis/issues/2442) -- [coreApiShareUpdateToShares/updateShare.feature:197](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L197) +- [coreApiShareUpdateToShares/updateShare.feature:129](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L129) - [coreApiShareUpdateToShares/updateShare.feature:161](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L161) - [coreApiShareManagementToShares/mergeShare.feature:131](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/mergeShare.feature#L131) #### [Sharing folder and sub-folder with same user but different permission,the permission of sub-folder is not obeyed ](https://github.com/owncloud/ocis/issues/2440) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:262](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L262) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:297](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L297) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:406](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L406) -- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:441](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L441) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:222](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L222) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:253](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L253) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:352](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L352) +- [coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature:383](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareCreateSpecialToShares2/createShareReceivedInMultipleWays.feature#L383) #### [copying the file inside Shares folder returns 404](https://github.com/owncloud/ocis/issues/3874) - [coreApiWebdavProperties1/copyFile.feature:409](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavProperties1/copyFile.feature#L409) @@ -601,7 +601,7 @@ _ocs: api compatibility, return correct status code_ #### [Shares to deleted group listed in the response](https://github.com/owncloud/ocis/issues/2441) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:530](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L530) -- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:534](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L534) +- [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:533](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L533) ### [Share path in the response is different between share states](https://github.com/owncloud/ocis/issues/2540) - [coreApiShareManagementToShares/acceptShares.feature:28](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/acceptShares.feature#L28) @@ -740,16 +740,16 @@ _ocs: api compatibility, return correct status code_ - [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:74](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L74) - [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:96](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L96) - [coreApiWebdavEtagPropagation2/restoreFromTrash.feature:97](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavEtagPropagation2/restoreFromTrash.feature#L97) -- [coreApiVersions/fileVersions.feature:239](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L239) +- [coreApiVersions/fileVersions.feature:232](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L232) #### [`meta` requests have empty responses with master branch](https://github.com/cs3org/reva/issues/2897) -- [coreApiVersions/fileVersions.feature:199](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L199) -- [coreApiVersions/fileVersions.feature:205](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L205) -- [coreApiVersions/fileVersions.feature:212](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L212) -- [coreApiVersions/fileVersions.feature:220](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L220) +- [coreApiVersions/fileVersions.feature:196](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L196) +- [coreApiVersions/fileVersions.feature:202](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L202) +- [coreApiVersions/fileVersions.feature:209](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L209) +- [coreApiVersions/fileVersions.feature:217](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L217) +- [coreApiVersions/fileVersions.feature:230](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L230) +- [coreApiVersions/fileVersions.feature:231](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L231) - [coreApiVersions/fileVersions.feature:233](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L233) -- [coreApiVersions/fileVersions.feature:234](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L234) -- [coreApiVersions/fileVersions.feature:235](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L235) - [coreApiVersions/fileVersions.feature:236](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L236) #### [WebDAV MOVE with body returns 400 rather than 415](https://github.com/cs3org/reva/issues/3119) @@ -780,8 +780,8 @@ The problem has been fixed in reva edge branch but not in reva master #### [Updating the role of a public link to internal gives returns 400] - [coreApiSharePublicLink3/updatePublicLinkShare.feature:483](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L483) - [coreApiSharePublicLink3/updatePublicLinkShare.feature:484](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L484) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:485](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L485) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:486](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L486) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:481](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L481) +- [coreApiSharePublicLink3/updatePublicLinkShare.feature:482](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L482) #### [Default capabilities for normal user and admin user not same as in oC-core](https://github.com/owncloud/ocis/issues/1285) - [coreApiCapabilities/capabilities.feature:11](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiCapabilities/capabilities.feature#L11) diff --git a/tests/ocis b/tests/ocis index a8ff963166..ca45cb8b46 160000 --- a/tests/ocis +++ b/tests/ocis @@ -1 +1 @@ -Subproject commit a8ff963166ecd9adf3f44aa6fa9fe68f53517d05 +Subproject commit ca45cb8b461752289ba0f8a57baed9289f7e35a7 From 6997a51d6efd1cb0957497bbef5700398f02afa1 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 6 Jul 2023 17:44:41 +0200 Subject: [PATCH 39/48] Added /app/notify endpoint for logging/tracking apps (#4044) --- changelog/unreleased/app-notif.md | 7 ++++ .../http/services/appprovider/appprovider.go | 37 +++++++++++++++++++ pkg/app/provider/wopi/wopi.go | 1 - 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/app-notif.md diff --git a/changelog/unreleased/app-notif.md b/changelog/unreleased/app-notif.md new file mode 100644 index 0000000000..f40064a213 --- /dev/null +++ b/changelog/unreleased/app-notif.md @@ -0,0 +1,7 @@ +Enhancement: added an /app/notify endpoint for logging/tracking apps + +The new endpoint serves to probe the health state of apps such as +Microsoft Office Online, and it is expected to be called by the frontend +upon successful loading of the document by the underlying app + +https://github.com/cs3org/reva/pull/4044 diff --git a/internal/http/services/appprovider/appprovider.go b/internal/http/services/appprovider/appprovider.go index 049b8e83a3..cb0d84698b 100644 --- a/internal/http/services/appprovider/appprovider.go +++ b/internal/http/services/appprovider/appprovider.go @@ -30,6 +30,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/datagateway" + "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp" @@ -90,6 +91,7 @@ func (s *svc) routerInit() error { s.router.Get("/list", s.handleList) s.router.Post("/new", s.handleNew) s.router.Post("/open", s.handleOpen) + s.router.Post("/notify", s.handleNotify) return nil } @@ -418,6 +420,9 @@ func (s *svc) handleOpen(w http.ResponseWriter, r *http.Request) { return } + log := appctx.GetLogger(ctx) + log.Info().Str("url", openRes.AppUrl.AppUrl).Interface("resource", fileRef).Msg("returning app URL for file") + w.Header().Set("Content-Type", "application/json") if _, err = w.Write(js); err != nil { writeError(w, r, appErrorServerError, "Internal error with JSON payload", @@ -426,6 +431,38 @@ func (s *svc) handleOpen(w http.ResponseWriter, r *http.Request) { } } +func (s *svc) handleNotify(w http.ResponseWriter, r *http.Request) { + err := r.ParseForm() + if err != nil { + writeError(w, r, appErrorInvalidParameter, "parameters could not be parsed", nil) + } + + fileID := r.Form.Get("file_id") + var fileRef provider.Reference + if fileID == "" { + path := r.Form.Get("path") + if path == "" { + writeError(w, r, appErrorInvalidParameter, "missing file ID or path", nil) + return + } + fileRef.Path = path + } else { + resourceID := resourceid.OwnCloudResourceIDUnwrap(fileID) + if resourceID == nil { + writeError(w, r, appErrorInvalidParameter, "invalid file ID", nil) + return + } + fileRef.ResourceId = resourceID + } + + // log the fileid for later correlation / monitoring + ctx := r.Context() + log := appctx.GetLogger(ctx) + log.Info().Interface("resource", fileRef).Msg("file successfully opened in app") + + w.WriteHeader(http.StatusOK) +} + func filterAppsByUserAgent(mimeTypes []*appregistry.MimeTypeInfo, userAgent string) []*appregistry.MimeTypeInfo { ua := ua.Parse(userAgent) res := []*appregistry.MimeTypeInfo{} diff --git a/pkg/app/provider/wopi/wopi.go b/pkg/app/provider/wopi/wopi.go index 86bdf2c07b..0ecdb42073 100644 --- a/pkg/app/provider/wopi/wopi.go +++ b/pkg/app/provider/wopi/wopi.go @@ -332,7 +332,6 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc } } - log.Info().Str("url", appFullURL).Str("resource", resource.Path).Msg("wopi: returning URL for file") return &appprovider.OpenInAppURL{ AppUrl: appFullURL, Method: method, From b38215dd8772079332e7e6ef6d91bf1d422bfed9 Mon Sep 17 00:00:00 2001 From: Javier Ferrer Date: Tue, 11 Jul 2023 15:21:03 +0200 Subject: [PATCH 40/48] Improve error checking code and fix sql script (#4041) --- changelog/unreleased/notifications-changes.md | 3 +++ .../services/notifications/notifications.go | 15 +++++++-------- pkg/notification/db_changes.sql | 19 ++++++++++--------- pkg/notification/notification.go | 4 ++-- pkg/notification/template/template.go | 2 +- 5 files changed, 23 insertions(+), 20 deletions(-) create mode 100644 changelog/unreleased/notifications-changes.md diff --git a/changelog/unreleased/notifications-changes.md b/changelog/unreleased/notifications-changes.md new file mode 100644 index 0000000000..49e05ae82b --- /dev/null +++ b/changelog/unreleased/notifications-changes.md @@ -0,0 +1,3 @@ +Change: Clean up notifications error checking code, fix sql creation script + +https://github.com/cs3org/reva/pull/4041 diff --git a/internal/serverless/services/notifications/notifications.go b/internal/serverless/services/notifications/notifications.go index a1df4b066a..22256b90c0 100644 --- a/internal/serverless/services/notifications/notifications.go +++ b/internal/serverless/services/notifications/notifications.go @@ -221,14 +221,13 @@ func (s *svc) handleMsgTemplate(msg []byte) { name, err := s.templates.Put(msg, s.handlers) if err != nil { - s.log.Error().Err(err).Msgf("template registration failed %v", err) + s.log.Error().Err(err).Msg("template registration failed") // If a template file was not found, delete that template from the registry altogether, // this way we ensure templates that are deleted from the config are deleted from the // store too. - wrappedErr := errors.Unwrap(errors.Unwrap(err)) - _, isFileNotFoundError := wrappedErr.(*template.FileNotFoundError) - if isFileNotFoundError && name != "" { + var e *template.FileNotFoundError + if errors.As(err, &e) && name != "" { err := s.kv.Purge(name) if err != nil { s.log.Error().Err(err).Msgf("deletion of template %s from store failed", name) @@ -274,8 +273,8 @@ func (s *svc) handleMsgUnregisterNotification(msg *nats.Msg) { err := s.nm.DeleteNotification(ref) if err != nil { - _, isNotFoundError := err.(*notification.NotFoundError) - if isNotFoundError { + var e *notification.NotFoundError + if errors.As(err, &e) { s.log.Debug().Msgf("a notification with ref %s does not exist", ref) } else { s.log.Error().Err(err).Msgf("notification unregister failed") @@ -322,8 +321,8 @@ func (s *svc) handleMsgTrigger(msg *nats.Msg) { if notif == nil { notif, err = s.nm.GetNotification(tr.Ref) if err != nil { - _, isNotFoundError := err.(*notification.NotFoundError) - if isNotFoundError { + var e *notification.NotFoundError + if errors.As(err, &e) { s.log.Debug().Msgf("trigger %s does not have a notification attached", tr.Ref) return } diff --git a/pkg/notification/db_changes.sql b/pkg/notification/db_changes.sql index a240b63300..234c2a2558 100644 --- a/pkg/notification/db_changes.sql +++ b/pkg/notification/db_changes.sql @@ -19,7 +19,7 @@ -- This file can be used to make the required changes to the MySQL DB. This is -- not a proper migration but it should work on most situations. -CREATE TABLE `cbox_notifications` ( +CREATE TABLE `notifications` ( `id` INT PRIMARY KEY AUTO_INCREMENT, `ref` VARCHAR(3072) UNIQUE NOT NULL, `template_name` VARCHAR(320) NOT NULL @@ -27,26 +27,27 @@ CREATE TABLE `cbox_notifications` ( COMMIT; -CREATE TABLE `cbox_notification_recipients` ( +CREATE TABLE `notification_recipients` ( `id` INT PRIMARY KEY AUTO_INCREMENT, `notification_id` INT NOT NULL, `recipient` VARCHAR(320) NOT NULL, FOREIGN KEY (notification_id) - REFERENCES cbox_notifications (id) + REFERENCES notifications (id) ON DELETE CASCADE ); COMMIT; -CREATE INDEX `cbox_notifications_ix0` ON `cbox_notifications` (`ref`); +CREATE INDEX `notifications_ix0` ON `notifications` (`ref`); -CREATE INDEX `cbox_notification_recipients_ix0` ON `cbox_notification_recipients` (`notification_id`); -CREATE INDEX `cbox_notification_recipients_ix1` ON `cbox_notification_recipients` (`user_name`); +CREATE INDEX `notification_recipients_ix0` ON `notification_recipients` (`notification_id`); +CREATE INDEX `notification_recipients_ix1` ON `notification_recipients` (`recipient`); -- changes for added notifications on oc shares -ALTER TABLE cernboxngcopy.oc_share ADD notify_uploads BOOL DEFAULT false; +ALTER TABLE oc_share ADD notify_uploads BOOL DEFAULT false; +ALTER TABLE oc_share ADD notify_uploads_extra_recipients VARCHAR(2048); -UPDATE cernboxngcopy.oc_share SET notify_uploads = false; +UPDATE oc_share SET notify_uploads = false; -ALTER TABLE cernboxngcopy.oc_share MODIFY notify_uploads BOOL DEFAULT false NOT NULL; +ALTER TABLE oc_share MODIFY notify_uploads BOOL DEFAULT false NOT NULL; diff --git a/pkg/notification/notification.go b/pkg/notification/notification.go index fdf615f369..b7c2d50257 100644 --- a/pkg/notification/notification.go +++ b/pkg/notification/notification.go @@ -55,12 +55,12 @@ type InvalidNotificationError struct { } // Error returns the string error msg for NotFoundError. -func (n *NotFoundError) Error() string { +func (n NotFoundError) Error() string { return fmt.Sprintf("notification %s not found", n.Ref) } // Error returns the string error msg for InvalidNotificationError. -func (i *InvalidNotificationError) Error() string { +func (i InvalidNotificationError) Error() string { return i.Msg } diff --git a/pkg/notification/template/template.go b/pkg/notification/template/template.go index 39d5fc0c18..d1ad5f85ff 100644 --- a/pkg/notification/template/template.go +++ b/pkg/notification/template/template.go @@ -60,7 +60,7 @@ type FileNotFoundError struct { } // Error returns the string error msg for FileNotFoundError. -func (t *FileNotFoundError) Error() string { +func (t FileNotFoundError) Error() string { return fmt.Sprintf("template file %s not found", t.TemplateFileName) } From 8731461a6f6fac11b8d883dc39b68ab284c04e4f Mon Sep 17 00:00:00 2001 From: Nalem7 <61624650+nabim777@users.noreply.github.com> Date: Mon, 17 Jul 2023 19:44:57 +0545 Subject: [PATCH 41/48] [tests-only][full-ci]bump latest ocis commit id master (#4064) --- .drone.env | 2 +- .drone.star | 4 ++-- README.md | 2 +- tests/acceptance/expected-failures-on-OCIS-storage.md | 4 ++-- tests/acceptance/expected-failures-on-S3NG-storage.md | 4 ++-- tests/docker/docker-compose.yml | 4 ++-- tests/ocis | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.drone.env b/.drone.env index 8186d3e56f..57583443a2 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=941ef59cc97793ebd18df1c344a9b45cb96b46ff +APITESTS_COMMITID=6e45113e76c60965c5e1a3bad42e5b5459960571 APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/.drone.star b/.drone.star index f7e6c558d7..d90a8f9836 100644 --- a/.drone.star +++ b/.drone.star @@ -119,7 +119,7 @@ def ocisIntegrationTest(): "REVA_LDAP_HOSTNAME": "ldap", "TEST_REVA": "true", "SEND_SCENARIO_LINE_REFERENCES": "true", - "BEHAT_FILTER_TAGS": "~@provisioning_api-app-required&&~@preview-extension-required&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@skipOnGraph&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster", + "BEHAT_FILTER_TAGS": "~@provisioning_api-app-required&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@skipOnGraph&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster", "DIVIDE_INTO_NUM_PARTS": 6, "RUN_PART": 2, "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-OCIS-storage.md", @@ -190,7 +190,7 @@ def s3ngIntegrationTests(): "REVA_LDAP_HOSTNAME": "ldap", "TEST_REVA": "true", "SEND_SCENARIO_LINE_REFERENCES": "true", - "BEHAT_FILTER_TAGS": "~@provisioning_api-app-required&&~@preview-extension-required&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@skipOnGraph&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster", + "BEHAT_FILTER_TAGS": "~@provisioning_api-app-required&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@skipOnGraph&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster", "DIVIDE_INTO_NUM_PARTS": parallelRuns, "RUN_PART": runPart, "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-S3NG-storage.md", diff --git a/README.md b/README.md index 38c12f0cc8..bd8bca09c6 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ This will require some PHP-related tools to run, for instance on Ubuntu you will TEST_WITH_LDAP='true' \ REVA_LDAP_HOSTNAME='localhost' \ TEST_REVA='true' \ - BEHAT_FILTER_TAGS='~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis' \ + BEHAT_FILTER_TAGS='~systemtags-app-required&&~@provisioning_api-app-required&&~@skipOnOcis-OCIS-Storage&&~@skipOnOcis' \ EXPECTED_FAILURES_FILE=../reva/tests/acceptance/expected-failures-on-OCIS-storage.md \ make test-acceptance-from-core-api ``` diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index 031479832f..f11839c105 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -771,8 +771,8 @@ _ocs: api compatibility, return correct status code_ #### [PROPFIND on (password protected) public link returns invalid XML](https://github.com/owncloud/ocis/issues/39707) The problem has been fixed in reva edge branch but not in reva master -- [coreApiWebdavOperations/propfind.feature:64](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L64) -- [coreApiWebdavOperations/propfind.feature:76](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L76) +- [coreApiWebdavOperations/propfind.feature:65](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L65) +- [coreApiWebdavOperations/propfind.feature:77](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L77) #### [Updating the role of a public link to internal gives returns 400] - [coreApiSharePublicLink3/updatePublicLinkShare.feature:483](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L483) diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index df34b78db0..459776de5f 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -774,8 +774,8 @@ _ocs: api compatibility, return correct status code_ #### [PROPFIND on (password protected) public link returns invalid XML](https://github.com/owncloud/ocis/issues/39707) The problem has been fixed in reva edge branch but not in reva master -- [coreApiWebdavOperations/propfind.feature:64](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L64) -- [coreApiWebdavOperations/propfind.feature:76](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L76) +- [coreApiWebdavOperations/propfind.feature:65](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L65) +- [coreApiWebdavOperations/propfind.feature:77](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavOperations/propfind.feature#L77) #### [Updating the role of a public link to internal gives returns 400] - [coreApiSharePublicLink3/updatePublicLinkShare.feature:483](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L483) diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml index a4f4b131e0..580cbe065f 100644 --- a/tests/docker/docker-compose.yml +++ b/tests/docker/docker-compose.yml @@ -197,7 +197,7 @@ services: REVA_LDAP_HOSTNAME: ldap TEST_REVA: 'true' SEND_SCENARIO_LINE_REFERENCES: 'true' - BEHAT_FILTER_TAGS: ~@provisioning_api-app-required&&~@preview-extension-required&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@skipOnGraph&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster + BEHAT_FILTER_TAGS: ~@provisioning_api-app-required&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@skipOnGraph&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster DIVIDE_INTO_NUM_PARTS: ${PARTS:-1} RUN_PART: ${PART:-1} EXPECTED_FAILURES_FILE: /mnt/acceptance/expected-failures-on-OCIS-storage.md @@ -266,7 +266,7 @@ services: REVA_LDAP_HOSTNAME: ldap TEST_REVA: 'true' SEND_SCENARIO_LINE_REFERENCES: 'true' - BEHAT_FILTER_TAGS: ~@provisioning_api-app-required&&~@preview-extension-required&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~&&~@skipOnGraph&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster + BEHAT_FILTER_TAGS: ~@provisioning_api-app-required&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~&&~@skipOnGraph&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster DIVIDE_INTO_NUM_PARTS: ${PARTS:-1} RUN_PART: ${PART:-1} EXPECTED_FAILURES_FILE: /mnt/acceptance/expected-failures-on-S3NG-storage.md diff --git a/tests/ocis b/tests/ocis index ca45cb8b46..6e45113e76 160000 --- a/tests/ocis +++ b/tests/ocis @@ -1 +1 @@ -Subproject commit ca45cb8b461752289ba0f8a57baed9289f7e35a7 +Subproject commit 6e45113e76c60965c5e1a3bad42e5b5459960571 From 7c8eb90505b47d8992bb509797b6808d7287334b Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Thu, 20 Jul 2023 12:57:10 +0200 Subject: [PATCH 42/48] Plugins (#4073) --- Makefile | 4 +- changelog/unreleased/plugins.md | 7 + cmd/revad/main.go | 4 +- .../registry.go => cmd/revad/main/main.go | 12 +- docker/Dockerfile.revad | 2 +- docker/Dockerfile.revad-eos | 2 +- docs/content/en/docs/concepts/plugins.md | 126 ----------- examples/plugin/json/json.go | 174 -------------- examples/plugin/plugin.toml | 15 -- examples/plugin/users.demo.json | 103 --------- go.mod | 6 +- go.sum | 8 - .../applicationauth/applicationauth.go | 7 + .../grpc/services/appprovider/appprovider.go | 7 + .../grpc/services/appregistry/appregistry.go | 7 + .../services/authprovider/authprovider.go | 37 +-- .../services/authregistry/authregistry.go | 7 + internal/grpc/services/datatx/datatx.go | 7 + .../services/groupprovider/groupprovider.go | 7 + internal/grpc/services/ocmcore/ocmcore.go | 7 + .../ocminvitemanager/ocminvitemanager.go | 6 + .../ocmproviderauthorizer.go | 7 + .../ocmshareprovider/ocmshareprovider.go | 6 + .../grpc/services/permissions/permissions.go | 7 + .../grpc/services/preferences/preferences.go | 7 + .../publicshareprovider.go | 7 + .../storageprovider/storageprovider.go | 6 + .../storageregistry/storageregistry.go | 7 + .../services/userprovider/userprovider.go | 41 +--- .../usershareprovider/usershareprovider.go | 7 + internal/http/interceptors/loader/loader.go | 1 + internal/http/interceptors/plugins/plugins.go | 33 +++ internal/http/services/loader/loader.go | 1 + internal/http/services/plugins/plugins.go | 33 +++ internal/serverless/services/loader/loader.go | 1 + .../serverless/services/plugins/plugins.go | 33 +++ pkg/auth/auth.go | 2 - pkg/auth/rpc_auth.go | 118 ---------- pkg/plugin/loader.go | 127 ----------- pkg/plugin/plugin.go | 44 +++- pkg/user/rpc_user.go | 214 ------------------ pkg/user/user.go | 2 - pkg/utils/utils.go | 14 ++ plugins.go | 141 ++++++++++++ plugins_test.go | 149 ++++++++++++ 45 files changed, 593 insertions(+), 970 deletions(-) create mode 100644 changelog/unreleased/plugins.md rename pkg/plugin/registry.go => cmd/revad/main/main.go (75%) delete mode 100644 docs/content/en/docs/concepts/plugins.md delete mode 100644 examples/plugin/json/json.go delete mode 100644 examples/plugin/plugin.toml delete mode 100644 examples/plugin/users.demo.json create mode 100644 internal/http/interceptors/plugins/plugins.go create mode 100644 internal/http/services/plugins/plugins.go create mode 100644 internal/serverless/services/plugins/plugins.go delete mode 100644 pkg/auth/rpc_auth.go delete mode 100644 pkg/plugin/loader.go delete mode 100644 pkg/user/rpc_user.go create mode 100644 plugins.go create mode 100644 plugins_test.go diff --git a/Makefile b/Makefile index 234b872e43..c362c97be3 100644 --- a/Makefile +++ b/Makefile @@ -39,11 +39,11 @@ BUILD_FLAGS = "`[[ -z "$(STATIC)" ]] && echo "" || echo "-extldflags=-static"` - .PHONY: revad revad: - go build -ldflags $(BUILD_FLAGS) -o ./cmd/revad/revad ./cmd/revad + go build -ldflags $(BUILD_FLAGS) -o ./cmd/revad/revad ./cmd/revad/main .PHONY: revad-ceph revad-ceph: - go build -ldflags $(BUILD_FLAGS) -tags ceph -o ./cmd/revad/revad ./cmd/revad + go build -ldflags $(BUILD_FLAGS) -tags ceph -o ./cmd/revad/revad ./cmd/revad/main .PHONY: reva reva: diff --git a/changelog/unreleased/plugins.md b/changelog/unreleased/plugins.md new file mode 100644 index 0000000000..41564246ca --- /dev/null +++ b/changelog/unreleased/plugins.md @@ -0,0 +1,7 @@ +Enhancement: Plugins + +Adds a plugin system for allowing the creation of external +plugins for different plugable components in reva, +for example grpc drivers, http services and middlewares. + +https://github.com/cs3org/reva/pull/4073 diff --git a/cmd/revad/main.go b/cmd/revad/main.go index aad02a6baf..8041e19e9b 100644 --- a/cmd/revad/main.go +++ b/cmd/revad/main.go @@ -16,7 +16,7 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package main +package revadcmd import ( "flag" @@ -55,7 +55,7 @@ var ( revaProcs []*runtime.Reva ) -func main() { +func Main() { flag.Parse() // initialize the global system information diff --git a/pkg/plugin/registry.go b/cmd/revad/main/main.go similarity index 75% rename from pkg/plugin/registry.go rename to cmd/revad/main/main.go index fd96c19d1d..aab4599df4 100644 --- a/pkg/plugin/registry.go +++ b/cmd/revad/main/main.go @@ -16,14 +16,10 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -package plugin +package main -import "github.com/hashicorp/go-plugin" +import revadcmd "github.com/cs3org/reva/cmd/revad" -// PluginMap is a map containing all the plugins. -var PluginMap = map[string]plugin.Plugin{} - -// Register registers the plugin. -func Register(name string, plugin plugin.Plugin) { - PluginMap[name] = plugin +func main() { + revadcmd.Main() } diff --git a/docker/Dockerfile.revad b/docker/Dockerfile.revad index 6550ee9506..37a121799f 100644 --- a/docker/Dockerfile.revad +++ b/docker/Dockerfile.revad @@ -29,7 +29,7 @@ ENV CGO_ENABLED 1 RUN apk add --no-cache gcc musl-dev -RUN go build -ldflags "-X main.gitCommit=$GIT_COMMIT -X main.version=$VERSION -X main.goVersion=`go version | awk '{print $3}'` -X main.buildDate=`date +%FT%T%z`" -o ./cmd/revad/revad ./cmd/revad +RUN go build -ldflags "-X main.gitCommit=$GIT_COMMIT -X main.version=$VERSION -X main.goVersion=`go version | awk '{print $3}'` -X main.buildDate=`date +%FT%T%z`" -o ./cmd/revad/revad ./cmd/revad/main FROM alpine:3.16 diff --git a/docker/Dockerfile.revad-eos b/docker/Dockerfile.revad-eos index 92a6a075b6..bbde3b230c 100644 --- a/docker/Dockerfile.revad-eos +++ b/docker/Dockerfile.revad-eos @@ -29,7 +29,7 @@ ENV CGO_ENABLED 1 RUN apk add --no-cache gcc musl-dev -RUN go build -ldflags "-X main.gitCommit=$GIT_COMMIT -X main.version=$VERSION -X main.goVersion=`go version | awk '{print $3}'` -X main.buildDate=`date +%FT%T%z`" -o ./cmd/revad/revad ./cmd/revad +RUN go build -ldflags "-X main.gitCommit=$GIT_COMMIT -X main.version=$VERSION -X main.goVersion=`go version | awk '{print $3}'` -X main.buildDate=`date +%FT%T%z`" -o ./cmd/revad/revad ./cmd/revad/main FROM gitlab-registry.cern.ch/dss/eos/eos-all:4.8.91 diff --git a/docs/content/en/docs/concepts/plugins.md b/docs/content/en/docs/concepts/plugins.md deleted file mode 100644 index f9638a3544..0000000000 --- a/docs/content/en/docs/concepts/plugins.md +++ /dev/null @@ -1,126 +0,0 @@ ---- -title: "Runtime Plugins" -linkTitle: "Runtime Plugins" -weight: 10 -description: > - Guide for developing runtime plugin drivers for Reva ---- - -## Reva Plugins - -Reva Plugins allow new functionality to be added to Reva without modifying the core source code. Reva Plugins are able to add new custom drivers for various services to Reva and also at the same time enable loading existing plugins at runtime. - -This page serves as a guide for understanding and developing Reva plugins. - -### How Plugins Work - -A Reva plugin represents a *service driver*. - -Reva plugins are completely seperate, standalone applications that the core of Reva starts and communicates with. These plugin applications aren't meant to be run manually. Instead, Reva launches and communicates with them. - -Reva uses hashicorp's [go-plugin](https://github.com/hashicorp/go-plugin) framework to implement the plugin architecture. The plugin processes communicate with the Core using the go-plugin framework, which internally uses RPCs for the communication. The Reva core itself is responsible for launching and cleaning up the plugin processes. - -### Developing Plugins - -Reva plugins must be written in [Go](https://golang.org/), so you should be familiar with the language. - -As mentioned earlier, the components that can be created and used in a Reva Plugin are service drivers. These drivers can belong to any of the Reva service, eg: Userprovider, Storageprovider, Authprovider etc. Each service exposes an [interface](https://golang.org/doc/effective_go#interfaces_and_types) for the plugin to implement. - -All you need to do to create a plugin is: - -1. Create an implementation of the desired interface: A Plugin should implement the interface exposed by the corresponding service. Eg: [Userprovider](https://github.com/cs3org/reva/blob/master/pkg/user/user.go#L67) interface. -2. Serve the plugin using the [go-plugin's](https://github.com/hashicorp/go-plugin) `plugin.Serve` method. - -The core handles all of the communication details and go-plugin implementations inside the server. - -Your plugin must use the packages from the Reva core to implement the interfaces. You're encouraged to use whatever other packages you want in your plugin implementation. Because plugins are their own processes, there is no danger of colliding dependencies. - -- `github.com/cs3org/reva/pkg/`: Contains the interface that you have to implement for any give plugin. -- `github.com/hashicorp/go-plugin`: To serve the plugin over RPC. This handles all the inter-process communication. - -Basic example of serving your component is shown below. This example consists of a simple `JSON` plugin driver for the [Userprovider](https://github.com/cs3org/reva/blob/master/internal/grpc/services/userprovider/userprovider.go) service. You can find the example code [here](https://github.com/cs3org/reva/blob/master/examples/plugin/json/json.go). - -```go - -// main.go - -import ( - "github.com/cs3org/reva/pkg/user" - "github.com/hashicorp/go-plugin" - revaPlugin "github.com/cs3org/reva/pkg/plugin" -) - -// Assume this implements the user.Manager interface -type Manager struct{} - -func main() { - // plugin.Serve serves the implementation over RPC to the core - plugin.Serve(&plugin.ServeConfig{ - HandshakeConfig: revaPlugin.Handshake, - Plugins: map[string]plugin.Plugin{ - "userprovider": &user.ProviderPlugin{Impl: &Manager{}}, - }, - }) -} - -``` -The `plugin.Serve` method handles all the details of communicating with Reva core and serving your component over RPC. As long as your struct implements one of the exposed interfaces, Reva will be able to launch your plugin and use it. - -The `plugin.Serve` method takes in the plugin configuration, which you would have to define in your plugin source code: - -- `HandshakeConfig`: The handshake is defined in `github.com/cs3org/reva/pkg/plugin` - -```go -var Handshake = plugin.HandshakeConfig{ - ProtocolVersion: 1, - MagicCookieKey: "BASIC_PLUGIN", - MagicCookieValue: "reva", -} -``` - -- `Plugins`: Plugins is a map which maps the plugin implementation with the plugin name. Currently Reva supports 2 types of plugins (support for more to be added soon): - - `userprovider` - - `authprovider` - -The implementation should be mapped to the above provided plugin names. - - - -### Configuring and Loading Plugins - -Runtime plugin can be configured using `.toml` files. We have ensured backwards compatibilty, hence configuring Reva to use runtime-plugins should feel as natural as configuring in-memory plugin drivers. - -Reva provides 3 ways of loading plugins: - -1. Providing path to already compiled go binary: If you have already compiled plugin binary, you just need to provide the path to the binary and rest of the work of loading the plugin would be taken care by the reva core. - -``` -# Starting grpc userprovider service with json driver plugin -[grpc.service.userprovider] -driver = "/absolute/path/to/binary/json" - -[grpc.service.userprovider.drivers.json] -user = "user.demo.json" -``` - -2. Provide path to the plugin source code: If you want the reva core to compile the plugins and then load the binary, you need to point to the go package consisting the plugin source code: - -``` -# Starting grpc userprovider service with json driver plugin -[grpc.service.userprovider] -driver = "/absolute/path/to/source/json" - -[grpc.service.userprovider.drivers.json] -user = "user.demo.json" -``` - -3. Provide URL to plugin hosted on Github: If you provide Github URL, Reva would download the source code into a temporary directory, compile it into a binary and load that binary. - -``` -# Starting grpc userprovider service with json driver plugin -[grpc.service.userprovider] -driver = "https://github.com/jimil749/json" - -[grpc.service.userprovider.drivers.json] -user = "user.demo.json" -``` \ No newline at end of file diff --git a/examples/plugin/json/json.go b/examples/plugin/json/json.go deleted file mode 100644 index 7ebcc6f930..0000000000 --- a/examples/plugin/json/json.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package main - -import ( - "context" - "encoding/json" - "errors" - "os" - "strings" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/user" - "github.com/hashicorp/go-plugin" - "github.com/mitchellh/mapstructure" -) - -// Manager is a real implementation of Manager interface. -type Manager struct { - users []*userpb.User -} - -type config struct { - Users string `mapstructure:"users"` -} - -func (c *config) init() { - if c.Users == "" { - c.Users = "/etc/revad/users.json" - } -} - -func parseConfig(m map[string]interface{}) (*config, error) { - c := &config{} - if err := mapstructure.Decode(m, c); err != nil { - return nil, err - } - c.init() - return c, nil -} - -// Configure initializes the manager struct based on the configurations. -func (m *Manager) Configure(ml map[string]interface{}) error { - c, err := parseConfig(ml) - if err != nil { - return err - } - - f, err := os.ReadFile(c.Users) - if err != nil { - return err - } - - users := []*userpb.User{} - - err = json.Unmarshal(f, &users) - if err != nil { - return err - } - - m.users = users - - return nil -} - -// GetUser returns the user based on the uid. -func (m *Manager) GetUser(ctx context.Context, uid *userpb.UserId, skipFetchingGroups bool) (*userpb.User, error) { - for _, u := range m.users { - if (u.Id.GetOpaqueId() == uid.OpaqueId || u.Username == uid.OpaqueId) && (uid.Idp == "" || uid.Idp == u.Id.GetIdp()) { - user := *u - if skipFetchingGroups { - user.Groups = nil - } - return &user, nil - } - } - return nil, nil -} - -// GetUserByClaim returns user based on the claim -func (m *Manager) GetUserByClaim(ctx context.Context, claim, value string, skipFetchingGroups bool) (*userpb.User, error) { - for _, u := range m.users { - if userClaim, err := extractClaim(u, claim); err == nil && value == userClaim { - user := *u - if skipFetchingGroups { - user.Groups = nil - } - return &user, nil - } - } - return nil, errtypes.NotFound(value) -} - -func extractClaim(u *userpb.User, claim string) (string, error) { - switch claim { - case "mail": - return u.Mail, nil - case "username": - return u.Username, nil - case "uid": - if u.Opaque != nil && u.Opaque.Map != nil { - if uidObj, ok := u.Opaque.Map["uid"]; ok { - if uidObj.Decoder == "plain" { - return string(uidObj.Value), nil - } - } - } - } - return "", errors.New("json: invalid field") -} - -// TODO(jfd) search Opaque? compare sub? -func userContains(u *userpb.User, query string) bool { - query = strings.ToLower(query) - return strings.Contains(strings.ToLower(u.Username), query) || strings.Contains(strings.ToLower(u.DisplayName), query) || - strings.Contains(strings.ToLower(u.Mail), query) || strings.Contains(strings.ToLower(u.Id.OpaqueId), query) -} - -// FindUsers returns the user based on the query -func (m *Manager) FindUsers(ctx context.Context, query string, skipFetchingGroups bool) ([]*userpb.User, error) { - users := []*userpb.User{} - for _, u := range m.users { - if userContains(u, query) { - user := *u - if skipFetchingGroups { - user.Groups = nil - } - users = append(users, &user) - } - } - return users, nil -} - -// GetUserGroups returns the user groups -func (m *Manager) GetUserGroups(ctx context.Context, uid *userpb.UserId) ([]string, error) { - user, err := m.GetUser(ctx, uid, false) - if err != nil { - return nil, err - } - return user.Groups, nil -} - -// Handshake hashicorp go-plugin handshake -var Handshake = plugin.HandshakeConfig{ - ProtocolVersion: 1, - MagicCookieKey: "BASIC_PLUGIN", - MagicCookieValue: "hello", -} - -func main() { - plugin.Serve(&plugin.ServeConfig{ - HandshakeConfig: Handshake, - Plugins: map[string]plugin.Plugin{ - "userprovider": &user.ProviderPlugin{Impl: &Manager{}}, - }, - }) -} diff --git a/examples/plugin/plugin.toml b/examples/plugin/plugin.toml deleted file mode 100644 index 28bc78198c..0000000000 --- a/examples/plugin/plugin.toml +++ /dev/null @@ -1,15 +0,0 @@ -[shared] -gatewaysvc = "localhost:19000" -plugin = true - -[grpc] -address = "0.0.0.0:19000" - -[grpc.services.gateway] -userprovidersvc = "localhost:19000" - -[grpc.services.userprovider] -driver = "./json" - -[grpc.services.userprovider.drivers.json] -users = "users.demo.json" diff --git a/examples/plugin/users.demo.json b/examples/plugin/users.demo.json deleted file mode 100644 index 99a04a2554..0000000000 --- a/examples/plugin/users.demo.json +++ /dev/null @@ -1,103 +0,0 @@ -[ - { - "id": { - "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", - "idp": "https://cernbox.cern.ch", - "type": 1 - }, - "username": "einstein", - "secret": "relativity", - "mail": "einstein@cern.ch", - "display_name": "Albert Einstein", - "groups": ["sailing-lovers", "violin-haters", "physics-lovers"], - "opaque": { - "map": { - "gid": { - "_comment": "decodes to 987", - "decoder":"plain", - "value":"OTg3" - }, - "uid":{ - "_comment": "decodes to 123", - "decoder":"plain", - "value":"MTIz" - } - } - } - }, - { - "id": { - "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", - "idp": "cesnet.cz" - }, - "username": "marie", - "secret": "radioactivity", - "mail": "marie@cesnet.cz", - "display_name": "Marie Curie", - "groups": ["radium-lovers", "polonium-lovers", "physics-lovers"], - "opaque": { - "map": { - "gid": { - "_comment": "decodes to 987", - "decoder":"plain", - "value":"OTg3" - }, - "uid":{ - "_comment": "decodes to 456", - "decoder":"plain", - "value":"NDU2" - } - } - } - }, - { - "id": { - "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", - "idp": "example.org" - }, - "username": "richard", - "secret": "superfluidity", - "mail": "richard@example.org", - "display_name": "Richard Feynman", - "groups": ["quantum-lovers", "philosophy-haters", "physics-lovers"], - "opaque": { - "map": { - "gid": { - "_comment": "decodes to 135", - "decoder":"plain", - "value":"MTM1" - }, - "uid":{ - "_comment": "decodes to 246", - "decoder":"plain", - "value":"MjQ2" - } - } - } - }, - { - "id": { - "opaque_id": "932b4522-139b-4815-8ef4-42cdf82c3d51", - "idp": "example.com" - }, - "username": "test", - "secret": "test", - "mail": "test@example.com", - "display_name": "Test Testman", - "groups": ["quantum-lovers", "philosophy-haters", "physics-lovers"], - "opaque": { - "map": { - "gid": { - "_comment": "decodes to 135", - "decoder":"plain", - "value":"MTM1" - }, - "uid":{ - "_comment": "decodes to 468", - "decoder":"plain", - "value":"NDY4" - } - } - } - } -] diff --git a/go.mod b/go.mod index 6e25780e1f..f0fe32ae89 100644 --- a/go.mod +++ b/go.mod @@ -35,8 +35,6 @@ require ( github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 - github.com/hashicorp/go-hclog v1.5.0 - github.com/hashicorp/go-plugin v1.4.9 github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/juliangruber/go-intersect v1.1.0 github.com/mattn/go-sqlite3 v1.14.10 @@ -110,12 +108,12 @@ require ( github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/flatbuffers v2.0.6+incompatible // indirect + github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/hashicorp/go-immutable-radix v1.0.0 // indirect github.com/hashicorp/go-msgpack v1.1.5 // indirect github.com/hashicorp/go-msgpack/v2 v2.1.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/raft v1.4.0 // indirect - github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -134,7 +132,6 @@ require ( github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/hashstructure v1.1.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -144,7 +141,6 @@ require ( github.com/nats-io/nuid v1.0.1 // indirect github.com/nats-io/stan.go v0.10.4 // indirect github.com/nxadm/tail v1.4.8 // indirect - github.com/oklog/run v1.1.0 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 // indirect github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect diff --git a/go.sum b/go.sum index e3848a5c97..0f60a32e0a 100644 --- a/go.sum +++ b/go.sum @@ -652,8 +652,6 @@ github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YN github.com/hashicorp/go-msgpack/v2 v2.1.0 h1:J2g2hMyjSefUPTnkLRU2MnsLLsPRB1n4Z/wJRN07GuA= github.com/hashicorp/go-msgpack/v2 v2.1.0/go.mod h1:Tv81cKI2JmHZDjmzEmc1n+8h1DO5k+3pG6BPlNMQds0= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-plugin v1.4.9 h1:ESiK220/qE0aGxWdzKIvRH69iLiuN/PjoLTm69RoWtU= -github.com/hashicorp/go-plugin v1.4.9/go.mod h1:viDMjcLJuDui6pXb8U4HVfb8AamCWhHGUjr2IrTF67s= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= @@ -678,8 +676,6 @@ github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOn github.com/hashicorp/raft v1.4.0 h1:tn28S/AWv0BtRQgwZv/1NELu8sCvI0FixqL8C8MYKeY= github.com/hashicorp/raft v1.4.0/go.mod h1:nz64BIjXphDLATfKGG5RzHtNUPioLeKFsXEm88yTVew= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= -github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= @@ -698,7 +694,6 @@ github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PH github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -844,8 +839,6 @@ github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HK github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= -github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-vnc v0.0.0-20150629162542-723ed9867aed/go.mod h1:3rdaFaCv4AyBgu5ALFM0+tSuHrBh6v692nyQe3ikrq0= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= @@ -912,7 +905,6 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= diff --git a/internal/grpc/services/applicationauth/applicationauth.go b/internal/grpc/services/applicationauth/applicationauth.go index 387a75904e..2ccea16f5f 100644 --- a/internal/grpc/services/applicationauth/applicationauth.go +++ b/internal/grpc/services/applicationauth/applicationauth.go @@ -25,14 +25,21 @@ import ( "github.com/cs3org/reva/pkg/appauth" "github.com/cs3org/reva/pkg/appauth/manager/registry" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" + "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) func init() { rgrpc.Register("applicationauth", New) + plugin.RegisterNamespace("grpc.services.applicationauth.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type config struct { diff --git a/internal/grpc/services/appprovider/appprovider.go b/internal/grpc/services/appprovider/appprovider.go index 53f5b89a27..7028c4f4d2 100644 --- a/internal/grpc/services/appprovider/appprovider.go +++ b/internal/grpc/services/appprovider/appprovider.go @@ -36,10 +36,12 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/mime" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/sharedconf" + "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "github.com/juliangruber/go-intersect" "google.golang.org/grpc" @@ -47,6 +49,11 @@ import ( func init() { rgrpc.Register("appprovider", New) + plugin.RegisterNamespace("grpc.services.appprovider.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type service struct { diff --git a/internal/grpc/services/appregistry/appregistry.go b/internal/grpc/services/appregistry/appregistry.go index 0acea9c12f..68d9f00f1a 100644 --- a/internal/grpc/services/appregistry/appregistry.go +++ b/internal/grpc/services/appregistry/appregistry.go @@ -25,14 +25,21 @@ import ( "github.com/cs3org/reva/pkg/app" "github.com/cs3org/reva/pkg/app/registry/registry" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" + "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) func init() { rgrpc.Register("appregistry", New) + plugin.RegisterNamespace("grpc.services.appregistry.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type svc struct { diff --git a/internal/grpc/services/authprovider/authprovider.go b/internal/grpc/services/authprovider/authprovider.go index 71427f34cc..fdb6937014 100644 --- a/internal/grpc/services/authprovider/authprovider.go +++ b/internal/grpc/services/authprovider/authprovider.go @@ -21,14 +21,12 @@ package authprovider import ( "context" "fmt" - "path/filepath" provider "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/auth" "github.com/cs3org/reva/pkg/auth/manager/registry" "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/sharedconf" @@ -58,35 +56,18 @@ func (c *config) ApplyDefaults() { type service struct { authmgr auth.Manager conf *config - plugin *plugin.RevaPlugin blockedUsers user.BlockedUsers } -func getAuthManager(ctx context.Context, manager string, m map[string]map[string]interface{}) (auth.Manager, *plugin.RevaPlugin, error) { +func getAuthManager(ctx context.Context, manager string, m map[string]map[string]interface{}) (auth.Manager, error) { if manager == "" { - return nil, nil, errtypes.InternalError("authsvc: driver not configured for auth manager") + return nil, errtypes.InternalError("authsvc: driver not configured for auth manager") } - p, err := plugin.Load("authprovider", manager) - if err == nil { - authManager, ok := p.Plugin.(auth.Manager) - if !ok { - return nil, nil, fmt.Errorf("could not assert the loaded plugin") - } - pluginConfig := filepath.Base(manager) - err = authManager.Configure(m[pluginConfig]) - if err != nil { - return nil, nil, err - } - return authManager, p, nil - } else if _, ok := err.(errtypes.NotFound); ok { - if f, ok := registry.NewFuncs[manager]; ok { - authmgr, err := f(ctx, m[manager]) - return authmgr, nil, err - } - } else { - return nil, nil, err + if f, ok := registry.NewFuncs[manager]; ok { + authmgr, err := f(ctx, m[manager]) + return authmgr, err } - return nil, nil, errtypes.NotFound(fmt.Sprintf("authsvc: driver %s not found for auth manager", manager)) + return nil, errtypes.NotFound(fmt.Sprintf("authsvc: driver %s not found for auth manager", manager)) } // New returns a new AuthProviderServiceServer. @@ -96,7 +77,7 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { return nil, err } - authManager, plug, err := getAuthManager(ctx, c.AuthManager, c.AuthManagers) + authManager, err := getAuthManager(ctx, c.AuthManager, c.AuthManagers) if err != nil { return nil, err } @@ -104,7 +85,6 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { svc := &service{ conf: &c, authmgr: authManager, - plugin: plug, blockedUsers: user.NewBlockedUsersSet(c.blockedUsers), } @@ -112,9 +92,6 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { } func (s *service) Close() error { - if s.plugin != nil { - s.plugin.Kill() - } return nil } diff --git a/internal/grpc/services/authregistry/authregistry.go b/internal/grpc/services/authregistry/authregistry.go index 1e6969667f..1caaaf4d54 100644 --- a/internal/grpc/services/authregistry/authregistry.go +++ b/internal/grpc/services/authregistry/authregistry.go @@ -25,14 +25,21 @@ import ( "github.com/cs3org/reva/pkg/auth" "github.com/cs3org/reva/pkg/auth/registry/registry" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" + "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) func init() { rgrpc.Register("authregistry", New) + plugin.RegisterNamespace("grpc.services.authregistry.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type service struct { diff --git a/internal/grpc/services/datatx/datatx.go b/internal/grpc/services/datatx/datatx.go index ef953d744d..b927c89f2e 100644 --- a/internal/grpc/services/datatx/datatx.go +++ b/internal/grpc/services/datatx/datatx.go @@ -28,8 +28,10 @@ import ( txregistry "github.com/cs3org/reva/pkg/datatx/manager/registry" repoRegistry "github.com/cs3org/reva/pkg/datatx/repository/registry" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" + "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "google.golang.org/grpc" @@ -37,6 +39,11 @@ import ( func init() { rgrpc.Register("datatx", New) + plugin.RegisterNamespace("grpc.services.datatx.drivers", func(name string, newFunc any) { + var f txregistry.NewFunc + utils.Cast(newFunc, &f) + txregistry.Register(name, f) + }) } type config struct { diff --git a/internal/grpc/services/groupprovider/groupprovider.go b/internal/grpc/services/groupprovider/groupprovider.go index 7ab63af7bc..6a268a8f16 100644 --- a/internal/grpc/services/groupprovider/groupprovider.go +++ b/internal/grpc/services/groupprovider/groupprovider.go @@ -27,8 +27,10 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/group" "github.com/cs3org/reva/pkg/group/manager/registry" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" + "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "google.golang.org/grpc" @@ -36,6 +38,11 @@ import ( func init() { rgrpc.Register("groupprovider", New) + plugin.RegisterNamespace("grpc.services.groupprovider.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type config struct { diff --git a/internal/grpc/services/ocmcore/ocmcore.go b/internal/grpc/services/ocmcore/ocmcore.go index 54bf50c616..77a607dd57 100644 --- a/internal/grpc/services/ocmcore/ocmcore.go +++ b/internal/grpc/services/ocmcore/ocmcore.go @@ -30,14 +30,21 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/ocm/share" "github.com/cs3org/reva/pkg/ocm/share/repository/registry" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" + "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) func init() { rgrpc.Register("ocmcore", New) + plugin.RegisterNamespace("grpc.services.ocmcore.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type config struct { diff --git a/internal/grpc/services/ocminvitemanager/ocminvitemanager.go b/internal/grpc/services/ocminvitemanager/ocminvitemanager.go index 7906a38bc3..2acbecc185 100644 --- a/internal/grpc/services/ocminvitemanager/ocminvitemanager.go +++ b/internal/grpc/services/ocminvitemanager/ocminvitemanager.go @@ -31,6 +31,7 @@ import ( "github.com/cs3org/reva/pkg/ocm/client" "github.com/cs3org/reva/pkg/ocm/invite" "github.com/cs3org/reva/pkg/ocm/invite/repository/registry" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" @@ -43,6 +44,11 @@ import ( func init() { rgrpc.Register("ocminvitemanager", New) + plugin.RegisterNamespace("grpc.services.ocminvitemanager.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type config struct { diff --git a/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go b/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go index 18e436bee6..71d5d4cd60 100644 --- a/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go +++ b/internal/grpc/services/ocmproviderauthorizer/ocmproviderauthorizer.go @@ -26,14 +26,21 @@ import ( "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/ocm/provider" "github.com/cs3org/reva/pkg/ocm/provider/authorizer/registry" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" + "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) func init() { rgrpc.Register("ocmproviderauthorizer", New) + plugin.RegisterNamespace("grpc.services.ocmproviderauthorizer.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type config struct { diff --git a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go index acdc56f9c5..7f45e1dfdf 100644 --- a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go +++ b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go @@ -40,6 +40,7 @@ import ( "github.com/cs3org/reva/pkg/ocm/client" "github.com/cs3org/reva/pkg/ocm/share" "github.com/cs3org/reva/pkg/ocm/share/repository/registry" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" @@ -53,6 +54,11 @@ import ( func init() { rgrpc.Register("ocmshareprovider", New) + plugin.RegisterNamespace("grpc.services.ocmshareprovider.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type config struct { diff --git a/internal/grpc/services/permissions/permissions.go b/internal/grpc/services/permissions/permissions.go index eec9c7e22f..8969192cc4 100644 --- a/internal/grpc/services/permissions/permissions.go +++ b/internal/grpc/services/permissions/permissions.go @@ -26,13 +26,20 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" "github.com/cs3org/reva/pkg/permission" "github.com/cs3org/reva/pkg/permission/manager/registry" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" + "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) func init() { rgrpc.Register("permissions", New) + plugin.RegisterNamespace("grpc.services.permissions.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type config struct { diff --git a/internal/grpc/services/preferences/preferences.go b/internal/grpc/services/preferences/preferences.go index b45c6afbca..fc38b8f098 100644 --- a/internal/grpc/services/preferences/preferences.go +++ b/internal/grpc/services/preferences/preferences.go @@ -23,16 +23,23 @@ import ( preferencespb "github.com/cs3org/go-cs3apis/cs3/preferences/v1beta1" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/preferences" "github.com/cs3org/reva/pkg/preferences/registry" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" + "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) func init() { rgrpc.Register("preferences", New) + plugin.RegisterNamespace("grpc.services.preferences.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type config struct { diff --git a/internal/grpc/services/publicshareprovider/publicshareprovider.go b/internal/grpc/services/publicshareprovider/publicshareprovider.go index 91ce7a0547..09313d2f14 100644 --- a/internal/grpc/services/publicshareprovider/publicshareprovider.go +++ b/internal/grpc/services/publicshareprovider/publicshareprovider.go @@ -27,16 +27,23 @@ import ( "github.com/cs3org/reva/pkg/appctx" ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/publicshare" "github.com/cs3org/reva/pkg/publicshare/manager/registry" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" + "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) func init() { rgrpc.Register("publicshareprovider", New) + plugin.RegisterNamespace("grpc.services.publicshareprovider.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type config struct { diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index 0ef056cc53..8a70d81629 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -35,6 +35,7 @@ import ( "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/mime" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/rhttp/router" @@ -51,6 +52,11 @@ import ( func init() { rgrpc.Register("storageprovider", New) + plugin.RegisterNamespace("grpc.services.storageprovider.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type config struct { diff --git a/internal/grpc/services/storageregistry/storageregistry.go b/internal/grpc/services/storageregistry/storageregistry.go index e3a4ac3454..53afa2487a 100644 --- a/internal/grpc/services/storageregistry/storageregistry.go +++ b/internal/grpc/services/storageregistry/storageregistry.go @@ -24,16 +24,23 @@ import ( registrypb "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/registry/registry" + "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) func init() { rgrpc.Register("storageregistry", New) + plugin.RegisterNamespace("grpc.services.storageregistry.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type service struct { diff --git a/internal/grpc/services/userprovider/userprovider.go b/internal/grpc/services/userprovider/userprovider.go index d34ef48318..b3df5d98f6 100644 --- a/internal/grpc/services/userprovider/userprovider.go +++ b/internal/grpc/services/userprovider/userprovider.go @@ -21,7 +21,6 @@ package userprovider import ( "context" "fmt" - "path/filepath" "sort" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" @@ -31,6 +30,7 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/user" "github.com/cs3org/reva/pkg/user/manager/registry" + "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "github.com/pkg/errors" "google.golang.org/grpc" @@ -38,6 +38,11 @@ import ( func init() { rgrpc.Register("userprovider", New) + plugin.RegisterNamespace("grpc.services.userprovider.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type config struct { @@ -51,29 +56,12 @@ func (c *config) ApplyDefaults() { } } -func getDriver(ctx context.Context, c *config) (user.Manager, *plugin.RevaPlugin, error) { - p, err := plugin.Load("userprovider", c.Driver) - if err == nil { - manager, ok := p.Plugin.(user.Manager) - if !ok { - return nil, nil, fmt.Errorf("could not assert the loaded plugin") - } - pluginConfig := filepath.Base(c.Driver) - err = manager.Configure(c.Drivers[pluginConfig]) - if err != nil { - return nil, nil, err - } - return manager, p, nil - } else if _, ok := err.(errtypes.NotFound); ok { - // plugin not found, fetch the driver from the in-memory registry - if f, ok := registry.NewFuncs[c.Driver]; ok { - mgr, err := f(ctx, c.Drivers[c.Driver]) - return mgr, nil, err - } - } else { - return nil, nil, err +func getDriver(ctx context.Context, c *config) (user.Manager, error) { + if f, ok := registry.NewFuncs[c.Driver]; ok { + mgr, err := f(ctx, c.Drivers[c.Driver]) + return mgr, err } - return nil, nil, errtypes.NotFound(fmt.Sprintf("driver %s not found for user manager", c.Driver)) + return nil, errtypes.NotFound(fmt.Sprintf("driver %s not found for user manager", c.Driver)) } // New returns a new UserProviderServiceServer. @@ -82,13 +70,12 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { if err := cfg.Decode(m, &c); err != nil { return nil, err } - userManager, plug, err := getDriver(ctx, &c) + userManager, err := getDriver(ctx, &c) if err != nil { return nil, err } svc := &service{ usermgr: userManager, - plugin: plug, } return svc, nil @@ -96,13 +83,9 @@ func New(ctx context.Context, m map[string]interface{}) (rgrpc.Service, error) { type service struct { usermgr user.Manager - plugin *plugin.RevaPlugin } func (s *service) Close() error { - if s.plugin != nil { - s.plugin.Kill() - } return nil } diff --git a/internal/grpc/services/usershareprovider/usershareprovider.go b/internal/grpc/services/usershareprovider/usershareprovider.go index 8d37af562b..3de8bfa3c8 100644 --- a/internal/grpc/services/usershareprovider/usershareprovider.go +++ b/internal/grpc/services/usershareprovider/usershareprovider.go @@ -28,16 +28,23 @@ import ( "github.com/cs3org/reva/pkg/appctx" ctxpkg "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/plugin" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rgrpc/status" "github.com/cs3org/reva/pkg/share" "github.com/cs3org/reva/pkg/share/manager/registry" + "github.com/cs3org/reva/pkg/utils" "github.com/cs3org/reva/pkg/utils/cfg" "google.golang.org/grpc" ) func init() { rgrpc.Register("usershareprovider", New) + plugin.RegisterNamespace("grpc.services.usershareprovider.drivers", func(name string, newFunc any) { + var f registry.NewFunc + utils.Cast(newFunc, &f) + registry.Register(name, f) + }) } type config struct { diff --git a/internal/http/interceptors/loader/loader.go b/internal/http/interceptors/loader/loader.go index f063179431..722ac23d94 100644 --- a/internal/http/interceptors/loader/loader.go +++ b/internal/http/interceptors/loader/loader.go @@ -21,5 +21,6 @@ package loader import ( // Load core HTTP middlewares. _ "github.com/cs3org/reva/internal/http/interceptors/cors" + _ "github.com/cs3org/reva/internal/http/interceptors/plugins" // Add your own middleware. ) diff --git a/internal/http/interceptors/plugins/plugins.go b/internal/http/interceptors/plugins/plugins.go new file mode 100644 index 0000000000..16284eb790 --- /dev/null +++ b/internal/http/interceptors/plugins/plugins.go @@ -0,0 +1,33 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package plugins + +import ( + "github.com/cs3org/reva/pkg/plugin" + "github.com/cs3org/reva/pkg/rhttp/global" + "github.com/cs3org/reva/pkg/utils" +) + +func init() { + plugin.RegisterNamespace("http.middlewares", func(name string, newFunc any) { + var f global.NewMiddleware + utils.Cast(newFunc, &f) + global.RegisterMiddleware(name, f) + }) +} diff --git a/internal/http/services/loader/loader.go b/internal/http/services/loader/loader.go index 50ad252d9a..3fc02ce9bb 100644 --- a/internal/http/services/loader/loader.go +++ b/internal/http/services/loader/loader.go @@ -31,6 +31,7 @@ import ( _ "github.com/cs3org/reva/internal/http/services/ocmprovider" _ "github.com/cs3org/reva/internal/http/services/owncloud/ocdav" _ "github.com/cs3org/reva/internal/http/services/owncloud/ocs" + _ "github.com/cs3org/reva/internal/http/services/plugins" _ "github.com/cs3org/reva/internal/http/services/preferences" _ "github.com/cs3org/reva/internal/http/services/prometheus" _ "github.com/cs3org/reva/internal/http/services/reverseproxy" diff --git a/internal/http/services/plugins/plugins.go b/internal/http/services/plugins/plugins.go new file mode 100644 index 0000000000..827216e328 --- /dev/null +++ b/internal/http/services/plugins/plugins.go @@ -0,0 +1,33 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package plugins + +import ( + "github.com/cs3org/reva/pkg/plugin" + "github.com/cs3org/reva/pkg/rhttp/global" + "github.com/cs3org/reva/pkg/utils" +) + +func init() { + plugin.RegisterNamespace("http.services", func(name string, newFunc any) { + var f global.NewService + utils.Cast(newFunc, &f) + global.Register(name, f) + }) +} diff --git a/internal/serverless/services/loader/loader.go b/internal/serverless/services/loader/loader.go index 71ab9afb1d..f72462f600 100644 --- a/internal/serverless/services/loader/loader.go +++ b/internal/serverless/services/loader/loader.go @@ -22,5 +22,6 @@ import ( // Load core serverless services. _ "github.com/cs3org/reva/internal/serverless/services/helloworld" _ "github.com/cs3org/reva/internal/serverless/services/notifications" + _ "github.com/cs3org/reva/internal/serverless/services/plugins" // Add your own service here. ) diff --git a/internal/serverless/services/plugins/plugins.go b/internal/serverless/services/plugins/plugins.go new file mode 100644 index 0000000000..125ea28658 --- /dev/null +++ b/internal/serverless/services/plugins/plugins.go @@ -0,0 +1,33 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package plugins + +import ( + "github.com/cs3org/reva/pkg/plugin" + "github.com/cs3org/reva/pkg/rserverless" + "github.com/cs3org/reva/pkg/utils" +) + +func init() { + plugin.RegisterNamespace("serverless.services", func(name string, newFunc any) { + var f rserverless.NewService + utils.Cast(newFunc, &f) + rserverless.Register(name, f) + }) +} diff --git a/pkg/auth/auth.go b/pkg/auth/auth.go index fd7a26858e..f79e959bd3 100644 --- a/pkg/auth/auth.go +++ b/pkg/auth/auth.go @@ -25,12 +25,10 @@ import ( authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1" registry "github.com/cs3org/go-cs3apis/cs3/auth/registry/v1beta1" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - "github.com/cs3org/reva/pkg/plugin" ) // Manager is the interface to implement to authenticate users. type Manager interface { - plugin.Plugin Authenticate(ctx context.Context, clientID, clientSecret string) (*user.User, map[string]*authpb.Scope, error) } diff --git a/pkg/auth/rpc_auth.go b/pkg/auth/rpc_auth.go deleted file mode 100644 index 5ffbaf3b48..0000000000 --- a/pkg/auth/rpc_auth.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package auth - -import ( - "context" - "net/rpc" - - authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1" - user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/plugin" - hcplugin "github.com/hashicorp/go-plugin" -) - -func init() { - plugin.Register("authprovider", &ProviderPlugin{}) -} - -// ProviderPlugin is the implementation of plugin.Plugin so we can serve/consume this. -type ProviderPlugin struct { - Impl Manager -} - -// Server returns the RPC Server which serves the methods that the Client calls over net/rpc. -func (p *ProviderPlugin) Server(*hcplugin.MuxBroker) (interface{}, error) { - return &RPCServer{Impl: p.Impl}, nil -} - -// Client returns interface implementation for the plugin that communicates to the server end of the plugin. -func (p *ProviderPlugin) Client(b *hcplugin.MuxBroker, c *rpc.Client) (interface{}, error) { - return &RPCClient{Client: c}, nil -} - -// RPCClient is an implementation of Manager that talks over RPC. -type RPCClient struct{ Client *rpc.Client } - -// ConfigureArg for RPC. -type ConfigureArg struct { - Ml map[string]interface{} -} - -// ConfigureReply for RPC. -type ConfigureReply struct { - Err error -} - -// Configure RPCClient configure method. -func (m *RPCClient) Configure(ml map[string]interface{}) error { - args := ConfigureArg{Ml: ml} - resp := ConfigureReply{} - err := m.Client.Call("Plugin.Configure", args, &resp) - if err != nil { - return err - } - return resp.Err -} - -// AuthenticateArgs for RPC. -type AuthenticateArgs struct { - Ctx map[interface{}]interface{} - ClientID string - ClientSecret string -} - -// AuthenticateReply for RPC. -type AuthenticateReply struct { - User *user.User - Auth map[string]*authpb.Scope - Error error -} - -// Authenticate RPCClient Authenticate method. -func (m *RPCClient) Authenticate(ctx context.Context, clientID, clientSecret string) (*user.User, map[string]*authpb.Scope, error) { - ctxVal := appctx.GetKeyValuesFromCtx(ctx) - args := AuthenticateArgs{Ctx: ctxVal, ClientID: clientID, ClientSecret: clientSecret} - reply := AuthenticateReply{} - err := m.Client.Call("Plugin.Authenticate", args, &reply) - if err != nil { - return nil, nil, err - } - return reply.User, reply.Auth, reply.Error -} - -// RPCServer is the server that RPCClient talks to, conforming to the requirements of net/rpc. -type RPCServer struct { - // This is the real implementation - Impl Manager -} - -// Configure RPCServer Configure method. -func (m *RPCServer) Configure(args ConfigureArg, resp *ConfigureReply) error { - resp.Err = m.Impl.Configure(args.Ml) - return nil -} - -// Authenticate RPCServer Authenticate method. -func (m *RPCServer) Authenticate(args AuthenticateArgs, resp *AuthenticateReply) error { - ctx := appctx.PutKeyValuesToCtx(args.Ctx) - resp.User, resp.Auth, resp.Error = m.Impl.Authenticate(ctx, args.ClientID, args.ClientSecret) - return nil -} diff --git a/pkg/plugin/loader.go b/pkg/plugin/loader.go deleted file mode 100644 index e4bf5865d0..0000000000 --- a/pkg/plugin/loader.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package plugin - -import ( - "bytes" - "fmt" - "os" - "os/exec" - "path/filepath" - "regexp" - - "github.com/cs3org/reva/pkg/errtypes" - "github.com/hashicorp/go-hclog" - "github.com/hashicorp/go-plugin" -) - -// RevaPlugin represents the runtime plugin. -type RevaPlugin struct { - Plugin interface{} - Client *plugin.Client -} - -const dirname = "/var/tmp/reva" - -var isAlphaNum = regexp.MustCompile(`^[A-Za-z0-9]+$`).MatchString - -// Kill kills the plugin process. -func (plug *RevaPlugin) Kill() { - plug.Client.Kill() -} - -var handshake = plugin.HandshakeConfig{ - ProtocolVersion: 1, - MagicCookieKey: "BASIC_PLUGIN", - MagicCookieValue: "hello", -} - -func compile(pluginType string, path string) (string, error) { - var errb bytes.Buffer - binaryPath := filepath.Join(dirname, "bin", pluginType, filepath.Base(path)) - command := fmt.Sprintf("go build -o %s %s", binaryPath, path) - cmd := exec.Command("bash", "-c", command) - cmd.Stderr = &errb - err := cmd.Run() - if err != nil { - return "", fmt.Errorf("%v: %w", errb.String(), err) - } - return binaryPath, nil -} - -// checkDir checks and compiles plugin if the configuration points to a directory. -func checkDirAndCompile(pluginType, driver string) (string, error) { - bin := driver - file, err := os.Stat(driver) - if err != nil { - return "", err - } - // compile if we point to a package - if file.IsDir() { - bin, err = compile(pluginType, driver) - if err != nil { - return "", err - } - } - return bin, nil -} - -// Load loads the plugin using the hashicorp go-plugin system. -func Load(pluginType, driver string) (*RevaPlugin, error) { - if isAlphaNum(driver) { - return nil, errtypes.NotFound(driver) - } - bin, err := checkDirAndCompile(pluginType, driver) - if err != nil { - return nil, err - } - - logger := hclog.New(&hclog.LoggerOptions{ - Name: "plugin", - Output: os.Stdout, - Level: hclog.Trace, - }) - - client := plugin.NewClient(&plugin.ClientConfig{ - HandshakeConfig: handshake, - Plugins: PluginMap, - Cmd: exec.Command(bin), - AllowedProtocols: []plugin.Protocol{ - plugin.ProtocolNetRPC, - }, - Logger: logger, - }) - - rpcClient, err := client.Client() - if err != nil { - return nil, err - } - - raw, err := rpcClient.Dispense(pluginType) - if err != nil { - return nil, err - } - - revaPlugin := &RevaPlugin{ - Plugin: raw, - Client: client, - } - - return revaPlugin, nil -} diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index 4ee00f24c3..d4bfa9251c 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -18,7 +18,45 @@ package plugin -// Plugin is the interface used to configure plugins. -type Plugin interface { - Configure(m map[string]interface{}) error +import "reflect" + +// RegistryFunc is the func a component that is pluggable +// must define to register the new func in its own registry. +// It is responsibility of the component to type assert the +// new func with the expected one and panic if not. +type RegistryFunc func(name string, newFunc any) + +var registry = map[string]RegistryFunc{} // key is the namespace + +// RegisterNamespace is the function called by a component +// that is pluggable, to register its namespace and a function +// to register the plugins. +func RegisterNamespace(ns string, f RegistryFunc) { + if ns == "" { + panic("namespace cannot be empty") + } + registry[ns] = f +} + +// RegisterPlugin is called to register a new plugin in the +// given namespace. Its called internally by reva, and should +// not be used by external plugins. +func RegisterPlugin(ns, name string, newFunc any) { + if ns == "" { + panic("namespace cannot be empty") + } + if name == "" { + panic("name cannot be empty") + } + if newFunc == nil { + panic("new func cannot be nil") + } + if reflect.TypeOf(newFunc).Kind() != reflect.Func { + panic("type must be a function") + } + r, ok := registry[ns] + if !ok { + panic("namespace does not exist") + } + r(name, newFunc) } diff --git a/pkg/user/rpc_user.go b/pkg/user/rpc_user.go deleted file mode 100644 index 750718d457..0000000000 --- a/pkg/user/rpc_user.go +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package user - -import ( - "context" - "encoding/gob" - "net/rpc" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - "github.com/cs3org/reva/pkg/appctx" - "github.com/cs3org/reva/pkg/plugin" - hcplugin "github.com/hashicorp/go-plugin" -) - -func init() { - gob.Register(&userpb.User{}) - plugin.Register("userprovider", &ProviderPlugin{}) -} - -// ProviderPlugin is the implementation of plugin.Plugin so we can serve/consume this. -type ProviderPlugin struct { - Impl Manager -} - -// Server returns the RPC Server which serves the methods that the Client calls over net/rpc. -func (p *ProviderPlugin) Server(*hcplugin.MuxBroker) (interface{}, error) { - return &RPCServer{Impl: p.Impl}, nil -} - -// Client returns interface implementation for the plugin that communicates to the server end of the plugin. -func (p *ProviderPlugin) Client(b *hcplugin.MuxBroker, c *rpc.Client) (interface{}, error) { - return &RPCClient{Client: c}, nil -} - -// RPCClient is an implementation of Manager that talks over RPC. -type RPCClient struct{ Client *rpc.Client } - -// ConfigureArg for RPC. -type ConfigureArg struct { - Ml map[string]interface{} -} - -// ConfigureReply for RPC. -type ConfigureReply struct { - Err error -} - -// Configure RPCClient configure method. -func (m *RPCClient) Configure(ml map[string]interface{}) error { - args := ConfigureArg{Ml: ml} - resp := ConfigureReply{} - err := m.Client.Call("Plugin.Configure", args, &resp) - if err != nil { - return err - } - return resp.Err -} - -// GetUserArg for RPC. -type GetUserArg struct { - Ctx map[interface{}]interface{} - UID *userpb.UserId - SkipFetchingGroups bool -} - -// GetUserReply for RPC. -type GetUserReply struct { - User *userpb.User - Err error -} - -// GetUser RPCClient GetUser method. -func (m *RPCClient) GetUser(ctx context.Context, uid *userpb.UserId, skipFetchingGroups bool) (*userpb.User, error) { - ctxVal := appctx.GetKeyValuesFromCtx(ctx) - args := GetUserArg{Ctx: ctxVal, UID: uid, SkipFetchingGroups: skipFetchingGroups} - resp := GetUserReply{} - err := m.Client.Call("Plugin.GetUser", args, &resp) - if err != nil { - return nil, err - } - return resp.User, resp.Err -} - -// GetUserByClaimArg for RPC. -type GetUserByClaimArg struct { - Ctx map[interface{}]interface{} - Claim string - Value string - SkipFetchingGroups bool -} - -// GetUserByClaimReply for RPC. -type GetUserByClaimReply struct { - User *userpb.User - Err error -} - -// GetUserByClaim RPCClient GetUserByClaim method. -func (m *RPCClient) GetUserByClaim(ctx context.Context, claim, value string, skipFetchingGroups bool) (*userpb.User, error) { - ctxVal := appctx.GetKeyValuesFromCtx(ctx) - args := GetUserByClaimArg{Ctx: ctxVal, Claim: claim, Value: value, SkipFetchingGroups: skipFetchingGroups} - resp := GetUserByClaimReply{} - err := m.Client.Call("Plugin.GetUserByClaim", args, &resp) - if err != nil { - return nil, err - } - return resp.User, resp.Err -} - -// GetUserGroupsArg for RPC. -type GetUserGroupsArg struct { - Ctx map[interface{}]interface{} - User *userpb.UserId -} - -// GetUserGroupsReply for RPC. -type GetUserGroupsReply struct { - Group []string - Err error -} - -// GetUserGroups RPCClient GetUserGroups method. -func (m *RPCClient) GetUserGroups(ctx context.Context, user *userpb.UserId) ([]string, error) { - ctxVal := appctx.GetKeyValuesFromCtx(ctx) - args := GetUserGroupsArg{Ctx: ctxVal, User: user} - resp := GetUserGroupsReply{} - err := m.Client.Call("Plugin.GetUserGroups", args, &resp) - if err != nil { - return nil, err - } - return resp.Group, resp.Err -} - -// FindUsersArg for RPC. -type FindUsersArg struct { - Ctx map[interface{}]interface{} - Query string - SkipFetchingGroups bool -} - -// FindUsersReply for RPC. -type FindUsersReply struct { - User []*userpb.User - Err error -} - -// FindUsers RPCClient FindUsers method. -func (m *RPCClient) FindUsers(ctx context.Context, query string, skipFetchingGroups bool) ([]*userpb.User, error) { - ctxVal := appctx.GetKeyValuesFromCtx(ctx) - args := FindUsersArg{Ctx: ctxVal, Query: query, SkipFetchingGroups: skipFetchingGroups} - resp := FindUsersReply{} - err := m.Client.Call("Plugin.FindUsers", args, &resp) - if err != nil { - return nil, err - } - return resp.User, resp.Err -} - -// RPCServer is the server that RPCClient talks to, conforming to the requirements of net/rpc. -type RPCServer struct { - // This is the real implementation - Impl Manager -} - -// Configure RPCServer Configure method. -func (m *RPCServer) Configure(args ConfigureArg, resp *ConfigureReply) error { - resp.Err = m.Impl.Configure(args.Ml) - return nil -} - -// GetUser RPCServer GetUser method. -func (m *RPCServer) GetUser(args GetUserArg, resp *GetUserReply) error { - ctx := appctx.PutKeyValuesToCtx(args.Ctx) - resp.User, resp.Err = m.Impl.GetUser(ctx, args.UID, args.SkipFetchingGroups) - return nil -} - -// GetUserByClaim RPCServer GetUserByClaim method. -func (m *RPCServer) GetUserByClaim(args GetUserByClaimArg, resp *GetUserByClaimReply) error { - ctx := appctx.PutKeyValuesToCtx(args.Ctx) - resp.User, resp.Err = m.Impl.GetUserByClaim(ctx, args.Claim, args.Value, args.SkipFetchingGroups) - return nil -} - -// GetUserGroups RPCServer GetUserGroups method. -func (m *RPCServer) GetUserGroups(args GetUserGroupsArg, resp *GetUserGroupsReply) error { - ctx := appctx.PutKeyValuesToCtx(args.Ctx) - resp.Group, resp.Err = m.Impl.GetUserGroups(ctx, args.User) - return nil -} - -// FindUsers RPCServer FindUsers method. -func (m *RPCServer) FindUsers(args FindUsersArg, resp *FindUsersReply) error { - ctx := appctx.PutKeyValuesToCtx(args.Ctx) - resp.User, resp.Err = m.Impl.FindUsers(ctx, args.Query, args.SkipFetchingGroups) - return nil -} diff --git a/pkg/user/user.go b/pkg/user/user.go index e011b84567..d53233e104 100644 --- a/pkg/user/user.go +++ b/pkg/user/user.go @@ -22,12 +22,10 @@ import ( "context" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - "github.com/cs3org/reva/pkg/plugin" ) // Manager is the interface to implement to manipulate users. type Manager interface { - plugin.Plugin // GetUser returns the user metadata identified by a uid. // The groups of the user are omitted if specified, as these might not be required for certain operations // and might involve computational overhead. diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 41b3af44ad..4c7ce12133 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -418,3 +418,17 @@ func UserIsLightweight(u *userpb.User) bool { return u.Id.Type == userpb.UserType_USER_TYPE_FEDERATED || u.Id.Type == userpb.UserType_USER_TYPE_LIGHTWEIGHT } + +// Cast casts a value `v` to the value `to`. +// `v` is expected to be the underlying type of `to`. +// For example, if the type A is defined as func() +// and v is func(), to is of type A. +// to must be a pointer, otherwise the method panics. +func Cast(v any, to any) { + toVal := reflect.ValueOf(to) + if toVal.Type().Kind() != reflect.Pointer { + panic("cast: destination must be a pointer") + } + toVal = toVal.Elem() + toVal.Set(reflect.ValueOf(v)) +} diff --git a/plugins.go b/plugins.go new file mode 100644 index 0000000000..72c7479350 --- /dev/null +++ b/plugins.go @@ -0,0 +1,141 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package reva + +import ( + "sort" + "strings" + + "github.com/cs3org/reva/pkg/plugin" +) + +// Plugin is a type used as reva plugin. +// The type may implement something useful, depending +// on what is possible to plug in. +type Plugin interface { + // RevaPlugin returns the plugin info, like the ID + // in the form of ., and a New func + // used to create the plugin. + // The namespace can be only one defined by reva, + // depending on the scope of the plugin, while the name + // can be whatever, but unique in the namespace. + RevaPlugin() PluginInfo +} + +// PluginInfo holds the information of a reva plugin. +type PluginInfo struct { + // ID is the full name of the plugin, in the form + // .. It must be unique. + ID PluginID + // New is the constructor of the plugin. We rely + // on the developer to correctly provide a valid + // construct depending on the plugin. + New any +} + +// String return a string representation of the PluginInfo. +func (pi PluginInfo) String() string { return string(pi.ID) } + +// PluginID is the string that uniquely identify a reva plugin. +// It consists of a dot-separated labels. The last label is the +// name of the plugin, while the labels before represents the +// namespace. +// A pluginID is in the form . +// Neither the name nor the namespace can be empty. +// The name cannot contain dots. +// ModuleIDs shuld be lowercase and use underscores (_) instead +// of spaces. +type PluginID string + +var registry = map[string]Plugin{} + +// Name returns the name of the Plugin ID (i.e. the last name). +func (i PluginID) Name() string { + parts := strings.Split(string(i), ".") + if len(parts) <= 1 { + panic("plugin id must be .") + } + return parts[len(parts)-1] +} + +// Namespace returns the namespace of the plugin ID, which is +// all but the last label of the ID. +func (i PluginID) Namespace() string { + idx := strings.LastIndex(string(i), ".") + if idx < 0 { + panic("plugin id must be .") + } + return string(i)[:idx] +} + +// RegisterPlugin registers a reva plugin. For registration +// this method should be called in the init() method. +func RegisterPlugin(p Plugin) { + if p == nil { + panic("plugin cannot be nil") + } + plug := p.RevaPlugin() + if plug.ID == "" { + panic("plugin id cannot be nil") + } + if plug.New == nil { + panic("plugin new func cannot be nil") + } + + name := plug.ID.Name() + ns := plug.ID.Namespace() + registry[string(p.RevaPlugin().ID)] = p + plugin.RegisterPlugin(ns, name, plug.New) +} + +func hasPrefixSlices(s, prefix []string) bool { + if len(prefix) > len(s) { + return false + } + for i := range prefix { + if s[i] != prefix[i] { + return false + } + } + return true +} + +// GetPlugins returns all the plugins in the given namespace, +// and their descendants. +// For example, a namespace "foo" returns modules with id "foo", +// "foo.bar", "foo.bar.foo", but not "bar". +func GetPlugins(ns string) []PluginInfo { + prefix := strings.Split(ns, ".") + if ns == "" { + prefix = []string{} + } + + var plugs []PluginInfo + for ns, p := range registry { + nsParts := strings.Split(ns, ".") + if hasPrefixSlices(nsParts, prefix) { + plugs = append(plugs, p.RevaPlugin()) + } + } + + sort.SliceStable(plugs, func(i, j int) bool { + return plugs[i].ID < plugs[j].ID + }) + return plugs +} diff --git a/plugins_test.go b/plugins_test.go new file mode 100644 index 0000000000..1fcf2e4f80 --- /dev/null +++ b/plugins_test.go @@ -0,0 +1,149 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package reva + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +type mockPlugin struct { + id string +} + +func (p mockPlugin) RevaPlugin() PluginInfo { + return PluginInfo{ID: PluginID(p.id)} +} + +func TestGetPlugins(t *testing.T) { + registry = map[string]Plugin{ + "a": mockPlugin{id: "a"}, + "a.b": mockPlugin{id: "a.b"}, + "a.b.c": mockPlugin{id: "a.b.c"}, + "a.b.cd": mockPlugin{id: "a.b.cd"}, + "a.c": mockPlugin{id: "a.c"}, + "a.d": mockPlugin{id: "a.d"}, + "b": mockPlugin{id: "b"}, + "b.a": mockPlugin{id: "b.a"}, + "b.b": mockPlugin{id: "b.b"}, + "b.a.c": mockPlugin{id: "b.a.c"}, + "c": mockPlugin{id: "c"}, + } + + tests := []struct { + scope string + exp []PluginInfo + }{ + { + scope: "", + exp: []PluginInfo{ + {ID: "a"}, + {ID: "a.b"}, + {ID: "a.b.c"}, + {ID: "a.b.cd"}, + {ID: "a.c"}, + {ID: "a.d"}, + {ID: "b"}, + {ID: "b.a"}, + {ID: "b.a.c"}, + {ID: "b.b"}, + {ID: "c"}, + }, + }, + { + scope: "a", + exp: []PluginInfo{ + {ID: "a"}, + {ID: "a.b"}, + {ID: "a.b.c"}, + {ID: "a.b.cd"}, + {ID: "a.c"}, + {ID: "a.d"}, + }, + }, + { + scope: "a.b.c", + exp: []PluginInfo{ + {ID: "a.b.c"}, + }, + }, + { + scope: "b.a", + exp: []PluginInfo{ + {ID: "b.a"}, + {ID: "b.a.c"}, + }, + }, + } + + for i, tt := range tests { + got := GetPlugins(tt.scope) + if !reflect.DeepEqual(got, tt.exp) { + t.Fatalf("test %d: expected %v got %v", i+1, tt.exp, got) + } + } +} + +func TestNameNamespace(t *testing.T) { + tests := []struct { + id PluginID + name string + ns string + valid bool + }{ + { + id: "", + valid: false, + }, + { + id: "a", + valid: false, + }, + { + id: "a.b", + name: "b", + ns: "a", + valid: true, + }, + { + id: "aa.bb.cc", + name: "cc", + ns: "aa.bb", + valid: true, + }, + } + + for i, tt := range tests { + if !tt.valid { + assert.Panics(t, func() { + tt.id.Name() + }, "test %d should have paniced", i) + assert.Panics(t, func() { + tt.id.Namespace() + }, "test %d should have paniced", i) + } else { + name := tt.id.Name() + ns := tt.id.Namespace() + assert.Equal(t, tt.name, name, "test %d", i) + assert.Equal(t, tt.ns, ns, "test %d", i) + } + } +} From 0ded646022507e831b1d4eaf91d63a2392b9980e Mon Sep 17 00:00:00 2001 From: Karun Atreya <33852651+KarunAtreya@users.noreply.github.com> Date: Tue, 25 Jul 2023 16:25:50 +0545 Subject: [PATCH 43/48] bump latest ocis commit in reva master (#4079) --- .drone.env | 2 +- tests/ocis | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.env b/.drone.env index 57583443a2..62893b1bd5 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=6e45113e76c60965c5e1a3bad42e5b5459960571 +APITESTS_COMMITID=dc18f58693cacb0fb41503f572462560f8fcc984 APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/tests/ocis b/tests/ocis index 6e45113e76..dc18f58693 160000 --- a/tests/ocis +++ b/tests/ocis @@ -1 +1 @@ -Subproject commit 6e45113e76c60965c5e1a3bad42e5b5459960571 +Subproject commit dc18f58693cacb0fb41503f572462560f8fcc984 From a839778fa628f6fa8e403b7cc72f2b676aaa1106 Mon Sep 17 00:00:00 2001 From: Sawjan Gurung Date: Mon, 31 Jul 2023 14:18:44 +0545 Subject: [PATCH 44/48] [tests-only][full-ci] bump ocis commit id for tests - master (#4088) * bump ocis commit id for tests * bump ocis commit id for tests * update ocis submodule --- .drone.env | 2 +- .../expected-failures-on-OCIS-storage.md | 20 ---------------- .../expected-failures-on-S3NG-storage.md | 23 ------------------- tests/ocis | 2 +- 4 files changed, 2 insertions(+), 45 deletions(-) diff --git a/.drone.env b/.drone.env index 62893b1bd5..a6a14775e0 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=dc18f58693cacb0fb41503f572462560f8fcc984 +APITESTS_COMMITID=82045b885073959ba35c14f568e27f10aa6ccabc APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index f11839c105..92cfa7ab29 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -204,20 +204,6 @@ File and sync features in a shared scenario - [coreApiSharePublicLink2/copyFromPublicLink.feature:192](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L192) - [coreApiSharePublicLink2/copyFromPublicLink.feature:193](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L193) -#### [Increasing permission of a public link of a folder that was initially shared with share+read permissions is allowed](https://github.com/owncloud/ocis/issues/3881) - -- [coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature:159](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature#L159) -- [coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature:160](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature#L160) -- [coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature:181](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature#L181) -- [coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature:182](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature#L182) - -#### [Adding public upload to a read only shared folder as a receipient is allowed ](https://github.com/owncloud/ocis/issues/2164) - -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:267](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L267) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:268](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L268) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:309](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L309) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:310](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L310) - #### [Upload-only shares must not overwrite but create a separate file](https://github.com/owncloud/ocis/issues/1267) - [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:13](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L13) @@ -229,13 +215,7 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:87](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L87) - [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:96](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L96) -#### [share permissions are not enforced](https://github.com/owncloud/product/issues/270) //todo -- [coreApiShareReshareToShares3/reShareUpdate.feature:63](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L63) -- [coreApiShareReshareToShares3/reShareUpdate.feature:64](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L64) - #### [path property in pending shares gives only filename](https://github.com/owncloud/ocis/issues/2156) -- [coreApiShareReshareToShares2/reShareSubfolder.feature:143](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares2/reShareSubfolder.feature#L143) -- [coreApiShareReshareToShares2/reShareSubfolder.feature:144](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares2/reShareSubfolder.feature#L144) - [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:61](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L61) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:733](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L733) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:734](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L734) diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 459776de5f..7a56b962dc 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -229,20 +229,6 @@ File and sync features in a shared scenario - [coreApiSharePublicLink2/copyFromPublicLink.feature:192](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L192) - [coreApiSharePublicLink2/copyFromPublicLink.feature:193](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/copyFromPublicLink.feature#L193) -#### [Increasing permission of a public link of a folder that was initially shared with share+read permissions is allowed](https://github.com/owncloud/ocis/issues/3881) - -- [coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature:159](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature#L159) -- [coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature:160](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature#L160) -- [coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature:181](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature#L181) -- [coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature:182](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink2/reShareAsPublicLinkToSharesNewDav.feature#L182) - -#### [Adding public upload to a read only shared folder as a receipient is allowed ](https://github.com/owncloud/ocis/issues/2164) - -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:267](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L267) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:268](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L268) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:309](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L309) -- [coreApiSharePublicLink3/updatePublicLinkShare.feature:310](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/updatePublicLinkShare.feature#L310) - #### [Upload-only shares must not overwrite but create a separate file](https://github.com/owncloud/ocis/issues/1267) - [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:13](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L13) @@ -254,15 +240,6 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:87](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L87) - [coreApiSharePublicLink3/uploadToPublicLinkShare.feature:96](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiSharePublicLink3/uploadToPublicLinkShare.feature#L96) -#### [share permissions are not enforced](https://github.com/owncloud/product/issues/270) //todo - -- [coreApiShareReshareToShares3/reShareUpdate.feature:63](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L63) -- [coreApiShareReshareToShares3/reShareUpdate.feature:64](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares3/reShareUpdate.feature#L64) - -#### [file_target in share response](https://github.com/owncloud/product/issues/203) //todo -- [coreApiShareReshareToShares2/reShareSubfolder.feature:143](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares2/reShareSubfolder.feature#L143) -- [coreApiShareReshareToShares2/reShareSubfolder.feature:144](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareReshareToShares2/reShareSubfolder.feature#L144) - #### [deleting a file inside a received shared folder is moved to the trash-bin of the sharer not the receiver](https://github.com/owncloud/ocis/issues/1124) - [coreApiTrashbin/trashbinSharingToShares.feature:47](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiTrashbin/trashbinSharingToShares.feature#L47) diff --git a/tests/ocis b/tests/ocis index dc18f58693..82045b8850 160000 --- a/tests/ocis +++ b/tests/ocis @@ -1 +1 @@ -Subproject commit dc18f58693cacb0fb41503f572462560f8fcc984 +Subproject commit 82045b885073959ba35c14f568e27f10aa6ccabc From a06384a7a98aac2f94bb00b070ae23e8d781bbaa Mon Sep 17 00:00:00 2001 From: Sabin Panta <64484313+S-Panta@users.noreply.github.com> Date: Wed, 2 Aug 2023 12:44:45 +0545 Subject: [PATCH 45/48] bumped ocis commit id for tests (#4094) --- .drone.env | 2 +- tests/ocis | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.env b/.drone.env index a6a14775e0..563780c1a8 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=82045b885073959ba35c14f568e27f10aa6ccabc +APITESTS_COMMITID=43ee37d02ed652b89e88c03c6bbeb5c89a62ce16 APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/tests/ocis b/tests/ocis index 82045b8850..43ee37d02e 160000 --- a/tests/ocis +++ b/tests/ocis @@ -1 +1 @@ -Subproject commit 82045b885073959ba35c14f568e27f10aa6ccabc +Subproject commit 43ee37d02ed652b89e88c03c6bbeb5c89a62ce16 From 9cfa2739aa1d23a1592212ae2c1685cfca8c7ed9 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 2 Aug 2023 17:27:42 +0200 Subject: [PATCH 46/48] Fixes on notifications (#4061) * Fixes following recent changes * Reverted to bool to match the rest of the stack * changelog * Fixed tests --- changelog/unreleased/notif-fixes.md | 5 +++++ pkg/notification/db_changes.sql | 6 +----- pkg/notification/db_sqlite.sql | 16 ++++++++-------- pkg/notification/manager/sql/sql.go | 10 +++++----- pkg/notification/manager/sql/sql_test.go | 18 +++++++++--------- pkg/notification/manager/sql/test.sqlite | Bin 32768 -> 32768 bytes 6 files changed, 28 insertions(+), 27 deletions(-) create mode 100644 changelog/unreleased/notif-fixes.md diff --git a/changelog/unreleased/notif-fixes.md b/changelog/unreleased/notif-fixes.md new file mode 100644 index 0000000000..24a7e5ceb6 --- /dev/null +++ b/changelog/unreleased/notif-fixes.md @@ -0,0 +1,5 @@ +Bugfix: Fixes on notifications + +This is to align the code to the latest schema for notifications + +https://github.com/cs3org/reva/pull/4061 diff --git a/pkg/notification/db_changes.sql b/pkg/notification/db_changes.sql index 234c2a2558..8b2ccc804d 100644 --- a/pkg/notification/db_changes.sql +++ b/pkg/notification/db_changes.sql @@ -45,9 +45,5 @@ CREATE INDEX `notification_recipients_ix1` ON `notification_recipients` (`recipi -- changes for added notifications on oc shares -ALTER TABLE oc_share ADD notify_uploads BOOL DEFAULT false; +ALTER TABLE oc_share ADD notify_uploads BOOL DEFAULT false NOT NULL; ALTER TABLE oc_share ADD notify_uploads_extra_recipients VARCHAR(2048); - -UPDATE oc_share SET notify_uploads = false; - -ALTER TABLE oc_share MODIFY notify_uploads BOOL DEFAULT false NOT NULL; diff --git a/pkg/notification/db_sqlite.sql b/pkg/notification/db_sqlite.sql index 8e110fe153..3d691352e1 100644 --- a/pkg/notification/db_sqlite.sql +++ b/pkg/notification/db_sqlite.sql @@ -19,7 +19,7 @@ -- This file can be used to quickstart a SQLite DB for running the tests in -- ./manager/sql/sql_test.go -CREATE TABLE `cbox_notifications` ( +CREATE TABLE `notifications` ( `id` INTEGER PRIMARY KEY AUTOINCREMENT, `ref` VARCHAR(3072) UNIQUE NOT NULL, `template_name` VARCHAR(320) NOT NULL @@ -27,25 +27,25 @@ CREATE TABLE `cbox_notifications` ( COMMIT; -CREATE TABLE `cbox_notification_recipients` ( +CREATE TABLE `notification_recipients` ( `id` INTEGER PRIMARY KEY AUTOINCREMENT, `notification_id` INTEGER NOT NULL, `recipient` VARCHAR(320) NOT NULL, FOREIGN KEY (notification_id) - REFERENCES cbox_notifications (id) + REFERENCES notifications (id) ON DELETE CASCADE ); COMMIT; -CREATE INDEX `cbox_notifications_ix0` ON `cbox_notifications` (`ref`); +CREATE INDEX `notifications_ix0` ON `notifications` (`ref`); -CREATE INDEX `cbox_notification_recipients_ix0` ON `cbox_notification_recipients` (`notification_id`); -CREATE INDEX `cbox_notification_recipients_ix1` ON `cbox_notification_recipients` (`recipient`); +CREATE INDEX `notification_recipients_ix0` ON `notification_recipients` (`notification_id`); +CREATE INDEX `notification_recipients_ix1` ON `notification_recipients` (`recipient`); COMMIT; -INSERT INTO `cbox_notifications` (`id`, `ref`, `template_name`) VALUES (1, "notification-test", "notification-template-test"); -INSERT INTO `cbox_notification_recipients` (`id`, `notification_id`, `recipient`) VALUES (1, 1, "jdoe"), (2, 1, "testuser"); +INSERT INTO `notifications` (`id`, `ref`, `template_name`) VALUES (1, "notification-test", "notification-template-test"); +INSERT INTO `notification_recipients` (`id`, `notification_id`, `recipient`) VALUES (1, 1, "jdoe"), (2, 1, "testuser"); COMMIT; diff --git a/pkg/notification/manager/sql/sql.go b/pkg/notification/manager/sql/sql.go index c1af5b15a9..d0760bdf48 100644 --- a/pkg/notification/manager/sql/sql.go +++ b/pkg/notification/manager/sql/sql.go @@ -80,7 +80,7 @@ func (m *mgr) UpsertNotification(n notification.Notification) error { } // Create/update notification - stmt, err := m.db.Prepare("REPLACE INTO cbox_notifications (ref, template_name) VALUES (?, ?)") + stmt, err := m.db.Prepare("REPLACE INTO notifications (ref, template_name) VALUES (?, ?)") if err != nil { return err } @@ -99,7 +99,7 @@ func (m *mgr) UpsertNotification(n notification.Notification) error { return err } - stmt, err = tx.Prepare("REPLACE INTO cbox_notification_recipients (notification_id, recipient) VALUES (?, ?)") + stmt, err = tx.Prepare("REPLACE INTO notification_recipients (notification_id, recipient) VALUES (?, ?)") if err != nil { _ = tx.Rollback() return err @@ -126,8 +126,8 @@ func (m *mgr) UpsertNotification(n notification.Notification) error { func (m *mgr) GetNotification(ref string) (*notification.Notification, error) { query := ` SELECT n.id, n.ref, n.template_name, nr.recipient - FROM cbox_notifications AS n - JOIN cbox_notification_recipients AS nr ON n.id = nr.notification_id + FROM notifications AS n + JOIN notification_recipients AS nr ON n.id = nr.notification_id WHERE n.ref = ? ` @@ -171,7 +171,7 @@ func (m *mgr) DeleteNotification(ref string) error { } // Delete notification - stmt, err := m.db.Prepare("DELETE FROM cbox_notifications WHERE ref = ?") + stmt, err := m.db.Prepare("DELETE FROM notifications WHERE ref = ?") if err != nil { return err } diff --git a/pkg/notification/manager/sql/sql_test.go b/pkg/notification/manager/sql/sql_test.go index 60080ffab0..5bfe1f3378 100644 --- a/pkg/notification/manager/sql/sql_test.go +++ b/pkg/notification/manager/sql/sql_test.go @@ -48,8 +48,8 @@ var _ = Describe("SQL manager for notifications", func() { nn *notification.Notification ref string err error - selectNotificationsSQL = "SELECT ref, template_name FROM cbox_notifications WHERE ref = ?" - selectNotificationRecipientsSQL = "SELECT COUNT(*) FROM cbox_notification_recipients WHERE notification_id = ?" + selectNotificationsSQL = "SELECT ref, template_name FROM notifications WHERE ref = ?" + selectNotificationRecipientsSQL = "SELECT COUNT(*) FROM notification_recipients WHERE notification_id = ?" ) AfterEach(func() { @@ -103,7 +103,7 @@ var _ = Describe("SQL manager for notifications", func() { It("should create notification recipients entries", func() { var notificationID int - err = db.QueryRow("SELECT id FROM cbox_notifications WHERE ref = ?", n2.Ref).Scan(¬ificationID) + err = db.QueryRow("SELECT id FROM notifications WHERE ref = ?", n2.Ref).Scan(¬ificationID) Expect(err).ToNot(HaveOccurred()) var newRecipientCount int err = db.QueryRow(selectNotificationRecipientsSQL, notificationID).Scan(&newRecipientCount) @@ -129,7 +129,7 @@ var _ = Describe("SQL manager for notifications", func() { It("should not increase the number of entries in the notification table", func() { var count int - err = db.QueryRow("SELECT COUNT(*) FROM cbox_notifications").Scan(&count) + err = db.QueryRow("SELECT COUNT(*) FROM notifications").Scan(&count) Expect(err).ToNot(HaveOccurred()) Expect(count).To(Equal(1)) }) @@ -143,14 +143,14 @@ var _ = Describe("SQL manager for notifications", func() { It("should delete old entries in notification recipients", func() { var count int - err = db.QueryRow("SELECT COUNT(*) FROM cbox_notification_recipients WHERE recipient = 'testuser'").Scan(&count) + err = db.QueryRow("SELECT COUNT(*) FROM notification_recipients WHERE recipient = 'testuser'").Scan(&count) Expect(err).ToNot(HaveOccurred()) Expect(count).To(BeZero()) }) It("should create new entries in notification recipients", func() { var notificationID int - err = db.QueryRow("SELECT id FROM cbox_notifications WHERE ref = ?", m.Ref).Scan(¬ificationID) + err = db.QueryRow("SELECT id FROM notifications WHERE ref = ?", m.Ref).Scan(¬ificationID) Expect(err).ToNot(HaveOccurred()) var newRecipientCount int err = db.QueryRow(selectNotificationRecipientsSQL, notificationID).Scan(&newRecipientCount) @@ -217,14 +217,14 @@ var _ = Describe("SQL manager for notifications", func() { It("should delete the notification from the database", func() { var count int - err = db.QueryRow("SELECT COUNT(*) FROM cbox_notifications WHERE ref = ?", ref).Scan(&count) + err = db.QueryRow("SELECT COUNT(*) FROM notifications WHERE ref = ?", ref).Scan(&count) Expect(err).ToNot(HaveOccurred()) Expect(count).To(BeZero()) }) It("should cascade the deletions to notification_recipients table", func() { var count int - err = db.QueryRow("SELECT COUNT(*) FROM cbox_notification_recipients WHERE notification_id = ?", 1).Scan(&count) + err = db.QueryRow("SELECT COUNT(*) FROM notification_recipients WHERE notification_id = ?", 1).Scan(&count) Expect(err).ToNot(HaveOccurred()) Expect(count).To(BeZero()) }) @@ -241,7 +241,7 @@ var _ = Describe("SQL manager for notifications", func() { isNotFoundError, _ := err.(*notification.NotFoundError) Expect(isNotFoundError).ToNot(BeNil()) var count int - err = db.QueryRow("SELECT COUNT(*) FROM cbox_notifications").Scan(&count) + err = db.QueryRow("SELECT COUNT(*) FROM notifications").Scan(&count) Expect(err).ToNot(HaveOccurred()) Expect(count).To(Equal(1)) }) diff --git a/pkg/notification/manager/sql/test.sqlite b/pkg/notification/manager/sql/test.sqlite index 7bf9289cdda218362bae3993b3e552879705b4f0..e00cc5378734cb5e835a91e515d5cdf564decd79 100644 GIT binary patch delta 490 zcmZo@U}|V!njkHx&A`CG0mLvMI#I`1TAM*H+>4j*1_L{{6$4)z??!$`o+kc2?%Ui} zn*{~BxmbWw$0s_<$^q%EFr6^EQI3;c+}E11(Q&c^zu4pgZvM#=dBi6NaGNkHfmo7C z3K|JTsmYlInW=dt37U;691!)sVD%TE>VL!3tANz!<(FipWhN(r@oh^vCLW3aQM ziz}CAV-O3wxVk!HQ!>~z6n&Hb@k#Te88!Jk?;~a#2F;0$s`i$wY~sG^j0~B1DXA63 zg*llesqu-WCHY`}Jc?2AhA3QSV2}k(Z1m$SU}qP%RA+4D1%`n1y$7At6Erf70;a{BuzxVZYbhPWy?I|e&Dy0~&_ zHYT#LiyIm;Hsyki#MBEGnS7R4S|kBeGGX#}-bboG7~;W3L=!SX2b&QJ8e9qxP{qzJ zZfVHaC=LmdbNph$m{zh)ZseAqe4n2y6T4x+Ku$~0bY^7}4>x3F0Gm`?n3GwO8lPBN zk`Ly`W11Roh$+SlOakDPut|VTf$ca0|4aVkz}Q~KKY5eAD60Y!vnwMg2Dq6-8JP_^ M!TL9Uw&!&K0ICVM#sB~S From 9266d8ed710e07388d26310d4c6029aac0289e76 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 3 Aug 2023 09:51:16 +0200 Subject: [PATCH 47/48] Handle target in OpenInApp response (#4077) --- changelog/unreleased/openappintab.md | 10 ++++ .../grpc/services/appprovider/_index.md | 4 +- .../grpc/services/ocminvitemanager/_index.md | 2 +- .../grpc/services/ocmshareprovider/_index.md | 2 +- .../grpc/services/permissions/_index.md | 4 +- .../grpc/services/storageprovider/_index.md | 18 +++--- .../http/services/appprovider/_index.md | 2 +- .../http/services/datagateway/_index.md | 2 +- .../http/services/owncloud/ocdav/_index.md | 4 +- .../handler/emailhandler/_index.md | 10 ++-- .../ocm/provider/authorizer/mentix/_index.md | 2 +- .../packages/storage/fs/local/_index.md | 4 +- .../packages/storage/fs/localhome/_index.md | 6 +- .../services/notifications/_index.md | 16 +++--- go.mod | 2 +- go.sum | 4 +- .../grpc/services/appprovider/appprovider.go | 3 +- .../http/services/appprovider/appprovider.go | 55 ++++++++++++++----- internal/http/services/archiver/handler.go | 26 ++++++--- pkg/app/provider/demo/demo.go | 6 +- pkg/app/provider/wopi/wopi.go | 1 + pkg/errtypes/errtypes.go | 6 +- pkg/rgrpc/status/status.go | 2 + 23 files changed, 122 insertions(+), 69 deletions(-) create mode 100644 changelog/unreleased/openappintab.md diff --git a/changelog/unreleased/openappintab.md b/changelog/unreleased/openappintab.md new file mode 100644 index 0000000000..9091a34bf9 --- /dev/null +++ b/changelog/unreleased/openappintab.md @@ -0,0 +1,10 @@ +Enhancement: Handle target in OpenInApp response + +This PR adds the OpenInApp.target and AppProviderInfo.action properties +to the respective responses (/app/open and /app/list), to support +different app integrations. +In addition, the archiver was extended to use the name of the file/folder +as opposed to "download", and to include a query parameter to +override the archive type, as it will be used in an upcoming app. + +https://github.com/cs3org/reva/pull/4077 diff --git a/docs/content/en/docs/config/grpc/services/appprovider/_index.md b/docs/content/en/docs/config/grpc/services/appprovider/_index.md index db3ebb3287..d06d011b76 100644 --- a/docs/content/en/docs/config/grpc/services/appprovider/_index.md +++ b/docs/content/en/docs/config/grpc/services/appprovider/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="mime_types" type="[]string" default=nil %}} -A list of mime types supported by this app. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/appprovider/appprovider.go#L62) +A list of mime types supported by this app. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/appprovider/appprovider.go#L68) {{< highlight toml >}} [grpc.services.appprovider] mime_types = nil @@ -17,7 +17,7 @@ mime_types = nil {{% /dir %}} {{% dir name="custom_mime_types_json" type="string" default="nil" %}} -An optional mapping file with the list of supported custom file extensions and corresponding mime types. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/appprovider/appprovider.go#L63) +An optional mapping file with the list of supported custom file extensions and corresponding mime types. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/appprovider/appprovider.go#L69) {{< highlight toml >}} [grpc.services.appprovider] custom_mime_types_json = "nil" diff --git a/docs/content/en/docs/config/grpc/services/ocminvitemanager/_index.md b/docs/content/en/docs/config/grpc/services/ocminvitemanager/_index.md index cc72d58efe..363d33b540 100644 --- a/docs/content/en/docs/config/grpc/services/ocminvitemanager/_index.md +++ b/docs/content/en/docs/config/grpc/services/ocminvitemanager/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="provider_domain" type="string" default="The same domain registered in the provider authorizer" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/ocminvitemanager/ocminvitemanager.go#L55) + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/ocminvitemanager/ocminvitemanager.go#L61) {{< highlight toml >}} [grpc.services.ocminvitemanager] provider_domain = "The same domain registered in the provider authorizer" diff --git a/docs/content/en/docs/config/grpc/services/ocmshareprovider/_index.md b/docs/content/en/docs/config/grpc/services/ocmshareprovider/_index.md index 3de755abf5..08235771f3 100644 --- a/docs/content/en/docs/config/grpc/services/ocmshareprovider/_index.md +++ b/docs/content/en/docs/config/grpc/services/ocmshareprovider/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="provider_domain" type="string" default="The same domain registered in the provider authorizer" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/ocmshareprovider/ocmshareprovider.go#L64) + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/ocmshareprovider/ocmshareprovider.go#L70) {{< highlight toml >}} [grpc.services.ocmshareprovider] provider_domain = "The same domain registered in the provider authorizer" diff --git a/docs/content/en/docs/config/grpc/services/permissions/_index.md b/docs/content/en/docs/config/grpc/services/permissions/_index.md index af355d5064..09e9e45090 100644 --- a/docs/content/en/docs/config/grpc/services/permissions/_index.md +++ b/docs/content/en/docs/config/grpc/services/permissions/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="driver" type="string" default="localhome" %}} -The permission driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/permissions/permissions.go#L40) +The permission driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/permissions/permissions.go#L46) {{< highlight toml >}} [grpc.services.permissions] driver = "localhome" @@ -17,7 +17,7 @@ driver = "localhome" {{% /dir %}} {{% dir name="drivers" type="map[string]map[string]interface{}" default="permission" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/permissions/permissions.go#L41) + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/permissions/permissions.go#L47) {{< highlight toml >}} [grpc.services.permissions.drivers.permission] diff --git a/docs/content/en/docs/config/grpc/services/storageprovider/_index.md b/docs/content/en/docs/config/grpc/services/storageprovider/_index.md index 0f14db6708..ecd6f8eb34 100644 --- a/docs/content/en/docs/config/grpc/services/storageprovider/_index.md +++ b/docs/content/en/docs/config/grpc/services/storageprovider/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="mount_path" type="string" default="/" %}} -The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L57) +The path where the file system would be mounted. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L63) {{< highlight toml >}} [grpc.services.storageprovider] mount_path = "/" @@ -17,7 +17,7 @@ mount_path = "/" {{% /dir %}} {{% dir name="mount_id" type="string" default="-" %}} -The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L58) +The ID of the mounted file system. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L64) {{< highlight toml >}} [grpc.services.storageprovider] mount_id = "-" @@ -25,7 +25,7 @@ mount_id = "-" {{% /dir %}} {{% dir name="driver" type="string" default="localhome" %}} -The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L59) +The storage driver to be used. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L65) {{< highlight toml >}} [grpc.services.storageprovider] driver = "localhome" @@ -33,7 +33,7 @@ driver = "localhome" {{% /dir %}} {{% dir name="drivers" type="map[string]map[string]interface{}" default="localhome" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L60) + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L66) {{< highlight toml >}} [grpc.services.storageprovider.drivers.localhome] root = "/var/tmp/reva/" @@ -44,7 +44,7 @@ user_layout = "{{.Username}}" {{% /dir %}} {{% dir name="tmp_folder" type="string" default="/var/tmp" %}} -Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L61) +Path to temporary folder. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L67) {{< highlight toml >}} [grpc.services.storageprovider] tmp_folder = "/var/tmp" @@ -52,7 +52,7 @@ tmp_folder = "/var/tmp" {{% /dir %}} {{% dir name="data_server_url" type="string" default="http://localhost/data" %}} -The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L62) +The URL for the data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L68) {{< highlight toml >}} [grpc.services.storageprovider] data_server_url = "http://localhost/data" @@ -60,7 +60,7 @@ data_server_url = "http://localhost/data" {{% /dir %}} {{% dir name="expose_data_server" type="bool" default=false %}} -Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L63) +Whether to expose data server. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L69) {{< highlight toml >}} [grpc.services.storageprovider] expose_data_server = false @@ -68,7 +68,7 @@ expose_data_server = false {{% /dir %}} {{% dir name="available_checksums" type="map[string]uint32" default=nil %}} -List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L64) +List of available checksums. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L70) {{< highlight toml >}} [grpc.services.storageprovider] available_checksums = nil @@ -76,7 +76,7 @@ available_checksums = nil {{% /dir %}} {{% dir name="custom_mime_types_json" type="string" default="nil" %}} -An optional mapping file with the list of supported custom file extensions and corresponding mime types. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L65) +An optional mapping file with the list of supported custom file extensions and corresponding mime types. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/grpc/services/storageprovider/storageprovider.go#L71) {{< highlight toml >}} [grpc.services.storageprovider] custom_mime_types_json = "nil" diff --git a/docs/content/en/docs/config/http/services/appprovider/_index.md b/docs/content/en/docs/config/http/services/appprovider/_index.md index c0088e049d..bda4901bfc 100644 --- a/docs/content/en/docs/config/http/services/appprovider/_index.md +++ b/docs/content/en/docs/config/http/services/appprovider/_index.md @@ -9,7 +9,7 @@ description: > # _struct: Config_ {{% dir name="insecure" type="bool" default=false %}} -Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/appprovider/appprovider.go#L54) +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/appprovider/appprovider.go#L56) {{< highlight toml >}} [http.services.appprovider] insecure = false diff --git a/docs/content/en/docs/config/http/services/datagateway/_index.md b/docs/content/en/docs/config/http/services/datagateway/_index.md index 2e70830e53..e293489e5d 100644 --- a/docs/content/en/docs/config/http/services/datagateway/_index.md +++ b/docs/content/en/docs/config/http/services/datagateway/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="insecure" type="bool" default=false %}} -Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/datagateway/datagateway.go#L62) +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/datagateway/datagateway.go#L61) {{< highlight toml >}} [http.services.datagateway] insecure = false diff --git a/docs/content/en/docs/config/http/services/owncloud/ocdav/_index.md b/docs/content/en/docs/config/http/services/owncloud/ocdav/_index.md index e269611bb1..874587adfb 100644 --- a/docs/content/en/docs/config/http/services/owncloud/ocdav/_index.md +++ b/docs/content/en/docs/config/http/services/owncloud/ocdav/_index.md @@ -9,7 +9,7 @@ description: > # _struct: Config_ {{% dir name="insecure" type="bool" default=false %}} -Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/owncloud/ocdav/ocdav.go#L104) +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/owncloud/ocdav/ocdav.go#L103) {{< highlight toml >}} [http.services.owncloud.ocdav] insecure = false @@ -17,7 +17,7 @@ insecure = false {{% /dir %}} {{% dir name="notifications" type="map[string]interface{}" default=Settingsg for the Notification Helper %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/owncloud/ocdav/ocdav.go#L115) + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/http/services/owncloud/ocdav/ocdav.go#L114) {{< highlight toml >}} [http.services.owncloud.ocdav] notifications = Settingsg for the Notification Helper diff --git a/docs/content/en/docs/config/packages/notification/handler/emailhandler/_index.md b/docs/content/en/docs/config/packages/notification/handler/emailhandler/_index.md index fa5c7ca72e..80138d85cd 100644 --- a/docs/content/en/docs/config/packages/notification/handler/emailhandler/_index.md +++ b/docs/content/en/docs/config/packages/notification/handler/emailhandler/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="smtp_server" type="string" default="" %}} -The hostname and port of the SMTP server. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/handler/emailhandler/emailhandler.go#L44) +The hostname and port of the SMTP server. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/handler/emailhandler/emailhandler.go#L46) {{< highlight toml >}} [notification.handler.emailhandler] smtp_server = "" @@ -17,7 +17,7 @@ smtp_server = "" {{% /dir %}} {{% dir name="sender_login" type="string" default="" %}} -The email to be used to send mails. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/handler/emailhandler/emailhandler.go#L45) +The email to be used to send mails. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/handler/emailhandler/emailhandler.go#L47) {{< highlight toml >}} [notification.handler.emailhandler] sender_login = "" @@ -25,7 +25,7 @@ sender_login = "" {{% /dir %}} {{% dir name="sender_password" type="string" default="" %}} -The sender's password. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/handler/emailhandler/emailhandler.go#L46) +The sender's password. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/handler/emailhandler/emailhandler.go#L48) {{< highlight toml >}} [notification.handler.emailhandler] sender_password = "" @@ -33,7 +33,7 @@ sender_password = "" {{% /dir %}} {{% dir name="disable_auth" type="bool" default=false %}} -Whether to disable SMTP auth. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/handler/emailhandler/emailhandler.go#L47) +Whether to disable SMTP auth. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/handler/emailhandler/emailhandler.go#L49) {{< highlight toml >}} [notification.handler.emailhandler] disable_auth = false @@ -41,7 +41,7 @@ disable_auth = false {{% /dir %}} {{% dir name="default_sender" type="string" default="no-reply@cernbox.cern.ch" %}} -Default sender when not specified in the trigger. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/handler/emailhandler/emailhandler.go#L48) +Default sender when not specified in the trigger. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/notification/handler/emailhandler/emailhandler.go#L50) {{< highlight toml >}} [notification.handler.emailhandler] default_sender = "no-reply@cernbox.cern.ch" diff --git a/docs/content/en/docs/config/packages/ocm/provider/authorizer/mentix/_index.md b/docs/content/en/docs/config/packages/ocm/provider/authorizer/mentix/_index.md index 1194dc447b..fe12cab3b7 100644 --- a/docs/content/en/docs/config/packages/ocm/provider/authorizer/mentix/_index.md +++ b/docs/content/en/docs/config/packages/ocm/provider/authorizer/mentix/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="insecure" type="bool" default=false %}} -Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/ocm/provider/authorizer/mentix/mentix.go#L81) +Whether to skip certificate checks when sending requests. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/ocm/provider/authorizer/mentix/mentix.go#L79) {{< highlight toml >}} [ocm.provider.authorizer.mentix] insecure = false diff --git a/docs/content/en/docs/config/packages/storage/fs/local/_index.md b/docs/content/en/docs/config/packages/storage/fs/local/_index.md index 2443783511..550016e895 100644 --- a/docs/content/en/docs/config/packages/storage/fs/local/_index.md +++ b/docs/content/en/docs/config/packages/storage/fs/local/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="root" type="string" default="/var/tmp/reva/" %}} -Path of root directory for user storage. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/local/local.go#L34) +Path of root directory for user storage. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/local/local.go#L35) {{< highlight toml >}} [storage.fs.local] root = "/var/tmp/reva/" @@ -17,7 +17,7 @@ root = "/var/tmp/reva/" {{% /dir %}} {{% dir name="share_folder" type="string" default="/MyShares" %}} -Path for storing share references. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/local/local.go#L35) +Path for storing share references. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/local/local.go#L36) {{< highlight toml >}} [storage.fs.local] share_folder = "/MyShares" diff --git a/docs/content/en/docs/config/packages/storage/fs/localhome/_index.md b/docs/content/en/docs/config/packages/storage/fs/localhome/_index.md index 7a1d3626da..0288e9e6ae 100644 --- a/docs/content/en/docs/config/packages/storage/fs/localhome/_index.md +++ b/docs/content/en/docs/config/packages/storage/fs/localhome/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="root" type="string" default="/var/tmp/reva/" %}} -Path of root directory for user storage. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/localhome/localhome.go#L34) +Path of root directory for user storage. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/localhome/localhome.go#L36) {{< highlight toml >}} [storage.fs.localhome] root = "/var/tmp/reva/" @@ -17,7 +17,7 @@ root = "/var/tmp/reva/" {{% /dir %}} {{% dir name="share_folder" type="string" default="/MyShares" %}} -Path for storing share references. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/localhome/localhome.go#L35) +Path for storing share references. [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/localhome/localhome.go#L37) {{< highlight toml >}} [storage.fs.localhome] share_folder = "/MyShares" @@ -25,7 +25,7 @@ share_folder = "/MyShares" {{% /dir %}} {{% dir name="user_layout" type="string" default="{{.Username}}" %}} -Template for user home directories [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/localhome/localhome.go#L36) +Template for user home directories [[Ref]](https://github.com/cs3org/reva/tree/master/pkg/storage/fs/localhome/localhome.go#L38) {{< highlight toml >}} [storage.fs.localhome] user_layout = "{{.Username}}" diff --git a/docs/content/en/docs/config/serverless/services/notifications/_index.md b/docs/content/en/docs/config/serverless/services/notifications/_index.md index f4478eb33e..fe98ffbd5f 100644 --- a/docs/content/en/docs/config/serverless/services/notifications/_index.md +++ b/docs/content/en/docs/config/serverless/services/notifications/_index.md @@ -9,7 +9,7 @@ description: > # _struct: config_ {{% dir name="nats_address" type="string" default="" %}} -The NATS server address. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L46) +The NATS server address. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L47) {{< highlight toml >}} [serverless.services.notifications] nats_address = "" @@ -17,7 +17,7 @@ nats_address = "" {{% /dir %}} {{% dir name="nats_token" type="string" default="The token to authenticate against the NATS server" %}} - [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L47) + [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L48) {{< highlight toml >}} [serverless.services.notifications] nats_token = "The token to authenticate against the NATS server" @@ -25,15 +25,15 @@ nats_token = "The token to authenticate against the NATS server" {{% /dir %}} {{% dir name="nats_prefix" type="string" default="reva-notifications" %}} -The notifications NATS stream. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L48) +The notifications NATS stream. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L49) {{< highlight toml >}} [serverless.services.notifications] nats_prefix = "reva-notifications" {{< /highlight >}} {{% /dir %}} -{{% dir name="handlers" type="map[string]interface{}" default= %}} -Settings for the different notification handlers. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L49) +{{% dir name="handlers" type="map[string]map[string]interface{}" default= %}} +Settings for the different notification handlers. [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L50) {{< highlight toml >}} [serverless.services.notifications] handlers = @@ -41,7 +41,7 @@ handlers = {{% /dir %}} {{% dir name="grouping_interval" type="int" default=60 %}} -Time in seconds to group incoming notification triggers [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L50) +Time in seconds to group incoming notification triggers [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L51) {{< highlight toml >}} [serverless.services.notifications] grouping_interval = 60 @@ -49,7 +49,7 @@ grouping_interval = 60 {{% /dir %}} {{% dir name="grouping_max_size" type="int" default=100 %}} -Maximum number of notifications to group [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L51) +Maximum number of notifications to group [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L52) {{< highlight toml >}} [serverless.services.notifications] grouping_max_size = 100 @@ -57,7 +57,7 @@ grouping_max_size = 100 {{% /dir %}} {{% dir name="storage_driver" type="string" default="mysql" %}} -The driver used to store notifications [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L52) +The driver used to store notifications [[Ref]](https://github.com/cs3org/reva/tree/master/internal/serverless/services/notifications/notifications.go#L53) {{< highlight toml >}} [serverless.services.notifications] storage_driver = "mysql" diff --git a/go.mod b/go.mod index f0fe32ae89..03f3d43c07 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/coreos/go-oidc/v3 v3.5.0 github.com/creasty/defaults v1.7.0 github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e - github.com/cs3org/go-cs3apis v0.0.0-20230606135123-b799d47a6648 + github.com/cs3org/go-cs3apis v0.0.0-20230727093620-0f4399be4543 github.com/dgraph-io/ristretto v0.1.1 github.com/dolthub/go-mysql-server v0.14.0 github.com/eventials/go-tus v0.0.0-20200718001131-45c7ec8f5d59 diff --git a/go.sum b/go.sum index 0f60a32e0a..9dd5cc889a 100644 --- a/go.sum +++ b/go.sum @@ -308,8 +308,8 @@ github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdB github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e h1:tqSPWQeueWTKnJVMJffz4pz0o1WuQxJ28+5x5JgaHD8= github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= -github.com/cs3org/go-cs3apis v0.0.0-20230606135123-b799d47a6648 h1:gBz1JSC2u6o/TkUhWSdJZvacyTsVUzDouegRzvrJye4= -github.com/cs3org/go-cs3apis v0.0.0-20230606135123-b799d47a6648/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/go-cs3apis v0.0.0-20230727093620-0f4399be4543 h1:IFo6dj0XEOIA6i2baRWMC3vd+fAmuIUAVfSf77ZhoQg= +github.com/cs3org/go-cs3apis v0.0.0-20230727093620-0f4399be4543/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/internal/grpc/services/appprovider/appprovider.go b/internal/grpc/services/appprovider/appprovider.go index 7028c4f4d2..42283be1ef 100644 --- a/internal/grpc/services/appprovider/appprovider.go +++ b/internal/grpc/services/appprovider/appprovider.go @@ -21,7 +21,6 @@ package appprovider import ( "context" "encoding/json" - "errors" "fmt" "os" "strconv" @@ -211,7 +210,7 @@ func (s *service) OpenInApp(ctx context.Context, req *providerpb.OpenInAppReques appURL, err := s.provider.GetAppURL(ctx, req.ResourceInfo, req.ViewMode, req.AccessToken, req.Opaque.Map, s.conf.Language) if err != nil { res := &providerpb.OpenInAppResponse{ - Status: status.NewInternal(ctx, errors.New("appprovider: error calling GetAppURL"), err.Error()), + Status: status.NewStatusFromErrType(ctx, "appprovider: error calling GetAppURL", err), } return res, nil } diff --git a/internal/http/services/appprovider/appprovider.go b/internal/http/services/appprovider/appprovider.go index cb0d84698b..24fb6d3dc4 100644 --- a/internal/http/services/appprovider/appprovider.go +++ b/internal/http/services/appprovider/appprovider.go @@ -24,10 +24,11 @@ import ( "net/http" "path" + apppb "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" appregistry "github.com/cs3org/go-cs3apis/cs3/app/registry/v1beta1" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + storagepb "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/datagateway" "github.com/cs3org/reva/pkg/appctx" @@ -158,8 +159,8 @@ func (s *svc) handleNew(w http.ResponseWriter, r *http.Request) { return } - statParentContainerReq := &provider.StatRequest{ - Ref: &provider.Reference{ + statParentContainerReq := &storagepb.StatRequest{ + Ref: &storagepb.Reference{ ResourceId: parentContainerRef, }, } @@ -174,16 +175,16 @@ func (s *svc) handleNew(w http.ResponseWriter, r *http.Request) { return } - if parentContainer.Info.Type != provider.ResourceType_RESOURCE_TYPE_CONTAINER { + if parentContainer.Info.Type != storagepb.ResourceType_RESOURCE_TYPE_CONTAINER { writeError(w, r, appErrorInvalidParameter, "the parent container id does not point to a container", nil) return } - fileRef := &provider.Reference{ + fileRef := &storagepb.Reference{ Path: path.Join(parentContainer.Info.Path, utils.MakeRelativePath(filename)), } - statFileReq := &provider.StatRequest{ + statFileReq := &storagepb.StatRequest{ Ref: fileRef, } statFileRes, err := client.Stat(ctx, statFileReq) @@ -202,7 +203,7 @@ func (s *svc) handleNew(w http.ResponseWriter, r *http.Request) { } // Create empty file via storageprovider - createReq := &provider.InitiateFileUploadRequest{ + createReq := &storagepb.InitiateFileUploadRequest{ Ref: fileRef, Opaque: &typespb.Opaque{ Map: map[string]*typespb.OpaqueEntry{ @@ -269,7 +270,7 @@ func (s *svc) handleNew(w http.ResponseWriter, r *http.Request) { return } - if statRes.Info.Type != provider.ResourceType_RESOURCE_TYPE_FILE { + if statRes.Info.Type != storagepb.ResourceType_RESOURCE_TYPE_FILE { writeError(w, r, appErrorInvalidParameter, "the given file id does not point to a file", nil) return } @@ -339,7 +340,7 @@ func (s *svc) handleOpen(w http.ResponseWriter, r *http.Request) { fileID := r.Form.Get("file_id") - var fileRef provider.Reference + var fileRef storagepb.Reference if fileID == "" { path := r.Form.Get("path") if path == "" { @@ -356,7 +357,7 @@ func (s *svc) handleOpen(w http.ResponseWriter, r *http.Request) { fileRef.ResourceId = resourceID } - statRes, err := client.Stat(ctx, &provider.StatRequest{Ref: &fileRef}) + statRes, err := client.Stat(ctx, &storagepb.StatRequest{Ref: &fileRef}) if err != nil { writeError(w, r, appErrorServerError, "Internal error accessing the file, please try again later", err) return @@ -370,7 +371,7 @@ func (s *svc) handleOpen(w http.ResponseWriter, r *http.Request) { return } - if statRes.Info.Type != provider.ResourceType_RESOURCE_TYPE_FILE { + if statRes.Info.Type != storagepb.ResourceType_RESOURCE_TYPE_FILE { writeError(w, r, appErrorInvalidParameter, "the given file id does not point to a file", nil) return } @@ -408,12 +409,25 @@ func (s *svc) handleOpen(w http.ResponseWriter, r *http.Request) { writeError(w, r, appErrorNotFound, openRes.Status.Message, nil) return } + if openRes.Status.Code == rpc.Code_CODE_ALREADY_EXISTS { + writeError(w, r, appErrorAlreadyExists, openRes.Status.Message, nil) + return + } writeError(w, r, appErrorServerError, openRes.Status.Message, status.NewErrorFromCode(openRes.Status.Code, "error calling OpenInApp")) return } - js, err := json.Marshal(openRes.AppUrl) + // recreate the structure to be able to marshal the AppUrl.Target as a string + js, err := json.Marshal( + map[string]interface{}{ + "app_url": openRes.AppUrl.AppUrl, + "method": openRes.AppUrl.Method, + "form_parameters": openRes.AppUrl.FormParameters, + "headers": openRes.AppUrl.Headers, + "target": appTargetToString(openRes.AppUrl.Target), + }, + ) if err != nil { writeError(w, r, appErrorServerError, "Internal error with JSON payload", errors.Wrap(err, "error marshalling JSON response")) @@ -421,7 +435,7 @@ func (s *svc) handleOpen(w http.ResponseWriter, r *http.Request) { } log := appctx.GetLogger(ctx) - log.Info().Str("url", openRes.AppUrl.AppUrl).Interface("resource", fileRef).Msg("returning app URL for file") + log.Info().Interface("resource", fileRef).Str("url", openRes.AppUrl.AppUrl).Str("method", openRes.AppUrl.Method).Interface("target", openRes.AppUrl.Target).Msg("returning app URL for file") w.Header().Set("Content-Type", "application/json") if _, err = w.Write(js); err != nil { @@ -438,7 +452,7 @@ func (s *svc) handleNotify(w http.ResponseWriter, r *http.Request) { } fileID := r.Form.Get("file_id") - var fileRef provider.Reference + var fileRef storagepb.Reference if fileID == "" { path := r.Form.Get("path") if path == "" { @@ -484,7 +498,7 @@ func filterAppsByUserAgent(mimeTypes []*appregistry.MimeTypeInfo, userAgent stri return res } -func resolveViewMode(res *provider.ResourceInfo, vm string) gateway.OpenInAppRequest_ViewMode { +func resolveViewMode(res *storagepb.ResourceInfo, vm string) gateway.OpenInAppRequest_ViewMode { var viewMode gateway.OpenInAppRequest_ViewMode if vm != "" { viewMode = utils.GetViewMode(vm) @@ -508,3 +522,14 @@ func resolveViewMode(res *provider.ResourceInfo, vm string) gateway.OpenInAppReq } return viewMode } + +func appTargetToString(t apppb.Target) string { + switch t { + case apppb.Target_TARGET_IFRAME: + return "iframe" + case apppb.Target_TARGET_BLANK: + return "blank" + default: + return "invalid" + } +} diff --git a/internal/http/services/archiver/handler.go b/internal/http/services/archiver/handler.go index 4ca94eb230..2f43e2b1e1 100644 --- a/internal/http/services/archiver/handler.go +++ b/internal/http/services/archiver/handler.go @@ -23,7 +23,9 @@ import ( "errors" "fmt" "net/http" + "path/filepath" "regexp" + "strings" "time" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" @@ -102,7 +104,7 @@ func New(ctx context.Context, conf map[string]interface{}) (global.Service, erro func (c *Config) ApplyDefaults() { if c.Prefix == "" { - c.Prefix = "download_archive" + c.Prefix = "archiver" } if c.Name == "" { @@ -229,13 +231,23 @@ func (s *svc) Handler() http.Handler { return } - userAgent := ua.Parse(r.Header.Get("User-Agent")) + archType := v.Get("arch_type") // optional, either "tar" or "zip" + if archType == "" || archType != "tar" && archType != "zip" { + // in case of missing or bogus arch_type, detect it via user-agent + userAgent := ua.Parse(r.Header.Get("User-Agent")) + if userAgent.OS == ua.Windows { + archType = "zip" + } else { + archType = "tar" + } + } - archName := s.config.Name - if userAgent.OS == ua.Windows { - archName += ".zip" + var archName string + if len(files) == 1 { + archName = strings.TrimSuffix(filepath.Base(files[0]), filepath.Ext(files[0])) + "." + archType } else { - archName += ".tar" + // TODO(lopresti) we may want to generate a meaningful name out of the list + archName = s.config.Name + "." + archType } log.Debug().Msg("Requested the following files/folders to archive: " + render.Render(files)) @@ -244,7 +256,7 @@ func (s *svc) Handler() http.Handler { rw.Header().Set("Content-Transfer-Encoding", "binary") // create the archive - if userAgent.OS == ua.Windows { + if archType == "zip" { err = arch.CreateZip(ctx, rw) } else { err = arch.CreateTar(ctx, rw) diff --git a/pkg/app/provider/demo/demo.go b/pkg/app/provider/demo/demo.go index c17403fe12..7ce885ee5e 100644 --- a/pkg/app/provider/demo/demo.go +++ b/pkg/app/provider/demo/demo.go @@ -45,12 +45,16 @@ func (p *demoProvider) GetAppURL(ctx context.Context, resource *provider.Resourc return &appprovider.OpenInAppURL{ AppUrl: url, Method: http.MethodGet, + Target: appprovider.Target_TARGET_IFRAME, // alternatively, appprovider.Target_TARGET_BLANK }, nil } func (p *demoProvider) GetAppProviderInfo(ctx context.Context) (*appregistry.ProviderInfo, error) { return &appregistry.ProviderInfo{ - Name: "demo-app", + Name: "demo-app", + Description: "A dummy app provider", + MimeTypes: []string{}, + Action: "Demo open", }, nil } diff --git a/pkg/app/provider/wopi/wopi.go b/pkg/app/provider/wopi/wopi.go index 0ecdb42073..690711e14d 100644 --- a/pkg/app/provider/wopi/wopi.go +++ b/pkg/app/provider/wopi/wopi.go @@ -336,6 +336,7 @@ func (p *wopiProvider) GetAppURL(ctx context.Context, resource *provider.Resourc AppUrl: appFullURL, Method: method, FormParameters: formParams, + Target: appprovider.Target_TARGET_IFRAME, }, nil } diff --git a/pkg/errtypes/errtypes.go b/pkg/errtypes/errtypes.go index 9af7d2b535..bf54dd00c0 100644 --- a/pkg/errtypes/errtypes.go +++ b/pkg/errtypes/errtypes.go @@ -22,7 +22,7 @@ // and error is a reserved word :) package errtypes -// NotFound is the error to use when a something is not found. +// NotFound is the error to use when something is not found. type NotFound string func (e NotFound) Error() string { return "error: not found: " + string(e) } @@ -46,7 +46,7 @@ func (e PermissionDenied) Error() string { return "error: permission denied: " + // IsPermissionDenied implements the IsPermissionDenied interface. func (e PermissionDenied) IsPermissionDenied() {} -// AlreadyExists is the error to use when a resource something is not found. +// AlreadyExists is the error to use when a resource already exists and can't be overwritten. type AlreadyExists string func (e AlreadyExists) Error() string { return "error: already exists: " + string(e) } @@ -54,7 +54,7 @@ func (e AlreadyExists) Error() string { return "error: already exists: " + strin // IsAlreadyExists implements the IsAlreadyExists interface. func (e AlreadyExists) IsAlreadyExists() {} -// UserRequired represents an error when a resource is not found. +// UserRequired represents an error when a user could not be found from the context. type UserRequired string func (e UserRequired) Error() string { return "error: user required: " + string(e) } diff --git a/pkg/rgrpc/status/status.go b/pkg/rgrpc/status/status.go index 80197e0552..27c285bcbc 100644 --- a/pkg/rgrpc/status/status.go +++ b/pkg/rgrpc/status/status.go @@ -181,6 +181,8 @@ func NewStatusFromErrType(ctx context.Context, msg string, err error) *rpc.Statu return NewUnimplemented(ctx, err, "gateway: "+msg+":"+err.Error()) case errtypes.BadRequest: return NewInvalidArg(ctx, "gateway: "+msg+":"+err.Error()) + case errtypes.AlreadyExists: + return NewAlreadyExists(ctx, err, "gateway: "+msg+":"+err.Error()) } // map GRPC status codes coming from the auth middleware From 9160d1628ca3c9bb9bc0d6563f5f9f06f77a8b1e Mon Sep 17 00:00:00 2001 From: nirajacharya2 <122071597+nirajacharya2@users.noreply.github.com> Date: Thu, 10 Aug 2023 11:16:28 +0545 Subject: [PATCH 48/48] bumped ocis commit id reva master (#4105) --- .drone.env | 2 +- tests/ocis | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.env b/.drone.env index 563780c1a8..90f64b2bad 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=43ee37d02ed652b89e88c03c6bbeb5c89a62ce16 +APITESTS_COMMITID=cbe945af4a97b1bafbc903ab5ac2905258f19a62 APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/tests/ocis b/tests/ocis index 43ee37d02e..cbe945af4a 160000 --- a/tests/ocis +++ b/tests/ocis @@ -1 +1 @@ -Subproject commit 43ee37d02ed652b89e88c03c6bbeb5c89a62ce16 +Subproject commit cbe945af4a97b1bafbc903ab5ac2905258f19a62