diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..72ac82a32 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.git/ +.github/ +log/ +misc/ +dev/ +_build/ +rel/pkg/ +rel/riak-cs/ diff --git a/.github/workflows/erlang.yml b/.github/workflows/erlang.yml new file mode 100644 index 000000000..d7c222f62 --- /dev/null +++ b/.github/workflows/erlang.yml @@ -0,0 +1,33 @@ +name: Erlang CI + +on: + push: + branches: [ develop ] + pull_request: + branches: [ develop ] + + +jobs: + + build: + + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + otp: + - "22" + - "24" + - "25" + - "26" + + container: + image: erlang:${{ matrix.otp }} + + steps: + - uses: actions/checkout@v3 + - name: Compile + run: make compile + - name: Run checks + run: make check diff --git a/.gitignore b/.gitignore index e505080ec..5576dd63e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,21 +1,6 @@ -.eunit/* -deps/* +_build/ +_checkouts/ +log/ rel/riak-cs -rel/vars/dev*_vars.config -ebin/* -riak_test/ebin/* -doc dev/ -/package/ -log/ -*~ -current_counterexample.eqc -.client_test/ -pkg.vars.config -client_tests/python/ceph_tests/s3-tests -riak-cs.png -.local_dialyzer_plt -dialyzer.ignore-warnings -dialyzer_warnings -dialyzer_unhandled_warnings -riak_test/src/downgrade_bitcask.erl +*-riak-cs-debug.tar.gz diff --git a/.plugins/rebar_test_plugin.erl b/.plugins/rebar_test_plugin.erl deleted file mode 100644 index 05116bbc5..000000000 --- a/.plugins/rebar_test_plugin.erl +++ /dev/null @@ -1,123 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(rebar_test_plugin). - --export([ - client_test_clean/2, - client_test_compile/2, - client_test_run/2, - riak_test_clean/2, - riak_test_compile/2 -]). - -%% =================================================================== -%% Public API -%% =================================================================== -client_test_clean(Config, AppFile) -> - case should_i_run(Config) of - false -> ok; - _ -> test_clean(client_test, Config, AppFile) - end. - -client_test_compile(Config, AppFile) -> - case should_i_run(Config) of - false -> ok; - _ -> test_compile(client_test, Config, AppFile) - end. - -client_test_run(Config, AppFile) -> - case should_i_run(Config) of - false -> ok; - _ -> test_run(client_test, Config, AppFile) - end. - -riak_test_clean(Config, AppFile) -> - case should_i_run(Config) of - false -> ok; - _ -> test_clean(riak_test, Config, AppFile) - end. - -riak_test_compile(Config, AppFile) -> - case should_i_run(Config) of - false -> ok; - _ -> test_compile(riak_test, Config, AppFile) - end. - -%% =================================================================== -%% Private Functions - pronounced Funk-tee-owns, not funk-ee-towns -%% =================================================================== -should_i_run(Config) -> - rebar_utils:processing_base_dir(Config). - -option(TestType, Key, Config) -> - case proplists:get_value(TestType, element(3, Config), not_configured) of - not_configured -> {error, not_configured}; - TestConfig -> - proplists:get_value(Key, TestConfig, {error, not_set}) - end. - -test_clean(TestType, Config, _AppFile) -> - case option(TestType, test_output, Config) of - {error, not_set} -> - io:format("No test_output directory set, check your rebar.config"); - TestOutputDir -> - io:format("Removing test_output dir ~s~n", [TestOutputDir]), - rebar_file_utils:rm_rf(TestOutputDir) - end, - ok. - -test_compile(TestType, Config, AppFile) -> - CompilationConfig = compilation_config(TestType, Config), - OutputDir = option(TestType, test_output, Config), - rebar_erlc_compiler:compile(CompilationConfig, AppFile), - ok. - -test_run(TestType, Config, _AppFile) -> - OutputDir = option(TestType, test_output, Config), - Cwd = rebar_utils:get_cwd(), - ok = file:set_cwd([Cwd, $/, OutputDir]), - EunitResult = (catch eunit:test("./")), - %% Return to original working dir - ok = file:set_cwd(Cwd), - EunitResult. - - -compilation_config(TestType, Conf) -> - C1 = rebar_config:set(Conf, TestType, undefined), - C2 = rebar_config:set(C1, plugins, undefined), - ErlOpts = rebar_utils:erl_opts(Conf), - ErlOpts1 = proplists:delete(src_dirs, ErlOpts), - ErlOpts2 = - case eqc_present() of - true -> - [{d, 'TEST'}, {d, 'EQC'}, {outdir, option(TestType, test_output, Conf)}, {src_dirs, option(TestType, test_paths, Conf)} | ErlOpts1]; - false -> - [{d, 'TEST'}, {outdir, option(TestType, test_output, Conf)}, {src_dirs, option(TestType, test_paths, Conf)} | ErlOpts1] - end, - rebar_config:set(C2, erl_opts, ErlOpts2). - -eqc_present() -> - case catch eqc:version() of - {'EXIT', {undef, _}} -> - false; - _ -> - true - end. diff --git a/CONTRIBUTORS b/CONTRIBUTORS new file mode 100644 index 000000000..697328d84 --- /dev/null +++ b/CONTRIBUTORS @@ -0,0 +1,57 @@ +Riak CS was created at Basho Tecnologies, Inc. by various contributors +employed by that company during 2007-15. Committers' names, as +extracted from git history, are as follows: + + Alex Wilson + Andrew J. Stone + Andrew Thompson + Andy Gross + Bryan Fink + Charlie Voiselle + Christian Dahlqvist + Christopher Meiklejohn + Christopher Molozian + Dave Parfitt + Dave Smith + Drew Pirrone-Brusse + Gabriel Nicolas Avellaneda + Greg Cymbalski + Hector Castro + Ian Plosker + Jared Morrow + Jimmy Mullen + Joe Caswell + Joe DeVivo + John Daily + Jon Anderson + Jon Glick + Jordan West + Junya Namai + Justin Pease + Kazuhiro Suzuki + Kelly McLaughlin + Kenji Rikitake + Lauren Rother + Luc Perkins + Macneil Shonle + Magnus Kessler + Marcel Neuhausler + Mark Allen + Pavlo Baron + Paul Hagan + Reid Draper + Scott Lystig Fritchie + Sean Cribbs + Seema Jethani + Shunichi Shinohara + Ted Burghart + Uenishi Kota + mexicat + +Since the days of Basho, development and maintenance work has been +done with contributions from: + + Andriy Zavada + Martin Cox + Nicholas Adams + Peter Clark diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..1e4b72100 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +# To override the default Erlang version used in the build, use: +# $ docker build --build-arg erlang_version=22 ... + +ARG erlang_version="25" +FROM erlang:${erlang_version} AS compile-image + +EXPOSE 8080 8085 + +WORKDIR /usr/src/riak_cs +COPY . /usr/src/riak_cs + +# When running in a docker container, ideally we would want our app to +# be configurable via environment variables (option --env-file to +# docker run). For that reason, We use a pared-down, cuttlefish-less +# rebar.config. Configuration from environment now becomes possible, +# via rebar's own method of generating sys.config from +# /sys.config.src. +RUN make rel-docker + +FROM debian:latest AS runtime-image + +COPY --from=compile-image /usr/src/riak_cs/rel/riak-cs /opt/riak-cs + +ENTRYPOINT [ "/opt/riak-cs/bin/riak-cs" "foreground" ] diff --git a/Makefile b/Makefile index 74d95df8a..b8deb7990 100644 --- a/Makefile +++ b/Makefile @@ -1,123 +1,80 @@ -REPO ?= riak_cs -PKG_REVISION ?= $(shell git describe --tags) -PKG_VERSION ?= $(shell git describe --tags | tr - .) -PKG_ID = riak-cs-$(PKG_VERSION) +REPO := riak_cs +HEAD_REVISION := $(shell git describe --tags --exact-match HEAD 2>/dev/null) +PKG_REVISION := $(shell git describe --tags 2>/dev/null) +PKG_VERSION := $(shell git describe --tags | tr - .) +REPO_TAG := $(PKG_REVISION) +OTP_VER = $(shell erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().' -noshell) +PKG_ID := "$(REPO_TAG)-OTP$(OTP_VER)" PKG_BUILD = 1 BASE_DIR = $(shell pwd) -ERLANG_BIN = $(shell dirname $(shell which erl)) -REBAR ?= $(BASE_DIR)/rebar -OVERLAY_VARS ?= -CS_HTTP_PORT ?= 8080 +ERLANG_BIN = $(shell dirname $(shell which erl 2>/dev/null) 2>/dev/null) +REBAR ?= $(BASE_DIR)/rebar3 PULSE_TESTS = riak_cs_get_fsm_pulse -.PHONY: rel stagedevrel deps test depgraph graphviz all compile compile-src +.PHONY: all rel compile clean distclean check stagedevrel test depgraph graphviz package pkg-clean sbom all: compile -compile: compile-riak-test +compile: + @$(REBAR) compile -compile-src: deps - @(./rebar compile) - -compile-client-test: all - @./rebar client_test_compile - -bitcask-downgrade-script: riak_test/src/downgrade_bitcask.erl - -## KLUDGE, as downgrade script is not included in the release. -riak_test/src/downgrade_bitcask.erl: - @wget --no-check-certificate https://raw.githubusercontent.com/basho/bitcask/develop/priv/scripts/downgrade_bitcask.erl \ - -O riak_test/src/downgrade_bitcask.erl - -compile-riak-test: compile-src bitcask-downgrade-script - @./rebar skip_deps=true riak_test_compile - ## There are some Riak CS internal modules that our riak_test - ## test would like to use. But riak_test doesn't have a nice - ## way of adding the -pz argument + code path that we need. - ## So we'll copy the BEAM files to a place that riak_test is - ## already using. - cp ebin/riak_cs_wm_utils.beam riak_test/ebin - cp ebin/twop_set.beam riak_test/ebin +clean: + @$(REBAR) clean -clean-client-test: - @./rebar client_test_clean +distclean: devclean relclean + @$(REBAR) clean -a -clean-riak-test: - @./rebar riak_test_clean skip_deps=true +check: + @$(REBAR) eunit + @$(REBAR) xref + @$(REBAR) dialyzer -deps: - @./rebar get-deps +sbom: + @$(REBAR) sbom ## -## Lock Targets +## Release targets ## -## see https://github.com/seth/rebar_lock_deps_plugin -lock: deps compile - ./rebar lock-deps - -locked-all: locked-deps compile - -locked-deps: - @echo "Using rebar.config.lock file to fetch dependencies" - ./rebar -C rebar.config.lock get-deps - -clean: - @./rebar clean - -distclean: clean - @./rebar delete-deps - @rm -rf $(PKG_ID).tar.gz - -## Create a dependency graph png -depgraph: graphviz - @echo "Note: If you have nothing in deps/ this might be boring" - @echo "Creating dependency graph..." - @misc/mapdeps.erl | dot -Tpng -oriak-cs.png - @echo "Dependency graph created as riak-cs.png" -graphviz: - $(if $(shell which dot),,$(error "To make the depgraph, you need graphviz installed")) +rel: compile + @$(REBAR) as rel release + @cp -a _build/rel/rel/riak-cs rel/ -pulse: all - @rm -rf $(BASE_DIR)/.eunit - @./rebar -D PULSE eunit skip_deps=true suites=$(PULSE_TESTS) - -test-client: test-clojure test-boto test-ceph test-erlang test-ruby test-php test-go +rel-rpm: compile + @$(REBAR) as rpm release + @cp -a _build/rpm/rel/riak-cs rel/ -test-python: - @cd client_tests/python/ && make CS_HTTP_PORT=$(CS_HTTP_PORT) +rel-deb: compile + @$(REBAR) as deb release + @cp -a _build/deb/rel/riak-cs rel/ -test-boto: - @cd client_tests/python/ && make boto_tests CS_HTTP_PORT=$(CS_HTTP_PORT) +rel-fbsdng: compile relclean + @$(REBAR) as fbsdng release + @cp -a _build/fbsdng/rel/riak-cs rel/ -test-ceph: - @cd client_tests/python/ && make ceph_tests CS_HTTP_PORT=$(CS_HTTP_PORT) +rel-osx: compile relclean + @$(REBAR) as osx release + @cp -a _build/osx/rel/riak-cs rel/ -test-ruby: - @bundle --gemfile client_tests/ruby/Gemfile --path vendor - @cd client_tests/ruby && bundle exec rake spec +rel-alpine: compile relclean + @$(REBAR) as alpine release + @(cd _build/alpine/rel/riak-cs/usr/bin && mv riak-cs.nosu riak-cs) + @cp -a _build/alpine/rel/riak-cs rel/ -test-erlang: compile-client-test - @./rebar skip_deps=true client_test_run +rel-docker: compile relclean + @REBAR_CONFIG=rebar.docker.config $(REBAR) release + @cp -a _build/default/rel/riak-cs rel/ -test-clojure: - @command -v lein >/dev/null 2>&1 || { echo >&2 "I require lein but it's not installed. \ - Please read client_tests/clojure/clj-s3/README."; exit 1; } - @cd client_tests/clojure/clj-s3 && lein do deps, midje - -test-php: - @cd client_tests/php && make - -test-go: - @cd client_tests/go && make +relclean: + rm -rf _build/default/rel rel/riak-cs ## -## Release targets +## test targets ## -rel: deps compile - @./rebar generate skip_deps=true $(OVERLAY_VARS) +test: + $(REBAR) eunit + @rm -f log@nonode* data@nonode* + $(REBAR) dialyzer -relclean: - rm -rf rel/riak-cs ## ## Developer targets @@ -134,70 +91,80 @@ relclean: DEVNODES ?= 8 # 'seq' is not available on all *BSD, so using an alternate in awk -SEQ = $(shell awk 'BEGIN { for (i = 1; i < '$(DEVNODES)'; i++) printf("%i ", i); print i ;exit(0);}') +SEQ = $(shell seq $(DEVNODES)) $(eval stagedevrel : $(foreach n,$(SEQ),stagedev$(n))) $(eval devrel : $(foreach n,$(SEQ),dev$(n))) dev% : all - mkdir -p dev - rel/gen_dev $@ rel/vars/dev_vars.config.src rel/vars/$@_vars.config - (cd rel && ../rebar generate target_dir=../dev/$@ overlay_vars=vars/$@_vars.config) + rel/gen_dev dev$* rel/vars/dev_vars.config.src rel/vars/$*_vars.config + $(REBAR) as dev release -o dev/dev$* --overlay_vars rel/vars/$*_vars.config -stagedev% : dev% - $(foreach app,$(wildcard apps/*), rm -rf dev/$^/lib/$(shell basename $(app))* && ln -sf $(abspath $(app)) dev/$^/lib;) - $(foreach dep,$(wildcard deps/*), rm -rf dev/$^/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) dev/$^/lib;) +stagedev% : all + rel/gen_dev dev$* rel/vars/dev_vars.config.src rel/vars/$*_vars.config + $(REBAR) as dev release -o dev/dev$* --overlay_vars rel/vars/$*_vars.config devclean: clean rm -rf dev stage : rel - $(foreach app,$(wildcard apps/*), rm -rf rel/riak-cs/lib/$(shell basename $(app))* && ln -sf $(abspath $(app)) rel/riak-cs/lib;) - $(foreach dep,$(wildcard deps/*), rm -rf rel/riak-cs/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) rel/riak-cs/lib;) + $(foreach app,$(wildcard apps/*), rm -rf rel/riak-cs/lib/$(shell basename $(app))* && ln -sf $(abspath $(app)) rel/riak-cs/lib;) + $(foreach dep,$(wildcard _build/default/lib/*), rm -rf rel/riak-cs/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) rel/riak-cs/lib;) + + +## Create a dependency graph png +depgraph: graphviz + @echo "Note: If you have nothing in deps/ this might be boring" + @echo "Creating dependency graph..." + @misc/mapdeps.erl | dot -Tpng -oriak-cs.png + @echo "Dependency graph created as riak-cs.png" +graphviz: + $(if $(shell which dot),,$(error "To make the depgraph, you need graphviz installed")) + +pulse: all + @rm -rf $(BASE_DIR)/.eunit + @ERLOPTS="-D PULSE" $(REBAR) eunit --module=$(PULSE_TESTS) + + + +## +## Version and naming variables for distribution and packaging +## +# Split off repo name +# Changes to 1.0.3 or 1.1.0pre1-27-g1170096 from example above +REVISION = $(shell echo $(REPO_TAG) | sed -e 's/^$(REPO)-//') -DIALYZER_APPS = kernel stdlib sasl erts ssl tools os_mon runtime_tools crypto inets \ - webtool eunit syntax_tools compiler -PLT ?= $(HOME)/.riak-cs_dialyzer_plt +# Primary version identifier, strip off commmit information +# Changes to 1.0.3 or 1.1.0pre1 from example above +MAJOR_VERSION ?= $(shell echo $(REVISION) | sed -e 's/\([0-9.]*\)-.*/\1/') ## ## Packaging targets ## + +# Yes another variable, this one is repo-- +PKG_VERSION = $(shell echo $(PKG_ID) | sed -e 's/^$(REPO)-//') + +package: + $(REBAR) get-deps + mkdir -p rel/pkg/out/riak_cs-$(PKG_ID) + git archive --format=tar HEAD | gzip >rel/pkg/out/$(PKG_ID).tar.gz + $(MAKE) -f rel/pkg/Makefile + +packageclean: + rm -rf rel/pkg/out/* + + .PHONY: package export PKG_VERSION PKG_ID PKG_BUILD BASE_DIR ERLANG_BIN REBAR OVERLAY_VARS RELEASE -## Do not export RIAK_CS_EE_DEPS unless it is set, since even an empty -## variable will affect the build and 'export' by default makes it empty -## if it is unset -BUILD_EE = $(shell test -n "$${RIAK_CS_EE_DEPS+x}" && echo "true" || echo "false") -ifeq ($(BUILD_EE),true) -export RIAK_CS_EE_DEPS=true -endif - - -package.src: deps - mkdir -p package - rm -rf package/$(PKG_ID) - git archive --format=tar --prefix=$(PKG_ID)/ $(PKG_REVISION)| (cd package && tar -xf -) - cp rebar.config.script package/$(PKG_ID) - make -C package/$(PKG_ID) deps - mkdir -p package/$(PKG_ID)/priv - git --git-dir=.git describe --tags >package/$(PKG_ID)/priv/vsn.git - for dep in package/$(PKG_ID)/deps/*; do \ - echo "Processing dep: $${dep}"; \ - mkdir -p $${dep}/priv; \ - git --git-dir=$${dep}/.git describe --tags >$${dep}/priv/vsn.git; \ - done - find package/$(PKG_ID) -depth -name ".git" -exec rm -rf {} \; - tar -C package -czf package/$(PKG_ID).tar.gz $(PKG_ID) - -dist: package.src - cp package/$(PKG_ID).tar.gz . - -package: package.src - make -C package -f $(PKG_ID)/deps/node_package/Makefile - -pkgclean: distclean - rm -rf package - -include tools.mk +# Package up a devrel to save time later rebuilding it +pkg-devrel: devrel + echo -n $(PKG_REVISION) > VERSION + tar -czf $(PKG_ID)-devrel.tar.gz dev/ VERSION + rm -rf VERSION + +pkg-rel: rel + tar -czf $(PKG_ID)-rel.tar.gz -C rel/ . diff --git a/README.md b/README.md index e8df40f87..c216cdc6d 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -Welcome to Riak CS. +Welcome to Riak CS or Riak S2. # Overview -Riak CS is an object storage system built on top of Riak. It -facilitates storing large objects in Riak and presents an +Riak CS is an object storage system built on top of Riak KV. It +facilitates storing large objects in Riak KV and presents an S3-compatible interface. It also provides multi-tenancy features such as user accounts, authentication, access control mechanisms, and per account usage reporting. @@ -13,6 +13,82 @@ using Riak CS. For more information, browse the following files: - README: this file - LICENSE: the license under which Riak CS is released +- RELEASE-NOTES: new features and changes for each release. + +# Operation + +The full suite of Riak CS includes: + * one or more nodes of Riak CS proper; + * prior to version 3.1.0, a single instance of Stanchion; + * a Riak cluster; + * optionally, a node running Riak CS Control. + +## Configuation + +These components need to be properly configured, at a minimum: + + * Riak nodes must have these items in `riak.conf`: + ``` + backend = multi + buckets.default.allow_mult = true + buckets.default.merge_strategy = 2 + ``` + and, in `advanced.config`: + ``` + {riak_kv, [ + {multi_backend, + [{be_default, riak_kv_eleveldb_backend, + [{max_open_files, 20}]}, + {be_blocks, riak_kv_bitcask_backend, + []}]}, + {multi_backend_default, be_default}, + {multi_backend_prefix_list, [{<<"0b:">>, be_blocks}]}, + {storage_backend, riak_kv_multi_backend} + ]} + ``` + There is a convenience script, misc/prepare-riak-for-cs, which will + apply these changes to a factory `riak.conf`. + + * `riak-cs.conf` and `stanchion.conf` must have the actual address:port + of nodes (same or different) of the underlying riak cluster; + + * likewise, in `riak-cs.conf`, you should set the address:port of the + stanchion node. + +## Starting order + +Riak must be started before Riak CS and Stanchion. Also, Stanchion +must be running before Riak CS can begin to serve user requests. + +## Users and administrative access + +On a fresh install, in order to create an admin user, you will need +to start Riak CS with authorization disabled. This can be done with +this block in `riak-cs/advanced.config`: +``` +{riak_cs, [{admin_auth_enabled, false}]} +``` +At this point, you should block any traffic from your _real_ users. + +Now you can create a user, which will become your admin user, with a +`POST` to `$RIAK_CS_ADDRESS:$RIAK_CS_PORT/riak-cs/user`, like so: +``` +curl -X POST 172.17.0.2:8080/riak-cs/user \ + -H 'Content-Type: application/json' \ + --data '{"name": "fafa", "email": "fa@keke.org"}' +``` + +Copy the `key_id` from the returned json and set `admin.key` +option to its value in `riak-cs.conf` and `stanchion.conf`. Then disable +the auth bypass by changing `admin_auth_enabled` setting to `true`, +and restart Riak CS and Stanchion. Your Riak CS suite is ready +for production use; now external incoming requests can be unblocked. + +All the configuration steps and procedures are scripted in Riak CS as +a [Docker service](https://github.com/TI-Tokyo/riak_cs_service_bundle), +which you can, incidentally, use not only to acquaint yourself with the +operations, but also as a quick and minimal, yet fully functional, +local installation of a full Riak CS suite. # Compatible clients and libraries @@ -25,14 +101,19 @@ The following is a sample of the clients and libraries that have been tested with Riak CS: - [s3cmd](https://github.com/s3tools/s3cmd) +- [boto3](https://github.com/boto/boto3) +- [erlcloud](https://github.com/basho/erlcloud) (an old fork; current + upstream version not tested) +- [AWS Ruby SDK](http://aws.amazon.com/sdkforruby/) + +The following clients have been tested with 2.x and may or may not +work with 3.0: + - [s3curl](http://aws.amazon.com/code/128) -- [boto](https://github.com/boto/boto) -- [erlcloud](https://github.com/basho/erlcloud) -- [AWS Java SDK](http://aws.amazon.com/sdk-for-java/) -- [AWS Ruby SDK](http://aws.amazon.com/sdk-for-ruby/) +- [AWS Java SDK](http://aws.amazon.com/sdkforjava/) - [Fog](http://fog.io/) -## Administrative interface +# Administrative interface The Riak CS administrative interface is accessible via HTTP just like the object storage interface. All functions can be accessed under the @@ -62,19 +143,14 @@ should be used with /caution/ and it is only recommended when the administrative requests are handled by a private interface that only system administrators may access. -## Repo organization +# Repo organization Riak CS uses the [git-flow](http://nvie.com/posts/a-successful-git-branching-model/) branching model and we have the default -branch on this repo set to `develop` so that when you browse here on +branch on this repo set to `develop-3.0` so that when you browse here on GitHub you always see the latest changes and to serve as a reminder when opening pull requests for new features. -All releases are tagged off of the `master` branch so to access a -specific release from the git repo simply checkout the appropriate -tag. *e.g.* For the *1.3.0* release use the following command: -`git checkout 1.3.0`. - # Where to find more Below, you'll find a direct link to a guide to quickly getting started @@ -128,8 +204,10 @@ The unit tests for each subproject can be run with `make` or make test ``` +or, if you only want eunit tests: + ``` -./rebar skip_deps=true eunit +./rebar3 eunit ``` ### Running dialyzer @@ -150,7 +228,7 @@ Dialyzer can be run with `make`: make dialyzer ``` -### `riak_test` modules +### Integration tests -Riak CS has a number of integration and regression tests in the -`/riak_test/` subdirectory. Please refer `README.md` under the directory. +Riak CS has a number of integration and regression tests, in a +[separate project](https://github.com/TI-Tokyo/riak_cs_test). diff --git a/RELEASE-NOTES.ja.md b/RELEASE-NOTES.ja.md index 873729cb7..ec78bb7b0 100644 --- a/RELEASE-NOTES.ja.md +++ b/RELEASE-NOTES.ja.md @@ -686,6 +686,656 @@ Stanchion 1.5.x へダウングレードするには次の手順を各ノード ダウングレードはサポートされていません。これによりダウングレードにはデータファイルの 変換スクリプトが必要です。こちらもご覧ください。[2.0 ダウングレードノート][downgrade_notes]. + +# Riak CS 1.5.4 リリースノート + +## 修正されたバグ + +- バックプレッシャーのスリープ発動後に取得済みのRiakオブジェクトを破棄 + [riak_cs/#1041](https://github.com/basho/riak_cs/pull/1041)。 + これは次の場合に起こり得る Sibling の増加を防ぎます。 + (a) 高い同時実行アップロードによるバックプレッシャーが起動しており、かつ + (b) バックプレッシャーによるスリープ中にアップロードがインターリーブするとき。 + この問題はマルチパートアップロードへは影響しません。 +- 不要なURLデコードを引き起こす S3 API における不正確なURLパスの rewrite 処理。 + [riak_cs/#1040](https://github.com/basho/riak_cs/pull/1040). + URLエンコード・デコードが不正確な事により、 + `%[0-9a-fA-F][0-9a-fA-F]` (正規表現) や `+` を含むオブジェクト名は + 誤ったデコードが実施されていました。この結果、前者は異なるバイナリへ、 + 後者は ` ` (空白) へと置き換わり、どちらの場合でも暗黙的にデータを + 上書きする可能性があります。例えば後者のケースでは、 キー に `+` を含む + オブジェクト(例：`foo+bar`) は、`+` が ` ` に置き換わっただけの、 + ほぼ同じ名前のオブジェクト(`foo bar`)に上書きされます。逆も起こり得ます。 + この修正は次の問題にも関連します： + [riak_cs/#910](https://github.com/basho/riak_cs/pull/910) + [riak_cs/#977](https://github.com/basho/riak_cs/pull/977). + +## アップグレード時の注意 + +Riak CS 1.5.4 へアップグレードすると、デフォルト設定のままでは、 +キーに `%[0-9a-fA-F][0-9a-fA-F]` や `+` を含むオブジェクトは見えなくなり、 +違うオブジェクト名で見えるようになります。 +前者は余分にデコードされたオブジェクトとして参照され、 +後者は ` ` を `+` で置き換えたキー(例： `foo bar`)で参照されるようになります。 + +下記の表はアップグレードの前後で +`%[0-9a-fA-F][0-9a-fA-F]` を含むURLがどう振る舞うかの例です。 + + + | アップグレード前 | アップグレード後 | +:------------|:-------------------------|:------------------| + 書き込み時 | `a%2Fkey` | - | + 読み込み時 | `a%2Fkey` または `a/key` | `a/key` | +リスト表示時 | `a/key` | `a/key` | + +`+` や ` ` を含むオブジェクトのアップグレード前後の例： + + | アップグレード前 | アップグレード後 | +:------------|:-------------------------|:------------------| + 書き込み時 | `a+key` | - | + 読み込み時 | `a+key` または `a key` | `a key` | +リスト表示時 | `a key` | `a key` | + + | アップグレード前 | アップグレード後 | +:------------|:-------------------------|:------------------| + 書き込み時 | `a key` | - | + 読み込み時 | `a+key` または `a key` | `a key` | +リスト表示時 | `a key` | `a key` | + +またこの修正によりアクセスログのフォーマットも単一のURLエンコードから二重エンコードスタイルへ変わります。 +下記は変更前の例です： + +``` +127.0.0.1 - - [07/Jan/2015:08:27:07 +0000] "PUT /buckets/test/objects/path1%2Fpath2%2Fte%2Bst.txt HTTP/1.1" 200 0 "" "" +``` + +そしてこちらが新しいフォーマットです。 + +``` +127.0.0.1 - - [07/Jan/2015:08:27:07 +0000] "PUT /buckets/test/objects/path1%2Fpath2%2Fte%252Bst.txt HTTP/1.1" 200 0 "" "" +``` + +この例から分かるように、オブジェクトのパスが `path1%2Fpath2%2Fte%252Bst.txt` +から `path1%2Fpath2%2Fte%2Bst.txt` へ変わることに注意して下さい。 + +もし Riak CS を利用するアプリケーション側の都合で +以前の挙動のままにしたい場合、アップグレード時に +Riak CSの設定を変更すればそれが可能です。 +この場合、`rewrite_module` 設定を下記のように変更してください： + +```erlang +{riak_cs, [ + %% Other settings + {rewrite_module, riak_cs_s3_rewrite_legacy}, + %% Other settings +]} +``` + +**注意**: 以前の挙動は技術的に不適切であり、 +前述したように暗黙的なデータの上書きが起こり得ます。 +注意の上でご利用下さい。 + +# Riak CS 1.5.3 リリースノート + +## 新規追加 + +- read_before_last_manifest_writeオプションの追加。 + 一部のkeyへの高頻度かつ多並列でのアクセスによるSibling explosionの回避に有効。 + [riak_cs/#1011](https://github.com/basho/riak_cs/pull/1011) +- タイムアウト設定の追加。Riak - Riak CS 間の全アクセスに対してタイムアウトを設定可能にし、運用に柔軟性を提供。 + [riak_cs/#1021](https://github.com/basho/riak_cs/pull/1021) + +## 修正されたバグ + +- ストレージ統計の集計結果に削除済バケットのデータが含まれ得る問題を修正。 + [riak_cs/#996](https://github.com/basho/riak_cs/pull/996) + +# Riak CS 1.5.2 リリースノート + +## 新規追加 + +- Riakに対する接続失敗に関するロギングの改善 + [riak_cs/#987](https://github.com/basho/riak_cs/pull/987). +- Riakに対してアクセス統計情報の保存に失敗した際のログを追加 + [riak_cs/#988](https://github.com/basho/riak_cs/pull/988). + これは一時的な Riak - Riak CS 間の接続エラーによるアクセス統計ログの消失を防ぎます。 + アクセスログは `console.log` へ `warning` レベルで保存されます。 +- 不正なガベージコレクション manifest の修復スクリプトの追加 + [riak_cs/#983](https://github.com/basho/riak_cs/pull/983)。 + active manifest が GCバケットへ保存される際に + [既知の問題](https://github.com/basho/riak_cs/issues/827) があります。 + このスクリプトは不正な状態を正常な状態へ変更します。 + +## 修正されたバグ + +- プロトコルバッファのコネクションプール (`pbc_pool_master`) のリークを修正 + [riak_cs/#986](https://github.com/basho/riak_cs/pull/986) 。 + 存在しないバケットに対する認証ヘッダ無しのリクエストや、ユーザ一覧のリクエストが + コネクションプールのリークを引き起こし、プールは結果的に空になります。このバグは1.5.0から含まれます。 + +# Riak CS 1.5.1 リリースノート + +## 新規追加 + +- Sibling Explosionを避けるために sleep-after-update を追加 [riak_cs/#959](https://github.com/basho/riak_cs/pull/959) +- `riak-cs-debug` の multibag サポート [riak_cs/#930](https://github.com/basho/riak_cs/pull/930) +- Riak CS におけるバケット数に上限を追加 [riak_cs/#950](https://github.com/basho/riak_cs/pull/950) +- バケットの衝突解決を効率化 [riak_cs/#951](https://github.com/basho/riak_cs/pull/951) + +## 修正されたバグ + +- `riak_cs_delete_fsm` のデッドロックによるGCの停止 [riak_cs/#949](https://github.com/basho/riak_cs/pull/949) +- `riak-cs-debug` がログを収集するディレクトリのパスを修正 [riak_cs/#953](https://github.com/basho/riak_cs/pull/953) +- DST-awareなローカルタイムからGMTへの変換を回避 [riak_cs/#954](https://github.com/basho/riak_cs/pull/954) +- Secretの代わりに UUID をカノニカルID生成時のシードに利用 [riak_cs/#956](https://github.com/basho/riak_cs/pull/956) +- マルチパートアップロードにおけるパート数の上限を追加 [riak_cs/#957](https://github.com/basho/riak_cs/pull/957) +- タイムアウトをデフォルトの 5000ms から無限に設定 [riak_cs/#963](https://github.com/basho/riak_cs/pull/963) +- GC バケット内の無効な状態のマニフェストをスキップ [riak_cs/#964](https://github.com/basho/riak_cs/pull/964) + +## アップグレード時の注意点 + +### ユーザー毎のバケット数 + +Riak CS 1.5.1 を使うと、ユーザーが作ることのできるバケット数を制限することができます。 +デフォルトでこの最大値は 100 です。この制限はユーザーの新たなバケット作成を禁止しますが、 +既に制限数を超えているユーザーが実施する、バケット削除を含む他の操作へは影響しません。 +デフォルトの制限を変更するには `app.config` の `riak_cs` セクションで次の箇所を変更してください: + +```erlang +{riak_cs, [ + %% ... + {max_buckets_per_user, 5000}, + %% ... + ]} +``` + +この制限を利用しない場合は `max_buckets_per_user` を `unlimited` に設定してください。 + +# Riak CS 1.5.0 リリースノート + +## 新規追加 + +* `cluster-info` 取得を含む新規コマンド `riak-cs-debug` を追加 [riak_cs/#769](https://github.com/basho/riak_cs/pull/769), [riak_cs/#832](https://github.com/basho/riak_cs/pull/832) +* 既存コマンド群を新規コマンド `riak-cs-admin` へ統合 [riak_cs/#839](https://github.com/basho/riak_cs/pull/839) +* Stanchion の IP、ポートを変更する新規コマンド `riak-cs-admin stanchion` を追加 [riak_cs/#657](https://github.com/basho/riak_cs/pull/657) +* 並行 GC によるガベージコレクション性能の向上 [riak_cs/#830](https://github.com/basho/riak_cs/pull/830) +* Iterator refresh [riak_cs/#805](https://github.com/basho/riak_cs/pull/805) +* `fold_objects_for_list_keys` 設定をデフォルト有効に変更 [riak_cs/#737](https://github.com/basho/riak_cs/pull/737), [riak_cs/#785](https://github.com/basho/riak_cs/pull/785) +* Cache-Control ヘッダーのサポートを追加 [riak_cs/#821](https://github.com/basho/riak_cs/pull/821) +* 猶予期間(`leeway_seconds`)内でもオブジェクトをガベージコレクション可能にする変更 [riak_cs/#470](https://github.com/basho/riak_cs/pull/470) +* オブジェクト、マルチパートともに PUT Copy API を追加 [riak_cs/#548](https://github.com/basho/riak_cs/pull/548) +* lager 2.0.3 へ更新 +* R16B0x をビルド環境に追加 (リリースは R15B01 でビルド) +* `gc_paginated_index` 設定をデフォルト有効に変更 [riak_cs/#881](https://github.com/basho/riak_cs/issues/881) +* 新規 API: Delete Multiple Objects の追加[riak_cs/#728](https://github.com/basho/riak_cs/pull/728) +* マニフェストに対して siblings, バイト、履歴の肥大化を警告するログ追加 [riak_cs/#915](https://github.com/basho/riak_cs/pull/915) + +## 修正されたバグ + +* `ERL_MAX_PORTS` を Riak のデフォルトに合わせ 64000 へ変更 [riak_cs/#636](https://github.com/basho/riak_cs/pull/636) +* Riak CS 管理リソースを OpenStack API でも利用可能にする修正 [riak_cs/#666](https://github.com/basho/riak_cs/pull/666) +* Solaris でのソースビルドのバグ修正のため、パス代入コードの変更 [riak_cs/#733](https://github.com/basho/riak_cs/pull/733) +* `riakc_pb_socket` エラー時の `sanity_check(true,false)` バグを修正 [riak_cs/#683](https://github.com/basho/riak_cs/pull/683) +* Riak-CS-GC のスケジューラタイムスタンプが 2013 ではなく 0043 になるバグを修正 [riak_cs/#713](https://github.com/basho/riak_cs/pull/713) fixed by [riak_cs/#676](https://github.com/basho/riak_cs/pull/676) +* OTP code_server プロセスを過剰に呼び出すバグを修正 [riak_cs/#675](https://github.com/basho/riak_cs/pull/675) +* content-md5 が一致しない場合に HTTP 400 を返すよう修正 [riak_cs/#596](https://github.com/basho/riak_cs/pull/596) +* `/riak-cs/stats` が `admin_auth_enabled=false` の時に動作しないバグを修正. [riak_cs/#719](https://github.com/basho/riak_cs/pull/719) +* ストレージ計算で tombstone および undefined の manifest.props を処理できないバグを修正 [riak_cs/#849](https://github.com/basho/riak_cs/pull/849) +* 未完了のマルチパートオブジェクトが、バケットの削除、作成後にも残るバグを修正 [riak_cs/#857](https://github.com/basho/riak_cs/pull/857) and [stanchion/#78](https://github.com/basho/stanchion/pull/78) +* list multipart upload の空クエリパラメータの扱いを修正 [riak_cs/#843](https://github.com/basho/riak_cs/pull/843) +* PUT Object 時にヘッダ指定の ACL が設定されないバグを修正 [riak_cs/#631](https://github.com/basho/riak_cs/pull/631) +* ping リクエストの poolboy タイムアウト処理を改善 [riak_cs/#763](https://github.com/basho/riak_cs/pull/763) +* 匿名アクセス時の不要なログを削除 [riak_cs/#876](https://github.com/basho/riak_cs/issues/876) +* マルチパートでアップロードされたオブジェクトの ETag 不正を修正 [riak_cs/#855](https://github.com/basho/riak_cs/issues/855) +* PUT Bucket Policy のポリシーバージョン確認の不具合を修正[riak_cs/#911](https://github.com/basho/riak_cs/issues/911) +* コマンド成功時に終了コード 0 を返すよう修正 [riak_cs/#908](https://github.com/basho/riak_cs/issues/908) +* `{error, disconnected}` が内部で notfound に書き換えられる問題を修正 [riak_cs/#929](https://github.com/basho/riak_cs/issues/929) + +## アップグレードに関する注意事項 + +### Riak Version + +このリリースは Riak 1.4.10 上でテストされました。 +[互換性マトリクス](http://docs.basho.com/riakcs/latest/cookbooks/Version-Compatibility/) +を参考に、正しいバージョンを使用していることを確認してください。 + +### 未完了のマルチパートアップロード + +[riak_cs/#475](https://github.com/basho/riak_cs/issues/475) はセキュリティ +に関する問題で、以前に作られた同名のバケットに +対する未完了のマルチパートアップロードが、新しく作成されたバケットに +含まれてしまう可能性があります。これは次のように修正されました。 + +- バケット作成時には、有効なマルチパートが存在するかを確認し、 + 存在する場合には 500 エラーをクライアントに返します。 + +- バケット削除時には、まず存在する有効なマルチパートの削除を試みた後に、 + 有効なマルチパートが存在するかを(Stanchion 上で)再度確認します。 + 存在する場合には 409 エラーをクライアントに返します。 + +1.4.x (またはそれより前のバージョン)から 1.5.0 へのアップグレード後には +いくつかの操作が必要です。 + +- すべてのバケットを正常な状態にするため、 `riak-cs-admin + cleanup-orphan-multipart` を実行します。マルチパートアップロードとバ + ケット削除が競合したときに発生しうるコーナーケースを避けるために、こ + のコマンドは `2014-07-30T11:09:30.000Z`のような、 ISO 8601 形式の日付 + を引数として指定することができます。この引数があるとき、バケットのク + リーンアップ操作はそのタイムスタンプよりも新しいマルチパートアップロー + ドを削除しません。もしこれを指定する場合は、全てのCSノードのアップグ + レードが終わって以降の時間がよいでしょう。 + +- 上記操作が終了するまでの期間は、削除済みのバケットで、未完了のマルチ + パートアップロードを含むバケットは再作成が出来ない場合があります。こ + のような再作成の失敗は [critical] ログ (`"Multipart upload remains + in deleted bucket "`) で確認可能です。 + +### ガベージコレクションの猶予期間(Leeway seconds)とディスク空き容量 + +[riak_cs/#470](https://github.com/basho/riak_cs/pull/470) は、 +オブジェクト削除とガベージコレクションの振る舞いを次のように変更します。 +これまで、ガベージコレクションバケットのタイムスタンプはオブジェクトが +回収される将来の時刻でしたが、削除された時刻そのものへと変わります。 +同時に、ガベージコレクターは現在の時刻までのタイムスタンプを回収していましたが、 +猶予期間(`leeway_seconds`)だけ過去のタイムスタンプまでだけを回収するようになります。 + +以前(- 1.4.x): + +``` + t1 t2 +-----------+--------------------------+-------------------> + DELETE object: GC 実行: + "t1 + leeway" "t2" までの + とマークされる オブジェクトを回収 +``` + +今後(1.5.0-): + +``` + t1 t2 +-----------+--------------------------+-------------------> + DELETE object: GC 実行: + "t1" "t2 - leeway" までの + とマークされる オブジェクトを回収 +``` + +これにより、1.5.0 へのアップグレード直後(仮に`t0`とします）にはオブジェ +クトが回収されない期間ができます。つまり `t0` から `t0 + leeway` までの +期間です。そして `t0` 直前に削除されたオブジェクトは `t0 + 2 * leeway` +時点で回収可能になります。 + +ローリングアップグレードに際しては、GC を実行している CS ノードを +**最初に** アップグレードする必要があります。 +GC を実行しない CS ノードは、猶予期間が正しく動作するために、その後から +アップグレードして下さい。 +また、`riak-cs-admin gc set-interval infinity` コマンドを実行して +ガベージコレクションを無効にしておくと、ノードの順序を +気にすることなくアップグレードが可能です。 + +マルチデータセンター構成のクラスタは、より慎重になる必要があります。 +ガベージコレクションを確実に無効化してからアップグレードしてください。 + +## 既知の問題と制限事項 + +* コピーを実行中にクライアントが次のリクエストを送信するとコピーは中断 + されます。これはクライアントの切断を検出してコピーを中止する機構の副 + 作用です。詳しくは [#932](https://github.com/basho/riak_cs/pull/932) + をご覧ください。 + +* OOSインターフェースでのコピーはサポートされていません。 + +* Multibag はオブジェクトのマニフェストとブロックを複数の異なるクラスタ + に分けて格納する機能です。これは Riak CS Enterprise の機能として追加 + されましたが、技術プレビューの段階にあります。クラスタ間レプリケーショ + ンによる `proxy_get` はサポートされておりません。Multibagは今のところ、 + ひとつのDCでのみ動作するように設計されています。 + +# Riak CS 1.4.5 リリースノート + +## 修正されたバグ + +* list objects v2 fsm のいくつかのデータが「見えない」バグを修正 [riak_cs/788](https://github.com/basho/riak_cs/pull/788) +* HEADリクエスト時にアクセス集計していた問題を修正 [riak_cs/791](https://github.com/basho/riak_cs/pull/791) +* POST/PUTリクエスト時のXML中の空白文字の対処 [riak_cs/795](https://github.com/basho/riak_cs/pull/795) +* ストレージ使用量計算時の誤ったバケット名を修正 [riak_cs/800](https://github.com/basho/riak_cs/pull/800) + Riak CS 1.4.4 で混入したバグにより、そのバージョンを使用している期間の + ストレージ計算はバケット名が文字列 "struct" に置き換わった結果となっていました。 + 本バージョン 1.4.5 でこのバグ自体は修正されましたが、すでに計算済みの古い結果を + さかのぼって修正することは不可能です。バケット名が "struct" に置き換わってしまった + 計算結果では、個別バケットの使用量を知ることはできませんが、その場合であっても + 個々のユーザに関して所有バケットにわたる合計は正しい数字を示します。 +* Unicodeのユーザ名とXMLの対応 [riak_cs/807](https://github.com/basho/riak_cs/pull/807) +* ストレージ使用量で必要なXMLフィールドを追加 [riak_cs/808](https://github.com/basho/riak_cs/pull/808) +* オブジェクトのfoldのタイムアウトを揃えた [riak_cs/811](https://github.com/basho/riak_cs/pull/811) +* 削除されたバケットをユーザーのレコードから削除 [riak_cs/812](https://github.com/basho/riak_cs/pull/812) + +## 新規追加 + +* オブジェクト一覧表示のv2 FSMでプレフィクスを使用する最適化を追加 [riak_cs/804](https://github.com/basho/riak_cs/pull/804) + +# Riak CS 1.4.4 リリースノート + +これはバグフィックスのためのリリースです。統計計算の修正が含まれています。 + +## 修正されたバグ + +* basho-patches ディレクトリが作成されなかった問題を修正 [riak_cs/775](https://github.com/basho/riak_cs/issues/775) . + +* `sum_bucket` のタイムアウトが全ての容量計算をクラッシュさせていた問題を修正 [riak_cs/759](https://github.com/basho/riak_cs/issues/759) . + +* アクセス集計のスロットリング失敗を修正 [riak_cs/758](https://github.com/basho/riak_cs/issues/758) . + +* アクセス集計のクラッシュを修正 [riak_cs/747](https://github.com/basho/riak_cs/issues/747) . + + +# Riak CS 1.4.3 リリースノート + +## 修正された問題 + +- schedule_delete状態のマニフェストがpending_deleteやactive状態へ復帰するバグを修正。 +- 上書きによって既に削除されたマニフェストをカウントしない。 +- 誤ったmd5による上書き操作で、既存バージョンのオブジェクトを削除しない。 + +## 新規追加 + +- マニフェストプルーニングのパフォーマンス改善。 +- GCにおける2iのページングオプションを追加。GC対象データ収集時のタイムアウト対策。 +- ブロック取得処理における接続断のハンドリングを改善。 +- lager 2.0.1へのアップデート。 +- 時刻によるマニフェストプルーニングに個数オプションを追加。 +- 複数アクセスアーカイブプロセスの並行実行を許可。 + +# Riak CS 1.4.2 リリースノート + +## 修正された問題 + +- Debian Linux 上の Enterprise 版ビルドの問題を修正。 +- ソース tarball ビルドの問題を修正。 +- アクセス統計において、正常アクセスがエラーと扱われてしまうバグを修正。 +- Riak バージョン 1.4 以前とあわせて動作するよう、バケットリスト + map フェーズのログを lager バージョンに依存しないよう変更。 +- Riak CS 1.3.0 以前で保存されたマニフェストについて、 `props` フィールド + の `undefined` を正しく扱うよう修正。 + +## 新規追加 + +- 最初のガベージコレクションの遅延を設定する `initial_gc_delay` オプションを追加。 +- ガベージコレクションバケットのキーにランダムなサフィックスを追加し、 + ホットキーの回避と削除の性能を向上。 +- マニフェストに cluster id が指定されていない場合に用いる + `default_proxy_cluster_id` オプションを追加。OSS 版から Enterprise 版への + 移行が容易になる。 + +# Riak CS 1.4.1 リリースノート + +## 修正された問題 + +- 最初の1002個のキーがpending delete状態だったときにlist objectsがクラッシュ + する問題を修正 +- GCデーモンがクラッシュする問題を解決 +- node_packageをアップデートしパッケージ作成の問題を解決 + +# Riak CS 1.4.0 リリースノート + +## 修正された問題 + +- GCバケットで使われていないキーを削除 +- マルチパートアップロードのクエリ文字での認証を修正 +- マルチパートでアップロードされたオブジェクトのストレージクラスを修正 +- マルチパートアップロードされたオブジェクトのetagsを修正 +- Riak CSのマルチバックエンドのインデックス修正をサポート +- GETリクエストの際、通信が遅い場合のメモリ増大を修正 +- アクセス統計処理のメモリ使用量を削減 +- オブジェクトのACL HEADリクエストの際の500を修正 +- マルチパートでアップロードされたオブジェクトの並列アップロードや削除の際の競合の問題を解決 +- Content-md5のヘッダがあった場合に整合性をチェックするように修正 +- Riakとのコネクションが切れた際のハンドリングを修正 + +## 新規追加 + +- Swift APIとKeystone認証のサポートを試験的に追加 +- Riak 1.4.0以降と併用された場合のオブジェクト一覧取得のパフォーマンスを改善 +- ユーザーアカウント名とメールアドレスは変更可能に +- データセンタ間レプリケーションv3のサポートを追加 +- Riakとのコネクションタイムアウトを変更可能に +- Lagerのsyslogサポートを追加 +- データブロックへのリクエスト時は1つのvnodeへアクセス + +# Riak CS 1.3.1 Release Notes + +## Bugs Fixed + +- Fix bug in handling of active object manifests in the case of + overwrite or delete that could lead to old object versions being + resurrected. +- Fix improper capitalization of user metadata header names. +- Fix issue where the S3 rewrite module omits any query parameters + that are not S3 subresources. Also correct handling of query + parameters so that parameter values are not URL decoded twice. This + primarily affects pre-signed URLs because the access key and request + signature are included as query parameters. +- Fix for issue with init script stop. + +# Riak CS 1.3.0 Release Notes + +## Bugs Fixed + +- Fix handling of cases where buckets have siblings. Previously this + resulted in 500 errors returned to the client. +- Reduce likelihood of sibling creation when creating a bucket. +- Return a 404 instead of a 403 when accessing a deleted object. +- Unquote URLs to accommodate clients that URL encode `/` characters + in URLs. +- Deny anonymous service-level requests to avoid unnecessary error + messages trying to list the buckets owned by an undefined user. + +## Additions + +- Support for multipart file uploads. Parts must be in the range of + 5MB-5GB. +- Support for bucket policies using a restricted set of principals and + conditions. +- Support for returning bytes ranges of a file using the Range header. +- Administrative commands may be segrated onto a separate interface. +- Authentication for administrative commands may be disabled. +- Performance and stability improvements for listing the contents of + buckets. +- Support for the prefix, delimiter, and marker options when listing + the contents of a bucket. +- Support for using Webmachine's access logging features in + conjunction with the Riak CS internal access logging mechanism. +- Moved all administrative resources under /riak-cs. +- Riak CS now supports packaging for FreeBSD, SmartOS, and Solaris. + +# Riak CS 1.2.2 Release Notes + +## Bugs Fixed + +- Fix problem where objects with utf-8 unicode key cannot be listed + nor fetched. +- Speed up bucket_empty check and fix process leak. This bug was + originally found when a user was having trouble with `s3cmd + rb s3://foo --recursive`. The operation first tries to delete the + (potentially large) bucket, which triggers our bucket empty + check. If the bucket has more than 32k items, we run out of + processes unless +P is set higher (because of the leak). + +## Additions + +- Full support for MDC replication + +# Riak CS 1.2.1 Release Notes + +## Bugs Fixed + +- Return 403 instead of 404 when a user attempts to list contents of + nonexistent bucket. +- Do not do bucket list for HEAD or ?versioning or ?location request. + +## Additions + +- Add reduce phase for listing bucket contents to provide backpressure + when executing the MapReduce job. +- Use prereduce during storage calculations. +- Return 403 instead of 404 when a user attempts to list contents of + nonexistent bucket. + +# Riak CS 1.2.0 Release Notes + +## Bugs Fixed + +- Do not expose stack traces to users on 500 errors +- Fix issue with sibling creation on user record updates +- Fix crash in terminate state when fsm state is not fully populated +- Script fixes and updates in response to node_package updates + +## Additions + +- Add preliminary support for MDC replication +- Quickcheck test to exercise the erlcloud library against Riak CS +- Basic support for riak_test integration + +# Riak CS 1.1.0 Release Notes + +## Bugs Fixed + +- Check for timeout when checking out a connection from poolboy. +- PUT object now returns 200 instead of 204. +- Fixes for Dialyzer errors and warnings. +- Return readable error message with 500 errors instead of large webmachine backtraces. + +## Additions + +- Update user creation to accept a JSON or XML document for user + creation instead of URL encoded text string. +- Configuration option to allow anonymous users to create accounts. In + the default mode, only the administrator is allowed to create + accounts. +- Ping resource for health checks. +- Support for user-specified metadata headers. +- User accounts may be disabled by the administrator. +- A new key_secret can be issued for a user by the administrator. +- Administrator can now list all system users and optionally filter by + enabled or disabled account status. +- Garbage collection for deleted and overwritten objects. +- Separate connection pool for object listings with a default of 5 + connections. +- Improved performance for listing all objects in a bucket. +- Statistics collection and querying. +- DTrace probing. + +# Riak CS 1.0.2 Release Notes + +## Additions + +- Support query parameter authentication as specified in [[http://docs.amazonwebservices.com/AmazonS3/latest/dev/RESTAuthentication.html][Signing and Authenticating REST Requests]]. + +# Riak CS 1.0.1 Release Notes + +## Bugs Fixed + +- Default content-type is not passed into function to handle PUT + request body +- Requests hang when a node in the Riak cluster is unavailable +- Correct inappropriate use of riak_moss_utils:get_user by + riak_moss_acl_utils:get_owner_data + +# Riak CS 1.0.0 Release Notes + +## Bugs Fixed + +- Fix PUTs for zero-byte files +- Fix fsm initialization race conditions +- Canonicalize the entire path if there is no host header, but there are + tokens +- Fix process and socket leaks in get fsm + +## Other Additions + +- Subsystem for calculating user access and storage usage +- Fixed-size connection pool of Riak connections +- Use a single Riak connection per request to avoid deadlock conditions +- Object ACLs +- Management for multiple versions of a file manifest +- Configurable block size and max content length +- Support specifying non-default ACL at bucket creation time + +# Riak CS 0.1.2 Release Notes + +## Bugs Fixed + +- Return 403 instead of 503 for invalid anonymous or signed requests. +- Properly clean up processes and connections on object requests. + +# Riak CS 0.1.1 Release Notes + +## Bugs Fixed + +- HEAD requests always result in a `403 Forbidden`. +- `s3cmd info` on a bucket object results in an error due to missing + ACL document. +- Incorrect atom specified in `riak_moss_wm_utils:parse_auth_header`. +- Bad match condition used in `riak_moss_acl:has_permission/2`. + +# Riak CS 0.1.0 Release Notes + +## Bugs Fixed + +- `s3cmd info` fails due to missing `'last-modified` key in return document. +- `s3cmd get` of 0 byte file fails. +- Bucket creation fails with status code `415` using the AWS Java SDK. + +## Other Additions + +- Bucket-level access control lists +- User records have been modified so that an system-wide unique email + address is required to create a user. +- User creation requests are serialized through `stanchion` to be + certain the email address is unique. +- Bucket creation and deletion requests are serialized through + `stanchion` to ensure bucket names are unique in the system. +- The `stanchion` serialization service is now required to be installed + and running for the system to be fully operational. +- The concept of an administrative user has been added to the system. The credentials of the + administrative user must be added to the app.config files for `moss` and `stanchion`. +- User credentials are now created using a url-safe base64 encoding module. + +## Known Issues + +- Object-level access control lists have not yet been implemented. + +# Riak CS 0.0.3 Release Notes + +## Bugs Fixed + +- URL decode keys on put so they are represented correctly. This + eliminates confusion when objects with spaces in their names are + listed and when attempting to access them. +- Properly handle zero-byte files +- Reap all processes during file puts + +## Other Additions + +- Support for the s3cmd subcommands sync, du, and rb + + - Return valid size and checksum for each object when listing bucket objects. + - Changes so that a bucket may be deleted if it is empty. + +- Changes so a subdirectory path can be specified when storing or retrieving files. +- Make buckets private by default +- Support the prefix query parameter +- Enhance process dependencies for improved failure handling + +## Known Issues + +- Buckets are marked as /private/ by default, but globally-unique + bucket names are not enforced. This means that two users may + create the same bucket and this could result in unauthorized + access and unintentional overwriting of files. This will be + addressed in a future release by ensuring that bucket names are + unique across the system. + + [riak_1.4_release_notes]: https://github.com/basho/riak/blob/1.4/RELEASE-NOTES.ja.md [riak_2.0_release_notes]: https://github.com/basho/riak/blob/2.0/RELEASE-NOTES.ja.md [riak_2.0_release_notes_bitcask]: https://github.com/basho/riak/blob/2.0/RELEASE-NOTES.ja.md#bitcask diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index a27374f6f..24deee518 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,542 @@ +# Riak CS 3.2.5 Release Notes + +Released May 7, 2024. + +## General + +This release extends OTP compatibility to versions 22 and 26. + +## Bug fixes + +* `ListAttachedRolePolicies` and `ListAttachedUserPolicies` fixed for + the case when `PathPrefix` parameter is empty. + +## Changes + +### User-visible changes + +* Configuration parameter `gc_paginated_indexes`, long slated for + removal, was removed. +* Convenence script `misc/prepare-riak-for-cs` now always copies + CS-specific beams to where Riak can find them (previously, when both + Riak and Riak CS are on the same host, this was achieved via adding + `add_paths` parameter to `riak_kv` app in advanced.config, which + became an issue on OTP-22 for tools/internal/offline_delete.erl). + +### Other changes + +* Dependencies have been updated to their latest versions, except for + exometer\_core, which was downgraded from 1.6.2 to 1.6.1 on account + of eunit failures with 1.6.2 on otp-22. Rebar3 was updated to 3.23.0. + + +# Riak CS 3.2.4 Release Notes + +Released January 25, 2024. + +## General + +Bugfix/minor feature release. + +## New features + +* /riak-cs/info has a new field, `storage_info`, with the following + bits: `backend_data_total_size`, `df_available`, `df_total`, + `n_val`, gathered from each nodes in the riak cluster. + +## Bug fixes + +* `AssumeRoleWithSAML`, when called on a non-existing + `SAMLProvider`, now correctly returns a 404 (not 500). +* Listing objects with an explicit `PathPrefix` value of "" now correctly + includes items with empty path. + + +# Riak CS 3.2.3 Release Notes + +Released December 24, 2023. + +## General + +Incremental release, with features complementary to the more +substantive release of Riak CS Control 3.2.3. + +## New features + +* New IAM call `ListAttachedRolePolicies`. +* New admin API endpoint /riak-cs/temp-sessions, listing the currently + active assumed roles. + +## Bug fixes + +* `GetSAMLProvider` now includes `SAMLMetadataDocument` field as it + should have. +* `CreateSAMLProvider` fixed for the case when request is made without + tags. +* `CreateBucket` by federated users now properly returns 400 (and + not 500). +* Fix `ListBucket` for federated users with appropriate policy. +* Federated users' policies are now correctly evaluated in operations + on objects. +* Operations involving role update do not double-base64 encode its + `AssumeRolePolicyDocument` anymore. +* Improved locking when updating users and roles in `AttachUserPolicy` + and `DetachuserPolicy` (same for roles). + + + +# Riak CS 3.2.2 Release Notes + +Released November 4, 2023. + +## General + +This is mostly a bug-fix release, with a smattering of small +improvements enabling it to operate with the new Riak CS Control +(version 3.2.2, rewritten in Elm). + +## New features + +* New admin API endpoint, /riak-cs/info, currently reporting Riak CS version + and uptime. +* New riak-cs admin command, `create-admin-user`, intended to help + create an admin user on a fresh install of Riak CS. +* New helper script, `prepare-riak-for-cs`, which prepares riak + configuration files in /etc/riak to work with locally co-installed + Riak CS. This script is installed in /usr/lib/riak-cs/bin (or + /usr/lib64/... on other distros). + +## Bug fixes + +### Critical bugs + +* In 3.2.x, admin user signature was not verified in IAM calls. + +### Less critical bugs + +* Fix DeleteUserPolicy when called on a policy with a non-zero + attachment count. +* GetUser now includes PermissionsBoundary in its XML response. +* Fix parsing of IsAttachable parameter in *Policy calls. +* Fix updating user email via PUT to /riak-cs/user. + + +# Riak CS 3.2.1 Release Notes + +Released September 3, 2023. + +## Bug fixes + +This is a bugfix release addressing the issue of temporary sessions +created for federated users as a direct result of AssumeRoleWithSAML, +not expiring after DurationSeconds if the Riak CS node which received +this request is restarted. + + +# Riak CS 3.2.0 Release Notes + +Released August 25, 2023. + +## New Features + +This release includes basic support for IAM entities such as roles and +standalone policies, and federated users. The following new API calls are implemented: + +* IAM: + * CreateRole, GetRole, DeleteRole, ListRoles; + * CreatePolicy, GetPolicy, DeletePolicy, ListPolicies; + * AttachRolePolicy, AttachUserPolicy, DetachRolePolicy, + DetachUserPolicy, ListAttachedUserPolicies; + * CreateSAMLProvider, GetSAMLProvider, DeleteSAMLProvider, + ListSAMLProviders. +* STS: + * AssumeRoleWithSAML (with [this specific use case](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_saml.html#CreatingSAML-configuring) +in mind). + +Note that as a Service Provider, Riak CS does not currently accept +encrypted SAML assertions. + +## Changes + +### User-visible changes + +* A script, tools/create-admin, is provided to facilitate the creation + of admin user. This script will also create a policy permitting + all S3, IAM and STS calls and attach it to the admin user. The + script requires python3 with httplib2 and boto3. + +* Changed configuration items: + * `fold_objects_for_list_keys` was removed, along with legacy + riak\_cs\_list\_objects\_fsm.erl, long slated for removal in favor of riak\_cs\_list\_objects\_fsm_v2.erl. + * New key, `iam_create_user_default_email_host`, that defines the domain + used to construct (artificial) email from the UserName parameter + to the iam:CreateUser call. + * `cs_root_host` renamed to `s3_root_host`. + * Default values for keys `rewrite_module`, `auth_module` have + changed to "riak\_cs\_aws\_rewrite" and "riak\_cs\_aws\_auth". + * `dtrace` key was removed, along with code supporting it. + +* `riak-cs stop` now prints "ok" on success, as in good old Basho times. + +### Other changes + +* Rebar was upgraded to version 3.22. +* riak\_cs\_multibag, a companion application previously existing in + its own repo, has been incorporated into Riak CS. +* Users are now stored by their arns as keys, as do all IAM entities. +* Many files and structures have been renamed to reflect the fact that + now Riak CS provides IAM and STS services, in addition to S3 around + which it was originally designed. +* Throughout the entire code base, calls to mochijson2 facilities were + replaced with corresponding jsx and jason functions. +* In many internal structures, strings were replaced with binaries, + and timestamps previously stored as `calendar:datetime()`, have been + converted to plain unixtime, in milliseconds. + +## Compatibility + +This release is forward-compatible with 3.1.0. Users and data written +by 3.1.0 can be read in 3.2.0. User records updated or created in 3.1 +will have a new layout (rcs\_user\_v3), and *will not* be accessible +from older versions of Riak CS. + + +# Riak CS 3.1.0 Release Notes + +Released March 15, 2023. + +## General + +This release is centred around a single architectural change: +the merge of Stanchion into Riak CS. There are no substantial additions +in the scope of supported S3 methods, and no changes to the behaviour +or feature set otherwise. + +## New features + +* Stanchion, which was a separate Erlang application serving the + purpose of serializing CRUD operations on users and buckets, and + therefore had to be deployed and run alongside Riak CS nodes on a + dedicated node, is now colocated on one of the Riak CS nodes. + + In `auto` mode, it is dynamically created at the Riak CS node + receiving the first request that needs to be serialized. This node + will then store Stanchion details (ip:port) in a special service + bucket on a configured Riak KV node. Riak CS nodes will read that + ip:port and send subsequent Stanchion requests to that endpoint. If + a Riak CS node finds that Stanchion is unreachable, it will spawn a + new instance on its premises and update the details in Riak KV. When + a node that previously hosted Stanchion, after being temporarily + unavailable, sees the Stanchion ip:port has changed, it will stop + its Stanchion instance. + +* Riak CS now can be built on OTP-22 through 25. + +* A new subcommand, `supps` has been added to `riak-cs admin`, which + will produce a ps-like output for the processes in the riak_cs main + supervisor tree with some stats. + +## Changes + +### User-visible changes + +* New configuration parameters: + + - `stanchion_hosting_mode`, with acceptable values: `auto`, + `riak_cs_with_stanchion`, `riak_cs_only`, `stanchion_only` + (default is `auto`). + + - `tussle_voss_riak_host` ("voss" stands for "VOlatile, Storage, + Serialized"), which can be set to `auto` or a fqdn:port at which + riak_cs will store Stanchion details. A value of `auto` is + equivalent to setting it to `riak_host`. The purpose of this + parameter is to enable users operating in suboptimal networking + conditions to set it to a dedicated, single-node riak cluster on + a separate network, which can be made more reliable than the one + carrying S3 traffic. + + - `stanchion_port`, port at which Stanchion instance will + listen (if/when this node gets to start it). + + - `stanchion_subnet` and `stanchion_netmask` (with default values of + "127.0.0.1" and "255.255.255.255" respectively), to use when selecting which + network to place Stanchion on. + +* `riak-cs admin stanchion switch` command has been removed. The + purpose of this command was to enable operators to change the + ip:port of Stanchion endpoint without restarting the node. With + Stanchion location now being set dynamically and discovered + automatically, there is no need to expose an option for operators to + intervene in this process. + +### Other changes + +* A fair bit of work has gone to uplift riak\_cs\_test, hopefully + making life easier for the next decade. Specifically: + + - we switched erlcloud from an ancient, Basho-patched fork to + upstream (tag 3.6.7), incidentally triggereing (and fixing) a bug + with /stats endpoint, which previously only accepted v2 + signatures. + + - riak\_cs\_test can be built with OTP-24, and has lager replaced + by the standard kernel logger facility. + + - php and ruby tests have been updated and re-included in external + client tests. + +* [Riak CS Service + Bundle](https://github.com/TI-Tokyo/riak_cs_service_bundle) has been + updated to accommodate stanchion-less version of Riak CS. + + +# Riak CS 3.0.1 Release Notes + +Released June 10, 2022. + +## General + +This is a correction release that includes one feature that slipped +from 3.0.0. + +## New features + +* Support for fqdn data type for `riak_host` and `stanchion_host` + configuration items in riak-cs.conf. + +## Changes + +### User-visible changes + +* S3 request signature v4 is now the default. The old (v2) signatures + continue to be supported. +* A change of internal structures needed to support object versions, + meaning downgrade to 2.x is no longer possible (even if the objects + created with 3.0 have no versions). Upgrade from 2.x is possible. +* The rpm and deb packages now rely on systemd (old-style SysV init + scripts are no longer included). + +### Other changes + +* Riak CS and Stanchion now require OTP-22 and rebar3. +* Riak CS Test framework: + - The framework, along with a suite of tests (also the [multibag + additions](https://github.com/TI-Tokyo/riak_cs_multibag)), has been + upgraded to OTP-22/rebar3 and moved into a separate project, + [riak_cs_test](https://github.com/TI-Tokyo/riak_cs_test). + - A new battery of tests is written for `s3cmd` as a client. + - The Python client tests have been upgraded to boto3 and python-3.9. +* A refactoring of code shared between Riak CS and stanchion resulted + in that code being collected into a separate dependency, + [rcs_common](https://github.com/TI-Tokyo/rcs_common). +* [Riak CS Control](https://github.com/TI-Tokyo/riak_cs_control) + application has been upgraded to OTP-22/rebar3, too, however without + any new features. +* All EQC tests have been converted to use PropEr (no shortcuts taken, + all coverage is preserved). + +## Upgrading + +Existing data in the riak cluster underlying a 2.x instance of Riak CS +can be used with Riak CS 3.0 without any modification. + +*Note:* Once a new object is written into a database by Riak CS 3.0, +that database cannot be used again with 2.x. + +## Compatibility + + +# Riak CS 3.0.0 Release Notes + +Released May 30, 2022. + +## General + +This release was originally envisaged as an uplift of 2.1.2 to OTP-22 +and rebar3. There were no known critical bugs that needed fixing. We did, +however, identifiy and implement a new S3 API call +(`ListObjectVersions` and related), to give more substance to the +major version bump. + +We also provide Dockerfiles, and a [Riak CS Docker Service +Bundle](https://github.com/TI-Tokyo/riak_cs_service_bundle), as a +convenient way to set up the full Riak CS suite locally. + +All principal repositories are in [TI Tokyo +org](https://github.com/TI-Tokyo) on Github. + +This release was [presented](https://youtu.be/CmmeYza5HPg) at Code +BEAM America 2021. + +## New features + +* Support for **[object + versions](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html)**, + including new S3 API calls: + - [`GetBucketVersioning`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketVersioning.html), + [`PutBucketVersioning`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketVersioning.html) and + [`ListObjectVersions`](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectVersions.html). + - For buckets with versioning enabled, `GetObject` will return the + specific version if it is given in the request, or the `null` + version if it is not. + - As a Riak CS extension, rather than generating a random id for the + new version, `PutObject` will read a `versionId` from header + `x-rcs-versionid`, and use that instead. +* Riak CS Suite as a [Docker + service](https://github.com/TI-Tokyo/riak_cs_service_bundle), + allowing the (prospective) users quickly to bring up a fully functional Riak + CS cluster running locally, complete with Riak CS Control, and + - properly configured and set up with a new user, whose credentials + will be shown; + - with riak data persisted; + - ready for a [load-test](https://github.com/TI-Tokyo/s3-benchmark). +* Packaging: + - **New packages** are provided for FreeBSD 13 and OSX 14 (in the latter + case, the package is the result of `make rel` tarred; no special + user is created). + - Packages have been verified for: + * RPM-based: Centos 7 and 8, Amazon Linux 2, SLES 15, Oracle + Linux 8; + * DEB-based: Debian 8 and 11, Ubuntu Bionic and Xenial; + * Other: FreeBSD 13, OSX 14; Alpine Linux 3.15. +* A **Dockerfile**, bypassing cuttlefish mechanism to enable run-time + configuration via environment variables. +* `riak-cs-admin` now has a new option, **`test`**, which creates a bucket + and performs a basic write-read-delete cycle in it (useful to test + that the riak cluster is configured properly for use with Riak CS). + +## Changes + +### User-visible changes + +* S3 request signature v4 is now the default. The old (v2) signatures + continue to be supported. +* A change of internal structures needed to support object versions, + meaning downgrade to 2.x is no longer possible (even if the objects + created with 3.0 have no versions). Upgrade from 2.x is possible. +* The rpm and deb packages now rely on systemd (old-style SysV init + scripts are no longer included). + +### Other changes + +* Riak CS and Stanchion now require OTP-22 and rebar3. +* Riak CS Test framework: + - The framework, along with a suite of tests (also the [multibag + additions](https://github.com/TI-Tokyo/riak_cs_multibag)), has been + upgraded to OTP-22/rebar3 and moved into a separate project, + [riak_cs_test](https://github.com/TI-Tokyo/riak_cs_test). + - A new battery of tests is written for `s3cmd` as a client. + - The Python client tests have been upgraded to boto3 and python-3.9. +* A refactoring of code shared between Riak CS and stanchion resulted + in that code being collected into a separate dependency, + [rcs_common](https://github.com/TI-Tokyo/rcs_common). +* [Riak CS Control](https://github.com/TI-Tokyo/riak_cs_control) + application has been upgraded to OTP-22/rebar3, too, however without + any new features. +* All EQC tests have been converted to use PropEr (no shortcuts taken, + all coverage is preserved). + +## Upgrading + +Existing data in the riak cluster underlying a 2.x instance of Riak CS +can be used with Riak CS 3.0 without any modification. + +*Note:* Once a new object is written into a database by Riak CS 3.0, +that database cannot be used again with 2.x. + +## Compatibility + +Riak CS 3.0 has been tested with Riak versions 2.2.6, 2.9.8 through +.10, and 3.0.7 and .9. It requires Stanchion 3.0.0 (2.x versions not +supported due to changes in the manifest record). + + +# Riak CS 2.1.2 Release Notes + +Released April 7, 2019. + +This is a backwards-compatible* release that updates node_package to address a recent [Product Advisory](http://docs.basho.com/riak/latest/community/product-advisories/codeinjectioninitfiles/), as well as fixes several bugs. + +Riak CS 2.1 is designed to work with Riak KV 2.1.1+. + +>*This release is backwards compatible only with the Riak CS 2.x series. + +### Upgrading + +**For anyone updating to 2.1.2 from 2.1.1 and older versions.** + +During the update to 2.1.2, a '==' omitted upload ID might be passed to a Riak CS node running an older version of CS. This may lead to process-crash by failing on decoding upload ID. + +## Changes +* For s3cmd users, experimental signature_v4 support has been made available through a simple on/off toggle in riak-cs.conf. With a default setting of "off", it allows in-situ upgrades without the need to change s3cfg files until after all nodes have been upgraded. Note: this function is currently unfinished and suffers from compatibility issues with some clients ([#1058](https://github.com/basho/riak_cs/issues/1058) / [#1060](https://github.com/basho/riak_cs/issues/1060)) and one potential security issue ([#1059](https://github.com/basho/riak_cs/issues/1059) +* Experimental support for Leveled (the alternative to LevelDB to be released with Riak KV 2.9) has been successfully tested with the Riak KV 2.9.0 Release Candidates. +* Due to a recent [Product Advisory](http://docs.basho.com/riak/latest/community/product-advisories/codeinjectioninitfiles/), node_package was bumped to version 3.0.0 to prevent a potential code injection on the riak init file. [[Issue 1297](https://github.com/basho/riak_cs/issues/1297), [PR 1306](https://github.com/basho/riak_cs/pull/1306), & [PR 109](https://github.com/basho/stanchion/pull/109)] +* Multipart upload IDs no longer contain trailing '=' characters, which caused trouble for some clients. This change also makes upload IDs URL-safe. [[PR 1316](https://github.com/basho/riak_cs/pull/1316)] +* When Riak is unavailable due to network partition or node being offline, a 500 error is returned. [[PR 1298](https://github.com/basho/riak_cs/pull/1298)] +* Switched from `make` to `${MAKE}` to facilitate easier building on FreeBSD and related platforms + +## Bugs Fixed + +* [[Issue 1100](https://github.com/basho/riak_cs/issues/1100)/[PR 1304](https://github.com/basho/riak_cs/pull/1304)] When a multipart completion request was empty, Riak CS would return a "Completed" message. Now, during a multipart upload, the "complete" message contains a list of part IDs and MD5 hashes. If it is empty, it returns a "400 Bad Request, Malformed" XML message like S3. +* [[Issue 1288](https://github.com/basho/riak_cs/issues/1288)/[PR 1302](https://github.com/basho/riak_cs/pull/1302)] The `root_host` configuration parameter is now used to populate the `Location` response element. +* [[Issue 972](https://github.com/basho/riak_cs/issues/972)/[PR 1296](https://github.com/basho/riak_cs/pull/1296)] 'adderss' now reads 'address' in Stanchion console output. +* [[Issue 1025](https://github.com/basho/riak_cs/issues/1025)/[PR 1300](https://github.com/basho/riak_cs/pull/1300)] Copying an object used to fail when connecting via HTTPS. + + +# Riak S2 (Riak CS) 2.1.1 Release Notes + +## General Information +This is a bugfix release. + +## Bug Fixes + +* Remove blocks and manifest in cases of client-side failure, eg: + incomplete upload, network issues, or other unexpected transfer failures. + If the manifest was writing and no overwrite happened after upload canceled, + then the partially uploaded blocks and manifest are left as is, occupying + disk space. This is fixed by adding an error handling routine to move the + manifest to the garbage collection bucket by catching the socket error. + Partially uploaded blocks and manifests will eventually be deleted by + garbage collection. ([#770](https://github.com/basho/riak_cs/issues/770) / + [PR#1280](https://github.com/basho/riak_cs/pull/1280)) +* Remove `admin.secret` from riak-cs.conf and stanchion.conf. Both + Riak CS and Stanchion searched riak-cs.conf and stanchion.conf to + find who was the administrator using `admin.key` and `admin.secret`. + Writing down `admin.secret` in non-encrypted form compromises the + security of the system. The administrator is able to (1) list all + users, (2) disable/enable other users, (3) changing user accounts, + and (4) read access and storage statistics of all users. A workaround + for this is encrypting the partition that includes the /etc/riak-cs + directory. ([#1274](https://github.com/basho/riak_cs/issues/1274) / + [PR#1279](https://github.com/basho/riak_cs/issues/1279) / + [PR#108](https://github.com/basho/stanchion/pull/108)) +* Use /etc/riak-cs/app.config when no generated config file is found. + This prevents an error when using app.config/vm.args instead of + riak.conf, such as when upgrading from 1.5.x. + ([#1261](https://github.com/basho/riak_cs/issues/1261)/ + [PR#1263](https://github.com/basho/riak_cs/pull/1263) + as [PR#1266](https://github.com/basho/riak_cs/pull/1266)) +* Allow any bytes (including non-UTF8 ones) in List Objects response + XML. List Objects API had been failing in cases where an object with + non-UTF8 characters in its name was included in the result. With this + change, such keys are represented in the result of List Objects as + raw binaries, although it may not be proper XML 1.0. + ([#974](https://github.com/basho/riak_cs/issues/974) / + [PR#1255](https://github.com/basho/riak_cs/pull/1255) / + [PR#1275](https://github.com/basho/riak_cs/pull/1275)) + +## Notes on Upgrade + +* We strongly recommend removing `admin.secret` from + riak-cs.conf and stanchion.conf, and `admin_secret` from the + `advanced.config` files of Riak CS and Stanchion. +* On startup, Riak CS now verifies that the value of `admin.key` + is either a valid Riak CS user key or the placeholder value + `admin-key`. For an user with a non-existent or invlaid user + key, Riak CS process won't start. Also, any `admin.secret` will + be ignored. The initial procedure to create the very first + account, starting `anonymous_user_creation = true`, does not change. + #Riak S2 (Riak CS) 2.1.0 Release Notes Released October 13, 2015. @@ -19,8 +558,8 @@ cs_version = 20100 If you need storage calculation, you will still require the `add_paths` config to load MapReduce codes into Riak KV. -##New Features -###Metrics +## New Features +### Metrics New metrics have been added that enable you to determine the health of your Riak S2 system, as well as get reports on your storage utilization per bucket or user. The following stats items are available: * All calls, latencies, and counters in the S3 API * All calls, latencies, and counters in Stanchion @@ -698,6 +1237,674 @@ Bitcask data files is not. For this reason downgrading requires a script to translate data files. See also the [2.0 downgrade notes][downgrade_notes]. +# Riak CS 1.5.4 Release Notes + +## Bugs Fixed + +- Disable previous Riak object after backpressure sleep is triggered + [riak_cs/#1041](https://github.com/basho/riak_cs/pull/1041). This + change prevents unnecessary siblings growth in cases where (a) + backpressure is triggered under high upload concurrency and (b) + uploads are interleaved during backpressure sleep. This issue does not + affect multipart uploads. +- Fix an incorrect path rewrite in the S3 API caused by unnecessary URL + decoding + [riak_cs/#1040](https://github.com/basho/riak_cs/pull/1040). Due to + the incorrect handling of URL encoding/decoding, object keys including + `%[0-9a-fA-F][0-9a-fA-F]` (as a regular expression) or `+` had been + mistakenly decoded. As a consequence, the former case was decoded to + some other binary and for the latter case (`+`) was replaced with ` ` + (space). In both cases, there was a possibility of an implicit data + overwrite. For the latter case, an overwrite occurs for an object + including `+` in its key (e.g. `foo+bar`) by a different object with a + name that is largely similar but replaced with ` ` (space, e.g. `foo + bar`), and vice versa. This fix also addresses + [riak_cs/#910](https://github.com/basho/riak_cs/pull/910) and + [riak_cs/#977](https://github.com/basho/riak_cs/pull/977). + +## Notes on upgrading + +After upgrading to Riak CS 1.5.4, objects including +`%[0-9a-fA-F][0-9a-fA-F]` or `+` in their key (e.g. `foo+bar`) become +invisible and can be seen as objects with a different name. For the +former case, objects will be referred as unnecessary decoded key. For +the latter case, those objects will be referred as keys `+` replaced +with ` ` (e.g. `foo bar`) by default. + +The table below provides examples for URLs including +`%[0-9a-fA-F][0-9a-fA-F]` and how they will work before and after the +upgrade. + + | before upgrade | after upgrade | +:-----------|:-------------------|:--------------| + written as | `a%2Fkey` | - | + read as | `a%2Fkey`or`a/key` | `a/key` | + listed as | `a/key` | `a/key` | + +Examples on unique objects including `+` or ` ` through upgrade: + + | before upgrade | after upgrade | +:-----------|------------------|---------------| + written as | `a+key` | - | + read as | `a+key`or`a key` | `a key` | + listed as | `a key` | `a key` | + + | before upgrade | after upgrade | +------------|------------------|---------------| + written as | `a key` | - | + read as | `a+key`or`a key` | `a key` | + listed as | `a key` | `a key` | + +This fix also changes the path format in access logs from the single +URL-encoded style to the doubly-encoded URL style. Below is an example +of the old style: + +``` +127.0.0.1 - - [07/Jan/2015:08:27:07 +0000] "PUT /buckets/test/objects/path1%2Fpath2%2Fte%2Bst.txt HTTP/1.1" 200 0 "" "" +``` + +Below is an example of the new style: + +``` +127.0.0.1 - - [07/Jan/2015:08:27:07 +0000] "PUT /buckets/test/objects/path1%2Fpath2%2Fte%252Bst.txt HTTP/1.1" 200 0 "" "" +``` + +Note that the object path has changed from +`path1%2Fpath2%2Fte%2Bst.txt` to `path1%2Fpath2%2Fte%252Bst.txt` between +the two examples shown. + +If the old behavior is preferred, perhaps because +applications using Riak CS have been written to use it, you can retain +that behavior by modifying your Riak CS configuration upon upgrade. +Change the `rewrite_module` setting as follows: + +```erlang +{riak_cs, [ + %% Other settings + {rewrite_module, riak_cs_s3_rewrite_legacy}, + %% Other settings +]} +``` + +**Note**: The old behavior is technically incorrect and implicitly +overwrites data in the ways described above, so please retain the old +behavior with caution. + +# Riak CS 1.5.3 Release Notes + +## Additions + +- Add read_before_last_manifest_write option to help avoid sibling + explosion for use cases involving high churn and concurrency on a + fixed set of keys. [riak_cs/#1011](https://github.com/basho/riak_cs/pull/1011) +- Add configurable timeouts for all Riak CS interactions with Riak to + provide more flexibility in operational + situations. [riak_cs/#1021](https://github.com/basho/riak_cs/pull/1021) + +## Bugs Fixed + +- Fix storage usage calculation bug where data for deleted buckets + would be included in the calculation + results. [riak_cs/#996](https://github.com/basho/riak_cs/pull/996) + +# Riak CS 1.5.2 Release Notes + +## Additions + +- Improved logging around connection failures with Riak + [riak_cs/#987](https://github.com/basho/riak_cs/pull/987). +- Add amendment log output when storing access stats into Riak failed + [riak_cs/#988](https://github.com/basho/riak_cs/pull/988). This + prevents losing access stats logs in cases of temporary connection + failure between Riak and Riak CS. Access logs are stored in + `console.log` at the `warning` level. +- Add script to repair invalid garbage collection manifests + [riak_cs/#983](https://github.com/basho/riak_cs/pull/983). There is + a [known issue](https://github.com/basho/riak_cs/issues/827) where + an active manifest would be stored in the GC bucket. This script + changes invalid state to valid state. + +## Bugs Fixed + +- Fix Protocol Buffer connection pool (`pbc_pool_master`) leak + [riak_cs/#986](https://github.com/basho/riak_cs/pull/986) . Requests + for non-existent buckets without an authorization header and + requests asking for listing users make connections leak from the + pool, causing the pool to eventually go empty. This bug was introduced + in release 1.5.0. + +# Riak CS 1.5.1 Release Notes + +## Additions + +- Add sleep-after-update manifests to avoid sibling explosion [riak_cs/#959](https://github.com/basho/riak_cs/pull/959) +- Multibag support on `riak-cs-debug` [riak_cs/#930](https://github.com/basho/riak_cs/pull/930) +- Add bucket number limit check in Riak CS process [riak_cs/#950](https://github.com/basho/riak_cs/pull/950) +- More efficient bucket resolution [riak_cs/#951](https://github.com/basho/riak_cs/pull/951) + +## Bugs Fixed + +- GC may stall due to `riak_cs_delete_fsm` deadlock [riak_cs/#949](https://github.com/basho/riak_cs/pull/949) +- Fix wrong log directory for gathering logs on `riak-cs-debug` [riak_cs/#953](https://github.com/basho/riak_cs/pull/953) +- Avoid DST-aware translation from local time to GMT [riak_cs/#954](https://github.com/basho/riak_cs/pull/954) +- Use new UUID for seed of canonical ID instead of secret [riak_cs/#956](https://github.com/basho/riak_cs/pull/956) +- Add max part number limitation [riak_cs/#957](https://github.com/basho/riak_cs/pull/957) +- Set timeout as infinity to replace the default of 5000ms [riak_cs/#963](https://github.com/basho/riak_cs/pull/963) +- Skip invalid state manifests in GC bucket [riak_cs/#964](https://github.com/basho/riak_cs/pull/964) + +## Notes on Upgrading + +### Bucket number per user + +Beginning with Riak CS 1.5.1, you can limit the number of buckets that +can be created per user. The default maximum number is 100. While this +limitation prohibits the creation of new buckets by users, users that +exceed the limit can still perform other operations, including bucket +deletion. To change the default limit, add the following line to the +`riak_cs` section of `app.config`: + + +```erlang +{riak_cs, [ + %% ... + {max_buckets_per_user, 5000}, + %% ... + ]} +``` + +To avoid having a limit, set `max_buckets_per_user` to `unlimited`. + +# Riak CS 1.5.0 Release Notes + +## Additions + +* A new command `riak-cs-debug` including `cluster-info` [riak_cs/#769](https://github.com/basho/riak_cs/pull/769), [riak_cs/#832](https://github.com/basho/riak_cs/pull/832) +* Tie up all existing commands into a new command `riak-cs-admin` [riak_cs/#839](https://github.com/basho/riak_cs/pull/839) +* Add a command `riak-cs-admin stanchion` to switch Stanchion IP and port manually [riak_cs/#657](https://github.com/basho/riak_cs/pull/657) +* Performance of garbage collection has been improved via Concurrent GC [riak_cs/#830](https://github.com/basho/riak_cs/pull/830) +* Iterator refresh [riak_cs/#805](https://github.com/basho/riak_cs/pull/805) +* `fold_objects_for_list_keys` made default in Riak CS [riak_cs/#737](https://github.com/basho/riak_cs/pull/737), [riak_cs/#785](https://github.com/basho/riak_cs/pull/785) +* Add support for Cache-Control header [riak_cs/#821](https://github.com/basho/riak_cs/pull/821) +* Allow objects to be reaped sooner than leeway interval. [riak_cs/#470](https://github.com/basho/riak_cs/pull/470) +* PUT Copy on both objects and upload parts [riak_cs/#548](https://github.com/basho/riak_cs/pull/548) +* Update to lager 2.0.3 +* Compiles with R16B0x (Releases still by R15B01) +* Change default value of `gc_paginated_index` to `true` [riak_cs/#881](https://github.com/basho/riak_cs/issues/881) +* Add new API: Delete Multiple Objects [riak_cs/#728](https://github.com/basho/riak_cs/pull/728) +* Add warning logs for manifests, siblings, bytes and history [riak_cs/#915](https://github.com/basho/riak_cs/pull/915) + +## Bugs Fixed + +* Align `ERL_MAX_PORTS` with Riak default: 64000 [riak_cs/#636](https://github.com/basho/riak_cs/pull/636) +* Allow Riak CS admin resources to be used with OpenStack API [riak_cs/#666](https://github.com/basho/riak_cs/pull/666) +* Fix path substitution code to fix Solaris source builds [riak_cs/#733](https://github.com/basho/riak_cs/pull/733) +* `sanity_check(true,false)` logs invalid error on `riakc_pb_socket` error [riak_cs/#683](https://github.com/basho/riak_cs/pull/683) +* Riak-CS-GC timestamp for scheduler is in the year 0043, not 2013. [riak_cs/#713](https://github.com/basho/riak_cs/pull/713) fixed by [riak_cs/#676](https://github.com/basho/riak_cs/pull/676) +* Excessive calls to OTP code_server process #669 fixed by [riak_cs/#675](https://github.com/basho/riak_cs/pull/675) +* Return HTTP 400 if content-md5 does not match [riak_cs/#596](https://github.com/basho/riak_cs/pull/596) +* `/riak-cs/stats` and `admin_auth_enabled=false` don't work together correctly. [riak_cs/#719](https://github.com/basho/riak_cs/pull/719) +* Storage calculation doesn't handle tombstones, nor handle undefined manifest.props [riak_cs/#849](https://github.com/basho/riak_cs/pull/849) +* MP initiated objects remains after delete/create buckets #475 fixed by [riak_cs/#857](https://github.com/basho/riak_cs/pull/857) and [stanchion/#78](https://github.com/basho/stanchion/pull/78) +* handling empty query string on list multipart upload [riak_cs/#843](https://github.com/basho/riak_cs/pull/843) +* Setting ACLs via headers at PUT Object creation [riak_cs/#631](https://github.com/basho/riak_cs/pull/631) +* Improve handling of poolboy timeouts during ping requests [riak_cs/#763](https://github.com/basho/riak_cs/pull/763) +* Remove unnecessary log message on anonymous access [riak_cs/#876](https://github.com/basho/riak_cs/issues/876) +* Fix inconsistent ETag on objects uploaded by multipart [riak_cs/#855](https://github.com/basho/riak_cs/issues/855) +* Fix policy version validation in PUT Bucket Policy [riak_cs/#911](https://github.com/basho/riak_cs/issues/911) +* Fix return code of several commands, to return 0 for success [riak_cs/#908](https://github.com/basho/riak_cs/issues/908) +* Fix `{error, disconnected}` repainted with notfound [riak_cs/#929](https://github.com/basho/riak_cs/issues/929) + +## Notes on Upgrading + +### Riak Version + +This release of Riak CS was tested with Riak 1.4.10. Be sure to +consult the +[Compatibility Matrix](http://docs.basho.com/riakcs/latest/cookbooks/Version-Compatibility/) +to ensure that you are using the correct version. + +### Incomplete multipart uploads + +[riak_cs/#475](https://github.com/basho/riak_cs/issues/475) was a +security issue where a newly created bucket may include unaborted or +incomplete multipart uploads which was created in previous epoch of +the bucket with same name. This was fixed by: + +- on creating buckets; checking if live multipart exists and if + exists, return 500 failure to client. + +- on deleting buckets; trying to clean up all live multipart remains, + and checking if live multipart remains (in stanchion). if exists, + return 409 failure to client. + +Note that a few operations are needed after upgrading from 1.4.x (or +former) to 1.5.0. + +- run `riak-cs-admin cleanup-orphan-multipart` to cleanup all + buckets. To avoid some corner cases where multipart uploads can + conflict with bucket deletion, this command can also be run with a + timestamp with ISO 8601 format such as `2014-07-30T11:09:30.000Z` as + an argument. When this argument is provided, the cleanup operation + will not clean up multipart uploads that are newer than the provided + timestamp. If used, this should be set to a time when you expect + your upgrade to be completed. + +- there might be a time period until above cleanup finished, where no + client can create bucket if unfinished multipart upload remains + under deleted bucket. You can find [critical] log (`"Multipart + upload remains in deleted bucket "`) if such bucket + creation is attempted. + +### Leeway seconds and disk space + +[riak_cs/#470](https://github.com/basho/riak_cs/pull/470) changed the +behaviour of object deletion and garbage collection. The timestamps in +garbage collection bucket were changed from the future time when the +object is to be deleted, to the current time when the object is +deleted, Garbage collector was also changed to collect objects until +'now - leeway seconds', from collecting objects until 'now'. + +Before (-1.4.x): + +``` + t1 t2 +-----------+--------------------------+-------------------> + DELETE object: GC triggered: + marked as collects objects + "t1+leeway" marked as "t2" +``` + +After (1.5.0-): + +``` + t1 t2 +-----------+--------------------------+-------------------> + DELETE object: GC triggered: + marked as "t1" collects objects + in GC bucket marked as "t2 - leeway" +``` + +This means that there will be a period where no objects are collected +immediately following an upgrade to 1.5.0. If your cluster is upgraded +at `t0`, no objects will be cleaned up until `t0 + leeway` . Objects +deleted just before `t0` won't be collected until `t0 + 2*leeway` . + +Also, all CS nodes which run GC should be upgraded *first.* CS nodes +which do not run GC should be upgraded later, to ensure the leeway +setting is intiated properly. Alternatively, you may stop GC while +upgrading, by running `riak-cs-admin gc set-interval infinity` . + +Multi data center cluster should be upgraded more carefully, as to +make sure GC is not running while upgrading. + +## Known Issues and Limitations + +* If a second client request is made using the same connection while a + copy operation is in progress, the copy will be aborted. This is a + side effect of the way Riak CS currently handles client disconnect + detection. See [#932](https://github.com/basho/riak_cs/pull/932) for + further information. + +* Copying objects in OOS interface is not yet implemented. + +* Multibag, the ability to store object manifests and blocks in + separate clusters or groups of clusters, has been added as + Enterprise feature, but it is in early preview status. `proxy_get` + has not yet been implemented for this preview feature. Multibag is + intended for single DC only at this time. + +# Riak CS 1.4.5 Release Notes + +## Bugs Fixed + +* Fix several 'data hiding' bugs with the v2 list objects FSM [riak_cs/788](https://github.com/basho/riak_cs/pull/788) +* Don't treat HEAD requests toward BytesOut in access statistics [riak_cs/791](https://github.com/basho/riak_cs/pull/791) +* Handle whitespace in POST/PUT XML documents [riak_cs/795](https://github.com/basho/riak_cs/pull/795) +* Fix bad bucketname in storage usage [riak_cs/800](https://github.com/basho/riak_cs/pull/800) + Riak CS 1.4.4 introduced a bug where storage calculations made while running + that version would have the bucket-name replaced by the string "struct". This + version fixes the bug, but can't go back and retroactively fix the old + storage calculations. Aggregations on an entire user-account should still + be accurate, but you won't be able to break-down storage by bucket, as they + will all share the name "struct". +* Handle unicode user-names and XML [riak_cs/807](https://github.com/basho/riak_cs/pull/807) +* Fix missing XML fields on storage usage [riak_cs/808](https://github.com/basho/riak_cs/pull/808) +* Adjust fold-objects timeout [riak_cs/811](https://github.com/basho/riak_cs/pull/811) +* Prune deleted buckets from user record [riak_cs/812](https://github.com/basho/riak_cs/pull/812) + +## Additions + +* Optimize the list objects v2 FSM for prefix requests [riak_cs/804](https://github.com/basho/riak_cs/pull/804) + +# Riak CS 1.4.4 Release Notes + +This is a bugfix release. The major fixes are to the storage calculation. + +## Bugs Fixed + +* Create basho-patches directory [riak_cs/775](https://github.com/basho/riak_cs/issues/775) . + +* `sum_bucket` timeout crashes all storage calculation is fixed by [riak_cs/759](https://github.com/basho/riak_cs/issues/759) . + +* Failure to throttle access archiver is fixed by [riak_cs/758](https://github.com/basho/riak_cs/issues/758) . + +* Access archiver crash is fixed by [riak_cs/747](https://github.com/basho/riak_cs/issues/747) . + + +# Riak CS 1.4.3 Release Notes + +## Bugs Fixed + +- Fix bug that reverted manifests in the scheduled_delete state to the + pending_delete or active state. +- Don't count already deleted manifests as overwritten +- Don't delete current object version on overwrite with incorrect md5 + +## Additions + +- Improve performance of manifest pruning +- Optionally use paginated 2i for the GC daemon. This is to help prevent + timeouts when collecting data that can be garbage collected. +- Improve handling of Riak disconnects on block fetches +- Update to lager 2.0.1 +- Optionally prune manifests based on count, in addition to time +- Allow multiple access archiver processes to run concurrently + +# Riak CS 1.4.2 Release Notes + +## Bugs Fixed + +- Fix issue with Enterprise build on Debian Linux distributions. +- Fix source tarball build. +- Fix access statistics bug that caused all accesses to be treated as + errors. +- Make logging in bucket listing map phase function lager version + agnostic to avoid issues when using versions of Riak older than 1.4. +- Handle undefined `props` field in manifests to fix issue accessing + objects written with a version of Riak CS older than 1.3.0. + +## Additions + +- Add option to delay initial GC sweep on a node using the + initial_gc_delay configuration option. +- Append random suffix to GC bucket keys to avoid hot keys and improve + performance during periods of frequent deletion. +- Add default_proxy_cluster_id option to provide a way to specify a + default cluster id to be used when the cluster id is undefined. This is + to facilitate migration from the OSS version to the + Enterprise version. + +# Riak CS 1.4.1 Release Notes + +## Bugs Fixed + +- Fix list objects crash when more than the first 1001 keys are in + the pending delete state +- Fix crash in garbage collection daemon +- Fix packaging bug by updating node_package dependency + +# Riak CS 1.4.0 Release Notes + +## Bugs Fixed + +- Remove unnecessary keys in GC bucket +- Fix query-string authentication for multi-part uploads +- Fix Storage Class for multi-part uploaded objects +- Fix etags for multi-part uploads +- Support reformat indexes in the Riak CS multi-backend +- Fix unbounded memory-growth on GET requests with a slow connection +- Reduce access-archiver memory use +- Fix 500 on object ACL HEAD request +- Fix semantics for concurrent upload and delete of the same key with a + multi-part upload +- Verify content-md5 header if supplied +- Handle transient Riak connection failures + +## Additions + +- Add preliminary support for the Swift API and Keystone authentication +- Improve performance of object listing when using Riak 1.4.0 or greater +- Add ability to edit user account name and email address +- Add support for v3 multi-data-center replication +- Add configurable Riak connection timeouts +- Add syslog support via Lager +- Only contact one vnode for immutable block requests + +# Riak CS 1.3.1 Release Notes + +## Bugs Fixed +- Fix bug in handling of active object manifests in the case of + overwrite or delete that could lead to old object versions being + resurrected. +- Fix improper capitalization of user metadata header names. +- Fix issue where the S3 rewrite module omits any query parameters + that are not S3 subresources. Also correct handling of query + parameters so that parameter values are not URL decoded twice. This + primarily affects pre-signed URLs because the access key and request + signature are included as query parameters. +- Fix for issue with init script stop. + +# Riak CS 1.3.0 Release Notes + +## Bugs Fixed + +- Fix handling of cases where buckets have siblings. Previously this + resulted in 500 errors returned to the client. +- Reduce likelihood of sibling creation when creating a bucket. +- Return a 404 instead of a 403 when accessing a deleted object. +- Unquote URLs to accommodate clients that URL encode `/` characters + in URLs. +- Deny anonymous service-level requests to avoid unnecessary error + messages trying to list the buckets owned by an undefined user. + +## Additions + +- Support for multipart file uploads. Parts must be in the range of + 5MB-5GB. +- Support for bucket policies using a restricted set of principals and + conditions. +- Support for returning bytes ranges of a file using the Range header. +- Administrative commands may be segrated onto a separate interface. +- Authentication for administrative commands may be disabled. +- Performance and stability improvements for listing the contents of + buckets. +- Support for the prefix, delimiter, and marker options when listing + the contents of a bucket. +- Support for using Webmachine's access logging features in + conjunction with the Riak CS internal access logging mechanism. +- Moved all administrative resources under /riak-cs. +- Riak CS now supports packaging for FreeBSD, SmartOS, and Solaris. + +# Riak CS 1.2.2 Release Notes + +## Bugs Fixed + +- Fix problem where objects with utf-8 unicode key cannot be listed + nor fetched. +- Speed up bucket_empty check and fix process leak. This bug was + originally found when a user was having trouble with `s3cmd + rb s3://foo --recursive`. The operation first tries to delete the + (potentially large) bucket, which triggers our bucket empty + check. If the bucket has more than 32k items, we run out of + processes unless +P is set higher (because of the leak). + +## Additions + +- Full support for MDC replication + +# Riak CS 1.2.1 Release Notes + +## Bugs Fixed + +- Return 403 instead of 404 when a user attempts to list contents of + nonexistent bucket. +- Do not do bucket list for HEAD or ?versioning or ?location request. + +## Additions + +- Add reduce phase for listing bucket contents to provide backpressure + when executing the MapReduce job. +- Use prereduce during storage calculations. +- Return 403 instead of 404 when a user attempts to list contents of + nonexistent bucket. + +# Riak CS 1.2.0 Release Notes + +## Bugs Fixed + +- Do not expose stack traces to users on 500 errors +- Fix issue with sibling creation on user record updates +- Fix crash in terminate state when fsm state is not fully populated +- Script fixes and updates in response to node_package updates + +## Additions + +- Add preliminary support for MDC replication +- Quickcheck test to exercise the erlcloud library against Riak CS +- Basic support for riak_test integration + +# Riak CS 1.1.0 Release Notes + +## Bugs Fixed + +- Check for timeout when checking out a connection from poolboy. +- PUT object now returns 200 instead of 204. +- Fixes for Dialyzer errors and warnings. +- Return readable error message with 500 errors instead of large webmachine backtraces. + +## Additions + +- Update user creation to accept a JSON or XML document for user + creation instead of URL encoded text string. +- Configuration option to allow anonymous users to create accounts. In + the default mode, only the administrator is allowed to create + accounts. +- Ping resource for health checks. +- Support for user-specified metadata headers. +- User accounts may be disabled by the administrator. +- A new key_secret can be issued for a user by the administrator. +- Administrator can now list all system users and optionally filter by + enabled or disabled account status. +- Garbage collection for deleted and overwritten objects. +- Separate connection pool for object listings with a default of 5 + connections. +- Improved performance for listing all objects in a bucket. +- Statistics collection and querying. +- DTrace probing. + +# Riak CS 1.0.2 Release Notes + +## Additions + +- Support query parameter authentication as specified in [Signing and Authenticating REST Requests](http://docs.amazonwebservices.com/AmazonS3/latest/dev/RESTAuthentication.html) + +# Riak CS 1.0.1 Release Notes + +## Bugs Fixed + +- Default content-type is not passed into function to handle PUT + request body +- Requests hang when a node in the Riak cluster is unavailable +- Correct inappropriate use of riak_moss_utils:get_user by + riak_moss_acl_utils:get_owner_data + +# Riak CS 1.0.0 Release Notes + +## Bugs Fixed + +- Fix PUTs for zero-byte files +- Fix fsm initialization race conditions +- Canonicalize the entire path if there is no host header, but there are + tokens +- Fix process and socket leaks in get fsm + +## Other Additions + +- Subsystem for calculating user access and storage usage +- Fixed-size connection pool of Riak connections +- Use a single Riak connection per request to avoid deadlock conditions +- Object ACLs +- Management for multiple versions of a file manifest +- Configurable block size and max content length +- Support specifying non-default ACL at bucket creation time + +# Riak CS 0.1.2 Release Notes + +## Bugs Fixed + +- Return 403 instead of 503 for invalid anonymous or signed requests. +- Properly clean up processes and connections on object requests. + +# Riak CS 0.1.1 Release Notes + +## Bugs Fixed + +- HEAD requests always result in a `403 Forbidden`. +- `s3cmd info` on a bucket object results in an error due to missing + ACL document. +- Incorrect atom specified in `riak_moss_wm_utils:parse_auth_header`. +- Bad match condition used in `riak_moss_acl:has_permission/2`. + +# Riak CS 0.1.0 Release Notes + +## Bugs Fixed + +- `s3cmd info` fails due to missing `'last-modified` key in return document. +- `s3cmd get` of 0 byte file fails. +- Bucket creation fails with status code `415` using the AWS Java SDK. + +## Other Additions + +- Bucket-level access control lists +- User records have been modified so that an system-wide unique email + address is required to create a user. +- User creation requests are serialized through `stanchion` to be + certain the email address is unique. +- Bucket creation and deletion requests are serialized through + `stanchion` to ensure bucket names are unique in the system. +- The `stanchion` serialization service is now required to be installed + and running for the system to be fully operational. +- The concept of an administrative user has been added to the system. The credentials of the + administrative user must be added to the app.config files for `moss` and `stanchion`. +- User credentials are now created using a url-safe base64 encoding module. + +## Known Issues + +- Object-level access control lists have not yet been implemented. + +# Riak CS 0.0.3 Release Notes + +## Bugs Fixed + +- URL decode keys on put so they are represented correctly. This + eliminates confusion when objects with spaces in their names are + listed and when attempting to access them. +- Properly handle zero-byte files +- Reap all processes during file puts + +## Other Additions + +- Support for the s3cmd subcommands sync, du, and rb + + - Return valid size and checksum for each object when listing bucket objects. + - Changes so that a bucket may be deleted if it is empty. + +- Changes so a subdirectory path can be specified when storing or retrieving files. +- Make buckets private by default +- Support the prefix query parameter + +- Enhance process dependencies for improved failure handling + +## Known Issues + +- Buckets are marked as /private/ by default, but globally-unique + bucket names are not enforced. This means that two users may + create the same bucket and this could result in unauthorized + access and unintentional overwriting of files. This will be + addressed in a future release by ensuring that bucket names are + unique across the system. + + [riak_1.4_release_notes]: https://github.com/basho/riak/blob/1.4/RELEASE-NOTES.md [riak_2.0_release_notes]: https://github.com/basho/riak/blob/2.0/RELEASE-NOTES.md @@ -715,5 +1922,3 @@ translate data files. See also the [2.0 downgrade notes][downgrade_notes]. [upgrading_your_configuration]: http://docs.basho.com/riak/2.0.5/upgrade-v20/#Upgrading-Your-Configuration-System [storage_statistics]: http://docs.basho.com/riakcs/latest/cookbooks/Usage-and-Billing-Data/#Storage-Statistics [downgrade_notes]: https://github.com/basho/riak/wiki/2.0-downgrade-notes - - diff --git a/apps/riak_cs/ebin b/apps/riak_cs/ebin deleted file mode 120000 index 4037435d7..000000000 --- a/apps/riak_cs/ebin +++ /dev/null @@ -1 +0,0 @@ -../../ebin/ \ No newline at end of file diff --git a/apps/riak_cs/include b/apps/riak_cs/include deleted file mode 120000 index 3611dd266..000000000 --- a/apps/riak_cs/include +++ /dev/null @@ -1 +0,0 @@ -../../include/ \ No newline at end of file diff --git a/apps/riak_cs/include/aws_api.hrl b/apps/riak_cs/include/aws_api.hrl new file mode 100644 index 000000000..936973c1c --- /dev/null +++ b/apps/riak_cs/include/aws_api.hrl @@ -0,0 +1,368 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-ifndef(RIAK_CS_AWS_API_HRL). +-define(RIAK_CS_AWS_API_HRL, included). + +-include_lib("public_key/include/public_key.hrl"). + +-type aws_service() :: s3 | iam | sts. + + +%% ACL ============= + +-type acl_perm() :: 'READ' | 'WRITE' | 'READ_ACP' | 'WRITE_ACP' | 'FULL_CONTROL'. +-type group_grant() :: 'AllUsers' | 'AuthUsers'. + +-type acl_owner_very_old() :: {string(), string()}. +-type acl_owner_old() :: {string(), string(), string()}. +-type acl_owner() :: #{display_name => undefined | binary(), + canonical_id => undefined | binary(), + email => undefined | binary(), + key_id => undefined | binary()}. + +-type acl_grantee_very_old() :: acl_owner_very_old() | group_grant(). +-type acl_grantee_old() :: acl_owner_old() | group_grant(). +-type acl_grantee() :: acl_owner() | group_grant(). + +-record(acl_grant_v2, { grantee :: undefined | binary() | acl_grantee() + , perms = [] :: [binary() | acl_perm()] + } + ). + +-type acl_grant_old() :: {acl_grantee_old() | acl_grantee_very_old(), [acl_perm()]}. +-type acl_grant() :: #acl_grant_v2{}. +-define(ACL_GRANT, #acl_grant_v2). + + +-record(acl_v1, {owner = {"", ""} :: acl_owner_very_old(), + grants = [] :: [acl_grant_old()], + creation_time = erlang:timestamp() :: erlang:timestamp()}). + +%% %% acl_v2 owner fields: {DisplayName, CanonicalId, KeyId} +-record(acl_v2, {owner = {"", "", ""} :: acl_owner_old(), + grants = [] :: [acl_grant_old()], + creation_time = erlang:timestamp() :: erlang:timestamp()}). + +-record(acl_v3, { owner :: undefined | acl_owner() + , grants = [] :: [#{} | acl_grant()] + , creation_time = os:system_time(millisecond) :: non_neg_integer() + } + ). + +-type acl() :: #acl_v3{}. +-define(ACL, #acl_v3). + + +%% Policies ============= + +-define(SUBRESOURCES, ["acl", "location", "logging", "notification", "partNumber", + "policy", "requestPayment", "torrent", "uploadId", "uploads", + "versionId", "versioning", "versions", "website", + "delete", "lifecycle"]). + +%% type and record definitions for S3 policy API +-type s3_object_action() :: 's3:GetObject' | 's3:GetObjectVersion' + | 's3:GetObjectAcl' | 's3:GetObjectVersionAcl' + | 's3:PutObject' | 's3:PutObjectAcl' + | 's3:PutObjectVersionAcl' + | 's3:DeleteObject' | 's3:DeleteObjectVersion' + | 's3:ListObjectVersions' + | 's3:ListMultipartUploadParts' + | 's3:AbortMultipartUpload' + %%| 's3:GetObjectTorrent' we never do this + %%| 's3:GetObjectVersionTorrent' we never do this + | 's3:RestoreObject'. + +-define(SUPPORTED_OBJECT_ACTIONS, + [ 's3:GetObject', 's3:GetObjectAcl', 's3:PutObject', 's3:PutObjectAcl', + 's3:DeleteObject', + 's3:ListObjectVersions', + 's3:ListMultipartUploadParts', 's3:AbortMultipartUpload' + ]). + +-type s3_bucket_action() :: 's3:CreateBucket' + | 's3:DeleteBucket' + | 's3:ListBucket' + | 's3:ListBucketVersions' + | 's3:ListAllMyBuckets' + | 's3:ListBucketMultipartUploads' + | 's3:GetBucketAcl' | 's3:PutBucketAcl' + | 's3:GetBucketVersioning' | 's3:PutBucketVersioning' + | 's3:GetBucketRequestPayment' | 's3:PutBucketRequestPayment' + | 's3:GetBucketLocation' + | 's3:GetBucketPolicy' | 's3:DeleteBucketPolicy' | 's3:PutBucketPolicy' + | 's3:GetBucketNotification' | 's3:PutBucketNotification' + | 's3:GetBucketLogging' | 's3:PutBucketLogging' + | 's3:GetBucketWebsite' | 's3:PutBucketWebsite' | 's3:DeleteBucketWebsite' + | 's3:GetLifecycleConfiguration' | 's3:PutLifecycleConfiguration'. + +-define(SUPPORTED_BUCKET_ACTIONS, + [ 's3:CreateBucket', 's3:DeleteBucket', 's3:ListBucket', 's3:ListAllMyBuckets', + 's3:GetBucketAcl', 's3:PutBucketAcl', + 's3:GetBucketPolicy', 's3:DeleteBucketPolicy', 's3:PutBucketPolicy', + 's3:GetBucketVersioning', 's3:PutBucketVersioning', + 's3:ListBucketMultipartUploads']). + +-type s3_action() :: s3_bucket_action() | s3_object_action(). + +-define(SUPPORTED_S3_ACTIONS, ?SUPPORTED_BUCKET_ACTIONS ++ ?SUPPORTED_OBJECT_ACTIONS). + + +-type iam_action() :: 'iam:CreateUser' | 'iam:GetUser' | 'iam:DeleteUser' | 'iam:ListUsers' + | 'iam:CreateRole' | 'iam:GetRole' | 'iam:DeleteRole' | 'iam:ListRoles' + | 'iam:CreatePolicy' | 'iam:GetPolicy' | 'iam:DeletePolicy' | 'iam:ListPolicies' + | 'iam:AttachRolePolicy' | 'iam:DetachRolePolicy' + | 'iam:AttachUserPolicy' | 'iam:DetachUserPolicy' + | 'iam:CreateSAMLProvider' | 'iam:GetSAMLProvider' | 'iam:DeleteSAMLProvider' | 'iam:ListSAMLProviders'. + +-define(SUPPORTED_IAM_ACTIONS, + [ 'iam:CreateUser', 'iam:GetUser', 'iam:DeleteUser', 'iam:ListUsers' + , 'iam:CreateRole', 'iam:GetRole', 'iam:DeleteRole', 'iam:ListRoles' + , 'iam:CreatePolicy', 'iam:GetPolicy', 'iam:DeletePolicy', 'iam:ListPolicies' + , 'iam:AttachRolePolicy', 'iam:DetachRolePolicy' + , 'iam:AttachUserPolicy', 'iam:DetachUserPolicy' + , 'iam:CreateSAMLProvider', 'iam:GetSAMLProvider', 'iam:DeleteSAMLProvider', 'iam:ListSAMLProviders' + ] + ). + +-type sts_action() :: 'sts:AssumeRoleWithSAML'. + +-define(SUPPORTED_STS_ACTIONS, + [ 'sts:AssumeRoleWithSAML' + ] + ). + +-type aws_action() :: s3_action() | iam_action() | sts_action() + | binary(). %% actions like "s3:Get*' + +-define(SUPPORTED_ACTIONS, ?SUPPORTED_S3_ACTIONS ++ ?SUPPORTED_IAM_ACTIONS ++ ?SUPPORTED_STS_ACTIONS). + + +%% one of string, numeric, date&time, boolean, IP address, ARN and existence of condition keys +-type string_condition_type() :: 'StringEquals' | streq | 'StringNotEquals' | strneq + | 'StringEqualsIgnoreCase' | streqi | 'StringNotEqualsIgnoreCase' | strneqi + | 'StringLike' | strl | 'StringNotLike' | strnl. + +-define(STRING_CONDITION_ATOMS, + [ 'StringEquals' , streq, 'StringNotEquals', strneq, + 'StringEqualsIgnoreCase', streqi, 'StringNotEqualsIgnoreCase', strneqi, + 'StringLike', strl, 'StringNotLike' , strnl]). + +-type numeric_condition_type() :: 'NumericEquals' | numeq | 'NumericNotEquals' | numneq + | 'NumericLessThan' | numlt | 'NumericLessThanEquals' | numlteq + | 'NumericGreaterThan' | numgt | 'NumericGreaterThanEquals' | numgteq. + +-define(NUMERIC_CONDITION_ATOMS, + [ 'NumericEquals', numeq, 'NumericNotEquals', numneq, + 'NumericLessThan' , numlt, 'NumericLessThanEquals', numlteq, + 'NumericGreaterThan', numgt, 'NumericGreaterThanEquals', numgteq]). + +-type date_condition_type() :: 'DateEquals' | dateeq + | 'DateNotEquals' | dateneq + | 'DateLessThan' | datelt + | 'DateLessThanEquals' | datelteq + | 'DateGreaterThan' | dategt + | 'DateGreaterThanEquals' | dategteq. + +-define(DATE_CONDITION_ATOMS, + [ 'DateEquals', dateeq + , 'DateNotEquals', dateneq + , 'DateLessThan', datelt + , 'DateLessThanEquals', datelteq + , 'DateGreaterThan', dategt + , 'DateGreaterThanEquals', dategteq + ] + ). + + +-type ip_addr_condition_type() :: 'IpAddress' | 'NotIpAddress'. + +-define(IP_ADDR_CONDITION_ATOMS, + ['IpAddress', 'NotIpAddress']). + +-type condition_pair() :: {date_condition_type(), [{'aws:CurrentTime', binary()}]} + | {numeric_condition_type(), [{'aws:EpochTime', non_neg_integer()}]} + | {boolean(), 'aws:SecureTransport'} + | {ip_addr_condition_type(), [{'aws:SourceIp', {IP::inet:ip_address(), inet:ip_address()}}]} + | {string_condition_type(), [{'aws:UserAgent', binary()}]} + | {string_condition_type(), [{'aws:Referer', binary()}]}. + + +-record(arn_v1, { provider = aws :: aws + , service = s3 :: aws_service() + , region :: binary() + , id :: binary() + , path :: binary() + } + ). + +-type arn() :: #arn_v1{}. +-define(S3_ARN, #arn_v1). + +-type flat_arn() :: binary(). + +-type principal_id() :: '*' | binary(). +-type one_or_many(A) :: A | [A]. +-type principal() :: '*' + | [ {canonical_user, one_or_many(principal_id())} + | {federated, one_or_many(principal_id())} + | {service, one_or_many(principal_id())} + | {aws, one_or_many(principal_id())} + ]. + +-record(statement, { sid :: undefined | binary() + , effect = deny :: allow | deny + , principal = [] :: principal() + , action = [] :: aws_action() | [aws_action()] + , not_action = [] :: aws_action() | [aws_action()] + , resource = [] :: [ flat_arn() ] | '*' + , condition_block = [] :: [ condition_pair() ] + } + ). +-define(S3_STATEMENT, #statement). + +-define(POLICY_VERSION_2008, <<"2008-10-17">>). +-define(POLICY_VERSION_2012, <<"2012-10-17">>). +-define(POLICY_VERSION_2020, <<"2020-10-17">>). + +-record(policy, { version = ?POLICY_VERSION_2012 :: binary() + , id = undefined :: undefined | binary() % had better use uuid: should be UNIQUE + , statement = [] :: [#statement{}] + , creation_time = os:system_time(millisecond) :: non_neg_integer() + }). +-type policy() :: #policy{}. +-define(POLICY, #policy). + + +%% IAM entities ============= + +-record(iam_policy, { arn :: undefined | flat_arn() + , path = <<"/">> :: binary() + , attachment_count = 0 :: non_neg_integer() + , create_date = os:system_time(millisecond) :: non_neg_integer() + , default_version_id = <<"v1">> :: binary() + , description = <<>> :: binary() + , is_attachable = true :: boolean() + , permissions_boundary_usage_count = 0 :: non_neg_integer() + , policy_document :: undefined | binary() + , policy_id :: undefined | binary() + , policy_name :: undefined | binary() + , tags = [] :: [#{} | tag()] + , update_date = os:system_time(millisecond) :: non_neg_integer() + }). +-type iam_policy() :: #iam_policy{}. +-define(IAM_POLICY, #iam_policy). + + +-record(permissions_boundary, { permissions_boundary_arn :: undefined | flat_arn() + , permissions_boundary_type = <<"Policy">> :: undefined | binary() + } +). +-type permissions_boundary() :: #permissions_boundary{}. +-define(IAM_PERMISSION_BOUNDARY, #permissions_boundary). + + +-record(tag, { key :: undefined | binary() + , value :: undefined | binary() + } + ). +-type tag() :: #tag{}. +-define(IAM_TAG, #tag). + + +-record(role_last_used, { last_used_date :: undefined | non_neg_integer() + , region :: undefined | binary() + } + ). +-type role_last_used() :: #role_last_used{}. +-define(IAM_ROLE_LAST_USED, #role_last_used). + +-record(role_v1, { arn :: undefined | flat_arn() + , path = <<"/">> :: binary() + , assume_role_policy_document :: undefined | binary() + , create_date = os:system_time(millisecond) :: non_neg_integer() + , description :: undefined | binary() + , max_session_duration :: undefined | non_neg_integer() + , permissions_boundary :: undefined | #{} | flat_arn() %% permissions_boundary() + , role_id :: undefined | binary() + , role_last_used :: undefined | #{} | role_last_used() + , role_name :: undefined | binary() + , tags = [] :: [#{} | tag()] + , attached_policies :: undefined | [flat_arn()] + } + ). +-type role() :: #role_v1{}. +-define(IAM_ROLE, #role_v1). + +-record(saml_provider_v1, { arn :: undefined | flat_arn() + , saml_metadata_document :: undefined | binary() + , tags = [] :: [#{} | tag()] + , name :: undefined | binary() + %% fields populated with values extracted from MD document + , create_date = os:system_time(millisecond) :: non_neg_integer() + , valid_until :: undefined | non_neg_integer() + , entity_id :: undefined | binary() + , consume_uri :: undefined | binary() + , certificates :: undefined | [{signing|encryption, #'OTPCertificate'{}, FP::binary()}] + } + ). +-type saml_provider() :: #saml_provider_v1{}. +-define(IAM_SAML_PROVIDER, #saml_provider_v1). + + +-record(assumed_role_user, { arn :: undefined | flat_arn() + , assumed_role_id :: undefined | binary() + } + ). +-type assumed_role_user() :: #assumed_role_user{}. + +-record(credentials, { access_key_id :: undefined | binary() + , expiration :: undefined | non_neg_integer() + , secret_access_key :: undefined | binary() + , session_token :: undefined | binary() + } + ). +-type credentials() :: #credentials{}. + + +-type iam_entity() :: role | policy | user. + +-define(ROLE_ID_PREFIX, "AROA"). +-define(USER_ID_PREFIX, "AIDA"). +-define(POLICY_ID_PREFIX, "ANPA"). + +-define(IAM_ENTITY_ID_LENGTH, 21). %% length("AROAJQABLZS4A3QDU576Q"). + + +-define(S3_ROOT_HOST, "s3.amazonaws.com"). +-define(IAM_ROOT_HOST, "iam.amazonaws.com"). +-define(STS_ROOT_HOST, "sts.amazonaws.com"). + +-define(DEFAULT_REGION, "us-east-1"). + +-define(AUTH_USERS_GROUP, "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"). +-define(ALL_USERS_GROUP, "http://acs.amazonaws.com/groups/global/AllUsers"). +-define(LOG_DELIVERY_GROUP, "http://acs.amazonaws.com/groups/s3/LogDelivery"). + +-define(IAM_CREATE_USER_DEFAULT_EMAIL_HOST, "my-riak-cs-megacorp.com"). + +-endif. diff --git a/apps/riak_cs/include/manifest.hrl b/apps/riak_cs/include/manifest.hrl new file mode 100644 index 000000000..4dc8d81bd --- /dev/null +++ b/apps/riak_cs/include/manifest.hrl @@ -0,0 +1,387 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-ifndef(RCS_COMMON_MANIFEST_HRL). +-define(RCS_COMMON_MANIFEST_HRL, included). + +-include("aws_api.hrl"). + + +-define(MANIFEST, #lfs_manifest_v5). +-define(MULTIPART_MANIFEST, #multipart_manifest_v2). +-define(PART_MANIFEST, #part_manifest_v3). +-define(MULTIPART_DESCR, #multipart_descr_v2). +-define(PART_DESCR, #part_descr_v2). + +-define(LFS_DEFAULT_OBJECT_VERSION, <<"null">>). + + +-type lfs_manifest_state() :: writing | active | + pending_delete | scheduled_delete | deleted. + + +-type cluster_id() :: undefined | binary(). %% flattened string as binary +-type cs_uuid() :: binary(). +-type bag_id() :: undefined | binary(). + +-record(lfs_manifest_v2, { + version = 2 :: 2, + block_size :: integer(), + bkey :: {binary(), binary()}, + metadata :: orddict:orddict(), + created :: string(), + uuid :: cs_uuid(), + content_length :: non_neg_integer(), + content_type :: binary(), + content_md5 :: term(), + state = undefined :: undefined | lfs_manifest_state(), + write_start_time :: term(), %% immutable + last_block_written_time :: term(), + write_blocks_remaining :: ordsets:ordset(integer()), + delete_marked_time :: term(), + last_block_deleted_time :: term(), + delete_blocks_remaining :: ordsets:ordset(integer()), + acl :: #acl_v1{} | #acl_v2{}, + props = [] :: proplists:proplist(), + cluster_id :: cluster_id() + }). + +-record(lfs_manifest_v3, { + version = 3 :: 3, + block_size :: undefined | integer(), + bkey :: {binary(), binary()}, + metadata :: orddict:orddict(), + created :: string(), + uuid :: cs_uuid(), + content_length :: non_neg_integer(), + content_type :: binary(), + content_md5 :: term(), + state :: undefined | lfs_manifest_state(), + write_start_time :: term(), %% immutable + last_block_written_time :: term(), + write_blocks_remaining :: undefined | ordsets:ordset(integer()), + delete_marked_time :: term(), + last_block_deleted_time :: term(), + delete_blocks_remaining :: undefined | ordsets:ordset(integer()), + scheduled_delete_time :: term(), %% new in v3 + acl = no_acl_yet :: #acl_v1{} | #acl_v2{} | no_acl_yet, + props = [] :: undefined | proplists:proplist(), + cluster_id :: cluster_id() + }). + +-record(lfs_manifest_v4, { + version = 4 :: 4, + block_size :: undefined | integer(), + bkey :: {binary(), binary()}, + vsn = ?LFS_DEFAULT_OBJECT_VERSION :: binary(), + metadata :: orddict:orddict(), + created :: string(), + uuid :: cs_uuid(), + content_length :: non_neg_integer(), + content_type :: binary(), + content_md5 :: term(), + state :: undefined | lfs_manifest_state(), + write_start_time :: term(), %% immutable + last_block_written_time :: term(), + write_blocks_remaining :: undefined | ordsets:ordset(integer()), + delete_marked_time :: term(), + last_block_deleted_time :: term(), + delete_blocks_remaining :: undefined | ordsets:ordset(integer()), + scheduled_delete_time :: term(), + acl = no_acl_yet :: #acl_v2{} | no_acl_yet, + props = [] :: proplists:proplist(), + cluster_id :: cluster_id() + }). + +-record(lfs_manifest_v5, { + %% the block_size setting when this manifest + %% was written. Needed if the user + %% ever changes the block size after writing + %% data + block_size :: undefined | integer(), + + %% identifying properties + %% ----------------------------------------------------------------- + bkey :: {binary(), binary()}, + %% added in v4: + %% there's always a primary version, which is head of a + %% double-linked list of all versions + vsn = ?LFS_DEFAULT_OBJECT_VERSION :: binary(), + + %% user metadata that would normally + %% be placed on the riak_object. We avoid + %% putting it on the riak_object so that + %% we can use that metadata ourselves + metadata :: orddict:orddict(), + + %% the date the manifest was created. + %% not sure if we need both this and + %% write_start_time. My thought was that + %% write_start_time would have millisecond + %% resolution, but I suppose there's no + %% reason we can't change created + %% to have millisecond as well. + %% created :: string(), + + uuid :: cs_uuid(), + + %% content properties + %% ----------------------------------------------------------------- + content_length :: non_neg_integer(), + content_type :: binary(), + content_md5 :: term(), + + %% state properties + %% ----------------------------------------------------------------- + state :: undefined | lfs_manifest_state(), + + %% writing/active state + %% ----------------------------------------------------------------- + write_start_time = os:system_time(millisecond) :: non_neg_integer(), %% immutable + + %% used for two purposes + %% 1. to mark when a file has finished uploading + %% 2. to decide if a write crashed before completing + %% and needs to be garbage collected + last_block_written_time :: undefined | non_neg_integer(), + + %% a shrink-only (during resolution) + %% set to denote which blocks still + %% need to be written. We use a shrinking + %% (rather than growing) set to that the + %% set is empty after the write has completed, + %% which should be most of the lifespan on disk + write_blocks_remaining :: undefined | ordsets:ordset(integer()), + + %% pending_delete/deleted state + %% ----------------------------------------------------------------- + %% set to the current time + %% when a manifest is marked as deleted + %% and enters the pending_delete state + delete_marked_time :: undefined | non_neg_integer(), + + %% the timestamp serves a similar + %% purpose to last_block_written_time, + %% in that it's used for figuring out + %% when delete processes have died + %% and garbage collection needs to + %% pick up where they left off. + last_block_deleted_time :: undefined | non_neg_integer(), + + %% a shrink-only (during resolution) + %% set to denote which blocks + %% still need to be deleted. + %% See write_blocks_remaining for + %% an explanation of why we chose + %% a shrinking set + delete_blocks_remaining :: undefined | ordsets:ordset(integer()), + + %% the time the manifest was put + %% into the scheduled_delete + %% state + scheduled_delete_time :: undefined | non_neg_integer(), + + %% The ACL for the version of the object represented + %% by this manifest. + acl = no_acl_yet :: #acl_v3{} | no_acl_yet, + + %% There are a couple of cases where we want to add record + %% member'ish data without adding new members to the record, + %% e.g. + %% 1. Data for which the common value is 'undefined' or not + %% used/set for this particular manifest + %% 2. Cases where we do want to change the structure of the + %% record but don't want to go through the full code + %% refactoring and backward-compatibility tap dance + %% until sometime later. + %% 'undefined' is for backward compatibility with v3 manifests + %% written with Riak CS 1.2.2 or earlier. + props = [] :: proplists:proplist(), + + %% cluster_id: A couple of uses, both short- and longer-term + %% possibilities: + %% + %% 1. We don't have a good story in early 2012 for how to + %% build a stable 2,000 node Riak cluster. If MOSS can + %% talk to multiple Riak clusters, then each individual + %% cluster can be a size that we're comfortable + %% supporting. + %% + %% 2. We may soon have Riak EE's replication have full + %% plumbing to make it feasible to forward arbitrary + %% traffic between clusters. Then if a slave cluster is + %% missing a data block, and read-repair cannot + %% automagically fix the 'not_found' problem, then perhaps + %% forwarding a get request to the source Riak cluster can + %% fetch us the missing data. + cluster_id :: cluster_id() + }). + +-type lfs_manifest() :: #lfs_manifest_v5{}. + +-type cs_uuid_and_manifest() :: {cs_uuid(), lfs_manifest()}. +-type wrapped_manifest() :: orddict:orddict(cs_uuid(), lfs_manifest()). + +-record(part_manifest_v1, { + bucket :: binary(), + key :: binary(), + start_time :: erlang:timestamp(), + part_number :: integer(), + part_id :: binary(), + content_length :: integer(), + content_md5 :: undefined | binary(), + block_size :: integer() +}). + +-record(part_manifest_v2, { + bucket :: binary(), + key :: binary(), + %% new in v2 + vsn = ?LFS_DEFAULT_OBJECT_VERSION :: binary(), + start_time :: erlang:timestamp(), + part_number :: integer(), + part_id :: binary(), + content_length :: integer(), + content_md5 :: undefined | binary(), + block_size :: integer() +}). + +-record(part_manifest_v3, { + bucket :: binary(), + key :: binary(), + + vsn = ?LFS_DEFAULT_OBJECT_VERSION :: binary(), + + %% used to judge races between concurrent uploads + %% of the same part_number + start_time :: non_neg_integer(), + + %% one-of 1-10000, inclusive + part_number :: integer(), + + %% a UUID to prevent conflicts with concurrent + %% uploads of the same {upload_id, part_number}. + part_id :: binary(), + + %% each individual part upload always has a content-length + %% content_md5 is used for the part ETag, alas. + content_length :: integer(), + content_md5 :: undefined | binary(), + + %% block size just like in `lfs_manifest_v2'. Concievably, + %% parts for the same upload id could have different block_sizes. + block_size :: integer() +}). +-type part_manifest() :: #part_manifest_v3{}. + + +-record(multipart_manifest_v1, { + upload_id :: binary(), + owner :: acl_owner_old(), + parts = ordsets:new() :: ordsets:ordset(#part_manifest_v2{}), + done_parts = ordsets:new() :: ordsets:ordset({binary(), binary()}), + cleanup_parts = ordsets:new() :: ordsets:ordset(#part_manifest_v2{}), + props = [] :: proplists:proplist() +}). + +-record(multipart_manifest_v2, { + upload_id :: binary(), + owner :: acl_owner(), + + %% since we don't have any point of strong + %% consistency (other than stanchion), we + %% can get concurrent `complete' and `abort' + %% requests. There are still some details to + %% work out, but what we observe here will + %% affect whether we accept future `complete' + %% or `abort' requests. + + %% Stores references to all of the parts uploaded + %% with this `upload_id' so far. A part + %% can be uploaded more than once with the same + %% part number. type = #part_manifest_vX + parts = ordsets:new() :: ordsets:ordset(#part_manifest_v3{}), + %% List of UUIDs for parts that are done uploading. + %% The part number is redundant, so we only store + %% {UUID::binary(), PartETag::binary()} here. + done_parts = ordsets:new() :: ordsets:ordset({binary(), binary()}), + %% type = #part_manifest_vX + cleanup_parts = ordsets:new() :: ordsets:ordset(#part_manifest_v3{}), + + %% a place to stuff future information + %% without having to change + %% the record format + props = [] :: proplists:proplist() +}). + +-type multipart_manifest() :: #multipart_manifest_v2{}. + + +%% Basis of list multipart uploads output +-record(multipart_descr_v1, { + key :: binary(), + upload_id :: binary(), + owner_display :: string(), + owner_key_id :: string(), + storage_class = standard, + initiated :: string() +}). + +-record(multipart_descr_v2, { + %% Object key for the multipart upload + key :: binary(), + + %% UUID of the multipart upload + upload_id :: binary(), + + %% User that initiated the upload + owner_display :: binary(), + owner_key_id :: binary(), + + %% storage class: no real options here + storage_class = standard, + + %% Time that the upload was initiated + initiated :: non_neg_integer() +}). + +-type multipart_descr() :: #multipart_descr_v2{}. + + +%% Basis of multipart list parts output +-record(part_descr_v1, { + part_number :: integer(), + last_modified :: erlang:timestamp(), + etag :: binary(), + size :: integer() +}). + +-record(part_descr_v2, { + part_number :: integer(), + last_modified :: non_neg_integer(), + etag :: binary(), + size :: integer() +}). + +-type part_descr() :: #part_descr_v2{}. + + +-endif. diff --git a/apps/riak_cs/include/moss.hrl b/apps/riak_cs/include/moss.hrl new file mode 100644 index 000000000..a7e0c5cce --- /dev/null +++ b/apps/riak_cs/include/moss.hrl @@ -0,0 +1,137 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-ifndef(RCS_COMMON_MOSS_HRL). +-define(RCS_COMMON_MOSS_HRL, included). + +-include("aws_api.hrl"). + +%% All those cases of `| undefined | #{} | binary()` to proper field +%% types in various records in this file are for the sake of +%% exprec. Because in the exprec-generated frommap_thing, fields can +%% briefly be assigned a value of `undefined` or an unconverted +%% binary, dialyzer rightfully takes issue with it. Further, in custom +%% record conversion functions like riak_cs_iam:exprec_role, some +%% other fields exist as maps (before they are converted into proper +%% sub-records). Hence this type dilution and overall +%% kludginess. There surely is a less painful solution, of which I am, +%% as of releasing 3.2, sadly unaware. + +%% User +-record(moss_user, { name :: string() + , key_id :: string() + , key_secret :: string() + , buckets = [] + }). + +-record(moss_user_v1, { name :: string() + , display_name :: string() + , email :: string() + , key_id :: string() + , key_secret :: string() + , canonical_id :: string() + , buckets = [] :: [cs_bucket()] + }). + +-record(rcs_user_v2, { name :: string() + , display_name :: string() + , email :: string() + , key_id :: string() + , key_secret :: string() + , canonical_id :: string() + , buckets = [] :: [cs_bucket()] + , status = enabled :: undefined | enabled | disabled + }). + +%% this now in part logically belongs in aws_api.hrl +-record(rcs_user_v3, { arn :: undefined | flat_arn() + , path = <<"/">> :: binary() + , create_date = os:system_time(millisecond) :: non_neg_integer() + %% , user_id :: binary() %% maps to id + %% , user_name :: binary() %% maps to name + , password_last_used :: undefined | null | non_neg_integer() + , permissions_boundary :: undefined | #{} | permissions_boundary() + , tags = [] :: [#{} | tag()] + , attached_policies = [] :: [flat_arn()] + + , name :: undefined | binary() + , display_name :: undefined | binary() + , email :: undefined | binary() + , key_id :: undefined | binary() + , key_secret :: undefined | binary() + , id :: undefined | binary() + , buckets = [] :: [#{} | cs_bucket()] + , status = enabled :: enabled | disabled | binary() + }). + +-type moss_user() :: #rcs_user_v2{} | #moss_user_v1{}. +-type rcs_user() :: #rcs_user_v3{}. +-define(IAM_USER, #rcs_user_v3). +-define(RCS_USER, #rcs_user_v3). + + +%% Bucket +-record(moss_bucket, { name :: string() + , creation_date :: term() + , acl :: #acl_v1{}}). + +-record(moss_bucket_v1, { name :: string() | binary() + , last_action :: undefined | created | deleted + , creation_date :: undefined | string() + , modification_time :: undefined | erlang:timestamp() + , acl :: undefined | #acl_v2{} + }). + +-record(moss_bucket_v2, { name :: undefined | binary() + , last_action :: undefined | created | deleted | binary() + , creation_date = os:system_time(millisecond) :: non_neg_integer() + , modification_time :: undefined | non_neg_integer() + , acl :: undefined | null | #{} | acl() + }). + +-type cs_bucket() :: #moss_bucket_v2{}. +-define(RCS_BUCKET, #moss_bucket_v2). + +-type bucket_operation() :: create | delete | update_acl | update_policy + | delete_policy | update_versioning. +-type bucket_action() :: created | deleted. + + +%% federated users + +-record(temp_session, { assumed_role_user :: assumed_role_user() + , role :: role() + , credentials :: credentials() + , duration_seconds :: non_neg_integer() + , created = os:system_time(millisecond) :: non_neg_integer() + , inline_policy :: undefined | flat_arn() + , session_policies :: [flat_arn()] + , subject :: binary() + , source_identity :: binary() %% both this and the following can provide the email + , email :: binary() %% for our internal rcs_user + , user_id :: binary() + , canonical_id :: binary() + } + ). +-type temp_session() :: #temp_session{}. +-define(TEMP_SESSION, #temp_session). + +-endif. diff --git a/include/oos_api.hrl b/apps/riak_cs/include/oos_api.hrl similarity index 79% rename from include/oos_api.hrl rename to apps/riak_cs/include/oos_api.hrl index 9958921cf..d3d9da560 100644 --- a/include/oos_api.hrl +++ b/apps/riak_cs/include/oos_api.hrl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,6 +19,9 @@ %% %% --------------------------------------------------------------------- +-ifndef(RIAK_CS_OOS_API_HRL). +-define(RIAK_CS_OOS_API_HRL, included). + -define(DEFAULT_OS_AUTH_URL, "http://localhost:35357/v2.0/"). -define(DEFAULT_TOKENS_RESOURCE, "tokens/"). -define(DEFAULT_S3_TOKENS_RESOURCE, "s3tokens/"). @@ -25,9 +29,11 @@ -define(DEFAULT_OS_ADMIN_TOKEN, "ADMIN"). -define(DEFAULT_OS_OPERATOR_ROLES, [<<"admin">>, <<"swiftoperator">>]). --record(keystone_s3_auth_req_v1, { +-record(keystone_aws_auth_req_v1, { access :: binary(), signature :: binary(), token :: binary()}). --type keystone_s3_auth_req() :: #keystone_s3_auth_req_v1{}. --define(KEYSTONE_S3_AUTH_REQ, #keystone_s3_auth_req_v1). +-type keystone_aws_auth_req() :: #keystone_aws_auth_req_v1{}. +-define(KEYSTONE_S3_AUTH_REQ, #keystone_aws_auth_req_v1). + +-endif. diff --git a/apps/riak_cs/include/riak_cs.hrl b/apps/riak_cs/include/riak_cs.hrl new file mode 100644 index 000000000..4863cff14 --- /dev/null +++ b/apps/riak_cs/include/riak_cs.hrl @@ -0,0 +1,147 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2024 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-ifndef(RIAK_CS_HRL). +-define(RIAK_CS_HRL, included). + +-include("aws_api.hrl"). +-include("riak_cs_web.hrl"). + +-define(RCS_VERSION, 030205). +-define(RCS_VERSION_STRING, "3.2.5"). + +-define(DEFAULT_ADMIN_NAME, <<"admin">>). +-define(DEFAULT_ADMIN_KEY, <<"admin-key">>). + +-type riak_client() :: pid(). + +-define(AWS_API_MOD, riak_cs_aws_rewrite). +-define(OOS_API_MOD, riak_cs_oos_rewrite). +-define(AWS_RESPONSE_MOD, riak_cs_aws_response). +-define(OOS_RESPONSE_MOD, riak_cs_oos_response). + +-define(DEFAULT_STANCHION_IP, "127.0.0.1"). +-define(DEFAULT_STANCHION_PORT, 8085). +-define(DEFAULT_STANCHION_SSL, true). + +-define(DEFAULT_MAX_BUCKETS_PER_USER, 100). +-define(DEFAULT_MAX_CONTENT_LENGTH, 5368709120). %% 5 GB +-define(DEFAULT_LFS_BLOCK_SIZE, 1048576).%% 1 MB + +-define(XML_PROLOG, ""). +-define(S3_XMLNS, "http://s3.amazonaws.com/doc/2006-03-01/"). +-define(IAM_XMLNS, "https://iam.amazonaws.com/doc/2010-05-08/"). +-define(STS_XMLNS, "https://sts.amazonaws.com/doc/2011-06-15/"). + +-define(USER_BUCKET, <<"moss.users">>). +-define(ACCESS_BUCKET, <<"moss.access">>). +-define(STORAGE_BUCKET, <<"moss.storage">>). +-define(BUCKETS_BUCKET, <<"moss.buckets">>). +-define(SERVICE_BUCKET, <<"moss.service">>). +-define(IAM_USER_BUCKET, ?USER_BUCKET). +-define(IAM_ROLE_BUCKET, <<"moss.iam.roles">>). +-define(IAM_POLICY_BUCKET, <<"moss.iam.policies">>). +-define(IAM_SAMLPROVIDER_BUCKET, <<"moss.iam.samlproviders">>). +-define(TEMP_SESSIONS_BUCKET, <<"moss.sts.temp-sessions">>). + +-define(OBJECT_LOCK_BUCKET, <<"moss.object.lock">>). +-define(GC_BUCKET, <<"riak-cs-gc">>). +-define(FREE_BUCKET_MARKER, <<"0">>). +-define(DELETED_MARKER, <<"0">>). + +-define(MD_BAG, <<"X-Rcs-Bag">>). +-define(MD_ACL, <<"X-Moss-Acl">>). +-define(MD_POLICY, <<"X-Rcs-Policy">>). +-define(MD_VERSIONING, <<"X-Rcs-Versioning">>). + +-define(USERMETA_BUCKET, "RCS-bucket"). +-define(USERMETA_KEY, "RCS-key"). +-define(USERMETA_BCSUM, "RCS-bcsum"). + +-define(USER_EMAIL_INDEX, <<"user_email_bin">>). +-define(USER_PATH_INDEX, <<"user_path_bin">>). +-define(USER_NAME_INDEX, <<"user_name_bin">>). +-define(USER_KEYID_INDEX, <<"user_keyid_bin">>). +-define(USER_ID_INDEX, <<"user_id_bin">>). +-define(KEY_INDEX, <<"$key">>). +-define(ROLE_ID_INDEX, <<"role_id_bin">>). +-define(ROLE_PATH_INDEX, <<"role_path_bin">>). +-define(ROLE_NAME_INDEX, <<"role_name_bin">>). +-define(POLICY_ID_INDEX, <<"policy_id_bin">>). +-define(POLICY_PATH_INDEX, <<"policy_path_bin">>). +-define(POLICY_NAME_INDEX, <<"policy_name_bin">>). +-define(SAMLPROVIDER_NAME_INDEX, <<"samlprovider_name_bin">>). +-define(SAMLPROVIDER_ENTITYID_INDEX, <<"samlprovider_entityid_bin">>). +-define(TEMP_SESSION_ID_INDEX, <<"tmpsession_keyid_bin">>). + +-define(CONSISTENT_READ_OPTIONS, [{r, all}, {pr, all}, {notfound_ok, false}]). +-define(WEAK_READ_OPTIONS, [{r, quorum}, {pr, one}, {notfound_ok, false}]). +-define(CONSISTENT_WRITE_OPTIONS, [{w, all}, {pw, all}]). +-define(CONSISTENT_DELETE_OPTIONS, [{dw, all}]). + +-define(STANCHION_DETAILS_KEY, <<"stanchion">>). + +-define(DEFAULT_FETCH_CONCURRENCY, 1). +-define(DEFAULT_PUT_CONCURRENCY, 1). +-define(DEFAULT_DELETE_CONCURRENCY, 1). +%% A number to multiplied with the block size +%% to determine the PUT buffer size. +%% ex. 2 would mean BlockSize * 2 +-define(DEFAULT_PUT_BUFFER_FACTOR, 1). +%% Similar to above, but for fetching +%% This is also max ram per fetch request +-define(DEFAULT_FETCH_BUFFER_FACTOR, 32). +-define(N_VAL_1_GET_REQUESTS, true). +-define(DEFAULT_PING_TIMEOUT, 5000). + +-define(COMPRESS_TERMS, false). + +-define(USER_BUCKETS_PRUNE_TIME, 86400). %% one-day in seconds +-define(DEFAULT_CLUSTER_ID_TIMEOUT,5000). +-define(DEFAULT_LIST_OBJECTS_MAX_KEYS, 1000). +-define(DEFAULT_MD5_CHUNK_SIZE, 1048576). %% 1 MB +-define(DEFAULT_MANIFEST_WARN_SIBLINGS, 20). +-define(DEFAULT_MANIFEST_WARN_BYTES, 5*1024*1024). %% 5MB +-define(DEFAULT_MANIFEST_WARN_HISTORY, 30). +-define(DEFAULT_MAX_PART_NUMBER, 10000). + +%% timeout hitting Riak PB API +-define(DEFAULT_RIAK_TIMEOUT, 60000). + +%% General system info +-define(WORD_SIZE, erlang:system_info(wordsize)). + +-define(OBJECT_BUCKET_PREFIX, <<"0o:">>). % Version # = 0 +-define(BLOCK_BUCKET_PREFIX, <<"0b:">>). % Version # = 0 + +-define(MAX_S3_KEY_LENGTH, 1024). + +-define(VERSIONED_KEY_SEPARATOR, <<5>>). + +-type riak_obj_error() :: notfound | term(). +-type reportable_error_reason() :: atom() + | {riak_connect_failed, term()} + | {malformed_policy_version, binary()} + | {invalid_argument, binary()} + | {key_too_long, pos_integer()} + | {unsatisfied_constraint, binary()}. + +-endif. diff --git a/include/riak_cs_gc.hrl b/apps/riak_cs/include/riak_cs_gc.hrl similarity index 89% rename from include/riak_cs_gc.hrl rename to apps/riak_cs/include/riak_cs_gc.hrl index 8e0bfb0e0..60cd74200 100644 --- a/include/riak_cs_gc.hrl +++ b/apps/riak_cs/include/riak_cs_gc.hrl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,6 +19,9 @@ %% %% --------------------------------------------------------------------- +-ifndef(RIAK_CS_GC_HRL). +-define(RIAK_CS_GC_HRL, included). + -include("riak_cs.hrl"). -include_lib("riakc/include/riakc.hrl"). @@ -48,16 +52,16 @@ %% Used for paginated 2I querying of GC bucket key_list_state :: undefined | gc_key_list_state(), %% Options to use when start workers - bag_id :: binary() + bag_id :: undefined | binary() }). -record(gc_worker_state, { %% Riak connection pid riak_client :: undefined | riak_client(), bag_id :: bag_id(), - current_files :: [lfs_manifest()], - current_fileset :: twop_set:twop_set(), - current_riak_object :: riakc_obj:riakc_obj(), + current_files = [] :: [lfs_manifest()], + current_fileset = twop_set:new() :: twop_set:twop_set(), + current_riak_object :: undefined | riakc_obj:riakc_obj(), %% Count of filesets collected successfully batch_count=0 :: non_neg_integer(), %% Count of filesets skipped in this batch @@ -65,7 +69,7 @@ batch=[] :: undefined | [binary()], % `undefined' only for testing manif_count=0 :: non_neg_integer(), block_count=0 :: non_neg_integer(), - delete_fsm_pid :: pid() + delete_fsm_pid :: undefined | pid() }). -record(gc_key_list_state, { @@ -105,6 +109,8 @@ batch_history = [] :: list(#gc_batch_state{}), current_batch :: undefined | #gc_batch_state{}, interval = ?DEFAULT_GC_INTERVAL:: non_neg_integer() | infinity, - initial_delay :: non_neg_integer(), + initial_delay :: undefined | non_neg_integer(), timer_ref :: undefined | reference() }). + +-endif. diff --git a/apps/riak_cs/include/riak_cs_web.hrl b/apps/riak_cs/include/riak_cs_web.hrl new file mode 100644 index 000000000..5e9582093 --- /dev/null +++ b/apps/riak_cs/include/riak_cs_web.hrl @@ -0,0 +1,443 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-ifndef(RIAK_CS_WEB_HRL). +-define(RIAK_CS_WEB_HRL, included). + +-include("manifest.hrl"). +-include("moss.hrl"). +-include("aws_api.hrl"). +-include_lib("webmachine/include/webmachine.hrl"). + +-type mochiweb_headers() :: gb_trees:tree(). + +-define(DEFAULT_AUTH_MODULE, riak_cs_aws_auth). +-define(DEFAULT_POLICY_MODULE, riak_cs_aws_policy). +-define(DEFAULT_RESPONSE_MODULE, riak_cs_aws_response). + +-define(RCS_REWRITE_HEADER, "x-rcs-rewrite-path"). +-define(RCS_RAW_URL_HEADER, "x-rcs-raw-url"). +-define(OOS_API_VSN_HEADER, "x-oos-api-version"). +-define(OOS_ACCOUNT_HEADER, "x-oos-account"). + +-define(JSON_TYPE, "application/json"). +-define(XML_TYPE, "application/xml"). +-define(WWWFORM_TYPE, "application/x-www-form-urlencoded"). + +-type api() :: aws | oos. + +-type policy_module() :: ?DEFAULT_POLICY_MODULE. + +-record(rcs_web_context, {start_time = os:system_time(millisecond) :: non_neg_integer(), + request_id = riak_cs_wm_utils:make_request_id() :: binary(), + auth_bypass = false :: boolean(), + admin_access = false :: boolean(), + user :: undefined | rcs_user(), + user_object :: undefined | riakc_obj:riakc_obj(), + role :: undefined | role(), + bucket :: undefined | binary(), + acl :: undefined | acl(), + requested_perm :: undefined | acl_perm(), + riak_client :: undefined | pid(), + rc_pool = request_pool:: atom(), % pool name which riak_client belongs to + auto_rc_close = true :: boolean(), + submodule :: module(), + exports_fun :: undefined | function(), + auth_module = ?DEFAULT_AUTH_MODULE :: module(), + policy_module = ?DEFAULT_POLICY_MODULE :: policy_module(), + response_module = ?DEFAULT_RESPONSE_MODULE :: module(), + %% Key for API rate and latency stats. + %% If `stats_prefix' or `stats_key' is `no_stats', no stats + %% will be gathered by riak_cs_wm_common. + %% The prefix is defined by `stats_prefix()' callback of sub-module. + %% If sub-module provides only `stats_prefix' (almost the case), + %% stats key is [Prefix, HttpMethod]. Otherwise, sum-module + %% can set specific `stats_key' by any callback that returns + %% this context. + stats_prefix = no_stats :: atom(), + stats_key = prefix_and_method :: prefix_and_method | + no_stats | + riak_cs_stats:key(), + local_context :: term(), + api :: atom() + }). + +-record(key_context, {manifest :: undefined | notfound | lfs_manifest(), + upload_id :: undefined | binary(), + part_number :: undefined | integer(), + part_uuid :: undefined | binary(), + get_fsm_pid :: undefined | pid(), + putctype :: undefined | string(), + bucket :: undefined | binary(), + bucket_object :: undefined | notfound | riakc_obj:riakc_obj(), + key :: undefined | binary(), + obj_vsn = ?LFS_DEFAULT_OBJECT_VERSION :: binary(), + owner :: undefined | binary(), + size :: undefined | non_neg_integer(), + content_md5 :: undefined | binary(), + update_metadata = false :: boolean()}). + +-type action_target() :: object | object_part | object_acl + | bucket | bucket_acl | bucket_policy + | bucket_request_payment | bucket_location | bucket_uploads | no_bucket + | iam_entity + | sts_entity. + +-record(access_v1, { method :: undefined | 'PUT' | 'GET' | 'POST' | 'DELETE' | 'HEAD' + , target :: undefined | action_target() + , id :: undefined | binary() + , bucket :: undefined | binary() + , key = <<>> :: undefined | binary() + , action :: undefined | aws_action() + , req :: undefined | #wm_reqdata{} + } + ). +-type access() :: #access_v1{}. + + +%% === objects === + +-type next_marker() :: undefined | binary(). + +-type list_objects_req_type() :: objects | versions. + +-record(list_objects_request, + { + req_type :: list_objects_req_type(), + + %% the name of the bucket + name :: binary(), + + %% how many keys to return in the response + max_keys :: non_neg_integer(), + + %% a 'starts-with' parameter + prefix :: binary() | undefined, + + %% a binary to group keys by + delimiter :: binary() | undefined, + + %% the key and version_id to start with + marker :: binary() | undefined, + version_id_marker :: binary() | undefined + }). +-type list_object_request() :: #list_objects_request{}. +-define(LOREQ, #list_objects_request). + +-record(list_objects_response, + { + %% Params just echoed back from the request -------------------------- + + %% the name of the bucket + name :: binary(), + + %% how many keys were requested to be + %% returned in the response + max_keys :: non_neg_integer(), + + %% a 'starts-with' parameter + prefix :: binary() | undefined, + + %% a binary to group keys by + delimiter :: binary() | undefined, + + %% the marker used in the _request_ + marker :: binary() | undefined, + %% the (optional) marker to use for pagination + %% in the _next_ request + next_marker :: next_marker(), + %% (marker and next_marker not used in ListObjectsV2) + + %% The actual response ----------------------------------------------- + is_truncated :: boolean(), + + contents :: [list_objects_key_content()], + + common_prefixes :: list_objects_common_prefixes() + }). + +-type list_objects_response() :: #list_objects_response{}. +-define(LORESP, #list_objects_response). + +-record(list_objects_key_content, + { + key :: binary(), + last_modified :: binary(), + etag :: binary(), + size :: non_neg_integer(), + owner :: list_objects_owner(), + storage_class :: binary() + }). +-type list_objects_key_content() :: #list_objects_key_content{}. +-define(LOKC, #list_objects_key_content). + + +-record(list_object_versions_response, + { + %% Params just echoed back from the request -------------------------- + + %% the name of the bucket + name :: binary(), + + %% how many keys were requested to be + %% returned in the response + max_keys :: non_neg_integer(), + + %% a 'starts-with' parameter + prefix :: binary() | undefined, + + %% a binary to group keys by + delimiter :: binary() | undefined, + + %% the marker used in the _request_ + key_marker :: binary() | undefined, + version_id_marker :: binary() | undefined, + + %% the (optional) marker to use for pagination + %% in the _next_ request + next_key_marker :: next_marker(), + next_version_id_marker :: next_marker(), + + %% The actual response ----------------------------------------------- + is_truncated :: boolean(), + + contents :: [list_object_versions_key_content()], + + common_prefixes :: list_objects_common_prefixes() + }). + +-type list_object_versions_response() :: #list_object_versions_response{}. +-define(LOVRESP, #list_object_versions_response). + +-record(list_object_versions_key_content, + { + key :: binary(), + last_modified :: binary(), + etag :: binary(), + is_latest :: boolean(), + version_id :: binary(), + size :: non_neg_integer(), + owner :: list_objects_owner(), + storage_class :: binary() + }). +-type list_object_versions_key_content() :: #list_object_versions_key_content{}. +-define(LOVKC, #list_object_versions_key_content). + +-record(list_objects_owner, { + id :: binary(), + display_name :: binary()}). +-type list_objects_owner() :: #list_objects_owner{}. + +-type list_objects_common_prefixes() :: list(binary()). + +-define(LIST_OBJECTS_CACHE, list_objects_cache). +-define(ENABLE_CACHE, true). +-define(CACHE_TIMEOUT, timer:minutes(15)). +-define(MIN_KEYS_TO_CACHE, 2000). +-define(MAX_CACHE_BYTES, 104857600). % 100MB + + +%% === buckets === + +-record(list_buckets_response, + { + %% the user record + user :: rcs_user(), + + %% the list of bucket records + buckets :: [cs_bucket()] + }). +-type list_buckets_response() :: #list_buckets_response{}. +-define(LBRESP, #list_buckets_response). + +-record(bucket_versioning, { status = suspended :: enabled | suspended | binary() + , mfa_delete = disabled :: disabled | enabled | binary() + %% Riak CS extensions (still to be implemented) + , use_subversioning = false :: boolean() + , can_update_versions = false :: boolean() + , repl_siblings = true :: boolean() + } + ). +-type bucket_versioning() :: #bucket_versioning{}. + + +%% === IAM === + +-record(create_user_response, { user :: rcs_user() + , request_id :: binary() + } + ). +-record(get_user_response, { user :: rcs_user() + , request_id :: binary() + } + ). +-record(delete_user_response, { request_id :: binary() + } + ). +-record(list_users_request, { max_items = 1000 :: non_neg_integer() + , path_prefix :: binary() | undefined + , marker :: binary() | undefined + , request_id :: binary() + } + ). +-record(list_users_response, { marker :: binary() | undefined + , is_truncated :: boolean() + , users :: [rcs_user()] + , request_id :: binary() + } + ). + + +-record(create_role_response, { role :: role() + , request_id :: binary() + } + ). +-record(get_role_response, { role :: role() + , request_id :: binary() + } + ). +-record(delete_role_response, { request_id :: binary() + } + ). +-record(list_roles_request, { max_items = 1000 :: non_neg_integer() + , path_prefix :: binary() | undefined + , marker :: binary() | undefined + , request_id :: binary() + } + ). +-record(list_roles_response, { marker :: binary() | undefined + , is_truncated :: boolean() + , roles :: [role()] + , request_id :: binary() + } + ). + + +-record(create_policy_response, { policy :: iam_policy() + , request_id :: binary() + } + ). +-record(get_policy_response, { policy :: iam_policy() + , request_id :: binary() + } + ). +-record(delete_policy_response, { request_id :: binary() + } + ). +-record(list_policies_request, { max_items = 1000 :: non_neg_integer() + , only_attached = false :: boolean() + , policy_usage_filter = 'All' :: 'All' | 'PermissionsPolicy' | 'PermissionsBoundary' + , scope = all :: 'All' | 'AWS' | 'Local' + , path_prefix :: binary() | undefined + , marker :: binary() | undefined + , request_id :: binary() + } + ). +-record(list_policies_response, { marker :: binary() | undefined + , is_truncated :: boolean() + , policies :: [iam_policy()] + , request_id :: binary() + } + ). + + +-record(create_saml_provider_response, { saml_provider_arn :: flat_arn() + , tags :: [tag()] + , request_id :: binary() + } + ). +-record(get_saml_provider_response, { create_date :: non_neg_integer() + , valid_until :: non_neg_integer() + , saml_metadata_document :: binary() + , tags :: [tag()] + , request_id :: binary() + } + ). +-record(delete_saml_provider_response, { request_id :: binary() + } + ). +-record(list_saml_providers_request, { request_id :: binary() + } + ). +-record(saml_provider_list_entry, { create_date :: non_neg_integer() + , valid_until :: non_neg_integer() + , arn :: flat_arn() + } + ). +-record(list_saml_providers_response, { saml_provider_list :: [#saml_provider_list_entry{}] + , request_id :: binary() + } + ). + +-record(assume_role_with_saml_response, { assumed_role_user :: assumed_role_user() + , audience :: binary() + , credentials :: credentials() + , issuer :: binary() + , name_qualifier :: binary() + , packed_policy_size :: non_neg_integer() + , source_identity :: binary() + , subject :: binary() + , subject_type :: binary() + , request_id :: binary() + } + ). + +-record(attach_role_policy_response, { request_id :: binary() + } + ). +-record(detach_role_policy_response, { request_id :: binary() + } + ). + +-record(attach_user_policy_response, { request_id :: binary() + } + ). +-record(detach_user_policy_response, { request_id :: binary() + } + ). + +-record(list_attached_user_policies_response, { policies :: [{flat_arn(), binary()}] + , marker :: binary() | undefined + , is_truncated :: boolean() + , request_id :: binary() + } + ). + +-record(list_attached_role_policies_response, { policies :: [{flat_arn(), binary()}] + , marker :: binary() | undefined + , is_truncated :: boolean() + , request_id :: binary() + } + ). + +%% this is not an official AWS request +-record(list_temp_sessions_request, { max_items = 1000 :: non_neg_integer() + , marker :: binary() | undefined + , request_id :: binary() + } + ). +-record(list_temp_sessions_response, { temp_sessions :: [#temp_session{}] + , request_id :: binary() + , marker :: binary() | undefined + , is_truncated :: boolean() + } + ). + +-endif. diff --git a/include/rts.hrl b/apps/riak_cs/include/rts.hrl similarity index 96% rename from include/rts.hrl rename to apps/riak_cs/include/rts.hrl index 5899f0ecf..8719a53f4 100644 --- a/include/rts.hrl +++ b/apps/riak_cs/include/rts.hrl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,6 +19,9 @@ %% %% --------------------------------------------------------------------- +-ifndef(RIAK_CS_RTS_HRL). +-define(RIAK_CS_RTS_HRL, included). + %% JSON keys used by rts module -define(START_TIME, <<"StartTime">>). -define(END_TIME, <<"EndTime">>). @@ -59,3 +63,5 @@ <<"PendingDeleteOldObjects">>,<<"PendingDeleteOldBytes">>, <<"PendingDeleteOldBlocks">>, <<"PendingDeleteNewObjects">>,<<"PendingDeleteNewBytes">>, <<"PendingDeleteNewBlocks">>, <<"ActiveInvisibleObjects">>,<<"ActiveInvisibleBytes">>, <<"ActiveInvisibleBlocks">>]). + +-endif. diff --git a/riak_test/tests/list_objects_test.erl b/apps/riak_cs/include/stanchion.hrl similarity index 54% rename from riak_test/tests/list_objects_test.erl rename to apps/riak_cs/include/stanchion.hrl index c38072731..8eef2e952 100644 --- a/riak_test/tests/list_objects_test.erl +++ b/apps/riak_cs/include/stanchion.hrl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,20 +19,23 @@ %% %% --------------------------------------------------------------------- --module(list_objects_test). +-ifndef(STANCHION_HRL). +-define(STANCHION_HRL, included). -%% @doc Integration test for list the contents of a bucket +-record(stanchion_context, {auth_bypass :: boolean(), + bucket :: undefined | binary(), + owner_id :: undefined | all | string()}). --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). +-define(TURNAROUND_TIME(Call), + begin + StartTime_____tat = os:timestamp(), + Result_____tat = (Call), + EndTime_____tat = os:timestamp(), + {Result_____tat, + timer:now_diff(EndTime_____tat, + StartTime_____tat)} + end). --define(TEST_BUCKET, "riak-test-bucket"). +-define(STANCHION_OWN_PBC_TABLE, stanchion_own_pbc_table). -confirm() -> - rtcs:set_conf(cs, [{"fold_objects_for_list_keys", "off"}]), - {UserConfig, {_RiakNodes, CSNodes, _Stanchion}} = rtcs:setup(1), - assert_v1(CSNodes), - list_objects_test_helper:test(UserConfig). - -assert_v1(CSNodes) -> - false =:= rpc:call(hd(CSNodes), riak_cs_list_objects_utils, fold_objects_for_list_keys, []). +-endif. diff --git a/apps/riak_cs/priv b/apps/riak_cs/priv deleted file mode 120000 index e38c3d37a..000000000 --- a/apps/riak_cs/priv +++ /dev/null @@ -1 +0,0 @@ -../../priv \ No newline at end of file diff --git a/apps/riak_cs/priv/riak_cs-sp-saml-metadata.xml b/apps/riak_cs/priv/riak_cs-sp-saml-metadata.xml new file mode 100644 index 000000000..84470456a --- /dev/null +++ b/apps/riak_cs/priv/riak_cs-sp-saml-metadata.xml @@ -0,0 +1,100 @@ + + + + + + + MIIDezCCAmMCFEuXK3UJNKFNDU1+ZshGQRaQ8xDTMA0GCSqGSIb3DQEBCwUAMHox + CzAJBgNVBAYTAkpQMREwDwYDVQQIDAhUb2t5by10bzENMAsGA1UEBwwEY2l0eTER + MA8GA1UECgwIVEkgVG9reW8xEDAOBgNVBAMMB3Rpb3QuanAxJDAiBgkqhkiG9w0B + CQEWFWFuZHJlaS56YXZhZGFAdGlvdC5qcDAeFw0yMzAzMjExNDI1NDRaFw0zMzAz + MTgxNDI1NDRaMHoxCzAJBgNVBAYTAkpQMREwDwYDVQQIDAhUb2t5by10bzENMAsG + A1UEBwwEY2l0eTERMA8GA1UECgwIVEkgVG9reW8xEDAOBgNVBAMMB3Rpb3QuanAx + JDAiBgkqhkiG9w0BCQEWFWFuZHJlaS56YXZhZGFAdGlvdC5qcDCCASIwDQYJKoZI + hvcNAQEBBQADggEPADCCAQoCggEBAKl90zzn+Ww6fBZLsw2NOmdT1z5dG+8Tnysf + V6f4NCtMcnD26aNFNuAe7b/XIT8YdYrBR/YEyl6dY/c281HBfiSJiXf62vXRlSXD + r1c7515CFy1hSOg8IwFE0qPQizpmIwK8E1S38vq/aj4smWLnpTI/UhliaRgyUaII + M3FBxy79r0gdndACW1l4ajFIufgcidPvtvRxdhkVKMXMPK2hXSM24+8POwfTvFUh + Y7enwYn4eYlo2IcM2O+U7ELDXI4GXHckqmch3rwYlJIAXJDqLoqagWtCnJBLWzlw + YajHXXae59kp6YGuepIjdKDsZmDegZxo+8KRWOXYhugutE4eih8CAwEAATANBgkq + hkiG9w0BAQsFAAOCAQEAhD89Gxg8NqeSkpubyqHKygpAFXwBYTDky6UdIYA6HSXC + YtulTcsaNO+8+aAdLrEXfiOfLADIFAhsyLNzB8+4W2jDbuMrLeviW4p2UvQn9lpu + SaZuT+oudArnwvp4I5S4Dbej46L1aZO3gFCSRwtiWwiDVIWoswGxLHBciMy54yCQ + oq1QHg8y+F1pJ7xloSIEJ3JBJ6sCn59CaXcM+75KXVsdNbmNn7zaZ0NxbDjYlsMH + taDfFmoMmb46QGjIvbW+Vzfm1kUI1Mk7AtKDeyCrexN9MsrxVRJlC+8PGSVQYkjd + I7S6DycXmAOQCNjKbP5xa6Qosj996PDdTNgQ0ICciA== + + + + + urn:oasis:names:tc:SAML:2.0:nameid-format:transient + urn:oasis:names:tc:SAML:2.0:nameid-format:persistent + urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress + urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified + urn:oasis:names:tc:SAML:1.1:nameid-format:X509SubjectName + urn:oasis:names:tc:SAML:1.1:nameid-format:WindowsDomainQualifiedName + urn:oasis:names:tc:SAML:2.0:nameid-format:kerberos + urn:oasis:names:tc:SAML:2.0:nameid-format:entity + + + + + + + + + + + + + + + + + + + + + + + + + + + + + RCS Club Management Console Single Sign-On + + + + + + + + + + + + + + + + + + + + + + + Riak CS Enthusiast Club. + + https://aws.amazon.com + + diff --git a/rel/files/riak_cs.schema b/apps/riak_cs/priv/riak_cs.schema similarity index 59% rename from rel/files/riak_cs.schema rename to apps/riak_cs/priv/riak_cs.schema index 401da7003..ce062da43 100644 --- a/rel/files/riak_cs.schema +++ b/apps/riak_cs/priv/riak_cs.schema @@ -1,10 +1,34 @@ %% -*- erlang -*- -%% ex: ft=erlang ts=4 sw=4 et %% Riak CS configuration %% == Basic Configuration == +%% @doc Whether to run stanchion on this node (auto means we will +%% check if stanchion is already started on any nodes in our tussle, and +%% only start it here if it is not). +{mapping, "stanchion_hosting_mode", "riak_cs.stanchion_hosting_mode", [ + {default, {{ stanchion_hosting_mode }} }, + {datatype, {enum, [auto, riak_cs_with_stanchion, + riak_cs_only, stanchion_only]}}, + {validators, ["valid_stanchion_hosting_mode"]} +]}. + +%% @doc Riak node to store volatile tussle data (ip:port of the node +%% running stanchion, currently). +{mapping, "tussle_voss_riak_host", "riak_cs.tussle_voss_riak_host", [ + {default, "{{ tussle_voss_riak_host }}" }, + {datatype, [fqdn, ip, {enum, [auto]}]}, + {validators, ["valid_host_or_auto"]} +]}. + +{validator, + "valid_stanchion_hosting_mode", + "should be one of: auto, riak_cs_with_stanchion, riak_cs_only, stanchion_only", + fun(A) -> lists:member(A, [auto, riak_cs_with_stanchion, + riak_cs_only, stanchion_only]) end +}. + %% @doc Riak CS http/https port and IP address to listen at for object %% storage activity {mapping, "listener", "riak_cs.listener", [ @@ -18,19 +42,51 @@ "should be a valid host", fun({Host, Port}) -> is_list(Host) andalso 0 < Port andalso Port < 65536 end}. +{validator, + "valid_host_or_auto", + "should be a valid host or 'auto'", + fun({Host, Port}) -> is_list(Host) andalso 0 < Port andalso Port < 65536; + (Auto) -> Auto == auto end}. + %% @doc Riak node to which Riak CS accesses {mapping, "riak_host", "riak_cs.riak_host", [ {default, {"{{riak_ip}}", {{riak_pb_port}} }}, - {datatype, ip}, + {datatype, [fqdn, ip]}, {validators, ["valid_host"]} ]}. -%% @doc Configuration for access to request -%% serialization service -{mapping, "stanchion_host", "riak_cs.stanchion_host", [ - {default, {"{{stanchion_ip}}", {{stanchion_port}} }}, - {datatype, ip}, - {validators, ["valid_host"]} +%% @doc Default cert location for https can be overridden +%% with the ssl config variable, for example: +{mapping, "ssl.certfile", "riak_cs.ssl.certfile", [ + {datatype, file}, + {commented, "{{platform_etc_dir}}/cert.pem"} +]}. + +%% @doc Default key location for https can be overridden with the ssl +%% config variable, for example: +{mapping, "ssl.keyfile", "riak_cs.ssl.keyfile", [ + {datatype, file}, + {commented, "{{platform_etc_dir}}/key.pem"} +]}. + + +%% @doc Stanchion http/https port to listen at. The IP address will be +%% selected based on stanchion_subnet and stanchion_netmask. +{mapping, "stanchion_port", "riak_cs.stanchion_port", [ + {default, {{stanchion_port}} }, + {datatype, integer}, + {validators, ["valid_port"]} +]}. + +%% @doc Subnet to use when selecting which network to place stanchion on +{mapping, "stanchion_subnet", "riak_cs.stanchion_subnet", [ + {default, "127.0.0.1"}, + {datatype, string} +]}. +%% @doc Netmask to use when selecting which network to place stanchion on +{mapping, "stanchion_netmask", "riak_cs.stanchion_netmask", [ + {default, "255.255.255.255"}, + {datatype, string} ]}. %% @doc SSL configuration for access to request serialization @@ -42,18 +98,20 @@ %% @doc Default cert location for https can be overridden %% with the ssl config variable, for example: -{mapping, "ssl.certfile", "riak_cs.ssl.certfile", [ +{mapping, "stanchion.ssl_certfile", "riak_cs.stanchion_ssl_certfile", [ {datatype, file}, - {commented, "$(platform_etc_dir)/cert.pem"} + {commented, "$(platform_etc_dir)/stanchion_cert.pem"} ]}. %% @doc Default key location for https can be overridden with the ssl %% config variable, for example: -{mapping, "ssl.keyfile", "riak_cs.ssl.keyfile", [ +{mapping, "stanchion.ssl_keyfile", "riak_cs.stanchion_ssl_keyfile", [ {datatype, file}, - {commented, "$(platform_etc_dir)/key.pem"} + {commented, "$(platform_etc_dir)/stanchion_key.pem"} ]}. + + %% @doc Enable this to allow the creation of an admin user when %% setting up a system. It is recommended to only enable this %% temporarily unless your use-case specifically dictates letting @@ -72,9 +130,7 @@ ]}. %% @doc Admin user credentials. Admin access like /riak-cs/stats -%% requires this entry to be set properly. The credentials specified -%% here must match the admin credentials specified in the -%% stanchion.conf for the system to function properly. +%% requires this entry to be set properly. {mapping, "admin.key", "riak_cs.admin_key", [ {default, "{{admin_key}}"}, {datatype, string} @@ -86,14 +142,50 @@ hidden ]}. -%% @doc Root host name which Riak CS accepts. +%% @doc {auth_bypass, {{auth_bypass}} } , +{mapping, "auth_bypass", "riak_cs.auth_bypass", [ + {datatype, {flag, on, off}}, + {default, off}, + hidden +]}. + +%% @doc Enable experimental signature_v4 compatibility. +%% Changing this setting to on will allow s3cmd to utilise +%% signature_v4 and thus function without the need to manually add +%% v2_signature support to your .s3cfg file, +%% Note: this function is unfinished and suffers from issues: +%% #1058, #1059, #1060. Use at your own risk. +{mapping, "auth_v4", "riak_cs.auth_v4_enabled", [ + {default, on}, + {datatype, flag} +]}. + +%% @doc A string identifying this host. It will appear in error responses. +{mapping, "host_id", "riak_cs.host_id", [ + {default, binary_to_list(base64:encode("riak_cs-is-not-amazonaws.com"))}, + {datatype, string} +]}. + +%% @doc Root host name which Riak CS accepts for S3 requests. %% Your CS bucket at s3.example.com will be accessible %% via URL like http://bucket.s3.example.com/object/name -{mapping, "root_host", "riak_cs.cs_root_host", [ +{mapping, "s3_root_host", "riak_cs.s3_root_host", [ {default, "s3.amazonaws.com"}, {datatype, string} ]}. +%% @doc Per +%% https://docs.aws.amazon.com/IAM/latest/APIReference/API_CreateUser.html, +%% CreateUser IAM call has no "Email" parameter. Because in Riak CS, +%% users do have email as a required parameter, we need to provide a +%% valid email, which we can construct from the user name @ the email +%% domain. +{mapping, "iam_create_user_default_email_host", "riak_cs.iam_create_user_default_email_host", [ + {default, "my-riak-cs-megacorp.com"}, + {datatype, string} +]}. + + %% @doc Fixed pool size of primary connection pool which is used %% to service the majority of API requests related to the upload %% or retrieval of objects. @@ -195,14 +287,6 @@ {datatype, {duration, s}} ]}. -%% @doc Set this to false if you're running -%% Riak nodes prior to version 1.4.0. -{mapping, "gc.paginated_indexes", "riak_cs.gc_paginated_indexes", [ - {default, on}, - {datatype, flag}, - hidden -]}. - {mapping, "gc.max_workers", "riak_cs.gc_max_workers", [ {default, 2}, {datatype, integer}, @@ -333,7 +417,7 @@ %% @doc Access log directory. {mapping, "log.access.dir", "webmachine.log_handlers", [ - {default, "$(platform_log_dir)" }, + {default, "{{platform_log_dir}}" }, {datatype, string} ]}. @@ -353,7 +437,7 @@ %% @doc URL rewrite module. {mapping, "rewrite_module", "riak_cs.rewrite_module", [ - {commented, riak_cs_s3_rewrite}, + {commented, riak_cs_aws_rewrite}, {datatype, atom}, {validators, ["valid_rewrite_module"]} ]}. @@ -361,15 +445,14 @@ {validator, "valid_rewrite_module", "should be a valid rewrite module", - fun(riak_cs_s3_rewrite) -> true; - (riak_cs_s3_rewrite_legacy) -> true; + fun(riak_cs_aws_rewrite) -> true; (riak_cs_oos_rewrite) -> true; (_) -> false end}. %% @doc Authentication module. {mapping, "auth_module", "riak_cs.auth_module", [ - {commented, riak_cs_s3_auth}, + {commented, riak_cs_aws_auth}, {datatype, atom}, {validators, ["valid_auth_module"]} ]}. @@ -377,20 +460,11 @@ {validator, "valid_auth_module", "should be a valid auth module", - fun(riak_cs_s3_auth) -> true; + fun(riak_cs_aws_auth) -> true; (riak_cs_keystone_auth) -> true; (_) -> false end}. -%% == Bucket (List Objects) == - -%% @doc Set this to false if you're running -%% Riak nodes prior to version 1.4.0. -{mapping, "fold_objects_for_list_keys", "riak_cs.fold_objects_for_list_keys", [ - {default, on}, - {datatype, flag}, - hidden -]}. %% == Rolling upgrade support == @@ -415,262 +489,108 @@ {datatype, flag} ]}. -%% == DTrace == - -%% @doc If your Erlang virtual machine supports DTrace (or -%% user-space SystemTap), set dtrace_support to true. -{mapping, "dtrace", "riak_cs.dtrace_support", [ - {default, off}, - {datatype, flag} -]}. - -%% @doc Where to emit the default log messages (typically at 'info' -%% severity): -%% off: disabled -%% file: the file specified by log.console.file -%% console: to standard output (seen when using `riak attach-direct`) -%% both: log.console.file and standard out. -{mapping, "log.console", "lager.handlers", [ - {default, {{console_log_default}} }, - {datatype, {enum, [off, file, console, both]}} -]}. +%% == logger == %% @doc The severity level of the console log, default is 'info'. -{mapping, "log.console.level", "lager.handlers", [ - {default, info}, +{mapping, "logger.level", "kernel.logger_level", [ + {default, {{log_level}} }, {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}} ]}. -%% @doc When 'log.console' is set to 'file' or 'both', the file where -%% console messages will be logged. -{mapping, "log.console.file", "lager.handlers", [ - {default, "$(platform_log_dir)/console.log"}, - {datatype, file} -]}. - -%% @doc Maximum size of the console log in bytes, before it is rotated -{mapping, "log.console.size", "lager.handlers", [ - {default, "10MB"}, - {datatype, bytesize} -]}. - -%% @doc The schedule on which to rotate the console log. For more -%% information see: -%% https://github.com/basho/lager/blob/master/README.md#internal-log-rotation -{mapping, "log.console.rotation", "lager.handlers", [ - {default, "$D0"} +%% @doc Format string for the messages emitted to default log. The string is passed into +%% the handler as a logger_formatter template, the format is a list containing strings +%% and atoms. The atoms denote the keys for retrieving metadata from the logger events. +%% More information on default metadata can be found here (https://www.erlang.org/doc/man/logger_formatter.html#type-template). +{mapping, "logger.format", "kernel.logger", [ + {default, "[time,\" [\",level,\"] \",pid,\"@\",mfa,\":\",line,\" \",msg,\"\\n\"]."} ]}. -%% @doc The number of rotated console logs to keep. When set to -%% 'current', only the current open log file is kept. -{mapping, "log.console.rotation.keep", "lager.handlers", [ - {default, 5}, - {datatype, [integer, {atom, current}]}, - {validators, ["rotation_count"]} -]}. - -%% @doc The file where error messages will be logged. -{mapping, "log.error.file", "lager.handlers", [ - {default, "$(platform_log_dir)/error.log"}, +%% @doc Filename to use for log files. +{mapping, "logger.file", "kernel.logger", [ + {default, "{{platform_log_dir}}/console.log"}, {datatype, file} ]}. -%% @doc Maximum size of the error log in bytes, before it is rotated -{mapping, "log.error.size", "lager.handlers", [ - {default, "10MB"}, - {datatype, bytesize} -]}. - -%% @doc The schedule on which to rotate the error log. For more -%% information see: -%% https://github.com/basho/lager/blob/master/README.md#internal-log-rotation -{mapping, "log.error.rotation", "lager.handlers", [ - {default, "$D0"} -]}. - -%% @doc The number of rotated error logs to keep. When set to -%% 'current', only the current open log file is kept. -{mapping, "log.error.rotation.keep", "lager.handlers", [ - {default, 5}, - {datatype, [integer, {atom, current}]}, - {validators, ["rotation_count"]} -]}. - -%% @doc When set to 'on', enables log output to syslog. -{mapping, "log.syslog", "lager.handlers", [ - {default, off}, - {datatype, flag} -]}. - -%% @doc When set to 'on', enables log output to syslog. -{mapping, "log.syslog.ident", "lager.handlers", [ - {default, "riak-cs"}, - hidden -]}. - -%% @doc Syslog facility to log entries from Riak CS. -{mapping, "log.syslog.facility", "lager.handlers", [ - {default, daemon}, - {datatype, {enum,[kern, user, mail, daemon, auth, syslog, - lpr, news, uucp, clock, authpriv, ftp, - cron, local0, local1, local2, local3, - local4, local5, local6, local7]}}, - hidden -]}. - -%% @doc The severity level at which to log entries to syslog, default is 'info'. -{mapping, "log.syslog.level", "lager.handlers", [ - {default, info}, - {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}}, - hidden -]}. - -{translation, - "lager.handlers", - fun(Conf) -> - SyslogHandler = case cuttlefish:conf_get("log.syslog", Conf) of - true -> - Ident = cuttlefish:conf_get("log.syslog.ident", Conf), - Facility = cuttlefish:conf_get("log.syslog.facility", Conf), - LogLevel = cuttlefish:conf_get("log.syslog.level", Conf), - [{lager_syslog_backend, [Ident, Facility, LogLevel]}]; - _ -> [] - end, - - TranslateKeep = fun(current) -> 0; - (Int) -> Int - end, - ErrorHandler = case cuttlefish:conf_get("log.error.file", Conf) of - undefined -> []; - ErrorLogFilename -> - ErrorLogRotation = cuttlefish:conf_get("log.error.rotation", Conf), - ErrorLogRotationKeep = TranslateKeep(cuttlefish:conf_get("log.error.rotation.keep", Conf)), - - ErrorLogSize = cuttlefish:conf_get("log.error.size", Conf), - [{lager_file_backend, [{file, ErrorLogFilename}, - {level, error}, - {size, ErrorLogSize}, - {date, ErrorLogRotation}, - {count, ErrorLogRotationKeep}]}] - end, - - ConsoleLogLevel = cuttlefish:conf_get("log.console.level", Conf), - ConsoleLogFile = cuttlefish:conf_get("log.console.file", Conf), - ConsoleLogSize = cuttlefish:conf_get("log.console.size", Conf), - ConsoleLogRotation = cuttlefish:conf_get("log.console.rotation", Conf), - ConsoleLogRotationKeep = TranslateKeep(cuttlefish:conf_get("log.console.rotation.keep", Conf)), - - ConsoleHandler = {lager_console_backend, ConsoleLogLevel}, - ConsoleFileHandler = {lager_file_backend, [{file, ConsoleLogFile}, - {level, ConsoleLogLevel}, - {size, ConsoleLogSize}, - {date, ConsoleLogRotation}, - {count, ConsoleLogRotationKeep}]}, - - ConsoleHandlers = case cuttlefish:conf_get("log.console", Conf) of - off -> []; - file -> [ConsoleFileHandler]; - console -> [ConsoleHandler]; - both -> [ConsoleHandler, ConsoleFileHandler]; - _ -> [] - end, - SyslogHandler ++ ConsoleHandlers ++ ErrorHandler - end -}. - -%% @doc Whether to enable Erlang's built-in error logger. -{mapping, "sasl", "sasl.sasl_error_logger", [ - {default, off}, - {datatype, flag}, - hidden -]}. - -%% @doc Whether to enable the crash log. -{mapping, "log.crash", "lager.crash_log", [ - {default, on}, - {datatype, flag} -]}. - -%% @doc If the crash log is enabled, the file where its messages will -%% be written. -{mapping, "log.crash.file", "lager.crash_log", [ - {default, "$(platform_log_dir)/crash.log"}, - {datatype, file} -]}. - -{translation, - "lager.crash_log", - fun(Conf) -> - case cuttlefish:conf_get("log.crash", Conf) of - false -> undefined; - _ -> - cuttlefish:conf_get("log.crash.file", Conf, "{{platform_log_dir}}/crash.log") - end - end}. - -%% @doc Maximum size in bytes of individual messages in the crash log -{mapping, "log.crash.maximum_message_size", "lager.crash_log_msg_size", [ - {default, "64KB"}, - {datatype, bytesize} +%% @doc With log rotation enabled, this decides the maximum number of log +%% files to store. +{mapping, "logger.max_files", "kernel.logger", [ + {default, 10}, + {datatype, integer} ]}. -%% @doc Maximum size of the crash log in bytes, before it is rotated -{mapping, "log.crash.size", "lager.crash_log_size", [ - {default, "10MB"}, +%% @doc The maximum size of a single log file. Total size used for log files will +%% be max_file_size * max_files. +{mapping, "logger.max_file_size", "kernel.logger", [ + {default, "1MB"}, {datatype, bytesize} ]}. -%% @doc The schedule on which to rotate the crash log. For more -%% information see: -%% https://github.com/basho/lager/blob/master/README.md#internal-log-rotation -{mapping, "log.crash.rotation", "lager.crash_log_date", [ - {default, "$D0"} +%% @doc Whether to enable SASL reports. +{mapping, "logger.sasl.enabled", "kernel.logger", [ + {default, {{logger_sasl_enabled}}}, + {datatype, {flag, on, off}} ]}. -%% @doc The number of rotated crash logs to keep. When set to -%% 'current', only the current open log file is kept. -{mapping, "log.crash.rotation.keep", "lager.crash_log_count", [ - {default, 5}, - {datatype, [integer, {atom, current}]}, - {validators, ["rotation_count"]} +%% @doc Filename to use for SASL reports. +{mapping, "logger.sasl.file", "kernel.logger", [ + {default, "{{platform_log_dir}}/reports.log"}, + {datatype, file} ]}. -{validator, - "rotation_count", - "must be 'current' or a positive integer", - fun(current) -> true; - (Int) when is_integer(Int) andalso Int >= 0 -> true; - (_) -> false - end}. - {translation, - "lager.crash_log_count", + "kernel.logger", fun(Conf) -> - case cuttlefish:conf_get("log.crash.rotation.keep", Conf) of - current -> 0; - Int -> Int - end - end}. - -%% @doc Whether to redirect error_logger messages into lager - -%% defaults to true -{mapping, "log.error.redirect", "lager.error_logger_redirect", [ - {default, on}, - {datatype, flag}, - hidden -]}. - -%% @doc Maximum number of error_logger messages to handle in a second -{mapping, "log.error.messages_per_second", "lager.error_logger_hwm", [ - {default, 100}, - {datatype, integer}, - hidden -]}. + LogFile = cuttlefish:conf_get("logger.file", Conf), + MaxNumBytes = cuttlefish:conf_get("logger.max_file_size", Conf), + MaxNumFiles = cuttlefish:conf_get("logger.max_files", Conf), + + DefaultFormatStr = cuttlefish:conf_get("logger.format", Conf), + DefaultFormatTerm = + case erl_scan:string(DefaultFormatStr) of + {ok, DefaultTokens, _} -> + case erl_parse:parse_term(DefaultTokens) of + {ok, DefaultTerm} -> + DefaultTerm; + {error, {_, _, DefaultError}} -> + cuttlefish:error(foo) + end; + {error, {_, _, DefaultScanError}} -> + cuttlefish:error(foo) + end, + ConfigMap0 = #{config => #{file => LogFile, + max_no_bytes => MaxNumBytes, + max_no_files => MaxNumFiles}, + filters => [{no_sasl, {fun logger_filters:domain/2, {stop, super, [otp, sasl]}}}], + formatter => {logger_formatter, + #{template => DefaultFormatTerm} + } + }, + + StandardCfg = [{handler, default, logger_std_h, ConfigMap0}], + + SaslCfg = + case cuttlefish:conf_get("logger.sasl.enabled", Conf) of + true -> + SaslLogFile = cuttlefish:conf_get("logger.sasl.file", Conf), + SaslConfigMap = #{config => #{file => SaslLogFile, + max_no_bytes => MaxNumBytes, %% reuse + max_no_files => MaxNumFiles}, + filter_default => stop, + filters => [{sasl_here, {fun logger_filters:domain/2, {log, equal, [otp, sasl]}}}], + formatter => {logger_formatter, + #{legacy_header => true, + single_line => false} + } + }, + [{handler, sasl, logger_std_h, SaslConfigMap}]; + _ -> + [] + end, + + StandardCfg ++ SaslCfg + end +}. -{mapping, "platform_log_dir", "riak_cs.platform_log_dir", [ - {datatype, directory}, - {default, "{{platform_log_dir}}"} -]}. %% == Supercluster aka Multi-bug Support == @@ -678,9 +598,9 @@ %% each line, in which the last token of key is member identifier and %% value is a PB address of Riak node to connect. {mapping, "supercluster.member.$member_id", "riak_cs.supercluster_members", [ - {datatype, ip}, + {datatype, [ip, fqdn]}, {include_default, "bag-A"}, - {commented, {"192.168.0.101", 8087}}, + {commented, {"riak-A1.example.com:8087", 8087}}, hidden ]}. @@ -700,7 +620,7 @@ }. %% @doc Interval to refresh weight information for every supercluster member. -{mapping, "supercluster.weight_refresh_interval", +{mapping, "supercluster.weight_refresh_interval", "riak_cs.supercluster_weight_refresh_interval", [ {default, "15m"}, {datatype, [{duration, s}]}, @@ -714,6 +634,12 @@ {default, "riak"} ]}. +%% override zdbbl from 1mb to 32mb +{mapping, "erlang.distribution_buffer_size", "vm_args.+zdbbl", [ + {default, "32MB"}, + merge +]}. + %% VM scheduler collapse, part 1 of 2 {mapping, "erlang.schedulers.force_wakeup_interval", "vm_args.+sfwi", [ {default, 500}, diff --git a/apps/riak_cs/src b/apps/riak_cs/src deleted file mode 120000 index a8d2a6755..000000000 --- a/apps/riak_cs/src +++ /dev/null @@ -1 +0,0 @@ -../../src/ \ No newline at end of file diff --git a/apps/riak_cs/src/common/rcs_common_manifest.erl b/apps/riak_cs/src/common/rcs_common_manifest.erl new file mode 100644 index 000000000..4e7796020 --- /dev/null +++ b/apps/riak_cs/src/common/rcs_common_manifest.erl @@ -0,0 +1,48 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(rcs_common_manifest). + +-export([make_versioned_key/2, + decompose_versioned_key/1]). + +-include("riak_cs.hrl"). +-include("manifest.hrl"). + + +-spec make_versioned_key(binary(), binary()) -> binary(). +make_versioned_key(Key, ?LFS_DEFAULT_OBJECT_VERSION) -> +%% old keys written without a version should continue to be accessible +%% for reads with the default version + Key; +make_versioned_key(Key, Vsn) -> + <>. + + +-spec decompose_versioned_key(binary()) -> {binary(), binary()}. +decompose_versioned_key(VK) -> + case binary:split(VK, ?VERSIONED_KEY_SEPARATOR) of + [K, V] -> + {K, V}; + [K] -> + {K, ?LFS_DEFAULT_OBJECT_VERSION} + end. + diff --git a/src/riak_cs_manifest_resolution.erl b/apps/riak_cs/src/common/rcs_common_manifest_resolution.erl similarity index 90% rename from src/riak_cs_manifest_resolution.erl rename to apps/riak_cs/src/common/rcs_common_manifest_resolution.erl index e647c8292..0dd1a25ca 100644 --- a/src/riak_cs_manifest_resolution.erl +++ b/apps/riak_cs/src/common/rcs_common_manifest_resolution.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -20,9 +21,9 @@ %% @doc Module to resolve siblings with manifest records --module(riak_cs_manifest_resolution). +-module(rcs_common_manifest_resolution). --include("riak_cs.hrl"). +-include("manifest.hrl"). %% export Public API -export([resolve/1]). @@ -37,7 +38,7 @@ %% and values are dictionaries whose %% keys are UUIDs and whose values %% are manifests. --spec resolve(list()) -> term(). +-spec resolve([wrapped_manifest()]) -> wrapped_manifest(). resolve(Siblings) -> lists:foldl(fun resolve_dicts/2, orddict:new(), Siblings). @@ -48,14 +49,12 @@ resolve(Siblings) -> %% @doc Take two dictionaries %% of manifests and resolve them. %% @private --spec resolve_dicts(term(), term()) -> term(). resolve_dicts(A, B) -> orddict:merge(fun resolve_manifests/3, A, B). %% @doc Take two manifests with %% the same UUID and resolve them %% @private --spec resolve_manifests(term(), term(), term()) -> term(). resolve_manifests(_Key, A, B) -> AState = state_to_stage_number(A?MANIFEST.state), BState = state_to_stage_number(B?MANIFEST.state), @@ -72,8 +71,6 @@ state_to_stage_number(scheduled_delete) -> 40. %% The third and fourth args, A, B, are the %% manifests themselves. %% @private --spec resolve_manifests(integer(), integer(), term(), term()) -> term(). - resolve_manifests(StageA, StageB, A, _B) when StageA > StageB -> A; resolve_manifests(StageA, StageB, _A, B) when StageB > StageA -> @@ -86,13 +83,13 @@ resolve_manifests(_, _, WriteBlocksRemaining = resolve_written_blocks(A, B), LastBlockWrittenTime = resolve_last_written_time(A, B), Props = resolve_props(A, B), - A?MANIFEST{write_blocks_remaining=WriteBlocksRemaining, - last_block_written_time=LastBlockWrittenTime, + A?MANIFEST{write_blocks_remaining = WriteBlocksRemaining, + last_block_written_time = LastBlockWrittenTime, props = Props}; resolve_manifests(_, _, - ?MANIFEST{state = active,acl=A1Acl} = A, - ?MANIFEST{state = active,acl=A2Acl} = B) -> + ?MANIFEST{state = active, acl = A1Acl} = A, + ?MANIFEST{state = active, acl = A2Acl} = B) -> Props = resolve_props(A, B), case A1Acl?ACL.creation_time >= A2Acl?ACL.creation_time of true -> @@ -106,16 +103,16 @@ resolve_manifests(_, _, ?MANIFEST{state = pending_delete} = B) -> BlocksLeftToDelete = resolve_deleted_blocks(A, B), LastDeletedTime = resolve_last_deleted_time(A, B), - A?MANIFEST{delete_blocks_remaining=BlocksLeftToDelete, - last_block_deleted_time=LastDeletedTime}; + A?MANIFEST{delete_blocks_remaining = BlocksLeftToDelete, + last_block_deleted_time = LastDeletedTime}; resolve_manifests(_, _, ?MANIFEST{state = scheduled_delete} = A, ?MANIFEST{state = scheduled_delete} = B) -> BlocksLeftToDelete = resolve_deleted_blocks(A, B), LastDeletedTime = resolve_last_deleted_time(A, B), - A?MANIFEST{delete_blocks_remaining=BlocksLeftToDelete, - last_block_deleted_time=LastDeletedTime}. + A?MANIFEST{delete_blocks_remaining = BlocksLeftToDelete, + last_block_deleted_time = LastDeletedTime}. resolve_written_blocks(A, B) -> AWritten = A?MANIFEST.write_blocks_remaining, diff --git a/apps/riak_cs/src/common/rcs_common_manifest_utils.erl b/apps/riak_cs/src/common/rcs_common_manifest_utils.erl new file mode 100644 index 000000000..de8b422fc --- /dev/null +++ b/apps/riak_cs/src/common/rcs_common_manifest_utils.erl @@ -0,0 +1,467 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc Module for choosing and manipulating lists (well, orddict) of manifests + +-module(rcs_common_manifest_utils). + +-include("manifest.hrl"). + +-export([new_dict/2, + active_manifest/1, + active_manifests/1, + active_and_writing_manifests/1, + deleted_while_writing/1, + filter_manifests_by_state/2, + mark_deleted/2, + mark_pending_delete/2, + mark_scheduled_delete/2, + needs_pruning/3, + prune/4, + upgrade_wrapped_manifests/1, + upgrade_manifest/1]). + +%% @doc Return a new orddict of manifest (only +%% one in this case). Used when storing something +%% in Riak when the previous GET returned notfound, +%% so we're (maybe) creating a new object. +-spec new_dict(binary(), lfs_manifest()) -> orddict:orddict(). +new_dict(UUID, Manifest) -> + orddict:store(UUID, Manifest, orddict:new()). + + +-spec upgrade_wrapped_manifests([wrapped_manifest()]) -> [wrapped_manifest()]. +upgrade_wrapped_manifests(ListofOrdDicts) -> + DictMapFun = fun(_Key, Value) -> upgrade_manifest(Value) end, + MapFun = fun(Value) -> orddict:map(DictMapFun, Value) end, + lists:map(MapFun, ListofOrdDicts). + +%% @doc Upgrade the manifest to the most recent +%% version of the manifest record. This is so that +%% _most_ of the codebase only has to deal with +%% the most recent version of the record. +-spec upgrade_manifest(#lfs_manifest_v2{} | + #lfs_manifest_v3{} | + #lfs_manifest_v4{} | + #lfs_manifest_v5{}) -> + lfs_manifest(). +upgrade_manifest(#lfs_manifest_v5{} = A) -> + A; +upgrade_manifest(#lfs_manifest_v4{block_size = BlockSize, + bkey = Bkey, + vsn = Vsn, + metadata = Metadata, + created = _Created, + uuid = UUID, + content_length = ContentLength, + content_type = ContentType, + content_md5 = ContentMd5, + state = State, + write_start_time = WriteStartTime, + last_block_written_time = LastBlockWrittenTime, + write_blocks_remaining = WriteBlocksRemaining, + delete_marked_time = DeleteMarkedTime, + last_block_deleted_time = LastBlockDeletedTime, + delete_blocks_remaining = DeleteBlocksRemaining, + scheduled_delete_time = ScheduledDeleteTime, + acl = Acl, + props = Props, + cluster_id = ClusterID}) -> + #lfs_manifest_v5{block_size = BlockSize, + bkey = Bkey, + vsn = Vsn, + metadata = Metadata, + uuid = UUID, + content_length = ContentLength, + content_type = ContentType, + content_md5 = ContentMd5, + state = State, + write_start_time = maybe_now_to_system_time(WriteStartTime), + last_block_written_time = maybe_now_to_system_time(LastBlockWrittenTime), + write_blocks_remaining = WriteBlocksRemaining, + delete_marked_time = maybe_now_to_system_time(DeleteMarkedTime), + last_block_deleted_time = maybe_now_to_system_time(LastBlockDeletedTime), + delete_blocks_remaining = DeleteBlocksRemaining, + scheduled_delete_time = maybe_now_to_system_time(ScheduledDeleteTime), + acl = riak_cs_acl:upgrade_acl_record(Acl), + props = upgrade_props(Props), + cluster_id = ClusterID}; + +upgrade_manifest(#lfs_manifest_v3{block_size = BlockSize, + bkey = Bkey, + metadata = Metadata, + uuid = UUID, + content_length = ContentLength, + content_type = ContentType, + content_md5 = ContentMd5, + state = State, + write_start_time = WriteStartTime, + last_block_written_time = LastBlockWrittenTime, + write_blocks_remaining = WriteBlocksRemaining, + delete_marked_time = DeleteMarkedTime, + last_block_deleted_time = LastBlockDeletedTime, + delete_blocks_remaining = DeleteBlocksRemaining, + scheduled_delete_time = ScheduledDeleteTime, + acl = Acl, + props = Props, + cluster_id = ClusterID}) -> + #lfs_manifest_v5{block_size = BlockSize, + bkey = Bkey, + vsn = ?LFS_DEFAULT_OBJECT_VERSION, + metadata = Metadata, + uuid = UUID, + content_length = ContentLength, + content_type = ContentType, + content_md5 = ContentMd5, + state = State, + write_start_time = maybe_now_to_system_time(WriteStartTime), + last_block_written_time = maybe_now_to_system_time(LastBlockWrittenTime), + write_blocks_remaining = WriteBlocksRemaining, + delete_marked_time = maybe_now_to_system_time(DeleteMarkedTime), + last_block_deleted_time = maybe_now_to_system_time(LastBlockDeletedTime), + delete_blocks_remaining = DeleteBlocksRemaining, + scheduled_delete_time = maybe_now_to_system_time(ScheduledDeleteTime), + acl = riak_cs_acl:upgrade_acl_record(Acl), + props = upgrade_props(fixup_props(Props)), + cluster_id = ClusterID}; + +upgrade_manifest(#lfs_manifest_v2{block_size = BlockSize, + bkey = Bkey, + metadata = Metadata, + uuid = UUID, + content_length = ContentLength, + content_type = ContentType, + content_md5 = ContentMd5, + state = State, + write_start_time = WriteStartTime, + last_block_written_time = LastBlockWrittenTime, + write_blocks_remaining = WriteBlocksRemaining, + delete_marked_time = DeleteMarkedTime, + last_block_deleted_time = LastBlockDeletedTime, + delete_blocks_remaining = DeleteBlocksRemaining, + acl = Acl, + props = Props, + cluster_id = ClusterID}) -> + #lfs_manifest_v5{block_size = BlockSize, + bkey = Bkey, + metadata = Metadata, + uuid = UUID, + content_length = ContentLength, + content_type = ContentType, + content_md5 = ContentMd5, + state = State, + write_start_time = maybe_now_to_system_time(WriteStartTime), + last_block_written_time = maybe_now_to_system_time(LastBlockWrittenTime), + write_blocks_remaining = WriteBlocksRemaining, + delete_marked_time = maybe_now_to_system_time(DeleteMarkedTime), + last_block_deleted_time = maybe_now_to_system_time(LastBlockDeletedTime), + delete_blocks_remaining = DeleteBlocksRemaining, + acl = riak_cs_acl:upgrade_acl_record(Acl), + props = upgrade_props(fixup_props(Props)), + cluster_id = ClusterID}. + +upgrade_props(PP) -> + case proplists:get_value(multipart, PP, undefined) of + undefined -> + PP; + MPM -> + lists:keyreplace(multipart, 1, PP, {multipart, upgrade_multipart_manifest(MPM)}) + end. + + +upgrade_multipart_manifest(undefined) -> + undefined; +upgrade_multipart_manifest(#multipart_manifest_v2{} = A) -> + A; +upgrade_multipart_manifest(#multipart_manifest_v1{upload_id = UploadId, + owner = Owner, + parts = Parts, + done_parts = DoneParts, + cleanup_parts = CleanupParts, + props = Props}) -> + #multipart_manifest_v2{upload_id = UploadId, + owner = riak_cs_acl:upgrade_owner(Owner), + parts = upgrade_parts(Parts), + done_parts = DoneParts, + cleanup_parts = upgrade_parts(CleanupParts), + props = Props}. + +upgrade_parts(A) -> + ordsets:from_list( + [upgrade_part(P) || P <- ordsets:to_list(A)]). +upgrade_part(P = #part_manifest_v3{}) -> + P; +upgrade_part(#part_manifest_v2{bucket = Bucket, + key = Key, + vsn = Vsn, + start_time = StartTime, + part_number = PartNumber, + part_id = PartId, + content_length = ContentLength, + content_md5 = ContentMD5, + block_size = BlockSize}) -> + #part_manifest_v3{bucket = Bucket, + key = Key, + vsn = Vsn, + start_time = maybe_now_to_system_time(StartTime), + part_number = PartNumber, + part_id = PartId, + content_length = ContentLength, + content_md5 = ContentMD5, + block_size = BlockSize}. + + +maybe_now_to_system_time({M1, M2, M3}) -> + M1 * 1000000 + M2 + M3 div 1000; +maybe_now_to_system_time(M) -> + M. + +fixup_props(undefined) -> + []; +fixup_props(Props) when is_list(Props) -> + Props. + + +%% @doc Return the current active manifest +%% from an orddict of manifests. +-spec active_manifest(orddict:orddict()) -> {ok, lfs_manifest()} | {error, no_active_manifest}. +active_manifest(Dict) -> + live_manifest(lists:foldl(fun most_recent_active_manifest/2, + {no_active_manifest, undefined}, orddict_values(Dict))). + +%% @doc Ensure the manifest hasn't been deleted during upload. +live_manifest({no_active_manifest, _}) -> + {error, no_active_manifest}; +live_manifest({Manifest, undefined}) -> + {ok, Manifest}; +live_manifest({Manifest, DeleteTime}) -> + case DeleteTime > Manifest?MANIFEST.write_start_time of + true -> + {error, no_active_manifest}; + false -> + {ok, Manifest} + end. + +%% NOTE: This is a foldl function, initial acc = {no_active_manifest, undefined} +%% Return the most recent active manifest as well as the most recent manifest delete time +most_recent_active_manifest(Manifest = ?MANIFEST{state = scheduled_delete}, + {MostRecent, undefined}) -> + {MostRecent, delete_time(Manifest)}; +most_recent_active_manifest(Manifest = ?MANIFEST{state = scheduled_delete}, + {MostRecent, DeleteTime}) -> + Dt = delete_time(Manifest), + {MostRecent, later(Dt, DeleteTime)}; +most_recent_active_manifest(Manifest = ?MANIFEST{state = pending_delete}, + {MostRecent, undefined}) -> + {MostRecent, delete_time(Manifest)}; +most_recent_active_manifest(Manifest = ?MANIFEST{state = pending_delete}, + {MostRecent, DeleteTime}) -> + Dt = delete_time(Manifest), + {MostRecent, later(Dt, DeleteTime)}; +most_recent_active_manifest(Manifest = ?MANIFEST{state = active}, + {no_active_manifest, undefined}) -> + {Manifest, undefined}; +most_recent_active_manifest(Man1 = ?MANIFEST{state = active}, + {Man2 = ?MANIFEST{state = active}, DeleteTime}) + when Man1?MANIFEST.write_start_time > Man2?MANIFEST.write_start_time -> + {Man1, DeleteTime}; +most_recent_active_manifest(_Man1 = ?MANIFEST{state = active}, + {Man2 = ?MANIFEST{state = active}, DeleteTime}) -> + {Man2, DeleteTime}; +most_recent_active_manifest(Man1 = ?MANIFEST{state = active}, + {no_active_manifest, DeleteTime}) -> + {Man1, DeleteTime}; +most_recent_active_manifest(_Man1, {Man2 = ?MANIFEST{state = active}, DeleteTime}) -> + {Man2, DeleteTime}; +most_recent_active_manifest(_Manifest, {MostRecent, DeleteTime}) -> + {MostRecent, DeleteTime}. + +delete_time(Manifest) -> + case proplists:is_defined(deleted, Manifest?MANIFEST.props) of + true -> + Manifest?MANIFEST.delete_marked_time; + false -> + undefined + end. + +%% @doc Return the later of two times, accounting for undefined +later(undefined, undefined) -> + undefined; +later(undefined, DeleteTime2) -> + DeleteTime2; +later(DeleteTime1, undefined) -> + DeleteTime1; +later(DeleteTime1, DeleteTime2) -> + case DeleteTime1 > DeleteTime2 of + true -> + DeleteTime1; + false -> + DeleteTime2 + end. + + +%% @doc Return all active manifests +-spec active_manifests(orddict:orddict()) -> [lfs_manifest()] | []. +active_manifests(Dict) -> + lists:filter(fun manifest_is_active/1, orddict_values(Dict)). + +manifest_is_active(?MANIFEST{state = active}) -> true; +manifest_is_active(_Manifest) -> false. + + + +%% @doc Return a list of all manifests in the +%% `active' or `writing' state +-spec active_and_writing_manifests(orddict:orddict()) -> [{binary(), lfs_manifest()}]. +active_and_writing_manifests(Dict) -> + orddict:to_list(filter_manifests_by_state(Dict, [active, writing])). + + + +%% @doc Return `Dict' with the manifests in +%% `UUIDsToMark' with their state changed to +%% `pending_delete' +-spec mark_pending_delete(orddict:orddict(), list(binary())) -> + orddict:orddict(). +mark_pending_delete(Dict, UUIDsToMark) -> + MapFun = fun(K, V) -> + case lists:member(K, UUIDsToMark) of + true -> + V?MANIFEST{state=pending_delete, + delete_marked_time = os:system_time(millisecond)}; + false -> + V + end + end, + orddict:map(MapFun, Dict). + +%% @doc Return `Dict' with the manifests in +%% `UUIDsToMark' with their state changed to +%% `pending_delete' and {deleted, true} added to props. +-spec mark_deleted(orddict:orddict(), list(binary())) -> + orddict:orddict(). +mark_deleted(Dict, UUIDsToMark) -> + MapFun = fun(K, V) -> + case lists:member(K, UUIDsToMark) of + true -> + V?MANIFEST{state = pending_delete, + delete_marked_time = os:system_time(millisecond), + props=[{deleted, true} | V?MANIFEST.props]}; + false -> + V + end + end, + orddict:map(MapFun, Dict). + +%% @doc Return `Dict' with the manifests in +%% `UUIDsToMark' with their state changed to +%% `scheduled_delete' +-spec mark_scheduled_delete(orddict:orddict(), list(cs_uuid())) -> + orddict:orddict(). +mark_scheduled_delete(Dict, UUIDsToMark) -> + MapFun = fun(K, V) -> + case lists:member(K, UUIDsToMark) of + true -> + V?MANIFEST{state = scheduled_delete, + scheduled_delete_time = os:system_time(millisecond)}; + false -> + V + end + end, + orddict:map(MapFun, Dict). + + + +%% @doc Return all active manifests that have timestamps before the latest deletion +%% This happens when a manifest is still uploading while it is deleted. The upload +%% is allowed to complete, but is not visible afterwards. +-spec deleted_while_writing(orddict:orddict()) -> [binary()]. +deleted_while_writing(Manifests) -> + ManifestList = orddict_values(Manifests), + DeleteTime = latest_delete_time(ManifestList), + find_deleted_active_manifests(ManifestList, DeleteTime). + +-spec find_deleted_active_manifests([lfs_manifest()], term()) -> [cs_uuid()]. +find_deleted_active_manifests(_Manifests, undefined) -> + []; +find_deleted_active_manifests(Manifests, DeleteTime) -> + [M?MANIFEST.uuid || M <- Manifests, M?MANIFEST.state =:= active, + M?MANIFEST.write_start_time < DeleteTime]. + + + +-spec prune(orddict:orddict(), + non_neg_integer(), + unlimited | non_neg_integer(), non_neg_integer()) -> orddict:orddict(). +prune(Dict, Time, MaxCount, LeewaySeconds) -> + Filtered = orddict:filter(fun (_Key, Value) -> not needs_pruning(Value, Time, LeewaySeconds) end, + Dict), + prune_count(Filtered, MaxCount). + +-spec prune_count(orddict:orddict(), unlimited | non_neg_integer()) -> + orddict:orddict(). +prune_count(Manifests, unlimited) -> + Manifests; +prune_count(Manifests, MaxCount) -> + ScheduledDelete = filter_manifests_by_state(Manifests, [scheduled_delete]), + UUIDAndTime = [{M?MANIFEST.uuid, M?MANIFEST.scheduled_delete_time} || + {_UUID, M} <- ScheduledDelete], + case length(UUIDAndTime) > MaxCount of + true -> + SortedByTimeRecentFirst = lists:keysort(2, UUIDAndTime), + UUIDsToPrune = sets:from_list([UUID || {UUID, _ScheduledDeleteTime} <- + lists:nthtail(MaxCount, SortedByTimeRecentFirst)]), + orddict:filter(fun (UUID, _Value) -> not sets:is_element(UUID, UUIDsToPrune) end, + Manifests); + false -> + Manifests + end. + +-spec needs_pruning(lfs_manifest(), non_neg_integer(), non_neg_integer()) -> boolean(). +needs_pruning(?MANIFEST{state = scheduled_delete, + scheduled_delete_time = ScheduledDeleteTime}, Time, LeewaySeconds) -> + (Time - ScheduledDeleteTime) > LeewaySeconds * 1000; +needs_pruning(_Manifest, _, _) -> + false. + + + +latest_delete_time(Manifests) -> + lists:foldl(fun(M, Acc) -> + DeleteTime = delete_time(M), + later(DeleteTime, Acc) + end, undefined, Manifests). + + + +%% @doc Filter an orddict manifests and accept only manifests whose +%% current state is specified in the `AcceptedStates' list. +filter_manifests_by_state(Dict, AcceptedStates) -> + AcceptManifest = + fun(_, ?MANIFEST{state = State}) -> + lists:member(State, AcceptedStates) + end, + orddict:filter(AcceptManifest, Dict). + + + +orddict_values(OrdDict) -> + [V || {_K, V} <- orddict:to_list(OrdDict)]. diff --git a/apps/riak_cs/src/lib/base64url.erl b/apps/riak_cs/src/lib/base64url.erl new file mode 100644 index 000000000..0aa70dfaa --- /dev/null +++ b/apps/riak_cs/src/lib/base64url.erl @@ -0,0 +1,173 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +%% @doc base64url is a wrapper around the base64 module to produce +%% base64-compatible encodings that are URL safe. +%% The / character in normal base64 encoding is replaced with +%% the _ character, + is replaced with -,and = is omitted. +%% This replacement scheme is named "base64url" by +%% http://en.wikipedia.org/wiki/Base64 + +-module(base64url). + +-export([decode/1, + decode_to_string/1, + encode/1, + encode_to_string/1]). + +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +decode(Base64url) -> + base64:decode(append_equals(urldecode(Base64url))). + +decode_to_string(Base64url) -> + base64:decode_to_string(append_equals(urldecode(Base64url))). + +encode(Data) -> + urlencode(strip_equals(base64:encode(Data))). + +encode_to_string(Data) -> + urlencode(strip_equals(base64:encode_to_string(Data))). + +-spec strip_equals(binary() | string()) -> binary()|string(). +%% @private Strip off trailing '=' characters. +strip_equals(Str) when is_list(Str) -> + string:strip(Str, right, $=); +strip_equals(Bin) when is_binary(Bin) -> + LCS = binary:longest_common_suffix([Bin, <<"===">>]), + binary:part(Bin, 0, byte_size(Bin)-LCS). + +-spec append_equals(binary()|string()) -> binary()|string(). +%% @private Append trailing '=' characters to make result legal Base64 length. +%% The most common use case will be a B64-encoded UUID, requiring the addition +%% of 2 characters, so that's the first check. We assume 0 and 3 are equally +%% likely. +%% Because B64 encoding spans all bytes across two characters, the remainder +%% of (length / 4) can never be 1 with a valid encoding, so we throw a badarg +%% with the argument here rather than letting it percolate up from stdlib with +%% no worthwhile information. +append_equals(Str) when is_list(Str) -> + case length(Str) rem 4 of + 2 -> + Str ++ "=="; + 0 -> + Str; + 3 -> + Str ++ "="; + 1 -> + erlang:error(badarg, [Str]) + end; +append_equals(Bin) when is_binary(Bin) -> + case byte_size(Bin) rem 4 of + 2 -> + <>; + 0 -> + Bin; + 3 -> + <>, <<"/">>, <<"=">>]), + {ok, StrRE} = re:compile("[\\+/=]"), + {BinRE, StrRE}. + +encode_decode_test() -> + crypto:start(), + % Make sure Rand is at least twice as long as the highest count, but + % because we're depleting the entropy pool don't go overboard! + RandLen = 2050, + % crypto:rand:bytes/1 would be fine for us, but it's gone in OTP-19, + % so use the good stuff rather than bothering with a version check. + BinRand = crypto:strong_rand_bytes(RandLen), + % Swap halves so the binary and string tests don't use the same sequences. + HalfLen = (RandLen div 2), + RandLo = binary_part(BinRand, 0, HalfLen), + RandHi = binary_part(BinRand, HalfLen, HalfLen), + StrRand = << RandHi/binary, RandLo/binary >>, + {BinRE, StrRE} = illegal_char_REs(), + test_encode_decode_uuid(256, BinRE, StrRE), + test_encode_decode_binary(1024, BinRE, StrRE, BinRand), + test_encode_decode_string(384, BinRE, StrRE, StrRand). + +test_encode_decode_uuid(0, _, _) -> + ok; +test_encode_decode_uuid(Count, BinRE, StrRE) -> + test_encode_decode(uuid:get_v4(), BinRE, StrRE), + test_encode_decode_uuid((Count - 1), BinRE, StrRE). + +test_encode_decode_binary(0, _, _, _) -> + ok; +test_encode_decode_binary(Count, BinRE, StrRE, Rand) -> + test_encode_decode(binary:part(Rand, Count, Count), BinRE, StrRE), + test_encode_decode_binary((Count - 1), BinRE, StrRE, Rand). + +test_encode_decode_string(0, _, _, _) -> + ok; +test_encode_decode_string(Count, BinRE, StrRE, Rand) -> + test_encode_decode(binary:bin_to_list(Rand, Count, Count), BinRE, StrRE), + test_encode_decode_string((Count - 1), BinRE, StrRE, Rand). + +test_encode_decode(Data, BinRE, StrRE) when is_binary(Data) -> + EncBin = base64url:encode(Data), + EncStr = base64url:encode_to_string(Data), + ?assertEqual(EncStr, binary_to_list(EncBin)), + ?assertEqual(nomatch, binary:match(EncBin, BinRE)), + ?assertEqual(nomatch, re:run(EncStr, StrRE)), + ?assertEqual(Data, base64url:decode(EncBin)), + ?assertEqual(Data, base64url:decode(EncStr)); + +test_encode_decode(Data, BinRE, StrRE) when is_list(Data) -> + EncBin = base64url:encode(Data), + EncStr = base64url:encode_to_string(Data), + ?assertEqual(EncStr, binary_to_list(EncBin)), + ?assertEqual(nomatch, binary:match(EncBin, BinRE)), + ?assertEqual(nomatch, re:run(EncStr, StrRE)), + ?assertEqual(Data, base64url:decode_to_string(EncBin)), + ?assertEqual(Data, base64url:decode_to_string(EncStr)). + +-endif. diff --git a/test/riak_cs_wm_key_test.erl b/apps/riak_cs/src/lib/exprec.erl similarity index 53% rename from test/riak_cs_wm_key_test.erl rename to apps/riak_cs/src/lib/exprec.erl index 1d86ec406..12972475c 100644 --- a/test/riak_cs_wm_key_test.erl +++ b/apps/riak_cs/src/lib/exprec.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,28 +19,35 @@ %% %% --------------------------------------------------------------------- --module(riak_cs_wm_key_test). - --compile(export_all). +-module(exprec). -include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). --include_lib("eunit/include/eunit.hrl"). +-include("stanchion.hrl"). +-include("aws_api.hrl"). + +-compile([export_all, nowarn_export_all]). + +-define(ALL_RECORDS, + [ rcs_user_v3 + , moss_bucket_v2 + , acl_v3 + , acl_grant_v2 + , bucket_versioning + %% AWS records + , iam_policy + , statement + , tag + , role_last_used + , permissions_boundary + , role_v1 + , saml_provider_v1 + , credentials + ] + ). -key_test_() -> - {setup, - fun riak_cs_wm_test_utils:setup/0, - fun riak_cs_wm_test_utils:teardown/1, - [fun get_object/0]}. +-export_records(?ALL_RECORDS). -get_object() -> - %% XXX TODO: MAKE THESE ACTUALLY TEST SOMETHING - %% We use this instead of setting - %% path info the wm_reqdata because - %% riak_cs_wm_utils:ensure_doc uses - %% it. - _Ctx= #key_context{bucket="keytest", key="foo"}, - _RD = #wm_reqdata{}, - ?assert(true). -%% {Object, _, _} = riak_cs_wm_key:produce_body(RD, Ctx), -%% ?assertEqual(<<>>, Object). +-exprecs_prefix(["", operation, ""]). +-exprecs_fname([prefix, "_", record]). +-exprecs_vfname([fname, "__", version]). +-compile({parse_transform, exprecs}). diff --git a/apps/riak_cs/src/lib/netutils.erl b/apps/riak_cs/src/lib/netutils.erl new file mode 100644 index 000000000..a2f75824e --- /dev/null +++ b/apps/riak_cs/src/lib/netutils.erl @@ -0,0 +1,101 @@ +-module(netutils). +-author('Juan Jose Comellas '). + +%% API +-export([get_local_ip_from_subnet/1, get_ip_from_subnet/2, + cidr_network/1, cidr_netmask/1]). + +%% @type ipv4() = {integer(), integer(), integer(), integer()} + + +%%-------------------------------------------------------------------- +%% @spec get_local_ip_from_subnet({Network :: ipv4(), Netmask :: ipv4()}) -> +%% {ok, ipv4()} | undefined | {error, Reason, Data} +%% @doc Returns an IP address that is assigned to a local network interface and +%% belongs to the specified subnet. +%%-------------------------------------------------------------------- +get_local_ip_from_subnet({{_I1, _I2, _I3, _I4}, {_N1, _N2, _N3, _N4}} = Subnet) -> + case inet:getif() of + {ok, AddrList} when is_list(AddrList) -> + get_ip_from_subnet(Subnet, [Ip || {Ip, _Broadcast, _Netmask} <- AddrList]); + InvalidAddrList -> + {error, invalid_network_interface, [InvalidAddrList]} + end; +get_local_ip_from_subnet({{_I1, _I2, _I3, _I4} = Ip, Bits}) + when is_integer(Bits), Bits >= 0, Bits =< 32 -> + get_local_ip_from_subnet({Ip, cidr_netmask(Bits)}). + + + +%%-------------------------------------------------------------------- +%% @spec get_ip_from_subnet({Network :: ipv4(), Netmask :: ipv4(), [ipv4()]}) -> +%% {ok, ipv4()} | undefined | {error, Reason, Data} +%% @doc Returns the first IP address in the list received as argument that +%% belongs to the specified subnet. +%%-------------------------------------------------------------------- +get_ip_from_subnet({{I1, I2, I3, I4}, {N1, N2, N3, N4} = Netmask}, AddrList) -> + get_ip_from_normalized_subnet({{I1 band N1, I2 band N2, I3 band N3, I4 band N4}, + Netmask}, AddrList); +get_ip_from_subnet({{_I1, _I2, _I3, _I4} = Ip, Bits}, AddrList) + when is_integer(Bits), Bits >= 0, Bits =< 32 -> + get_ip_from_subnet({Ip, cidr_netmask(Bits)}, AddrList). + + +get_ip_from_normalized_subnet({{I1, I2, I3, I4}, {N1, N2, N3, N4}} = Subnet, + [{A1, A2, A3, A4} = Addr | Tail]) -> + if ((A1 band N1) =:= I1 andalso + (A2 band N2) =:= I2 andalso + (A3 band N3) =:= I3 andalso + (A4 band N4) =:= I4) -> + {ok, Addr}; + true -> + get_ip_from_normalized_subnet(Subnet, Tail) + end; +get_ip_from_normalized_subnet(_Subnet, []) -> + undefined. + + +%%-------------------------------------------------------------------- +%% @spec cidr_network({Addr :: ipv4(), Bits :: integer()}) -> ipv4() +%% @doc Return the subnet corresponding the the IP address and network bits +%% received in CIDR format. +%%-------------------------------------------------------------------- +cidr_network({{I1, I2, I3, I4}, Bits}) when is_integer(Bits) andalso Bits =< 32 -> + ZeroBits = 8 - (Bits rem 8), + Last = (16#ff bsr ZeroBits) bsl ZeroBits, + + case (Bits div 8) of + 0 -> + {(I1 band Last), 0, 0, 0}; + 1 -> + {I1, (I2 band Last), 0, 0}; + 2 -> + {I1, I2, (I3 band Last), 0}; + 3 -> + {I1, I2, I3, (I4 band Last)}; + 4 -> + {I1, I2, I3, I4} + end. + + +%%-------------------------------------------------------------------- +%% @spec cidr_netmask(Bits :: integer()) -> ipv4() +%% @doc Return the netmask corresponding to the network bits received in CIDR +%% format. +%%-------------------------------------------------------------------- +cidr_netmask(Bits) when is_integer(Bits) andalso Bits =< 32 -> + ZeroBits = 8 - (Bits rem 8), + Last = (16#ff bsr ZeroBits) bsl ZeroBits, + + case (Bits div 8) of + 0 -> + {(255 band Last), 0, 0, 0}; + 1 -> + {255, (255 band Last), 0, 0}; + 2 -> + {255, 255, (255 band Last), 0}; + 3 -> + {255, 255, 255, (255 band Last)}; + 4 -> + {255, 255, 255, 255} + end. diff --git a/src/rts.erl b/apps/riak_cs/src/lib/rts.erl similarity index 91% rename from src/rts.erl rename to apps/riak_cs/src/lib/rts.erl index 9971c0dac..020e546a2 100644 --- a/src/rts.erl +++ b/apps/riak_cs/src/lib/rts.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -46,37 +47,38 @@ find_samples/4, slice_containing/2, next_slice/2, - iso8601/1, + iso8601/1, iso8601_s/1, datetime/1 ]). -include("rts.hrl"). -ifdef(TEST). --ifdef(EQC). +-ifdef(PROPER). -compile([export_all]). --include_lib("eqc/include/eqc.hrl"). +-include_lib("proper/include/proper.hrl"). -endif. -include_lib("eunit/include/eunit.hrl"). -endif. +-include_lib("kernel/include/logger.hrl"). + -export_type([slice/0]). -type datetime() :: calendar:datetime(). -type slice() :: {Start :: datetime(), End :: datetime()}. --type mochijson2() :: term(). %% @doc Just create the new sample object (don't store it). --spec new_sample(binary(), iolist(), +-spec new_sample(binary(), binary(), datetime(), datetime(), - integer(), mochijson2()) + integer(), riakc_obj:riakc_object()) -> riakc_obj:riakc_obj(). new_sample(Bucket, KeyPostfix, Start, End, Period, Data) -> Slice = slice_containing(Start, Period), Key = slice_key(Slice, KeyPostfix), - MJSON = {struct, [{?START_TIME, iso8601(Start)}, - {?END_TIME, iso8601(End)} - |Data]}, - Body = iolist_to_binary(mochijson2:encode(MJSON)), + MJSON = [{?START_TIME, iso8601(Start)}, + {?END_TIME, iso8601(End)} + |Data], + Body = jsx:encode(MJSON), riakc_obj:new(Bucket, Key, Body, "application/json"). %% @doc Fetch all of the samples from riak that overlap the specified @@ -86,7 +88,7 @@ new_sample(Bucket, KeyPostfix, Start, End, Period, Data) -> %% extraction/etc. on the client side. It would be a a trivial %% modification to do this via MapReduce instead. -spec find_samples(fun(), datetime(), datetime(), integer()) -> - {Samples::[mochijson2()], Errors::[{slice(), Reason::term()}]}. + {Samples::list(), Errors::[{slice(), Reason::term()}]}. find_samples(Puller, Start, End, Period) -> Slices = slices_filling(Start, End, Period), {Samples, Errors} = lists:foldl(Puller, {[], []}, Slices), @@ -95,8 +97,6 @@ find_samples(Puller, Start, End, Period) -> %% @doc Make a thunk that lists:filter can use to filter samples for a %% given time period. Samples are stored in groups, an a user may %% request some, but not all, samples from a group. --spec sample_in_bounds(datetime(), datetime()) - -> fun( (list()) -> boolean() ). sample_in_bounds(Start, End) -> Start8601 = iso8601(Start), End8601 = iso8601(End), @@ -111,7 +111,7 @@ sample_in_bounds(Start, End) -> %% @doc Make the key for this slice+postfix. Note: this must be the %% actual slice, not just any two times (the times are not realigned %% to slice boundaries before making the key). --spec slice_key(slice(), iolist()) -> binary(). +-spec slice_key(slice(), binary()) -> binary(). slice_key({SliceStart, _}, Postfix) -> iolist_to_binary([iso8601(SliceStart),".",Postfix]). @@ -161,11 +161,20 @@ dtgs(DT) -> calendar:datetime_to_gregorian_seconds(DT). gsdt(S) -> calendar:gregorian_seconds_to_datetime(S). %% @doc Produce an ISO8601-compatible representation of the given time. --spec iso8601(calendar:datetime()) -> binary(). +-spec iso8601(calendar:datetime() | non_neg_integer()) -> binary(). +iso8601(TS) when is_integer(TS) -> + iso8601(calendar:system_time_to_universal_time(TS, millisecond)); iso8601({{Y,M,D},{H,I,S}}) -> iolist_to_binary( io_lib:format("~4..0b~2..0b~2..0bT~2..0b~2..0b~2..0bZ", [Y, M, D, H, I, S])). +-spec iso8601_s(calendar:datetime() | non_neg_integer()) -> string(). +iso8601_s(TS) when is_integer(TS) -> + iso8601_s(calendar:system_time_to_universal_time(TS, millisecond)); +iso8601_s({{Y,M,D},{H,I,S}}) -> + lists:flatten( + io_lib:format("~4..0b~2..0b~2..0bT~2..0b~2..0b~2..0bZ", + [Y, M, D, H, I, S])). %% @doc Produce a datetime tuple from a ISO8601 string -spec datetime(binary()|string()) -> {ok, calendar:datetime()} | error. @@ -298,8 +307,8 @@ make_object_prop() -> {SliceStart,_} = slice_containing(Start, Period), Obj = new_sample(Bucket, Postfix, Start, End, Period, []), - {struct, MJ} = mochijson2:decode( - riakc_obj:get_update_value(Obj)), + MJ = jsx:decode( + riakc_obj:get_update_value(Obj, [{return_maps, false}])), ?WHENFAIL( io:format(user, "keys: ~p~n", [MJ]), diff --git a/apps/riak_cs/src/lib/supps.erl b/apps/riak_cs/src/lib/supps.erl new file mode 100644 index 000000000..131a401f1 --- /dev/null +++ b/apps/riak_cs/src/lib/supps.erl @@ -0,0 +1,258 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2022, 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + + +-module(supps). + +-export([q/1, p/1, p/0]). + +-ignore_xref([q/1, p/1, p/0]). + +-record(p, {name :: atom() | tuple(), + info :: undefined | proplists:proplist(), + total_mem = 0 :: non_neg_integer(), + type :: worker | supervisor, + pid :: pid() | undefined, + children = [] + } + ). + +usage() -> + "A ps-like output of process_info items (memory, message_queue_len\n" + "etc) of processes under the supervisor trees of riak_cs_sup.\n" + "\n" + "Options:\n" + " --format flat|tree, selects the output format (default is 'tree');\n" + " --depth Depth, print children up to Depth level deep (default\n" + " is 2, meaning top-level sups with their immediate children). Depth\n" + " can be 'max';\n" + " --filter Regex, filter on process names to use (default is \".+\"):\n" + " * if a sup's name matches, all its children are shown;\n" + " * if a sup's name doesn't match, it is only shown if a match is\n" + " found in its children or below;\n" + " --order_by, ProcessInfoItem, sort by this process_info item, or 'none'\n" + " to preserve the order children are created (default is 'memory').\n". + + +sups() -> + [riak_cs_sup]. + + +-spec p(proplists:proplist()) -> ok. +p(Opts) -> + io:put_chars(q(Opts)). + +-spec p() -> ok. +p() -> + io:put_chars(q([])). + +-spec q(string() | proplists:proplist()) -> string(). +q(["--help"]) -> + usage(); +q([O|_] = Options) when not is_tuple(O) -> + q(validate_options(Options, [])); +q(Options0) -> + Nodes = [node() | nodes()], + Options = extract_options(Options0), + + erlang:put(supps_output, []), + lists:foreach( + fun(Node) -> + acc(io_lib:format("============ Node: ~s ===========================\n", [Node])), + acc(io_lib:format("~11s\t~5s\t~8s\t~.14s~s\n" + "-------------------------------------------------------------\n", + [mem, mq, ths, pid, process])), + print( + reformat([get_info(Node, P) || P <- sups()], Options), + Options, 0) + end, Nodes + ), + lists:flatten(lists:reverse(erlang:get(supps_output))). + +validate_options([], Good) -> + Good; +validate_options(["--depth", "max" | R], Q) -> + validate_options(R, [{depth, max} | Q]); +validate_options(["--depth", A | R], Q) -> + validate_options(R, [{depth, list_to_integer(A)} | Q]); +validate_options(["--format", "flat" | R], Q) -> + validate_options(R, [{format, flat} | Q]); +validate_options(["--format", "tree" | R], Q) -> + validate_options(R, [{format, tree} | Q]); +validate_options(["--filter", A | R], Q) -> + validate_options(R, [{filter, A} | Q]); +validate_options(["--order_by", A | R], Q) when A == "none"; + A == "memory" -> + validate_options(R, [{order_by, list_to_atom(A)} | Q]); +validate_options([K], Q) -> + io:format("Option requires a value: ~s\n", [K]), + validate_options([], Q); +validate_options([K, V | R], Q) -> + io:format("Invalid option ~s or value ~s\n", [K, V]), + validate_options(R, Q). + +extract_options(PL) -> + Depth = + case proplists:get_value(depth, PL, 1) of + max -> + 9999; + V -> + V + end, + #{filter => proplists:get_value(filter, PL, ".+"), + format => proplists:get_value(format, PL, tree), + order_by => proplists:get_value(order_by, PL, memory), + depth => Depth + }. + + +get_info(Node, Name) -> + FF = + lists:foldl( + fun({SubName, Pid, worker, _MM}, Q) -> + Info = rpc:call(Node, erlang, process_info, + [Pid, [memory, message_queue_len, messages, total_heap_size]]), + [#p{name = SubName, info = Info, type = worker, pid = Pid, + total_mem = proplists:get_value(memory, Info, 0)} | Q]; + ({SubName, _Pid, supervisor, _MM}, Q) -> + [get_info(Node, SubName) | Q] + end, + [], + case rpc:call(Node, supervisor, which_children, [Name]) of + Children when is_list(Children) -> + Children; + _ -> + [] + end + ), + #p{name = Name, + total_mem = lists:foldl(fun(#p{total_mem = TM}, Q) -> + Q + TM + end, 0, FF), + type = supervisor, + children = FF}. + + +print(_p, #{depth := MaxDepth}, Depth) when MaxDepth < Depth -> + ok; +print(PP, Options, Depth) when is_list(PP) -> + lists:foreach(fun(P) -> print(P, Options, Depth) end, PP); +print(#p{name = Name, info = Info, type = worker, pid = Pid}, + #{filter := Filter}, Depth) -> + case re:run(printable(Name), Filter) of + nomatch -> + skip; + _ -> + Mem = integer_or_blank(memory, Info), + THS = integer_or_blank(total_heap_size, Info), + MQ = integer_or_blank(message_queue_len, Info), + acc(io_lib:format("~11s\t~5s\t~8s\t~.14s~s~s\n", [Mem, MQ, THS, pid_to_list(Pid), pad(Depth * 2), printable(Name)])) + end; +print(#p{name = Name, children = FF, total_mem = Mem} = P, + Options = #{filter := Filter}, Depth) -> + case has_printable_children(P, Filter) of + no -> + skip; + Yes -> + acc(io_lib:format("~11b\t~5s\t~8s\t~14s~s~s (~b)\n", + [Mem, "", "", "", pad(Depth * 2), printable(Name), length(FF)])), + lists:foreach( + fun(F) -> print(F, Options#{filter => maybe_drop_filter(Yes, Filter)}, Depth + 1) end, + FF + ) + end. +maybe_drop_filter(all, _) -> ".+"; +maybe_drop_filter(yes, Filter) -> Filter. + + +integer_or_blank(F, Info) -> + A_ = proplists:get_value(F, Info, ""), + [integer_to_list(A_)||is_integer(A_)]. + +pad(N) -> + lists:duplicate(N, $ ). + +printable(Name) when is_atom(Name) -> atom_to_list(Name); +printable(Name) -> io_lib:format("~p", [Name]). + + +has_printable_children(#p{name = Name, type = worker}, Filter) -> + case re:run(printable(Name), Filter) of + nomatch -> + no; + _ -> + yes + end; +has_printable_children(#p{name = Name, children = FF}, Filter) -> + case re:run(printable(Name), Filter) of + nomatch -> + case lists:any( + fun(P) -> no /= has_printable_children(P, Filter) end, + FF) of + true -> + yes; + false -> + no + end; + _ -> + all + end. + +reformat(PP, #{format := tree, order_by := none}) -> + PP; +reformat(PP, #{format := tree, order_by := memory} = Options) -> + lists:map( + fun(P = #p{children = []}) -> + P; + (P = #p{children = FF}) -> + P#p{children = lists:sort( + fun(#p{total_mem = M1}, #p{total_mem = M2}) -> + M1 > M2 + end, + reformat(FF, Options))} + end, + PP); +reformat(#p{children = PP}, Options) -> + reformat(PP, Options); +reformat(PP, #{format := flat, order_by := OrderBy} = Options) -> + PP1 = + lists:flatten( + lists:foldl( + fun(#p{type = worker} = P, Q) -> + [P | Q]; + (#p{type = supervisor, children = FF}, Q) -> + lists:concat([reformat(FF, Options), Q]) + end, + [], + PP + ) + ), + case OrderBy of + none -> + PP1; + memory -> + lists:sort(fun(#p{total_mem = M1}, #p{total_mem = M2}) -> + M1 > M2 + end, PP1) + end. + +acc(L) -> + Q = erlang:get(supps_output), + erlang:put(supps_output, [L|Q]). diff --git a/src/table.erl b/apps/riak_cs/src/lib/table.erl similarity index 73% rename from src/table.erl rename to apps/riak_cs/src/lib/table.erl index 8f8dd8af5..281f0d405 100644 --- a/src/table.erl +++ b/apps/riak_cs/src/lib/table.erl @@ -1,8 +1,28 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + -module(table). %% API --export([print/2, - create_table/2]). +-export([print/2]). -spec print(list(), list()) -> ok. print(Spec, Rows) -> @@ -30,7 +50,7 @@ create_table(Spec, [Row | Rows], Length, IoList) -> -spec get_row_length(list(tuple())) -> non_neg_integer(). get_row_length(Spec) -> lists:foldl(fun({_Name, Size}, Total) -> - Total + Size + 2 + Total + Size + 2 end, 0, Spec) + 2. -spec row(list(), list(string())) -> iolist(). @@ -56,9 +76,9 @@ align(Str, Size) when is_binary(Str) -> align(binary_to_list(Str), Size); align(Str, Size) when is_atom(Str) -> align(atom_to_list(Str), Size); -align(Str, Size) when is_list(Str), length(Str) > Size -> +align(Str, Size) when is_list(Str), length(Str) > Size -> Truncated = lists:sublist(Str, Size), - Truncated ++ " |"; + Truncated ++ " |"; align(Str, Size) when is_list(Str), length(Str) =:= Size -> Str ++ " |"; align(Str, Size) when is_list(Str) -> @@ -81,4 +101,3 @@ spaces(Length) -> -spec char_seq(non_neg_integer(), char()) -> string(). char_seq(Length, Char) -> [Char || _ <- lists:seq(1, Length)]. - diff --git a/src/twop_set.erl b/apps/riak_cs/src/lib/twop_set.erl similarity index 97% rename from src/twop_set.erl rename to apps/riak_cs/src/lib/twop_set.erl index 0a7a0daf9..0b1382917 100644 --- a/src/twop_set.erl +++ b/apps/riak_cs/src/lib/twop_set.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -39,25 +40,20 @@ -ifdef(TEST). -compile(export_all). +-compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). -endif. %% export Public API --export([ - new/0, +-export([new/0, size/1, to_list/1, - is_element/2, add_element/2, del_element/2, resolve/1 ]). --ifdef(namespaced_types). -type stdlib_set() :: sets:set(). --else. --type stdlib_set() :: set(). --endif. -type twop_set() :: {stdlib_set(), stdlib_set()}. -export_type([twop_set/0]). @@ -77,8 +73,10 @@ size(Set) -> to_list(Set) -> sets:to_list(minus_deletes(Set)). +-ifdef(TEST). is_element(Element, Set) -> sets:is_element(Element, minus_deletes(Set)). +-endif. add_element(Element, Set={Adds,Dels}) -> case sets:is_element(Element, Dels) of diff --git a/test/riak_cs_wm_test_utils.erl b/apps/riak_cs/src/riak_cs.app.src similarity index 51% rename from test/riak_cs_wm_test_utils.erl rename to apps/riak_cs/src/riak_cs.app.src index 4ff879ed0..194f47d51 100644 --- a/test/riak_cs_wm_test_utils.erl +++ b/apps/riak_cs/src/riak_cs.app.src @@ -1,6 +1,8 @@ +%%-*- mode: erlang -*- %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,24 +20,40 @@ %% %% --------------------------------------------------------------------- --module(riak_cs_wm_test_utils). - --export([setup/0, teardown/1]). - -setup() -> - %% Start erlang node - application:start(sasl), - TestNode = list_to_atom("testnode" ++ integer_to_list(element(3, now()))), - net_kernel:start([TestNode, shortnames]), - application:start(lager), - application:start(riakc), - application:start(inets), - application:start(mochiweb), - application:start(crypto), - application:start(webmachine), - application:start(riak_cs). - -%% TODO: -%% Implement this -teardown(_) -> - ok. +{application, riak_cs, + [ + {description, "riak_cs"}, + {vsn, "3.2.5"}, + {modules, []}, + {registered, []}, + {applications, + [kernel, + stdlib, + inets, + crypto, + sasl, + syntax_tools, + uuid, + jsx, + jason, + getopt, + esaml, + mochiweb, + webmachine, + poolboy, + cluster_info, + exometer_core, + riakc, + riak_repl_pb_api, + riak_cs_multibag + ]}, + {mod, {riak_cs_app, []}}, + {env, + [{access_archive_period, 3600}, + {access_log_flush_factor, 1}, + {access_log_flush_size, 1000000}, + {access_archiver_max_backlog, 2}, + {storage_archive_period, 86400}, + {usage_request_limit, 744} + ]} + ]}. diff --git a/src/riak_cs_access.erl b/apps/riak_cs/src/riak_cs_access.erl similarity index 89% rename from src/riak_cs_access.erl rename to apps/riak_cs/src/riak_cs_access.erl index 249935117..711e89849 100644 --- a/src/riak_cs_access.erl +++ b/apps/riak_cs/src/riak_cs_access.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -32,7 +33,6 @@ flush_access_object_to_log/3 ]). --include("riak_cs.hrl"). -ifdef(TEST). -ifdef(EQC). -compile([export_all]). @@ -41,6 +41,9 @@ -include_lib("eunit/include/eunit.hrl"). -endif. +-include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). + -export_type([slice/0]). -type slice() :: {Start :: calendar:datetime(), @@ -104,20 +107,16 @@ max_flush_size() -> %% The keys of the proplist must be either atoms or binaries, to be %% encoded as JSON keys. The values of the proplists must be numbers, %% as the values for each key will be summed in the stored object. --spec make_object(iodata(), - [[{atom()|binary(), number()}]], - slice()) - -> riakc_obj:riakc_obj(). +-spec make_object(binary(), [[{atom()|binary(), number()}]], slice()) -> + riakc_obj:riakc_obj(). make_object(User, Accesses, {Start, End}) -> {ok, Period} = archive_period(), Aggregate = aggregate_accesses(Accesses), rts:new_sample(?ACCESS_BUCKET, User, Start, End, Period, - [{?NODEKEY, node()}|Aggregate]). + [{?NODEKEY, node()} | Aggregate]). aggregate_accesses(Accesses) -> - Merged = lists:foldl(fun merge_ops/2, [], Accesses), - %% now mochijson-ify - [ {OpName, {struct, Stats}} || {OpName, Stats} <- Merged ]. + lists:foldl(fun merge_ops/2, [], Accesses). merge_ops({OpName, Stats}, Acc) -> case lists:keytake(OpName, 1, Acc) of @@ -136,15 +135,15 @@ merge_stats(Stats, Acc) -> %% which the keys are Riak CS node names. The value for each key is a %% list of samples. Each sample is an orddict full of metrics. -spec get_usage(riak_client(), - term(), %% TODO: riak_cs:user_key() type doesn't exist + binary(), boolean(), %% Not used in this module calendar:datetime(), calendar:datetime()) -> - {Usage::orddict:orddict(), Errors::[{slice(), term()}]}. -get_usage(RcPid, User, _AdminAccess, Start, End) -> + {Usage::orddict:orddict(), Errors::[{slice(), term()}]}. +get_usage(RcPid, UserArn, _AdminAccess, Start, End) -> {ok, Period} = archive_period(), RtsPuller = riak_cs_riak_client:rts_puller( - RcPid, ?ACCESS_BUCKET, User, [riakc, get_access]), + RcPid, ?ACCESS_BUCKET, UserArn, [riakc, get_access]), {Usage, Errors} = rts:find_samples(RtsPuller, Start, End, Period), {group_by_node(Usage), Errors}. @@ -167,17 +166,19 @@ flush_to_log('$end_of_table', _, _) -> ok; flush_to_log(User, Table, Slice) -> Accesses = [ A || {_, A} <- ets:lookup(Table, User) ], - RiakObj = riak_cs_access:make_object(User, Accesses, Slice), + RiakObj = make_object(iolist_to_binary([User]), Accesses, Slice), flush_access_object_to_log(User, RiakObj, Slice), flush_to_log(ets:next(Table, User), Table, Slice). +-spec flush_access_object_to_log(binary(), riakc_obj:riakc_obj(), slice()) -> + ok. flush_access_object_to_log(User, RiakObj, Slice) -> {Start0, End0} = Slice, Start = rts:iso8601(Start0), End = rts:iso8601(End0), Accesses = riakc_obj:get_update_value(RiakObj), - _ = lager:warning("lost access stat: User=~s, Slice=(~s, ~s), Accesses:'~s'", - [User, Start, End, Accesses]). + logger:warning("lost access stat: User=~s, Slice=(~s, ~s), Accesses:'~s'", + [User, Start, End, Accesses]). -ifdef(TEST). @@ -231,11 +232,9 @@ make_object_prop() -> [ if is_atom(K) -> atom_to_binary(K, latin1); is_binary(K) -> K end || {K, _V} <- lists:flatten(Accesses)]), - {struct, MJ} = mochijson2:decode( - riakc_obj:get_update_value(Obj)), + MJ = jsx:decode(riakc_obj:get_update_value(Obj), [{return_maps, false}]), - Paired = [{{struct, sum_access(K, Accesses)}, - proplists:get_value(K, MJ)} + Paired = [{sum_access(K, Accesses), proplists:get_value(K, MJ)} || K <- Unique], ?WHENFAIL( diff --git a/src/riak_cs_access_archiver.erl b/apps/riak_cs/src/riak_cs_access_archiver.erl similarity index 94% rename from src/riak_cs_access_archiver.erl rename to apps/riak_cs/src/riak_cs_access_archiver.erl index 7802a9aee..cc3027b25 100644 --- a/src/riak_cs_access_archiver.erl +++ b/apps/riak_cs/src/riak_cs_access_archiver.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -45,6 +46,7 @@ code_change/4]). -include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). -record(state, { @@ -156,8 +158,8 @@ terminate(Reason, StateName, #state{table=Table, terminate(_Reason, _StateName, #state{table=Table, riak_client=RcPid, mon=Mon}) -> - _ = lager:warning("Access archiver stopping with work left to do;" - " logs will be dropped"), + logger:warning("Access archiver stopping with work left to do;" + " logs will be dropped"), cleanup(Table, RcPid, Mon), ok. @@ -197,7 +199,8 @@ continue(State) -> archive_user(User, RcPid, Table, Slice) -> Accesses = [ A || {_, A} <- ets:lookup(Table, User) ], - Record = riak_cs_access:make_object(User, Accesses, Slice), + Record = riak_cs_access:make_object( + iolist_to_binary([User]), Accesses, Slice), store(User, RcPid, Record, Slice). store(User, RcPid, Record, Slice) -> @@ -205,8 +208,7 @@ store(User, RcPid, Record, Slice) -> Timeout = riak_cs_config:put_access_timeout(), case catch riak_cs_pbc:put(MasterPbc, Record, Timeout, [riakc, put_access]) of ok -> - ok = lager:debug("Archived access stats for ~s ~p", - [User, Slice]); + ?LOG_DEBUG("Archived access stats for ~s ~p", [User, Slice]); {error, Reason} -> riak_cs_pbc:check_connection_status(MasterPbc, "riak_cs_access_archiver:store/4"), diff --git a/src/riak_cs_access_archiver_manager.erl b/apps/riak_cs/src/riak_cs_access_archiver_manager.erl similarity index 87% rename from src/riak_cs_access_archiver_manager.erl rename to apps/riak_cs/src/riak_cs_access_archiver_manager.erl index 0f3d3f4ee..fa93e9d44 100644 --- a/src/riak_cs_access_archiver_manager.erl +++ b/apps/riak_cs/src/riak_cs_access_archiver_manager.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -38,6 +39,7 @@ code_change/3]). -include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). -define(DEFAULT_MAX_BACKLOG, 2). % 2 ~= forced and next -define(DEFAULT_MAX_ARCHIVERS, 2). @@ -69,8 +71,8 @@ archive(Table, Slice) -> ets:give_away(Table, Pid, Slice) catch error:badarg -> - _ = lager:error("~p was not available, access stats for ~p lost", - [?MODULE, Slice]), + logger:error("~p was not available, access stats for ~p lost", + [?MODULE, Slice]), riak_cs_access:flush_to_log(Table, Slice), %% if the archiver had been alive just now, but crashed %% during operation, the stats also would have been lost, @@ -83,8 +85,8 @@ archive(Table, Slice) -> false %% opposite of ets:give_away/3 success end; _ -> - _ = lager:error("~p was not available, access stats for ~p lost", - [?MODULE, Slice]), + logger:error("~p was not available, access stats for ~p lost", + [?MODULE, Slice]), riak_cs_access:flush_to_log(Table, Slice), ets:delete(Table), false @@ -124,20 +126,18 @@ init([]) -> riak_cs, access_archiver_max_workers) of {ok, Workers} when is_integer(Workers) -> Workers; _ -> - _ = lager:warning( - "access_archiver_max_workers was unset or" - " invalid; overriding with default of ~b", - [?DEFAULT_MAX_ARCHIVERS]), + logger:warning("access_archiver_max_workers was unset or" + " invalid; overriding with default of ~b", + [?DEFAULT_MAX_ARCHIVERS]), ?DEFAULT_MAX_ARCHIVERS end, MaxBacklog = case application:get_env( riak_cs, access_archiver_max_backlog) of {ok, MB} when is_integer(MB) -> MB; _ -> - _ = lager:warning( - "access_archiver_max_backlog was unset or" - " invalid; overriding with default of ~b", - [?DEFAULT_MAX_BACKLOG]), + logger:warning("access_archiver_max_backlog was unset or" + " invalid; overriding with default of ~b", + [?DEFAULT_MAX_BACKLOG]), ?DEFAULT_MAX_BACKLOG end, {ok, #state{max_workers=MaxWorkers, @@ -180,9 +180,8 @@ handle_info({'ETS-TRANSFER', Table, _From, Slice}, State) -> false -> %% too much in the backlog, drop the first item in the backlog [{_DropTable, DropSlice}|RestBacklog] = Backlog, - ok = lager:error("Skipping archival of accesses ~p to" - " catch up on backlog", - [DropSlice]), + logger:error("Skipping archival of accesses ~p to catch up on backlog", + [DropSlice]), State#state{backlog=RestBacklog++[{Table, Slice}]} end, {noreply, NewState}; @@ -192,8 +191,7 @@ handle_info(_Info, State) -> terminate(_Reason, #state{backlog=[]}) -> ok; terminate(_Reason, _State) -> - _ = lager:warning("Access archiver manager stopping with a backlog;" - " logs will be dropped"), + logger:warning("Access archiver manager stopping with a backlog; logs will be dropped"), ok. code_change(_OldVsn, State, _Extra) -> diff --git a/src/riak_cs_access_console.erl b/apps/riak_cs/src/riak_cs_access_console.erl similarity index 98% rename from src/riak_cs_access_console.erl rename to apps/riak_cs/src/riak_cs_access_console.erl index a67bf89e3..32fd0bc12 100644 --- a/src/riak_cs_access_console.erl +++ b/apps/riak_cs/src/riak_cs_access_console.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/src/riak_cs_access_log_handler.erl b/apps/riak_cs/src/riak_cs_access_log_handler.erl similarity index 92% rename from src/riak_cs_access_log_handler.erl rename to apps/riak_cs/src/riak_cs_access_log_handler.erl index 258c484d3..676d9f348 100644 --- a/src/riak_cs_access_log_handler.erl +++ b/apps/riak_cs/src/riak_cs_access_log_handler.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -24,12 +25,12 @@ %% I/O stats are expected as notes in the webmachine log data, with %% keys of the form `{access, KEY}'. I/O stats are only logged if a %% note is included with the key `user' and a value that is a -%% `#rcs_user_v2{}' record. +%% `#rcs_user_v3{}' record. %% %% That is, to log I/O stats for a request, call %% %% ``` -%% wrq:add_note({access, user}, User=#rcs_user_v2{}, RD) +%% wrq:add_note({access, user}, User=#rcs_user_v3{}, RD) %% ''' %% %% somewhere in your resource. To add another stat, for instance @@ -73,6 +74,7 @@ code_change/3]). -include_lib("webmachine/include/webmachine_logger.hrl"). +-include_lib("kernel/include/logger.hrl"). -include("riak_cs.hrl"). -ifdef(TEST). @@ -85,8 +87,8 @@ size :: integer(), %% num. accesses since last archival current :: {calendar:datetime(), calendar:datetime()}, %% current agg. slice - archive :: reference(), %% reference for archive msg - table :: ets:tid() %% the table aggregating stats + archive :: undefined | reference(), %% reference for archive msg + table :: ets:tid() %% the table aggregating stats }). -type state() :: #state{}. @@ -112,10 +114,10 @@ flush(Timeout) -> %% @doc Set the Riak CS user for this request. Stats are not recorded if %% the user is not set. -set_user(KeyID, RD) when is_list(KeyID) -> - wrq:add_note(?STAT(user), KeyID, RD); -set_user(?RCS_USER{key_id=KeyID}, RD) -> - wrq:add_note(?STAT(user), KeyID, RD); +set_user(Arn, RD) when is_binary(Arn) -> + wrq:add_note(?STAT(user), Arn, RD); +set_user(?RCS_USER{arn = Arn}, RD) -> + wrq:add_note(?STAT(user), Arn, RD); set_user(undefined, RD) -> RD; set_user(unknown, RD) -> @@ -163,15 +165,17 @@ init(_) -> end, {ok, SchedState}; {{error, Reason}, _} -> - _ = lager:error("Error starting access logger: ~s", [Reason]), + logger:error("Error starting access logger: ~s", [Reason]), %% can't simply {error, Reason} out here, because %% webmachine/mochiweb will just ignore the failed %% startup; using init:stop/0 here so that the user isn't %% suprised later when there are no logs - init:stop(); + init:stop(), + {error, doesnt_matter_as_node_is_going_down}; {_, {error, Reason}} -> - _ = lager:error("Error starting access logger: ~s", [Reason]), - init:stop() + logger:error("Error starting access logger: ~s", [Reason]), + init:stop(), + {error, doesnt_matter_as_node_is_going_down} end. %% @private @@ -185,7 +189,7 @@ handle_call(_Request, State) -> handle_event({log_access, #wm_log_data{notes=undefined, method=Method, path=Path, headers=Headers}}, State) -> - lager:debug("No WM route: ~p ~s ~p\n", [Method, Path, Headers]), + ?LOG_DEBUG("No WM route: ~p ~s ~p", [Method, Path, Headers]), {ok, State}; handle_event({log_access, LogData}, #state{table=T, size=S, max_size=MaxS}=State) -> @@ -229,9 +233,9 @@ handle_info({archive, Ref}, #state{archive=Ref}=State) -> %% simple "missed window" is too lossy [{message_queue_len, MessageCount}] = process_info(self(), [message_queue_len]), - _ = lager:error("Access logger is running ~b seconds behind," - " skipping ~p log messages to catch up", - [Behind, MessageCount]), + logger:error("Access logger is running ~b seconds behind," + " skipping ~p log messages to catch up", + [Behind, MessageCount]), remove_handler end; handle_info(_Info, State) -> @@ -265,7 +269,7 @@ schedule_archival(#state{current={_,E}}=State) -> TL = calendar:datetime_to_gregorian_seconds(E)-Now, case TL < 0 of false -> - _ = lager:debug("Next access archival in ~b seconds", [TL]), + ?LOG_DEBUG("Next access archival in ~b seconds", [TL]), %% time left is in seconds, we need milliseconds erlang:send_after(TL*1000, self(), {archive, Ref}), @@ -285,9 +289,8 @@ force_archive(#state{current=C}=State, FlushEnd) -> %% @doc Send the current slice's accumulated accesses to the archiver %% for storage. Create a clean table to store the next slice's accesses. --spec do_archive(state()) -> state(). do_archive(#state{period=P, table=T, current=C}=State) -> - _ = lager:debug("Rolling access for ~p", [C]), + ?LOG_DEBUG("Rolling access for ~p", [C]), %% archiver takes ownership of the table, and deletes it when done riak_cs_access_archiver_manager:archive(T, C), diff --git a/src/riak_cs_acl.erl b/apps/riak_cs/src/riak_cs_acl.erl similarity index 73% rename from src/riak_cs_acl.erl rename to apps/riak_cs/src/riak_cs_acl.erl index 0b04779b4..7dcc97a68 100644 --- a/src/riak_cs_acl.erl +++ b/apps/riak_cs/src/riak_cs_acl.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -25,12 +26,12 @@ -include("riak_cs.hrl"). -include_lib("riak_pb/include/riak_pb_kv_codec.hrl"). +-include_lib("kernel/include/logger.hrl"). -ifdef(TEST). - -compile(export_all). +-compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). - -endif. %% Public API @@ -45,7 +46,10 @@ bucket_acl_from_contents/2, object_access/5, object_access/6, - owner_id/2 + owner_id/2, + exprec_acl/1, + upgrade_acl_record/1, + upgrade_owner/1 ]). -define(ACL_UNDEF, {error, acl_undefined}). @@ -55,12 +59,12 @@ %% =================================================================== %% @doc Determine if anonymous access is set for the bucket. --spec anonymous_bucket_access(binary(), atom(), riak_client()) -> {true, string()} | false. +-spec anonymous_bucket_access(binary(), atom(), riak_client()) -> {true, binary()} | false. anonymous_bucket_access(Bucket, RequestedAccess, RcPid) -> anonymous_bucket_access(Bucket, RequestedAccess, RcPid, undefined). -spec anonymous_bucket_access(binary(), atom(), riak_client(), acl()|undefined) - -> {true, string()} | false. + -> {true, binary()} | false. anonymous_bucket_access(_Bucket, undefined, _, _) -> false; anonymous_bucket_access(Bucket, RequestedAccess, RcPid, undefined) -> @@ -70,7 +74,7 @@ anonymous_bucket_access(Bucket, RequestedAccess, RcPid, undefined) -> {error, Reason} -> %% @TODO Think about bubbling this error up and providing %% feedback to requester. - _ = lager:error("Anonymous bucket access check failed due to error. Reason: ~p", [Reason]), + logger:error("Anonymous bucket access check failed due to error. Reason: ~p", [Reason]), false end; anonymous_bucket_access(_Bucket, RequestedAccess, RcPid, BucketAcl) -> @@ -90,14 +94,14 @@ anonymous_bucket_access(_Bucket, RequestedAccess, RcPid, BucketAcl) -> %% @doc Determine if anonymous access is set for the object. %% @TODO Enhance when doing object ACLs -spec anonymous_object_access(riakc_obj:riakc_obj(), acl(), atom(), riak_client()) -> - {true, string()} | - false. + {true, binary()} | + false. anonymous_object_access(BucketObj, ObjAcl, RequestedAccess, RcPid) -> anonymous_object_access(BucketObj, ObjAcl, RequestedAccess, RcPid, undefined). -spec anonymous_object_access(riakc_obj:riakc_obj(), acl(), atom(), riak_client(), acl()|undefined) -> - {true, string()} | false. + {true, binary()} | false. anonymous_object_access(_BucketObj, _ObjAcl, undefined, _, _) -> false; anonymous_object_access(BucketObj, ObjAcl, 'WRITE', RcPid, undefined) -> @@ -109,7 +113,7 @@ anonymous_object_access(BucketObj, ObjAcl, 'WRITE', RcPid, undefined) -> {error, Reason} -> %% @TODO Think about bubbling this error up and providing %% feedback to requester. - _ = lager:error("Anonymous object access check failed due to error. Reason: ~p", [Reason]), + logger:error("Anonymous object access check failed due to error. Reason: ~p", [Reason]), false end; anonymous_object_access(_BucketObj, _ObjAcl, 'WRITE', RcPid, BucketAcl) -> @@ -130,13 +134,13 @@ anonymous_object_access(_BucketObj, ObjAcl, RequestedAccess, RcPid, _) -> end. %% @doc Determine if a user has the requested access to a bucket. --spec bucket_access(binary(), atom(), string(), riak_client()) -> - boolean() | {true, string()}. +-spec bucket_access(binary(), atom(), binary(), riak_client()) -> + boolean() | {true, binary()}. bucket_access(Bucket, RequestedAccess, CanonicalId, RcPid) -> bucket_access(Bucket, RequestedAccess, CanonicalId, RcPid, undefined). --spec bucket_access(binary(), atom(), string(), riak_client(), acl()|undefined ) -> - boolean() | {true, string()}. +-spec bucket_access(binary(), atom(), binary(), riak_client(), acl() | undefined ) -> + boolean() | {true, binary()}. bucket_access(_Bucket, undefined, _CanonicalId, _, _) -> false; bucket_access(Bucket, RequestedAccess, CanonicalId, RcPid, undefined) -> @@ -151,7 +155,7 @@ bucket_access(Bucket, RequestedAccess, CanonicalId, RcPid, undefined) -> {error, Reason} -> %% @TODO Think about bubbling this error up and providing %% feedback to requester. - _ = lager:error("Bucket access check failed due to error. Reason: ~p", [Reason]), + logger:error("Bucket access check failed due to error. Reason: ~p", [Reason]), false end; bucket_access(_, RequestedAccess, CanonicalId, RcPid, Acl) -> @@ -181,9 +185,8 @@ fetch_bucket_acl(Bucket, RcPid) -> {ok, Obj} -> bucket_acl(Obj); {error, Reason} -> - _ = lager:debug("Failed to fetch ACL. Bucket ~p " - " does not exist. Reason: ~p", - [Bucket, Reason]), + logger:warning("Failed to fetch ACL. Bucket ~p does not exist. Reason: ~p", + [Bucket, Reason]), {error, notfound} end. @@ -201,8 +204,7 @@ bucket_acl(BucketObj) -> %% We attempt resolution, but intentionally do not write back a resolved %% value. Instead the fact that the bucket has siblings is logged, but the %% condition should be rare so we avoid updating the value at this time. --spec bucket_acl_from_contents(binary(), riakc_obj:contents()) -> - bucket_acl_result(). +-spec bucket_acl_from_contents(binary(), riakc_obj:contents()) -> {ok, acl()} | ?ACL_UNDEF. bucket_acl_from_contents(_, [{MD, _}]) -> MetaVals = dict:fetch(?MD_USERMETA, MD), acl_from_meta(MetaVals); @@ -213,22 +215,17 @@ bucket_acl_from_contents(Bucket, Contents) -> riak_cs_bucket:maybe_log_bucket_owner_error(Bucket, UniqueVals), resolve_bucket_metadata(UserMetas, UniqueVals). --spec resolve_bucket_metadata(list(riakc_obj:metadata()), - list(riakc_obj:value())) -> bucket_acl_result(). resolve_bucket_metadata(Metas, [_Val]) -> Acls = [acl_from_meta(M) || M <- Metas], resolve_bucket_acls(Acls); resolve_bucket_metadata(_Metas, _) -> {error, multiple_bucket_owners}. --spec resolve_bucket_acls(list(acl_from_meta_result())) -> acl_from_meta_result(). resolve_bucket_acls([Acl]) -> Acl; resolve_bucket_acls(Acls) -> lists:foldl(fun newer_acl/2, ?ACL_UNDEF, Acls). --spec newer_acl(acl_from_meta_result(), acl_from_meta_result()) -> - acl_from_meta_result(). newer_acl(Acl1, ?ACL_UNDEF) -> Acl1; newer_acl({ok, Acl1}, {ok, Acl2}) @@ -241,14 +238,14 @@ newer_acl(_, Acl2) -> %% @TODO Enhance when doing object-level ACL work. This is a bit %% patchy until object ACLs are done. The bucket owner gets full %% control, but bucket-level ACLs only matter for writes otherwise. --spec object_access(riakc_obj:riakc_obj(), acl(), atom(), string(), riak_client()) - -> boolean() | {true, string()}. +-spec object_access(riakc_obj:riakc_obj(), acl(), atom(), binary(), riak_client()) -> + boolean() | {true, binary()}. object_access(BucketObj, ObjAcl, RequestedAccess, CanonicalId, RcPid) -> object_access(BucketObj, ObjAcl, RequestedAccess, CanonicalId, RcPid, undefined). --spec object_access(riakc_obj:riakc_obj(), acl(), atom(), string(), riak_client(), undefined|acl()) - -> boolean() | {true, string()}. +-spec object_access(riakc_obj:riakc_obj(), acl(), atom(), binary(), riak_client(), undefined|acl()) -> + boolean() | {true, binary()}. object_access(_BucketObj, _ObjAcl, undefined, _CanonicalId, _, _) -> false; object_access(_BucketObj, _ObjAcl, _RequestedAccess, undefined, _RcPid, _) -> @@ -261,7 +258,7 @@ object_access(BucketObj, ObjAcl, 'WRITE', CanonicalId, RcPid, undefined) -> {error, Reason} -> %% @TODO Think about bubbling this error up and providing %% feedback to requester. - _ = lager:error("Object access check failed due to error. Reason: ~p", [Reason]), + logger:error("Object access check failed due to error. Reason: ~p", [Reason]), false end; object_access(_BucketObj, _ObjAcl, 'WRITE', CanonicalId, RcPid, BucketAcl) -> @@ -279,13 +276,10 @@ object_access(_BucketObj, _ObjAcl, 'WRITE', CanonicalId, RcPid, BucketAcl) -> false end; object_access(_BucketObj, ObjAcl, RequestedAccess, CanonicalId, RcPid, _) -> - _ = lager:debug("ObjAcl: ~p~nCanonicalId: ~p", [ObjAcl, CanonicalId]), IsObjOwner = is_owner(ObjAcl, CanonicalId), HasObjPerm = has_permission(acl_grants(ObjAcl), RequestedAccess, CanonicalId), - _ = lager:debug("IsObjOwner: ~p", [IsObjOwner]), - _ = lager:debug("HasObjPerm: ~p", [HasObjPerm]), if (RequestedAccess == 'READ_ACP' orelse RequestedAccess == 'WRITE_ACP') andalso @@ -303,21 +297,26 @@ object_access(_BucketObj, ObjAcl, RequestedAccess, CanonicalId, RcPid, _) -> end. %% @doc Get the canonical id of the owner of an entity. --spec owner_id(acl(), riak_client()) -> string(). -owner_id(?ACL{owner=Owner}, _) -> - {_, _, OwnerId} = Owner, - OwnerId; -owner_id(#acl_v1{owner=OwnerData}, RcPid) -> - {Name, CanonicalId} = OwnerData, - case riak_cs_user:get_user_by_index(?ID_INDEX, - list_to_binary(CanonicalId), - RcPid) of - {ok, {Owner, _}} -> - Owner?RCS_USER.key_id; - {error, _} -> - _ = lager:warning("Failed to retrieve key_id for user ~p with canonical_id ~p", [Name, CanonicalId]), - [] - end. +-spec owner_id(acl(), riak_client()) -> binary(). +owner_id(?ACL{owner = #{key_id := OwnerKeyId}}, _) -> + OwnerKeyId. + +-spec exprec_acl(maps:map()) -> ?ACL{}. +exprec_acl(Map) -> + Acl0 = ?ACL{grants = GG0} = exprec:frommap_acl_v3(Map), + GG = [exprec_grant(G) || G <- GG0], + Acl0?ACL{grants = GG}. +exprec_grant(Map) -> + G0 = ?ACL_GRANT{perms = Perms0, + grantee = Grantee0} = exprec:frommap_acl_grant_v2(Map), + G0?ACL_GRANT{perms = [binary_to_existing_atom(P, latin1) || P <- Perms0], + grantee = case Grantee0 of + #{} -> + Grantee0; + GroupGrantee when is_binary(GroupGrantee) -> + binary_to_existing_atom(GroupGrantee, latin1) + end + }. %% =================================================================== %% Internal functions @@ -326,27 +325,22 @@ owner_id(#acl_v1{owner=OwnerData}, RcPid) -> %% @doc Find the ACL in a list of metadata values and %% convert it to an erlang term representation. Return %% `undefined' if an ACL is not found. --spec acl_from_meta([{string(), term()}]) -> acl_from_meta_result(). acl_from_meta([]) -> ?ACL_UNDEF; acl_from_meta([{?MD_ACL, Acl} | _]) -> - {ok, binary_to_term(Acl)}; + {ok, upgrade_acl_record(binary_to_term(Acl))}; acl_from_meta([_ | RestMD]) -> acl_from_meta(RestMD). %% @doc Get the grants from an ACL --spec acl_grants(acl()) -> [acl_grant()]. -acl_grants(?ACL{grants=Grants}) -> - Grants; -acl_grants(#acl_v1{grants=Grants}) -> +acl_grants(?ACL{grants = Grants}) -> Grants. %% @doc Iterate through a list of ACL grants and return %% any group grants. --spec group_grants([acl_grant()], [acl_grant()]) -> [acl_grant()]. group_grants([], GroupGrants) -> GroupGrants; -group_grants([HeadGrant={Grantee, _} | RestGrants], +group_grants([HeadGrant = ?ACL_GRANT{grantee = Grantee} | RestGrants], GroupGrants) when is_atom(Grantee) -> group_grants(RestGrants, [HeadGrant | GroupGrants]); group_grants([_ | RestGrants], _GroupGrants) -> @@ -354,10 +348,9 @@ group_grants([_ | RestGrants], _GroupGrants) -> %% @doc Determine if the ACL grants group access %% for the requestsed permission type. --spec has_group_permission([{group_grant() | {term(), term()}, term()}], atom()) -> boolean(). has_group_permission([], _RequestedAccess) -> false; -has_group_permission([{_, Perms} | RestGrants], RequestedAccess) -> +has_group_permission([?ACL_GRANT{perms = Perms} | RestGrants], RequestedAccess) -> case check_permission(RequestedAccess, Perms) of true -> true; @@ -367,10 +360,10 @@ has_group_permission([{_, Perms} | RestGrants], RequestedAccess) -> %% @doc Determine if the ACL grants anonymous access %% for the requestsed permission type. --spec has_permission([acl_grant()], atom()) -> boolean(). has_permission(Grants, RequestedAccess) -> GroupGrants = group_grants(Grants, []), - case [Perms || {Grantee, Perms} <- GroupGrants, + case [Perms || ?ACL_GRANT{grantee = Grantee, + perms = Perms} <- GroupGrants, Grantee =:= 'AllUsers'] of [] -> false; @@ -380,31 +373,25 @@ has_permission(Grants, RequestedAccess) -> %% @doc Determine if a user has the requested permission %% granted in an ACL. --spec has_permission([acl_grant()], atom(), string()) -> boolean(). +-spec has_permission([acl_grant()], atom(), binary()) -> boolean(). has_permission(Grants, RequestedAccess, CanonicalId) -> GroupGrants = group_grants(Grants, []), case user_grant(Grants, CanonicalId) of undefined -> has_group_permission(GroupGrants, RequestedAccess); - {_, Perms} -> + ?ACL_GRANT{perms = Perms} -> check_permission(RequestedAccess, Perms) orelse has_group_permission(GroupGrants, RequestedAccess) end. %% @doc Determine if a user is the owner of a system entity. --spec is_owner(acl(), string()) -> boolean(). -is_owner(?ACL{owner={_, CanonicalId, _}}, CanonicalId) -> +is_owner(?ACL{owner = #{canonical_id := CanonicalId}}, CanonicalId) -> true; is_owner(?ACL{}, _) -> - false; -is_owner(#acl_v1{owner={_, CanonicalId}}, CanonicalId) -> - true; -is_owner(#acl_v1{}, _) -> false. %% @doc Check if a list of ACL permissions contains a specific permission %% or the `FULL_CONTROL' permission. --spec check_permission(acl_perm(), acl_perms()) -> boolean(). check_permission(_, []) -> false; check_permission(Permission, [Permission | _]) -> @@ -417,12 +404,55 @@ check_permission(_Permission, [_ | RestPerms]) -> %% @doc Iterate through a list of ACL grants and determine %% if there is an entry for the specified user's id. Ignore %% any group grants. --spec user_grant([acl_grant()], string()) -> undefined | acl_grant(). user_grant([], _) -> undefined; -user_grant([{Grantee, _} | RestGrants], _CanonicalId) when is_atom(Grantee) -> - user_grant(RestGrants, _CanonicalId); -user_grant([HeadGrant={{_, CanonicalId}, _} | _], CanonicalId) -> +user_grant([?ACL_GRANT{grantee = Grantee} | RestGrants], CanonicalId) when is_atom(Grantee) -> + user_grant(RestGrants, CanonicalId); +user_grant([HeadGrant = ?ACL_GRANT{grantee = #{canonical_id := CanonicalId}} | _], CanonicalId) -> HeadGrant; user_grant([_ | RestGrants], _CanonicalId) -> user_grant(RestGrants, _CanonicalId). + + +-spec upgrade_acl_record(undefined | #acl_v1{} | #acl_v2{} | #acl_v3{}) -> undefined | acl(). +upgrade_acl_record(undefined) -> + undefined; +upgrade_acl_record(#acl_v3{} = A) -> + A; +upgrade_acl_record(#acl_v2{owner = Owner, + grants = Grants, + creation_time = {T1, T2, T3}}) -> + #acl_v3{owner = upgrade_owner(Owner), + grants = [upgrade_grant(G) || G <- Grants], + creation_time = T1 * 1000000 + T2 + T3 div 1000}; +upgrade_acl_record(#acl_v1{owner = Owner, + grants = Grants, + creation_time = {T1, T2, T3}}) -> + #acl_v3{owner = upgrade_owner(Owner), + grants = [upgrade_grant(G) || G <- Grants], + creation_time = T1 * 1000000 + T2 + T3 div 1000}. + +upgrade_owner({DisplayName, CanonicalId, KeyId}) -> + #{display_name => list_to_binary(DisplayName), + canonical_id => list_to_binary(CanonicalId), + key_id => list_to_binary(KeyId), + email => undefined}; +upgrade_owner({DisplayName, CanonicalId}) -> + #{display_name => list_to_binary(DisplayName), + canonical_id => list_to_binary(CanonicalId), + key_id => <<"FaKeKeyIdWontWorkWillFailReplaceIt">>, + email => undefined}. + +upgrade_grant({Grantee, Perms}) -> + #acl_grant_v2{grantee = upgrade_grantee(Grantee), + perms = Perms}. + +upgrade_grantee(GroupGrant) when is_atom(GroupGrant) -> + GroupGrant; +upgrade_grantee({A, B}) -> + #{display_name => list_to_binary(A), + canonical_id => list_to_binary(B)}; +upgrade_grantee({A, B, C}) -> + #{display_name => list_to_binary(A), + canonical_id => list_to_binary(B), + key_id => list_to_binary(C)}. diff --git a/apps/riak_cs/src/riak_cs_acl_utils.erl b/apps/riak_cs/src/riak_cs_acl_utils.erl new file mode 100644 index 000000000..b4cbd2afe --- /dev/null +++ b/apps/riak_cs/src/riak_cs_acl_utils.erl @@ -0,0 +1,909 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc ACL utility functions + +-module(riak_cs_acl_utils). + +-include("riak_cs.hrl"). +-include_lib("xmerl/include/xmerl.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +%% Public API +-export([default_acl/1, default_acl/3, + canned_acl/3, + specific_acl_grant/3, + acl_from_xml/3, + empty_acl_xml/0, + requested_access/2, + check_grants/4, + check_grants/5, + validate_acl/2 + ]). + +%% =================================================================== +%% Public API +%% =================================================================== + +%% @doc Construct a default acl. The structure is the same for buckets +%% and objects. +-spec default_acl(binary(), binary(), binary()) -> acl(). +default_acl(DisplayName, CanonicalId, KeyId) -> + default_acl(#{display_name => DisplayName, + canonical_id => CanonicalId, + key_id => KeyId}). +-spec default_acl(maps:map()) -> acl(). +default_acl(Owner) -> + ?ACL{owner = Owner, + grants = [?ACL_GRANT{grantee = Owner, + perms = ['FULL_CONTROL']}] + }. + +%% @doc Map a x-amz-acl header value to an +%% internal acl representation. +-spec canned_acl(undefined | string(), + acl_owner(), + undefined | acl_owner()) -> acl(). +canned_acl(undefined, Owner, _) -> + default_acl(Owner); +canned_acl(HeaderVal, Owner, BucketOwner) -> + ?ACL{owner = Owner, + grants = canned_acl_grants(HeaderVal, + Owner, + BucketOwner)}. + +%% @doc Turn a list of header-name, value pairs into an ACL. If the header +%% values don't parse correctly, return `{error, invalid_argument}'. If +%% the header references an email address that cannot be mapped to a +%% canonical id, return `{error, unresolved_grant_email}'. Otherwise +%% return an acl record. +-spec specific_acl_grant(Owner :: acl_owner(), + [{HeaderName :: acl_perm(), + HeaderValue :: string()}], + riak_client()) -> + {ok, acl()} | + {error, invalid_argument | unresolved_grant_email | unresolved_grant_canonical_id}. +specific_acl_grant(Owner, Headers, RcPid) -> + %% TODO: this function is getting a bit long and confusing + Grants = [{HeaderName, parse_grant_header_value(GrantString)} + || {HeaderName, GrantString} <- Headers], + case promote_failure([G || {_HeaderName, G} <- Grants]) of + {error, invalid_argument} = E -> + E; + {ok, _GoodGrants} -> + EmailsTranslated = [{HeaderName, emails_to_ids(G, RcPid)} + || {HeaderName, {ok, G}} <- Grants], + case promote_failure([EmailOk || {_H, EmailOk} <- EmailsTranslated]) of + {error, unresolved_grant_email} = E -> + E; + {ok, _GoodEmails} -> + case valid_headers_to_grants( + [{HeaderName, Val} || {HeaderName, {ok, Val}} <- EmailsTranslated], + RcPid) of + {ok, AclGrants} -> + {ok, ?ACL{owner = Owner, + grants = AclGrants} + }; + {error, _} = E -> + E + end + end + end. + +%% @doc Attempt to parse a list of ACL headers into a list +%% of `acl_grant()'s. +valid_headers_to_grants(Pairs, RcPid) -> + MaybeGrants = [header_to_acl_grants(HeaderName, Grants, RcPid) || + {HeaderName, Grants} <- Pairs], + case promote_failure(MaybeGrants) of + {ok, Grants} -> + {ok, lists:foldl(fun add_grant/2, [], lists:flatten(Grants))}; + {error, _} = E -> + E + end. + +%% @doc Attempt to turn a `acl_perm()' and list of grants +%% into a list of `acl_grant()'s. At this point, email +%% addresses have already been resolved, and headers parsed. +header_to_acl_grants(HeaderName, Grants, RcPid) -> + MaybeGrantList = lists:map( + fun (Id) -> header_to_grant(HeaderName, Id, RcPid) end, + Grants), + case promote_failure(MaybeGrantList) of + {ok, GrantList} -> + {ok, lists:foldl(fun add_grant/2, [], GrantList)}; + {error, _} = E -> + E + end. + +%% Attempt to turn an `acl_perm()' and `grant_user_identifier()' +%% into an `acl_grant()'. If the `grant_user_identifier()' uses an +%% id, and the name can't be found, returns `{error, invalid_argument}'. +header_to_grant(Permission, {id, ID}, RcPid) -> + case name_for_canonical(ID, RcPid) of + {ok, DisplayName} -> + {ok, {{DisplayName, ID}, [Permission]}}; + {error, _} = E -> + E + end; +header_to_grant(Permission, {uri, URI}, _RcPid) -> + case URI of + ?ALL_USERS_GROUP -> + {ok, ?ACL_GRANT{grantee = 'AllUsers', + perms = [Permission]}}; + ?AUTH_USERS_GROUP -> + {ok, ?ACL_GRANT{grantee = 'AuthUsers', + perms = [Permission]}} + end. + +%% @doc Attempt to parse a header into +%% a list of grant identifiers and strings. +-type grant_user_identifier() :: 'emailAddress' | 'id' | 'uri'. +-spec parse_grant_header_value(string()) -> + {ok, [{grant_user_identifier(), string()}]} | + {error, invalid_argument | unresolved_grant_email | unresolved_grant_canonical_id}. +parse_grant_header_value(HeaderValue) -> + Mappings = split_header_values_and_strip(HeaderValue), + promote_failure(lists:map(fun parse_mapping/1, Mappings)). + +%% @doc split a string like: +%% `"emailAddress=\"xyz@amazon.com\", emailAddress=\"abc@amazon.com\""' +%% into: +%% `["emailAddress=\"xyz@amazon.com\"", +%% "emailAddress=\"abc@amazon.com\""]' +-spec split_header_values_and_strip(string()) -> [string()]. +split_header_values_and_strip(Value) -> + [string:strip(V) || V <- string:tokens(Value, ",")]. + +%% @doc Attempt to parse a single grant, like: +%% `"emailAddress=\"name@example.com\""' +%% If the value can't be parsed, return +%% `{error, invalid_argument}'. +-spec parse_mapping(string()) -> + {ok, {grant_user_identifier(), Value :: string()}} | + {error, invalid_argument}. +parse_mapping("emailAddress=" ++ QuotedEmail) -> + wrap('emailAddress', remove_quotes(QuotedEmail)); +parse_mapping("id=" ++ QuotedID) -> + wrap('id', remove_quotes(QuotedID)); +parse_mapping("uri=" ++ QuotedURI) -> + case remove_quotes(QuotedURI) of + {ok, NoQuote}=OK -> + case valid_uri(NoQuote) of + true -> + wrap('uri', OK); + false -> + {error, 'invalid_argument'} + end; + {error, invalid_argument}=E -> + E + end; +parse_mapping(_Else) -> + {error, invalid_argument}. + +%% @doc Return true if `URI' is a valid group grant URI. +valid_uri(URI) -> + %% log delivery is not yet a supported option + lists:member(URI, [?ALL_USERS_GROUP, ?AUTH_USERS_GROUP]). + +%% @doc Combine the first and second argument, if the second +%% is wrapped in `ok'. Otherwise return the second argugment. +wrap(_Atom, {error, invalid_argument}=E) -> + E; +wrap(Atom, {ok, Value}) -> + {ok, {Atom, Value}}. + +%% If `String' is enclosed in quotation marks, remove them. Otherwise +%% return an error. +remove_quotes(String) -> + case starts_and_ends_with_quotes(String) of + false -> + {error, invalid_argument}; + true -> + {ok, string:sub_string(String, 2, length(String) - 1)} + end. + +%% @doc Return true if `String' is enclosed in quotation +%% marks. The enclosed string must also be non-empty. +starts_and_ends_with_quotes(String) -> + length(String) > 2 andalso + hd(String) =:= 34 andalso + lists:last(String) =:= 34. + +%% @doc Attempt to turn a list of grants that use email addresses +%% into a list of grants that only use canonical ids. Returns an error +%% if any of the emails cannot be turned into canonical ids. +emails_to_ids(Grants, RcPid) -> + {EmailGrants, RestGrants} = lists:partition(fun email_grant/1, Grants), + Ids = [canonical_for_email(EmailAddress, RcPid) || + {emailAddress, EmailAddress} <- EmailGrants], + case promote_failure(Ids) of + {error, unresolved_grant_email}=E -> + E; + {ok, AllIds} -> + {ok, RestGrants ++ [{id, ID} || ID <- AllIds]} + end. + +email_grant({Atom, _Val}) -> + Atom =:= 'emailAddress'. + +%% @doc Turn a list of ok-values or errors into either +%% an ok of list, or an error. Returns the latter is any +%% of the values in the input list are an error. +promote_failure(List) -> + %% this will reverse the list, but we don't care + %% about order + case lists:foldl(fun fail_either/2, {ok, []}, List) of + {{error, _Reason}=E, _Acc} -> + E; + {ok, _Acc}=Ok -> + Ok + end. + +%% @doc Return an error if either argument is an error. Otherwise, +%% cons the value from the first argument onto the accumulator +%% in the second. +fail_either(_Elem, {{error, _Reason}=E, Acc}) -> + {E, Acc}; +fail_either(E={error, _Reason}, {_OkOrError, Acc}) -> + %% don't cons the error onto the acc + {E, Acc}; +fail_either({ok, Val}, {_OkOrError, Acc}) -> + {ok, [Val | Acc]}. + +%% @doc Convert an XML document representing an ACL into +%% an internal representation. +-spec acl_from_xml(string(), binary(), riak_client()) -> + {ok, acl()} | {error, invalid_argument | unresolved_grant_email | malformed_acl_error}. +acl_from_xml(Xml, KeyId, RcPid) -> + case riak_cs_xml:scan(Xml) of + {error, malformed_xml} -> {error, malformed_acl_error}; + {ok, ParsedData} -> + BareAcl = ?ACL{owner = #{display_name => undefined, + key_id => KeyId}}, + process_acl_contents(ParsedData#xmlElement.content, BareAcl, RcPid) + end. + +%% @doc Convert an internal representation of an ACL +%% into XML. +-spec empty_acl_xml() -> binary(). +empty_acl_xml() -> + XmlDoc = [{'AccessControlPolicy',[]}], + unicode:characters_to_binary( + xmerl:export_simple(XmlDoc, xmerl_xml, [{prolog, ?XML_PROLOG}])). + +%% @doc Map a request type to the type of ACL permissions needed +%% to complete the request. +-type request_method() :: 'GET' | 'HEAD' | 'PUT' | 'POST' | + 'DELETE' | 'Dialyzer happiness'. +-spec requested_access(request_method(), boolean()) -> acl_perm(). +requested_access(Method, AclRequest) -> + if + Method == 'GET' + andalso + AclRequest == true-> + 'READ_ACP'; + (Method == 'GET' + orelse + Method == 'HEAD') -> + 'READ'; + Method == 'PUT' + andalso + AclRequest == true-> + 'WRITE_ACP'; + (Method == 'POST' + orelse + Method == 'DELETE') + andalso + AclRequest == true-> + undefined; + Method == 'PUT' + orelse + Method == 'POST' + orelse + Method == 'DELETE' -> + 'WRITE'; + Method == 'Dialyzer happiness' -> + 'FULL_CONTROL'; + true -> + undefined + end. + +-spec check_grants(undefined | rcs_user(), binary(), atom(), riak_client()) -> + boolean() | {true, string()}. +check_grants(User, Bucket, RequestedAccess, RcPid) -> + check_grants(User, Bucket, RequestedAccess, RcPid, undefined). + +-spec check_grants(undefined | rcs_user(), binary(), atom(), riak_client(), acl() | undefined) -> + boolean() | {true, binary()}. +check_grants(undefined, Bucket, RequestedAccess, RcPid, BucketAcl) -> + riak_cs_acl:anonymous_bucket_access(Bucket, RequestedAccess, RcPid, BucketAcl); +check_grants(User, Bucket, RequestedAccess, RcPid, BucketAcl) -> + riak_cs_acl:bucket_access(Bucket, + RequestedAccess, + User?RCS_USER.id, + RcPid, + BucketAcl). + +-spec validate_acl({ok, acl()} | {error, term()}, binary()) -> + {ok, acl()} | {error, access_denied}. +validate_acl({ok, Acl = ?ACL{owner = #{canonical_id := Id}}}, Id) -> + {ok, Acl}; +validate_acl({ok, _}, _) -> + {error, access_denied}; +validate_acl({error, _} = Error, _) -> + Error. + + +%% =================================================================== +%% Internal functions +%% =================================================================== + +%% @doc Update the permissions for a grant in the provided +%% list of grants if an entry exists with matching grantee +%% data or add a grant to a list of grants. +add_grant(NewGrant, Grants) -> + ?ACL_GRANT{grantee = NewGrantee, + perms = NewPerms} = NewGrant, + SplitFun = fun(?ACL_GRANT{grantee = Grantee}) -> + Grantee =:= NewGrantee + end, + {GranteeGrants, OtherGrants} = lists:partition(SplitFun, Grants), + case GranteeGrants of + [] -> + [NewGrant | Grants]; + _ -> + %% `GranteeGrants' will nearly always be a single + %% item list, but use a fold just in case. + %% The combined list of perms should be small so + %% using usort should not be too expensive. + FoldFun = fun(?ACL_GRANT{perms = Perms}, Acc) -> + lists:usort(Perms ++ Acc) + end, + UpdPerms = lists:foldl(FoldFun, NewPerms, GranteeGrants), + [?ACL_GRANT{grantee = NewGrantee, + perms = UpdPerms} | OtherGrants] + end. + +%% @doc Get the list of grants for a canned ACL +-spec canned_acl_grants(string(), + acl_owner(), + undefined | acl_owner()) -> [acl_grant()]. +canned_acl_grants("public-read", Owner, _) -> + [?ACL_GRANT{grantee = Owner, perms = ['FULL_CONTROL']}, + ?ACL_GRANT{grantee = 'AllUsers', perms = ['READ']}]; +canned_acl_grants("public-read-write", Owner, _) -> + [?ACL_GRANT{grantee = Owner, perms = ['FULL_CONTROL']}, + ?ACL_GRANT{grantee = 'AllUsers', perms = ['READ', 'WRITE']}]; +canned_acl_grants("authenticated-read", Owner, _) -> + [?ACL_GRANT{grantee = Owner, perms = ['FULL_CONTROL']}, + ?ACL_GRANT{grantee = 'AuthUsers', perms = ['READ']}]; +canned_acl_grants("bucket-owner-read", Owner, undefined) -> + canned_acl_grants("private", Owner, undefined); +canned_acl_grants("bucket-owner-read", Owner, Owner) -> + [?ACL_GRANT{grantee = Owner, perms = ['FULL_CONTROL']}]; +canned_acl_grants("bucket-owner-read", Owner, BucketOwner) -> + [?ACL_GRANT{grantee = Owner, perms = ['FULL_CONTROL']}, + ?ACL_GRANT{grantee = BucketOwner, perms = ['READ']}]; +canned_acl_grants("bucket-owner-full-control", Owner, undefined) -> + canned_acl_grants("private", Owner, undefined); +canned_acl_grants("bucket-owner-full-control", Owner, Owner) -> + [?ACL_GRANT{grantee = Owner, perms = ['FULL_CONTROL']}]; +canned_acl_grants("bucket-owner-full-control", Owner, BucketOwner) -> + [?ACL_GRANT{grantee = Owner, perms = ['FULL_CONTROL']}, + ?ACL_GRANT{grantee = BucketOwner, perms = ['FULL_CONTROL']}]; +canned_acl_grants(_, Owner, _) -> + [?ACL_GRANT{grantee = Owner, perms = ['FULL_CONTROL']}]. + + +%% @doc Get the canonical id of the user associated with +%% a given email address. +canonical_for_email(Email, RcPid) -> + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:find_user(#{email => Email}, Pbc) of + {ok, {User, _}} -> + {ok, User?RCS_USER.id}; + {error, Reason} -> + logger:notice("Failed to find user with email ~s: ~p", [Email, Reason]), + {error, unresolved_grant_email} + end. + +%% @doc Get the display name of the user associated with +%% a given canonical id. +name_for_canonical(Id, RcPid) -> + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:find_user(#{canonical_id => Id}, Pbc) of + {ok, {User, _}} -> + {ok, User?RCS_USER.display_name}; + {error, Reason} -> + logger:notice("Failed to find user with canonical_id ~s: ~p", [Id, Reason]), + {error, unresolved_grant_canonical_id} + end. + +user_details_for_canonical(Id, RcPid) -> + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:find_user(#{canonical_id => Id}, Pbc) of + {ok, {?RCS_USER{email = Email, display_name = DisplayName}, _}} -> + {ok, {Email, DisplayName}}; + {error, Reason} -> + logger:notice("Failed to find user with canonical_id ~s: ~p", [Id, Reason]), + {error, unresolved_grant_id} + end. + +%% @doc Process the top-level elements of the +process_acl_contents([], Acl, _) -> + {ok, Acl}; +process_acl_contents([#xmlElement{content=Content, + name=ElementName} + | RestElements], Acl, RcPid) -> + UpdAclRes = + case ElementName of + 'Owner' -> + process_owner(Content, Acl, RcPid); + 'AccessControlList' -> + process_grants(Content, Acl, RcPid); + _ -> + logger:notice("Unexpected element encountered while processing ACL content: ~p", [ElementName]), + Acl + end, + case UpdAclRes of + {ok, UpdAcl} -> + process_acl_contents(RestElements, UpdAcl, RcPid); + {error, _}=Error -> + Error + end; +process_acl_contents([#xmlComment{} | RestElements], Acl, RcPid) -> + process_acl_contents(RestElements, Acl, RcPid); +process_acl_contents([#xmlText{} | RestElements], Acl, RcPid) -> + %% skip normalized space + process_acl_contents(RestElements, Acl, RcPid). + +%% @doc Process an XML element containing acl owner information. +process_owner([], Acl = ?ACL{owner = #{canonical_id := CanonicalId} = Owner}, RcPid) -> + case maps:get(display_name, Owner, undefined) of + undefined -> + case name_for_canonical(CanonicalId, RcPid) of + {ok, DisplayName} -> + {ok, Acl?ACL{owner = Owner#{display_name => DisplayName}}}; + {error, _} = Error -> + Error + end; + _ -> + {ok, Acl} + end; +process_owner([], Acl, _) -> + {ok, Acl}; +process_owner([#xmlElement{content = [Content], + name = ElementName} | + RestElements], Acl, RcPid) -> + Owner = Acl?ACL.owner, + case Content of + #xmlText{value = Value} -> + UpdOwner = + case ElementName of + 'ID' -> + Owner#{canonical_id => list_to_binary(Value)}; + 'DisplayName' -> + Owner#{display_name => list_to_binary(Value)}; + _ -> + logger:warning("Encountered unexpected element: ~p", [ElementName]), + Owner + end, + process_owner(RestElements, Acl?ACL{owner = UpdOwner}, RcPid); + _ -> + process_owner(RestElements, Acl, RcPid) + end; +process_owner([_ | RestElements], Acl, RcPid) -> + %% this pattern matches with text, comment, etc.. + process_owner(RestElements, Acl, RcPid). + +%% @doc Process an XML element containing the grants for the acl. +process_grants([], Acl, _) -> + {ok, Acl}; +process_grants([#xmlElement{content = Content, + name = ElementName} | + RestElements], Acl, RcPid) -> + UpdAcl = + case ElementName of + 'Grant' -> + Grant = process_grant( + Content, + ?ACL_GRANT{grantee = #{display_name => undefined}, + perms = []}, + Acl?ACL.owner, RcPid), + case Grant of + {error, _} -> + Grant; + _ -> + Acl?ACL{grants = add_grant(Grant, Acl?ACL.grants)} + end; + _ -> + logger:warning("Encountered unexpected grants element: ~p", [ElementName]), + Acl + end, + case UpdAcl of + {error, _} -> UpdAcl; + _ -> process_grants(RestElements, UpdAcl, RcPid) + end; +process_grants([ #xmlComment{} | RestElements], Acl, RcPid) -> + process_grants(RestElements, Acl, RcPid); +process_grants([ #xmlText{} | RestElements], Acl, RcPid) -> + process_grants(RestElements, Acl, RcPid). + +%% @doc Process an XML element containing the grants for the acl. +process_grant([], Grant, _, _) -> + Grant; +process_grant([#xmlElement{content = Content, + name = ElementName} | + RestElements], Grant, AclOwner, RcPid) -> + UpdGrant = + case ElementName of + 'Grantee' -> + process_grantee(Content, Grant, AclOwner, RcPid); + 'Permission' -> + process_permission(Content, Grant); + _ -> + logger:warning("Encountered unexpected grant element: ~p", [ElementName]), + {error, invalid_argument} + end, + case UpdGrant of + {error, _} = Error -> + Error; + _ -> + process_grant(RestElements, UpdGrant, AclOwner, RcPid) + end; +process_grant([#xmlComment{}|RestElements], Grant, Owner, RcPid) -> + process_grant(RestElements, Grant, Owner, RcPid); +process_grant([#xmlText{}|RestElements], Grant, Owner, RcPid) -> + process_grant(RestElements, Grant, Owner, RcPid). + +%% @doc Process an XML element containing information about +%% an ACL permission grantee. +process_grantee([], G0 = ?ACL_GRANT{grantee = Gee0}, AclOwner, RcPid) -> + case Gee0 of + GroupGrantee when GroupGrantee == 'AllUsers'; + GroupGrantee == 'AuthUsers' -> + G0; + #{email := Email, + canonical_id := CanonicalId, + display_name := _DisplayName} when is_binary(Email), + is_binary(CanonicalId) -> + G0; + #{email := Email} = M -> + MaybeConflictingId = + case maps:get(canonical_id, M, undefined) of + undefined -> + undefined; + Defined -> + Defined + end, + case canonical_for_email(Email, RcPid) of + {ok, CanonicalId} when MaybeConflictingId /= undefined, + CanonicalId /= MaybeConflictingId -> + logger:notice("ACL has both Email (~s) and ID (~s) but the user with this email has a different canonical_id; " + "treating this ACL as invalid", [Email, MaybeConflictingId]), + {error, conflicting_grantee_canonical_id}; + {ok, CanonicalId} -> + G0?ACL_GRANT{grantee = Gee0#{canonical_id => CanonicalId}}; + ER -> + ER + end; + #{canonical_id := CanonicalId} -> + case AclOwner of + #{display_name := DisplayName} = M -> + G0?ACL_GRANT{grantee = Gee0#{email => maps:get(email, M, undefined), + display_name => DisplayName}}; + _ -> + case user_details_for_canonical(CanonicalId, RcPid) of + {ok, {Email, DisplayName}} -> + G0?ACL_GRANT{grantee = Gee0#{email => Email, + display_name => DisplayName}}; + ER -> + ER + end + end; + _DialyzersGoHere -> + G0 + end; +process_grantee([#xmlElement{content = [Content], + name = ElementName} | + RestElements], ?ACL_GRANT{grantee = Grantee} = G, AclOwner, RcPid) -> + Value = Content#xmlText.value, + UpdGrant = + case ElementName of + 'ID' -> + G?ACL_GRANT{grantee = Grantee#{canonical_id => list_to_binary(Value)}}; + 'EmailAddress' -> + G?ACL_GRANT{grantee = Grantee#{email => list_to_binary(Value)}}; + 'URI' -> + case Value of + ?AUTH_USERS_GROUP -> + G?ACL_GRANT{grantee = 'AuthUsers'}; + ?ALL_USERS_GROUP -> + G?ACL_GRANT{grantee = 'AllUsers'}; + _ -> + %% Not yet supporting log delivery group + G + end; + _ -> + G + end, + process_grantee(RestElements, UpdGrant, AclOwner, RcPid); +process_grantee([#xmlText{} | RestElements], Grant, Owner, RcPid) -> + process_grantee(RestElements, Grant, Owner, RcPid); +process_grantee([#xmlComment{} | RestElements], Grant, Owner, RcPid) -> + process_grantee(RestElements, Grant, Owner, RcPid). + +%% @doc Process an XML element containing information about +%% an ACL permission. +process_permission([Content], G = ?ACL_GRANT{perms = Perms0}) -> + Value = list_to_existing_atom(Content#xmlText.value), + Perms9 = case lists:member(Value, Perms0) of + true -> Perms0; + false -> [Value | Perms0] + end, + G?ACL_GRANT{perms = Perms9}. + + + +%% =================================================================== +%% Eunit tests +%% =================================================================== + +-ifdef(TEST). + +default_acl_test() -> + ExpectedXml = <<"TESTID1tester1TESTID1tester1FULL_CONTROL">>, + DefaultAcl = default_acl(<<"tester1">>, + <<"TESTID1">>, + <<"TESTKEYID1">>), + ?assertMatch(?ACL{owner = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>, + key_id := <<"TESTKEYID1">>}, + grants = [?ACL_GRANT{grantee = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>}, + perms = ['FULL_CONTROL']} + ] + }, + DefaultAcl), + ?assertEqual(ExpectedXml, riak_cs_xml:to_xml(DefaultAcl)). + +acl_from_xml_test() -> + Xml = "TESTID1tester1TESTID1tester1FULL_CONTROL", + DefaultAcl = default_acl(<<"tester1">>, <<"TESTID1">>, <<"TESTKEYID1">>), + {ok, Acl} = acl_from_xml(Xml, <<"TESTKEYID1">>, undefined), + #{display_name := ExpectedOwnerName, + canonical_id := ExpectedOwnerId} = DefaultAcl?ACL.owner, + #{display_name := ActualOwnerName, + canonical_id := ActualOwnerId} = Acl?ACL.owner, + ?ACL{grants = [?ACL_GRANT{grantee = #{display_name := ExpectedOwnerName, + canonical_id := ExpectedOwnerId}, + perms = ExpectedPerms}]} = DefaultAcl, + ?ACL{grants = [?ACL_GRANT{grantee = #{display_name := ActualOwnerName, + canonical_id := ActualOwnerId}, + perms = ActualPerms}]} = Acl, + ?assertEqual(ExpectedOwnerName, ActualOwnerName), + ?assertEqual(ExpectedOwnerId, ActualOwnerId), + ?assertEqual(ExpectedPerms, ActualPerms). + +roundtrip_test() -> + Xml1 = "TESTID1tester1TESTID1tester1FULL_CONTROL", + Xml2 = "TESTID1tester1TESTID1tester1FULL_CONTROLhttp://acs.amazonaws.com/groups/global/AuthenticatedUsersREAD", + {ok, AclFromXml1} = acl_from_xml(Xml1, "TESTKEYID1", undefined), + {ok, AclFromXml2} = acl_from_xml(Xml2, "TESTKEYID2", undefined), + ?assertEqual(Xml1, binary_to_list(riak_cs_xml:to_xml(AclFromXml1))), + ?assertEqual(Xml2, binary_to_list(riak_cs_xml:to_xml(AclFromXml2))). + +requested_access_test() -> + ?assertEqual('READ', requested_access('GET', false)), + ?assertEqual('READ_ACP', requested_access('GET', true)), + ?assertEqual('WRITE', requested_access('PUT', false)), + ?assertEqual('WRITE_ACP', requested_access('PUT', true)), + ?assertEqual('WRITE', requested_access('POST', false)), + ?assertEqual('WRITE', requested_access('DELETE', false)), + ?assertEqual(undefined, requested_access('POST', true)), + ?assertEqual(undefined, requested_access('DELETE', true)), + ?assertEqual(undefined, requested_access('GARBAGE', false)), + ?assertEqual(undefined, requested_access('GARBAGE', true)). + +canned_acl_test() -> + Owner = #{display_name => <<"tester1">>, + canonical_id => <<"TESTID1">>, + key_id => <<"TESTKEYID1">>}, + BucketOwner = #{display_name => <<"owner1">>, + canonical_id => <<"OWNERID1">>, + key_id => <<"OWNERKEYID1">>}, + DefaultAcl = canned_acl(undefined, Owner, undefined), + PrivateAcl = canned_acl("private", Owner, undefined), + PublicReadAcl = canned_acl("public-read", Owner, undefined), + PublicRWAcl = canned_acl("public-read-write", Owner, undefined), + AuthReadAcl = canned_acl("authenticated-read", Owner, undefined), + BucketOwnerReadAcl1 = canned_acl("bucket-owner-read", Owner, undefined), + BucketOwnerReadAcl2 = canned_acl("bucket-owner-read", Owner, BucketOwner), + BucketOwnerReadAcl3 = canned_acl("bucket-owner-read", Owner, Owner), + BucketOwnerFCAcl1 = canned_acl("bucket-owner-full-control", Owner, undefined), + BucketOwnerFCAcl2 = canned_acl("bucket-owner-full-control", Owner, BucketOwner), + BucketOwnerFCAcl3 = canned_acl("bucket-owner-full-control", Owner, Owner), + + ?assertMatch(?ACL{owner = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>, + key_id := <<"TESTKEYID1">>}, + grants = [?ACL_GRANT{grantee = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>}, + perms = ['FULL_CONTROL']}] + }, + DefaultAcl), + ?assertMatch(?ACL{owner = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>, + key_id := <<"TESTKEYID1">>}, + grants = [?ACL_GRANT{grantee = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>}, + perms = ['FULL_CONTROL']}] + }, + PrivateAcl), + ?assertMatch(?ACL{owner = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>, + key_id := <<"TESTKEYID1">>}, + grants = [?ACL_GRANT{grantee = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>}, + perms = ['FULL_CONTROL']}, + ?ACL_GRANT{grantee = 'AllUsers', + perms = ['READ']} + ] + }, + PublicReadAcl), + ?assertMatch(?ACL{owner = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>, + key_id := <<"TESTKEYID1">>}, + grants = [?ACL_GRANT{grantee = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>}, + perms = ['FULL_CONTROL']}, + ?ACL_GRANT{grantee = 'AllUsers', + perms = ['READ', 'WRITE'] + } + ] + }, + PublicRWAcl), + ?assertMatch(?ACL{owner = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>, + key_id := <<"TESTKEYID1">>}, + grants = [?ACL_GRANT{grantee = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>}, + perms = ['FULL_CONTROL']}, + ?ACL_GRANT{grantee = 'AuthUsers', + perms = ['READ']} + ] + }, + AuthReadAcl), + ?assertMatch(?ACL{owner = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>, + key_id := <<"TESTKEYID1">>}, + grants = [?ACL_GRANT{grantee = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>}, + perms = ['FULL_CONTROL']}] + }, + BucketOwnerReadAcl1), + ?assertMatch(?ACL{owner = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>, + key_id := <<"TESTKEYID1">>}, + grants = [?ACL_GRANT{grantee = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>}, + perms = ['FULL_CONTROL']}, + ?ACL_GRANT{grantee = #{display_name := <<"owner1">>, + canonical_id := <<"OWNERID1">>}, + perms = ['READ']} + ] + }, + BucketOwnerReadAcl2), + ?assertMatch(?ACL{owner = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>, + key_id := <<"TESTKEYID1">>}, + grants = [?ACL_GRANT{grantee = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>}, + perms = ['FULL_CONTROL']} + ] + }, + BucketOwnerReadAcl3), + ?assertMatch(?ACL{owner = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>, + key_id := <<"TESTKEYID1">>}, + grants = [?ACL_GRANT{grantee = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>}, + perms = ['FULL_CONTROL']} + ] + }, + BucketOwnerFCAcl1), + ?assertMatch(?ACL{owner = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>, + key_id := <<"TESTKEYID1">>}, + grants = [?ACL_GRANT{grantee = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>}, + perms = ['FULL_CONTROL']}, + ?ACL_GRANT{grantee = #{display_name := <<"owner1">>, + canonical_id := <<"OWNERID1">>}, + perms = ['FULL_CONTROL']} + ] + }, BucketOwnerFCAcl2), + ?assertMatch(?ACL{owner = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>, + key_id := <<"TESTKEYID1">>}, + grants = [?ACL_GRANT{grantee = #{display_name := <<"tester1">>, + canonical_id := <<"TESTID1">>}, + perms = ['FULL_CONTROL']}] + }, + BucketOwnerFCAcl3). + + +indented_xml_with_comments() -> + Xml=" " + " " + " eb874c6afce06925157eda682f1b3c6eb0f3b983bbee3673ae62f41cce21f6b1" + " admin " + " " + " " + " " + " eb874c6afce06925157eda682f1b3c6eb0f3b983bbee3673ae62f41cce21f6b1 " + " admin " + " " + " FULL_CONTROL " + " " + " " + " ", + Xml. + +comment_space_test() -> + Xml = indented_xml_with_comments(), + %% if cs782 alive, error:{badrecord,xmlElement} thrown here. + {ok, ?ACL{} = Acl} = riak_cs_acl_utils:acl_from_xml(Xml, boom, foo), + %% Compare the result with the one from XML without comments and extra spaces + StrippedXml0 = re:replace(Xml, "", "", [global]), + StrippedXml1 = re:replace(StrippedXml0, " *<", "<", [global]), + StrippedXml = binary_to_list(iolist_to_binary(re:replace(StrippedXml1, " *$", "", [global]))), + {ok, ?ACL{} = AclFromStripped} = riak_cs_acl_utils:acl_from_xml(StrippedXml, boom, foo), + ?assertEqual(AclFromStripped?ACL{creation_time = Acl?ACL.creation_time}, + Acl), + ok. + +acl_to_from_json_term_test() -> + CreationTime = os:system_time(millisecond), + Acl0 = ?ACL{owner = #{display_name => <<"tester1">>, + canonical_id => <<"TESTID1">>, + key_id => <<"TESTKEYID1">>}, + grants = [?ACL_GRANT{grantee = #{display_name => <<"tester1">>, + canonical_id => <<"TESTID1">>}, + perms = ['READ']}, + ?ACL_GRANT{grantee = #{display_name => <<"tester2">>, + canonical_id => <<"TESTID2">>}, + perms = ['WRITE']}], + creation_time = CreationTime}, + Json = riak_cs_json:to_json(Acl0), + Acl9 = riak_cs_acl:exprec_acl( + jsx:decode(Json, [{labels, atom}])), + ?assertEqual(Acl0, Acl9). + +-endif. diff --git a/apps/riak_cs/src/riak_cs_api.erl b/apps/riak_cs/src/riak_cs_api.erl new file mode 100644 index 000000000..41977feca --- /dev/null +++ b/apps/riak_cs/src/riak_cs_api.erl @@ -0,0 +1,51 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_api). + +-export([list_buckets/1, + list_objects/5 + ]). + +-include("riak_cs.hrl"). +-include("riak_cs_web.hrl"). +-include_lib("kernel/include/logger.hrl"). + +%% @doc Return a user's buckets. +-spec list_buckets(rcs_user()) -> ?LBRESP{}. +list_buckets(User = ?RCS_USER{buckets=Buckets}) -> + ?LBRESP{user = User, + buckets = [Bucket || Bucket <- Buckets, + Bucket?RCS_BUCKET.last_action /= deleted]}. + +-spec list_objects(list_objects_req_type(), binary(), non_neg_integer(), proplists:proplist(), riak_client()) -> + {ok, list_objects_response() | list_object_versions_response()} | {error, term()}. +list_objects(_, _Bucket, {error, _} = Error, _Options, _RcPid) -> + Error; +list_objects(ReqType, Bucket, MaxKeys, Options, RcPid) -> + Request = riak_cs_list_objects:new_request( + ReqType, Bucket, MaxKeys, Options), + case riak_cs_list_objects_fsm_v2:start_link(RcPid, Request) of + {ok, ListFSMPid} -> + riak_cs_list_objects_utils:get_object_list(ListFSMPid); + {error, _} = Error -> + Error + end. diff --git a/apps/riak_cs/src/riak_cs_app.erl b/apps/riak_cs/src/riak_cs_app.erl new file mode 100644 index 000000000..138fee739 --- /dev/null +++ b/apps/riak_cs/src/riak_cs_app.erl @@ -0,0 +1,226 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc Callbacks for the riak_cs application. + +-module(riak_cs_app). + +-behaviour(application). + +%% application API +-export([start/2, + stop/1]). +-export([atoms_for_check_bucket_props/0]). + +-include("riak_cs.hrl"). +-include_lib("riakc/include/riakc.hrl"). +-include_lib("riak_pb/include/riak_pb_kv_codec.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-type start_type() :: normal | {takeover, node()} | {failover, node()}. +-type start_args() :: term(). + +%% =================================================================== +%% Public API +%% =================================================================== + +%% @doc application start callback for riak_cs. +-spec start(start_type(), start_args()) -> {ok, pid()} | + {error, term()}. +start(_Type, _StartArgs) -> + riak_cs_config:warnings(), + {ok, Pbc} = riak_connection(), + ok = is_config_valid(), + ok = ensure_bucket_props(Pbc), + {ok, PostFun} = check_admin_creds(Pbc), + ok = try_connect_to_riak_node(), + Ret = {ok, _Pid} = riak_cs_sup:start_link(), + ok = PostFun(), + ok = riakc_pb_socket:stop(Pbc), + ok = stanchion_lock:cleanup(), + Ret. + + +try_connect_to_riak_node() -> + Node = riak_cs_utils:guess_riak_node_name(), + case net_kernel:connect_node(Node) of + true -> + ok; + false -> + logger:notice("Failed to connect to riak node ~s. Rpc calls won't work.", [Node]), + ok + end. + + +%% @doc application stop callback for riak_cs. +-spec stop(term()) -> ok. +stop(_State) -> + ok. + + +is_config_valid() -> + case application:get_env(riak_cs, connection_pools) of + {ok, _} -> + ok; + _ -> + logger:error("connection_pools is not set"), + not_ok + end. + + +ensure_bucket_props(Pbc) -> + BucketsWithMultiTrue = [?ACCESS_BUCKET, + ?STORAGE_BUCKET], + BucketsWithMultiFalse = [?USER_BUCKET, + ?BUCKETS_BUCKET, + ?SERVICE_BUCKET, + ?IAM_ROLE_BUCKET, + ?IAM_POLICY_BUCKET, + ?IAM_SAMLPROVIDER_BUCKET, + ?TEMP_SESSIONS_BUCKET, + ?OBJECT_LOCK_BUCKET], + [ok = riakc_pb_socket:set_bucket(Pbc, B, [{allow_mult, true}]) || B <- BucketsWithMultiTrue], + [ok = riakc_pb_socket:set_bucket(Pbc, B, [{allow_mult, false}]) || B <- BucketsWithMultiFalse], + ok. + +%% Put atoms into atom table to suppress warning logs in `check_bucket_props' +atoms_for_check_bucket_props() -> + [riak_core_util, chash_std_keyfun, + riak_kv_wm_link_walker, mapreduce_linkfun]. + +check_admin_creds(Pbc) -> + case riak_cs_config:admin_creds() of + {ok, {?DEFAULT_ADMIN_KEY, _}} -> + %% The default key + logger:warning("admin.key has not been set. Please create an admin user (e.g., `riak-cs admin create-admin-user`)", []), + application:set_env(riak_cs, admin_secret, <<"admin-secret">>), + {ok, nop()}; + {ok, {undefined, _}} -> + logger:error("The admin user's key id has not been specified."), + {error, admin_key_undefined}; + {ok, {<<>>, _}} -> + logger:error("The admin user's key id has not been specified."), + {error, admin_key_undefined}; + {ok, {Key, undefined}} -> + fetch_and_cache_admin_creds(Key, Pbc); + {ok, {Key, <<>>}} -> + fetch_and_cache_admin_creds(Key, Pbc); + {ok, {Key, _}} -> + logger:warning("The admin user's secret is specified. Ignoring."), + fetch_and_cache_admin_creds(Key, Pbc) + end. + +fetch_and_cache_admin_creds(Key, Pbc) -> + fetch_and_cache_admin_creds(Key, Pbc, _MaxRetries = 5, no_error). +fetch_and_cache_admin_creds(_Key, _, 0, Error) -> + Error; +fetch_and_cache_admin_creds(Key, Pbc, Attempt, _Error) -> + case find_user(Key, Pbc) of + {ok, ?IAM_USER{key_secret = SAK}, PostFun} -> + application:set_env(riak_cs, admin_secret, SAK), + {ok, PostFun}; + Error -> + logger:warning("Couldn't get admin user (~s) record: ~p. Will retry ~b more times", + [Key, Error, Attempt]), + timer:sleep(3000), + fetch_and_cache_admin_creds(Key, Pbc, Attempt - 1, Error) + end. + + +find_user(A, Pbc) -> + case riakc_pb_socket:get_index_eq(Pbc, ?USER_BUCKET, ?USER_KEYID_INDEX, A) of + {ok, ?INDEX_RESULTS{keys = []}} -> + %% try_get_user_pre_3_2 + maybe_get_3_1_user_with_policy(A, Pbc); + {ok, ?INDEX_RESULTS{keys = [Arn|_]}} -> + case get_user(Arn, Pbc) of + {ok, User} -> + {ok, User, nop()}; + ER -> + ER + end; + {error, Reason} -> + logger:warning("Riak client connection error while finding user ~s: ~p", [A, Reason]), + {error, Reason} + end. + +maybe_get_3_1_user_with_policy(KeyForArn, Pbc) -> + case get_user(KeyForArn, Pbc) of + {ok, OldAdminUser = ?IAM_USER{arn = AdminArn, + name = AdminUserName, + key_id = KeyId}} -> + logger:notice("Found pre-3.2 admin user; " + "converting it to rcs_user_v3, attaching an admin policy and saving", []), + ?LOG_DEBUG("Converted admin: ~p", [OldAdminUser]), + + ?LOG_DEBUG("deleting old admin user", []), + ok = riakc_pb_socket:delete(Pbc, ?USER_BUCKET, KeyId, ?CONSISTENT_DELETE_OPTIONS), + ?LOG_DEBUG("recreating it as rcs_user_v3"), + Meta = dict:store(?MD_INDEX, riak_cs_utils:object_indices(OldAdminUser), dict:new()), + Obj = riakc_obj:update_metadata( + riakc_obj:new(?USER_BUCKET, AdminArn, term_to_binary(OldAdminUser)), + Meta), + ok = riakc_pb_socket:put(Pbc, Obj, ?CONSISTENT_WRITE_OPTIONS), + + CompleteAdminConversion = + fun() -> + ?LOG_DEBUG("Attaching admin policy to the recreated admin"), + AdminPolicyDocument = + #{<<"Version">> => <<"2012-10-17">>, + <<"Statement">> => [#{<<"Principal">> => <<"*">>, + <<"Effect">> => <<"Allow">>, + <<"Action">> => [<<"sts:*">>, <<"iam:*">>, <<"s3:*">>], + <<"Resource">> => <<"*">> + } + ] + }, + {ok, ?IAM_POLICY{arn = PolicyArn}} = + riak_cs_iam:create_policy(#{policy_name => <<"AdminPolicy">>, + policy_document => D = jsx:encode(AdminPolicyDocument)}), + ?LOG_DEBUG("Attached policy: ~p", [D]), + ok = riak_cs_iam:attach_user_policy(PolicyArn, AdminUserName, Pbc), + ok + end, + {ok, OldAdminUser, CompleteAdminConversion}; + ER -> + ER + end. + +get_user(Arn, Pbc) -> + case riak_cs_pbc:get_sans_stats(Pbc, ?USER_BUCKET, Arn, + [{notfound_ok, false}], + riak_cs_config:get_user_timeout()) of + {ok, Obj} -> + {ok, riak_cs_user:from_riakc_obj(Obj, _KeepDeletedBuckets = false)}; + ER -> + ER + end. + + +riak_connection() -> + {Host, Port} = riak_cs_config:riak_host_port(), + Timeout = application:get_env(riak_cs, riakc_connect_timeout, 10000), + StartOptions = [{connect_timeout, Timeout}, + {auto_reconnect, true}], + riakc_pb_socket:start_link(Host, Port, StartOptions). + +nop() -> + fun() -> ok end. diff --git a/src/riak_cs_auth.erl b/apps/riak_cs/src/riak_cs_auth.erl similarity index 79% rename from src/riak_cs_auth.erl rename to apps/riak_cs/src/riak_cs_auth.erl index 0e5fcae57..69eed0122 100644 --- a/src/riak_cs_auth.erl +++ b/apps/riak_cs/src/riak_cs_auth.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -22,18 +23,15 @@ -include("riak_cs.hrl"). --type wm_reqdata() :: tuple(). --type wm_context() :: tuple(). - %% TODO: arguments of `identify/2', and 3rd and 4th arguments of %% authenticate/4 are actually `#wm_reqdata{}' and `#context{}' %% from webmachine, but can't compile after `webmachine.hrl' import. --callback identify(RD :: wm_reqdata(), Ctx :: wm_context()) -> +-callback identify(#wm_reqdata{}, #rcs_web_context{}) -> failed | {failed, Reason::term()} | - {string() | undefined, string() | tuple()} | - {string(), undefined}. + {binary() | undefined, binary() | tuple()} | + {binary(), undefined}. -callback authenticate(rcs_user(), string() | {string(), term()} | undefined, - wm_reqdata(), wm_context()) -> - ok | {error, atom()}. + #wm_reqdata{}, #rcs_web_context{}) -> + ok | {error, atom()}. diff --git a/src/riak_cs_s3_auth.erl b/apps/riak_cs/src/riak_cs_aws_auth.erl similarity index 78% rename from src/riak_cs_s3_auth.erl rename to apps/riak_cs/src/riak_cs_aws_auth.erl index 073c8d7f4..fa55821d9 100644 --- a/src/riak_cs_s3_auth.erl +++ b/apps/riak_cs/src/riak_cs_aws_auth.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,23 +19,27 @@ %% %% --------------------------------------------------------------------- --module(riak_cs_s3_auth). +-module(riak_cs_aws_auth). -behavior(riak_cs_auth). -export([identify/2, authenticate/4]). +-export([parse_auth_header/1]). +-export([calculate_signature_v2/2]). %% lend it to stanchion_auth -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). --export([calculate_signature_v2/2]). -endif. -include("riak_cs.hrl"). --include("s3_api.hrl"). --include_lib("webmachine/include/webmachine.hrl"). +-include("aws_api.hrl"). +-include_lib("kernel/include/logger.hrl"). -type v4_attrs() :: [{string(), string()}]. +-type sigv2_quirk() :: none | boto3_1_26_36. +-define(SIGV2_QUIRKS, [none, boto3_1_26_36]). + -define(QS_KEYID, "AWSAccessKeyId"). -define(QS_SIGNATURE, "Signature"). @@ -50,11 +55,11 @@ %% Public API %% =================================================================== --spec identify(RD::term(), #context{}) -> - {string() | undefined, - string() | {v4, v4_attrs()} | undefined} | - {failed, Reason::atom()}. -identify(RD,_Ctx) -> +-spec identify(#wm_reqdata{}, #rcs_web_context{}) -> + {binary() | undefined, + binary() | {v4, v4_attrs()} | undefined} | + {failed, Reason::atom()}. +identify(RD, _Ctx) -> case wrq:get_req_header("authorization", RD) of undefined -> identify_by_query_string(RD); @@ -62,8 +67,8 @@ identify(RD,_Ctx) -> parse_auth_header(AuthHeader) end. --spec authenticate(rcs_user(), string() | {v4, v4_attrs()}, RD::term(), #context{}) -> - ok | {error, atom()}. +-spec authenticate(rcs_user(), string() | {v4, v4_attrs()}, #wm_reqdata{}, #rcs_web_context{}) -> + ok | {error, atom()}. authenticate(User, Signature, RD, Ctx) -> case wrq:get_req_header("authorization", RD) of undefined -> @@ -78,21 +83,21 @@ authenticate(User, Signature, RD, Ctx) -> end end. --spec authenticate_1(rcs_user(), string() | {v4, v4_attrs()}, RD::term(), #context{}) -> - ok | {error, atom()}. authenticate_1(User, {v4, Attributes}, RD, _Ctx) -> authenticate_v4(User, Attributes, RD); authenticate_1(User, Signature, RD, _Ctx) -> - CalculatedSignature = calculate_signature_v2(User?RCS_USER.key_secret, RD), - case check_auth(Signature, CalculatedSignature) of - true -> + authenticate_1(User, Signature, RD, _Ctx, ?SIGV2_QUIRKS). +authenticate_1(_, _, _, _, []) -> + {error, invalid_authentication}; +authenticate_1(User, Signature, RD, Ctx = #rcs_web_context{request_id = RequestId}, [Quirk|MoreQuirks]) -> + CalculatedSignature = calculate_signature_v2(User?RCS_USER.key_secret, RD, Quirk), + if Signature == CalculatedSignature -> Expires = wrq:get_qs_value("Expires", RD), case Expires of undefined -> ok; _ -> - {MegaSecs, Secs, _} = os:timestamp(), - Now = (MegaSecs * 1000000) + Secs, + Now = os:system_time(second), case Now > list_to_integer(Expires) of true -> %% @TODO Not sure if this is the proper error @@ -102,19 +107,15 @@ authenticate_1(User, Signature, RD, _Ctx) -> ok end end; - _ -> - {error, invalid_authentication} + el/=se -> + logger:notice("Bad signature: received ~s vs calculated ~s on request_id ~s", + [Signature, CalculatedSignature, RequestId]), + authenticate_1(User, Signature, RD, Ctx, MoreQuirks) end. -%% =================================================================== -%% Internal functions -%% =================================================================== - -%% Idintify user by query string. -%% Currently support signature v2 only, does NOT support signature v4. -identify_by_query_string(RD) -> - {wrq:get_qs_value(?QS_KEYID, RD), wrq:get_qs_value(?QS_SIGNATURE, RD)}. - +-spec parse_auth_header(string()) -> + {binary(), binary()} | {undefined, undefined} + | {failed, {auth_not_supported, string()}}. parse_auth_header("AWS4-HMAC-SHA256 " ++ String) -> case riak_cs_config:auth_v4_enabled() of true -> @@ -125,21 +126,31 @@ parse_auth_header("AWS4-HMAC-SHA256 " ++ String) -> parse_auth_header("AWS " ++ Key) -> case string:tokens(Key, ":") of [KeyId, KeyData] -> - {KeyId, KeyData}; + {list_to_binary(KeyId), + list_to_binary(KeyData)}; _ -> {undefined, undefined} end; parse_auth_header(_) -> {undefined, undefined}. --spec parse_auth_v4_header(string()) -> {string(), {v4, [{string(), string()}]}}. +%% =================================================================== +%% Internal functions +%% =================================================================== + +%% Idintify user by query string. +%% Currently support signature v2 only, does NOT support signature v4. +identify_by_query_string(RD) -> + {case wrq:get_qs_value(?QS_KEYID, RD) of undefined -> undefined; A -> list_to_binary(A) end, + case wrq:get_qs_value(?QS_SIGNATURE, RD) of undefined -> undefined; A -> list_to_binary(A) end}. + parse_auth_v4_header(String) -> KVs = string:tokens(String, ", "), parse_auth_v4_header(KVs, undefined, []). parse_auth_v4_header([], UserId, Acc) -> - {UserId, {v4, lists:reverse(Acc)}}; + {list_to_binary(UserId), {v4, lists:reverse(Acc)}}; parse_auth_v4_header([KV | KVs], UserId, Acc) -> - lager:debug("Auth header ~p~n", [KV]), + %% ?LOG_DEBUG("Auth header ~p", [KV]), case string:tokens(KV, "=") of [Key, Value] -> case Key of @@ -154,23 +165,29 @@ parse_auth_v4_header([KV | KVs], UserId, Acc) -> parse_auth_v4_header(KVs, UserId, Acc) end. --spec calculate_signature_v2(string(), #wm_reqdata{}) -> string(). +-spec calculate_signature_v2(binary(), #wm_reqdata{}) -> binary(). calculate_signature_v2(KeyData, RD) -> + calculate_signature_v2(KeyData, RD, none). + +-spec calculate_signature_v2(binary(), #wm_reqdata{}, sigv2_quirk()) -> binary(). +calculate_signature_v2(KeyData, RD, Quirk) -> Headers = riak_cs_wm_utils:normalize_headers(RD), AmazonHeaders = riak_cs_wm_utils:extract_amazon_headers(Headers), - OriginalResource = riak_cs_s3_rewrite:original_resource(RD), - Resource = case OriginalResource of - undefined -> []; %% TODO: get noisy here? - {Path,QS} -> [Path, canonicalize_qs(v2, QS)] + {Path, QS} = riak_cs_aws_rewrite:original_resource(RD), + Resource = case Quirk of + boto3_1_26_36 -> + apply_sigv2_quirk(boto3_1_26_36, {Path, QS}); + none -> + [Path, canonicalize_qs(v2, QS)] end, Date = case wrq:get_qs_value("Expires", RD) of undefined -> - case proplists:is_defined("x-amz-date", Headers) of - true -> "\n"; - false -> [wrq:get_req_header("date", RD), "\n"] + case wrq:get_req_header("x-amz-date", RD) of + undefined -> wrq:get_req_header("date", RD); + XAmzDate -> XAmzDate end; Expires -> - Expires ++ "\n" + Expires end, CMD5 = case wrq:get_req_header("content-md5", RD) of undefined -> []; @@ -186,35 +203,66 @@ calculate_signature_v2(KeyData, RD) -> ContentType, "\n", Date, + "\n", AmazonHeaders, Resource], - _ = lager:debug("STS: ~p", [STS]), - - base64:encode_to_string(riak_cs_utils:sha_mac(KeyData, STS)). + %%?LOG_DEBUG("STS: ~p", [STS]), + + base64:encode(riak_cs_utils:sha_mac(KeyData, STS)). + +apply_sigv2_quirk(boto3_1_26_36, {Path, QS}) -> + logger:notice("applying boto3-1.26.36 quirk to QS ~p", [QS]), + CQ = + case QS of + [{"versions", []}, {"encoding-type", _}] -> + "?versions?versions"; + [{"delete", []}] -> + "?delete?delete"; + [{"acl", []}] -> + "?acl?acl"; + [{"acl", []}, {"versionId", V}] -> + "?acl?acl&versionId="++V; + [{"uploads", []}] -> + "?uploads?uploads"; + [{"policy", []}] -> + "?policy?policy"; + [{"versioning", []}] -> + "?versioning?versioning"; + _ -> + canonicalize_qs(v2, QS) + end, + [drop_slash(Path), CQ]. + +drop_slash("") -> + ""; +drop_slash(A) -> + case lists:last(A) of + $/ -> + drop_slash(lists:droplast(A)); + _ -> + A + end. --spec authenticate_v4(rcs_user(), v4_attrs(), RD::term()) -> - ok | - {error, {unmatched_signature, - Presented::string(), Calculated::string()}}. authenticate_v4(?RCS_USER{key_secret = SecretAccessKey} = _User, AuthAttrs, RD) -> Method = wrq:method(RD), - {Path, Qs} = riak_cs_s3_rewrite:raw_url(RD), + {Path, Qs} = riak_cs_aws_s3_rewrite:raw_url(RD), AllHeaders = riak_cs_wm_utils:normalize_headers(RD), authenticate_v4(SecretAccessKey, AuthAttrs, Method, Path, Qs, AllHeaders). authenticate_v4(SecretAccessKey, AuthAttrs, Method, Path, Qs, AllHeaders) -> CanonicalRequest = canonical_request_v4(AuthAttrs, Method, Path, Qs, AllHeaders), - _ = lager:debug("CanonicalRequest(v4):~n~s~nEND-OF-CANONICAL-REQUEST", [CanonicalRequest]), + %% ?LOG_DEBUG("CanonicalRequest(v4): ~s", [CanonicalRequest]), {StringToSign, Scope} = string_to_sign_v4(AuthAttrs, AllHeaders, CanonicalRequest), - _ = lager:debug("StringToSign(v4):~n~s~nEND-OF-STRING-TO-SIGN", [StringToSign]), - CalculatedSignature = calculate_signature_v4(SecretAccessKey, Scope, StringToSign), - _ = lager:debug("CalculatedSignature(v4): ~s", [CalculatedSignature]), - {"Signature", PresentedSignature} = lists:keyfind("Signature", 1, AuthAttrs), - _ = lager:debug("PresentedSignature(v4) : ~s", [PresentedSignature]), - case CalculatedSignature of - PresentedSignature -> ok; - _ -> {error, {unmatched_signature, PresentedSignature, CalculatedSignature}} + %%?LOG_DEBUG("StringToSign(v4): ~s", [StringToSign]), + Calculated = calculate_signature_v4(SecretAccessKey, Scope, StringToSign), + {"Signature", Presented} = lists:keyfind("Signature", 1, AuthAttrs), + case Calculated == Presented of + true -> + ok; + _ -> + ?LOG_NOTICE("Bad signature: ~s (expected: ~s)", [Presented, Calculated]), + {error, bad_signature} end. canonical_request_v4(AuthAttrs, Method, Path, Qs, AllHeaders) -> @@ -224,12 +272,14 @@ canonical_request_v4(AuthAttrs, Method, Path, Qs, AllHeaders) -> string:tokens(SignedHeaders, [$;])), {"x-amz-content-sha256", HashedPayload} = lists:keyfind("x-amz-content-sha256", 1, AllHeaders), + %%?LOG_DEBUG("HashedPayload ~p", [HashedPayload]), CanonicalRequest = [atom_to_list(Method), $\n, strict_url_encode_for_path(Path), $\n, CanonicalQs, $\n, CanonicalHeaders, $\n, SignedHeaders, $\n, HashedPayload], + %%?LOG_DEBUG("CanonicalRequest: ~p", [CanonicalRequest]), CanonicalRequest. canonical_headers_v4(AllHeaders, HeaderNames) -> @@ -258,7 +308,7 @@ string_to_sign_v4(AuthAttrs, AllHeaders, CanonicalRequest) -> %% Are there any good points to check `AwsRegion' to be region in app env? %% So far, it does not improve security (at least for CS) but %% introduces some complexity for client (config/coding/etc). - [_UserId, CredDate, AwsRegion, "s3" = AwsService, "aws4_request" = AwsRequest] = + [_UserId, CredDate, AwsRegion, AwsService, "aws4_request" = AwsRequest] = string:tokens(Cred, [$/]), %% TODO: Validate `CredDate' be within 7 days Scope = [CredDate, $/, AwsRegion, $/, AwsService, $/, AwsRequest], @@ -277,19 +327,16 @@ calculate_signature_v4(SecretAccessKey, mochihex:to_hex(hmac_sha256(SigningKey, StringToSign)). hmac_sha256(Key, Data) -> - crypto:hmac(sha256, Key, Data). + crypto:mac(hmac, sha256, Key, Data). hex_sha256hash(Data) -> mochihex:to_hex(crypto:hash(sha256, Data)). -check_auth(PresentedSignature, CalculatedSignature) -> - PresentedSignature == CalculatedSignature. - canonicalize_qs(Version, QS) -> %% The QS must be sorted be canonicalized, %% and since `canonicalize_qs/2` builds up the %% accumulator with cons, it comes back in reverse - %% order. So we'll sort then reverise, so cons'ing + %% order. So we'll sort then reverse, so cons'ing %% actually puts it back in the correct order ReversedSorted = lists:reverse(lists:sort(QS)), case Version of @@ -312,7 +359,7 @@ canonicalize_qs_v2([{K, []}|T], Acc) -> canonicalize_qs_v2(T, Acc) end; canonicalize_qs_v2([{K, V}|T], Acc) -> - case lists:member(K, ?SUBRESOURCES) of + case lists:member(K, ?SUBRESOURCES ++ ["list-type"]) of true -> Amp = if Acc == [] -> ""; true -> "&" @@ -403,17 +450,17 @@ auth_test_() -> setup() -> application:set_env(riak_cs, verify_client_clock_skew, false), - application:set_env(riak_cs, cs_root_host, ?ROOT_HOST). + application:set_env(riak_cs, cs_root_host, ?S3_ROOT_HOST). teardown(_) -> application:unset_env(riak_cs, verify_client_clock_skew), application:unset_env(riak_cs, cs_root_host). test_fun(Desc, ExpectedSignature, CalculatedSignature) -> - {Desc, ?_assert(check_auth(ExpectedSignature,CalculatedSignature))}. + {Desc, ?_assert(ExpectedSignature == CalculatedSignature)}. example_get_object() -> - KeyData = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o", + KeyData = <<"uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o">>, Method = 'GET', Version = {1, 1}, OrigPath = "/johnsmith/photos/puppy.jpg", @@ -423,12 +470,12 @@ example_get_object() -> {"Date", "Tue, 27 Mar 2007 19:36:42 +0000"}, {"x-rcs-rewrite-path", OrigPath}]), RD = wrq:create(Method, Version, Path, Headers), - ExpectedSignature = "xXjDGYUmKxnwqr5KXNPGldn5LbA=", + ExpectedSignature = <<"xXjDGYUmKxnwqr5KXNPGldn5LbA=">>, CalculatedSignature = calculate_signature_v2(KeyData, RD), test_fun("example get object test", ExpectedSignature, CalculatedSignature). example_put_object() -> - KeyData = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o", + KeyData = <<"uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o">>, Method = 'PUT', Version = {1, 1}, OrigPath = "/johnsmith/photos/puppy.jpg", @@ -440,12 +487,12 @@ example_put_object() -> {"Content-Length", 94328}, {"Date", "Tue, 27 Mar 2007 21:15:45 +0000"}]), RD = wrq:create(Method, Version, Path, Headers), - ExpectedSignature = "hcicpDDvL9SsO6AkvxqmIWkmOuQ=", + ExpectedSignature = <<"hcicpDDvL9SsO6AkvxqmIWkmOuQ=">>, CalculatedSignature = calculate_signature_v2(KeyData, RD), test_fun("example put object test", ExpectedSignature, CalculatedSignature). example_list() -> - KeyData = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o", + KeyData = <<"uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o">>, Method = 'GET', Version = {1, 1}, OrigPath = "/johnsmith/?prefix=photos&max-keys=50&marker=puppy", @@ -456,12 +503,12 @@ example_list() -> {"x-rcs-rewrite-path", OrigPath}, {"Date", "Tue, 27 Mar 2007 19:42:41 +0000"}]), RD = wrq:create(Method, Version, Path, Headers), - ExpectedSignature = "jsRt/rhG+Vtp88HrYL706QhE4w4=", + ExpectedSignature = <<"jsRt/rhG+Vtp88HrYL706QhE4w4=">>, CalculatedSignature = calculate_signature_v2(KeyData, RD), test_fun("example list test", ExpectedSignature, CalculatedSignature). example_fetch() -> - KeyData = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o", + KeyData = <<"uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o">>, Method = 'GET', Version = {1, 1}, OrigPath = "/johnsmith/?acl", @@ -471,12 +518,12 @@ example_fetch() -> {"x-rcs-rewrite-path", OrigPath}, {"Date", "Tue, 27 Mar 2007 19:44:46 +0000"}]), RD = wrq:create(Method, Version, Path, Headers), - ExpectedSignature = "thdUi9VAkzhkniLj96JIrOPGi0g=", + ExpectedSignature = <<"thdUi9VAkzhkniLj96JIrOPGi0g=">>, CalculatedSignature = calculate_signature_v2(KeyData, RD), test_fun("example fetch test", ExpectedSignature, CalculatedSignature). example_delete() -> - KeyData = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o", + KeyData = <<"uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o">>, Method = 'DELETE', Version = {1, 1}, OrigPath = "/johnsmith/photos/puppy.jpg", @@ -488,7 +535,7 @@ example_delete() -> {"Date", "Tue, 27 Mar 2007 21:20:27 +0000"}, {"x-amz-date", "Tue, 27 Mar 2007 21:20:26 +0000"}]), RD = wrq:create(Method, Version, Path, Headers), - ExpectedSignature = "k3nL7gH3+PadhTEVn5Ip83xlYzk=", + ExpectedSignature = <<"amG66fXy2/BJ/HvrN2jUKINWfrE=">>, CalculatedSignature = calculate_signature_v2(KeyData, RD), test_fun("example delete test", ExpectedSignature, CalculatedSignature). @@ -502,7 +549,7 @@ example_delete() -> %% is specified using a singled X-Amz-Meta-ReviewedBy header with %% multiple field values. example_upload() -> - KeyData = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o", + KeyData = <<"uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o">>, Method = 'PUT', Version = {1, 1}, OrigPath = "/static.johnsmith.net/db-backup.dat.gz", @@ -523,12 +570,12 @@ example_upload() -> {"Content-Encoding", "gzip"}, {"Content-Length", 5913339}]), RD = wrq:create(Method, Version, Path, Headers), - ExpectedSignature = "C0FlOtU8Ylb9KDTpZqYkZPX91iI=", + ExpectedSignature = <<"C0FlOtU8Ylb9KDTpZqYkZPX91iI=">>, CalculatedSignature = calculate_signature_v2(KeyData, RD), test_fun("example upload test", ExpectedSignature, CalculatedSignature). example_list_all_buckets() -> - KeyData = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o", + KeyData = <<"uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o">>, Method = 'GET', Version = {1, 1}, Path = "/", @@ -537,12 +584,12 @@ example_list_all_buckets() -> {"x-rcs-rewrite-path", Path}, {"Date", "Wed, 28 Mar 2007 01:29:59 +0000"}]), RD = wrq:create(Method, Version, Path, Headers), - ExpectedSignature = "Db+gepJSUbZKwpx1FR0DLtEYoZA=", + ExpectedSignature = <<"Db+gepJSUbZKwpx1FR0DLtEYoZA=">>, CalculatedSignature = calculate_signature_v2(KeyData, RD), test_fun("example list all buckts test", ExpectedSignature, CalculatedSignature). example_unicode_keys() -> - KeyData = "uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o", + KeyData = <<"uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o">>, Method = 'GET', Version = {1, 1}, OrigPath = "/dictionary/fran%C3%A7ais/pr%c3%a9f%c3%a8re", @@ -552,7 +599,7 @@ example_unicode_keys() -> {"x-rcs-rewrite-path", OrigPath}, {"Date", "Wed, 28 Mar 2007 01:49:49 +0000"}]), RD = wrq:create(Method, Version, Path, Headers), - ExpectedSignature = "dxhSBHoI6eVSPcXJqEghlUzZMnY=", + ExpectedSignature = <<"dxhSBHoI6eVSPcXJqEghlUzZMnY=">>, CalculatedSignature = calculate_signature_v2(KeyData, RD), test_fun("example unicode keys test", ExpectedSignature, CalculatedSignature). diff --git a/apps/riak_cs/src/riak_cs_aws_iam_rewrite.erl b/apps/riak_cs/src/riak_cs_aws_iam_rewrite.erl new file mode 100644 index 000000000..7179c99fa --- /dev/null +++ b/apps/riak_cs/src/riak_cs_aws_iam_rewrite.erl @@ -0,0 +1,65 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_aws_iam_rewrite). +-behaviour(riak_cs_rewrite). + +-export([rewrite/5, + original_resource/1, + raw_url/1]). + +-include("riak_cs.hrl"). +-include("aws_api.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-spec rewrite(atom(), atom(), {integer(), integer()}, mochiweb_headers(), string()) -> + {mochiweb_headers(), string()}. +rewrite(Method, _Scheme, _Vsn, Headers, Url) -> + {Path, QueryString, _} = mochiweb_util:urlsplit_path(Url), + rewrite_path_and_headers(Method, Headers, Url, Path, QueryString). + + +-spec original_resource(#wm_reqdata{}) -> undefined | {string(), [{term(),term()}]}. +original_resource(RD) -> + riak_cs_rewrite:original_resource(RD). + +-spec raw_url(#wm_reqdata{}) -> undefined | {string(), [{term(), term()}]}. +raw_url(RD) -> + riak_cs_rewrite:raw_url(RD). + +rewrite_path_and_headers(Method, Headers, Url, Path, QueryString) -> + RewrittenPath = + rewrite_path(Method, Path, QueryString), + RewrittenHeaders = + mochiweb_headers:default( + ?RCS_RAW_URL_HEADER, Url, + mochiweb_headers:default( + ?RCS_REWRITE_HEADER, Url, + Headers)), + {RewrittenHeaders, RewrittenPath}. + + +%% All IAM requests are POSTs without a resource path. Instead of +%% attempting to rewrite path and map those POSTs to something we may +%% deem more logical, let's just skip rewriting altogether. In +%% riak_cs_wm_iam.erl, we read the www-form and handle the request +%% based on presented Action. +rewrite_path(_Method, "/", _QS) -> + "/iam". diff --git a/src/riak_cs_s3_policy.erl b/apps/riak_cs/src/riak_cs_aws_policy.erl similarity index 63% rename from src/riak_cs_s3_policy.erl rename to apps/riak_cs/src/riak_cs_aws_policy.erl index a33b49b5e..702cabaab 100644 --- a/src/riak_cs_s3_policy.erl +++ b/apps/riak_cs/src/riak_cs_aws_policy.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -21,19 +22,12 @@ %% @doc policy utility functions %% A (principal) is/isn't allowed (effect) to do B (action) to C (resource) where D (condition) applies --module(riak_cs_s3_policy). +-module(riak_cs_aws_policy). -behaviour(riak_cs_policy). --include("riak_cs.hrl"). --include("s3_api.hrl"). --include_lib("webmachine/include/wm_reqdata.hrl"). --include_lib("webmachine/include/wm_reqstate.hrl"). --include_lib("riak_pb/include/riak_pb_kv_codec.hrl"). - %% Public API --export([ - fetch_bucket_policy/2, +-export([fetch_bucket_policy/2, bucket_policy/1, bucket_policy_from_contents/2, eval/2, @@ -41,13 +35,15 @@ reqdata_to_access/3, policy_from_json/1, policy_to_json_term/1, - supported_object_action/0, - supported_bucket_action/0, log_supported_actions/0 ]). --ifdef(TEST). +-include("riak_cs.hrl"). +-include_lib("webmachine/include/wm_reqstate.hrl"). +-include_lib("riak_pb/include/riak_pb_kv_codec.hrl"). +-include_lib("kernel/include/logger.hrl"). +-ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -export([eval_all_ip_addr/2, eval_ip_address/2, @@ -58,18 +54,10 @@ print_arns/1, parse_ip/1, print_ip/1, - statement_eq/2 %% for test use ]). - -endif. --type policy1() :: ?POLICY{}. - --export_type([policy1/0]). - --define(AMZ_POLICY_OLD_VERSION, <<"2008-10-17">>). --define(AMZ_DEFAULT_VERSION, <<"2012-10-17">>). -define(POLICY_UNDEF, {error, policy_undefined}). %% =================================================================== @@ -77,9 +65,9 @@ %% =================================================================== %% @doc evaluates the policy and returns the policy allows, denies or -%% not says nothing about this access. Usually in case of undefined, +%% not says anything about this access. Usually in case of undefined, %% the owner access must be accepted and others must be refused. --spec eval(access(), policy1() | undefined | binary() ) -> boolean() | undefined. +-spec eval(access(), policy() | undefined | binary()) -> boolean() | undefined. eval(_, undefined) -> undefined; eval(Access, JSON) when is_binary(JSON) -> case policy_from_json(JSON) of @@ -102,12 +90,13 @@ aggregate_evaluation(Access, [Stmt|Stmts]) -> % @doc semantic validation of policy --spec check_policy(access(), policy1()) -> ok | {error, atom()}. -check_policy(#access_v1{bucket=B} = _Access, - Policy) -> - +-spec check_policy(access(), policy()) -> + ok | {error, {malformed_policy_version, binary()} + | malformed_policy_resource | malformed_policy_action}. +check_policy(#access_v1{bucket = B}, Policy) -> case check_version(Policy) of - false -> {error, {malformed_policy_version, Policy?POLICY.version}}; + false -> + {error, {malformed_policy_version, Policy?POLICY.version}}; true -> case check_all_resources(B, Policy) of false -> {error, malformed_policy_resource}; @@ -123,34 +112,34 @@ check_policy(#access_v1{bucket=B} = _Access, end end. --spec check_version(policy1()) -> boolean(). -check_version(?POLICY{version = ?AMZ_DEFAULT_VERSION}) -> - true; -check_version(?POLICY{version = ?AMZ_POLICY_OLD_VERSION}) -> - true; -check_version(?POLICY{version = Version}) -> - _ = lager:debug("unknown version: ~p", [Version]), - false. +check_version(?POLICY{version = V}) -> + case lists:member(V, [?POLICY_VERSION_2008, + ?POLICY_VERSION_2012, + ?POLICY_VERSION_2020]) of + true -> + true; + false -> + logger:notice("Unknown policy version: ~p", [V]), + false + end. % @doc confirm if forbidden action included in policy % s3:CreateBucket and s3:ListAllMyBuckets are prohibited at S3 --spec check_actions([#statement{}]) -> boolean(). check_actions([]) -> true; check_actions([Stmt|Stmts]) -> case Stmt#statement.action of - '*' -> check_actions(Stmts); + Globbing when is_binary(Globbing) -> + check_actions(Stmts); Actions -> - case not lists:member('s3:CreateBucket', Actions) of + case lists:member('s3:CreateBucket', Actions) orelse + lists:member('s3:ListAllMyBuckets', Actions) of true -> - case not lists:member('s3:ListAllMyBuckets', Actions) of - true -> check_actions(Stmts); - false -> false - end; - false -> false + false; + false -> + check_actions(Stmts) end end. --spec check_principals([#statement{}]) -> boolean(). check_principals([]) -> false; check_principals([Stmt|Stmts]) -> case check_principal(Stmt#statement.principal) of @@ -158,12 +147,18 @@ check_principals([Stmt|Stmts]) -> false -> check_principals(Stmts) end. --spec check_principal(any()) -> boolean(). check_principal('*') -> true; check_principal([]) -> false; -check_principal([{canonical_id, _Id}|_]) -> %% TODO: do we check all canonical ids exist? +check_principal([{canonical_user, _Id}|_]) -> %% TODO: do we check all canonical ids exist? + %% no. Checking that such a user exists now will only create a + %% false sense of consistency. That user can be deleted at a later + %% point while the policy referencing it will continue to exist. + true; +check_principal([{federated, _Id}|_]) -> + true; +check_principal([{service, _Id}|_]) -> true; check_principal([{aws, '*'}|_]) -> true; @@ -171,104 +166,104 @@ check_principal([_|T]) -> check_principal(T). % @doc check if the policy is set to proper bucket by checking arn -check_all_resources(BucketBin, ?POLICY{statement=Stmts} = _Policy) -> +check_all_resources(Bucket, ?POLICY{statement = Stmts}) -> CheckFun = fun(Stmt) -> - check_all_resources(BucketBin, Stmt) + check_all_resources(Bucket, Stmt) end, lists:all(CheckFun, Stmts); -check_all_resources(BucketBin, #statement{resource=Resources} = _Stmt) -> +check_all_resources(Bucket, #statement{resource = Resources}) -> CheckFun = fun(Resource) -> - check_all_resources(BucketBin, Resource) + check_all_resources(Bucket, Resource) end, lists:all(CheckFun, Resources); -check_all_resources(BucketBin, #arn_v1{path=Path} = _Resource) -> - [B|_] = string:tokens(Path, "/"), - B =:= binary_to_list(BucketBin). +check_all_resources(Bucket, #arn_v1{path = Path}) -> + [B|_] = binary:split(Path, <<"/">>), + B =:= Bucket. + --spec reqdata_to_access(#wm_reqdata{}, Target::atom(), ID::binary()|undefined) -> access(). +-spec reqdata_to_access(#wm_reqdata{}, action_target(), binary() | undefined) -> access(). reqdata_to_access(RD, Target, ID) -> Key = case wrq:path_info(object, RD) of undefined -> undefined; RawKey -> list_to_binary(mochiweb_util:unquote(mochiweb_util:unquote(RawKey))) end, - #access_v1{ - method = wrq:method(RD), - target = Target, - id = ID, req = RD, - bucket = list_to_binary(wrq:path_info(bucket, RD)), - key = Key - }. - --spec policy_from_json(JSON::binary()) -> {ok, policy1()} | {error, term()}. + #access_v1{method = wrq:method(RD), + target = Target, + id = ID, + req = RD, + bucket = case wrq:path_info(bucket, RD) of + undefined -> undefined; + B -> list_to_binary(B) + end, + key = Key, + action = riak_cs_wm_utils:aws_service_action(RD, Target) + }. + +-spec policy_from_json(JSON::binary()) -> {ok, policy()} | {error, term()}. policy_from_json(JSON) -> - %% TODO: stop using exception and start some monadic validation and parsing. - case catch(mochijson2:decode(JSON)) of - {struct, Pairs} -> - Version = proplists:get_value(<<"Version">>, Pairs), - ID = proplists:get_value(<<"Id">>, Pairs), - Stmts0 = proplists:get_value(<<"Statement">>, Pairs), - - case catch(lists:map(fun({struct,S})-> - statement_from_pairs(S, #statement{}) - end, Stmts0)) of - - {error, _Reason} = E -> - E; - [] -> - {error, malformed_policy_missing}; - - Stmts -> - case {Version, ID} of - {undefined, <<"undefined">>} -> - {ok, ?POLICY{statement=Stmts}}; - {undefined, _} -> - {ok, ?POLICY{id=ID, statement=Stmts}}; - {_, <<"undefined">>} -> - {ok, ?POLICY{version=Version, statement=Stmts}}; - _ -> - {ok, ?POLICY{id=ID, version=Version, statement=Stmts}} - end - end; - {error, _Reason} = E -> - E; - {'EXIT', {{case_clause, B}, _}} when is_binary(B) -> - %% mochiweb failed to parse the JSON - _ = lager:debug("~p", [B]), + try + case jsx:decode(JSON) of + #{<<"Statement">> := Stmts0} = Map -> + case [statement_from_pairs(maps:to_list(S), #statement{}) || S <- Stmts0] of + [] -> + {error, missing_principal}; + Stmts -> + Version = maps:get(<<"Version">>, Map, ?POLICY_VERSION_2020), + ID = n2u(maps:get(<<"Id">>, Map, null)), + CreationTime = maps:get(<<"CreationTime">>, Map, os:system_time(millisecond)), + {ok, ?POLICY{id = ID, + version = Version, + statement = Stmts, + creation_time = CreationTime}} + end; + #{} -> + logger:warning("Policy document missing required fields: ~s", [JSON]), + {error, malformed_policy_json} + end + catch + throw:{error, SpecificError} -> + logger:notice("Bad Policy JSON (~p): ~s", [SpecificError, JSON]), + {error, SpecificError}; + T:E:ST -> + logger:notice("Malformed policy json: ~p:~p", [T, E]), + ?LOG_DEBUG("Stacktrace: ~p", [ST]), {error, malformed_policy_json} end. - --spec policy_to_json_term(policy1()) -> JSON::binary(). -policy_to_json_term( ?POLICY{ version = Version, - id = ID, statement = Stmts0}) - when Version =:= ?AMZ_POLICY_OLD_VERSION - orelse Version =:= ?AMZ_DEFAULT_VERSION -> +n2u(null) -> undefined; +n2u(A) -> A. + +-spec policy_to_json_term(policy()) -> JSON::binary(). +policy_to_json_term(?POLICY{version = Version, + id = ID, + statement = Stmts0, + creation_time = CreationTime}) + when Version =:= ?POLICY_VERSION_2008; + Version =:= ?POLICY_VERSION_2012; + Version =:= ?POLICY_VERSION_2020 -> Stmts = lists:map(fun statement_to_pairs/1, Stmts0), - % hope no unicode included - Policy = [{"Version", Version},{"Id", ID},{"Statement",Stmts}], - unicode:characters_to_binary(mochijson2:encode(Policy), unicode). - - --spec supported_object_action() -> [s3_object_action()]. -supported_object_action() -> ?SUPPORTED_OBJECT_ACTION. - --spec supported_bucket_action() -> [s3_bucket_action()]. -supported_bucket_action() -> ?SUPPORTED_BUCKET_ACTION. + Policy = #{<<"Version">> => Version, + <<"Id">> => u2n(ID), + <<"Statement">> => Stmts, + <<"CreationTime">> => CreationTime}, + jsx:encode(Policy). +u2n(undefined) -> null; +u2n(A) -> A. %% @doc put required atoms into atom table %% to make policy json parser safer by using erlang:binary_to_existing_atom/2. -spec log_supported_actions() -> ok. -log_supported_actions()-> - _ = lager:info("supported object actions: ~p", - [lists:map(fun atom_to_list/1, supported_object_action())]), - _ = lager:info("supported bucket actions: ~p", - [lists:map(fun atom_to_list/1, supported_bucket_action())]), +log_supported_actions() -> + logger:info("supported object actions: ~p", + [lists:map(fun atom_to_list/1, ?SUPPORTED_OBJECT_ACTIONS)]), + logger:info("supported bucket actions: ~p", + [lists:map(fun atom_to_list/1, ?SUPPORTED_BUCKET_ACTIONS)]), ok. %% @doc Fetch the policy for a bucket --type policy_from_meta_result() :: {'ok', policy()} | {'error', 'policy_undefined'}. +-type policy_from_meta_result() :: {ok, policy()} | {error, policy_undefined}. -type bucket_policy_result() :: policy_from_meta_result() | - {'error', 'notfound'} | - {'error', 'multiple_bucket_owners'}. + {error, notfound} | + {error, multiple_bucket_owners}. -spec fetch_bucket_policy(binary(), riak_client()) -> bucket_policy_result(). fetch_bucket_policy(Bucket, RcPid) -> case riak_cs_bucket:fetch_bucket_object(Bucket, RcPid) of @@ -279,9 +274,8 @@ fetch_bucket_policy(Bucket, RcPid) -> Contents = riakc_obj:get_contents(Obj), bucket_policy_from_contents(Bucket, Contents); {error, Reason} -> - _ = lager:debug("Failed to fetch policy. Bucket ~p " - " does not exist. Reason: ~p", - [Bucket, Reason]), + logger:warning("Failed to fetch policy. Bucket ~p does not exist. Reason: ~p", + [Bucket, Reason]), {error, notfound} end. @@ -300,7 +294,7 @@ bucket_policy(BucketObj) -> %% value. Instead the fact that the bucket has siblings is logged, but the %% condition should be rare so we avoid updating the value at this time. -spec bucket_policy_from_contents(binary(), riakc_obj:contents()) -> - bucket_policy_result(). + bucket_policy_result(). bucket_policy_from_contents(_, [{MD, _}]) -> MetaVals = dict:fetch(?MD_USERMETA, MD), policy_from_meta(MetaVals); @@ -312,7 +306,7 @@ bucket_policy_from_contents(Bucket, Contents) -> resolve_bucket_metadata(UserMetas, UniqueVals). -spec resolve_bucket_metadata(list(riakc_obj:metadata()), - list(riakc_obj:value())) -> bucket_policy_result(). + list(riakc_obj:value())) -> bucket_policy_result(). resolve_bucket_metadata(Metas, [_Val]) -> Policies = [policy_from_meta(M) || M <- Metas], resolve_bucket_policies(Policies); @@ -349,93 +343,76 @@ policy_from_meta([_ | RestMD]) -> %% =================================================================== %% internal API --spec resource_matches(binary(), binary()|undefined|list(), #statement{}) -> boolean(). -resource_matches(_, _, #statement{resource='*'} = _Stmt ) -> true; -resource_matches(BucketBin, KeyBin, #statement{resource=Resources}) +resource_matches(_, _, #statement{resource = '*'} = _Stmt ) -> + true; +resource_matches(Bucket, KeyBin, #statement{resource = Resources}) when KeyBin =:= undefined orelse is_binary(KeyBin) -> - Bucket = binary_to_list(BucketBin), % @TODO in case of using unicode object keys Path0 = case KeyBin of undefined -> Bucket; - _ when is_binary(KeyBin) -> - unicode:characters_to_list(<>, unicode) + <<>> -> + Bucket; + _ -> + <> end, - lists:any(fun(#arn_v1{path="*"}) -> true; - (#arn_v1{path=Path}) -> - case Path of - Bucket -> true; - - %% only prefix matching - Path -> - [B|_] = string:tokens(Path, "*"), - Len = length(B), - B =:= string:substr(Path0, 1, Len) + lists:any(fun(#arn_v1{path = Path}) -> + case binary:last(Path) of + $* -> + PfxSz = size(Path)-1, + if size(Path) < size(Path0) -> + %% only prefix matching + <> = Path, + <> = Path0, + M1 =:= M2; + el/=se -> + false + end; + + _ -> + maybe_drop_last_slash(Path) =:= maybe_drop_last_slash(Path0) end; (_) -> false end, Resources). +maybe_drop_last_slash(A) -> + case binary:last(A) of + $/ -> + maybe_drop_last_slash(binary:part(A, 0, byte_size(A)-1)); + _ -> + A + end. + % functions to eval: -spec eval_statement(access(), #statement{}) -> boolean() | undefined. -eval_statement(#access_v1{method=M, target=T, req=Req, bucket=B, key=K} = _Access, - #statement{effect=E, condition_block=Conds, action=As} = Stmt) -> - {ok, A} = make_action(M, T), +eval_statement(#access_v1{req = Req, + bucket = B, + key = K, + action = A}, + #statement{effect = E, + condition_block = Conds, + action = As, + not_action = NAs} = Stmt) -> ResourceMatch = resource_matches(B, K, Stmt), - IsRelated = (As =:= '*') - orelse (A =:= As) - orelse (is_list(As) andalso lists:member(A, As)), + IsRelated = (lists:member(<<"s3:*">>, As) + orelse (lists:member(<<"s3:Get*">>, As)) + orelse (lists:member(A, As)) andalso not (lists:member(A, NAs) + orelse lists:member(<<"s3:*">>, NAs) + orelse lists:member(<<"s3:Get*">>, NAs)) + ), case {IsRelated, ResourceMatch} of {false, _} -> undefined; {_, false} -> undefined; {true, true} -> Match = lists:all(fun(Cond) -> eval_condition(Req, Cond) end, Conds), case {E, Match} of - {allow, true} -> true; + {allow, true} -> + true; {deny, true} -> false; {_, false} -> undefined %% matches nothing end end. -make_action(Method, Target) -> - case {Method, Target} of - {'PUT', object} -> {ok, 's3:PutObject'}; - {'PUT', object_part} -> {ok, 's3:PutObject'}; - {'PUT', object_acl} -> {ok, 's3:PutObjectAcl'}; - {'PUT', bucket_acl} -> {ok, 's3:PutBucketAcl'}; - {'PUT', bucket_policy} -> {ok, 's3:PutBucketPolicy'}; - {'PUT', bucket_request_payment} -> {ok, 's3:PutBucketRequestPayment'}; - - {'GET', object} -> {ok, 's3:GetObject'}; - {'GET', object_part} -> {ok, 's3:ListMultipartUploadParts'}; - {'GET', object_acl} -> {ok, 's3:GetObjectAcl'}; - {'GET', bucket} -> {ok, 's3:ListBucket'}; - {'GET', no_bucket } -> {ok, 's3:ListAllMyBuckets'}; - {'GET', bucket_acl} -> {ok, 's3:GetBucketAcl'}; - {'GET', bucket_policy} -> {ok, 's3:GetBucketPolicy'}; - {'GET', bucket_location} -> {ok, 's3:GetBucketLocation'}; - {'GET', bucket_request_payment} -> {ok, 's3:GetBucketRequestPayment'}; - {'GET', bucket_uploads} -> {ok, 's3:ListBucketMultipartUploads'}; - - {'DELETE', object} -> {ok, 's3:DeleteObject'}; - {'DELETE', object_part} -> {ok, 's3:AbortMultipartUpload'}; - {'DELETE', bucket} -> {ok, 's3:DeleteBucket'}; - {'DELETE', bucket_policy} -> {ok, 's3:DeleteBucketPolicy'}; - - {'HEAD', object} -> {ok, 's3:GetObject'}; % no HeadObject - - %% PUT Object includes POST Object, - %% including Initiate Multipart Upload, Upload Part, Complete Multipart Upload - {'POST', object} -> {ok, 's3:PutObject'}; - {'POST', object_part} -> {ok, 's3:PutObject'}; - - %% same as {'GET' bucket} - {'HEAD', bucket} -> {ok, 's3:ListBucket'}; - - %% 400 (MalformedPolicy): Policy has invalid action - {'PUT', bucket} -> {ok, 's3:CreateBucket'}; - - {'HEAD', _} -> {error, no_action} - end. eval_condition(Req, {AtomKey, Cond}) -> case AtomKey of @@ -550,11 +527,9 @@ ipv4_band({A,B,C,D}, {E,F,G,H}) -> ipv4_eq({A,B,C,D}, {E,F,G,H}) -> (A =:= E) andalso (B =:= F) andalso (C =:= G) andalso (D =:= H). --type json_term() :: [{binary(), json_term()}] | integer() | float() | binary(). % =========================================== % functions to convert policy record to JSON: --spec statement_to_pairs(#statement{}) -> [{binary(), any()}]. statement_to_pairs(#statement{sid=Sid, effect=E, principal=P, action=A, not_action=NA, resource=R, condition_block=Cs})-> AtomE = case E of @@ -562,13 +537,14 @@ statement_to_pairs(#statement{sid=Sid, effect=E, principal=P, action=A, deny -> <<"Deny">> end, Conds = lists:map(fun condition_block_from_condition_pair/1, Cs), - [{<<"Sid">>, Sid}, {<<"Effect">>, AtomE}, - {<<"Principal">>, print_principal(P)}, - {<<"Action">>, A}, {<<"NotAction">>, NA}, - {<<"Resource">>, print_arns(R)}, - {<<"Condition">>, Conds}]. + #{<<"Sid">> => Sid, + <<"Effect">> => AtomE, + <<"Principal">> => print_principal(P), + <<"Action">> => A, + <<"NotAction">> => NA, + <<"Resource">> => print_arns(R), + <<"Condition">> => Conds}. --spec condition_block_from_condition_pair(condition_pair()) -> {binary(), list()}. condition_block_from_condition_pair({AtomKey, Conds})-> Fun = fun({'aws:SourceIp', IPs}) when is_list(IPs) -> {'aws:SourceIp', [print_ip(IP) || IP <- IPs]}; @@ -576,7 +552,7 @@ condition_block_from_condition_pair({AtomKey, Conds})-> {'aws:SourceIp', print_ip(IP)}; (Cond) -> Cond end, - {atom_to_binary(AtomKey, latin1), lists:map(Fun, Conds)}. + {atom_to_binary(AtomKey, latin1), lists:map(Fun, Conds)}. % inverse of parse_ip/1 -spec print_ip({inet:ip_address(), inet:ip_address()}) -> binary(). @@ -620,107 +596,122 @@ int_to_prefix(0) -> 0. % =========================================================== % functions to convert (parse) JSON to create a policy record: --spec statement_from_pairs(list(), #statement{})-> #statement{}. statement_from_pairs([], Stmt) -> case Stmt#statement.principal of [] -> %% TODO: there're a lot to do: S3 describes the %% details of error, in xml. with , and - throw({error, malformed_policy_missing}); + throw({error, missing_principal}); _ -> Stmt end; -statement_from_pairs([{<<"Sid">>, <<>>} |_], _) -> +statement_from_pairs([{<<"Sid">>, <<>>} |_], _) -> throw({error, malformed_policy_resource}); -statement_from_pairs([{<<"Sid">>,Sid} |T], Stmt) -> - statement_from_pairs(T, Stmt#statement{sid=Sid}); +statement_from_pairs([{<<"Sid">>, Sid} | T], Stmt) -> + statement_from_pairs(T, Stmt#statement{sid = Sid}); -statement_from_pairs([{<<"Effect">>,<<"Allow">>}|T], Stmt) -> - statement_from_pairs(T, Stmt#statement{effect=allow}); -statement_from_pairs([{<<"Effect">>,<<"Deny">>}|T], Stmt) -> - statement_from_pairs(T, Stmt#statement{effect=deny}); +statement_from_pairs([{<<"Effect">>, <<"Allow">>} | T], Stmt) -> + statement_from_pairs(T, Stmt#statement{effect = allow}); +statement_from_pairs([{<<"Effect">>, <<"Deny">>} | T], Stmt) -> + statement_from_pairs(T, Stmt#statement{effect = deny}); -statement_from_pairs([{<<"Principal">>,P} |T], Stmt) -> +statement_from_pairs([{<<"Principal">>, P} | T], Stmt) -> Principal = parse_principal(P), statement_from_pairs(T, Stmt#statement{principal=Principal}); -statement_from_pairs([{<<"Action">>,As} |T], Stmt) -> +statement_from_pairs([{<<"Action">>, As} | T], Stmt) -> Atoms = case As of As when is_list(As) -> lists:map(fun binary_to_action/1, As); Bin when is_binary(Bin) -> - binary_to_existing_atom(Bin, latin1) + [binary_to_action(Bin)] end, statement_from_pairs(T, Stmt#statement{action=Atoms}); -statement_from_pairs([{<<"NotAction">>,As} |T], Stmt) when is_list(As) -> - Atoms = lists:map(fun binary_to_action/1, As), - statement_from_pairs(T, Stmt#statement{not_action=Atoms}); +statement_from_pairs([{<<"NotAction">>, As} | T], Stmt) -> + Atoms = case As of + As when is_list(As) -> + lists:map(fun binary_to_action/1, As); + Bin when is_binary(Bin) -> + [binary_to_action(Bin)] + end, + statement_from_pairs(T, Stmt#statement{not_action = Atoms}); -statement_from_pairs([{<<"Resource">>,R} |T], Stmt) -> +statement_from_pairs([{<<"Resource">>, R} | T], Stmt) -> case parse_arns(R) of {ok, ARN} -> - statement_from_pairs(T, Stmt#statement{resource=ARN}); + statement_from_pairs(T, Stmt#statement{resource = ARN}); {error, _} -> throw({error, malformed_policy_resource}) end; -statement_from_pairs([{<<"Condition">>,[]} |T], Stmt) -> +statement_from_pairs([{<<"Condition">>, []} | T], Stmt) -> %% empty conditions statement_from_pairs(T, Stmt#statement{condition_block=[]}); -statement_from_pairs([{<<"Condition">>,{struct, Cs}} |T], Stmt) -> - Conditions = lists:map(fun condition_block_to_condition_pair/1, Cs), - statement_from_pairs(T, Stmt#statement{condition_block=Conditions}). +statement_from_pairs([{<<"Condition">>, Cs} | T], Stmt) -> + Conditions = lists:map(fun condition_block_to_condition_pair/1, maps:to_list(Cs)), + statement_from_pairs(T, Stmt#statement{condition_block = Conditions}). --spec binary_to_action(binary()) -> s3_object_action() | s3_bucket_action(). -binary_to_action(Bin)-> - binary_to_existing_atom(Bin, latin1). +binary_to_action(Bin) -> + A = binary_to_atom(Bin, latin1), + case lists:member(A, ?SUPPORTED_ACTIONS) of + true -> + A; + false -> + case re:run(Bin, <<"((s3|iam|sts):|)[a-zA-Z]*\\*">>) of + {match, _} -> + Bin; + nomatch -> + throw({error, malformed_policy_action}) + end + end. -% @TODO: error processing --spec parse_principal(binary() | [binary()]) -> principal(). +parse_principal([]) -> throw({error, missing_principal}); parse_principal(<<"*">>) -> '*'; -parse_principal({struct, List}) when is_list(List) -> - parse_principals(List, []); -parse_principal([{struct, List}]) when is_list(List) -> - parse_principals(List, []). - - -parse_principals([], Principal) -> Principal; -parse_principals([{<<"AWS">>,[<<"*">>]}|TL], Principal) -> - parse_principals(TL, [{aws, '*'}|Principal]); -parse_principals([{<<"AWS">>,<<"*">>}|TL], Principal) -> - parse_principals(TL, [{aws, '*'}|Principal]). -%% TODO: CanonicalUser as principal is not yet supported, -%% Because to use at Riak CS, key_id is better to specify user, because -%% getting canonical ID from Riak is not enough efficient -%% ``` -%% parse_principals([{<<"CanonicalUser">>,CanonicalIds}|TL], Principal) -> -%% case CanonicalIds of -%% [H|_] when is_binary(H) -> -%% %% in case of list of strings ["CAFEBABE...", "BA6DAAD..."] -%% CanonicalUsers = lists:map(fun(CanonicalId) -> -%% {canonical_id, binary_to_list(CanonicalId)} -%% end, -%% CanonicalIds), -%% parse_principals(TL, CanonicalUsers ++ Principal); -%% CanonicalId when is_binary(CanonicalId) -> -%% %% in case of just a string ["CAFEBABE..."] -%% parse_principals(TL, [{canonical_id, binary_to_list(CanonicalId)}|Principal]) -%% end. -%% ''' +parse_principal(#{} = A) -> + parse_principals(maps:to_list(A), []). + +-define(IS_VALID_PRINCIPAL(P), + P =:= <<"AWS">> orelse + P =:= <<"Federated">> orelse + P =:= <<"CanonicalUser">> orelse + P =:= <<"Service">>). +-define(IS_ASTERISK(A), + A =:= <<"*">> orelse A =:= [<<"*">>]). +parse_principals([], Q) -> Q; +parse_principals([{P, Id} | TL], Q) + when ?IS_VALID_PRINCIPAL(P) -> + parse_principals(TL, [{s2principal(P), s2id(Id)} | Q]). +s2principal(<<"AWS">>) -> aws; +s2principal(<<"Federated">>) -> federated; +s2principal(<<"Service">>) -> service; +s2principal(<<"CanonicalUser">>) -> canonical_user; +s2principal(Invalid) -> throw({error, {invalid_principal, Invalid}}). + +s2id(<<"*">>) -> '*'; +s2id(Ids) when is_list(Ids) -> + [s2id_flat(Id) || Id <- Ids]; +s2id(Id) -> Id. + +s2id_flat(Id) when is_binary(Id) -> Id; +s2id_flat(A) -> throw({error, {invalid_principal_id, A}}). print_principal('*') -> <<"*">>; -print_principal({aws, '*'}) -> - [{<<"AWS">>, <<"*">>}]; -%%print_principal({canonical_id, Id}) -> -%% {"CanonicalUser", Id}; +print_principal({P, A}) -> + [{principal2s(P), a2s(A)}]; print_principal(Principals) when is_list(Principals) -> - PrintFun = fun(Principal) -> print_principal(Principal) end, - lists:map(PrintFun, Principals). + lists:flatten(lists:map(fun print_principal/1, Principals)). +principal2s(aws) -> <<"AWS">>; +principal2s(federated) -> <<"Federated">>; +principal2s(service) -> <<"Service">>; +principal2s(canonical_user) -> <<"CanonicalUser">>. +a2s('*') -> <<"*">>; +a2s(Id) -> Id. --spec parse_arns(binary()|[binary()]) -> {ok, arn()} | {error, bad_arn}. + +-spec parse_arns(binary()|[binary()]) -> {ok, [arn()]} | {error, bad_arn}. parse_arns(<<"*">>) -> {ok, '*'}; parse_arns(Bin) when is_binary(Bin) -> Str = binary_to_list(Bin), @@ -734,7 +725,7 @@ parse_arns(List) when is_list(List) -> {ok, ARN} -> {ok, ARN ++ Acc0}; {error, bad_arn} -> {error, bad_arn} end; - ({error, bad_arn}, _) -> {error, bad_arn} + (_, {error, bad_arn}) -> {error, bad_arn} end, case lists:foldl(AccFun, {ok, []}, List) of {ok, ARNs} -> {ok, lists:reverse(ARNs)}; @@ -747,9 +738,9 @@ parse_arn(Str) -> ["arn", "aws", "s3", Region, ID, Path] -> {ok, #arn_v1{provider = aws, service = s3, - region = Region, + region = list_to_binary(Region), id = list_to_binary(ID), - path = Path}}; + path = list_to_binary(Path)}}; _ -> {error, bad_arn} end. @@ -765,20 +756,20 @@ my_split(Ch, [Ch0|TL], Acc, L) -> -spec print_arns( '*'|[arn()]) -> [binary()] | binary(). print_arns('*') -> <<"*">>; -print_arns(#arn_v1{region=R, id=ID, path=Path} = _ARN) -> - StringPath = unicode:characters_to_list(Path), - StringID = binary_to_list(ID), - list_to_binary(string:join(["arn", "aws", "s3", R, StringID, StringPath], ":")); +print_arns(#arn_v1{region = R, id = ID, path = Path} = _ARN) -> + list_to_binary(string:join(["arn", "aws", "s3", + binary_to_list(R), + binary_to_list(ID), + unicode:characters_to_list(Path)], ":")); print_arns(ARNs) when is_list(ARNs)-> PrintARN = fun(ARN) -> print_arns(ARN) end, lists:map(PrintARN, ARNs). --spec condition_block_to_condition_pair({binary(), {struct, json_term()}}) -> condition_pair(). -condition_block_to_condition_pair({Key,{struct,Cond}}) -> +condition_block_to_condition_pair({Key, Cond}) -> % all key should be defined in stanchion.hrl AtomKey = binary_to_existing_atom(Key, latin1), - {AtomKey, lists:map(fun condition_/1, Cond)}. + {AtomKey, lists:map(fun condition_/1, maps:to_list(Cond))}. % TODO: more strict condition - currenttime only for date_condition, and so on condition_({<<"aws:CurrentTime">>, Bin}) when is_binary(Bin) -> @@ -883,9 +874,9 @@ principal_eq({aws, H}, [{aws, H}]) -> true; principal_eq([{aws, H}], {aws, H}) -> true; principal_eq(LHS, RHS) -> LHS =:= RHS. -resource_eq(?ARN{} = LHS, [?ARN{} = LHS]) -> true; -resource_eq([?ARN{} = LHS], ?ARN{} = LHS) -> true; -resource_eq(LHS, RHS) -> LHS =:= RHS. +resource_eq(?S3_ARN{} = LHS, [?S3_ARN{} = LHS]) -> true; +resource_eq([?S3_ARN{} = LHS], ?S3_ARN{} = LHS) -> true; +resource_eq(LHS, RHS) -> LHS =:= RHS. statement_eq(LHS, RHS) -> (LHS#statement.sid =:= RHS#statement.sid) @@ -902,4 +893,20 @@ statement_eq(LHS, RHS) -> andalso (LHS#statement.condition_block =:= RHS#statement.condition_block). +valid_arn_parse_test() -> + A1 = <<"arn:aws:s3::123456789012:user/Development/product_1234/*">>, + E1 = ?S3_ARN{provider = aws, + service = s3, + region = <<>>, + id = <<"123456789012">>, + path = <<"user/Development/product_1234/*">>}, + AAEE = [{{ok, [E1]}, A1}, + {{ok, [E1]}, [A1]}, + {{ok, [E1, E1]}, [A1, A1]} + ], + lists:foreach(fun({E, A}) -> ?assertEqual(E, parse_arns(A)) end, AAEE). + +invalid_arn_parse_test() -> + ?assertEqual({error, bad_arn}, parse_arns([<<"arn:aws:s2::123456789012:user/Development/product_1234/*">>])). + -endif. diff --git a/apps/riak_cs/src/riak_cs_aws_response.erl b/apps/riak_cs/src/riak_cs_aws_response.erl new file mode 100644 index 000000000..8060a5c52 --- /dev/null +++ b/apps/riak_cs/src/riak_cs_aws_response.erl @@ -0,0 +1,445 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_aws_response). + +-export([api_error/3, + status_code/1, + respond/3, + respond/4, + error_message/1, + error_code/1, + copy_object_response/3, + copy_part_response/3, + no_such_upload_response/3, + invalid_digest_response/3]). + +-include("riak_cs.hrl"). +-include_lib("xmerl/include/xmerl.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-spec error_message(reportable_error_reason()) -> string(). +error_message(invalid_access_key_id) -> + "The AWS Access Key Id you provided does not exist in our records."; +error_message(invalid_email_address) -> + "The email address you provided is not a valid."; +error_message(user_has_buckets) -> + "User has buckets and cannot be deleted."; +error_message(user_has_attached_policies) -> + "User has attached policies and cannot be deleted."; +error_message(policy_in_use) -> + "Policy is attached to other entities and cannot be deleted."; +error_message(bad_signature) -> + "The request signature that the server calculated does not match the signature that you provided."; +error_message(access_denied) -> + "Access Denied"; +error_message(user_disabled) -> + "User is disabled"; +error_message(copy_source_access_denied) -> + "Access Denied"; +error_message(reqtime_tooskewed) -> + "The difference between the request time and the current time is too large."; +error_message(bucket_not_empty) -> + "The bucket you tried to delete is not empty."; +error_message(bucket_already_exists) -> + "The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again."; +error_message(toomanybuckets) -> + "You have attempted to create more buckets than allowed"; +error_message({key_too_long, _}) -> + "Your key is too long"; +error_message(user_already_exists) -> + "The specified email address has already been registered. Email addresses must be unique among all users of the system. Please try again with a different email address."; +error_message(entity_too_large) -> + "Your proposed upload exceeds the maximum allowed object size."; +error_message(entity_too_small) -> + "Your proposed upload is smaller than the minimum allowed object size. Each part must be at least 5 MB in size, except the last part."; +error_message(invalid_user_update) -> + "The user update you requested was invalid."; +error_message(no_updates_for_federated_users) -> + "Federated users cannot be updated."; +error_message(no_bucket_create_for_federated_users) -> + "Federated users cannot create buckets."; +error_message(invalid_role_parameters) -> + "Incomplete or invalid role parameters."; +error_message(no_such_user) -> + "The specified user does not exist."; +error_message(no_such_bucket) -> + "The specified bucket does not exist."; +error_message({riak_connect_failed, Reason}) -> + io_lib:format("Unable to establish connection to Riak. Reason: ~p", [Reason]); +error_message(admin_key_undefined) -> "Please reduce your request rate."; +error_message(admin_secret_undefined) -> "Please reduce your request rate."; +error_message(bucket_owner_unavailable) -> "The user record for the bucket owner was unavailable. Try again later."; +error_message(econnrefused) -> "Please reduce your request rate."; +error_message(malformed_policy_json) -> "JSON parsing error"; +error_message({malformed_policy_version, Version}) -> + io_lib:format("Document is invalid: Invalid Version ~s", [Version]); +error_message({auth_not_supported, AuthType}) -> + io_lib:format("The authorization mechanism you have provided (~s) is not supported.", [AuthType]); +error_message(malformed_policy_missing) -> "Policy is missing required element"; +error_message(malformed_policy_resource) -> "Policy has invalid resource"; +error_message(malformed_policy_principal) -> "Invalid principal in policy"; +error_message(malformed_policy_action) -> "Policy has invalid action"; +error_message(malformed_policy_condition) -> "Policy has invalid condition"; +error_message(no_such_key) -> "The specified key does not exist."; +error_message(no_copy_source_key) -> "The specified key does not exist."; +error_message(no_such_bucket_policy) -> "The specified bucket does not have a bucket policy."; +error_message(no_such_upload) -> + "The specified upload does not exist. The upload ID may be invalid, " + "or the upload may have been aborted or completed."; +error_message(invalid_digest) -> + "The Content-MD5 you specified was invalid."; +error_message(bad_request) -> "Bad Request"; +error_message(invalid_argument) -> "Invalid Argument"; +error_message({invalid_argument, "x-amz-metadata-directive"}) -> + "Unknown metadata directive."; +error_message(unresolved_grant_email) -> "The e-mail address you provided does not match any account on record."; +error_message(unresolved_grant_canonical_id) -> "The Canonical Id you provided does not match any account on record."; +error_message(invalid_range) -> "The requested range is not satisfiable"; +error_message(invalid_bucket_name) -> "The specified bucket is not valid."; +error_message(invalid_part_number) -> "Part number must be an integer between 1 and 10000, inclusive"; +error_message(unexpected_content) -> "This request does not support content"; +error_message(conflicting_grantee_canonical_id) -> "ACL grantee with Email and defined ID that does not match user's canonical_id on record"; +error_message(canned_acl_and_header_grant) -> "Specifying both Canned ACLs and Header Grants is not allowed"; +error_message(malformed_xml) -> "The XML you provided was not well-formed or did not validate against our published schema"; +error_message(no_such_policy) -> "No such policy"; +error_message(no_such_role) -> "No such role"; +error_message(no_such_saml_provider) -> "No such SAML provider"; +error_message(role_already_exists) -> "Role already exists"; +error_message(policy_already_exists) -> "Policy already exists"; +error_message(policy_not_attachable) -> "Service role policies can only be attached to the service-linked role for that service"; +error_message(unmodifiable_entity) -> "Service-linked roles are protected"; +error_message(saml_provider_already_exists) -> "SAML provider already exists"; +error_message(invalid_metadata_document) -> "IdP provided invalid or unacceptable metadata document"; +error_message(idp_rejected_claim) -> "The IdP reported that authentication failed, or the certificate in the SAML assertion is invalid"; +error_message(remaining_multipart_upload) -> "Concurrent multipart upload initiation detected. Please stop it to delete bucket."; +error_message(disconnected) -> "Please contact administrator."; +error_message(stanchion_recovery_failure) -> "Service is temporarily unavailable because the node running stanchion is unreachable. Please report this to your administrator."; +error_message(invalid_action) -> "This Action is invalid or not yet supported"; +error_message(invalid_parameter_value) -> "Unacceptable parameter value"; +error_message(missing_parameter) -> "Missing parameter"; +error_message({unsatisfied_constraint, Constraint}) -> + io_lib:format("Unable to complete operation due to ~s constraint violation.", [Constraint]); +error_message(not_implemented) -> "A request you provided implies functionality that is not implemented"; +error_message(ErrorName) -> + logger:warning("Unknown error: ~p", [ErrorName]), + "Please reduce your request rate.". + +-spec error_code(reportable_error_reason()) -> string(). +error_code(invalid_access_key_id) -> "InvalidAccessKeyId"; +error_code(user_has_buckets) -> "DeleteConflict"; +error_code(user_has_attached_policies) -> "DeleteConflict"; +error_code(policy_in_use) -> "DeleteConflict"; +error_code(bad_signature) -> "SignatureDoesNotMatch"; +error_code(access_denied) -> "AccessDenied"; +error_code(user_disabled) -> "AccessDenied"; +error_code(copy_source_access_denied) -> "AccessDenied"; +error_code(reqtime_tooskewed) -> "RequestTimeTooSkewed"; +error_code(bucket_not_empty) -> "BucketNotEmpty"; +error_code(bucket_already_exists) -> "BucketAlreadyExists"; +error_code(toomanybuckets) -> "TooManyBuckets"; +error_code({key_too_long, _}) -> "KeyTooLongError"; +error_code(user_already_exists) -> "UserAlreadyExists"; +error_code(entity_too_large) -> "EntityTooLarge"; +error_code(entity_too_small) -> "EntityTooSmall"; +error_code(bad_etag) -> "InvalidPart"; +error_code(bad_etag_order) -> "InvalidPartOrder"; +error_code(invalid_user_update) -> "InvalidUserUpdate"; +error_code(no_updates_for_federated_users) -> "InvalidUserUpdate"; +error_code(no_bucket_create_for_federated_users) -> "NoBucketCreateForFederatedUsers"; +error_code(no_such_user) -> "NoSuchUser"; +error_code(no_such_bucket) -> "NoSuchBucket"; +error_code(no_such_key) -> "NoSuchKey"; +error_code(no_copy_source_key) -> "NoSuchKey"; +error_code({riak_connect_failed, _}) -> "RiakConnectFailed"; +error_code({unsatisfied_constraint, _}) -> "UnsatisfiedConstraint"; +error_code(admin_key_undefined) -> "ServiceUnavailable"; +error_code(admin_secret_undefined) -> "ServiceUnavailable"; +error_code(bucket_owner_unavailable) -> "ServiceUnavailable"; +error_code(econnrefused) -> "ServiceUnavailable"; +error_code(malformed_policy_json) -> "MalformedPolicy"; +error_code(malformed_policy_missing) -> "MalformedPolicy"; +error_code({malformed_policy_version, _}) -> "MalformedPolicy"; +error_code({auth_not_supported, _}) -> "InvalidRequest"; +error_code(malformed_policy_resource) -> "MalformedPolicy"; +error_code(malformed_policy_principal) -> "MalformedPolicy"; +error_code(malformed_policy_action) -> "MalformedPolicy"; +error_code(malformed_policy_condition) -> "MalformedPolicy"; +error_code(no_such_bucket_policy) -> "NoSuchBucketPolicy"; +error_code(no_such_upload) -> "NoSuchUpload"; +error_code(invalid_digest) -> "InvalidDigest"; +error_code(bad_request) -> "BadRequest"; +error_code(invalid_argument) -> "InvalidArgument"; +error_code(invalid_range) -> "InvalidRange"; +error_code(invalid_bucket_name) -> "InvalidBucketName"; +error_code(invalid_part_number) -> "InvalidArgument"; +error_code(unresolved_grant_email) -> "UnresolvableGrantByEmailAddress"; +error_code(unresolved_grant_canonical_id) -> "UnresolvableGrantByCanonicalId"; +error_code(unexpected_content) -> "UnexpectedContent"; +error_code(conflicting_grantee_canonical_id) -> "ConflictingGranteeCanonicalId"; +error_code(canned_acl_and_header_grant) -> "InvalidRequest"; +error_code(malformed_acl_error) -> "MalformedACLError"; +error_code(malformed_xml) -> "MalformedXML"; +error_code(no_such_policy) -> "NoSuchEntity"; +error_code(no_such_role) -> "NoSuchEntity"; +error_code(no_such_saml_provider) -> "NoSuchEntity"; +error_code(role_already_exists) -> "EntityAlreadyExists"; +error_code(policy_already_exists) -> "EntityAlreadyExists"; +error_code(policy_not_attachable) -> "PolicyNotAttachable"; +error_code(unmodifiable_entity) -> "UnmodifiableEntity"; +error_code(saml_provider_already_exists) -> "EntityAlreadyExists"; +error_code(invalid_metadata_document) -> "InvalidInput"; +error_code(idp_rejected_claim) -> "IDPRejectedClaim"; +error_code(remaining_multipart_upload) -> "MultipartUploadRemaining"; +error_code(disconnected) -> "ServiceUnavailable"; +error_code(stanchion_recovery_failure) -> "ServiceDegraded"; +error_code(invalid_action) -> "InvalidAction"; +error_code(invalid_parameter_value) -> "InvalidParameterValue"; +error_code(missing_parameter) -> "MissingParameter"; +error_code(not_implemented) -> "NotImplemented"; +error_code(ErrorName) -> + logger:warning("Unknown error: ~p", [ErrorName]), + "ServiceUnavailable". + +%% These should match: +%% http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html + +-spec status_code(reportable_error_reason()) -> pos_integer(). +status_code(invalid_access_key_id) -> 403; +status_code(invalid_email_address) -> 400; +status_code(user_has_buckets) -> 409; +status_code(user_has_attached_policies) -> 409; +status_code(policy_in_use) -> 409; +status_code(bad_signature) -> 403; +status_code(access_denied) -> 403; +status_code(user_disabled) -> 403; +status_code(copy_source_access_denied) -> 403; +status_code(reqtime_tooskewed) -> 403; +status_code(bucket_not_empty) -> 409; +status_code(bucket_already_exists) -> 409; +status_code(user_already_exists) -> 409; +status_code(toomanybuckets) -> 400; +status_code({key_too_long, _}) -> 400; +%% yes, 400, really, not 413 +status_code(entity_too_large) -> 400; +status_code(entity_too_small) -> 400; +status_code(bad_etag) -> 400; +status_code(bad_etag_order) -> 400; +status_code(invalid_user_update) -> 400; +status_code(no_updates_for_federated_users) -> 400; +status_code(no_bucket_create_for_federated_users) -> 400; +status_code(no_such_user) -> 404; +status_code(no_such_bucket) -> 404; +status_code(no_such_key) -> 404; +status_code(no_copy_source_key) -> 404; +status_code(stanchion_recovery_failure) -> 503; +status_code({riak_connect_failed, _}) -> 503; +status_code(admin_key_undefined) -> 503; +status_code(admin_secret_undefined) -> 503; +status_code(bucket_owner_unavailable) -> 503; +status_code(multiple_bucket_owners) -> 503; +status_code(econnrefused) -> 503; +status_code(unsatisfied_constraint) -> 503; +status_code(malformed_policy_json) -> 400; +status_code({malformed_policy_version, _}) -> 400; +status_code(malformed_policy_missing) -> 400; +status_code(malformed_policy_resource) -> 400; +status_code(malformed_policy_principal) -> 400; +status_code(malformed_policy_action) -> 400; +status_code(malformed_policy_condition) -> 400; +status_code({auth_not_supported, _}) -> 400; +status_code(no_such_bucket_policy) -> 404; +status_code(no_such_upload) -> 404; +status_code(invalid_digest) -> 400; +status_code(bad_request) -> 400; +status_code(invalid_argument) -> 400; +status_code(unresolved_grant_canonical_id) -> 400; +status_code(unresolved_grant_display_name) -> 400; +status_code(invalid_range) -> 416; +status_code(invalid_bucket_name) -> 400; +status_code(invalid_part_number) -> 400; +status_code(unexpected_content) -> 400; +status_code(conflicting_grantee_canonical_id) -> 400; +status_code(canned_acl_and_header_grant) -> 400; +status_code(malformed_acl_error) -> 400; +status_code(malformed_xml) -> 400; +status_code(no_such_policy) -> 404; +status_code(no_such_role) -> 404; +status_code(no_such_saml_provider) -> 404; +status_code(role_already_exists) -> 409; +status_code(policy_already_exists) -> 409; +status_code(policy_not_attachable) -> 400; +status_code(unmodifiable_entity) -> 400; +status_code(saml_provider_already_exists) -> 409; +status_code(invalid_metadata_document) -> 400; +status_code(idp_rejected_claim) -> 403; +status_code(remaining_multipart_upload) -> 409; +status_code(disconnected) -> 500; +status_code(invalid_action) -> 400; +status_code(invalid_parameter_value) -> 400; +status_code(missing_parameter) -> 400; +status_code(not_implemented) -> 501; +status_code(ErrorName) -> + logger:warning("Unknown error: ~p", [ErrorName]), + 503. + +-spec respond(term(), #wm_reqdata{}, #rcs_web_context{}) -> + {binary(), #wm_reqdata{}, #rcs_web_context{}}. +respond(?LBRESP{} = Response, RD, Ctx) -> + {riak_cs_xml:to_xml(Response), RD, Ctx}; +respond({ok, ?LORESP{} = Response}, RD, Ctx) -> + {riak_cs_xml:to_xml(Response), RD, Ctx}; +respond({ok, ?LOVRESP{} = Response}, RD, Ctx) -> + {riak_cs_xml:to_xml(Response), RD, Ctx}; +respond({error, _} = Error, RD, Ctx) -> + api_error(Error, RD, Ctx). + +respond(404 = _StatusCode, Body, ReqData, Ctx) -> + respond({404, "Not Found"}, Body, ReqData, Ctx); +respond(StatusCode, Body, ReqData, Ctx) -> + UpdReqData = wrq:set_resp_body(Body, + wrq:set_resp_header("Content-Type", + ?XML_TYPE, + ReqData)), + {{halt, StatusCode}, UpdReqData, Ctx}. + +api_error({toomanybuckets, Current, BucketLimit}, RD, Ctx) -> + toomanybuckets_response(Current, BucketLimit, RD, Ctx); +api_error({invalid_argument, Name, Value}, RD, Ctx) -> + invalid_argument_response(Name, Value, RD, Ctx); +api_error({key_too_long, Len}, RD, Ctx) -> + key_too_long(Len, RD, Ctx); +api_error(stanchion_recovery_failure, RD, Ctx) -> + stanchion_recovery_failure_response(RD, Ctx); +api_error({error, Reason}, RD, Ctx) -> + api_error(Reason, RD, Ctx); +api_error(Tag, RD, Ctx) -> + error_response(Tag, + error_code(Tag), + error_message(Tag), + RD, + Ctx). + +error_response(Tag, Code, Message, RD, Ctx) -> + XmlDoc = [{'Error', [{'Code', [Code]}, + {'Message', [Message]} + ] ++ common_response_items(Tag, RD, Ctx)} + ], + respond(status_code(Tag), riak_cs_xml:to_xml(XmlDoc), RD, Ctx). + + +toomanybuckets_response(Current, BucketLimit, RD, Ctx) -> + XmlDoc = [{'Error', [{'Code', [error_code(toomanybuckets)]}, + {'Message', [error_message(toomanybuckets)]}, + {'CurrentNumberOfBuckets', [Current]}, + {'AllowedNumberOfBuckets', [BucketLimit]} + ] ++ common_response_items(toomanybuckets, RD, Ctx)} + ], + respond(status_code(toomanybuckets), riak_cs_xml:to_xml(XmlDoc), RD, Ctx). + +invalid_argument_response(Name, Value, RD, Ctx) -> + XmlDoc = [{'Error', [{'Code', [error_code(invalid_argument)]}, + {'Message', [error_message({invalid_argument, Name})]}, + {'ArgumentName', [Name]}, + {'ArgumentValue', [Value]} + ] ++ common_response_items(invalid_argument, RD, Ctx)} + ], + respond(status_code(invalid_argument), riak_cs_xml:to_xml(XmlDoc), RD, Ctx). + +key_too_long(Len, RD, Ctx) -> + XmlDoc = [{'Error', [{'Code', [error_code({key_too_long, Len})]}, + {'Message', [error_message({key_too_long, Len})]}, + {'Size', [Len]}, + {'MaxSizeAllowed', [riak_cs_config:max_key_length()]} + ] ++ common_response_items(key_too_long, RD, Ctx)} + ], + respond(status_code(invalid_argument), riak_cs_xml:to_xml(XmlDoc), RD, Ctx). + +stanchion_recovery_failure_response(RD, Ctx) -> + XmlDoc = [{'Error', [{'Code', [error_code(stanchion_recovery_failure)]}, + {'Message', [error_message(stanchion_recovery_failure)]} + ] ++ common_response_items(stanchion_recovery_failure, RD, Ctx)} + ], + respond(status_code(invalid_argument), riak_cs_xml:to_xml(XmlDoc), RD, Ctx). + +copy_object_response(Manifest, RD, Ctx) -> + copy_response(Manifest, 'CopyObjectResult', RD, Ctx). + +copy_part_response(Manifest, RD, Ctx) -> + copy_response(Manifest, 'CopyPartResult', RD, Ctx). + +copy_response(Manifest, TagName, RD, Ctx) -> + LastModified = rts:iso8601_s(Manifest?MANIFEST.write_start_time), + ETag = riak_cs_manifest:etag(Manifest), + XmlDoc = [{TagName, [{'LastModified', [LastModified]}, + {'ETag', [ETag]} + ] ++ common_response_items(TagName, RD, Ctx)} + ], + respond(200, riak_cs_xml:to_xml(XmlDoc), RD, Ctx). + + +no_such_upload_response(InternalUploadId, RD, Ctx) -> + UploadId = case InternalUploadId of + {raw, ReqUploadId} -> ReqUploadId; + _ -> base64url:encode(InternalUploadId) + end, + XmlDoc = [{'Error', [{'Code', [error_code(no_such_upload)]}, + {'Message', [error_message(no_such_upload)]}, + {'UploadId', [UploadId]} + ] ++ common_response_items(no_such_upload, RD, Ctx)} + ], + respond(status_code(no_such_upload), riak_cs_xml:to_xml(XmlDoc), RD, Ctx). + +invalid_digest_response(ContentMd5, RD, Ctx) -> + XmlDoc = [{'Error', [{'Code', [error_code(invalid_digest)]}, + {'Message', [error_message(invalid_digest)]}, + {'Content-MD5', [ContentMd5]} + ] ++ common_response_items(invalid_digest, RD, Ctx)} + ], + respond(status_code(invalid_digest), riak_cs_xml:to_xml(XmlDoc), RD, Ctx). + + +common_response_items(Tag, RD, #rcs_web_context{request_id = RequestId, + user = User}) -> + [{'RequestId', [binary_to_list(RequestId)]}, + {'Resource', [error_resource(Tag, RD)]}, + {'AWSAccessKeyId', [user_access_key(User)]}, + {'HostId', [riak_cs_config:host_id()]} + ]. + +error_resource(Tag, RD) + when Tag =:= no_copy_source_key; + Tag =:= copy_source_access_denied-> + {B, K, V} = riak_cs_copy_object:get_copy_source(RD), + case V of + ?LFS_DEFAULT_OBJECT_VERSION -> + <<$/, B/binary, $/, K/binary>>; + _ -> + <<$/, B/binary, $/, K/binary, $/, V/binary>> + end; +error_resource(_Tag, RD) -> + {OrigResource, _} = riak_cs_rewrite:original_resource(RD), + OrigResource. + +user_access_key(?RCS_USER{key_id = KeyId}) when KeyId /= undefined -> + KeyId; +user_access_key(_) -> + "NoKeyId". diff --git a/apps/riak_cs/src/riak_cs_aws_rewrite.erl b/apps/riak_cs/src/riak_cs_aws_rewrite.erl new file mode 100644 index 000000000..1c705fda0 --- /dev/null +++ b/apps/riak_cs/src/riak_cs_aws_rewrite.erl @@ -0,0 +1,104 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_aws_rewrite). +-behaviour(riak_cs_rewrite). + +-export([rewrite/5, + original_resource/1, + raw_url/1 + ]). + +-include("riak_cs.hrl"). +-include("riak_cs_web.hrl"). +-include_lib("kernel/include/logger.hrl"). + + +-spec rewrite(atom(), atom(), {integer(), integer()}, mochiweb_headers(), string()) -> + {mochiweb_headers(), string()}. +rewrite(Method, Scheme, Vsn, Headers, Url) -> + Host = mochiweb_headers:get_value("host", Headers), + case service_from_host(Host) of + likely_s3_or_native -> + case service_from_url(Url) of + unrecognized -> + case needs_rewriting(Url) of + true -> + riak_cs_aws_s3_rewrite:rewrite(Method, Scheme, Vsn, Headers, Url); + false -> + riak_cs_rewrite:rewrite(Method, Scheme, Vsn, Headers, Url) + end; + _Mod -> + riak_cs_rewrite:rewrite(Method, Scheme, Vsn, Headers, Url) + end; + {unsupported, A} -> + logger:warning("Service ~s is not supported", [A]), + {Headers, Url}; + Mod -> + Mod:rewrite(Method, Scheme, Vsn, Headers, Url) + end. + +service_from_host(Host) -> + {AttempRewrite, Third, Fourth} = + case lists:reverse( + string:split(string:to_lower(Host), ".", all)) of + ["com", "amazonaws", A] -> + {true, A, ""}; + ["com", "amazonaws", A, B|_] -> + {true, A, B}; + _ -> + {false, "", ""} + end, + case AttempRewrite of + true -> + case aws_service_submodule(Third) of + {unsupported, _} -> + %% third item is a region, then fourth must be it + aws_service_submodule(Fourth); + Service -> + Service + end; + false -> + likely_s3_or_native + end. + +service_from_url("/iam?"++_) -> riak_cs_aws_iam_rewrite; +service_from_url("/sts?"++_) -> riak_cs_aws_sts_rewrite; +service_from_url(_) -> unrecognized. + +aws_service_submodule("s3") -> riak_cs_aws_s3_rewrite; +aws_service_submodule("iam") -> riak_cs_aws_iam_rewrite; +aws_service_submodule("sts") -> riak_cs_aws_sts_rewrite; +aws_service_submodule(A) -> {unsupported, A}. + +needs_rewriting(Url) -> + case string:split(string:to_lower(Url), "/", all) of + [[], "buckets" | _] -> false; + [[], "riak-cs" | _] -> false; + _ -> true + end. + +-spec original_resource(#wm_reqdata{}) -> {string(), [{term(),term()}]}. +original_resource(RD) -> + riak_cs_rewrite:original_resource(RD). + +-spec raw_url(#wm_reqdata{}) -> undefined | {string(), [{term(), term()}]}. +raw_url(RD) -> + riak_cs_rewrite:raw_url(RD). diff --git a/src/riak_cs_s3_rewrite.erl b/apps/riak_cs/src/riak_cs_aws_s3_rewrite.erl similarity index 67% rename from src/riak_cs_s3_rewrite.erl rename to apps/riak_cs/src/riak_cs_aws_s3_rewrite.erl index d228c0035..575283549 100644 --- a/src/riak_cs_s3_rewrite.erl +++ b/apps/riak_cs/src/riak_cs_aws_s3_rewrite.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -41,53 +42,46 @@ --module(riak_cs_s3_rewrite). +-module(riak_cs_aws_s3_rewrite). +-behaviour(riak_cs_rewrite). --export([rewrite/5, original_resource/1, raw_url/1]). --export([rewrite_path_and_headers/5]). +-export([rewrite/5, + original_resource/1, + raw_url/1 + ]). -include("riak_cs.hrl"). --include("s3_api.hrl"). - --define(RCS_REWRITE_HEADER, "x-rcs-rewrite-path"). --define(RCS_RAW_URL_HEADER, "x-rcs-raw-url"). +-include("riak_cs_web.hrl"). +-include("aws_api.hrl"). +-include_lib("kernel/include/logger.hrl"). -ifdef(TEST). --compile(export_all). +-compile([export_all, nowarn_export_all]). +-export([bucket_from_host/1]). +-include_lib("eunit/include/eunit.hrl"). -endif. -type subresource() :: {string(), string()}. -type query_params() :: [{string(), string()}]. -type subresources() :: [subresource()]. + %% @doc Function to rewrite headers prior to processing by webmachine. -spec rewrite(atom(), atom(), {integer(), integer()}, mochiweb_headers(), string()) -> - {mochiweb_headers(), string()}. + {mochiweb_headers(), string()}. rewrite(Method, _Scheme, _Vsn, Headers, Url) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"rewrite">>), {Path, QueryString, _} = mochiweb_util:urlsplit_path(Url), rewrite_path_and_headers(Method, Headers, Url, Path, QueryString). --spec original_resource(term()) -> undefined | {string(), [{term(),term()}]}. +-spec original_resource(#wm_reqdata{}) -> undefined | {string(), [{term(),term()}]}. original_resource(RD) -> - case wrq:get_req_header(?RCS_REWRITE_HEADER, RD) of - undefined -> undefined; - RawPath -> - {Path, QS, _} = mochiweb_util:urlsplit_path(RawPath), - {Path, mochiweb_util:parse_qs(QS)} - end. + riak_cs_rewrite:original_resource(RD). --spec raw_url(term()) -> undefined | {string(), [{term(), term()}]}. +-spec raw_url(#wm_reqdata{}) -> undefined | {string(), [{term(), term()}]}. raw_url(RD) -> - case wrq:get_req_header(?RCS_RAW_URL_HEADER, RD) of - undefined -> undefined; - RawUrl -> - {Path, QS, _} = mochiweb_util:urlsplit_path(RawUrl), - {Path, mochiweb_util:parse_qs(QS)} - end. + riak_cs_rewrite:raw_url(RD). + --spec rewrite_path_and_headers(atom(), mochiweb_headers(), string(), string(), string()) -> - {mochiweb_headers(), string()}. rewrite_path_and_headers(Method, Headers, Url, Path, QueryString) -> Host = mochiweb_headers:get_value("host", Headers), HostBucket = bucket_from_host(Host), @@ -104,7 +98,6 @@ rewrite_path_and_headers(Method, Headers, Url, Path, QueryString) -> %% @doc Internal function to handle rewriting the URL --spec rewrite_path(atom(),string(), string(), undefined | string()) -> string(). rewrite_path(_Method, "/", _QS, undefined) -> "/buckets"; rewrite_path(Method, Path, QS, undefined) -> @@ -138,34 +131,31 @@ rcs_rewrite_header(RawPath, Bucket) -> %% @doc Extract the bucket name that may have been prepended to the %% host name in the Host header value. --spec bucket_from_host(undefined | string()) -> undefined | string(). bucket_from_host(undefined) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"bucket_from_host">>), undefined; bucket_from_host(HostHeader) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"bucket_from_host">>), - {ok, RootHost} = application:get_env(riak_cs, cs_root_host), - bucket_from_host(HostHeader, RootHost). - -bucket_from_host(HostHeader, RootHost) -> - HostNoPort = case string:tokens(HostHeader, ":") of - [] -> HostHeader; - [H|_] -> H - end, - extract_bucket_from_host(HostNoPort, - string:rstr(HostNoPort, RootHost)). - -%% @doc Extract the bucket name from the `Host' header value if a -%% bucket name is present. --spec extract_bucket_from_host(string(), non_neg_integer()) -> undefined | string(). -extract_bucket_from_host(_Host, 0) -> - undefined; -extract_bucket_from_host(_Host, 1) -> - undefined; -extract_bucket_from_host(Host, RootHostIndex) -> + [HostNoPort|_] = string:tokens(HostHeader, ":"), + extract_bucket_from_host(HostNoPort). + +extract_bucket_from_host(Host) -> %% Take the substring of the everything up to %% the '.' preceding the root host - string:sub_string(Host, 1, RootHostIndex-2). + Bucket = + case re:run(Host, "(.+\.|)s3\.(.+\.|)?amazonaws\.com", + [{capture, all_but_first, list}]) of + {match, [M1, _M2]} -> + M1; + {match, [M1]} -> + M1; + _ -> + [] + end, + case Bucket of + [] -> + undefined; + _ -> + lists:droplast(Bucket) + end. %% @doc Separate the bucket name from the rest of the raw path in the %% case where the bucket name is included in the path. @@ -197,26 +187,47 @@ format_bucket_qs(_Method, QueryParams, SubResources) -> %% rewrite rules. -spec format_object_qs({subresources(), query_params()}) -> string(). format_object_qs({SubResources, QueryParams}) -> + HaveAcl = (proplists:get_value("acl", SubResources) /= undefined), + HaveUploads = (proplists:get_value("uploads", SubResources) /= undefined), UploadId = proplists:get_value("uploadId", SubResources, []), PartNum = proplists:get_value("partNumber", SubResources, []), - format_object_qs(SubResources, QueryParams, UploadId, PartNum). + VersionId = proplists:get_value("versionId", SubResources, binary_to_list(?LFS_DEFAULT_OBJECT_VERSION)), + format_object_qs(SubResources, QueryParams, #{have_acl => HaveAcl, + have_uploads => HaveUploads, + version_id => VersionId, + upload_id => UploadId, + part_num => PartNum}). %% @doc Format an object operation query string to conform the the %% rewrite rules. --spec format_object_qs(subresources(), query_params(), string(), string()) -> string(). -format_object_qs(SubResources, QueryParams, [], []) -> - [format_subresources(SubResources), format_query_params(QueryParams)]; -format_object_qs(_SubResources, QueryParams, UploadId, []) -> - ["/uploads/", UploadId, format_query_params(QueryParams)]; -format_object_qs(_SubResources, QueryParams, UploadId, PartNum) -> - ["/uploads/", UploadId, format_query_params([{"partNumber", PartNum} | QueryParams])]. +format_object_qs(_SubResources, QueryParams, #{version_id := VersionId, + have_acl := true}) -> + ["/versions/", VersionId, "/acl", format_query_params(QueryParams)]; +format_object_qs(_SubResources, QueryParams, #{have_uploads := true, + version_id := VersionId}) -> + ["/versions/", VersionId, "/uploads", format_query_params(QueryParams)]; + +format_object_qs(SubResources, QueryParams, #{version_id := VersionId, + upload_id := [], + part_num := []}) -> + ["/versions/", VersionId, format_subresources(SubResources), format_query_params(QueryParams)]; +format_object_qs(_SubResources, QueryParams, #{version_id := VersionId, + upload_id := UploadId, + part_num := []}) -> + ["/versions/", VersionId, "/uploads/", UploadId, format_query_params(QueryParams)]; +format_object_qs(_SubResources, QueryParams, #{version_id := VersionId, + upload_id := UploadId, + part_num := PartNum}) -> + ["/versions/", VersionId, "/uploads/", UploadId, format_query_params([{"partNumber", PartNum} | QueryParams])]. %% @doc Format a string that expresses the subresource request %% that can be appended to the URL. -spec format_subresources(subresources()) -> string(). format_subresources([]) -> []; -format_subresources([{Key, []} | _]) -> +format_subresources([{"versionId", _}]) -> + []; +format_subresources([{Key, []}]) -> ["/", Key]. %% @doc Format a proplist of query parameters into a string @@ -249,3 +260,19 @@ get_subresources(QueryString) -> valid_subresource({Key, _}) -> lists:member(Key, ?SUBRESOURCES). + +-ifdef(TEST). + +extract_bucket_from_host_test() -> + Cases = [ {"asdf.s3.amazonaws.com", "asdf"} + , {"a.s.df.s3.amazonaws.com", "a.s.df"} + , {"a-s.df.s3.amazonaws.com", "a-s.df"} + , {"asdfff.s3.eu-west-1.amazonaws.com", "asdfff"} + , {"a.s.df.s2.amazonaws.com", undefined} + , {"a.s.df.s3.amazonaws.org", undefined} + ], + lists:foreach(fun({A, E}) -> ?assertEqual(bucket_from_host(A), E) end, Cases), + lists:foreach(fun({A, E}) -> ?assertEqual(bucket_from_host(A), E) end, Cases), + ok. + +-endif. diff --git a/apps/riak_cs/src/riak_cs_aws_sts_rewrite.erl b/apps/riak_cs/src/riak_cs_aws_sts_rewrite.erl new file mode 100644 index 000000000..ce4444879 --- /dev/null +++ b/apps/riak_cs/src/riak_cs_aws_sts_rewrite.erl @@ -0,0 +1,60 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_aws_sts_rewrite). +-behaviour(riak_cs_rewrite). + +-export([rewrite/5, + original_resource/1, + raw_url/1]). + +-include("riak_cs.hrl"). +-include("aws_api.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-spec rewrite(atom(), atom(), {integer(), integer()}, mochiweb_headers(), string()) -> + {mochiweb_headers(), string()}. +rewrite(Method, _Scheme, _Vsn, Headers, Url) -> + {Path, QueryString, _} = mochiweb_util:urlsplit_path(Url), + rewrite_path_and_headers(Method, Headers, Url, Path, QueryString). + + +-spec original_resource(#wm_reqdata{}) -> undefined | {string(), [{term(),term()}]}. +original_resource(RD) -> + riak_cs_rewrite:original_resource(RD). + +-spec raw_url(#wm_reqdata{}) -> undefined | {string(), [{term(), term()}]}. +raw_url(RD) -> + riak_cs_rewrite:raw_url(RD). + +rewrite_path_and_headers(Method, Headers, Url, Path, QueryString) -> + RewrittenPath = + rewrite_path(Method, Path, QueryString), + RewrittenHeaders = + mochiweb_headers:default( + ?RCS_RAW_URL_HEADER, Url, + mochiweb_headers:default( + ?RCS_REWRITE_HEADER, Url, + Headers)), + {RewrittenHeaders, RewrittenPath}. + + +rewrite_path(_Method, "/", _QS) -> + "/sts". diff --git a/apps/riak_cs/src/riak_cs_aws_utils.erl b/apps/riak_cs/src/riak_cs_aws_utils.erl new file mode 100644 index 000000000..745405d9c --- /dev/null +++ b/apps/riak_cs/src/riak_cs_aws_utils.erl @@ -0,0 +1,186 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_aws_utils). + +-export([aws_service_from_url/1, + make_id/1, + make_id/2, + make_unique_index_id/2, + make_user_arn/2, + make_role_arn/2, + make_policy_arn/2, + make_provider_arn/1, + make_assumed_role_user_arn/2, + generate_access_creds/1, + generate_secret/2, + generate_canonical_id/1 + ]). + +-include("riak_cs.hrl"). +-include("aws_api.hrl"). +-include_lib("riakc/include/riakc.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-define(AWS_ID_CHARSET, "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"). +-define(AWS_ID_EXT_CHARSET, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"). +-define(ACCESS_KEY_LENGTH, 20). + +-spec make_unique_index_id(iam_entity(), pid()) -> binary(). +make_unique_index_id(role, Pbc) -> + case try_unique_index_id( + make_id(?IAM_ENTITY_ID_LENGTH, ?ROLE_ID_PREFIX), + ?IAM_ROLE_BUCKET, ?ROLE_ID_INDEX, Pbc) of + retry -> + make_unique_index_id(role, Pbc); + Id -> + Id + end; +make_unique_index_id(user, Pbc) -> + case try_unique_index_id( + make_id(?IAM_ENTITY_ID_LENGTH, ?USER_ID_PREFIX), + ?IAM_USER_BUCKET, ?USER_ID_INDEX, Pbc) of + retry -> + make_unique_index_id(user, Pbc); + Id -> + Id + end; +make_unique_index_id(policy, Pbc) -> + case try_unique_index_id( + make_id(?IAM_ENTITY_ID_LENGTH, ?POLICY_ID_PREFIX), + ?IAM_POLICY_BUCKET, ?POLICY_ID_INDEX, Pbc) of + retry -> + make_unique_index_id(policy, Pbc); + Id -> + Id + end. + +try_unique_index_id(Id, Bucket, Index, Pbc) -> + case riakc_pb_socket:get_index_eq(Pbc, Bucket, Index, Id) of + {ok, ?INDEX_RESULTS{keys=[]}} -> + Id; + _ -> + retry + end. + + +-spec aws_service_from_url(string()) -> aws_service() | unsupported. +aws_service_from_url(Host) -> + {Third, Fourth} = + case lists:reverse( + string:split(string:to_lower(Host), ".", all)) of + ["com", "amazonaws", A] -> + {A, ""}; + ["com", "amazonaws", A, B|_] -> + {A, B}; + _ -> + {"", ""} + end, + case aws_service(Third) of + unsupported -> + %% third item is a region, then fourth must be it + aws_service(Fourth); + Service -> + Service + end. + +aws_service("s3") -> s3; +aws_service("iam") -> iam; +aws_service("sts") -> sts; +aws_service(_) -> unsupported. + + + +-spec make_id(non_neg_integer(), string()) -> binary(). +make_id(Length) -> + make_id(Length, []). +make_id(Length, Prefix) -> + make_id(Length, Prefix, ?AWS_ID_CHARSET). +make_id(Length, Prefix, CharSet) -> + iolist_to_binary([Prefix, fill(Length - length(Prefix), [], CharSet)]). + +make_user_arn(Name, Path) -> + single_slash( + iolist_to_binary(["arn:aws:iam::", fill(12, [], "0123456789"), ":user", Path, $/, Name])). + +make_role_arn(Name, Path) -> + single_slash( + iolist_to_binary(["arn:aws:iam::", fill(12, [], "0123456789"), ":role", Path, $/, Name])). + +make_policy_arn(Name, Path) -> + single_slash( + iolist_to_binary(["arn:aws:iam::", fill(12, [], "0123456789"), ":policy", Path, $/, Name])). + +make_provider_arn(Name) -> + single_slash( + iolist_to_binary(["arn:aws:iam::", fill(12, "", "0123456789"), ":saml-provider/", Name])). + +make_assumed_role_user_arn(RoleName, SessionName) -> + single_slash( + iolist_to_binary(["arn:aws:sts::", fill(12, "", "0123456789"), ":assumed-role/", RoleName, $/, SessionName])). +single_slash(A) -> + iolist_to_binary(re:replace(A, <<"/+">>, <<"/">>)). + +%% @doc Generate a new set of access credentials for user. +-spec generate_access_creds(binary()) -> {iodata(), iodata()}. +generate_access_creds(UserId) -> + KeyId = generate_key(UserId), + Secret = generate_secret(UserId, KeyId), + {KeyId, Secret}. + +%% @doc Generate the canonical id for a user. +-spec generate_canonical_id(binary()) -> binary(). +generate_canonical_id(KeyID) -> + Bytes = 16, + Id1 = riak_cs_utils:md5(KeyID), + Id2 = riak_cs_utils:md5(uuid:get_v4()), + list_to_binary( + riak_cs_utils:binary_to_hexlist( + iolist_to_binary(<< Id1:Bytes/binary, + Id2:Bytes/binary >>))). + +%% @doc Generate an access key for a user +-spec generate_key(binary()) -> binary(). +generate_key(UserName) -> + Ctx = crypto:mac_init(hmac, sha, UserName), + Ctx1 = crypto:mac_update(Ctx, uuid:get_v4()), + Key = crypto:mac_finalN(Ctx1, 15), + list_to_binary(string:to_upper(base64url:encode_to_string(Key))). + +%% @doc Generate a secret access token for a user +-spec generate_secret(binary(), binary()) -> binary(). +generate_secret(UserId, Key) -> + Bytes = 14, + Ctx = crypto:mac_init(hmac, sha, UserId), + Ctx1 = crypto:mac_update(Ctx, Key), + SecretPart1 = crypto:mac_finalN(Ctx1, Bytes), + Ctx2 = crypto:mac_init(hmac, sha, UserId), + Ctx3 = crypto:mac_update(Ctx2, uuid:get_v4()), + SecretPart2 = crypto:mac_finalN(Ctx3, Bytes), + base64url:encode( + << SecretPart1:Bytes/binary, + SecretPart2:Bytes/binary >>). + + +fill(0, Q, _) -> + Q; +fill(N, Q, CharSet) -> + fill(N-1, [lists:nth(rand:uniform(length(CharSet)), CharSet) | Q], CharSet). + diff --git a/src/riak_cs_block_server.erl b/apps/riak_cs/src/riak_cs_block_server.erl similarity index 85% rename from src/riak_cs_block_server.erl rename to apps/riak_cs/src/riak_cs_block_server.erl index 07d71b4f3..99889abba 100644 --- a/src/riak_cs_block_server.erl +++ b/apps/riak_cs/src/riak_cs_block_server.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -50,7 +51,7 @@ code_change/3, format_status/2]). --record(state, {riak_client :: riak_client(), +-record(state, {riak_client :: undefined | riak_client(), close_riak_connection=true :: boolean(), bag_id :: bag_id()}). @@ -184,45 +185,41 @@ handle_cast({get_block, ReplyPid, Bucket, Key, ClusterID, UUID, BlockNumber}, {noreply, State}; handle_cast({put_block, ReplyPid, Bucket, Key, UUID, BlockNumber, Value, BCSum}, State=#state{riak_client=RcPid}) -> - dt_entry(<<"put_block">>, [BlockNumber], [Bucket, Key]), {FullBucket, FullKey} = full_bkey(Bucket, Key, UUID, BlockNumber), MD = make_md_usermeta([{?USERMETA_BUCKET, Bucket}, {?USERMETA_KEY, Key}, {?USERMETA_BCSUM, BCSum}]), FailFun = fun(Error) -> - _ = lager:error("Put ~p ~p UUID ~p block ~p failed: ~p\n", - [Bucket, Key, UUID, BlockNumber, Error]) + logger:error("Put ~p ~p UUID ~p block ~p failed: ~p", + [Bucket, Key, UUID, BlockNumber, Error]) end, %% TODO: Handle put failure here. ok = do_put_block(FullBucket, FullKey, <<>>, Value, MD, RcPid, [riakc, put_block], FailFun), riak_cs_put_fsm:block_written(ReplyPid, BlockNumber), - dt_return(<<"put_block">>, [BlockNumber], [Bucket, Key]), {noreply, State}; handle_cast({delete_block, ReplyPid, Bucket, Key, UUID, BlockNumber}, State=#state{riak_client=RcPid}) -> - dt_entry(<<"delete_block">>, [BlockNumber], [Bucket, Key]), {FullBucket, FullKey} = full_bkey(Bucket, Key, UUID, BlockNumber), Timeout = riak_cs_config:get_block_timeout(), %% do a get first to get the vclock (only do a head request though) GetOptions = [head | pr_quorum_options()], - _ = case riak_cs_pbc:get(block_pbc(RcPid), FullBucket, FullKey, - GetOptions, Timeout, [riakc, head_block]) of - {ok, RiakObject} -> - ok = delete_block(RcPid, ReplyPid, RiakObject, {UUID, BlockNumber}); - {error, notfound} -> - %% If the block isn't found, assume it's been - %% previously deleted by another delete FSM, and - %% move on to the next block. - riak_cs_delete_fsm:block_deleted(ReplyPid, {ok, {UUID, BlockNumber}}); - {error, _} = Error -> - %% Report errors in HEADs to prevent crashing block - %% servers, as crash logs forces lager to sync log - %% files at each line. - Result = format_delete_result(Error, {UUID, BlockNumber}), - riak_cs_delete_fsm:block_deleted(ReplyPid, Result) - end, - dt_return(<<"delete_block">>, [BlockNumber], [Bucket, Key]), + case riak_cs_pbc:get(block_pbc(RcPid), FullBucket, FullKey, + GetOptions, Timeout, [riakc, head_block]) of + {ok, RiakObject} -> + ok = delete_block(RcPid, ReplyPid, RiakObject, {UUID, BlockNumber}); + {error, notfound} -> + %% If the block isn't found, assume it's been + %% previously deleted by another delete FSM, and + %% move on to the next block. + riak_cs_delete_fsm:block_deleted(ReplyPid, {ok, {UUID, BlockNumber}}); + {error, _} = Error -> + %% Report errors in HEADs to prevent crashing block + %% servers, as crash logs forces lager to sync log + %% files at each line. + Result = format_delete_result(Error, {UUID, BlockNumber}), + riak_cs_delete_fsm:block_deleted(ReplyPid, Result) + end, {noreply, State}; handle_cast(_Msg, State) -> {noreply, State}. @@ -253,29 +250,27 @@ do_get_block(ReplyPid, _Bucket, _Key, _ClusterID, _UseProxyGet, _ProxyActive, when is_list(ErrorReasons) andalso is_atom(LastReason) -> %% Not worth retrying, 'failure' comes as LastReason Sorry = {error, ErrorReasons}, - _ = lager:error("do_get_block/11 failed. Errors: ~p", [ErrorReasons]), + logger:error("do_get_block/11 failed. Errors: ~p", [ErrorReasons]), ok = riak_cs_get_fsm:chunk(ReplyPid, {UUID, BlockNumber}, Sorry); do_get_block(ReplyPid, _Bucket, _Key, _ClusterID, _UseProxyGet, _ProxyActive, UUID, BlockNumber, _RcPid, MaxRetries, ErrorReasons) when is_list(ErrorReasons) andalso length(ErrorReasons) > MaxRetries -> Sorry = {error, ErrorReasons}, - _ = lager:error("do_get_block/11 failed. Errors: ~p", [ErrorReasons]), + logger:error("do_get_block/11 failed. Errors: ~p", [ErrorReasons]), ok = riak_cs_get_fsm:chunk(ReplyPid, {UUID, BlockNumber}, Sorry); do_get_block(ReplyPid, Bucket, Key, ClusterID, UseProxyGet, ProxyActive, UUID, BlockNumber, RcPid, MaxRetries, ErrorReasons) -> ok = sleep_retries(length(ErrorReasons)), - dt_entry(<<"get_block">>, [BlockNumber], [Bucket, Key]), {FullBucket, FullKey} = full_bkey(Bucket, Key, UUID, BlockNumber), GetOptions1 = n_val_one_options(), GetOptions2 = r_one_options(), ProceedFun = fun(OkReply) -> - ok = riak_cs_get_fsm:chunk(ReplyPid, {UUID, BlockNumber}, OkReply), - dt_return(<<"get_block">>, [BlockNumber], [Bucket, Key]) + ok = riak_cs_get_fsm:chunk(ReplyPid, {UUID, BlockNumber}, OkReply) end, RetryFun = fun(NewPause) -> do_get_block(ReplyPid, Bucket, Key, ClusterID, UseProxyGet, @@ -306,10 +301,15 @@ try_local_get(RcPid, FullBucket, FullKey, GetOptions1, GetOptions2, [{local_one, Why}|ErrorReasons], UseProxyGet, ProxyActive, ClusterID); {error, Other} -> - _ = lager:error("do_get_block: other error 1: ~p\n", [Other]), + logger:error("do_get_block: other error 1: ~p", [Other]), RetryFun({failure, [{local_one, Other}|ErrorReasons]}) end. + +%% having spent three days trying to deal with this warning in a +%% non-violent manner: +-dialyzer([{no_match, handle_local_notfound/10}]). + handle_local_notfound(RcPid, FullBucket, FullKey, GetOptions2, ProceedFun, RetryFun, ErrorReasons, UseProxyGet, ProxyActive, ClusterID) -> @@ -322,7 +322,7 @@ handle_local_notfound(RcPid, FullBucket, FullKey, GetOptions2, {error, Why} when Why == disconnected; Why == timeout -> - _ = lager:debug("get_block_local() failed: {error, ~p}", [Why]), + logger:warning("get_block_local() failed: ~p", [Why]), _ = riak_cs_pbc:pause_to_reconnect(block_pbc(RcPid), Why, Timeout), RetryFun([{local_quorum, Why}|ErrorReasons]); @@ -341,13 +341,10 @@ handle_local_notfound(RcPid, FullBucket, FullKey, GetOptions2, {error, notfound} -> RetryFun({failure, [{local_quorum, notfound}|ErrorReasons]}); {error, Other} -> - _ = lager:error("do_get_block: other error 2: ~p\n", [Other]), + logger:error("do_get_block: other error 2: ~p", [Other]), RetryFun({failure, [{local_quorum, Other}|ErrorReasons]}) end. --spec get_block_local(riak_client(), binary(), binary(), list(), - timeout(), riak_cs_stats:key()) -> - {ok, binary()} | {error, term()}. get_block_local(RcPid, FullBucket, FullKey, GetOptions, Timeout, StatsKey) -> case riak_cs_pbc:get(block_pbc(RcPid), FullBucket, FullKey, GetOptions, Timeout, StatsKey) of @@ -357,9 +354,8 @@ get_block_local(RcPid, FullBucket, FullKey, GetOptions, Timeout, StatsKey) -> Else end. --spec get_block_remote(riak_client(), binary(), binary(), binary(), get_options(), - riak_cs_stats:key()) -> - {ok, binary()} | {error, term()}. +-dialyzer([{no_match, get_block_remote/6}]). + get_block_remote(RcPid, FullBucket, FullKey, ClusterID, GetOptions0, StatsKey) -> %% replace get_block_timeout with proxy_get_block_timeout GetOptions = proplists:delete(timeout, GetOptions0), @@ -376,7 +372,6 @@ get_block_remote(RcPid, FullBucket, FullKey, ClusterID, GetOptions0, StatsKey) - %% to modify n-val per GET request. get_block_legacy(ReplyPid, Bucket, Key, ClusterID, UseProxyGet, UUID, BlockNumber, RcPid) -> - dt_entry(<<"get_block_legacy">>, [BlockNumber], [Bucket, Key]), {FullBucket, FullKey} = full_bkey(Bucket, Key, UUID, BlockNumber), GetOptions = r_one_options(), ChunkValue = @@ -390,8 +385,7 @@ get_block_legacy(ReplyPid, Bucket, Key, ClusterID, UseProxyGet, UUID, get_block_remote(RcPid, FullBucket, FullKey, ClusterID, GetOptions, [riakc, get_block_legacy_remote]) end, - ok = riak_cs_get_fsm:chunk(ReplyPid, {UUID, BlockNumber}, ChunkValue), - dt_return(<<"get_block_legacy">>, [BlockNumber], [Bucket, Key]). + ok = riak_cs_get_fsm:chunk(ReplyPid, {UUID, BlockNumber}, ChunkValue). delete_block(RcPid, ReplyPid, RiakObject, BlockId) -> Result = constrained_delete(RcPid, RiakObject, BlockId), @@ -413,7 +407,7 @@ secondary_delete_check({error, {unsatisfied_constraint, _, _}}, RcPid, RiakObjec StatsKey = [riakc, delete_block_secondary], riak_cs_pbc:delete_obj(block_pbc(RcPid), RiakObject, [], Timeout, StatsKey); secondary_delete_check({error, Reason} = E, _, _) -> - _ = lager:warning("Constrained block deletion failed. Reason: ~p", [Reason]), + logger:warning("Constrained block deletion failed. Reason: ~p", [Reason]), E; secondary_delete_check(_, _, _) -> ok. @@ -497,35 +491,32 @@ full_bkey(Bucket, Key, UUID, BlockId) -> find_md_usermeta(MD) -> dict:find(?MD_USERMETA, MD). --spec resolve_block_object(riakc_obj:riakc_obj(), riak_client()) -> - {ok, binary()} | {error, notfound}. resolve_block_object(RObj, RcPid) -> {{MD, Value}, NeedRepair} = riak_cs_utils:resolve_robj_siblings(riakc_obj:get_contents(RObj)), - _ = if NeedRepair andalso is_binary(Value) -> + if NeedRepair andalso is_binary(Value) -> RBucket = riakc_obj:bucket(RObj), RKey = riakc_obj:key(RObj), [MD1|_] = riakc_obj:get_metadatas(RObj), S3Info = case find_md_usermeta(MD1) of - {ok, Ps} -> - Ps; - error -> - [] - end, - _ = lager:info("Repairing riak ~p ~p for ~p\n", - [RBucket, RKey, S3Info]), + {ok, Ps} -> + Ps; + error -> + [] + end, + logger:info("Repairing riak ~p ~p for ~p", [RBucket, RKey, S3Info]), Bucket = proplists:get_value(<>, S3Info), Key = proplists:get_value(<>, S3Info), VClock = riakc_obj:vclock(RObj), FailFun = fun(Error) -> - _ = lager:error("Put S3 ~p ~p Riak ~p ~p failed: ~p\n", - [Bucket, Key, RBucket, RKey, Error]) - end, + logger:error("Put S3 ~p ~p Riak ~p ~p failed: ~p", + [Bucket, Key, RBucket, RKey, Error]) + end, do_put_block(RBucket, RKey, VClock, Value, MD, RcPid, [riakc, put_block_resolved], FailFun); - NeedRepair andalso not is_binary(Value) -> - _ = lager:error("All checksums fail: ~P\n", [RObj, 200]); - true -> + NeedRepair andalso not is_binary(Value) -> + logger:error("All checksums fail: ~P", [RObj]); + true -> ok end, if is_binary(Value) -> @@ -551,7 +542,7 @@ do_put_block(FullBucket, FullKey, VClock, Value, MD, RcPid, StatsKey, FailFun) - Else end. --spec sleep_retries(integer()) -> 'ok'. +-spec sleep_retries(non_neg_integer()) -> 'ok'. sleep_retries(N) -> timer:sleep(num_retries_to_sleep_millis(N)). @@ -580,12 +571,6 @@ use_proxy_get(SourceClusterId, BagId) when is_binary(SourceClusterId) -> LocalClusterID = riak_cs_mb_helper:cluster_id(BagId), LocalClusterID =/= SourceClusterId. -dt_entry(Func, Ints, Strings) -> - riak_cs_dtrace:dtrace(?DT_BLOCK_OP, 1, Ints, ?MODULE, Func, Strings). - -dt_return(Func, Ints, Strings) -> - riak_cs_dtrace:dtrace(?DT_BLOCK_OP, 2, Ints, ?MODULE, Func, Strings). - block_pbc(RcPid) -> {ok, BlockPbc} = riak_cs_riak_client:block_pbc(RcPid), BlockPbc. diff --git a/src/riak_cs_bucket.erl b/apps/riak_cs/src/riak_cs_bucket.erl similarity index 58% rename from src/riak_cs_bucket.erl rename to apps/riak_cs/src/riak_cs_bucket.erl index b5f656214..66b26111c 100644 --- a/src/riak_cs_bucket.erl +++ b/apps/riak_cs/src/riak_cs_bucket.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -24,30 +25,34 @@ -module(riak_cs_bucket). %% Public API --export([ - fetch_bucket_object/2, - create_bucket/6, +-export([fetch_bucket_object/2, + create_bucket/5, delete_bucket/4, get_buckets/1, - set_bucket_acl/5, - set_bucket_policy/5, - delete_bucket_policy/4, + set_bucket_acl/4, + set_bucket_policy/4, + delete_bucket_policy/3, get_bucket_acl_policy/3, + set_bucket_versioning/4, + get_bucket_versioning/2, maybe_log_bucket_owner_error/2, resolve_buckets/3, - update_bucket_record/1, delete_all_uploads/2, delete_old_uploads/3, fold_all_buckets/3, - fetch_bucket_keys/1 + fetch_bucket_keys/1, + exprec_bucket_versioning/1, + upgrade_bucket_record/1 ]). -include("riak_cs.hrl"). -include_lib("riak_pb/include/riak_pb_kv_codec.hrl"). -include_lib("riakc/include/riakc.hrl"). +-include_lib("kernel/include/logger.hrl"). -ifdef(TEST). -compile(export_all). +-compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). -endif. @@ -57,10 +62,11 @@ %% @doc Create a bucket in the global namespace or return %% an error if it already exists. --spec create_bucket(rcs_user(), term(), binary(), bag_id(), acl(), riak_client()) -> - ok | - {error, term()}. -create_bucket(User, UserObj, Bucket, BagId, ACL, RcPid) -> +-spec create_bucket(rcs_user(), riakc_object:riakc_object() | undefined, binary(), bag_id(), acl()) -> + ok | {error, term()}. +create_bucket(_User, undefined, _Bucket, _BagId, _ACL) -> + {error, no_bucket_create_for_federated_users}; +create_bucket(User, _UserObj, Bucket, BagId, ACL) -> CurrentBuckets = get_buckets(User), %% Do not attempt to create bucket if the user already owns it @@ -77,10 +83,8 @@ create_bucket(User, UserObj, Bucket, BagId, ACL, RcPid) -> BagId, ACL, User, - UserObj, create, - [velvet, create_bucket], - RcPid); + [velvet, create_bucket]); false -> {error, invalid_bucket_name} end; @@ -121,7 +125,6 @@ middle_chars(B) -> ByteSize = byte_size(B), [binary:at(B, Position) || Position <- lists:seq(1, ByteSize - 2)]. --spec is_bucket_ip_addr(string()) -> boolean(). is_bucket_ip_addr(Bucket) -> case inet_parse:ipv4strict_address(Bucket) of {ok, _} -> @@ -154,14 +157,13 @@ dash_char(Char) -> %% @doc Delete a bucket -spec delete_bucket(rcs_user(), riakc_obj:riakc_obj(), binary(), riak_client()) -> - ok | - {error, remaining_multipart_upload}. -delete_bucket(User, UserObj, Bucket, RcPid) -> + ok | {error, remaining_multipart_upload}. +delete_bucket(User, _UserObj, Bucket, RcPid) -> CurrentBuckets = get_buckets(User), %% Buckets can only be deleted if they exist {AttemptDelete, LocalError} = - case bucket_exists(CurrentBuckets, binary_to_list(Bucket)) of + case bucket_exists(CurrentBuckets, Bucket) of true -> case bucket_empty(Bucket, RcPid) of {ok, true} -> {true, ok}; @@ -174,7 +176,7 @@ delete_bucket(User, UserObj, Bucket, RcPid) -> %% TODO: output log if failed in cleaning up existing uploads. %% The number of retry is hardcoded. {ok, Count} = delete_all_uploads(Bucket, RcPid), - _ = lager:debug("deleted ~p multiparts before bucket deletion.", [Count]), + ?LOG_DEBUG("deleted ~p multiparts before bucket deletion.", [Count]), %% This call still may return {error, remaining_multipart_upload} %% even if all uploads cleaned up above, because concurrent %% multiple deletion may happen. Then Riak CS returns 409 confliction @@ -182,10 +184,8 @@ delete_bucket(User, UserObj, Bucket, RcPid) -> serialized_bucket_op(Bucket, ?ACL{}, User, - UserObj, delete, - [velvet, delete_bucket], - RcPid); + [velvet, delete_bucket]); false -> LocalError end. @@ -207,13 +207,12 @@ delete_old_uploads(Bucket, RcPid, Timestamp) when is_binary(Timestamp) -> fold_delete_uploads(Bucket, RcPid, Ds, Timestamp, 0). fold_delete_uploads(_Bucket, _RcPid, [], _Timestamp, Count) -> {ok, Count}; -fold_delete_uploads(Bucket, RcPid, [D|Ds], Timestamp, Count)-> - Key = D?MULTIPART_DESCR.key, - +fold_delete_uploads(Bucket, RcPid, [?MULTIPART_DESCR{key = VKey, + upload_id = UploadId} | Ds], + Timestamp, Count) -> %% cannot fail here - {ok, Obj, Manifests} = riak_cs_manifest:get_manifests(RcPid, Bucket, Key), - - UploadId = D?MULTIPART_DESCR.upload_id, + {Key, Vsn} = rcs_common_manifest:decompose_versioned_key(VKey), + {ok, Obj, Manifests} = riak_cs_manifest:get_manifests(RcPid, Bucket, Key, Vsn), %% find_manifest_with_uploadid case lists:keyfind(UploadId, 1, Manifests) of @@ -221,19 +220,19 @@ fold_delete_uploads(Bucket, RcPid, [D|Ds], Timestamp, Count)-> {UploadId, M} when M?MANIFEST.state == writing %% comparing timestamp here, like %% <<"2012-02-17T18:22:50.000Z">> < <<"2014-05-11-....">> => true - andalso M?MANIFEST.created < Timestamp -> + andalso M?MANIFEST.write_start_time < Timestamp -> case riak_cs_gc:gc_specific_manifests( - [M?MANIFEST.uuid], Obj, Bucket, Key, RcPid) of + [M?MANIFEST.uuid], Obj, Bucket, RcPid) of {ok, _NewObj} -> fold_delete_uploads(Bucket, RcPid, Ds, Timestamp, Count+1); E -> - lager:debug("cannot delete multipart manifest: ~p ~p (~p)", - [{Bucket, Key}, M?MANIFEST.uuid, E]), + logger:notice("cannot delete multipart manifest: ~s (~s/~s:~s): ~p", + [M?MANIFEST.uuid, Bucket, Key, Vsn, E]), E end; _E -> - lager:debug("skipping multipart manifest: ~p ~p (~p)", - [{Bucket, Key}, UploadId, _E]), + ?LOG_DEBUG("skipping multipart manifest: ~p ~p (~p)", + [{Bucket, Key}, UploadId, _E]), fold_delete_uploads(Bucket, RcPid, Ds, Timestamp, Count) end. @@ -241,8 +240,6 @@ fold_delete_uploads(Bucket, RcPid, [D|Ds], Timestamp, Count)-> fold_all_buckets(Fun, Acc0, RcPid) when is_function(Fun) -> iterate_csbuckets(RcPid, Acc0, Fun, undefined). --spec iterate_csbuckets(riak_client(), term(), fun(), binary()|undefined) -> - {ok, term()} | {error, any()}. iterate_csbuckets(RcPid, Acc0, Fun, Cont0) -> Options = case Cont0 of @@ -254,7 +251,8 @@ iterate_csbuckets(RcPid, Acc0, Fun, Cont0) -> case riak_cs_pbc:get_index_range(MasterPbc, ?BUCKETS_BUCKET, <<"$key">>, <<0>>, <<255>>, Options, [riakc, get_cs_buckets_by_index]) of - {ok, ?INDEX_RESULTS{keys=BucketNames, continuation=Cont}} -> + {ok, ?INDEX_RESULTS{keys = BucketNames, + continuation = Cont}} -> Foldfun = iterate_csbuckets_fold_fun(Fun), Acc2 = lists:foldl(Foldfun, Acc0, BucketNames), case Cont of @@ -265,7 +263,7 @@ iterate_csbuckets(RcPid, Acc0, Fun, Cont0) -> end; Error -> - _ = lager:error("iterating CS buckets: ~p", [Error]), + logger:error("iterating CS buckets: ~p", [Error]), {error, {Error, Acc0}} end. @@ -287,44 +285,38 @@ get_buckets(?RCS_USER{buckets=Buckets}) -> %% @doc Set the ACL for a bucket. Existing ACLs are only %% replaced, they cannot be updated. --spec set_bucket_acl(rcs_user(), riakc_obj:riakc_obj(), binary(), acl(), riak_client()) -> - ok | {error, term()}. -set_bucket_acl(User, UserObj, Bucket, ACL, RcPid) -> +-spec set_bucket_acl(rcs_user(), riakc_obj:riakc_obj(), binary(), acl()) -> + ok | {error, term()}. +set_bucket_acl(User, _UserObj, Bucket, ACL) -> serialized_bucket_op(Bucket, ACL, User, - UserObj, update_acl, - [velvet, set_bucket_acl], - RcPid). + [velvet, set_bucket_acl]). %% @doc Set the policy for a bucket. Existing policy is only overwritten. --spec set_bucket_policy(rcs_user(), riakc_obj:riakc_obj(), binary(), []|policy()|acl(), riak_client()) -> - ok | {error, term()}. -set_bucket_policy(User, UserObj, Bucket, PolicyJson, RcPid) -> +-spec set_bucket_policy(rcs_user(), riakc_obj:riakc_obj(), binary(), binary()) -> + ok | {error, term()}. +set_bucket_policy(User, _UserObj, Bucket, PolicyJson) -> serialized_bucket_op(Bucket, PolicyJson, User, - UserObj, update_policy, - [velvet, set_bucket_policy], - RcPid). + [velvet, set_bucket_policy]). %% @doc Set the policy for a bucket. Existing policy is only overwritten. --spec delete_bucket_policy(rcs_user(), riakc_obj:riakc_obj(), binary(), riak_client()) -> - ok | {error, term()}. -delete_bucket_policy(User, UserObj, Bucket, RcPid) -> +-spec delete_bucket_policy(rcs_user(), riakc_obj:riakc_obj(), binary()) -> + ok | {error, term()}. +delete_bucket_policy(User, _UserObj, Bucket) -> serialized_bucket_op(Bucket, [], User, - UserObj, delete_policy, - [velvet, delete_bucket_policy], - RcPid). + [velvet, delete_bucket_policy]). %% @doc fetch moss.bucket and return acl and policy -spec get_bucket_acl_policy(binary(), atom(), riak_client()) -> - {acl(), policy()} | {error, term()}. + {ok, {acl(), policy()}} | {error, term()}. get_bucket_acl_policy(Bucket, PolicyMod, RcPid) -> case fetch_bucket_object(Bucket, RcPid) of {ok, Obj} -> @@ -335,24 +327,58 @@ get_bucket_acl_policy(Bucket, PolicyMod, RcPid) -> Acl = riak_cs_acl:bucket_acl_from_contents(Bucket, Contents), Policy = PolicyMod:bucket_policy_from_contents(Bucket, Contents), format_acl_policy_response(Acl, Policy); - {error, _}=Error -> + {error, _} = Error -> Error end. --type policy_from_meta_result() :: {'ok', policy()} | {'error', 'policy_undefined'}. --type bucket_policy_result() :: policy_from_meta_result() | {'error', 'multiple_bucket_owners'}. --type acl_from_meta_result() :: {'ok', acl()} | {'error', 'acl_undefined'}. --type bucket_acl_result() :: acl_from_meta_result() | {'error', 'multiple_bucket_owners'}. --spec format_acl_policy_response(bucket_acl_result(), bucket_policy_result()) -> - {error, atom()} | {acl(), 'undefined' | policy()}. format_acl_policy_response({error, _}=Error, _) -> Error; format_acl_policy_response(_, {error, multiple_bucket_owners}=Error) -> Error; format_acl_policy_response({ok, Acl}, {error, policy_undefined}) -> - {Acl, undefined}; + {ok, {Acl, undefined}}; format_acl_policy_response({ok, Acl}, {ok, Policy}) -> - {Acl, Policy}. + {ok, {Acl, Policy}}. + +-spec set_bucket_versioning(rcs_user(), riakc_obj:riakc_obj(), + binary(), bucket_versioning() | #{}) -> + ok | {error, term()}. +set_bucket_versioning(User, _UserObj, Bucket, Specs = #{}) -> + set_bucket_versioning(User, _UserObj, Bucket, exprec_bucket_versioning(Specs)); +set_bucket_versioning(User, _UserObj, Bucket, Options) -> + serialized_bucket_op(Bucket, + Options, + User, + update_versioning, + [velvet, set_bucket_versioning]). + +-spec get_bucket_versioning(binary(), riak_client()) -> + {ok, bucket_versioning()} | {error, term()}. +get_bucket_versioning(Bucket, RcPid) -> + case fetch_bucket_object(Bucket, RcPid) of + {ok, Obj} -> + MD = riakc_obj:get_metadata(Obj), + case dict:find(?MD_USERMETA, MD) of + {ok, UM} -> + case proplists:get_value(?MD_VERSIONING, UM) of + undefined -> + {ok, #bucket_versioning{status = suspended}}; + Defined -> + {ok, binary_to_term(Defined)} + end; + error -> + {ok, #bucket_versioning{status = suspended}} + end; + {error, _} = Error -> + Error + end. + +-spec exprec_bucket_versioning(#{}) -> bucket_versioning(). +exprec_bucket_versioning(FF) -> + BV0 = #bucket_versioning{status = S0, + mfa_delete = M0} = exprec:frommap_bucket_versioning(FF), + BV0#bucket_versioning{status = binary_to_atom(S0, latin1), + mfa_delete = binary_to_atom(M0, latin1)}. %% =================================================================== @@ -361,24 +387,27 @@ format_acl_policy_response({ok, Acl}, {ok, Policy}) -> %% @doc Generate a JSON document to use for a bucket %% ACL request. --spec bucket_acl_json(acl(), string()) -> string(). -bucket_acl_json(ACL, KeyId) -> - binary_to_list( - iolist_to_binary( - mochijson2:encode({struct, [{<<"requester">>, list_to_binary(KeyId)}, - riak_cs_acl_utils:acl_to_json_term(ACL)]}))). +bucket_acl_json(ACL, KeyId) -> + list_to_binary( + jason:encode([{requester, KeyId}, + {acl, ACL}], + [{records, [{acl_v3, record_info(fields, acl_v3)}, + {acl_grant_v2, record_info(fields, acl_grant_v2)}]}])). %% @doc Generate a JSON document to use for a bucket --spec bucket_policy_json(binary(), string()) -> string(). -bucket_policy_json(PolicyJson, KeyId) -> - binary_to_list( - iolist_to_binary( - mochijson2:encode({struct, [{<<"requester">>, list_to_binary(KeyId)}, - {<<"policy">>, PolicyJson}] - }))). +bucket_policy_json(PolicyJson, KeyId) -> + list_to_binary( + jason:encode([{requester, KeyId}, + {policy, base64:encode(PolicyJson)}])). + +%% @doc Generate a JSON document to use in setting bucket versioning option +bucket_versioning_json(BV, KeyId) -> + list_to_binary( + jason:encode([{requester, KeyId}, + {versioning, BV}], + [{records, [{bucket_versioning, record_info(fields, bucket_versioning)}]}])). %% @doc Check if a bucket is empty --spec bucket_empty(binary(), riak_client()) -> {ok, boolean()} | {error, term()}. bucket_empty(Bucket, RcPid) -> ManifestBucket = riak_cs_utils:to_bucket_name(objects, Bucket), %% @TODO Use `stream_list_keys' instead @@ -388,10 +417,6 @@ bucket_empty(Bucket, RcPid) -> [riakc, list_all_manifest_keys]), {ok, bucket_empty_handle_list_keys(RcPid, Bucket, ListKeysResult)}. --spec bucket_empty_handle_list_keys(riak_client(), binary(), - {ok, list()} | - {error, term()}) -> - boolean(). bucket_empty_handle_list_keys(RcPid, Bucket, {ok, Keys}) -> AnyPred = bucket_empty_any_pred(RcPid, Bucket), %% `lists:any/2' will break out early as soon @@ -400,8 +425,6 @@ bucket_empty_handle_list_keys(RcPid, Bucket, {ok, Keys}) -> bucket_empty_handle_list_keys(_RcPid, _Bucket, _Error) -> false. --spec bucket_empty_any_pred(riak_client(), Bucket :: binary()) -> - fun((Key :: binary()) -> boolean()). bucket_empty_any_pred(RcPid, Bucket) -> fun (Key) -> riak_cs_utils:key_exists(RcPid, Bucket, Key) @@ -409,7 +432,7 @@ bucket_empty_any_pred(RcPid, Bucket) -> %% @doc Fetches the bucket object and verify its status. -spec fetch_bucket_object(binary(), riak_client()) -> - {ok, riakc_obj:riakc_obj()} | {error, term()}. + {ok, riakc_obj:riakc_obj()} | {error, term()}. fetch_bucket_object(BucketName, RcPid) -> case fetch_bucket_object_raw(BucketName, RcPid) of {ok, Obj} -> @@ -420,36 +443,36 @@ fetch_bucket_object(BucketName, RcPid) -> _ -> {ok, Obj} end; - {error, _}=Error -> + {error, notfound} -> + {error, no_such_bucket}; + {error, _} = Error -> Error end. %% @doc Fetches the bucket object, even it is marked as free --spec fetch_bucket_object_raw(binary(), riak_client()) -> - {ok, riakc_obj:riakc_obj()} | {error, term()}. -fetch_bucket_object_raw(BucketName, RcPid) -> - case riak_cs_riak_client:get_bucket(RcPid, BucketName) of +fetch_bucket_object_raw(BucketName, Pbc) -> + case riak_cs_riak_client:get_bucket(Pbc, BucketName) of {ok, Obj} -> Values = riakc_obj:get_values(Obj), maybe_log_sibling_warning(BucketName, Values), {ok, Obj}; - {error, _}=Error -> + {error, _} = Error -> Error end. -spec maybe_log_sibling_warning(binary(), list(riakc_obj:value())) -> ok. maybe_log_sibling_warning(Bucket, Values) when length(Values) > 1 -> - _ = lager:warning("The bucket ~s has ~b siblings that may need resolution.", - [binary_to_list(Bucket), length(Values)]), + logger:warning("The bucket ~s has ~b siblings that may need resolution.", + [binary_to_list(Bucket), length(Values)]), ok; maybe_log_sibling_warning(_, _) -> ok. -spec maybe_log_bucket_owner_error(binary(), list(riakc_obj:value())) -> ok. maybe_log_bucket_owner_error(Bucket, Values) when length(Values) > 1 -> - _ = lager:error("The bucket ~s has ~b owners." - " This situation requires administrator intervention.", - [binary_to_list(Bucket), length(Values)]), + logger:error("The bucket ~s has ~b owners." + " This situation requires administrator intervention.", + [binary_to_list(Bucket), length(Values)]), ok; maybe_log_bucket_owner_error(_, _) -> ok. @@ -457,7 +480,6 @@ maybe_log_bucket_owner_error(_, _) -> %% @doc Check if a bucket exists in a list of the user's buckets. %% @TODO This will need to change once globally unique buckets %% are enforced. --spec bucket_exists([cs_bucket()], string()) -> boolean(). bucket_exists(Buckets, CheckBucket) -> SearchResults = [Bucket || Bucket <- Buckets, Bucket?RCS_BUCKET.name =:= CheckBucket andalso @@ -472,100 +494,67 @@ bucket_exists(Buckets, CheckBucket) -> %% @doc Return a closure over a specific function %% call to the stanchion client module for either %% bucket creation or deletion. --spec bucket_fun(bucket_operation(), - binary(), - bag_id(), - [] | policy() | acl(), - string(), - {string(), string()}, - {string(), pos_integer(), boolean()}) -> function(). -bucket_fun(create, Bucket, BagId, ACL, KeyId, AdminCreds, StanchionData) -> - {StanchionIp, StanchionPort, StanchionSSL} = StanchionData, +bucket_fun(create, Bucket, BagId, ACL, KeyId, AdminCreds) -> %% Generate the bucket JSON document BucketDoc = bucket_json(Bucket, BagId, ACL, KeyId), fun() -> - velvet:create_bucket(StanchionIp, - StanchionPort, - "application/json", + velvet:create_bucket("application/json", BucketDoc, - [{ssl, StanchionSSL}, - {auth_creds, AdminCreds}]) + [{auth_creds, AdminCreds}]) end; -bucket_fun(update_acl, Bucket, _BagId, ACL, KeyId, AdminCreds, StanchionData) -> - {StanchionIp, StanchionPort, StanchionSSL} = StanchionData, +bucket_fun(update_acl, Bucket, _BagId, ACL, KeyId, AdminCreds) -> %% Generate the bucket JSON document for the ACL request AclDoc = bucket_acl_json(ACL, KeyId), fun() -> - velvet:set_bucket_acl(StanchionIp, - StanchionPort, - Bucket, + velvet:set_bucket_acl(Bucket, "application/json", AclDoc, - [{ssl, StanchionSSL}, - {auth_creds, AdminCreds}]) + [{auth_creds, AdminCreds}]) end; -bucket_fun(update_policy, Bucket, _BagId, PolicyJson, KeyId, AdminCreds, StanchionData) -> - {StanchionIp, StanchionPort, StanchionSSL} = StanchionData, +bucket_fun(update_policy, Bucket, _BagId, PolicyJson, KeyId, AdminCreds) -> %% Generate the bucket JSON document for the ACL request PolicyDoc = bucket_policy_json(PolicyJson, KeyId), fun() -> - velvet:set_bucket_policy(StanchionIp, - StanchionPort, - Bucket, + velvet:set_bucket_policy(Bucket, "application/json", PolicyDoc, - [{ssl, StanchionSSL}, - {auth_creds, AdminCreds}]) + [{auth_creds, AdminCreds}]) + end; +bucket_fun(update_versioning, Bucket, _BagId, VsnOption, KeyId, AdminCreds) -> + Doc = bucket_versioning_json(VsnOption, KeyId), + fun() -> + velvet:set_bucket_versioning(Bucket, + "application/json", + Doc, + [{auth_creds, AdminCreds}]) end; -bucket_fun(delete_policy, Bucket, _BagId, _, KeyId, AdminCreds, StanchionData) -> - {StanchionIp, StanchionPort, StanchionSSL} = StanchionData, +bucket_fun(delete_policy, Bucket, _BagId, _, KeyId, AdminCreds) -> %% Generate the bucket JSON document for the ACL request fun() -> - velvet:delete_bucket_policy(StanchionIp, - StanchionPort, - Bucket, + velvet:delete_bucket_policy(Bucket, KeyId, - [{ssl, StanchionSSL}, - {auth_creds, AdminCreds}]) + [{auth_creds, AdminCreds}]) end; -bucket_fun(delete, Bucket, _BagId, _ACL, KeyId, AdminCreds, StanchionData) -> - {StanchionIp, StanchionPort, StanchionSSL} = StanchionData, +bucket_fun(delete, Bucket, _BagId, _ACL, KeyId, AdminCreds) -> fun() -> - velvet:delete_bucket(StanchionIp, - StanchionPort, - Bucket, + velvet:delete_bucket(Bucket, KeyId, - [{ssl, StanchionSSL}, - {auth_creds, AdminCreds}]) + [{auth_creds, AdminCreds}]) end. %% @doc Generate a JSON document to use for a bucket %% creation request. --spec bucket_json(binary(), bag_id(), acl(), string()) -> string(). bucket_json(Bucket, BagId, ACL, KeyId) -> BagElement = case BagId of undefined -> []; - _ -> [{<<"bag">>, BagId}] + _ -> [{bag, BagId}] end, - binary_to_list( - iolist_to_binary( - mochijson2:encode({struct, [{<<"bucket">>, Bucket}, - {<<"requester">>, list_to_binary(KeyId)}, - riak_cs_acl_utils:acl_to_json_term(ACL)] ++ - BagElement}))). - -%% @doc Return a bucket record for the specified bucket name. --spec bucket_record(binary(), bucket_operation()) -> cs_bucket(). -bucket_record(Name, Operation) -> - Action = case Operation of - create -> created; - delete -> deleted; - _ -> undefined - end, - ?RCS_BUCKET{name=binary_to_list(Name), - last_action=Action, - creation_date=riak_cs_wm_utils:iso_8601_datetime(), - modification_time=os:timestamp()}. + list_to_binary( + jason:encode([{bucket, Bucket}, + {requester, KeyId}, + {acl, ACL}] ++ BagElement, + [{records, [{acl_v3, record_info(fields, acl_v3)}, + {acl_grant_v2, record_info(fields, acl_grant_v2)}]}])). %% @doc Check for and resolve any conflict between %% a bucket record from a user record sibling and @@ -606,14 +595,13 @@ bucket_sorter(?RCS_BUCKET{name=Bucket1}, -spec cleanup_bucket(cs_bucket()) -> boolean(). cleanup_bucket(?RCS_BUCKET{last_action=created}) -> false; -cleanup_bucket(?RCS_BUCKET{last_action=deleted, - modification_time=ModTime}) -> +cleanup_bucket(?RCS_BUCKET{last_action = deleted, + modification_time = ModTime}) -> %% the prune-time is specified in seconds, so we must %% convert Erlang timestamps to seconds first - NowSeconds = riak_cs_utils:second_resolution_timestamp(os:timestamp()), - ModTimeSeconds = riak_cs_utils:second_resolution_timestamp(ModTime), - (NowSeconds - ModTimeSeconds) > - riak_cs_config:user_buckets_prune_time(). + Now = os:system_time(millisecond), + (Now - ModTime) > + riak_cs_config:user_buckets_prune_time() * 1000. %% @doc Determine if an existing bucket from the resolution list %% should be kept or replaced when a conflict occurs. @@ -655,127 +643,25 @@ resolve_buckets([HeadUserRec | RestUserRecs], Buckets, _KeepDeleted) -> resolve_buckets(RestUserRecs, UpdBuckets, _KeepDeleted). %% @doc Shared code used when doing a bucket creation or deletion. --spec serialized_bucket_op(binary(), - [] | acl() | policy(), - rcs_user(), - riakc_obj:riakc_obj(), - bucket_operation(), - riak_cs_stats:key(), - riak_client()) -> - ok | - {error, term()}. -serialized_bucket_op(Bucket, ACL, User, UserObj, BucketOp, StatKey, RcPid) -> - serialized_bucket_op(Bucket, undefined, ACL, User, UserObj, - BucketOp, StatKey, RcPid). +serialized_bucket_op(Bucket, Arg, User, BucketOp, StatKey) -> + serialized_bucket_op(Bucket, undefined, Arg, User, BucketOp, StatKey). -%% @doc Shared code used when doing a bucket creation or deletion. --spec serialized_bucket_op(binary(), - bag_id(), - [] | acl() | policy(), - rcs_user(), - riakc_obj:riakc_obj(), - bucket_operation(), - riak_cs_stats:key(), - riak_client()) -> - ok | - {error, term()}. -serialized_bucket_op(Bucket, BagId, ACL, User, UserObj, BucketOp, StatsKey, RcPid) -> - StartTime = os:timestamp(), +serialized_bucket_op(Bucket, BagId, Arg, User, BucketOp, StatsKey) -> + StartTime = os:system_time(millisecond), _ = riak_cs_stats:inflow(StatsKey), {ok, AdminCreds} = riak_cs_config:admin_creds(), BucketFun = bucket_fun(BucketOp, Bucket, BagId, - ACL, + Arg, User?RCS_USER.key_id, - AdminCreds, - riak_cs_utils:stanchion_data()), + AdminCreds), %% Make a call to the request serialization service. OpResult = BucketFun(), _ = riak_cs_stats:update_with_start(StatsKey, StartTime, OpResult), - case OpResult of - ok -> - BucketRecord = bucket_record(Bucket, BucketOp), - case update_user_buckets(User, BucketRecord) of - {ok, ignore} when BucketOp == update_acl -> - OpResult; - {ok, ignore} -> - OpResult; - {ok, UpdUser} -> - riak_cs_user:save_user(UpdUser, UserObj, RcPid) - end; - {error, {error_status, Status, _, ErrorDoc}} -> - handle_stanchion_response(Status, ErrorDoc, BucketOp, Bucket); - {error, _} -> - OpResult - end. + OpResult. -%% @doc needs retry for delete op. 409 assumes -%% MultipartUploadRemaining for now if a new feature that needs retry -%% could come up, add branch here. See tests in -%% tests/riak_cs_bucket_test.erl --spec handle_stanchion_response(200..503, string(), delete|create, binary()) -> - {error, remaining_multipart_upload} | - {error, atom()}. -handle_stanchion_response(409, ErrorDoc, Op, Bucket) - when Op =:= delete orelse Op =:= create -> - - Value = riak_cs_s3_response:xml_error_code(ErrorDoc), - case {lists:flatten(Value), Op} of - {"MultipartUploadRemaining", delete} -> - _ = lager:error("Concurrent multipart upload might have" - " happened on deleting bucket '~s'.", [Bucket]), - {error, remaining_multipart_upload}; - {"MultipartUploadRemaining", create} -> - %% might be security issue - _ = lager:critical("Multipart upload remains in deleted bucket (~s)" - " Clean up the deleted buckets now.", [Bucket]), - %% Broken, returns 500 - throw({remaining_multipart_upload_on_deleted_bucket, Bucket}); - Other -> - _ = lager:debug("errordoc: ~p => ~s", [Other, ErrorDoc]), - riak_cs_s3_response:error_response(ErrorDoc) - end; -handle_stanchion_response(_C, ErrorDoc, _M, _) -> - %% _ = lager:error("unexpected errordoc: (~p, ~p) ~s", [_C, _M, ErrorDoc]), - riak_cs_s3_response:error_response(ErrorDoc). - -%% @doc Update a bucket record to convert the name from binary -%% to string if necessary. --spec update_bucket_record(term()) -> cs_bucket(). -update_bucket_record(Bucket=?RCS_BUCKET{name=Name}) when is_binary(Name) -> - Bucket?RCS_BUCKET{name=binary_to_list(Name)}; -update_bucket_record(Bucket) -> - Bucket. - -%% @doc Check if a user already has an ownership of -%% a bucket and update the bucket list if needed. --spec update_user_buckets(rcs_user(), cs_bucket()) -> - {ok, ignore} | {ok, rcs_user()}. -update_user_buckets(User, Bucket) -> - Buckets = User?RCS_USER.buckets, - %% At this point any siblings from the read of the - %% user record have been resolved so the user bucket - %% list should have 0 or 1 buckets that share a name - %% with `Bucket'. - case [B || B <- Buckets, B?RCS_BUCKET.name =:= Bucket?RCS_BUCKET.name] of - [] -> - {ok, User?RCS_USER{buckets=[Bucket | Buckets]}}; - [ExistingBucket] -> - case - (Bucket?RCS_BUCKET.last_action == deleted andalso - ExistingBucket?RCS_BUCKET.last_action == created) - orelse - (Bucket?RCS_BUCKET.last_action == created andalso - ExistingBucket?RCS_BUCKET.last_action == deleted) of - true -> - UpdBuckets = [Bucket | lists:delete(ExistingBucket, Buckets)], - {ok, User?RCS_USER{buckets=UpdBuckets}}; - false -> - {ok, ignore} - end - end. %% @doc Grab the whole list of Riak CS bucket keys. -spec fetch_bucket_keys(riak_client()) -> {ok, [binary()]} | {error, term()}. @@ -784,3 +670,24 @@ fetch_bucket_keys(RcPid) -> Timeout = riak_cs_config:list_keys_list_buckets_timeout(), riak_cs_pbc:list_keys(MasterPbc, ?BUCKETS_BUCKET, Timeout, [riakc, list_all_bucket_keys]). + +-spec upgrade_bucket_record(#moss_bucket{} | #moss_bucket_v1{} | #moss_bucket_v2{}) -> ?RCS_BUCKET{}. +upgrade_bucket_record(#moss_bucket_v2{} = B) -> + B; +upgrade_bucket_record(#moss_bucket_v1{name = Name, + last_action = LastAction, + creation_date = CreationDate, + modification_time = {M1, M2, M3}, + acl = Acl} = A) -> + ?LOG_DEBUG("Upgrading moss_bucket_v1 ~p", [A]), + #moss_bucket_v2{name = list_to_binary(Name), + last_action = LastAction, + creation_date = calendar:rfc3339_to_system_time(CreationDate, [{unit, millisecond}]), + modification_time = M1 * 1000000 + M2 + M3 div 1000, + acl = riak_cs_acl:upgrade_acl_record(Acl)}; +upgrade_bucket_record(#moss_bucket{name = Name, + creation_date = CreationDate, + acl = Acl}) -> + #moss_bucket_v2{name = iolist_to_binary([Name]), + creation_date = calendar:rfc3339_to_system_time(CreationDate, [{unit, millisecond}]), + acl = riak_cs_acl:upgrade_acl_record(Acl)}. diff --git a/src/riak_cs_config.erl b/apps/riak_cs/src/riak_cs_config.erl similarity index 65% rename from src/riak_cs_config.erl rename to apps/riak_cs/src/riak_cs_config.erl index c7667b467..3f119e4fb 100644 --- a/src/riak_cs_config.erl +++ b/apps/riak_cs/src/riak_cs_config.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -22,11 +23,13 @@ -export([ warnings/0, + host_id/0, admin_creds/0, anonymous_user_creation/0, api/0, auth_bypass/0, admin_auth_enabled/0, + control_ui_enabled/0, auth_module/0, auth_v4_enabled/0, cluster_id/1, @@ -35,10 +38,7 @@ enforce_multipart_part_size/0, gc_batch_size/0, get_env/3, - key_list_multiplier/0, - set_key_list_multiplier/1, md5_chunk_size/0, - gc_paginated_indexes/0, policy_module/0, proxy_get_active/0, response_module/0, @@ -61,12 +61,18 @@ max_key_length/0, read_before_last_manifest_write/0, region/0, - stanchion/0, + set_stanchion/2, set_stanchion/3, use_2i_for_storage_calc/0, detailed_storage_calc/0, quota_modules/0, active_delete_threshold/0, - fast_user_get/0 + fast_user_get/0, + s3_root_host/0, + iam_create_user_default_email_host/0, + stanchion/0, + stanchion_subnet_and_netmask/0, + stanchion_hosting_mode/0, + tussle_voss_riak_host/0 ]). %% Timeouts hitting Riak @@ -110,32 +116,31 @@ -include("riak_cs_gc.hrl"). -include("oos_api.hrl"). --include("s3_api.hrl"). --include("list_objects.hrl"). +-include("aws_api.hrl"). -define(MAYBE_WARN(Bool, Msg), case (Bool) of - true -> _ = lager:warning((Msg)); - _ -> ok + true -> + logger:warning((Msg)); + _ -> + ok end). -spec warnings() -> ok. warnings() -> - ?MAYBE_WARN(not riak_cs_list_objects_utils:fold_objects_for_list_keys(), - "`fold_objects_for_list_keys` is set as false." - " This will be removed at next major version."), ?MAYBE_WARN(anonymous_user_creation(), "`anonymous_user_creation` is set as true. Set this as false" " when this CS nodes is populated as public service."), - ?MAYBE_WARN(not gc_paginated_indexes(), - "`gc_paginated_indexes` is set as false. " - " This will be removed at next major version."), ok. %% =================================================================== %% General config options %% =================================================================== +-spec host_id() -> string(). +host_id() -> + get_env(riak_cs, host_id, binary_to_list(base64:encode("riak_cs-is-not-amazonaws.com"))). + %% @doc Return the value of the `anonymous_user_creation' application %% environment variable. -spec anonymous_user_creation() -> boolean(). @@ -143,10 +148,10 @@ anonymous_user_creation() -> get_env(riak_cs, anonymous_user_creation, false). %% @doc Return the credentials of the admin user --spec admin_creds() -> {ok, {string()|undefined, string()|undefined}}. +-spec admin_creds() -> {ok, {binary()|undefined, binary()|undefined}}. admin_creds() -> - {ok, {get_env(riak_cs, admin_key, undefined), - get_env(riak_cs, admin_secret, undefined)}}. + {ok, {maybe_bin(get_env(riak_cs, admin_key, undefined)), + maybe_bin(get_env(riak_cs, admin_secret, undefined))}}. %% @doc Get the active version of Riak CS to use in checks to %% determine if new features should be enabled. @@ -154,19 +159,15 @@ admin_creds() -> cs_version() -> get_env(riak_cs, cs_version, undefined). --spec api() -> s3 | oos. +-spec api() -> api(). api() -> - api(get_env(riak_cs, rewrite_module, ?S3_API_MOD)). + api(get_env(riak_cs, rewrite_module, ?AWS_API_MOD)). --spec api(atom() | undefined) -> s3 | oos | undefined. -api(?S3_API_MOD) -> - s3; -api(?S3_LEGACY_API_MOD) -> - s3; +-spec api(atom() | undefined) -> api(). +api(?AWS_API_MOD) -> + aws; api(?OOS_API_MOD) -> - oos; -api(_) -> - undefined. + oos. -spec auth_bypass() -> boolean(). auth_bypass() -> @@ -176,6 +177,10 @@ auth_bypass() -> admin_auth_enabled() -> get_env(riak_cs, admin_auth_enabled, true). +-spec control_ui_enabled() -> boolean(). +control_ui_enabled() -> + get_env(riak_cs, control_ui_enabled, true). + -spec auth_module() -> atom(). auth_module() -> get_env(riak_cs, auth_module, ?DEFAULT_AUTH_MODULE). @@ -196,25 +201,10 @@ enforce_multipart_part_size() -> gc_batch_size() -> get_env(riak_cs, gc_batch_size, ?DEFAULT_GC_BATCH_SIZE). --spec key_list_multiplier() -> float(). -key_list_multiplier() -> - get_env(riak_cs, key_list_multiplier, ?KEY_LIST_MULTIPLIER). - --spec set_key_list_multiplier(float()) -> 'ok'. -set_key_list_multiplier(Multiplier) -> - application:set_env(riak_cs, key_list_multiplier, Multiplier). - -spec policy_module() -> atom(). policy_module() -> get_env(riak_cs, policy_module, ?DEFAULT_POLICY_MODULE). -%% @doc paginated 2i is supported after Riak 1.4 -%% When using Riak CS `>= 1.5' with Riak `=< 1.3' (it rarely happens) -%% this should be set as false at app.config. --spec gc_paginated_indexes() -> atom(). -gc_paginated_indexes() -> - get_env(riak_cs, gc_paginated_indexes, true). - -spec response_module() -> atom(). response_module() -> response_module(api()). @@ -223,7 +213,7 @@ response_module() -> response_module(oos) -> ?OOS_RESPONSE_MOD; response_module(_) -> - ?S3_RESPONSE_MOD. + ?AWS_RESPONSE_MOD. -spec use_t2b_compression() -> boolean(). use_t2b_compression() -> @@ -268,8 +258,7 @@ proxy_get_active() -> disabled -> false; Flag when is_boolean(Flag) -> Flag; Other -> - _ = lager:warning("proxy_get value in advanced.config is invalid: ~p", - [Other]), + logger:warning("proxy_get value in advanced.config is invalid: ~p", [Other]), false end. @@ -279,7 +268,7 @@ trust_x_forwarded_for() -> {ok, true} -> true; {ok, false} -> false; {ok, _} -> - _ = lager:warning("trust_x_forwarded_for value in app.config is invalid"), + logger:warning("trust_x_forwarded_for value in app.config is invalid"), false; undefined -> false %% secure by default! end. @@ -371,20 +360,29 @@ max_key_length() -> %% @doc Return `stanchion' configuration data. -spec stanchion() -> {string(), pos_integer(), boolean()}. stanchion() -> - {Host, Port} = case application:get_env(riak_cs, stanchion_host) of - {ok, HostPort} -> HostPort; - undefined -> - _ = lager:warning("No stanchion access defined. Using default."), - {?DEFAULT_STANCHION_IP, ?DEFAULT_STANCHION_PORT} - end, - SSL = case application:get_env(riak_cs, stanchion_ssl) of - {ok, SSL0} -> SSL0; - undefined -> - _ = lager:warning("No ssl flag for stanchion access defined. Using default."), - ?DEFAULT_STANCHION_SSL - end, + {Host, Port} = application:get_env(riak_cs, stanchion_host, {"127.0.0.1", 8085}), + SSL = application:get_env(riak_cs, stanchion_ssl, false), {Host, Port, SSL}. +-spec stanchion_subnet_and_netmask() -> {string(), string()}. +stanchion_subnet_and_netmask() -> + {ok, Subnet} = application:get_env(riak_cs, stanchion_subnet), + {ok, Netmask} = application:get_env(riak_cs, stanchion_netmask), + {Subnet, Netmask}. + +-spec set_stanchion(string(), inet:port()) -> ok. +set_stanchion(Host, Port) -> + application:set_env(riak_cs, stanchion_host, {Host, Port}), + ok. + +-spec set_stanchion(string(), inet:port(), boolean()) -> ok. +set_stanchion(Host, Port, Ssl) -> + application:set_env(riak_cs, stanchion_host, {Host, Port}), + application:set_env(riak_cs, stanchion_ssl, Ssl), + ok. + + + %% @doc This options is useful for use case involving high churn and %% concurrency on a fixed set of keys and when not using a Riak %% version >= 2.0.0 with DVVs enabled. It helps to avoid sibling @@ -428,53 +426,116 @@ active_delete_threshold() -> fast_user_get() -> get_env(riak_cs, fast_user_get, false). +-spec s3_root_host() -> string(). +s3_root_host() -> + get_env(riak_cs, s3_root_host, ?S3_ROOT_HOST). + +-spec iam_create_user_default_email_host() -> string(). +iam_create_user_default_email_host() -> + get_env(riak_cs, iam_create_user_default_email, ?IAM_CREATE_USER_DEFAULT_EMAIL_HOST). + +-spec stanchion_hosting_mode() -> auto | riak_cs_only | stanchion_only | riak_cs_with_stanchion. +stanchion_hosting_mode() -> + {ok, A} = application:get_env(riak_cs, stanchion_hosting_mode), + true = (A == auto orelse A == riak_cs_only orelse A == stanchion_only orelse A == riak_cs_with_stanchion), + A. + +-spec tussle_voss_riak_host() -> {string(), string()} | auto. +tussle_voss_riak_host() -> + get_env(riak_cs, tussle_voss_riak_host, auto). + %% =================================================================== %% ALL Timeouts hitting Riak %% =================================================================== --define(TIMEOUT_CONFIG_FUNC(ConfigName), - ConfigName() -> - get_env(riak_cs, ConfigName, - get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT))). - %% @doc Return the configured ping timeout. Default is 5 seconds. -spec ping_timeout() -> pos_integer(). ping_timeout() -> get_env(riak_cs, ping_timeout, ?DEFAULT_PING_TIMEOUT). %% timeouts in milliseconds -?TIMEOUT_CONFIG_FUNC(get_user_timeout). -?TIMEOUT_CONFIG_FUNC(get_bucket_timeout). -?TIMEOUT_CONFIG_FUNC(get_manifest_timeout). -?TIMEOUT_CONFIG_FUNC(get_block_timeout). - local_get_block_timeout() -> get_env(riak_cs, local_get_block_timeout, timer:seconds(5)). -?TIMEOUT_CONFIG_FUNC(proxy_get_block_timeout). -?TIMEOUT_CONFIG_FUNC(get_access_timeout). -?TIMEOUT_CONFIG_FUNC(get_gckey_timeout). -?TIMEOUT_CONFIG_FUNC(put_user_timeout). -?TIMEOUT_CONFIG_FUNC(put_manifest_timeout). -?TIMEOUT_CONFIG_FUNC(put_block_timeout). -?TIMEOUT_CONFIG_FUNC(put_access_timeout). -?TIMEOUT_CONFIG_FUNC(put_gckey_timeout). -?TIMEOUT_CONFIG_FUNC(put_user_usage_timeout). -?TIMEOUT_CONFIG_FUNC(delete_manifest_timeout). -?TIMEOUT_CONFIG_FUNC(delete_block_timeout). -?TIMEOUT_CONFIG_FUNC(delete_gckey_timeout). -?TIMEOUT_CONFIG_FUNC(list_keys_list_objects_timeout). -?TIMEOUT_CONFIG_FUNC(list_keys_list_users_timeout). -?TIMEOUT_CONFIG_FUNC(list_keys_list_buckets_timeout). -?TIMEOUT_CONFIG_FUNC(storage_calc_timeout). -?TIMEOUT_CONFIG_FUNC(list_objects_timeout). -?TIMEOUT_CONFIG_FUNC(fold_objects_timeout). -?TIMEOUT_CONFIG_FUNC(get_index_range_gckeys_timeout). -?TIMEOUT_CONFIG_FUNC(get_index_range_gckeys_call_timeout). -?TIMEOUT_CONFIG_FUNC(get_index_list_multipart_uploads_timeout). -?TIMEOUT_CONFIG_FUNC(cluster_id_timeout). - --undef(TIMEOUT_CONFIG_FUNC). +-spec get_user_timeout() -> non_neg_integer(). +get_user_timeout() -> + get_env(riak_cs, get_user_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec get_bucket_timeout() -> non_neg_integer(). +get_bucket_timeout() -> + get_env(riak_cs, get_bucket_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec get_manifest_timeout() -> non_neg_integer(). +get_manifest_timeout() -> + get_env(riak_cs, get_manifest_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec get_block_timeout() -> non_neg_integer(). +get_block_timeout() -> + get_env(riak_cs, get_block_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). + +-spec proxy_get_block_timeout() -> non_neg_integer(). +proxy_get_block_timeout() -> + get_env(riak_cs, proxy_get_block_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec get_access_timeout() -> non_neg_integer(). +get_access_timeout() -> + get_env(riak_cs, get_access_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec get_gckey_timeout() -> non_neg_integer(). +get_gckey_timeout() -> + get_env(riak_cs, get_gckey_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec put_user_timeout() -> non_neg_integer(). +put_user_timeout() -> + get_env(riak_cs, put_user_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec put_manifest_timeout() -> non_neg_integer(). +put_manifest_timeout() -> + get_env(riak_cs, put_manifest_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec put_block_timeout() -> non_neg_integer(). +put_block_timeout() -> + get_env(riak_cs, put_block_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec put_access_timeout() -> non_neg_integer(). +put_access_timeout() -> + get_env(riak_cs, put_access_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec put_gckey_timeout() -> non_neg_integer(). +put_gckey_timeout() -> + get_env(riak_cs, put_gckey_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec put_user_usage_timeout() -> non_neg_integer(). +put_user_usage_timeout() -> + get_env(riak_cs, put_user_usage_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec delete_manifest_timeout() -> non_neg_integer(). +delete_manifest_timeout() -> + get_env(riak_cs, delete_manifest_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec delete_block_timeout() -> non_neg_integer(). +delete_block_timeout() -> + get_env(riak_cs, delete_block_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec delete_gckey_timeout() -> non_neg_integer(). +delete_gckey_timeout() -> + get_env(riak_cs, delete_gckey_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec list_keys_list_objects_timeout() -> non_neg_integer(). +list_keys_list_objects_timeout() -> + get_env(riak_cs, list_keys_list_objects_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec list_keys_list_users_timeout() -> non_neg_integer(). +list_keys_list_users_timeout() -> + get_env(riak_cs, list_keys_list_users_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec list_keys_list_buckets_timeout() -> non_neg_integer(). +list_keys_list_buckets_timeout() -> + get_env(riak_cs, list_keys_list_buckets_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec storage_calc_timeout() -> non_neg_integer(). +storage_calc_timeout() -> + get_env(riak_cs, storage_calc_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec list_objects_timeout() -> non_neg_integer(). +list_objects_timeout() -> + get_env(riak_cs, list_objects_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec fold_objects_timeout() -> non_neg_integer(). +fold_objects_timeout() -> + get_env(riak_cs, fold_objects_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec get_index_range_gckeys_timeout() -> non_neg_integer(). +get_index_range_gckeys_timeout() -> + get_env(riak_cs, get_index_range_gckeys_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec get_index_range_gckeys_call_timeout() -> non_neg_integer(). +get_index_range_gckeys_call_timeout() -> + get_env(riak_cs, get_index_range_gckeys_call_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec get_index_list_multipart_uploads_timeout() -> non_neg_integer(). +get_index_list_multipart_uploads_timeout() -> + get_env(riak_cs, get_index_list_multipart_uploads_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). +-spec cluster_id_timeout() -> non_neg_integer(). +cluster_id_timeout() -> + get_env(riak_cs, cluster_id_timeout, get_env(riak_cs, riakc_timeouts, ?DEFAULT_RIAK_TIMEOUT)). %% =================================================================== %% S3 config options @@ -532,3 +593,7 @@ get_env(App, Key, Default) -> _ -> Default end. + + +maybe_bin(A) when is_list(A) -> list_to_binary(A); +maybe_bin(A) -> A. diff --git a/src/riak_cs_console.erl b/apps/riak_cs/src/riak_cs_console.erl similarity index 57% rename from src/riak_cs_console.erl rename to apps/riak_cs/src/riak_cs_console.erl index 77a1c5661..e6c4501c4 100644 --- a/src/riak_cs_console.erl +++ b/apps/riak_cs/src/riak_cs_console.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -20,8 +21,11 @@ -module(riak_cs_console). --export([ +-export([create_admin/1, + version/1, status/1, + supps/1, + test/1, cluster_info/1, audit_bucket_ownership/1, cleanup_orphan_multipart/0, @@ -32,17 +36,202 @@ -include("riak_cs.hrl"). -include_lib("riakc/include/riakc.hrl"). +-include_lib("kernel/include/logger.hrl"). %%%=================================================================== %%% Public API %%%=================================================================== +create_admin(Options_) -> + GetoptSpecs = [{terse, $t, "terse", boolean, + "produce machine-readable output (just \"KeyId Secret Id\" on a single line)"}, + {ignored_admin_option, $n, "dont-touch-riak-cs-conf", boolean, + "don't try to sed KeyId into /etc/riak-cs/riak-cs.conf"}], + case getopt:parse(GetoptSpecs, Options_) of + {ok, {Options, _Args}} -> + Email = "admin@me.com", + Terse = proplists:get_value(terse, Options, false), + case riak_cs_config:admin_creds() of + {ok, {?DEFAULT_ADMIN_KEY, _}} -> + do_create_admin(Email, Terse); + {ok, {SAK, Secret}} -> + io:format("An admin user already exists, with these creds:\n" + " KeyId: ~s\n" + " Secret: ~s\n", + [SAK, Secret]), + ok + end; + _ -> + getopt:usage(GetoptSpecs, "riak-cs admin create-admin-user") + end. + +do_create_admin(Email, Terse) -> + case riak_cs_user:create_user( + ?DEFAULT_ADMIN_NAME, list_to_binary(Email)) of + {ok, ?RCS_USER{key_id = SAK, + key_secret = Secret, + id = Id}} -> + application:set_env(riak_cs, admin_key, SAK), + application:set_env(riak_cs, admin_secret, Secret), + if Terse -> + io:format("~s ~s ~s\n", [SAK, Secret, Id]); + el/=se -> + io:format("Admin user created:\n\n" + " KeyId: ~s\n" + " Secret: ~s\n\n" + "Next steps:\n\n" + "1. Copy these details for use with your clients.\n" + "2. For all Riak CS nodes backed by the Riak cluster this node is connected to,\n" + " edit riak-cs.conf and set `admin.key` to the value shown above.\n" + "3. Restart all affected Riak CS nodes.\n", + [SAK, Secret]) + end, + create_and_attach_admin_policy(); + {error, user_already_exists} -> + {ok, Pbc} = riak_cs_utils:riak_connection(), + {ok, {?RCS_USER{key_id = KeyId, + key_secret = KeySecret}, _}} = + riak_cs_iam:find_user(#{name => ?DEFAULT_ADMIN_NAME}, Pbc), + ok = riak_cs_utils:close_riak_connection(Pbc), + io:format("An admin user has been created before with these creds:\n" + " KeyId: ~s\n" + " KeySecret: ~s\n" + "but your riak-cs.conf has not been updated.\n", + [KeyId, KeySecret]); + {error, Reason} -> + io:format("Failed to create admin user: ~p\n", [Reason]) + end. + +create_and_attach_admin_policy() -> + PD = #{<<"Version">> => <<"2012-10-17">>, + <<"Statement">> => + [ #{ <<"Principal">> => <<"*">>, + <<"Effect">> => <<"Allow">>, + <<"Action">> => [<<"sts:*">>, <<"iam:*">>, <<"s3:*">>], + <<"Resource">> => <<"*">> + } + ] + }, + {ok, ?IAM_POLICY{arn = Arn}} = + riak_cs_iam:create_policy( + #{path => <<"/">>, + description => <<"Admin Policy allowing access to all S3. IAM and STS actions">>, + policy_name => <<"Admin policy">>, + policy_document => jsx:encode(PD)}), + {ok, Pbc} = riak_cs_utils:riak_connection(), + ok = riak_cs_iam:attach_user_policy(Arn, ?DEFAULT_ADMIN_NAME, Pbc), + ok = riak_cs_utils:close_riak_connection(Pbc), + ok. + + +version([]) -> + {ok, Vsn} = application:get_env(riak_cs, cs_version), + io:format("version : ~p~n", [Vsn]), + ok. + status([]) -> - Stats = riak_cs_stats:get_stats(), + Stats = riak_cs_stats:get_stats() ++ stanchion_stats:get_stats(), _ = [io:format("~p : ~p~n", [Name, Value]) || {Name, Value} <- Stats], ok. +supps(Opts) -> + supps:p(Opts). + + +test([]) -> + UserName = <<"ADMINTESTUSER">>, + Bucket = <<"testbucket">>, + try + {ok, RcPid} = riak_cs_riak_client:start_link([]), + {User, UserObj} = get_test_user(UserName, RcPid), + + BagId = riak_cs_mb_helper:choose_bag_id(manifest, Bucket), + + Acl = riak_cs_acl_utils:default_acl( + User?RCS_USER.display_name, + User?RCS_USER.id, + User?RCS_USER.key_id), + + io:format("creating bucket\n", []), + ok = riak_cs_bucket:create_bucket( + User, UserObj, Bucket, BagId, Acl), + + Key = <<"testkey">>, + Value = <<"testvalue">>, + io:format("putting object\n", []), + ok = put_object(Bucket, Key, Value, Acl, RcPid), + io:format("reading object back\n", []), + ok = verify_object(Bucket, Key, Value, RcPid), + io:format("deleting object\n", []), + ok = delete_object(Bucket, Key, RcPid), + + io:format("deleting bucket\n", []), + ok = riak_cs_bucket:delete_bucket( + User, UserObj, Bucket, RcPid), + ok = riak_cs_iam:delete_user(User), + ok = riak_cs_riak_client:stop(RcPid), + ok + catch + E:R:T -> + logger:error("self-test failed: ~p:~p", [E, R]), + io:format("self-test failed: ~p:~p\nStacktrace: ~p\n", [E, R, T]) + end. + +get_test_user(Name, RcPid) -> + Email = <>, + case riak_cs_user:create_user(Name, Email) of + {ok, ?RCS_USER{key_id = KeyId}} -> + {ok, UU} = + riak_cs_user:get_user(KeyId, RcPid), + UU; + {error, user_already_exists} -> + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + {ok, UU} = + riak_cs_iam:find_user(#{email => Email}, Pbc), + UU + end. + + +put_object(Bucket, Key, Value, Acl, RcPid) -> + Args = [{Bucket, Key, ?LFS_DEFAULT_OBJECT_VERSION, + size(Value), <<"application/octet-stream">>, + [], riak_cs_lfs_utils:block_size(), Acl, timer:seconds(60), self(), RcPid}], + {ok, Pid} = riak_cs_put_fsm_sup:start_put_fsm(node(), Args), + ok = riak_cs_put_fsm:augment_data(Pid, Value), + {ok, _Mfst} = riak_cs_put_fsm:finalize(Pid, base64:encode(crypto:hash(md5, Value))), + ok. + +verify_object(Bucket, Key, Value, RcPid) -> + FetchConcurrency = riak_cs_lfs_utils:fetch_concurrency(), + BufferFactor = riak_cs_lfs_utils:get_fsm_buffer_size_factor(), + {ok, Pid} = + riak_cs_get_fsm_sup:start_get_fsm( + node(), Bucket, Key, ?LFS_DEFAULT_OBJECT_VERSION, self(), RcPid, + FetchConcurrency, BufferFactor), + + _Manifest = riak_cs_get_fsm:get_manifest(Pid), + riak_cs_get_fsm:continue(Pid, {0, size(Value) - 1}), + Value = read_object(Pid, <<>>), + + ok = riak_cs_get_fsm:stop(Pid), + ok. + +read_object(Pid, Q) -> + case riak_cs_get_fsm:get_next_chunk(Pid) of + {done, Chunk} -> + <>; + {chunk, Chunk} -> + read_object(Pid, <>) + end. + +delete_object(Bucket, Key, RcPid) -> + {ok, _UUIDsMarkedforDelete} = + riak_cs_utils:delete_object(Bucket, Key, ?LFS_DEFAULT_OBJECT_VERSION, RcPid), + ok. + + + %% in progress. cluster_info([OutFile]) -> try @@ -60,7 +249,7 @@ cluster_info([OutFile]) -> error:{badmatch, {error, enotdir}} -> io:format("Cluster_info failed, not a directory ~p~n", [filename:dirname(OutFile)]); Exception:Reason -> - lager:error("Cluster_info failed ~p:~p", [Exception, Reason]), + logger:error("Cluster_info failed ~p:~p", [Exception, Reason]), io:format("Cluster_info failed, see log for details~n"), error end. @@ -86,28 +275,24 @@ audit_bucket_ownership() -> audit_bucket_ownership0(RcPid) -> FromUsers = ownership_from_users(RcPid), {FromBuckets, OwnedBy} = ownership_from_buckets(RcPid), - lager:debug("FromUsers: ~p~n", [lists:usort(gb_sets:to_list(FromUsers))]), - lager:debug("FromBuckets: ~p~n", [lists:usort(gb_sets:to_list(FromBuckets))]), - lager:debug("OwnedBy: ~p~n", [lists:usort(gb_trees:to_list(OwnedBy))]), + ?LOG_DEBUG("FromUsers: ~p", [lists:usort(gb_sets:to_list(FromUsers))]), + ?LOG_DEBUG("FromBuckets: ~p", [lists:usort(gb_sets:to_list(FromBuckets))]), + ?LOG_DEBUG("OwnedBy: ~p", [lists:usort(gb_trees:to_list(OwnedBy))]), Inconsistencies0 = gb_sets:fold( fun ({U, B}, Acc) -> - lager:info( - "Bucket is not tracked by user: {User, Bucket} = {~s, ~s}", - [U, B]), + logger:info("Bucket is not tracked by user: {User, Bucket} = {~s, ~s}", [U, B]), [{not_tracked, {U, B}} | Acc] end, [], gb_sets:subtract(FromBuckets, FromUsers)), gb_sets:fold( fun({U,B}, Acc) -> case gb_trees:lookup(B, OwnedBy) of none -> - lager:info( - "Bucket does not exist: {User, Bucket} = {~s, ~s}", [U, B]), + logger:info("Bucket does not exist: {User, Bucket} = {~s, ~s}", [U, B]), [{no_bucket_object, {U, B}} | Acc]; {value, DifferentUser} -> - lager:info( - "Bucket is owned by different user: {User, Bucket, DifferentUser} = " - "{~s, ~s, ~s}", [U, B, DifferentUser]), + logger:info("Bucket is owned by different user: {User, Bucket, DifferentUser} = " + "{~s, ~s, ~s}", [U, B, DifferentUser]), [{different_user, {U, B, DifferentUser}} | Acc] end end, Inconsistencies0, gb_sets:subtract(FromUsers, FromBuckets)). @@ -116,12 +301,11 @@ ownership_from_users(RcPid) -> {ok, UserKeys} = riak_cs_user:fetch_user_keys(RcPid), lists:foldl( fun(UserKey, Ownership) -> - UserStr = binary_to_list(UserKey), - {ok, {RCSUser, _Obj}} = riak_cs_user:get_user(UserStr, RcPid), - ?RCS_USER{buckets=Buckets} = RCSUser, + {ok, {RCSUser, _Obj}} = riak_cs_user:get_user(UserKey, RcPid), + ?RCS_USER{buckets = Buckets} = RCSUser, lists:foldl( - fun(?RCS_BUCKET{name=BucketStr}, InnerOwnership) -> - gb_sets:add_element({UserKey, list_to_binary(BucketStr)}, + fun(?RCS_BUCKET{name = Bucket}, InnerOwnership) -> + gb_sets:add_element({UserKey, Bucket}, InnerOwnership) end, Ownership, Buckets) end, gb_sets:new(), UserKeys). @@ -171,8 +355,8 @@ cleanup_orphan_multipart(Timestamp) when is_list(Timestamp) -> cleanup_orphan_multipart(Timestamp) when is_binary(Timestamp) -> {ok, RcPid} = riak_cs_riak_client:start_link([]), - _ = lager:info("cleaning up with timestamp ~s", [Timestamp]), - _ = io:format("cleaning up with timestamp ~s", [Timestamp]), + logger:info("cleaning up with timestamp ~s", [Timestamp]), + io:format("cleaning up with timestamp ~s", [Timestamp]), Fun = fun(RcPidForOneBucket, BucketName, GetResult, Acc0) -> ok = maybe_cleanup_csbucket(RcPidForOneBucket, BucketName, GetResult, Timestamp), @@ -181,8 +365,8 @@ cleanup_orphan_multipart(Timestamp) when is_binary(Timestamp) -> _ = riak_cs_bucket:fold_all_buckets(Fun, [], RcPid), ok = riak_cs_riak_client:stop(RcPid), - _ = lager:info("All old unaborted orphan multipart uploads have been deleted.", []), - _ = io:format("~nAll old unaborted orphan multipart uploads have been deleted.~n", []). + logger:info("All old unaborted orphan multipart uploads have been deleted.", []), + io:format("~nAll old unaborted orphan multipart uploads have been deleted.~n", []). -spec resolve_siblings(binary(), binary()) -> any(). @@ -225,24 +409,23 @@ resolve_siblings(Pid, RawBucket, RawKey) -> resolve_siblings(Pid, RawBucket, RawKey, GetOptions, GetTimeout, PutOptions, PutTimeout) when is_integer(GetTimeout) andalso is_integer(PutTimeout) -> - _ = lager:info("Getting ~p:~p~n", [RawBucket, RawKey]), case riakc_pb_socket:get(Pid, RawBucket, RawKey, GetOptions, GetTimeout) of {ok, RiakObj} -> - _ = lager:info("Trying to resolve ~p sibling(s) of ~p:~p", - [riakc_obj:value_count(RiakObj), RawBucket, RawKey]), + logger:info("Trying to resolve ~p sibling(s) of ~p:~p", + [riakc_obj:value_count(RiakObj), RawBucket, RawKey]), case resolve_ro_siblings(RiakObj, RawBucket, RawKey) of {ok, RiakObj2} -> R = riakc_pb_socket:put(Pid, RiakObj2, PutOptions, PutTimeout), - _ = lager:info("Resolution result: ~p~n", [R]), + logger:info("Resolution result: ~p", [R]), R; ok -> - lager:info("No siblings in ~p:~p~n", [RawBucket, RawKey]); + logger:info("No siblings in ~p:~p", [RawBucket, RawKey]); {error, _} = E -> - _ = lager:info("Not updating ~p:~p: ~p~n", [RawBucket, RawKey, E]), + logger:info("Not updating ~p:~p: ~p", [RawBucket, RawKey, E]), E end; {error, Reason} = E -> - _ = lager:info("Failed to get an object before resolution: ~p~n", [Reason]), + logger:info("Failed to get an object before resolution: ~p", [Reason]), E end. @@ -283,7 +466,7 @@ resolve_ro_siblings(RO, <<"0b:", _/binary>>, _) -> RO1 = riakc_obj:update_metadata(RO, MD), {ok, riakc_obj:update_value(RO1, Value)}; {E, true} -> - _ = lager:info("Cannot resolve: ~p~n", [E]), + logger:info("Cannot resolve: ~p", [E]), {error, E}; {_, false} -> ok @@ -291,13 +474,12 @@ resolve_ro_siblings(RO, <<"0b:", _/binary>>, _) -> resolve_ro_siblings(RiakObject, <<"0o:", _/binary>>, _RawKey) -> [{_, Manifest}|_] = Manifests = riak_cs_manifest:manifests_from_riak_object(RiakObject), - _ = lager:info("Number of histories after sibling resolution: ~p.~n", - [length(Manifests)]), + logger:info("Number of histories after sibling resolution: ~p.", [length(Manifests)]), ObjectToWrite0 = riak_cs_utils:update_obj_value( RiakObject, riak_cs_utils:encode_term(Manifests)), - {B, K} = Manifest?MANIFEST.bkey, - RO = riak_cs_manifest_fsm:update_md_with_multipart_2i(ObjectToWrite0, Manifests, B, K), + {B, _K} = Manifest?MANIFEST.bkey, + RO = riak_cs_manifest_fsm:update_md_with_multipart_2i(ObjectToWrite0, Manifests, B), {ok, RO}. -spec maybe_cleanup_csbucket(riak_client(), @@ -315,8 +497,8 @@ maybe_cleanup_csbucket(RcPidForOneBucket, BucketName, {ok, RiakObj}, Timestamp) {ok, Count} -> io:format(" aborted ~p uploads.~n", [Count]); Error -> - lager:warning("Error in deleting old uploads: ~p~n", [Error]), - io:format("Error in deleting old uploads: ~p <<< ~n", [Error]), + logger:warning("Error in deleting old uploads: ~p", [Error]), + io:format("Error in deleting old uploads: ~p <<< \n", [Error]), Error end; @@ -332,6 +514,6 @@ maybe_cleanup_csbucket(RcPidForOneBucket, BucketName, {ok, RiakObj}, Timestamp) maybe_cleanup_csbucket(_, _, {error, notfound}, _) -> ok; maybe_cleanup_csbucket(_, BucketName, {error, _} = Error, _) -> - lager:error("~p on processing ~s", [Error, BucketName]), + logger:error("~p on processing ~s", [Error, BucketName]), io:format("Error: ~p on processing ~s\n", [Error, BucketName]), Error. diff --git a/src/riak_cs_copy_object.erl b/apps/riak_cs/src/riak_cs_copy_object.erl similarity index 78% rename from src/riak_cs_copy_object.erl rename to apps/riak_cs/src/riak_cs_copy_object.erl index 48375be41..9f65124ae 100644 --- a/src/riak_cs_copy_object.erl +++ b/apps/riak_cs/src/riak_cs_copy_object.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -25,44 +26,49 @@ -module(riak_cs_copy_object). -include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). -export([test_condition_and_permission/4, malformed_request/1, new_metadata/2, get_copy_source/1, - start_put_fsm/7, + start_put_fsm/8, copy/4, copy/5, copy_range/2]). %% Do not use this -export([connection_checker/1]). --include_lib("webmachine/include/webmachine.hrl"). - %% @doc tests condition and permission on source object: %% * x-amz-copy-source-if-* stuff %% * bucket/object acl %% * bucket policy --spec test_condition_and_permission(riak_client(), lfs_manifest(), #wm_reqdata{}, #context{}) -> - {boolean()|{halt, non_neg_integer()}, #wm_reqdata{}, #context{}}. +-spec test_condition_and_permission(riak_client(), lfs_manifest(), #wm_reqdata{}, #rcs_web_context{}) -> + {boolean()|{halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. test_condition_and_permission(RcPid, SrcManifest, RD, Ctx) -> ETag = riak_cs_manifest:etag(SrcManifest), - LastUpdate = SrcManifest?MANIFEST.created, + LastUpdate = rts:iso8601_s(SrcManifest?MANIFEST.write_start_time), %% TODO: write tests around these conditions, any kind of test is okay case condition_check(RD, ETag, LastUpdate) of ok -> - authorize_on_src(RcPid, SrcManifest, RD, Ctx); + case {authorize_on_src(RcPid, SrcManifest, RD, Ctx), + authorize_on_dst(RD, Ctx)} of + {{false, _, _} = Allow, {false, _, _}} -> + Allow; + {_1, _2} -> + ?LOG_DEBUG("put_copy denied due to either ~p (src) or ~p (dst)", [_1, _2]), + riak_cs_wm_utils:deny_access(RD, Ctx) + end; Other -> {Other, RD, Ctx} end. %% @doc tests permission on acl, policy --spec authorize_on_src(riak_client(), lfs_manifest(), #wm_reqdata{}, #context{}) -> - {boolean()|{halt, non_neg_integer()}, #wm_reqdata{}, #context{}}. authorize_on_src(RcPid, SrcManifest, RD, - #context{auth_module=AuthMod, local_context=LocalCtx} = Ctx) -> + #rcs_web_context{auth_module = AuthMod, + local_context = LocalCtx} = Ctx) -> {SrcBucket, SrcKey} = SrcManifest?MANIFEST.bkey, {ok, SrcBucketObj} = riak_cs_bucket:fetch_bucket_object(SrcBucket, RcPid), {ok, SrcBucketAcl} = riak_cs_acl:bucket_acl(SrcBucketObj), @@ -76,23 +82,23 @@ authorize_on_src(RcPid, SrcManifest, RD, {undefined, undefined} end, - _ = lager:debug("UserKey: ~p / ~p", [UserKey, User]), - %% Build fake context for checking read access to the src object - OtherLocalCtx = LocalCtx#key_context{bucket=SrcBucket, - key=binary_to_list(SrcKey), - bucket_object=SrcBucketObj, - manifest=SrcManifest}, - OtherCtx = Ctx#context{local_context=OtherLocalCtx, - acl=SrcBucketAcl, - user=User, user_object=UserObj}, + OtherLocalCtx = LocalCtx#key_context{bucket = SrcBucket, + key = SrcKey, + bucket_object = SrcBucketObj, + manifest = SrcManifest}, + OtherCtx = Ctx#rcs_web_context{local_context = OtherLocalCtx, + acl = SrcBucketAcl, + user = User, + bucket = SrcBucket, + user_object = UserObj}, %% Build fake ReqData for checking read access to the src object %% TODO: use unicode module for for key? or is this encoded? Path = string:join(["/buckets", binary_to_list(SrcBucket), "objects", binary_to_list(SrcKey)], "/"), - _ = lager:debug("raw path: ~p => ~p", [wrq:raw_path(RD), Path]), - _ = lager:debug("src bucket obj: ~p", [SrcBucketObj]), + ?LOG_DEBUG("raw path: ~p => ~p", [wrq:raw_path(RD), Path]), + ?LOG_DEBUG("src bucket obj: ~p", [SrcBucketObj]), OtherRD0 = wrq:create('GET', wrq:version(RD), Path, wrq:req_headers(RD)), OtherRD = wrq:load_dispatch_data([{bucket, binary_to_list(SrcBucket)}, {object, binary_to_list(SrcKey)}], @@ -100,8 +106,12 @@ authorize_on_src(RcPid, SrcManifest, RD, wrq:app_root(RD), wrq:disp_path(RD), OtherRD0), %% check the permission on ACL and Policy - _ = riak_cs_wm_utils:object_access_authorize_helper(object, false, - OtherRD, OtherCtx). + riak_cs_wm_utils:object_access_authorize_helper(object, false, + OtherRD, OtherCtx). + +authorize_on_dst(RD, Ctx) -> + riak_cs_wm_utils:object_access_authorize_helper(object, false, + RD, Ctx). -spec malformed_request(#wm_reqdata{}) -> false | @@ -130,13 +140,14 @@ new_metadata(SrcManifest, RD) -> orddict:to_list(SrcManifest?MANIFEST.metadata)} end. --spec start_put_fsm(binary(), binary(), non_neg_integer(), binary(), +-spec start_put_fsm(binary(), binary(), binary(), non_neg_integer(), binary(), proplists:proplist(), acl(), riak_client()) -> {ok, pid()}. -start_put_fsm(Bucket, Key, ContentLength, ContentType, Metadata, Acl, RcPid) -> +start_put_fsm(Bucket, Key, Vsn, ContentLength, ContentType, Metadata, Acl, RcPid) -> BlockSize = riak_cs_lfs_utils:block_size(), riak_cs_put_fsm_sup:start_put_fsm(node(), [{Bucket, Key, + Vsn, ContentLength, ContentType, Metadata, @@ -150,7 +161,7 @@ start_put_fsm(Bucket, Key, ContentLength, ContentType, Metadata, Acl, RcPid) -> %% @doc check "x-amz-copy-source" to know whether it requests copying %% from another object -spec get_copy_source(#wm_reqdata{}) -> undefined | - {binary(), binary()} | + {binary(), binary(), binary()} | {error, atom()}. get_copy_source(RD) -> %% for oos (TODO) @@ -166,22 +177,13 @@ handle_copy_source([$/|Path]) -> handle_copy_source(Path); handle_copy_source(Path0) when is_list(Path0) -> Path = mochiweb_util:unquote(Path0), - Length = length(Path), - case string:chr(Path, $/) of - 0 -> - {error, bad_request}; - Length -> - {error, bad_request}; - Offset -> - Bucket = string:substr(Path, 1, Offset-1), - SplitKey = string:substr(Path, Offset+1), - case SplitKey of - [] -> - {error, bad_request}; - _ -> - {iolist_to_binary(Bucket), - iolist_to_binary(SplitKey)} - end + case re:split(Path, "/+", [{return, binary}]) of + [Bucket, Key] when size(Key) > 0 -> + {Bucket, Key, ?LFS_DEFAULT_OBJECT_VERSION}; + [Bucket, Key, VId] when size(VId) > 0 -> + {Bucket, Key, VId}; + _ -> + {error, bad_request} end. %% @doc runs copy @@ -197,12 +199,12 @@ copy(PutFsmPid, SrcManifest, ReadRcPid, ContFun) -> %% disconnect the TCP connection while waiting for copying large %% objects. But mochiweb/webmachine cannot notify nor stop copying, %% thus some fd watcher is expected here. --spec copy(pid(), lfs_manifest(), riak_client(), fun(()->boolean()), - {non_neg_integer(), non_neg_integer()}|fail|undefined) -> +-spec copy(pid(), lfs_manifest(), riak_client(), fun(() -> boolean()), + {non_neg_integer(), non_neg_integer()} | fail | undefined) -> {ok, DstManifest::lfs_manifest()} | {error, term()}. copy(_, _, _, _, fail) -> {error, bad_request}; -copy(PutFsmPid, ?MANIFEST{content_length=0} = _SrcManifest, +copy(PutFsmPid, ?MANIFEST{content_length = 0} = _SrcManifest, _ReadRcPid, _, _) -> %% Zero-size copy will successfully create zero-size object riak_cs_put_fsm:finalize(PutFsmPid, undefined); @@ -226,14 +228,11 @@ copy(PutFsmPid, SrcManifest, ReadRcPid, ContFun, Range0) -> riak_cs_get_fsm:continue(GetFsmPid, Range), get_and_put(GetFsmPid, PutFsmPid, MD5, ContFun). --spec get_content_md5(tuple() | binary()) -> string(). get_content_md5({_MD5, _Str}) -> undefined; get_content_md5(MD5) -> - base64:encode_to_string(MD5). + base64:encode(MD5). --spec get_and_put(pid(), pid(), list(), fun(() -> boolean())) - -> {ok, lfs_manifest()} | {error, term()}. get_and_put(GetPid, PutPid, MD5, ContFun) -> case ContFun() of true -> @@ -246,20 +245,21 @@ get_and_put(GetPid, PutPid, MD5, ContFun) -> get_and_put(GetPid, PutPid, MD5, ContFun) end; false -> - _ = lager:debug("Connection lost during a copy", []), + logger:warning("Connection lost during a copy", []), catch riak_cs_get_fsm:stop(GetPid), catch riak_cs_put_fsm:force_stop(PutPid), {error, connection_lost} end. -spec start_get_fsm(lfs_manifest(), riak_client()) -> {ok, pid()}. -start_get_fsm(SrcManifest, ReadRcPid) -> - {Bucket, Key} = SrcManifest?MANIFEST.bkey, +start_get_fsm(?MANIFEST{bkey = {Bucket, Key}, + vsn = Vsn}, ReadRcPid) -> FetchConcurrency = riak_cs_lfs_utils:fetch_concurrency(), BufferFactor = riak_cs_lfs_utils:get_fsm_buffer_size_factor(), riak_cs_get_fsm_sup:start_get_fsm(node(), Bucket, Key, + Vsn, self(), ReadRcPid, FetchConcurrency, @@ -332,16 +332,17 @@ copy_range(RD, ?MANIFEST{content_length=Len}) -> end. -%% @doc nasty hack, do not use this other than for disconnect +%% @doc nasty hack, do not use this other than for disconnect %% detection in copying objects. --spec connection_checker(inet:socket()) -> fun(() -> boolean()). +-type mochiweb_socket() :: inet:socket() | {ssl, ssl:sslsocket()}. +-spec connection_checker(mochiweb_socket()) -> fun(() -> boolean()). connection_checker(Socket) -> fun() -> - case inet:peername(Socket) of + case mochiweb_socket:peername(Socket) of {error,_E} -> false; {ok,_} -> - case gen_tcp:recv(Socket, 0, 0) of + case mochiweb_socket:recv(Socket, 0, 0) of {error, timeout} -> true; {error, _E} -> diff --git a/src/riak_cs_delete_fsm.erl b/apps/riak_cs/src/riak_cs_delete_fsm.erl similarity index 64% rename from src/riak_cs_delete_fsm.erl rename to apps/riak_cs/src/riak_cs_delete_fsm.erl index 7e825c82a..043c1a100 100644 --- a/src/riak_cs_delete_fsm.erl +++ b/apps/riak_cs/src/riak_cs_delete_fsm.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -25,6 +26,7 @@ -behaviour(gen_fsm). -include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -58,8 +60,8 @@ %% For active deletion, not used. %% Set only once at init and unchanged. Used only for logs. gc_key :: binary(), - delete_blocks_remaining :: ordsets:ordset({binary(), integer()}), - unacked_deletes=ordsets:new() :: ordsets:ordset(integer()), + delete_blocks_remaining :: undefined | ordsets:ordset({binary(), integer()}), + unacked_deletes = ordsets:new() :: ordsets:ordset(integer()), all_delete_workers=[] :: list(pid()), free_deleters = ordsets:new() :: ordsets:ordset(pid()), deleted_blocks = 0 :: non_neg_integer(), @@ -93,21 +95,20 @@ block_deleted(Pid, Response) -> %% gen_fsm callbacks %% ==================================================================== -init([BagId, {UUID, Manifest}, FinishedCallback, GCKey, Options]) -> - {Bucket, Key} = Manifest?MANIFEST.bkey, +init([BagId, {UUID, Manifest = ?MANIFEST{bkey = {Bucket, Key}}}, FinishedCallback, GCKey, Options]) -> {ok, RcPid} = riak_cs_riak_client:checkout(), ok = riak_cs_riak_client:set_manifest_bag(RcPid, BagId), ok = riak_cs_riak_client:set_manifest(RcPid, Manifest), CleanupManifests = proplists:get_value(cleanup_manifests, Options, true), - State = #state{bucket=Bucket, - key=Key, - manifest=Manifest, - uuid=UUID, - riak_client=RcPid, - finished_callback=FinishedCallback, - gc_key=GCKey, - cleanup_manifests=CleanupManifests}, + State = #state{bucket = Bucket, + key = Key, + manifest = Manifest, + uuid = UUID, + riak_client = RcPid, + finished_callback = FinishedCallback, + gc_key = GCKey, + cleanup_manifests = CleanupManifests}, {ok, prepare, State}. %% @TODO Make sure we avoid any race conditions here @@ -116,15 +117,15 @@ prepare(delete, State) -> handle_receiving_manifest(State). prepare(delete, From, State) -> - handle_receiving_manifest(State#state{caller=From}). + handle_receiving_manifest(State#state{caller = From}). deleting({block_deleted, {ok, BlockID}, DeleterPid}, - State=#state{deleted_blocks=DeletedBlocks}) -> + State = #state{deleted_blocks = DeletedBlocks}) -> UpdState = deleting_state_update(BlockID, DeleterPid, DeletedBlocks+1, State), ManifestState = UpdState#state.manifest?MANIFEST.state, deleting_state_result(ManifestState, UpdState); deleting({block_deleted, {error, {unsatisfied_constraint, _, BlockID}}, DeleterPid}, - State=#state{deleted_blocks=DeletedBlocks}) -> + State = #state{deleted_blocks = DeletedBlocks}) -> UpdState = deleting_state_update(BlockID, DeleterPid, DeletedBlocks, State), ManifestState = UpdState#state.manifest?MANIFEST.state, deleting_state_result(ManifestState, UpdState); @@ -141,15 +142,15 @@ handle_info(_Info, StateName, State) -> {next_state, StateName, State}. terminate(Reason, _StateName, - #state{all_delete_workers=AllDeleteWorkers, - manifest=?MANIFEST{state=ManifestState}, - bucket=Bucket, - key=Key, - uuid=UUID, - riak_client=RcPid, - cleanup_manifests=CleanupManifests} = State) -> + #state{all_delete_workers = AllDeleteWorkers, + manifest = ?MANIFEST{state = ManifestState, vsn = ObjVsn}, + bucket = Bucket, + key = Key, + uuid = UUID, + riak_client = RcPid, + cleanup_manifests = CleanupManifests} = State) -> if CleanupManifests -> - manifest_cleanup(ManifestState, Bucket, Key, UUID, RcPid); + manifest_cleanup(ManifestState, Bucket, Key, ObjVsn, UUID, RcPid); true -> noop end, @@ -167,20 +168,20 @@ code_change(_OldVsn, StateName, State, _Extra) -> %% @doc Update the state record following notification of the %% completion of a block deletion. -spec deleting_state_update(pos_integer(), pid(), non_neg_integer(), #state{}) -> - #state{}. + #state{}. deleting_state_update(BlockID, DeleterPid, DeletedBlocks, - State=#state{manifest=Manifest, - free_deleters=FreeDeleters, - unacked_deletes=UnackedDeletes}) -> + State = #state{manifest = Manifest, + free_deleters = FreeDeleters, + unacked_deletes = UnackedDeletes}) -> NewManifest = riak_cs_lfs_utils:remove_delete_block(Manifest, BlockID), - State#state{free_deleters=ordsets:add_element(DeleterPid, - FreeDeleters), - unacked_deletes=ordsets:del_element(BlockID, - UnackedDeletes), - manifest=NewManifest, - deleted_blocks=DeletedBlocks}. + State#state{free_deleters = ordsets:add_element(DeleterPid, + FreeDeleters), + unacked_deletes = ordsets:del_element(BlockID, + UnackedDeletes), + manifest = NewManifest, + deleted_blocks = DeletedBlocks}. %% @doc Determine the appropriate `deleting' state %% fsm callback result based on the given manifest state. @@ -193,20 +194,21 @@ deleting_state_result(_, State) -> -spec handle_receiving_manifest(state()) -> {next_state, atom(), state()} | {stop, normal, state()}. -handle_receiving_manifest(State=#state{manifest=Manifest, - gc_key=GCKey}) -> +handle_receiving_manifest(State = #state{manifest = Manifest, + gc_key = GCKey}) -> case blocks_to_delete_from_manifest(Manifest) of {ok, {NewManifest, BlocksToDelete}} -> BlockCount = ordsets:size(BlocksToDelete), - NewState = State#state{manifest=NewManifest, - delete_blocks_remaining=BlocksToDelete, - total_blocks=BlockCount}, + NewState = State#state{manifest = NewManifest, + delete_blocks_remaining = BlocksToDelete, + total_blocks = BlockCount}, start_block_servers(NewState); {error, invalid_state} -> {Bucket, Key} = Manifest?MANIFEST.bkey, - _ = lager:warning("Invalid state manifest in GC bucket at ~p, " - "bucket=~p key=~p: ~p", - [GCKey, Bucket, Key, Manifest]), + ObjVsn = Manifest?MANIFEST.vsn, + logger:warning("Invalid state manifest in GC bucket at ~p, " + "b/k:v \"~s/~s:~s\": ~p", + [GCKey, Bucket, Key, ObjVsn, Manifest]), %% If total blocks and deleted blocks are the same, %% gc worker attempt to delete the manifest in fileset. %% Then manifests and blocks becomes orphan. @@ -214,14 +216,14 @@ handle_receiving_manifest(State=#state{manifest=Manifest, %% For now delete FSM stops with pseudo-normal termination to %% let other valid manifests be collected, as the root cause %% of #827 is still unidentified. - {stop, normal, State#state{total_blocks=1}} + {stop, normal, State#state{total_blocks = 1}} end. -spec start_block_servers(state()) -> {next_state, atom(), state()} | {stop, normal, state()}. -start_block_servers(#state{riak_client=RcPid, - manifest=Manifest, - delete_blocks_remaining=BlocksToDelete} = State) -> +start_block_servers(#state{riak_client = RcPid, + manifest = Manifest, + delete_blocks_remaining = BlocksToDelete} = State) -> %% Handle the case where there are 0 blocks to delete, %% i.e. content length of 0, %% and can not check-out any workers. @@ -236,8 +238,8 @@ start_block_servers(#state{riak_client=RcPid, {stop, normal, State}; _ -> FreeDeleters = ordsets:from_list(AllDeleteWorkers), - NewState = State#state{all_delete_workers=AllDeleteWorkers, - free_deleters=FreeDeleters}, + NewState = State#state{all_delete_workers = AllDeleteWorkers, + free_deleters = FreeDeleters}, StateAfterDeleteStart = maybe_delete_blocks(NewState), {next_state, deleting, StateAfterDeleteStart} end; @@ -245,62 +247,61 @@ start_block_servers(#state{riak_client=RcPid, {stop, normal, State} end. -maybe_delete_blocks(State=#state{free_deleters=[]}) -> +maybe_delete_blocks(State = #state{free_deleters = []}) -> State; -maybe_delete_blocks(State=#state{delete_blocks_remaining=[]}) -> +maybe_delete_blocks(State = #state{delete_blocks_remaining = []}) -> State; -maybe_delete_blocks(State=#state{bucket=Bucket, - key=Key, - free_deleters=FreeDeleters=[DeleterPid | _Rest], - unacked_deletes=UnackedDeletes, - delete_blocks_remaining=DeleteBlocksRemaining= - [BlockID | _RestBlocks]}) -> +maybe_delete_blocks(State = #state{bucket = Bucket, + key = Key, + manifest = ?MANIFEST{vsn = ObjVsn}, + free_deleters = FreeDeleters = [DeleterPid | _Rest], + unacked_deletes = UnackedDeletes, + delete_blocks_remaining = DeleteBlocksRemaining = [BlockID | _RestBlocks]}) -> NewUnackedDeletes = ordsets:add_element(BlockID, UnackedDeletes), NewDeleteBlocksRemaining = ordsets:del_element(BlockID, DeleteBlocksRemaining), {UUID, Seq} = BlockID, - _ = lager:debug("Deleting block: ~p ~p ~p ~p", [Bucket, Key, UUID, Seq]), + ?LOG_DEBUG("Deleting block #~b (~p) of \"~s/~s:~s\"", [Seq, UUID, Bucket, Key, ObjVsn]), riak_cs_block_server:delete_block(DeleterPid, Bucket, Key, UUID, Seq), NewFreeDeleters = ordsets:del_element(DeleterPid, FreeDeleters), - maybe_delete_blocks(State#state{unacked_deletes=NewUnackedDeletes, - free_deleters=NewFreeDeleters, - delete_blocks_remaining=NewDeleteBlocksRemaining}). + maybe_delete_blocks(State#state{unacked_deletes = NewUnackedDeletes, + free_deleters = NewFreeDeleters, + delete_blocks_remaining = NewDeleteBlocksRemaining}). -reply_or_callback(Reason, #state{caller=Caller}=State) when Caller =/= undefined -> +reply_or_callback(Reason, #state{caller = Caller} = State) when Caller =/= undefined -> gen_fsm:reply(Caller, notification_msg(Reason, State)); -reply_or_callback(Reason, #state{finished_callback=Callback}=State) -> +reply_or_callback(Reason, #state{finished_callback = Callback} = State) -> Callback(notification_msg(Reason, State)). --spec notification_msg(term(), state()) -> {pid(), - {ok, {non_neg_integer(), non_neg_integer()}} | - {error, term()}}. -notification_msg(normal, #state{ - bucket=Bucket, - key=Key, - uuid=UUID, - deleted_blocks = DeletedBlocks, - total_blocks = TotalBlocks}) -> - Reply = {ok, {Bucket, Key, UUID, DeletedBlocks, TotalBlocks}}, +-spec notification_msg(term(), state()) -> + {pid(), {ok, {non_neg_integer(), non_neg_integer()}} | {error, term()}}. +notification_msg(normal, #state{bucket = Bucket, + key = Key, + manifest = ?MANIFEST{vsn = ObjVsn}, + uuid = UUID, + deleted_blocks = DeletedBlocks, + total_blocks = TotalBlocks}) -> + Reply = {ok, {Bucket, Key, ObjVsn, UUID, DeletedBlocks, TotalBlocks}}, {self(), Reply}; notification_msg(Reason, _State) -> {self(), {error, Reason}}. --spec manifest_cleanup(atom(), binary(), binary(), binary(), riak_client()) -> ok. -manifest_cleanup(deleted, Bucket, Key, UUID, RcPid) -> - {ok, ManiFsmPid} = riak_cs_manifest_fsm:start_link(Bucket, Key, RcPid), +-spec manifest_cleanup(atom(), binary(), binary(), binary(), binary(), riak_client()) -> ok. +manifest_cleanup(deleted, Bucket, Key, ObjVsn, UUID, RcPid) -> + {ok, ManiFsmPid} = riak_cs_manifest_fsm:start_link(Bucket, Key, ObjVsn, RcPid), _ = try _ = riak_cs_manifest_fsm:delete_specific_manifest(ManiFsmPid, UUID) after _ = riak_cs_manifest_fsm:stop(ManiFsmPid) end, ok; -manifest_cleanup(_, _, _, _, _) -> +manifest_cleanup(_, _, _, _, _, _) -> ok. -spec blocks_to_delete_from_manifest(lfs_manifest()) -> - {ok, {lfs_manifest(), ordsets:ordset(integer())}} | - {error, term()}. -blocks_to_delete_from_manifest(Manifest=?MANIFEST{state=State, - delete_blocks_remaining=undefined}) + {ok, {lfs_manifest(), ordsets:ordset(integer())}} | + {error, term()}. +blocks_to_delete_from_manifest(Manifest = ?MANIFEST{state = State, + delete_blocks_remaining = undefined}) when State =:= pending_delete;State =:= writing; State =:= scheduled_delete -> {UpdState, Blocks} = case riak_cs_lfs_utils:block_sequences_for_manifest(Manifest) of @@ -310,10 +311,10 @@ blocks_to_delete_from_manifest(Manifest=?MANIFEST{state=State, {State, BlockSequence} end, - UpdManifest = Manifest?MANIFEST{delete_blocks_remaining=Blocks, - state=UpdState}, + UpdManifest = Manifest?MANIFEST{delete_blocks_remaining = Blocks, + state = UpdState}, {ok, {UpdManifest, Blocks}}; -blocks_to_delete_from_manifest(?MANIFEST{delete_blocks_remaining=undefined}) -> +blocks_to_delete_from_manifest(?MANIFEST{delete_blocks_remaining = undefined}) -> {error, invalid_state}; blocks_to_delete_from_manifest(Manifest) -> {ok, {Manifest, Manifest?MANIFEST.delete_blocks_remaining}}. diff --git a/src/riak_cs_delete_fsm_sup.erl b/apps/riak_cs/src/riak_cs_delete_fsm_sup.erl similarity index 97% rename from src/riak_cs_delete_fsm_sup.erl rename to apps/riak_cs/src/riak_cs_delete_fsm_sup.erl index 370303614..79d468283 100644 --- a/src/riak_cs_delete_fsm_sup.erl +++ b/apps/riak_cs/src/riak_cs_delete_fsm_sup.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/src/riak_cs_diags.erl b/apps/riak_cs/src/riak_cs_diags.erl similarity index 58% rename from src/riak_cs_diags.erl rename to apps/riak_cs/src/riak_cs_diags.erl index 6d21a98d0..5d1ec6c4b 100644 --- a/src/riak_cs_diags.erl +++ b/apps/riak_cs/src/riak_cs_diags.erl @@ -1,12 +1,41 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + -module(riak_cs_diags). -behaviour(gen_server). +-dialyzer([ {nowarn_function, pr/1} + , {nowarn_function, pr/2} + , {nowarn_function, rec_to_proplist/1} + , {nowarn_function, print_field/2} + , {nowarn_function, spaces/1} + , {nowarn_function, print_multipart_manifest/2} + ]). + -include("riak_cs.hrl"). %% API -export([start_link/0, - print_manifests/2, - print_manifest/3]). + print_manifests/3, + print_manifest/4]). -define(INDENT_LEVEL, 4). @@ -20,21 +49,21 @@ start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). --spec print_manifests(binary() | string(), binary() | string()) -> ok. -print_manifests(Bucket, Key) when is_list(Bucket), is_list(Key) -> - print_manifests(list_to_binary(Bucket), list_to_binary(Key)); -print_manifests(Bucket, Key) -> - Manifests = gen_server:call(?MODULE, {get_manifests, Bucket, Key}), +-spec print_manifests(binary() | string(), binary() | string(), binary() | string()) -> ok. +print_manifests(Bucket, Key, Vsn) when is_list(Bucket) -> + print_manifests(list_to_binary(Bucket), list_to_binary(Key), list_to_binary(Vsn)); +print_manifests(Bucket, Key, Vsn) -> + Manifests = gen_server:call(?MODULE, {get_manifests, Bucket, Key, Vsn}), Rows = manifest_rows(orddict_values(Manifests)), table:print(manifest_table_spec(), Rows). --spec print_manifest(binary() | string(), binary() | string(), binary() | string()) -> ok. -print_manifest(Bucket, Key, Uuid) when is_list(Bucket), is_list(Key) -> - print_manifest(list_to_binary(Bucket), list_to_binary(Key), Uuid); -print_manifest(Bucket, Key, Uuid) when is_list(Uuid) -> - print_manifest(Bucket, Key, mochihex:to_bin(Uuid)); -print_manifest(Bucket, Key, Uuid) -> - Manifests = gen_server:call(?MODULE, {get_manifests, Bucket, Key}), +-spec print_manifest(binary() | string(), binary() | string(), binary() | string(), binary() | string()) -> ok. +print_manifest(Bucket, Key, Vsn, Uuid) when is_list(Bucket) -> + print_manifest(list_to_binary(Bucket), list_to_binary(Key), list_to_binary(Vsn), Uuid); +print_manifest(Bucket, Key, Vsn, Uuid) when is_list(Uuid) -> + print_manifest(Bucket, Key, Vsn, mochihex:to_bin(Uuid)); +print_manifest(Bucket, Key, Vsn, Uuid) -> + Manifests = gen_server:call(?MODULE, {get_manifests, Bucket, Key, Vsn}), {ok, Manifest} = orddict:find(Uuid, Manifests), io:format("\n~s", [pr(Manifest)]). @@ -44,10 +73,14 @@ pr(Record) -> -spec pr(tuple(), non_neg_integer()) -> iolist(). pr(Record, Indent) -> - {'$lager_record', RecordName, Zipped} = lager:pr(Record, ?MODULE), - ["\n", spaces(Indent), "#", atom_to_list(RecordName), - "\n", spaces(Indent), "--------------------\n", - [print_field(Field, Indent) || Field <- Zipped]]. + Zipped = rec_to_proplist(Record), + ["\n", spaces(Indent), "#", atom_to_list(hd(tuple_to_list(Record))), + "\n", spaces(Indent), "--------------------\n", + [print_field(FV, Indent) || FV <- Zipped]]. + +rec_to_proplist(R) -> + RFF = exprec:info(R), + lists:zip(RFF, tl(tuple_to_list(R))). print_field({_, undefined}, _) -> ""; @@ -62,10 +95,10 @@ print_field({part_id, Value}, Indent) when is_binary(Value) -> print_field({acl, Value}, Indent) -> io_lib:format("~s~s = ~s\n\n", [spaces(Indent), acl, pr(Value, Indent + 1)]); print_field({props, Props}, Indent) -> - io_lib:format("~s~s = ~s\n\n", [spaces(Indent), multipart, + io_lib:format("~s~s = ~s\n\n", [spaces(Indent), multipart, print_multipart_manifest(Props, Indent)]); print_field({parts, Parts}, Indent) -> - io_lib:format("~s~s = ~s\n\n", [spaces(Indent), parts, + io_lib:format("~s~s = ~s\n\n", [spaces(Indent), parts, [pr(P, Indent + 1) || P <- Parts]]); print_field({Key, Value}, Indent) -> io_lib:format("~s~s = ~p\n", [spaces(Indent), Key, Value]). @@ -80,18 +113,20 @@ spaces(Num) -> %% Table Specifications and Record to Row conversions %% ==================================================================== manifest_table_spec() -> - [{state, 20}, {deleted, 8}, {mp, 6}, {created, 28}, {uuid, 36}, + [{state, 20}, {deleted, 8}, {mp, 6}, {created, 28}, {uuid, 36}, {write_start_time, 23}, {delete_marked_time, 23}]. manifest_rows(Manifests) -> [ [M?MANIFEST.state, deleted(M?MANIFEST.props), - riak_cs_mp_utils:is_multipart_manifest(M), - M?MANIFEST.created, mochihex:to_hex(M?MANIFEST.uuid), + riak_cs_mp_utils:is_multipart_manifest(M), + rts:iso8601( + calendar:system_time_to_universal_time(M?MANIFEST.write_start_time, millisecond)), + mochihex:to_hex(M?MANIFEST.uuid), M?MANIFEST.write_start_time, M?MANIFEST.delete_marked_time] || M <- Manifests]. print_multipart_manifest(Props, Indent) -> case lists:keyfind(multipart, 1, Props) of - {multipart, MpManifest} -> + {multipart, MpManifest} -> pr(MpManifest, Indent + 1); _ -> "" @@ -107,10 +142,10 @@ deleted(Props) -> init([]) -> {ok, #state{}}. -handle_call({get_manifests, Bucket, Key}, _From, State) -> +handle_call({get_manifests, Bucket, Key, Vsn}, _From, State) -> {ok, Pid} = riak_cs_utils:riak_connection(), try - {ok, _, Manifests} = riak_cs_manifest:get_manifests(Pid, Bucket, Key), + {ok, _, Manifests} = riak_cs_manifest:get_manifests(Pid, Bucket, Key, Vsn), {reply, Manifests, State} catch _:_=E -> {reply, {error, E}, State} @@ -129,4 +164,3 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. - diff --git a/src/riak_cs_gc.erl b/apps/riak_cs/src/riak_cs_gc.erl similarity index 74% rename from src/riak_cs_gc.erl rename to apps/riak_cs/src/riak_cs_gc.erl index 361fa0989..19c06a48d 100644 --- a/src/riak_cs_gc.erl +++ b/apps/riak_cs/src/riak_cs_gc.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reseved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -23,8 +24,11 @@ -module(riak_cs_gc). -include("riak_cs_gc.hrl"). +-include_lib("kernel/include/logger.hrl"). + -ifdef(TEST). -compile(export_all). +-compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). -endif. @@ -35,18 +39,17 @@ initial_gc_delay/0, gc_retry_interval/0, gc_max_workers/0, - gc_active_manifests/3, - gc_specific_manifests/5, + gc_active_manifests/4, + gc_specific_manifests/4, epoch_start/0, leeway_seconds/0, set_leeway_seconds/1, max_scheduled_delete_manifests/0, move_manifests_to_gc_bucket/2, - timestamp/0, default_batch_end/2]). %% export for repl debugging and testing --export([get_active_manifests/3]). +-export([get_active_manifests/4]). %%%=================================================================== %%% Public API @@ -61,25 +64,25 @@ %% Note that any error is irrespective of the current position of the GC states. %% Some manifests may have been GC'd and then an error occurs. In this case the %% client will only get the error response. --spec gc_active_manifests(binary(), binary(), riak_client()) -> +-spec gc_active_manifests(binary(), binary(), binary(), riak_client()) -> {ok, [binary()]} | {error, term()}. -gc_active_manifests(Bucket, Key, RcPid) -> - gc_active_manifests(Bucket, Key, RcPid, []). +gc_active_manifests(Bucket, Key, ObjVsn, RcPid) -> + gc_active_manifests(Bucket, Key, ObjVsn, RcPid, []). %% @private --spec gc_active_manifests(binary(), binary(), riak_client(), [binary]) -> +-spec gc_active_manifests(binary(), binary(), binary(), riak_client(), [binary()]) -> {ok, [binary()]} | {error, term()}. -gc_active_manifests(Bucket, Key, RcPid, UUIDs) -> - case get_active_manifests(Bucket, Key, RcPid) of +gc_active_manifests(Bucket, Key, ObjVsn, RcPid, UUIDs) -> + case get_active_manifests(Bucket, Key, ObjVsn, RcPid) of {ok, _RiakObject, []} -> {ok, UUIDs}; {ok, RiakObject, Manifests} -> UnchangedManifests = clean_manifests(Manifests, RcPid), - case gc_manifests(UnchangedManifests, RiakObject, Bucket, Key, RcPid) of + case gc_manifests(UnchangedManifests, RiakObject, Bucket, RcPid) of {error, _}=Error -> Error; NewUUIDs -> - gc_active_manifests(Bucket, Key, RcPid, UUIDs ++ NewUUIDs) + gc_active_manifests(Bucket, Key, ObjVsn, RcPid, UUIDs ++ NewUUIDs) end; {error, notfound} -> {ok, UUIDs}; @@ -87,18 +90,19 @@ gc_active_manifests(Bucket, Key, RcPid, UUIDs) -> Error end. --spec get_active_manifests(binary(), binary(), riak_client()) -> +-spec get_active_manifests(binary(), binary(), binary(), riak_client()) -> {ok, riakc_obj:riakc_obj(), [lfs_manifest()]} | {error, term()}. -get_active_manifests(Bucket, Key, RcPid) -> - active_manifests(riak_cs_manifest:get_manifests(RcPid, Bucket, Key)). +get_active_manifests(Bucket, Key, Vsn, RcPid) -> + active_manifests( + riak_cs_manifest:get_manifests(RcPid, Bucket, Key, Vsn)). -spec active_manifests({ok, riakc_obj:riakc_obj(), [lfs_manifest()]}) -> {ok, riakc_obj:riakc_obj(), [lfs_manifest()]}; ({error, term()}) -> {error, term()}. active_manifests({ok, RiakObject, Manifests}) -> - {ok, RiakObject, riak_cs_manifest_utils:active_manifests(Manifests)}; -active_manifests({error, _}=Error) -> + {ok, RiakObject, rcs_common_manifest_utils:active_manifests(Manifests)}; +active_manifests({error, _} = Error) -> Error. -spec clean_manifests([lfs_manifest()], riak_client()) -> [lfs_manifest()]. @@ -117,27 +121,25 @@ is_multipart_clean(updated) -> -spec gc_manifests(Manifests :: [lfs_manifest()], RiakObject :: riakc_obj:riakc_obj(), Bucket :: binary(), - Key :: binary(), RcPid :: riak_client()) -> [binary()] | {error, term()}. -gc_manifests(Manifests, RiakObject, Bucket, Key, RcPid) -> - F = fun(_M, {error, _}=Error) -> +gc_manifests(Manifests, RiakObject, Bucket, RcPid) -> + F = fun(_M, {error, _} = Error) -> Error; (M, UUIDs) -> - gc_manifest(M, RiakObject, Bucket, Key, RcPid, UUIDs) + gc_manifest(M, RiakObject, Bucket, RcPid, UUIDs) end, lists:foldl(F, [], Manifests). -spec gc_manifest(M :: lfs_manifest(), RiakObject :: riakc_obj:riakc_obj(), Bucket :: binary(), - Key :: binary(), RcPid :: riak_client(), UUIDs :: [binary()]) -> [binary()] | no_return(). -gc_manifest(M, RiakObject, Bucket, Key, RcPid, UUIDs) -> +gc_manifest(M, RiakObject, Bucket, RcPid, UUIDs) -> UUID = M?MANIFEST.uuid, - check(gc_specific_manifests_to_delete([UUID], RiakObject, Bucket, Key, RcPid), [UUID | UUIDs]). + check(gc_specific_manifests_to_delete([UUID], RiakObject, Bucket, RcPid), [UUID | UUIDs]). check({ok, _}, Val) -> Val; @@ -147,39 +149,36 @@ check({error, _}=Error, _Val) -> -spec gc_specific_manifests_to_delete(UUIDsToMark :: [binary()], RiakObject :: riakc_obj:riakc_obj(), Bucket :: binary(), - Key :: binary(), RcPid :: riak_client()) -> {error, term()} | {ok, riakc_obj:riakc_obj()}. -gc_specific_manifests_to_delete(UUIDsToMark, RiakObject, Bucket, Key, RcPid) -> - MarkedResult = mark_as_deleted(UUIDsToMark, RiakObject, Bucket, Key, RcPid), - handle_mark_as_pending_delete(MarkedResult, Bucket, Key, UUIDsToMark, RcPid). +gc_specific_manifests_to_delete(UUIDsToMark, RiakObject, Bucket, RcPid) -> + MarkedResult = mark_as_deleted(UUIDsToMark, RiakObject, Bucket, RcPid), + handle_mark_as_pending_delete(MarkedResult, Bucket, UUIDsToMark, RcPid). %% @private -spec gc_specific_manifests(UUIDsToMark :: [binary()], - RiakObject :: riakc_obj:riakc_obj(), - Bucket :: binary(), - Key :: binary(), - RcPid :: riak_client()) -> + RiakObject :: riakc_obj:riakc_obj(), + Bucket :: binary(), + RcPid :: riak_client()) -> {error, term()} | {ok, riakc_obj:riakc_obj()}. -gc_specific_manifests([], RiakObject, _Bucket, _Key, _RcPid) -> +gc_specific_manifests([], RiakObject, _Bucket, _RcPid) -> {ok, RiakObject}; -gc_specific_manifests(UUIDsToMark, RiakObject, Bucket, Key, RcPid) -> +gc_specific_manifests(UUIDsToMark, RiakObject, Bucket, RcPid) -> MarkedResult = mark_as_pending_delete(UUIDsToMark, RiakObject, - Bucket, Key, + Bucket, RcPid), - handle_mark_as_pending_delete(MarkedResult, Bucket, Key, UUIDsToMark, RcPid). + handle_mark_as_pending_delete(MarkedResult, Bucket, UUIDsToMark, RcPid). %% @private -spec handle_mark_as_pending_delete({ok, riakc_obj:riakc_obj()}, - binary(), binary(), + binary(), [binary()], riak_client()) -> - {error, term()} | {ok, riakc_obj:riakc_obj()}; - - ({error, term()}, binary(), binary(), [binary()], riak_client()) -> - {error, term()} | {ok, riakc_obj:riakc_obj()}. -handle_mark_as_pending_delete({ok, RiakObject}, Bucket, Key, UUIDsToMark, RcPid) -> + {error, term()} | {ok, riakc_obj:riakc_obj()}; + ({error, term()}, binary(), [binary()], riak_client()) -> + {error, term()} | {ok, riakc_obj:riakc_obj()}. +handle_mark_as_pending_delete({ok, RiakObject}, Bucket, UUIDsToMark, RcPid) -> Manifests = riak_cs_manifest:manifests_from_riak_object(RiakObject), PDManifests0 = riak_cs_manifest_utils:manifests_to_gc(UUIDsToMark, Manifests), {ToGC, DeletedUUIDs} = @@ -212,13 +211,13 @@ handle_mark_as_pending_delete({ok, RiakObject}, Bucket, Key, UUIDsToMark, RcPid) case move_manifests_to_gc_bucket(ToGC, RcPid) of ok -> PDUUIDs = [UUID || {UUID, _} <- ToGC], - mark_as_scheduled_delete(PDUUIDs ++ DeletedUUIDs, RiakObject, Bucket, Key, RcPid); + mark_as_scheduled_delete(PDUUIDs ++ DeletedUUIDs, RiakObject, Bucket, RcPid); {error, _} = Error -> Error end; -handle_mark_as_pending_delete({error, _Error}=Error, _Bucket, _Key, _UUIDsToMark, _RcPid) -> - _ = lager:warning("Failed to mark as pending_delete, reason: ~p", [Error]), +handle_mark_as_pending_delete({error, _Error}=Error, _Bucket, _UUIDsToMark, _RcPid) -> + logger:warning("Failed to mark as pending_delete, reason: ~p", [Error]), Error. %% @doc Return the number of seconds to wait after finishing garbage @@ -273,7 +272,7 @@ gc_max_workers() -> end. %% @doc Return the start of GC epoch represented as a binary. -%% This is the time that the GC manager uses to begin collecting keys +%% This is the time that the GC manager uses to begin collecting keys %% from the `riak-cs-gc' bucket. -spec epoch_start() -> non_neg_integer(). epoch_start() -> @@ -316,15 +315,6 @@ max_scheduled_delete_manifests() -> MaxCount end. -%% @doc Generate a key for storing a set of manifests for deletion. --spec timestamp() -> non_neg_integer(). -timestamp() -> - timestamp(os:timestamp()). - --spec timestamp(erlang:timestamp()) -> non_neg_integer(). -timestamp(ErlangTime) -> - riak_cs_utils:second_resolution_timestamp(ErlangTime). - -spec default_batch_end(non_neg_integer(), non_neg_integer()) -> non_neg_integer(). default_batch_end(BatchStart, Leeway) -> BatchStart - Leeway. @@ -336,44 +326,36 @@ default_batch_end(BatchStart, Leeway) -> %% @doc Mark a list of manifests as `pending_delete' based upon the %% UUIDs specified, and also add {deleted, true} to the props member %% to signify an actual delete, and not an overwrite. --spec mark_as_deleted([binary()], riakc_obj:riakc_obj(), binary(), binary(), riak_client()) -> - {ok, riakc_obj:riakc_obj()} | {error, term()}. -mark_as_deleted(UUIDsToMark, RiakObject, Bucket, Key, RcPid) -> - mark_manifests(RiakObject, Bucket, Key, UUIDsToMark, - fun riak_cs_manifest_utils:mark_deleted/2, +mark_as_deleted(UUIDsToMark, RiakObject, Bucket, RcPid) -> + mark_manifests(RiakObject, Bucket, UUIDsToMark, + fun rcs_common_manifest_utils:mark_deleted/2, RcPid). %% @doc Mark a list of manifests as `pending_delete' based upon the %% UUIDs specified. --spec mark_as_pending_delete([binary()], riakc_obj:riakc_obj(), binary(), binary(), riak_client()) -> - {ok, riakc_obj:riakc_obj()} | {error, term()}. -mark_as_pending_delete(UUIDsToMark, RiakObject, Bucket, Key, RcPid) -> - mark_manifests(RiakObject, Bucket, Key, UUIDsToMark, - fun riak_cs_manifest_utils:mark_pending_delete/2, +mark_as_pending_delete(UUIDsToMark, RiakObject, Bucket, RcPid) -> + mark_manifests(RiakObject, Bucket, UUIDsToMark, + fun rcs_common_manifest_utils:mark_pending_delete/2, RcPid). %% @doc Mark a list of manifests as `scheduled_delete' based upon the %% UUIDs specified. --spec mark_as_scheduled_delete([cs_uuid()], riakc_obj:riakc_obj(), binary(), binary(), riak_client()) -> - {ok, riakc_obj:riakc_obj()} | {error, term()}. -mark_as_scheduled_delete(UUIDsToMark, RiakObject, Bucket, Key, RcPid) -> - mark_manifests(RiakObject, Bucket, Key, UUIDsToMark, - fun riak_cs_manifest_utils:mark_scheduled_delete/2, +mark_as_scheduled_delete(UUIDsToMark, RiakObject, Bucket, RcPid) -> + mark_manifests(RiakObject, Bucket, UUIDsToMark, + fun rcs_common_manifest_utils:mark_scheduled_delete/2, RcPid). %% @doc Call a `riak_cs_manifest_utils' function on a set of manifests %% to update the state of the manifests specified by `UUIDsToMark' %% and then write the updated values to riak. --spec mark_manifests(riakc_obj:riakc_obj(), binary(), binary(), [binary()], fun(), riak_client()) -> - {ok, riakc_obj:riakc_obj()} | {error, term()}. -mark_manifests(RiakObject, Bucket, Key, UUIDsToMark, ManiFunction, RcPid) -> +mark_manifests(RiakObject, Bucket, UUIDsToMark, ManiFunction, RcPid) -> Manifests = riak_cs_manifest:manifests_from_riak_object(RiakObject), Marked = ManiFunction(Manifests, UUIDsToMark), UpdObj0 = riak_cs_utils:update_obj_value(RiakObject, riak_cs_utils:encode_term(Marked)), UpdObj = riak_cs_manifest_fsm:update_md_with_multipart_2i( - UpdObj0, Marked, Bucket, Key), + UpdObj0, Marked, Bucket), %% use [returnbody] so that we get back the object %% with vector clock. This allows us to do a PUT @@ -383,8 +365,6 @@ mark_manifests(RiakObject, Bucket, Key, UUIDsToMark, ManiFunction, RcPid) -> riak_cs_config:put_manifest_timeout(), [riakc, put_manifest]). --spec maybe_delete_small_objects([cs_uuid_and_manifest()], riak_client(), non_neg_integer()) -> - {[cs_uuid_and_manifest()], [cs_uuid()]}. maybe_delete_small_objects(Manifests, RcPid, Threshold) -> {ok, BagId} = riak_cs_riak_client:get_manifest_bag(RcPid), DelFun= fun({UUID, Manifest = ?MANIFEST{state=pending_delete, @@ -393,7 +373,7 @@ maybe_delete_small_objects(Manifests, RcPid, Threshold) -> when ContentLength < Threshold -> %% actually this won't be scheduled :P UUIDManifest = {UUID, Manifest?MANIFEST{state=scheduled_delete}}, - _ = lager:debug("trying to delete ~p at ~p", [UUIDManifest, BagId]), + ?LOG_DEBUG("trying to delete ~p at ~p", [UUIDManifest, BagId]), case try_delete_blocks(BagId, UUIDManifest) of ok -> {Survivors, [UUID|UUIDsToDelete]}; @@ -403,34 +383,31 @@ maybe_delete_small_objects(Manifests, RcPid, Threshold) -> end; ({UUID, M}, {Survivors, UUIDsToDelete}) -> ContentLength = M?MANIFEST.content_length, - _ = lager:debug("~p is not being deleted: (CL, threshold)=(~p, ~p)", - [UUID, ContentLength, Threshold]), + ?LOG_DEBUG("~p is not being deleted: (CL, threshold)=(~p, ~p)", + [UUID, ContentLength, Threshold]), {[{UUID, M}|Survivors], UUIDsToDelete} end, %% Obtain a new history! lists:foldl(DelFun, {[], []}, Manifests). --spec try_delete_blocks(binary(), cs_uuid_and_manifest()) -> ok | {error, term()}. try_delete_blocks(BagId, {UUID, _} = UUIDManifest) -> Args = [BagId, UUIDManifest, undefined, dummy_gc_key_in_sync_delete, [{cleanup_manifests, false}]], {ok, Pid} = riak_cs_delete_fsm_sup:start_delete_fsm(node(), Args), case riak_cs_delete_fsm:sync_delete(Pid) of - {Pid, {ok, {_, _, _, TotalBlocks, TotalBlocks}}} -> + {Pid, {ok, {_, _, _, _, TotalBlocks, TotalBlocks}}} -> %% all the blocks are successfully deleted - _ = lager:debug("Active deletion of ~p succeeded", [UUID]), + ?LOG_DEBUG("Active deletion of ~p succeeded", [UUID]), ok; - {Pid, {ok, {_, _, _, NumDeleted, TotalBlocks}}} -> - _ = lager:debug("Only ~p/~p blocks of ~p deleted", - [NumDeleted, TotalBlocks, UUID]), + {Pid, {ok, {_, _, _, _, NumDeleted, TotalBlocks}}} -> + ?LOG_DEBUG("Only ~p/~p blocks of ~p deleted", [NumDeleted, TotalBlocks, UUID]), {error, partially_deleted}; {Pid, {error, _} = E} -> - _ = lager:warning("Active deletion of ~p failed. Reason: ~p", - [UUID, E]), + logger:warning("Active deletion of ~p failed. Reason: ~p", [UUID, E]), E; Other -> - _ = lager:error("Active deletion failed. Reason: ~p", [Other]), + logger:error("Active deletion failed. Reason: ~p", [Other]), {error, Other} end. @@ -463,26 +440,24 @@ move_manifests_to_gc_bucket(Manifests, RcPid) -> end, %% Create a set from the list of manifests - _ = lager:debug("Manifests scheduled for deletion: ~p", [ManifestSet]), + ?LOG_DEBUG("Manifests scheduled for deletion: ~p", [Manifests]), Timeout1 = riak_cs_config:put_gckey_timeout(), riak_cs_pbc:put(ManifestPbc, ObjectToWrite, [], Timeout1, [riakc, put_gc_manifest_set]). --spec build_manifest_set([cs_uuid_and_manifest()]) -> twop_set:twop_set(). build_manifest_set(Manifests) -> lists:foldl(fun twop_set:add_element/2, twop_set:new(), Manifests). %% @doc Generate a key for storing a set of manifests in the %% garbage collection bucket. --spec generate_key() -> binary(). generate_key() -> - list_to_binary([integer_to_list(timestamp()), + A = os:system_time(millisecond), + list_to_binary([integer_to_list(A), $_, - key_suffix(os:timestamp())]). + key_suffix(A)]). --spec key_suffix(erlang:timestamp()) -> string(). key_suffix(Time) -> - _ = random:seed(Time), - integer_to_list(random:uniform(riak_cs_config:gc_key_suffix_max())). + _ = rand:seed(exsss, Time), + integer_to_list(rand:uniform(riak_cs_config:gc_key_suffix_max())). %% @doc Given a list of riakc_obj-flavored object (with potentially %% many siblings and perhaps a tombstone), decode and merge them. diff --git a/src/riak_cs_gc_batch.erl b/apps/riak_cs/src/riak_cs_gc_batch.erl similarity index 83% rename from src/riak_cs_gc_batch.erl rename to apps/riak_cs/src/riak_cs_gc_batch.erl index 6bec9d2ea..fdbf43b34 100644 --- a/src/riak_cs_gc_batch.erl +++ b/apps/riak_cs/src/riak_cs_gc_batch.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -50,6 +51,7 @@ code_change/4]). -include("riak_cs_gc.hrl"). +-include_lib("kernel/include/logger.hrl"). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -100,22 +102,19 @@ init([#gc_batch_state{ 1000000000 =< EndKey -> %% StartKey < EndKey %% EndKey <= BatchStart - Leeway - _ = lager:info("Starting garbage collection: " - "(start, end) = (~p, ~p), " - "leeway=~p, batch_start=~p, max_workers=~p, page-size=~p", - [StartKey, EndKey, Leeway, BatchStart, MaxWorkers, BatchSize]), + logger:info("Starting garbage collection: (start, end) = (~p, ~p), " + "leeway=~p, batch_start=~p, max_workers=~p, page-size=~p", + [StartKey, EndKey, Leeway, BatchStart, MaxWorkers, BatchSize]), {ok, prepare, State, 0}; DefaultEndKey -> - _ = lager:error("GC did not start: " - "End of GC target period was too recent (~p > ~p)", - [EndKey, DefaultEndKey]), + logger:error("GC did not start: End of GC target period was too recent (~p > ~p)", + [EndKey, DefaultEndKey]), {stop, {error, invalid_gc_end_key}} end; init([#gc_batch_state{start_key=StartKey, end_key=EndKey}]) -> - _ = lager:error("GC did not start due to wrong GC target period: " - "(start, end) = (~p, ~p)", - [StartKey, EndKey]), + logger:error("GC did not start due to wrong GC target period: (start, end) = (~p, ~p)", + [StartKey, EndKey]), {stop, {error, invalid_gc_start_key}}. @@ -177,20 +176,20 @@ handle_info(_Info, StateName, State) -> %% @doc TODO: log warnings if this fsm is asked to terminate in the %% middle of running a gc batch terminate(normal, _StateName, State) -> - _ = lager:info("Finished garbage collection: " - "~b seconds, ~p batch_count, ~p batch_skips, " - "~p manif_count, ~p block_count\n", + logger:info("Finished garbage collection: " + "~b msec, ~p batch_count, ~p batch_skips, " + "~p manif_count, ~p block_count", + [elapsed(State?STATE.batch_start), State?STATE.batch_count, + State?STATE.batch_skips, State?STATE.manif_count, + State?STATE.block_count]), + riak_cs_gc_manager:finished(State); +terminate(cancel, _StateName, State) -> + logger:warning("Garbage collection has been canceled: " + "~b msec, ~p batch_count, ~p batch_skips, " + "~p manif_count, ~p block_count", [elapsed(State?STATE.batch_start), State?STATE.batch_count, State?STATE.batch_skips, State?STATE.manif_count, State?STATE.block_count]), - riak_cs_gc_manager:finished(State); -terminate(cancel, _StateName, State) -> - _ = lager:warning("Garbage collection has been canceled: " - "~b seconds, ~p batch_count, ~p batch_skips, " - "~p manif_count, ~p block_count\n", - [elapsed(State?STATE.batch_start), State?STATE.batch_count, - State?STATE.batch_skips, State?STATE.manif_count, - State?STATE.block_count]), ok; terminate(_Reason, _StateName, _State) -> ok. @@ -230,16 +229,14 @@ fetch_first_keys(?STATE{batch_start=_BatchStart, {KeyListRes, KeyListState} = riak_cs_gc_key_list:new(StartKey, EndKey, BatchSize), #gc_key_list_result{bag_id=BagId, batch=Batch} = KeyListRes, - _ = lager:debug("Initial batch keys: ~p", [Batch]), + ?LOG_DEBUG("Initial batch keys: ~p", [Batch]), State?STATE{batch=Batch, key_list_state=KeyListState, bag_id=BagId}. %% @doc Handle a `batch_complete' event from a GC worker process. --spec handle_batch_complete(pid(), #gc_worker_state{}, ?STATE{}) -> ?STATE{}. handle_batch_complete(WorkerPid, WorkerState, State) -> - ?STATE{ - worker_pids=WorkerPids, + ?STATE{worker_pids=WorkerPids, batch_count=BatchCount, batch_skips=BatchSkips, manif_count=ManifestCount, @@ -248,12 +245,11 @@ handle_batch_complete(WorkerPid, WorkerState, State) -> batch_skips=WorkerBatchSkips, manif_count=WorkerManifestCount, block_count=WorkerBlockCount} = WorkerState, - _ = lager:debug("~p completed (~p)", [WorkerPid, WorkerState]), + ?LOG_DEBUG("~p completed (~p)", [WorkerPid, WorkerState]), UpdWorkerPids = lists:delete(WorkerPid, WorkerPids), %% @TODO Workout the terminiology for these stats. i.e. Is batch %% count just an increment or represenative of something else. - State?STATE{ - worker_pids=UpdWorkerPids, + State?STATE{worker_pids=UpdWorkerPids, batch_count=BatchCount + WorkerBatchCount, batch_skips=BatchSkips + WorkerBatchSkips, manif_count=ManifestCount + WorkerManifestCount, @@ -261,13 +257,12 @@ handle_batch_complete(WorkerPid, WorkerState, State) -> %% @doc Start a GC worker and return the apprpriate next state and %% updated state record. --spec start_worker(?STATE{}) -> ?STATE{}. start_worker(?STATE{batch=[NextBatch|RestBatches], bag_id=BagId, worker_pids=WorkerPids} = State) -> case ?GC_WORKER:start_link(BagId, NextBatch) of {ok, Pid} -> - _ = lager:debug("GC worker ~p for bag ~p has started", [Pid, BagId]), + ?LOG_DEBUG("GC worker ~p for bag ~p has started", [Pid, BagId]), State?STATE{batch=RestBatches, worker_pids=[Pid | WorkerPids]}; {error, _Reason} -> @@ -275,15 +270,13 @@ start_worker(?STATE{batch=[NextBatch|RestBatches], end. %% @doc Cancel the current batch of files set for garbage collection. --spec cancel_batch(?STATE{}) -> any(). cancel_batch(?STATE{batch_start=BatchStart, worker_pids=WorkerPids}=_State) -> %% Interrupt the batch of deletes - _ = lager:info("Canceled garbage collection batch after ~b seconds.", - [elapsed(BatchStart)]), + logger:info("Canceled garbage collection batch after ~b msec", + [elapsed(BatchStart)]), [riak_cs_gc_worker:stop(P) || P <- WorkerPids]. --spec ok_reply(atom(), ?STATE{}) -> {reply, ok, atom(), ?STATE{}}. ok_reply(NextState, NextStateData) -> {reply, ok, NextState, NextStateData}. @@ -305,7 +298,7 @@ maybe_start_workers(?STATE{max_workers=MaxWorkers, %% Fetch the next set of manifests for deletion {KeyListRes, UpdKeyListState} = riak_cs_gc_key_list:next(KeyListState), #gc_key_list_result{bag_id=BagId, batch=Batch} = KeyListRes, - _ = lager:debug("Next batch keys: ~p", [Batch]), + ?LOG_DEBUG("Next batch keys: ~p", [Batch]), State2 = State?STATE{batch=Batch, key_list_state=UpdKeyListState, bag_id=BagId}, @@ -317,7 +310,7 @@ maybe_start_workers(?STATE{max_workers=MaxWorkers, worker_pids=WorkerPids, batch=Batch} = State) when MaxWorkers > length(WorkerPids) -> - _ = lager:debug("Batch: ~p", [Batch, WorkerPids]), + ?LOG_DEBUG("Batch: ~p, WorkerPids: ~p", [Batch, WorkerPids]), State2 = start_worker(State), maybe_start_workers(State2). @@ -332,11 +325,10 @@ status_data(State) -> end}]. %% @doc How many seconds have passed from `Time' to now. --spec elapsed(undefined | non_neg_integer()) -> non_neg_integer(). elapsed(undefined) -> - riak_cs_gc:timestamp(); + os:system_time(millisecond); elapsed(Time) -> - Now = riak_cs_gc:timestamp(), + Now = os:system_time(millisecond), case (Diff = Now - Time) > 0 of true -> Diff; diff --git a/src/riak_cs_gc_console.erl b/apps/riak_cs/src/riak_cs_gc_console.erl similarity index 96% rename from src/riak_cs_gc_console.erl rename to apps/riak_cs/src/riak_cs_gc_console.erl index 5a5f5580a..93682327c 100644 --- a/src/riak_cs_gc_console.erl +++ b/apps/riak_cs/src/riak_cs_gc_console.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -37,9 +38,9 @@ try Code catch - Type:Reason -> - io:format("~s failed:~n ~p:~p~n~p~n", - [Description, Type, Reason, erlang:get_stacktrace()]), + Type:Reason:ST -> + io:format("~s failed:\n ~p:~p\n~p\n", + [Description, Type, Reason, ST]), error end). @@ -222,8 +223,8 @@ human_detail(Name, Value) -> -spec human_time(non_neg_integer()|undefined) -> binary(). human_time(undefined) -> "unknown/never"; -human_time(Seconds) -> - Seconds0 = Seconds + ?DAYS_FROM_0_TO_1970*?SECONDS_PER_DAY, +human_time(Millis) -> + Seconds0 = Millis div 1000 + ?DAYS_FROM_0_TO_1970*?SECONDS_PER_DAY, rts:iso8601(calendar:gregorian_seconds_to_datetime(Seconds0)). parse_batch_opts([Leeway]) -> @@ -273,7 +274,6 @@ convert(Options) -> error({bad_arg, BadArg}) end, Options). --spec iso8601_to_epoch(string()) -> non_neg_integer(). iso8601_to_epoch(S) -> {ok, Datetime} = rts:datetime(S), GregorianSeconds = calendar:datetime_to_gregorian_seconds(Datetime), diff --git a/src/riak_cs_gc_key_list.erl b/apps/riak_cs/src/riak_cs_gc_key_list.erl similarity index 63% rename from src/riak_cs_gc_key_list.erl rename to apps/riak_cs/src/riak_cs_gc_key_list.erl index 586e0de9f..e91ac56a8 100644 --- a/src/riak_cs_gc_key_list.erl +++ b/apps/riak_cs/src/riak_cs_gc_key_list.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2014 Basho Technologies, Inc. +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -28,6 +29,7 @@ -export([find_oldest_entries/1]). -include("riak_cs_gc.hrl"). +-include_lib("kernel/include/logger.hrl"). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -57,7 +59,7 @@ next(#gc_key_list_state{current_riak_client=RcPid, continuation=Continuation} = State) -> {Batch, UpdContinuation} = fetch_eligible_manifest_keys(RcPid, StartKey, EndKey, BatchSize, Continuation), - lager:debug("next Batch: ~p~n", [Batch]), + ?LOG_DEBUG("next Batch: ~p", [Batch]), {#gc_key_list_result{bag_id=BagId, batch=Batch}, State#gc_key_list_state{continuation=UpdContinuation}}. @@ -68,7 +70,6 @@ has_next(_) -> true. %% @doc Fetch next key list and returns it with updated state --spec next_pool(gc_key_list_state()) -> {gc_key_list_result(), gc_key_list_state()|undefined}. next_pool(#gc_key_list_state{remaining_bags=[]}) -> {#gc_key_list_result{bag_id=undefined, batch=[]}, undefined}; @@ -81,88 +82,48 @@ next_pool(#gc_key_list_state{ ok = riak_cs_riak_client:set_manifest_bag(RcPid, BagId), {Batch, Continuation} = fetch_eligible_manifest_keys(RcPid, StartKey, EndKey, BatchSize, undefined), - lager:debug("next_bag ~s Batch: ~p~n", [BagId, Batch]), + ?LOG_DEBUG("next_bag ~s Batch: ~p", [BagId, Batch]), {#gc_key_list_result{bag_id=BagId, batch=Batch}, State#gc_key_list_state{remaining_bags=Rest, current_riak_client=RcPid, current_bag_id=BagId, continuation=Continuation}}; {error, Reason} -> - lager:error("Connection error for bag ~s in garbage collection: ~p", - [BagId, Reason]), + logger:error("Connection error for bag ~s in garbage collection: ~p", + [BagId, Reason]), next_pool(State#gc_key_list_state{remaining_bags=Rest}) end. %% @doc Fetch the list of keys for file manifests that are eligible %% for delete. --spec fetch_eligible_manifest_keys(riak_client(), binary(), binary(), pos_integer(), continuation()) -> - {[index_result_keys()], continuation()}. fetch_eligible_manifest_keys(RcPid, StartKey, EndKey, BatchSize, Continuation) -> - UsePaginatedIndexes = riak_cs_config:gc_paginated_indexes(), QueryResults = gc_index_query(RcPid, StartKey, EndKey, BatchSize, - Continuation, - UsePaginatedIndexes), - {eligible_manifest_keys(QueryResults, UsePaginatedIndexes, BatchSize), + Continuation), + {eligible_manifest_keys(QueryResults, BatchSize), continuation(QueryResults)}. --spec eligible_manifest_keys({{ok, index_results()} | {error, term()}, {binary(), binary()}}, - UsePaginatedIndexes::boolean(), pos_integer()) -> - [index_result_keys()]. -eligible_manifest_keys({{ok, ?INDEX_RESULTS{keys=Keys}}, _}, - true, _) -> +eligible_manifest_keys({{ok, ?INDEX_RESULTS{keys=Keys}}, _}, _) -> case Keys of [] -> []; _ -> [Keys] end; -eligible_manifest_keys({{ok, ?INDEX_RESULTS{keys=Keys}}, _}, - false, BatchSize) -> - split_eligible_manifest_keys(BatchSize, Keys, []); -eligible_manifest_keys({{error, Reason}, {StartKey, EndKey}}, _, _) -> - _ = lager:warning("Error occurred trying to query from time ~p to ~p" - "in gc key index. Reason: ~p", - [StartKey, EndKey, Reason]), +eligible_manifest_keys({{error, Reason}, {StartKey, EndKey}}, _) -> + logger:warning("Error occurred trying to query from time ~p to ~p" + "in gc key index. Reason: ~p", [StartKey, EndKey, Reason]), []. -%% @doc Break a list of gc-eligible keys from the GC bucket into smaller sets -%% to be processed by different GC workers. --spec split_eligible_manifest_keys(non_neg_integer(), index_result_keys(), [index_result_keys()]) -> - [index_result_keys()]. -split_eligible_manifest_keys(_BatchSize, [], Acc) -> - lists:reverse(Acc); -split_eligible_manifest_keys(BatchSize, Keys, Acc) -> - {Batch, Rest} = split_at_most_n(BatchSize, Keys, []), - split_eligible_manifest_keys(BatchSize, Rest, [Batch | Acc]). - -split_at_most_n(_, [], Acc) -> - {lists:reverse(Acc), []}; -split_at_most_n(0, L, Acc) -> - {lists:reverse(Acc), L}; -split_at_most_n(N, [H|T], Acc) -> - split_at_most_n(N-1, T, [H|Acc]). - --spec continuation({{ok, index_results()} | {error, term()}, - {binary(), binary()}}) -> - continuation() | undefined. continuation({{ok, ?INDEX_RESULTS{continuation=Continuation}}, _EndTime}) -> Continuation; continuation({{error, _}, _EndTime}) -> undefined. --spec gc_index_query(riak_client(), binary(), binary(), non_neg_integer(), continuation(), boolean()) -> - {{ok, index_results()} | {error, term()}, - {binary(), binary()}}. -gc_index_query(RcPid, StartKey, EndKey, BatchSize, Continuation, UsePaginatedIndexes) -> - Options = case UsePaginatedIndexes of - true -> - [{max_results, BatchSize}, - {continuation, Continuation}]; - false -> - [] - end, +gc_index_query(RcPid, StartKey, EndKey, BatchSize, Continuation) -> + Options = [{max_results, BatchSize}, + {continuation, Continuation}], {ok, ManifestPbc} = riak_cs_riak_client:manifest_pbc(RcPid), Timeout = riak_cs_config:get_index_range_gckeys_timeout(), @@ -185,19 +146,18 @@ gc_index_query(RcPid, StartKey, EndKey, BatchSize, Continuation, UsePaginatedInd {QueryResult, {StartKey, EndKey}}. -spec find_oldest_entries(BagId::binary()|master) -> - {ok, [{string(), [pos_integer()]}]} | - {error, term()}. + {ok, [{string(), [pos_integer()]}]} | {error, term()}. find_oldest_entries(BagId) -> %% walk around {ok, RcPid} = riak_cs_riak_client:start_link([]), try ok = riak_cs_riak_client:set_manifest_bag(RcPid, BagId), Start = riak_cs_gc:epoch_start(), - End = riak_cs_gc:default_batch_end(riak_cs_gc:timestamp(), 0), + End = riak_cs_gc:default_batch_end(os:system_time(millisecond), 0), {QueryResult, _} = gc_index_query(RcPid, int2bin(Start), int2bin(End), riak_cs_config:gc_batch_size(), - undefined, true), + undefined), case QueryResult of {ok, ?INDEX_RESULTS{keys=Keys}} -> List = correlate([ gc_key_to_datetime(Key) || Key <- Keys]), @@ -209,7 +169,6 @@ find_oldest_entries(BagId) -> riak_cs_riak_client:stop(RcPid) end. --spec gc_key_to_datetime(binary()) -> {string(), integer()}. gc_key_to_datetime(Key) -> [Str|Suffix] = string:tokens(binary_to_list(Key), "_"), Datetime = binary_to_list(riak_cs_gc_console:human_time(list_to_integer(Str))), @@ -227,7 +186,6 @@ correlate(Pairs) -> end, lists:reverse(lists:foldl(F, [], Pairs)). --spec int2bin(non_neg_integer()) -> binary(). int2bin(I) -> list_to_binary(integer_to_list(I)). @@ -244,13 +202,4 @@ correlate_test() -> {b, []}, {c, [23, 434, 435]}], non_neg_only(correlate(Data))). -split_eligible_manifest_keys_test() -> - ?assertEqual([], split_eligible_manifest_keys(3, [], [])), - ?assertEqual([[1]], split_eligible_manifest_keys(3, [1], [])), - ?assertEqual([[1,2,3]], split_eligible_manifest_keys(3, lists:seq(1,3), [])), - ?assertEqual([[1,2,3],[4]], split_eligible_manifest_keys(3, lists:seq(1,4), [])), - ?assertEqual([[1,2,3],[4,5,6]], split_eligible_manifest_keys(3, lists:seq(1,6), [])), - ?assertEqual([[1,2,3],[4,5,6],[7,8,9],[10]], - split_eligible_manifest_keys(3, lists:seq(1,10), [])). - -endif. diff --git a/src/riak_cs_gc_manager.erl b/apps/riak_cs/src/riak_cs_gc_manager.erl similarity index 91% rename from src/riak_cs_gc_manager.erl rename to apps/riak_cs/src/riak_cs_gc_manager.erl index a97e8bd44..d7b52c0eb 100644 --- a/src/riak_cs_gc_manager.erl +++ b/apps/riak_cs/src/riak_cs_gc_manager.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -66,6 +67,7 @@ -export_type([statename/0]). -include("riak_cs_gc.hrl"). +-include_lib("kernel/include/logger.hrl"). -define(SERVER, ?MODULE). @@ -128,28 +130,35 @@ finished(Report) -> %%% gen_fsm callbacks %%%=================================================================== +-ifndef(TEST). + init([]) -> process_flag(trap_exit, true), - InitialDelay = riak_cs_gc:initial_gc_delay(), + InitialDelay = riak_cs_gc:initial_gc_delay() * 1000, State = case riak_cs_gc:gc_interval() of infinity -> #gc_manager_state{}; Interval when is_integer(Interval) -> - Interval2 = Interval + InitialDelay, - Next = riak_cs_gc:timestamp() + Interval2, - TimerRef = erlang:send_after(Interval2 * 1000, self(), {start, []}), - _ = lager:info("Scheduled next batch at ~s", - [riak_cs_gc_console:human_time(Next)]), + Interval2 = Interval * 1000 + InitialDelay, + Next = os:system_time(millisecond) + Interval2, + TimerRef = erlang:send_after(Interval2, self(), {start, []}), + logger:info("Scheduled next batch at ~s", + [riak_cs_gc_console:human_time(Next)]), #gc_manager_state{next=Next, interval=Interval, initial_delay=InitialDelay, timer_ref=TimerRef} end, - {ok, idle, State}; + {ok, idle, State}. + +-else. + init([testing]) -> {ok, idle, #gc_manager_state{}}. +-endif. + %%-------------------------------------------------------------------- %% @private %% @doc All gen_fsm:send_event/2 call should be ignored @@ -215,7 +224,7 @@ handle_info({'EXIT', Pid, Reason}, _StateName, #gc_manager_state{gc_batch_pid=Pid} = State) -> case Reason of Reason when Reason =/= normal andalso Reason =/= cancel -> - _ = lager:warning("GC batch has terminated for reason: ~p", [Reason]); + logger:warning("GC batch has terminated for reason: ~p", [Reason]); _ -> ok end, @@ -226,7 +235,7 @@ handle_info({start, Options}, idle, State) -> {ok, NextState} -> {next_state, running, NextState}; Error -> - _ = lager:error("Cannot start batch. Reason: ~p", [Error]), + logger:error("Cannot start batch. Reason: ~p", [Error]), NextState = schedule_next(State), {next_state, idle, NextState} end; @@ -235,8 +244,8 @@ handle_info({start, _}, running, State) -> {next_state, running, State}; handle_info(Info, StateName, State) -> %% This is really unexpected and unknown - warning. - _ = lager:warning("Unexpected message received at GC process (~p): ~p", - [StateName, Info]), + logger:warning("Unexpected message received at GC process (~p): ~p", + [StateName, Info]), {next_state, StateName, State}. %% @private @@ -261,7 +270,7 @@ start_batch(State, Options) -> %% All these Items should be non_neg_integer() MaxWorkers = proplists:get_value('max-workers', Options, riak_cs_gc:gc_max_workers()), - BatchStart = riak_cs_gc:timestamp(), + BatchStart = os:system_time(millisecond), Leeway = proplists:get_value(leeway, Options, riak_cs_gc:leeway_seconds()), StartKey = proplists:get_value(start, Options,riak_cs_gc:epoch_start()), @@ -320,7 +329,7 @@ schedule_next(#gc_manager_state{timer_ref=Ref}=State) false -> schedule_next(State#gc_manager_state{timer_ref=undefined}); _ -> - _ = lager:debug("Timer is already scheduled, maybe manually triggered?"), + ?LOG_DEBUG("Timer is already scheduled, maybe manually triggered?"), %% Timer is already scheduled, do nothing State end; @@ -328,11 +337,11 @@ schedule_next(#gc_manager_state{interval=infinity}=State) -> %% nothing to schedule, all triggers manual State#gc_manager_state{next=undefined}; schedule_next(#gc_manager_state{interval=Interval}=State) -> - RevisedNext = riak_cs_gc:timestamp() + Interval, + RevisedNext = os:system_time(millisecond) + Interval, TimerValue = Interval * 1000, TimerRef = erlang:send_after(TimerValue, self(), {start, []}), - _ = lager:info("Scheduled next batch at ~s", - [riak_cs_gc_console:human_time(RevisedNext)]), + logger:info("Scheduled next batch at ~s", + [riak_cs_gc_console:human_time(RevisedNext)]), State#gc_manager_state{next=RevisedNext, timer_ref=TimerRef}. diff --git a/src/riak_cs_gc_worker.erl b/apps/riak_cs/src/riak_cs_gc_worker.erl similarity index 96% rename from src/riak_cs_gc_worker.erl rename to apps/riak_cs/src/riak_cs_gc_worker.erl index 6798ac551..891ca4894 100644 --- a/src/riak_cs_gc_worker.erl +++ b/apps/riak_cs/src/riak_cs_gc_worker.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -45,6 +46,7 @@ code_change/4]). -include("riak_cs_gc.hrl"). +-include_lib("kernel/include/logger.hrl"). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -235,9 +237,8 @@ fetch_next_fileset(ManifestSetKey, RcPid) -> {error, notfound}=Error -> Error; {error, Reason}=Error -> - _ = lager:info("Error occurred trying to read the fileset" - " for ~p for gc. Reason: ~p", - [ManifestSetKey, Reason]), + logger:info("Error occurred trying to read the fileset" + " for ~p for gc. Reason: ~p", [ManifestSetKey, Reason]), riak_cs_pbc:check_connection_status(ManifestPbc, fetch_next_fileset), Error end. @@ -257,7 +258,7 @@ finish_file_delete(0, _, RiakObj, RcPid) -> Timeout, [riakc, delete_gc_manifest_set]), ok; finish_file_delete(_, FileSet, _RiakObj, _RcPid) -> - _ = lager:debug("Remaining file keys: ~p", [twop_set:to_list(FileSet)]), + ?LOG_DEBUG("Remaining file keys: ~p", [twop_set:to_list(FileSet)]), %% NOTE: we used to do a PUT here, but now with multidc replication %% we run garbage collection seprarately on each cluster, so we don't %% want to send this update to another data center. When we delete this @@ -272,7 +273,7 @@ ok_reply(NextState, NextStateData) -> %% Refactor TODO: %% 1. delete_fsm_pid=undefined is desirable in both ok & error cases? %% 2. It's correct to *not* change pause_state? -handle_delete_fsm_reply({ok, {_, _, _, TotalBlocks, TotalBlocks}}, +handle_delete_fsm_reply({ok, {_, _, _, _, TotalBlocks, TotalBlocks}}, ?STATE{current_files=[CurrentManifest | RestManifests], current_fileset=FileSet, block_count=BlockCount} = State) -> @@ -282,7 +283,7 @@ handle_delete_fsm_reply({ok, {_, _, _, TotalBlocks, TotalBlocks}}, current_fileset=UpdFileSet, current_files=RestManifests, block_count=BlockCount+TotalBlocks}; -handle_delete_fsm_reply({ok, {_, _, _, NumDeleted, _TotalBlocks}}, +handle_delete_fsm_reply({ok, {_, _, _, _, NumDeleted, _TotalBlocks}}, ?STATE{current_files=[_CurrentManifest | RestManifests], block_count=BlockCount} = State) -> ok = continue(), diff --git a/src/riak_cs_get_fsm.erl b/apps/riak_cs/src/riak_cs_get_fsm.erl similarity index 70% rename from src/riak_cs_get_fsm.erl rename to apps/riak_cs/src/riak_cs_get_fsm.erl index 52bb4176a..54815e888 100644 --- a/src/riak_cs_get_fsm.erl +++ b/apps/riak_cs/src/riak_cs_get_fsm.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -35,13 +36,13 @@ %% Test API -export([test_link/6]). - -endif. -include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). %% API --export([start_link/6, +-export([start_link/7, stop/1, continue/2, manifest/2, @@ -67,44 +68,39 @@ -type block_name() :: {binary(), integer()}. --ifdef(namespaced_types). -type block_queue() :: queue:queue(). --else. --type block_queue() :: queue(). --endif. --record(state, {from :: {pid(), reference()}, +-record(state, {from :: undefined | {pid(), reference()}, riak_client :: riak_client(), - mani_fsm_pid :: pid(), - bucket :: term(), + mani_fsm_pid :: undefined | pid(), + bucket :: binary(), caller :: reference(), key :: term(), + obj_vsn :: binary(), fetch_concurrency :: pos_integer(), buffer_factor :: pos_integer(), got_blocks=orddict:new() :: orddict:orddict(), manifest :: term(), - blocks_order :: [block_name()], - blocks_intransit=queue:new() :: block_queue(), - test=false :: boolean(), - total_blocks :: pos_integer(), + blocks_order = [] :: [block_name()], + blocks_intransit = queue:new() :: block_queue(), + test = false :: boolean(), + total_blocks :: undefined | pos_integer(), num_sent=0 :: non_neg_integer(), - initial_block :: block_name(), - final_block :: block_name(), - skip_bytes_initial :: non_neg_integer(), - keep_bytes_final :: non_neg_integer(), - free_readers :: [pid()], - all_reader_pids :: [pid()]}). --type state() :: #state{}. + initial_block :: undefined | block_name(), + final_block :: undefined | block_name(), + skip_bytes_initial = 0 :: non_neg_integer(), + keep_bytes_final = 0 :: non_neg_integer(), + free_readers :: undefined | [pid()], + all_reader_pids :: undefined | [pid()]}). %% =================================================================== %% Public API %% =================================================================== --spec start_link(binary(), binary(), pid(), riak_client(), pos_integer(), +-spec start_link(binary(), binary(), binary(), pid(), riak_client(), pos_integer(), pos_integer()) -> {ok, pid()} | {error, term()}. - -start_link(Bucket, Key, Caller, RcPid, FetchConcurrency, BufferFactor) -> - gen_fsm:start_link(?MODULE, [Bucket, Key, Caller, RcPid, +start_link(Bucket, Key, ObjVsn, Caller, RcPid, FetchConcurrency, BufferFactor) -> + gen_fsm:start_link(?MODULE, [Bucket, Key, ObjVsn, Caller, RcPid, FetchConcurrency, BufferFactor], []). stop(Pid) -> @@ -129,7 +125,9 @@ chunk(Pid, ChunkSeq, ChunkValue) -> %% gen_fsm callbacks %% ==================================================================== -init([Bucket, Key, Caller, RcPid, FetchConcurrency, BufferFactor]) +-ifndef(TEST). + +init([Bucket, Key, ObjVsn, Caller, RcPid, FetchConcurrency, BufferFactor]) when is_binary(Bucket), is_binary(Key), is_pid(Caller), is_pid(RcPid), FetchConcurrency > 0, BufferFactor > 0 -> @@ -152,16 +150,37 @@ init([Bucket, Key, Caller, RcPid, FetchConcurrency, BufferFactor]) %% an exit Reason of `noproc` process_flag(trap_exit, true), - State = #state{bucket=Bucket, - caller=CallerRef, - key=Key, - riak_client=RcPid, - buffer_factor=BufferFactor, - fetch_concurrency=FetchConcurrency}, + State = #state{bucket = Bucket, + key = Key, + obj_vsn = ObjVsn, + caller = CallerRef, + riak_client = RcPid, + buffer_factor = BufferFactor, + fetch_concurrency = FetchConcurrency}, + {ok, prepare, State, 0}. + +-else. + +init([Bucket, Key, ObjVsn, Caller, RcPid, FetchConcurrency, BufferFactor]) + when is_binary(Bucket), is_binary(Key), is_pid(Caller), + is_pid(RcPid), + FetchConcurrency > 0, BufferFactor > 0 -> + + CallerRef = erlang:monitor(process, Caller), + process_flag(trap_exit, true), + + State = #state{bucket = Bucket, + key = Key, + obj_vsn = ObjVsn, + caller = CallerRef, + riak_client = RcPid, + buffer_factor = BufferFactor, + fetch_concurrency = FetchConcurrency}, {ok, prepare, State, 0}; -init([test, Bucket, Key, Caller, ContentLength, BlockSize, FetchConcurrency, - BufferFactor]) -> - {ok, prepare, State1, 0} = init([Bucket, Key, Caller, self(), + +init([test, Bucket, Key, Caller, ContentLength, + BlockSize, FetchConcurrency, BufferFactor]) -> + {ok, prepare, State1, 0} = init([Bucket, Key, ?LFS_DEFAULT_OBJECT_VERSION, Caller, self(), FetchConcurrency, BufferFactor]), %% purposely have the timeout happen @@ -172,6 +191,7 @@ init([test, Bucket, Key, Caller, ContentLength, BlockSize, FetchConcurrency, riak_cs_dummy_reader:start_link([self(), Bucket, Key, + ?LFS_DEFAULT_OBJECT_VERSION, ContentLength, BlockSize]), link(ReaderPid), @@ -182,6 +202,8 @@ init([test, Bucket, Key, Caller, ContentLength, BlockSize, FetchConcurrency, manifest=Manifest, test=true}}. +-endif. + prepare(timeout, State) -> NewState = prepare(State), {next_state, waiting_value, NewState}. @@ -211,19 +233,20 @@ waiting_continue_or_stop(timeout, State) -> {stop, normal, State}; waiting_continue_or_stop(stop, State) -> {stop, normal, State}; -waiting_continue_or_stop({continue, Range}, #state{manifest=Manifest, - bucket=BucketName, - key=Key, - fetch_concurrency=FetchConcurrency, - free_readers=Readers, - riak_client=RcPid}=State) -> +waiting_continue_or_stop({continue, Range}, #state{manifest = Manifest, + bucket = BucketName, + key = Key, + obj_vsn = ObjVsn, + fetch_concurrency = FetchConcurrency, + free_readers = Readers, + riak_client = RcPid} = State) -> {BlocksOrder, SkipInitial, KeepFinal} = riak_cs_lfs_utils:block_sequences_for_manifest(Manifest, Range), case BlocksOrder of [] -> %% We should never get here because empty %% files are handled by the wm resource. - _ = lager:warning("~p:~p has no blocks", [BucketName, Key]), + logger:warning("~p/~p:~p has no blocks", [BucketName, Key, ObjVsn]), {stop, normal, State}; [InitialBlock|_] -> TotalBlocks = length(BlocksOrder), @@ -234,34 +257,33 @@ waiting_continue_or_stop({continue, Range}, #state{manifest=Manifest, FreeReaders = riak_cs_block_server:start_block_servers(Manifest, RcPid, FetchConcurrency), - _ = lager:debug("Block Servers: ~p", [FreeReaders]); + ?LOG_DEBUG("Block Servers: ~p", [FreeReaders]); _ -> FreeReaders = Readers end, %% start retrieving the first set of blocks - UpdState = State#state{blocks_order=BlocksOrder, - total_blocks=TotalBlocks, - initial_block=InitialBlock, - final_block=lists:last(BlocksOrder), - skip_bytes_initial=SkipInitial, - keep_bytes_final=KeepFinal, - free_readers=FreeReaders}, + UpdState = State#state{blocks_order = BlocksOrder, + total_blocks = TotalBlocks, + initial_block = InitialBlock, + final_block = lists:last(BlocksOrder), + skip_bytes_initial = SkipInitial, + keep_bytes_final = KeepFinal, + free_readers = FreeReaders}, {next_state, waiting_chunks, read_blocks(UpdState)} end. waiting_continue_or_stop(Event, From, State) -> - _ = lager:info("Pid ~p got unknown event ~p from ~p\n", - [self(), Event, From]), + logger:info("Pid ~p got unknown event ~p from ~p", [self(), Event, From]), {next_state, waiting_continue_or_stop, State}. -waiting_chunks(get_next_chunk, From, State=#state{num_sent=TotalNumBlocks, - total_blocks=TotalNumBlocks}) -> +waiting_chunks(get_next_chunk, From, State=#state{num_sent = TotalNumBlocks, + total_blocks = TotalNumBlocks}) -> _ = gen_fsm:reply(From, {done, <<>>}), {stop, normal, State}; waiting_chunks(get_next_chunk, From, State) -> case perhaps_send_to_user(From, State) of done -> - UpdState = State#state{from=From}, + UpdState = State#state{from = From}, {next_state, waiting_chunks, read_blocks(UpdState)}; {sent, UpdState} -> Got = UpdState#state.got_blocks, @@ -276,22 +298,22 @@ waiting_chunks(get_next_chunk, From, State) -> {next_state, waiting_chunks, read_blocks(UpdState)} end. -perhaps_send_to_user(From, #state{got_blocks=Got, - num_sent=NumSent, - blocks_intransit=Intransit}=State) -> +perhaps_send_to_user(From, #state{got_blocks = Got, + num_sent = NumSent, + blocks_intransit = Intransit} = State) -> case queue:out(Intransit) of {empty, _} -> done; {{value, NextBlock}, UpdIntransit} -> case orddict:find(NextBlock, Got) of {ok, Block} -> - _ = lager:debug("Returning block ~p to client", [NextBlock]), + ?LOG_DEBUG("Returning block ~p to client", [NextBlock]), %% Must use gen_fsm:reply/2 here! We are shared %% with an async event func and must return next_state. gen_fsm:reply(From, {chunk, Block}), - {sent, State#state{got_blocks=orddict:erase(NextBlock, Got), - num_sent=NumSent+1, - blocks_intransit=UpdIntransit}}; + {sent, State#state{got_blocks = orddict:erase(NextBlock, Got), + num_sent = NumSent+1, + blocks_intransit = UpdIntransit}}; error -> {not_sent, State#state{from=From}} end @@ -301,25 +323,25 @@ waiting_chunks(stop, State) -> {stop, normal, State}; waiting_chunks(timeout, State = #state{got_blocks = Got}) -> GotSize = orddict:size(Got), - _ = lager:debug("starting fetch again with ~p left in queue", [GotSize]), + ?LOG_DEBUG("starting fetch again with ~p left in queue", [GotSize]), UpdState = read_blocks(State), {next_state, waiting_chunks, UpdState}; waiting_chunks({chunk, Pid, {NextBlock, BlockReturnValue}}, - #state{from=From, - got_blocks=Got, - free_readers=FreeReaders, - initial_block=InitialBlock, - final_block=FinalBlock, - skip_bytes_initial=SkipInitial, - keep_bytes_final=KeepFinal - }=State) -> - _ = lager:debug("Retrieved block ~p", [NextBlock]), + #state{from = From, + got_blocks = Got, + free_readers = FreeReaders, + initial_block = InitialBlock, + final_block = FinalBlock, + skip_bytes_initial = SkipInitial, + keep_bytes_final = KeepFinal + } = State) -> + ?LOG_DEBUG("Retrieved block ~p", [NextBlock]), case BlockReturnValue of {error, _} = ErrorRes -> - #state{bucket=Bucket, key=Key} = State, - _ = lager:error("~p: Cannot get S3 ~p ~p block# ~p: ~p\n", - [?MODULE, Bucket, Key, NextBlock, ErrorRes]), + #state{bucket = Bucket, key = Key, obj_vsn = ObjVsn} = State, + logger:error("Cannot get S3 ~s/~s:~s block# ~p: ~p", + [Bucket, Key, ObjVsn, NextBlock, ErrorRes]), %% Our terminate() will explicitly stop dependent processes, %% we don't need an abnormal exit to kill them for us. exit(normal); @@ -332,7 +354,7 @@ waiting_chunks({chunk, Pid, {NextBlock, BlockReturnValue}}, {InitialBlock, FinalBlock}, {SkipInitial, KeepFinal}), UpdGot = orddict:store(NextBlock, BlockValue, Got), - %% TODO: _ = lager:debug("BlocksLeft: ~p", [BlocksLeft]), + %% TODO: ?LOG_DEBUG("BlocksLeft: ~p", [BlocksLeft]), GotSize = orddict:size(UpdGot), UpdState0 = State#state{got_blocks = UpdGot, free_readers = [Pid|FreeReaders]}, MaxGotSize = riak_cs_lfs_utils:get_fsm_buffer_size_factor(), @@ -347,7 +369,7 @@ waiting_chunks({chunk, Pid, {NextBlock, BlockReturnValue}}, true -> case perhaps_send_to_user(From, UpdState) of {sent, Upd2State} -> - {next_state, waiting_chunks, Upd2State#state{from=undefined}}; + {next_state, waiting_chunks, Upd2State#state{from = undefined}}; {not_sent, Upd2State} -> {next_state, waiting_chunks, Upd2State} end @@ -373,11 +395,11 @@ handle_info(request_timeout, StateName, StateData) -> %% we have no reason to stick around %% %% @TODO Also handle reader pid death -handle_info({'EXIT', ManiPid, _Reason}, _StateName, StateData=#state{mani_fsm_pid=ManiPid}) -> +handle_info({'EXIT', ManiPid, _Reason}, _StateName, StateData = #state{mani_fsm_pid = ManiPid}) -> {stop, normal, StateData}; handle_info({'DOWN', CallerRef, process, _Pid, Reason}, _StateName, - State=#state{caller=CallerRef}) -> + State = #state{caller = CallerRef}) -> {stop, Reason, State}; handle_info({'EXIT', _Pid, normal}, StateName, StateData) -> %% TODO: who is _Pid when clean_multipart_unused_parts returns updated? @@ -386,9 +408,9 @@ handle_info(_Info, _StateName, StateData) -> {stop, {badmsg, _Info}, StateData}. %% @private -terminate(_Reason, _StateName, #state{test=false, - all_reader_pids=BlockServerPids, - mani_fsm_pid=ManiPid}) -> +terminate(_Reason, _StateName, #state{test = false, + all_reader_pids = BlockServerPids, + mani_fsm_pid = ManiPid}) -> riak_cs_manifest_fsm:maybe_stop_manifest_fsm(ManiPid), riak_cs_block_server:maybe_stop_block_servers(BlockServerPids), ok; @@ -402,31 +424,28 @@ code_change(_OldVsn, StateName, State, _Extra) -> {ok, StateName, State}. %% Internal functions %% =================================================================== --spec prepare(#state{}) -> #state{}. -prepare(#state{bucket=Bucket, - key=Key, - riak_client=RcPid}=State) -> +prepare(#state{bucket = Bucket, + key = Key, + obj_vsn = Vsn, + riak_client = RcPid} = State) -> %% start the process that will %% fetch the value, be it manifest %% or regular object - {ok, ManiPid} = riak_cs_manifest_fsm:start_link(Bucket, Key, RcPid), + {ok, ManiPid} = riak_cs_manifest_fsm:start_link(Bucket, Key, Vsn, RcPid), case riak_cs_manifest_fsm:get_active_manifest(ManiPid) of {ok, Manifest} -> - _ = lager:debug("Manifest: ~p", [Manifest]), case riak_cs_mp_utils:clean_multipart_unused_parts(Manifest, RcPid) of same -> - State#state{manifest=Manifest, - mani_fsm_pid=ManiPid}; + State#state{manifest = Manifest, + mani_fsm_pid = ManiPid}; updated -> riak_cs_manifest_fsm:stop(ManiPid), prepare(State) end; {error, notfound} -> - State#state{mani_fsm_pid=ManiPid} + State#state{mani_fsm_pid = ManiPid} end. --spec read_blocks(state()) -> state(). - read_blocks(#state{free_readers=[]} = State) -> State; read_blocks(#state{blocks_order=[]} = State) -> @@ -439,6 +458,8 @@ read_blocks(#state{manifest=Manifest, blocks_intransit=Intransit} = State) -> ClusterID = cluster_id_or_default(Manifest?MANIFEST.cluster_id), {UUID, Seq} = NextBlock, + %% no need to thread ObjVsnName to block server as it forms its + %% keys from uuids (in turn from corresponding version manifests) riak_cs_block_server:get_block(ReaderPid, Bucket, Key, ClusterID, UUID, Seq), read_blocks(State#state{free_readers=RestFreeReaders, blocks_order=BlocksOrder, diff --git a/src/riak_cs_get_fsm_sup.erl b/apps/riak_cs/src/riak_cs_get_fsm_sup.erl similarity index 64% rename from src/riak_cs_get_fsm_sup.erl rename to apps/riak_cs/src/riak_cs_get_fsm_sup.erl index 7a672356b..d111cb2cc 100644 --- a/src/riak_cs_get_fsm_sup.erl +++ b/apps/riak_cs/src/riak_cs_get_fsm_sup.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -25,7 +26,7 @@ -behaviour(supervisor). %% API --export([start_get_fsm/7]). +-export([start_get_fsm/8]). -export([start_link/0]). %% Supervisor callbacks @@ -43,13 +44,13 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). %% @doc Start a `riak_cs_get_fsm' child process. --spec start_get_fsm(node(), binary(), binary(), pid(), riak_client(), pos_integer(), +-spec start_get_fsm(node(), binary(), binary(), binary(), pid(), riak_client(), pos_integer(), pos_integer()) -> - {ok, pid()} | {error, term()}. %% SLF: R14B04's supervisor:startchild_ret() is broken? -start_get_fsm(Node, Bucket, Key, Caller, RcPid, + {ok, pid()} | {error, term()}. %% SLF: R14B04's supervisor:startchild_ret() is broken? +start_get_fsm(Node, Bucket, Key, ObjVsn, Caller, RcPid, FetchConcurrency, BufferFactor) -> - supervisor:start_child({?MODULE, Node}, [Bucket, Key, Caller, RcPid, - FetchConcurrency, BufferFactor]). + supervisor:start_child({?MODULE, Node}, [Bucket, Key, ObjVsn, Caller, RcPid, + FetchConcurrency, BufferFactor]). %% =================================================================== %% Supervisor callbacks @@ -57,23 +58,15 @@ start_get_fsm(Node, Bucket, Key, Caller, RcPid, %% @doc Initialize this supervisor. This is a `simple_one_for_one', %% whose child spec is for starting `riak_cs_get_fsm' processes. --spec init([]) -> {ok, {{supervisor:strategy(), - pos_integer(), - pos_integer()}, - [supervisor:child_spec()]}}. +-spec init([]) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. init([]) -> - RestartStrategy = simple_one_for_one, - MaxRestarts = 1000, - MaxSecondsBetweenRestarts = 3600, + SupFlags = #{strategy => simple_one_for_one, + intensity => 1000, + period => 3600}, - SupFlags = {RestartStrategy, MaxRestarts, MaxSecondsBetweenRestarts}, - - Restart = temporary, - Shutdown = 2000, - Type = worker, - - PutFsmSpec = {undefined, - {riak_cs_get_fsm, start_link, []}, - Restart, Shutdown, Type, [riak_cs_get_fsm]}, + PutFsmSpec = #{id => get_fsm, + start => {riak_cs_get_fsm, start_link, []}, + restart => temporary, + shutdown => 2000}, {ok, {SupFlags, [PutFsmSpec]}}. diff --git a/apps/riak_cs/src/riak_cs_iam.erl b/apps/riak_cs/src/riak_cs_iam.erl new file mode 100644 index 000000000..e75af00d6 --- /dev/null +++ b/apps/riak_cs/src/riak_cs_iam.erl @@ -0,0 +1,805 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_iam). + +-export([create_user/1, + delete_user/1, + get_user/2, + find_user/2, + update_user/1, + list_attached_user_policies/3, + list_users/2, + + create_policy/1, + update_policy/1, + delete_policy/2, + get_policy/2, + find_policy/2, + list_policies/2, + attach_role_policy/3, + detach_role_policy/3, + attach_user_policy/3, + detach_user_policy/3, + express_policies/2, + + create_role/1, + update_role/1, + delete_role/1, + get_role/2, + list_roles/2, + find_role/2, + list_attached_role_policies/3, + + create_saml_provider/1, + delete_saml_provider/1, + get_saml_provider/2, + list_saml_providers/2, + find_saml_provider/2, + parse_saml_provider_idp_metadata/1, + + fix_permissions_boundary/1, + exprec_user/1, + exprec_bucket/1, + exprec_role/1, + exprec_iam_policy/1, + exprec_saml_provider/1, + unarm/1 + ]). + +-include("moss.hrl"). +-include("riak_cs.hrl"). +-include("aws_api.hrl"). +-include_lib("riakc/include/riakc.hrl"). +-include_lib("xmerl/include/xmerl.hrl"). +-include_lib("kernel/include/logger.hrl"). + + +%% ------------ users + +-spec create_user(maps:map()) -> {ok, rcs_user()} | {error, already_exists | term()}. +create_user(Specs = #{user_name := Name}) -> + Email = iolist_to_binary([Name, $@, riak_cs_config:iam_create_user_default_email_host()]), + riak_cs_user:create_user(Name, Email, Specs). + +-spec delete_user(rcs_user()) -> ok | {error, term()}. +delete_user(?IAM_USER{attached_policies = PP}) when PP /= [] -> + {error, user_has_attached_policies}; +delete_user(?IAM_USER{buckets = BB}) when BB /= [] -> + {error, user_has_buckets}; +delete_user(?IAM_USER{arn = Arn}) -> + {ok, AdminCreds} = riak_cs_config:admin_creds(), + velvet:delete_user(base64:encode(Arn), [{auth_creds, AdminCreds}]). + +-spec get_user(flat_arn(), pid()) -> {ok, {rcs_user(), riakc_obj:riakc_obj()}} | {error, notfound}. +get_user(Arn, Pbc) -> + case riak_cs_pbc:get(Pbc, ?USER_BUCKET, Arn, get_cs_user) of + {ok, Obj} -> + {ok, {riak_cs_user:from_riakc_obj(Obj, _KeepDeletedBuckets = false), Obj}}; + {weak_ok, Obj} -> + {ok, {riak_cs_user:from_riakc_obj(Obj, true), Obj}}; + ER -> + ER + end. + +-spec find_user(maps:map(), pid()) -> {ok, {rcs_user(), riakc_obj:riakc_obj()}} | {error, notfound | term()}. +find_user(#{name := A}, Pbc) -> + find_user(?USER_NAME_INDEX, A, Pbc); +find_user(#{canonical_id := A}, Pbc) -> + find_user(?USER_ID_INDEX, A, Pbc); +find_user(#{key_id := A}, Pbc) -> + case find_user(?USER_KEYID_INDEX, A, Pbc) of + {ok, _} = Found -> + Found; + _ -> + ?LOG_DEBUG("Trying to read 3.1 user by KeyId ~s", [A]), + get_user(A, Pbc) + end; +find_user(#{email := A}, Pbc) -> + find_user(?USER_EMAIL_INDEX, A, Pbc). + +find_user(Index, A, Pbc) -> + Res = riakc_pb_socket:get_index_eq(Pbc, ?USER_BUCKET, Index, A), + case Res of + {ok, ?INDEX_RESULTS{keys = []}} -> + {error, notfound}; + {ok, ?INDEX_RESULTS{keys = [Arn|_]}} -> + get_user(Arn, Pbc); + {error, Reason} -> + logger:notice("Riak client connection error while finding user ~s in ~s: ~p", [A, Index, Reason]), + {error, Reason} + end. + +-spec update_user(rcs_user()) -> {ok, rcs_user()} | {error, reportable_error_reason()}. +update_user(U = ?IAM_USER{name = Name}) -> + UserKeyForLocking = <<"u", Name/binary>>, + Lock = stanchion_lock:acquire(UserKeyForLocking), + try + update_user_already_locked(U) + after + stanchion_lock:release(UserKeyForLocking, Lock) + end. + +update_user_already_locked(U = ?IAM_USER{key_id = KeyId}) -> + {ok, AdminCreds} = riak_cs_config:admin_creds(), + velvet:update_user("application/json", + KeyId, + riak_cs_json:to_json(U), + [{auth_creds, AdminCreds}]). + +-spec list_attached_user_policies(binary(), binary(), pid()) -> + {ok, [{flat_arn(), PolicyName::binary()}]} | {error, term()}. +list_attached_user_policies(UserName, PathPrefix_, Pbc) -> + PathPrefix = sanitize_path_prefix(PathPrefix_), + case find_user(#{name => UserName}, Pbc) of + {ok, {?IAM_USER{attached_policies = AA}, _}} -> + AANN = [begin + try + {ok, ?IAM_POLICY{path = Path, + policy_name = N}} = get_policy(A, Pbc), + case 0 < binary:longest_common_prefix([Path, PathPrefix]) of + true -> + {A, N}; + false -> + [] + end + catch + error:badmatch -> + logger:error("Policy ~s not found while it is still attached to user ~s", + [A, UserName]), + [] + end + end || A <- AA], + {ok, lists:flatten(AANN)}; + ER -> + ER + end. + +-spec list_users(riak_client(), #list_users_request{}) -> + {ok, maps:map()} | {error, term()}. +list_users(RcPid, #list_users_request{path_prefix = PathPrefix, + max_items = MaxItems, + marker = Marker}) -> + Arg = #{path_prefix => PathPrefix, + max_items => MaxItems, + marker => Marker}, + {ok, MasterPbc} = riak_cs_riak_client:master_pbc(RcPid), + case riakc_pb_socket:mapred_bucket( + MasterPbc, ?IAM_USER_BUCKET, riak_cs_riak_mapred:query(users, Arg)) of + {ok, Batches} -> + {ok, #{users => extract_objects(Batches, []), + marker => undefined, + is_truncated => false}}; + {error, _} = ER -> + ER + end. + + + +%% ------------ policies + +-spec create_policy(maps:map()) -> {ok, iam_policy()} | {error, reportable_error_reason()}. +create_policy(Specs = #{policy_document := D}) -> + case riak_cs_aws_policy:policy_from_json(D) of %% this is to validate PolicyDocument + {ok, _} -> + Encoded = riak_cs_json:to_json(exprec_iam_policy(Specs)), + {ok, AdminCreds} = riak_cs_config:admin_creds(), + velvet:create_policy("application/json", + Encoded, + [{auth_creds, AdminCreds}]); + {error, _} = ER -> + ER + end. + +-spec delete_policy(binary(), pid()) -> ok | {error, reportable_error_reason()}. +delete_policy(Arn, Pbc) -> + {ok, AdminCreds} = riak_cs_config:admin_creds(), + case get_policy(Arn, Pbc) of + {ok, ?IAM_POLICY{attachment_count = AC}} when AC > 0 -> + {error, policy_in_use}; + {ok, _} -> + velvet:delete_policy(Arn, [{auth_creds, AdminCreds}]); + {error, notfound} -> + {error, no_such_policy} + end. + +-spec get_policy(binary(), pid()) -> {ok, ?IAM_POLICY{}} | {error, term()}. +get_policy(Arn, Pbc) -> + case riak_cs_pbc:get(Pbc, ?IAM_POLICY_BUCKET, Arn, get_cs_policy) of + {OK, Obj} when OK =:= ok; + OK =:= weak_ok -> + from_riakc_obj(Obj); + ER -> + ER + end. + +-spec find_policy(maps:map() | binary(), pid()) -> {ok, policy()} | {error, notfound | term()}. +find_policy(Name, Pbc) when is_binary(Name) -> + find_policy(#{name => Name}, Pbc); +find_policy(#{name := Name}, Pbc) -> + Res = riakc_pb_socket:get_index_eq(Pbc, ?IAM_POLICY_BUCKET, ?POLICY_NAME_INDEX, Name), + case Res of + {ok, ?INDEX_RESULTS{keys = []}} -> + {error, notfound}; + {ok, ?INDEX_RESULTS{keys = [Key|_]}} -> + get_policy(Key, Pbc); + {error, Reason} -> + {error, Reason} + end. + +-spec list_policies(riak_client(), #list_policies_request{}) -> + {ok, maps:map()} | {error, term()}. +list_policies(RcPid, #list_policies_request{path_prefix = PathPrefix, + only_attached = OnlyAttached, + policy_usage_filter = PolicyUsageFilter, + scope = Scope, + max_items = MaxItems, + marker = Marker}) -> + Arg = #{path_prefix => PathPrefix, + only_attached => OnlyAttached, + policy_usage_filter => PolicyUsageFilter, + scope => Scope, + max_items => MaxItems, + marker => Marker}, + {ok, MasterPbc} = riak_cs_riak_client:master_pbc(RcPid), + case riakc_pb_socket:mapred_bucket( + MasterPbc, ?IAM_POLICY_BUCKET, riak_cs_riak_mapred:query(policies, Arg)) of + {ok, Batches} -> + {ok, #{policies => extract_objects(Batches, []), + marker => undefined, + is_truncated => false}}; + {error, _} = ER -> + ER + end. + +-spec attach_role_policy(binary(), binary(), pid()) -> + ok | {error, reportable_error_reason()}. +attach_role_policy(PolicyArn, RoleName, Pbc) -> + RoleKeyForLocking = <<"r", RoleName/binary>>, + Lock1 = stanchion_lock:acquire(RoleKeyForLocking), + Lock2 = stanchion_lock:acquire(PolicyArn), + try find_role(#{name => RoleName}, Pbc) of + {ok, Role = ?IAM_ROLE{attached_policies = PP}} -> + case lists:member(PolicyArn, PP) of + true -> + ok; + false -> + case get_policy(PolicyArn, Pbc) of + {ok, Policy = ?IAM_POLICY{is_attachable = true, + attachment_count = AC}} -> + case update_role(Role?IAM_ROLE{attached_policies = lists:usort([PolicyArn | PP])}) of + ok -> + update_policy(Policy?IAM_POLICY{attachment_count = AC + 1}); + ER1 -> + ER1 + end; + {ok, ?IAM_POLICY{}} -> + {error, policy_not_attachable}; + {error, notfound} -> + {error, no_such_policy} + end + end; + {error, notfound} -> + {error, no_such_role} + after + stanchion_lock:release(RoleKeyForLocking, Lock1), + stanchion_lock:release(PolicyArn, Lock2) + end. + +-spec detach_role_policy(binary(), binary(), pid()) -> + ok | {error, unmodifiable_entity}. +detach_role_policy(PolicyArn, RoleName, Pbc) -> + RoleKeyForLocking = <<"r", RoleName/binary>>, + Lock1 = stanchion_lock:acquire(RoleKeyForLocking), + Lock2 = stanchion_lock:acquire(PolicyArn), + try find_role(#{name => RoleName}, Pbc) of + {ok, Role = ?IAM_ROLE{attached_policies = PP}} -> + case lists:member(PolicyArn, PP) of + false -> + ok; + true -> + case get_policy(PolicyArn, Pbc) of + {ok, Policy = ?IAM_POLICY{attachment_count = AC}} -> + case update_role(Role?IAM_ROLE{attached_policies = lists:delete(PolicyArn, PP)}) of + ok -> + update_policy(Policy?IAM_POLICY{attachment_count = AC - 1}); + ER1 -> + ER1 + end; + {error, notfound} -> + {error, no_such_policy} + end + end; + {error, notfound} -> + {error, no_such_role} + after + stanchion_lock:release(RoleKeyForLocking, Lock1), + stanchion_lock:release(PolicyArn, Lock2) + end. + +-spec attach_user_policy(binary(), binary(), pid()) -> + ok | {error, reportable_error_reason()}. +attach_user_policy(PolicyArn, UserName, Pbc) -> + UserKeyForLocking = <<"u", UserName/binary>>, + Lock1 = stanchion_lock:acquire(UserKeyForLocking), + Lock2 = stanchion_lock:acquire(PolicyArn), + try find_user(#{name => UserName}, Pbc) of + {ok, {User = ?RCS_USER{attached_policies = PP}, _}} -> + case lists:member(PolicyArn, PP) of + true -> + ok; + false -> + case get_policy(PolicyArn, Pbc) of + {ok, Policy = ?IAM_POLICY{is_attachable = true, + attachment_count = AC}} -> + case update_user_already_locked( + User?RCS_USER{attached_policies = lists:usort([PolicyArn | PP])}) of + {ok, _} -> + update_policy(Policy?IAM_POLICY{attachment_count = AC + 1}); + {error, _} = ER -> + ER + end; + {ok, ?IAM_POLICY{}} -> + {error, policy_not_attachable}; + {error, notfound} -> + {error, no_such_policy} + end + end; + {error, notfound} -> + {error, no_such_user} + after + stanchion_lock:release(PolicyArn, Lock2), + stanchion_lock:release(UserKeyForLocking, Lock1) + end. + +-spec detach_user_policy(binary(), binary(), pid()) -> + ok | {error, unmodifiable_entity}. +detach_user_policy(PolicyArn, UserName, Pbc) -> + UserKeyForLocking = <<"u", UserName/binary>>, + Lock1 = stanchion_lock:acquire(UserKeyForLocking), + Lock2 = stanchion_lock:acquire(PolicyArn), + try find_user(#{name => UserName}, Pbc) of + {ok, {User = ?RCS_USER{attached_policies = PP}, _}} -> + case lists:member(PolicyArn, PP) of + false -> + ok; + true -> + case get_policy(PolicyArn, Pbc) of + {ok, Policy = ?IAM_POLICY{attachment_count = AC}} -> + case update_user_already_locked( + User?RCS_USER{attached_policies = lists:delete(PolicyArn, PP)}) of + {ok, _} -> + update_policy(Policy?IAM_POLICY{attachment_count = AC - 1}); + {error, _} = ER -> + ER + end; + {error, notfound} -> + {error, no_such_policy} + end + end; + {error, notfound} -> + {error, no_such_user} + after + ?LOG_DEBUG("Are we even releasing?"), + stanchion_lock:release(PolicyArn, Lock2), + stanchion_lock:release(UserKeyForLocking, Lock1) + end. + +-spec update_role(role()) -> ok | {error, reportable_error_reason()}. +update_role(R = ?IAM_ROLE{arn = Arn}) -> + Encoded = riak_cs_json:to_json(R), + {ok, AdminCreds} = riak_cs_config:admin_creds(), + velvet:update_role("application/json", + Arn, + Encoded, + [{auth_creds, AdminCreds}]). + +-spec update_policy(iam_policy()) -> ok | {error, reportable_error_reason()}. +update_policy(A = ?IAM_POLICY{arn = Arn}) -> + Encoded = riak_cs_json:to_json(A), + {ok, AdminCreds} = riak_cs_config:admin_creds(), + velvet:update_policy("application/json", + Arn, + Encoded, + [{auth_creds, AdminCreds}]). + + +-spec express_policies([flat_arn()], pid()) -> [policy()]. +express_policies(AA, Pbc) -> + lists:flatten( + [begin + case get_policy(Arn, Pbc) of + {ok, ?IAM_POLICY{policy_document = D}} -> + case riak_cs_aws_policy:policy_from_json(D) of + {ok, P} -> + P; + {error, _ReasonAlreadyReported} -> + logger:notice("Managed policy ~s has invalid policy document", [Arn]), + [] + end; + {error, _} -> + logger:notice("Managed policy ~s is not available", [Arn]), + [] + end + end || Arn <- AA] + ). + +%% CreateRole takes a string for PermissionsBoundary parameter, which +%% needs to become part of a structure (and handled and exported thus), so: +-spec fix_permissions_boundary(maps:map()) -> maps:map(). +fix_permissions_boundary(#{permissions_boundary := A} = Map) when A /= null, + A /= undefined -> + maps:update(permissions_boundary, #{permissions_boundary_arn => A}, Map); +fix_permissions_boundary(Map) -> + Map. + + +%% ------------ roles + +-spec create_role(maps:map()) -> {ok, role()} | {error, reportable_error_reason()}. +create_role(Specs) -> + case validate_role_specs(Specs) of + ok -> + Encoded = riak_cs_json:to_json(exprec_role(Specs)), + {ok, AdminCreds} = riak_cs_config:admin_creds(), + velvet:create_role("application/json", + Encoded, + [{auth_creds, AdminCreds}]); + ER -> + ER + end. + +validate_role_specs(#{assume_role_policy_document := A}) -> + case riak_cs_aws_policy:policy_from_json(A) of + {ok, _} -> + ok; + ER -> + ER + end; +validate_role_specs(#{}) -> + {error, missing_parameter}. + + +-spec delete_role(binary()) -> ok | {error, reportable_error_reason()}. +delete_role(Arn) -> + {ok, AdminCreds} = riak_cs_config:admin_creds(), + velvet:delete_role(Arn, [{auth_creds, AdminCreds}]). + +-spec get_role(binary(), pid()) -> {ok, role()} | {error, term()}. +get_role(Arn, Pbc) -> + case riak_cs_pbc:get(Pbc, ?IAM_ROLE_BUCKET, Arn, get_cs_role) of + {OK, Obj} when OK =:= ok; + OK =:= weak_ok -> + from_riakc_obj(Obj); + Error -> + Error + end. + +-spec find_role(maps:map() | binary(), pid()) -> {ok, role()} | {error, notfound | term()}. +find_role(Name, Pbc) when is_binary(Name) -> + find_role(#{name => Name}, Pbc); +find_role(#{name := A}, Pbc) -> + find_role(?ROLE_NAME_INDEX, A, Pbc); +find_role(#{path := A}, Pbc) -> + find_role(?ROLE_PATH_INDEX, A, Pbc); +find_role(#{id := A}, Pbc) -> + find_role(?ROLE_ID_INDEX, A, Pbc). +find_role(Index, A, Pbc) -> + Res = riakc_pb_socket:get_index_eq(Pbc, ?IAM_ROLE_BUCKET, Index, A), + case Res of + {ok, ?INDEX_RESULTS{keys = []}} -> + {error, notfound}; + {ok, ?INDEX_RESULTS{keys = [Key|_]}} -> + get_role(Key, Pbc); + {error, Reason} -> + logger:notice("Riak client connection error while finding role ~s in ~s: ~p", [A, Index, Reason]), + {error, Reason} + end. + +-spec list_roles(riak_client(), #list_roles_request{}) -> + {ok, maps:map()} | {error, term()}. +list_roles(RcPid, #list_roles_request{path_prefix = PathPrefix, + max_items = MaxItems, + marker = Marker}) -> + Arg = #{path_prefix => PathPrefix, + max_items => MaxItems, + marker => Marker}, + {ok, MasterPbc} = riak_cs_riak_client:master_pbc(RcPid), + case riakc_pb_socket:mapred_bucket( + MasterPbc, ?IAM_ROLE_BUCKET, riak_cs_riak_mapred:query(roles, Arg)) of + {ok, Batches} -> + {ok, #{roles => extract_objects(Batches, []), + marker => undefined, + is_truncated => false}}; + {error, _} = ER -> + ER + end. + + +-spec list_attached_role_policies(binary(), binary(), pid()) -> + {ok, [{flat_arn(), PolicyName::binary()}]} | {error, term()}. +list_attached_role_policies(RoleName, PathPrefix_, Pbc) -> + PathPrefix = sanitize_path_prefix(PathPrefix_), + case find_role(#{name => RoleName}, Pbc) of + {ok, ?IAM_ROLE{attached_policies = AA}} -> + AANN = [begin + try + {ok, ?IAM_POLICY{path = Path, + policy_name = N}} = get_policy(A, Pbc), + case 0 < binary:longest_common_prefix([Path, PathPrefix]) of + true -> + {A, N}; + false -> + [] + end + catch + error:badmatch -> + logger:notice("Policy ~s not found while it is still attached to role ~s", + [A, RoleName]), + [] + end + end || A <- AA], + {ok, lists:flatten(AANN)}; + ER -> + ER + end. + + + +%% ------------ SAML providers + +-spec create_saml_provider(maps:map()) -> {ok, {Arn::binary(), [tag()]}} | {error, reportable_error_reason()}. +create_saml_provider(Specs) -> + Encoded = riak_cs_json:to_json(exprec_saml_provider(Specs)), + {ok, AdminCreds} = riak_cs_config:admin_creds(), + velvet:create_saml_provider("application/json", + Encoded, + [{auth_creds, AdminCreds}]). + + +-spec delete_saml_provider(binary()) -> ok | {error, reportable_error_reason()}. +delete_saml_provider(Arn) -> + {ok, AdminCreds} = riak_cs_config:admin_creds(), + velvet:delete_saml_provider(Arn, [{auth_creds, AdminCreds}]). + + +-spec get_saml_provider(binary(), pid()) -> {ok, saml_provider()} | {error, term()}. +get_saml_provider(Arn, Pbc) -> + case riak_cs_pbc:get(Pbc, ?IAM_SAMLPROVIDER_BUCKET, Arn, get_cs_saml_provider) of + {OK, Obj} when OK =:= ok; + OK =:= weak_ok -> + from_riakc_obj(Obj); + Error -> + Error + end. + +-spec list_saml_providers(riak_client(), #list_saml_providers_request{}) -> + {ok, maps:map()} | {error, term()}. +list_saml_providers(RcPid, #list_saml_providers_request{}) -> + Arg = #{}, + {ok, MasterPbc} = riak_cs_riak_client:master_pbc(RcPid), + case riakc_pb_socket:mapred_bucket( + MasterPbc, ?IAM_SAMLPROVIDER_BUCKET, riak_cs_riak_mapred:query(saml_providers, Arg)) of + {ok, Batches} -> + {ok, #{saml_providers => extract_objects(Batches, [])}}; + {error, _} = ER -> + ER + end. + +-spec find_saml_provider(maps:map() | binary(), pid()) -> + {ok, saml_provider()} | {error, notfound | term()}. +find_saml_provider(Name, Pbc) when is_binary(Name) -> + find_saml_provider(#{name => Name}, Pbc); +find_saml_provider(#{name := Name}, Pbc) -> + Res = riakc_pb_socket:get_index_eq(Pbc, ?IAM_SAMLPROVIDER_BUCKET, ?SAMLPROVIDER_NAME_INDEX, Name), + case Res of + {ok, ?INDEX_RESULTS{keys = []}} -> + {error, notfound}; + {ok, ?INDEX_RESULTS{keys = [Key|_]}} -> + get_saml_provider(Key, Pbc); + {error, Reason} -> + {error, Reason} + end; +find_saml_provider(#{entity_id := EntityID}, Pbc) -> + Res = riakc_pb_socket:get_index_eq(Pbc, ?IAM_SAMLPROVIDER_BUCKET, ?SAMLPROVIDER_ENTITYID_INDEX, EntityID), + case Res of + {ok, ?INDEX_RESULTS{keys = []}} -> + {error, notfound}; + {ok, ?INDEX_RESULTS{keys = [Key|_]}} -> + get_saml_provider(Key, Pbc); + {error, Reason} -> + {error, Reason} + end. + + +-spec parse_saml_provider_idp_metadata(saml_provider()) -> + {ok, saml_provider()} | {error, invalid_metadata_document}. +parse_saml_provider_idp_metadata(?IAM_SAML_PROVIDER{saml_metadata_document = D} = P) -> + {Xml, _} = xmerl_scan:string(binary_to_list(D)), + Ns = [{namespace, [{"md", "urn:oasis:names:tc:SAML:2.0:metadata"}]}], + [#xmlAttribute{value = EntityID}] = xmerl_xpath:string("/md:EntityDescriptor/@entityID", Xml, Ns), + [#xmlAttribute{value = ValidUntil}] = xmerl_xpath:string("/md:EntityDescriptor/@validUntil", Xml, Ns), + [#xmlAttribute{value = ConsumeUri}] = xmerl_xpath:string("/md:EntityDescriptor/md:IDPSSODescriptor/md:SingleSignOnService/@Location", Xml, Ns), + case extract_certs( + xmerl_xpath:string("/md:EntityDescriptor/md:IDPSSODescriptor/md:KeyDescriptor", Xml, Ns), []) of + {ok, Certs} -> + {ok, P?IAM_SAML_PROVIDER{entity_id = list_to_binary(EntityID), + valid_until = calendar:rfc3339_to_system_time(ValidUntil, [{unit, millisecond}]), + consume_uri = list_to_binary(ConsumeUri), + certificates = Certs}}; + {error, Reason} -> + logger:warning("Problem parsing certificate in IdP metadata: ~p", [Reason]), + {error, invalid_metadata_document} + end. + +extract_certs([], Q) -> + {ok, Q}; +extract_certs([#xmlElement{content = RootContent, + attributes = RootAttributes} | Rest], Q) -> + [#xmlElement{content = KeyInfoContent}|_] = riak_cs_xml:find_elements('ds:KeyInfo', RootContent), + [#xmlElement{content = X509DataContent}|_] = riak_cs_xml:find_elements('ds:X509Data', KeyInfoContent), + [#xmlElement{content = X509CertificateContent}|_] = riak_cs_xml:find_elements('ds:X509Certificate', X509DataContent), + [#xmlText{value = CertDataS}] = [T || T = #xmlText{} <- X509CertificateContent], + [#xmlAttribute{value = TypeS}|_] = [A || A = #xmlAttribute{name = use} <- RootAttributes], + Type = list_to_atom(TypeS), + FP = esaml_util:convert_fingerprints( + [crypto:hash(sha, base64:decode(CertDataS))]), + case riak_cs_utils:parse_x509_cert(list_to_binary(CertDataS)) of + {ok, Certs} -> + extract_certs(Rest, [{Type, Certs, FP} | Q]); + ER -> + ER + end. + + +from_riakc_obj(Obj) -> + case riakc_obj:value_count(Obj) of + 1 -> + case riakc_obj:get_value(Obj) of + ?DELETED_MARKER -> + {error, notfound}; + V -> + {ok, binary_to_term(V)} + end; + 0 -> + error(no_value); + N -> + logger:warning("object with key ~p has ~b siblings", [riakc_obj:key(Obj), N]), + Values = [V || V <- riakc_obj:get_values(Obj), + V /= <<>>, %% tombstone + V /= ?DELETED_MARKER], + case length(Values) of + 0 -> + {error, notfound}; + _ -> + {ok, binary_to_term(hd(Values))} + end + end. + + +-spec exprec_user(maps:map()) -> ?IAM_USER{}. +exprec_user(Map) -> + U0 = ?IAM_USER{status = S, + attached_policies = AP0, + password_last_used = PLU0, + permissions_boundary = PB0, + tags = TT0, + buckets = BB} = exprec:frommap_rcs_user_v3(Map), + TT = [exprec:frommap_tag(T) || is_list(TT0), T <- TT0], + PB = case PB0 of + Undefined when Undefined =:= null; + Undefined =:= undefined -> + undefined; + _ -> + exprec:frommap_permissions_boundary(PB0) + end, + U0?IAM_USER{status = status_from_binary(S), + attached_policies = [A || AP0 /= <<>>, A <- AP0], + password_last_used = maybe_int(PLU0), + permissions_boundary = PB, + tags = TT, + buckets = [exprec_bucket(B) || BB /= <<>>, B <- BB]}. +status_from_binary(<<"enabled">>) -> enabled; +status_from_binary(<<"disabled">>) -> disabled. +maybe_int(null) -> undefined; +maybe_int(undefined) -> undefined; +maybe_int(A) -> A. + + +-spec exprec_bucket(maps:map()) -> ?RCS_BUCKET{}. +exprec_bucket(Map) -> + B0 = ?RCS_BUCKET{last_action = LA0, + acl = A0} = exprec:frommap_moss_bucket_v2(Map), + B0?RCS_BUCKET{last_action = last_action_from_binary(LA0), + acl = maybe_exprec_acl(A0)}. +last_action_from_binary(<<"undefined">>) -> undefined; +last_action_from_binary(<<"created">>) -> created; +last_action_from_binary(<<"deleted">>) -> deleted. +maybe_exprec_acl(null) -> undefined; +maybe_exprec_acl(undefined) -> undefined; +maybe_exprec_acl(A) -> exprec:frommap_acl_v3(A). + + +-spec exprec_role(maps:map()) -> ?IAM_ROLE{}. +exprec_role(Map) -> + Role0 = ?IAM_ROLE{permissions_boundary = PB0, + role_last_used = LU0, + attached_policies = AP0, + tags = TT0} = exprec:frommap_role_v1(Map), + PB = case PB0 of + Undefined when Undefined =:= null; + Undefined =:= undefined -> + undefined; + _ -> + exprec:frommap_permissions_boundary(PB0) + end, + LU = case LU0 of + Undefined1 when Undefined1 =:= null; + Undefined1 =:= undefined -> + undefined; + _ -> + exprec:frommap_role_last_used(LU0) + end, + AP = case AP0 of + Undefined2 when Undefined2 =:= null; + Undefined2 =:= undefined; + Undefined2 =:= <<>> -> + []; + Defined -> + Defined + end, + TT = [exprec:frommap_tag(T) || is_list(TT0), T <- TT0], + Role0?IAM_ROLE{permissions_boundary = PB, + role_last_used = LU, + attached_policies = AP, + tags = TT}. + +-spec exprec_iam_policy(maps:map()) -> ?IAM_POLICY{}. +exprec_iam_policy(Map) -> + Policy0 = ?IAM_POLICY{tags = TT0} = exprec:frommap_iam_policy(Map), + TT = [exprec:frommap_tag(T) || is_list(TT0), T <- TT0], + Policy0?IAM_POLICY{tags = TT}. + +-spec exprec_saml_provider(maps:map()) -> ?IAM_SAML_PROVIDER{}. +exprec_saml_provider(Map) -> + P0 = ?IAM_SAML_PROVIDER{tags = TT0} = exprec:frommap_saml_provider_v1(Map), + TT = [exprec:frommap_tag(T) || is_list(TT0), T <- TT0], + P0?IAM_SAML_PROVIDER{tags = TT}. + +-spec unarm(A) -> A when A :: rcs_user() | role() | iam_policy() | saml_provider(). +unarm(A = ?IAM_USER{}) -> + A; +unarm(A = ?IAM_POLICY{policy_document = D}) -> + A?IAM_POLICY{policy_document = base64:decode(D)}; +unarm(A = ?IAM_ROLE{assume_role_policy_document = D}) + when is_binary(D), size(D) > 0 -> + A?IAM_ROLE{assume_role_policy_document = base64:decode(D)}; +unarm(A = ?IAM_SAML_PROVIDER{saml_metadata_document = D}) -> + A?IAM_SAML_PROVIDER{saml_metadata_document = base64:decode(D)}. + + +extract_objects([], Q) -> + Q; +extract_objects([{_N, RR}|Rest], Q) -> + extract_objects(Rest, Q ++ RR). + + +sanitize_path_prefix(<<>>) -> <<"/">>; +sanitize_path_prefix(A) -> A. diff --git a/src/riak_cs_json.erl b/apps/riak_cs/src/riak_cs_json.erl similarity index 51% rename from src/riak_cs_json.erl rename to apps/riak_cs/src/riak_cs_json.erl index dda831d97..a7c30aa1d 100644 --- a/src/riak_cs_json.erl +++ b/apps/riak_cs/src/riak_cs_json.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -24,8 +25,10 @@ -module(riak_cs_json). -include("riak_cs.hrl"). --include("list_objects.hrl"). +-include("riak_cs_web.hrl"). -include("oos_api.hrl"). +-include("aws_api.hrl"). +-include_lib("kernel/include/logger.hrl"). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -41,14 +44,15 @@ -type external_node() :: {atom(), [string()]}. -type internal_node() :: {atom(), [internal_node() | external_node()]} | {atom(), attributes(), [internal_node() | external_node()]}. +-export_type([attributes/0, external_node/0, internal_node/0]). %% =================================================================== %% Public API %% =================================================================== --spec from_json(string()) -> {struct, term()} | [term()] | {error, decode_failed}. +-spec from_json(string()) -> proplists:proplist() | {error, decode_failed}. from_json(JsonString) -> - case catch mochijson2:decode(JsonString) of + case catch jsx:decode(JsonString, [{return_maps, false}]) of {'EXIT', _} -> {error, decode_failed}; Result -> @@ -58,37 +62,98 @@ from_json(JsonString) -> -type match_spec() :: {index, non_neg_integer()} | {key, binary(), binary()}. -type path_query() :: {find, match_spec()}. -type path() :: [binary() | tuple() | path_query()]. --spec get({struct, term()} | [term()] | undefined, path()) -> term(). -get({struct, _}=Object, Path) -> - follow_path(Object, Path); -get(Array, [{find, Query} | RestPath]) when is_list(Array) -> - follow_path(find(Array, Query), RestPath); -get(_Array, [{find, _Query} | _RestPath]) -> - {error, invalid_path}; +-spec get(proplists:proplist() | undefined, path()) -> term(). get(undefined, _) -> {error, not_found}; get(not_found, _) -> - {error, not_found}. - --spec to_json(term()) -> binary(). -to_json(?KEYSTONE_S3_AUTH_REQ{}=Req) -> - Inner = {struct, [{<<"access">>, Req?KEYSTONE_S3_AUTH_REQ.access}, - {<<"signature">>, Req?KEYSTONE_S3_AUTH_REQ.signature}, - {<<"token">>, Req?KEYSTONE_S3_AUTH_REQ.token}]}, - iolist_to_binary(mochijson2:encode({struct, [{<<"credentials">>, Inner}]})); -to_json(?RCS_USER{}=Req) -> - iolist_to_binary(mochijson2:encode(user_object(Req))); -to_json({users, Users}) -> - UserList = [user_object(User) || User <- Users], - iolist_to_binary(mochijson2:encode(UserList)); + {error, not_found}; +get(Array, [{find, Query} | RestPath]) -> + follow_path(find(Array, Query), RestPath); +get(Object, Path) -> + follow_path(Object, Path). + +-spec to_json(tuple() | undefined | []) -> binary(). +to_json(?KEYSTONE_S3_AUTH_REQ{} = A) -> + list_to_binary( + jason:encode(A, + [{records, [{keystone_aws_auth_req_v1, record_info(fields, keystone_aws_auth_req_v1)}]}])); +to_json(?RCS_USER{} = A) -> + list_to_binary( + fix_quotes_for_empty_lists( + user, + jason:encode(A, + [{records, [{rcs_user_v3, record_info(fields, rcs_user_v3)}, + {permissions_boundary, record_info(fields, permissions_boundary)}, + {moss_bucket_v2, record_info(fields, moss_bucket_v2)}, + {acl_grant_v2, record_info(fields, acl_grant_v2)}, + {acl_v3, record_info(fields, acl_v3)}, + {tag, record_info(fields, tag)}]}]))); +to_json({users, AA}) -> + list_to_binary( + fix_quotes_for_empty_lists( + user, + jason:encode(AA, + [{records, [{rcs_user_v3, record_info(fields, rcs_user_v3)}, + {permissions_boundary, record_info(fields, permissions_boundary)}, + {moss_bucket_v2, record_info(fields, moss_bucket_v2)}, + {acl_grant_v2, record_info(fields, acl_grant_v2)}, + {acl_v3, record_info(fields, acl_v3)}, + {tag, record_info(fields, tag)}]}]))); +to_json(?IAM_ROLE{assume_role_policy_document = D} = A) -> + list_to_binary( + fix_quotes_for_empty_lists( + role, + jason:encode(A?IAM_ROLE{assume_role_policy_document = base64:encode(D)}, + [{records, [{role_v1, record_info(fields, role_v1)}, + {role_last_used, record_info(fields, role_last_used)}, + {permissions_boundary, record_info(fields, permissions_boundary)}, + {tag, record_info(fields, tag)}]}]))); +to_json(?IAM_POLICY{policy_document = D} = A) -> + list_to_binary( + fix_quotes_for_empty_lists( + policy, + jason:encode(A?IAM_POLICY{policy_document = base64:encode(D)}, + [{records, [{iam_policy, record_info(fields, iam_policy)}, + {tag, record_info(fields, tag)}]}]))); +to_json(?IAM_SAML_PROVIDER{saml_metadata_document = D} = A) -> + list_to_binary( + fix_quotes_for_empty_lists( + saml_provider, + jason:encode(A?IAM_SAML_PROVIDER{saml_metadata_document = base64:encode(D)}, + [{records, [{saml_provider_v1, record_info(fields, saml_provider_v1)}, + {tag, record_info(fields, tag)}]}]))); +to_json(?ACL{} = A) -> + list_to_binary( + jason:encode(A, + [{records, [{acl_v3, record_info(fields, acl_v3)}, + {acl_grant_v2, record_info(fields, acl_grant_v2)}]}])); + to_json(undefined) -> - []; + <<>>; to_json([]) -> - []. + <<>>. + +fix_quotes_for_empty_lists(A, S) -> + lists:foldl(fun({RE, Replacement}, Q) -> re:replace(Q, RE, Replacement, [global]) end, S, res(A)). +res(user) -> + [{"\"buckets\": \"\"", "\"buckets\": []"}, + {"\"attached_policies\": \"\"", "\"attached_policies\": []"}, + {"\"tags\": \"\"", "\"tags\": []"}]; +res(role) -> + [{"\"attached_policies\": \"\"", "\"attached_policies\": []"}, + {"\"tags\": \"\"", "\"tags\": []"}]; +res(policy) -> + [{"\"statement\": \"\"", "\"statement\": []"}, + {"\"tags\": \"\"", "\"tags\": []"}]; +res(saml_provider) -> + [{"\"certificates\": \"\"", "\"certificates\": []"}, + {"\"tags\": \"\"", "\"tags\": []"}]. + + -spec value_or_default({ok, term()} | {error, term()}, term()) -> term(). value_or_default({error, Reason}, Default) -> - _ = lager:debug("JSON error: ~p", [Reason]), + ?LOG_DEBUG("JSON error: ~p", [Reason]), Default; value_or_default({ok, Value}, _) -> Value. @@ -97,8 +162,6 @@ value_or_default({ok, Value}, _) -> %% Internal functions %% =================================================================== --spec follow_path(tuple() | [term()] | undefined, path()) -> - {ok, term()} | {error, not_found}. follow_path(undefined, _) -> {error, not_found}; follow_path(Value, []) -> @@ -107,15 +170,14 @@ follow_path(JsonItems, [{find, Query}]) -> follow_path(find(JsonItems, Query), []); follow_path(JsonItems, [{find, Query} | RestPath]) -> get(find(JsonItems, Query), RestPath); -follow_path({struct, JsonItems}, [Key]) when is_tuple(Key) -> +follow_path(JsonItems, [Key]) when is_tuple(Key) -> follow_path(target_tuple_values(Key, JsonItems), []); -follow_path({struct, JsonItems}, [Key]) -> +follow_path(JsonItems, [Key]) -> follow_path(proplists:get_value(Key, JsonItems), []); -follow_path({struct, JsonItems}, [Key | RestPath]) -> +follow_path(JsonItems, [Key | RestPath]) -> Value = proplists:get_value(Key, JsonItems), follow_path(Value, RestPath). --spec find([term()], match_spec()) -> undefined | {struct, term()}. find(Array, {key, Key, Value}) -> lists:foldl(key_folder(Key, Value), not_found, Array); find(Array, {index, Index}) when Index =< length(Array) -> @@ -124,15 +186,13 @@ find(_, {index, _}) -> undefined. key_folder(Key, Value) -> - fun({struct, Items}=X, Acc) -> - case lists:keyfind(Key, 1, Items) of + fun(X, Acc) -> + case lists:keyfind(Key, 1, X) of {Key, Value} -> X; _ -> Acc - end; - (_, Acc) -> - Acc + end end. -spec target_tuple_values(tuple(), proplists:proplist()) -> tuple(). @@ -141,28 +201,6 @@ target_tuple_values(Keys, JsonItems) -> [proplists:get_value(element(Index, Keys), JsonItems) || Index <- lists:seq(1, tuple_size(Keys))]). --spec user_object(rcs_user()) -> {struct, proplists:proplist()}. -user_object(?RCS_USER{email=Email, - display_name=DisplayName, - name=Name, - key_id=KeyID, - key_secret=KeySecret, - canonical_id=CanonicalID, - status=Status}) -> - StatusBin = case Status of - enabled -> - <<"enabled">>; - _ -> - <<"disabled">> - end, - UserData = [{email, list_to_binary(Email)}, - {display_name, list_to_binary(DisplayName)}, - {name, list_to_binary(Name)}, - {key_id, list_to_binary(KeyID)}, - {key_secret, list_to_binary(KeySecret)}, - {id, list_to_binary(CanonicalID)}, - {status, StatusBin}], - {struct, UserData}. %% =================================================================== %% Eunit tests @@ -170,8 +208,8 @@ user_object(?RCS_USER{email=Email, -ifdef(TEST). get_single_key_test() -> - Object1 = "{\"abc\":\"123\", \"def\":\"456\", \"ghi\":\"789\"}", - Object2 = "{\"test\":{\"abc\":\"123\", \"def\":\"456\", \"ghi\":\"789\"}}", + Object1 = <<"{\"abc\":\"123\", \"def\":\"456\", \"ghi\":\"789\"}">>, + Object2 = <<"{\"test\":{\"abc\":\"123\", \"def\":\"456\", \"ghi\":\"789\"}}">>, ?assertEqual({ok, <<"123">>}, get(from_json(Object1), [<<"abc">>])), ?assertEqual({ok, <<"456">>}, get(from_json(Object1), [<<"def">>])), ?assertEqual({ok, <<"789">>}, get(from_json(Object1), [<<"ghi">>])), @@ -182,7 +220,7 @@ get_single_key_test() -> ?assertEqual({error, not_found}, get(from_json(Object2), [<<"test">>, <<"zzz">>])). get_array_test() -> - Array = "[\"abc\", \"123\", \"def\", \"456\", 7]", + Array = <<"[\"abc\", \"123\", \"def\", \"456\", 7]">>, ?assertEqual({ok, <<"abc">>}, get(from_json(Array), [{find, {index, 1}}])), ?assertEqual({ok, <<"123">>}, get(from_json(Array), [{find, {index, 2}}])), ?assertEqual({ok, <<"def">>}, get(from_json(Array), [{find, {index, 3}}])), @@ -191,44 +229,44 @@ get_array_test() -> ?assertEqual({error, not_found}, get(from_json(Array), [{find, {index, 6}}])). get_multi_key_test() -> - Object1 = "{\"test\":{\"abc\":\"123\", \"def\":\"456\", \"ghi\":\"789\"}}", - Object2 = "{\"test\":{\"abc\":{\"123\":123,\"456\":456,\"789\":789},\"def\"" - ":{\"123\":123,\"456\":456,\"789\":789},\"ghi\":{\"123\":123,\"456\"" - ":456,\"789\":789}}}", + Object1 = <<"{\"test\":{\"abc\":\"123\", \"def\":\"456\", \"ghi\":\"789\"}}">>, + Object2 = <<"{\"test\":{\"abc\":{\"123\":123,\"456\":456,\"789\":789},\"def\"" + ":{\"123\":123,\"456\":456,\"789\":789},\"ghi\":{\"123\":123,\"456\"" + ":456,\"789\":789}}}">>, ?assertEqual({ok, {<<"123">>, <<"789">>}}, get(from_json(Object1), [<<"test">>, {<<"abc">>, <<"ghi">>}])), ?assertEqual({ok, {123, 789}}, get(from_json(Object2), [<<"test">>, <<"abc">>, {<<"123">>, <<"789">>}])), ?assertEqual({ok, {123, 789}}, get(from_json(Object2), [<<"test">>, <<"def">>, {<<"123">>, <<"789">>}])), ?assertEqual({ok, {123, 789}}, get(from_json(Object2), [<<"test">>, <<"ghi">>, {<<"123">>, <<"789">>}])). get_embedded_key_from_array_test() -> - Object = "{\"test\":{\"objects\":[{\"key1\":\"a1\",\"key2\":\"a2\",\"key3\"" - ":\"a3\"},{\"key1\":\"b1\",\"key2\":\"b2\",\"key3\":\"b3\"},{\"key1\"" - ":\"c1\",\"key2\":\"c2\",\"key3\":\"c3\"}]}}", - ?assertEqual({ok, {struct, [{<<"key1">>, <<"a1">>}, {<<"key2">>, <<"a2">>}, {<<"key3">>, <<"a3">>}]}}, + Object = <<"{\"test\":{\"objects\":[{\"key1\":\"a1\",\"key2\":\"a2\",\"key3\"" + ":\"a3\"},{\"key1\":\"b1\",\"key2\":\"b2\",\"key3\":\"b3\"},{\"key1\"" + ":\"c1\",\"key2\":\"c2\",\"key3\":\"c3\"}]}}">>, + ?assertEqual({ok, [{<<"key1">>, <<"a1">>}, {<<"key2">>, <<"a2">>}, {<<"key3">>, <<"a3">>}]}, get(from_json(Object), [<<"test">>, <<"objects">>, {find, {key, <<"key1">>, <<"a1">>}}])), - ?assertEqual({ok, {struct, [{<<"key1">>, <<"a1">>}, {<<"key2">>, <<"a2">>}, {<<"key3">>, <<"a3">>}]}}, + ?assertEqual({ok, [{<<"key1">>, <<"a1">>}, {<<"key2">>, <<"a2">>}, {<<"key3">>, <<"a3">>}]}, get(from_json(Object), [<<"test">>, <<"objects">>, {find, {key, <<"key2">>, <<"a2">>}}])), - ?assertEqual({ok, {struct, [{<<"key1">>, <<"a1">>}, {<<"key2">>, <<"a2">>}, {<<"key3">>, <<"a3">>}]}}, + ?assertEqual({ok, [{<<"key1">>, <<"a1">>}, {<<"key2">>, <<"a2">>}, {<<"key3">>, <<"a3">>}]}, get(from_json(Object), [<<"test">>, <<"objects">>, {find, {key, <<"key3">>, <<"a3">>}}])), - ?assertEqual({ok, {struct, [{<<"key1">>, <<"b1">>}, {<<"key2">>, <<"b2">>}, {<<"key3">>, <<"b3">>}]}}, + ?assertEqual({ok, [{<<"key1">>, <<"b1">>}, {<<"key2">>, <<"b2">>}, {<<"key3">>, <<"b3">>}]}, get(from_json(Object), [<<"test">>, <<"objects">>, {find, {key, <<"key1">>, <<"b1">>}}])), - ?assertEqual({ok, {struct, [{<<"key1">>, <<"b1">>}, {<<"key2">>, <<"b2">>}, {<<"key3">>, <<"b3">>}]}}, + ?assertEqual({ok, [{<<"key1">>, <<"b1">>}, {<<"key2">>, <<"b2">>}, {<<"key3">>, <<"b3">>}]}, get(from_json(Object), [<<"test">>, <<"objects">>, {find, {key, <<"key2">>, <<"b2">>}}])), - ?assertEqual({ok, {struct, [{<<"key1">>, <<"b1">>}, {<<"key2">>, <<"b2">>}, {<<"key3">>, <<"b3">>}]}}, + ?assertEqual({ok, [{<<"key1">>, <<"b1">>}, {<<"key2">>, <<"b2">>}, {<<"key3">>, <<"b3">>}]}, get(from_json(Object), [<<"test">>, <<"objects">>, {find, {key, <<"key3">>, <<"b3">>}}])), - ?assertEqual({ok, {struct, [{<<"key1">>, <<"c1">>}, {<<"key2">>, <<"c2">>}, {<<"key3">>, <<"c3">>}]}}, + ?assertEqual({ok, [{<<"key1">>, <<"c1">>}, {<<"key2">>, <<"c2">>}, {<<"key3">>, <<"c3">>}]}, get(from_json(Object), [<<"test">>, <<"objects">>, {find, {key, <<"key1">>, <<"c1">>}}])), - ?assertEqual({ok, {struct, [{<<"key1">>, <<"c1">>}, {<<"key2">>, <<"c2">>}, {<<"key3">>, <<"c3">>}]}}, + ?assertEqual({ok, [{<<"key1">>, <<"c1">>}, {<<"key2">>, <<"c2">>}, {<<"key3">>, <<"c3">>}]}, get(from_json(Object), [<<"test">>, <<"objects">>, {find, {key, <<"key2">>, <<"c2">>}}])), - ?assertEqual({ok, {struct, [{<<"key1">>, <<"c1">>}, {<<"key2">>, <<"c2">>}, {<<"key3">>, <<"c3">>}]}}, + ?assertEqual({ok, [{<<"key1">>, <<"c1">>}, {<<"key2">>, <<"c2">>}, {<<"key3">>, <<"c3">>}]}, get(from_json(Object), [<<"test">>, <<"objects">>, {find, {key, <<"key3">>, <<"c3">>}}])). diff --git a/src/riak_cs_keystone_auth.erl b/apps/riak_cs/src/riak_cs_keystone_auth.erl similarity index 72% rename from src/riak_cs_keystone_auth.erl rename to apps/riak_cs/src/riak_cs_keystone_auth.erl index 7728ba54e..230e08639 100644 --- a/src/riak_cs_keystone_auth.erl +++ b/apps/riak_cs/src/riak_cs_keystone_auth.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -22,14 +23,11 @@ -behavior(riak_cs_auth). --compile(export_all). - -export([identify/2, authenticate/4]). -include("riak_cs.hrl"). --include("s3_api.hrl"). +-include("aws_api.hrl"). -include("oos_api.hrl"). --include_lib("webmachine/include/webmachine.hrl"). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -42,16 +40,16 @@ %% Public API %% =================================================================== --spec identify(#wm_reqdata{}, #context{}) -> failed | {string() | undefined , string()}. -identify(RD, #context{api=s3}) -> - validate_token(s3, RD); -identify(RD, #context{api=oos}) -> +-spec identify(#wm_reqdata{}, #rcs_web_context{}) -> failed | {binary() | undefined , binary()}. +identify(RD, #rcs_web_context{api = aws}) -> + validate_token(aws, RD); +identify(RD, #rcs_web_context{api = oos}) -> validate_token(oos, wrq:get_req_header("x-auth-token", RD)). -spec authenticate(rcs_user(), {string(), term()}|tuple(), - #wm_reqdata{}, #context{}) -> - ok | {error, invalid_authentication}. + #wm_reqdata{}, #rcs_web_context{}) -> + ok | {error, invalid_authentication}. authenticate(_User, {_, TokenItems}, _RD, _Ctx) -> %% @TODO Expand authentication check for non-operators who may %% have access @@ -79,9 +77,8 @@ token_names(undefined) -> ordsets:new(); token_names(Roles) -> ordsets:from_list( - [proplists:get_value(<<"name">>, Role, []) || {struct, Role} <- Roles]). + [proplists:get_value(<<"name">>, Role, []) || Role <- Roles]). --spec validate_token(s3 | oos, undefined | string()) -> failed | {term(), term()}. validate_token(_, undefined) -> failed; validate_token(Api, AuthToken) -> @@ -94,24 +91,24 @@ validate_token(Api, AuthToken) -> handle_token_info_response( request_keystone_token_info(Api, AuthToken)). --spec request_keystone_token_info(s3 | oos, string() | {term(), term()}) -> term(). request_keystone_token_info(oos, AuthToken) -> RequestURI = riak_cs_config:os_tokens_url() ++ AuthToken, RequestHeaders = [{"X-Auth-Token", riak_cs_config:os_admin_token()}], httpc:request(get, {RequestURI, RequestHeaders}, [], []); -request_keystone_token_info(s3, RD) -> - {KeyId, Signature} = case wrq:get_req_header("authorization", RD) of - undefined -> - {wrq:get_qs_value(?QS_KEYID, RD), wrq:get_qs_value(?QS_SIGNATURE, RD)}; - AuthHeader -> - parse_auth_header(AuthHeader) - end, +request_keystone_token_info(aws, RD) -> + {KeyId, Signature} = + case wrq:get_req_header("authorization", RD) of + undefined -> + {wrq:get_qs_value(?QS_KEYID, RD), wrq:get_qs_value(?QS_SIGNATURE, RD)}; + AuthHeader -> + parse_auth_header(AuthHeader) + end, RequestURI = riak_cs_config:os_s3_tokens_url(), STS = base64url:encode_to_string(calculate_sts(RD)), - RequestBody = riak_cs_json:to_json(?KEYSTONE_S3_AUTH_REQ{ - access=list_to_binary(KeyId), - signature=list_to_binary(Signature), - token=list_to_binary(STS)}), + RequestBody = riak_cs_json:to_json( + ?KEYSTONE_S3_AUTH_REQ{access = list_to_binary(KeyId), + signature = list_to_binary(Signature), + token = list_to_binary(STS)}), RequestHeaders = [{"X-Auth-Token", riak_cs_config:os_admin_token()}], httpc:request(post, {RequestURI, RequestHeaders, "application/json", RequestBody}, [], []). @@ -132,8 +129,7 @@ handle_token_info_response({ok, {{_HTTPVer, _Status, _StatusLine}, _, TokenInfo} handle_token_info_response({ok, {{_HTTPVer, _Status, _StatusLine}, _, _}}) -> failed; handle_token_info_response({error, Reason}) -> - _ = lager:warning("Error occurred requesting token from keystone. Reason: ~p", - [Reason]), + logger:warning("Error occurred requesting token from keystone. Reason: ~p", [Reason]), failed. parse_auth_header("AWS " ++ Key) -> @@ -145,15 +141,11 @@ parse_auth_header("AWS " ++ Key) -> parse_auth_header(_) -> {undefined, undefined}. --spec calculate_sts(term()) -> binary(). calculate_sts(RD) -> Headers = riak_cs_wm_utils:normalize_headers(RD), AmazonHeaders = riak_cs_wm_utils:extract_amazon_headers(Headers), - OriginalResource = riak_cs_s3_rewrite:original_resource(RD), - Resource = case OriginalResource of - undefined -> []; %% TODO: get noisy here? - {Path,QS} -> [Path, canonicalize_qs(lists:sort(QS))] - end, + {Path,QS} = riak_cs_aws_rewrite:original_resource(RD), + Resource = [Path, canonicalize_qs(lists:sort(QS))], Expires = wrq:get_qs_value("Expires", RD), case Expires of undefined -> @@ -216,16 +208,18 @@ canonicalize_qs([{K, V}|T], Acc) -> -ifdef(TEST). tenant_id_test() -> - Token = "{\"access\":{\"token\":{\"expires\":\"2012-02-05T00:00:00\"," - "\"id\":\"887665443383838\", \"tenant\":{\"id\":\"1\", \"name\"" - ":\"customer-x\"}}, \"user\":{\"name\":\"joeuser\", \"tenantName\"" - ":\"customer-x\", \"id\":\"1\", \"roles\":[{\"serviceId\":\"1\"," - "\"id\":\"3\", \"name\":\"Member\"}], \"tenantId\":\"1\"}}}", - InvalidToken = "{\"access\":{\"token\":{\"expires\":\"2012-02-05T00:00:00\"," - "\"id\":\"887665443383838\", \"tenant\":{\"id\":\"1\", \"name\"" - ":\"customer-x\"}}, \"user\":{\"name\":\"joeuser\", \"tenantName\"" - ":\"customer-x\", \"id\":\"1\", \"roles\":[{\"serviceId\":\"1\"," - "\"id\":\"3\", \"name\":\"Member\"}]}}}", + Token = + <<"{\"access\":{\"token\":{\"expires\":\"2012-02-05T00:00:00\"," + "\"id\":\"887665443383838\", \"tenant\":{\"id\":\"1\", \"name\"" + ":\"customer-x\"}}, \"user\":{\"name\":\"joeuser\", \"tenantName\"" + ":\"customer-x\", \"id\":\"1\", \"roles\":[{\"serviceId\":\"1\"," + "\"id\":\"3\", \"name\":\"Member\"}], \"tenantId\":\"1\"}}}">>, + InvalidToken = + <<"{\"access\":{\"token\":{\"expires\":\"2012-02-05T00:00:00\"," + "\"id\":\"887665443383838\", \"tenant\":{\"id\":\"1\", \"name\"" + ":\"customer-x\"}}, \"user\":{\"name\":\"joeuser\", \"tenantName\"" + ":\"customer-x\", \"id\":\"1\", \"roles\":[{\"serviceId\":\"1\"," + "\"id\":\"3\", \"name\":\"Member\"}]}}}">>, ?assertEqual({ok, <<"1">>}, riak_cs_json:get(riak_cs_json:from_json(Token), [<<"access">>, <<"user">>, <<"tenantId">>])), diff --git a/src/riak_cs_kv_multi_backend.erl b/apps/riak_cs/src/riak_cs_kv_multi_backend.erl similarity index 95% rename from src/riak_cs_kv_multi_backend.erl rename to apps/riak_cs/src/riak_cs_kv_multi_backend.erl index 9b8939e18..c58e8fcea 100644 --- a/src/riak_cs_kv_multi_backend.erl +++ b/apps/riak_cs/src/riak_cs_kv_multi_backend.erl @@ -2,7 +2,8 @@ %% %% riak_cs_kv_multi_backend: switching between multiple storage engines %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -86,6 +87,7 @@ fold_buckets/4, fold_keys/4, fold_objects/4, + head/3, is_empty/1, data_size/1, status/1, @@ -232,11 +234,28 @@ stop(#state{backends=Backends}) -> _ = [Module:stop(SubState) || {_, Module, SubState} <- Backends], ok. +%% @doc Do everything get/3 does, without actually fetching the object +-spec head(riak_object:bucket(), riak_object:key(), state()) -> + {ok, any(), state()} | + {ok, not_found, state()} | + {error, term(), state()}. +head(Bucket, Key, State) -> + {Name, Mod, SubState} = get_backend(Bucket, State), + case Mod:head(Bucket, Key, SubState) of + {ok, Value, NewSubState} -> + NewState = update_backend_state(Name, Mod, NewSubState, State), + {ok, Value, NewState}; + {error, Reason, NewSubState} -> + NewState = update_backend_state(Name, Mod, NewSubState, State), + {error, Reason, NewState} + end. + + %% @doc Retrieve an object from the backend -spec get(riak_object:bucket(), riak_object:key(), state()) -> - {ok, any(), state()} | - {ok, not_found, state()} | - {error, term(), state()}. + {ok, any(), state()} | + {ok, not_found, state()} | + {error, term(), state()}. get(Bucket, Key, State) -> {Name, Module, SubState} = get_backend(Bucket, State), case Module:get(Bucket, Key, SubState) of @@ -250,10 +269,11 @@ get(Bucket, Key, State) -> %% @doc Insert an object with secondary index %% information into the kv backend --type index_spec() :: {add, Index, SecondaryKey} | {remove, Index, SecondaryKey}. +-type index_spec() :: {add, Index::binary(), SecondaryKey::binary()} + | {remove, Index::binary(), SecondaryKey::binary()}. -spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> - {ok, state()} | - {error, term(), state()}. + {ok, state()} | + {error, term(), state()}. put(Bucket, PrimaryKey, IndexSpecs, Value, State) -> {Name, Module, SubState} = get_backend(Bucket, State), case Module:put(Bucket, PrimaryKey, IndexSpecs, Value, SubState) of @@ -267,8 +287,8 @@ put(Bucket, PrimaryKey, IndexSpecs, Value, State) -> %% @doc Delete an object from the backend -spec delete(riak_object:bucket(), riak_object:key(), [index_spec()], state()) -> - {ok, state()} | - {error, term(), state()}. + {ok, state()} | + {error, term(), state()}. delete(Bucket, Key, IndexSpecs, State) -> {Name, Module, SubState} = get_backend(Bucket, State), case Module:delete(Bucket, Key, IndexSpecs, SubState) of @@ -403,18 +423,18 @@ maybe_mark_indexes_fixed(Mod, ModState, ForUpgrade) -> end. fix_index(BKeys, ForUpgrade, State) -> - % Group keys per bucket + % Group keys per bucket PerBucket = lists:foldl(fun(BK={B,_},D) -> dict:append(B,BK,D) end, dict:new(), BKeys), - Result = + Result = dict:fold( fun(Bucket, StorageKey, Acc = {Success, Ignore, Errors}) -> {_, Mod, ModState} = Backend = get_backend(Bucket, State), case backend_can_index_reformat(Mod, ModState) of - true -> - {S, I, E} = backend_fix_index(Backend, Bucket, + true -> + {S, I, E} = backend_fix_index(Backend, Bucket, StorageKey, ForUpgrade), {Success + S, Ignore + I, Errors + E}; - false -> + false -> Acc end end, {0, 0, 0}, PerBucket), @@ -422,11 +442,11 @@ fix_index(BKeys, ForUpgrade, State) -> backend_fix_index({_, Mod, ModState}, Bucket, StorageKey, ForUpgrade) -> case Mod:fix_index(StorageKey, ForUpgrade, ModState) of - {reply, Reply, _UpModState} -> + {reply, Reply, _UpModState} -> Reply; {error, Reason} -> - _ = lager:error("Failed to fix index for bucket ~p, key ~p, backend ~p: ~p", - [Bucket, StorageKey, Mod, Reason]), + logger:error("Failed to fix index for bucket ~p, key ~p, backend ~p: ~p", + [Bucket, StorageKey, Mod, Reason]), {0, 0, length(StorageKey)} end. diff --git a/src/riak_cs_lfs_utils.erl b/apps/riak_cs/src/riak_cs_lfs_utils.erl similarity index 80% rename from src/riak_cs_lfs_utils.erl rename to apps/riak_cs/src/riak_cs_lfs_utils.erl index 722f850d6..5f5b0d114 100644 --- a/src/riak_cs_lfs_utils.erl +++ b/apps/riak_cs/src/riak_cs_lfs_utils.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -20,8 +21,6 @@ -module(riak_cs_lfs_utils). --include("riak_cs.hrl"). - -export([block_count/2, block_keynames/3, block_name/3, @@ -37,11 +36,14 @@ initial_blocks/2, block_sequences_for_manifest/1, block_sequences_for_manifest/2, - new_manifest/12, + new_manifest/13, set_bag_id/2, remove_write_block/2, remove_delete_block/2]). +-include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). + %% ------------------------------------------------------------------- %% Public API %% ------------------------------------------------------------------- @@ -109,26 +111,24 @@ initial_blocks(ContentLength, BlockSize) -> lists:seq(0, (UpperBound - 1)). -spec initial_blocks(integer(), integer(), binary()) -> - [{binary(), integer()}]. + [{binary(), integer()}]. initial_blocks(ContentLength, SafeBlockSize, UUID) -> Bs = initial_blocks(ContentLength, SafeBlockSize), [{UUID, B} || B <- Bs]. -spec range_blocks(integer(), integer(), integer(), binary()) -> - {[{binary(), integer()}], integer(), integer()}. + {[{binary(), integer()}], integer(), integer()}. range_blocks(Start, End, SafeBlockSize, UUID) -> SkipInitial = Start rem SafeBlockSize, KeepFinal = (End rem SafeBlockSize) + 1, - _ = lager:debug("InitialBlock: ~p, FinalBlock: ~p~n", - [Start div SafeBlockSize, End div SafeBlockSize]), - _ = lager:debug("SkipInitial: ~p, KeepFinal: ~p~n", [SkipInitial, KeepFinal]), + ?LOG_DEBUG("InitialBlock: ~p, FinalBlock: ~p", + [Start div SafeBlockSize, End div SafeBlockSize]), + ?LOG_DEBUG("SkipInitial: ~p, KeepFinal: ~p", [SkipInitial, KeepFinal]), {[{UUID, B} || B <- lists:seq(Start div SafeBlockSize, End div SafeBlockSize)], SkipInitial, KeepFinal}. -spec block_sequences_for_manifest(lfs_manifest()) -> - ordsets:ordset({binary(), integer()}). -block_sequences_for_manifest(?MANIFEST{props=undefined}=Manifest) -> - block_sequences_for_manifest(Manifest?MANIFEST{props=[]}); + ordsets:ordset({binary(), integer()}). block_sequences_for_manifest(?MANIFEST{uuid=UUID, content_length=ContentLength}=Manifest)-> SafeBlockSize = safe_block_size_from_manifest(Manifest), @@ -143,9 +143,7 @@ block_sequences_for_manifest(?MANIFEST{uuid=UUID, end. -spec block_sequences_for_manifest(lfs_manifest(), {integer(), integer()}) -> - {[{binary(), integer()}], integer(), integer()}. -block_sequences_for_manifest(?MANIFEST{props=undefined}=Manifest, {Start, End}) -> - block_sequences_for_manifest(Manifest?MANIFEST{props=[]}, {Start, End}); + {[{binary(), integer()}], integer(), integer()}. block_sequences_for_manifest(?MANIFEST{uuid=UUID}=Manifest, {Start, End})-> SafeBlockSize = safe_block_size_from_manifest(Manifest), @@ -171,8 +169,8 @@ block_sort_fun(SafeBlockSize) -> block_sequences_for_part_manifests_skip(SafeBlockSize, [PM | Rest], StartOffset, EndOffset) -> - _ = lager:debug("StartOffset: ~p, EndOffset: ~p, PartLength: ~p~n", - [StartOffset, EndOffset, PM?PART_MANIFEST.content_length]), + ?LOG_DEBUG("StartOffset: ~p, EndOffset: ~p, PartLength: ~p", + [StartOffset, EndOffset, PM?PART_MANIFEST.content_length]), case PM?PART_MANIFEST.content_length of %% Skipped PartLength when PartLength =< StartOffset -> @@ -195,8 +193,8 @@ block_sequences_for_part_manifests_skip(SafeBlockSize, [PM | Rest], block_sequences_for_part_manifests_keep(SafeBlockSize, SkipInitial, [PM | Rest], EndOffset, ListOfBlocks) -> - _ = lager:debug("EndOffset: EndOffset: ~p, PartLength: ~p~n", - [EndOffset, PM?PART_MANIFEST.content_length]), + ?LOG_DEBUG("EndOffset: EndOffset: ~p, PartLength: ~p", + [EndOffset, PM?PART_MANIFEST.content_length]), case PM?PART_MANIFEST.content_length of %% More blocks needed PartLength when PartLength =< EndOffset -> @@ -268,35 +266,31 @@ get_fsm_buffer_size_factor() -> end. %% @doc Initialize a new file manifest --spec new_manifest(binary(), - binary(), - binary(), - non_neg_integer(), - binary(), - term(), - term(), - pos_integer(), - acl() | no_acl_yet, - proplists:proplist(), - cluster_id(), - bag_id()) -> lfs_manifest(). -new_manifest(Bucket, FileName, UUID, ContentLength, ContentType, ContentMd5, - MetaData, BlockSize, Acl, Props, ClusterID, BagId) -> +-spec new_manifest(binary(), binary(), binary(), cs_uuid(), + non_neg_integer(), binary(), term(), + orddict:orddict(), pos_integer(), acl() | no_acl_yet, + proplists:proplist(), cluster_id(), bag_id()) -> lfs_manifest(). +new_manifest(Bucket, FileName, Vsn, UUID, + ContentLength, ContentType, ContentMd5, + MetaData, BlockSize, Acl, + Props, ClusterID, BagId) -> Blocks = ordsets:from_list(initial_blocks(ContentLength, BlockSize)), - Manifest = ?MANIFEST{bkey={Bucket, FileName}, - uuid=UUID, - state=writing, - content_length=ContentLength, - content_type=ContentType, - content_md5=ContentMd5, - block_size=BlockSize, - write_blocks_remaining=Blocks, - metadata=MetaData, - acl=Acl, - props=Props, - cluster_id=ClusterID}, + Manifest = ?MANIFEST{bkey = {Bucket, FileName}, + vsn = Vsn, + uuid = UUID, + state = writing, + content_length = ContentLength, + content_type = ContentType, + content_md5 = ContentMd5, + block_size = BlockSize, + write_blocks_remaining = Blocks, + metadata = MetaData, + acl = Acl, + props = Props, + cluster_id = ClusterID}, set_bag_id(BagId, Manifest). + -spec set_bag_id(bag_id(), lfs_manifest()) -> lfs_manifest(). set_bag_id(BagId, Manifest) -> riak_cs_mb_helper:set_bag_id_to_manifest(BagId, Manifest). @@ -312,9 +306,9 @@ remove_write_block(Manifest, Chunk) -> _ -> writing end, - Manifest?MANIFEST{write_blocks_remaining=Updated, - state=ManiState, - last_block_written_time=os:timestamp()}. + Manifest?MANIFEST{write_blocks_remaining = Updated, + state = ManiState, + last_block_written_time = os:system_time(millisecond)}. %% @doc Remove a chunk from the `delete_blocks_remaining' %% field of `Manifest' @@ -327,6 +321,6 @@ remove_delete_block(Manifest, Chunk) -> _ -> scheduled_delete end, - Manifest?MANIFEST{delete_blocks_remaining=Updated, - state=ManiState, - last_block_deleted_time=os:timestamp()}. + Manifest?MANIFEST{delete_blocks_remaining = Updated, + state = ManiState, + last_block_deleted_time = os:system_time(millisecond)}. diff --git a/apps/riak_cs/src/riak_cs_list_objects.erl b/apps/riak_cs/src/riak_cs_list_objects.erl new file mode 100644 index 000000000..ab0a605ab --- /dev/null +++ b/apps/riak_cs/src/riak_cs_list_objects.erl @@ -0,0 +1,167 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc + +-module(riak_cs_list_objects). + +-include("riak_cs.hrl"). +-include("riak_cs_web.hrl"). + +%% API +-export([new_request/2, + new_request/4, + new_response/5, + manifest_to_keycontent/2]). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%% Request +%%-------------------------------------------------------------------- + +-spec new_request(list_objects_req_type(), binary()) -> list_object_request(). +new_request(Type, Name) -> + new_request(Type, Name, 1000, []). + +-spec new_request(list_objects_req_type(), binary(), pos_integer(), list()) -> list_object_request(). +new_request(Type, Name, MaxKeys, Options) -> + process_options(#list_objects_request{req_type = Type, + name = Name, + max_keys = MaxKeys}, + Options). + +%% @private +-spec process_options(list_object_request(), list()) -> + list_object_request(). +process_options(Request, Options) -> + lists:foldl(fun process_options_helper/2, + Request, + Options). + +process_options_helper({prefix, Val}, Req) -> + Req#list_objects_request{prefix = Val}; +process_options_helper({delimiter, Val}, Req) -> + Req#list_objects_request{delimiter = Val}; +process_options_helper({marker, Val}, Req) -> + Req#list_objects_request{marker = Val}. + +%% Response +%%-------------------------------------------------------------------- + +-spec new_response(list_object_request(), + IsTruncated :: boolean(), + NextMarker :: next_marker(), + CommonPrefixes :: list(list_objects_common_prefixes()), + ObjectContents :: list(list_objects_key_content())) -> + list_objects_response() | list_object_versions_response(). +new_response(?LOREQ{req_type = objects, + name = Name, + max_keys = MaxKeys, + prefix = Prefix, + delimiter = Delimiter, + marker = Marker}, + IsTruncated, NextMarker, CommonPrefixes, ObjectContents) -> + ?LORESP{name = Name, + max_keys = MaxKeys, + prefix = Prefix, + delimiter = Delimiter, + marker = Marker, + next_marker = NextMarker, + is_truncated = IsTruncated, + contents = ObjectContents, + common_prefixes = CommonPrefixes}; + +new_response(?LOREQ{req_type = versions, + name = Name, + max_keys = MaxKeys, + prefix = Prefix, + delimiter = Delimiter, + marker = Marker}, + IsTruncated, NextMarker, CommonPrefixes, ObjectContents) -> + {KeyMarker, VersionIdMarker} = safe_decompose_key(Marker), + {NextKeyMarker, NextVersionIdMarker} = safe_decompose_key(NextMarker), + ?LOVRESP{name = Name, + max_keys = MaxKeys, + prefix = Prefix, + delimiter = Delimiter, + key_marker = KeyMarker, + version_id_marker = VersionIdMarker, + next_key_marker = NextKeyMarker, + next_version_id_marker = NextVersionIdMarker, + is_truncated = IsTruncated, + contents = ObjectContents, + common_prefixes = CommonPrefixes}. + +safe_decompose_key(undefined) -> {undefined, undefined}; +safe_decompose_key(K) -> rcs_common_manifest:decompose_versioned_key(K). + +%% Rest +%%-------------------------------------------------------------------- + +-spec manifest_to_keycontent(list_objects_req_type(), lfs_manifest()) -> + list_objects_key_content() | list_object_versions_key_content(). +manifest_to_keycontent(ReqType, ?MANIFEST{bkey = {_Bucket, Key}, + write_start_time = Created, + content_md5 = ContentMd5, + content_length = ContentLength, + vsn = Vsn, + acl = ACL}) -> + + LastModified = rts:iso8601(Created), + + %% Etag + ETagString = riak_cs_utils:etag_from_binary(ContentMd5), + Etag = list_to_binary(ETagString), + + Size = ContentLength, + Owner = acl_to_owner(ACL), + %% hardcoded since we don't support reduced redundancy or glacier + StorageClass = <<"STANDARD">>, + + case ReqType of + versions -> + ?LOVKC{key = Key, + last_modified = LastModified, + etag = Etag, + is_latest = true, + version_id = Vsn, + size = ContentLength, + owner = Owner, + storage_class = StorageClass}; + objects -> + ?LOKC{key = Key, + last_modified = LastModified, + etag = Etag, + size = Size, + owner = Owner, + storage_class = StorageClass} + end. + +%% ==================================================================== +%% Internal functions +%% ==================================================================== + +acl_to_owner(?ACL{owner = #{display_name := DisplayName, + canonical_id := CanonicalId}}) -> + #list_objects_owner{id = CanonicalId, + display_name = DisplayName}. diff --git a/src/riak_cs_list_objects_fsm_v2.erl b/apps/riak_cs/src/riak_cs_list_objects_fsm_v2.erl similarity index 79% rename from src/riak_cs_list_objects_fsm_v2.erl rename to apps/riak_cs/src/riak_cs_list_objects_fsm_v2.erl index 7e05ecea3..de67a4620 100644 --- a/src/riak_cs_list_objects_fsm_v2.erl +++ b/apps/riak_cs/src/riak_cs_list_objects_fsm_v2.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -26,7 +27,8 @@ -behaviour(gen_fsm). -include("riak_cs.hrl"). --include("list_objects.hrl"). +-include("riak_cs_web.hrl"). +-include_lib("kernel/include/logger.hrl"). %%%=================================================================== %%% Exports @@ -34,6 +36,7 @@ -ifdef(TEST). -compile(export_all). +-compile(nowarn_export_all). -endif. %% API @@ -61,12 +64,12 @@ temp_fold_objects_request :: undefined | {Request :: {StartKey :: binary(), EndKey :: binary()}, - StartTime :: erlang:timestamp()}, + StartTime :: non_neg_integer()}, fold_objects_requests=[] :: [{Request :: {StartKey :: binary(), EndKey :: binary()}, NumKeysReturned :: non_neg_integer(), - Timing :: {StartTime :: erlang:timestamp(), - EndTime :: erlang:timestamp()}}]}). + Timing :: {StartTime :: non_neg_integer(), + EndTime :: non_neg_integer()}}]}). -type profiling() :: #profiling{}. @@ -84,7 +87,7 @@ object_list_ranges=[] :: object_list_ranges(), profiling=#profiling{} :: profiling(), response :: undefined | - {ok, list_object_response()} | + {ok, list_objects_response() | list_object_versions_response()} | {error, term()}, common_prefixes=ordsets:new() :: list_objects_common_prefixes()}). @@ -98,7 +101,7 @@ -type continuation() :: binary() | 'undefined'. -type list_objects_event() :: {ReqID :: reference(), {done, continuation()}} | - {ReqID :: reference(), {objects, list()}} | + {ReqID :: reference(), {ok, list()}} | {ReqID :: reference(), {error, term()}}. %% `Start' and `End' are inclusive @@ -127,7 +130,6 @@ start_link(RcPid, ListKeysRequest, FoldObjectsBatchSize) -> -spec init(list()) -> {ok, prepare, state(), 0}. init([RcPid, Request, FoldObjectsBatchSize]) -> - State = #state{riak_client=RcPid, fold_objects_batch_size=FoldObjectsBatchSize, req=Request}, @@ -144,12 +146,9 @@ prepare(timeout, State=#state{riak_client=RcPid}) -> end. -spec waiting_object_list(list_objects_event(), state()) -> fsm_state_return(). -waiting_object_list({ReqId, {ok, _}=Objs}, State) -> - waiting_object_list({ReqId, [Objs, undefined]}, State); -waiting_object_list({ReqId, [{ok, ObjectList} | _]}, - State=#state{object_list_req_id=ReqId, - object_buffer=ObjectBuffer}) -> - NewStateData = State#state{object_buffer=ObjectBuffer ++ ObjectList}, +waiting_object_list({ReqId, {ok, ObjectList}}, State = #state{object_list_req_id = ReqId, + object_buffer = ObjectBuffer}) -> + NewStateData = State#state{object_buffer = ObjectBuffer ++ ObjectList}, {next_state, waiting_object_list, NewStateData}; waiting_object_list({ReqId, {done, _Continuation}}, State=#state{object_list_req_id=ReqId}) -> handle_done(State); @@ -172,7 +171,7 @@ handle_sync_event(get_internal_state, _From, StateName, State) -> Reply = {StateName, State}, {reply, Reply, StateName, State}; handle_sync_event(Event, _From, StateName, State) -> - _ = lager:debug("got unknown event ~p in state ~p", [Event, StateName]), + logger:warning("got unknown event ~p in state ~p", [Event, StateName]), Reply = ok, {reply, Reply, StateName, State}. @@ -182,12 +181,11 @@ handle_sync_event(Event, _From, StateName, State) -> handle_info(Info, waiting_object_list, State) -> waiting_object_list(Info, State); handle_info(Info, StateName, _State) -> - _ = lager:debug("Received unknown info message ~p" - "in state ~p", [Info, StateName]), + logger:warning("Received unknown info message ~p in state ~p", [Info, StateName]), ok. terminate(normal, _StateName, State) -> - lager:debug(format_profiling_from_state(State)); + ?LOG_DEBUG(format_profiling_from_state(State)); terminate(_Reason, _StateName, _State) -> ok. @@ -228,13 +226,8 @@ handle_done(State=#state{object_buffer=ObjectBuffer, common_prefixes=NewPrefixes, reached_end_of_keyspace=ReachedEnd, object_buffer=[]}, - _ = lager:debug("Ranges: ~p", [NewStateData#state.object_list_ranges]), respond(NewStateData, NewManis, NewPrefixes). --spec reached_end_of_keyspace(non_neg_integer(), - undefined | pos_integer(), - list(lfs_manifest()), - undefined | binary()) -> boolean(). reached_end_of_keyspace(BufferLength, NumKeysRequested, _, _) when BufferLength < NumKeysRequested -> true; @@ -258,15 +251,11 @@ handle_error(Error, #state{profiling=Profiling} = State) -> _ = riak_cs_stats:update_error_with_start([riakc, fold_manifest_objs], StartTime), try_reply(Error, State). --spec update_profiling_and_last_request(state(), list(), integer()) -> - state(). update_profiling_and_last_request(State, ObjectBuffer, ObjectBufferLength) -> - State2 = update_profiling_state_with_end(State, os:timestamp(), + State2 = update_profiling_state_with_end(State, os:system_time(millisecond), ObjectBufferLength), update_last_request_state(State2, ObjectBuffer). --spec respond(state(), list(), list_objects_common_prefixes()) -> - fsm_state_return(). respond(StateData=#state{req=Request=?LOREQ{max_keys=UserMaxKeys, delimiter=Delimiter}}, Manifests, Prefixes) -> @@ -281,10 +270,8 @@ respond(StateData=#state{req=Request=?LOREQ{max_keys=UserMaxKeys, {NewManis, NewPrefixes} = riak_cs_list_objects_utils:untagged_manifest_and_prefix(SlicedTaggedItems), Response = - response_from_manifests_and_common_prefixes(Request, - Truncated, - NextMarker, - {NewManis, NewPrefixes}), + response_from_manifests_and_common_prefixes( + Request, Truncated, NextMarker, {NewManis, NewPrefixes}), try_reply({ok, Response}, StateData); false -> RcPid = StateData#state.riak_client, @@ -297,7 +284,6 @@ respond(StateData=#state{req=Request=?LOREQ{max_keys=UserMaxKeys, end end. --spec truncated(non_neg_integer(), {list(), ordsets:ordset(term())}) -> boolean(). truncated(NumKeysRequested, ObjectsAndPrefixes) -> NumKeysRequested < riak_cs_list_objects_utils:manifests_and_prefix_length(ObjectsAndPrefixes) andalso %% this is because (strangely) S3 returns `false' for @@ -305,7 +291,6 @@ truncated(NumKeysRequested, ObjectsAndPrefixes) -> %% The `Ceph' tests were nice to find this. NumKeysRequested =/= 0. --spec enough_results(state()) -> boolean(). %% @doc Return a `boolean' determining whether enough results have been %% returned from the fold objects queries to return to the user. In order %% to tell if the result-set is truncated, we either need one more result @@ -319,9 +304,6 @@ enough_results(#state{req=?LOREQ{max_keys=UserMaxKeys}, > UserMaxKeys orelse EndOfKeyspace. --spec next_marker(undefined | binary(), - riak_cs_list_objects_utils:tagged_item_list()) -> - next_marker(). next_marker(undefined, _List) -> undefined; next_marker(_Delimiter, []) -> @@ -329,8 +311,6 @@ next_marker(_Delimiter, []) -> next_marker(_Delimiter, List) -> next_marker_from_element(lists:last(List)). --spec next_marker_from_element(riak_cs_list_objects_utils:tagged_item()) -> - next_marker(). next_marker_from_element({prefix, Name}) -> Name; next_marker_from_element({manifest, ?MANIFEST{bkey={_Bucket, Key}}}) -> @@ -338,20 +318,18 @@ next_marker_from_element({manifest, ?MANIFEST{bkey={_Bucket, Key}}}) -> next_marker_from_element({manifest, {Key, ?MANIFEST{}}}) -> Key. -response_from_manifests_and_common_prefixes(Request, +response_from_manifests_and_common_prefixes(?LOREQ{req_type = ReqType} = Request, Truncated, NextMarker, {Manifests, CommonPrefixes}) -> - KeyContent = lists:map(fun riak_cs_list_objects:manifest_to_keycontent/1, + KeyContent = lists:map(fun(M) -> riak_cs_list_objects:manifest_to_keycontent(ReqType, M) end, Manifests), riak_cs_list_objects:new_response(Request, Truncated, NextMarker, CommonPrefixes, KeyContent). --spec make_2i_request(riak_client(), state()) -> - {state(), {ok, reference()} | {error, term()}}. -make_2i_request(RcPid, State=#state{req=?LOREQ{name=BucketName,prefix=Prefix}, - fold_objects_batch_size=BatchSize}) -> +make_2i_request(RcPid, State=#state{req = ?LOREQ{name = BucketName, prefix = Prefix}, + fold_objects_batch_size = BatchSize}) -> ManifestBucket = riak_cs_utils:to_bucket_name(objects, BucketName), StartKey = make_start_key(State), EndKey = riak_cs_utils:big_end_key(Prefix), @@ -361,25 +339,22 @@ make_2i_request(RcPid, State=#state{req=?LOREQ{name=BucketName,prefix=Prefix}, NewStateData2 = update_profiling_state_with_start(NewStateData, StartKey, EndKey, - os:timestamp()), + os:system_time(millisecond)), Opts = [{max_results, BatchSize}, {start_key, StartKey}, {end_key, EndKey}, - {timeout, riak_cs_list_objects_utils:fold_objects_timeout()}], + {timeout, riak_cs_config:fold_objects_timeout()}], {ok, ManifestPbc} = riak_cs_riak_client:manifest_pbc(RcPid), FoldResult = riakc_pb_socket:cs_bucket_fold(ManifestPbc, ManifestBucket, Opts), {NewStateData2, FoldResult}. --spec last_result_is_common_prefix(state()) -> boolean(). last_result_is_common_prefix(#state{object_list_ranges=Ranges, req=Request}) -> Key = element(2, lists:last(Ranges)), key_is_common_prefix(Key, Request). --spec key_is_common_prefix(binary(), list_object_request()) -> - boolean(). key_is_common_prefix(_Key, ?LOREQ{delimiter=undefined}) -> false; key_is_common_prefix(Key, ?LOREQ{prefix=Prefix, @@ -391,7 +366,6 @@ key_is_common_prefix(Key, ?LOREQ{prefix=Prefix, handle_prefix(Key, Prefix, Delimiter) end. --spec handle_undefined_prefix(binary(), binary()) -> boolean(). handle_undefined_prefix(Key, Delimiter) -> case binary:match(Key, [Delimiter]) of nomatch -> @@ -400,7 +374,6 @@ handle_undefined_prefix(Key, Delimiter) -> true end. --spec handle_prefix(binary(), binary(), binary()) -> boolean(). handle_prefix(Key, Prefix, Delimiter) -> PrefixLen = byte_size(Prefix), case Key of @@ -415,10 +388,6 @@ handle_prefix(Key, Prefix, Delimiter) -> false end. --spec common_prefix_from_key(Key :: binary(), - Prefix :: binary(), - Delimiter :: binary()) -> - binary(). %% @doc Extract common prefix from `Key'. `Key' must contain `Delimiter', so %% you must first check with `key_is_common_prefix'. common_prefix_from_key(Key, Prefix, Delimiter) -> @@ -432,7 +401,6 @@ common_prefix_from_key(Key, Prefix, Delimiter) -> <> end. --spec make_start_key(state()) -> binary(). make_start_key(#state{object_list_ranges=[], req=Request}) -> make_start_key_from_marker_and_prefix(Request); make_start_key(State=#state{object_list_ranges=PrevRanges, @@ -453,7 +421,6 @@ make_start_key(State=#state{object_list_ranges=PrevRanges, Key end. --spec make_start_key_from_marker_and_prefix(list_object_request()) -> binary(). make_start_key_from_marker_and_prefix(?LOREQ{marker=undefined, prefix=undefined}) -> <<0:8/integer>>; @@ -476,14 +443,11 @@ make_start_key_from_marker_and_prefix(?LOREQ{marker=Marker, Marker end. --spec map_active_manifests([orddict:orddict()]) -> list(lfs_manifest()). map_active_manifests(Manifests) -> - ActiveTuples = [riak_cs_manifest_utils:active_manifest(M) || + ActiveTuples = [rcs_common_manifest_utils:active_manifest(M) || M <- Manifests], [A || {ok, A} <- ActiveTuples]. --spec exclude_key_from_state(state(), list(riakc_obj:riakc_obj())) -> - list(riakc_obj:riakc_obj()). exclude_key_from_state(_State, []) -> []; exclude_key_from_state(#state{object_list_ranges=[], @@ -492,14 +456,11 @@ exclude_key_from_state(#state{object_list_ranges=[], exclude_key_from_state(#state{last_request_start_key=StartKey}, Objects) -> exclude_key(StartKey, Objects). --spec exclude_marker(list_object_request(), list()) -> list(). exclude_marker(?LOREQ{marker=undefined}, Objects) -> Objects; exclude_marker(?LOREQ{marker=Marker}, Objects) -> exclude_key(Marker, Objects). --spec exclude_key(binary(), list(riakc_obj:riakc_obj())) -> - list(riakc_obj:riakc_obj()). exclude_key(Key, [H | T]=Objects) -> case riakc_obj:key(H) == Key of true -> @@ -508,7 +469,6 @@ exclude_key(Key, [H | T]=Objects) -> Objects end. --spec skip_past_prefix_and_delimiter(binary()) -> binary(). skip_past_prefix_and_delimiter(<<>>) -> <<0:8/integer>>; skip_past_prefix_and_delimiter(Key) -> @@ -523,15 +483,10 @@ next_byte(<>=Byte) when Integer == 255 -> next_byte(<>) -> <<(Integer+1):8/integer>>. --spec try_reply({ok, list_object_response()} | {error, term()}, - state()) -> - fsm_state_return(). try_reply(Response, State) -> NewStateData = State#state{response=Response}, reply_or_wait(Response, NewStateData). --spec reply_or_wait({ok, list_object_response()} | {error, term()}, state()) -> - fsm_state_return(). reply_or_wait(_Response, State=#state{reply_ref=undefined}) -> {next_state, waiting_req, State}; reply_or_wait(Response, State=#state{reply_ref=Ref}) -> @@ -539,8 +494,6 @@ reply_or_wait(Response, State=#state{reply_ref=Ref}) -> Reason = make_reason(Response), {stop, Reason, State}. --spec make_reason({ok, list_object_response()} | {error, term()}) -> - normal | term(). make_reason({ok, _Response}) -> normal; make_reason({error, Reason}) -> @@ -562,10 +515,6 @@ update_last_request_state(State=#state{last_request_start_key=StartKey, %% Profiling helper functions %%-------------------------------------------------------------------- --spec update_profiling_state_with_start(state(), StartKey :: binary(), - EndKey :: binary(), - StartTime :: erlang:timestamp()) -> - state(). update_profiling_state_with_start(State=#state{profiling=Profiling}, StartKey, EndKey, StartTime) -> _ = riak_cs_stats:inflow([riakc, fold_manifest_objs]), @@ -574,9 +523,6 @@ update_profiling_state_with_start(State=#state{profiling=Profiling}, NewProfiling = Profiling#profiling{temp_fold_objects_request=TempData}, State#state{profiling=NewProfiling}. --spec update_profiling_state_with_end(state(), EndTime :: erlang:timestamp(), - NumKeysReturned :: non_neg_integer()) -> - state(). update_profiling_state_with_end(State=#state{profiling=Profiling}, EndTime, NumKeysReturned) -> {KeyRange, StartTime} = Profiling#profiling.temp_fold_objects_request, @@ -588,42 +534,35 @@ update_profiling_state_with_end(State=#state{profiling=Profiling}, [NewRequest | OldRequests]}, State#state{profiling=NewProfiling}. --spec extract_timings(list()) -> [{Millis :: number(), - NumResults :: non_neg_integer()}]. extract_timings(Requests) -> [extract_timing(R) || R <- Requests]. %% TODO: time to make legit types out of these --spec extract_timing({term(), non_neg_integer(), - {erlang:timestamp(), erlang:timestamp()}}) -> - {number(), non_neg_integer()}. extract_timing({_Range, NumKeysReturned, {StartTime, EndTime}}) -> - MillisecondDiff = riak_cs_utils:timestamp_to_milliseconds(EndTime) - - riak_cs_utils:timestamp_to_milliseconds(StartTime), + MillisecondDiff = EndTime - StartTime, {MillisecondDiff, NumKeysReturned}. --spec format_profiling_from_state(state()) -> string(). -format_profiling_from_state(#state{req=Request, - response={ok, Response}, - profiling=Profiling}) -> +format_profiling_from_state(#state{req = Request, + response = {ok, Response}, + profiling = Profiling}) -> format_profiling(Request, Response, Profiling, self()). --spec format_profiling(list_object_request(), - list_object_response(), - profiling(), - pid()) -> string(). -format_profiling(?LOREQ{max_keys=MaxKeys}, - ?LORESP{contents=Contents, common_prefixes=CommonPrefixes}, - #profiling{fold_objects_requests=Requests}, +format_profiling(Request, ?LORESP{contents = Contents, common_prefixes = CommonPrefixes}, + Profiling, Pid) -> + format_profiling(Request, Contents, CommonPrefixes, Profiling, Pid); +format_profiling(Request, ?LOVRESP{contents = Contents, common_prefixes = CommonPrefixes}, + Profiling, Pid) -> + format_profiling(Request, Contents, CommonPrefixes, Profiling, Pid). + +format_profiling(?LOREQ{max_keys = MaxKeys}, + Contents, CommonPrefixes, + #profiling{fold_objects_requests = Requests}, Pid) -> string:join([io_lib:format("~p: User requested ~p keys", [Pid, MaxKeys]), - io_lib:format("~p: We returned ~p objects", [Pid, length(Contents)]), - io_lib:format("~p: We returned ~p common prefixes", [Pid, ordsets:size(CommonPrefixes)]), - io_lib:format("~p: With fold objects timings: {Millis, NumObjects}: ~p", %% We reverse the Requests in here because they %% were cons'd as they happened. diff --git a/src/riak_cs_list_objects_utils.erl b/apps/riak_cs/src/riak_cs_list_objects_utils.erl similarity index 76% rename from src/riak_cs_list_objects_utils.erl rename to apps/riak_cs/src/riak_cs_list_objects_utils.erl index 3c7a96802..f4397a73e 100644 --- a/src/riak_cs_list_objects_utils.erl +++ b/apps/riak_cs/src/riak_cs_list_objects_utils.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -23,7 +24,6 @@ -module(riak_cs_list_objects_utils). -include("riak_cs.hrl"). --include("list_objects.hrl"). %%%=================================================================== %%% Types @@ -46,8 +46,7 @@ %%%=================================================================== %% API --export([start_link/5, - get_object_list/1, +-export([get_object_list/1, get_internal_state/1]). %% Shared Helpers @@ -58,36 +57,13 @@ filter_prefix_keys/2, extract_group/2]). -%% Observability / Configuration --export([get_key_list_multiplier/0, - set_key_list_multiplier/1, - fold_objects_for_list_keys/0, - fold_objects_timeout/0]). - %%%=================================================================== %%% API %%%=================================================================== --spec start_link(riak_client(), pid(), list_object_request(), term(), - UseCache :: boolean()) -> - {ok, pid()} | {error, term()}. -%% @doc An abstraction between the old and new list-keys mechanism. Uses the -%% old mechanism if `fold_objects_for_list_keys' is false, otherwise uses -%% the new one. After getting a pid back, the API is the same, so users don't -%% need to differentiate. -start_link(RcPid, CallerPid, ListKeysRequest, CacheKey, UseCache) -> - case fold_objects_for_list_keys() of - true -> - riak_cs_list_objects_fsm_v2:start_link(RcPid, ListKeysRequest); - false -> - riak_cs_list_objects_fsm:start_link(RcPid, CallerPid, - ListKeysRequest, CacheKey, - UseCache) - end. - -spec get_object_list(pid()) -> - {ok, list_object_response()} | + {ok, list_objects_response() | list_object_versions_response()} | {error, term()}. get_object_list(FSMPid) -> gen_fsm:sync_send_all_state_event(FSMPid, get_object_list, infinity). @@ -208,29 +184,3 @@ update_keys_and_prefixes({ManifestList, Prefixes}, _, Prefix, PrefixLen, Group) -> NewPrefix = << Prefix:PrefixLen/binary, Group/binary >>, {ManifestList, ordsets:add_element(NewPrefix, Prefixes)}. - - - -%%%=================================================================== -%%% Observability / Configuration -%%%=================================================================== - --spec get_key_list_multiplier() -> float(). -get_key_list_multiplier() -> - riak_cs_config:get_env(riak_cs, key_list_multiplier, - ?KEY_LIST_MULTIPLIER). - --spec set_key_list_multiplier(float()) -> 'ok'. -set_key_list_multiplier(Multiplier) -> - application:set_env(riak_cs, key_list_multiplier, - Multiplier). - - --spec fold_objects_for_list_keys() -> boolean(). -fold_objects_for_list_keys() -> - riak_cs_config:get_env(riak_cs, fold_objects_for_list_keys, - ?FOLD_OBJECTS_FOR_LIST_KEYS). - --spec fold_objects_timeout() -> non_neg_integer(). -fold_objects_timeout() -> - riak_cs_config:fold_objects_timeout(). diff --git a/src/riak_cs_manifest.erl b/apps/riak_cs/src/riak_cs_manifest.erl similarity index 71% rename from src/riak_cs_manifest.erl rename to apps/riak_cs/src/riak_cs_manifest.erl index 72e196624..08f15d598 100644 --- a/src/riak_cs_manifest.erl +++ b/apps/riak_cs/src/riak_cs_manifest.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -20,8 +21,8 @@ -module(riak_cs_manifest). --export([fetch/3, - get_manifests/3, +-export([fetch/4, + get_manifests/4, manifests_from_riak_object/1, etag/1, etag_no_quotes/1, @@ -29,29 +30,30 @@ -include("riak_cs.hrl"). --spec fetch(pid(), binary(), binary()) -> {ok, lfs_manifest()} | {error, term()}. -fetch(RcPid, Bucket, Key) -> - case riak_cs_manifest:get_manifests(RcPid, Bucket, Key) of +-spec fetch(pid(), binary(), binary(), binary()) -> {ok, lfs_manifest()} | {error, term()}. +fetch(RcPid, Bucket, Key, ObjVsn) -> + case get_manifests(RcPid, Bucket, Key, ObjVsn) of {ok, _, Manifests} -> - riak_cs_manifest_utils:active_manifest(orddict:from_list(Manifests)); + rcs_common_manifest_utils:active_manifest(orddict:from_list(Manifests)); Error -> Error end. --spec get_manifests(riak_client(), binary(), binary()) -> - {ok, term(), term()} | {error, term()}. -get_manifests(RcPid, Bucket, Key) -> - case get_manifests_raw(RcPid, Bucket, Key) of +-spec get_manifests(riak_client(), binary(), binary(), binary()) -> + {ok, riakc_obj:riakc_obj(), wrapped_manifest()} | {error, term()}. +get_manifests(RcPid, Bucket, Key, ObjVsn) -> + case get_manifests_raw(RcPid, Bucket, Key, ObjVsn) of {ok, Object} -> Manifests = manifests_from_riak_object(Object), maybe_warn_bloated_manifests(Bucket, Key, Object, Manifests), - _ = gc_deleted_while_writing_manifests(Object, Manifests, Bucket, Key, RcPid), + _ = gc_deleted_while_writing_manifests(Object, Manifests, Bucket, RcPid), {ok, Object, Manifests}; - {error, _Reason}=Error -> + {error, _Reason} = Error -> Error end. --spec manifests_from_riak_object(riakc_obj:riakc_obj()) -> orddict:orddict(). + +-spec manifests_from_riak_object(riakc_obj:riakc_obj()) -> wrapped_manifest(). manifests_from_riak_object(RiakObject) -> %% For example, riak_cs_manifest_fsm:get_and_update/4 may wish to %% update the #riakc_obj without a roundtrip to Riak first. So we @@ -66,33 +68,33 @@ manifests_from_riak_object(RiakObject) -> riakc_obj:get_contents(RiakObject) end, DecodedSiblings = [binary_to_term(V) || - {_, V}=Content <- Contents, + {_, V} = Content <- Contents, not riak_cs_utils:has_tombstone(Content)], %% Upgrade the manifests to be the latest erlang %% record version - Upgraded = riak_cs_manifest_utils:upgrade_wrapped_manifests(DecodedSiblings), + Upgraded = rcs_common_manifest_utils:upgrade_wrapped_manifests(DecodedSiblings), %% resolve the siblings - Resolved = riak_cs_manifest_resolution:resolve(Upgraded), + Resolved = rcs_common_manifest_resolution:resolve(Upgraded), %% prune old scheduled_delete manifests riak_cs_manifest_utils:prune(Resolved). -spec etag(lfs_manifest()) -> string(). -etag(?MANIFEST{content_md5={MD5, Suffix}}) -> +etag(?MANIFEST{content_md5 = {MD5, Suffix}}) -> riak_cs_utils:etag_from_binary(MD5, Suffix); -etag(?MANIFEST{content_md5=MD5}) -> +etag(?MANIFEST{content_md5 = MD5}) -> riak_cs_utils:etag_from_binary(MD5). -spec etag_no_quotes(lfs_manifest()) -> string(). -etag_no_quotes(?MANIFEST{content_md5=ContentMD5}) -> +etag_no_quotes(?MANIFEST{content_md5 = ContentMD5}) -> riak_cs_utils:etag_from_binary_no_quotes(ContentMD5). --spec object_acl(notfound|lfs_manifest()) -> undefined|acl(). +-spec object_acl(notfound | lfs_manifest()) -> undefined | acl(). object_acl(notfound) -> undefined; -object_acl(?MANIFEST{acl=Acl}) -> +object_acl(?MANIFEST{acl = Acl}) -> Acl. @@ -100,16 +102,16 @@ object_acl(?MANIFEST{acl=Acl}) -> %% Internal functions %% =================================================================== -%% Retrieve the riak object at a bucket/key --spec get_manifests_raw(riak_client(), binary(), binary()) -> - {ok, riakc_obj:riakc_obj()} | {error, term()}. -get_manifests_raw(RcPid, Bucket, Key) -> +%% Retrieve the riak object at a bucket/key/version +get_manifests_raw(RcPid, Bucket, Key, Vsn) -> ManifestBucket = riak_cs_utils:to_bucket_name(objects, Bucket), ok = riak_cs_riak_client:set_bucket_name(RcPid, Bucket), {ok, ManifestPbc} = riak_cs_riak_client:manifest_pbc(RcPid), Timeout = riak_cs_config:get_manifest_timeout(), - case riakc_pb_socket:get(ManifestPbc, ManifestBucket, Key, Timeout) of - {ok, _} = Result -> Result; + case riakc_pb_socket:get(ManifestPbc, ManifestBucket, + rcs_common_manifest:make_versioned_key(Key, Vsn), Timeout) of + {ok, _} = Result -> + Result; {error, disconnected} -> riak_cs_pbc:check_connection_status(ManifestPbc, get_manifests_raw), {error, disconnected}; @@ -117,9 +119,9 @@ get_manifests_raw(RcPid, Bucket, Key) -> Error end. -gc_deleted_while_writing_manifests(Object, Manifests, Bucket, Key, RcPid) -> - UUIDs = riak_cs_manifest_utils:deleted_while_writing(Manifests), - riak_cs_gc:gc_specific_manifests(UUIDs, Object, Bucket, Key, RcPid). +gc_deleted_while_writing_manifests(Object, Manifests, Bucket, RcPid) -> + UUIDs = rcs_common_manifest_utils:deleted_while_writing(Manifests), + riak_cs_gc:gc_specific_manifests(UUIDs, Object, Bucket, RcPid). -spec maybe_warn_bloated_manifests(binary(), binary(), riakc_obj:riakc_obj(), [term()]) -> ok. maybe_warn_bloated_manifests(Bucket, Key, Object, Manifests) -> @@ -150,6 +152,6 @@ maybe_warn_bloated_manifests(Bucket, Key, Actual, Threshold, Message, Unit) -> case Threshold of disabled -> ok; _ when Actual < Threshold -> ok; - _ -> _ = lager:warning("~s (~p ~s) for bucket=~p key=~p", - [Message, Actual, Unit, Bucket, Key]) + _ -> logger:warning("~s (~p ~s) for bucket=~p key=~p", + [Message, Actual, Unit, Bucket, Key]) end. diff --git a/src/riak_cs_manifest_fsm.erl b/apps/riak_cs/src/riak_cs_manifest_fsm.erl similarity index 71% rename from src/riak_cs_manifest_fsm.erl rename to apps/riak_cs/src/riak_cs_manifest_fsm.erl index de193073f..474bcdd99 100644 --- a/src/riak_cs_manifest_fsm.erl +++ b/apps/riak_cs/src/riak_cs_manifest_fsm.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -20,20 +21,8 @@ -module(riak_cs_manifest_fsm). --include("riak_cs.hrl"). - --behaviour(gen_fsm). - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -%% Test API --export([test_link/2]). - --endif. - %% API --export([start_link/3, +-export([start_link/4, get_all_manifests/1, get_active_manifest/1, get_specific_manifest/2, @@ -46,7 +35,7 @@ update_manifests_with_confirmation/2, maybe_stop_manifest_fsm/1, stop/1]). --export([update_md_with_multipart_2i/4]). +-export([update_md_with_multipart_2i/3]). %% gen_fsm callbacks -export([init/1, @@ -66,10 +55,24 @@ terminate/3, code_change/4]). +-include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-behaviour(gen_fsm). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +%% Test API +-export([test_link/2]). + +-endif. + -define(SERVER, ?MODULE). -record(state, {bucket :: binary(), key :: binary(), + obj_vsn :: binary(), riak_object :: term(), manifests :: term(), % an orddict mapping UUID -> Manifest riak_client :: riak_client() @@ -87,8 +90,8 @@ %% %% @end %%-------------------------------------------------------------------- -start_link(Bucket, Key, RcPid) -> - gen_fsm:start_link(?MODULE, [Bucket, Key, RcPid], []). +start_link(Bucket, Key, ObjVsn, RcPid) -> + gen_fsm:start_link(?MODULE, [Bucket, Key, ObjVsn, RcPid], []). get_all_manifests(Pid) -> gen_fsm:sync_send_event(Pid, get_manifests, infinity). @@ -101,24 +104,26 @@ get_specific_manifest(Pid, UUID) -> case gen_fsm:sync_send_event(Pid, get_manifests, infinity) of {ok, Manifests} -> case orddict:fetch(UUID, Manifests) of - {ok, _}=Result -> + {ok, _} = Result -> Result; error -> {error, notfound} end; - {error, notfound}=NotFound -> + {error, notfound} = NotFound -> NotFound end. add_new_manifest(Pid, Manifest) -> - Dict = riak_cs_manifest_utils:new_dict(Manifest?MANIFEST.uuid, Manifest), + Dict = rcs_common_manifest_utils:new_dict(Manifest?MANIFEST.uuid, Manifest), gen_fsm:send_event(Pid, {add_new_dict, Dict}). +-spec update_manifests(pid(), wrapped_manifest()) -> ok. update_manifests(Pid, Manifests) -> gen_fsm:send_event(Pid, {update_manifests, Manifests}). +-spec update_manifest(pid(), lfs_manifest()) -> ok. update_manifest(Pid, Manifest) -> - Dict = riak_cs_manifest_utils:new_dict(Manifest?MANIFEST.uuid, Manifest), + Dict = rcs_common_manifest_utils:new_dict(Manifest?MANIFEST.uuid, Manifest), update_manifests(Pid, Dict). %% @doc Delete a specific manifest version from a manifest and @@ -143,7 +148,7 @@ update_manifests_with_confirmation(Pid, Manifests) -> -spec update_manifest_with_confirmation(pid(), lfs_manifest()) -> ok | {error, term()}. update_manifest_with_confirmation(Pid, Manifest) -> - Dict = riak_cs_manifest_utils:new_dict(Manifest?MANIFEST.uuid, Manifest), + Dict = rcs_common_manifest_utils:new_dict(Manifest?MANIFEST.uuid, Manifest), update_manifests_with_confirmation(Pid, Dict). -spec maybe_stop_manifest_fsm(undefined | pid()) -> ok. @@ -160,16 +165,30 @@ stop(Pid) -> %%% gen_fsm callbacks %%%=================================================================== -init([Bucket, Key, RcPid]) -> +-ifndef(TEST). +init([Bucket, Key, ObjVsn, RcPid]) -> + process_flag(trap_exit, true), + {ok, waiting_command, #state{bucket = Bucket, + key = Key, + obj_vsn = ObjVsn, + riak_client = RcPid}}. +-else. +init([Bucket, Key, ObjVsn, RcPid]) -> process_flag(trap_exit, true), - {ok, waiting_command, #state{bucket=Bucket, - key=Key, - riak_client=RcPid}}; + {ok, waiting_command, #state{bucket = Bucket, + key = Key, + obj_vsn = ObjVsn, + riak_client = RcPid}}; init([test, Bucket, Key]) -> %% creating the "mock" riakc_pb_socket %% gen_server here {ok, FakePbc} = riakc_pb_socket_fake:start_link(), - {ok, waiting_command, #state{bucket=Bucket, key=Key, riak_client=FakePbc}}. + {ok, waiting_command, #state{bucket = Bucket, + key = Key, + obj_vsn = ?LFS_DEFAULT_OBJECT_VERSION, + riak_client = FakePbc}}. +-endif. + %% This clause is for adding a new %% manifest that doesn't exist yet. @@ -177,47 +196,50 @@ init([test, Bucket, Key]) -> %% with a particular UUID, update_manifest %% should be used from then on out. waiting_command({add_new_dict, WrappedManifest}, - State=#state{riak_client=RcPid, - bucket=Bucket, - key=Key}) -> - {_, RiakObj, Manifests} = get_and_update(RcPid, WrappedManifest, Bucket, Key), - UpdState = State#state{riak_object=RiakObj, manifests=Manifests}, + State = #state{riak_client = RcPid, + bucket = Bucket, + key = Key, + obj_vsn = ObjVsn}) -> + {_, RiakObj, Manifests} = get_and_update(RcPid, WrappedManifest, Bucket, Key, ObjVsn), + UpdState = State#state{riak_object = RiakObj, manifests = Manifests}, {next_state, waiting_update_command, UpdState}. waiting_update_command({update_manifests, WrappedManifests}, - State=#state{riak_client=RcPid, - bucket=Bucket, - key=Key, - riak_object=undefined, - manifests=undefined}) -> - _Res = get_and_update(RcPid, WrappedManifests, Bucket, Key), + State = #state{riak_client = RcPid, + bucket = Bucket, + key = Key, + obj_vsn = ObjVsn, + riak_object = undefined, + manifests = undefined}) -> + _Res = get_and_update(RcPid, WrappedManifests, Bucket, Key, ObjVsn), {next_state, waiting_update_command, State}; waiting_update_command({update_manifests, WrappedManifests}, - State=#state{riak_client=RcPid, - bucket=Bucket, - key=Key, - riak_object=PreviousRiakObject, - manifests=PreviousManifests}) -> + State = #state{riak_client = RcPid, + bucket = Bucket, + riak_object = PreviousRiakObject, + manifests = PreviousManifests}) -> _ = update_from_previous_read(RcPid, PreviousRiakObject, - Bucket, Key, + Bucket, PreviousManifests, WrappedManifests), - {next_state, waiting_update_command, State#state{riak_object=undefined, manifests=undefined}}. + {next_state, waiting_update_command, State#state{riak_object = undefined, + manifests = undefined}}. waiting_command(get_manifests, _From, State) -> {Reply, NewState} = handle_get_manifests(State), {reply, Reply, waiting_update_command, NewState}; waiting_command({delete_manifest, UUID}, - _From, - State=#state{riak_client=RcPid, - bucket=Bucket, - key=Key, - riak_object=undefined, - manifests=undefined}) -> - Reply = get_and_delete(RcPid, UUID, Bucket, Key), + _From, + State = #state{riak_client = RcPid, + bucket = Bucket, + key = Key, + obj_vsn = ObjVsn, + riak_object = undefined, + manifests = undefined}) -> + Reply = get_and_delete(RcPid, UUID, Bucket, Key, ObjVsn), {reply, Reply, waiting_update_command, State}; -waiting_command({update_manifests_with_confirmation, _}=Cmd, From, State) -> +waiting_command({update_manifests_with_confirmation, _} = Cmd, From, State) -> %% Used by multipart commit: this FSM was just started a moment %% ago, and we don't need this FSM to re-do work that multipart %% commit has already done. @@ -225,54 +247,50 @@ waiting_command({update_manifests_with_confirmation, _}=Cmd, From, State) -> waiting_update_command({update_manifests_with_confirmation, WrappedManifests}, _From, - State=#state{riak_client=RcPid, - bucket=Bucket, - key=Key, - riak_object=undefined, - manifests=undefined}) -> - {Reply, _, _} = get_and_update(RcPid, WrappedManifests, Bucket, Key), + State = #state{riak_client = RcPid, + bucket = Bucket, + key = Key, + obj_vsn = ObjVsn, + riak_object = undefined, + manifests = undefined}) -> + {Reply, _, _} = get_and_update(RcPid, WrappedManifests, Bucket, Key, ObjVsn), {reply, Reply, waiting_update_command, State}; waiting_update_command({update_manifests_with_confirmation, WrappedManifests}, _From, - State=#state{riak_client=RcPid, - bucket=Bucket, - key=Key, - riak_object=PreviousRiakObject, - manifests=PreviousManifests}) -> + State = #state{riak_client = RcPid, + bucket = Bucket, + key = Key, + obj_vsn = ObjVsn, + riak_object = PreviousRiakObject, + manifests = PreviousManifests}) -> Reply = case riak_cs_config:read_before_last_manifest_write() of true -> - {R, _, _} = get_and_update(RcPid, WrappedManifests, Bucket, Key), + {R, _, _} = get_and_update(RcPid, WrappedManifests, Bucket, Key, ObjVsn), R; false -> - update_from_previous_read(RcPid, - PreviousRiakObject, - Bucket, - Key, - PreviousManifests, - WrappedManifests) + update_from_previous_read( + RcPid, PreviousRiakObject, Bucket, + PreviousManifests, WrappedManifests) end, - {reply, Reply, waiting_update_command, State#state{riak_object=undefined, - manifests=undefined}}; + {reply, Reply, waiting_update_command, State#state{riak_object = undefined, + manifests = undefined}}; waiting_update_command({gc_specific_manifest, UUID}, _From, - #state{ - riak_object = RiakObj0, - bucket = Bucket, - key = Key, - riak_client = RcPid - } = State) -> + State = #state{riak_object = RiakObj0, + bucket = Bucket, + key = Key, + obj_vsn = ObjVsn, + riak_client = RcPid}) -> %% put_fsm has issued delete_manifest caused by force_stop Res = case RiakObj0 of undefined -> - case riak_cs_manifest:get_manifests(RcPid, Bucket, Key) of + case riak_cs_manifest:get_manifests(RcPid, Bucket, Key, ObjVsn) of {ok, RiakObj, _} -> - riak_cs_gc:gc_specific_manifests([UUID], RiakObj, - Bucket, Key, RcPid); + riak_cs_gc:gc_specific_manifests([UUID], RiakObj, Bucket, RcPid); Error -> Error end; RiakObj -> - riak_cs_gc:gc_specific_manifests([UUID], RiakObj, - Bucket, Key, RcPid) + riak_cs_gc:gc_specific_manifests([UUID], RiakObj, Bucket, RcPid) end, {stop, normal, Res, State}. @@ -299,15 +317,17 @@ code_change(_OldVsn, StateName, State, _Extra) -> %% @doc Return all (resolved) manifests, or notfound -spec handle_get_manifests(#state{}) -> {{ok, [lfs_manifest()]}, #state{}} | {{error, notfound}, #state{}}. -handle_get_manifests(State=#state{riak_client=RcPid, - bucket=Bucket, - key=Key}) -> - case riak_cs_manifest:get_manifests(RcPid, Bucket, Key) of +handle_get_manifests(State = #state{riak_client = RcPid, + bucket = Bucket, + key = Key, + obj_vsn = ObjVsn}) -> + case riak_cs_manifest:get_manifests(RcPid, Bucket, Key, ObjVsn) of {ok, RiakObject, Resolved} -> Reply = {ok, Resolved}, - NewState = State#state{riak_object=RiakObject, manifests=Resolved}, + NewState = State#state{riak_object = RiakObject, + manifests = Resolved}, {Reply, NewState}; - {error, notfound}=NotFound -> + {error, notfound} = NotFound -> {NotFound, State} end. @@ -315,13 +335,10 @@ handle_get_manifests(State=#state{riak_client=RcPid, %% delete the manifest corresponding to `UUID', and then %% write the value back to Riak or delete the manifest value %% if there are no manifests remaining. --spec get_and_delete(riak_client(), binary(), binary(), binary()) -> ok | - {error, term()}. -get_and_delete(RcPid, UUID, Bucket, Key) -> - case riak_cs_manifest:get_manifests(RcPid, Bucket, Key) of +get_and_delete(RcPid, UUID, Bucket, Key, Vsn) -> + case riak_cs_manifest:get_manifests(RcPid, Bucket, Key, Vsn) of {ok, RiakObject, Manifests} -> - ResolvedManifests = riak_cs_manifest_resolution:resolve([Manifests]), - UpdatedManifests = orddict:erase(UUID, ResolvedManifests), + UpdatedManifests = orddict:erase(UUID, Manifests), case UpdatedManifests of [] -> DeleteTimeout = riak_cs_config:delete_manifest_timeout(), @@ -332,7 +349,7 @@ get_and_delete(RcPid, UUID, Bucket, Key) -> riak_cs_utils:update_obj_value( RiakObject, riak_cs_utils:encode_term(UpdatedManifests)), ObjectToWrite = update_md_with_multipart_2i( - ObjectToWrite0, UpdatedManifests, Bucket, Key), + ObjectToWrite0, UpdatedManifests, Bucket), PutTimeout = riak_cs_config:put_manifest_timeout(), riak_cs_pbc:put(manifest_pbc(RcPid), ObjectToWrite, PutTimeout, [riakc, put_manifest]) @@ -341,13 +358,11 @@ get_and_delete(RcPid, UUID, Bucket, Key) -> ok end. --spec get_and_update(riak_client(), orddict:orddict(), binary(), binary()) -> - {ok | error, undefined | riakc_obj:riakc_obj(), - undefined | orddict:orddict()}. -get_and_update(RcPid, WrappedManifests, Bucket, Key) -> - case riak_cs_manifest:get_manifests(RcPid, Bucket, Key) of + +get_and_update(RcPid, WrappedManifests, Bucket, Key, ObjVsn) -> + case riak_cs_manifest:get_manifests(RcPid, Bucket, Key, ObjVsn) of {ok, RiakObject, Manifests} -> - case update(RcPid, Manifests, RiakObject, WrappedManifests, Bucket, Key) of + case update(RcPid, Manifests, RiakObject, WrappedManifests, Bucket) of {ok, _, _} = Res -> case maybe_backpressure_sleep(riakc_obj:value_count(RiakObject)) of true -> @@ -363,9 +378,11 @@ get_and_update(RcPid, WrappedManifests, Bucket, Key) -> end; {error, notfound} -> ManifestBucket = riak_cs_utils:to_bucket_name(objects, Bucket), - ObjectToWrite0 = riakc_obj:new(ManifestBucket, Key, riak_cs_utils:encode_term(WrappedManifests)), + ObjectToWrite0 = riakc_obj:new(ManifestBucket, + rcs_common_manifest:make_versioned_key(Key, ObjVsn), + riak_cs_utils:encode_term(WrappedManifests)), ObjectToWrite = update_md_with_multipart_2i( - ObjectToWrite0, WrappedManifests, Bucket, Key), + ObjectToWrite0, WrappedManifests, Bucket), Timeout = riak_cs_config:put_manifest_timeout(), PutResult = riak_cs_pbc:put(manifest_pbc(RcPid), ObjectToWrite, Timeout, [riakc, put_manifest]), @@ -389,14 +406,14 @@ maybe_backpressure_sleep(Siblings, _BackpressureThreshold) -> Coefficient = riak_cs_config:get_env(riak_cs, manifest_siblings_bp_coefficient, 200), MeanSleepMS = min(Coefficient * Siblings, MaxSleep), Delta = MeanSleepMS div 2, - SleepMS = crypto:rand_uniform(MeanSleepMS - Delta, MeanSleepMS + Delta), - lager:debug("maybe_backpressure_sleep: Siblings=~p, SleepMS=~p~n", [Siblings, SleepMS]), + SleepMS = MeanSleepMS - Delta + rand:uniform(MeanSleepMS + Delta), + ?LOG_DEBUG("maybe_backpressure_sleep: Siblings=~p, SleepMS=~p", [Siblings, SleepMS]), ok = riak_cs_stats:countup([manifest, siblings_bp_sleep]), ok = timer:sleep(SleepMS), true. -update(RcPid, OldManifests, OldRiakObject, WrappedManifests, Bucket, Key) -> - NewManiAdded = riak_cs_manifest_resolution:resolve([WrappedManifests, OldManifests]), +update(RcPid, OldManifests, OldRiakObject, WrappedManifests, Bucket) -> + NewManiAdded = rcs_common_manifest_resolution:resolve([OldManifests, WrappedManifests]), %% Update the object here so that if there are any %% overwritten UUIDs, then gc_specific_manifests() will %% operate on NewManiAdded and save it to Riak when it is @@ -404,7 +421,7 @@ update(RcPid, OldManifests, OldRiakObject, WrappedManifests, Bucket, Key) -> ObjectToWrite0 = riak_cs_utils:update_obj_value( OldRiakObject, riak_cs_utils:encode_term(NewManiAdded)), ObjectToWrite = update_md_with_multipart_2i( - ObjectToWrite0, NewManiAdded, Bucket, Key), + ObjectToWrite0, NewManiAdded, Bucket), {Result, NewRiakObject} = case riak_cs_manifest_utils:overwritten_UUIDs(NewManiAdded) of [] -> @@ -414,7 +431,7 @@ update(RcPid, OldManifests, OldRiakObject, WrappedManifests, Bucket, Key) -> OverwrittenUUIDs -> riak_cs_gc:gc_specific_manifests(OverwrittenUUIDs, ObjectToWrite, - Bucket, Key, + Bucket, RcPid) end, UpdatedManifests = riak_cs_manifest:manifests_from_riak_object(NewRiakObject), @@ -424,18 +441,11 @@ manifest_pbc(RcPid) -> {ok, ManifestPbc} = riak_cs_riak_client:manifest_pbc(RcPid), ManifestPbc. --spec update_from_previous_read(riak_client(), riakc_obj:riakc_obj(), - binary(), binary(), - orddict:orddict(), orddict:orddict()) -> - ok | {error, term()}. -update_from_previous_read(RcPid, RiakObject, Bucket, Key, - PreviousManifests, NewManifests) -> - Resolved = riak_cs_manifest_resolution:resolve([PreviousManifests, - NewManifests]), +update_from_previous_read(RcPid, RiakObject, Bucket, PreviousManifests, NewManifests) -> + Resolved = rcs_common_manifest_resolution:resolve([PreviousManifests, NewManifests]), NewRiakObject0 = riak_cs_utils:update_obj_value(RiakObject, riak_cs_utils:encode_term(Resolved)), - NewRiakObject = update_md_with_multipart_2i(NewRiakObject0, Resolved, - Bucket, Key), + NewRiakObject = update_md_with_multipart_2i(NewRiakObject0, Resolved, Bucket), %% TODO: %% currently we don't do %% anything to make sure @@ -444,7 +454,7 @@ update_from_previous_read(RcPid, RiakObject, Bucket, Key, riak_cs_pbc:put(manifest_pbc(RcPid), NewRiakObject, [], Timeout, [riakc, put_manifest]). -update_md_with_multipart_2i(RiakObject, WrappedManifests, Bucket, Key) -> +update_md_with_multipart_2i(RiakObject, WrappedManifests, Bucket) -> %% During testing, it's handy to delete Riak keys in the %% S3 bucket, e.g., cleaning up from a previous test. %% Let's not trip over tombstones here. @@ -456,7 +466,7 @@ update_md_with_multipart_2i(RiakObject, WrappedManifests, Bucket, Key) -> merge_dicts(MDs) end, {K_i, V_i} = riak_cs_mp_utils:calc_multipart_2i_dict( - [M || {_, M} <- WrappedManifests], Bucket, Key), + [M || {_, M} <- WrappedManifests], Bucket), MD = dict:store(K_i, V_i, MD0), riakc_obj:update_metadata(RiakObject, MD). diff --git a/apps/riak_cs/src/riak_cs_manifest_utils.erl b/apps/riak_cs/src/riak_cs_manifest_utils.erl new file mode 100644 index 000000000..a168fd922 --- /dev/null +++ b/apps/riak_cs/src/riak_cs_manifest_utils.erl @@ -0,0 +1,201 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc Module for choosing and manipulating lists (well, orddict) of manifests + +-module(riak_cs_manifest_utils). + +-include("riak_cs.hrl"). +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +%% export Public API +-export([overwritten_UUIDs/1, + manifests_to_gc/2, + prune/1]). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%% @doc Extract all manifests that are not "the most active" +%% and not actively writing (within the leeway period). +-spec overwritten_UUIDs(orddict:orddict()) -> term(). +overwritten_UUIDs(Dict) -> + case rcs_common_manifest_utils:active_manifest(Dict) of + {error, no_active_manifest} -> + []; + {ok, Active} -> + lists:foldl(overwritten_UUIDs_active_fold_helper(Active), + [], + orddict:to_list(Dict)) + end. + +overwritten_UUIDs_active_fold_helper(Active) -> + fun({UUID, Manifest}, Acc) -> + update_acc(UUID, Manifest, Acc, Active =:= Manifest) + end. + +update_acc(_UUID, _Manifest, Acc, true) -> + Acc; +update_acc(UUID, ?MANIFEST{state = active}, Acc, false) -> + [UUID | Acc]; +update_acc(UUID, ?MANIFEST{state = writing, + last_block_written_time = LBWT, + write_start_time = WST}, Acc, _) -> + acc_leeway_helper(UUID, Acc, LBWT, WST); +update_acc(_, _, Acc, _) -> + Acc. + +acc_leeway_helper(UUID, Acc, undefined, WST) -> + acc_leeway_helper(UUID, Acc, WST); +acc_leeway_helper(UUID, Acc, LBWT, _) -> + acc_leeway_helper(UUID, Acc, LBWT). + +acc_leeway_helper(UUID, Acc, Time) -> + handle_leeway_elaped_time(leeway_elapsed(Time), UUID, Acc). + +handle_leeway_elaped_time(true, UUID, Acc) -> + [UUID | Acc]; +handle_leeway_elaped_time(false, _UUID, Acc) -> + Acc. + +%% @doc Return a list of manifests that are either +%% in `PendingDeleteUUIDs' or are in the `pending_delete' +%% state and have been there for longer than the retry +%% interval. +-spec manifests_to_gc([cs_uuid()], orddict:orddict()) -> [cs_uuid_and_manifest()]. +manifests_to_gc(PendingDeleteUUIDs, Manifests) -> + FilterFun = pending_delete_helper(PendingDeleteUUIDs), + orddict:to_list(orddict:filter(FilterFun, Manifests)). + +%% @private +%% Return a function for use in `orddict:filter/2' +%% that will return true if the manifest key is +%% in `UUIDs' or the manifest should be retried +%% moving to the GC bucket +pending_delete_helper(UUIDs) -> + fun(Key, Manifest) -> + lists:member(Key, UUIDs) orelse retry_manifest(Manifest) + end. + +%% @private +%% Return true if this manifest should be retried +%% moving to the GC bucket +retry_manifest(?MANIFEST{state = pending_delete, + delete_marked_time = MarkedTime}) -> + retry_from_marked_time(MarkedTime, os:system_time(millisecond)); +retry_manifest(_Manifest) -> + false. + +%% @private +%% Return true if the time elapsed between +%% `MarkedTime' and `Now' is greater than +%% `riak_cs_gc:gc_retry_interval()'. +retry_from_marked_time(MarkedTime, Now) -> + Now > (MarkedTime + riak_cs_gc:gc_retry_interval() * 1000). + +%% @doc Remove all manifests that require pruning, +%% see needs_pruning() for definition of needing pruning. +-spec prune(orddict:orddict()) -> orddict:orddict(). +prune(Dict) -> + MaxCount = riak_cs_gc:max_scheduled_delete_manifests(), + rcs_common_manifest_utils:prune(Dict, os:system_time(millisecond), MaxCount, riak_cs_gc:leeway_seconds()). + + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + + +leeway_elapsed(Timestamp) -> + Now = os:system_time(millisecond), + Now > Timestamp + riak_cs_gc:leeway_seconds() * 1000. + + +%% =================================================================== +%% EUnit tests +%% =================================================================== +-ifdef(TEST). + +new_mani_helper() -> + riak_cs_lfs_utils:new_manifest( + <<"bucket">>, <<"key">>, <<"1.0">>, <<"uuid">>, + 100, %% content-length + <<"ctype">>, + undefined, %% md5 + orddict:new(), + 10, + undefined, + [], + undefined, + undefined). + +manifest_test_() -> + {setup, + fun setup/0, + fun cleanup/1, + [fun wrong_state_for_pruning/0, + fun wrong_state_for_pruning_2/0, + fun does_need_pruning/0, + fun not_old_enough_for_pruning/0] + }. + +setup() -> + ok. + +cleanup(_Ctx) -> + ok. + +wrong_state_for_pruning() -> + Mani = new_mani_helper(), + Mani2 = Mani?MANIFEST{state = active}, + ?assert(not rcs_common_manifest_utils:needs_pruning(Mani2, os:system_time(millisecond), 5)). + +wrong_state_for_pruning_2() -> + Mani = new_mani_helper(), + Mani2 = Mani?MANIFEST{state = pending_delete}, + ?assert(not rcs_common_manifest_utils:needs_pruning(Mani2, os:system_time(millisecond), 5)). + +does_need_pruning() -> + application:set_env(riak_cs, leeway_seconds, 1), + %% 1000000 second diff + Now = os:system_time(millisecond), + ScheduledDeleteTime = Now - 24 * 3600 * 1000, + Mani = new_mani_helper(), + Mani2 = Mani?MANIFEST{state = scheduled_delete, + scheduled_delete_time = ScheduledDeleteTime}, + ?assert(rcs_common_manifest_utils:needs_pruning(Mani2, Now, 5)). + +not_old_enough_for_pruning() -> + application:set_env(riak_cs, leeway_seconds, 2), + %$ 1 second diff + Now = os:system_time(millisecond), + ScheduledDeleteTime = Now - 1 * 1000, + Mani = new_mani_helper(), + Mani2 = Mani?MANIFEST{state = scheduled_delete, + scheduled_delete_time = ScheduledDeleteTime}, + ?assert(not rcs_common_manifest_utils:needs_pruning(Mani2, Now, 5)). + +-endif. diff --git a/src/riak_cs_mb_helper.erl b/apps/riak_cs/src/riak_cs_mb_helper.erl similarity index 88% rename from src/riak_cs_mb_helper.erl rename to apps/riak_cs/src/riak_cs_mb_helper.erl index 9d01ad895..244941b79 100644 --- a/src/riak_cs_mb_helper.erl +++ b/apps/riak_cs/src/riak_cs_mb_helper.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -24,13 +25,13 @@ -export([process_specs/0, bags/0, cluster_id/1, - get_cluster_id/1, choose_bag_id/2, set_bag_id_to_manifest/2, bag_id_from_manifest/1]). -include("riak_cs.hrl"). -include_lib("riak_pb/include/riak_pb_kv_codec.hrl"). +-include_lib("kernel/include/logger.hrl"). -define(MB_ENABLED(Normal, Multibag), case riak_cs_config:is_multibag_enabled() of @@ -57,14 +58,10 @@ set_bag_id_to_manifest(BagId, ?MANIFEST{props = Props} = Manifest) -spec bag_id_from_manifest(lfs_manifest()) -> bag_id(). bag_id_from_manifest(?MANIFEST{props = Props}) -> - case Props of - undefined -> - undefined; - _ -> - case lists:keyfind(block_bag, 1, Props) of - false -> undefined; - {block_bag, BagId} -> BagId - end + case lists:keyfind(block_bag, 1, Props) of + false -> undefined; + {block_bag, BagId} -> BagId + end. -spec cluster_id(bag_id()) -> cluster_id(). @@ -84,7 +81,7 @@ get_cluster_id(BagId) -> {ok, Pbc} = riak_cs_utils:riak_connection(PbcPool), try ClusterId = riak_cs_pbc:get_cluster_id(Pbc), - lager:info("Cluster ID for bag ~p : ~p", [BagId, ClusterId]), + ?LOG_DEBUG("Cluster ID for bag ~s: ~s", [BagId, ClusterId]), ClusterId after riak_cs_utils:close_riak_connection(PbcPool, Pbc) diff --git a/src/riak_cs_mp_utils.erl b/apps/riak_cs/src/riak_cs_mp_utils.erl similarity index 76% rename from src/riak_cs_mp_utils.erl rename to apps/riak_cs/src/riak_cs_mp_utils.erl index 834ab37a3..758ae58ea 100644 --- a/src/riak_cs_mp_utils.erl +++ b/apps/riak_cs/src/riak_cs_mp_utils.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -22,15 +23,32 @@ -module(riak_cs_mp_utils). +%% export Public API +-export([abort_multipart_upload/6, + calc_multipart_2i_dict/2, + clean_multipart_unused_parts/2, + complete_multipart_upload/7, + get_mp_manifest/1, + initiate_multipart_upload/7, + is_multipart_manifest/1, + list_all_multipart_uploads/3, + list_multipart_uploads/4, + list_parts/7, + make_content_types_accepted/2, + make_content_types_accepted/3, + upload_part/8, + upload_part_1blob/2, + upload_part_finished/9]). + -include("riak_cs.hrl"). -include_lib("riak_pb/include/riak_pb_kv_codec.hrl"). -include_lib("riakc/include/riakc.hrl"). +-include_lib("kernel/include/logger.hrl"). -ifdef(TEST). -compile(export_all). --ifdef(EQC). --include_lib("eqc/include/eqc.hrl"). --endif. +-compile(nowarn_export_all). +-include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). -endif. @@ -38,30 +56,13 @@ -define(PID(WrappedRcPid), get_riak_client_pid(WrappedRcPid)). -%% export Public API --export([ - abort_multipart_upload/4, abort_multipart_upload/5, - calc_multipart_2i_dict/3, - clean_multipart_unused_parts/2, - complete_multipart_upload/5, complete_multipart_upload/6, - initiate_multipart_upload/5, initiate_multipart_upload/6, - list_all_multipart_uploads/3, - list_multipart_uploads/3, list_multipart_uploads/4, - list_parts/5, list_parts/6, - make_content_types_accepted/2, - make_content_types_accepted/3, - upload_part/6, upload_part/7, - upload_part_1blob/2, - upload_part_finished/7, upload_part_finished/8, - is_multipart_manifest/1 - ]). --export([get_mp_manifest/1]). - %%%=================================================================== %%% API %%%=================================================================== -calc_multipart_2i_dict(Ms, Bucket, _Key) when is_list(Ms) -> +-spec calc_multipart_2i_dict([lfs_manifest()], binary()) -> + {binary(), [{binary(), binary()}]}. +calc_multipart_2i_dict(Ms, Bucket) when is_list(Ms) -> %% According to API Version 2006-03-01, page 139-140, bucket %% owners have some privileges for multipart uploads performed by %% other users, i.e, see those MP uploads via list multipart uploads, @@ -73,20 +74,25 @@ calc_multipart_2i_dict(Ms, Bucket, _Key) when is_list(Ms) -> case get_mp_manifest(M) of undefined -> []; - MpM when is_record(MpM, ?MULTIPART_MANIFEST_RECNAME) -> - [{make_2i_key(Bucket, MpM?MULTIPART_MANIFEST.owner), <<"1">>}, + ?MULTIPART_MANIFEST{owner = Owner} -> + [{make_2i_key(Bucket, Owner), <<"1">>}, {make_2i_key(Bucket), <<"1">>}] end || M <- Ms, M?MANIFEST.state == writing], {?MD_INDEX, lists:usort(lists:append(L_2i))}. -abort_multipart_upload(Bucket, Key, UploadId, Caller) -> - abort_multipart_upload(Bucket, Key, UploadId, Caller, nopid). -abort_multipart_upload(Bucket, Key, UploadId, Caller, RcPidUnW) -> - do_part_common(abort, Bucket, Key, UploadId, Caller, [], RcPidUnW). +-spec abort_multipart_upload(binary(), binary(), binary(), + binary(), acl_owner(), nopid | pid()) -> + ok | {error, term()}. +abort_multipart_upload(Bucket, Key, ObjVsn, UploadId, Caller, RcPidUnW) -> + do_part_common(abort, Bucket, Key, ObjVsn, UploadId, Caller, [], RcPidUnW). -clean_multipart_unused_parts(?MANIFEST{bkey=BKey, props=Props} = Manifest, RcPid) -> +-spec clean_multipart_unused_parts(lfs_manifest(), nopid | pid()) -> + same | updated. +clean_multipart_unused_parts(?MANIFEST{bkey = {Bucket, Key}, + vsn = ObjVsn, + props = Props} = Manifest, RcPid) -> case get_mp_manifest(Manifest) of undefined -> same; @@ -97,16 +103,15 @@ clean_multipart_unused_parts(?MANIFEST{bkey=BKey, props=Props} = Manifest, RcPid same; {false, PartsToDelete} -> _ = try - {Bucket, Key} = BKey, BagId = riak_cs_mb_helper:bag_id_from_manifest(Manifest), - ok = move_dead_parts_to_gc(Bucket, Key, BagId, + ok = move_dead_parts_to_gc(Bucket, Key, ObjVsn, BagId, PartsToDelete, RcPid), - UpdManifest = Manifest?MANIFEST{props=[multipart_clean|Props]}, + UpdManifest = Manifest?MANIFEST{props = [multipart_clean|Props]}, ok = update_manifest_with_confirmation(RcPid, UpdManifest) - catch X:Y -> - lager:debug("clean_multipart_unused_parts: " - "bkey ~p: ~p ~p @ ~p\n", - [BKey, X, Y, erlang:get_stacktrace()]) + catch X:Y:ST -> + ?LOG_DEBUG("clean_multipart_unused_parts: " + "b/key:vsn ~s/~s:~s : ~p ~p @ ~p", + [Bucket, Key, ObjVsn, X, Y, ST]) end, %% Return same value to caller, regardless of ok/catch updated; @@ -115,61 +120,28 @@ clean_multipart_unused_parts(?MANIFEST{bkey=BKey, props=Props} = Manifest, RcPid end end. -complete_multipart_upload(Bucket, Key, UploadId, PartETags, Caller) -> - complete_multipart_upload(Bucket, Key, UploadId, PartETags, Caller, nopid). -complete_multipart_upload(Bucket, Key, UploadId, PartETags, Caller, RcPidUnW) -> +-spec complete_multipart_upload(binary(), binary(), binary(), + binary(), [{integer(), binary()}], acl_owner(), nopid | pid()) -> + {ok, lfs_manifest()} | {error, atom()}. +complete_multipart_upload(Bucket, Key, Vsn, UploadId, PartETags, Caller, RcPidUnW) -> Extra = {PartETags}, - do_part_common(complete, Bucket, Key, UploadId, Caller, [{complete, Extra}], + do_part_common(complete, Bucket, Key, Vsn, UploadId, Caller, [{complete, Extra}], RcPidUnW). -initiate_multipart_upload(Bucket, Key, ContentType, Owner, Opts) -> - initiate_multipart_upload(Bucket, Key, ContentType, Owner, Opts, nopid). -initiate_multipart_upload(Bucket, Key, ContentType, {_,_,_} = Owner, +-spec initiate_multipart_upload(binary(), binary(), binary(), + binary(), acl_owner(), proplists:proplist(), nopid | pid()) -> + {ok, binary()} | {error, term()}. +initiate_multipart_upload(Bucket, Key, Vsn, ContentType, Owner, Opts, RcPidUnW) -> - write_new_manifest(new_manifest(Bucket, Key, ContentType, Owner, Opts), + write_new_manifest(new_manifest(Bucket, Key, Vsn, ContentType, Owner, Opts), Opts, RcPidUnW). -make_content_types_accepted(RD, Ctx) -> - make_content_types_accepted(RD, Ctx, unused_callback). -make_content_types_accepted(RD, Ctx, Callback) -> - make_content_types_accepted(wrq:get_req_header("Content-Type", RD), - RD, - Ctx, - Callback). - -make_content_types_accepted(CT, RD, Ctx, Callback) - when CT =:= undefined; - CT =:= [] -> - make_content_types_accepted("application/octet-stream", RD, Ctx, Callback); -make_content_types_accepted(CT, RD, Ctx=#context{local_context=LocalCtx0}, Callback) -> - %% This was shamelessly ripped out of - %% https://github.com/basho/riak_kv/blob/0d91ca641a309f2962a216daa0cee869c82ffe26/src/riak_kv_wm_object.erl#L492 - {Media, _Params} = mochiweb_util:parse_header(CT), - case string:tokens(Media, "/") of - [_Type, _Subtype] -> - %% accept whatever the user says - LocalCtx = LocalCtx0#key_context{putctype=Media}, - {[{Media, Callback}], RD, Ctx#context{local_context=LocalCtx}}; - _ -> - {[], - wrq:set_resp_header( - "Content-Type", - "text/plain", - wrq:set_resp_body( - ["\"", Media, "\"" - " is not a valid media type" - " for the Content-type header.\n"], - RD)), - Ctx} - end. - -list_multipart_uploads(Bucket, Caller, Opts) -> - list_multipart_uploads(Bucket, Caller, Opts, nopid). - -list_multipart_uploads(Bucket, {_Display, _Canon, CallerKeyId} = Caller, +-spec list_multipart_uploads(binary(), acl_owner(), proplists:proplist(), nopid | pid()) -> + {ok, {[multipart_descr()], [ordsets:ordset()]}} | {error, term()}. +list_multipart_uploads(Bucket, #{key_id := CallerKeyId} = Caller, Opts, RcPidUnW) -> case wrap_riak_client(RcPidUnW) of {ok, RcPid} -> @@ -204,61 +176,39 @@ list_multipart_uploads_with_2ikey(Bucket, Opts, RcPid, Key2i) -> case riak_cs_pbc:get_index_eq(ManifestPbc, HashBucket, Key2i, <<"1">>, [{timeout, Timeout}], [riakc, get_uploads_by_index]) of - {ok, ?INDEX_RESULTS{keys=Names}} -> + {ok, ?INDEX_RESULTS{keys = Names}} -> {ok, list_multipart_uploads2(Bucket, RcPid, Names, Opts)}; Else2 -> Else2 end. -list_parts(Bucket, Key, UploadId, Caller, Opts) -> - list_parts(Bucket, Key, UploadId, Caller, Opts, nopid). -list_parts(Bucket, Key, UploadId, Caller, Opts, RcPidUnW) -> +-spec list_parts(binary(), binary(), binary(), + binary(), acl_owner(), proplists:proplist(), nopid | pid()) -> + {ok, [part_descr()]} | {error, term()}. +list_parts(Bucket, Key, ObjVsn, UploadId, Caller, Opts, RcPidUnW) -> Extra = {Opts}, - do_part_common(list, Bucket, Key, UploadId, Caller, [{list, Extra}], RcPidUnW). - -%% @doc --spec new_manifest(binary(), binary(), binary(), acl_owner(), list()) -> lfs_manifest(). -new_manifest(Bucket, Key, ContentType, {_, _, _} = Owner, Opts) -> - UUID = druuid:v4(), - %% TODO: add object metadata here, e.g. content-disposition et al. - MetaData = case proplists:get_value(meta_data, Opts) of - undefined -> []; - AsIsHdrs -> AsIsHdrs - end, - M = riak_cs_lfs_utils:new_manifest(Bucket, - Key, - UUID, - 0, - ContentType, - %% we won't know the md5 of a multipart - undefined, - MetaData, - riak_cs_lfs_utils:block_size(), - %% ACL: needs Riak client pid, so we wait - no_acl_yet, - [], - %% Cluster ID and Bag ID are added later - undefined, - undefined), - MpM = ?MULTIPART_MANIFEST{upload_id = UUID, - owner = Owner}, - M?MANIFEST{props = replace_mp_manifest(MpM, M?MANIFEST.props)}. + do_part_common(list, Bucket, Key, ObjVsn, UploadId, Caller, [{list, Extra}], RcPidUnW). -upload_part(Bucket, Key, UploadId, PartNumber, Size, Caller) -> - upload_part(Bucket, Key, UploadId, PartNumber, Size, Caller, nopid). -upload_part(Bucket, Key, UploadId, PartNumber, Size, Caller, RcPidUnW) -> - Extra = {Bucket, Key, UploadId, Caller, PartNumber, Size}, - do_part_common(upload_part, Bucket, Key, UploadId, Caller, +-spec upload_part(binary(), binary(), binary(), + binary(), non_neg_integer(), non_neg_integer(), acl_owner(), pid()) -> + {upload_part_ready, binary(), pid()} | {error, riak_unavailable | notfound}. +upload_part(Bucket, Key, ObjVsn, UploadId, PartNumber, Size, Caller, RcPidUnW) -> + Extra = {Bucket, Key, ObjVsn, UploadId, Caller, PartNumber, Size}, + do_part_common(upload_part, Bucket, Key, ObjVsn, UploadId, Caller, [{upload_part, Extra}], RcPidUnW). + +-spec upload_part_1blob(pid(), binary()) -> + {ok, lfs_manifest()}. upload_part_1blob(PutPid, Blob) -> ok = riak_cs_put_fsm:augment_data(PutPid, Blob), {ok, M} = riak_cs_put_fsm:finalize(PutPid, undefined), {ok, M?MANIFEST.content_md5}. + %% Once upon a time, in a naive land far away, I thought that it would %% be sufficient to use each part's UUID as the ETag when the part %% upload was finished, and thus the client would use that UUID to @@ -268,17 +218,71 @@ upload_part_1blob(PutPid, Blob) -> %% must thread the MD5 value through upload_part_finished and update %% the ?MULTIPART_MANIFEST in a mergeable way. {sigh} -upload_part_finished(Bucket, Key, UploadId, _PartNumber, PartUUID, MD5, Caller) -> - upload_part_finished(Bucket, Key, UploadId, _PartNumber, PartUUID, MD5, - Caller, nopid). - -upload_part_finished(Bucket, Key, UploadId, _PartNumber, PartUUID, MD5, +-spec upload_part_finished(binary(), binary(), binary(), + binary(), non_neg_integer(), binary(), term(), acl_owner(), pid()) -> + ok | {error, any()}. +upload_part_finished(Bucket, Key, ObjVsn, + UploadId, _PartNumber, PartUUID, MD5, Caller, RcPidUnW) -> Extra = {PartUUID, MD5}, - do_part_common(upload_part_finished, Bucket, Key, UploadId, + do_part_common(upload_part_finished, Bucket, Key, ObjVsn, UploadId, Caller, [{upload_part_finished, Extra}], RcPidUnW). -write_new_manifest(?MANIFEST{bkey={Bucket, Key}, uuid=UUID}=M, Opts, RcPidUnW) -> + + +-spec is_multipart_manifest(?MANIFEST{}) -> boolean(). +is_multipart_manifest(?MANIFEST{props = Props}) -> + case proplists:get_value(multipart, Props) of + undefined -> + false; + _ -> + true + end. + + +make_content_types_accepted(RD, Ctx) -> + make_content_types_accepted(RD, Ctx, unused_callback). + +make_content_types_accepted(RD, Ctx, Callback) -> + make_content_types_accepted(wrq:get_req_header("Content-Type", RD), + RD, + Ctx, + Callback). + +make_content_types_accepted(CT, RD, Ctx, Callback) + when CT =:= undefined; + CT =:= [] -> + make_content_types_accepted("application/octet-stream", RD, Ctx, Callback); +make_content_types_accepted(CT, RD, Ctx = #rcs_web_context{local_context = LocalCtx0}, Callback) -> + %% This was shamelessly ripped out of + %% https://github.com/basho/riak_kv/blob/0d91ca641a309f2962a216daa0cee869c82ffe26/src/riak_kv_wm_object.erl#L492 + {Media, _Params} = mochiweb_util:parse_header(CT), + case string:tokens(Media, "/") of + [_Type, _Subtype] -> + %% accept whatever the user says + LocalCtx = LocalCtx0#key_context{putctype = Media}, + {[{Media, Callback}], RD, Ctx#rcs_web_context{local_context = LocalCtx}}; + _ -> + {[], + wrq:set_resp_header( + "Content-Type", + "text/plain", + wrq:set_resp_body( + ["\"", Media, "\"" + " is not a valid media type" + " for the Content-type header.\n"], + RD)), + Ctx} + end. + + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +write_new_manifest(?MANIFEST{bkey = {Bucket, Key}, + vsn = Vsn, + uuid = UUID} = M, Opts, RcPidUnW) -> MpM = get_mp_manifest(M), Owner = MpM?MULTIPART_MANIFEST.owner, case wrap_riak_client(RcPidUnW) of @@ -294,9 +298,9 @@ write_new_manifest(?MANIFEST{bkey={Bucket, Key}, uuid=UUID}=M, Opts, RcPidUnW) - M2 = riak_cs_lfs_utils:set_bag_id(BagId, M), ClusterId = riak_cs_mb_helper:cluster_id(BagId), M3 = M2?MANIFEST{acl = Acl, - cluster_id=ClusterId, - write_start_time=os:timestamp()}, - {ok, ManiPid} = riak_cs_manifest_fsm:start_link(Bucket, Key, + cluster_id = ClusterId, + write_start_time = os:system_time(millisecond)}, + {ok, ManiPid} = riak_cs_manifest_fsm:start_link(Bucket, Key, Vsn, ?PID(RcPid)), try ok = riak_cs_manifest_fsm:add_new_manifest(ManiPid, M3), @@ -311,22 +315,47 @@ write_new_manifest(?MANIFEST{bkey={Bucket, Key}, uuid=UUID}=M, Opts, RcPidUnW) - Else end. -%%%=================================================================== -%%% Internal functions -%%%=================================================================== +new_manifest(Bucket, Key, Vsn, ContentType, Owner, Opts) -> + UUID = uuid:get_v4(), + %% TODO: add object metadata here, e.g. content-disposition et al. + MetaData = case proplists:get_value(meta_data, Opts) of + undefined -> []; + AsIsHdrs -> AsIsHdrs + end, + M0 = riak_cs_lfs_utils:new_manifest( + Bucket, Key, Vsn, UUID, + 0, + ContentType, + %% we won't know the md5 of a multipart + undefined, + MetaData, + riak_cs_lfs_utils:block_size(), + %% ACL: needs Riak client pid, so we wait + no_acl_yet, + [], + %% Cluster ID and Bag ID are added later + undefined, + undefined), + MpM = ?MULTIPART_MANIFEST{upload_id = UUID, + owner = Owner}, + M9 = M0?MANIFEST{props = replace_mp_manifest(MpM, M0?MANIFEST.props)}, + ?LOG_DEBUG("created mp manifest for ~s/~s:~s", [Bucket, Key, Vsn]), + + M9. -do_part_common(Op, Bucket, Key, UploadId, {_,_,CallerKeyId} = _Caller, Props, RcPidUnW) -> + +do_part_common(Op, Bucket, Key, ObjVsn, UploadId, #{key_id := CallerKeyId}, Props, RcPidUnW) -> case wrap_riak_client(RcPidUnW) of {ok, RcPid} -> try - case riak_cs_manifest:get_manifests(?PID(RcPid), Bucket, Key) of + case riak_cs_manifest:get_manifests(?PID(RcPid), Bucket, Key, ObjVsn) of {ok, Obj, Manifests} -> case find_manifest_with_uploadid(UploadId, Manifests) of false -> {error, notfound}; M when M?MANIFEST.state == writing -> MpM = get_mp_manifest(M), - {_, _, MpMOwner} = MpM?MULTIPART_MANIFEST.owner, + #{key_id := MpMOwner} = MpM?MULTIPART_MANIFEST.owner, case CallerKeyId == MpMOwner of true -> do_part_common2(Op, ?PID(RcPid), @@ -351,17 +380,20 @@ do_part_common(Op, Bucket, Key, UploadId, {_,_,CallerKeyId} = _Caller, Props, Rc Else end. -do_part_common2(abort, RcPid, M, Obj, _Mpm, _Props) -> - {Bucket, Key} = M?MANIFEST.bkey, +do_part_common2(abort, RcPid, ?MANIFEST{uuid = UUID, + bkey = {Bucket, _Key}}, Obj, _Mpm, _Props) -> case riak_cs_gc:gc_specific_manifests( - [M?MANIFEST.uuid], Obj, Bucket, Key, RcPid) of + [UUID], Obj, Bucket, RcPid) of {ok, _NewObj} -> ok; - Else3 -> - Else3 + Else -> + Else end; do_part_common2(complete, RcPid, - ?MANIFEST{uuid = _UUID, props = MProps} = Manifest, + ?MANIFEST{uuid = _UUID, + bkey = {Bucket, Key}, + vsn = ObjVsn, + props = MProps} = Manifest, _Obj, MpM, Props) -> %% The content_md5 is used by WM to create the ETags header. %% However/fortunately/sigh-of-relief, Amazon's S3 doesn't use @@ -376,8 +408,7 @@ do_part_common2(complete, RcPid, %% BogoMD5 = iolist_to_binary([UUID, "-1"]), {PartETags} = proplists:get_value(complete, Props), try - {Bucket, Key} = Manifest?MANIFEST.bkey, - {ok, ManiPid} = riak_cs_manifest_fsm:start_link(Bucket, Key, RcPid), + {ok, ManiPid} = riak_cs_manifest_fsm:start_link(Bucket, Key, ObjVsn, RcPid), try {Bytes, OverAllMD5, PartsToKeep, PartsToDelete} = comb_parts(MpM, PartETags), true = enforce_part_size(PartsToKeep), @@ -407,7 +438,7 @@ do_part_common2(complete, RcPid, %% then pass them to the GC monster for immediate %% deletion. BagId = riak_cs_mb_helper:bag_id_from_manifest(NewManifest), - ok = move_dead_parts_to_gc(Bucket, Key, BagId, + ok = move_dead_parts_to_gc(Bucket, Key, ObjVsn, BagId, PartsToDelete, RcPid), MProps3 = [multipart_clean|MProps2], New2Manifest = NewManifest?MANIFEST{props = MProps3}, @@ -442,18 +473,18 @@ do_part_common2(list, _RcPid, _M, _Obj, MpM, Props) -> end, [], Parts), Ds = [?PART_DESCR{part_number = P?PART_MANIFEST.part_number, %% TODO: technically, start_time /= last_modified - last_modified = riak_cs_wm_utils:iso_8601_datetime(calendar:now_to_local_time(P?PART_MANIFEST.start_time)), + last_modified = P?PART_MANIFEST.start_time, etag = ETag, size = P?PART_MANIFEST.content_length} || {ETag, P} <- ETagPs], {ok, Ds}; do_part_common2(upload_part, RcPid, M, _Obj, MpM, Props) -> - {Bucket, Key, _UploadId, _Caller, PartNumber, Size} = + {Bucket, Key, ObjVsn, _UploadId, _Caller, PartNumber, Size} = proplists:get_value(upload_part, Props), BlockSize = riak_cs_lfs_utils:block_size(), BagId = riak_cs_mb_helper:bag_id_from_manifest(M), {ok, PutPid} = riak_cs_put_fsm:start_link( - {Bucket, Key, Size, <<"x-riak/multipart-part">>, + {Bucket, Key, ObjVsn, Size, <<"x-riak/multipart-part">>, orddict:new(), BlockSize, M?MANIFEST.acl, infinity, self(), RcPid}, false, BagId), @@ -464,7 +495,8 @@ do_part_common2(upload_part, RcPid, M, _Obj, MpM, Props) -> PartUUID = riak_cs_put_fsm:get_uuid(PutPid), PM = ?PART_MANIFEST{bucket = Bucket, key = Key, - start_time = os:timestamp(), + vsn = ObjVsn, + start_time = os:system_time(millisecond), part_number = PartNumber, part_id = PartUUID, content_length = Size, @@ -479,6 +511,7 @@ do_part_common2(upload_part, RcPid, M, _Obj, MpM, Props) -> riak_cs_put_fsm:force_stop(PutPid), {error, riak_unavailable} end; + do_part_common2(upload_part_finished, RcPid, M, _Obj, MpM, Props) -> {PartUUID, MD5} = proplists:get_value(upload_part_finished, Props), try @@ -491,11 +524,10 @@ do_part_common2(upload_part_finished, RcPid, M, _Obj, MpM, Props) -> {error, notfound}; {_, true} -> {error, notfound}; - {PM, false} when is_record(PM, ?PART_MANIFEST_RECNAME) -> + {?PART_MANIFEST{}, false} -> ?MANIFEST{props = MProps} = M, - NewMpM = MpM?MULTIPART_MANIFEST{ - done_parts = ordsets:add_element({PartUUID, MD5}, - DoneParts)}, + NewMpM = MpM?MULTIPART_MANIFEST{done_parts = ordsets:add_element({PartUUID, MD5}, + DoneParts)}, NewM = M?MANIFEST{props = replace_mp_manifest(NewMpM, MProps)}, ok = update_manifest_with_confirmation(RcPid, NewM) end @@ -503,40 +535,39 @@ do_part_common2(upload_part_finished, RcPid, M, _Obj, MpM, Props) -> {error, riak_unavailable} end. -update_manifest_with_confirmation(RcPid, Manifest) -> - {Bucket, Key} = Manifest?MANIFEST.bkey, - {m_umwc, {ok, ManiPid}} = {m_umwc, - riak_cs_manifest_fsm:start_link(Bucket, Key, - RcPid)}, + +update_manifest_with_confirmation(RcPid, M = ?MANIFEST{bkey = {Bucket, Key}, + vsn = ObjVsn}) -> + {ok, ManiPid} = + riak_cs_manifest_fsm:start_link(Bucket, Key, ObjVsn, RcPid), try - ok = riak_cs_manifest_fsm:update_manifest_with_confirmation(ManiPid, - Manifest) + ok = riak_cs_manifest_fsm:update_manifest_with_confirmation(ManiPid, M) after ok = riak_cs_manifest_fsm:stop(ManiPid) end. --spec make_2i_key(binary()) -> binary(). + make_2i_key(Bucket) -> make_2i_key2(Bucket, ""). --spec make_2i_key(binary(), acl_owner()) -> binary(). -make_2i_key(Bucket, {_, _, OwnerStr}) -> +make_2i_key(Bucket, #{key_id := OwnerStr}) -> make_2i_key2(Bucket, OwnerStr). --spec make_2i_key2(binary(), string()) -> binary(). make_2i_key2(Bucket, OwnerStr) -> iolist_to_binary(["rcs@", OwnerStr, "@", Bucket, "_bin"]). + list_multipart_uploads2(Bucket, RcPid, Names, Opts) -> FilterFun = - fun(K, Acc) -> - filter_uploads_list(Bucket, K, Opts, RcPid, Acc) + fun(VK, Acc) -> + {Key, Vsn} = rcs_common_manifest:decompose_versioned_key(VK), + filter_uploads_list(Bucket, Key, Vsn, Opts, RcPid, Acc) end, {Manifests, Prefixes} = lists:foldl(FilterFun, {[], ordsets:new()}, Names), {lists:sort(Manifests), ordsets:to_list(Prefixes)}. -filter_uploads_list(Bucket, Key, Opts, RcPid, Acc) -> - multipart_manifests_for_key(Bucket, Key, Opts, Acc, RcPid). +filter_uploads_list(Bucket, Key, Vsn, Opts, RcPid, Acc) -> + multipart_manifests_for_key(Bucket, Key, Vsn, Opts, Acc, RcPid). parameter_filter(M, Acc, _, _, KeyMarker, _) when M?MULTIPART_DESCR.key =< KeyMarker-> @@ -558,7 +589,7 @@ parameter_filter(M, {Manifests, Prefixes}, Prefix, undefined, _, _) -> _ -> {Manifests, Prefixes} end; -parameter_filter(M, {Manifests, Prefixes}=Acc, Prefix, Delimiter, _, _) -> +parameter_filter(M, {Manifests, Prefixes} = Acc, Prefix, Delimiter, _, _) -> PrefixLen = byte_size(Prefix), case M?MULTIPART_DESCR.key of << Prefix:PrefixLen/binary, Rest/binary >> -> @@ -582,10 +613,10 @@ update_keys_and_prefixes({Keys, Prefixes}, _, Prefix, PrefixLen, Group) -> NewPrefix = << Prefix:PrefixLen/binary, Group/binary >>, {Keys, ordsets:add_element(NewPrefix, Prefixes)}. -multipart_manifests_for_key(Bucket, Key, Opts, Acc, RcPid) -> +multipart_manifests_for_key(Bucket, Key, Vsn, Opts, Acc, RcPid) -> ParameterFilter = build_parameter_filter(Opts), Manifests = handle_get_manifests_result( - riak_cs_manifest:get_manifests(RcPid, Bucket, Key)), + riak_cs_manifest:get_manifests(RcPid, Bucket, Key, Vsn)), lists:foldl(ParameterFilter, Acc, Manifests). build_parameter_filter(Opts) -> @@ -605,28 +636,22 @@ handle_get_manifests_result({ok, _Obj, Manifests}) -> [multipart_description(M) || {_, M} <- Manifests, M?MANIFEST.state == writing, - is_multipart_manifest(M)]; -handle_get_manifests_result(_) -> - []. - --spec is_multipart_manifest(?MANIFEST{}) -> boolean(). -is_multipart_manifest(?MANIFEST{props=Props}) -> - case proplists:get_value(multipart, Props) of - undefined -> - false; - _ -> - true - end. - --spec multipart_description(?MANIFEST{}) -> ?MULTIPART_DESCR{}. -multipart_description(Manifest) -> - MpM = proplists:get_value(multipart, Manifest?MANIFEST.props), - ?MULTIPART_DESCR{ - key = element(2, Manifest?MANIFEST.bkey), - upload_id = Manifest?MANIFEST.uuid, - owner_key_id = element(3, MpM?MULTIPART_MANIFEST.owner), - owner_display = element(1, MpM?MULTIPART_MANIFEST.owner), - initiated = Manifest?MANIFEST.created}. + is_multipart_manifest(M)]. + + +multipart_description(?MANIFEST{bkey = {_, Key}, + vsn = Vsn, + uuid = UUID, + props = Props, + write_start_time = Created}) -> + ?MULTIPART_MANIFEST{owner = #{display_name := OwnerDisplay, + key_id := OwnerKeyId}} = + proplists:get_value(multipart, Props), + ?MULTIPART_DESCR{key = rcs_common_manifest:make_versioned_key(Key, Vsn), + upload_id = UUID, + owner_key_id = OwnerKeyId, + owner_display = OwnerDisplay, + initiated = Created}. %% @doc Will cause error:{badmatch, {m_ibco, _}} if CallerKeyId does not exist @@ -684,7 +709,7 @@ comb_parts(MpM, PartETags) -> KeepPartIDs = [PM?PART_MANIFEST.part_id || PM <- KeepPMs], ToDelete = [PM || PM <- Parts, not lists:member(PM?PART_MANIFEST.part_id, KeepPartIDs)], - lager:debug("Part count to be deleted at completion = ~p~n", [length(ToDelete)]), + ?LOG_DEBUG("Part count to be deleted at completion = ~p", [length(ToDelete)]), {KeepBytes, riak_cs_utils:md5_final(MD5Context), lists:reverse(KeepPMs), ToDelete}. comb_parts_fold({LastPartNum, LastPartETag} = _K, @@ -705,21 +730,20 @@ comb_parts_fold({PartNum, ETag} = K, throw(bad_etag) end. -move_dead_parts_to_gc(Bucket, Key, BagId, PartsToDelete, RcPid) -> +move_dead_parts_to_gc(Bucket, Key, Vsn, BagId, PartsToDelete, RcPid) -> PartDelMs = [{P_UUID, riak_cs_lfs_utils:new_manifest( - Bucket, - Key, - P_UUID, - ContentLength, - <<"x-delete/now">>, - undefined, - [], - P_BlockSize, - no_acl_yet, - [], - undefined, - BagId)} || + Bucket, Key, Vsn, P_UUID, + ContentLength, + <<"x-delete/now">>, + undefined, + [], + P_BlockSize, + no_acl_yet, + [], + undefined, + BagId) + } || ?PART_MANIFEST{part_id=P_UUID, content_length=ContentLength, block_size=P_BlockSize} <- PartsToDelete], @@ -775,10 +799,6 @@ get_riak_client_pid({remote_pid, RcPid}) -> -spec get_mp_manifest(lfs_manifest()) -> multipart_manifest() | 'undefined'. get_mp_manifest(?MANIFEST{props = Props}) when is_list(Props) -> - %% TODO: When the version number of the multipart_manifest_v1 changes - %% to version v2 and beyond, this might be a good place to add - %% a record conversion function to handle older versions of - %% the multipart record? proplists:get_value(multipart, Props, undefined); get_mp_manifest(_) -> undefined. @@ -786,6 +806,9 @@ get_mp_manifest(_) -> replace_mp_manifest(MpM, Props) -> [{multipart, MpM}|proplists:delete(multipart, Props)]. + + + %% =================================================================== %% EUnit tests %% =================================================================== @@ -857,12 +880,10 @@ comb_parts_test() -> ok. --ifdef(EQC). +eval_part_sizes_eqc_test() -> + true = proper:quickcheck(numtests(500, prop_part_sizes())). -eval_part_sizes_eqc_test() -> - true = eqc:quickcheck(eqc:numtests(500, prop_part_sizes())). - -prop_part_sizes() -> +prop_part_sizes() -> Min = ?MIN_MP_PART_SIZE, Min_1 = Min - 1, MinMinus100 = Min - 100, @@ -873,6 +894,4 @@ prop_part_sizes() false == eval_part_sizes_wrapper(L ++ [Min_1] ++ L ++ [Either]) ). --endif. % EQC - -endif. diff --git a/src/riak_cs_oos_response.erl b/apps/riak_cs/src/riak_cs_oos_response.erl similarity index 90% rename from src/riak_cs_oos_response.erl rename to apps/riak_cs/src/riak_cs_oos_response.erl index 9a841136c..b86096688 100644 --- a/src/riak_cs_oos_response.erl +++ b/apps/riak_cs/src/riak_cs_oos_response.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -27,27 +28,17 @@ status_code/1]). -include("riak_cs.hrl"). --include("riak_cs_api.hrl"). --include("list_objects.hrl"). --include_lib("webmachine/include/webmachine.hrl"). --spec respond(term(), #wm_reqdata{}, #context{}) -> - {string() | {halt, non_neg_integer()} , #wm_reqdata{}, #context{}}. -respond(?LBRESP{}=Response, RD, Ctx) -> +-spec respond(?LBRESP{}, #wm_reqdata{}, #rcs_web_context{}) -> + {string() | {halt, non_neg_integer()} , #wm_reqdata{}, #rcs_web_context{}}. +respond(?LBRESP{} = Response, RD, Ctx) -> BucketsDoc = - [begin - case is_binary(B?RCS_BUCKET.name) of - true -> - binary_to_list(B?RCS_BUCKET.name) ++ "\n"; - false -> - B?RCS_BUCKET.name ++ "\n" - end - end || B <- Response?LBRESP.buckets], + [binary_to_list(B?RCS_BUCKET.name) ++ "\n" || B <- Response?LBRESP.buckets], UpdRD = wrq:set_resp_header("Content-Type", "text/plain", RD), {BucketsDoc, UpdRD, Ctx}; -respond({ok, ?LORESP{}=Response}, RD, Ctx) -> +respond({ok, ?LORESP{} = Response}, RD, Ctx) -> %% @TODO Expand to handle common prefixes UpdRD = wrq:set_resp_header("Content-Type", "text/plain", @@ -56,6 +47,16 @@ respond({ok, ?LORESP{}=Response}, RD, Ctx) -> KeyContent <- Response?LORESP.contents, KeyContent /= undefined], {ResponseBody, UpdRD, Ctx}; +respond({ok, ?LOVRESP{} = Response}, RD, Ctx) -> + %% @TODO Expand to handle common prefixes + UpdRD = wrq:set_resp_header("Content-Type", + "text/plain", + RD), + ResponseBody = [binary_to_list(KeyContent?LOVKC.key) ++ ":" ++ + binary_to_list(KeyContent?LOVKC.version_id) ++ "\n" || + KeyContent <- Response?LOVRESP.contents, + KeyContent /= undefined], + {ResponseBody, UpdRD, Ctx}; respond({error, _}=Error, RD, Ctx) -> api_error(Error, RD, Ctx). @@ -156,7 +157,7 @@ error_message(invalid_range) -> "The requested range is not satisfiable"; error_message(invalid_bucket_name) -> "The specified bucket is not valid."; error_message(not_implemented) -> "A request you provided implies functionality that is not implemented"; error_message(ErrorName) -> - _ = lager:debug("Unknown error: ~p", [ErrorName]), + logger:warning("Unknown error: ~p", [ErrorName]), "Please reduce your request rate.". -spec error_code(atom() | {'riak_connect_failed', term()}) -> string(). @@ -192,7 +193,7 @@ error_code(invalid_bucket_name) -> "InvalidBucketName"; error_code(unresolved_grant_email) -> "UnresolvableGrantByEmailAddress"; error_code(not_implemented) -> "NotImplemented"; error_code(ErrorName) -> - _ = lager:debug("Unknown error: ~p", [ErrorName]), + logger:warning("Unknown error: ~p", [ErrorName]), "ServiceUnavailable". %% These should match: @@ -234,5 +235,5 @@ status_code(invalid_range) -> 416; status_code(invalid_bucket_name) -> 400; status_code(not_implemented) -> 501; status_code(ErrorName) -> - _ = lager:debug("Unknown error: ~p", [ErrorName]), + logger:warning("Unknown error: ~p", [ErrorName]), 503. diff --git a/src/riak_cs_oos_rewrite.erl b/apps/riak_cs/src/riak_cs_oos_rewrite.erl similarity index 93% rename from src/riak_cs_oos_rewrite.erl rename to apps/riak_cs/src/riak_cs_oos_rewrite.erl index bb71729fa..0f1b97da3 100644 --- a/src/riak_cs_oos_rewrite.erl +++ b/apps/riak_cs/src/riak_cs_oos_rewrite.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -23,21 +24,19 @@ -export([rewrite/5, original_resource/1]). -include("riak_cs.hrl"). +-include("riak_cs_web.hrl"). --define(RCS_REWRITE_HEADER, "x-rcs-rewrite-path"). --define(OOS_API_VSN_HEADER, "x-oos-api-version"). --define(OOS_ACCOUNT_HEADER, "x-oos-account"). -ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). -compile(export_all). +-compile(nowarn_export_all). +-include_lib("eunit/include/eunit.hrl"). -endif. %% @doc Function to rewrite headers prior to processing by webmachine. -spec rewrite(atom(), atom(), {integer(), integer()}, mochiweb_headers(), string()) -> {mochiweb_headers(), string()}. rewrite(Method, _Scheme, _Vsn, Headers, RawPath) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"rewrite">>), {Path, QueryString, _} = mochiweb_util:urlsplit_path(RawPath), {ApiVsn, Account, RestPath} = parse_path(Path), RewrittenHeaders = rewrite_headers(Headers, RawPath, ApiVsn, Account), @@ -45,12 +44,8 @@ rewrite(Method, _Scheme, _Vsn, Headers, RawPath) -> -spec original_resource(term()) -> undefined | {string(), [{term(),term()}]}. original_resource(RD) -> - case wrq:get_req_header(?RCS_REWRITE_HEADER, RD) of - undefined -> undefined; - RawPath -> - {Path, QS, _} = mochiweb_util:urlsplit_path(RawPath), - {Path, mochiweb_util:parse_qs(QS)} - end. + riak_cs_rewrite:original_resource(RD). + %% @doc Parse the path string into its component parts: the API version, %% the account identifier, and the remainder of the path information diff --git a/src/riak_cs_oos_utils.erl b/apps/riak_cs/src/riak_cs_oos_utils.erl similarity index 93% rename from src/riak_cs_oos_utils.erl rename to apps/riak_cs/src/riak_cs_oos_utils.erl index ef272ec85..a9455d8df 100644 --- a/src/riak_cs_oos_utils.erl +++ b/apps/riak_cs/src/riak_cs_oos_utils.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -83,8 +84,7 @@ handle_user_data_response({ok, {{_HTTPVer, _Status, _StatusLine}, _, _}}) -> %% @TODO Log error undefined; handle_user_data_response({error, Reason}) -> - _ = lager:warning("Error occurred requesting user data from keystone. Reason: ~p", - [Reason]), + logger:warning("Error occurred requesting user data from keystone. Reason: ~p", [Reason]), undefined. handle_ec2_creds_response({ok, {{_HTTPVer, _Status, _StatusLine}, _, CredsInfo}}, TenantId) @@ -95,8 +95,7 @@ handle_ec2_creds_response({ok, {{_HTTPVer, _Status, _StatusLine}, _, _}}, _) -> %% @TODO Log error {undefined, []}; handle_ec2_creds_response({error, Reason}, _) -> - _ = lager:warning("Error occurred requesting user EC2 credentials from keystone. Reason: ~p", - [Reason]), + logger:warning("Error occurred requesting user EC2 credentials from keystone. Reason: ~p", [Reason]), {undefined, []}. ec2_creds_for_tenant({error, decode_failed}, _) -> @@ -109,7 +108,7 @@ ec2_creds_for_tenant(Creds, TenantId) -> {undefined, []} -> Res; {KeyBin, SecretBin} -> - {binary_to_list(KeyBin), binary_to_list(SecretBin)} + {KeyBin, SecretBin} end. %% =================================================================== diff --git a/src/riak_cs_pbc.erl b/apps/riak_cs/src/riak_cs_pbc.erl similarity index 75% rename from src/riak_cs_pbc.erl rename to apps/riak_cs/src/riak_cs_pbc.erl index 218c6c206..01ffdcf75 100644 --- a/src/riak_cs_pbc.erl +++ b/apps/riak_cs/src/riak_cs_pbc.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -23,7 +24,7 @@ -module(riak_cs_pbc). -export([ping/3, - get/6, + get/4, get/6, repl_get/7, put/4, put/5, @@ -47,12 +48,14 @@ list_keys_sans_stats/3 ]). +-include("riak_cs.hrl"). -include_lib("riakc/include/riakc.hrl"). +-include_lib("kernel/include/logger.hrl"). -define(WITH_STATS(StatsKey, Statement), begin _ = riak_cs_stats:inflow(StatsKey), - StartTime__with_stats = os:timestamp(), + StartTime__with_stats = os:system_time(millisecond), Result__with_stats = Statement, _ = riak_cs_stats:update_with_start(StatsKey, StartTime__with_stats, Result__with_stats), @@ -62,7 +65,7 @@ -spec ping(pid(), timeout(), riak_cs_stats:key()) -> pong. ping(PbcPid, Timeout, StatsKey) -> _ = riak_cs_stats:inflow(StatsKey), - StartTime = os:timestamp(), + StartTime = os:system_time(millisecond), Result = riakc_pb_socket:ping(PbcPid, Timeout), case Result of pong -> @@ -74,19 +77,56 @@ ping(PbcPid, Timeout, StatsKey) -> %% @doc Get an object from Riak -spec get_sans_stats(pid(), binary(), binary(), proplists:proplist(), timeout()) -> - {ok, riakc_obj:riakc_obj()} | {error, term()}. -get_sans_stats(PbcPid, BucketName, Key, Opts, Timeout) -> + {ok, riakc_obj:riakc_obj()} | {error, term()}. +get_sans_stats(PbcPid, BucketName, Key, Opts, Timeout) -> riakc_pb_socket:get(PbcPid, BucketName, Key, Opts, Timeout). --spec get(pid(), binary(), binary(), proplists:proplist(), timeout(), - riak_cs_stats:key()) -> - {ok, riakc_obj:riakc_obj()} | {error, term()}. + +%% @doc Perform an initial read attempt with R=PR=N. If the initial +%% read fails retry using R=quorum and PR=1, but indicate that read +%% has not been consistent. This can then be used by the caller (notably, +%% riak_cs_user:get_user, which in this case should not +%% clean up deleted buckets). +-spec get(pid(), binary(), binary(), proplists:proplist(), timeout(), riak_cs_stats:key()) -> + {ok, riakc_obj:riakc_obj()} | {error, term()}. get(PbcPid, BucketName, Key, Opts, Timeout, StatsKey) -> ?WITH_STATS(StatsKey, get_sans_stats(PbcPid, BucketName, Key, Opts, Timeout)). +-spec get(pid(), binary(), binary(), atom()) -> + {ok | weak_ok, riakc_obj:riakc_obj()} | {error, term()}. +get(Pbc, Bucket, Key, StatsItem) -> + Timeout = riak_cs_config:get_bucket_timeout(), + Options = + case riak_cs_config:fast_user_get() of + true -> + ?WEAK_READ_OPTIONS; + false -> + ?CONSISTENT_READ_OPTIONS + end, + case get(Pbc, Bucket, Key, Options, Timeout, + [riakc, StatsItem]) of + {ok, Obj} -> + {ok, Obj}; + {error, <<"{pr_val_unsatisfied,", _/binary>>} + when Options == ?CONSISTENT_READ_OPTIONS -> + logger:notice("Failed to read key ~s from bucket ~s with consistent options; " + "retrying with weak options", + [Key, Bucket]), + case get(Pbc, Bucket, Key, ?WEAK_READ_OPTIONS, Timeout, [riakc, StatsItem]) of + {ok, Obj} -> + {weak_ok, Obj}; + ER -> + ER + end; + {error, Reason} -> + {error, Reason} + end. + + + -spec repl_get(pid(), binary(), binary(), binary(), - proplists:proplist(), timeout(), riak_cs_stats:key()) -> - {ok, riakc_obj:riakc_obj()} | {error, term()}. + proplists:proplist(), timeout(), riak_cs_stats:key()) -> + {ok, riakc_obj:riakc_obj()} | {error, term()}. repl_get(PbcPid, BucketName, Key, ClusterID, Opts, Timeout, StatsKey) -> ?WITH_STATS(StatsKey, riak_repl_pb_api:get(PbcPid, BucketName, Key, ClusterID, Opts, Timeout)). @@ -95,9 +135,9 @@ repl_get(PbcPid, BucketName, Key, ClusterID, Opts, Timeout, StatsKey) -> %% TODO: two `put_object' are without stats yet. -spec put_object(pid(), binary(), undefined | binary(), binary(), [term()]) -> ok | {error, term()}. put_object(_PbcPid, BucketName, undefined, Value, Metadata) -> - error_logger:warning_msg("Attempt to put object into ~p with undefined key " - "and value ~P and dict ~p\n", - [BucketName, Value, 30, Metadata]), + logger:warning("Attempt to put object into ~p with undefined key " + "and value ~P and dict ~p", + [BucketName, Value, Metadata]), {error, bad_key}; put_object(PbcPid, BucketName, Key, Value, Metadata) -> RiakObject = riakc_obj:new(BucketName, Key, Value), @@ -178,6 +218,8 @@ get_index_range(PbcPid, Bucket, Index, StartKey, EndKey, Opts, StatsKey) -> riakc_pb_socket:get_index_range(PbcPid, Bucket, Index, StartKey, EndKey, Opts)). +-dialyzer({[no_match], get_cluster_id/1}). + %% @doc Attempt to determine the cluster id -spec get_cluster_id(pid()) -> undefined | binary(). get_cluster_id(Pbc) -> @@ -188,17 +230,10 @@ get_cluster_id(Pbc) -> {error, _} -> undefined end. +-spec get_cluster_id_sans_stats(pid()) -> {ok, binary()} | {error, term()}. get_cluster_id_sans_stats(Pbc) -> Timeout = riak_cs_config:cluster_id_timeout(), - try - riak_repl_pb_api:get_clusterid(Pbc, Timeout) - catch C:R -> - %% Disable `proxy_get' so we do not repeatedly have to - %% handle this same exception. This would happen if an OSS - %% install has `proxy_get' enabled. - application:set_env(riak_cs, proxy_get, disabled), - {error, {C, R}} - end. + riak_repl_pb_api:get_clusterid(Pbc, Timeout). %% @doc don't reuse return value -spec check_connection_status(pid(), term()) -> any(). @@ -207,13 +242,11 @@ check_connection_status(Pbc, Where) -> case riakc_pb_socket:is_connected(Pbc) of true -> ok; Other -> - _ = lager:warning("Connection status of ~p at ~p: ~p", - [Pbc, Where, Other]) + logger:warning("Connection status of ~p at ~p: ~p", [Pbc, Where, Other]) end catch Type:Error -> - _ = lager:warning("Connection status of ~p at ~p: ~p", - [Pbc, Where, {Type, Error}]) + logger:warning("Connection status of ~p at ~p: ~p", [Pbc, Where, {Type, Error}]) end. %% @doc Pause for a while so that underlying `riaic_pb_socket' can have @@ -225,16 +258,16 @@ check_connection_status(Pbc, Where) -> -spec pause_to_reconnect(pid(), term(), non_neg_integer()) -> ok. pause_to_reconnect(Pbc, Reason, Timeout) when Reason =:= timeout orelse Reason =:= disconnected -> - pause_to_reconnect0(Pbc, Timeout, os:timestamp()); + pause_to_reconnect0(Pbc, Timeout, os:system_time(millisecond)); pause_to_reconnect(_Pbc, _Other, _Timeout) -> ok. pause_to_reconnect0(Pbc, Timeout, Start) -> - lager:debug("riak_cs_pbc:pause_to_reconnect0"), + logger:info("pausing to reconnect to riak"), case riakc_pb_socket:is_connected(Pbc, ?FIRST_RECONNECT_INTERVAL) of true -> ok; {false, _} -> - Remaining = Timeout - timer:now_diff(os:timestamp(), Start) div 1000, + Remaining = Timeout - (os:system_time(millisecond) - Start), case Remaining of Positive when 0 < Positive -> %% sleep to avoid busy-loop calls of `is_connected' diff --git a/src/riak_cs_policy.erl b/apps/riak_cs/src/riak_cs_policy.erl similarity index 89% rename from src/riak_cs_policy.erl rename to apps/riak_cs/src/riak_cs_policy.erl index 5a4db7a7b..e34ead5a4 100644 --- a/src/riak_cs_policy.erl +++ b/apps/riak_cs/src/riak_cs_policy.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -25,7 +26,7 @@ -callback fetch_bucket_policy(binary(), pid()) -> {ok, policy()} | {error, term()}. -callback bucket_policy(riakc_obj:riakc_obj()) -> {ok, policy()} | {error, term()}. -callback eval(access(), policy() | undefined | binary() ) -> boolean() | undefined. --callback check_policy(access(), policy()) -> ok | {error, atom()}. +-callback check_policy(access(), policy()) -> ok | {error, reportable_error_reason()}. -callback reqdata_to_access(RD :: term(), Target::atom(), ID::binary()|undefined) -> access(). -callback policy_from_json(JSON::binary()) -> {ok, policy()} | {error, term()}. -callback policy_to_json_term(policy()) -> JSON::binary(). diff --git a/src/riak_cs_put_fsm.erl b/apps/riak_cs/src/riak_cs_put_fsm.erl similarity index 78% rename from src/riak_cs_put_fsm.erl rename to apps/riak_cs/src/riak_cs_put_fsm.erl index 3646e2b50..229131018 100644 --- a/src/riak_cs_put_fsm.erl +++ b/apps/riak_cs/src/riak_cs_put_fsm.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -24,8 +25,6 @@ -behaviour(gen_fsm). --include("riak_cs.hrl"). - %% API -export([start_link/1, start_link/3, get_uuid/1, @@ -50,6 +49,9 @@ terminate/3, code_change/4]). +-include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). + -define(SERVER, ?MODULE). -define(EMPTYORDSET, ordsets:new()). -define(MD5_CHUNK_SIZE, 64*1024). @@ -58,23 +60,24 @@ block_size :: pos_integer(), caller :: reference(), uuid :: binary(), - md5 :: crypto_context() | digest(), - reported_md5 :: undefined | string(), - reply_pid :: {pid(), reference()}, + md5 :: crypto:hash_state() | binary(), + reported_md5 :: undefined | binary(), + reply_pid :: undefined | {pid(), reference()}, riak_client :: riak_client(), mani_pid :: undefined | pid(), make_new_manifest_p :: boolean(), bag_id :: bag_id(), - timer_ref :: reference(), + timer_ref :: undefined | reference(), bucket :: binary(), key :: binary(), + obj_vsn :: binary(), metadata :: term(), acl :: acl(), - manifest :: lfs_manifest(), + manifest :: undefined | lfs_manifest(), content_length :: non_neg_integer(), content_type :: binary(), num_bytes_received=0 :: non_neg_integer(), - max_buffer_size :: non_neg_integer(), + max_buffer_size :: undefined | non_neg_integer(), current_buffer_size=0 :: non_neg_integer(), buffer_queue=[] :: [binary()], %% not actually a queue, but we treat it like one remainder_data :: undefined | binary(), @@ -87,18 +90,19 @@ %%% API %%%=================================================================== --spec start_link({binary(), binary(), non_neg_integer(), binary(), +-spec start_link({binary(), binary(), binary(), non_neg_integer(), binary(), term(), pos_integer(), acl(), timeout(), pid(), riak_client()}) -> {ok, pid()} | {error, term()}. start_link(Tuple) when is_tuple(Tuple) -> start_link(Tuple, true, undefined). --spec start_link({binary(), binary(), non_neg_integer(), binary(), +-spec start_link({binary(), binary(), binary(), non_neg_integer(), binary(), term(), pos_integer(), acl(), timeout(), pid(), riak_client()}, boolean(), bag_id()) -> {ok, pid()} | {error, term()}. start_link({_Bucket, _Key, + _OVN, _ContentLength, _ContentType, _Metadata, @@ -106,7 +110,7 @@ start_link({_Bucket, _Acl, _Timeout, _Caller, - _RcPid}=Arg1, + _RcPid} = Arg1, MakeNewManifestP, BagId) -> gen_fsm:start_link(?MODULE, {Arg1, MakeNewManifestP, BagId}, []). @@ -141,11 +145,11 @@ block_written(Pid, BlockID) -> %% so that I can be thinking about how it %% might be implemented. Does it actually %% make things more confusing? --spec init({{binary(), binary(), non_neg_integer(), binary(), +-spec init({{binary(), binary(), binary(), non_neg_integer(), binary(), term(), pos_integer(), acl(), timeout(), pid(), riak_client()}, boolean(), bag_id()}) -> {ok, prepare, #state{}, timeout()}. -init({{Bucket, Key, ContentLength, ContentType, +init({{Bucket, Key, ObjVsn, ContentLength, ContentType, Metadata, BlockSize, Acl, Timeout, Caller, RcPid}, MakeNewManifestP, BagId0}) -> %% We need to do this (the monitor) for two reasons @@ -158,60 +162,63 @@ init({{Bucket, Key, ContentLength, ContentType, %% live. CallerRef = erlang:monitor(process, Caller), - UUID = druuid:v4(), + Md5 = riak_cs_utils:md5_init(), + UUID = uuid:get_v4(), BagId = case BagId0 of undefined -> riak_cs_mb_helper:choose_bag_id(block, {Bucket, Key, UUID}); _ -> BagId0 end, - {ok, prepare, #state{bucket=Bucket, - key=Key, - block_size=BlockSize, - caller=CallerRef, - uuid=UUID, - metadata=Metadata, - acl=Acl, - content_length=ContentLength, - content_type=ContentType, - riak_client=RcPid, - make_new_manifest_p=MakeNewManifestP, - bag_id=BagId, - timeout=Timeout}, + {ok, prepare, #state{bucket = Bucket, + key = Key, + obj_vsn = ObjVsn, + md5 = Md5, + block_size = BlockSize, + caller = CallerRef, + uuid = UUID, + metadata = Metadata, + acl = Acl, + content_length = ContentLength, + content_type = ContentType, + riak_client = RcPid, + make_new_manifest_p = MakeNewManifestP, + bag_id = BagId, + timeout = Timeout}, 0}. %%-------------------------------------------------------------------- %% %%-------------------------------------------------------------------- -prepare(timeout, State=#state{content_length=0}) -> +prepare(timeout, State = #state{content_length = 0}) -> NewState = prepare(State), Md5 = riak_cs_utils:md5_final(NewState#state.md5), - NewManifest = NewState#state.manifest?MANIFEST{content_md5=Md5, - state=active, - last_block_written_time=os:timestamp()}, - {next_state, done, NewState#state{md5=Md5, manifest=NewManifest}}; + NewManifest = NewState#state.manifest?MANIFEST{content_md5 = Md5, + state = active, + last_block_written_time = os:system_time(millisecond)}, + {next_state, done, NewState#state{md5 = Md5, manifest = NewManifest}}; prepare(timeout, State) -> NewState = prepare(State), {next_state, not_full, NewState}. -prepare({finalize, ContentMD5}, From, State=#state{content_length=0}) -> +prepare({finalize, ContentMD5}, From, State = #state{content_length = 0}) -> NewState = prepare(State), Md5 = riak_cs_utils:md5_final(NewState#state.md5), - NewManifest = NewState#state.manifest?MANIFEST{content_md5=Md5, - state=active, - last_block_written_time=os:timestamp()}, - done(finalize, From, NewState#state{md5=Md5, - manifest=NewManifest, - reported_md5=ContentMD5}); + NewManifest = NewState#state.manifest?MANIFEST{content_md5 = Md5, + state = active, + last_block_written_time = os:system_time(millisecond)}, + done(finalize, From, NewState#state{md5 = Md5, + manifest = NewManifest, + reported_md5 = ContentMD5}); prepare({get_uuid}, _From, State) -> {reply, State#state.uuid, prepare, State}; prepare({augment_data, NewData}, From, State) -> - #state{content_length=CLength, - num_bytes_received=NumBytesReceived, - current_buffer_size=CurrentBufferSize, - max_buffer_size=MaxBufferSize} = NewState = prepare(State), + #state{content_length = CLength, + num_bytes_received = NumBytesReceived, + current_buffer_size = CurrentBufferSize, + max_buffer_size = MaxBufferSize} = NewState = prepare(State), case handle_chunk(CLength, NumBytesReceived, size(NewData), CurrentBufferSize, MaxBufferSize) of accept -> @@ -229,7 +236,7 @@ not_full({block_written, BlockID, WriterPid}, State) -> NewState = state_from_block_written(BlockID, WriterPid, State), {next_state, not_full, NewState}. -full({block_written, BlockID, WriterPid}, State=#state{reply_pid=Waiter}) -> +full({block_written, BlockID, WriterPid}, State = #state{reply_pid = Waiter}) -> NewState = state_from_block_written(BlockID, WriterPid, State), gen_fsm:reply(Waiter, ok), {next_state, not_full, NewState#state{reply_pid=undefined}}. @@ -238,8 +245,8 @@ all_received({augment_data, <<>>}, State) -> {next_state, all_received, State}; all_received({block_written, BlockID, WriterPid}, State) -> NewState = state_from_block_written(BlockID, WriterPid, State), - Manifest = NewState#state.manifest?MANIFEST{state=active}, - NewState2 = NewState#state{manifest=Manifest}, + Manifest = NewState#state.manifest?MANIFEST{state = active}, + NewState2 = NewState#state{manifest = Manifest}, case ordsets:size(NewState#state.unacked_writes) of 0 -> case State#state.reply_pid of @@ -266,10 +273,10 @@ not_full({get_uuid}, _From, State) -> {reply, State#state.uuid, not_full, State}; not_full({augment_data, NewData}, From, - State=#state{content_length=CLength, - num_bytes_received=NumBytesReceived, - current_buffer_size=CurrentBufferSize, - max_buffer_size=MaxBufferSize}) -> + State = #state{content_length = CLength, + num_bytes_received = NumBytesReceived, + current_buffer_size = CurrentBufferSize, + max_buffer_size = MaxBufferSize}) -> case handle_chunk(CLength, NumBytesReceived, size(NewData), CurrentBufferSize, MaxBufferSize) of @@ -287,33 +294,33 @@ all_received({finalize, ContentMD5}, From, State) -> %% 1. stash the From pid into our %% state so that we know to reply %% later with the finished manifest - {next_state, all_received, State#state{reply_pid=From, - reported_md5=ContentMD5}}. + {next_state, all_received, State#state{reply_pid = From, + reported_md5 = ContentMD5}}. -done({finalize, ReportedMD5}, _From, State=#state{md5=MD5}) -> +done({finalize, ReportedMD5}, _From, State = #state{md5 = MD5}) -> done(finalize, is_digest_valid(MD5, ReportedMD5), _From, State); -done(finalize, _From, State=#state{md5=MD5, - reported_md5=ReportedMD5}) -> +done(finalize, _From, State = #state{md5 = MD5, + reported_md5 = ReportedMD5}) -> done(finalize, is_digest_valid(MD5, ReportedMD5), _From, State). -done(finalize, false, From, State=#state{manifest=Manifest, - mani_pid=ManiPid, - timer_ref=TimerRef}) -> +done(finalize, false, From, State = #state{manifest = Manifest, + mani_pid = ManiPid, + timer_ref = TimerRef}) -> _ = erlang:cancel_timer(TimerRef), %% reset the state from `active' to `pending_delete', this means %% that it's never persisted in the active state Props = Manifest?MANIFEST.props, - NotActive = Manifest?MANIFEST{state=pending_delete, - props=[{bad_checksum, true} | Props]}, + NotActive = Manifest?MANIFEST{state = pending_delete, + props = [{bad_checksum, true} | Props]}, ok = maybe_update_manifest_with_confirmation(ManiPid, NotActive), gen_fsm:reply(From, {error, invalid_digest}), - _ = lager:debug("Invalid digest in the PUT FSM"), + logger:warning("Invalid digest in the PUT FSM"), {stop, normal, State}; -done(finalize, true, From, State=#state{manifest=Manifest, - mani_pid=ManiPid, - timer_ref=TimerRef}) -> +done(finalize, true, From, State = #state{manifest = Manifest, + mani_pid = ManiPid, + timer_ref = TimerRef}) -> %% 1. reply immediately with the finished manifest _ = erlang:cancel_timer(TimerRef), case maybe_update_manifest_with_confirmation(ManiPid, Manifest) of @@ -325,16 +332,14 @@ done(finalize, true, From, State=#state{manifest=Manifest, {stop, Error, State} end. --spec is_digest_valid(binary(), undefined | string()) -> boolean(). -is_digest_valid(D1, undefined) -> +is_digest_valid(_D1, undefined) -> %% reported MD5 is not in request header - _ = lager:debug("Calculated = ~p, Reported = undefined~n", [D1]), + %% ?LOG_DEBUG("Calculated = ~p, Reported = undefined", [_D1]), true; -is_digest_valid(CalculatedMD5, ReportedMD5) -> - StringCalculatedMD5 = base64:encode(CalculatedMD5), - _ = lager:debug("Calculated = ~p, Reported = ~p~n", - [StringCalculatedMD5, ReportedMD5]), - StringCalculatedMD5 =:= list_to_binary(ReportedMD5). +is_digest_valid(CalculatedMD5Raw, ReportedMD5) -> + CalculatedMD5 = base64:encode(CalculatedMD5Raw), + %% ?LOG_DEBUG("Calculated = ~p, Reported = ~p", [CalculatedMD5, ReportedMD5]), + CalculatedMD5 =:= ReportedMD5. %%-------------------------------------------------------------------- %% @@ -348,10 +353,10 @@ handle_event(_Event, StateName, State) -> handle_sync_event(current_state, _From, StateName, State) -> Reply = {StateName, State}, {reply, Reply, StateName, State}; -handle_sync_event(force_stop, _From, _StateName, State = #state{mani_pid=ManiPid, - uuid=UUID}) -> +handle_sync_event(force_stop, _From, _StateName, State = #state{mani_pid = ManiPid, + uuid = UUID}) -> Res = riak_cs_manifest_fsm:gc_specific_manifest(ManiPid, UUID), - lager:debug("Manifest collection on upload failure: ~p", [Res]), + ?LOG_DEBUG("Manifest collection on upload failure: ~p", [Res]), {stop, normal, Res, State}; handle_sync_event(_Event, _From, StateName, State) -> Reply = ok, @@ -360,8 +365,8 @@ handle_sync_event(_Event, _From, StateName, State) -> %%-------------------------------------------------------------------- %% %%-------------------------------------------------------------------- -handle_info(save_manifest, StateName, State=#state{mani_pid=ManiPid, - manifest=Manifest}) -> +handle_info(save_manifest, StateName, State=#state{mani_pid = ManiPid, + manifest = Manifest}) -> %% 1. save the manifest maybe_update_manifest(ManiPid, Manifest), TRef = erlang:send_after(60000, self(), save_manifest), @@ -369,14 +374,14 @@ handle_info(save_manifest, StateName, State=#state{mani_pid=ManiPid, %% TODO: %% add a clause for handling down %% messages from the blocks gen_servers -handle_info({'DOWN', CallerRef, process, _Pid, Reason}, _StateName, State=#state{caller=CallerRef}) -> +handle_info({'DOWN', CallerRef, process, _Pid, Reason}, _StateName, State = #state{caller = CallerRef}) -> {stop, Reason, State}. %%-------------------------------------------------------------------- %% %%-------------------------------------------------------------------- -terminate(_Reason, _StateName, #state{mani_pid=ManiPid, - all_writer_pids=BlockServerPids}) -> +terminate(_Reason, _StateName, #state{mani_pid = ManiPid, + all_writer_pids = BlockServerPids}) -> riak_cs_manifest_fsm:maybe_stop_manifest_fsm(ManiPid), riak_cs_block_server:maybe_stop_block_servers(BlockServerPids), ok. @@ -393,38 +398,33 @@ code_change(_OldVsn, StateName, State, _Extra) -> %% @doc Handle expensive initialization operations required for the put_fsm. -spec prepare(#state{}) -> #state{}. -prepare(State=#state{bucket=Bucket, - key=Key, - block_size=BlockSize, - uuid=UUID, - content_length=ContentLength, - content_type=ContentType, - metadata=Metadata, - acl=Acl, - riak_client=RcPid, - make_new_manifest_p=MakeNewManifestP, - bag_id=BagId}) +prepare(State=#state{bucket = Bucket, + key = Key, + obj_vsn = Vsn, + block_size = BlockSize, + uuid = UUID, + content_length = ContentLength, + content_type = ContentType, + metadata = Metadata, + acl = Acl, + riak_client = RcPid, + make_new_manifest_p = MakeNewManifestP, + bag_id = BagId}) when is_integer(ContentLength), ContentLength >= 0 -> + ClusterID = riak_cs_mb_helper:cluster_id(BagId), + + %% 0. prepare manifest + Manifest1 = riak_cs_lfs_utils:new_manifest( + Bucket, Key, Vsn, UUID, + ContentLength, ContentType, undefined, %% we don't know the md5 yet + Metadata, BlockSize, Acl, [], ClusterID, BagId), + %% 1. start the manifest_fsm proc {ok, ManiPid} = maybe_riak_cs_manifest_fsm_start_link( - MakeNewManifestP, Bucket, Key, RcPid), - ClusterID = riak_cs_mb_helper:cluster_id(BagId), - Manifest = riak_cs_lfs_utils:new_manifest(Bucket, - Key, - UUID, - ContentLength, - ContentType, - %% we don't know the md5 yet - undefined, - Metadata, - BlockSize, - Acl, - [], - ClusterID, - BagId), - NewManifest = Manifest?MANIFEST{write_start_time=os:timestamp()}, + MakeNewManifestP, Bucket, Key, Vsn, RcPid), + + Manifest2 = Manifest1?MANIFEST{write_start_time = os:system_time(millisecond)}, - Md5 = riak_cs_utils:md5_init(), WriterPids = case ContentLength of 0 -> %% Don't start any writers @@ -433,7 +433,7 @@ prepare(State=#state{bucket=Bucket, []; _ -> riak_cs_block_server:start_block_servers( - NewManifest, + Manifest2, RcPid, riak_cs_lfs_utils:put_concurrency()) end, @@ -446,14 +446,16 @@ prepare(State=#state{bucket=Bucket, %% and if it is, what should %% it be? TRef = erlang:send_after(60000, self(), save_manifest), - ok = maybe_add_new_manifest(ManiPid, NewManifest), - State#state{manifest=NewManifest, - md5=Md5, - timer_ref=TRef, - mani_pid=ManiPid, - max_buffer_size=MaxBufferSize, - all_writer_pids=WriterPids, - free_writers=FreeWriters}. + ok = maybe_add_new_manifest(ManiPid, Manifest2), + State#state{manifest = Manifest2, + %% possibly null, ignoring user-supplied Vsn0 unless a + %% primary version already exists + obj_vsn = Vsn, + timer_ref = TRef, + mani_pid = ManiPid, + max_buffer_size = MaxBufferSize, + all_writer_pids = WriterPids, + free_writers = FreeWriters}. handle_chunk(ContentLength, NumBytesReceived, NewDataSize, CurrentBufferSize, MaxBufferSize) -> if @@ -643,10 +645,10 @@ handle_receiving_last_chunk(NewData, State=#state{buffer_queue=BufferQueue, Reply = ok, {reply, Reply, all_received, NewStateData}. -maybe_riak_cs_manifest_fsm_start_link(false, _Bucket, _Key, _RcPid) -> +maybe_riak_cs_manifest_fsm_start_link(false, _Bucket, _Key, _ObjVsn, _RcPid) -> {ok, undefined}; -maybe_riak_cs_manifest_fsm_start_link(true, Bucket, Key, RcPid) -> - riak_cs_manifest_fsm:start_link(Bucket, Key, RcPid). +maybe_riak_cs_manifest_fsm_start_link(true, Bucket, Key, ObjVsn, RcPid) -> + riak_cs_manifest_fsm:start_link(Bucket, Key, ObjVsn, RcPid). maybe_add_new_manifest(undefined, _NewManifest) -> ok; diff --git a/src/riak_cs_put_fsm_sup.erl b/apps/riak_cs/src/riak_cs_put_fsm_sup.erl similarity index 72% rename from src/riak_cs_put_fsm_sup.erl rename to apps/riak_cs/src/riak_cs_put_fsm_sup.erl index 9a70d1435..419cbf1c4 100644 --- a/src/riak_cs_put_fsm_sup.erl +++ b/apps/riak_cs/src/riak_cs_put_fsm_sup.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -45,9 +46,9 @@ start_link() -> %% @doc Start a `riak_cs_put_fsm' child process. -spec start_put_fsm(node(), - [{binary(), binary(), non_neg_integer(), binary(), - term(), pos_integer(), acl(), timeout(), pid(), pid()}])-> - {ok, pid()} | {error, term()}. + [{binary(), binary(), binary(), non_neg_integer(), binary(), + term(), pos_integer(), acl(), timeout(), pid(), pid()}]) -> + {ok, pid()} | {error, term()}. start_put_fsm(Node, ArgList) -> supervisor:start_child({?MODULE, Node}, ArgList). @@ -57,23 +58,15 @@ start_put_fsm(Node, ArgList) -> %% @doc Initialize this supervisor. This is a `simple_one_for_one', %% whose child spec is for starting `riak_cs_put_fsm' processes. --spec init([]) -> {ok, {{supervisor:strategy(), - pos_integer(), - pos_integer()}, - [supervisor:child_spec()]}}. +-spec init([]) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. init([]) -> - RestartStrategy = simple_one_for_one, - MaxRestarts = 1000, - MaxSecondsBetweenRestarts = 3600, - - SupFlags = {RestartStrategy, MaxRestarts, MaxSecondsBetweenRestarts}, - - Restart = temporary, - Shutdown = 2000, - Type = worker, - - PutFsmSpec = {undefined, - {riak_cs_put_fsm, start_link, []}, - Restart, Shutdown, Type, [riak_cs_put_fsm]}, + SupFlags = #{strategy => simple_one_for_one, + intensity => 1000, + period => 3600}, + + PutFsmSpec = #{id => put_fsm, + start => {riak_cs_put_fsm, start_link, []}, + restart => temporary, + shutdown => 2000}, {ok, {SupFlags, [PutFsmSpec]}}. diff --git a/src/riak_cs_quota.erl b/apps/riak_cs/src/riak_cs_quota.erl similarity index 90% rename from src/riak_cs_quota.erl rename to apps/riak_cs/src/riak_cs_quota.erl index cadd6cccd..dd6a7d1d0 100644 --- a/src/riak_cs_quota.erl +++ b/apps/riak_cs/src/riak_cs_quota.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -70,8 +71,9 @@ handle_error/4]). -include("riak_cs.hrl"). +-include("riak_cs_web.hrl"). -include_lib("webmachine/include/webmachine_logger.hrl"). --include_lib("webmachine/include/wm_reqdata.hrl"). +-include_lib("kernel/include/logger.hrl"). %% Callbacks as a behaviour @@ -91,13 +93,13 @@ | string()|binary(), %% Other authenticated user Access :: access(), %% including accessor, bucket, key, RD Context::term()) -> - {ok, Request::#wm_reqdata{}, Context::#context{}} | + {ok, Request::#wm_reqdata{}, Context::#rcs_web_context{}} | {error, Reason::term()}. -callback error_response(Reason :: term(), - #wm_reqdata{}, #context{}) -> + #wm_reqdata{}, #rcs_web_context{}) -> {non_neg_integer() | {non_neg_integer(), string()}, - #wm_reqdata{}, #context{}}. + #wm_reqdata{}, #rcs_web_context{}}. %% @doc for now, to know the accessor, RD should be retrieved from %% headers in LogData (not knowing whether he's authenticated or not) @@ -117,19 +119,19 @@ invoke_all_callbacks(Owner, Access, Ctx0) -> {ok, _, _} = R-> R; {error, Reason} -> - lager:debug("quota check failed at ~p: ~p", [Module, Reason]), + logger:info("quota check failed at ~p: ~p", [Module, Reason]), {error, Module, Reason, RD, Ctx} end; (_, Other) -> Other end, {ok, Access#access_v1.req, Ctx0}, Modules). --spec update_all_states(iolist(), #wm_log_data{}) -> no_return(). +-spec update_all_states(binary(), #wm_log_data{}) -> no_return(). update_all_states(User, LogData) -> Modules = riak_cs_config:quota_modules(), [begin - lager:debug("quota update at ~p: ~p", [Module, User]), - (catch Module:update(list_to_binary(User), LogData)) + ?LOG_DEBUG("quota update at ~p: ~p", [Module, User]), + (catch Module:update(User, LogData)) end || Module <- Modules]. handle_error(Module, Reason, RD0, Ctx0) -> diff --git a/src/riak_cs_quota_sup.erl b/apps/riak_cs/src/riak_cs_quota_sup.erl similarity index 95% rename from src/riak_cs_quota_sup.erl rename to apps/riak_cs/src/riak_cs_quota_sup.erl index 6db2eb3a5..7802b1c01 100644 --- a/src/riak_cs_quota_sup.erl +++ b/apps/riak_cs/src/riak_cs_quota_sup.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/apps/riak_cs/src/riak_cs_rewrite.erl b/apps/riak_cs/src/riak_cs_rewrite.erl new file mode 100644 index 000000000..0edb17e8b --- /dev/null +++ b/apps/riak_cs/src/riak_cs_rewrite.erl @@ -0,0 +1,67 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_rewrite). + +-callback rewrite(Method::atom(), Scheme::atom(), Vsn::{integer(), integer()}, + mochiweb_headers(), Url::string()) -> + {mochiweb_headers(), string()}. + +-export([rewrite/5, + original_resource/1, + raw_url/1 + ]). + +-include("riak_cs.hrl"). +-include("riak_cs_web.hrl"). +-include_lib("kernel/include/logger.hrl"). + + +%% no-rewrite, for calls directly to riak_cs +-spec rewrite(Method::atom(), Scheme::atom(), Vsn::{integer(), integer()}, + mochiweb_headers(), Url::string()) -> + {mochiweb_headers(), string()}. +rewrite(_Method, _Scheme, _Vsn, Headers, Url) -> + {Path, _QS, _} = mochiweb_util:urlsplit_path(Url), + RewrittenHeaders = mochiweb_headers:default( + ?RCS_RAW_URL_HEADER, Url, Headers), + {RewrittenHeaders, Path}. + + +-spec original_resource(#wm_reqdata{}) -> {string(), [{term(), term()}]}. +original_resource(RD) -> + RawPath = + case wrq:get_req_header(?RCS_REWRITE_HEADER, RD) of + undefined -> + wrq:raw_path(RD); + A -> + A + end, + {Path, QS, _} = mochiweb_util:urlsplit_path(RawPath), + {Path, mochiweb_util:parse_qs(QS)}. + +-spec raw_url(#wm_reqdata{}) -> undefined | {string(), [{term(), term()}]}. +raw_url(RD) -> + case wrq:get_req_header(?RCS_RAW_URL_HEADER, RD) of + undefined -> undefined; + RawUrl -> + {Path, QS, _} = mochiweb_util:urlsplit_path(RawUrl), + {Path, mochiweb_util:parse_qs(QS)} + end. diff --git a/src/riak_cs_riak_client.erl b/apps/riak_cs/src/riak_cs_riak_client.erl similarity index 64% rename from src/riak_cs_riak_client.erl rename to apps/riak_cs/src/riak_cs_riak_client.erl index c0c961fb7..e157ffe16 100644 --- a/src/riak_cs_riak_client.erl +++ b/apps/riak_cs/src/riak_cs_riak_client.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -32,8 +33,6 @@ stop/1, get_bucket/2, set_bucket_name/2, - get_user/2, - save_user/3, set_manifest_bag/2, get_manifest_bag/1, set_manifest/2, @@ -48,13 +47,12 @@ terminate/2, code_change/3]). %% exported for other `riak_client' implementations --export([get_bucket_with_pbc/2, - get_user_with_pbc/2, - save_user_with_pbc/3]). +-export([get_bucket_with_pbc/2]). -include("riak_cs.hrl"). -include_lib("riak_pb/include/riak_pb_kv_codec.hrl"). -include_lib("riakc/include/riakc.hrl"). +-include_lib("kernel/include/logger.hrl"). -define(SERVER, ?MODULE). @@ -65,17 +63,12 @@ }). start_link(_Args) -> - case application:get_env(riak_cs, riak_client) of - {ok, Mod} -> - gen_server:start_link(Mod, [], []); - undefined -> - Mod = case riak_cs_config:is_multibag_enabled() of - true -> riak_cs_multibag_riak_client; - false -> ?MODULE - end, - application:set_env(riak_cs, riak_client, Mod), - gen_server:start_link(Mod, [], []) - end. + Mod = case riak_cs_config:is_multibag_enabled() of + true -> riak_cs_multibag_riak_client; + false -> ?MODULE + end, + application:set_env(riak_cs, riak_client, Mod), + gen_server:start_link(Mod, [], []). stop(Pid) -> gen_server:call(Pid, stop). @@ -120,16 +113,16 @@ pbc_pool_name(BagId) when is_binary(BagId) -> list_to_atom(lists:flatten(io_lib:format("pbc_pool_~s", [BagId]))). %% @doc Make a thunk that looks up samples for a given bucket and suffix. --spec rts_puller(riak_client(), binary(), iolist(), riak_cs_stats:key()) -> fun(). +-spec rts_puller(riak_client(), binary(), binary(), riak_cs_stats:key()) -> fun(). rts_puller(RcPid, Bucket, Suffix, StatsKey) -> fun(Slice, {Samples, Errors}) -> - {ok, MasterPbc} = riak_cs_riak_client:master_pbc(RcPid), + {ok, MasterPbc} = master_pbc(RcPid), Timeout = riak_cs_config:get_access_timeout(), case riak_cs_pbc:get(MasterPbc, Bucket, rts:slice_key(Slice, Suffix), [], Timeout, StatsKey) of {ok, Object} -> RawSamples = - [ catch element(2, {struct,_}=mochijson2:decode(V)) + [ jsx:decode(V, [{return_maps, false}]) || V <- riakc_obj:get_values(Object) ], {NewSamples, EncodingErrors} = lists:partition(fun({'EXIT',_}) -> false; @@ -157,22 +150,6 @@ get_bucket(RcPid, BucketName) when is_binary(BucketName) -> set_bucket_name(RcPid, BucketName) when is_binary(BucketName) -> gen_server:call(RcPid, {set_bucket_name, BucketName}, infinity). -%% @doc Perform an initial read attempt with R=PR=N. -%% If the initial read fails retry using -%% R=quorum and PR=1, but indicate that bucket deletion -%% indicators should not be cleaned up. --spec get_user(riak_client(), - UserKey :: binary()) -> - {ok, {riakc_obj:riakc_obj(), KeepDeletedBuckets :: boolean()}} | - {error, term()}. -get_user(RcPid, UserKey) when is_binary(UserKey) -> - gen_server:call(RcPid, {get_user, UserKey}, infinity). - --spec save_user(riak_client(), rcs_user(), riakc_obj:riakc_obj()) -> ok | {error, term()}. -save_user(RcPid, User, OldUserObj) -> - gen_server:call(RcPid, {save_user, User, OldUserObj}, infinity). - - -spec set_manifest(riak_client(), lfs_manifest()) -> ok | {error, term()}. set_manifest(RcPid, Manifest) -> gen_server:call(RcPid, {set_manifest, {Manifest?MANIFEST.uuid, Manifest}}). @@ -221,22 +198,6 @@ handle_call({get_bucket, BucketName}, _From, State) -> end; handle_call({set_bucket_name, _BucketName}, _From, State) -> {reply, ok, State}; -handle_call({get_user, UserKey}, _From, State) -> - case ensure_master_pbc(State) of - {ok, #state{master_pbc=MasterPbc} = NewState} -> - Res = get_user_with_pbc(MasterPbc, UserKey), - {reply, Res, NewState}; - {error, Reason} -> - {reply, {error, Reason}, State} - end; -handle_call({save_user, User, OldUserObj}, _From, State) -> - case ensure_master_pbc(State) of - {ok, #state{master_pbc=MasterPbc} = NewState} -> - Res = save_user_with_pbc(MasterPbc, User, OldUserObj), - {reply, Res, NewState}; - {error, Reason} -> - {reply, {error, Reason}, State} - end; handle_call(master_pbc, _From, State) -> case ensure_master_pbc(State) of {ok, #state{master_pbc=MasterPbc} = NewState} -> @@ -298,10 +259,11 @@ stop_pbc(Pbc) when is_pid(Pbc) -> do_get_bucket(State) -> case ensure_master_pbc(State) of - {ok, #state{master_pbc=MasterPbc, bucket_name=BucketName} = NewState} -> + {ok, #state{master_pbc = MasterPbc, + bucket_name = BucketName} = NewState} -> case get_bucket_with_pbc(MasterPbc, BucketName) of {ok, Obj} -> - {ok, NewState#state{bucket_obj=Obj}}; + {ok, NewState#state{bucket_obj = Obj}}; {error, Reason} -> {error, Reason, NewState} end; @@ -322,62 +284,3 @@ get_bucket_with_pbc(MasterPbc, BucketName) -> Timeout = riak_cs_config:get_bucket_timeout(), riak_cs_pbc:get(MasterPbc, ?BUCKETS_BUCKET, BucketName, [], Timeout, [riakc, get_cs_bucket]). - -get_user_with_pbc(MasterPbc, Key) -> - get_user_with_pbc(MasterPbc, Key, riak_cs_config:fast_user_get()). - -get_user_with_pbc(MasterPbc, Key, true) -> - weak_get_user_with_pbc(MasterPbc, Key); -get_user_with_pbc(MasterPbc, Key, false) -> - case strong_get_user_with_pbc(MasterPbc, Key) of - {ok, _} = OK -> OK; - {error, <<"{pr_val_unsatisfied,", _/binary>>} -> - weak_get_user_with_pbc(MasterPbc, Key); - {error, Reason} -> - _ = lager:warning("Fetching user record with strong option failed: ~p", [Reason]), - Timeout = riak_cs_config:get_user_timeout(), - _ = riak_cs_pbc:pause_to_reconnect(MasterPbc, Reason, Timeout), - weak_get_user_with_pbc(MasterPbc, Key) - end. - -strong_get_user_with_pbc(MasterPbc, Key) -> - StrongOptions = [{r, all}, {pr, all}, {notfound_ok, false}], - Timeout = riak_cs_config:get_user_timeout(), - case riak_cs_pbc:get(MasterPbc, ?USER_BUCKET, Key, StrongOptions, - Timeout, [riakc, get_cs_user_strong]) of - {ok, Obj} -> - %% since we read from all primaries, we're less concerned - %% with there being an 'out-of-date' replica that we might - %% conflict with (and not be able to properly resolve - %% conflicts). - KeepDeletedBuckets = false, - {ok, {Obj, KeepDeletedBuckets}}; - {error, _} = Error -> - Error - end. - -weak_get_user_with_pbc(MasterPbc, Key) -> - Timeout = riak_cs_config:get_user_timeout(), - WeakOptions = [{r, quorum}, {pr, one}, {notfound_ok, false}], - case riak_cs_pbc:get(MasterPbc, ?USER_BUCKET, Key, WeakOptions, - Timeout, [riakc, get_cs_user]) of - {ok, Obj} -> - %% We weren't able to read from all primary vnodes, so - %% don't risk losing information by pruning the bucket - %% list. - KeepDeletedBuckets = true, - {ok, {Obj, KeepDeletedBuckets}}; - {error, Reason} -> - {error, Reason} - end. - -save_user_with_pbc(MasterPbc, User, OldUserObj) -> - Indexes = [{?EMAIL_INDEX, User?RCS_USER.email}, - {?ID_INDEX, User?RCS_USER.canonical_id}], - MD = dict:store(?MD_INDEX, Indexes, dict:new()), - UpdUserObj = riakc_obj:update_metadata( - riakc_obj:update_value(OldUserObj, - riak_cs_utils:encode_term(User)), - MD), - Timeout = riak_cs_config:put_user_timeout(), - riak_cs_pbc:put(MasterPbc, UpdUserObj, Timeout, [riakc, put_cs_user]). diff --git a/apps/riak_cs/src/riak_cs_riak_mapred.erl b/apps/riak_cs/src/riak_cs_riak_mapred.erl new file mode 100644 index 000000000..2c27d4dc0 --- /dev/null +++ b/apps/riak_cs/src/riak_cs_riak_mapred.erl @@ -0,0 +1,223 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_riak_mapred). + +%% mapreduce functions that run on a riak node + +-export([map_keys_and_manifests/3, + maybe_process_resolved/3, + reduce_keys_and_manifests/2, + map_users/3, + reduce_users/2, + map_roles/3, + reduce_roles/2, + map_policies/3, + reduce_policies/2, + map_saml_providers/3, + reduce_saml_providers/2, + map_temp_sessions/3, + reduce_temp_sessions/2, + + query/2 + ]). + +-include("moss.hrl"). + + +query(users, Arg) -> + [{map, {modfun, ?MODULE, map_users}, + Arg, false}, + {reduce, {modfun, ?MODULE, reduce_users}, + Arg, true}]; +query(roles, Arg) -> + [{map, {modfun, ?MODULE, map_roles}, + Arg, false}, + {reduce, {modfun, ?MODULE, reduce_roles}, + Arg, true}]; +query(policies, Arg) -> + [{map, {modfun, ?MODULE, map_policies}, + Arg, false}, + {reduce, {modfun, ?MODULE, reduce_policies}, + Arg, true}]; +query(saml_providers, Arg) -> + [{map, {modfun, ?MODULE, map_saml_providers}, + Arg, false}, + {reduce, {modfun, ?MODULE, reduce_saml_providers}, + Arg, true}]; +query(temp_sessions, Arg) -> + [{map, {modfun, ?MODULE, map_temp_sessions}, + Arg, false}, + {reduce, {modfun, ?MODULE, reduce_temp_sessions}, + Arg, true}]. + + +%% MapReduce function, runs on the Riak nodes, should therefore use +%% riak_object, not riakc_obj. +map_keys_and_manifests({error, notfound}, _, _) -> + []; +map_keys_and_manifests(Object, _, _) -> + Handler = fun(Resolved) -> + case rcs_common_manifest_utils:active_manifest(Resolved) of + {ok, Manifest} -> + [{riak_object:key(Object), {ok, Manifest}}]; + _ -> + [] + end + end, + maybe_process_resolved(Object, Handler, []). + +maybe_process_resolved(Object, ResolvedManifestsHandler, ErrorReturn) -> + try + AllManifests = [ binary_to_term(V) + || {_, V} = Content <- riak_object:get_contents(Object), + not riak_cs_utils:has_tombstone(Content) ], + Upgraded = rcs_common_manifest_utils:upgrade_wrapped_manifests(AllManifests), + Resolved = rcs_common_manifest_resolution:resolve(Upgraded), + ResolvedManifestsHandler(Resolved) + catch Type:Reason:StackTrace -> + logger:error("Riak CS object mapreduce failed for ~p:~p with reason ~p:~p at ~p", + [riak_object:bucket(Object), + riak_object:key(Object), + Type, + Reason, + StackTrace]), + ErrorReturn + end. + +%% Pipe all the bucket listing results through a passthrough reduce +%% phase. This is just a temporary kludge until the sink backpressure +%% work is done. +reduce_keys_and_manifests(Acc, _) -> + Acc. + + +map_users({error, notfound}, _, _) -> + []; +map_users(Object, _2, Args) -> + #{path_prefix := PathPrefix} = Args, + case riak_object:get_values(Object) of + [] -> + []; + [<<>>|_] -> + []; + [RBin|_] -> + ?IAM_USER{path = Path} = R = binary_to_term(RBin), + case prefix_match(Path, PathPrefix) of + false -> + []; + true -> + [R] + end + end. + +map_roles({error, notfound}, _, _) -> + []; +map_roles(Object, _2, Args) -> + #{path_prefix := PathPrefix} = Args, + case riak_object:get_values(Object) of + [] -> + []; + [<<>>|_] -> + []; + [RBin|_] -> + ?IAM_ROLE{path = Path} = R = binary_to_term(RBin), + case prefix_match(Path, PathPrefix) of + false -> + []; + true -> + [R] + end + end. + +reduce_users(Acc, _) -> + Acc. +reduce_roles(Acc, _) -> + Acc. + +map_policies({error, notfound}, _, _) -> + []; +map_policies(Object, _2, Args) -> + #{path_prefix := PathPrefix, + only_attached := OnlyAttached, + policy_usage_filter := PolicyUsageFilter, + scope := Scope} = Args, + logger:notice("list_roles: Ignoring parameters PolicyUsageFilter (~s) and Scope (~s)", [PolicyUsageFilter, Scope]), + case riak_object:get_values(Object) of + [] -> + []; + [<<>>|_] -> + []; + [PBin|_] -> + ?IAM_POLICY{path = Path, + attachment_count = AttachmentCount} = P = binary_to_term(PBin), + case prefix_match(Path, PathPrefix) and + ((true == OnlyAttached andalso AttachmentCount > 0) orelse false == OnlyAttached) of + false -> + []; + true -> + [P] + end + end. + +reduce_policies(Acc, _) -> + Acc. + +map_saml_providers({error, notfound}, _, _) -> + []; +map_saml_providers(Object, _2, _Args) -> + case riak_object:get_values(Object) of + [] -> + []; + [<<>>|_] -> + []; + [PBin|_] -> + ?IAM_SAML_PROVIDER{} = P = binary_to_term(PBin), + [P] + end. + +reduce_saml_providers(Acc, _) -> + Acc. + +map_temp_sessions({error, notfound}, _, _) -> + []; +map_temp_sessions(Object, _2, _Args) -> + case riak_object:get_values(Object) of + [] -> + []; + [<<>>|_] -> + []; + [PBin|_] -> + ?TEMP_SESSION{created = Created, + duration_seconds = DurationSeconds} = P = binary_to_term(PBin), + case Created + DurationSeconds * 1000 > os:system_time(millisecond) of + true -> + [P]; + false -> + [] + end + end. + +reduce_temp_sessions(Acc, _) -> + Acc. + + +prefix_match(_, <<>>) -> true; +prefix_match(A, P) -> 0 < binary:longest_common_prefix([A, P]). diff --git a/src/riak_cs_riakc_pool_worker.erl b/apps/riak_cs/src/riak_cs_riakc_pool_worker.erl similarity index 95% rename from src/riak_cs_riakc_pool_worker.erl rename to apps/riak_cs/src/riak_cs_riakc_pool_worker.erl index 844c2d794..2d3eda48d 100644 --- a/src/riak_cs_riakc_pool_worker.erl +++ b/apps/riak_cs/src/riak_cs_riakc_pool_worker.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/src/riak_cs_s3_passthru_auth.erl b/apps/riak_cs/src/riak_cs_s3_passthru_auth.erl similarity index 89% rename from src/riak_cs_s3_passthru_auth.erl rename to apps/riak_cs/src/riak_cs_s3_passthru_auth.erl index 106087681..be6bd9d2f 100644 --- a/src/riak_cs_s3_passthru_auth.erl +++ b/apps/riak_cs/src/riak_cs_s3_passthru_auth.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -26,7 +27,7 @@ -export([identify/2, authenticate/4]). --spec identify(term(),term()) -> {string() | undefined, undefined}. +-spec identify(term(),term()) -> {binary() | undefined, undefined}. identify(RD,_Ctx) -> case wrq:get_req_header("authorization", RD) of undefined -> {[], undefined}; diff --git a/src/riak_cs_simple_bwlimiter.erl b/apps/riak_cs/src/riak_cs_simple_bwlimiter.erl similarity index 88% rename from src/riak_cs_simple_bwlimiter.erl rename to apps/riak_cs/src/riak_cs_simple_bwlimiter.erl index d2a69bfb3..560fb0852 100644 --- a/src/riak_cs_simple_bwlimiter.erl +++ b/apps/riak_cs/src/riak_cs_simple_bwlimiter.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -44,14 +45,13 @@ -behaviour(riak_cs_quota). --include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine_logger.hrl"). --include_lib("webmachine/include/wm_reqdata.hrl"). - -export([state/0, reset/0, set_limits/3]). - -export([start_link/0, allow/3, update/2, error_response/3]). +-include("riak_cs.hrl"). +-include_lib("webmachine/include/webmachine_logger.hrl"). +-include_lib("kernel/include/logger.hrl"). + -record(user_state, { user :: binary(), bandwidth_acc = 0 :: non_neg_integer(), %% Bytes usage per refresh rate @@ -113,21 +113,20 @@ refresher() -> end, receive reset -> - lager:debug("reset received: ~p", [?MODULE]), + ?LOG_DEBUG("reset received: ~p", [?MODULE]), refresher(); _ -> ets:delete(?MODULE) after IntervalSec * 1000 -> - lager:debug("~p refresh in ~p secs", [?MODULE, IntervalSec]), + ?LOG_DEBUG("~p refresh in ~p secs", [?MODULE, IntervalSec]), ets:delete_all_objects(?MODULE), refresher() end. --spec allow(rcs_user(), access(), #context{}) -> {ok, #wm_reqdata{}, #context{}}. -allow(Owner, #access_v1{req = RD} = _Access, Ctx) -> +-spec allow(rcs_user(), access(), #rcs_web_context{}) -> {ok, #wm_reqdata{}, #rcs_web_context{}}. +allow(?RCS_USER{key_id = OwnerKey}, #access_v1{req = RD} = _Access, Ctx) -> - OwnerKey = list_to_binary(riak_cs_user:key_id(Owner)), - lager:debug("access => ~p", [OwnerKey]), + ?LOG_DEBUG("access => ~p", [OwnerKey]), UserState = case ets:lookup(?MODULE, OwnerKey) of [UserState0] -> UserState0; [] -> new_user_state(OwnerKey) @@ -143,20 +142,20 @@ allow(Owner, #access_v1{req = RD} = _Access, Ctx) -> ok -> {ok, RD, Ctx}; {error, {_, Current, Threshold}} = Error -> - lager:warning("User ~p has exceeded access limit: usage, limit = ~p, ~p (/sec)", - [OwnerKey, Current, Threshold]), + logger:info("User ~p has exceeded access limit: usage, limit = ~p, ~p (/sec)", + [OwnerKey, Current, Threshold]), Error end; {error, {_, Current, Threshold}} = Error2 -> - lager:warning("User ~p has exceeded bandwidth limit: usage, limit = ~p, ~p (bytes/sec)", - [OwnerKey, Current, Threshold]), + logger:info("User ~p has exceeded bandwidth limit: usage, limit = ~p, ~p (bytes/sec)", + [OwnerKey, Current, Threshold]), Error2 end. -spec new_user_state(binary()) -> #user_state{}. new_user_state(User) -> UserState = #user_state{user = User}, - lager:debug("quota init: ~p => ~p", [User, UserState]), + ?LOG_DEBUG("quota init: ~p => ~p", [User, UserState]), %% Here's a race condition where if so many concurrent access %% come, each access can yield a new fresh state and then %% receives not access limitation. @@ -207,19 +206,19 @@ update(User, {#user_state.access_count, 1}], try case ets:update_counter(?MODULE, User, UpdateOps) of - [_, _] = R -> + [_, _] -> ok; Error0 -> - lager:warning("something wrong? ~p", [Error0]), + logger:warning("something wrong? ~p", [Error0]), {error, Error0} end catch error:badarg -> %% record not just found here - lager:debug("Cache of ~p (maybe not found)", [User]), + ?LOG_DEBUG("Cache of ~p (maybe not found)", [User]), ok; Type:Error -> %% TODO: show out stacktrace heah - _ = lager:warning("something wrong? ~p", [Error]), + logger:warning("something wrong? ~p", [Error]), {error, {Type, Error}} end. diff --git a/src/riak_cs_simple_quota.erl b/apps/riak_cs/src/riak_cs_simple_quota.erl similarity index 83% rename from src/riak_cs_simple_quota.erl rename to apps/riak_cs/src/riak_cs_simple_quota.erl index 49a032516..1dc0f47f9 100644 --- a/src/riak_cs_simple_quota.erl +++ b/apps/riak_cs/src/riak_cs_simple_quota.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -49,14 +50,13 @@ -behaviour(riak_cs_quota). --include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine_logger.hrl"). --include_lib("webmachine/include/wm_reqdata.hrl"). - -export([state/0, reset/0, set_params/2]). - -export([start_link/0, allow/3, update/2, error_response/3]). +-include("riak_cs.hrl"). +-include_lib("webmachine/include/webmachine_logger.hrl"). +-include_lib("kernel/include/logger.hrl"). + %% Quota not set -define(DEFAULT_SIMPLE_QUOTA, -1). %% A day @@ -101,24 +101,23 @@ refresher() -> end, receive reset -> - lager:debug("reset received: ~p", [?MODULE]), + ?LOG_DEBUG("reset received: ~p", [?MODULE]), refresher(); _ -> ets:delete(?MODULE) after IntervalSec * 1000 -> - lager:debug("~p refresh in ~p secs", [?MODULE, IntervalSec]), + ?LOG_DEBUG("~p refresh in ~p secs", [?MODULE, IntervalSec]), ets:delete_all_objects(?MODULE), refresher() end. %% @doc Only puts affected --spec allow(rcs_user(), access(), #context{}) -> - {ok, #wm_reqdata{}, #context{}} | +-spec allow(rcs_user(), access(), #rcs_web_context{}) -> + {ok, #wm_reqdata{}, #rcs_web_context{}} | {error, {disk_quota, non_neg_integer(), non_neg_integer()}}. -allow(Owner, #access_v1{req = RD, method = 'PUT'} = _Access, Ctx) -> - OwnerKey = iolist_to_binary(riak_cs_user:key_id(Owner)), - lager:debug("access => ~p", [OwnerKey]), +allow(?RCS_USER{key_id = OwnerKey}, #access_v1{req = RD, method = 'PUT'} = _Access, Ctx) -> + ?LOG_DEBUG("access => ~p", [OwnerKey]), {_, Usage} = case ets:lookup(?MODULE, OwnerKey) of [{OwnerKey, _Usage} = UserState0] -> UserState0; @@ -133,22 +132,21 @@ allow(Owner, #access_v1{req = RD, method = 'PUT'} = _Access, Ctx) -> Quota < 0 -> {ok, RD, Ctx}; Usage < Quota -> {ok, RD, Ctx}; true -> - lager:warning("User ~s has exceeded it's quota: usage, quota = ~p, ~p (bytes)", - [OwnerKey, Usage, Quota]), + logger:info("User ~s has exceeded its quota: usage, quota = ~p, ~p (bytes)", + [OwnerKey, Usage, Quota]), {error, {disk_quota, Usage, Quota}} end; allow(_Owner, #access_v1{req = RD} = _Access, Ctx) -> {ok, RD, Ctx}. --spec maybe_usage(binary(), #context{}) -> non_neg_integer(). -maybe_usage(_, _Ctx = #context{riak_client=undefined}) -> +-spec maybe_usage(binary(), #rcs_web_context{}) -> non_neg_integer(). +maybe_usage(_, _Ctx = #rcs_web_context{riak_client=undefined}) -> %% can't happen here error(no_riak_client); -maybe_usage(User0, _Ctx = #context{riak_client=RiakClient}) -> - User = binary_to_list(User0), +maybe_usage(User, _Ctx = #rcs_web_context{riak_client=RiakClient}) -> Usage = case get_latest_usage(RiakClient, User) of {error, notfound} -> - lager:warning("No storage stats data was found. Starting as no usage."), + logger:warning("No storage stats data was found. Starting as no usage."), 0; {ok, Usages} -> sum_all_buckets(lists:last(Usages)) @@ -157,7 +155,7 @@ maybe_usage(User0, _Ctx = #context{riak_client=RiakClient}) -> %% come, each access can yield a new fresh state and then %% receives not access limitation. try - ets:insert_new(?MODULE, {User0, Usage}), + ets:insert_new(?MODULE, {User, Usage}), Usage catch _:_ -> Usage @@ -185,7 +183,7 @@ update(User, ets:update_counter(?MODULE, User, Bytes) catch Type:Error -> %% TODO: show out stacktrace heah - lager:warning("something wrong? ~p", [Error]), + logger:warning("something wrong? ~p", [Error]), {error, {Type, Error}} end; update(_, _) -> @@ -205,17 +203,17 @@ error_response({disk_quota, Current, Limit}, RD, Ctx) -> UpdReqData = wrq:set_resp_body(Body, ReqData), {StatusCode, UpdReqData, Ctx}. --spec get_latest_usage(pid(), string()) -> {ok, list()} | {error, notfound}. get_latest_usage(Pid, User) -> - Now = calendar:now_to_datetime(os:timestamp()), - NowASec = calendar:datetime_to_gregorian_seconds(Now), + Now = calendar:datetime_to_gregorian_seconds( + calendar:system_time_to_universal_time( + os:system_time(millisecond), millisecond)), ArchivePeriod = case riak_cs_storage:archive_period() of {ok, Period} -> Period; _ -> 86400 %% A day is hard coded end, - get_latest_usage(Pid, User, ArchivePeriod, NowASec, 0, 10). + get_latest_usage(Pid, User, ArchivePeriod, Now, 0, 10). --spec get_latest_usage(pid(), string(), non_neg_integer(), +-spec get_latest_usage(pid(), binary(), non_neg_integer(), non_neg_integer(), non_neg_integer(), non_neg_integer()) -> {ok, list()} | {error, notfound}. get_latest_usage(_Pid, _User, _, _, N, N) -> {error, notfound}; @@ -223,7 +221,8 @@ get_latest_usage(Pid, User, ArchivePeriod, EndSec, N, Max) -> End = calendar:gregorian_seconds_to_datetime(EndSec), EndSec2 = EndSec - ArchivePeriod, ADayAgo = calendar:gregorian_seconds_to_datetime(EndSec2), - case riak_cs_storage:get_usage(Pid, User, false, ADayAgo, End) of + case riak_cs_storage:get_usage( + Pid, User, false, ADayAgo, End) of {[], _} -> get_latest_usage(Pid, User, ArchivePeriod, EndSec2, N+1, Max); {Res, _} -> @@ -234,7 +233,7 @@ get_latest_usage(Pid, User, ArchivePeriod, EndSec, N, Max) -> sum_all_buckets(UsageList) -> lists:foldl(fun({<<"StartTime">>, _}, Sum) -> Sum; ({<<"EndTime">>, _}, Sum) -> Sum; - ({_BucketName, {struct, Data}}, Sum) -> + ({_BucketName, Data}, Sum) -> case proplists:get_value(<<"Bytes">>, Data) of I when is_integer(I) -> Sum + I; _ -> Sum diff --git a/src/riak_cs_stanchion_console.erl b/apps/riak_cs/src/riak_cs_stanchion_console.erl similarity index 58% rename from src/riak_cs_stanchion_console.erl rename to apps/riak_cs/src/riak_cs_stanchion_console.erl index 77465b241..6b2bb9c09 100644 --- a/src/riak_cs_stanchion_console.erl +++ b/apps/riak_cs/src/riak_cs_stanchion_console.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -23,7 +24,6 @@ -module(riak_cs_stanchion_console). -export([ - switch/1, show/1 ]). @@ -44,29 +44,6 @@ %%% Public API %%%=================================================================== -%% @doc Switch Stanchion to a new one. Can be used for disabling. -switch([Host, Port]) -> - Msg = io_lib:format("Switching stanchion to ~s:~s", [Host, Port]), - ?SAFELY(begin - NewPort = list_to_integer(Port), - true = (0 < NewPort andalso NewPort < 65536), - %% {_, _, SSL} = riak_cs_utils:stanchion_data(), - %% currently this does not work due to bad path generation of velvet:ping/3 - %% ok = velvet:ping(Host, NewPort, SSL), - - %% Or, set this configuration to dummy host/port - %% to block bucket/user creation/deletion - %% in case of multiple stanchion working. - ok = application:set_env(riak_cs, stanchion_host, {Host, NewPort}), - Msg2 = io_lib:format("Succesfully switched stanchion to ~s:~s: This change is only effective until restart.", - [Host, Port]), - _ = lager:info(Msg2), - io:format("~s~nTo make permanent change, be sure to edit configuration file.~n", [Msg2]) - end, Msg); -switch(_) -> - io:format("Usage: riak-cs-admin stanchion switch IP Port~n"), - error. - show([]) -> ?SAFELY(begin {Host, Port, SSL} = riak_cs_utils:stanchion_data(), @@ -74,6 +51,6 @@ show([]) -> true -> "https://"; false -> "http://" end, - io:format("Current Stanchion Adderss: ~s~s:~s~n", + io:format("Current Stanchion Address: ~s~s:~s~n", [Scheme, Host, integer_to_list(Port)]) end, "Retrieving Stanchion info"). diff --git a/src/riak_cs_stats.erl b/apps/riak_cs/src/riak_cs_stats.erl similarity index 92% rename from src/riak_cs_stats.erl rename to apps/riak_cs/src/riak_cs_stats.erl index 2689197c5..a94b6186e 100644 --- a/src/riak_cs_stats.erl +++ b/apps/riak_cs/src/riak_cs_stats.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -65,10 +66,12 @@ duration_metrics() -> [bucket_policy, delete], [bucket_location, get], [bucket_versioning, get], + [bucket_versioning, put], [bucket_request_payment, get], [list_uploads, get], [multiple_delete, post], [list_objects, get], + [list_object_versions, get], [object, get], [object, put], @@ -92,6 +95,7 @@ duration_metrics() -> [velvet, set_bucket_acl], [velvet, set_bucket_policy], [velvet, delete_bucket_policy], + [velvet, set_bucket_versioning], %% Riak PB client, per key operations [riakc, ping], @@ -99,6 +103,9 @@ duration_metrics() -> [riakc, get_cs_user_strong], [riakc, get_cs_user], [riakc, put_cs_user], + [riakc, get_cs_role], + [riakc, get_cs_policy], + [riakc, get_cs_saml_provider], [riakc, get_manifest], [riakc, put_manifest], @@ -163,7 +170,7 @@ counting_subkeys() -> inflow(Key) -> safe_update([riak_cs, in | Key], 1). --spec update_with_start(key(), erlang:timestamp(), ok_error_res()) -> ok. +-spec update_with_start(key(), non_neg_integer(), ok_error_res()) -> ok. update_with_start(Key, StartTime, ok) -> update_with_start(Key, StartTime); update_with_start(Key, StartTime, {ok, _}) -> @@ -171,25 +178,25 @@ update_with_start(Key, StartTime, {ok, _}) -> update_with_start(Key, StartTime, {error, _}) -> update_error_with_start(Key, StartTime). --spec update_with_start(key(), erlang:timestamp()) -> ok. +-spec update_with_start(key(), non_neg_integer()) -> ok. update_with_start(Key, StartTime) -> - update(Key, timer:now_diff(os:timestamp(), StartTime)). + update(Key, os:system_time(millisecond) - StartTime). --spec update_error_with_start(key(), erlang:timestamp()) -> ok. +-spec update_error_with_start(key(), non_neg_integer()) -> ok. update_error_with_start(Key, StartTime) -> - update([error | Key], timer:now_diff(os:timestamp(), StartTime)). + update([error | Key], os:system_time(millisecond) - StartTime). -spec countup(key()) -> ok. countup(Key) -> safe_update([riak_cs | Key], 1). --spec report_json() -> string(). +-spec report_json() -> binary(). report_json() -> - lists:flatten(mochijson2:encode({struct, get_stats()})). + jsx:encode(get_stats()). --spec report_pretty_json() -> string(). +-spec report_pretty_json() -> binary(). report_pretty_json() -> - lists:flatten(riak_cs_utils:json_pp_print(report_json())). + jsx:prettify(report_json()). -spec get_stats() -> proplists:proplist(). get_stats() -> @@ -231,7 +238,7 @@ safe_update(ExometerKey, Arg) -> case exometer:update(ExometerKey, Arg) of ok -> ok; {error, Reason} -> - lager:warning("Stats update for key ~p error: ~p", [ExometerKey, Reason]), + logger:warning("Stats update for key ~p error: ~p", [ExometerKey, Reason]), ok end. @@ -331,11 +338,10 @@ metric_to_atom(Key, Suffix) -> -include_lib("eunit/include/eunit.hrl"). stats_test_() -> - Apps = [setup, compiler, syntax_tools, goldrush, lager, exometer_core], + Apps = [setup, compiler, syntax_tools, exometer_core], {setup, fun() -> - application:set_env(lager, handlers, []), - [catch (application:start(App)) || App <- Apps], + [application:ensure_all_started(App) || App <- Apps], ok = init() end, fun(_) -> diff --git a/src/riak_cs_storage.erl b/apps/riak_cs/src/riak_cs_storage.erl similarity index 76% rename from src/riak_cs_storage.erl rename to apps/riak_cs/src/riak_cs_storage.erl index eb58a5da2..5793903d7 100644 --- a/src/riak_cs_storage.erl +++ b/apps/riak_cs/src/riak_cs_storage.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -22,8 +23,6 @@ -module(riak_cs_storage). --include("riak_cs.hrl"). - -export([ sum_user/4, sum_bucket/3, @@ -37,24 +36,26 @@ object_size_reduce/2 ]). +-include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). + -ifdef(TEST). -compile(export_all). +-compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). -endif. %% @doc Sum the number of bytes stored in active files in all of the %% given user's directories. The result is a list of pairs of %% `{BucketName, Bytes}'. --spec sum_user(riak_client(), string(), boolean(), erlang:timestamp()) -> - {ok, [{string(), integer()}]} - | {error, term()}. -sum_user(RcPid, User, Detailed, LeewayEdge) when is_binary(User) -> - sum_user(RcPid, binary_to_list(User), Detailed, LeewayEdge); -sum_user(RcPid, User, Detailed, LeewayEdge) when is_list(User) -> - case riak_cs_user:get_user(User, RcPid) of +-spec sum_user(riak_client(), binary(), boolean(), non_neg_integer()) -> + {ok, [{string(), integer()}]} | {error, term()}. +sum_user(RcPid, Arn, Detailed, LeewayEdge) -> + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:get_user(Arn, Pbc) of {ok, {UserRecord, _UserObj}} -> Buckets = riak_cs_bucket:get_buckets(UserRecord), - BucketUsages = [maybe_sum_bucket(User, B, Detailed, LeewayEdge) || + BucketUsages = [maybe_sum_bucket(Arn, B, Detailed, LeewayEdge) || B <- Buckets], {ok, BucketUsages}; {error, Error} -> @@ -65,21 +66,12 @@ sum_user(RcPid, User, Detailed, LeewayEdge) when is_list(User) -> %% This log is *very* important because unless this log %% there are no other way for operator to know a calculation %% which riak_cs_storage_d failed. --spec maybe_sum_bucket(string(), cs_bucket(), boolean(), erlang:timestamp()) -> - {binary(), [{binary(), integer()}]} | - {binary(), binary()}. -maybe_sum_bucket(User, ?RCS_BUCKET{name=Name} = Bucket, Detailed, LeewayEdge) - when is_list(Name) -> - maybe_sum_bucket(User, Bucket?RCS_BUCKET{name=list_to_binary(Name)}, - Detailed, LeewayEdge); -maybe_sum_bucket(User, ?RCS_BUCKET{name=Name} = _Bucket, Detailed, LeewayEdge) - when is_binary(Name) -> +maybe_sum_bucket(User, ?RCS_BUCKET{name = Name}, Detailed, LeewayEdge) -> case sum_bucket(Name, Detailed, LeewayEdge) of - {struct, _} = BucketUsage -> {Name, BucketUsage}; + {ok, BucketUsage} -> {Name, BucketUsage}; {error, _} = E -> - _ = lager:error("failed to calculate usage of " - "bucket '~s' of user '~s'. Reason: ~p", - [Name, User, E]), + logger:error("failed to calculate usage of bucket \"~s\" of user \"~s\". Reason: ~p", + [Name, User, E]), {Name, iolist_to_binary(io_lib:format("~p", [E]))} end. @@ -95,9 +87,8 @@ maybe_sum_bucket(User, ?RCS_BUCKET{name=Name} = _Bucket, Detailed, LeewayEdge) %% which is the number of objects that were counted in the bucket, and %% `Bytes', which is the total size of all of those objects. More %% fields are included for detailed calculation. --spec sum_bucket(binary(), boolean(), erlang:timestamp()) -> - {struct, [{binary(), integer()}]} - | {error, term()}. +-spec sum_bucket(binary(), boolean(), non_neg_integer()) -> + {ok, [{binary(), integer()}]} | {error, term()}. sum_bucket(BucketName, Detailed, LeewayEdge) -> Query = case Detailed of false -> @@ -127,7 +118,7 @@ sum_bucket(BucketName, Detailed, LeewayEdge) -> case riak_cs_pbc:mapred(ManifestPbc, Input, Query, Timeout, [riakc, mapred_storage]) of {ok, MRRes} -> - extract_summary(MRRes, Detailed); + {ok, extract_summary(MRRes, Detailed)}; {error, Error} -> {error, Error} end @@ -143,14 +134,14 @@ object_size_reduce(Values, Args) -> extract_summary(MRRes, false) -> {1, [{Objects, Bytes}]} = lists:keyfind(1, 1, MRRes), - {struct, [{<<"Objects">>, Objects}, - {<<"Bytes">>, Bytes}]}; + [{<<"Objects">>, Objects}, + {<<"Bytes">>, Bytes}]; extract_summary(MRRes, true) -> Summary = case lists:keyfind(1, 1, MRRes) of {1, [[]]} -> riak_cs_storage_mr:empty_summary(); {1, [NonEmptyValue]} -> NonEmptyValue end, - {struct, detailed_result_json_struct(Summary, [])}. + detailed_result_json_struct(Summary, []). detailed_result_json_struct([], Acc) -> Acc; @@ -182,14 +173,14 @@ make_object(User, BucketList, SampleStart, SampleEnd) -> rts:new_sample(?STORAGE_BUCKET, User, SampleStart, SampleEnd, Period, BucketList). --spec get_usage(riak_client(), string(), +-spec get_usage(riak_client(), binary(), boolean(), calendar:datetime(), calendar:datetime()) -> {list(), list()}. -get_usage(RcPid, User, AdminAccess, Start, End) -> +get_usage(RcPid, UserArn, AdminAccess, Start, End) -> {ok, Period} = archive_period(), RtsPuller = riak_cs_riak_client:rts_puller( - RcPid, ?STORAGE_BUCKET, User, [riakc, get_storage]), + RcPid, ?STORAGE_BUCKET, UserArn, [riakc, get_storage]), {Samples, Errors} = rts:find_samples(RtsPuller, Start, End, Period), case AdminAccess of true -> {Samples, Errors}; @@ -201,9 +192,9 @@ filter_internal_usage([], Acc) -> filter_internal_usage([{K, _V}=T | Rest], Acc) when K =:= <<"StartTime">> orelse K =:= <<"EndTime">> -> filter_internal_usage(Rest, [T|Acc]); -filter_internal_usage([{Bucket, {struct, UsageList}} | Rest], Acc) -> +filter_internal_usage([{_Bucket, <<"{error,", _/binary>>} = T | Rest], Acc) -> + filter_internal_usage(Rest, [T|Acc]); +filter_internal_usage([{Bucket, UsageList} | Rest], Acc) -> Objects = lists:keyfind(<<"Objects">>, 1, UsageList), Bytes = lists:keyfind(<<"Bytes">>, 1, UsageList), - filter_internal_usage(Rest, [{Bucket, {struct, [Objects, Bytes]}} | Acc]); -filter_internal_usage([{_Bucket, _ErrorBin}=T | Rest], Acc) -> - filter_internal_usage(Rest, [T|Acc]). + filter_internal_usage(Rest, [{Bucket, [Objects, Bytes]} | Acc]). diff --git a/src/riak_cs_storage_console.erl b/apps/riak_cs/src/riak_cs_storage_console.erl similarity index 98% rename from src/riak_cs_storage_console.erl rename to apps/riak_cs/src/riak_cs_storage_console.erl index fe8da476d..6a30e1d98 100644 --- a/src/riak_cs_storage_console.erl +++ b/apps/riak_cs/src/riak_cs_storage_console.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/src/riak_cs_storage_d.erl b/apps/riak_cs/src/riak_cs_storage_d.erl similarity index 82% rename from src/riak_cs_storage_d.erl rename to apps/riak_cs/src/riak_cs_storage_d.erl index a0914f5e3..1c7c42449 100644 --- a/src/riak_cs_storage_d.erl +++ b/apps/riak_cs/src/riak_cs_storage_d.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -48,6 +49,7 @@ code_change/4]). -include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). -define(SERVER, ?MODULE). @@ -58,17 +60,15 @@ next, %% the next scheduled time riak_client :: undefined | pid(), %% client we're currently using - batch_start, %% the time we actually started - batch_count=0, %% count of users processed so far - batch_skips=0, %% count of users skipped so far - batch=[] :: [string()], %% users left to process in this batch + batch_start, %% the time we actually started + batch_count = 0, %% count of users processed so far + batch_skips = 0, %% count of users skipped so far + batch = [] :: [binary()], %% users left to process in this batch recalc, %% recalculate a user's storage for this period? detailed, %% calculate more than counts and total sizes leeway_edge %% prior to this clock, gc can reclain objects }). --type state() :: #state{}. - %%%=================================================================== %%% API %%%=================================================================== @@ -149,8 +149,8 @@ idle(_, State) -> %% from the outside world (like `status'). calculating(continue, #state{batch=[], current=Current}=State) -> %% finished with this batch - _ = lager:info("Finished storage calculation in ~b seconds.", - [elapsed(State#state.batch_start)]), + logger:info("Finished storage calculation in ~b seconds", + [elapsed(State#state.batch_start)]), riak_cs_riak_client:stop(State#state.riak_client), NewState = State#state{riak_client=undefined, last=Current, @@ -200,12 +200,12 @@ calculating({manual_batch, _Options}, _From, State) -> %% this is the manual user request to begin a batch {reply, {error, already_calculating}, calculating, State}; calculating(pause_batch, _From, State) -> - _ = lager:info("Pausing storage calcluation"), + logger:info("Pausing storage calcluation"), {reply, ok, paused, State}; calculating(cancel_batch, _From, #state{current=Current}=State) -> %% finished with this batch - _ = lager:info("Canceled storage calculation after ~b seconds.", - [elapsed(State#state.batch_start)]), + logger:info("Canceled storage calculation after ~b seconds", + [elapsed(State#state.batch_start)]), riak_cs_riak_client:stop(State#state.riak_client), NewState = State#state{riak_client=undefined, last=Current, @@ -219,7 +219,7 @@ paused(status, From, State) -> {reply, {ok, {_, Status}}, _, State} = calculating(status, From, State), {reply, {ok, {paused, Status}}, paused, State}; paused(resume_batch, _From, State) -> - _ = lager:info("Resuming storage calculation"), + logger:info("Resuming storage calculation"), gen_fsm:send_event(?SERVER, continue), {reply, ok, calculating, State}; paused(cancel_batch, From, State) -> @@ -243,9 +243,9 @@ handle_info({start_batch, Next}, idle, #state{next=Next}=State) -> {next_state, calculating, NewState}; handle_info({start_batch, Next}, InBatch, #state{next=Next, current=Current}=State) -> - _ = lager:error("Unable to start storage calculation for ~p" - " because ~p is still working. Skipping forward...", - [Next, Current]), + logger:error("Unable to start storage calculation for ~p" + " because ~p is still working. Skipping forward...", + [Next, Current]), NewState = schedule_next(State, Next), {next_state, InBatch, NewState}; handle_info(_Info, StateName, State) -> @@ -264,8 +264,6 @@ code_change(_OldVsn, StateName, State, _Extra) -> %%% Internal functions %%%=================================================================== --spec try_prepare(state()) -> {next_state, atom(), state()} | - {next_state, atom(), state(), pos_integer()}. try_prepare(State) -> Schedule = read_storage_schedule(), SchedState = schedule_next(State#state{schedule=Schedule}, @@ -280,8 +278,8 @@ read_storage_schedule() -> read_storage_schedule1() -> case application:get_env(riak_cs, storage_schedule) of undefined -> - _ = lager:warning("No storage schedule defined." - " Calculation must be triggered manually."), + logger:warning("No storage schedule defined." + " Calculation must be triggered manually."), []; {ok, Sched} -> case catch parse_time(Sched) of @@ -293,23 +291,19 @@ read_storage_schedule1() -> _ = case [ X || {X,{'EXIT',_}} <- Times ] of [] -> ok; Bad -> - _ = lager:error( - "Ignoring bad storage schedule elements ~p", - [Bad]) + logger:error("Ignoring bad storage schedule elements ~p", [Bad]) end, case [ Parsed || {_, {ok, Parsed}} <- Times] of [] -> - _ = lager:warning( - "No storage schedule defined." - " Calculation must be triggered manually."), + logger:warning("No storage schedule defined." + " Calculation must be triggered manually."), []; Good -> Good end; _ -> - _ = lager:error( - "Invalid storage schedule defined." - " Calculation must be triggered manually."), + logger:error("Invalid storage schedule defined." + " Calculation must be triggered manually."), [] end end. @@ -335,18 +329,15 @@ start_batch(Options, Time, State) -> Recalc = true == proplists:get_value(recalc, Options), Detailed = proplists:get_value(detailed, Options, riak_cs_config:detailed_storage_calc()), - Now = riak_cs_utils:second_resolution_timestamp(os:timestamp()), - LeewayEdgeTs = Now - riak_cs_gc:leeway_seconds(), - LeewayEdge = {LeewayEdgeTs div 1000000, LeewayEdgeTs rem 1000000, 0}, - _ = case Detailed of - true -> - lager:info("Starting storage calculation: " - "recalc=~p, detailed=~p, leeway edge=~p", - [Recalc, Detailed, - calendar:now_to_universal_time(LeewayEdge)]); - _ -> - lager:info("Starting storage calculation: recalc=~p", [Recalc]) - end, + Now = os:system_time(millisecond), + LeewayEdge = Now - riak_cs_gc:leeway_seconds() * 1000, + case Detailed of + true -> + logger:info("Starting storage calculation: recalc=~p, detailed=~p, leeway edge=~p", + [Recalc, Detailed, calendar:system_time_to_universal_time(LeewayEdge, millisecond)]); + _ -> + logger:info("Starting storage calculation: recalc=~p", [Recalc]) + end, %% TODO: probably want to do this fetch streaming, to avoid %% accidental memory pressure at other points @@ -361,9 +352,7 @@ start_batch(Options, Time, State) -> case riak_cs_user:fetch_user_keys(RcPid) of {ok, UserKeys} -> UserKeys; {error, Error} -> - _ = lager:error("Storage calculator was unable" - " to fetch list of users (~p)", - [Error]), + logger:error("Storage calculator was unable to fetch list of users (~p)", [Error]), [] end, @@ -380,20 +369,19 @@ start_batch(Options, Time, State) -> %% @doc Compute storage for the next user in the batch. calculate_next_user(#state{riak_client=RcPid, - batch=[User|Rest], + batch=[UserArn|Rest], recalc=Recalc, detailed=Detailed, leeway_edge=LeewayEdge}=State) -> Start = calendar:universal_time(), - case recalc(Recalc, RcPid, User, Start) of + case recalc(Recalc, RcPid, UserArn, Start) of true -> - _ = case riak_cs_storage:sum_user(RcPid, User, Detailed, LeewayEdge) of + _ = case riak_cs_storage:sum_user(RcPid, UserArn, Detailed, LeewayEdge) of {ok, BucketList} -> End = calendar:universal_time(), - store_user(State, User, BucketList, Start, End); + store_user(State, UserArn, BucketList, Start, End); {error, Error} -> - _ = lager:error("Error computing storage for user ~s (~p)", - [User, Error]) + logger:error("Error computing storage for user ~s (~p)", [UserArn, Error]) end, State#state{batch=Rest, batch_count=1+State#state.batch_count}; false -> @@ -417,15 +405,15 @@ recalc(false, RcPid, User, Time) -> end. %% @doc Archive a user's storage calculation. -store_user(#state{riak_client=RcPid}, User, BucketList, Start, End) -> - Obj = riak_cs_storage:make_object(User, BucketList, Start, End), +store_user(#state{riak_client = RcPid}, User, BucketList, Start, End) -> + Obj = riak_cs_storage:make_object( + iolist_to_binary([User]), BucketList, Start, End), {ok, MasterPbc} = riak_cs_riak_client:master_pbc(RcPid), Timeout = riak_cs_config:put_user_usage_timeout(), case riak_cs_pbc:put(MasterPbc, Obj, Timeout, [riakc, put_storage]) of ok -> ok; {error, Error} -> - _ = lager:error("Error storing storage for user ~s (~p)", - [User, Error]) + logger:error("Error storing storage for user ~s (~p)", [User, Error]) end. %% @doc How many seconds have passed from `Time' to now. @@ -451,14 +439,12 @@ schedule_next(#state{schedule=Schedule}=State, Last) -> NextTime = next_target_time(Last, Schedule), case elapsed(calendar:universal_time(), NextTime) of D when D > 0 -> - _ = lager:info("Scheduling next storage calculation for ~p", - [NextTime]), + logger:info("Scheduling next storage calculation for ~p", [NextTime]), erlang:send_after(D*1000, self(), {start_batch, NextTime}), State#state{next=NextTime}; _ -> - _ = lager:error("Missed start time for storage calculation at ~p," - " skipping to next scheduled time...", - [NextTime]), + logger:error("Missed start time for storage calculation at ~p," + " skipping to next scheduled time...", [NextTime]), %% just skip everything until the next scheduled time from now schedule_next(State, calendar:universal_time()) end. diff --git a/src/riak_cs_storage_mr.erl b/apps/riak_cs/src/riak_cs_storage_mr.erl similarity index 93% rename from src/riak_cs_storage_mr.erl rename to apps/riak_cs/src/riak_cs_storage_mr.erl index 8fc48a75e..cfc875673 100644 --- a/src/riak_cs_storage_mr.erl +++ b/apps/riak_cs/src/riak_cs_storage_mr.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -74,7 +75,7 @@ bucket_summary_map({error, notfound}, _, _Args) -> []; bucket_summary_map(Object, _, Args) -> LeewayEdge = proplists:get_value(leeway_edge, Args), - Summary = riak_cs_utils:maybe_process_resolved( + Summary = riak_cs_riak_mapred:maybe_process_resolved( Object, fun(History) -> sum_objs(LeewayEdge, History) end, #sum{}), Res = summary_to_list(Summary), [Res]. @@ -83,7 +84,7 @@ object_size_map({error, notfound}, _, _) -> []; object_size_map(Object, _, _) -> Handler = fun(Resolved) -> object_size(Resolved) end, - riak_cs_utils:maybe_process_resolved(Object, Handler, []). + riak_cs_riak_mapred:maybe_process_resolved(Object, Handler, []). object_size_reduce(Sizes, _) -> {Objects,Bytes} = lists:unzip(Sizes), @@ -93,7 +94,7 @@ object_size_reduce(Sizes, _) -> -spec sum_objs(erlang:timestamp(), [cs_uuid_and_manifest()]) -> sum(). sum_objs(LeewayEdge, History) -> - case riak_cs_manifest_utils:active_manifest(History) of + case rcs_common_manifest_utils:active_manifest(History) of {ok, Active} -> {_, _, OBB} = objs_bytes_and_blocks(Active), Sum0 = add_to(#sum{}, #sum.user, OBB), @@ -162,7 +163,7 @@ objs_bytes_and_blocks_non_mp(?MANIFEST{state=State, content_length=CL, block_siz BlockCount = riak_cs_lfs_utils:block_count(CL, BS), {non_mp, State, {1, CL, BlockCount}}; objs_bytes_and_blocks_non_mp(?MANIFEST{state=State} = _M) -> - lager:debug("Strange manifest: ~p~n", [_M]), + logger:info("Strange manifest: ~p", [_M]), %% The branch above is for content_length is properly set. This %% is true for non-MP v2 auth case but not always true for v4 of %% streaming sha256 check of writing. To avoid error, ignore @@ -209,10 +210,7 @@ blocks_mp_parts(?MULTIPART_MANIFEST{parts=PartMs}) -> P?PART_MANIFEST.content_length, P?PART_MANIFEST.block_size) || P <- PartMs]). -% @doc Returns `new' if Timestamp is 3-tuple and greater than `LeewayEdge', -% otherwise `old'. --spec new_or_old(erlang:timestamp(), erlang:timestamp()) -> new | old. -new_or_old(LeewayEdge, {_,_,_} = Timestamp) when LeewayEdge < Timestamp -> new; +new_or_old(LeewayEdge, Timestamp) when LeewayEdge < Timestamp -> new; new_or_old(_, _) -> old. -spec add_to(sum(), pos_integer(), @@ -235,7 +233,7 @@ summary_to_list([F|Fields], [{C, By, Bl}|Triples], Acc) -> object_size(Resolved) -> {MPparts, MPbytes} = count_multipart_parts(Resolved), - case riak_cs_manifest_utils:active_manifest(Resolved) of + case rcs_common_manifest_utils:active_manifest(Resolved) of {ok, ?MANIFEST{content_length=Length}} -> [{1 + MPparts, Length + MPbytes}]; _ -> @@ -263,9 +261,8 @@ count_multipart_parts({_UUID, ?MANIFEST{props=Props, state=writing} = M}, Acc; Other -> %% strange thing happened - _ = lager:log(warning, self(), - "strange writing multipart manifest detected at ~p: ~p", - [M?MANIFEST.bkey, Other]), + logger:warning("strange writing multipart manifest detected at ~p: ~p", + [M?MANIFEST.bkey, Other]), Acc end; count_multipart_parts(_, Acc) -> diff --git a/apps/riak_cs/src/riak_cs_sts.erl b/apps/riak_cs/src/riak_cs_sts.erl new file mode 100644 index 000000000..bfd80cd1c --- /dev/null +++ b/apps/riak_cs/src/riak_cs_sts.erl @@ -0,0 +1,321 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_sts). + +-export([assume_role_with_saml/2 + ]). + +-include("riak_cs.hrl"). +-include("aws_api.hrl"). +-include_lib("xmerl/include/xmerl.hrl"). +-include_lib("esaml/include/esaml.hrl"). +-include_lib("kernel/include/logger.hrl"). + + +-type assume_role_with_saml_error() :: expired_token + | idp_rejected_claim + | invalid_identity_token + | malformed_policy_document + | packed_policy_too_large + | region_disabled. + +-spec assume_role_with_saml(maps:map(), pid()) -> {ok, maps:map()} | {error, assume_role_with_saml_error()}. +assume_role_with_saml(Specs, Pbc) -> + Res = lists:foldl( + fun(StepF, State) -> StepF(State) end, + #{pbc => Pbc, + specs => Specs}, + [fun validate_args/1, + fun check_role/1, + fun parse_saml_assertion_claims/1, + fun check_with_saml_provider/1, + fun create_session_and_issue_temp_creds/1]), + case Res of + #{status := ok} -> + {ok, Res}; + #{status := NotOk} -> + NotOk + end. + +validate_args(#{specs := Specs} = State) -> + case lists:foldl(fun(_Fun, {error, _} = E) -> E; + (Fun, ok) -> Fun(Specs) end, + ok, + [fun validate_duration_seconds/1, + fun validate_policy/1, + fun validate_policy_arns/1, + fun validate_principal_arn/1, + fun validate_role_arn/1, + fun validate_saml_assertion/1]) of + ok -> + State#{status => ok}; + {error, Reason} -> + State#{status => {error, Reason}} + end. +validate_duration_seconds(#{duration_seconds := A}) -> + case A >= 900 andalso A =< 43200 of + true -> + ok; + false -> + logger:warning("Unacceptable value for DurationSeconds: ~p", [A]), + {error, invalid_parameter_value} + end; +validate_duration_seconds(#{}) -> + ok. + +validate_policy(#{policy := A}) -> + case nomatch /= re:run(A, "[\u0009\u000A\u000D\u0020-\u00FF]+") + andalso size(A) >= 1 + andalso size(A) =< 2048 of + true -> + ok; + false -> + logger:warning("Unacceptable value for Policy: ~p", [A]), + {error, invalid_parameter_value} + end; +validate_policy(#{}) -> + ok. + +validate_policy_arns(#{policy_arns := AA}) -> + case length(AA) =< 10 andalso + lists:all(fun is_valid_arn/1, AA) of + true -> + ok; + false -> + logger:warning("Invalid or too many PolicyArn members", []), + {error, invalid_parameter_value} + end; +validate_policy_arns(#{}) -> + ok. + +validate_principal_arn(#{principal_arn := A}) -> + case is_valid_arn(A) + andalso size(A) >= 20 + andalso size(A) =< 2048 of + true -> + ok; + false -> + logger:warning("Unacceptable value for PrincipalArn: ~p", [A]), + {error, invalid_parameter_value} + end; +validate_principal_arn(#{}) -> + logger:warning("Missing PrincipalArn parameter"), + {error, missing_parameter}. + +validate_role_arn(#{role_arn := A}) -> + case is_valid_arn(A) + andalso size(A) >= 20 + andalso size(A) =< 2048 of + true -> + ok; + false -> + logger:warning("Unacceptable value for RoleArn: ~p", [A]), + {error, invalid_parameter_value} + end. +validate_saml_assertion(#{saml_assertion := A}) -> + case size(A) >= 4 andalso size(A) =< 100000 of + true -> + ok; + false -> + logger:warning("Unacceptable value for SAMLAssertion: ~p", [A]), + {error, invalid_parameter_value} + end; +validate_saml_assertion(#{}) -> + logger:warning("Missing SAMLAssertion parameter"), + {error, missing_parameter}. + + + +check_role(#{status := {error, _}} = PreviousStepFailed) -> + PreviousStepFailed; +check_role(#{pbc := Pbc, + specs := #{role_arn := RoleArn}} = State) -> + case riak_cs_iam:get_role(RoleArn, Pbc) of + {ok, Role} -> + State#{status => ok, + role => Role}; + ER -> + State#{status => ER} + end. + + +%% Since we have IdP metadata (from previous calls to +%% CreateSAMLProvider), we can, and will, use esaml decoding and +%% validation facilities to emulate a SP without actuallay talking to +%% the IdP described in the SAMLMetadataDocument. This appears to be +%% sufficient except for the case of encrypted SAML assertions, for +%% which we need a private key which we currently have no way to +%% obtain via an IAM or STS call. + +parse_saml_assertion_claims(#{status := {error, _}} = PreviousStepFailed) -> + PreviousStepFailed; +parse_saml_assertion_claims(#{specs := #{request_id := RequestId, + saml_assertion := SAMLAssertion_}} = State0) -> + SAMLAssertion = base64:decode(SAMLAssertion_), + {Doc, _} = xmerl_scan:string(binary_to_list(SAMLAssertion)), + case esaml:decode_response(Doc) of + {ok, #esaml_response{} = SAMLResponse} -> + parse_saml_assertion_claims( + SAMLResponse, State0#{response_doc => Doc}); + {error, Reason} -> + logger:warning("Failed to parse Response document in request ~s: ~p", [RequestId, Reason]), + State0#{status => {error, idp_rejected_claim}} + end. +parse_saml_assertion_claims(#esaml_response{issuer = Issuer, + assertion = Assertion, + version = _Version}, State0) -> + #esaml_assertion{subject = #esaml_subject{name = SubjectName, + name_format = SubjectNameFormat}, + conditions = Conditions, + attributes = Attributes} = Assertion, + + %% optional fields + RoleSessionName = + proplists:get_value("https://aws.amazon.com/SAML/Attributes/RoleSessionName", + Attributes), + SessionDuration = + proplists:get_value("https://aws.amazon.com/SAML/Attributes/SessionDuration", + Attributes), + SourceIdentity = + proplists:get_value("https://aws.amazon.com/SAML/Attributes/SourceIdentity", + Attributes), + Role = + proplists:get_value("https://aws.amazon.com/SAML/Attributes/Role", + Attributes, []), + Email = + proplists:get_value("mail", + Attributes, []), + + Audience = proplists:get_value(audience, Conditions), + + State1 = State0#{status => ok, + issuer => list_to_binary(Issuer), + subject => list_to_binary(SubjectName), + subject_type => list_to_binary(amazonize(SubjectNameFormat)), + audience => list_to_binary(Audience)}, + maybe_update_state_with([{role_session_name, maybe_list_to_binary(RoleSessionName)}, + {source_identity, maybe_list_to_binary(SourceIdentity)}, + {email, maybe_list_to_binary(Email)}, + {session_duration, maybe_list_to_integer(SessionDuration)}, + {claims_role, [maybe_list_to_binary(A) || A <- Role]}], State1). + +maybe_list_to_integer(undefined) -> undefined; +maybe_list_to_integer(A) -> list_to_integer(A). +maybe_list_to_binary(undefined) -> undefined; +maybe_list_to_binary(A) -> list_to_binary(A). + +amazonize("urn:oasis:names:tc:SAML:2.0:nameid-format:" ++ A) -> A; +amazonize(A) -> A. + + + +check_with_saml_provider(#{status := {error, _}} = PreviousStepFailed) -> + PreviousStepFailed; +check_with_saml_provider(#{pbc := Pbc, + response_doc := ResponseDoc, + specs := #{request_id := RequestId, + principal_arn := PrincipalArn} + } = State) -> + case riak_cs_iam:get_saml_provider(PrincipalArn, Pbc) of + {ok, SP = ?IAM_SAML_PROVIDER{name = SAMLProviderName}} -> + State#{status => validate_assertion(ResponseDoc, SP, RequestId), + saml_provider_name => SAMLProviderName}; + {error, notfound} -> + State#{status => {error, no_such_saml_provider}} + end. + +validate_assertion(ResponseDoc, ?IAM_SAML_PROVIDER{certificates = Certs, + consume_uri = ConsumeUri, + entity_id = EntityId}, + RequestId) -> + FPs = lists:flatten([FP || {signing, _, FP} <- Certs]), + SP = #esaml_sp{trusted_fingerprints = FPs, + entity_id = binary_to_list(EntityId), + idp_signs_assertions = false, + idp_signs_envelopes = false, + consume_uri = binary_to_list(ConsumeUri)}, + case esaml_sp:validate_assertion(ResponseDoc, SP) of + {ok, _} -> + ok; + {error, Reason} -> + logger:warning("Failed to validate SAML Assertion for AssumeRoleWithSAML call on request ~s: ~p", [RequestId, Reason]), + {error, idp_rejected_claim} + end. + +create_session_and_issue_temp_creds(#{status := {error, _}} = PreviousStepFailed) -> + PreviousStepFailed; +create_session_and_issue_temp_creds(#{specs := #{duration_seconds := DurationSeconds} = Specs, + issuer := Issuer, + saml_provider_name := SAMLProviderName, + audience := Audience, + role := Role, + subject := Subject, + subject_type := SubjectType, + pbc := Pbc} = State) -> + SourceIdentity = maps:get(source_identity, State, <<>>), + Email = maps:get(email, State, <<>>), + InlinePolicy = maps:get(policy, Specs, undefined), + PolicyArns = maps:get(policy_arns, Specs, []), + + case riak_cs_temp_sessions:create( + Role, Subject, SourceIdentity, Email, DurationSeconds, InlinePolicy, PolicyArns, Pbc) of + {ok, #temp_session{assumed_role_user = AssumedRoleUser, + credentials = Credentials}} -> + State#{status => ok, + assumed_role_user => AssumedRoleUser, + audience => Audience, + credentials => Credentials, + name_qualifier => base64:encode( + crypto:hash( + sha, iolist_to_binary( + [Issuer, account_id_of_role(Role), $/, SAMLProviderName]))), + packed_policy_size => 6, + subject => Subject, + subject_type => SubjectType, + source_identity => SourceIdentity}; + ER -> + State#{status => ER} + end. + +account_id_of_role(?IAM_ROLE{} = R) -> + ?LOG_DEBUG("STUB ~p", [R]), + + "123456789012". + + +maybe_update_state_with([], State) -> + State; +maybe_update_state_with([{_, undefined}|Rest], State) -> + maybe_update_state_with(Rest, State); +maybe_update_state_with([{P, V}|Rest], State) -> + maybe_update_state_with(Rest, maps:put(P, V, State)). + + + + +is_valid_arn(A) -> + nomatch /= re:run(A, "[\u0009\u000A\u000D\u0020-\u007E\u0085\u00A0-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]+"). + + +-ifdef(TEST). +-compile([export_all, nowarn_export_all]). +-include_lib("eunit/include/eunit.hrl"). +-endif. diff --git a/apps/riak_cs/src/riak_cs_sup.erl b/apps/riak_cs/src/riak_cs_sup.erl new file mode 100644 index 000000000..fc7a6a24d --- /dev/null +++ b/apps/riak_cs/src/riak_cs_sup.erl @@ -0,0 +1,221 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc Supervisor for the riak_cs application. + +-module(riak_cs_sup). + +-behaviour(supervisor). + +-export([start_link/0]). +-export([init/1]). + +-include("riak_cs.hrl"). + +-define(OPTIONS, [connection_pools, + listener, + admin_listener, + ssl, + admin_ssl, + {stanchion_hosting_mode, auto}]). + +-spec start_link() -> supervisor:startlink_ret(). +start_link() -> + riak_cs_stats:init(), + + RewriteMod = application:get_env(riak_cs, rewrite_module, ?AWS_API_MOD), + ok = application:set_env(webmachine_mochiweb, rewrite_modules, [{object_web, RewriteMod}]), + + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + + +-spec init([]) -> {ok, {supervisor:sup_flags(), + [supervisor:child_spec()]}}. +init([]) -> + Options = [get_option_val(Option) || Option <- ?OPTIONS], + init2(Options). + +init2(Options) -> + Mode = proplists:get_value(stanchion_hosting_mode, Options), + RCSChildren = + case Mode of + M when M == auto; + M == riak_cs_only; + M == riak_cs_with_stanchion -> + pool_specs(Options) ++ rcs_process_specs() ++ web_specs(Options); + _ -> + [] + end, + {ok, {#{strategy => one_for_one, + intensity => 10, + period => 10}, RCSChildren + }}. + +rcs_process_specs() -> + [ #{id => riak_cs_access_archiver_manager, + start => {riak_cs_access_archiver_manager, start_link, []}}, + + #{id => riak_cs_storage_d, + start => {riak_cs_storage_d, start_link, []}}, + + #{id => riak_cs_gc_manager, + start => {riak_cs_gc_manager, start_link, []}}, + + #{id => riak_cs_delete_fsm_sup, + start => {riak_cs_delete_fsm_sup, start_link, []}, + type => supervisor, + modules => dynamic}, + + #{id => riak_cs_get_fsm_sup, + start => {riak_cs_get_fsm_sup, start_link, []}, + modules => dynamic}, + + #{id => riak_cs_put_fsm_sup, + start => {riak_cs_put_fsm_sup, start_link, []}, + type => supervisor, + modules => dynamic}, + + #{id => riak_cs_diags, + start => {riak_cs_diags, start_link, []}, + modules => dynamic}, + + #{id => riak_cs_quota_sup, + start => {riak_cs_quota_sup, start_link, []}, + type => supervisor, + modules => dynamic}, + + #{id => stanchion_sup, + start => {stanchion_sup, start_link, []}, + type => supervisor, + modules => dynamic}, + + #{id => stanchion_lock, + start => {stanchion_lock, start_link, []}} + ] + ++ riak_cs_mb_helper:process_specs(). + +get_option_val({Option, Default}) -> + handle_get_env_result(Option, get_env(Option), Default); +get_option_val(Option) -> + get_option_val({Option, undefined}). + +get_env(Key) -> + application:get_env(riak_cs, Key). + +handle_get_env_result(Option, {ok, Value}, _) -> + {Option, Value}; +handle_get_env_result(Option, undefined, Default) -> + {Option, Default}. + +web_specs(Options) -> + WebConfigs = + case single_web_interface(proplists:get_value(admin_listener, Options)) of + true -> + [{riak_cs_object_web, add_admin_dispatch_table(object_web_config(Options))}]; + false -> + [{riak_cs_admin_web, admin_web_config(Options)}, + {riak_cs_object_web, object_web_config(Options)}] + end, + [web_spec(Name, Config) || {Name, Config} <- WebConfigs]. + +pool_specs(Options) -> + rc_pool_specs(Options) ++ + pbc_pool_specs(Options). + +rc_pool_specs(Options) -> + WorkerStop = fun(Worker) -> riak_cs_riak_client:stop(Worker) end, + MasterPools = proplists:get_value(connection_pools, Options), + [#{id => Name, + start => {poolboy, start_link, [[{name, {local, Name}}, + {worker_module, riak_cs_riak_client}, + {size, Workers}, + {max_overflow, Overflow}, + {stop_fun, WorkerStop}], + []]}} + || {Name, {Workers, Overflow}} <- MasterPools]. + +pbc_pool_specs(Options) -> + WorkerStop = fun(Worker) -> riak_cs_riakc_pool_worker:stop(Worker) end, + %% Use sums of fixed/overflow for pbc pool + MasterPools = proplists:get_value(connection_pools, Options), + {FixedSum, OverflowSum} = lists:foldl(fun({_, {Fixed, Overflow}}, {FAcc, OAcc}) -> + {Fixed + FAcc, Overflow + OAcc} + end, + {0, 0}, MasterPools), + riak_cs_config:set_multibag_appenv(), + Bags = riak_cs_mb_helper:bags(), + [pbc_pool_spec(BagId, FixedSum, OverflowSum, Address, Port, WorkerStop) + || {BagId, Address, Port} <- Bags]. + +pbc_pool_spec(BagId, Fixed, Overflow, Address, Port, WorkerStop) -> + Name = riak_cs_riak_client:pbc_pool_name(BagId), + #{id => Name, + start => {poolboy, start_link, [[{name, {local, Name}}, + {worker_module, riak_cs_riakc_pool_worker}, + {size, Fixed}, + {max_overflow, Overflow}, + {stop_fun, WorkerStop}], + [{address, Address}, + {port, Port}]]}}. + +web_spec(Name, Config) -> + #{id => Name, + start => {webmachine_mochiweb, start, [Config]}, + modules => dynamic}. + +object_web_config(Options) -> + {IP, Port} = proplists:get_value(listener, Options), + [{dispatch, riak_cs_web:object_api_dispatch_table()}, + {name, object_web}, + {dispatch_group, object_web}, + {ip, IP}, + {port, Port}, + {nodelay, true}, + {error_handler, riak_cs_wm_error_handler}, + {resource_module_option, submodule}] ++ + maybe_add_ssl_opts(proplists:get_value(ssl, Options)). + +admin_web_config(Options) -> + {IP, Port} = proplists:get_value(admin_listener, + Options, {"127.0.0.1", 8000}), + [{dispatch, riak_cs_web:admin_api_dispatch_table()}, + {name, admin_web}, + {dispatch_group, admin_web}, + {ip, IP}, {port, Port}, + {nodelay, true}, + {error_handler, riak_cs_wm_error_handler}] ++ + maybe_add_ssl_opts(proplists:get_value(admin_ssl, Options)). + +single_web_interface(undefined) -> + true; +single_web_interface(_) -> + false. + +maybe_add_ssl_opts(undefined) -> + []; +maybe_add_ssl_opts(SSLOpts) -> + [{ssl, true}, {ssl_opts, SSLOpts}]. + +add_admin_dispatch_table(Config) -> + UpdDispatchTable = proplists:get_value(dispatch, Config) ++ + riak_cs_web:admin_api_dispatch_table(), + [{dispatch, UpdDispatchTable} | proplists:delete(dispatch, Config)]. + diff --git a/apps/riak_cs/src/riak_cs_temp_sessions.erl b/apps/riak_cs/src/riak_cs_temp_sessions.erl new file mode 100644 index 000000000..556ba712e --- /dev/null +++ b/apps/riak_cs/src/riak_cs_temp_sessions.erl @@ -0,0 +1,196 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_temp_sessions). + +-export([create/8, + get/1, get/2, + list/2, + effective_policies/2, + close_session/1 + ]). + +-include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-define(USER_ID_LENGTH, 16). %% length("ARO456EXAMPLE789"). + + +-spec create(role(), binary(), binary(), binary(), non_neg_integer(), binary(), [binary()], pid()) -> + {ok, temp_session()} | {error, term()}. +create(?IAM_ROLE{role_id = RoleId, + role_name = RoleName} = Role, + Subject, SourceIdentity, Email, DurationSeconds, InlinePolicy_, PolicyArns, Pbc) -> + + UserId = riak_cs_aws_utils:make_id(?USER_ID_LENGTH, ?USER_ID_PREFIX), + {KeyId, AccessKey} = riak_cs_aws_utils:generate_access_creds(UserId), + CanonicalId = riak_cs_aws_utils:generate_canonical_id(KeyId), + + SessionName = riak_cs_aws_utils:make_id(16), + Arn = riak_cs_aws_utils:make_assumed_role_user_arn(RoleName, SessionName), + AssumedRoleUser = #assumed_role_user{arn = Arn, + assumed_role_id = <>}, + + InlinePolicy = case InlinePolicy_ of + undefined -> + undefined; + _ -> + base64:decode(InlinePolicy_) + end, + Session = #temp_session{assumed_role_user = AssumedRoleUser, + role = Role, + credentials = #credentials{access_key_id = KeyId, + secret_access_key = AccessKey, + expiration = os:system_time(millisecond) + DurationSeconds * 1000, + session_token = make_session_token()}, + duration_seconds = DurationSeconds, + subject = Subject, + source_identity = SourceIdentity, + email = Email, + user_id = UserId, + canonical_id = CanonicalId, + inline_policy = InlinePolicy, + session_policies = PolicyArns}, + + %% rather than using an arn for key, consistently with all other + %% things in IAM, let's use access_key_id here, to make it easier + %% to check for existing assumed role sessions from + %% riak_cs_user:get_user/2 where we only have the key_id. + Obj = riakc_obj:new(?TEMP_SESSIONS_BUCKET, KeyId, term_to_binary(Session)), + case riakc_pb_socket:put(Pbc, Obj, ?CONSISTENT_WRITE_OPTIONS) of + ok -> + logger:info("Opened new temp session for user ~s with key_id ~s", [UserId, KeyId]), + {ok, Session}; + {error, Reason} = ER -> + logger:error("Failed to save temp session: ~p", [Reason]), + ER + end. + + +-spec list(riak_client(), #list_temp_sessions_request{}) -> + {ok, maps:map()} | {error, term()}. +list(RcPid, #list_temp_sessions_request{max_items = MaxItems, + marker = Marker}) -> + Arg = #{max_items => MaxItems, + marker => Marker}, + {ok, MasterPbc} = riak_cs_riak_client:master_pbc(RcPid), + case riakc_pb_socket:mapred_bucket( + MasterPbc, ?TEMP_SESSIONS_BUCKET, riak_cs_riak_mapred:query(temp_sessions, Arg)) of + {ok, Batches} -> + {ok, #{temp_sessions => extract_objects(Batches, []), + marker => undefined, + is_truncated => false}}; + {error, _} = ER -> + ER + end. + +extract_objects([], Q) -> + Q; +extract_objects([{_N, RR}|Rest], Q) -> + extract_objects(Rest, Q ++ RR). + + +-spec effective_policies(#temp_session{}, pid()) -> {[policy()], PermissionsBoundary::policy() | []}. +effective_policies(#temp_session{inline_policy = InlinePolicy, + session_policies = SessionPolicies, + role = ?IAM_ROLE{assume_role_policy_document = AssumeRolePolicyDocument, + permissions_boundary = PermissionsBoundary, + attached_policies = RoleAttachedPolicies}}, + Pbc) -> + AttachedPolicies = + riak_cs_iam:express_policies(SessionPolicies ++ RoleAttachedPolicies, Pbc), + AllPolicies = lists:flatten( + [maybe_include(P) || P <- [AssumeRolePolicyDocument, InlinePolicy | AttachedPolicies]]), + ?LOG_DEBUG("Effective policies: ~p", [AllPolicies]), + {AllPolicies, maybe_include(PermissionsBoundary)}. + +maybe_include(undefined) -> []; +maybe_include(A) -> + case riak_cs_aws_policy:policy_from_json(A) of + {ok, P} -> + P; + {error, _} -> + [] + end. + + +-spec get(binary(), pid()) -> {ok, temp_session()} | {error, term()}. +get(KeyId) -> + {ok, Pbc} = riak_cs_utils:riak_connection(), + try + get(KeyId, Pbc) + after + riak_cs_utils:close_riak_connection(Pbc) + end. + +get(KeyId, Pbc) -> + case riakc_pb_socket:get(Pbc, ?TEMP_SESSIONS_BUCKET, KeyId) of + {ok, Obj} -> + check_expired( + KeyId, session_from_riakc_obj(Obj)); + ER -> + ER + end. +check_expired(KeyId, {ok, #temp_session{created = Created, + duration_seconds = DurationSeconds} = A}) -> + case os:system_time(millisecond) > Created + DurationSeconds * 1000 of + true -> + _ = close_session(KeyId), + {error, notfound}; + false -> + {ok, A} + end; +check_expired(_, ER) -> + ER. + + +session_from_riakc_obj(Obj) -> + case [binary_to_term(Value) || Value <- riakc_obj:get_values(Obj), + Value /= <<>>] of + [] -> + {error, notfound}; + [S] -> + {ok, S}; + [S|_] = VV -> + logger:warning("Temp session object for user ~s has ~b siblings", + [S#temp_session.user_id, length(VV)]), + {ok, S} + end. + +make_session_token() -> + ?LOG_DEBUG("STUB"), + + riak_cs_aws_utils:make_id(80). + +close_session(Id) -> + {ok, Pbc} = riak_cs_utils:riak_connection(), + try + case riakc_pb_socket:delete(Pbc, ?TEMP_SESSIONS_BUCKET, Id, ?CONSISTENT_DELETE_OPTIONS) of + ok -> + logger:info("Deleted temp session for user with key_id ~s", [Id]), + ok; + {error, Reason} -> + logger:warning("Failed to delete temp session with key_id ~s: ~p", [Id, Reason]), + still_ok + end + after + riak_cs_utils:close_riak_connection(Pbc) + end. + diff --git a/apps/riak_cs/src/riak_cs_user.erl b/apps/riak_cs/src/riak_cs_user.erl new file mode 100644 index 000000000..4789b4aac --- /dev/null +++ b/apps/riak_cs/src/riak_cs_user.erl @@ -0,0 +1,260 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc riak_cs user related functions + +-module(riak_cs_user). + +%% Public API +-export([create_user/2, + create_user/3, + create_user/5, + display_name/1, + is_admin/1, + get_user/2, + get_cs_user/2, + from_riakc_obj/2, + to_3tuple/1, + update_key_secret/1, + fetch_user_keys/1 + ]). + +-include("riak_cs.hrl"). +-include_lib("riakc/include/riakc.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. + +%% =================================================================== +%% Public API +%% =================================================================== + +%% @doc Create a new Riak CS user +-spec create_user(binary(), binary()) -> {ok, rcs_user()} | {error, term()}. +create_user(Name, Email) -> + create_user(Name, Email, #{}). + +-spec create_user(binary(), binary(), maps:map()) -> {ok, rcs_user()} | {error, term()}. +create_user(Name, Email, IAMExtra) -> + {KeyId, Secret} = riak_cs_aws_utils:generate_access_creds(Email), + create_user(Name, Email, KeyId, Secret, IAMExtra). + +%% @doc Create a new Riak CS/IAM user +-spec create_user(binary(), binary(), binary(), binary(), maps:map()) -> + {ok, rcs_user()} | {error, term()}. +create_user(Name, Email, KeyId, Secret, IAMExtra) -> + Precious = stanchion_lock:acquire(KeyId), + try validate_email(Email) of + ok -> + CanonicalId = riak_cs_aws_utils:generate_canonical_id(KeyId), + DisplayName = display_name(Email), + Path = maps:get(path, IAMExtra, <<"/">>), + Arn = riak_cs_aws_utils:make_user_arn(Name, Path), + User = ?RCS_USER{arn = Arn, + name = Name, + path = Path, + permissions_boundary = maps:get(permissions_boundary, IAMExtra, undefined), + tags = [exprec:frommap_tag(T) || T <- maps:get(tags, IAMExtra, [])], + display_name = DisplayName, + email = Email, + key_id = KeyId, + key_secret = Secret, + id = CanonicalId}, + create_credentialed_user(User); + {error, _Reason} = Error -> + Error + after + stanchion_lock:release(KeyId, Precious) + end. + +create_credentialed_user(User) -> + %% Make a call to the user request serialization service. + {ok, AdminCreds} = riak_cs_config:admin_creds(), + StatsKey = [velvet, create_user], + _ = riak_cs_stats:inflow(StatsKey), + StartTime = os:system_time(millisecond), + Result = velvet:create_user("application/json", + riak_cs_json:to_json(User), + [{auth_creds, AdminCreds}]), + _ = riak_cs_stats:update_with_start(StatsKey, StartTime, Result), + handle_create_user(Result, User). + +handle_create_user(ok, User) -> + {ok, User}; +handle_create_user({error, _} = Error, _User) -> + Error. + + +%% @doc Retrieve a Riak CS user's information based on their id string. +-spec get_user(undefined | binary(), riak_client()) -> + {ok, {rcs_user(), undefined | riakc_obj:riakc_obj()}} | {error, no_user_key | riak_obj_error()}. +get_user(undefined, _) -> + {error, no_user_key}; +get_user(KeyId, RcPid) -> + case riak_cs_temp_sessions:get(KeyId) of + {ok, #temp_session{assumed_role_user = #assumed_role_user{arn = AssumedRoleUserArn}, + credentials = #credentials{secret_access_key = SecretKey}, + canonical_id = CanonicalId, + subject = Subject, + source_identity = SourceIdentity, + email = Email}} -> + {ok, {?RCS_USER{arn = AssumedRoleUserArn, + attached_policies = [], + name = Subject, + email = select_email([SourceIdentity, Email]), + display_name = Subject, + id = CanonicalId, + key_id = KeyId, + key_secret = SecretKey, + buckets = []}, + _UserObject = undefined}}; + {error, notfound} -> + get_cs_user(KeyId, RcPid) + end. + + +get_cs_user(KeyId, RcPid) -> + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + riak_cs_iam:find_user(#{key_id => KeyId}, Pbc). + + +-spec from_riakc_obj(riakc_obj:riakc_obj(), boolean()) -> rcs_user(). +from_riakc_obj(Obj, KeepDeletedBuckets) -> + case riakc_obj:value_count(Obj) of + 1 -> + Value = binary_to_term(riakc_obj:get_value(Obj)), + User = upgrade_user_record(Value), + Buckets = riak_cs_bucket:resolve_buckets([User], [], KeepDeletedBuckets), + User?RCS_USER{buckets = Buckets}; + 0 -> + error(no_value); + N -> + Values = [binary_to_term(Value) || + Value <- riakc_obj:get_values(Obj), + Value /= <<>> % tombstone + ], + [User|_] = Users = [upgrade_user_record(A) || A <- Values], + + KeyId = User?RCS_USER.key_id, + logger:warning("User object of '~s' has ~p siblings", [KeyId, N]), + + Buckets = riak_cs_bucket:resolve_buckets(Users, [], KeepDeletedBuckets), + User?RCS_USER{buckets = Buckets} + end. + +%% @doc Determine if the specified user account is a system admin. +-spec is_admin(rcs_user()) -> boolean(). +is_admin(User) -> + is_admin(User, riak_cs_config:admin_creds()). +is_admin(?RCS_USER{key_id = KeyId, key_secret = KeySecret}, + {ok, {KeyId, KeySecret}}) -> + true; +is_admin(_, _) -> + false. + +-spec to_3tuple(rcs_user()) -> acl_owner(). +to_3tuple(?RCS_USER{display_name = DisplayName, + id = CanonicalId, + key_id = KeyId}) -> + %% acl_owner3: {display name, canonical id, key id} + #{display_name => DisplayName, + canonical_id => CanonicalId, + key_id => KeyId}. + + +%% @doc Generate a new `key_secret' for a user record. +-spec update_key_secret(rcs_user()) -> rcs_user(). +update_key_secret(User = ?RCS_USER{email = Email, + key_id = KeyId}) -> + User?RCS_USER{key_secret = riak_cs_aws_utils:generate_secret(Email, KeyId)}. + +%% @doc Strip off the user name portion of an email address +-spec display_name(binary()) -> binary(). +display_name(Email) -> + hd(binary:split(Email, <<"@">>)). + +%% @doc Grab the whole list of Riak CS user keys. +-spec fetch_user_keys(riak_client()) -> {ok, [binary()]} | {error, term()}. +fetch_user_keys(RcPid) -> + {ok, MasterPbc} = riak_cs_riak_client:master_pbc(RcPid), + Timeout = riak_cs_config:list_keys_list_users_timeout(), + riak_cs_pbc:list_keys(MasterPbc, ?USER_BUCKET, Timeout, + [riakc, list_all_user_keys]). + +%% =================================================================== +%% Internal functions +%% =================================================================== + +validate_email(EmailAddr) -> + case re:run(EmailAddr, "^[a-z0-9]+[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,17}$", [caseless]) of + nomatch -> + {error, invalid_email_address}; + _ -> + ok + end. + +select_email([]) -> + <<"unspecified">>; +select_email([A|T]) -> + case validate_email(A) of + ok -> + A; + _ -> + select_email(T) + end. + +upgrade_user_record(#rcs_user_v3{} = A) -> + A; +upgrade_user_record(#rcs_user_v2{name = Name, + display_name = DisplayName, + email = Email, + key_id = KeyId, + key_secret = KeySecret, + canonical_id = CanonicalId, + buckets = Buckets}) -> + ?LOG_DEBUG("Upgrading rcs_user_v2", []), + #rcs_user_v3{arn = riak_cs_aws_utils:make_user_arn(Name, <<"/">>), + name = list_to_binary(Name), + display_name = list_to_binary(DisplayName), + email = list_to_binary(Email), + key_id = list_to_binary(KeyId), + key_secret = list_to_binary(KeySecret), + id = list_to_binary(CanonicalId), + buckets = [riak_cs_bucket:upgrade_bucket_record(B) || B <- Buckets]}; +upgrade_user_record(#moss_user_v1{name = Name, + display_name = DisplayName, + email = Email, + key_id = KeyId, + key_secret = KeySecret, + canonical_id = CanonicalId, + buckets = Buckets}) -> + ?LOG_DEBUG("Upgrading moss_user_v1", []), + #rcs_user_v3{arn = riak_cs_aws_utils:make_user_arn(Name, <<"/">>), + name = list_to_binary(Name), + display_name = list_to_binary(DisplayName), + email = list_to_binary(Email), + key_id = list_to_binary(KeyId), + key_secret = list_to_binary(KeySecret), + id = list_to_binary(CanonicalId), + buckets = [riak_cs_bucket:upgrade_bucket_record(B) || B <- Buckets]}. diff --git a/src/riak_cs_utils.erl b/apps/riak_cs/src/riak_cs_utils.erl similarity index 70% rename from src/riak_cs_utils.erl rename to apps/riak_cs/src/riak_cs_utils.erl index 8b37a1fbb..a82b60e4f 100644 --- a/src/riak_cs_utils.erl +++ b/apps/riak_cs/src/riak_cs_utils.erl @@ -1,6 +1,7 @@ -%% --------------------------------------------------------------------- +%% ------------------------------------------------------------------- %% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2024 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -16,7 +17,7 @@ %% specific language governing permissions and limitations %% under the License. %% -%% --------------------------------------------------------------------- +%% ------------------------------------------------------------------- %% @doc riak_cs utility functions @@ -28,22 +29,18 @@ etag_from_binary_no_quotes/1, close_riak_connection/1, close_riak_connection/2, - delete_object/3, + delete_object/4, encode_term/1, has_tombstone/1, - map_keys_and_manifests/3, - maybe_process_resolved/3, sha_mac/2, sha/1, md5/1, md5_init/0, md5_update/2, md5_final/1, - reduce_keys_and_manifests/2, active_manifest_from_response/1, hexlist_to_binary/1, binary_to_hexlist/1, - json_pp_print/1, key_exists/3, n_val_1_get_requests/0, pow/2, @@ -51,30 +48,34 @@ resolve_robj_siblings/1, riak_connection/0, riak_connection/1, - safe_base64_decode/1, safe_base64url_decode/1, safe_list_to_integer/1, - set_object_acl/5, - second_resolution_timestamp/1, - timestamp_to_seconds/1, - timestamp_to_milliseconds/1, + set_object_acl/6, update_obj_value/2, pid_to_binary/1, + parse_x509_cert/1, from_bucket_name/1, to_bucket_name/2, big_end_key/1, big_end_key/0, stanchion_data/0, camel_case/1, - capitalize/1 + capitalize/1, + object_indices/1, + gather_disk_usage_on_connected_riak_nodes/0, + get_storage_info/0, + guess_riak_node_name/0 ]). -include("riak_cs.hrl"). +-include_lib("public_key/include/public_key.hrl"). -include_lib("riak_pb/include/riak_pb_kv_codec.hrl"). -include_lib("riakc/include/riakc.hrl"). +-include_lib("kernel/include/logger.hrl"). -ifdef(TEST). -compile(export_all). +-compile(nowarn_export_all). -include_lib("eunit/include/eunit.hrl"). -endif. @@ -84,6 +85,8 @@ -define(is_indent(C), (C == 91) orelse (C == 123)). % [, { -define(is_undent(C), (C == 93) orelse (C == 125)). % ], } +-type digest() :: binary(). + %% =================================================================== %% Public API %% =================================================================== @@ -159,10 +162,10 @@ close_riak_connection(Pool, Pid) -> %% Garbage collection. Otherwise returns an error. Note, %% {error, notfound} counts as success in this case, %% with the list of UUIDs being []. --spec delete_object(binary(), binary(), riak_client()) -> +-spec delete_object(binary(), binary(), binary(), riak_client()) -> {ok, [binary()]} | {error, term()}. -delete_object(Bucket, Key, RcPid) -> - riak_cs_gc:gc_active_manifests(Bucket, Key, RcPid). +delete_object(Bucket, Key, ObjVsn, RcPid) -> + riak_cs_gc:gc_active_manifests(Bucket, Key, ObjVsn, RcPid). -spec encode_term(term()) -> binary(). encode_term(Term) -> @@ -173,51 +176,8 @@ encode_term(Term) -> term_to_binary(Term) end. -%% MapReduce function, runs on the Riak nodes, should therefore use -%% riak_object, not riakc_obj. -map_keys_and_manifests({error, notfound}, _, _) -> - []; -map_keys_and_manifests(Object, _, _) -> - Handler = fun(Resolved) -> - case riak_cs_manifest_utils:active_manifest(Resolved) of - {ok, Manifest} -> - [{riak_object:key(Object), {ok, Manifest}}]; - _ -> - [] - end - end, - maybe_process_resolved(Object, Handler, []). - -maybe_process_resolved(Object, ResolvedManifestsHandler, ErrorReturn) -> - try - AllManifests = [ binary_to_term(V) - || {_, V} = Content <- riak_object:get_contents(Object), - not has_tombstone(Content) ], - Upgraded = riak_cs_manifest_utils:upgrade_wrapped_manifests(AllManifests), - Resolved = riak_cs_manifest_resolution:resolve(Upgraded), - ResolvedManifestsHandler(Resolved) - catch Type:Reason -> - StackTrace = erlang:get_stacktrace(), - _ = lager:log(error, - self(), - "Riak CS object mapreduce failed for ~p:~p with reason ~p:~p" - "at ~p", - [riak_object:bucket(Object), - riak_object:key(Object), - Type, - Reason, - StackTrace]), - ErrorReturn - end. - -%% Pipe all the bucket listing results through a passthrough reduce -%% phase. This is just a temporary kludge until the sink backpressure -%% work is done. -reduce_keys_and_manifests(Acc, _) -> - Acc. - -spec sha_mac(iolist() | binary(), iolist() | binary()) -> binary(). -sha_mac(Key,STS) -> crypto:hmac(sha, Key,STS). +sha_mac(Key,STS) -> crypto:mac(hmac, sha, Key,STS). -spec sha(binary()) -> binary(). sha(Bin) -> crypto:hash(sha, Bin). @@ -228,30 +188,25 @@ md5(IOData) -> -define(MAX_UPDATE_SIZE, (32*1024)). --spec md5_init() -> crypto_context(). +-spec md5_init() -> crypto:hash_state(). md5_init() -> crypto:hash_init(md5). --spec md5_update(crypto_context(), binary()) -> crypto_context(). +-spec md5_update(crypto:hash_state(), binary()) -> crypto:hash_state(). md5_update(Ctx, Bin) when size(Bin) =< ?MAX_UPDATE_SIZE -> crypto:hash_update(Ctx, Bin); md5_update(Ctx, <>) -> md5_update(crypto:hash_update(Ctx, Part), Rest). --spec md5_final(crypto_context()) -> digest(). +-spec md5_final(crypto:hash_state()) -> digest(). md5_final(Ctx) -> crypto:hash_final(Ctx). -spec active_manifest_from_response({ok, orddict:orddict()} | {error, notfound}) -> {ok, lfs_manifest()} | {error, notfound}. active_manifest_from_response({ok, Manifests}) -> - handle_active_manifests(riak_cs_manifest_utils:active_manifest(Manifests)); + handle_active_manifests(rcs_common_manifest_utils:active_manifest(Manifests)); active_manifest_from_response({error, notfound}=NotFound) -> NotFound. - -%% @private --spec handle_active_manifests({ok, lfs_manifest()} | - {error, no_active_manifest}) -> - {ok, lfs_manifest()} | {error, notfound}. handle_active_manifests({ok, _Active}=ActiveReply) -> ActiveReply; handle_active_manifests({error, no_active_manifest}) -> @@ -264,30 +219,6 @@ has_tombstone({_, <<>>}) -> has_tombstone({MD, _V}) -> dict:is_key(?MD_DELETED, MD) =:= true. -%% @doc Pretty-print a JSON string ... from riak_core's json_pp.erl -json_pp_print(Str) when is_list(Str) -> - json_pp_print(Str, 0, undefined, []). - -json_pp_print([$\\, C| Rest], I, C, Acc) -> % in quote - json_pp_print(Rest, I, C, [C, $\\| Acc]); -json_pp_print([C| Rest], I, undefined, Acc) when ?is_quote(C) -> - json_pp_print(Rest, I, C, [C| Acc]); -json_pp_print([C| Rest], I, C, Acc) -> % in quote - json_pp_print(Rest, I, undefined, [C| Acc]); -json_pp_print([C| Rest], I, undefined, Acc) when ?is_indent(C) -> - json_pp_print(Rest, I+1, undefined, [json_pp_indent(I+1), $\n, C| Acc]); -json_pp_print([C| Rest], I, undefined, Acc) when ?is_undent(C) -> - json_pp_print(Rest, I-1, undefined, [C, json_pp_indent(I-1), $\n| Acc]); -json_pp_print([$,| Rest], I, undefined, Acc) -> - json_pp_print(Rest, I, undefined, [json_pp_indent(I), $\n, $,| Acc]); -json_pp_print([$:| Rest], I, undefined, Acc) -> - json_pp_print(Rest, I, undefined, [?SPACE, $:| Acc]); -json_pp_print([C|Rest], I, Q, Acc) -> - json_pp_print(Rest, I, Q, [C| Acc]); -json_pp_print([], _I, _Q, Acc) -> % done - lists:reverse(Acc). - -json_pp_indent(I) -> lists:duplicate(I*4, ?SPACE). -spec n_val_1_get_requests() -> boolean(). n_val_1_get_requests() -> @@ -312,8 +243,7 @@ pow(Base, Power, Acc) -> -type resolve_ok() :: {term(), binary()}. -type resolve_error() :: {atom(), atom()}. -spec resolve_robj_siblings(RObj::term()) -> - {resolve_ok() | resolve_error(), NeedsRepair::boolean()}. - + {resolve_ok() | resolve_error(), NeedsRepair::boolean()}. resolve_robj_siblings(Cs) -> [{BestRating, BestMDV}|Rest] = lists:sort([{rate_a_dict(MD, V), MDV} || {MD, V} = MDV <- Cs]), @@ -378,10 +308,10 @@ riak_connection(Pool) -> %% @doc Set the ACL for an object. Existing ACLs are only %% replaced, they cannot be updated. --spec set_object_acl(binary(), binary(), lfs_manifest(), acl(), riak_client()) -> - ok | {error, term()}. -set_object_acl(Bucket, Key, Manifest, Acl, RcPid) -> - {ok, ManiPid} = riak_cs_manifest_fsm:start_link(Bucket, Key, RcPid), +-spec set_object_acl(binary(), binary(), binary(), lfs_manifest(), acl(), riak_client()) -> + ok | {error, term()}. +set_object_acl(Bucket, Key, Vsn, Manifest, Acl, RcPid) -> + {ok, ManiPid} = riak_cs_manifest_fsm:start_link(Bucket, Key, Vsn, RcPid), try _ActiveMfst = riak_cs_manifest_fsm:get_active_manifest(ManiPid), UpdManifest = Manifest?MANIFEST{acl=Acl}, @@ -390,25 +320,6 @@ set_object_acl(Bucket, Key, Manifest, Acl, RcPid) -> riak_cs_manifest_fsm:stop(ManiPid) end. --spec second_resolution_timestamp(erlang:timestamp()) -> non_neg_integer(). -%% @doc Return the number of seconds this timestamp represents. Truncated to -%% seconds, as an integer. -second_resolution_timestamp({MegaSecs, Secs, _MicroSecs}) -> - (MegaSecs * 1000000) + Secs. - -%% same as timestamp_to_milliseconds below --spec timestamp_to_seconds(erlang:timestamp()) -> number(). -timestamp_to_seconds({MegaSecs, Secs, MicroSecs}) -> - (MegaSecs * 1000000) + Secs + (MicroSecs / 1000000). - -%% riak_cs_utils.erl:991: Invalid type specification for function riak_cs_utils:timestamp_to_milliseconds/1. The success typing is ({number(),number(),number()}) -> float() -%% this is also a derp that dialyzer shows above message when defined -%% like this, as manpage says it's three-integer tuple : -%% -spec timestamp_to_milliseconds(erlang:timestamp()) -> integer(). --spec timestamp_to_milliseconds(erlang:timestamp()) -> number(). -timestamp_to_milliseconds(Timestamp) -> - timestamp_to_seconds(Timestamp) * 1000. - %% Get the proper bucket name for either the Riak CS object %% bucket or the data block bucket. -spec to_bucket_name(objects | blocks, binary()) -> binary(). @@ -433,7 +344,8 @@ update_obj_value(Obj, Value) when is_binary(Value) -> %% we'll take care of calling `to_bucket_name' -spec key_exists(riak_client(), binary(), binary()) -> boolean(). key_exists(RcPid, Bucket, Key) -> - key_exists_handle_get_manifests(riak_cs_manifest:get_manifests(RcPid, Bucket, Key)). + key_exists_handle_get_manifests( + riak_cs_manifest:get_manifests(RcPid, Bucket, Key, ?LFS_DEFAULT_OBJECT_VERSION)). -spec big_end_key() -> binary(). @@ -501,15 +413,6 @@ active_to_bool({error, notfound}) -> pid_to_binary(Pid) -> list_to_binary(pid_to_list(Pid)). --spec safe_base64_decode(binary() | string()) -> {ok, binary()} | bad. -safe_base64_decode(Str) -> - try - X = base64:decode(Str), - {ok, X} - catch _:_ -> - bad - end. - -spec safe_base64url_decode(binary() | string()) -> {ok, binary()} | bad. safe_base64url_decode(Str) -> try @@ -539,6 +442,141 @@ camel_case(String) when is_list(String) -> capitalize("") -> ""; capitalize([H|T]) -> string:to_upper([H]) ++ T. + +-spec parse_x509_cert(binary()) -> {ok, [#'OTPCertificate'{}]} | {error, term()}. +parse_x509_cert(X509Certificate) -> + try + CC = public_key:pem_decode( + iolist_to_binary(["-----BEGIN CERTIFICATE-----\n", + X509Certificate, + "\n-----END CERTIFICATE-----\n"])), + parse_certs(CC, []) + catch + _E:R -> + {error, R} + end. + +parse_certs([], Q) -> + {ok, lists:reverse(Q)}; +parse_certs([{'Certificate', EntryData, _encrypted_or_not} | Rest], Q) -> + Entry = public_key:pkix_decode_cert(EntryData, otp), + parse_certs(Rest, [Entry | Q]). + + +object_indices(?RCS_USER{path = Path, + id = Id, + email = Email, + name = Name, + key_id = KeyId}) -> + [{?USER_NAME_INDEX, Name}, + {?USER_PATH_INDEX, Path}, + {?USER_KEYID_INDEX, KeyId}, + {?USER_EMAIL_INDEX, Email}, + {?USER_ID_INDEX, Id}]; +object_indices(?IAM_ROLE{role_id = Id, + path = Path, + role_name = Name}) -> + [{?ROLE_ID_INDEX, Id}, + {?ROLE_PATH_INDEX, Path}, + {?ROLE_NAME_INDEX, Name}]; +object_indices(?IAM_POLICY{policy_id = Id, + path = Path, + policy_name = Name}) -> + [{?POLICY_ID_INDEX, Id}, + {?POLICY_PATH_INDEX, Path}, + {?POLICY_NAME_INDEX, Name}]; +object_indices(?IAM_SAML_PROVIDER{name = Name, + entity_id = EntityId}) -> + [{?SAMLPROVIDER_NAME_INDEX, Name}, + {?SAMLPROVIDER_ENTITYID_INDEX, EntityId}]. + + +-spec gather_disk_usage_on_connected_riak_nodes() -> [#{node := atom(), + df_total := integer(), + df_available := integer(), + backend_data_total_size := integer(), + n_val := integer()}]. +gather_disk_usage_on_connected_riak_nodes() -> + [begin + Info = + case rpc:call(N, riak_cs_utils, get_storage_info, []) of + #{} = A -> + A; + _ -> + logger:warning("failed to get storage info via rpc from node ~s", [N]), + #{} + end, + maps:merge(#{node => N}, Info) + end || N <- nodes(), is_riak_node(atom_to_list(N))]. +is_riak_node([$r, $i, $a, $k, $@ | _]) -> true; +is_riak_node([$d, $e, $v, N, $@ | _]) when N >= $1, N =< $9 -> true; +is_riak_node(_) -> false. + + + +-spec get_storage_info() -> #{}. +get_storage_info() -> + lists:foldl(fun maps:merge/2, #{}, + [get_df(), get_nval(), get_backend_data_du()]). + +get_nval() -> + {ok, DefBP} = application:get_env(riak_core, default_bucket_props), + #{n_val => proplists:get_value(n_val, DefBP)}. + +get_backend_data_du() -> + {ok, DataDir} = application:get_env(riak_core, platform_data_dir), + #{backend_data_total_size => + filelib:fold_files( + DataDir, ".*\.sst$", true, + fun(F, Q) -> filelib:file_size(F) + Q end, 0)}. + +%% this function will be called on riak nodes +-if (OTP_26). +get_df() -> + case disksup:get_disk_info("./data") of + {ok, {Total, _, _, _}, Remaining} -> + {Total, Remaining}; + _ -> + #{df_total => -1, + df_available => -1} + end. +-else. +get_df() -> + [_Header, Line, _] = string:split(os:cmd("df ./data"), "\n", all), + case parse_df(Line) of + {ok, {Total, _, _, _}, Remaining} -> + #{df_total => Total, + df_available => Remaining}; + _ -> + #{df_total => -1, + df_available => -1} + end. +parse_df(L) -> + try + [_Device, OneKblocks, _Used, Available, _UsedPC, MountPoint] = + string:tokens(L, " "), + KiBTotal = list_to_integer(OneKblocks), + KiBAvailable = list_to_integer(Available), + Capacity = list_to_integer(OneKblocks), + {ok, {KiBTotal, KiBAvailable, Capacity, MountPoint}, KiBAvailable} + catch + error:badarg -> + {error, parse_df} + end. + +-endif. + + +guess_riak_node_name() -> + {H, _} = riak_cs_config:riak_host_port(), + list_to_atom(guess_riak_node_name(atom_to_list(node())) ++ "@" ++ H). + +guess_riak_node_name([$r, $c, $s, $-, $d, $e, $v, N, $@ | _]) -> + "dev" ++ [N]; +guess_riak_node_name(_) -> + logger:info("will use standard node name \"riak\" for rpc calls"), + "riak". + -ifdef(TEST). camel_case_test() -> diff --git a/src/riak_cs_web.erl b/apps/riak_cs/src/riak_cs_web.erl similarity index 53% rename from src/riak_cs_web.erl rename to apps/riak_cs/src/riak_cs_web.erl index 16f3b52e2..64f4846bf 100644 --- a/src/riak_cs_web.erl +++ b/apps/riak_cs/src/riak_cs_web.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -40,14 +41,14 @@ object_api_dispatch_table() -> base_resources() ++ one_three_resources(riak_cs_config:cs_version()). --spec props(atom()) -> [term()]. +-spec props(atom()) -> proplists:proplist(). props(Mod) -> [{auth_bypass, riak_cs_config:auth_bypass()}, {auth_module, riak_cs_config:auth_module()}, {policy_module, riak_cs_config:policy_module()}, {submodule, Mod}]. --spec stats_props() -> [term()]. +-spec stats_props() -> proplists:proplist(). stats_props() -> [{admin_auth_enabled, riak_cs_config:admin_auth_enabled()}, {auth_bypass, riak_cs_config:auth_bypass()}]. @@ -57,7 +58,9 @@ admin_resources(Props) -> [ {["riak-cs", "stats"], riak_cs_wm_stats, Props}, {["riak-cs", "ping"], riak_cs_wm_ping, []}, + {["riak-cs", "info"], riak_cs_wm_info, []}, {["riak-cs", "users"], riak_cs_wm_users, Props}, + {["riak-cs", "temp-sessions"], riak_cs_wm_temp_sessions, Props}, {["riak-cs", "user", '*'], riak_cs_wm_user, Props}, {["riak-cs", "usage", '*'], riak_cs_wm_usage, Props} ]. @@ -66,24 +69,29 @@ admin_resources(Props) -> base_resources() -> [ %% Bucket resources - {["buckets"], riak_cs_wm_common, props(riak_cs_wm_buckets)}, - {["buckets", bucket], riak_cs_wm_common, props(riak_cs_wm_bucket)}, - {["buckets", bucket, "objects"], riak_cs_wm_common, props(riak_cs_wm_objects)}, - {["buckets", bucket, "delete"], riak_cs_wm_common, props(riak_cs_wm_bucket_delete)}, - {["buckets", bucket, "acl"], riak_cs_wm_common, props(riak_cs_wm_bucket_acl)}, - {["buckets", bucket, "location"], riak_cs_wm_common, props(riak_cs_wm_bucket_location)}, + {["buckets"], riak_cs_wm_s3_common, props(riak_cs_wm_buckets)}, + {["buckets", bucket], riak_cs_wm_s3_common, props(riak_cs_wm_bucket)}, + {["buckets", bucket, "objects"], riak_cs_wm_s3_common, props(riak_cs_wm_objects)}, + {["buckets", bucket, "delete"], riak_cs_wm_s3_common, props(riak_cs_wm_bucket_delete)}, + {["buckets", bucket, "acl"], riak_cs_wm_s3_common, props(riak_cs_wm_bucket_acl)}, + {["buckets", bucket, "policy"], riak_cs_wm_s3_common, props(riak_cs_wm_bucket_policy)}, + {["buckets", bucket, "location"], riak_cs_wm_s3_common, props(riak_cs_wm_bucket_location)}, + {["buckets", bucket, "versioning"], riak_cs_wm_s3_common, props(riak_cs_wm_bucket_versioning)}, + {["buckets", bucket, "versions"], riak_cs_wm_s3_common, props(riak_cs_wm_object_versions)}, %% No dynamic contents, almost stub - {["buckets", bucket, "versioning"], riak_cs_wm_common, props(riak_cs_wm_bucket_versioning)}, - {["buckets", bucket, "requestPayment"], riak_cs_wm_common, props(riak_cs_wm_bucket_request_payment)}, + {["buckets", bucket, "requestPayment"], riak_cs_wm_s3_common, props(riak_cs_wm_bucket_request_payment)}, %% NYI - {["buckets", bucket, "versions"], riak_cs_wm_common, props(riak_cs_wm_not_implemented)}, - {["buckets", bucket, "lifecycle"], riak_cs_wm_common, props(riak_cs_wm_not_implemented)}, + {["buckets", bucket, "lifecycle"], riak_cs_wm_s3_common, props(riak_cs_wm_not_implemented)}, %% Object resources - {["buckets", bucket, "objects", object], riak_cs_wm_common, props(riak_cs_wm_object)}, - {["buckets", bucket, "objects", object, "acl"], riak_cs_wm_common, props(riak_cs_wm_object_acl)} + {["buckets", bucket, "objects", object, "versions", versionId], riak_cs_wm_s3_common, props(riak_cs_wm_object)}, + {["buckets", bucket, "objects", object, "versions", versionId, "uploads", uploadId], riak_cs_wm_s3_common, props(riak_cs_wm_object_upload_part)}, + {["buckets", bucket, "objects", object, "versions", versionId, "uploads"], riak_cs_wm_s3_common, props(riak_cs_wm_object_upload)}, + {["buckets", bucket, "objects", object, "versions", versionId, "acl"], riak_cs_wm_s3_common, props(riak_cs_wm_object_acl)}, + %% catch-all modules for services using POSTs for all their requests + {["iam"], riak_cs_wm_iam, props(no_submodule)}, + {["sts"], riak_cs_wm_sts, props(no_submodule)} ]. --spec one_three_resources(undefined | pos_integer()) -> [dispatch_rule()]. one_three_resources(undefined) -> []; one_three_resources(Version) when Version < 010300 -> @@ -91,9 +99,9 @@ one_three_resources(Version) when Version < 010300 -> one_three_resources(_Version) -> [ %% Bucket resources - {["buckets", bucket, "uploads"], riak_cs_wm_common, props(riak_cs_wm_bucket_uploads)}, - {["buckets", bucket, "policy"], riak_cs_wm_common, props(riak_cs_wm_bucket_policy)}, + {["buckets", bucket, "uploads"], riak_cs_wm_s3_common, props(riak_cs_wm_bucket_uploads)}, + {["buckets", bucket, "policy"], riak_cs_s3_wm_common, props(riak_cs_wm_bucket_policy)}, %% Object resources - {["buckets", bucket, "objects", object, "uploads", uploadId], riak_cs_wm_common, props(riak_cs_wm_object_upload_part)}, - {["buckets", bucket, "objects", object, "uploads"], riak_cs_wm_common, props(riak_cs_wm_object_upload)} + {["buckets", bucket, "objects", object, "uploads", uploadId], riak_cs_wm_s3_common, props(riak_cs_wm_object_upload_part)}, + {["buckets", bucket, "objects", object, "uploads"], riak_cs_wm_s3_common, props(riak_cs_wm_object_upload)} ]. diff --git a/apps/riak_cs/src/riak_cs_wm_bucket.erl b/apps/riak_cs/src/riak_cs_wm_bucket.erl new file mode 100644 index 000000000..771834ecc --- /dev/null +++ b/apps/riak_cs/src/riak_cs_wm_bucket.erl @@ -0,0 +1,157 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_wm_bucket). + +-export([accept_body/2, + allowed_methods/0, + authorize/2, + content_types_accepted/2, + content_types_provided/2, + delete_resource/2, + malformed_request/2, + stats_prefix/0, + to_xml/2 + ]). + +-ignore_xref([accept_body/2, + allowed_methods/0, + authorize/2, + content_types_accepted/2, + content_types_provided/2, + delete_resource/2, + malformed_request/2, + stats_prefix/0, + to_xml/2 + ]). + +-include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-spec stats_prefix() -> bucket. +stats_prefix() -> bucket. + +%% @doc Get the list of methods this resource supports. +-spec allowed_methods() -> [atom()]. +allowed_methods() -> + ['HEAD', 'PUT', 'DELETE']. + +-spec malformed_request(#wm_reqdata{}, #rcs_web_context{}) -> {false, #wm_reqdata{}, #rcs_web_context{}}. +malformed_request(RD, Ctx) -> + case riak_cs_wm_utils:has_canned_acl_and_header_grant(RD) of + true -> + riak_cs_aws_response:api_error(canned_acl_and_header_grant, RD, Ctx); + false -> + {false, RD, Ctx} + end. + +-spec content_types_provided(#wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_provided(RD, Ctx) -> + {[{"application/xml", to_xml}], RD, Ctx}. + +-spec content_types_accepted(#wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_accepted(RD, Ctx) -> + content_types_accepted(wrq:get_req_header("content-type", RD), RD, Ctx). + +-spec content_types_accepted(undefined | string(), #wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_accepted(CT, RD, Ctx) when CT =:= undefined; + CT =:= [] -> + content_types_accepted("application/octet-stream", RD, Ctx); +content_types_accepted(CT, RD, Ctx) -> + {Media, _Params} = mochiweb_util:parse_header(CT), + {[{Media, add_acl_to_context_then_accept}], RD, Ctx}. + +-spec authorize(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +authorize(RD, #rcs_web_context{user=User}=Ctx) -> + Method = wrq:method(RD), + RequestedAccess = + riak_cs_acl_utils:requested_access(Method, false), + Bucket = list_to_binary(wrq:path_info(bucket, RD)), + PermCtx = Ctx#rcs_web_context{bucket = Bucket, + requested_perm = RequestedAccess}, + + case {Method, RequestedAccess} of + {_, 'WRITE'} when User == undefined -> + %% unauthed users may neither create nor delete buckets + riak_cs_wm_utils:deny_access(RD, PermCtx); + {'PUT', 'WRITE'} -> + %% authed users are always allowed to attempt bucket creation + AccessRD = riak_cs_access_log_handler:set_user(User, RD), + {false, AccessRD, PermCtx}; + _ -> + riak_cs_wm_utils:bucket_access_authorize_helper(bucket, true, RD, Ctx) + end. + + +-spec to_xml(#wm_reqdata{}, #rcs_web_context{}) -> + {binary() | {'halt', term()}, #wm_reqdata{}, #rcs_web_context{}}. +to_xml(RD, Ctx) -> + handle_read_request(RD, Ctx). + +%% @private +handle_read_request(RD, Ctx=#rcs_web_context{user = User, + bucket = Bucket}) -> + %% override the content-type on HEAD + HeadRD = wrq:set_resp_header("content-type", "text/html", RD), + case [B || B <- riak_cs_bucket:get_buckets(User), + B?RCS_BUCKET.name =:= Bucket] of + [] -> + {{halt, 404}, HeadRD, Ctx}; + [_BucketRecord] -> + {{halt, 200}, HeadRD, Ctx} + end. + +%% @doc Process request body on `PUT' request. +-spec accept_body(#wm_reqdata{}, #rcs_web_context{}) -> {{halt, integer()}, #wm_reqdata{}, #rcs_web_context{}}. +accept_body(RD, Ctx = #rcs_web_context{user = User, + acl = ACL, + user_object = UserObj, + bucket = Bucket, + response_module = ResponseMod}) -> + BagId = riak_cs_mb_helper:choose_bag_id(manifest, Bucket), + case riak_cs_bucket:create_bucket( + User, UserObj, Bucket, BagId, ACL) of + ok -> + {{halt, 200}, RD, Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end. + +%% @doc Callback for deleting a bucket. +-spec delete_resource(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {'halt', term()}, #wm_reqdata{}, #rcs_web_context{}}. +delete_resource(RD, Ctx = #rcs_web_context{user = User, + user_object = UserObj, + response_module = ResponseMod, + bucket = Bucket, + riak_client = RcPid}) -> + case riak_cs_bucket:delete_bucket(User, + UserObj, + Bucket, + RcPid) of + ok -> + {true, RD, Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end. diff --git a/apps/riak_cs/src/riak_cs_wm_bucket_acl.erl b/apps/riak_cs/src/riak_cs_wm_bucket_acl.erl new file mode 100644 index 000000000..bb826f90d --- /dev/null +++ b/apps/riak_cs/src/riak_cs_wm_bucket_acl.erl @@ -0,0 +1,128 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_wm_bucket_acl). + +-export([stats_prefix/0, + content_types_provided/2, + to_xml/2, + allowed_methods/0, + malformed_request/2, + content_types_accepted/2, + accept_body/2 + ]). + +-ignore_xref([stats_prefix/0, + content_types_provided/2, + to_xml/2, + allowed_methods/0, + malformed_request/2, + content_types_accepted/2, + accept_body/2 + ]). + +-export([authorize/2]). + +%% TODO: DELETE? + +-include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). + + +-spec stats_prefix() -> bucket_acl. +stats_prefix() -> bucket_acl. + +%% @doc Get the list of methods this resource supports. +-spec allowed_methods() -> [atom()]. +allowed_methods() -> + ['GET', 'PUT']. + +-spec malformed_request(#wm_reqdata{}, #rcs_web_context{}) -> {false, #wm_reqdata{}, #rcs_web_context{}}. +malformed_request(RD, Ctx) -> + case riak_cs_wm_utils:has_acl_header_and_body(RD) of + true -> + riak_cs_aws_response:api_error(unexpected_content, RD, Ctx); + false -> + {false, RD, Ctx} + end. + +-spec content_types_provided(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_provided(RD, Ctx) -> + {[{"application/xml", to_xml}], RD, Ctx}. + +-spec content_types_accepted(#wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_accepted(RD, Ctx) -> + case wrq:get_req_header("content-type", RD) of + undefined -> + {[{"application/octet-stream", add_acl_to_context_then_accept}], RD, Ctx}; + CType -> + {Media, _Params} = mochiweb_util:parse_header(CType), + {[{Media, add_acl_to_context_then_accept}], RD, Ctx} + end. + +-spec authorize(#wm_reqdata{}, #rcs_web_context{}) -> {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +authorize(RD, Ctx) -> + riak_cs_wm_utils:bucket_access_authorize_helper(bucket_acl, true, RD, Ctx). + + +-spec to_xml(#wm_reqdata{}, #rcs_web_context{}) -> + {binary() | {'halt', non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +to_xml(RD, Ctx = #rcs_web_context{bucket = Bucket, + riak_client = RcPid}) -> + case riak_cs_acl:fetch_bucket_acl(Bucket, RcPid) of + {ok, Acl} -> + {riak_cs_xml:to_xml(Acl), RD, Ctx}; + {error, Reason} -> + riak_cs_aws_response:api_error(Reason, RD, Ctx) + end. + +%% @doc Process request body on `PUT' request. +-spec accept_body(#wm_reqdata{}, #rcs_web_context{}) -> {{halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +accept_body(RD, Ctx = #rcs_web_context{user = User, + user_object = UserObj, + acl = AclFromHeadersOrDefault, + bucket = Bucket, + riak_client = RcPid}) -> + Body = binary_to_list(wrq:req_body(RD)), + AclRes = + case Body of + [] -> + {ok, AclFromHeadersOrDefault}; + _ -> + RawAcl = riak_cs_acl_utils:acl_from_xml( + Body, User?RCS_USER.key_id, RcPid), + riak_cs_acl_utils:validate_acl(RawAcl, User?RCS_USER.id) + end, + case AclRes of + {ok, ACL} -> + case riak_cs_bucket:set_bucket_acl(User, + UserObj, + Bucket, + ACL) of + ok -> + {{halt, 200}, RD, Ctx}; + {error, Reason} -> + riak_cs_aws_response:api_error(Reason, RD, Ctx) + end; + {error, Reason2} -> + riak_cs_aws_response:api_error(Reason2, RD, Ctx) + end. diff --git a/apps/riak_cs/src/riak_cs_wm_bucket_delete.erl b/apps/riak_cs/src/riak_cs_wm_bucket_delete.erl new file mode 100644 index 000000000..e8c669414 --- /dev/null +++ b/apps/riak_cs/src/riak_cs_wm_bucket_delete.erl @@ -0,0 +1,204 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc WM resouce for Delete Multiple Objects + +-module(riak_cs_wm_bucket_delete). + +-export([init/1, + stats_prefix/0, + allowed_methods/0, + post_is_create/2, + process_post/2 + ]). + +-ignore_xref([init/1, + stats_prefix/0, + allowed_methods/0, + post_is_create/2, + process_post/2 + ]). + +-export([authorize/2]). + +-include("riak_cs.hrl"). +-include_lib("xmerl/include/xmerl.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +-define(RIAKCPOOL, request_pool). + +-spec init(#rcs_web_context{}) -> {ok, #rcs_web_context{}}. +init(Ctx) -> + {ok, Ctx#rcs_web_context{rc_pool=?RIAKCPOOL}}. + +-spec stats_prefix() -> multiple_delete. +stats_prefix() -> multiple_delete. + +-spec allowed_methods() -> [atom()]. +allowed_methods() -> + %% POST is for Delete Multiple Objects + ['POST']. + +%% TODO: change to authorize/spec/cleanup unneeded cases +%% TODO: requires update for multi-delete +-spec authorize(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +authorize(RD, Ctx) -> + Bucket = list_to_binary(wrq:path_info(bucket, RD)), + {false, RD, Ctx#rcs_web_context{bucket = Bucket}}. + +post_is_create(RD, Ctx) -> + {false, RD, Ctx}. + +-spec process_post(#wm_reqdata{}, #rcs_web_context{}) -> {term(), #wm_reqdata{}, #rcs_web_context{}}. +process_post(RD, Ctx = #rcs_web_context{bucket = Bucket, + riak_client = RcPid}) -> + handle_with_bucket_obj(riak_cs_bucket:fetch_bucket_object(Bucket, RcPid), RD, Ctx). + +handle_with_bucket_obj({error, notfound}, RD, + #rcs_web_context{response_module = ResponseMod} = Ctx) -> + ResponseMod:api_error(no_such_bucket, RD, Ctx); + +handle_with_bucket_obj({error, _} = Error, RD, + #rcs_web_context{response_module = ResponseMod} = Ctx) -> + ?LOG_DEBUG("bucket error: ~p", [Error]), + ResponseMod:api_error(Error, RD, Ctx); + +handle_with_bucket_obj({ok, BucketObj}, + RD, Ctx = #rcs_web_context{bucket = Bucket, + user = ?IAM_USER{id = CanonicalId}, + riak_client = RcPid, + response_module = ResponseMod, + policy_module = PolicyMod}) -> + + case parse_body(binary_to_list(wrq:req_body(RD))) of + {error, _} = Error -> + ResponseMod:api_error(Error, RD, Ctx); + {ok, Keys} when length(Keys) > 1000 -> + %% Delete Multiple Objects accepts a request to delete up to 1000 Objects. + ResponseMod:api_error(malformed_xml, RD, Ctx); + {ok, Keys} -> + ?LOG_DEBUG("deleting keys at ~p: ~p", [Bucket, Keys]), + + BucketPolicy = riak_cs_wm_utils:translate_bucket_policy(PolicyMod, BucketObj), + {UserPolicies, PermissionsBoundary} = + riak_cs_wm_utils:get_user_policies_or_halt(Ctx), + ApplicablePolicies = [BucketPolicy | UserPolicies], + Access0 = PolicyMod:reqdata_to_access(RD, object, CanonicalId), + + %% map: keys => delete_results => xmlElements + Results = + lists:map(fun({Key, Vsn} = VKey) -> + handle_key(RcPid, Bucket, VKey, + check_permission( + Key, Vsn, + Access0, + ApplicablePolicies, PermissionsBoundary, + BucketObj, + RD, Ctx)) + end, Keys), + + %% xmlDoc => return body. + Xml = riak_cs_xml:to_xml([{'DeleteResult', [{'xmlns', ?S3_XMLNS}], Results}]), + + RD2 = wrq:set_resp_body(Xml, RD), + {true, RD2, Ctx} + end. + +check_permission(Key, Vsn, + Access0, Policies, PermissionsBoundary, BucketObj, + RD, Ctx = #rcs_web_context{bucket = Bucket, + riak_client = RcPid}) -> + case riak_cs_manifest:fetch(RcPid, Bucket, Key, Vsn) of + {ok, Manifest} -> + ObjectAcl = riak_cs_manifest:object_acl(Manifest), + Access = Access0#access_v1{key = Key, + method = 'DELETE', + target = object}, + + case riak_cs_wm_utils:check_object_authorization( + Access, false, ObjectAcl, Policies, PermissionsBoundary, BucketObj, RD, Ctx) of + {ok, _} -> ok; + {error, _} -> {error, access_denied} + end; + E -> + E + end. + +%% bucket/key => delete => xml indicating each result +handle_key(_RcPid, _Bucket, {Key, Vsn}, {error, notfound}) -> + %% delete is RESTful, thus this is success + {'Deleted', [{'Key', [Key]}, {'VersionId', [Vsn]}]}; +handle_key(_RcPid, _Bucket, {Key, Vsn}, {error, no_active_manifest}) -> + {'Deleted', [{'Key', [Key]}, {'VersionId', [Vsn]}]}; +handle_key(_RcPid, _Bucket, {Key, Vsn}, {error, Error}) -> + {'Error', + [{'Key', [Key]}, + {'VersionId', [Vsn]}, + {'Code', [riak_cs_aws_response:error_code(Error)]}, + {'Message', [riak_cs_aws_response:error_message(Error)]}]}; +handle_key(RcPid, Bucket, {Key, Vsn}, ok) -> + case riak_cs_utils:delete_object(Bucket, Key, Vsn, RcPid) of + {ok, _UUIDsMarkedforDelete} -> + {'Deleted', [{'Key', [Key]}, {'VersionId', [Vsn]}]}; + Error -> + handle_key(RcPid, Bucket, {Key, Vsn}, Error) + end. + +parse_body(Body) -> + case riak_cs_xml:scan(Body) of + {ok, #xmlElement{name='Delete'} = ParsedData} -> + KKVV = [ key_and_version_from_xml_node(Node) + || Node <- xmerl_xpath:string("//Delete/Object", ParsedData), + is_record(Node, xmlElement) ], + case lists:member(malformed_xml, KKVV) of + true -> + {error, malformed_xml}; + false -> + {ok, KKVV} + end; + {ok, _ParsedData} -> + {error, malformed_xml}; + Error -> + Error + end. + +key_and_version_from_xml_node(Node) -> + case {xmerl_xpath:string("//Key/text()", Node), + xmerl_xpath:string("//VersionId/text()", Node)} of + {[#xmlText{value = K}], [#xmlText{value = V}]} -> + {unicode:characters_to_binary(K), unicode:characters_to_binary(V)}; + {[#xmlText{value = K}], _} -> + {unicode:characters_to_binary(K), ?LFS_DEFAULT_OBJECT_VERSION}; + _ -> + malformed_xml + end. + +-ifdef(TEST). + +parse_body_test() -> + Body = " </Key> ", + ?assertEqual({ok, [{<<"">>, ?LFS_DEFAULT_OBJECT_VERSION}]}, parse_body(Body)). + +-endif. diff --git a/src/riak_cs_wm_bucket_location.erl b/apps/riak_cs/src/riak_cs_wm_bucket_location.erl similarity index 67% rename from src/riak_cs_wm_bucket_location.erl rename to apps/riak_cs/src/riak_cs_wm_bucket_location.erl index 6fef58044..b82660f25 100644 --- a/src/riak_cs_wm_bucket_location.erl +++ b/apps/riak_cs/src/riak_cs_wm_bucket_location.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -26,11 +27,15 @@ to_xml/2, allowed_methods/0 ]). +-ignore_xref([stats_prefix/0, + content_types_provided/2, + to_xml/2, + allowed_methods/0 + ]). -export([authorize/2]). -include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). -spec stats_prefix() -> bucket_location. stats_prefix() -> bucket_location. @@ -40,28 +45,26 @@ stats_prefix() -> bucket_location. allowed_methods() -> ['GET']. --spec content_types_provided(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. +-spec content_types_provided(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. content_types_provided(RD, Ctx) -> {[{"application/xml", to_xml}], RD, Ctx}. --spec authorize(#wm_reqdata{}, #context{}) -> - {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #context{}}. +-spec authorize(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. authorize(RD, Ctx) -> riak_cs_wm_utils:bucket_access_authorize_helper(bucket_location, false, RD, Ctx). --spec to_xml(#wm_reqdata{}, #context{}) -> - {binary() | {'halt', term()}, #wm_reqdata{}, #context{}}. -to_xml(RD, Ctx=#context{user=User,bucket=Bucket}) -> - StrBucket = binary_to_list(Bucket), +-spec to_xml(#wm_reqdata{}, #rcs_web_context{}) -> + {binary() | {'halt', term()}, #wm_reqdata{}, #rcs_web_context{}}. +to_xml(RD, Ctx = #rcs_web_context{user = User, + bucket = Bucket}) -> case [B || B <- riak_cs_bucket:get_buckets(User), - B?RCS_BUCKET.name =:= StrBucket] of + B?RCS_BUCKET.name =:= Bucket] of [] -> - riak_cs_s3_response:api_error(no_such_bucket, RD, Ctx); + riak_cs_aws_response:api_error(no_such_bucket, RD, Ctx); [_BucketRecord] -> Doc = [{'LocationConstraint', [{xmlns, "http://s3.amazonaws.com/doc/2006-03-01/"}], [riak_cs_config:region()]}], {riak_cs_xml:to_xml(Doc), RD, Ctx} end. - - diff --git a/apps/riak_cs/src/riak_cs_wm_bucket_policy.erl b/apps/riak_cs/src/riak_cs_wm_bucket_policy.erl new file mode 100644 index 000000000..3cf2c129a --- /dev/null +++ b/apps/riak_cs/src/riak_cs_wm_bucket_policy.erl @@ -0,0 +1,135 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_wm_bucket_policy). + +-export([stats_prefix/0, + content_types_provided/2, + to_json/2, + allowed_methods/0, + content_types_accepted/2, + accept_body/2, + delete_resource/2 + ]). +-ignore_xref([stats_prefix/0, + content_types_provided/2, + to_json/2, + allowed_methods/0, + content_types_accepted/2, + accept_body/2, + delete_resource/2 + ]). + +-export([authorize/2]). + +%% TODO: DELETE? + +-include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). + + +-spec stats_prefix() -> bucket_policy. +stats_prefix() -> bucket_policy. + +%% @doc Get the list of methods this resource supports. +-spec allowed_methods() -> [atom()]. +allowed_methods() -> + ['GET', 'PUT', 'DELETE']. + +-spec content_types_provided(#wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_provided(RD, Ctx) -> + {[{"application/json", to_json}], RD, Ctx}. + +-spec content_types_accepted(#wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_accepted(RD, Ctx) -> + case wrq:get_req_header("content-type", RD) of + undefined -> + {[{"application/json", accept_body}], RD, Ctx}; + "application/json" -> + {[{"application/json", accept_body}], RD, Ctx}; + _ -> + {false, RD, Ctx} + end. + +-spec authorize(#wm_reqdata{}, #rcs_web_context{}) -> {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +authorize(RD, Ctx) -> + riak_cs_wm_utils:bucket_access_authorize_helper(bucket_policy, true, RD, Ctx). + + +-spec to_json(#wm_reqdata{}, #rcs_web_context{}) -> + {binary() | {'halt', non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +to_json(RD, Ctx = #rcs_web_context{bucket = Bucket, + response_module = RespMod, + policy_module = PolicyMod, + riak_client = RcPid}) -> + case PolicyMod:fetch_bucket_policy(Bucket, RcPid) of + {ok, PolicyJson} -> + {PolicyJson, RD, Ctx}; + {error, policy_undefined} -> + % S3 error: 404 (NoSuchBucketPolicy): The bucket policy does not exist + RespMod:api_error(no_such_bucket_policy, RD, Ctx); + {error, Reason} -> + RespMod:api_error(Reason, RD, Ctx) + end. + +%% @doc Process request body on `PUT' request. +-spec accept_body(#wm_reqdata{}, #rcs_web_context{}) -> + {{halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +accept_body(RD, Ctx = #rcs_web_context{user = User, + user_object = UserObj, + bucket = Bucket, + policy_module = PolicyMod, + response_module = RespMod}) -> + PolicyJson = wrq:req_body(RD), + case PolicyMod:policy_from_json(PolicyJson) of + {ok, Policy} -> + Access = PolicyMod:reqdata_to_access(RD, bucket_policy, User?RCS_USER.id), + case PolicyMod:check_policy(Access, Policy) of + ok -> + case riak_cs_bucket:set_bucket_policy(User, UserObj, Bucket, PolicyJson) of + ok -> + {{halt, 200}, RD, Ctx}; + {error, Reason} -> + RespMod:api_error(Reason, RD, Ctx) + end; + {error, Reason} -> %% good JSON, but bad as IAM policy + RespMod:api_error(Reason, RD, Ctx) + end; + {error, Reason} -> %% Broken as JSON + RespMod:api_error(Reason, RD, Ctx) + end. + + +%% @doc Callback for deleting policy. +-spec delete_resource(#wm_reqdata{}, #rcs_web_context{}) -> {true, #wm_reqdata{}, #rcs_web_context{}} | + {{halt, 200}, #wm_reqdata{}, #rcs_web_context{}}. +delete_resource(RD, Ctx = #rcs_web_context{user = User, + user_object = UserObj, + bucket = Bucket, + response_module = RespMod}) -> + case riak_cs_bucket:delete_bucket_policy(User, UserObj, Bucket) of + ok -> + {{halt, 200}, RD, Ctx}; + {error, Reason} -> + RespMod:api_error(Reason, RD, Ctx) + end. diff --git a/src/riak_cs_wm_bucket_request_payment.erl b/apps/riak_cs/src/riak_cs_wm_bucket_request_payment.erl similarity index 69% rename from src/riak_cs_wm_bucket_request_payment.erl rename to apps/riak_cs/src/riak_cs_wm_bucket_request_payment.erl index 68f16c2b2..48a7b68a4 100644 --- a/src/riak_cs_wm_bucket_request_payment.erl +++ b/apps/riak_cs/src/riak_cs_wm_bucket_request_payment.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -23,12 +24,17 @@ -export([stats_prefix/0, content_types_provided/2, to_xml/2, - allowed_methods/0]). + allowed_methods/0 + ]). +-ignore_xref([stats_prefix/0, + content_types_provided/2, + to_xml/2, + allowed_methods/0 + ]). -export([authorize/2]). -include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). -spec stats_prefix() -> bucket_request_payment. stats_prefix() -> bucket_request_payment. @@ -39,28 +45,26 @@ stats_prefix() -> bucket_request_payment. allowed_methods() -> ['GET']. --spec content_types_provided(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. +-spec content_types_provided(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. content_types_provided(RD, Ctx) -> {[{"application/xml", to_xml}], RD, Ctx}. --spec authorize(#wm_reqdata{}, #context{}) -> - {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #context{}}. +-spec authorize(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. authorize(RD, Ctx) -> riak_cs_wm_utils:bucket_access_authorize_helper(bucket_request_payment, false, RD, Ctx). --spec to_xml(#wm_reqdata{}, #context{}) -> - {binary() | {'halt', term()}, #wm_reqdata{}, #context{}}. -to_xml(RD, Ctx=#context{user=User,bucket=Bucket}) -> - StrBucket = binary_to_list(Bucket), +-spec to_xml(#wm_reqdata{}, #rcs_web_context{}) -> + {binary() | {'halt', term()}, #wm_reqdata{}, #rcs_web_context{}}. +to_xml(RD, Ctx = #rcs_web_context{user = User, + bucket = Bucket}) -> case [B || B <- riak_cs_bucket:get_buckets(User), - B?RCS_BUCKET.name =:= StrBucket] of + B?RCS_BUCKET.name =:= Bucket] of [] -> - riak_cs_s3_response:api_error(no_such_bucket, RD, Ctx); + riak_cs_aws_response:api_error(no_such_bucket, RD, Ctx); [_BucketRecord] -> Doc = [{'RequestPaymentConfiguration', [{xmlns, "http://s3.amazonaws.com/doc/2006-03-01/"}], [{'Payer', [<<"BucketOwner">>]}]}], {riak_cs_xml:to_xml(Doc), RD, Ctx} end. - - diff --git a/src/riak_cs_wm_bucket_uploads.erl b/apps/riak_cs/src/riak_cs_wm_bucket_uploads.erl similarity index 74% rename from src/riak_cs_wm_bucket_uploads.erl rename to apps/riak_cs/src/riak_cs_wm_bucket_uploads.erl index 4d37470b7..d249060c1 100644 --- a/src/riak_cs_wm_bucket_uploads.erl +++ b/apps/riak_cs/src/riak_cs_wm_bucket_uploads.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -35,27 +36,39 @@ allowed_methods/0, malformed_request/2, content_types_accepted/2, - finish_request/2]). + finish_request/2 + ]). +-ignore_xref([init/1, + stats_prefix/0, + authorize/2, + content_types_provided/2, + to_xml/2, + allowed_methods/0, + malformed_request/2, + content_types_accepted/2, + finish_request/2 + ]). -include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). +-include_lib("kernel/include/logger.hrl"). -define(RIAKCPOOL, bucket_list_pool). --spec init(#context{}) -> {ok, #context{}}. +-spec init(#rcs_web_context{}) -> {ok, #rcs_web_context{}}. init(Ctx) -> - {ok, Ctx#context{local_context=#key_context{}}}. + {ok, Ctx#rcs_web_context{local_context = #key_context{}}}. -spec stats_prefix() -> list_uploads. -stats_prefix() -> list_uploads. +stats_prefix() -> + list_uploads. --spec malformed_request(#wm_reqdata{}, #context{}) -> {false, #wm_reqdata{}, #context{}}. -malformed_request(RD,Ctx=#context{local_context=LocalCtx0}) -> +-spec malformed_request(#wm_reqdata{}, #rcs_web_context{}) -> {false, #wm_reqdata{}, #rcs_web_context{}}. +malformed_request(RD, Ctx = #rcs_web_context{local_context = LocalCtx0}) -> Bucket = list_to_binary(wrq:path_info(bucket, RD)), - LocalCtx = LocalCtx0#key_context{bucket=Bucket}, - {false, RD, Ctx#context{local_context=LocalCtx}}. + LocalCtx = LocalCtx0#key_context{bucket = Bucket}, + {false, RD, Ctx#rcs_web_context{local_context = LocalCtx}}. --spec authorize(#wm_reqdata{}, #context{}) -> {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #context{}}. +-spec authorize(#wm_reqdata{}, #rcs_web_context{}) -> {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. authorize(RD, Ctx) -> riak_cs_wm_utils:bucket_access_authorize_helper(bucket_uploads, false, RD, Ctx). @@ -64,10 +77,10 @@ authorize(RD, Ctx) -> allowed_methods() -> ['GET']. -to_xml(RD, Ctx=#context{local_context=LocalCtx, - riak_client=RcPid}) -> - #key_context{bucket=Bucket} = LocalCtx, - User = riak_cs_user:to_3tuple(Ctx#context.user), +to_xml(RD, Ctx=#rcs_web_context{local_context = LocalCtx, + riak_client = RcPid}) -> + #key_context{bucket = Bucket} = LocalCtx, + User = riak_cs_user:to_3tuple(Ctx#rcs_web_context.user), Opts = make_list_mp_uploads_opts(RD), case riak_cs_mp_utils:list_multipart_uploads(Bucket, User, Opts, RcPid) of {ok, {Ds, Commons}} -> @@ -86,7 +99,7 @@ to_xml(RD, Ctx=#context{local_context=LocalCtx, %% Just ignore the value in `D?MULTIPART_DESCR.storage_class', %% since there was a bug where it was writen as `regular'. {'StorageClass', ["STANDARD"]}, - {'Initiated', [D?MULTIPART_DESCR.initiated]} + {'Initiated', [binary_to_list(rts:iso8601(D?MULTIPART_DESCR.initiated))]} ] } || D <- Ds], Cs = [{'CommonPrefixes', @@ -115,15 +128,15 @@ to_xml(RD, Ctx=#context{local_context=LocalCtx, Body = riak_cs_xml:to_xml([XmlDoc]), {Body, RD, Ctx}; {error, Reason} -> - riak_cs_s3_response:api_error(Reason, RD, Ctx) + riak_cs_aws_response:api_error(Reason, RD, Ctx) end. finish_request(RD, Ctx) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"finish_request">>, [0], []), {true, RD, Ctx}. --spec content_types_provided(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_provided(RD, Ctx=#context{}) -> +-spec content_types_provided(#wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_provided(RD, Ctx = #rcs_web_context{}) -> Method = wrq:method(RD), if Method == 'GET' -> {[{?XML_TYPE, to_xml}], RD, Ctx}; @@ -133,7 +146,7 @@ content_types_provided(RD, Ctx=#context{}) -> {[{"text/plain", unused_callback}], RD, Ctx} end. --spec content_types_accepted(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. +-spec content_types_accepted(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. content_types_accepted(RD, Ctx) -> riak_cs_mp_utils:make_content_types_accepted(RD, Ctx). diff --git a/apps/riak_cs/src/riak_cs_wm_bucket_versioning.erl b/apps/riak_cs/src/riak_cs_wm_bucket_versioning.erl new file mode 100644 index 000000000..96e4a76aa --- /dev/null +++ b/apps/riak_cs/src/riak_cs_wm_bucket_versioning.erl @@ -0,0 +1,220 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_wm_bucket_versioning). + +-export([stats_prefix/0, + content_types_provided/2, + content_types_accepted/2, + to_xml/2, + accept_body/2, + allowed_methods/0 + ]). + +-ignore_xref([stats_prefix/0, + content_types_provided/2, + content_types_accepted/2, + to_xml/2, + accept_body/2, + allowed_methods/0 + ]). + +-export([authorize/2]). + +-include("riak_cs.hrl"). +-include_lib("xmerl/include/xmerl.hrl"). + +-spec stats_prefix() -> bucket_versioning. +stats_prefix() -> bucket_versioning. + +%% @doc Get the list of methods this resource supports. +-spec allowed_methods() -> [atom()]. +allowed_methods() -> + ['GET', 'PUT']. + +-spec content_types_provided(#wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_provided(RD, Ctx) -> + {[{"application/xml", to_xml}], RD, Ctx}. + +-spec content_types_accepted(#wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_accepted(RD, Ctx) -> + case wrq:get_req_header("content-type", RD) of + undefined -> + {[{"application/octet-stream", add_acl_to_context_then_accept}], RD, Ctx}; + CType -> + {Media, _Params} = mochiweb_util:parse_header(CType), + {[{Media, add_acl_to_context_then_accept}], RD, Ctx} + end. + +-spec authorize(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, term()}, #wm_reqdata{}, #rcs_web_context{}}. +authorize(RD, Ctx) -> + riak_cs_wm_utils:bucket_access_authorize_helper(bucket_versioning, false, RD, Ctx). + + +-spec to_xml(#wm_reqdata{}, #rcs_web_context{}) -> + {binary() | {halt, term()}, #wm_reqdata{}, #rcs_web_context{}}. +to_xml(RD, Ctx=#rcs_web_context{user = User, + bucket = Bucket, + riak_client = RcPid}) -> + case [B || B <- riak_cs_bucket:get_buckets(User), + B?RCS_BUCKET.name =:= Bucket] of + [] -> + riak_cs_aws_response:api_error(no_such_bucket, RD, Ctx); + [_BucketRecord] -> + {ok, #bucket_versioning{status = Status, + mfa_delete = MFADelete, + use_subversioning = UseSubVersioning, + can_update_versions = CanUpdateVersions, + repl_siblings = ReplSiblings}} = + riak_cs_bucket:get_bucket_versioning(Bucket, RcPid), + {iolist_to_binary(["", + "", to_string(status, Status), "", + "", to_string(mfa_delete, MFADelete), "", + "", to_string(bool, UseSubVersioning), "", + "", to_string(bool, CanUpdateVersions), "", + "", to_string(bool, ReplSiblings), "", + ""]), + RD, Ctx} + end. + +to_string(status, enabled) -> "Enabled"; +to_string(status, suspended) -> "Suspended"; +to_string(mfa_delete, enabled) -> "Enabled"; +to_string(mfa_delete, disabled) -> "Disabled"; +to_string(bool, true) -> "True"; +to_string(bool, false) -> "False". + +-spec accept_body(#wm_reqdata{}, #rcs_web_context{}) -> {{halt, integer()}, #wm_reqdata{}, #rcs_web_context{}}. +accept_body(RD, Ctx = #rcs_web_context{user = User, + user_object = UserObj, + bucket = Bucket, + response_module = ResponseMod, + riak_client = RcPid}) -> + {ok, OldV} = riak_cs_bucket:get_bucket_versioning(Bucket, RcPid), + case riak_cs_xml:scan(binary_to_list(wrq:req_body(RD))) of + {ok, Doc} -> + {NewV, IsUpdated} = + update_versioning_struct_from_headers( + update_versioning_struct_from_xml(OldV, Doc), + RD), + case IsUpdated of + true -> + riak_cs_bucket:set_bucket_versioning( + User, UserObj, Bucket, NewV), + {{halt, 200}, RD, Ctx}; + false -> + {{halt, 200}, RD, Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end. + +update_versioning_struct_from_xml(Old, #xmlElement{name = 'VersioningConfiguration', + content = Content}) -> + MaybeNew = + lists:foldl( + fun(#xmlElement{name = 'Status', content = C}, Acc) -> + Acc#bucket_versioning{status = from_xml_node_content(status, C, Old#bucket_versioning.status)}; + (#xmlElement{name = 'MFADelete', content = C}, Acc) -> + Acc#bucket_versioning{mfa_delete = from_xml_node_content(mfa_delete, C, Old#bucket_versioning.mfa_delete)}; + (#xmlElement{name = 'UseSubVersioning', content = C}, Acc) -> + Acc#bucket_versioning{use_subversioning = from_xml_node_content(use_subversioning, C, Old#bucket_versioning.use_subversioning)}; + (#xmlElement{name = 'CanUpdateVersions', content = C}, Acc) -> + Acc#bucket_versioning{can_update_versions = from_xml_node_content(can_update_versions, C, Old#bucket_versioning.can_update_versions)}; + (#xmlElement{name = 'ReplSiblings', content = C}, Acc) -> + Acc#bucket_versioning{repl_siblings = from_xml_node_content(repl_siblings, C, Old#bucket_versioning.repl_siblings)}; + (#xmlElement{}, Acc) -> + Acc + end, + Old, + Content), + case Old == MaybeNew of + true -> + {Old, false}; + false -> + {MaybeNew, true} + end; +update_versioning_struct_from_xml(Old, _) -> + {Old, {error, malformed_xml}}. + +from_xml_node_content(status, CC, Old) -> + case lists:search(fun(#xmlText{}) -> true; (_) -> false end, CC) of + {value, #xmlText{value = "Enabled"}} -> + enabled; + {value, #xmlText{value = "Suspended"}} -> + suspended; + _ -> + Old + end; +from_xml_node_content(mfa_delete, CC, Old) -> + case lists:search(fun(#xmlText{}) -> true; (_) -> false end, CC) of + {value, #xmlText{value = "Enabled"}} -> + enabled; + {value, #xmlText{value = "Disabled"}} -> + disabled; + _ -> + Old + end; +from_xml_node_content(_, CC, Old) -> + case lists:search(fun(#xmlText{}) -> true; (_) -> false end, CC) of + {value, #xmlText{value = V}} when V == "True"; + V == "true" -> + true; + {value, #xmlText{value = V}} when V == "False"; + V == "false" -> + false; + _ -> + Old + end. + +update_versioning_struct_from_headers({OldV, IsUpdated0}, RD) -> + MaybeNew = + lists:foldl(fun({H, F}, Q) -> maybe_set_field(F, to_bool(wrq:get_req_header(H, RD)), Q) end, + OldV, + [{"x-rcs-versioning-use_subversioning", #bucket_versioning.use_subversioning}, + {"x-rcs-versioning-can_update_versions", #bucket_versioning.can_update_versions}, + {"x-rcs-versioning-repl_siblings", #bucket_versioning.repl_siblings}]), + if MaybeNew == OldV -> + {OldV, IsUpdated0}; + el/=se -> + {MaybeNew, true} + end. +maybe_set_field(_, undefined, OldV) -> + OldV; +maybe_set_field(F, V, OldV) -> + if element(F, OldV) == V -> + OldV; + el/=se -> + setelement(F, OldV, V) + end. + +to_bool("True") -> true; +to_bool("true") -> true; +to_bool("1") -> true; +to_bool("False") -> false; +to_bool("false") -> false; +to_bool("0") -> false; +to_bool(_) -> undefined. diff --git a/src/riak_cs_wm_buckets.erl b/apps/riak_cs/src/riak_cs_wm_buckets.erl similarity index 70% rename from src/riak_cs_wm_buckets.erl rename to apps/riak_cs/src/riak_cs_wm_buckets.erl index 1bc9e2a4a..5449f570d 100644 --- a/src/riak_cs_wm_buckets.erl +++ b/apps/riak_cs/src/riak_cs_wm_buckets.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -23,11 +24,18 @@ -export([stats_prefix/0, allowed_methods/0, api_request/2, - anon_ok/0]). + anon_ok/0 + ]). + +-ignore_xref([stats_prefix/0, + allowed_methods/0, + api_request/2, + anon_ok/0 + ]). -include("riak_cs.hrl"). --include("riak_cs_api.hrl"). --include_lib("webmachine/include/webmachine.hrl"). +-include("riak_cs_web.hrl"). +-include_lib("kernel/include/logger.hrl"). -spec stats_prefix() -> service. stats_prefix() -> service. @@ -37,13 +45,9 @@ stats_prefix() -> service. allowed_methods() -> ['GET']. --spec api_request(#wm_reqdata{}, #context{}) -> ?LBRESP{}. -api_request(_RD, #context{user=User}) -> - UserName = riak_cs_wm_utils:extract_name(User), - riak_cs_dtrace:dt_service_entry(?MODULE, <<"service_get_buckets">>, [], [UserName]), - Res = riak_cs_api:list_buckets(User), - riak_cs_dtrace:dt_service_return(?MODULE, <<"service_get_buckets">>, [], [UserName]), - Res. +-spec api_request(#wm_reqdata{}, #rcs_web_context{}) -> ?LBRESP{}. +api_request(_RD, #rcs_web_context{user = User}) -> + riak_cs_api:list_buckets(User). -spec anon_ok() -> boolean(). anon_ok() -> diff --git a/src/riak_cs_wm_error_handler.erl b/apps/riak_cs/src/riak_cs_wm_error_handler.erl similarity index 86% rename from src/riak_cs_wm_error_handler.erl rename to apps/riak_cs/src/riak_cs_wm_error_handler.erl index 86d49f433..119ab7bd5 100644 --- a/src/riak_cs_wm_error_handler.erl +++ b/apps/riak_cs/src/riak_cs_wm_error_handler.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -19,15 +20,17 @@ %% --------------------------------------------------------------------- -module(riak_cs_wm_error_handler). + +-compile({nowarn_deprecated_function, [{erlang, get_stacktrace, 0}]}). + -export([render_error/3, xml_error_body/4]). -include("riak_cs.hrl"). render_error(500, Req, Reason) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"render_error">>), {ok, ReqState} = Req:add_response_header("Content-Type", "text/html"), {Path,_} = Req:path(), - error_logger:error_msg("webmachine error: path=~p~n~p~n", [Path, {error, {error, Reason, erlang:get_stacktrace()}}]), + logger:error("webmachine error: path=~p; ~p", [Path, Reason]), ErrorOne = <<"500 Internal Server Error">>, ErrorTwo = <<"

Internal Server Error

">>, @@ -36,12 +39,10 @@ render_error(500, Req, Reason) -> IOList = [ErrorOne, ErrorTwo, ErrorThree, ErrorFour], {erlang:iolist_to_binary(IOList), ReqState}; render_error(405, Req, _Reason) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"render_error">>), {ok, ReqState} = Req:add_response_header("Content-Type", "application/xml"), {Path,_} = Req:path(), {xml_error_body(Path, <<"MethodNotAllowed">>, <<"The specified method is not allowed against this resource.">>, <<"12345">>), ReqState}; render_error(412, Req, _Reason) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"render_error">>), {ok, ReqState} = Req:add_response_header("Content-Type", "application/xml"), {Path,_} = Req:path(), {xml_error_body(Path, @@ -49,7 +50,6 @@ render_error(412, Req, _Reason) -> <<"At least one of the pre-conditions you specified did not hold">>, <<"12345">>), ReqState}; render_error(_Code, Req, _Reason) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"render_error">>), Req:response_body(). xml_error_body(Resource, Code, Message, RequestId) -> diff --git a/apps/riak_cs/src/riak_cs_wm_iam.erl b/apps/riak_cs/src/riak_cs_wm_iam.erl new file mode 100644 index 000000000..8cdf50de5 --- /dev/null +++ b/apps/riak_cs/src/riak_cs_wm_iam.erl @@ -0,0 +1,769 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc WM resource for IAM requests. + +-module(riak_cs_wm_iam). + +-export([init/1, + service_available/2, + forbidden/2, + options/2, + content_types_accepted/2, + generate_etag/2, + last_modified/2, + valid_entity_length/2, + multiple_choices/2, + accept_wwwform/2, + allowed_methods/2, + post_is_create/2, + create_path/2, + finish_request/2 + ]). + +-export([finish_tags/1, + add_tag/3]). +-ignore_xref([init/1, + service_available/2, + forbidden/2, + options/2, + content_types_accepted/2, + generate_etag/2, + last_modified/2, + multiple_choices/2, + accept_wwwform/2, + allowed_methods/2, + valid_entity_length/2, + post_is_create/2, + create_path/2, + finish_request/2 + ]). + +-include("riak_cs_web.hrl"). +-include_lib("xmerl/include/xmerl.hrl"). +-include_lib("kernel/include/logger.hrl"). + +%% ------------------------------------------------------------------- +%% Webmachine callbacks +%% ------------------------------------------------------------------- + +-spec init([proplists:proplist()]) -> {ok, #rcs_web_context{}}. +init(Config) -> + %% Check if authentication is disabled and set that in the context. + AuthBypass = proplists:get_value(auth_bypass, Config), + AuthModule = proplists:get_value(auth_module, Config), + Api = riak_cs_config:api(), + RespModule = riak_cs_config:response_module(Api), + StatsPrefix = no_stats, + Ctx = #rcs_web_context{auth_bypass = AuthBypass, + auth_module = AuthModule, + response_module = RespModule, + stats_prefix = StatsPrefix, + api = Api}, + {ok, Ctx}. + + +-spec options(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), string()}], #wm_reqdata{}, #rcs_web_context{}}. +options(RD, Ctx) -> + {riak_cs_wm_utils:cors_headers(), + RD, Ctx}. + +-spec service_available(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +service_available(RD, Ctx) -> + riak_cs_wm_utils:service_available( + wrq:set_resp_headers(riak_cs_wm_utils:cors_headers(), RD), Ctx). + + +-spec valid_entity_length(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +valid_entity_length(RD, Ctx) -> + {true, RD, Ctx}. + + +-spec forbidden(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +forbidden(RD, Ctx) -> + case wrq:method(RD) of + 'OPTIONS' -> + {false, RD, Ctx}; + 'POST' -> + riak_cs_wm_utils:forbidden(RD, Ctx, iam_entity) + end. + +-spec allowed_methods(#wm_reqdata{}, #rcs_web_context{}) -> {[atom()], #wm_reqdata{}, #rcs_web_context{}}. +allowed_methods(RD, Ctx) -> + {['POST', 'OPTIONS'], RD, Ctx}. + + +-spec content_types_accepted(#wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), module()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_accepted(RD, Ctx) -> + {[{?WWWFORM_TYPE, accept_wwwform}], RD, Ctx}. + + +-spec generate_etag(#wm_reqdata{}, #rcs_web_context{}) -> + {undefined|string(), #wm_reqdata{}, #rcs_web_context{}}. +generate_etag(RD, Ctx) -> + {undefined, RD, Ctx}. + + +-spec last_modified(#wm_reqdata{}, #rcs_web_context{}) -> + {undefined|string(), #wm_reqdata{}, #rcs_web_context{}}. +last_modified(RD, Ctx) -> + {undefined, RD, Ctx}. + +-spec post_is_create(#wm_reqdata{}, #rcs_web_context{}) -> + {true, #wm_reqdata{}, #rcs_web_context{}}. +post_is_create(RD, Ctx) -> + {true, RD, Ctx}. + + +-spec create_path(#wm_reqdata{}, #rcs_web_context{}) -> + {string(), #wm_reqdata{}, #rcs_web_context{}}. +create_path(RD, Ctx) -> + {wrq:disp_path(RD), RD, Ctx}. + + +-spec multiple_choices(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +multiple_choices(RD, Ctx) -> + {false, RD, Ctx}. + + +-spec accept_wwwform(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, term()}, term(), term()}. +accept_wwwform(RD, Ctx) -> + Form = mochiweb_util:parse_qs(wrq:req_body(RD)), + Action = proplists:get_value("Action", Form), + do_action(Action, Form, RD, Ctx). + +-spec finish_request(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, term()}, term(), term()}. +finish_request(RD, Ctx=#rcs_web_context{riak_client = undefined}) -> + {true, RD, Ctx}; +finish_request(RD, Ctx=#rcs_web_context{riak_client=RcPid}) -> + riak_cs_riak_client:checkin(RcPid), + {true, RD, Ctx#rcs_web_context{riak_client = undefined}}. + + +%% ------------------------------------------------------------------- +%% Internal functions +%% ------------------------------------------------------------------- + +do_action("CreateUser", + Form, RD, Ctx = #rcs_web_context{response_module = ResponseMod, + request_id = RequestId}) -> + Specs = finish_tags( + lists:foldl(fun create_user_fields_filter/2, #{}, Form)), + case create_user_require_fields(Specs) of + false -> + ResponseMod:api_error(missing_parameter, RD, Ctx); + true -> + case riak_cs_iam:create_user(Specs) of + {ok, User} -> + logger:info("Created user ~s \"~s\" (~s) on request_id ~s", + [User?IAM_USER.id, User?IAM_USER.name, User?IAM_USER.arn, RequestId]), + Doc = riak_cs_xml:to_xml( + #create_user_response{user = User, + request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end + end; + +do_action("GetUser", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + UserName = proplists:get_value("UserName", Form), + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:find_user(#{name => UserName}, Pbc) of + {ok, {User, _}} -> + Doc = riak_cs_xml:to_xml( + #get_user_response{user = User, + request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, notfound} -> + ResponseMod:api_error(no_such_user, RD, Ctx); + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action("DeleteUser", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + Name = proplists:get_value("UserName", Form), + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:find_user(#{name => list_to_binary(Name)}, Pbc) of + {ok, {?IAM_USER{arn = Arn} = User, _}} -> + case riak_cs_iam:delete_user(User) of + ok -> + logger:info("Deleted user \"~s\" (~s) on request_id ~s", [Name, Arn, RequestId]), + Doc = riak_cs_xml:to_xml( + #delete_user_response{request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + {error, notfound} -> + ResponseMod:api_error(no_such_user, RD, Ctx); + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action("ListUsers", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + PathPrefix = proplists:get_value("PathPrefix", Form, ""), + MaxItems = proplists:get_value("MaxItems", Form), + Marker = proplists:get_value("Marker", Form), + case riak_cs_iam:list_users( + RcPid, #list_users_request{request_id = RequestId, + path_prefix = list_to_binary(PathPrefix), + max_items = MaxItems, + marker = Marker}) of + {ok, #{users := Users, + marker := NewMarker, + is_truncated := IsTruncated}} -> + Doc = riak_cs_xml:to_xml( + #list_users_response{users = Users, + request_id = RequestId, + marker = NewMarker, + is_truncated = IsTruncated}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + + +do_action("CreateRole", + Form, RD, Ctx = #rcs_web_context{response_module = ResponseMod, + request_id = RequestId}) -> + Specs = finish_tags( + lists:foldl(fun create_role_fields_filter/2, #{}, Form)), + case create_role_require_fields(Specs) of + false -> + ResponseMod:api_error(missing_parameter, RD, Ctx); + true -> + case riak_cs_iam:create_role(Specs) of + {ok, Role} -> + logger:info("Created role ~s \"~s\" (~s) on request_id ~s", + [Role?IAM_ROLE.role_id, Role?IAM_ROLE.role_name, Role?IAM_ROLE.arn, RequestId]), + Doc = riak_cs_xml:to_xml( + #create_role_response{role = Role, + request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end + end; + +do_action("GetRole", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + RoleName = proplists:get_value("RoleName", Form), + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:find_role(#{name => RoleName}, Pbc) of + {ok, Role} -> + Doc = riak_cs_xml:to_xml( + #get_role_response{role = Role, + request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, notfound} -> + ResponseMod:api_error(no_such_role, RD, Ctx); + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action("DeleteRole", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + Name = proplists:get_value("RoleName", Form), + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:find_role(#{name => list_to_binary(Name)}, Pbc) of + {ok, ?IAM_ROLE{arn = Arn}} -> + case riak_cs_iam:delete_role(Arn) of + ok -> + logger:info("Deleted role \"~s\" (~s) on request_id ~s", [Name, Arn, RequestId]), + Doc = riak_cs_xml:to_xml( + #delete_role_response{request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + {error, notfound} -> + ResponseMod:api_error(no_such_role, RD, Ctx); + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action("ListRoles", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + PathPrefix = proplists:get_value("PathPrefix", Form, ""), + MaxItems = proplists:get_value("MaxItems", Form), + Marker = proplists:get_value("Marker", Form), + case riak_cs_iam:list_roles( + RcPid, #list_roles_request{request_id = RequestId, + path_prefix = list_to_binary(PathPrefix), + max_items = MaxItems, + marker = Marker}) of + {ok, #{roles := Roles, + marker := NewMarker, + is_truncated := IsTruncated}} -> + Doc = riak_cs_xml:to_xml( + #list_roles_response{roles = Roles, + request_id = RequestId, + marker = NewMarker, + is_truncated = IsTruncated}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + + +do_action("CreatePolicy", + Form, RD, Ctx = #rcs_web_context{response_module = ResponseMod, + request_id = RequestId}) -> + Specs = finish_tags( + lists:foldl(fun create_policy_fields_filter/2, #{}, Form)), + case create_policy_require_fields(Specs) of + false -> + ResponseMod:api_error(missing_parameter, RD, Ctx); + true -> + case riak_cs_iam:create_policy(Specs) of + {ok, Policy} -> + logger:info("Created managed policy \"~s\" (~s) on request_id ~s", + [Policy?IAM_POLICY.policy_id, Policy?IAM_POLICY.arn, RequestId]), + Doc = riak_cs_xml:to_xml( + #create_policy_response{policy = Policy, + request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end + end; + +do_action("GetPolicy", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + Arn = proplists:get_value("PolicyArn", Form), + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:get_policy(list_to_binary(Arn), Pbc) of + {ok, Policy} -> + Doc = riak_cs_xml:to_xml( + #get_policy_response{policy = Policy, + request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, notfound} -> + ResponseMod:api_error(no_such_policy, RD, Ctx); + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action("DeletePolicy", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + Arn = proplists:get_value("PolicyArn", Form), + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:delete_policy(list_to_binary(Arn), Pbc) of + ok -> + logger:info("Deleted policy with arn ~s on request_id ~s", [Arn, RequestId]), + Doc = riak_cs_xml:to_xml( + #delete_policy_response{request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, notfound} -> + ResponseMod:api_error(no_such_policy, RD, Ctx); + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action("ListPolicies", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + PathPrefix = proplists:get_value("PathPrefix", Form, ""), + OnlyAttached = proplists:get_value("OnlyAttached", Form, "false"), + PolicyUsageFilter = proplists:get_value("PolicyUsageFilter", Form, "All"), + Scope = proplists:get_value("Scope", Form, "All"), + MaxItems = proplists:get_value("MaxItems", Form), + Marker = proplists:get_value("Marker", Form), + case riak_cs_iam:list_policies( + RcPid, #list_policies_request{request_id = RequestId, + path_prefix = list_to_binary(PathPrefix), + only_attached = list_to_atom(OnlyAttached), + scope = list_to_atom(Scope), + policy_usage_filter = list_to_atom(PolicyUsageFilter), + max_items = MaxItems, + marker = Marker}) of + {ok, #{policies := Policies, + marker := NewMarker, + is_truncated := IsTruncated}} -> + Doc = riak_cs_xml:to_xml( + #list_policies_response{policies = Policies, + request_id = RequestId, + marker = NewMarker, + is_truncated = IsTruncated}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action("ListAttachedUserPolicies", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + PathPrefix = proplists:get_value("PathPrefix", Form, ""), + UserName = proplists:get_value("UserName", Form), + %% _MaxItems = proplists:get_value("MaxItems", Form), + %% _Marker = proplists:get_value("Marker", Form), + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:list_attached_user_policies( + list_to_binary(UserName), + list_to_binary(PathPrefix), Pbc) of + {ok, AANN} -> + NewMarker = undefined, + IsTruncated = false, + Doc = riak_cs_xml:to_xml( + #list_attached_user_policies_response{policies = AANN, + request_id = RequestId, + marker = NewMarker, + is_truncated = IsTruncated}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action("ListAttachedRolePolicies", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + PathPrefix = proplists:get_value("PathPrefix", Form, ""), + RoleName = proplists:get_value("RoleName", Form), + %% _MaxItems = proplists:get_value("MaxItems", Form), + %% _Marker = proplists:get_value("Marker", Form), + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:list_attached_role_policies( + list_to_binary(RoleName), + list_to_binary(PathPrefix), Pbc) of + {ok, AANN} -> + NewMarker = undefined, + IsTruncated = false, + Doc = riak_cs_xml:to_xml( + #list_attached_role_policies_response{policies = AANN, + request_id = RequestId, + marker = NewMarker, + is_truncated = IsTruncated}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action("AttachRolePolicy", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + PolicyArn = proplists:get_value("PolicyArn", Form), + RoleName = proplists:get_value("RoleName", Form), + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:attach_role_policy(list_to_binary(PolicyArn), + list_to_binary(RoleName), Pbc) of + ok -> + logger:info("Attached policy ~s to role ~s on request_id ~s", + [PolicyArn, RoleName, RequestId]), + Doc = riak_cs_xml:to_xml( + #attach_role_policy_response{request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action("AttachUserPolicy", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + UserName = proplists:get_value("UserName", Form), + PolicyArn = proplists:get_value("PolicyArn", Form), + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:attach_user_policy(list_to_binary(PolicyArn), + list_to_binary(UserName), Pbc) of + ok -> + logger:info("Attached policy ~s to user ~s on request_id ~s", + [PolicyArn, UserName, RequestId]), + Doc = riak_cs_xml:to_xml( + #attach_user_policy_response{request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action("DetachRolePolicy", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + RoleName = proplists:get_value("RoleName", Form), + PolicyArn = proplists:get_value("PolicyArn", Form), + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:detach_role_policy(list_to_binary(PolicyArn), + list_to_binary(RoleName), Pbc) of + ok -> + logger:info("Detached policy ~s from role ~s on request_id ~s", + [PolicyArn, RoleName, RequestId]), + Doc = riak_cs_xml:to_xml( + #detach_role_policy_response{request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action("DetachUserPolicy", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + PolicyArn = proplists:get_value("PolicyArn", Form), + UserName = proplists:get_value("UserName", Form), + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:detach_user_policy(list_to_binary(PolicyArn), + list_to_binary(UserName), Pbc) of + ok -> + logger:info("Detached policy ~s from user ~s on request_id ~s", + [PolicyArn, UserName, RequestId]), + Doc = riak_cs_xml:to_xml( + #detach_role_policy_response{request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + + +do_action("CreateSAMLProvider", + Form, RD, Ctx = #rcs_web_context{response_module = ResponseMod, + request_id = RequestId}) -> + Specs = finish_tags( + lists:foldl(fun create_saml_provider_fields_filter/2, #{}, Form)), + case create_saml_provider_require_fields(Specs) of + false -> + ResponseMod:api_error(missing_parameter, RD, Ctx); + true -> + case riak_cs_iam:create_saml_provider(Specs) of + {ok, {Arn, Tags}} -> + logger:info("Created SAML Provider \"~s\" (~s) on request_id ~s", + [maps:get(name, Specs), Arn, RequestId]), + Doc = riak_cs_xml:to_xml( + #create_saml_provider_response{saml_provider_arn = Arn, + tags = Tags, + request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end + end; + +do_action("GetSAMLProvider", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + Arn = proplists:get_value("SAMLProviderArn", Form), + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:get_saml_provider(list_to_binary(Arn), Pbc) of + {ok, ?IAM_SAML_PROVIDER{create_date = CreateDate, + valid_until = ValidUntil, + saml_metadata_document = SAMLMetadataDocument, + tags = Tags}} -> + Doc = riak_cs_xml:to_xml( + #get_saml_provider_response{create_date = CreateDate, + valid_until = ValidUntil, + saml_metadata_document = SAMLMetadataDocument, + tags = Tags, + request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, notfound} -> + ResponseMod:api_error(no_such_saml_provider, RD, Ctx); + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action("DeleteSAMLProvider", + Form, RD, Ctx = #rcs_web_context{response_module = ResponseMod, + request_id = RequestId}) -> + Arn = proplists:get_value("SAMLProviderArn", Form), + case riak_cs_iam:delete_saml_provider(list_to_binary(Arn)) of + ok -> + logger:info("Deleted SAML Provider with arn ~s on request_id ~s", [Arn, RequestId]), + Doc = riak_cs_xml:to_xml( + #delete_saml_provider_response{request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, notfound} -> + ResponseMod:api_error(no_such_saml_provider, RD, Ctx); + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action("ListSAMLProviders", + _Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + case riak_cs_iam:list_saml_providers( + RcPid, #list_saml_providers_request{request_id = RequestId}) of + {ok, #{saml_providers := PP}} -> + PPEE = [#saml_provider_list_entry{arn = Arn, + create_date = CreateDate, + valid_until = ValidUntil} + || ?IAM_SAML_PROVIDER{arn = Arn, + create_date = CreateDate, + valid_until = ValidUntil} <- PP], + Doc = riak_cs_xml:to_xml( + #list_saml_providers_response{saml_provider_list = PPEE, + request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action(Unsupported, _Form, RD, Ctx = #rcs_web_context{response_module = ResponseMod}) -> + logger:warning("IAM action ~s not supported yet; ignoring request", [Unsupported]), + ResponseMod:api_error(invalid_action, RD, Ctx). + + +create_user_fields_filter({K, V}, Acc) -> + case K of + "Path" -> + maps:put(path, list_to_binary(V), Acc); + "PermissionsBoundary" -> + maps:put(permissions_boundary, list_to_binary(V), Acc); + "UserName" -> + maps:put(user_name, list_to_binary(V), Acc); + "Tags.member." ++ TagMember -> + add_tag(TagMember, V, Acc); + CommonParameter when CommonParameter == "Action"; + CommonParameter == "Version" -> + Acc; + Unrecognized -> + logger:warning("Unrecognized parameter for CreateUser: ~s", [Unrecognized]), + Acc + end. +create_user_require_fields(FF) -> + lists:all(fun(A) -> lists:member(A, maps:keys(FF)) end, + [user_name]). + +create_role_fields_filter({K, V}, Acc) -> + case K of + "AssumeRolePolicyDocument" -> + maps:put(assume_role_policy_document, list_to_binary(V), Acc); + "Description" -> + maps:put(description, list_to_binary(V), Acc); + "MaxSessionDuration" -> + maps:put(max_session_duration, list_to_integer(V), Acc); + "Path" -> + maps:put(path, list_to_binary(V), Acc); + "PermissionsBoundary" -> + maps:put(permissions_boundary, list_to_binary(V), Acc); + "RoleName" -> + maps:put(role_name, list_to_binary(V), Acc); + "Tags.member." ++ TagMember -> + add_tag(TagMember, V, Acc); + CommonParameter when CommonParameter == "Action"; + CommonParameter == "Version" -> + Acc; + Unrecognized -> + logger:warning("Unrecognized parameter for CreateRole: ~s", [Unrecognized]), + Acc + end. +create_role_require_fields(FF) -> + lists:all(fun(A) -> lists:member(A, maps:keys(FF)) end, + [assume_role_policy_document, + role_name]). + +create_policy_fields_filter({K, V}, Acc) -> + case K of + "Description" -> + maps:put(description, list_to_binary(V), Acc); + "Path" -> + maps:put(path, list_to_binary(V), Acc); + "PolicyDocument" -> + maps:put(policy_document, list_to_binary(V), Acc); + "PolicyName" -> + maps:put(policy_name, list_to_binary(V), Acc); + "Tags.member." ++ TagMember -> + add_tag(TagMember, V, Acc); + CommonParameter when CommonParameter == "Action"; + CommonParameter == "Version" -> + Acc; + Unrecognized -> + logger:warning("Unrecognized parameter for CreatePolicy: ~s", [Unrecognized]), + Acc + end. +create_policy_require_fields(FF) -> + lists:all(fun(A) -> lists:member(A, maps:keys(FF)) end, + [policy_name, + policy_document]). + +create_saml_provider_fields_filter({K, V}, Acc) -> + case K of + "Name" -> + maps:put(name, list_to_binary(V), Acc); + "SAMLMetadataDocument" -> + maps:put(saml_metadata_document, list_to_binary(V), Acc); + "Tags.member." ++ TagMember -> + add_tag(TagMember, V, Acc); + CommonParameter when CommonParameter == "Action"; + CommonParameter == "Version" -> + Acc; + Unrecognized -> + logger:warning("Unrecognized parameter in call to CreateSAMLProvider: ~s", [Unrecognized]), + Acc + end. +create_saml_provider_require_fields(FF) -> + lists:all(fun(A) -> lists:member(A, maps:keys(FF)) end, + [name, + saml_metadata_document]). + +add_tag(A, V, Acc) -> + Tags0 = maps:get(tags, Acc, []), + Tags = + case string:tokens(A, ".") of + [N, "Key"] -> + lists:keystore({k, N}, 1, Tags0, {{k, N}, list_to_binary(V)}); + [N, "Value"] -> + lists:keystore({v, N}, 1, Tags0, {{v, N}, list_to_binary(V)}); + _ -> + logger:warning("Malformed Tags item", []) + end, + maps:put(tags, Tags, Acc). + +finish_tags(Acc) -> + Tags0 = lists:sort(maps:get(tags, Acc, [])), + Tags1 = lists:foldl( + fun({{k, N}, A}, Q) -> + lists:keystore(A, 1, Q, {{swap_me, N}, A}); + ({{v, N}, A}, Q) -> + {_, K} = lists:keyfind({swap_me, N}, 1, Q), + lists:keyreplace({swap_me, N}, 1, Q, {K, A}) + end, + [], Tags0), + Tags2 = [#{key => K, value => V} || {K, V} <- Tags1], + maps:put(tags, Tags2, Acc). diff --git a/apps/riak_cs/src/riak_cs_wm_info.erl b/apps/riak_cs/src/riak_cs_wm_info.erl new file mode 100644 index 000000000..8dc28695a --- /dev/null +++ b/apps/riak_cs/src/riak_cs_wm_info.erl @@ -0,0 +1,105 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023-2024 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_wm_info). + +-export([init/1, + allowed_methods/2, + service_available/2, + options/2, + forbidden/2, + content_types_provided/2, + to_json/2 + ]). + +-ignore_xref([init/1, + allowed_methods/2, + service_available/2, + options/2, + forbidden/2, + content_types_provided/2, + to_json/2 + ]). + +-include("riak_cs.hrl"). + +init(_Config) -> + Api = riak_cs_config:api(), + RespModule = riak_cs_config:response_module(Api), + {ok, #rcs_web_context{api = Api, + response_module = RespModule, + auth_bypass = riak_cs_config:auth_bypass()}}. + +-spec options(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), string()}], #wm_reqdata{}, #rcs_web_context{}}. +options(RD, Ctx) -> + {riak_cs_wm_utils:cors_headers(), RD, Ctx}. + +-spec service_available(#wm_reqdata{}, #rcs_web_context{}) -> {true, #wm_reqdata{}, #rcs_web_context{}}. +service_available(RD, Ctx) -> + riak_cs_wm_utils:service_available( + wrq:set_resp_headers(riak_cs_wm_utils:cors_headers(), RD), Ctx). + +-spec allowed_methods(#wm_reqdata{}, #rcs_web_context{}) -> {[atom()], #wm_reqdata{}, #rcs_web_context{}}. +allowed_methods(RD, Ctx) -> + {['GET', 'OPTIONS'], RD, Ctx}. + +-spec forbidden(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +forbidden(RD, Ctx) -> + case wrq:method(RD) of + 'OPTIONS' -> + {false, RD, Ctx}; + _ -> + forbidden2(RD, Ctx) + end. +forbidden2(RD, Ctx = #rcs_web_context{auth_bypass = AuthBypass}) -> + riak_cs_wm_utils:find_and_auth_admin(RD, Ctx, AuthBypass). + +content_types_provided(RD, Ctx) -> + {[{?JSON_TYPE, to_json}], RD, Ctx}. + +-spec to_json(#wm_reqdata{}, #rcs_web_context{}) -> {binary(), #wm_reqdata{}, #rcs_web_context{}}. +to_json(RD, Ctx) -> + {jsx:encode(gather_info()), RD, Ctx}. + +gather_info() -> + {MS, _} = erlang:statistics(wall_clock), + St = MS div 1000, + S = St rem 60, + Mt = St div 60, + M = Mt rem 60, + Ht = Mt div 60, + H = Ht rem 24, + Dt = Ht div 24, + D = Dt, + Str = case {D, H, M} of + {A, _, _} when A > 0 -> io_lib:format("~b day~s, ~b hour~s, ~b minute~s, ~b sec", [D, s(D), H, s(H), M, s(M), S]); + {_, A, _} when A > 0 -> io_lib:format("~b hour~s, ~b minute~s, ~b sec", [H, s(H), M, s(M), S]); + {_, _, A} when A > 0 -> io_lib:format("~b minute~s, ~b sec", [M, s(M), S]); + _ -> io_lib:format("~b sec", [S]) + end, + #{version => list_to_binary(?RCS_VERSION_STRING), + system_version => list_to_binary(lists:droplast(erlang:system_info(system_version))), + uptime => iolist_to_binary(Str), + storage_info => riak_cs_utils:gather_disk_usage_on_connected_riak_nodes() + }. + +s(1) -> ""; +s(_) -> "s". diff --git a/src/riak_cs_wm_not_implemented.erl b/apps/riak_cs/src/riak_cs_wm_not_implemented.erl similarity index 75% rename from src/riak_cs_wm_not_implemented.erl rename to apps/riak_cs/src/riak_cs_wm_not_implemented.erl index a5c74097b..b95ad9051 100644 --- a/src/riak_cs_wm_not_implemented.erl +++ b/apps/riak_cs/src/riak_cs_wm_not_implemented.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -23,16 +24,19 @@ -module(riak_cs_wm_not_implemented). -export([allowed_methods/0, - malformed_request/2]). + malformed_request/2 + ]). +-ignore_xref([allowed_methods/0, + malformed_request/2 + ]). -include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). -spec allowed_methods() -> [atom()]. allowed_methods() -> ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'CONNECT', 'OPTIONS']. --spec malformed_request(#wm_reqdata{}, #context{}) -> - {false | {halt, _}, #wm_reqdata{}, #context{}}. +-spec malformed_request(#wm_reqdata{}, #rcs_web_context{}) -> + {false | {halt, _}, #wm_reqdata{}, #rcs_web_context{}}. malformed_request(RD, Ctx) -> - riak_cs_s3_response:api_error(not_implemented, RD, Ctx). + riak_cs_aws_response:api_error(not_implemented, RD, Ctx). diff --git a/src/riak_cs_wm_object.erl b/apps/riak_cs/src/riak_cs_wm_object.erl similarity index 59% rename from src/riak_cs_wm_object.erl rename to apps/riak_cs/src/riak_cs_wm_object.erl index e8254e292..b6b3ef049 100644 --- a/src/riak_cs_wm_object.erl +++ b/apps/riak_cs/src/riak_cs_wm_object.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -32,21 +33,38 @@ content_types_accepted/2, accept_body/2, delete_resource/2, - valid_entity_length/2]). + valid_entity_length/2 + ]). + +-ignore_xref([init/1, + stats_prefix/0, + authorize/2, + content_types_provided/2, + generate_etag/2, + last_modified/2, + produce_body/2, + allowed_methods/0, + malformed_request/2, + content_types_accepted/2, + accept_body/2, + delete_resource/2, + valid_entity_length/2 + ]). -include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). +-include("riak_cs_web.hrl"). -include_lib("webmachine/include/wm_reqstate.hrl"). +-include_lib("kernel/include/logger.hrl"). --spec init(#context{}) -> {ok, #context{}}. +-spec init(#rcs_web_context{}) -> {ok, #rcs_web_context{}}. init(Ctx) -> - {ok, Ctx#context{local_context=#key_context{}}}. + {ok, Ctx#rcs_web_context{local_context = #key_context{}}}. -spec stats_prefix() -> object. stats_prefix() -> object. --spec malformed_request(#wm_reqdata{}, #context{}) -> {false, #wm_reqdata{}, #context{}}. -malformed_request(RD, #context{response_module=ResponseMod} = Ctx) -> +-spec malformed_request(#wm_reqdata{}, #rcs_web_context{}) -> {false, #wm_reqdata{}, #rcs_web_context{}}. +malformed_request(RD, #rcs_web_context{response_module=ResponseMod} = Ctx) -> case riak_cs_wm_utils:extract_key(RD, Ctx) of {error, Reason} -> ResponseMod:api_error(Reason, RD, Ctx); @@ -69,15 +87,16 @@ malformed_request(RD, #context{response_module=ResponseMod} = Ctx) -> %% object ACL and compare the permission requested with the permission %% granted, and allow or deny access. Returns a result suitable for %% directly returning from the {@link forbidden/2} webmachine export. --spec authorize(#wm_reqdata{}, #context{}) -> - {boolean() | {halt, term()}, #wm_reqdata{}, #context{}}. -authorize(RD, Ctx0=#context{local_context=LocalCtx0, - riak_client=RcPid}) -> +-spec authorize(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, term()}, #wm_reqdata{}, #rcs_web_context{}}. +authorize(RD, Ctx0 = #rcs_web_context{local_context = LocalCtx0, + riak_client = RcPid}) -> Method = wrq:method(RD), RequestedAccess = riak_cs_acl_utils:requested_access(Method, false), LocalCtx = riak_cs_wm_utils:ensure_doc(LocalCtx0, RcPid), - Ctx = Ctx0#context{requested_perm=RequestedAccess, local_context=LocalCtx}, + Ctx = Ctx0#rcs_web_context{requested_perm = RequestedAccess, + local_context = LocalCtx}, authorize(RD, Ctx, LocalCtx#key_context.bucket_object, Method, LocalCtx#key_context.manifest). @@ -97,7 +116,7 @@ allowed_methods() -> %% TODO: POST ['HEAD', 'GET', 'DELETE', 'PUT']. --spec valid_entity_length(#wm_reqdata{}, #context{}) -> {boolean(), #wm_reqdata{}, #context{}}. +-spec valid_entity_length(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. valid_entity_length(RD, Ctx) -> MaxLen = riak_cs_lfs_utils:max_content_len(), case riak_cs_wm_utils:valid_entity_length(MaxLen, RD, Ctx) of @@ -108,9 +127,9 @@ valid_entity_length(RD, Ctx) -> Other end. --spec content_types_provided(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_provided(RD, Ctx=#context{local_context=LocalCtx, - riak_client=RcPid}) -> +-spec content_types_provided(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_provided(RD, Ctx = #rcs_web_context{local_context = LocalCtx, + riak_client = RcPid}) -> Mfst = LocalCtx#key_context.manifest, %% TODO: %% As I understand S3, the content types provided @@ -123,7 +142,7 @@ content_types_provided(RD, Ctx=#context{local_context=LocalCtx, ContentType = binary_to_list(Mfst?MANIFEST.content_type), case ContentType of _ -> - UpdCtx = Ctx#context{local_context=UpdLocalCtx}, + UpdCtx = Ctx#rcs_web_context{local_context = UpdLocalCtx}, {[{ContentType, produce_body}], RD, UpdCtx} end; true -> @@ -132,23 +151,25 @@ content_types_provided(RD, Ctx=#context{local_context=LocalCtx, {[{"text/plain", produce_body}], RD, Ctx} end. --spec generate_etag(#wm_reqdata{}, #context{}) -> {string(), #wm_reqdata{}, #context{}}. -generate_etag(RD, Ctx=#context{local_context=LocalCtx}) -> +-spec generate_etag(#wm_reqdata{}, #rcs_web_context{}) -> {string(), #wm_reqdata{}, #rcs_web_context{}}. +generate_etag(RD, Ctx = #rcs_web_context{local_context = LocalCtx}) -> Mfst = LocalCtx#key_context.manifest, ETag = riak_cs_manifest:etag_no_quotes(Mfst), {ETag, RD, Ctx}. --spec last_modified(#wm_reqdata{}, #context{}) -> {calendar:datetime(), #wm_reqdata{}, #context{}}. -last_modified(RD, Ctx=#context{local_context=LocalCtx}) -> - Mfst = LocalCtx#key_context.manifest, - ErlDate = riak_cs_wm_utils:iso_8601_to_erl_date(Mfst?MANIFEST.created), +-spec last_modified(#wm_reqdata{}, #rcs_web_context{}) -> {calendar:datetime(), #wm_reqdata{}, #rcs_web_context{}}. +last_modified(RD, Ctx = #rcs_web_context{local_context + = #key_context{manifest + = ?MANIFEST{write_start_time = WST}}}) -> + ErlDate = calendar:system_time_to_universal_time(WST, millisecond), {ErlDate, RD, Ctx}. --spec produce_body(#wm_reqdata{}, #context{}) -> - {{known_length_stream, non_neg_integer(), {<<>>, function()}}, #wm_reqdata{}, #context{}}. -produce_body(RD, Ctx=#context{local_context=LocalCtx, - response_module=ResponseMod}) -> - #key_context{get_fsm_pid=GetFsmPid, manifest=Mfst} = LocalCtx, +-spec produce_body(#wm_reqdata{}, #rcs_web_context{}) -> + {{known_length_stream, non_neg_integer(), {<<>>, function()}}, #wm_reqdata{}, #rcs_web_context{}}. +produce_body(RD, Ctx = #rcs_web_context{local_context = LocalCtx, + response_module = ResponseMod}) -> + #key_context{get_fsm_pid = GetFsmPid, + manifest = Mfst} = LocalCtx, ResourceLength = Mfst?MANIFEST.content_length, case parse_range(RD, ResourceLength) of invalid_range -> @@ -162,30 +183,24 @@ produce_body(RD, Ctx=#context{local_context=LocalCtx, produce_body(RD, Ctx, RangeIndexes, RespRange) end. -produce_body(RD, Ctx=#context{rc_pool=RcPool, - riak_client=RcPid, - local_context=LocalCtx, - start_time=StartTime, - user=User}, +produce_body(RD, Ctx = #rcs_web_context{rc_pool = RcPool, + riak_client = RcPid, + local_context = LocalCtx, + start_time = StartTime}, {Start, End}, RespRange) -> - #key_context{get_fsm_pid=GetFsmPid, manifest=Mfst} = LocalCtx, - {Bucket, File} = Mfst?MANIFEST.bkey, - ResourceLength = Mfst?MANIFEST.content_length, - BFile_str = [Bucket, $,, File], - UserName = riak_cs_wm_utils:extract_name(User), + #key_context{get_fsm_pid = GetFsmPid, + manifest = ?MANIFEST{write_start_time = Created, + content_length = ResourceLength, + metadata = Metadata} = Mfst} = LocalCtx, Method = wrq:method(RD), - Func = case Method of - 'HEAD' -> <<"object_head">>; - _ -> <<"object_get">> - end, - riak_cs_dtrace:dt_object_entry(?MODULE, Func, [], [UserName, BFile_str]), - LastModified = riak_cs_wm_utils:to_rfc_1123(Mfst?MANIFEST.created), + LastModified = webmachine_util:rfc1123_date( + calendar:system_time_to_universal_time(Created, millisecond)), ETag = riak_cs_manifest:etag(Mfst), NewRQ1 = lists:foldl(fun({K, V}, Rq) -> wrq:set_resp_header(K, V, Rq) end, RD, [{"ETag", ETag}, {"Last-Modified", LastModified} - ] ++ Mfst?MANIFEST.metadata), + ] ++ Metadata), NewRQ2 = wrq:set_resp_range(RespRange, NewRQ1), NoBody = Method =:= 'HEAD' orelse ResourceLength =:= 0, {NewCtx, StreamBody} = @@ -198,18 +213,13 @@ produce_body(RD, Ctx=#context{rc_pool=RcPool, %% Streaming by `known_length_stream' and `StreamBody' function %% will be handled *after* WM's `finish_request' callback complets. %% Use `no_stats` to avoid auto stats update by `riak_cs_wm_common'. - {Ctx#context{auto_rc_close=false, stats_key=no_stats}, + {Ctx#rcs_web_context{auto_rc_close = false, + stats_key = no_stats}, {<<>>, fun() -> riak_cs_wm_utils:streaming_get( - RcPool, RcPid, GetFsmPid, StartTime, UserName, BFile_str) + RcPool, RcPid, GetFsmPid, StartTime) end}} end, - if Method == 'HEAD' -> - riak_cs_dtrace:dt_object_return(?MODULE, <<"object_head">>, - [], [UserName, BFile_str]); - true -> - ok - end, {{known_length_stream, ResourceLength, StreamBody}, NewRQ2, NewCtx}. parse_range(RD, ResourceLength) -> @@ -229,40 +239,35 @@ parse_range(RD, ResourceLength) -> end. %% @doc Callback for deleting an object. --spec delete_resource(#wm_reqdata{}, #context{}) -> {true, #wm_reqdata{}, #context{}}. -delete_resource(RD, Ctx=#context{local_context=LocalCtx, riak_client=RcPid}) -> - #key_context{bucket=Bucket, - key=Key, - get_fsm_pid=GetFsmPid} = LocalCtx, - BFile_str = [Bucket, $,, Key], - UserName = riak_cs_wm_utils:extract_name(Ctx#context.user), - riak_cs_dtrace:dt_object_entry(?MODULE, <<"object_delete">>, - [], [UserName, BFile_str]), +-spec delete_resource(#wm_reqdata{}, #rcs_web_context{}) -> {true, #wm_reqdata{}, #rcs_web_context{}}. +delete_resource(RD, Ctx = #rcs_web_context{local_context = LocalCtx, + riak_client = RcPid}) -> + #key_context{bucket = Bucket, + key = Key, + obj_vsn = ObjVsn, + get_fsm_pid = GetFsmPid} = LocalCtx, riak_cs_get_fsm:stop(GetFsmPid), - BinKey = list_to_binary(Key), - DeleteObjectResponse = riak_cs_utils:delete_object(Bucket, BinKey, RcPid), - handle_delete_object(DeleteObjectResponse, UserName, BFile_str, RD, Ctx). + DeleteObjectResponse = riak_cs_utils:delete_object(Bucket, Key, ObjVsn, RcPid), + handle_delete_object(DeleteObjectResponse, RD, Ctx). %% @private -handle_delete_object({error, Error}, UserName, BFile_str, RD, Ctx) -> - _ = lager:error("delete object failed with reason: ~p", [Error]), - riak_cs_dtrace:dt_object_return(?MODULE, <<"object_delete">>, [0], [UserName, BFile_str]), +handle_delete_object({error, Error}, RD, Ctx) -> + logger:error("delete object failed with reason: ~p", [Error]), {false, RD, Ctx}; -handle_delete_object({ok, _UUIDsMarkedforDelete}, UserName, BFile_str, RD, Ctx) -> - riak_cs_dtrace:dt_object_return(?MODULE, <<"object_delete">>, [1], [UserName, BFile_str]), +handle_delete_object({ok, _UUIDsMarkedforDelete}, RD, Ctx) -> {true, RD, Ctx}. --spec content_types_accepted(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. +-spec content_types_accepted(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. content_types_accepted(RD, Ctx) -> content_types_accepted(wrq:get_req_header("Content-Type", RD), RD, Ctx). --spec content_types_accepted(undefined | string(), #wm_reqdata{}, #context{}) -> - {[{string(), atom()}], #wm_reqdata{}, #context{}}. +-spec content_types_accepted(undefined | string(), #wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. content_types_accepted(CT, RD, Ctx) when CT =:= undefined; CT =:= [] -> content_types_accepted("application/octet-stream", RD, Ctx); -content_types_accepted(CT, RD, Ctx=#context{local_context=LocalCtx0}) -> +content_types_accepted(CT, RD, Ctx = #rcs_web_context{local_context = LocalCtx0}) -> %% This was shamelessly ripped out of %% https://github.com/basho/riak_kv/blob/0d91ca641a309f2962a216daa0cee869c82ffe26/src/riak_kv_wm_object.erl#L492 {Media, _Params} = mochiweb_util:parse_header(CT), @@ -270,7 +275,7 @@ content_types_accepted(CT, RD, Ctx=#context{local_context=LocalCtx0}) -> [_Type, _Subtype] -> %% accept whatever the user says LocalCtx = LocalCtx0#key_context{putctype=Media}, - {[{Media, add_acl_to_context_then_accept}], RD, Ctx#context{local_context=LocalCtx}}; + {[{Media, add_acl_to_context_then_accept}], RD, Ctx#rcs_web_context{local_context = LocalCtx}}; _ -> %% TODO: %% Maybe we should have caught @@ -287,18 +292,19 @@ content_types_accepted(CT, RD, Ctx=#context{local_context=LocalCtx0}) -> Ctx} end. --spec accept_body(#wm_reqdata{}, #context{}) -> - {{halt, integer()}, #wm_reqdata{}, #context{}}. -accept_body(RD, Ctx=#context{riak_client=RcPid, - local_context=LocalCtx, - response_module=ResponseMod}) +-spec accept_body(#wm_reqdata{}, #rcs_web_context{}) -> + {{halt, integer()}, #wm_reqdata{}, #rcs_web_context{}}. +accept_body(RD, Ctx = #rcs_web_context{riak_client=RcPid, + local_context=LocalCtx, + response_module=ResponseMod}) when LocalCtx#key_context.update_metadata == true -> %% zero-body put copy - just updating metadata - #key_context{bucket=Bucket, key=KeyStr, manifest=Mfst} = LocalCtx, + #key_context{bucket = Bucket, key = Key, obj_vsn = ObjVsn, + manifest = Mfst} = LocalCtx, Acl = Mfst?MANIFEST.acl, - NewAcl = Acl?ACL{creation_time = now()}, + NewAcl = Acl?ACL{creation_time = os:system_time(millisecond)}, {ContentType, Metadata} = riak_cs_copy_object:new_metadata(Mfst, RD), - case riak_cs_utils:set_object_acl(Bucket, list_to_binary(KeyStr), + case riak_cs_utils:set_object_acl(Bucket, Key, ObjVsn, Mfst?MANIFEST{metadata=Metadata, content_type=ContentType}, NewAcl, RcPid) of ok -> @@ -308,65 +314,78 @@ accept_body(RD, Ctx=#context{riak_client=RcPid, {error, Err} -> ResponseMod:api_error(Err, RD, Ctx) end; -accept_body(RD, #context{response_module=ResponseMod} = Ctx) -> +accept_body(RD, Ctx = #rcs_web_context{response_module = ResponseMod}) -> case riak_cs_copy_object:get_copy_source(RD) of undefined -> handle_normal_put(RD, Ctx); {error, _} = Err -> ResponseMod:api_error(Err, RD, Ctx); - {SrcBucket, SrcKey} -> - handle_copy_put(RD, Ctx#context{stats_key=[object, put_copy]}, - SrcBucket, SrcKey) + {SrcBucket, SrcKey, SrcObjVsn} -> + handle_copy_put(RD, Ctx#rcs_web_context{stats_key = [object, put_copy]}, + SrcBucket, SrcKey, SrcObjVsn) end. --spec handle_normal_put(#wm_reqdata{}, #context{}) -> - {{halt, integer()}, #wm_reqdata{}, #context{}}. -handle_normal_put(RD, Ctx) -> - #context{local_context=LocalCtx, - user=User, - acl=ACL, - riak_client=RcPid} = Ctx, - #key_context{bucket=Bucket, - key=Key, - putctype=ContentType, - size=Size, - get_fsm_pid=GetFsmPid} = LocalCtx, - - BFile_str = [Bucket, $,, Key], - UserName = riak_cs_wm_utils:extract_name(User), - riak_cs_dtrace:dt_object_entry(?MODULE, <<"object_put">>, - [], [UserName, BFile_str]), +handle_normal_put(RD, Ctx0) -> + #rcs_web_context{local_context = LocalCtx, + acl = ACL, + riak_client = RcPid} = Ctx0, + #key_context{bucket = Bucket, + key = Key, + obj_vsn = SuppliedVsn, + putctype = ContentType, + size = Size, + get_fsm_pid = GetFsmPid} = LocalCtx, + EventualVsn = determine_object_version(SuppliedVsn, Bucket, Key, RcPid), + Ctx1 = Ctx0#rcs_web_context{local_context = LocalCtx#key_context{obj_vsn = EventualVsn}}, + riak_cs_get_fsm:stop(GetFsmPid), Metadata = riak_cs_wm_utils:extract_user_metadata(RD), BlockSize = riak_cs_lfs_utils:block_size(), - Args = [{Bucket, list_to_binary(Key), Size, list_to_binary(ContentType), + Args = [{Bucket, Key, EventualVsn, Size, list_to_binary(ContentType), Metadata, BlockSize, ACL, timer:seconds(60), self(), RcPid}], {ok, Pid} = riak_cs_put_fsm_sup:start_put_fsm(node(), Args), try - accept_streambody(RD, Ctx, Pid, + accept_streambody(RD, Ctx1, Pid, wrq:stream_req_body(RD, riak_cs_lfs_utils:block_size())) catch - Type:Error -> + Type:Error:ST -> %% Want to catch mochiweb_socket:recv() returns {error, %% einval} or disconnected stuff, any errors prevents this %% manifests from being uploaded anymore - Res = riak_cs_put_fsm:force_stop(Pid), - _ = lager:debug("PUT FSM force_stop: ~p Reason: ~p", [Res, {Type, Error}]), + ?LOG_DEBUG("PUT FSM force_stop after ~p:~p. Stacktrace: ~p", [Type, Error, ST]), + _ = riak_cs_put_fsm:force_stop(Pid), error({Type, Error}) end. +determine_object_version(Vsn0, Bucket, Key, RcPid) -> + case {Vsn0, riak_cs_bucket:get_bucket_versioning(Bucket, RcPid)} of + {?LFS_DEFAULT_OBJECT_VERSION, {ok, #bucket_versioning{status = enabled}}} -> + Vsn1 = list_to_binary(riak_cs_utils:binary_to_hexlist(uuid:get_v4())), + logger:info("bucket \"~s\" has object versioning enabled," + " autogenerating version ~p for key ~p", [Bucket, Vsn1, Key]), + Vsn1; + {_, {ok, #bucket_versioning{status = enabled}}} -> + logger:info("bucket \"~s\" has object versioning enabled" + " but using ~p as supplied in request for key ~p", [Bucket, Vsn0, Key]), + Vsn0; + {?LFS_DEFAULT_OBJECT_VERSION, {ok, #bucket_versioning{status = suspended}}} -> + Vsn0; + {Vsn3, {ok, #bucket_versioning{status = suspended}}} -> + logger:warning("ignoring object version ~p in request for key ~p in bucket \"~s\"" + " as the bucket has object versioning suspended", [Vsn3, Key, Bucket]), + Vsn0 + end. + %% @doc the head is PUT copy path --spec handle_copy_put(#wm_reqdata{}, #context{}, binary(), binary()) -> - {boolean()|{halt, integer()}, #wm_reqdata{}, #context{}}. -handle_copy_put(RD, Ctx, SrcBucket, SrcKey) -> - #context{local_context=LocalCtx, - response_module=ResponseMod, - acl=Acl, - riak_client=RcPid} = Ctx, +handle_copy_put(RD, Ctx, SrcBucket, SrcKey, SrcObjVsn) -> + #rcs_web_context{local_context = LocalCtx, + response_module = ResponseMod, + acl = Acl, + riak_client = RcPid} = Ctx, %% manifest is always notfound|undefined here - #key_context{bucket=Bucket, key=KeyStr, get_fsm_pid=GetFsmPid} = LocalCtx, - Key = list_to_binary(KeyStr), + #key_context{bucket = Bucket, key = Key, obj_vsn = ObjVsn, + get_fsm_pid = GetFsmPid} = LocalCtx, {ok, ReadRcPid} = riak_cs_riak_client:checkout(), try @@ -374,7 +393,7 @@ handle_copy_put(RD, Ctx, SrcBucket, SrcKey) -> %% You'll also need permission to access source object, but RD and %% Ctx is of target object. Then access permission to source %% object has to be checked here. First of all, get manifest. - case riak_cs_manifest:fetch(ReadRcPid, SrcBucket, SrcKey) of + case riak_cs_manifest:fetch(ReadRcPid, SrcBucket, SrcKey, SrcObjVsn) of {ok, SrcManifest} -> EntityTooLarge = SrcManifest?MANIFEST.content_length > riak_cs_lfs_utils:max_content_len(), @@ -387,14 +406,14 @@ handle_copy_put(RD, Ctx, SrcBucket, SrcKey) -> {false, _, _} -> %% start copying - _ = lager:debug("copying! > ~s ~s => ~s ~s via ~p", - [SrcBucket, SrcKey, Bucket, Key, ReadRcPid]), + ?LOG_DEBUG("copying! > ~s/~s/~s => ~s/~s/~s via ~p", + [SrcBucket, SrcKey, SrcObjVsn, Bucket, Key, ObjVsn, ReadRcPid]), {ContentType, Metadata} = riak_cs_copy_object:new_metadata(SrcManifest, RD), - NewAcl = Acl?ACL{creation_time=os:timestamp()}, + NewAcl = Acl?ACL{creation_time = os:system_time(millisecond)}, {ok, PutFsmPid} = riak_cs_copy_object:start_put_fsm( - Bucket, Key, SrcManifest?MANIFEST.content_length, + Bucket, Key, ObjVsn, SrcManifest?MANIFEST.content_length, ContentType, Metadata, NewAcl, RcPid), %% Prepare for connection loss or client close @@ -406,17 +425,17 @@ handle_copy_put(RD, Ctx, SrcBucket, SrcKey) -> ETag = riak_cs_manifest:etag(DstManifest), RD2 = wrq:set_resp_header("ETag", ETag, RD), ResponseMod:copy_object_response(DstManifest, RD2, - Ctx#context{local_context=LocalCtx}); + Ctx#rcs_web_context{local_context = LocalCtx}); {true, _RD, _OtherCtx} -> %% access to source object not authorized %% TODO: check the return value / http status ResponseMod:api_error(copy_source_access_denied, RD, Ctx); - {{halt, 403}, _RD, _OtherCtx} = Error -> + {{halt, 403}, _RD, _OtherCtx} -> %% access to source object not authorized either, but %% in different return value ResponseMod:api_error(copy_source_access_denied, RD, Ctx); {Result, _, _} = Error -> - _ = lager:debug("~p on ~s ~s", [Result, SrcBucket, SrcKey]), + ?LOG_DEBUG("~p on ~s ~s", [Result, SrcBucket, SrcKey]), Error end; @@ -432,22 +451,13 @@ handle_copy_put(RD, Ctx, SrcBucket, SrcKey) -> riak_cs_riak_client:checkin(ReadRcPid) end. --spec accept_streambody(#wm_reqdata{}, #context{}, pid(), term()) -> {{halt, integer()}, #wm_reqdata{}, #context{}}. +-spec accept_streambody(#wm_reqdata{}, #rcs_web_context{}, pid(), term()) -> {{halt, integer()}, #wm_reqdata{}, #rcs_web_context{}}. accept_streambody(RD, - Ctx=#context{local_context=_LocalCtx=#key_context{size=0}}, + Ctx = #rcs_web_context{local_context = #key_context{size = 0}}, Pid, {_Data, _Next}) -> finalize_request(RD, Ctx, Pid); -accept_streambody(RD, - Ctx=#context{local_context=LocalCtx, - user=User}, - Pid, - {Data, Next}) -> - #key_context{bucket=Bucket, - key=Key} = LocalCtx, - BFile_str = [Bucket, $,, Key], - UserName = riak_cs_wm_utils:extract_name(User), - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"accept_streambody">>, [size(Data)], [UserName, BFile_str]), +accept_streambody(RD, Ctx, Pid, {Data, Next}) -> riak_cs_put_fsm:augment_data(Pid, Data), if is_function(Next) -> accept_streambody(RD, Ctx, Pid, Next()); @@ -458,19 +468,17 @@ accept_streambody(RD, %% TODO: %% We need to do some checking to make sure the bucket exists %% for the user who is doing this PUT --spec finalize_request(#wm_reqdata{}, #context{}, pid()) -> {{halt, 200}, #wm_reqdata{}, #context{}}. +-spec finalize_request(#wm_reqdata{}, #rcs_web_context{}, pid()) -> {{halt, 200}, #wm_reqdata{}, #rcs_web_context{}}. finalize_request(RD, - Ctx=#context{local_context=LocalCtx, - response_module=ResponseMod, - user=User}, + Ctx = #rcs_web_context{local_context = LocalCtx, + response_module = ResponseMod}, Pid) -> - #key_context{bucket=Bucket, - key=Key, - size=S} = LocalCtx, - BFile_str = [Bucket, $,, Key], - UserName = riak_cs_wm_utils:extract_name(User), - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"finalize_request">>, [S], [UserName, BFile_str]), - ContentMD5 = wrq:get_req_header("content-md5", RD), + #key_context{obj_vsn = Vsn, + size = S} = LocalCtx, + ContentMD5 = case wrq:get_req_header("content-md5", RD) of + undefined -> undefined; + A -> list_to_binary(A) + end, Response = case riak_cs_put_fsm:finalize(Pid, ContentMD5) of {ok, Manifest} -> @@ -478,31 +486,29 @@ finalize_request(RD, %% TODO: probably want something that counts actual bytes uploaded %% instead, to record partial/aborted uploads AccessRD = riak_cs_access_log_handler:set_bytes_in(S, RD), - {{halt, 200}, wrq:set_resp_header("ETag", ETag, AccessRD), Ctx}; + {{halt, 200}, wrq:set_resp_headers([{"ETag", ETag}, + {"x-amz-version-id", Vsn}], AccessRD), Ctx}; {error, invalid_digest} -> ResponseMod:invalid_digest_response(ContentMD5, RD, Ctx); {error, Reason} -> ResponseMod:api_error(Reason, RD, Ctx) end, - riak_cs_dtrace:dt_wm_return(?MODULE, <<"finalize_request">>, [S], [UserName, BFile_str]), - riak_cs_dtrace:dt_object_return(?MODULE, <<"object_put">>, [S], [UserName, BFile_str]), Response. -check_0length_metadata_update(Length, RD, Ctx=#context{local_context=LocalCtx}) -> +check_0length_metadata_update(Length, RD, Ctx = #rcs_web_context{local_context = LocalCtx}) -> %% The authorize() callback has already been called, which means %% that ensure_doc() has been called, so the local context %% manifest is up-to-date: the object exists or it doesn't. case (not is_atom(LocalCtx#key_context.manifest) andalso zero_length_metadata_update_p(Length, RD)) of false -> - UpdLocalCtx = LocalCtx#key_context{size=Length}, - {true, RD, Ctx#context{local_context=UpdLocalCtx}}; + UpdLocalCtx = LocalCtx#key_context{size = Length}, + {true, RD, Ctx#rcs_web_context{local_context = UpdLocalCtx}}; true -> - UpdLocalCtx = LocalCtx#key_context{size=Length, - update_metadata=true}, - {true, RD, Ctx#context{ - stats_key=[object, put_copy], - local_context=UpdLocalCtx}} + UpdLocalCtx = LocalCtx#key_context{size = Length, + update_metadata = true}, + {true, RD, Ctx#rcs_web_context{stats_key = [object, put_copy], + local_context = UpdLocalCtx}} end. zero_length_metadata_update_p(0, RD) -> diff --git a/src/riak_cs_wm_object_acl.erl b/apps/riak_cs/src/riak_cs_wm_object_acl.erl similarity index 58% rename from src/riak_cs_wm_object_acl.erl rename to apps/riak_cs/src/riak_cs_wm_object_acl.erl index 1cd06aa12..993783dd1 100644 --- a/src/riak_cs_wm_object_acl.erl +++ b/apps/riak_cs/src/riak_cs_wm_object_acl.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -28,19 +29,31 @@ content_types_accepted/2, content_types_provided/2, accept_body/2, - produce_body/2]). + produce_body/2 + ]). + +-ignore_xref([init/1, + stats_prefix/0, + allowed_methods/0, + malformed_request/2, + authorize/2, + content_types_accepted/2, + content_types_provided/2, + accept_body/2, + produce_body/2 + ]). -include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). +-include_lib("kernel/include/logger.hrl"). init(Ctx) -> - {ok, Ctx#context{local_context=#key_context{}}}. + {ok, Ctx#rcs_web_context{local_context = #key_context{}}}. -spec stats_prefix() -> object_acl. stats_prefix() -> object_acl. --spec malformed_request(#wm_reqdata{}, #context{}) -> {false, #wm_reqdata{}, #context{}}. -malformed_request(RD, #context{response_module=ResponseMod} = Ctx) -> +-spec malformed_request(#wm_reqdata{}, #rcs_web_context{}) -> {false, #wm_reqdata{}, #rcs_web_context{}}. +malformed_request(RD, #rcs_web_context{response_module = ResponseMod} = Ctx) -> case riak_cs_wm_utils:has_acl_header_and_body(RD) of true -> ResponseMod:api_error(unexpected_content, RD, Ctx); @@ -57,13 +70,15 @@ malformed_request(RD, #context{response_module=ResponseMod} = Ctx) -> %% object ACL and compare the permission requested with the permission %% granted, and allow or deny access. Returns a result suitable for %% directly returning from the {@link forbidden/2} webmachine export. -authorize(RD, Ctx0=#context{local_context=LocalCtx0, riak_client=RcPid}) -> +authorize(RD, Ctx0 = #rcs_web_context{local_context = LocalCtx0, + riak_client = RcPid}) -> Method = wrq:method(RD), RequestedAccess = %% This is really the only difference between authorize/2 in this module and riak_cs_wm_object riak_cs_acl_utils:requested_access(Method, true), LocalCtx = riak_cs_wm_utils:ensure_doc(LocalCtx0, RcPid), - Ctx = Ctx0#context{requested_perm=RequestedAccess,local_context=LocalCtx}, + Ctx = Ctx0#rcs_web_context{requested_perm =RequestedAccess, + local_context = LocalCtx}, authorize(RD, Ctx, LocalCtx#key_context.bucket_object, Method, LocalCtx#key_context.manifest). @@ -83,9 +98,10 @@ allowed_methods() -> ['GET', 'PUT']. --spec content_types_provided(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_provided(RD, Ctx=#context{local_context=LocalCtx, - riak_client=RcPid}) -> +-spec content_types_provided(#wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_provided(RD, Ctx = #rcs_web_context{local_context = LocalCtx, + riak_client = RcPid}) -> Mfst = LocalCtx#key_context.manifest, %% TODO: %% As I understand S3, the content types provided @@ -98,7 +114,7 @@ content_types_provided(RD, Ctx=#context{local_context=LocalCtx, ContentType = binary_to_list(Mfst?MANIFEST.content_type), case ContentType of _ -> - UpdCtx = Ctx#context{local_context=UpdLocalCtx}, + UpdCtx = Ctx#rcs_web_context{local_context = UpdLocalCtx}, {[{ContentType, produce_body}], RD, UpdCtx} end; true -> @@ -107,15 +123,15 @@ content_types_provided(RD, Ctx=#context{local_context=LocalCtx, {[{"text/plain", produce_body}], RD, Ctx} end. --spec content_types_accepted(term(), term()) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_accepted(RD, Ctx=#context{local_context=LocalCtx0}) -> +-spec content_types_accepted(term(), term()) -> {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_accepted(RD, Ctx = #rcs_web_context{local_context = LocalCtx0}) -> case wrq:get_req_header("Content-Type", RD) of undefined -> DefaultCType = "application/octet-stream", LocalCtx = LocalCtx0#key_context{putctype=DefaultCType}, {[{DefaultCType, add_acl_to_context_then_accept}], RD, - Ctx#context{local_context=LocalCtx}}; + Ctx#rcs_web_context{local_context = LocalCtx}}; %% This was shamelessly ripped out of %% https://github.com/basho/riak_kv/blob/0d91ca641a309f2962a216daa0cee869c82ffe26/src/riak_kv_wm_object.erl#L492 CType -> @@ -124,7 +140,8 @@ content_types_accepted(RD, Ctx=#context{local_context=LocalCtx0}) -> [_Type, _Subtype] -> %% accept whatever the user says LocalCtx = LocalCtx0#key_context{putctype=Media}, - {[{Media, add_acl_to_context_then_accept}], RD, Ctx#context{local_context=LocalCtx}}; + {[{Media, add_acl_to_context_then_accept}], + RD, Ctx#rcs_web_context{local_context = LocalCtx}}; _ -> %% TODO: %% Maybe we should have caught @@ -143,43 +160,32 @@ content_types_accepted(RD, Ctx=#context{local_context=LocalCtx0}) -> end. --spec produce_body(term(), term()) -> {iolist()|binary(), term(), term()}. -produce_body(RD, Ctx=#context{local_context=LocalCtx, - requested_perm='READ_ACP', - user=User}) -> - #key_context{get_fsm_pid=GetFsmPid, manifest=Mfst} = LocalCtx, - {Bucket, File} = Mfst?MANIFEST.bkey, - BFile_str = [Bucket, $,, File], - UserName = riak_cs_wm_utils:extract_name(User), - riak_cs_dtrace:dt_object_entry(?MODULE, <<"object_acl_get">>, - [], [UserName, BFile_str]), +-spec produce_body(#wm_reqdata{}, #rcs_web_context{}) -> {iolist()|binary(), term(), term()}. +produce_body(RD, Ctx = #rcs_web_context{local_context = LocalCtx, + requested_perm = 'READ_ACP'}) -> + #key_context{get_fsm_pid = GetFsmPid, + manifest = ?MANIFEST{acl = Acl}} = LocalCtx, riak_cs_get_fsm:stop(GetFsmPid), - Acl = Mfst?MANIFEST.acl, - {AclXml, DtraceTag} = case Acl of - undefined -> {riak_cs_acl_utils:empty_acl_xml(), -1}; - _ -> {riak_cs_xml:to_xml(Acl), -2} - end, - riak_cs_dtrace:dt_object_return(?MODULE, <<"object_acl_get">>, - [DtraceTag], [UserName, BFile_str]), + AclXml = case Acl of + no_acl_yet -> riak_cs_acl_utils:empty_acl_xml(); + _ -> riak_cs_xml:to_xml(Acl) + end, {AclXml, RD, Ctx}. --spec accept_body(term(), term()) -> +-spec accept_body(#wm_reqdata{}, #rcs_web_context{}) -> {boolean() | {halt, term()}, term(), term()}. -accept_body(RD, Ctx=#context{local_context=#key_context{get_fsm_pid=GetFsmPid, - manifest=Mfst, - key=KeyStr, - bucket=Bucket}, - user=User, - acl=AclFromHeadersOrDefault, - requested_perm='WRITE_ACP', - riak_client=RcPid}) when Bucket /= undefined, - KeyStr /= undefined, - Mfst /= undefined, - RcPid /= undefined -> - BFile_str = [Bucket, $,, KeyStr], - UserName = riak_cs_wm_utils:extract_name(User), - riak_cs_dtrace:dt_object_entry(?MODULE, <<"object_put_acl">>, - [], [UserName, BFile_str]), +accept_body(RD, Ctx = #rcs_web_context{local_context = #key_context{get_fsm_pid = GetFsmPid, + manifest = Mfst, + key = Key, + obj_vsn = Vsn, + bucket = Bucket}, + user = User, + acl = AclFromHeadersOrDefault, + requested_perm = 'WRITE_ACP', + riak_client = RcPid}) when Bucket /= undefined, + Key /= undefined, + Mfst /= undefined, + RcPid /= undefined -> riak_cs_get_fsm:stop(GetFsmPid), Body = binary_to_list(wrq:req_body(RD)), AclRes = @@ -187,30 +193,22 @@ accept_body(RD, Ctx=#context{local_context=#key_context{get_fsm_pid=GetFsmPid, [] -> {ok, AclFromHeadersOrDefault}; _ -> + A1 = riak_cs_acl_utils:acl_from_xml(Body, + User?RCS_USER.key_id, + RcPid), riak_cs_acl_utils:validate_acl( - riak_cs_acl_utils:acl_from_xml(Body, - User?RCS_USER.key_id, - RcPid), - User?RCS_USER.canonical_id) + A1, + User?RCS_USER.id) end, case AclRes of {ok, Acl} -> %% Write new ACL to active manifest - Key = list_to_binary(KeyStr), - case riak_cs_utils:set_object_acl(Bucket, Key, Mfst, Acl, RcPid) of + case riak_cs_utils:set_object_acl(Bucket, Key, Vsn, Mfst, Acl, RcPid) of ok -> - riak_cs_dtrace:dt_object_return(?MODULE, <<"object_acl_put">>, - [200], [UserName, BFile_str]), {{halt, 200}, RD, Ctx}; {error, Reason} -> - Code = riak_cs_s3_response:status_code(Reason), - riak_cs_dtrace:dt_object_return(?MODULE, <<"object_acl_put">>, - [Code], [UserName, BFile_str]), - riak_cs_s3_response:api_error(Reason, RD, Ctx) + riak_cs_aws_response:api_error(Reason, RD, Ctx) end; {error, Reason2} -> - Code = riak_cs_s3_response:status_code(Reason2), - riak_cs_dtrace:dt_object_return(?MODULE, <<"object_acl_put">>, - [Code], [UserName, BFile_str]), - riak_cs_s3_response:api_error(Reason2, RD, Ctx) + riak_cs_aws_response:api_error(Reason2, RD, Ctx) end. diff --git a/src/riak_cs_wm_object_upload.erl b/apps/riak_cs/src/riak_cs_wm_object_upload.erl similarity index 66% rename from src/riak_cs_wm_object_upload.erl rename to apps/riak_cs/src/riak_cs_wm_object_upload.erl index 91aca63ba..3f507fa85 100644 --- a/src/riak_cs_wm_object_upload.erl +++ b/apps/riak_cs/src/riak_cs_wm_object_upload.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -29,20 +30,32 @@ content_types_accepted/2, post_is_create/2, process_post/2, - valid_entity_length/2]). + valid_entity_length/2 + ]). + +-ignore_xref([init/1, + stats_prefix/0, + authorize/2, + content_types_provided/2, + allowed_methods/0, + malformed_request/2, + content_types_accepted/2, + post_is_create/2, + process_post/2, + valid_entity_length/2 + ]). -include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). --spec init(#context{}) -> {ok, #context{}}. +-spec init(#rcs_web_context{}) -> {ok, #rcs_web_context{}}. init(Ctx) -> - {ok, Ctx#context{local_context=#key_context{}}}. + {ok, Ctx#rcs_web_context{local_context = #key_context{}}}. -spec stats_prefix() -> multipart. stats_prefix() -> multipart. --spec malformed_request(#wm_reqdata{}, #context{}) -> {false, #wm_reqdata{}, #context{}}. -malformed_request(RD, #context{response_module=ResponseMod} = Ctx) -> +-spec malformed_request(#wm_reqdata{}, #rcs_web_context{}) -> {false, #wm_reqdata{}, #rcs_web_context{}}. +malformed_request(RD, Ctx = #rcs_web_context{response_module = ResponseMod}) -> case riak_cs_wm_utils:extract_key(RD, Ctx) of {error, Reason} -> ResponseMod:api_error(Reason, RD, Ctx); @@ -60,14 +73,16 @@ malformed_request(RD, #context{response_module=ResponseMod} = Ctx) -> %% object ACL and compare the permission requested with the permission %% granted, and allow or deny access. Returns a result suitable for %% directly returning from the {@link forbidden/2} webmachine export. --spec authorize(#wm_reqdata{}, #context{}) -> - {boolean() | {halt, term()}, #wm_reqdata{}, #context{}}. -authorize(RD, Ctx0=#context{local_context=LocalCtx0, riak_client=RcPid}) -> +-spec authorize(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, term()}, #wm_reqdata{}, #rcs_web_context{}}. +authorize(RD, Ctx0 = #rcs_web_context{local_context = LocalCtx0, + riak_client = RcPid}) -> Method = wrq:method(RD), RequestedAccess = riak_cs_acl_utils:requested_access(Method, false), LocalCtx = riak_cs_wm_utils:ensure_doc(LocalCtx0, RcPid), - Ctx = Ctx0#context{requested_perm=RequestedAccess,local_context=LocalCtx}, + Ctx = Ctx0#rcs_web_context{requested_perm = RequestedAccess, + local_context = LocalCtx}, authorize(RD, Ctx, LocalCtx#key_context.bucket_object). authorize(RD, Ctx, notfound = _BucketObj) -> @@ -91,21 +106,25 @@ process_post(RD, Ctx) -> HaltResponse end. -process_post_helper(RD, Ctx=#context{riak_client=RcPid, local_context=LocalCtx, acl=ACL}) -> - #key_context{bucket=Bucket, key=Key} = LocalCtx, +process_post_helper(RD, Ctx = #rcs_web_context{riak_client = RcPid, + local_context = #key_context{bucket = Bucket, + key = Key, + obj_vsn = ObjVsn}, + acl = ACL}) -> ContentType = try list_to_binary(wrq:get_req_header("Content-Type", RD)) catch error:badarg -> %% Per http://docs.amazonwebservices.com/AmazonS3/latest/API/mpUploadInitiate.html <<"binary/octet-stream">> end, - User = riak_cs_user:to_3tuple(Ctx#context.user), + User = riak_cs_user:to_3tuple(Ctx#rcs_web_context.user), Metadata = riak_cs_wm_utils:extract_user_metadata(RD), Opts = [{acl, ACL}, {meta_data, Metadata}], - case riak_cs_mp_utils:initiate_multipart_upload(Bucket, list_to_binary(Key), - ContentType, User, Opts, - RcPid) of + case riak_cs_mp_utils:initiate_multipart_upload( + Bucket, Key, ObjVsn, + ContentType, User, Opts, + RcPid) of {ok, UploadId} -> XmlDoc = {'InitiateMultipartUploadResult', [{'xmlns', "http://s3.amazonaws.com/doc/2006-03-01/"}], @@ -119,11 +138,11 @@ process_post_helper(RD, Ctx=#context{riak_client=RcPid, local_context=LocalCtx, RD2 = wrq:set_resp_body(Body, RD), {true, RD2, Ctx}; {error, Reason} -> - riak_cs_s3_response:api_error(Reason, RD, Ctx) + riak_cs_aws_response:api_error(Reason, RD, Ctx) end. --spec valid_entity_length(#wm_reqdata{}, #context{}) -> {boolean(), #wm_reqdata{}, #context{}}. -valid_entity_length(RD, Ctx=#context{local_context=LocalCtx}) -> +-spec valid_entity_length(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +valid_entity_length(RD, Ctx = #rcs_web_context{local_context = LocalCtx}) -> case wrq:method(RD) of 'PUT' -> case catch( @@ -132,11 +151,11 @@ valid_entity_length(RD, Ctx=#context{local_context=LocalCtx}) -> Length when is_integer(Length) -> case Length =< riak_cs_lfs_utils:max_content_len() of false -> - riak_cs_s3_response:api_error( + riak_cs_aws_response:api_error( entity_too_large, RD, Ctx); true -> - UpdLocalCtx = LocalCtx#key_context{size=Length}, - {true, RD, Ctx#context{local_context=UpdLocalCtx}} + UpdLocalCtx = LocalCtx#key_context{size = Length}, + {true, RD, Ctx#rcs_web_context{local_context = UpdLocalCtx}} end; _ -> {false, RD, Ctx} @@ -145,8 +164,8 @@ valid_entity_length(RD, Ctx=#context{local_context=LocalCtx}) -> {true, RD, Ctx} end. --spec content_types_provided(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_provided(RD, Ctx=#context{}) -> +-spec content_types_provided(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_provided(RD, Ctx = #rcs_web_context{}) -> Method = wrq:method(RD), if Method == 'POST' -> {[{?XML_TYPE, unused_callback}], RD, Ctx}; @@ -156,6 +175,6 @@ content_types_provided(RD, Ctx=#context{}) -> {[{"text/plain", unused_callback}], RD, Ctx} end. --spec content_types_accepted(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. +-spec content_types_accepted(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. content_types_accepted(RD, Ctx) -> riak_cs_mp_utils:make_content_types_accepted(RD, Ctx). diff --git a/src/riak_cs_wm_object_upload_part.erl b/apps/riak_cs/src/riak_cs_wm_object_upload_part.erl similarity index 60% rename from src/riak_cs_wm_object_upload_part.erl rename to apps/riak_cs/src/riak_cs_wm_object_upload_part.erl index 174733c54..b9fec480c 100644 --- a/src/riak_cs_wm_object_upload_part.erl +++ b/apps/riak_cs/src/riak_cs_wm_object_upload_part.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -32,24 +33,40 @@ valid_entity_length/2, delete_resource/2, accept_body/2, - to_xml/2]). + to_xml/2 + ]). + +-ignore_xref([init/1, + stats_prefix/0, + authorize/2, + content_types_provided/2, + allowed_methods/0, + malformed_request/2, + content_types_accepted/2, + post_is_create/2, + process_post/2, + valid_entity_length/2, + delete_resource/2, + accept_body/2, + to_xml/2 + ]). -include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). -include_lib("webmachine/include/wm_reqstate.hrl"). -include_lib("xmerl/include/xmerl.hrl"). +-include_lib("kernel/include/logger.hrl"). --spec init(#context{}) -> {ok, #context{}}. +-spec init(#rcs_web_context{}) -> {ok, #rcs_web_context{}}. init(Ctx) -> - %% {ok, Ctx#context{local_context=#key_context{}}}. - {ok, Ctx#context{local_context=#key_context{}}}. + %% {ok, Ctx#rcs_web_context{local_context=#key_context{}}}. + {ok, Ctx#rcs_web_context{local_context = #key_context{}}}. -spec stats_prefix() -> multipart_upload. stats_prefix() -> multipart_upload. --spec malformed_request(#wm_reqdata{}, #context{}) -> - {false, #wm_reqdata{}, #context{}} | {{halt, pos_integer()}, #wm_reqdata{}, #context{}}. -malformed_request(RD, #context{response_module=ResponseMod} = Ctx) -> +-spec malformed_request(#wm_reqdata{}, #rcs_web_context{}) -> + {false, #wm_reqdata{}, #rcs_web_context{}} | {{halt, pos_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +malformed_request(RD, Ctx = #rcs_web_context{response_module = ResponseMod}) -> Method = wrq:method(RD), case Method == 'PUT' andalso not valid_part_number(RD) of %% For multipart upload part, @@ -76,14 +93,16 @@ valid_part_number(RD) -> %% object ACL and compare the permission requested with the permission %% granted, and allow or deny access. Returns a result suitable for %% directly returning from the {@link forbidden/2} webmachine export. --spec authorize(#wm_reqdata{}, #context{}) -> - {boolean() | {halt, term()}, #wm_reqdata{}, #context{}}. -authorize(RD, Ctx0=#context{local_context=LocalCtx0, riak_client=RcPid}) -> +-spec authorize(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, term()}, #wm_reqdata{}, #rcs_web_context{}}. +authorize(RD, Ctx0 = #rcs_web_context{local_context = LocalCtx0, + riak_client = RcPid}) -> Method = wrq:method(RD), RequestedAccess = riak_cs_acl_utils:requested_access(Method, false), LocalCtx = riak_cs_wm_utils:ensure_doc(LocalCtx0, RcPid), - Ctx = Ctx0#context{requested_perm=RequestedAccess, local_context=LocalCtx}, + Ctx = Ctx0#rcs_web_context{requested_perm = RequestedAccess, + local_context = LocalCtx}, authorize(RD, Ctx, LocalCtx#key_context.bucket_object, Method, LocalCtx#key_context.manifest). @@ -109,26 +128,32 @@ allowed_methods() -> post_is_create(RD, Ctx) -> {false, RD, Ctx}. -process_post(RD, Ctx=#context{local_context=LocalCtx, riak_client=RcPid}) -> - #key_context{bucket=Bucket, key=Key} = LocalCtx, - User = riak_cs_user:to_3tuple(Ctx#context.user), +process_post(RD, Ctx = #rcs_web_context{riak_client = RcPid, + local_context = #key_context{bucket = Bucket, + key = Key, + obj_vsn = ObjVsn}}) -> + User = riak_cs_user:to_3tuple(Ctx#rcs_web_context.user), UploadId64 = re:replace(wrq:path(RD), ".*/uploads/", "", [{return, binary}]), Body = binary_to_list(wrq:req_body(RD)), case {parse_body(Body), catch base64url:decode(UploadId64)} of {bad, _} -> {{halt,477}, RD, Ctx}; + %% RCS-156 (gh #1100) return a 400 Bad Request, Malformed XML error + %% when there is a multipart upload without any parts in the upload + %% complete message + {[], _UploadId} -> + riak_cs_aws_response:api_error(malformed_xml, RD, Ctx); {PartETags, UploadId} -> case riak_cs_mp_utils:complete_multipart_upload( - Bucket, list_to_binary(Key), UploadId, PartETags, User, + Bucket, Key, ObjVsn, UploadId, PartETags, User, RcPid) of {ok, NewManifest} -> ETag = riak_cs_manifest:etag(NewManifest), - _ = lager:debug("checksum of all parts checksum: ~p", [ETag]), + ?LOG_DEBUG("checksum of all parts: ~p", [ETag]), XmlDoc = {'CompleteMultipartUploadResult', [{'xmlns', "http://s3.amazonaws.com/doc/2006-03-01/"}], [ - %% TODO: use cs_root from app.config - {'Location', [lists:append(["http://", binary_to_list(Bucket), ".s3.amazonaws.com/", Key])]}, + {'Location', [response_location(Bucket, Key)]}, {'Bucket', [Bucket]}, {'Key', [Key]}, {'ETag', [ETag]} @@ -138,42 +163,48 @@ process_post(RD, Ctx=#context{local_context=LocalCtx, riak_client=RcPid}) -> RD2 = wrq:set_resp_body(XmlBody, RD), {true, RD2, Ctx}; {error, notfound} -> - riak_cs_s3_response:no_such_upload_response(UploadId, RD, Ctx); + riak_cs_aws_response:no_such_upload_response(UploadId, RD, Ctx); {error, Reason} -> - riak_cs_s3_response:api_error(Reason, RD, Ctx) + riak_cs_aws_response:api_error(Reason, RD, Ctx) end end. --spec valid_entity_length(#wm_reqdata{}, #context{}) -> {boolean(), #wm_reqdata{}, #context{}}. +response_location(Bucket, Key) -> + iolist_to_binary(["http://", Bucket, ".", riak_cs_config:s3_root_host(), "/", Key]). + +-spec valid_entity_length(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. valid_entity_length(RD, Ctx) -> MaxLen = riak_cs_lfs_utils:max_content_len(), riak_cs_wm_utils:valid_entity_length(MaxLen, RD, Ctx). --spec delete_resource(#wm_reqdata{}, #context{}) -> - {boolean() | {'halt', term()}, #wm_reqdata{}, #context{}}. -delete_resource(RD, Ctx=#context{local_context=LocalCtx, - riak_client=RcPid}) -> +-spec delete_resource(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {'halt', term()}, #wm_reqdata{}, #rcs_web_context{}}. +delete_resource(RD, Ctx = #rcs_web_context{local_context = LocalCtx, + riak_client = RcPid}) -> ReqUploadId = wrq:path_info('uploadId', RD), case (catch base64url:decode(ReqUploadId)) of {'EXIT', _Reason} -> - riak_cs_s3_response:no_such_upload_response({raw, ReqUploadId}, RD, Ctx); + riak_cs_aws_response:no_such_upload_response({raw, ReqUploadId}, RD, Ctx); UploadId -> - #key_context{bucket=Bucket, key=KeyStr} = LocalCtx, - Key = list_to_binary(KeyStr), - User = riak_cs_user:to_3tuple(Ctx#context.user), - case riak_cs_mp_utils:abort_multipart_upload(Bucket, Key, UploadId, - User, RcPid) of + #key_context{bucket = Bucket, + key = Key, + obj_vsn = ObjVsn} = LocalCtx, + User = riak_cs_user:to_3tuple(Ctx#rcs_web_context.user), + case riak_cs_mp_utils:abort_multipart_upload( + Bucket, Key, ObjVsn, UploadId, + User, RcPid) of ok -> {true, RD, Ctx}; {error, notfound} -> - riak_cs_s3_response:no_such_upload_response(UploadId, RD, Ctx); + riak_cs_aws_response:no_such_upload_response(UploadId, RD, Ctx); {error, Reason} -> - riak_cs_s3_response:api_error(Reason, RD, Ctx) + riak_cs_aws_response:api_error(Reason, RD, Ctx) end end. --spec content_types_provided(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_provided(RD, Ctx=#context{}) -> +-spec content_types_provided(#wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_provided(RD, Ctx) -> Method = wrq:method(RD), if Method == 'GET' -> {[{?XML_TYPE, to_xml}], RD, Ctx}; @@ -187,7 +218,7 @@ content_types_provided(RD, Ctx=#context{}) -> {[{"text/plain", unused_callback2}], RD, Ctx} end. --spec content_types_accepted(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. +-spec content_types_accepted(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. content_types_accepted(RD, Ctx) -> %% For multipart upload part, %% e.g., PUT /ObjectName?partNumber=PartNumber&uploadId=UploadId @@ -207,44 +238,43 @@ parse_body(Body0) -> bad end. --spec accept_body(#wm_reqdata{}, #context{}) -> - {{halt, integer()}, #wm_reqdata{}, #context{}}. -accept_body(RD, #context{local_context=LocalCtx0} = Ctx0) -> +-spec accept_body(#wm_reqdata{}, #rcs_web_context{}) -> + {{halt, integer()}, #wm_reqdata{}, #rcs_web_context{}}. +accept_body(RD, Ctx0 = #rcs_web_context{local_context = LocalCtx0}) -> catch riak_cs_get_fsm:stop(LocalCtx0#key_context.get_fsm_pid), - try {t, {ok, UploadId}} = {t, riak_cs_utils:safe_base64url_decode( re:replace(wrq:path(RD), ".*/uploads/", "", [{return, binary}]))}, {t, {ok, PartNumber}} = {t, riak_cs_utils:safe_list_to_integer(wrq:get_qs_value("partNumber", RD))}, - LocalCtx = LocalCtx0#key_context{upload_id=UploadId, - part_number=PartNumber}, - Ctx = Ctx0#context{local_context=LocalCtx}, + LocalCtx = LocalCtx0#key_context{upload_id = UploadId, + part_number = PartNumber}, + Ctx = Ctx0#rcs_web_context{local_context = LocalCtx}, validate_copy_header(RD, Ctx) catch error:{badmatch, {t, _}} -> {{halt, 400}, RD, Ctx0} end. -validate_copy_header(RD, #context{response_module=ResponseMod, - local_context=LocalCtx} = Ctx) -> +validate_copy_header(RD, #rcs_web_context{response_module = ResponseMod, + local_context = LocalCtx} = Ctx) -> case riak_cs_copy_object:get_copy_source(RD) of {error, Reason} -> - riak_cs_s3_response:api_error(Reason, RD, Ctx); + riak_cs_aws_response:api_error(Reason, RD, Ctx); undefined -> validate_part_size(RD, Ctx, LocalCtx#key_context.size, undefined, undefined); - {SrcBucket, SrcKey} -> + {SrcBucket, SrcKey, SrcVsn} -> {ok, ReadRcPid} = riak_cs_riak_client:checkout(), try - case riak_cs_manifest:fetch(ReadRcPid, SrcBucket, SrcKey) of + case riak_cs_manifest:fetch(ReadRcPid, SrcBucket, SrcKey, SrcVsn) of {error, notfound} -> ResponseMod:api_error(no_copy_source_key, RD, Ctx); {ok, SrcManifest} -> {Start,End} = riak_cs_copy_object:copy_range(RD, SrcManifest), validate_part_size(RD, - Ctx#context{stats_key=[multipart_upload, put_copy]}, + Ctx#rcs_web_context{stats_key = [multipart_upload, put_copy]}, End - Start + 1, SrcManifest, ReadRcPid) end @@ -253,7 +283,7 @@ validate_copy_header(RD, #context{response_module=ResponseMod, end end. -validate_part_size(RD, #context{response_module=ResponseMod} = Ctx, +validate_part_size(RD, #rcs_web_context{response_module = ResponseMod} = Ctx, ExactSize, SrcManifest, ReadRcPid) -> case ExactSize =< riak_cs_lfs_utils:max_content_len() of false -> @@ -262,21 +292,21 @@ validate_part_size(RD, #context{response_module=ResponseMod} = Ctx, prepare_part_upload(RD, Ctx, ExactSize, SrcManifest, ReadRcPid) end. -prepare_part_upload(RD, #context{riak_client=RcPid, - local_context=LocalCtx0} = Ctx0, - ExactSize, SrcManifest, ReadRcPid) -> - #key_context{bucket=DstBucket, key=Key, - upload_id=UploadId, part_number=PartNumber} = LocalCtx0, - Caller = riak_cs_user:to_3tuple(Ctx0#context.user), - case riak_cs_mp_utils:upload_part(DstBucket, Key, UploadId, PartNumber, +prepare_part_upload(RD, #rcs_web_context{riak_client = RcPid, + local_context = LocalCtx0} = Ctx0, + ExactSize, SrcManifest, ReadRcPid) -> + #key_context{bucket = DstBucket, key = Key, obj_vsn = ObjVsn, + upload_id = UploadId, part_number = PartNumber} = LocalCtx0, + Caller = riak_cs_user:to_3tuple(Ctx0#rcs_web_context.user), + case riak_cs_mp_utils:upload_part(DstBucket, Key, ObjVsn, UploadId, PartNumber, ExactSize, Caller, RcPid) of {error, notfound} -> - riak_cs_s3_response:no_such_upload_response(UploadId, RD, Ctx0); + riak_cs_aws_response:no_such_upload_response(UploadId, RD, Ctx0); {error, Reason} -> - riak_cs_s3_response:api_error(Reason, RD, Ctx0); + riak_cs_aws_response:api_error(Reason, RD, Ctx0); {upload_part_ready, PartUUID, PutPid} -> - LocalCtx = LocalCtx0#key_context{part_uuid=PartUUID}, - Ctx = Ctx0#context{local_context=LocalCtx}, + LocalCtx = LocalCtx0#key_context{part_uuid = PartUUID}, + Ctx = Ctx0#rcs_web_context{local_context = LocalCtx}, case SrcManifest of undefined -> BlockSize = riak_cs_lfs_utils:block_size(), @@ -290,22 +320,12 @@ prepare_part_upload(RD, #context{riak_client=RcPid, end end. --spec accept_streambody(#wm_reqdata{}, #context{}, pid(), term()) -> {{halt, integer()}, #wm_reqdata{}, #context{}}. -accept_streambody(RD, - Ctx=#context{local_context=_LocalCtx=#key_context{size=0}}, - Pid, - {_Data, _Next}) -> +-spec accept_streambody(#wm_reqdata{}, #rcs_web_context{}, pid(), term()) -> {{halt, integer()}, #wm_reqdata{}, #rcs_web_context{}}. +accept_streambody(RD, Ctx = #rcs_web_context{local_context = #key_context{size = 0}}, + Pid, {_Data, _Next}) -> finalize_request(RD, Ctx, Pid); -accept_streambody(RD, - Ctx=#context{local_context=LocalCtx, - user=User}, - Pid, - {Data, Next}) -> - #key_context{bucket=Bucket, - key=Key} = LocalCtx, - BFile_str = [Bucket, $,, Key], - UserName = riak_cs_wm_utils:extract_name(User), - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"accept_streambody">>, [size(Data)], [UserName, BFile_str]), +accept_streambody(RD, Ctx, + Pid, {Data, Next}) -> riak_cs_put_fsm:augment_data(Pid, Data), if is_function(Next) -> accept_streambody(RD, Ctx, Pid, Next()); @@ -313,14 +333,16 @@ accept_streambody(RD, finalize_request(RD, Ctx, Pid) end. -to_xml(RD, Ctx=#context{local_context=LocalCtx, - riak_client=RcPid}) -> - #key_context{bucket=Bucket, key=Key} = LocalCtx, +to_xml(RD, Ctx = #rcs_web_context{local_context = #key_context{bucket = Bucket, + key = Key, + obj_vsn = Vsn}, + riak_client = RcPid}) -> UploadId = base64url:decode(re:replace(wrq:path(RD), ".*/uploads/", "", [{return, binary}])), - {UserDisplay, _Canon, UserKeyId} = User = - riak_cs_user:to_3tuple(Ctx#context.user), - case riak_cs_mp_utils:list_parts(Bucket, Key, UploadId, User, [], RcPid) of + #{display_name := UserDisplay, + key_id := UserKeyId} = User = + riak_cs_user:to_3tuple(Ctx#rcs_web_context.user), + case riak_cs_mp_utils:list_parts(Bucket, Key, Vsn, UploadId, User, [], RcPid) of {ok, Ps} -> Us = [{'Part', [ @@ -359,60 +381,61 @@ to_xml(RD, Ctx=#context{local_context=LocalCtx, Body = riak_cs_xml:to_xml([XmlDoc]), {Body, RD, Ctx}; {error, notfound} -> - riak_cs_s3_response:no_such_upload_response(UploadId, RD, Ctx); + riak_cs_aws_response:no_such_upload_response(UploadId, RD, Ctx); {error, Reason} -> - riak_cs_s3_response:api_error(Reason, RD, Ctx) + riak_cs_aws_response:api_error(Reason, RD, Ctx) end. -finalize_request(RD, Ctx=#context{local_context=LocalCtx, - response_module=ResponseMod, - riak_client=RcPid}, PutPid) -> - #key_context{bucket=Bucket, - key=Key, - upload_id=UploadId, - part_number=PartNumber, - part_uuid=PartUUID} = LocalCtx, - Caller = riak_cs_user:to_3tuple(Ctx#context.user), - ContentMD5 = wrq:get_req_header("content-md5", RD), +finalize_request(RD, Ctx=#rcs_web_context{local_context = LocalCtx, + response_module = ResponseMod, + riak_client = RcPid}, PutPid) -> + #key_context{bucket = Bucket, + key = Key, + obj_vsn = ObjVsn, + upload_id = UploadId, + part_number = PartNumber, + part_uuid = PartUUID} = LocalCtx, + Caller = riak_cs_user:to_3tuple(Ctx#rcs_web_context.user), + ContentMD5 = case wrq:get_req_header("content-md5", RD) of undefined -> undefined; + A -> list_to_binary(A) end, case riak_cs_put_fsm:finalize(PutPid, ContentMD5) of {ok, M} -> case riak_cs_mp_utils:upload_part_finished( - Bucket, Key, UploadId, PartNumber, PartUUID, + Bucket, Key, ObjVsn, UploadId, PartNumber, PartUUID, M?MANIFEST.content_md5, Caller, RcPid) of ok -> ETag = riak_cs_manifest:etag(M), RD2 = wrq:set_resp_header("ETag", ETag, RD), {{halt, 200}, RD2, Ctx}; {error, Reason} -> - riak_cs_s3_response:api_error(Reason, RD, Ctx) + riak_cs_aws_response:api_error(Reason, RD, Ctx) end; {error, invalid_digest} -> ResponseMod:invalid_digest_response(ContentMD5, RD, Ctx); {error, Reason1} -> - riak_cs_s3_response:api_error(Reason1, RD, Ctx) + riak_cs_aws_response:api_error(Reason1, RD, Ctx) end. --spec maybe_copy_part(pid(), lfs_manifest(), riak_client(), - #wm_reqdata{}, #context{}) -> - {{halt, integer()}, #wm_reqdata{}, #context{}}. maybe_copy_part(PutPid, - ?MANIFEST{bkey={SrcBucket, SrcKey}} = SrcManifest, + ?MANIFEST{bkey={SrcBucket, SrcKey}, + vsn = SrcVsn} = SrcManifest, ReadRcPid, - RD, #context{riak_client=RcPid, - local_context=LocalCtx, - user=User} = Ctx) -> - #key_context{bucket=DstBucket, key=Key, - upload_id=UploadId, - part_number=PartNumber, - part_uuid=PartUUID} = LocalCtx, - DstKey = list_to_binary(Key), + RD, #rcs_web_context{riak_client = RcPid, + local_context = LocalCtx, + user = User} = Ctx) -> + #key_context{bucket = DstBucket, + key = DstKey, + obj_vsn = DstVsn, + upload_id = UploadId, + part_number = PartNumber, + part_uuid = PartUUID} = LocalCtx, Caller = riak_cs_user:to_3tuple(User), case riak_cs_copy_object:test_condition_and_permission(ReadRcPid, SrcManifest, RD, Ctx) of {false, _, _} -> - _ = lager:debug("Start copying! > ~s ~s => ~s ~s via ~p", - [SrcBucket, SrcKey, DstBucket, DstKey, ReadRcPid]), + ?LOG_DEBUG("Start copying! > ~s/~s:~s => ~s/~s:~s via ~p", + [SrcBucket, SrcKey, SrcVsn, DstBucket, DstKey, DstVsn, ReadRcPid]), %% Prepare for connection loss or client close FDWatcher = riak_cs_copy_object:connection_checker((RD#wm_reqdata.wm_state)#wm_reqstate.socket), @@ -423,27 +446,27 @@ maybe_copy_part(PutPid, case riak_cs_copy_object:copy(PutPid, SrcManifest, ReadRcPid, FDWatcher, Range) of {ok, DstManifest} -> case riak_cs_mp_utils:upload_part_finished( - DstBucket, DstKey, UploadId, PartNumber, PartUUID, + DstBucket, DstKey, DstVsn, UploadId, PartNumber, PartUUID, DstManifest?MANIFEST.content_md5, Caller, RcPid) of ok -> ETag = riak_cs_manifest:etag(DstManifest), RD2 = wrq:set_resp_header("ETag", ETag, RD), - riak_cs_s3_response:copy_part_response(DstManifest, RD2, Ctx); + riak_cs_aws_response:copy_part_response(DstManifest, RD2, Ctx); {error, Reason0} -> - riak_cs_s3_response:api_error(Reason0, RD, Ctx) + riak_cs_aws_response:api_error(Reason0, RD, Ctx) end; {error, Reason} -> - riak_cs_s3_response:api_error(Reason, RD, Ctx) + riak_cs_aws_response:api_error(Reason, RD, Ctx) end; {true, _RD, _OtherCtx} -> %% access to source object not authorized %% TODO: check the return value - _ = lager:debug("access to source object denied (~s, ~s)", [SrcBucket, SrcKey]), + ?LOG_DEBUG("access to source object denied (~s/~s:~s)", [SrcBucket, SrcKey, SrcVsn]), {{halt, 403}, RD, Ctx}; Error -> - _ = lager:debug("unknown error: ~p", [Error]), - %% ResponseMod:api_error(Error, RD, Ctx#context{local_context=LocalCtx}) + ?LOG_DEBUG("unknown error: ~p", [Error]), + %% ResponseMod:api_error(Error, RD, Ctx#rcs_web_context{local_context=LocalCtx}) Error end. diff --git a/src/riak_cs_wm_objects.erl b/apps/riak_cs/src/riak_cs_wm_object_versions.erl similarity index 60% rename from src/riak_cs_wm_objects.erl rename to apps/riak_cs/src/riak_cs_wm_object_versions.erl index 11b4fc666..92e09bb2f 100644 --- a/src/riak_cs_wm_objects.erl +++ b/apps/riak_cs/src/riak_cs_wm_object_versions.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,58 +19,53 @@ %% %% --------------------------------------------------------------------- -%% @doc WM resource for object listing +%% @doc WM resource for object version listing --module(riak_cs_wm_objects). +-module(riak_cs_wm_object_versions). -export([init/1, + authorize/2, stats_prefix/0, allowed_methods/0, api_request/2 ]). --export([authorize/2]). +-ignore_xref([init/1, + authorize/2, + stats_prefix/0, + allowed_methods/0, + api_request/2 + ]). + -include("riak_cs.hrl"). --include("riak_cs_api.hrl"). --include("list_objects.hrl"). --include_lib("webmachine/include/webmachine.hrl"). +-include("riak_cs_web.hrl"). -define(RIAKCPOOL, bucket_list_pool). --spec init(#context{}) -> {ok, #context{}}. +-spec init(#rcs_web_context{}) -> {ok, #rcs_web_context{}}. init(Ctx) -> - {ok, Ctx#context{rc_pool=?RIAKCPOOL}}. + {ok, Ctx#rcs_web_context{rc_pool = ?RIAKCPOOL}}. --spec stats_prefix() -> list_objects. -stats_prefix() -> list_objects. +-spec stats_prefix() -> list_object_versions. +stats_prefix() -> list_object_versions. -spec allowed_methods() -> [atom()]. allowed_methods() -> - %% GET is for object listing ['GET']. -%% TODO: change to authorize/spec/cleanup unneeded cases -%% TODO: requires update for multi-delete --spec authorize(#wm_reqdata{}, #context{}) -> {boolean(), #wm_reqdata{}, #context{}}. +-spec authorize(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. authorize(RD, Ctx) -> riak_cs_wm_utils:bucket_access_authorize_helper(bucket, false, RD, Ctx). --spec api_request(#wm_reqdata{}, #context{}) -> {ok, ?LORESP{}} | {error, term()}. -api_request(RD, Ctx=#context{bucket=Bucket, - riak_client=RcPid, - user=User}) -> - UserName = riak_cs_wm_utils:extract_name(User), - riak_cs_dtrace:dt_bucket_entry(?MODULE, <<"list_keys">>, [], [UserName, Bucket]), - Res = riak_cs_api:list_objects( - [B || B <- riak_cs_bucket:get_buckets(User), - B?RCS_BUCKET.name =:= binary_to_list(Bucket)], - Ctx#context.bucket, - get_max_keys(RD), - get_options(RD), - RcPid), - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"list_keys">>, [200], [UserName, Bucket]), - Res. +-spec api_request(#wm_reqdata{}, #rcs_web_context{}) -> {ok, ?LOVRESP{}} | {error, term()}. +api_request(RD, Ctx = #rcs_web_context{riak_client = RcPid}) -> + riak_cs_api:list_objects( + versions, + Ctx#rcs_web_context.bucket, + get_max_keys(RD), + get_options(RD), + RcPid). -spec get_options(#wm_reqdata{}) -> [{atom(), 'undefined' | binary()}]. get_options(RD) -> diff --git a/apps/riak_cs/src/riak_cs_wm_objects.erl b/apps/riak_cs/src/riak_cs_wm_objects.erl new file mode 100644 index 000000000..9f970c314 --- /dev/null +++ b/apps/riak_cs/src/riak_cs_wm_objects.erl @@ -0,0 +1,99 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc WM resource for object listing + +-module(riak_cs_wm_objects). + +-export([init/1, + options/2, + authorize/2, + stats_prefix/0, + allowed_methods/0, + api_request/2 + ]). + +-ignore_xref([init/1, + authorize/2, + stats_prefix/0, + allowed_methods/0, + api_request/2 + ]). + +-include("riak_cs.hrl"). +-include("riak_cs_web.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-define(RIAKCPOOL, bucket_list_pool). + +-spec stats_prefix() -> list_objects. +stats_prefix() -> list_objects. + +-spec init(#rcs_web_context{}) -> {ok, #rcs_web_context{}}. +init(Ctx) -> + {ok, Ctx#rcs_web_context{rc_pool = ?RIAKCPOOL}}. + +-spec options(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), string()}], #wm_reqdata{}, #rcs_web_context{}}. +options(RD, Ctx) -> + {riak_cs_wm_utils:cors_headers(), RD, Ctx}. + +-spec allowed_methods() -> [atom()]. +allowed_methods() -> + ['GET', 'OPTIONS']. + +%% TODO: change to authorize/spec/cleanup unneeded cases +%% TODO: requires update for multi-delete +-spec authorize(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +authorize(RD, Ctx) -> + riak_cs_wm_utils:bucket_access_authorize_helper( + bucket, false, + wrq:set_resp_headers(riak_cs_wm_utils:cors_headers(), RD), Ctx). + +-spec api_request(#wm_reqdata{}, #rcs_web_context{}) -> {ok, ?LORESP{}} | {error, term()}. +api_request(RD, Ctx = #rcs_web_context{riak_client = RcPid}) -> + riak_cs_api:list_objects( + objects, + Ctx#rcs_web_context.bucket, + get_max_keys(RD), + get_options(RD), + RcPid). + +get_options(RD) -> + [get_option(list_to_atom(Opt), wrq:get_qs_value(Opt, RD)) || + Opt <- ["delimiter", "marker", "prefix"]]. + +get_option(Option, undefined) -> + {Option, undefined}; +get_option(Option, Value) -> + {Option, list_to_binary(Value)}. + +get_max_keys(RD) -> + case wrq:get_qs_value("max-keys", RD) of + undefined -> + ?DEFAULT_LIST_OBJECTS_MAX_KEYS; + StringKeys -> + try + erlang:min(list_to_integer(StringKeys), + ?DEFAULT_LIST_OBJECTS_MAX_KEYS) + catch _:_ -> + {error, invalid_argument} + end + end. diff --git a/apps/riak_cs/src/riak_cs_wm_ping.erl b/apps/riak_cs/src/riak_cs_wm_ping.erl new file mode 100644 index 000000000..45fdaba6b --- /dev/null +++ b/apps/riak_cs/src/riak_cs_wm_ping.erl @@ -0,0 +1,89 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_wm_ping). + +-export([init/1, + service_available/2, + allowed_methods/2, + to_html/2, + finish_request/2 + ]). + +-ignore_xref([init/1, + service_available/2, + allowed_methods/2, + to_html/2, + finish_request/2 + ]). + +-include("riak_cs.hrl"). + +-record(ping_context, {riak_client :: undefined | pid()}). + +%% ------------------------------------------------------------------- +%% Webmachine callbacks +%% ------------------------------------------------------------------- + +init(_Config) -> + {ok, #ping_context{}}. + +-spec service_available(#wm_reqdata{}, #ping_context{}) -> {boolean(), #wm_reqdata{}, #ping_context{}}. +service_available(RD, Ctx) -> + {Available, UpdCtx} = riak_ping(get_connection_pid(), Ctx), + {Available, RD, UpdCtx}. + +-spec allowed_methods(term(), term()) -> {[atom()], term(), term()}. +allowed_methods(RD, Ctx) -> + {['GET', 'HEAD'], RD, Ctx}. + +to_html(ReqData, Ctx) -> + {"OK", ReqData, Ctx}. + +finish_request(RD, Ctx = #ping_context{riak_client = undefined}) -> + {true, RD, Ctx}; +finish_request(RD, Ctx = #ping_context{riak_client = RcPid}) -> + riak_cs_riak_client:stop(RcPid), + {true, RD, Ctx#ping_context{riak_client = undefined}}. + +%% ------------------------------------------------------------------- +%% Internal functions +%% ------------------------------------------------------------------- + +get_connection_pid() -> + case riak_cs_riak_client:start_link([]) of + {ok, RcPid} -> + RcPid; + {error, Reason} -> + logger:error("Failed to obtain a riak_client: ~p", [Reason]), + undefined + end. + +riak_ping(RcPid, Ctx) -> + {ok, MasterPbc} = riak_cs_riak_client:master_pbc(RcPid), + Timeout = riak_cs_config:ping_timeout(), + Available = case catch riak_cs_pbc:ping(MasterPbc, Timeout, [riakc, ping]) of + pong -> + true; + _ -> + false + end, + {Available, Ctx}. diff --git a/apps/riak_cs/src/riak_cs_wm_s3_common.erl b/apps/riak_cs/src/riak_cs_wm_s3_common.erl new file mode 100644 index 000000000..a56b4d019 --- /dev/null +++ b/apps/riak_cs/src/riak_cs_wm_s3_common.erl @@ -0,0 +1,589 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_wm_s3_common). + +-export([init/1, + service_available/2, + forbidden/2, + content_types_accepted/2, + content_types_provided/2, + generate_etag/2, + last_modified/2, + valid_entity_length/2, + validate_content_checksum/2, + malformed_request/2, + options/2, + to_xml/2, + to_json/2, + post_is_create/2, + create_path/2, + process_post/2, + resp_body/2, + multiple_choices/2, + add_acl_to_context_then_accept/2, + accept_body/2, + produce_body/2, + allowed_methods/2, + delete_resource/2, + finish_request/2]). + +-export([default_allowed_methods/0, + default_stats_prefix/0, + default_content_types_accepted/2, + default_content_types_provided/2, + default_generate_etag/2, + default_last_modified/2, + default_finish_request/2, + default_init/1, + default_authorize/2, + default_malformed_request/2, + default_options/2, + default_valid_entity_length/2, + default_validate_content_checksum/2, + default_delete_resource/2, + default_anon_ok/0, + default_produce_body/2, + default_multiple_choices/2]). + +-include("riak_cs.hrl"). +-include("oos_api.hrl"). +-include_lib("webmachine/include/wm_reqstate.hrl"). +-include_lib("kernel/include/logger.hrl"). + +%% =================================================================== +%% Webmachine callbacks +%% =================================================================== + +-spec init([{atom(),term()}]) -> {ok, #rcs_web_context{}}. +init(Config) -> + Mod = proplists:get_value(submodule, Config), + %% Check if authentication is disabled and set that in the context. + AuthBypass = proplists:get_value(auth_bypass, Config), + AuthModule = proplists:get_value(auth_module, Config), + Api = riak_cs_config:api(), + RespModule = riak_cs_config:response_module(Api), + PolicyModule = proplists:get_value(policy_module, Config), + Exports = orddict:from_list(Mod:module_info(exports)), + ExportsFun = exports_fun(Exports), + StatsPrefix = resource_call(Mod, stats_prefix, [], ExportsFun), + Ctx = #rcs_web_context{auth_bypass = AuthBypass, + auth_module = AuthModule, + response_module = RespModule, + policy_module = PolicyModule, + exports_fun = ExportsFun, + stats_prefix = StatsPrefix, + submodule = Mod, + api = Api}, + resource_call(Mod, init, [Ctx], ExportsFun). + +-spec service_available(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +service_available(RD, #rcs_web_context{rc_pool = undefined} = Ctx) -> + service_available(RD, Ctx#rcs_web_context{rc_pool = request_pool}); +service_available(RD, #rcs_web_context{rc_pool = Pool} = Ctx) -> + case riak_cs_riak_client:checkout(Pool) of + {ok, RcPid} -> + {true, RD, Ctx#rcs_web_context{riak_client = RcPid}}; + {error, _Reason} -> + {false, RD, Ctx} + end. + +-spec malformed_request(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +malformed_request(RD, Ctx = #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun, + stats_prefix = StatsPrefix}) -> + %% Method is used in stats keys, updating inflow should be *after* + %% allowed_methods assertion. + _ = update_stats_inflow(RD, StatsPrefix), + resource_call(Mod, malformed_request, [RD, Ctx], ExportsFun). + + +-spec options(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), string()}], #wm_reqdata{}, #rcs_web_context{}}. +options(RD, Ctx = #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun}) -> + resource_call(Mod, options, [RD, Ctx], ExportsFun). + +-spec valid_entity_length(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +valid_entity_length(RD, #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun} = Ctx) -> + resource_call(Mod, valid_entity_length, [RD, Ctx], ExportsFun). + +-type validate_checksum_response() :: {error, term()} | + {halt, pos_integer()} | + boolean(). +-spec validate_content_checksum(#wm_reqdata{}, #rcs_web_context{}) -> + {validate_checksum_response(), #wm_reqdata{}, #rcs_web_context{}}. +validate_content_checksum(RD, Ctx = #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun}) -> + resource_call(Mod, validate_content_checksum, [RD, Ctx], ExportsFun). + +-spec forbidden(#wm_reqdata{}, #rcs_web_context{}) -> {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +forbidden(RD, Ctx) -> + case wrq:method(RD) of + 'OPTIONS' -> + {false, RD, Ctx}; + _ -> + forbidden2(RD, Ctx) + end. +forbidden2(RD, Ctx0 = #rcs_web_context{auth_module = AuthMod, + submodule = Mod, + riak_client = RcPid, + exports_fun = ExportsFun}) -> + {AuthResult, AnonOk, Ctx} = + case AuthMod:identify(RD, Ctx0) of + failed -> + {{error, no_such_key}, false, Ctx0}; + {failed, Reason} -> + {{error, Reason}, false, Ctx0}; + {UserKey, AuthData} -> + case maybe_create_user( + riak_cs_user:get_user(UserKey, RcPid), + UserKey, + Ctx0#rcs_web_context.api, + Ctx0#rcs_web_context.auth_module, + AuthData, + RcPid) of + {ok, {User, Obj}} -> + {authenticate( + User, Obj, + RD, Ctx0, + AuthData), + resource_call(Mod, anon_ok, [], ExportsFun), + Ctx0#rcs_web_context{admin_access = riak_cs_user:is_admin(User)}}; + Error -> + {Error, + resource_call(Mod, anon_ok, [], ExportsFun), Ctx0} + end + end, + post_authentication(AuthResult, RD, Ctx, AnonOk). + +maybe_create_user({ok, {_, _}} = UserResult, _, _, _, _, _) -> + UserResult; +maybe_create_user({error, NE}, KeyId, oos, _, {UserData, _}, RcPid) + when NE =:= not_found; + NE =:= notfound; + NE =:= no_user_key -> + {Name, Email, UserId} = UserData, + {_, Secret} = riak_cs_oos_utils:user_ec2_creds(UserId, KeyId), + %% Attempt to create a Riak CS user to represent the OS tenant + _ = riak_cs_user:create_user(Name, Email, KeyId, Secret, #{}), + riak_cs_user:get_user(KeyId, RcPid); +maybe_create_user({error, NE}, KeyId, aws, riak_cs_keystone_auth, {UserData, _}, RcPid) + when NE =:= not_found; + NE =:= notfound; + NE =:= no_user_key -> + {Name, Email, UserId} = UserData, + {_, Secret} = riak_cs_oos_utils:user_ec2_creds(UserId, KeyId), + %% Attempt to create a Riak CS user to represent the OS tenant + _ = riak_cs_user:create_user(Name, Email, KeyId, Secret, #{}), + riak_cs_user:get_user(KeyId, RcPid); +maybe_create_user({error, no_user_key} = Error, _, _, _, _, _) -> + %% Anonymous access may be authorized by ACL or policy afterwards, + %% no logging here. + Error; +maybe_create_user({error, disconnected} = Error, _, _, _, _, RcPid) -> + {ok, MasterPid} = riak_cs_riak_client:master_pbc(RcPid), + riak_cs_pbc:check_connection_status(MasterPid, maybe_create_user), + Error; +maybe_create_user({error, _Reason} = Error, _, _Api, _, _, _) -> + Error. + +%% @doc Get the list of methods a resource supports. +-spec allowed_methods(#wm_reqdata{}, #rcs_web_context{}) -> {[atom()], #wm_reqdata{}, #rcs_web_context{}}. +allowed_methods(RD, Ctx = #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun}) -> + Methods = resource_call(Mod, + allowed_methods, + [], + ExportsFun), + {Methods, RD, Ctx}. + +-spec content_types_accepted(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_accepted(RD, Ctx = #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun}) -> + resource_call(Mod, + content_types_accepted, + [RD,Ctx], + ExportsFun). + +-spec content_types_provided(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_provided(RD, Ctx = #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun}) -> + resource_call(Mod, + content_types_provided, + [RD,Ctx], + ExportsFun). + +-spec generate_etag(#wm_reqdata{}, #rcs_web_context{}) -> {string(), #wm_reqdata{}, #rcs_web_context{}}. +generate_etag(RD, Ctx = #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun}) -> + resource_call(Mod, + generate_etag, + [RD,Ctx], + ExportsFun). + +-spec last_modified(#wm_reqdata{}, #rcs_web_context{}) -> {calendar:datetime(), #wm_reqdata{}, #rcs_web_context{}}. +last_modified(RD, Ctx = #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun}) -> + resource_call(Mod, + last_modified, + [RD,Ctx], + ExportsFun). + +-spec delete_resource(#wm_reqdata{}, #rcs_web_context{}) -> {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +delete_resource(RD, Ctx = #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun}) -> + %% TODO: add dt_wm_return from subresource? + resource_call(Mod, + delete_resource, + [RD,Ctx], + ExportsFun). + +-spec to_xml(#wm_reqdata{}, #rcs_web_context{}) -> + {binary() | {'halt', non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +to_xml(RD, Ctx = #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun}) -> + resource_call(Mod, + to_xml, + [RD, Ctx], + ExportsFun). + +-spec to_json(#wm_reqdata{}, #rcs_web_context{}) -> + {binary() | {'halt', non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +to_json(RD, Ctx = #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun}) -> + resource_call(Mod, + to_json, + [RD, Ctx], + ExportsFun(to_json)). + +post_is_create(RD, Ctx = #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun}) -> + resource_call(Mod, post_is_create, [RD, Ctx], ExportsFun). + +create_path(RD, #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun} = Ctx) -> + resource_call(Mod, create_path, [RD, Ctx], ExportsFun). + +process_post(RD, #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun} = Ctx) -> + resource_call(Mod, process_post, [RD, Ctx], ExportsFun). + +resp_body(RD, #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun} = Ctx) -> + resource_call(Mod, resp_body, [RD, Ctx], ExportsFun). + +multiple_choices(RD, #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun} = Ctx) -> + try + resource_call(Mod, multiple_choices, [RD, Ctx], ExportsFun) + catch _:_ -> + {false, RD, Ctx} + end. + +%% @doc Add an ACL (or default ACL) to the context, parsed from headers. If +%% parsing the headers fails, halt the request. +add_acl_to_context_then_accept(RD, Ctx) -> + case riak_cs_wm_utils:maybe_update_context_with_acl_from_headers(RD, Ctx) of + {ok, ContextWithAcl} -> + accept_body(RD, ContextWithAcl); + {error, HaltResponse} -> + HaltResponse + end. + +-spec accept_body(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {'halt', non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +accept_body(RD, Ctx = #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun}) -> + resource_call(Mod, + accept_body, + [RD, Ctx], + ExportsFun). + %% TODO: extract response code and add to ints field + +-spec produce_body(#wm_reqdata{}, #rcs_web_context{}) -> + {iolist()|binary(), #wm_reqdata{}, #rcs_web_context{}} | + {{known_length_stream, non_neg_integer(), {<<>>, function()}}, #wm_reqdata{}, #rcs_web_context{}}. +produce_body(RD, Ctx = #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun}) -> + %% TODO: add dt_wm_return w/ content length + resource_call(Mod, + produce_body, + [RD, Ctx], + ExportsFun). + +-spec finish_request(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +finish_request(RD, Ctx = #rcs_web_context{riak_client = RcPid, + auto_rc_close = AutoRcClose, + submodule = Mod, + exports_fun = ExportsFun}) + when RcPid =:= undefined orelse AutoRcClose =:= false -> + Res = resource_call(Mod, + finish_request, + [RD, Ctx], + ExportsFun), + update_stats(RD, Ctx), + Res; +finish_request(RD, Ctx0 = #rcs_web_context{riak_client = RcPid, + rc_pool = Pool, + submodule = Mod, + exports_fun = ExportsFun}) -> + riak_cs_riak_client:checkin(Pool, RcPid), + Ctx = Ctx0#rcs_web_context{riak_client=undefined}, + Res = resource_call(Mod, + finish_request, + [RD, Ctx], + ExportsFun), + update_stats(RD, Ctx), + Res. + +%% =================================================================== +%% Helper functions +%% =================================================================== + +-spec authorize(#wm_reqdata{}, #rcs_web_context{}) -> {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +authorize(RD, Ctx = #rcs_web_context{submodule = Mod, + exports_fun = ExportsFun}) -> + resource_call(Mod, authorize, [RD,Ctx], ExportsFun). + +-spec authenticate(rcs_user(), riakc_obj:riakc_obj(), term(), term(), term()) -> + {ok, rcs_user(), riakc_obj:riakc_obj()} | {error, bad_auth}. +authenticate(User, UserObj, RD, Ctx = #rcs_web_context{auth_module = AuthMod}, AuthData) + when User?RCS_USER.status =:= enabled -> + case AuthMod:authenticate(User, AuthData, RD, Ctx) of + ok -> + {ok, User, UserObj}; + {error, reqtime_tooskewed} -> + {error, reqtime_tooskewed}; + {error, _Reason} -> + {error, bad_auth} + end; +authenticate(User, _UserObj, _RD, _Ctx, _AuthData) + when User?RCS_USER.status =/= enabled -> + %% {ok, _} -> %% disabled account, we are going to 403 + {error, bad_auth}. + +-spec exports_fun(orddict:orddict()) -> function(). +exports_fun(Exports) -> + fun(Function) -> + orddict:is_key(Function, Exports) + end. + + +resource_call(Mod, Fun, Args, true) -> + erlang:apply(Mod, Fun, Args); +resource_call(_Mod, Fun, Args, false) -> + erlang:apply(?MODULE, default(Fun), Args); +resource_call(Mod, Fun, Args, ExportsFun) -> + resource_call(Mod, Fun, Args, ExportsFun(Fun)). + + +post_authentication(AuthResult, RD, Ctx, AnonOk) -> + post_authentication(AuthResult, RD, Ctx, fun authorize/2, AnonOk). + +post_authentication({ok, ?RCS_USER{status = disabled}, _}, RD, Ctx, _, _) -> + riak_cs_wm_utils:deny_access(RD, Ctx); +post_authentication({ok, User, UserObj}, RD, Ctx, Authorize, _) -> + %% given keyid and signature matched, proceed + Authorize(RD, Ctx#rcs_web_context{user = User, + user_object = UserObj}); +post_authentication({error, no_user_key}, RD, Ctx, Authorize, true) -> + %% no keyid was given, proceed anonymously + ?LOG_DEBUG("No user key"), + Authorize(RD, Ctx); +post_authentication({error, no_user_key}, RD, Ctx, _, false) -> + %% no keyid was given, deny access + ?LOG_DEBUG("No user key, deny"), + riak_cs_wm_utils:deny_access(RD, Ctx); +post_authentication({error, bad_auth}, RD, Ctx, _, _) -> + %% given keyid was found, but signature didn't match + ?LOG_DEBUG("bad_auth"), + riak_cs_wm_utils:deny_access(RD, Ctx); +post_authentication({error, reqtime_tooskewed} = Error, RD, + #rcs_web_context{response_module = ResponseMod} = Ctx, _, _) -> + ?LOG_DEBUG("reqtime_tooskewed"), + ResponseMod:api_error(Error, RD, Ctx); +post_authentication({error, {auth_not_supported, AuthType}}, RD, + Ctx = #rcs_web_context{response_module = ResponseMod} = Ctx, _, _) -> + ?LOG_DEBUG("auth_not_supported: ~s", [AuthType]), + ResponseMod:api_error({auth_not_supported, AuthType}, RD, Ctx); +post_authentication({error, notfound}, RD, Ctx, _, _) -> + ?LOG_DEBUG("User does not exist"), + riak_cs_wm_utils:deny_invalid_key(RD, Ctx); +post_authentication({error, Reason}, RD, + #rcs_web_context{response_module = ResponseMod} = Ctx, _, _) -> + %% Lookup failed, basically due to disconnected stuff + ?LOG_DEBUG("Authentication error: ~p", [Reason]), + ResponseMod:api_error(Reason, RD, Ctx). + +update_stats_inflow(_RD, undefined = _StatsPrefix) -> + ok; +update_stats_inflow(_RD, no_stats = _StatsPrefix) -> + ok; +update_stats_inflow(RD, StatsPrefix) -> + case riak_cs_wm_utils:lower_case_method(wrq:method(RD)) of + options -> + ok; + Method -> + Key = [StatsPrefix, Method], + riak_cs_stats:inflow(Key) + end. + +update_stats(_RD, #rcs_web_context{stats_key = no_stats}) -> + ok; +update_stats(_RD, #rcs_web_context{stats_prefix = no_stats}) -> + ok; +update_stats(RD, #rcs_web_context{start_time = StartTime, + stats_prefix = StatsPrefix, + stats_key = StatsKey}) -> + update_stats(StartTime, + wrq:response_code(RD), + StatsPrefix, + riak_cs_wm_utils:lower_case_method(wrq:method(RD)), + StatsKey). + +update_stats(StartTime, Code, StatsPrefix, Method, StatsKey0) -> + StatsKey = case StatsKey0 of + prefix_and_method -> [StatsPrefix, Method]; + _ -> StatsKey0 + end, + case Code of + 405 -> + %% Method Not Allowed: don't update stats because unallowed + %% mothod may lead to notfound warning in updating stats + ok; + Success when is_integer(Success) andalso Success < 400 -> + riak_cs_stats:update_with_start(StatsKey, StartTime); + _Error -> + riak_cs_stats:update_error_with_start(StatsKey, StartTime) + end. + +%% =================================================================== +%% Resource function defaults +%% =================================================================== + +default(init) -> + default_init; +default(stats_prefix) -> + default_stats_prefix; +default(allowed_methods) -> + default_allowed_methods; +default(content_types_accepted) -> + default_content_types_accepted; +default(content_types_provided) -> + default_content_types_provided; +default(generate_etag) -> + default_generate_etag; +default(last_modified) -> + default_last_modified; +default(malformed_request) -> + default_malformed_request; +default(options) -> + default_options; +default(valid_entity_length) -> + default_valid_entity_length; +default(validate_content_checksum) -> + default_validate_content_checksum; +default(delete_resource) -> + default_delete_resource; +default(authorize) -> + default_authorize; +default(finish_request) -> + default_finish_request; +default(anon_ok) -> + default_anon_ok; +default(produce_body) -> + default_produce_body; +default(multiple_choices) -> + default_multiple_choices; +default(_) -> + undefined. + +default_init(Ctx) -> + {ok, Ctx}. + +default_stats_prefix() -> + no_stats. + +default_malformed_request(RD, Ctx) -> + {false, RD, Ctx}. + +default_options(RD, Ctx) -> + {[{"Access-Control-Allow-Origin", "*"}], RD, Ctx}. + +default_valid_entity_length(RD, Ctx) -> + {true, RD, Ctx}. + +default_validate_content_checksum(RD, Ctx) -> + {true, RD, Ctx}. + +default_content_types_accepted(RD, Ctx) -> + {[], RD, Ctx}. + +-spec default_content_types_provided(#wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), atom()}], + #wm_reqdata{}, + #rcs_web_context{}}. +default_content_types_provided(RD, Ctx = #rcs_web_context{api = oos}) -> + {[{"text/plain", produce_body}], RD, Ctx}; +default_content_types_provided(RD, Ctx) -> + {[{"application/xml", produce_body}], RD, Ctx}. + +default_generate_etag(RD, Ctx) -> + {undefined, RD, Ctx}. + +default_last_modified(RD, Ctx) -> + {undefined, RD, Ctx}. + +default_delete_resource(RD, Ctx) -> + {false, RD, Ctx}. + +default_allowed_methods() -> + ['OPTIONS']. + +default_finish_request(RD, Ctx) -> + {true, RD, Ctx}. + +default_anon_ok() -> + true. + +default_produce_body(RD, Ctx = #rcs_web_context{submodule = Mod, + response_module = ResponseMod, + exports_fun = ExportsFun}) -> + try + ResponseMod:respond( + resource_call(Mod, api_request, [RD, Ctx], ExportsFun), + RD, + Ctx) + catch error:{badmatch, {error, Reason}} -> + ResponseMod:api_error(Reason, RD, Ctx) + end. + +%% @doc this function will be called by `post_authenticate/2' if the user successfully +%% authenticates and the submodule does not provide an implementation +%% of authorize/2. The default implementation does not perform any authorization +%% and simply returns false to signify the request is not fobidden +-spec default_authorize(term(), term()) -> {false, term(), term()}. +default_authorize(RD, Ctx) -> + {false, RD, Ctx}. + +default_multiple_choices(RD, Ctx) -> + {false, RD, Ctx}. diff --git a/src/riak_cs_wm_stats.erl b/apps/riak_cs/src/riak_cs_wm_stats.erl similarity index 64% rename from src/riak_cs_wm_stats.erl rename to apps/riak_cs/src/riak_cs_wm_stats.erl index fbf277d65..31bca03bc 100644 --- a/src/riak_cs_wm_stats.erl +++ b/apps/riak_cs/src/riak_cs_wm_stats.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -21,25 +22,51 @@ -module(riak_cs_wm_stats). %% webmachine resource exports --export([ - init/1, +-export([init/1, encodings_provided/2, + options/2, + allowed_methods/2, content_types_provided/2, service_available/2, forbidden/2, - finish_request/2]). --export([ - produce_body/2, - pretty_print/2 + finish_request/2, + produce_body/2 + ]). +-ignore_xref([init/1, + encodings_provided/2, + content_types_provided/2, + service_available/2, + forbidden/2, + finish_request/2, + produce_body/2 + ]). + +-export([pretty_print/2 ]). -include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). init(Props) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"init">>), AuthBypass = not proplists:get_value(admin_auth_enabled, Props), - {ok, #context{auth_bypass = AuthBypass}}. + {ok, #rcs_web_context{auth_bypass = AuthBypass}}. + +-spec options(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), string()}], #wm_reqdata{}, #rcs_web_context{}}. +options(RD, Ctx) -> + {riak_cs_wm_utils:cors_headers(), RD, Ctx}. + +-spec service_available(#wm_reqdata{}, #rcs_web_context{}) -> {true, #wm_reqdata{}, #rcs_web_context{}}. +service_available(RD, Ctx) -> + case riak_cs_config:riak_cs_stat() of + false -> + {false, RD, Ctx}; + true -> + riak_cs_wm_utils:service_available( + wrq:set_resp_headers(riak_cs_wm_utils:cors_headers(), RD), Ctx) + end. + +-spec allowed_methods(#wm_reqdata{}, #rcs_web_context{}) -> {[atom()], #wm_reqdata{}, #rcs_web_context{}}. +allowed_methods(RD, Ctx) -> + {['GET', 'OPTIONS'], RD, Ctx}. %% @spec encodings_provided(webmachine:wrq(), context()) -> %% {[encoding()], webmachine:wrq(), context()} @@ -47,7 +74,6 @@ init(Props) -> %% "identity" is provided for all methods, and "gzip" is %% provided for GET as well encodings_provided(RD, Context) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"encodings_provided">>), case wrq:method(RD) of 'GET' -> {[{"identity", fun(X) -> X end}, @@ -68,52 +94,40 @@ encodings_provided(RD, Context) -> %% so s3cmd will only be able to get the JSON flavor. content_types_provided(RD, Context) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"content_types_provided">>), {[{"application/json", produce_body}, {"text/plain", pretty_print}], RD, Context}. -service_available(RD, Ctx) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"service_available">>), - case riak_cs_config:riak_cs_stat() of - false -> - {false, RD, Ctx}; - true -> - case riak_cs_riak_client:checkout() of - {ok, Pid} -> - {true, RD, Ctx#context{riak_client = Pid}}; - _ -> - {false, RD, Ctx} - end - end. - produce_body(RD, Ctx) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"produce_body">>), - Body = mochijson2:encode(get_stats()), + Body = jsx:encode(get_stats()), ETag = riak_cs_utils:etag_from_binary(riak_cs_utils:md5(Body)), RD2 = wrq:set_resp_header("ETag", ETag, RD), - riak_cs_dtrace:dt_wm_return(?MODULE, <<"produce_body">>), {Body, RD2, Ctx}. -forbidden(RD, Ctx=#context{auth_bypass=AuthBypass}) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"forbidden">>), +-spec forbidden(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +forbidden(RD, Ctx) -> + case wrq:method(RD) of + 'OPTIONS' -> + {false, RD, Ctx}; + _ -> + forbidden2(RD, Ctx) + end. +forbidden2(RD, Ctx = #rcs_web_context{auth_bypass = AuthBypass}) -> riak_cs_wm_utils:find_and_auth_admin(RD, Ctx, AuthBypass). -finish_request(RD, #context{riak_client=undefined}=Ctx) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"finish_request">>, [0], []), +finish_request(RD, Ctx = #rcs_web_context{riak_client = undefined}) -> {true, RD, Ctx}; -finish_request(RD, #context{riak_client=RcPid}=Ctx) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"finish_request">>, [1], []), +finish_request(RD, Ctx = #rcs_web_context{riak_client = RcPid}) -> riak_cs_riak_client:checkin(RcPid), - riak_cs_dtrace:dt_wm_return(?MODULE, <<"finish_request">>, [1], []), - {true, RD, Ctx#context{riak_client=undefined}}. + {true, RD, Ctx#rcs_web_context{riak_client = undefined}}. %% @spec pretty_print(webmachine:wrq(), context()) -> %% {string(), webmachine:wrq(), context()} %% @doc Format the respons JSON object is a "pretty-printed" style. -pretty_print(RD1, C1=#context{}) -> +pretty_print(RD1, C1 = #rcs_web_context{}) -> {Json, RD2, C2} = produce_body(RD1, C1), - Body = riak_cs_utils:json_pp_print(lists:flatten(Json)), + Body = jsx:prettify(Json), ETag = riak_cs_utils:etag_from_binary(riak_cs_utils:md5(term_to_binary(Body))), RD3 = wrq:set_resp_header("ETag", ETag, RD2), {Body, RD3, C2}. diff --git a/apps/riak_cs/src/riak_cs_wm_sts.erl b/apps/riak_cs/src/riak_cs_wm_sts.erl new file mode 100644 index 000000000..9e52b56aa --- /dev/null +++ b/apps/riak_cs/src/riak_cs_wm_sts.erl @@ -0,0 +1,259 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc WM resource for IAM requests. + +-module(riak_cs_wm_sts). + +-export([init/1, + service_available/2, + options/2, + malformed_request/2, + forbidden/2, + content_types_accepted/2, + generate_etag/2, + last_modified/2, + valid_entity_length/2, + multiple_choices/2, + accept_wwwform/2, + allowed_methods/2, + post_is_create/2, + create_path/2, + finish_request/2 + ]). + +-ignore_xref([init/1, + service_available/2, + options/2, + malformed_request/2, + forbidden/2, + content_types_accepted/2, + generate_etag/2, + last_modified/2, + multiple_choices/2, + accept_wwwform/2, + allowed_methods/2, + valid_entity_length/2, + post_is_create/2, + create_path/2, + finish_request/2 + ]). + +-include("riak_cs_web.hrl"). +-include_lib("xmerl/include/xmerl.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-define(UNSIGNED_API_CALLS, ["AssumeRoleWithSAML"]). + +%% ------------------------------------------------------------------- +%% Webmachine callbacks +%% ------------------------------------------------------------------- + +-spec init([proplists:proplist()]) -> {ok, #rcs_web_context{}}. +init(Config) -> + %% Check if authentication is disabled and set that in the context. + AuthBypass = proplists:get_value(auth_bypass, Config), + AuthModule = proplists:get_value(auth_module, Config), + Api = riak_cs_config:api(), + RespModule = riak_cs_config:response_module(Api), + StatsPrefix = no_stats, + Ctx = #rcs_web_context{auth_bypass = AuthBypass, + auth_module = AuthModule, + response_module = RespModule, + stats_prefix = StatsPrefix, + api = Api}, + {ok, Ctx}. + + +-spec options(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), string()}], #wm_reqdata{}, #rcs_web_context{}}. +options(RD, Ctx) -> + {riak_cs_wm_utils:cors_headers(), + RD, Ctx}. + +-spec service_available(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +service_available(RD, Ctx = #rcs_web_context{rc_pool = undefined}) -> + service_available(RD, Ctx#rcs_web_context{rc_pool = request_pool}); +service_available(RD, Ctx = #rcs_web_context{rc_pool = RcPool}) -> + case riak_cs_riak_client:checkout(RcPool) of + {ok, RcPid} -> + {true, wrq:set_resp_headers( + riak_cs_wm_utils:cors_headers(), RD), + Ctx#rcs_web_context{riak_client = RcPid}}; + {error, _Reason} -> + {false, RD, Ctx} + end. + +-spec malformed_request(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +malformed_request(RD, Ctx) -> + {false, RD, Ctx}. + + +-spec valid_entity_length(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +valid_entity_length(RD, Ctx) -> + {true, RD, Ctx}. + + +-spec forbidden(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +forbidden(RD, Ctx) -> + case wrq:method(RD) of + 'OPTIONS' -> + {false, RD, Ctx}; + 'POST' -> + forbidden2(RD, Ctx) + end. +forbidden2(RD, Ctx) -> + case unsigned_call_allowed(RD) of + true -> + {false, RD, Ctx}; + false -> + riak_cs_wm_utils:forbidden(RD, Ctx, sts_entity) + end. + +unsigned_call_allowed(RD) -> + Form = mochiweb_util:parse_qs(wrq:req_body(RD)), + lists:member(proplists:get_value("Action", Form), + ?UNSIGNED_API_CALLS). + + +-spec allowed_methods(#wm_reqdata{}, #rcs_web_context{}) -> {[atom()], #wm_reqdata{}, #rcs_web_context{}}. +allowed_methods(RD, Ctx) -> + {['POST'], RD, Ctx}. + + +-spec content_types_accepted(#wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_accepted(RD, Ctx) -> + {[{?WWWFORM_TYPE, accept_wwwform}], RD, Ctx}. + + +-spec generate_etag(#wm_reqdata{}, #rcs_web_context{}) -> + {undefined, #wm_reqdata{}, #rcs_web_context{}}. +generate_etag(RD, Ctx) -> + {undefined, RD, Ctx}. + + +-spec last_modified(#wm_reqdata{}, #rcs_web_context{}) -> + {undefined, #wm_reqdata{}, #rcs_web_context{}}. +last_modified(RD, Ctx) -> + {undefined, RD, Ctx}. + +-spec post_is_create(#wm_reqdata{}, #rcs_web_context{}) -> + {true, #wm_reqdata{}, #rcs_web_context{}}. +post_is_create(RD, Ctx) -> + {true, RD, Ctx}. + + +-spec create_path(#wm_reqdata{}, #rcs_web_context{}) -> + {string(), #wm_reqdata{}, #rcs_web_context{}}. +create_path(RD, Ctx) -> + {wrq:disp_path(RD), RD, Ctx}. + + +-spec multiple_choices(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +multiple_choices(RD, Ctx) -> + {false, RD, Ctx}. + + +-spec accept_wwwform(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, term()}, term(), term()}. +accept_wwwform(RD, Ctx) -> + Form = mochiweb_util:parse_qs(wrq:req_body(RD)), + Action = proplists:get_value("Action", Form), + do_action(Action, Form, RD, Ctx). + +-spec finish_request(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, term()}, term(), term()}. +finish_request(RD, Ctx=#rcs_web_context{riak_client = undefined}) -> + {true, RD, Ctx}; +finish_request(RD, Ctx=#rcs_web_context{riak_client = RcPid}) -> + riak_cs_riak_client:checkin(RcPid), + {true, RD, Ctx#rcs_web_context{riak_client = undefined}}. + + +%% ------------------------------------------------------------------- +%% Internal functions +%% ------------------------------------------------------------------- + +do_action("AssumeRoleWithSAML", + Form, RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + Specs = lists:foldl(fun assume_role_with_saml_fields_filter/2, + #{request_id => RequestId}, Form), + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_sts:assume_role_with_saml(Specs, Pbc) of + {ok, #{assumed_role_user := #assumed_role_user{assumed_role_id = AssumedRoleId} = AssumedRoleUser, + audience := Audience, + credentials := Credentials, + issuer := Issuer, + name_qualifier := NameQualifier, + packed_policy_size := PackedPolicySize, + source_identity := SourceIdentity, + subject := Subject, + subject_type := SubjectType}} -> + logger:info("AssumeRoleWithSAML completed for user ~s (key_id: ~s) on request_id ~s", + [AssumedRoleId, Credentials#credentials.access_key_id, RequestId]), + Doc = riak_cs_xml:to_xml( + #assume_role_with_saml_response{assumed_role_user = AssumedRoleUser, + audience = Audience, + credentials = Credentials, + issuer = Issuer, + name_qualifier = NameQualifier, + packed_policy_size = PackedPolicySize, + source_identity = SourceIdentity, + subject = Subject, + subject_type = SubjectType, + request_id = RequestId}), + {true, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end; + +do_action(Unsupported, _Form, RD, Ctx = #rcs_web_context{response_module = ResponseMod}) -> + logger:warning("STS action ~s not supported yet; ignoring request", [Unsupported]), + ResponseMod:api_error(invalid_action, RD, Ctx). + + +assume_role_with_saml_fields_filter({K, V}, Acc) -> + case K of + "DurationSeconds" -> + maps:put(duration_seconds, list_to_integer(V), Acc); + "Policy" -> + maps:put(policy, base64:encode(V), Acc); + "PrincipalArn" -> + maps:put(principal_arn, list_to_binary(V), Acc); + "RoleArn" -> + maps:put(role_arn, list_to_binary(V), Acc); + "SAMLAssertion" -> + maps:put(saml_assertion, list_to_binary(V), Acc); + "PolicyArns" -> + Acc; + "PolicyArns.member." ++ _MemberNo -> + AA = maps:get(policy_arns, Acc, []), + maps:put(policy_arns, AA ++ [list_to_binary(V)], Acc); + CommonParameter when CommonParameter == "Action"; + CommonParameter == "Version" -> + Acc; + Unrecognized -> + logger:warning("Unrecognized parameter in call to AssumeRoleWithSAML: ~s", [Unrecognized]), + Acc + end. diff --git a/apps/riak_cs/src/riak_cs_wm_temp_sessions.erl b/apps/riak_cs/src/riak_cs_wm_temp_sessions.erl new file mode 100644 index 000000000..20cc83f5a --- /dev/null +++ b/apps/riak_cs/src/riak_cs_wm_temp_sessions.erl @@ -0,0 +1,103 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_wm_temp_sessions). + +-export([init/1, + service_available/2, + forbidden/2, + options/2, + content_types_provided/2, + allowed_methods/2, + finish_request/2, + produce_xml/2 + ]). + +-include("riak_cs_web.hrl"). +-include_lib("xmerl/include/xmerl.hrl"). +-include_lib("kernel/include/logger.hrl"). + +init(Config) -> + AuthBypass = not proplists:get_value(admin_auth_enabled, Config), + Api = riak_cs_config:api(), + RespModule = riak_cs_config:response_module(Api), + {ok, #rcs_web_context{auth_bypass = AuthBypass, + api = Api, + response_module = RespModule}}. + +-spec options(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), string()}], #wm_reqdata{}, #rcs_web_context{}}. +options(RD, Ctx) -> + {riak_cs_wm_utils:cors_headers(), RD, Ctx}. + +-spec service_available(#wm_reqdata{}, #rcs_web_context{}) -> {true, #wm_reqdata{}, #rcs_web_context{}}. +service_available(RD, Ctx) -> + riak_cs_wm_utils:service_available( + wrq:set_resp_headers(riak_cs_wm_utils:cors_headers(), RD), Ctx). + +-spec allowed_methods(#wm_reqdata{}, #rcs_web_context{}) -> {[atom()], #wm_reqdata{}, #rcs_web_context{}}. +allowed_methods(RD, Ctx) -> + {['GET', 'OPTIONS'], RD, Ctx}. + + +-spec forbidden(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +forbidden(RD, Ctx) -> + case wrq:method(RD) of + 'OPTIONS' -> + {false, RD, Ctx}; + _ -> + forbidden2(RD, Ctx) + end. +forbidden2(RD, Ctx = #rcs_web_context{auth_bypass = AuthBypass}) -> + riak_cs_wm_utils:find_and_auth_admin(RD, Ctx, AuthBypass). + +content_types_provided(RD, Ctx) -> + {[{?XML_TYPE, produce_xml}], RD, Ctx}. + +-spec finish_request(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, term()}, term(), term()}. +finish_request(RD, Ctx = #rcs_web_context{riak_client = undefined}) -> + {true, RD, Ctx}; +finish_request(RD, Ctx = #rcs_web_context{riak_client = RcPid}) -> + riak_cs_riak_client:checkin(RcPid), + {true, RD, Ctx#rcs_web_context{riak_client = undefined}}. + + +produce_xml(RD, Ctx = #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + request_id = RequestId}) -> + MaxItems = list_to_integer(wrq:get_qs_value("MaxItems", "1000", RD)), + Marker = wrq:get_qs_value("Marker", RD), + case riak_cs_temp_sessions:list( + RcPid, #list_temp_sessions_request{request_id = RequestId, + max_items = MaxItems, + marker = Marker}) of + {ok, #{temp_sessions := TempSessions, + marker := NewMarker, + is_truncated := IsTruncated}} -> + Doc = riak_cs_xml:to_xml( + #list_temp_sessions_response{temp_sessions = TempSessions, + request_id = RequestId, + marker = NewMarker, + is_truncated = IsTruncated}), + {Doc, riak_cs_wm_utils:make_final_rd(Doc, RD), Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end. diff --git a/src/riak_cs_wm_usage.erl b/apps/riak_cs/src/riak_cs_wm_usage.erl similarity index 71% rename from src/riak_cs_wm_usage.erl rename to apps/riak_cs/src/riak_cs_wm_usage.erl index c645492fc..22723acfb 100644 --- a/src/riak_cs_wm_usage.erl +++ b/apps/riak_cs/src/riak_cs_wm_usage.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -103,9 +104,10 @@ -module(riak_cs_wm_usage). --export([ - init/1, +-export([init/1, + options/2, service_available/2, + allowed_methods/2, malformed_request/2, resource_exists/2, content_types_provided/2, @@ -115,19 +117,32 @@ produce_xml/2, finish_request/2 ]). + +-ignore_xref([init/1, + options/2, + service_available/2, + allowed_methods/2, + malformed_request/2, + resource_exists/2, + content_types_provided/2, + generate_etag/2, + forbidden/2, + produce_json/2, + produce_xml/2, + finish_request/2 + ]). + -on_load(on_load/0). -ifdef(TEST). --ifdef(EQC). --compile([export_all]). --include_lib("eqc/include/eqc.hrl"). --endif. +-compile([export_all, nowarn_export_all]). +-include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). -endif. --include_lib("webmachine/include/webmachine.hrl"). -include("rts.hrl"). -include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). %% Keys used in output - defined here to help keep JSON and XML output %% as similar as possible. @@ -149,16 +164,12 @@ -define(ATTR_START, 'StartTime'). -define(ATTR_END, 'EndTime'). --record(ctx, { - auth_bypass :: boolean(), - riak_client :: pid(), - user :: rcs_user(), - admin_access=false :: boolean(), - start_time :: calendar:datetime(), - end_time :: calendar:datetime(), - body :: iodata(), - etag :: iolist() - }). +-record(local_context, { start_time :: undefined | calendar:datetime() + , end_time :: undefined | calendar:datetime() + , body :: undefined | iodata() + , etag :: undefined | iolist() + } + ). on_load() -> %% put atoms into atom table, for binary_to_existing_atom/2 in xml_name/1 @@ -168,20 +179,25 @@ on_load() -> ok. init(Config) -> - %% Check if authentication is disabled and - %% set that in the context. AuthBypass = not proplists:get_value(admin_auth_enabled, Config), - {ok, #ctx{auth_bypass=AuthBypass}}. + {ok, #rcs_web_context{auth_bypass = AuthBypass, + local_context = #local_context{}}}. + +-spec options(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), string()}], #wm_reqdata{}, #rcs_web_context{}}. +options(RD, Ctx) -> + {riak_cs_wm_utils:cors_headers(), RD, Ctx}. +-spec service_available(#wm_reqdata{}, #rcs_web_context{}) -> {true, #wm_reqdata{}, #rcs_web_context{}}. service_available(RD, Ctx) -> - case riak_cs_riak_client:checkout() of - {ok, RcPid} -> - {true, RD, Ctx#ctx{riak_client=RcPid}}; - {error, _} -> - {false, error_msg(RD, <<"Usage database connection failed">>), Ctx} - end. + riak_cs_wm_utils:service_available( + wrq:set_resp_headers(riak_cs_wm_utils:cors_headers(), RD), Ctx). + +-spec allowed_methods(#wm_reqdata{}, #rcs_web_context{}) -> {[atom()], #wm_reqdata{}, #rcs_web_context{}}. +allowed_methods(RD, Ctx) -> + {['GET', 'OPTIONS'], RD, Ctx}. -malformed_request(RD, Ctx) -> +-spec malformed_request(#wm_reqdata{}, #rcs_web_context{}) -> {true, #wm_reqdata{}, #rcs_web_context{}}. +malformed_request(RD, #rcs_web_context{local_context = LCtx0} = Ctx) -> case parse_start_time(RD) of {ok, Start} -> case parse_end_time(RD, Start) of @@ -192,9 +208,10 @@ malformed_request(RD, Ctx) -> error_msg(RD, <<"Too much time requested">>), Ctx}; false -> + LCtx = LCtx0#local_context{start_time = lists:min([Start, End]), + end_time = lists:max([Start, End])}, {false, RD, - Ctx#ctx{start_time=lists:min([Start, End]), - end_time=lists:max([Start, End])}} + Ctx#rcs_web_context{local_context = LCtx}} end; error -> {true, error_msg(RD, <<"Invalid end-time format">>), Ctx} @@ -203,10 +220,10 @@ malformed_request(RD, Ctx) -> {true, error_msg(RD, <<"Invalid start-time format">>), Ctx} end. -resource_exists(RD, #ctx{riak_client=RcPid}=Ctx) -> +resource_exists(RD, #rcs_web_context{riak_client = RcPid} = Ctx) -> case riak_cs_user:get_user(user_key(RD), RcPid) of {ok, {User, _UserObj}} -> - {true, RD, Ctx#ctx{user=User}}; + {true, RD, Ctx#rcs_web_context{user = User}}; {error, _} -> {false, error_msg(RD, <<"Unknown user">>), Ctx} end. @@ -220,7 +237,7 @@ content_types_provided(RD, Ctx) -> end, {Types, RD, Ctx}. -generate_etag(RD, #ctx{etag=undefined}=Ctx) -> +generate_etag(RD, #rcs_web_context{local_context = #local_context{etag = undefined} = LCtx} = Ctx) -> case content_types_provided(RD, Ctx) of {[{_Type, Producer}], _, _} -> ok; {Choices, _, _} -> @@ -235,13 +252,24 @@ generate_etag(RD, #ctx{etag=undefined}=Ctx) -> end, {Body, NewRD, NewCtx} = ?MODULE:Producer(RD, Ctx), Etag = riak_cs_utils:etag_from_binary_no_quotes(riak_cs_utils:md5(Body)), - {Etag, NewRD, NewCtx#ctx{etag=Etag}}; -generate_etag(RD, #ctx{etag=Etag}=Ctx) -> + {Etag, NewRD, NewCtx#rcs_web_context{local_context = LCtx#local_context{etag = Etag}}}; +generate_etag(RD, #rcs_web_context{local_context = #local_context{etag = Etag}} = Ctx) -> {Etag, RD, Ctx}. -forbidden(RD, #ctx{auth_bypass=AuthBypass, riak_client=RcPid}=Ctx) -> - BogusContext = #context{auth_bypass=AuthBypass, riak_client=RcPid}, - Next = fun(NewRD, #context{user=User}) -> +-spec forbidden(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +forbidden(RD, Ctx) -> + case wrq:method(RD) of + 'OPTIONS' -> + {false, RD, Ctx}; + _ -> + forbidden2(RD, Ctx) + end. +forbidden2(RD, #rcs_web_context{auth_bypass = AuthBypass, + riak_client = RcPid} = Ctx) -> + BogusContext = #rcs_web_context{auth_bypass = AuthBypass, + riak_client = RcPid}, + Next = fun(NewRD, #rcs_web_context{user = User}) -> forbidden(NewRD, Ctx, User, AuthBypass) end, Conv2Ctx = fun(_) -> Ctx end, @@ -249,7 +277,7 @@ forbidden(RD, #ctx{auth_bypass=AuthBypass, riak_client=RcPid}=Ctx) -> forbidden(RD, Ctx, _, true) -> %% Treat AuthBypass=true as same as admin access - {false, RD, Ctx#ctx{admin_access=true}}; + {false, RD, Ctx#rcs_web_context{admin_access = true}}; forbidden(RD, Ctx, undefined, false) -> %% anonymous access disallowed riak_cs_wm_utils:deny_access(RD, Ctx); @@ -257,7 +285,7 @@ forbidden(RD, Ctx, User, false) -> case riak_cs_config:admin_creds() of {ok, {Admin, _}} when Admin == User?RCS_USER.key_id -> %% admin can access anyone's stats - {false, RD, Ctx#ctx{admin_access=true}}; + {false, RD, Ctx#rcs_web_context{admin_access = true}}; _ -> case user_key(RD) == User?RCS_USER.key_id of true -> @@ -270,62 +298,58 @@ forbidden(RD, Ctx, User, false) -> end end. -finish_request(RD, #ctx{riak_client=undefined}=Ctx) -> +finish_request(RD, #rcs_web_context{riak_client = undefined} = Ctx) -> {true, RD, Ctx}; -finish_request(RD, #ctx{riak_client=RcPid}=Ctx) -> +finish_request(RD, #rcs_web_context{riak_client = RcPid} = Ctx) -> riak_cs_riak_client:checkin(RcPid), - {true, RD, Ctx#ctx{riak_client=undefined}}. + {true, RD, Ctx#rcs_web_context{riak_client = undefined}}. %% JSON Production %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -produce_json(RD, #ctx{body=undefined}=Ctx) -> +produce_json(RD, #rcs_web_context{local_context = #local_context{body = undefined} = LCtx0} = Ctx) -> Access = maybe_access(RD, Ctx), Storage = maybe_storage(RD, Ctx), - MJ = {struct, [{?KEY_ACCESS, mochijson_access(Access)}, - {?KEY_STORAGE, mochijson_storage(Storage)}]}, - Body = mochijson2:encode(MJ), - {Body, RD, Ctx#ctx{body=Body}}; -produce_json(RD, #ctx{body=Body}=Ctx) -> + MJ = [{?KEY_ACCESS, json_access(Access)}, + {?KEY_STORAGE, json_storage(Storage)}], + Body = jsx:encode(MJ), + {Body, RD, Ctx#rcs_web_context{local_context = LCtx0#local_context{body = Body}}}; +produce_json(RD, #rcs_web_context{local_context = #local_context{body = Body}} = Ctx) -> {Body, RD, Ctx}. -mochijson_access(Msg) when is_atom(Msg) -> +json_access(Msg) when is_atom(Msg) -> Msg; -mochijson_access({Access, Errors}) -> - Nodes = [{struct, [{?KEY_NODE, Node}, - {?KEY_SAMPLES, [{struct, S} || S <- Samples]}]} +json_access({Access, Errors}) -> + Nodes = [[{?KEY_NODE, Node}, {?KEY_SAMPLES, [S || S <- Samples]}] || {Node, Samples} <- Access], - Errs = [ {struct, mochijson_sample_error(E)} || E <- Errors ], + Errs = [ json_sample_error(E) || E <- Errors ], [{?KEY_NODES, Nodes}, {?KEY_ERRORS, Errs}]. -mochijson_sample_error({{Start, End}, Reason}) -> +json_sample_error({{Start, End}, Reason}) -> [{?START_TIME, rts:iso8601(Start)}, {?END_TIME, rts:iso8601(End)}, - {?KEY_REASON, mochijson_reason(Reason)}]. + {?KEY_REASON, json_reason(Reason)}]. -mochijson_reason(Reason) -> +json_reason(Reason) -> if is_atom(Reason) -> atom_to_binary(Reason, latin1); is_binary(Reason) -> Reason; true -> list_to_binary(io_lib:format("~p", [Reason])) end. -mochijson_storage(Msg) when is_atom(Msg) -> +json_storage(Msg) when is_atom(Msg) -> Msg; -mochijson_storage({Storage, Errors}) -> - [{?KEY_SAMPLES, [ mochijson_storage_sample(S) || S <- Storage ]}, - {?KEY_ERRORS, [ mochijson_sample_error(E)|| E <- Errors ]}]. - -mochijson_storage_sample(Sample) -> - {struct, Sample}. +json_storage({Storage, Errors}) -> + [{?KEY_SAMPLES, Storage}, + {?KEY_ERRORS, [json_sample_error(E) || E <- Errors]}]. %% XML Production %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -produce_xml(RD, #ctx{body=undefined}=Ctx) -> +produce_xml(RD, #rcs_web_context{local_context = #local_context{body = undefined} = LCtx} = Ctx) -> Access = maybe_access(RD, Ctx), Storage = maybe_storage(RD, Ctx), Doc = [{?KEY_USAGE, [{?KEY_ACCESS, xml_access(Access)}, {?KEY_STORAGE, xml_storage(Storage)}]}], Body = riak_cs_xml:to_xml(Doc), - {Body, RD, Ctx#ctx{body=Body}}; -produce_xml(RD, #ctx{body=Body}=Ctx) -> + {Body, RD, Ctx#rcs_web_context{local_context = LCtx#local_context{body = Body}}}; +produce_xml(RD, #rcs_web_context{local_context = #local_context{body = Body}} = Ctx) -> {Body, RD, Ctx}. xml_access(Msg) when is_atom(Msg) -> @@ -348,7 +372,7 @@ xml_sample(Sample, SubType, TypeLabel) -> {?KEY_SAMPLE, [{xml_name(?START_TIME), S}, {xml_name(?END_TIME), E}], [{SubType, [{TypeLabel, OpName}], [{xml_name(K), [mochinum:digits(V)]} || {K, V} <- Stats]} - || {OpName, {struct, Stats}} <- Rest ]}. + || {OpName, Stats} <- Rest ]}. xml_sample_error({{Start, End}, Reason}, SubType, TypeLabel) -> %% cheat to make errors structured exactly like samples @@ -361,7 +385,6 @@ xml_sample_error({{Start, End}, Reason}, SubType, TypeLabel) -> %% @doc JSON deserializes with keys as binaries, but xmerl requires %% tag names to be atoms. --spec xml_name(binary()) -> usage_field_type() | ?ATTR_START | ?ATTR_END. xml_name(?START_TIME) -> ?ATTR_START; xml_name(?END_TIME) -> ?ATTR_END; xml_name(UsageFieldName) -> @@ -385,8 +408,8 @@ xml_storage({Storage, Errors}) -> %% Internals %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% user_key(RD) -> case path_tokens(RD) of - [KeyId|_] -> mochiweb_util:unquote(KeyId); - _ -> [] + [KeyId|_] -> list_to_binary(mochiweb_util:unquote(KeyId)); + _ -> <<>> end. maybe_access(RD, Ctx) -> @@ -395,12 +418,28 @@ maybe_access(RD, Ctx) -> maybe_storage(RD, Ctx) -> usage_if(RD, Ctx, "b", riak_cs_storage). -usage_if(RD, #ctx{riak_client=RcPid, admin_access=AdminAceess, - start_time=Start, end_time=End}, +usage_if(RD, #rcs_web_context{riak_client = RcPid, + admin_access = AdminAccess, + user = CallingUser, + local_context = #local_context{start_time = Start, + end_time = End}}, QParam, Module) -> case true_param(RD, QParam) of true -> - Module:get_usage(RcPid, user_key(RD), AdminAceess, Start, End); + TargetKeyId = user_key(RD), + case CallingUser of + ?RCS_USER{arn = Arn, + key_id = TargetKeyId} -> + Module:get_usage(RcPid, Arn, AdminAccess, Start, End); + _ -> + case riak_cs_user:get_user(TargetKeyId, RcPid) of + {ok, {?RCS_USER{arn = Arn}, _}} -> + Module:get_usage(RcPid, Arn, AdminAccess, Start, End); + _ -> + logger:notice("Usage request for non-existing user ~s", [TargetKeyId]), + not_requested + end + end; false -> not_requested end. @@ -456,7 +495,7 @@ time_param(RD, Param, N, Default) -> end. error_msg(RD, Message) -> - {CTP, _, _} = content_types_provided(RD, #ctx{}), + {CTP, _, _} = content_types_provided(RD, #rcs_web_context{}), PTypes = [Type || {Type,_Fun} <- CTP], AcceptHdr = wrq:get_req_header("accept", RD), case webmachine_util:choose_media_type(PTypes, AcceptHdr) of @@ -469,8 +508,8 @@ error_msg(RD, Message) -> wrq:set_resp_header("content-type", Type, wrq:set_resp_body(Body, RD)). json_error_msg(Message) -> - MJ = {struct, [{?KEY_ERROR, {struct, [{?KEY_MESSAGE, Message}]}}]}, - mochijson2:encode(MJ). + MJ = [{?KEY_ERROR, [{?KEY_MESSAGE, Message}]}], + jsx:encode(MJ). xml_error_msg(Message) when is_binary(Message) -> xml_error_msg(binary_to_list(Message)); @@ -484,7 +523,7 @@ xml_error_msg(Message) -> -> boolean(). too_many_periods(Start, End) -> Seconds = calendar:datetime_to_gregorian_seconds(End) - -calendar:datetime_to_gregorian_seconds(Start), + - calendar:datetime_to_gregorian_seconds(Start), {ok, Limit} = application:get_env(riak_cs, usage_request_limit), {ok, Access} = riak_cs_access:archive_period(), @@ -492,12 +531,11 @@ too_many_periods(Start, End) -> ((Seconds div Access) > Limit) orelse ((Seconds div Storage) > Limit). --ifdef(TEST). --ifdef(EQC). +-ifdef(TEST). datetime_test() -> - true = eqc:quickcheck(datetime_invalid_prop()). + true = proper:quickcheck(datetime_invalid_prop()). %% make sure that datetime correctly returns 'error' for invalid %% iso8601 date strings @@ -525,5 +563,4 @@ valid_iso8601(L) -> is_digit(C) -> C >= $0 andalso C =< $9. --endif. % EQC -endif. % TEST diff --git a/apps/riak_cs/src/riak_cs_wm_user.erl b/apps/riak_cs/src/riak_cs_wm_user.erl new file mode 100644 index 000000000..0f47da571 --- /dev/null +++ b/apps/riak_cs/src/riak_cs_wm_user.erl @@ -0,0 +1,392 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(riak_cs_wm_user). + +-export([init/1, + service_available/2, + forbidden/2, + content_types_provided/2, + content_types_accepted/2, + accept_json/2, + accept_xml/2, + delete_resource/2, + allowed_methods/2, + post_is_create/2, + create_path/2, + produce_json/2, + produce_xml/2, + finish_request/2 + ]). + +-ignore_xref([init/1, + service_available/2, + forbidden/2, + content_types_provided/2, + content_types_accepted/2, + accept_json/2, + accept_xml/2, + delete_resource/2, + allowed_methods/2, + post_is_create/2, + create_path/2, + produce_json/2, + produce_xml/2, + finish_request/2 + ]). + +-include("riak_cs.hrl"). +-include_lib("xmerl/include/xmerl.hrl"). +-include_lib("kernel/include/logger.hrl"). + +%% ------------------------------------------------------------------- +%% Webmachine callbacks +%% ------------------------------------------------------------------- + +init(Config) -> + %% Check if authentication is disabled and + %% set that in the context. + AuthBypass = not proplists:get_value(admin_auth_enabled, Config), + Api = riak_cs_config:api(), + RespModule = riak_cs_config:response_module(Api), + {ok, #rcs_web_context{auth_bypass = AuthBypass, + api = Api, + response_module = RespModule}}. + +-spec service_available(#wm_reqdata{}, #rcs_web_context{}) -> {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +service_available(RD, Ctx) -> + riak_cs_wm_utils:service_available( + wrq:set_resp_headers(riak_cs_wm_utils:cors_headers(), RD), Ctx). + +-spec allowed_methods(#wm_reqdata{}, #rcs_web_context{}) -> {[atom()], #wm_reqdata{}, #rcs_web_context{}}. +allowed_methods(RD, Ctx) -> + {['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'OPTIONS'], RD, Ctx}. + +-spec forbidden(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +forbidden(RD, Ctx) -> + case wrq:method(RD) of + 'OPTIONS' -> + {false, RD, Ctx}; + _ -> + forbidden2(RD, Ctx) + end. +forbidden2(RD, Ctx = #rcs_web_context{auth_bypass = AuthBypass}) -> + Method = wrq:method(RD), + AnonOk = ((Method =:= 'PUT' orelse Method =:= 'POST' orelse Method =:= 'DELETE') andalso + riak_cs_config:anonymous_user_creation()) + orelse AuthBypass, + Next = fun(NewRD, NewCtx = #rcs_web_context{user = User}) -> + forbidden(wrq:method(RD), + NewRD, + NewCtx, + User, + user_key(RD), + AnonOk) + end, + riak_cs_wm_utils:find_and_auth_user(RD, Ctx, Next). + +-spec content_types_accepted(#wm_reqdata{}, #rcs_web_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #rcs_web_context{}}. +content_types_accepted(RD, Ctx) -> + {[{?XML_TYPE, accept_xml}, {?JSON_TYPE, accept_json}], RD, Ctx}. + +content_types_provided(RD, Ctx) -> + {[{?XML_TYPE, produce_xml}, {?JSON_TYPE, produce_json}], RD, Ctx}. + +post_is_create(RD, Ctx) -> + {true, RD, Ctx}. + +create_path(RD, Ctx) -> + {"/riak-cs/user", RD, Ctx}. + +-spec accept_json(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, term()}, term(), term()}. +accept_json(RD, Ctx = #rcs_web_context{riak_client = RcPid}) -> + FF = jsx:decode(wrq:req_body(RD), [{labels, atom}]), + Method = wrq:method(RD), + case check_required_fields(Method, FF) of + ok -> + IAMExtra = #{permissions_boundary => maps:get(permissions_boundary, FF, undefined), + tags => maps:get(tags, FF, [])}, + Res = + case Method of + 'POST' -> + riak_cs_user:create_user(maps:get(name, FF, <<>>), + maps:get(email, FF, <<>>), + IAMExtra); + 'PUT' -> + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:find_user(#{key_id => user_key(RD)}, Pbc) of + {ok, {User, _}} -> + update_user(maps:to_list(maps:merge(FF, IAMExtra)), User); + {error, notfound} -> + {error, no_such_user} + end + end, + user_response(Res, ?JSON_TYPE, RD, Ctx); + ER -> + user_response(ER, ?JSON_TYPE, RD, Ctx) + end. +check_required_fields('POST', + #{email := Email, + name := Name}) when is_binary(Email), + is_binary(Name) -> + ok; +check_required_fields('PUT', + #{}) -> + ok; +check_required_fields(_, _) -> + {error, missing_parameter}. + + +-spec accept_xml(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, term()}, #wm_reqdata{}, #rcs_web_context{}}. +accept_xml(RD, Ctx = #rcs_web_context{riak_client = RcPid}) -> + Body = binary_to_list(wrq:req_body(RD)), + case riak_cs_xml:scan(Body) of + {error, malformed_xml} -> + riak_cs_aws_response:api_error(invalid_user_update, RD, Ctx); + {ok, Xml} -> + FF = maps:to_list( + lists:foldl(fun user_xml_filter/2, #{}, Xml#xmlElement.content)), + UserName = proplists:get_value(name, FF, ""), + Email = proplists:get_value(email, FF, ""), + Res = + case wrq:method(RD) of + 'POST' -> + riak_cs_user:create_user(UserName, Email); + 'PUT' -> + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + case riak_cs_iam:find_user(#{key_id => user_key(RD)}, Pbc) of + {ok, {User, _}} -> + update_user(FF, User); + {error, notfound} -> + {error, no_such_user} + end + end, + user_response(Res, ?XML_TYPE, RD, Ctx) + end. + +produce_json(RD, Ctx = #rcs_web_context{user = User}) -> + Body = riak_cs_json:to_json(User), + Etag = etag(Body), + RD2 = wrq:set_resp_header("ETag", Etag, RD), + {Body, RD2, Ctx}. + +produce_xml(RD, Ctx = #rcs_web_context{user = User}) -> + Body = riak_cs_xml:to_xml(User), + Etag = etag(Body), + RD2 = wrq:set_resp_header("ETag", Etag, RD), + {Body, RD2, Ctx}. + + +delete_resource(RD, Ctx = #rcs_web_context{user = User, + response_module = ResponseMod}) -> + case riak_cs_iam:delete_user(User) of + ok -> + {true, RD, Ctx}; + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end. + + +-spec finish_request(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean(), #wm_reqdata{}, #rcs_web_context{}}. +finish_request(RD, Ctx = #rcs_web_context{riak_client = undefined}) -> + {true, RD, Ctx}; +finish_request(RD, Ctx = #rcs_web_context{riak_client = RcPid}) -> + riak_cs_riak_client:checkin(RcPid), + {true, RD, Ctx#rcs_web_context{riak_client = undefined}}. + +%% ------------------------------------------------------------------- +%% Internal functions +%% ------------------------------------------------------------------- + +admin_check(true, RD, Ctx) -> + {false, RD, Ctx}; +admin_check(false, RD, Ctx) -> + riak_cs_wm_utils:deny_access(RD, Ctx). + +%% @doc Calculate the etag of a response body +etag(Body) -> + riak_cs_utils:etag_from_binary(riak_cs_utils:md5(Body)). + +forbidden(_Method, RD, Ctx, undefined, _UserPathKey, false) -> + %% anonymous access disallowed + riak_cs_wm_utils:deny_access(RD, Ctx); +forbidden(_, RD, Ctx, undefined, <<>>, true) -> + {false, RD, Ctx}; +forbidden(_, RD, Ctx, undefined, UserPathKey, true) -> + get_user({false, RD, Ctx}, UserPathKey); +forbidden(Method, RD, Ctx, User, <<>>, _) + when Method =:= 'PUT'; + Method =:= 'POST'; + Method =:= 'DELETE' -> + %% Admin is creating/updating/deleting a user + admin_check(riak_cs_user:is_admin(User), RD, Ctx); +forbidden(_Method, RD, Ctx, User, UserPathKey, _) when + UserPathKey =:= User?RCS_USER.key_id; + UserPathKey =:= <<>> -> + %% User is accessing own account + AccessRD = riak_cs_access_log_handler:set_user(User, RD), + {false, AccessRD, Ctx}; +forbidden(_Method, RD, Ctx, User, UserPathKey, _) -> + AdminCheckResult = admin_check(riak_cs_user:is_admin(User), RD, Ctx), + get_user(AdminCheckResult, UserPathKey). + +get_user({false, RD, Ctx}, UserPathKey) -> + handle_get_user_result( + riak_cs_user:get_user(UserPathKey, Ctx#rcs_web_context.riak_client), + RD, + Ctx); +get_user(AdminCheckResult, _) -> + AdminCheckResult. + +handle_get_user_result({ok, {User, UserObj}}, RD, Ctx) -> + {false, RD, Ctx#rcs_web_context{user = User, + user_object = UserObj}}; +handle_get_user_result({error, Reason}, RD, Ctx) -> + logger:warning("Failed to fetch user record. KeyId: ~s" + " Reason: ~p", [user_key(RD), Reason]), + riak_cs_aws_response:api_error(invalid_access_key_id, RD, Ctx). + +update_user(UpdateItems, User) -> + UpdateUserResult = update_user_record(User, UpdateItems, false), + handle_update_result(UpdateUserResult). + +update_user_record(User, [], U1) -> + {U1, User}; +update_user_record(User, [{status, Status} | RestUpdates], _) -> + update_user_record(User?RCS_USER{status = str_to_status(Status)}, RestUpdates, true); +update_user_record(User, [{name, Name} | RestUpdates], _) -> + update_user_record(User?RCS_USER{name = Name}, RestUpdates, true); +update_user_record(User, [{email, Email} | RestUpdates], _) -> + DisplayName = riak_cs_user:display_name(Email), + update_user_record(User?RCS_USER{email = Email, + display_name = DisplayName}, + RestUpdates, + true); +update_user_record(User, [{path, Path} | RestUpdates], _) -> + update_user_record(User?RCS_USER{path = Path}, RestUpdates, true); +update_user_record(User = ?RCS_USER{}, [{new_key_secret, true} | RestUpdates], _) -> + update_user_record(riak_cs_user:update_key_secret(User), RestUpdates, true); +update_user_record(User, [_ | RestUpdates], U1) -> + update_user_record(User, RestUpdates, U1). + +str_to_status(<<"enabled">>) -> enabled; +str_to_status(<<"disabled">>) -> disabled. + + +handle_update_result({false, _User}) -> + {halt, 200}; +handle_update_result({true, User}) -> + riak_cs_iam:update_user(User). + +set_resp_data(ContentType, RD, #rcs_web_context{user = User}) -> + UserDoc = format_user_record(User, ContentType), + wrq:set_resp_body(UserDoc, RD). + +user_key(RD) -> + case wrq:path_tokens(RD) of + [KeyId|_] -> + list_to_binary(mochiweb_util:unquote(KeyId)); + _ -> <<>> + end. + +user_xml_filter(#xmlText{}, Acc) -> + Acc; +user_xml_filter(Element, Acc) -> + case Element#xmlElement.name of + 'Email' -> + [Content | _] = Element#xmlElement.content, + case is_record(Content, xmlText) of + true -> + Acc#{email => list_to_binary(Content#xmlText.value)}; + false -> + Acc + end; + 'Name' -> + [Content | _] = Element#xmlElement.content, + case is_record(Content, xmlText) of + true -> + Acc#{name => list_to_binary(Content#xmlText.value)}; + false -> + Acc + end; + 'Id' -> + [Content | _] = Element#xmlElement.content, + case is_record(Content, xmlText) of + true -> + Acc#{id => list_to_binary(Content#xmlText.value)}; + false -> + Acc + end; + 'Path' -> + [Content | _] = Element#xmlElement.content, + case is_record(Content, xmlText) of + true -> + Acc#{path => list_to_binary(Content#xmlText.value)}; + false -> + Acc + end; + 'Status' -> + [Content | _] = Element#xmlElement.content, + case is_record(Content, xmlText) of + true -> + Acc#{status => list_to_binary(Content#xmlText.value)}; + false -> + Acc + end; + 'NewKeySecret' -> + [Content | _] = Element#xmlElement.content, + case is_record(Content, xmlText) of + true -> + case Content#xmlText.value of + "true" -> + Acc#{new_key_secret => true}; + "false" -> + Acc#{new_key_secret => false}; + _ -> + Acc + end; + false -> + Acc + end; + _ -> + Acc + end. + +user_response({ok, User}, ContentType, RD, Ctx) -> + UserDoc = format_user_record(User, ContentType), + WrittenRD = + wrq:set_resp_body(UserDoc, + wrq:set_resp_header("Content-Type", ContentType, RD)), + {true, WrittenRD, Ctx}; +user_response({halt, 200}, ContentType, RD, Ctx) -> + {{halt, 200}, set_resp_data(ContentType, RD, Ctx), Ctx}; +user_response({error, Reason}, _, RD, Ctx) -> + riak_cs_aws_response:api_error(Reason, RD, Ctx). + +format_user_record(User, ?JSON_TYPE) -> + riak_cs_json:to_json(User); +format_user_record(User, ?XML_TYPE) -> + riak_cs_xml:to_xml(User). diff --git a/src/riak_cs_wm_users.erl b/apps/riak_cs/src/riak_cs_wm_users.erl similarity index 70% rename from src/riak_cs_wm_users.erl rename to apps/riak_cs/src/riak_cs_wm_users.erl index 3bc13655b..15ebb52ea 100644 --- a/src/riak_cs_wm_users.erl +++ b/apps/riak_cs/src/riak_cs_wm_users.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -25,12 +26,25 @@ forbidden/2, content_types_provided/2, allowed_methods/2, + options/2, produce_json/2, produce_xml/2, - finish_request/2]). + finish_request/2 + ]). + +-ignore_xref([init/1, + service_available/2, + forbidden/2, + content_types_provided/2, + allowed_methods/2, + options/2, + produce_json/2, + produce_xml/2, + finish_request/2 + ]). -include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). +-include_lib("kernel/include/logger.hrl"). %% ------------------------------------------------------------------- %% Webmachine callbacks @@ -42,26 +56,40 @@ init(Config) -> AuthBypass = not proplists:get_value(admin_auth_enabled, Config), Api = riak_cs_config:api(), RespModule = riak_cs_config:response_module(Api), - {ok, #context{auth_bypass=AuthBypass, - api=Api, - response_module=RespModule}}. + {ok, #rcs_web_context{auth_bypass = AuthBypass, + api = Api, + response_module = RespModule}}. --spec service_available(term(), term()) -> {true, term(), term()}. +-spec options(#wm_reqdata{}, #rcs_web_context{}) -> {[{string(), string()}], #wm_reqdata{}, #rcs_web_context{}}. +options(RD, Ctx) -> + {riak_cs_wm_utils:cors_headers(), RD, Ctx}. + +-spec service_available(#wm_reqdata{}, #rcs_web_context{}) -> {true, #wm_reqdata{}, #rcs_web_context{}}. service_available(RD, Ctx) -> - riak_cs_wm_utils:service_available(RD, Ctx). + riak_cs_wm_utils:service_available( + wrq:set_resp_headers(riak_cs_wm_utils:cors_headers(), RD), Ctx). --spec allowed_methods(term(), term()) -> {[atom()], term(), term()}. +-spec allowed_methods(#wm_reqdata{}, #rcs_web_context{}) -> {[atom()], #wm_reqdata{}, #rcs_web_context{}}. allowed_methods(RD, Ctx) -> - {['GET', 'HEAD'], RD, Ctx}. + {['GET', 'HEAD', 'OPTIONS'], RD, Ctx}. + -forbidden(RD, Ctx=#context{auth_bypass=AuthBypass}) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"forbidden">>), +-spec forbidden(#wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +forbidden(RD, Ctx) -> + case wrq:method(RD) of + 'OPTIONS' -> + {false, RD, Ctx}; + _ -> + forbidden2(RD, Ctx) + end. +forbidden2(RD, Ctx = #rcs_web_context{auth_bypass = AuthBypass}) -> riak_cs_wm_utils:find_and_auth_admin(RD, Ctx, AuthBypass). content_types_provided(RD, Ctx) -> {[{?XML_TYPE, produce_xml}, {?JSON_TYPE, produce_json}], RD, Ctx}. -produce_json(RD, Ctx=#context{riak_client=RcPid}) -> +produce_json(RD, Ctx = #rcs_web_context{riak_client = RcPid}) -> Boundary = unique_id(), UpdRD = wrq:set_resp_header("Content-Type", "multipart/mixed; boundary="++Boundary, @@ -77,7 +105,7 @@ produce_json(RD, Ctx=#context{riak_client=RcPid}) -> end, {{stream, {<<>>, fun() -> stream_users(json, RcPid, Boundary, Status) end}}, UpdRD, Ctx}. -produce_xml(RD, Ctx=#context{riak_client=RcPid}) -> +produce_xml(RD, Ctx = #rcs_web_context{riak_client = RcPid}) -> Boundary = unique_id(), UpdRD = wrq:set_resp_header("Content-Type", "multipart/mixed; boundary="++Boundary, @@ -93,10 +121,10 @@ produce_xml(RD, Ctx=#context{riak_client=RcPid}) -> end, {{stream, {<<>>, fun() -> stream_users(xml, RcPid, Boundary, Status) end}}, UpdRD, Ctx}. -finish_request(RD, Ctx=#context{}) -> +finish_request(RD, Ctx) -> %% riak_client is still used for streaming response. %% So do not close it here. - {true, RD, Ctx#context{riak_client=undefined}}. + {true, RD, Ctx#rcs_web_context{riak_client = undefined}}. %% ------------------------------------------------------------------- %% Internal functions @@ -113,9 +141,9 @@ stream_users(Format, RcPid, Boundary, Status) -> wait_for_users(Format, RcPid, ReqId, Boundary, Status) -> _ = riak_cs_stats:inflow([riakc, list_users_receive_chunk]), - StartTime = os:timestamp(), + StartTime = os:system_time(millisecond), receive - {ReqId, {keys, UserIds}} -> + {ReqId, {keys, UserIds}} when UserIds =/= [] -> _ = riak_cs_stats:update_with_start( [riakc, list_users_receive_chunk], StartTime), FoldFun = user_fold_fun(RcPid, Status), @@ -146,8 +174,9 @@ users_doc(UserDocs, json, Boundary) -> %% @doc Return a fold function to retrieve and filter user accounts user_fold_fun(RcPid, Status) -> - fun(UserId, Users) -> - case riak_cs_user:get_user(binary_to_list(UserId), RcPid) of + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + fun(Arn, Users) -> + case riak_cs_iam:get_user(Arn, Pbc) of {ok, {User, _}} when User?RCS_USER.status =:= Status; Status =:= undefined -> [User | Users]; @@ -155,13 +184,13 @@ user_fold_fun(RcPid, Status) -> %% Status is defined and does not match the account status Users; {error, Reason} -> - _ = lager:warning("Failed to fetch user record. KeyId: ~p" - " Reason: ~p", [UserId, Reason]), + logger:warning("Failed to fetch user record ~s." + " Reason: ~p", [Arn, Reason]), Users end end. unique_id() -> - Rand = riak_cs_utils:sha(term_to_binary({make_ref(), now()})), + Rand = riak_cs_utils:sha(term_to_binary({make_ref(), erlang:timestamp()})), <> = Rand, integer_to_list(I, 36). diff --git a/src/riak_cs_wm_utils.erl b/apps/riak_cs/src/riak_cs_wm_utils.erl similarity index 52% rename from src/riak_cs_wm_utils.erl rename to apps/riak_cs/src/riak_cs_wm_utils.erl index 22ac1b7be..d861d4758 100644 --- a/src/riak_cs_wm_utils.erl +++ b/apps/riak_cs/src/riak_cs_wm_utils.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -22,6 +23,7 @@ -export([service_available/2, service_available/3, + forbidden/3, lower_case_method/1, iso_8601_datetime/0, iso_8601_datetime/1, @@ -29,23 +31,19 @@ iso_8601_to_rfc_1123/1, to_rfc_1123/1, iso_8601_to_erl_date/1, - streaming_get/6, + streaming_get/4, find_and_auth_admin/3, find_and_auth_user/3, find_and_auth_user/4, find_and_auth_user/5, - validate_auth_header/4, ensure_doc/2, respond_api_error/3, deny_access/2, deny_invalid_key/2, extract_key/2, extract_name/1, - extract_canonical_id/1, extract_object_acl/1, maybe_update_context_with_acl_from_headers/2, - maybe_acl_from_context_and_request/3, - acl_from_headers/4, extract_acl_headers/1, has_acl_header_and_body/1, has_acl_header/1, @@ -56,41 +54,102 @@ extract_amazon_headers/1, normalize_headers/1, extract_user_metadata/1, - shift_to_owner/4, bucket_access_authorize_helper/4, object_access_authorize_helper/4, object_access_authorize_helper/5, check_object_authorization/8, + get_user_policies_or_halt/1, translate_bucket_policy/2, fetch_bucket_owner/2, bucket_owner/1, extract_date/1, check_timeskew/1, content_length/1, - valid_entity_length/3 + valid_entity_length/3, + aws_service_action/2, + make_final_rd/2, + make_request_id/0, + cors_headers/0 ]). -include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). +-include_lib("webmachine/include/wm_reqstate.hrl"). +-include_lib("kernel/include/logger.hrl"). -define(QS_KEYID, "AWSAccessKeyId"). -define(QS_SIGNATURE, "Signature"). --type acl_or_error() :: {ok, #acl_v2{}} | - {error, 'invalid_argument'} | - {error, 'unresolved_grant_email'}. - %% =================================================================== %% Public API %% =================================================================== +-spec aws_service_from_path(string()) -> string(). +aws_service_from_path("/iam") -> "iam"; +aws_service_from_path("/sts") -> "sts"; +aws_service_from_path(_) -> "s3". + +-spec aws_service_action(#wm_reqdata{}, action_target()) -> aws_action() | no_action. +aws_service_action(RD, Target) -> + Path = wrq:path(RD), + case aws_service_from_path(Path) of + "s3" -> + make_s3_action(wrq:method(RD), Target); + ServiceDoingOnlyPOSTs -> + Form = mochiweb_util:parse_qs(wrq:req_body(RD)), + Action = proplists:get_value("Action", Form), + list_to_atom(ServiceDoingOnlyPOSTs ++ ":" ++ Action) + end. + +make_s3_action(Method, Target) -> + case {Method, Target} of + {'PUT', object} -> 's3:PutObject'; + {'PUT', object_part} -> 's3:PutObject'; + {'PUT', object_acl} -> 's3:PutObjectAcl'; + {'PUT', bucket_acl} -> 's3:PutBucketAcl'; + {'PUT', bucket_policy} -> 's3:PutBucketPolicy'; + {'PUT', bucket_versioning} -> 's3:PutBucketVersioning'; + {'PUT', bucket_request_payment} -> 's3:PutBucketRequestPayment'; + + {'GET', object} -> 's3:GetObject'; + {'GET', object_part} -> 's3:ListMultipartUploadParts'; + {'GET', object_acl} -> 's3:GetObjectAcl'; + {'GET', bucket} -> 's3:ListBucket'; + {'GET', no_bucket } -> 's3:ListAllMyBuckets'; + {'GET', bucket_acl} -> 's3:GetBucketAcl'; + {'GET', bucket_policy} -> 's3:GetBucketPolicy'; + {'GET', bucket_versioning} -> 's3:GetBucketVersioning'; + {'GET', bucket_location} -> 's3:GetBucketLocation'; + {'GET', bucket_request_payment} -> 's3:GetBucketRequestPayment'; + {'GET', bucket_uploads} -> 's3:ListBucketMultipartUploads'; + + {'DELETE', object} -> 's3:DeleteObject'; + {'DELETE', object_part} -> 's3:AbortMultipartUpload'; + {'DELETE', bucket} -> 's3:DeleteBucket'; + {'DELETE', bucket_policy} -> 's3:DeleteBucketPolicy'; + + {'HEAD', object} -> 's3:GetObject'; % no HeadObjet + + %% PUT Object includes POST Object, + %% including Initiate Multipart Upload, Upload Part, Complete Multipart Upload + {'POST', object} -> 's3:PutObject'; + {'POST', object_part} -> 's3:PutObject'; + + %% same as {'GET' bucket} + {'HEAD', bucket} -> 's3:ListBucket'; + + %% 400 (MalformedPolicy): Policy has invalid action + {'PUT', bucket} -> 's3:CreateBucket'; + + {'HEAD', _} -> no_action + end. + service_available(RD, Ctx) -> service_available(request_pool, RD, Ctx). service_available(Pool, RD, Ctx) -> case riak_cs_utils:riak_connection(Pool) of {ok, RcPid} -> - {true, RD, Ctx#context{riak_client=RcPid}}; + {true, RD, Ctx#rcs_web_context{riak_client = RcPid}}; {error, _Reason} -> {false, RD, Ctx} end. @@ -99,43 +158,32 @@ service_available(Pool, RD, Ctx) -> %% the appropriate module to use to authenticate the request. %% The passthru auth can be used either with a KeyID or %% anonymously by leving the header empty. --spec parse_auth_header(string(), boolean()) -> {atom(), - string() | undefined, - string() | undefined}. parse_auth_header(KeyId, true) when KeyId =/= undefined -> - {riak_cs_passthru_auth, KeyId, undefined}; -parse_auth_header("AWS " ++ Key, _) -> - case string:tokens(Key, ":") of - [KeyId, KeyData] -> - {riak_cs_s3_auth, KeyId, KeyData}; - Other -> Other - end; -parse_auth_header(_, _) -> - {riak_cs_blockall_auth, undefined, undefined}. + {riak_cs_s3_passthru_auth, KeyId, undefined}; +parse_auth_header(S, _) -> + {KeyId, Signature} = riak_cs_aws_auth:parse_auth_header(S), + {riak_cs_aws_auth, KeyId, Signature}. %% @doc Parse authentication query parameters and determine %% the appropriate module to use to authenticate the request. %% The passthru auth can be used either with a KeyID or %% anonymously by leving the header empty. --spec parse_auth_params(string(), string(), boolean()) -> {atom(), - string() | undefined, - string() | undefined}. parse_auth_params(KeyId, _, true) when KeyId =/= undefined -> - {riak_cs_passthru_auth, KeyId, undefined}; + {riak_cs_s3_passthru_auth, KeyId, undefined}; parse_auth_params(undefined, _, true) -> - {riak_cs_passthru_auth, undefined, undefined}; + {riak_cs_s3_passthru_auth, undefined, undefined}; parse_auth_params(undefined, _, false) -> {riak_cs_blockall_auth, undefined, undefined}; parse_auth_params(_, undefined, _) -> {riak_cs_blockall_auth, undefined, undefined}; parse_auth_params(KeyId, Signature, _) -> - {riak_cs_s3_auth, KeyId, Signature}. + {riak_cs_aws_auth, list_to_binary(KeyId), list_to_binary(Signature)}. %% @doc Lookup the user specified by the access headers, and call %% `Next(RD, NewCtx)' if there is no auth error. %% %% If a user was successfully authed, the `user' and `user_object' -%% fields in the `#context' record passed to `Next' will be filled. +%% fields in the `#rcs_web_context' record passed to `Next' will be filled. %% If the access is instead anonymous, those fields will be left as %% they were passed to this function. %% @@ -150,8 +198,8 @@ find_and_auth_user(RD, ICtx, Next, AnonymousOk) -> find_and_auth_user(RD, ICtx, Next, fun(X) -> X end, AnonymousOk). find_and_auth_user(RD, - #context{auth_bypass=AuthBypass, - riak_client=RcPid}=ICtx, + #rcs_web_context{auth_bypass = AuthBypass, + riak_client = RcPid} = ICtx, Next, Conv2KeyCtx, AnonymousOk) -> @@ -164,7 +212,7 @@ find_and_auth_user(RD, AnonymousOk). find_and_auth_admin(RD, Ctx, AuthBypass) -> - Next = fun(NewRD, NewCtx=#context{user=User}) -> + Next = fun(NewRD, NewCtx = #rcs_web_context{user = User}) -> handle_auth_admin(NewRD, NewCtx, User, @@ -174,70 +222,61 @@ find_and_auth_admin(RD, Ctx, AuthBypass) -> handle_validation_response({ok, User, UserObj}, RD, Ctx, Next, _, _) -> %% given keyid and signature matched, proceed - Next(RD, Ctx#context{user=User, - user_object=UserObj}); -handle_validation_response({error, disconnected}, RD, Ctx, _Next, _, _Bool) -> - {{halt, 503}, RD, Ctx}; -handle_validation_response({error, Reason}, RD, Ctx, Next, _, true) -> + Next(RD, Ctx#rcs_web_context{user = User, + user_object = UserObj}); +handle_validation_response({error, _Reason}, RD, Ctx, Next, _, true) -> %% no keyid was given, proceed anonymously - _ = lager:debug("No user key: ~p", [Reason]), + logger:notice("anonymous_user_creation is enabled, skipping auth (specific error was ~p)", [_Reason]), Next(RD, Ctx); handle_validation_response({error, no_user_key}, RD, Ctx, _, Conv2KeyCtx, false) -> %% no keyid was given, deny access - _ = lager:debug("No user key, deny"), + logger:notice("No user key, deny"), deny_access(RD, Conv2KeyCtx(Ctx)); handle_validation_response({error, bad_auth}, RD, Ctx, _, Conv2KeyCtx, _) -> - %% given keyid was found, but signature didn't match - _ = lager:debug("bad_auth"), + logger:notice("given key_id was found, but signature didn't match"), deny_access(RD, Conv2KeyCtx(Ctx)); handle_validation_response({error, notfound}, RD, Ctx, _, Conv2KeyCtx, _) -> %% no keyid was found - _ = lager:debug("key_id not found"), + logger:notice("key_id not found"), deny_access(RD, Conv2KeyCtx(Ctx)); handle_validation_response({error, Reason}, RD, Ctx, _, Conv2KeyCtx, _) -> %% no matching keyid was found, or lookup failed - _ = lager:debug("Authentication error: ~p", [Reason]), + logger:notice("Authentication error: ~p", [Reason]), deny_invalid_key(RD, Conv2KeyCtx(Ctx)). handle_auth_admin(RD, Ctx, undefined, true) -> {false, RD, Ctx}; handle_auth_admin(RD, Ctx, undefined, false) -> %% anonymous access disallowed - riak_cs_wm_utils:deny_access(RD, Ctx); + deny_access(RD, Ctx); handle_auth_admin(RD, Ctx, User, false) -> UserKeyId = User?RCS_USER.key_id, case riak_cs_config:admin_creds() of {ok, {Admin, _}} when Admin == UserKeyId -> - %% admin account is allowed - riak_cs_dtrace:dt_wm_return(?MODULE, <<"forbidden">>, - [], [<<"false">>, Admin]), - {false, RD, Ctx}; + logger:notice("admin access is allowed"), + {false, RD, Ctx#rcs_web_context{admin_access = true}}; _ -> %% non-admin account is not allowed -> 403 - Res = riak_cs_wm_utils:deny_access(RD, Ctx), - riak_cs_dtrace:dt_wm_return(?MODULE, <<"forbidden">>, [], [<<"true">>]), - Res + deny_access(RD, Ctx) end. %% @doc Look for an Authorization header in the request, and validate %% it if it exists. Returns `{ok, User, UserObj}' if validation %% succeeds, or `{error, KeyId, Reason}' if any step fails. --spec validate_auth_header(#wm_reqdata{}, term(), riak_client(), #context{}|undefined) -> - {ok, rcs_user(), riakc_obj:riakc_obj()} | - {error, bad_auth | notfound | no_user_key | term()}. validate_auth_header(RD, AuthBypass, RcPid, Ctx) -> AuthHeader = wrq:get_req_header("authorization", RD), - case AuthHeader of - undefined -> - %% Check for auth info presented as query params - KeyId0 = wrq:get_qs_value(?QS_KEYID, RD), - EncodedSig = wrq:get_qs_value(?QS_SIGNATURE, RD), - {AuthMod, KeyId, Signature} = parse_auth_params(KeyId0, - EncodedSig, - AuthBypass); - _ -> - {AuthMod, KeyId, Signature} = parse_auth_header(AuthHeader, AuthBypass) - end, + {AuthMod, KeyId, Signature} = + case AuthHeader of + undefined -> + %% Check for auth info presented as query params + KeyId0 = wrq:get_qs_value(?QS_KEYID, RD), + EncodedSig = wrq:get_qs_value(?QS_SIGNATURE, RD), + parse_auth_params(KeyId0, + EncodedSig, + AuthBypass); + _ -> + parse_auth_header(AuthHeader, AuthBypass) + end, case riak_cs_user:get_user(KeyId, RcPid) of {ok, {User, UserObj}} when User?RCS_USER.status =:= enabled -> case AuthMod:authenticate(User, Signature, RD, Ctx) of @@ -258,11 +297,104 @@ validate_auth_header(RD, AuthBypass, RcPid, Ctx) -> {error, NE}; {error, Reason} -> %% other failures, like Riak fetch timeout, be loud about - _ = lager:error("Retrieval of user record for ~p failed. Reason: ~p", - [KeyId, Reason]), + logger:error("Retrieval of user record for ~p failed. Reason: ~p", [KeyId, Reason]), {error, Reason} end. +forbidden(RD, #rcs_web_context{auth_module = AuthMod, + riak_client = RcPid} = Ctx, + Target) -> + {AuthResult, IsAdmin} = + case AuthMod:identify(RD, Ctx) of + failed -> + {{error, no_such_key}, false}; + {failed, Reason} -> + {{error, Reason}, false}; + {UserKey, AuthData} -> + case riak_cs_user:get_user(UserKey, RcPid) of + {ok, {?RCS_USER{status = enabled, + key_id = UserKey} = User, _}} = OkayedUserTuple -> + case AuthMod:authenticate(User, AuthData, RD, Ctx) of + ok -> + {ok, {AdminKey, _}} = riak_cs_config:admin_creds(), + {OkayedUserTuple, (AdminKey == UserKey)}; + NotOk -> + {NotOk, false} + end; + {ok, {?RCS_USER{status = disabled}, _}} -> + {{error, user_disabled}, false}; + {error, _} = Error -> + {Error, false} + end + end, + post_authentication( + AuthResult, RD, Ctx#rcs_web_context{admin_access = IsAdmin}, Target). + +post_authentication({ok, {User, UserObj}}, RD, Ctx, Target) -> + %% given keyid and signature matched, proceed + case {riak_cs_config:admin_creds(), + riak_cs_config:anonymous_user_creation(), + User} of + {{ok, {AdminKeyId, _}}, + true, + ?RCS_USER{key_id = AdminKeyId}} -> + logger:notice("Allowing admin user to execute this call ignoring policy checks", []), + logger:notice("Set `anonymous_user_creation` to `off` (after creating policies as appropriate) to squelch this message", []), + {false, RD, Ctx}; + _ -> + role_access_authorize_helper( + Target, RD, Ctx#rcs_web_context{user = User, + user_object = UserObj}) + end; +post_authentication({error, no_user_key}, RD, Ctx, Target) -> + %% no keyid was given, proceed anonymously + ?LOG_DEBUG("No user key"), + role_access_authorize_helper( + Target, RD, Ctx); +post_authentication({error, notfound}, RD, #rcs_web_context{response_module = ResponseMod} = Ctx, _) -> + ?LOG_DEBUG("User does not exist"), + ResponseMod:api_error(invalid_access_key_id, RD, Ctx); +post_authentication({error, Reason}, RD, + #rcs_web_context{response_module = ResponseMod} = Ctx, _) -> + %% Lookup failed, basically due to disconnected stuff + ?LOG_DEBUG("Authentication error: ~p", [Reason]), + ResponseMod:api_error(Reason, RD, Ctx). + +role_access_authorize_helper(Target, RD, + Ctx = #rcs_web_context{policy_module = PolicyMod, + user = User}) -> + Access = PolicyMod:reqdata_to_access(RD, Target, + User?RCS_USER.id), + case get_user_policies_or_halt(Ctx) of + user_session_expired -> + deny_access(RD, Ctx); + {UserPolicies, PermissionsBoundary} -> + PolicyVerdict = + lists:foldl( + fun(_, true) -> + true; + (P, _) -> + PolicyMod:eval(Access, P) + end, + undefined, + UserPolicies), + PermBoundaryVerdict = + case PermissionsBoundary of + ?POLICY{} -> + PolicyMod:eval(Access, PermissionsBoundary); + [] -> + undefined + end, + case PolicyVerdict == true andalso PermBoundaryVerdict /= false of + true -> + {false, RD, Ctx}; + false -> + deny_access(RD, Ctx) + end + end. + + + %% @doc Utility function for building #key_contest %% Spawns manifest FSM -spec ensure_doc(term(), riak_client()) -> term(). @@ -277,30 +409,30 @@ ensure_doc(KeyCtx=#key_context{bucket_object=undefined, ensure_doc(KeyCtx, _) -> KeyCtx. -setup_manifest(KeyCtx=#key_context{bucket=Bucket, - key=Key}, RcPid) -> +setup_manifest(KeyCtx = #key_context{bucket = Bucket, + key = Key, + obj_vsn = Vsn}, RcPid) -> %% start the get_fsm - BinKey = list_to_binary(Key), FetchConcurrency = riak_cs_lfs_utils:fetch_concurrency(), BufferFactor = riak_cs_lfs_utils:get_fsm_buffer_size_factor(), - {ok, FsmPid} = riak_cs_get_fsm_sup:start_get_fsm(node(), Bucket, BinKey, + {ok, FsmPid} = riak_cs_get_fsm_sup:start_get_fsm(node(), Bucket, Key, Vsn, self(), RcPid, FetchConcurrency, BufferFactor), Manifest = riak_cs_get_fsm:get_manifest(FsmPid), - KeyCtx#key_context{get_fsm_pid=FsmPid, - manifest=Manifest}. + KeyCtx#key_context{get_fsm_pid = FsmPid, + manifest = Manifest}. %% @doc Produce an api error by using response_module. respond_api_error(RD, Ctx, ErrorAtom) -> - ResponseMod = Ctx#context.response_module, + ResponseMod = Ctx#rcs_web_context.response_module, NewRD = maybe_log_user(RD, Ctx), ResponseMod:api_error(ErrorAtom, NewRD, Ctx). %% @doc Only set the user for the access logger to catch if there is a %% user to catch. maybe_log_user(RD, Context) -> - case Context#context.user of + case Context#rcs_web_context.user of undefined -> RD; User -> @@ -309,46 +441,42 @@ maybe_log_user(RD, Context) -> %% @doc Produce an access-denied error message from a webmachine %% resource's `forbidden/2' function. -deny_access(RD, Ctx=#context{response_module=ResponseMod}) -> +deny_access(RD, Ctx = #rcs_web_context{response_module = ResponseMod}) -> ResponseMod:api_error(access_denied, RD, Ctx); deny_access(RD, Ctx) -> - riak_cs_s3_response:api_error(access_denied, RD, Ctx). + riak_cs_aws_response:api_error(access_denied, RD, Ctx). -%% @doc Prodice an invalid-access-keyid error message from a +%% @doc Produce an invalid-access-keyid error message from a %% webmachine resource's `forbidden/2' function. -deny_invalid_key(RD, Ctx=#context{response_module=ResponseMod}) -> +deny_invalid_key(RD, Ctx = #rcs_web_context{response_module = ResponseMod}) -> ResponseMod:api_error(invalid_access_key_id, RD, Ctx). %% @doc In the case is a user is authorized to perform an operation on %% a bucket but is not the owner of that bucket this function can be used %% to switch to the owner's record if it can be retrieved --spec shift_to_owner(#wm_reqdata{}, #context{}, string(), riak_client()) -> - {boolean(), #wm_reqdata{}, #context{}}. -shift_to_owner(RD, Ctx=#context{response_module=ResponseMod}, OwnerId, RcPid) +shift_to_owner(RD, Ctx = #rcs_web_context{response_module = ResponseMod}, OwnerId, RcPid) when RcPid /= undefined -> case riak_cs_user:get_user(OwnerId, RcPid) of {ok, {Owner, OwnerObject}} when Owner?RCS_USER.status =:= enabled -> AccessRD = riak_cs_access_log_handler:set_user(Owner, RD), - {false, AccessRD, Ctx#context{user=Owner, - user_object=OwnerObject}}; + {false, AccessRD, Ctx#rcs_web_context{user = Owner, + user_object = OwnerObject}}; {ok, _} -> riak_cs_wm_utils:deny_access(RD, Ctx); {error, _} -> ResponseMod:api_error(bucket_owner_unavailable, RD, Ctx) end. -streaming_get(RcPool, RcPid, FsmPid, StartTime, UserName, BFile_str) -> +streaming_get(RcPool, RcPid, FsmPid, StartTime) -> case riak_cs_get_fsm:get_next_chunk(FsmPid) of {done, Chunk} -> ok = riak_cs_stats:update_with_start([object, get], StartTime), riak_cs_riak_client:checkin(RcPool, RcPid), - riak_cs_dtrace:dt_object_return(riak_cs_wm_object, <<"object_get">>, - [], [UserName, BFile_str]), {Chunk, done}; {chunk, Chunk} -> - {Chunk, fun() -> streaming_get(RcPool, RcPid, FsmPid, StartTime, UserName, BFile_str) end} + {Chunk, fun() -> streaming_get(RcPool, RcPid, FsmPid, StartTime) end} end. -spec lower_case_method(atom()) -> atom(). @@ -423,22 +551,48 @@ iso_8601_to_erl_date(Date) -> %% @doc Return a new context where the bucket and key for the s3 %% object have been inserted. It also does key length check. TODO: do %% we check if the key is valid Unicode string or not? --spec extract_key(#wm_reqdata{}, #context{}) -> - {ok, #context{}} | {error, {key_too_long, pos_integer()}}. -extract_key(RD,Ctx=#context{local_context=LocalCtx0}) -> +-spec extract_key(#wm_reqdata{}, #rcs_web_context{}) -> + {ok, #rcs_web_context{}} | + {error, {key_too_long, pos_integer()} | {vsn_too_long, pos_integer()}}. +extract_key(RD, Ctx = #rcs_web_context{local_context = LocalCtx0}) -> Bucket = list_to_binary(wrq:path_info(bucket, RD)), %% need to unquote twice since we re-urlencode the string during rewrite in %% order to trick webmachine dispatching MaxKeyLen = riak_cs_config:max_key_length(), case mochiweb_util:unquote(mochiweb_util:unquote(wrq:path_info(object, RD))) of Key when length(Key) =< MaxKeyLen -> - LocalCtx = LocalCtx0#key_context{bucket=Bucket, key=Key}, - {ok, Ctx#context{bucket=Bucket, - local_context=LocalCtx}}; + LocalCtx = LocalCtx0#key_context{bucket = Bucket, key = list_to_binary(Key)}, + extract_version_id(RD, Ctx#rcs_web_context{bucket = Bucket, + local_context = LocalCtx}); Key -> {error, {key_too_long, length(Key)}} end. +extract_version_id(RD, Ctx = #rcs_web_context{local_context = LocalCtx0}) -> + VsnId = + case {wrq:path_info(versionId, RD), rcs_version_id_from_headers(RD)} of + {undefined, undefined} -> + ?LFS_DEFAULT_OBJECT_VERSION; + {"null", Defined} when is_list(Defined) -> + %% emulating a versionId resource, if given, as a Riak + %% CS extension, for PutObject should probably be + %% better done in s3_rewrite, but doing so will be too + %% disruptive for the tidy rewriting flow + list_to_binary(Defined); + {V, _} -> + list_to_binary(mochiweb_util:unquote(mochiweb_util:unquote(V))) + end, + case size(VsnId) =< riak_cs_config:max_key_length() of + true -> + LocalCtx = LocalCtx0#key_context{obj_vsn = VsnId}, + {ok, Ctx#rcs_web_context{local_context = LocalCtx}}; + _ -> + {error, {key_too_long, size(VsnId)}} + end. + +rcs_version_id_from_headers(RD) -> + wrq:get_req_header("x-rcs-versionid", RD). + extract_name(User) when is_list(User) -> User; extract_name(?RCS_USER{name=Name}) -> @@ -449,37 +603,35 @@ extract_name(_) -> %% @doc Add an ACL to the context, from parsing the headers. If there is %% an error parsing the header, halt the request. If there is no ACL %% information in the headers, use the default ACL. --spec maybe_update_context_with_acl_from_headers(#wm_reqdata{}, #context{}) -> - {error, {{halt, term()}, #wm_reqdata{}, #context{}}} | - {ok, #context{}}. +-spec maybe_update_context_with_acl_from_headers(#wm_reqdata{}, #rcs_web_context{}) -> + {error, {{halt, term()}, #wm_reqdata{}, #rcs_web_context{}}} | + {ok, #rcs_web_context{}}. maybe_update_context_with_acl_from_headers(RD, - Ctx=#context{user=User, - bucket=BucketName, - local_context=LocalCtx, - riak_client=RcPid}) -> + Ctx=#rcs_web_context{user = User, + bucket = BucketName, + local_context = LocalCtx, + riak_client = RcPid}) -> case bucket_obj_from_local_context(LocalCtx, BucketName, RcPid) of {ok, BucketObject} -> case maybe_acl_from_context_and_request(RD, Ctx, BucketObject) of {ok, {error, BadAclReason}} -> - {error, riak_cs_s3_response:api_error(BadAclReason, RD, Ctx)}; + {error, riak_cs_aws_response:api_error(BadAclReason, RD, Ctx)}; %% pattern match on the ACL record type for a data-type %% sanity-check - {ok, {ok, Acl=?ACL{}}} -> - {ok, Ctx#context{acl=Acl}}; + {ok, {ok, Acl}} -> + {ok, Ctx#rcs_web_context{acl = Acl}}; error -> DefaultAcl = riak_cs_acl_utils:default_acl(User?RCS_USER.display_name, - User?RCS_USER.canonical_id, + User?RCS_USER.id, User?RCS_USER.key_id), - {ok, Ctx#context{acl=DefaultAcl}} + {ok, Ctx#rcs_web_context{acl = DefaultAcl}} end; {error, Reason} -> - _ = lager:error("Failed to retrieve bucket objects for reason ~p", [Reason]), + logger:error("Failed to retrieve bucket objects for reason ~p", [Reason]), {error, {{halt, 500}, RD, Ctx}} end. --spec bucket_obj_from_local_context(term(), binary(), riak_client()) -> - {ok, term()} | {'error', term()}. -bucket_obj_from_local_context(#key_context{bucket_object=BucketObject}, +bucket_obj_from_local_context(#key_context{bucket_object = BucketObject}, _BucketName, _RcPid) -> {ok, BucketObject}; bucket_obj_from_local_context(undefined, BucketName, RcPid) -> @@ -496,19 +648,16 @@ bucket_obj_from_local_context(undefined, BucketName, RcPid) -> %% are no ACL headers, return `error'. In this case, it's not unexpected %% to get the `error' value back, but it's name is used for convention. %% It could also reasonable be called `nothing'. --spec maybe_acl_from_context_and_request(#wm_reqdata{}, #context{}, - riakc_obj:riakc_obj()) -> - {ok, acl_or_error()} | error. -maybe_acl_from_context_and_request(RD, #context{user=User, - riak_client=RcPid}, +maybe_acl_from_context_and_request(RD, #rcs_web_context{user = User, + riak_client = RcPid}, BucketObj) -> case has_acl_header(RD) of true -> Headers = normalize_headers(RD), BucketOwner = bucket_owner(BucketObj), - Owner = {User?RCS_USER.display_name, - User?RCS_USER.canonical_id, - User?RCS_USER.key_id}, + Owner = #{display_name => User?RCS_USER.display_name, + canonical_id => User?RCS_USER.id, + key_id => User?RCS_USER.key_id}, {ok, acl_from_headers(Headers, Owner, BucketOwner, RcPid)}; false -> error @@ -519,11 +668,6 @@ maybe_acl_from_context_and_request(RD, #context{user=User, %% @doc Create an acl from the request headers. At this point, we should %% have already verified that there is only a canned acl header or specific %% header grants. --spec acl_from_headers(Headers :: list(), - Owner :: acl_owner(), - BucketOwner :: undefined | acl_owner(), - riak_client()) -> - acl_or_error(). acl_from_headers(Headers, Owner, BucketOwner, RcPid) -> %% TODO: time to make a macro for `"x-amz-acl"' %% `Headers' is an ordset. Is there a faster way to retrieve this? Or @@ -533,7 +677,9 @@ acl_from_headers(Headers, Owner, BucketOwner, RcPid) -> RenamedHeaders = extract_acl_headers(Headers), case RenamedHeaders of [] -> - {DisplayName, CanonicalId, KeyID} = Owner, + #{display_name := DisplayName, + canonical_id := CanonicalId, + key_id := KeyID} = Owner, {ok, riak_cs_acl_utils:default_acl(DisplayName, CanonicalId, KeyID)}; _Else -> riak_cs_acl_utils:specific_acl_grant(Owner, RenamedHeaders, RcPid) @@ -625,7 +771,6 @@ extract_amazon_headers(Headers) -> %% @doc Extract user metadata from request header %% Expires, Content-Disposition, Content-Encoding, Cache-Control and x-amz-meta-* %% TODO: pass in x-amz-server-side-encryption? -%% TODO: pass in x-amz-storage-class? %% TODO: pass in x-amz-grant-* headers? -spec extract_user_metadata(#wm_reqdata{}) -> proplists:proplist(). extract_user_metadata(RD) -> @@ -656,6 +801,9 @@ extract_user_metadata([{Name, Value} | Headers], Acc) when is_list(Name) -> "x-amz-meta" ++ _ -> extract_user_metadata( Headers, [{LowerName, unicode:characters_to_list(Value, utf8)} | Acc]); + "x-amz-storage-class" -> + extract_user_metadata( + Headers, [{LowerName, unicode:characters_to_list(Value, utf8)} | Acc]); _ -> extract_user_metadata(Headers, Acc) end; @@ -663,108 +811,182 @@ extract_user_metadata([_ | Headers], Acc) -> extract_user_metadata(Headers, Acc). -spec bucket_access_authorize_helper(AccessType::atom(), boolean(), - RD::term(), Ctx::#context{}) -> term(). -bucket_access_authorize_helper(AccessType, Deletable, RD, Ctx) -> - #context{riak_client=RcPid, - policy_module=PolicyMod} = Ctx, + #wm_reqdata{}, #rcs_web_context{}) -> + {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #rcs_web_context{}}. +bucket_access_authorize_helper(AccessType, Deletable, RD, + #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + policy_module = PolicyMod} = Ctx) -> + Bucket = list_to_binary(wrq:path_info(bucket, RD)), + case get_user_policies_or_halt(Ctx) of + user_session_expired -> + deny_access(RD, Ctx); + {UserPolicies, PermissionsBoundary} -> + case riak_cs_bucket:get_bucket_acl_policy(Bucket, PolicyMod, RcPid) of + {ok, {Acl, BucketPolicy}} -> + Policies = lists:filter(fun(P) -> P /= undefined end, [BucketPolicy | UserPolicies]), + policies_to_verdict(AccessType, Deletable, Bucket, + Acl, Policies, PermissionsBoundary, + RD, Ctx); + {error, Reason} -> + ResponseMod:api_error(Reason, RD, Ctx) + end + end. + +policies_to_verdict(AccessType, Deletable, Bucket, + Acl, Policies, PermissionsBoundary, + RD, #rcs_web_context{user = User, + request_id = RequestId} = Ctx) -> Method = wrq:method(RD), RequestedAccess = riak_cs_acl_utils:requested_access(Method, is_acl_request(AccessType)), - Bucket = list_to_binary(wrq:path_info(bucket, RD)), - PermCtx = Ctx#context{bucket=Bucket, - requested_perm=RequestedAccess}, - handle_bucket_acl_policy_response( - riak_cs_bucket:get_bucket_acl_policy(Bucket, PolicyMod, RcPid), - AccessType, - Deletable, - RD, - PermCtx). - -handle_bucket_acl_policy_response({error, notfound}, _, _, RD, Ctx) -> - ResponseMod = Ctx#context.response_module, - ResponseMod:api_error(no_such_bucket, RD, Ctx); -handle_bucket_acl_policy_response({error, Reason}, _, _, RD, Ctx) -> - ResponseMod = Ctx#context.response_module, - ResponseMod:api_error(Reason, RD, Ctx); -handle_bucket_acl_policy_response({Acl, Policy}, AccessType, DeleteEligible, RD, Ctx) -> - #context{bucket=Bucket, - riak_client=RcPid, - user=User, - requested_perm=RequestedAccess} = Ctx, - AclCheckRes = riak_cs_acl_utils:check_grants(User, - Bucket, - RequestedAccess, - RcPid, - Acl), + PermCtx = + Ctx#rcs_web_context{bucket = Bucket, + requested_perm = RequestedAccess}, + {UserOwnsBucket, UserName} = + case User of + undefined -> + {false, "anonymous"}; + ?RCS_USER{name = Name, + buckets = Buckets} -> + {lists:any(fun(?RCS_BUCKET{name = N}) -> Bucket == N end, Buckets), + Name} + end, + ?LOG_DEBUG("Bucket: ~s, Policies ~p", [Bucket, Policies]), + + {PolicyVerdict, VerdictRD1, _} = + case Policies of + [] when UserOwnsBucket -> + logger:info("No bucket or user-attached policies: granting ~s access to ~s on user's own bucket \"~s\" on request ~s", + [AccessType, UserName, Bucket, RequestId]), + AccessRD = riak_cs_access_log_handler:set_user(User, RD), + {false, AccessRD, Ctx}; + _pp -> + lists:foldl( + fun(_, {{halt, _}, _, _} = Q) -> + Q; + (P, _) -> + handle_bucket_acl_policy_response( + Acl, P, AccessType, Deletable, RD, PermCtx) + end, + {undefined, RD, Ctx}, + Policies) + end, + ?LOG_DEBUG("PolicyVerdict: ~p", [PolicyVerdict]), + {PermBoundaryVerdict, VerdictRD2, _} = + case PermissionsBoundary of + [] -> + {undefined, VerdictRD1, PermCtx}; + _ -> + handle_bucket_acl_policy_response( + Acl, PermissionsBoundary, AccessType, Deletable, VerdictRD1, PermCtx) + end, + UltimateVerdict = ultimate_verdict(PolicyVerdict, PermBoundaryVerdict), + {UltimateVerdict, VerdictRD2, PermCtx}. + +%% 'false' means allow +ultimate_verdict(undefined, undefined) -> undefined; +ultimate_verdict(undefined, {halt, _} = Deny) -> Deny; +ultimate_verdict(undefined, false) -> undefined; +ultimate_verdict(false, undefined) -> false; +ultimate_verdict(false, {halt, _} = Deny) -> Deny; +ultimate_verdict(false, false) -> false; +ultimate_verdict({halt, _} = Deny, _) -> Deny. + + + + +get_user_policies_or_halt(#rcs_web_context{user_object = undefined, + user = undefined}) -> + {[], []}; +get_user_policies_or_halt(#rcs_web_context{user_object = undefined, + user = ?RCS_USER{key_id = KeyId}, + riak_client = RcPid}) -> + case riak_cs_temp_sessions:get(KeyId) of + {ok, S} -> + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + riak_cs_temp_sessions:effective_policies(S, Pbc); + {error, notfound} -> + %% there was a call to temp_sessions:get a fraction of a + %% second ago as part of webmachine's serving of this + %% request. Still, races happen. + logger:notice("Denying an API request from user with key_id \"~s\" as their session has expired", + [KeyId]), + user_session_expired + end; +get_user_policies_or_halt(#rcs_web_context{user_object = _NotFederatedUser, + user = ?RCS_USER{attached_policies = PP}, + riak_client = RcPid}) -> + {ok, Pbc} = riak_cs_riak_client:master_pbc(RcPid), + {riak_cs_iam:express_policies(PP, Pbc), []}. + +handle_bucket_acl_policy_response(Acl, Policy, AccessType, DeleteEligible, RD, Ctx) -> + #rcs_web_context{bucket = Bucket, + riak_client = RcPid, + user = User, + requested_perm = RequestedAccess} = Ctx, + AclCheckRes = riak_cs_acl_utils:check_grants( + User, Bucket, RequestedAccess, RcPid, Acl), + ?LOG_DEBUG("AclCheckRes: ~p", [AclCheckRes]), Deletable = DeleteEligible andalso (RequestedAccess =:= 'WRITE'), handle_acl_check_result(AclCheckRes, Acl, Policy, AccessType, Deletable, RD, Ctx). handle_acl_check_result(true, _, undefined, _, _, RD, Ctx) -> %% because users are not allowed to create/destroy - %% buckets, we can assume that User is not + %% buckets (why?), we can assume that User is not %% undefined here - AccessRD = riak_cs_access_log_handler:set_user(Ctx#context.user, RD), + AccessRD = riak_cs_access_log_handler:set_user(Ctx#rcs_web_context.user, RD), {false, AccessRD, Ctx}; -handle_acl_check_result(true, _, Policy, AccessType, _, RD, Ctx) -> - %% because users are not allowed to create/destroy - %% buckets, we can assume that User is not - %% undefined here - User = Ctx#context.user, - PolicyMod = Ctx#context.policy_module, +handle_acl_check_result(true, _, Policy, AccessType, _, + RD, Ctx = #rcs_web_context{policy_module = PolicyMod, + user = User}) -> AccessRD = riak_cs_access_log_handler:set_user(User, RD), Access = PolicyMod:reqdata_to_access(RD, AccessType, - User?RCS_USER.canonical_id), + User?RCS_USER.id), case PolicyMod:eval(Access, Policy) of - false -> riak_cs_wm_utils:deny_access(AccessRD, Ctx); - _ -> {false, AccessRD, Ctx} + false -> + deny_access(AccessRD, Ctx); + _ -> + {false, AccessRD, Ctx} end; -handle_acl_check_result({true, _OwnerId}, _, _, _, true, RD, Ctx) -> - %% grants lied: this is a delete, and only the owner is allowed to - %% do that; setting user for the request anyway, so the error - %% tally is logged for them - AccessRD = riak_cs_access_log_handler:set_user(Ctx#context.user, RD), - riak_cs_wm_utils:deny_access(AccessRD, Ctx); -handle_acl_check_result({true, OwnerId}, _, _, _, _, RD, Ctx) -> - %% this operation is allowed, but we need to get the owner's - %% record, and log the access against them instead of the actor - riak_cs_wm_utils:shift_to_owner(RD, Ctx, OwnerId, Ctx#context.riak_client); +handle_acl_check_result({true, _OwnerId}, Acl, Policy, AccessType, Deletable, + RD, Ctx = #rcs_web_context{user = User}) -> + ?LOG_DEBUG("Actor (~s) is not owner (~s); applying policy checks regardless", [User?IAM_USER.id, _OwnerId]), + handle_acl_check_result(true, Acl, Policy, AccessType, Deletable, RD, Ctx); + handle_acl_check_result(false, _, undefined, _, _Deletable, RD, Ctx) -> %% No policy so emulate a policy eval failure to avoid code duplication - handle_policy_eval_result(Ctx#context.user, false, undefined, RD, Ctx); + handle_policy_eval_result(false, undefined, RD, Ctx); handle_acl_check_result(false, Acl, Policy, AccessType, _Deletable, RD, Ctx) -> - #context{riak_client=RcPid, - user=User0} = Ctx, - PolicyMod = Ctx#context.policy_module, - User = case User0 of - undefined -> undefined; - _ -> User0?RCS_USER.canonical_id - end, - Access = PolicyMod:reqdata_to_access(RD, AccessType, User), + #rcs_web_context{riak_client = RcPid, + user = User} = Ctx, + UserId = safely_extract_canonical_id(User), + PolicyMod = Ctx#rcs_web_context.policy_module, + Access = PolicyMod:reqdata_to_access(RD, AccessType, UserId), PolicyResult = PolicyMod:eval(Access, Policy), OwnerId = riak_cs_acl:owner_id(Acl, RcPid), - handle_policy_eval_result(User, PolicyResult, OwnerId, RD, Ctx). + handle_policy_eval_result(PolicyResult, OwnerId, RD, Ctx). -handle_policy_eval_result(_, true, OwnerId, RD, Ctx) -> +handle_policy_eval_result(true, OwnerId, RD, Ctx) -> %% Policy says yes while ACL says no - shift_to_owner(RD, Ctx, OwnerId, Ctx#context.riak_client); -handle_policy_eval_result(User, _, _, RD, Ctx) -> + shift_to_owner(RD, Ctx, OwnerId, Ctx#rcs_web_context.riak_client); +handle_policy_eval_result(_, _, RD, Ctx) -> %% Policy says no - #context{riak_client=RcPid, - response_module=ResponseMod, - user=User, - bucket=Bucket} = Ctx, + #rcs_web_context{riak_client = RcPid, + response_module = ResponseMod, + user = User, + bucket = Bucket} = Ctx, %% log bad requests against the actors that make them AccessRD = riak_cs_access_log_handler:set_user(User, RD), %% Check if the bucket actually exists so we can %% make the correct decision to return a 404 or 403 case riak_cs_bucket:fetch_bucket_object(Bucket, RcPid) of {ok, _} -> - riak_cs_wm_utils:deny_access(AccessRD, Ctx); + deny_access(AccessRD, Ctx); {error, Reason} -> ResponseMod:api_error(Reason, RD, Ctx) end. --spec is_acl_request(atom()) -> boolean(). is_acl_request(ReqType) when ReqType =:= bucket_acl orelse ReqType =:= object_acl -> true; @@ -772,31 +994,38 @@ is_acl_request(_) -> false. -type halt_or_bool() :: {halt, pos_integer()} | boolean(). --type authorized_response() :: {halt_or_bool(), RD :: #wm_reqdata{}, Ctx :: #context{}}. +-type authorized_response() :: {halt_or_bool(), #wm_reqdata{}, #rcs_web_context{}}. -spec object_access_authorize_helper(AccessType::atom(), boolean(), - RD:: #wm_reqdata{}, Ctx:: #context{}) -> - authorized_response(). + #wm_reqdata{}, #rcs_web_context{}) -> + authorized_response(). +object_access_authorize_helper(_AccessType, _Deletable, RD, #rcs_web_context{admin_access = true, + user = User} = Ctx) -> + AccessRD = riak_cs_access_log_handler:set_user(User, RD), + {false, AccessRD, Ctx}; object_access_authorize_helper(AccessType, Deletable, RD, Ctx) -> object_access_authorize_helper(AccessType, Deletable, false, RD, Ctx). -spec object_access_authorize_helper(AccessType::atom(), boolean(), boolean(), - RD:: #wm_reqdata{}, Ctx:: #context{}) -> - authorized_response(). + #wm_reqdata{}, #rcs_web_context{}) -> + authorized_response(). object_access_authorize_helper(AccessType, Deletable, SkipAcl, - RD, #context{policy_module=PolicyMod, - local_context=LocalCtx, - user=User, - riak_client=RcPid, - response_module=ResponseMod}=Ctx) + RD, #rcs_web_context{policy_module = PolicyMod, + local_context = LocalCtx, + user = User, + riak_client = RcPid, + response_module = ResponseMod} = Ctx) when ( AccessType =:= object_acl orelse AccessType =:= object_part orelse AccessType =:= object ) andalso is_boolean(Deletable) andalso is_boolean(SkipAcl) -> - #key_context{bucket_object=BucketObj} = LocalCtx, + + #key_context{bucket_object = BucketObj, + manifest = Manifest} = LocalCtx, + case translate_bucket_policy(PolicyMod, BucketObj) of - {error, multiple_bucket_owners=E} -> + {error, multiple_bucket_owners = E} -> %% We want to bail out early if there are siblings when %% retrieving the bucket policy ResponseMod:api_error(E, RD, Ctx); @@ -804,89 +1033,117 @@ object_access_authorize_helper(AccessType, Deletable, SkipAcl, %% The call to `fetch_bucket_object' returned `notfound' %% so we can assume to bucket does not exist. ResponseMod:api_error(no_such_bucket, RD, Ctx); - Policy -> + BucketPolicy -> + ?LOG_DEBUG("BucketPolicy: ~p", [BucketPolicy]), Method = wrq:method(RD), - CanonicalId = extract_canonical_id(User), + CanonicalId = safely_extract_canonical_id(User), Access = PolicyMod:reqdata_to_access(RD, AccessType, CanonicalId), - #key_context{bucket=_Bucket, bucket_object=BucketObj, manifest=Manifest} = LocalCtx, ObjectAcl = extract_object_acl(Manifest), - case check_object_authorization(Access, SkipAcl, ObjectAcl, - Policy, CanonicalId, - PolicyMod, RcPid, BucketObj) of - {error, actor_is_owner_but_denied_policy} -> - %% return forbidden or 404 based on the `Method' and `Deletable' - %% values - actor_is_owner_but_denied_policy(User, RD, Ctx, Method, Deletable); - {ok, actor_is_owner_and_allowed_policy} -> - %% actor is the owner - %% Quota hook here - case riak_cs_quota:invoke_all_callbacks(User, Access, Ctx) of - {ok, RD2, Ctx2} -> - actor_is_owner_and_allowed_policy(User, RD2, Ctx2, LocalCtx); - {error, Module, Reason, RD3, Ctx3} -> - riak_cs_quota:handle_error(Module, Reason, RD3, Ctx3) - end; - {error, {actor_is_not_owner_and_denied_policy, OwnerId}} -> - actor_is_not_owner_and_denied_policy(OwnerId, RD, Ctx, - Method, Deletable); - {ok, {actor_is_not_owner_but_allowed_policy, OwnerId}} -> - %% actor is not the owner - %% Quota hook here - case riak_cs_quota:invoke_all_callbacks(OwnerId, Access, Ctx) of - {ok, RD2, Ctx2} -> - actor_is_not_owner_but_allowed_policy(User, OwnerId, RD2, Ctx2, LocalCtx); - {error, Module, Reason, RD3, Ctx3} -> - riak_cs_quota:handle_error(Module, Reason, RD3, Ctx3) - end; - {ok, just_allowed_by_policy} -> - %% actor is not the owner, not permitted by ACL but permitted by policy - %% Quota hook here - OwnerId = riak_cs_acl:owner_id(ObjectAcl, RcPid), - case riak_cs_quota:invoke_all_callbacks(OwnerId, Access, Ctx) of - {ok, RD2, Ctx2} -> - just_allowed_by_policy(OwnerId, RD2, Ctx2, LocalCtx); - {error, Module, Reason, RD3, Ctx3} -> - riak_cs_quota:handle_error(Module, Reason, RD3, Ctx3) - end; - {error, access_denied} -> - riak_cs_wm_utils:deny_access(RD, Ctx) + case get_user_policies_or_halt(Ctx) of + user_session_expired -> + deny_access(RD, Ctx); + {UserPolicies, PermissionsBoundary} -> + ApplicablePolicies = [P || P <- [BucketPolicy | UserPolicies], + P /= undefined], + ?LOG_DEBUG("ApplicablePolicies ~p", [ApplicablePolicies]), + case check_object_authorization( + Access, SkipAcl, ObjectAcl, + ApplicablePolicies, PermissionsBoundary, + BucketObj, RD, Ctx) of + {error, actor_is_owner_but_denied_policy} -> + %% return forbidden or 404 based on the `Method' and `Deletable' + %% values + actor_is_owner_but_denied_policy(User, RD, Ctx, Method, Deletable); + {ok, actor_is_owner_and_allowed_policy} -> + %% actor is the owner + %% Quota hook here + case riak_cs_quota:invoke_all_callbacks(User, Access, Ctx) of + {ok, RD2, Ctx2} -> + actor_is_owner_and_allowed_policy(User, RD2, Ctx2, LocalCtx); + {error, Module, Reason, RD3, Ctx3} -> + riak_cs_quota:handle_error(Module, Reason, RD3, Ctx3) + end; + {error, {actor_is_not_owner_and_denied_policy, OwnerId}} -> + actor_is_not_owner_and_denied_policy(OwnerId, RD, Ctx, + Method, Deletable); + {ok, {actor_is_not_owner_but_allowed_policy, OwnerId}} -> + %% actor is not the owner + %% Quota hook here + case riak_cs_quota:invoke_all_callbacks(OwnerId, Access, Ctx) of + {ok, RD2, Ctx2} -> + actor_is_not_owner_but_allowed_policy(User, OwnerId, RD2, Ctx2, LocalCtx); + {error, Module, Reason, RD3, Ctx3} -> + riak_cs_quota:handle_error(Module, Reason, RD3, Ctx3) + end; + {ok, just_allowed_by_policy} -> + %% actor is not the owner, not permitted by ACL but permitted by policy + %% Quota hook here + OwnerId = riak_cs_acl:owner_id(ObjectAcl, RcPid), + case riak_cs_quota:invoke_all_callbacks(OwnerId, Access, Ctx) of + {ok, RD2, Ctx2} -> + just_allowed_by_policy(OwnerId, RD2, Ctx2, LocalCtx); + {error, Module, Reason, RD3, Ctx3} -> + riak_cs_quota:handle_error(Module, Reason, RD3, Ctx3) + end; + {error, access_denied} -> + deny_access(RD, Ctx) + end end end. - --spec check_object_authorization(access(), boolean(), undefined|acl(), policy(), - undefined|string(), atom(), riak_client(), riakc_obj:riakc_obj()) -> - {ok, actor_is_owner_and_allowed_policy | - {actor_is_not_owner_but_allowed_policy, string()} | - just_allowed_by_policy} | - {error, actor_is_owner_but_denied_policy | - {actor_is_not_owner_and_denied_policy, string()} | - access_denied}. -check_object_authorization(Access, SkipAcl, ObjectAcl, Policy, - CanonicalId, PolicyMod, - RcPid, BucketObj) -> +safely_extract_canonical_id(?IAM_USER{id = A}) -> A; +safely_extract_canonical_id(undefined) -> undefined. + + +-spec check_object_authorization( + access(), boolean(), undefined | acl(), [policy()], policy(), + riakc_obj:riakc_obj(), #wm_reqdata{}, #rcs_web_context{}) -> + {ok, actor_is_owner_and_allowed_policy | + {actor_is_not_owner_but_allowed_policy, binary()} | + just_allowed_by_policy} | + {error, actor_is_owner_but_denied_policy | + {actor_is_not_owner_and_denied_policy, binary()} | + access_denied}. +check_object_authorization(Access, SkipAcl, ObjectAcl, + Policies, PermissionsBoundary, + BucketObj, + RD, Ctx = #rcs_web_context{user = User, + bucket = Bucket, + riak_client = RcPid, + request_id = RequestId}) -> + CanonicalId = safely_extract_canonical_id(User), #access_v1{method = Method, target = AccessType} = Access, RequestedAccess = requested_access_helper(AccessType, Method), - Acl = case SkipAcl of - true -> true; - false -> riak_cs_acl:object_access(BucketObj, - ObjectAcl, - RequestedAccess, - CanonicalId, - RcPid) - end, - case {Acl, PolicyMod:eval(Access, Policy)} of - {true, false} -> + ?LOG_DEBUG("ObjectAcl: ~p, BucketName: ~p, RequestedAccess: ~p, Policies (~b) ~p", + [ObjectAcl, RequestedAccess, riakc_obj:key(BucketObj), length(Policies), Policies]), + ObjectAccess = + case SkipAcl of + true -> true; + false -> riak_cs_acl:object_access( + BucketObj, ObjectAcl, RequestedAccess, CanonicalId, RcPid) + end, + {Verdict, _, _} = policies_to_verdict( + AccessType, _Deletable = true, Bucket, + ObjectAcl, Policies, PermissionsBoundary, + RD, Ctx), + ?LOG_DEBUG("User: ~p, ObjectAccess: ~p, Verdict: ~p", [CanonicalId, ObjectAccess, Verdict]), + case {ObjectAccess, Verdict} of + {true, {halt, _}} -> + logger:info("caller the owner, but denied by policy (request_id: ~s)", + [RequestId]), {error, actor_is_owner_but_denied_policy}; - {true, _} -> + {true, _p} -> + ?LOG_DEBUG("actor is owner and (maybe implicitly: ~p) allowed by policy", [_p]), {ok, actor_is_owner_and_allowed_policy}; - {{true, OwnerId}, false} -> + {{true, OwnerId}, {halt, _}} -> + ?LOG_DEBUG("actor_is_not_owner_and_denied_policy"), {error, {actor_is_not_owner_and_denied_policy, OwnerId}}; - {{true, OwnerId}, _} -> + {{true, OwnerId}, _p} -> + ?LOG_DEBUG("actor is not owner and no policy (~p): implicitly allow", [_p]), {ok, {actor_is_not_owner_but_allowed_policy, OwnerId}}; - {false, true} -> - %% actor is not the owner, not permitted by ACL but permitted by policy + {false, false} -> + ?LOG_DEBUG("caller is not the owner, not permitted by ACL but permitted by policy (request_id: ~s)", [RequestId]), {ok, just_allowed_by_policy}; {false, _} -> %% policy says undefined or false @@ -897,15 +1154,8 @@ check_object_authorization(Access, SkipAcl, ObjectAcl, Policy, %% =================================================================== %% object_acces_authorize_helper helper functions --spec extract_canonical_id(rcs_user() | undefined) -> - undefined | string(). -extract_canonical_id(undefined) -> - undefined; -extract_canonical_id(?RCS_USER{canonical_id=CanonicalID}) -> - CanonicalID. - -spec requested_access_helper(object | object_part | object_acl, atom()) -> - acl_perm(). + acl_perm(). requested_access_helper(object, Method) -> riak_cs_acl_utils:requested_access(Method, false); requested_access_helper(object_part, Method) -> @@ -914,15 +1164,14 @@ requested_access_helper(object_acl, Method) -> riak_cs_acl_utils:requested_access(Method, true). -spec extract_object_acl(notfound | lfs_manifest()) -> - undefined | acl(). + undefined | acl(). extract_object_acl(Manifest) -> riak_cs_manifest:object_acl(Manifest). -spec translate_bucket_policy(atom(), riakc_obj:riakc_obj()) -> - policy() | - undefined | - {error, multiple_bucket_owners} | - {error, notfound}. + policy() | undefined | + {error, multiple_bucket_owners} | + {error, notfound}. translate_bucket_policy(PolicyMod, BucketObj) -> case PolicyMod:bucket_policy(BucketObj) of {ok, P} -> @@ -950,66 +1199,44 @@ actor_is_owner_but_denied_policy(User, RD, Ctx, Method, Deletable) Method =:= 'POST' orelse (Deletable andalso Method =:= 'DELETE') -> AccessRD = riak_cs_access_log_handler:set_user(User, RD), - riak_cs_wm_utils:deny_access(AccessRD, Ctx); + deny_access(AccessRD, Ctx); actor_is_owner_but_denied_policy(User, RD, Ctx, Method, Deletable) when Method =:= 'GET' orelse (Deletable andalso Method =:= 'HEAD') -> {{halt, {404, "Not Found"}}, riak_cs_access_log_handler:set_user(User, RD), Ctx}. --spec actor_is_owner_and_allowed_policy(User :: rcs_user(), - RD :: term(), - Ctx :: term(), - LocalCtx :: term()) -> - authorized_response(). actor_is_owner_and_allowed_policy(undefined, RD, Ctx, _LocalCtx) -> {false, RD, Ctx}; actor_is_owner_and_allowed_policy(User, RD, Ctx, LocalCtx) -> AccessRD = riak_cs_access_log_handler:set_user(User, RD), UpdLocalCtx = LocalCtx#key_context{owner=User?RCS_USER.key_id}, - {false, AccessRD, Ctx#context{local_context=UpdLocalCtx}}. - --spec actor_is_not_owner_and_denied_policy(OwnerId :: string(), - RD :: term(), - Ctx :: term(), - Method :: atom(), - Deletable :: boolean()) -> - authorized_response(). + {false, AccessRD, Ctx#rcs_web_context{local_context = UpdLocalCtx}}. + actor_is_not_owner_and_denied_policy(OwnerId, RD, Ctx, Method, Deletable) when Method =:= 'PUT' orelse (Deletable andalso Method =:= 'DELETE') -> AccessRD = riak_cs_access_log_handler:set_user(OwnerId, RD), - riak_cs_wm_utils:deny_access(AccessRD, Ctx); + deny_access(AccessRD, Ctx); actor_is_not_owner_and_denied_policy(_OwnerId, RD, Ctx, Method, Deletable) when Method =:= 'GET' orelse (Deletable andalso Method =:= 'HEAD') -> {{halt, {404, "Not Found"}}, RD, Ctx}. --spec actor_is_not_owner_but_allowed_policy(User :: rcs_user(), - OwnerId :: string(), - RD :: term(), - Ctx :: term(), - LocalCtx :: term()) -> - authorized_response(). actor_is_not_owner_but_allowed_policy(undefined, OwnerId, RD, Ctx, LocalCtx) -> %% This is an anonymous request so shift to the context of the %% owner for the remainder of the request. AccessRD = riak_cs_access_log_handler:set_user(OwnerId, RD), - UpdCtx = Ctx#context{local_context=LocalCtx#key_context{owner=OwnerId}}, - shift_to_owner(AccessRD, UpdCtx, OwnerId, Ctx#context.riak_client); + UpdCtx = Ctx#rcs_web_context{local_context = LocalCtx#key_context{owner = OwnerId}}, + shift_to_owner(AccessRD, UpdCtx, OwnerId, Ctx#rcs_web_context.riak_client); actor_is_not_owner_but_allowed_policy(_, OwnerId, RD, Ctx, LocalCtx) -> AccessRD = riak_cs_access_log_handler:set_user(OwnerId, RD), - UpdCtx = Ctx#context{local_context=LocalCtx#key_context{owner=OwnerId}}, + UpdCtx = Ctx#rcs_web_context{local_context = LocalCtx#key_context{owner = OwnerId}}, {false, AccessRD, UpdCtx}. --spec just_allowed_by_policy(OwnerId :: string(), - RD :: term(), - Ctx :: term(), - LocalCtx :: term()) -> - authorized_response(). just_allowed_by_policy(OwnerId, RD, Ctx, LocalCtx) -> AccessRD = riak_cs_access_log_handler:set_user(OwnerId, RD), - UpdLocalCtx = LocalCtx#key_context{owner=OwnerId}, - {false, AccessRD, Ctx#context{local_context=UpdLocalCtx}}. + UpdLocalCtx = LocalCtx#key_context{owner = OwnerId}, + {false, AccessRD, Ctx#rcs_web_context{local_context = UpdLocalCtx}}. -spec fetch_bucket_owner(binary(), riak_client()) -> undefined | acl_owner(). fetch_bucket_owner(Bucket, RcPid) -> @@ -1017,7 +1244,7 @@ fetch_bucket_owner(Bucket, RcPid) -> {ok, Acl} -> Acl?ACL.owner; {error, Reason} -> - _ = lager:debug("Failed to retrieve owner info for bucket ~p. Reason ~p", [Bucket, Reason]), + logger:warning("Failed to retrieve owner info for bucket ~p: ~p", [Bucket, Reason]), undefined end. @@ -1074,8 +1301,8 @@ content_length(RD) -> %% single request entity size. %% On the other hand, for PUT Copy, Content-Length is not mandatory. %% If it exists, however, it should be ZERO. -valid_entity_length(MaxLen, RD, #context{response_module=ResponseMod, - local_context=LocalCtx} = Ctx) -> +valid_entity_length(MaxLen, RD, #rcs_web_context{response_module = ResponseMod, + local_context = LocalCtx} = Ctx) -> case {wrq:method(RD), wrq:get_req_header("x-amz-copy-source", RD)} of {'PUT', undefined} -> MaxLen = riak_cs_lfs_utils:max_content_len(), @@ -1083,7 +1310,7 @@ valid_entity_length(MaxLen, RD, #context{response_module=ResponseMod, Length when is_integer(Length) andalso Length =< MaxLen -> UpdLocalCtx = LocalCtx#key_context{size=Length}, - {true, RD, Ctx#context{local_context=UpdLocalCtx}}; + {true, RD, Ctx#rcs_web_context{local_context = UpdLocalCtx}}; Length when is_integer(Length) -> ResponseMod:api_error(entity_too_large, RD, Ctx); _ -> {false, RD, Ctx} @@ -1092,13 +1319,48 @@ valid_entity_length(MaxLen, RD, #context{response_module=ResponseMod, case riak_cs_wm_utils:content_length(RD) of CL when CL =:= 0 orelse CL =:= undefined -> UpdLocalCtx = LocalCtx#key_context{size=0}, - {true, RD, Ctx#context{local_context=UpdLocalCtx}}; + {true, RD, Ctx#rcs_web_context{local_context = UpdLocalCtx}}; _ -> {false, RD, Ctx} end; _ -> {true, RD, Ctx} end. + + +make_final_rd(Body, RD) -> + wrq:set_resp_body( + Body, wrq:set_resp_header( + "ETag", etag(Body), + wrq:set_resp_header( + "Content-Type", ?XML_TYPE, RD))). + +make_request_id() -> + list_to_binary(uuid:uuid_to_string(uuid:get_v4())). + +etag(Body) -> + riak_cs_utils:etag_from_binary(riak_cs_utils:md5(Body)). + + +cors_headers() -> + [ {"Access-Control-Allow-Origin", "*"} + , {"Access-Control-Allow-Credentials", "true"} + , {"Access-Control-Allow-Methods", "POST,PUT,GET,OPTIONS,DELETE"} + , {"Access-Control-Allow-Headers", + "host," + "origin," + "authorization," + "x-amz-content-sha256," + "x-amz-date," + "content-type," + "content-md5," + "accept," + "accept-encoding" + } + ]. + + + %% =================================================================== %% Internal functions %% =================================================================== @@ -1127,3 +1389,4 @@ iso_8601_format(Year, Month, Day, Hour, Min, Sec) -> b2i(Bin) -> list_to_integer(binary_to_list(Bin)). + diff --git a/apps/riak_cs/src/riak_cs_xml.erl b/apps/riak_cs/src/riak_cs_xml.erl new file mode 100644 index 000000000..88774e247 --- /dev/null +++ b/apps/riak_cs/src/riak_cs_xml.erl @@ -0,0 +1,982 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc A collection functions for going to or from XML to an erlang +%% record type. + +-module(riak_cs_xml). + +-include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). +-include_lib("xmerl/include/xmerl.hrl"). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +%% Public API +-export([scan/1, + to_xml/1, + find_elements/2 + ]). + +-define(XML_SCHEMA_INSTANCE, "http://www.w3.org/2001/XMLSchema-instance"). + + +%% =================================================================== +%% Public API +%% =================================================================== + + +-spec find_elements(atom(), [#xmlElement{}]) -> [#xmlElement{}]. +find_elements(Name, EE) -> + [E || E = #xmlElement{name = N} <- EE, N == Name]. + + +%% @doc parse XML and produce xmlElement (other comments and else are bad) +%% in R15B03 (and maybe later version), xmerl_scan:string/2 may return any +%% xml nodes, such as defined as xmlNode() above. It is unsafe because +%% `String' is the Body sent from client, which can be anything. +-spec scan(string()) -> {ok, #xmlElement{}} | {error, malformed_xml}. +scan(String) -> + case catch xmerl_scan:string(String) of + {'EXIT', _E} -> {error, malformed_xml}; + { #xmlElement{} = ParsedData, _Rest} -> {ok, ParsedData}; + _E -> {error, malformed_xml} + end. + +-spec to_xml(term()) -> binary(). +to_xml(undefined) -> + []; +to_xml(SimpleForm) when is_list(SimpleForm) -> + simple_form_to_xml(SimpleForm); +to_xml(?ACL{}=Acl) -> + acl_to_xml(Acl); +to_xml(?LBRESP{}=ListBucketsResp) -> + list_buckets_response_to_xml(ListBucketsResp); +to_xml(Resp) when is_record(Resp, list_objects_response); + is_record(Resp, list_object_versions_response) -> + SimpleForm = list_objects_response_to_simple_form(Resp), + to_xml(SimpleForm); +to_xml(?RCS_USER{} = User) -> + user_record_to_xml(User); +to_xml({users, Users}) -> + user_records_to_xml(Users); + +to_xml(?IAM_ROLE{} = Role) -> + role_record_to_xml(Role); +to_xml({roles, RR}) -> + role_records_to_xml(RR); +to_xml(?IAM_SAML_PROVIDER{} = P) -> + saml_provider_record_to_xml(P); +to_xml({saml_providers, PP}) -> + saml_provider_records_to_xml(PP); + +to_xml(#create_user_response{} = R) -> + create_user_response_to_xml(R); +to_xml(#get_user_response{} = R) -> + get_user_response_to_xml(R); +to_xml(#delete_user_response{} = R) -> + delete_user_response_to_xml(R); +to_xml(#list_users_response{} = R) -> + list_users_response_to_xml(R); + +to_xml(#create_role_response{} = R) -> + create_role_response_to_xml(R); +to_xml(#get_role_response{} = R) -> + get_role_response_to_xml(R); +to_xml(#delete_role_response{} = R) -> + delete_role_response_to_xml(R); +to_xml(#list_roles_response{} = R) -> + list_roles_response_to_xml(R); + +to_xml(#create_policy_response{} = R) -> + create_policy_response_to_xml(R); +to_xml(#get_policy_response{} = R) -> + get_policy_response_to_xml(R); +to_xml(#delete_policy_response{} = R) -> + delete_policy_response_to_xml(R); +to_xml(#list_policies_response{} = R) -> + list_policies_response_to_xml(R); + +to_xml(#create_saml_provider_response{} = R) -> + create_saml_provider_response_to_xml(R); +to_xml(#get_saml_provider_response{} = R) -> + get_saml_provider_response_to_xml(R); +to_xml(#delete_saml_provider_response{} = R) -> + delete_saml_provider_response_to_xml(R); +to_xml(#list_saml_providers_response{} = R) -> + list_saml_providers_response_to_xml(R); + +to_xml(#attach_role_policy_response{} = R) -> + attach_role_policy_response_to_xml(R); +to_xml(#detach_role_policy_response{} = R) -> + detach_role_policy_response_to_xml(R); +to_xml(#attach_user_policy_response{} = R) -> + attach_user_policy_response_to_xml(R); +to_xml(#detach_user_policy_response{} = R) -> + detach_user_policy_response_to_xml(R); +to_xml(#list_attached_user_policies_response{} = R) -> + list_attached_user_policies_response_to_xml(R); +to_xml(#list_attached_role_policies_response{} = R) -> + list_attached_role_policies_response_to_xml(R); + +to_xml(#assume_role_with_saml_response{} = R) -> + assume_role_with_saml_response_to_xml(R); + +to_xml(#list_temp_sessions_response{} = R) -> + list_temp_sessions_response_to_xml(R). + + + + + +%% =================================================================== +%% Internal functions +%% =================================================================== + +export_xml(XmlDoc) -> + export_xml(XmlDoc, [{prolog, ?XML_PROLOG}]). +export_xml(XmlDoc, Opts) -> + list_to_binary(xmerl:export_simple(XmlDoc, xmerl_xml, Opts)). + +%% @doc Convert simple form into XML. +simple_form_to_xml(Elements) -> + XmlDoc = format_elements(Elements), + export_xml(XmlDoc). + +format_elements(Elements) -> + [format_element(E) || E <- Elements]. + +format_element({Tag, Elements}) -> + {Tag, format_elements(Elements)}; +format_element({Tag, Attrs, Elements}) -> + {Tag, Attrs, format_elements(Elements)}; +format_element(Value) -> + format_value(Value). + +%% @doc Convert an internal representation of an ACL into XML. +-spec acl_to_xml(acl()) -> binary(). +acl_to_xml(?ACL{owner = Owner, grants = Grants}) -> + Content = [make_internal_node('Owner', owner_content(Owner)), + make_internal_node('AccessControlList', make_grants(Grants))], + XmlDoc = [make_internal_node('AccessControlPolicy', Content)], + export_xml(XmlDoc). + +owner_content(#{display_name := Name, + canonical_id := Id}) -> + [make_external_node('ID', Id), + make_external_node('DisplayName', Name)]. + +list_objects_response_to_simple_form(?LORESP{contents = Contents, + common_prefixes = CommonPrefixes, + name = Name, + prefix = Prefix, + marker = Marker, + next_marker = NextMarker, + max_keys = MaxKeys, + delimiter = Delimiter, + is_truncated = IsTruncated}) -> + KeyContents = [{'Contents', key_content_to_simple_form(objects, Content)} || + Content <- Contents], + CCPP = [{'CommonPrefixes', [{'Prefix', [CommonPrefix]}]} || + CommonPrefix <- CommonPrefixes], + Body = [{'Name', [Name]}, + {'Prefix', [Prefix]}, + {'Marker', [Marker]}] ++ + %% use a list-comprehension trick to only include + %% the `NextMarker' element if it's not `undefined' + [{'NextMarker', [M]} || + M <- [NextMarker], + M =/= undefined, + IsTruncated] ++ + [{'MaxKeys', [MaxKeys]}, + {'Delimiter', [Delimiter]}, + {'IsTruncated', [IsTruncated]}] ++ + KeyContents ++ CCPP, + [{'ListBucketResult', [{'xmlns', ?S3_XMLNS}], Body}]; + +list_objects_response_to_simple_form(?LOVRESP{contents = Contents, + common_prefixes = CommonPrefixes, + name = Name, + prefix = Prefix, + key_marker = KeyMarker, + version_id_marker = VersionIdMarker, + next_key_marker = NextKeyMarker, + next_version_id_marker = NextVersionIdMarker, + max_keys = MaxKeys, + delimiter = Delimiter, + is_truncated = IsTruncated}) -> + KeyContents = [{'Version', key_content_to_simple_form(versions, Content)} || + Content <- Contents], + CommonPrefixes = [{'CommonPrefixes', [{'Prefix', [CommonPrefix]}]} || + CommonPrefix <- CommonPrefixes], + Body = [{'Name', [Name]}, + {'Prefix', [Prefix]}, + {'KeyMarker', [KeyMarker]}, + {'VersionIdMarker', [VersionIdMarker]}] ++ + [{'NextKeyMarker', [M]} || + M <- [NextKeyMarker], + M =/= undefined, + IsTruncated] ++ + [{'NextVersionIdMarker', [M]} || + M <- [NextVersionIdMarker], + M =/= undefined, + IsTruncated] ++ + [{'MaxKeys', [MaxKeys]}, + {'Delimiter', [Delimiter]}, + {'IsTruncated', [IsTruncated]}] ++ + KeyContents ++ CommonPrefixes, + [{'ListBucketResult', [{'xmlns', ?S3_XMLNS}], Body}]. + +key_content_to_simple_form(objects, KeyContent) -> + #list_objects_owner{id=Id, display_name=Name} = KeyContent?LOKC.owner, + [{'Key', [KeyContent?LOKC.key]}, + {'LastModified', [KeyContent?LOKC.last_modified]}, + {'ETag', [KeyContent?LOKC.etag]}, + {'Size', [KeyContent?LOKC.size]}, + {'StorageClass', [KeyContent?LOKC.storage_class]}, + {'Owner', [{'ID', [Id]}, + {'DisplayName', [Name]}]}]; + +key_content_to_simple_form(versions, KeyContent) -> + #list_objects_owner{id=Id, display_name=Name} = KeyContent?LOVKC.owner, + [{'Key', [KeyContent?LOVKC.key]}, + {'LastModified', [KeyContent?LOVKC.last_modified]}, + {'ETag', [KeyContent?LOVKC.etag]}, + {'Size', [KeyContent?LOVKC.size]}, + {'StorageClass', [KeyContent?LOVKC.storage_class]}, + {'IsLatest', [KeyContent?LOVKC.is_latest]}, + {'VersionId', [KeyContent?LOVKC.version_id]}, + {'Owner', [{'ID', [Id]}, + {'DisplayName', [Name]}]}]. + +list_buckets_response_to_xml(Resp) -> + BucketsContent = + make_internal_node('Buckets', + [bucket_to_xml(B?RCS_BUCKET.name, + B?RCS_BUCKET.creation_date) || + B <- Resp?LBRESP.buckets]), + UserContent = user_to_xml_owner(Resp?LBRESP.user), + Contents = [UserContent] ++ [BucketsContent], + export_xml([make_internal_node('ListAllMyBucketsResult', + [{'xmlns', ?S3_XMLNS}], + Contents)]). + +bucket_to_xml(Name, CreationDate) when is_binary(Name) -> + bucket_to_xml(binary_to_list(Name), CreationDate); +bucket_to_xml(Name, CreationDate) -> + make_internal_node('Bucket', + [make_external_node('Name', Name), + make_external_node('CreationDate', rts:iso8601_s(CreationDate))]). + +user_to_xml_owner(?RCS_USER{id = CanonicalId, + display_name = Name}) -> + make_internal_node('Owner', [make_external_node('ID', [CanonicalId]), + make_external_node('DisplayName', [Name])]). + +%% @doc Assemble the xml for the set of grantees for an acl. +make_grants(Grants) -> + make_grants(Grants, []). + +%% @doc Assemble the xml for the set of grantees for an acl. +make_grants([], Acc) -> + lists:flatten(Acc); +make_grants([?ACL_GRANT{grantee = #{display_name := GranteeName, + canonical_id := GranteeId}, + perms = Perms} | RestGrants], Acc) -> + Grant = [make_grant(GranteeName, GranteeId, Perm) || Perm <- Perms], + make_grants(RestGrants, [Grant | Acc]); +make_grants([?ACL_GRANT{grantee = Group, + perms = Perms} | RestGrants], Acc) -> + Grant = [make_grant(Group, Perm) || Perm <- Perms], + make_grants(RestGrants, [Grant | Acc]). + +%% @doc Assemble the xml for a group grantee for an acl. +make_grant(Group, Permission) -> + Attributes = [{'xmlns:xsi', ?XML_SCHEMA_INSTANCE}, + {'xsi:type', "Group"}], + GranteeContent = [make_external_node('URI', uri_for_group(Group))], + GrantContent = + [make_internal_node('Grantee', Attributes, GranteeContent), + make_external_node('Permission', Permission)], + make_internal_node('Grant', GrantContent). + +%% @doc Assemble the xml for a single grantee for an acl. +make_grant(DisplayName, CanonicalId, Permission) -> + Attributes = [{'xmlns:xsi', ?XML_SCHEMA_INSTANCE}, + {'xsi:type', "CanonicalUser"}], + GranteeContent = [make_external_node('ID', CanonicalId), + make_external_node('DisplayName', DisplayName)], + GrantContent = + [make_internal_node('Grantee', Attributes, GranteeContent), + make_external_node('Permission', Permission)], + make_internal_node('Grant', GrantContent). + +%% @doc Map a ACL group atom to its corresponding URI. +uri_for_group('AllUsers') -> + ?ALL_USERS_GROUP; +uri_for_group('AuthUsers') -> + ?AUTH_USERS_GROUP. + +%% @doc Convert a Riak CS user record to XML +user_record_to_xml(User) -> + export_xml([user_node(User)]). + +%% @doc Convert a set of Riak CS user records to XML +user_records_to_xml(Users) -> + UserNodes = [user_node(User) || User <- Users], + export_xml([make_internal_node('Users', UserNodes)]). + +user_node(?RCS_USER{name = Name, + email = Email, + display_name = DisplayName, + key_id = KeyID, + key_secret = KeySecret, + id = CanonicalID, + status = Status}) -> + StatusStr = case Status of + enabled -> + "enabled"; + _ -> + "disabled" + end, + Content = [make_external_node(K, V) + || {K, V} <- [{'Email', Email}, + {'DisplayName', DisplayName}, + {'Path', Name}, + {'Name', Name}, + {'KeyId', KeyID}, + {'KeySecret', KeySecret}, + {'Id', CanonicalID}, + {'Status', StatusStr}]], + make_internal_node('User', Content). + + +%% we stick to IAM specs with this one, in contrast to user_node which +%% serves CS-specific, legacy get_user call over plain http. +iam_user_node(?IAM_USER{arn = Arn, + name = UserName, + create_date = CreateDate, + path = Path, + id = UserId, + password_last_used = PasswordLastUsed, + permissions_boundary = PermissionsBoundary, + tags = Tags}) -> + C = lists:flatten( + [{'Arn', [make_arn(Arn)]}, + {'Path', [binary_to_list(Path)]}, + {'CreateDate', [rts:iso8601_s(CreateDate)]}, + {'UserName', [binary_to_list(UserName)]}, + {'UserId', [binary_to_list(UserId)]}, + [{'PasswordLastUsed', [rts:iso8601_s(PasswordLastUsed)]} + || PasswordLastUsed /= undefined, + PasswordLastUsed /= null], + [{'PermissionsBoundary', [{'xmlns:xsi', ?XML_SCHEMA_INSTANCE}, + {'xsi:type', "PermissionsBoundary"}], + make_permissions_boundary(PermissionsBoundary)} || PermissionsBoundary /= null, + PermissionsBoundary /= undefined], + {'Tags', [tag_node(T) || T <- Tags]} + ]), + {'User', C}. + +create_user_response_to_xml(#create_user_response{user = User, request_id = RequestId}) -> + CreateUserResult = iam_user_node(User), + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'CreateUserResult', [CreateUserResult]}, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('CreateUserResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + + +get_user_response_to_xml(#get_user_response{user = User, request_id = RequestId}) -> + GetUserResult = iam_user_node(User), + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'GetUserResult', [GetUserResult]}, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('GetUserResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + +delete_user_response_to_xml(#delete_user_response{request_id = RequestId}) -> + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('DeleteUserResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + +list_users_response_to_xml(#list_users_response{users = RR, + request_id = RequestId, + is_truncated = IsTruncated, + marker = Marker}) -> + ListUsersResult = + lists:flatten( + [{'Users', [iam_user_node(R) || R <- RR]}, + {'IsTruncated', [atom_to_list(IsTruncated)]}, + [{'Marker', Marker} || Marker /= undefined]]), + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'ListUsersResult', ListUsersResult}, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('ListUsersResponse', + [{'xmlns', ?IAM_XMLNS}], + lists:flatten(C))], []). + + + +role_record_to_xml(Role) -> + export_xml([role_node(Role)]). + +role_records_to_xml(Roles) -> + NN = [role_node(R) || R <- Roles], + export_xml([{'Roles', NN}]). + +role_node(?IAM_ROLE{arn = Arn, + assume_role_policy_document = AssumeRolePolicyDocument, + create_date = CreateDate, + description = Description, + max_session_duration = MaxSessionDuration, + path = Path, + permissions_boundary = PermissionsBoundary, + role_id = RoleId, + role_last_used = RoleLastUsed, + tags = Tags, + role_name = RoleName}) -> + C = lists:flatten( + [{'Arn', [make_arn(Arn)]}, + [{'AssumeRolePolicyDocument', [binary_to_list(AssumeRolePolicyDocument)]} || AssumeRolePolicyDocument /= undefined], + {'CreateDate', [rts:iso8601_s(CreateDate)]}, + [{'Description', [binary_to_list(Description)]} || Description /= undefined, + Description /= null], + [{'MaxSessionDuration', [integer_to_list(MaxSessionDuration)]} || MaxSessionDuration /= undefined, + MaxSessionDuration /= null], + {'Path', [binary_to_list(Path)]}, + [{'PermissionsBoundary', [{'xmlns:xsi', ?XML_SCHEMA_INSTANCE}, + {'xsi:type', "PermissionsBoundary"}], + make_permissions_boundary(PermissionsBoundary)} || PermissionsBoundary /= null, + PermissionsBoundary /= undefined], + {'RoleId', [binary_to_list(RoleId)]}, + [{'RoleLastUsed', [{'xmlns:xsi', ?XML_SCHEMA_INSTANCE}, + {'xsi:type', "RoleLastUsed"}], + make_role_last_used(RoleLastUsed)} || RoleLastUsed /= undefined], + {'RoleName', [binary_to_list(RoleName)]}, + {'Tags', [tag_node(T) || T <- Tags]} + ]), + {'Role', C}. + +make_arn(BareArn) when is_binary(BareArn) -> + binary_to_list(BareArn). +%% make_arn(?S3_ARN{provider = Provider, +%% service = Service, +%% region = Region, +%% id = Id, +%% path = Path}) -> +%% binary_to_list( +%% iolist_to_binary( +%% [atom_to_list(Provider), $:, atom_to_list(Service), $:, Region, $:, Id, $/, Path])). + +make_role_last_used(?IAM_ROLE_LAST_USED{last_used_date = LastUsedDate, + region = Region}) -> + [{'LastUsedDate', [rts:iso8601_s(LastUsedDate)]} || LastUsedDate =/= undefined ] + ++ [{'Region', [binary_to_list(Region)]} || Region =/= undefined]. + +make_permissions_boundary(BareArn) when is_binary(BareArn) -> + make_permissions_boundary(?IAM_PERMISSION_BOUNDARY{permissions_boundary_arn = BareArn}); +make_permissions_boundary(?IAM_PERMISSION_BOUNDARY{permissions_boundary_arn = PermissionsBoundaryArn, + permissions_boundary_type = PermissionsBoundaryType}) -> + [{'PermissionsBoundaryArn', [make_arn(PermissionsBoundaryArn)]}, + {'PermissionsBoundaryType', [binary_to_list(PermissionsBoundaryType)]} + ]. + + +create_role_response_to_xml(#create_role_response{role = Role, request_id = RequestId}) -> + CreateRoleResult = role_node(Role), + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'CreateRoleResult', [CreateRoleResult]}, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('CreateRoleResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + + +get_role_response_to_xml(#get_role_response{role = Role, request_id = RequestId}) -> + GetRoleResult = role_node(Role), + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'GetRoleResult', [GetRoleResult]}, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('GetRoleResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + +delete_role_response_to_xml(#delete_role_response{request_id = RequestId}) -> + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('DeleteRoleResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + +list_roles_response_to_xml(#list_roles_response{roles = RR, + request_id = RequestId, + is_truncated = IsTruncated, + marker = Marker}) -> + ListRolesResult = + lists:flatten( + [{'Roles', [role_node(R) || R <- RR]}, + {'IsTruncated', [atom_to_list(IsTruncated)]}, + [{'Marker', Marker} || Marker /= undefined]]), + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'ListRolesResult', ListRolesResult}, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('ListRolesResponse', + [{'xmlns', ?IAM_XMLNS}], + lists:flatten(C))], []). + + +policy_node(?IAM_POLICY{arn = Arn, + path = Path, + attachment_count = AttachmentCount, + create_date = CreateDate, + description = Description, + default_version_id = DefaultVersionId, + is_attachable = IsAttachable, + permissions_boundary_usage_count = PermissionsBoundaryUsageCount, + policy_document = PolicyDocument, + policy_id = PolicyId, + policy_name = PolicyName, + tags = Tags, + update_date = UpdateDate}) -> + C = lists:flatten( + [{'Arn', [make_arn(Arn)]}, + {'Path', [binary_to_list(Path)]}, + {'CreateDate', [rts:iso8601_s(CreateDate)]}, + [{'Description', [binary_to_list(Description)]} || Description /= undefined, + Description /= null], + {'DefaultVersionId', [binary_to_list(DefaultVersionId)]}, + {'AttachmentCount', [integer_to_list(AttachmentCount)]}, + {'IsAttachable', [atom_to_list(IsAttachable)]}, + {'PermissionsBoundaryUsageCount', [integer_to_list(PermissionsBoundaryUsageCount)]}, + {'PolicyDocument', [binary_to_list(PolicyDocument)]}, + {'PolicyId', [binary_to_list(PolicyId)]}, + {'PolicyName', [binary_to_list(PolicyName)]}, + {'Tags', [tag_node(T) || T <- Tags]}, + {'UpdateDate', [rts:iso8601_s(UpdateDate)]} + ]), + {'Policy', C}. + +create_policy_response_to_xml(#create_policy_response{policy = Policy, request_id = RequestId}) -> + CreatePolicyResult = policy_node(Policy), + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'CreatePolicyResult', [CreatePolicyResult]}, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('CreatePolicyResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + + +get_policy_response_to_xml(#get_policy_response{policy = Policy, request_id = RequestId}) -> + GetPolicyResult = policy_node(Policy), + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'GetPolicyResult', [GetPolicyResult]}, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('GetPolicyResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + +delete_policy_response_to_xml(#delete_policy_response{request_id = RequestId}) -> + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('DeletePolicyResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + +list_policies_response_to_xml(#list_policies_response{policies = RR, + request_id = RequestId, + is_truncated = IsTruncated, + marker = Marker}) -> + ListPoliciesResult = + lists:flatten( + [{'Policies', [policy_node(R) || R <- RR]}, + {'IsTruncated', [atom_to_list(IsTruncated)]}, + [{'Marker', Marker} || Marker /= undefined]]), + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'ListPoliciesResult', ListPoliciesResult}, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('ListPoliciesResponse', + [{'xmlns', ?IAM_XMLNS}], + lists:flatten(C))], []). + + + + +attach_role_policy_response_to_xml(#attach_role_policy_response{request_id = RequestId}) -> + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('AttachRolePolicyResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + +detach_role_policy_response_to_xml(#detach_role_policy_response{request_id = RequestId}) -> + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('DetachRolePolicyResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + +attach_user_policy_response_to_xml(#attach_user_policy_response{request_id = RequestId}) -> + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('AttachUserPolicyResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + +detach_user_policy_response_to_xml(#detach_user_policy_response{request_id = RequestId}) -> + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('DetachUserPolicyResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + + +policy_entry_for_laup_result(Arn, Name) -> + P = [{'PolicyArn', [make_arn(Arn)]}, + {'PolicyName', [binary_to_list(Name)]} + ], + {'member', P}. + +list_attached_user_policies_response_to_xml(#list_attached_user_policies_response{policies = RR, + request_id = RequestId, + is_truncated = IsTruncated, + marker = Marker}) -> + ListAttachedUserPoliciesResult = + lists:flatten( + [{'AttachedPolicies', [policy_entry_for_laup_result(A, N) || {A, N} <- RR]}, + {'IsTruncated', [atom_to_list(IsTruncated)]}, + [{'Marker', Marker} || Marker /= undefined]]), + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'ListAttachedUserPoliciesResult', ListAttachedUserPoliciesResult}, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('ListAttachedUserPoliciesResponse', + [{'xmlns', ?IAM_XMLNS}], + lists:flatten(C))], []). + + +list_attached_role_policies_response_to_xml(#list_attached_role_policies_response{policies = RR, + request_id = RequestId, + is_truncated = IsTruncated, + marker = Marker}) -> + ListAttachedRolePoliciesResult = + lists:flatten( + [{'AttachedPolicies', [policy_entry_for_laup_result(A, N) || {A, N} <- RR]}, + {'IsTruncated', [atom_to_list(IsTruncated)]}, + [{'Marker', Marker} || Marker /= undefined]]), + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'ListAttachedRolePoliciesResult', ListAttachedRolePoliciesResult}, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('ListAttachedRolePoliciesResponse', + [{'xmlns', ?IAM_XMLNS}], + lists:flatten(C))], []). + + +saml_provider_record_to_xml(P) -> + export_xml([saml_provider_node(P)]). + +saml_provider_records_to_xml(PP) -> + NN = [saml_provider_node(P) || P <- PP], + export_xml([make_internal_node('SAMLProviders', NN)]). + + +saml_provider_node(?IAM_SAML_PROVIDER{arn = Arn, + create_date = CreateDate, + saml_metadata_document = SAMLMetadataDocument, + tags = Tags, + valid_until = ValidUntil}) -> + C = [{'Arn', [binary_to_list(Arn)]}, + {'SAMLMetadataDocument', [binary_to_list(SAMLMetadataDocument)]}, + {'CreateDate', [rts:iso8601_s(CreateDate)]}, + {'ValidUntil', [rts:iso8601_s(ValidUntil)]}, + {'Tags', [tag_node(T) || T <- Tags]} + ], + {'SAMLProvider', C}. + +saml_provider_node_for_create(Arn, Tags) -> + C = [{'SAMLProviderArn', [binary_to_list(Arn)]}, + {'Tags', [], [tag_node(T) || T <- Tags]} + ], + {'CreateSAMLProviderResult', C}. + +saml_provider_node_for_get(CreateDate, ValidUntil, SAMLMetadataDocument, Tags) -> + C = [{'CreateDate', [rts:iso8601_s(CreateDate)]}, + {'ValidUntil', [rts:iso8601_s(ValidUntil)]}, + {'SAMLMetadataDocument', [binary_to_list(SAMLMetadataDocument)]}, + {'Tags', [], [tag_node(T) || T <- Tags]} + ], + {'GetSAMLProviderResult', C}. + +saml_provider_node_for_list(Arn, CreateDate, ValidUntil) -> + C = [{'Arn', [make_arn(Arn)]}, + {'CreateDate', [rts:iso8601_s(CreateDate)]}, + {'ValidUntil', [rts:iso8601_s(ValidUntil)]} + ], + {'SAMLProviderListEntry', C}. + +create_saml_provider_response_to_xml(#create_saml_provider_response{saml_provider_arn = BareArn, + tags = Tags, + request_id = RequestId}) -> + CreateSAMLProviderResult = saml_provider_node_for_create(BareArn, Tags), + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [CreateSAMLProviderResult, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('CreateSAMLProviderResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + +get_saml_provider_response_to_xml(#get_saml_provider_response{create_date = CreateDate, + valid_until = ValidUntil, + saml_metadata_document = SAMLMetadataDocument, + tags = Tags, + request_id = RequestId}) -> + GetSAMLProviderResult = saml_provider_node_for_get(CreateDate, ValidUntil, SAMLMetadataDocument, Tags), + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [GetSAMLProviderResult, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('GetSAMLProviderResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + +delete_saml_provider_response_to_xml(#delete_saml_provider_response{request_id = RequestId}) -> + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('DeleteSAMLProviderResponse', + [{'xmlns', ?IAM_XMLNS}], + C)], []). + +list_saml_providers_response_to_xml(#list_saml_providers_response{saml_provider_list = RR, + request_id = RequestId}) -> + ListSAMLProvidersResult = + [{'SAMLProviderList', [saml_provider_node_for_list(Arn, CreateDate, ValidUntil) + || #saml_provider_list_entry{arn = Arn, + create_date = CreateDate, + valid_until = ValidUntil} <- RR]}], + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'ListSAMLProvidersResult', ListSAMLProvidersResult}, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('ListSAMLProvidersResponse', + [{'xmlns', ?IAM_XMLNS}], + lists:flatten(C))], []). + +assume_role_with_saml_response_to_xml(#assume_role_with_saml_response{assumed_role_user = AssumedRoleUser, + audience = Audience, + credentials = Credentials, + issuer = Issuer, + name_qualifier = NameQualifier, + packed_policy_size = PackedPolicySize, + source_identity = SourceIdentity, + subject = Subject, + subject_type = SubjectType, + request_id = RequestId}) -> + AssumeRoleWithSAMLResult = + lists:flatten( + [{'AssumedRoleUser', make_assumed_role_user(AssumedRoleUser)}, + {'Audience', [binary_to_list(Audience)]}, + {'Credentials', make_credentials(Credentials)}, + {'Issuer', [binary_to_list(Issuer)]}, + {'NameQualifier', [binary_to_list(NameQualifier)]}, + {'PackedPolicySize', [integer_to_list(PackedPolicySize)]}, + [{'SourceIdentity', [binary_to_list(SourceIdentity)]} || SourceIdentity /= <<>>], + {'Subject', [binary_to_list(Subject)]}, + {'SubjectType', [binary_to_list(SubjectType)]} + ]), + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'AssumeRoleWithSAMLResult', AssumeRoleWithSAMLResult}, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('AssumeRoleWithSAMLResponse', + [{'xmlns', ?STS_XMLNS}], + C)], []). +make_assumed_role_user(#assumed_role_user{arn = Arn, + assumed_role_id = AssumedRoleId}) -> + [{'Arn', [binary_to_list(Arn)]}, + {'AssumedRoleId', [binary_to_list(AssumedRoleId)]}]. + +make_credentials(#credentials{access_key_id = AccessKeyId, + secret_access_key = SecretAccessKey, + session_token = SessionToken, + expiration = Expiration}) -> + [{'AccessKeyId', [binary_to_list(AccessKeyId)]}, + {'SecretAccessKey', [binary_to_list(SecretAccessKey)]}, + {'SessionToken', [binary_to_list(SessionToken)]}, + {'Expiration', [rts:iso8601_s(Expiration)]} + ]. + + +temp_session_node(#temp_session{ assumed_role_user = AssumedRoleUser + , role = Role + , credentials = Credentials + , duration_seconds = DurationSeconds + , created = Created + , inline_policy = InlinePolicy + , session_policies = SessionPolicies + , subject = Subject + , source_identity = SourceIdentity + , email = Email + , user_id = UserId + , canonical_id = CanonicalID + }) -> + C = lists:flatten( + [{'AssumedRoleUser', make_assumed_role_user(AssumedRoleUser)}, + role_node(Role), + {'Credentials', make_credentials(Credentials)}, + {'DurationSeconds', [integer_to_list(DurationSeconds)]}, + {'Created', [rts:iso8601_s(Created)]}, + [{'InlinePolicy', [binary_to_list(InlinePolicy) || InlinePolicy /= undefined]}], + {'SessionPolicies', [session_policy_node(A) || A <- SessionPolicies]}, + {'Subject', [binary_to_list(Subject)]}, + {'SourceIdentity', [binary_to_list(SourceIdentity)]}, + {'Email', [binary_to_list(Email)]}, + {'UserId', [binary_to_list(UserId)]}, + {'CanonicalID', [binary_to_list(CanonicalID)]} + ]), + {'TempSession', C}. + +session_policy_node(A) -> + {'SessionPolicy', [binary_to_list(A)]}. + + +list_temp_sessions_response_to_xml(#list_temp_sessions_response{temp_sessions = RR, + is_truncated = IsTruncated, + marker = Marker, + request_id = RequestId}) -> + ListTempSessionsResult = + lists:flatten( + [{'TempSessions', [temp_session_node(R) || R <- RR]}, + {'IsTruncated', [atom_to_list(IsTruncated)]}, + [{'Marker', Marker} || Marker /= undefined]]), + ResponseMetadata = make_internal_node('RequestId', [binary_to_list(RequestId)]), + C = [{'ListTempSessionsResult', ListTempSessionsResult}, + {'ResponseMetadata', [ResponseMetadata]}], + export_xml([make_internal_node('ListTempSessionsResponse', + [{'xmlns', ?IAM_XMLNS}], + lists:flatten(C))], []). + + +tag_node(?IAM_TAG{key = Key, + value = Value}) -> + {'Tag', [{'Key', [binary_to_list(Key)]}, + {'Value', [binary_to_list(Value)]}]}. + + +make_internal_node(Name, Content) -> + {Name, Content}. + +make_internal_node(Name, Attributes, Content) -> + {Name, Attributes, Content}. + +make_external_node(Name, Content) -> + {Name, [format_value(Content)]}. + + +%% @doc Convert value depending on its type into strings +format_value(undefined) -> + []; +format_value(Val) when is_atom(Val) -> + atom_to_list(Val); +format_value(Val) when is_binary(Val) -> + binary_to_list(Val); +format_value(Val) when is_integer(Val) -> + integer_to_list(Val); +format_value(Val) when is_list(Val) -> + Val; +format_value(Val) when is_float(Val) -> + io_lib:format("~p", [Val]). + + +%% =================================================================== +%% Eunit tests +%% =================================================================== + +-ifdef(TEST). + +acl_to_xml_test() -> + Xml = <<"TESTID1tester1TESTID2tester2WRITETESTID1tester1READ">>, + Grants1 = [?ACL_GRANT{grantee = #{display_name => <<"tester1">>, + canonical_id => <<"TESTID1">>}, + perms = ['READ']}, + ?ACL_GRANT{grantee = #{display_name => <<"tester2">>, + canonical_id => <<"TESTID2">>}, + perms = ['WRITE']}], + Grants2 = [?ACL_GRANT{grantee = #{display_name => <<"tester2">>, + canonical_id => <<"TESTID1">>}, + perms = ['READ']}, + ?ACL_GRANT{grantee = #{display_name => <<"tester1">>, + canonical_id => <<"TESTID2">>}, + perms = ['WRITE']}], + Now = os:system_time(millisecond), + Acl1 = ?ACL{owner = #{display_name => <<"tester1">>, + canonical_id => <<"TESTID1">>, + key_id => <<"TESTKEYID1">>}, + grants = Grants1, + creation_time = Now}, + Acl2 = ?ACL{owner = #{display_name => <<"tester1">>, + canonical_id => <<"TESTID1">>, + key_id => <<"TESTKEYID1">>}, + grants = Grants2, + creation_time = Now}, + ?assertEqual(Xml, riak_cs_xml:to_xml(Acl1)), + ?assertNotEqual(Xml, riak_cs_xml:to_xml(Acl2)). + +list_objects_response_to_xml_test() -> + Xml = <<"bucket1000falsetestkey12012-11-29T17:50:30.000Z\"fba9dede6af29711d7271245a35813428\"12345STANDARDTESTID1tester1testkey22012-11-29T17:52:30.000Z\"43433281b2f27731ccf53597645a3985\"54321STANDARDTESTID2tester2">>, + Owner1 = #list_objects_owner{id = <<"TESTID1">>, display_name = <<"tester1">>}, + Owner2 = #list_objects_owner{id = <<"TESTID2">>, display_name = <<"tester2">>}, + Content1 = ?LOKC{key = <<"testkey1">>, + last_modified = riak_cs_wm_utils:to_iso_8601("Thu, 29 Nov 2012 17:50:30 GMT"), + etag = <<"\"fba9dede6af29711d7271245a35813428\"">>, + size = 12345, + owner = Owner1, + storage_class = <<"STANDARD">>}, + Content2 = ?LOKC{key = <<"testkey2">>, + last_modified = riak_cs_wm_utils:to_iso_8601("Thu, 29 Nov 2012 17:52:30 GMT"), + etag = <<"\"43433281b2f27731ccf53597645a3985\"">>, + size = 54321, + owner = Owner2, + storage_class = <<"STANDARD">>}, + ListObjectsResponse = ?LORESP{name = <<"bucket">>, + max_keys = 1000, + prefix = undefined, + delimiter = undefined, + marker = undefined, + is_truncated = false, + contents = [Content1, Content2], + common_prefixes = []}, + ?assertEqual(Xml, riak_cs_xml:to_xml(ListObjectsResponse)). + +user_record_to_xml_test() -> + Xml = <<"barf@spaceballs.combarfbarfolomewbarfolomewbarf_keysecretsauce1234enabled">>, + User = ?RCS_USER{name="barfolomew", + display_name="barf", + email="barf@spaceballs.com", + key_id="barf_key", + key_secret="secretsauce", + id="1234", + status=enabled}, + io:format("~p", [riak_cs_xml:to_xml(User)]), + ?assertEqual(Xml, riak_cs_xml:to_xml(User)). + +-endif. diff --git a/src/riak_cs_yessir_riak_client.erl b/apps/riak_cs/src/riak_cs_yessir_riak_client.erl similarity index 71% rename from src/riak_cs_yessir_riak_client.erl rename to apps/riak_cs/src/riak_cs_yessir_riak_client.erl index a706ceab2..72e1b2f1a 100644 --- a/src/riak_cs_yessir_riak_client.erl +++ b/apps/riak_cs/src/riak_cs_yessir_riak_client.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -91,7 +92,7 @@ handle_call({req, #rpbcsbucketreq{max_results=_MaxResults, _Timeout, {ReqId, Caller}=_Ctx}, From, #state{bucket_name=BucketName, acl=Acl} = State) -> gen_server:reply(From, {ok, ReqId}), - RiakcObjs = manifests_to_robjs([new_manifest(BucketName, <<"yessir-key1">>, + RiakcObjs = manifests_to_robjs([new_manifest(BucketName, <<"yessir-key1">>, <<"1.0">>, 3141592, Acl)]), Caller ! {ReqId, {ok, RiakcObjs}}, Caller ! {ReqId, {done, continuation_ignored}}, @@ -124,7 +125,7 @@ handle_call(block_pbc, _From, State) -> {reply, {ok, self()}, State}; handle_call(Request, _From, State) -> - lager:warning("Unknown request: ~p~n", [Request]), + logger:warning("Unknown request: ~p", [Request]), Reply = {error, {invalid_request, Request}}, {reply, Reply, State}. @@ -142,93 +143,94 @@ code_change(_OldVsn, State, _Extra) -> %%% Internal functions -process_get_req(#rpbgetreq{bucket=RiakBucket, key=Key} = _RpbGetReq, - #state{bucket_name=BucketName, acl=Acl} = State) -> +process_get_req(#rpbgetreq{bucket = RiakBucket, key = VKey} = _RpbGetReq, + #state{bucket_name = BucketName, acl = Acl} = State) -> case which_bucket(RiakBucket) of objects -> - {M, RObj} = new_manifest_ro(BucketName, RiakBucket, Key, Acl), + {Key, Vsn} = rcs_common_manifest:decompose_versioned_key(VKey), + {M, RObj} = new_manifest_ro(BucketName, RiakBucket, Key, Vsn, Acl), {{ok, RObj}, State#state{manifest = M}}; blocks -> - UUIDSize = byte_size(Key) - 32 div 8, - <<_UUID:UUIDSize/binary, _BlockNumber:32>> = Key, - Obj = riakc_obj:new_obj(RiakBucket, Key, <<"vclock">>, + UUIDSize = byte_size(VKey) - 32 div 8, + <<_UUID:UUIDSize/binary, _BlockNumber:32>> = VKey, + Obj = riakc_obj:new_obj(RiakBucket, VKey, <<"vclock">>, [{dict:new(), binary:copy(<<"a">>, 10)}]), {{ok, Obj}, State}; Other -> - lager:warning("Unknown #rpbgetreq{} for ~p, bucket=~p, key=~p~n", - [Other, RiakBucket, Key]), - error({not_implemented, {rpbgetreq, Other, RiakBucket, Key}}) + logger:warning("Unknown #rpbgetreq{} for ~p, bucket=~p, key=~p", + [Other, RiakBucket, VKey]), + error({not_implemented, {rpbgetreq, Other, RiakBucket, VKey}}) end. -process_put_req(#rpbputreq{bucket=RiakBucket, key=Key, return_body=1} = _RpbPutReq, - #state{bucket_name=BucketName, acl=Acl} = State) -> +process_put_req(#rpbputreq{bucket = RiakBucket, key = VKey, return_body = 1} = _RpbPutReq, + #state{bucket_name = BucketName, acl = Acl} = State) -> case which_bucket(RiakBucket) of objects -> - {M, RObj} = new_manifest_ro(BucketName, RiakBucket, Key, Acl), + {Key, Vsn} = rcs_common_manifest:decompose_versioned_key(VKey), + {M, RObj} = new_manifest_ro(BucketName, RiakBucket, Key, Vsn, Acl), {{ok, RObj}, State#state{manifest = M}}; Other -> - lager:warning("Unknown #rpbgetreq{} with return_body for ~p," " bucket=~p, key=~p~n", - [Other, RiakBucket, Key]), - error({not_implemented, {rpbgetreq_with_return_body, Other, RiakBucket, Key}}) + logger:warning("Unknown #rpbgetreq{} with return_body for ~p, bucket=~p, key=~p", + [Other, RiakBucket, VKey]), + error({not_implemented, {rpbgetreq_with_return_body, Other, RiakBucket, VKey}}) end; process_put_req(_RpbPutReq, State) -> {ok, State}. -new_user(UserKeyBin) -> - UserKey = binary_to_list(UserKeyBin), - DisplayName = UserKey ++ "+DisplayName", - CanonicalId = UserKey ++ "+CanonicalId", +new_user(UserKey) -> + Name = <>, + DisplayName = <>, + CanonicalId = <>, DefaultAcl = default_acl(DisplayName, CanonicalId, UserKey), - {#rcs_user_v2{name = UserKey, - display_name = UserKey, - email = UserKey ++ "@example.com", - key_id = UserKey, - key_secret = UserKey ++ "+Secret", - canonical_id = CanonicalId, - buckets=[#moss_bucket_v1{name = UserKey ++ "-yessir-bucket-1", - acl = DefaultAcl}]}, + {?RCS_USER{arn = riak_cs_aws_utils:make_user_arn(UserKey, <<"/">>), + name = Name, + display_name = UserKey, + email = <>, + key_id = UserKey, + key_secret = <>, + id = CanonicalId, + buckets = [?RCS_BUCKET{name = <>, + acl = DefaultAcl}]}, DefaultAcl}. -user_to_robj(#rcs_user_v2{key_id=Key} = User) -> - UserKeyBin = list_to_binary(Key), - riakc_obj:new_obj(?USER_BUCKET, UserKeyBin, <<"vclock">>, +user_to_robj(?RCS_USER{arn = Arn} = User) -> + riakc_obj:new_obj(?USER_BUCKET, Arn, <<"vclock">>, [{dict:new(), term_to_binary(User)}]). default_acl(DisplayName, CannicalId, KeyId) -> riak_cs_acl_utils:default_acl(DisplayName, CannicalId, KeyId). -new_manifest_ro(BucketName, RiakBucket, Key, Acl) -> +new_manifest_ro(BucketName, RiakBucket, Key, Vsn, Acl) -> ContentLength = 10, - M = new_manifest(BucketName, Key, ContentLength, Acl), - Dict = riak_cs_manifest_utils:new_dict(M?MANIFEST.uuid, M), + M = new_manifest(BucketName, Key, Vsn, ContentLength, Acl), + Dict = rcs_common_manifest_utils:new_dict(M?MANIFEST.uuid, M), ValueBin = riak_cs_utils:encode_term(Dict), - RObj = riakc_obj:new_obj(RiakBucket, Key, <<"vclock">>, + RObj = riakc_obj:new_obj(RiakBucket, rcs_common_manifest:make_versioned_key(Key, Vsn), <<"vclock">>, [{dict:new(), ValueBin}]), {M, RObj}. -new_manifest(BucketName, Key, ContentLength, Acl) -> +new_manifest(BucketName, Key, Vsn, ContentLength, Acl) -> %% TODO: iteration is needed for large content length CMd5 = riak_cs_utils:md5(binary:copy(<<"a">>, ContentLength)), - ?MANIFEST{ - block_size = riak_cs_lfs_utils:block_size(), - bkey = {BucketName, Key}, - metadata = [], - created = riak_cs_wm_utils:iso_8601_datetime(), - uuid = druuid:v4(), + ?MANIFEST{block_size = riak_cs_lfs_utils:block_size(), + bkey = {BucketName, Key}, + vsn = Vsn, + metadata = [], + uuid = uuid:get_v4(), - content_length = ContentLength, - content_type = <<"application/octet-stream">>, - content_md5 = CMd5, - acl = Acl, + content_length = ContentLength, + content_type = <<"application/octet-stream">>, + content_md5 = CMd5, + acl = Acl, - state = active, - props = []}. + state = active, + props = []}. manifests_to_robjs(Manifests) -> [manifest_to_robj(M) || M <- Manifests]. manifest_to_robj(?MANIFEST{bkey={Bucket, Key}, uuid=UUID}=M) -> - Dict = riak_cs_manifest_utils:new_dict(UUID, M), + Dict = rcs_common_manifest_utils:new_dict(UUID, M), ManifestBucket = riak_cs_utils:to_bucket_name(objects, Bucket), riakc_obj:new(ManifestBucket, Key, riak_cs_utils:encode_term(Dict)). diff --git a/apps/riak_cs/src/stanchion_auth.erl b/apps/riak_cs/src/stanchion_auth.erl new file mode 100644 index 000000000..c703ad629 --- /dev/null +++ b/apps/riak_cs/src/stanchion_auth.erl @@ -0,0 +1,121 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_auth). + +-export([authenticate/2]). + +-include("stanchion.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +%% =================================================================== +%% Public API +%% =================================================================== + +-spec authenticate(term(), [string()]) -> ok | {error, atom()}. +authenticate(RD, [KeyId, Signature]) -> + {ok, {AdminKeyId, AdminSecret}} = riak_cs_config:admin_creds(), + case (list_to_binary(KeyId) == AdminKeyId) andalso + check_auth(Signature, signature(binary_to_list(AdminSecret), RD)) of + true -> + ok; + _ -> + {error, invalid_authentication} + end. + +%% =================================================================== +%% Internal functions +%% =================================================================== + +signature(KeyData, RD) -> + Headers = normalize_headers(get_request_headers(RD)), + BashoHeaders = extract_basho_headers(Headers), + Resource = wrq:path(RD), + case proplists:is_defined("x-basho-date", Headers) of + true -> + Date = "\n"; + false -> + Date = [wrq:get_req_header("date", RD), "\n"] + end, + case wrq:get_req_header("content-md5", RD) of + undefined -> + CMD5 = []; + CMD5 -> + ok + end, + case wrq:get_req_header("content-type", RD) of + undefined -> + ContentType = []; + ContentType -> + ok + end, + STS = [atom_to_list(wrq:method(RD)), "\n", + CMD5, + "\n", + ContentType, + "\n", + Date, + BashoHeaders, + Resource], + base64:encode_to_string(stanchion_utils:sha_mac(KeyData, STS)). + +check_auth(Signature, Calculated) when Signature /= Calculated -> + ?LOG_NOTICE("Bad signature presented: ~s (calculated: ~s)", [Signature, Calculated]), + false; +check_auth(_, _) -> + true. + + +get_request_headers(RD) -> + mochiweb_headers:to_list(wrq:req_headers(RD)). + +normalize_headers(Headers) -> + FilterFun = + fun({K, V}, Acc) -> + LowerKey = string:to_lower(any_to_list(K)), + [{LowerKey, V} | Acc] + end, + ordsets:from_list(lists:foldl(FilterFun, [], Headers)). + +extract_basho_headers(Headers) -> + FilterFun = + fun({K, V}, Acc) -> + case lists:prefix("x-basho-", K) of + true -> + [[K, ":", V, "\n"] | Acc]; + false -> + Acc + end + end, + ordsets:from_list(lists:foldl(FilterFun, [], Headers)). + +any_to_list(V) when is_list(V) -> + V; +any_to_list(V) when is_atom(V) -> + atom_to_list(V); +any_to_list(V) when is_binary(V) -> + binary_to_list(V); +any_to_list(V) when is_integer(V) -> + integer_to_list(V). diff --git a/include/riak_cs_api.hrl b/apps/riak_cs/src/stanchion_blockall_auth.erl similarity index 73% rename from include/riak_cs_api.hrl rename to apps/riak_cs/src/stanchion_blockall_auth.erl index d0926b9cb..175e6e9bb 100644 --- a/include/riak_cs_api.hrl +++ b/apps/riak_cs/src/stanchion_blockall_auth.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,12 +19,10 @@ %% %% --------------------------------------------------------------------- --record(list_buckets_response_v1, { - %% the user record - user :: rcs_user(), +-module(stanchion_blockall_auth). - %% the list of bucket records - buckets :: [cs_bucket()] - }). --type list_buckets_response() :: #list_buckets_response_v1{}. --define(LBRESP, #list_buckets_response_v1). +-export([authenticate/2]). + +-spec authenticate(term(), [string()]) -> {error, atom()}. +authenticate(_RD, [Reason]) -> + {error, Reason}. diff --git a/apps/riak_cs/src/stanchion_lock.erl b/apps/riak_cs/src/stanchion_lock.erl new file mode 100644 index 000000000..dc1795c0c --- /dev/null +++ b/apps/riak_cs/src/stanchion_lock.erl @@ -0,0 +1,182 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +%% Object locking: application-level lock-for-update, originally for +%% AttachRolePolicy and similar, which involve reading and updating of +%% multiple logically linked IAM entities. + +-module(stanchion_lock). + +-export([acquire/1, + acquire/2, + release/2, + cleanup/0, + precious/0 + ]). +-export([start_link/0 + ]). +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 + ]). + +-define(SERVER, ?MODULE). + +-include("riak_cs.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-define(ACQUIRE_WAIT_MSEC, 500). +-define(AUTO_RELEASE_MSEC, 15000). +-define(MAX_ACQUIRE_WAIT_MSEC, 5000). + +-spec acquire(binary()) -> binary() | busy. +acquire(A) -> + acquire(A, precious()). +acquire(A, Precious) -> + acquire(A, Precious, 0). +acquire(A, Precious, TT) -> + case gen_server:call(?SERVER, {acquire, A, Precious}, infinity) of + busy -> + ?LOG_DEBUG("waiting tick ~b to acquire lock on ~p", [TT, A]), + timer:sleep(?ACQUIRE_WAIT_MSEC), + acquire(A, Precious, TT + 1); + Precious -> + Precious + end. + +-spec release(binary(), binary()) -> ok. +release(A, Precious) -> + gen_server:call(?SERVER, {release, A, Precious}, infinity). + +-spec cleanup() -> ok. +cleanup() -> + gen_server:call(?SERVER, cleanup, infinity). + + +-record(state, {pbc :: undefined | pid()}). + + +-spec start_link() -> {ok, pid()} | {error, term()}. +start_link() -> + gen_server:start_link({local, ?MODULE}, ?SERVER, [], []). + +-spec init([]) -> {ok, #state{}}. +init([]) -> + {ok, Pbc} = riak_cs_utils:riak_connection(), + {ok, #state{pbc = Pbc}}. + +-spec handle_call(term(), {pid(), term()}, #state{}) -> + {reply, ok, #state{}}. +handle_call({acquire, A, Precious}, _From, State = #state{pbc = Pbc}) -> + {reply, do_acquire(A, Precious, Pbc), State}; +handle_call({release, A, B}, _From, State = #state{pbc = Pbc}) -> + {reply, do_release(A, B, Pbc), State}; +handle_call(cleanup, _From, State = #state{pbc = Pbc}) -> + {reply, do_cleanup(Pbc), State}; +handle_call(_Msg, _From, State) -> + ?LOG_WARNING("Unhandled call ~p from ~p", [_Msg, _From]), + {reply, ok, State}. + +-spec handle_cast(term(), #state{}) -> {noreply, #state{}}. +handle_cast(_Msg, State) -> + ?LOG_WARNING("Unhandled cast ~p", [_Msg]), + {noreply, State}. + +-spec handle_info(term(), #state{}) -> {noreply, #state{}}. +handle_info(_Info, State) -> + {noreply, State}. + +-spec terminate(term(), #state{}) -> ok. +terminate(_Reason, #state{pbc = Pbc}) -> + riak_cs_utils:close_riak_connection(Pbc), + ok. + +-spec code_change(term(), #state{}, term()) -> {ok, #state{}}. +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + + +do_acquire(A, Precious, Pbc) -> + case riakc_pb_socket:get(Pbc, ?OBJECT_LOCK_BUCKET, A, ?CONSISTENT_READ_OPTIONS) of + {ok, Obj} -> + case riakc_obj:get_value(Obj) of + Precious -> + ?LOG_DEBUG("lock on \"~s\" already acquired (~s)", [A, Precious]), + Precious; + _NotOurs -> + ?LOG_DEBUG("lock on \"~s\" in use (~s)", [A, _NotOurs]), + busy + end; + {error, notfound} -> + ?LOG_DEBUG("Placing lock on \"~s\" (precious: ~s)", [A, Precious]), + ok = riakc_pb_socket:put( + Pbc, riakc_obj:new(?OBJECT_LOCK_BUCKET, A, Precious), + ?CONSISTENT_WRITE_OPTIONS), + _ = spawn(fun() -> + timer:sleep(?AUTO_RELEASE_MSEC), + do_release(A, Precious, Pbc, auto) + end), + Precious + end. + +do_release(A, Precious, Pbc) -> + do_release(A, Precious, Pbc, normal). +do_release(A, PreciousOrig, Pbc, Mode) -> + case riakc_pb_socket:get(Pbc, ?OBJECT_LOCK_BUCKET, A, ?CONSISTENT_READ_OPTIONS) of + {ok, Obj} -> + case riakc_obj:get_value(Obj) of + PreciousOrig -> + ?LOG_DEBUG("lock on \"~s\" found (precious: ~s), releasing it", [A, PreciousOrig]); + _NotOurs -> + logger:error("found a lock on ~p overwritten by another process (precious: ~s)", [A, _NotOurs]) + end, + ok = riakc_pb_socket:delete(Pbc, ?OBJECT_LOCK_BUCKET, A, ?CONSISTENT_DELETE_OPTIONS); + {error, notfound} -> + case Mode of + normal -> + logger:notice("Lock on \"~s\" not found. The process that acquired it was holding it for too long (>~b msec) and it was autodeleted.", [A, ?AUTO_RELEASE_MSEC]), + ok; + auto -> + ok + end + end. + +do_cleanup(Pbc) -> + case riakc_pb_socket:list_keys(Pbc, ?OBJECT_LOCK_BUCKET) of + {ok, []} -> + ok; + {ok, KK} -> + logger:notice("found ~b stale locks; deleting them now", [length(KK)]), + delete_all(Pbc, KK) + end. + +delete_all(_, []) -> + ok; +delete_all(Pbc, [A | AA]) -> + ok = riakc_pb_socket:delete(Pbc, ?OBJECT_LOCK_BUCKET, A, ?CONSISTENT_DELETE_OPTIONS), + delete_all(Pbc, AA). + + +precious() -> + list_to_binary( + [$A + rand:uniform($Z-$A) || _ <- [a, a, a, a, a, a, a]]). diff --git a/apps/riak_cs/src/stanchion_migration.erl b/apps/riak_cs/src/stanchion_migration.erl new file mode 100644 index 000000000..26ade12d4 --- /dev/null +++ b/apps/riak_cs/src/stanchion_migration.erl @@ -0,0 +1,162 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2022, 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc Functions to locate and migrate stanchion. + +-module(stanchion_migration). + +-export([validate_stanchion/0, + adopt_stanchion/0, + apply_stanchion_details/1, + save_stanchion_data/1 + ]). +-export([stop_stanchion_here/0]). + +-include("riak_cs.hrl"). + +-define(REASONABLY_SMALL_TIMEOUT, 3000). + + +-spec validate_stanchion() -> ok. +validate_stanchion() -> + {ConfiguredIP, ConfiguredPort, _Ssl} = riak_cs_config:stanchion(), + case read_stanchion_data() of + {ok, {{Host, Port}, Node}} + when Host == ConfiguredIP, + Port == ConfiguredPort, + Node == node() -> + ok; + {ok, {{Host, Port}, Node}} -> + logger:info("stanchion details updated: ~s:~p on ~s", [Host, Port, Node]), + case lists:member(ConfiguredIP, this_host_addresses()) of + true when node() == Node -> + stop_stanchion_here(), + ok; + _ -> + ok + end, + apply_stanchion_details({Host, Port}); + {error, notfound} -> + logger:info("no previously saved stanchion details; adopting stanchion here"), + adopt_stanchion() + end. + +this_host_addresses() -> + {ok, Ifs} = inet:getifaddrs(), + lists:filtermap( + fun({_If, PL}) -> + case proplists:get_value(addr, PL) of + AA when AA /= undefined, + AA /= {0,0,0,0}, + size(AA) == 4 -> + {A1, A2, A3, A4} = AA, + {true, lists:flatten(io_lib:format("~b.~b.~b.~b", [A1, A2, A3, A4]))}; + _ -> + false + end + end, Ifs). + +select_addr_for_stanchion() -> + {Subnet_, Mask_} = riak_cs_config:stanchion_subnet_and_netmask(), + {ok, Subnet} = inet:parse_address(Subnet_), + {ok, Mask} = inet:parse_address(Mask_), + case netutils:get_local_ip_from_subnet({Subnet, Mask}) of + {ok, {A1, A2, A3, A4}} -> + lists:flatten(io_lib:format("~b.~b.~b.~b", [A1, A2, A3, A4])); + undefined -> + logger:warning("No network interfaces with assigned addresses matching ~s:" + " falling back to 127.0.0.1", [Mask_]), + "127.0.0.1" + end. + +adopt_stanchion() -> + case riak_cs_config:stanchion_hosting_mode() of + Adoptable when Adoptable =:= riak_cs_with_stanchion; + Adoptable =:= auto; + Adoptable =:= stanchion_only -> + Addr = select_addr_for_stanchion(), + {ok, Port} = application:get_env(riak_cs, stanchion_port), + ok = save_stanchion_data({Addr, Port}), + ok = apply_stanchion_details({Addr, Port}), + start_stanchion_here(), + ok; + M -> + logger:error("Riak CS stanchion_hosting_mode is ~s. Cannot adopt stanchion.", [M]), + {error, stanchion_not_relocatable} + end. + +start_stanchion_here() -> + case supervisor:which_children(stanchion_sup) of + [] -> + _ = [supervisor:start_child(stanchion_sup, F) || F <- stanchion_sup:stanchion_process_specs()]; + _ -> + already_running + end. + +stop_stanchion_here() -> + case supervisor:which_children(stanchion_sup) of + [] -> + already_stopped; + FF -> + logger:notice("Stopping stanchion on this node"), + [begin + ok = supervisor:terminate_child(stanchion_sup, Id), + ok = supervisor:delete_child(stanchion_sup, Id) + end || {Id, _, _, _} <- FF] + end. + + +apply_stanchion_details({Host, Port}) -> + riak_cs_config:set_stanchion(Host, Port). + +read_stanchion_data() -> + Pbc = stanchion_utils:get_pbc(), + case riak_cs_pbc:get_sans_stats(Pbc, ?SERVICE_BUCKET, ?STANCHION_DETAILS_KEY, + [{notfound_ok, false}], + ?REASONABLY_SMALL_TIMEOUT) of + {ok, Obj} -> + case riakc_obj:value_count(Obj) of + 1 -> + StanchionDetails = binary_to_term(riakc_obj:get_value(Obj)), + {ok, StanchionDetails}; + 0 -> + {error, notfound}; + N -> + Values = [binary_to_term(Value) || + Value <- riakc_obj:get_values(Obj), + Value /= <<>> % tombstone + ], + logger:warning("Read stanchion details riak object has ~b siblings." + " Please select a riak_cs node, reconfigure it with stanchion_hosting_mode = riak_cs_with_stanchion (or stanchion_only)," + " configure rest with stanchion_hosting_mode = riak_cs_only, and restart all nodes", [N]), + {ok, hd(Values)} + end; + _ -> + {error, notfound} + end. + +save_stanchion_data(HostPort) -> + Pbc = stanchion_utils:get_pbc(), + ok = riak_cs_pbc:put_sans_stats( + Pbc, riakc_obj:new(?SERVICE_BUCKET, ?STANCHION_DETAILS_KEY, + term_to_binary({HostPort, node()})), + ?REASONABLY_SMALL_TIMEOUT), + logger:info("saved stanchion details: ~p", [{HostPort, node()}]), + ok. diff --git a/apps/riak_cs/src/stanchion_multipart.erl b/apps/riak_cs/src/stanchion_multipart.erl new file mode 100644 index 000000000..a6e278a48 --- /dev/null +++ b/apps/riak_cs/src/stanchion_multipart.erl @@ -0,0 +1,79 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021, 2022 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_multipart). + +-export([check_no_multipart_uploads/2]). + +-include("stanchion.hrl"). +-include("manifest.hrl"). + + +check_no_multipart_uploads(Bucket, RiakPid) -> + HashBucket = stanchion_utils:to_bucket_name(objects, Bucket), + + {{ok, Keys}, TAT} = ?TURNAROUND_TIME(riakc_pb_socket:list_keys(RiakPid, HashBucket)), + stanchion_stats:update([riakc, list_all_manifest_keys], TAT), + + %% check all up + lists:all(fun(VKey) -> + {Key, Vsn} = rcs_common_manifest:decompose_versioned_key(VKey), + GetResult = stanchion_utils:get_manifests_raw(RiakPid, Bucket, Key, Vsn), + has_no_upload(GetResult) + end, Keys). + +has_no_upload({ok, Obj}) -> + Manifests = manifests_from_riak_object(Obj), + lists:all(fun({_UUID,Manifest}) -> + case Manifest?MANIFEST.state of + writing -> + %% if this is mp => false + not proplists:is_defined(multipart, Manifest?MANIFEST.props); + _ -> + true + end + end, Manifests); +has_no_upload({error, notfound}) -> true. + +-spec manifests_from_riak_object(riakc_obj:riakc_obj()) -> orddict:orddict(). +manifests_from_riak_object(RiakObject) -> + %% For example, riak_cs_manifest_fsm:get_and_update/4 may wish to + %% update the #riakc_obj without a roundtrip to Riak first. So we + %% need to see what the latest + Contents = try + %% get_update_value will return the updatevalue or + %% a single old original value. + [{riakc_obj:get_update_metadata(RiakObject), + riakc_obj:get_update_value(RiakObject)}] + catch throw:_ -> + %% Original value had many contents + riakc_obj:get_contents(RiakObject) + end, + DecodedSiblings = [binary_to_term(V) || + {_, V}=Content <- Contents, + not stanchion_utils:has_tombstone(Content)], + + %% Upgrade the manifests to be the latest erlang + %% record version + Upgraded = rcs_common_manifest_utils:upgrade_wrapped_manifests(DecodedSiblings), + + %% resolve the siblings + rcs_common_manifest_resolution:resolve(Upgraded). diff --git a/riak_test/tests/storage_stats_test_2.erl b/apps/riak_cs/src/stanchion_passthru_auth.erl similarity index 73% rename from riak_test/tests/storage_stats_test_2.erl rename to apps/riak_cs/src/stanchion_passthru_auth.erl index 32a2bc75b..fc064ae91 100644 --- a/riak_test/tests/storage_stats_test_2.erl +++ b/apps/riak_cs/src/stanchion_passthru_auth.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,11 +19,10 @@ %% %% --------------------------------------------------------------------- --module(storage_stats_test_2). -%% @doc Integration test for storage statistics. +-module(stanchion_passthru_auth). --compile(export_all). --export([confirm/0]). +-export([authenticate/2]). -confirm() -> - storage_stats_test:confirm_1(true). +-spec authenticate(term(), [string()]) -> ok. +authenticate(_RD, _) -> + ok. diff --git a/apps/riak_cs/src/stanchion_response.erl b/apps/riak_cs/src/stanchion_response.erl new file mode 100644 index 000000000..cfd19f349 --- /dev/null +++ b/apps/riak_cs/src/stanchion_response.erl @@ -0,0 +1,48 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_response). + +-export([api_error/3, + list_buckets_response/3]). + +-include("riak_cs.hrl"). +-include("stanchion.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-spec api_error(reportable_error_reason(), #wm_reqdata{}, #stanchion_context{}) -> + {{halt, 400..599}, #wm_reqdata{}, #stanchion_context{}}. +api_error(Error, RD, Ctx) -> + StatusCode = riak_cs_aws_response:status_code(Error), + ErrorDesc = jsx:encode(#{error_tag => base64:encode(term_to_binary(Error)), + resource => list_to_binary(wrq:path(RD))}), + {{halt, StatusCode}, wrq:set_resp_body(ErrorDesc, RD), Ctx}. + + +list_buckets_response(BucketData, RD, Ctx) -> + BucketsDoc = [{'Bucket', + [{'Name', [binary_to_list(Bucket)]}, + {'Owner', [binary_to_list(Owner)]}]} + || {Bucket, Owner} <- BucketData], + Contents = [{'Buckets', BucketsDoc}], + XmlDoc = [{'ListBucketsResult', Contents}], + riak_cs_aws_response:respond(200, riak_cs_xml:to_xml(XmlDoc), RD, Ctx). + diff --git a/apps/riak_cs/src/stanchion_server.erl b/apps/riak_cs/src/stanchion_server.erl new file mode 100644 index 000000000..76b820f24 --- /dev/null +++ b/apps/riak_cs/src/stanchion_server.erl @@ -0,0 +1,384 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc Module to process bucket creation requests. + +-module(stanchion_server). + +-behaviour(gen_server). + +%% API +-export([start_link/0, + create_bucket/1, + create_user/1, + delete_user/1, + delete_bucket/2, + set_bucket_acl/2, + set_bucket_policy/2, + set_bucket_versioning/2, + delete_bucket_policy/2, + stop/1, + update_user/1, + create_role/1, + update_role/1, + delete_role/1, + create_policy/1, + update_policy/1, + delete_policy/1, + create_saml_provider/1, + delete_saml_provider/1, + msgq_len/0]). + +%% gen_server callbacks +-export([init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3]). + +-include("moss.hrl"). +-include("stanchion.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +%% Test API +-export([test_link/0]). + +-endif. + +-record(state, {pbc :: undefined | pid()}). +-type state() :: #state{}. + +%% This ?TURNAROUND_TIME has another ?TURNAROUND_TIME at gen_server +%% process, to measure both waiting time and service time. +-define(MEASURE(Name, Call), + begin + {{Result_____, ServiceTime____}, + TATus_____} = ?TURNAROUND_TIME(Call), + WaitingTime____ = TATus_____ - ServiceTime____, + ?LOG_DEBUG("~p ~p ~p", [Name, Result_____, ServiceTime____]), + stanchion_stats:update(Name, ServiceTime____, WaitingTime____), + Result_____ + end). + +%% =================================================================== +%% Public API +%% =================================================================== + +%% @doc Start a `stanchion_server'. +-spec start_link() -> {ok, pid()} | {error, term()}. +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +%% @doc Attempt to create a bucket +-spec create_bucket([{term(), term()}]) -> ok | {error, term()}. +create_bucket(BucketData) -> + ?MEASURE([bucket, create], + gen_server:call(?MODULE, + {create_bucket, BucketData}, + infinity)). + +%% @doc Attempt to create a bucket +-spec create_user([{term(), term()}]) -> + ok | {error, term() | stanchion_utils:riak_connect_failed()}. +create_user(UserData) -> + ?MEASURE([user, create], + gen_server:call(?MODULE, + {create_user, UserData}, + infinity)). + +-spec delete_user([{term(), term()}]) -> + ok | {error, term() | stanchion_utils:riak_connect_failed()}. +delete_user(A) -> + ?MEASURE([user, delete], + gen_server:call(?MODULE, + {delete_user, A}, + infinity)). + +%% @doc Attempt to delete a bucket +-spec delete_bucket(binary(), binary()) -> ok | {error, term()}. +delete_bucket(Bucket, UserId) -> + ?MEASURE([bucket, delete], + gen_server:call(?MODULE, + {delete_bucket, Bucket, UserId}, + infinity)). + +%% @doc Set the ACL for a bucket +-spec set_bucket_acl(binary(), term()) -> ok | {error, term()}. +set_bucket_acl(Bucket, FieldList) -> + ?MEASURE([bucket, put_acl], + gen_server:call(?MODULE, + {set_acl, Bucket, FieldList}, + infinity)). + +%% @doc Set the policy for a bucket +-spec set_bucket_policy(binary(), term()) -> ok | {error, term()}. +set_bucket_policy(Bucket, FieldList) -> + ?MEASURE([bucket, set_policy], + gen_server:call(?MODULE, + {set_policy, Bucket, FieldList}, + infinity)). + +%% @doc Set the versioning option for a bucket +-spec set_bucket_versioning(binary(), term()) -> ok | {error, term()}. +set_bucket_versioning(Bucket, FieldList) -> + ?MEASURE([bucket, set_versioning], + gen_server:call(?MODULE, + {set_versioning, Bucket, FieldList}, + infinity)). + +%% @doc delete the policy for a bucket +-spec delete_bucket_policy(binary(), binary()) -> ok | {error, term()}. +delete_bucket_policy(Bucket, RequesterId) -> + ?MEASURE([bucket, delete_policy], + gen_server:call(?MODULE, + {delete_policy, Bucket, RequesterId}, + infinity)). + +-spec create_role(maps:map()) -> {ok, string()} | {error, term()}. +create_role(A) -> + ?MEASURE([role, create], + gen_server:call(?MODULE, + {create_role, A}, + infinity)). + +-spec update_role(maps:map()) -> ok | {error, term()}. +update_role(A) -> + ?MEASURE([role, update], + gen_server:call(?MODULE, + {update_role, A}, + infinity)). + +-spec delete_role(string()) -> ok | {error, term()}. +delete_role(A) -> + ?MEASURE([role, delete], + gen_server:call(?MODULE, + {delete_role, A}, + infinity)). + +-spec create_policy(maps:map()) -> {ok, string()} | {error, term()}. +create_policy(A) -> + ?MEASURE([policy, create], + gen_server:call(?MODULE, + {create_policy, A}, + infinity)). + +-spec update_policy(maps:map()) -> ok | {error, term()}. +update_policy(A) -> + ?MEASURE([policy, update], + gen_server:call(?MODULE, + {update_policy, A}, + infinity)). + +-spec delete_policy(string()) -> ok | {error, term()}. +delete_policy(A) -> + ?MEASURE([policy, delete], + gen_server:call(?MODULE, + {delete_policy, A}, + infinity)). + +-spec create_saml_provider(maps:map()) -> {ok, {flat_arn(), [tag()]}} | {error, term()}. +create_saml_provider(A) -> + ?MEASURE([saml_provider, create], + gen_server:call(?MODULE, + {create_saml_provider, A}, + infinity)). + +-spec delete_saml_provider(string()) -> ok | {error, term()}. +delete_saml_provider(A) -> + ?MEASURE([saml_provider, delete], + gen_server:call(?MODULE, + {delete_saml_provider, A}, + infinity)). + +stop(Pid) -> + gen_server:cast(Pid, stop). + +-spec update_user(maps:map()) -> {ok, rcs_user()} | {error, term() | stanchion_utils:riak_connect_failed()}. +update_user(A) -> + ?MEASURE([user, update], + gen_server:call(?MODULE, + {update_user, A}, + infinity)). + +-spec msgq_len() -> non_neg_integer(). +msgq_len() -> + case whereis(?MODULE) of + undefined -> + 0; %% server hasn't been started yet + Pid -> + {message_queue_len, Len} = erlang:process_info(Pid, message_queue_len), + Len + end. + +%% =================================================================== +%% gen_server callbacks +%% =================================================================== + +%% @doc Initialize the server. +-spec init([] | test) -> {ok, state()}. +init([]) -> + {ok, Pbc} = riak_cs_utils:riak_connection(), + {ok, #state{pbc = Pbc}}; +init(test) -> + {ok, #state{}}. + +%% @doc Handle synchronous commands issued via exported functions. +-spec handle_call(term(), {pid(), term()}, state()) -> + {reply, ok, state()}. +handle_call({create_bucket, BucketData}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:create_bucket(BucketData, Pbc)), + {reply, Result, State}; +handle_call({create_user, UserData}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:create_user(UserData, Pbc)), + {reply, Result, State}; +handle_call({delete_user, UserData}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:delete_user(UserData, Pbc)), + {reply, Result, State}; +handle_call({update_user, UserData}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:update_user(UserData, Pbc)), + {reply, Result, State}; +handle_call({delete_bucket, Bucket, OwnerId}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:delete_bucket(Bucket, OwnerId, Pbc)), + {reply, Result, State}; +handle_call({set_acl, Bucket, FieldList}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:set_bucket_acl(Bucket, FieldList, Pbc)), + {reply, Result, State}; +handle_call({set_policy, Bucket, FieldList}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:set_bucket_policy(Bucket, FieldList, Pbc)), + {reply, Result, State}; +handle_call({set_versioning, Bucket, FieldList}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:set_bucket_versioning(Bucket, FieldList, Pbc)), + {reply, Result, State}; +handle_call({delete_policy, Bucket, RequesterId}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:delete_bucket_policy(Bucket, RequesterId, Pbc)), + {reply, Result, State}; +handle_call({create_role, A}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:create_role(A, Pbc)), + {reply, Result, State}; +handle_call({update_role, A}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:update_role(A, Pbc)), + {reply, Result, State}; +handle_call({delete_role, A}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:delete_role(A, Pbc)), + {reply, Result, State}; +handle_call({create_policy, A}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:create_policy(A, Pbc)), + {reply, Result, State}; +handle_call({update_policy, A}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:update_policy(A, Pbc)), + {reply, Result, State}; +handle_call({delete_policy, A}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:delete_policy(A, Pbc)), + {reply, Result, State}; +handle_call({create_saml_provider, A}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:create_saml_provider(A, Pbc)), + {reply, Result, State}; +handle_call({delete_saml_provider, A}, + _From, + State=#state{pbc = Pbc}) -> + Result = ?TURNAROUND_TIME(stanchion_utils:delete_saml_provider(A, Pbc)), + {reply, Result, State}; +handle_call(_Msg, _From, State) -> + ?LOG_WARNING("Unhandled call ~p", [_Msg]), + {reply, ok, State}. + +%% @doc Handle asynchronous commands issued via +%% the exported functions. +-spec handle_cast(term(), state()) -> {noreply, state()}. +handle_cast(list_buckets, State) -> + %% @TODO Handle bucket listing and reply + {noreply, State}; +handle_cast(stop, State = #state{pbc = Pbc}) -> + riak_cs_utils:close_riak_connection(Pbc), + {stop, normal, State#state{pbc = undefined}}; +handle_cast(Event, State) -> + logger:warning("Received unknown cast event: ~p", [Event]), + {noreply, State}. + +%% @doc @TODO +-spec handle_info(term(), state()) -> {noreply, state()}. +handle_info(_Info, State) -> + {noreply, State}. + +%% @doc Unused. +-spec terminate(term(), state()) -> ok. +terminate(_Reason, #state{pbc = Pbc}) -> + riak_cs_utils:close_riak_connection(Pbc), + ok. + +%% @doc Unused. +-spec code_change(term(), state(), term()) -> + {ok, state()}. +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%% ==================================================================== +%% Internal functions +%% ==================================================================== + +%% =================================================================== +%% Test API +%% =================================================================== + +-ifdef(TEST). + +%% @doc Start a `stanchion_server' for testing. +-spec test_link() -> {ok, pid()} | {error, term()}. +test_link() -> + gen_server:start_link(?MODULE, test, []). + +-endif. diff --git a/test/riak_cs_wm_bucket_test.erl b/apps/riak_cs/src/stanchion_server_sup.erl similarity index 50% rename from test/riak_cs_wm_bucket_test.erl rename to apps/riak_cs/src/stanchion_server_sup.erl index 642615387..94dfc2755 100644 --- a/test/riak_cs_wm_bucket_test.erl +++ b/apps/riak_cs/src/stanchion_server_sup.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% %% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,30 +19,28 @@ %% %% --------------------------------------------------------------------- --module(riak_cs_wm_bucket_test). +-module(stanchion_server_sup). --compile(export_all). +-behaviour(supervisor). --include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). --include_lib("eunit/include/eunit.hrl"). +-export([start_link/0]). +-export([init/1]). -bucket_test_() -> - {setup, - fun riak_cs_wm_test_utils:setup/0, - fun riak_cs_wm_test_utils:teardown/1, - [fun create_bucket_and_list_keys/0]}. +-spec start_link() -> {ok, pid()} | ignore | {error, term()}. +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). -%% @doc Test to see that a newly created -%% bucket has no keys. +-spec init([]) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. +init([]) -> + SupFlags = #{strategy => one_for_one, + intensity => 1000, + period => 3600}, -%% XXX TODO: MAKE THESE ACTUALLY TEST SOMETHING -%% The state needed for this test -%% scares me -create_bucket_and_list_keys() -> - PathInfo = dict:from_list([{bucket, "create_bucket_test"}]), - _RD = #wm_reqdata{path_info = PathInfo}, - _Ctx = #context{}, - ?assert(true). -%% {Result, _, _} = riak_cs_wm_bucket:to_json(RD, Ctx), -%% ?assertEqual(mochijson2:encode([]), Result). + ServerSpec = #{id => stanchion_server, + start => {stanchion_server, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [stanchion_server]}, + + {ok, {SupFlags, [ServerSpec]}}. diff --git a/apps/riak_cs/src/stanchion_stats.erl b/apps/riak_cs/src/stanchion_stats.erl new file mode 100644 index 000000000..8d3cd37e5 --- /dev/null +++ b/apps/riak_cs/src/stanchion_stats.erl @@ -0,0 +1,290 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_stats). + +%% API +-export([safe_update/2, + update/2, + update/3, + update_with_start/2, + report_json/0, + %% report_pretty_json/0, + get_stats/0]). + +-export([init/0]). + +-include_lib("kernel/include/logger.hrl"). + +-type key() :: [atom()]. +-export_type([key/0]). + +-spec duration_metrics() -> [key()]. +duration_metrics() -> + [ + [bucket, create], + [bucket, delete], + [bucket, put_acl], + [bucket, set_policy], + [bucket, delete_policy], + [bucket, set_versioning], + [user, create], + [user, delete], + [user, update], + + [role, create], + [role, delete], + [role, update], + + [policy, create], + [policy, delete], + [policy, update], + + [saml_provider, create], + [saml_provider, delete], + + %% Riak PB client, per key operations + [riakc, get_cs_bucket], + [riakc, put_cs_bucket], %% Always strong put + [riakc, get_cs_user], + [riakc, get_cs_user_strong], + [riakc, put_cs_user], + [riakc, delete_cs_user], + [riakc, get_manifest], + [riakc, get_cs_role], + [riakc, get_cs_role_strong], + [riakc, put_cs_role], + [riakc, delete_cs_role], + [riakc, get_cs_policy], + [riakc, put_cs_policy], + [riakc, delete_cs_policy], + [riakc, get_cs_samlprovider], + [riakc, put_cs_samlprovider], + [riakc, delete_cs_samlprovider], + + %% Riak PB client, coverage operations + [riakc, list_all_manifest_keys], + [riakc, get_user_by_index] + ]. + +duration_only_metrics() -> + [[waiting]]. + +duration_subkeys() -> + [{[], spiral}, + {[time], histogram}]. + +duration_only_subkeys() -> + [{[time], histogram}]. + +%% ==================================================================== +%% API +%% ==================================================================== + + + +-spec safe_update(key(), integer()) -> ok | {error, any()}. +safe_update(BaseId, ElapsedUs) -> + %% Just in case those metrics happen to be not registered; should + %% be a bug and also should not interrupt handling requests by + %% crashing. + try + update(BaseId, ElapsedUs) + catch T:E -> + logger:error("Failed on storing some metrics: ~p,~p", [T,E]) + end. + +-spec update(key(), integer()) -> ok | {error, any()}. +update(BaseId, ElapsedUs) -> + ?LOG_DEBUG("Updating ~p (~p)", [BaseId, ElapsedUs]), + ok = exometer:update([stanchion|BaseId], 1), + ok = exometer:update([stanchion,time|BaseId], ElapsedUs). + +update(BaseId, ServiceTime, WaitingTime) -> + update(BaseId, ServiceTime), + ok = exometer:update([stanchion, time, waiting], WaitingTime). + +-spec update_with_start(key(), non_neg_integer()) -> + ok | {error, any()}. +update_with_start(BaseId, StartTime) -> + update(BaseId, os:system_time(millisecond) - StartTime). + +-spec report_json() -> binary(). +report_json() -> + jsx:encode(get_stats()). + +all_metrics() -> + Metrics0 = [{Subkey ++ Metric, Type, [], []} + || Metric <- duration_only_metrics(), + {Subkey, Type} <- duration_only_subkeys()], + Metrics1 = [{Subkey ++ Metric, Type, [], []} + || Metric <- duration_metrics(), + {Subkey, Type} <- duration_subkeys()], + Metrics0 ++ Metrics1. + +-spec get_stats() -> proplists:proplist(). +get_stats() -> + DurationStats = + [report_exometer_item(Key, SubKey, ExometerType) || + Key <- duration_metrics(), + {SubKey, ExometerType} <- duration_subkeys()], + DurationOnlyStats = + [report_exometer_item(Key, SubKey, ExometerType) || + Key <- duration_only_metrics(), + {SubKey, ExometerType} <- duration_only_subkeys()], + MsgStats = [{stanchion_server_msgq_len, stanchion_server:msgq_len()}], + lists:flatten([DurationStats, MsgStats, DurationOnlyStats, + report_mochiweb(), report_memory(), + report_system()]). + +init() -> + _ = [init_item(I) || I <- all_metrics()], + ok. + +%% ==================================================================== +%% Internal +%% ==================================================================== + +init_item({Name, Type, Opts, _Aliases}) -> + ok = exometer:re_register([stanchion|Name], Type, Opts). + +report_exometer_item(Key, SubKey, ExometerType) -> + AtomKeys = [metric_to_atom(Key ++ SubKey, Suffix) || + Suffix <- suffixes(ExometerType)], + {ok, Values} = exometer:get_value([stanchion | SubKey ++ Key], + datapoints(ExometerType)), + [{AtomKey, Value} || + {AtomKey, {_DP, Value}} <- lists:zip(AtomKeys, Values)]. + +datapoints(histogram) -> + [mean, median, 95, 99, max]; +datapoints(spiral) -> + [one, count]. + +suffixes(histogram) -> + ["mean", "median", "95", "99", "100"]; +suffixes(spiral) -> + ["one", "total"]. + +report_mochiweb() -> + [report_mochiweb(webmachine_mochiweb)]. + +report_mochiweb(Id) -> + Children = supervisor:which_children(stanchion_sup), + case lists:keyfind(Id, 1, Children) of + false -> []; + {_, Pid, _, _} -> report_mochiweb(Id, Pid) + end. + +report_mochiweb(Id, Pid) -> + [{metric_to_atom([Id], PropKey), gen_server:call(Pid, {get, PropKey})} || + PropKey <- [active_sockets, waiting_acceptors, port]]. + +report_memory() -> + lists:map(fun({K, V}) -> {metric_to_atom([memory], K), V} end, erlang:memory()). + +report_system() -> + [{nodename, erlang:node()}, + {connected_nodes, erlang:nodes()}, + {sys_driver_version, list_to_binary(erlang:system_info(driver_version))}, + {sys_heap_type, erlang:system_info(heap_type)}, + {sys_logical_processors, erlang:system_info(logical_processors)}, + {sys_monitor_count, system_monitor_count()}, + {sys_otp_release, list_to_binary(erlang:system_info(otp_release))}, + {sys_port_count, erlang:system_info(port_count)}, + {sys_process_count, erlang:system_info(process_count)}, + {sys_smp_support, erlang:system_info(smp_support)}, + {sys_system_version, system_version()}, + {sys_system_architecture, system_architecture()}, + {sys_threads_enabled, erlang:system_info(threads)}, + {sys_thread_pool_size, erlang:system_info(thread_pool_size)}, + {sys_wordsize, erlang:system_info(wordsize)}]. + +system_monitor_count() -> + lists:foldl(fun(Pid, Count) -> + case erlang:process_info(Pid, monitors) of + {monitors, Mons} -> + Count + length(Mons); + _ -> + Count + end + end, 0, processes()). + +system_version() -> + list_to_binary(string:strip(erlang:system_info(system_version), right, $\n)). + +system_architecture() -> + list_to_binary(erlang:system_info(system_architecture)). + +metric_to_atom(Key, Suffix) when is_atom(Suffix) -> + metric_to_atom(Key, atom_to_list(Suffix)); +metric_to_atom(Key, Suffix) -> + StringKey = string:join([atom_to_list(Token) || Token <- Key], "_"), + list_to_atom(lists:flatten([StringKey, $_, Suffix])). + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +stats_metric_test() -> + [begin + case hd(Key) of + time -> + ?assertEqual(histogram, Type); + _ -> + ?assertNotEqual(false, lists:keyfind(Key, 1, all_metrics())), + ?assertEqual(spiral, Type) + end, + ?assertEqual([], Options) + end || {Key, Type, Options, _} <- all_metrics()]. + +stats_test_() -> + Apps = [setup, compiler, syntax_tools, exometer_core], + {setup, + fun() -> + [application:ensure_all_started(App) || App <- Apps], + ok = stanchion_stats:init() + end, + fun(_) -> + [ok = application:stop(App) || App <- Apps] + end, + [{inparallel, [fun() -> + %% ?debugVal(Key), + case hd(Key) of + time -> ok; + _ -> stanchion_stats:update(Key, 16#deadbeef, 16#deadbeef) + end + end || {Key, _, _, _} <- all_metrics()]}, + fun() -> + [begin + ?debugVal(Key), + CountItems = report_exometer_item(Key, [], spiral), + DurationItems = report_exometer_item(Key, [time], histogram), + ?debugVal(CountItems), + ?debugVal(DurationItems), + ?assertMatch([1, 1], + [N || {_, N} <- CountItems]), + ?assertMatch([16#deadbeef, 16#deadbeef, 16#deadbeef, 16#deadbeef, 16#deadbeef], + [N || {_, N} <- DurationItems]) + end || Key <- duration_metrics()] + end]}. + +-endif. diff --git a/apps/riak_cs/src/stanchion_sup.erl b/apps/riak_cs/src/stanchion_sup.erl new file mode 100644 index 000000000..3703f84ec --- /dev/null +++ b/apps/riak_cs/src/stanchion_sup.erl @@ -0,0 +1,92 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2022, 2023 TI Tokyo, All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc Supervisor for the stanchion process tree. + +-module(stanchion_sup). + +-behaviour(supervisor). + +-export([start_link/0]). +-export([init/1]). +-export([stanchion_process_specs/0]). + +-include("stanchion.hrl"). +-include("riak_cs.hrl"). +-include("moss.hrl"). + +-spec start_link() -> supervisor:startlink_ret(). +start_link() -> + stanchion_stats:init(), + ets:new(?STANCHION_OWN_PBC_TABLE, [named_table]), + _Pbc = stanchion_utils:make_pbc(), + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + + +-spec init([]) -> {ok, {supervisor:sup_flags(), [supervisor:child_spec()]}}. +init([]) -> + Children = [], + {ok, {#{strategy => one_for_one, + intensity => 10, + period => 10}, Children + }}. + + + +stanchion_process_specs() -> + {Ip, Port, _SSL} = riak_cs_config:stanchion(), + + %% Hide any bags from user-facing parts. + case application:get_env(riak_cs, supercluster_members) of + undefined -> ok; + {ok, Bags} -> application:set_env(riak_cs, bags, Bags) + end, + + WebConfig1 = [{dispatch, stanchion_web:dispatch_table()}, + {ip, Ip}, + {port, Port}, + {nodelay, true}, + {log_dir, "log"}, + {error_handler, stanchion_wm_error_handler} + ], + WebConfig = + case application:get_env(riak_cs, stanchion_ssl) of + {ok, true} -> + {ok, CF} = application:get_env(riak_cs, stanchion_ssl_certfile), + {ok, KF} = application:get_env(riak_cs, stanchion_ssl_keyfile), + WebConfig1 ++ [{ssl, true}, + {ssl_opts, [{certfile, CF}, {keyfile, KF}]}]; + {ok, false} -> + WebConfig1 + end, + Web = + #{id => stanchion_webmachine, + start => {webmachine_mochiweb, start, [WebConfig]}, + restart => permanent, + shutdown => 5000, + modules => dynamic}, + ServerSup = + #{id => stanchion_server_sup, + start => {stanchion_server_sup, start_link, []}, + restart => permanent, + shutdown => 5000, + type => supervisor, + modules => dynamic}, + [ServerSup, Web]. diff --git a/apps/riak_cs/src/stanchion_utils.erl b/apps/riak_cs/src/stanchion_utils.erl new file mode 100644 index 000000000..ac3ad9bea --- /dev/null +++ b/apps/riak_cs/src/stanchion_utils.erl @@ -0,0 +1,854 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc stanchion utility functions + +-module(stanchion_utils). + +%% Public API +-export([create_bucket/2, + delete_bucket/3, + create_user/2, + delete_user/2, + update_user/2, + create_role/2, + update_role/2, + delete_role/2, + create_policy/2, + update_policy/2, + delete_policy/2, + create_saml_provider/2, + delete_saml_provider/2, + set_bucket_acl/3, + set_bucket_policy/3, + set_bucket_versioning/3, + delete_bucket_policy/3 + ]). +-export([get_manifests_raw/4, + get_pbc/0, + has_tombstone/1, + make_pbc/0, + to_bucket_name/2, + sha_mac/2 + ]). + +-include("riak_cs.hrl"). +-include("stanchion.hrl"). +-include("manifest.hrl"). +-include("moss.hrl"). +-include_lib("riakc/include/riakc.hrl"). +-include_lib("riak_pb/include/riak_pb_kv_codec.hrl"). +-include_lib("kernel/include/logger.hrl"). + + +%% this riak connection is separate, potentially to a different riak +%% endpoint, from the standard one obtained via riak_cs_utils +-spec make_pbc() -> pid(). +make_pbc() -> + {Host, Port} = + case riak_cs_config:tussle_voss_riak_host() of + auto -> + {H,P} = riak_cs_config:riak_host_port(), + logger:info("using main riak cluster for voss data at ~s:~b", [H, P]), + {H,P}; + Configured -> + Configured + end, + Timeout = application:get_env(riak_cs, riakc_connect_timeout, 10000), + StartOptions = [{connect_timeout, Timeout}, + {auto_reconnect, true}], + {ok, Pid} = riakc_pb_socket:start_link(Host, Port, StartOptions), + ets:insert(?STANCHION_OWN_PBC_TABLE, {pid, Pid}), + Pid. + +get_pbc() -> + [{pid, Pid}] = ets:lookup(?STANCHION_OWN_PBC_TABLE, pid), + case is_process_alive(Pid) of + true -> + Pid; + false -> + ?LOG_WARNING("voss riakc process ~p exited; spawning a new one." + " Check riak is reachable as configured (~p)", + [Pid, riak_cs_config:tussle_voss_riak_host()]), + make_pbc(), + timer:sleep(1000), + get_pbc() + end. + + +%% @doc Create a bucket in the global namespace or return +%% an error if it already exists. +-spec create_bucket(maps:map(), pid()) -> ok | {error, term()}. +create_bucket(#{bucket := Bucket, + requester := OwnerId, + acl := Acl_} = FF, Pbc) -> + Acl = riak_cs_acl:exprec_acl(Acl_), + BagId = maps:get(bag, FF, undefined), + OpResult1 = do_bucket_op(Bucket, OwnerId, [{acl, Acl}, {bag, BagId}], create, Pbc), + case OpResult1 of + ok -> + BucketRecord = bucket_record(Bucket, create), + case riak_cs_iam:find_user(#{key_id => OwnerId}, Pbc) of + {ok, {User, UserObj}} -> + UpdUser = update_user_buckets(add, User, BucketRecord), + save_user(UpdUser, UserObj, Pbc); + {error, _} = ER -> + ER + end; + {error, _} -> + OpResult1 + end. + +bucket_record(Name, Operation) -> + Action = case Operation of + create -> created; + delete -> deleted + end, + ?RCS_BUCKET{name = Name, + last_action = Action, + modification_time = os:system_time(millisecond)}. + + +%% @doc Delete a bucket +-spec delete_bucket(binary(), binary(), pid()) -> ok | {error, term()}. +delete_bucket(Bucket, OwnerId, Pbc) -> + OpResult1 = do_bucket_op(Bucket, OwnerId, [{acl, ?ACL{}}], delete, Pbc), + case OpResult1 of + ok -> + BucketRecord = bucket_record(Bucket, delete), + {ok, {User, UserObj}} = riak_cs_iam:find_user(#{key_id => OwnerId}, Pbc), + UpdUser = update_user_buckets(delete, User, BucketRecord), + save_user(UpdUser, UserObj, Pbc); + {error, _} -> + OpResult1 + end. + + +%% @doc Attempt to create a new user +-spec create_user(maps:map(), pid()) -> ok | {error, term()}. +create_user(FF = #{arn := Arn, + name := Name, + email := Email}, Pbc) -> + case email_and_name_available(Email, Name, Pbc) of + true -> + User = riak_cs_iam:unarm( + riak_cs_iam:exprec_user(FF)), + save_user(User, riakc_obj:new(?USER_BUCKET, Arn, term_to_binary(User)), Pbc); + {false, Reason} -> + logger:info("Refusing to create a user with email ~s: ~s", [Email, Reason]), + {error, user_already_exists} + end. + +-spec delete_user(binary(), pid()) -> ok | {error, term()}. +delete_user(Arn_, Pbc) -> + Arn = base64:decode(Arn_), + case ?TURNAROUND_TIME( + riakc_pb_socket:delete(Pbc, ?USER_BUCKET, Arn, ?CONSISTENT_DELETE_OPTIONS)) of + {ok, TAT} -> + stanchion_stats:update([riakc, delete_cs_user], TAT); + {{error, Reason} = ER, _} -> + logger:error("Failed to delete user object ~s: ~p", [Arn, Reason]), + ER + end. + +-spec update_user(maps:map(), pid()) -> {ok, rcs_user()} | {error, term()}. +update_user(FF, Pbc) -> + User = ?IAM_USER{arn = Arn, + key_id = KeyId, + email = Email} = + riak_cs_iam:exprec_user(FF), + RObj = + case riak_cs_iam:get_user(Arn, Pbc) of + {ok, {_, Obj32}} -> + Obj32; + {error, notfound} -> + case riak_cs_iam:get_user(KeyId, Pbc) of + {ok, {_, Obj31}} -> + Obj31; + {error, notfound} -> + logger:error("User ~s (~s) is gone while it is being updated. " + "Trying to create it as new.", [Arn, KeyId]), + riakc_obj:new(?USER_BUCKET, Arn, term_to_binary(User)) + end + end, + + CanProceed = + case riak_cs_iam:find_user(#{email => Email}, Pbc) of + {ok, {?IAM_USER{arn = Arn}, _}} -> + true; %% found self + {ok, {?IAM_USER{email = Email}, _}} -> + false; %% found some other user with this email + {error, notfound} -> + true + end, + if CanProceed -> + case save_user(User, RObj, Pbc) of + ok -> + {ok, User}; + ER -> + ER + end; + el/=se -> + {error, user_already_exists} + end. + + +%% @doc Determine if a user with the specified email +%% address already exists. There could be consistency +%% issues here since secondary index queries use +%% coverage and only consult a single vnode +%% for a particular key. +%% @TODO Consider other options that would give more +%% assurance that a particular email address is available. +email_and_name_available(Email, Name, Pbc) -> + case {riakc_pb_socket:get_index_eq(Pbc, ?USER_BUCKET, ?USER_EMAIL_INDEX, Email), + riakc_pb_socket:get_index_eq(Pbc, ?USER_BUCKET, ?USER_NAME_INDEX, Name)} of + {{ok, ?INDEX_RESULTS{keys = []}}, + {ok, ?INDEX_RESULTS{keys = []}}} -> + true; + {{ok, _}, {ok, _}} -> + {false, user_already_exists}; + {{error, Reason}, _} -> + %% @TODO Maybe bubble up this error info + logger:warning("Error occurred trying to check if the email ~p has been registered. Reason: ~p", + [Email, Reason]), + {false, Reason} + end. + + +-spec create_role(maps:map(), pid()) -> {ok, role()} | {error, already_exists|term()}. +create_role(Fields, Pbc) -> + R = ?IAM_ROLE{role_name = Name} = + riak_cs_iam:unarm( + riak_cs_iam:exprec_role( + riak_cs_iam:fix_permissions_boundary(Fields))), + case role_name_available(Name, Pbc) of + true -> + save_role(R, Pbc); + false -> + {error, role_already_exists} + end. + +-spec update_role(maps:map(), pid()) -> ok | {error, already_exists|term()}. +update_role(Fields, Pbc) -> + Role = + riak_cs_iam:unarm( + riak_cs_iam:exprec_role(Fields)), + save_role_directly(Role, Pbc). + +role_name_available(Name, Pbc) -> + (riak_cs_iam:find_role(#{name => Name}, Pbc) == {error, notfound}). + +save_role(Role0 = ?IAM_ROLE{role_name = Name, + path = Path}, Pbc) -> + Id = riak_cs_aws_utils:make_unique_index_id(role, Pbc), + Arn = riak_cs_aws_utils:make_role_arn(Name, Path), + Role1 = Role0?IAM_ROLE{arn = Arn, + role_id = Id}, + + Meta = dict:store(?MD_INDEX, riak_cs_utils:object_indices(Role1), dict:new()), + Obj = riakc_obj:update_metadata( + riakc_obj:new(?IAM_ROLE_BUCKET, Arn, term_to_binary(Role1)), + Meta), + {Res, TAT} = ?TURNAROUND_TIME(riakc_pb_socket:put(Pbc, Obj, ?CONSISTENT_WRITE_OPTIONS)), + case Res of + ok -> + ok = stanchion_stats:update([riakc, put_cs_role], TAT), + {ok, Role1}; + {error, Reason} -> + logger:error("Failed to save role \"~s\": ~p", [Name, Reason]), + Res + end. + +save_role_directly(Role = ?IAM_ROLE{arn = Arn, + role_name = Name}, Pbc) -> + Meta = dict:store(?MD_INDEX, riak_cs_utils:object_indices(Role), dict:new()), + Obj = riakc_obj:update_metadata( + riakc_obj:new(?IAM_ROLE_BUCKET, Arn, term_to_binary(Role)), + Meta), + {Res, TAT} = ?TURNAROUND_TIME(riakc_pb_socket:put(Pbc, Obj, ?CONSISTENT_WRITE_OPTIONS)), + case Res of + ok -> + ok = stanchion_stats:update([riakc, put_cs_role], TAT), + ok; + {error, Reason} -> + logger:error("Failed to save role \"~s\": ~p", [Name, Reason]), + Res + end. + +-spec delete_role(binary(), pid()) -> ok | {error, term()}. +delete_role(Arn, Pbc) -> + case ?TURNAROUND_TIME( + riakc_pb_socket:delete(Pbc, ?IAM_ROLE_BUCKET, Arn, ?CONSISTENT_DELETE_OPTIONS)) of + {ok, TAT} -> + stanchion_stats:update([riakc, delete_cs_role], TAT); + {{error, Reason} = ER, _} -> + logger:error("Failed to delete role object ~s: ~p", [Arn, Reason]), + ER + end. + + +-spec create_policy(maps:map(), pid()) -> {ok, iam_policy()} | {error, term()}. +create_policy(Fields, Pbc) -> + P = ?IAM_POLICY{policy_name = Name} = + riak_cs_iam:unarm( + riak_cs_iam:exprec_iam_policy(Fields)), + case policy_name_available(Name, Pbc) of + true -> + save_policy(P, Pbc); + false -> + {error, policy_already_exists} + end. + +-spec update_policy(maps:map(), pid()) -> ok | {error, term()}. +update_policy(Fields, Pbc) -> + Policy = riak_cs_iam:unarm( + riak_cs_iam:exprec_iam_policy(Fields)), + save_policy_directly(Policy, Pbc). + +policy_name_available(Name, Pbc) -> + {error, notfound} == riak_cs_iam:find_policy(#{name => Name}, Pbc). + +save_policy(Policy0 = ?IAM_POLICY{policy_name = Name, + path = Path}, Pbc) -> + Id = riak_cs_aws_utils:make_unique_index_id(policy, Pbc), + Arn = riak_cs_aws_utils:make_policy_arn(Name, Path), + Policy1 = Policy0?IAM_POLICY{arn = Arn, + policy_id = Id}, + + Meta = dict:store(?MD_INDEX, riak_cs_utils:object_indices(Policy1), dict:new()), + Obj = riakc_obj:update_metadata( + riakc_obj:new(?IAM_POLICY_BUCKET, Arn, term_to_binary(Policy1)), + Meta), + {Res, TAT} = ?TURNAROUND_TIME(riakc_pb_socket:put(Pbc, Obj, ?CONSISTENT_WRITE_OPTIONS)), + case Res of + ok -> + ok = stanchion_stats:update([riakc, put_cs_policy], TAT), + {ok, Policy1}; + {error, Reason} -> + logger:error("Failed to save managed policy \"~s\": ~p", [Name, Reason]), + Res + end. + +save_policy_directly(Policy = ?IAM_POLICY{arn = Arn, + policy_name = Name}, Pbc) -> + Meta = dict:store(?MD_INDEX, riak_cs_utils:object_indices(Policy), dict:new()), + Obj = riakc_obj:update_metadata( + riakc_obj:new(?IAM_POLICY_BUCKET, Arn, term_to_binary(Policy)), + Meta), + {Res, TAT} = ?TURNAROUND_TIME(riakc_pb_socket:put(Pbc, Obj, ?CONSISTENT_WRITE_OPTIONS)), + case Res of + ok -> + ok = stanchion_stats:update([riakc, put_cs_policy], TAT), + ok; + {error, Reason} -> + logger:error("Failed to save managed policy \"~s\": ~p", [Name, Reason]), + Res + end. + +-spec delete_policy(binary(), pid()) -> ok | {error, term()}. +delete_policy(Arn, Pbc) -> + case ?TURNAROUND_TIME( + riakc_pb_socket:delete(Pbc, ?IAM_POLICY_BUCKET, Arn, ?CONSISTENT_DELETE_OPTIONS)) of + {ok, TAT} -> + stanchion_stats:update([riakc, delete_cs_policy], TAT); + {{error, Reason} = ER, _} -> + logger:error("Failed to delete managed policy object ~s: ~p", [Arn, Reason]), + ER + end. + + +-spec create_saml_provider(maps:map(), pid()) -> {ok, {flat_arn(), [tag()]}} | {error, term()}. +create_saml_provider(Fields, Pbc) -> + P0 = ?IAM_SAML_PROVIDER{name = Name} = + riak_cs_iam:unarm( + riak_cs_iam:exprec_saml_provider(Fields)), + case riak_cs_iam:parse_saml_provider_idp_metadata(P0) of + {ok, P9} -> + case saml_provider_name_available(Name, Pbc) of + true -> + save_saml_provider(P9, Pbc); + false -> + {error, saml_provider_already_exists} + end; + ER -> + ER + end. + +saml_provider_name_available(Name, Pbc) -> + {error, notfound} == riak_cs_iam:find_saml_provider(#{name => Name}, Pbc). + +save_saml_provider(P0 = ?IAM_SAML_PROVIDER{name = Name, + tags = Tags}, Pbc) -> + Arn = riak_cs_aws_utils:make_provider_arn(Name), + P1 = P0?IAM_SAML_PROVIDER{arn = Arn}, + + Meta = dict:store(?MD_INDEX, riak_cs_utils:object_indices(P1), dict:new()), + Obj = riakc_obj:update_metadata( + riakc_obj:new(?IAM_SAMLPROVIDER_BUCKET, Arn, term_to_binary(P1)), + Meta), + {Res, TAT} = ?TURNAROUND_TIME(riakc_pb_socket:put(Pbc, Obj, ?CONSISTENT_WRITE_OPTIONS)), + case Res of + ok -> + ok = stanchion_stats:update([riakc, put_cs_samlprovider], TAT), + {ok, {Arn, Tags}}; + {error, Reason} -> + logger:error("Failed to save SAML provider \"~s\": ~p", [Reason]), + Res + end. + + +-spec delete_saml_provider(binary(), pid()) -> ok | {error, term()}. +delete_saml_provider(Arn, Pbc) -> + case ?TURNAROUND_TIME( + riakc_pb_socket:delete(Pbc, ?IAM_SAMLPROVIDER_BUCKET, Arn, ?CONSISTENT_DELETE_OPTIONS)) of + {ok, TAT} -> + stanchion_stats:update([riakc, put_cs_samlprovider], TAT); + {{error, Reason} = ER, _} -> + logger:error("Failed to delete saml_provider object ~s: ~p", [Arn, Reason]), + ER + end. + + +%% @doc +-spec get_manifests(pid(), binary(), binary(), binary()) -> + {ok, term(), term()} | {error, notfound}. +get_manifests(RiakcPid, Bucket, Key, Vsn) -> + case get_manifests_raw(RiakcPid, Bucket, Key, Vsn) of + {ok, Object} -> + DecodedSiblings = [binary_to_term(V) || + {_, V}=Content <- riakc_obj:get_contents(Object), + not has_tombstone(Content)], + + %% Upgrade the manifests to be the latest erlang + %% record version + Upgraded = rcs_common_manifest_utils:upgrade_wrapped_manifests(DecodedSiblings), + + %% resolve the siblings + Resolved = rcs_common_manifest_resolution:resolve(Upgraded), + + %% prune old scheduled_delete manifests + + %% commented out because we don't have the + %% riak_cs_gc module + Pruned = rcs_common_manifest_utils:prune( + Resolved, os:system_time(millisecond), + 50, %% riak_cs defaults for max_scheduled_delete_manifests and + 86400), %% leeway_seconds + {ok, Object, Pruned}; + {error, notfound} = NotFound -> + NotFound + end. + +%% @doc Determine if a set of contents of a riak object has a tombstone. +-spec has_tombstone({dict:dict(), binary()}) -> boolean(). +has_tombstone({_, <<>>}) -> + true; +has_tombstone({MD, _V}) -> + dict:is_key(<<"X-Riak-Deleted">>, MD) =:= true. + +%% @doc List the keys from a bucket +list_keys(BucketName, Pbc) -> + case ?TURNAROUND_TIME(riakc_pb_socket:list_keys(Pbc, BucketName)) of + {{ok, Keys}, TAT} -> + stanchion_stats:update([riakc, list_all_manifest_keys], TAT), + {ok, lists:sort(Keys)}; + {{error, _} = ER, _} -> + ER + end. + +%% @doc Set the ACL for a bucket +-spec set_bucket_acl(binary(), maps:map(), pid()) -> ok | {error, term()}. +set_bucket_acl(Bucket, #{requester := OwnerId, + acl := Acl}, Pbc) -> + do_bucket_op(Bucket, OwnerId, [{acl, Acl}], update_acl, Pbc). + +%% @doc add bucket policy in the global namespace +%% FieldList.policy has JSON-encoded policy from user +-spec set_bucket_policy(binary(), maps:map(), pid()) -> ok | {error, term()}. +set_bucket_policy(Bucket, #{requester := Requester, + policy := PolicyJson}, Pbc) -> + do_bucket_op(Bucket, Requester, [{policy, base64:decode(PolicyJson)}], update_policy, Pbc). + +%% @doc set bucket versioning option +-spec set_bucket_versioning(binary(), maps:map(), pid()) -> ok | {error, term()}. +set_bucket_versioning(Bucket, #{requester := Requester, + versioning := Versioning}, Pbc) -> + do_bucket_op(Bucket, Requester, [{versioning, riak_cs_bucket:exprec_bucket_versioning( + Versioning)}], update_versioning, Pbc). + + +%% @doc Delete a bucket +-spec delete_bucket_policy(binary(), binary(), pid()) -> ok | {error, term()}. +delete_bucket_policy(Bucket, OwnerId, Pbc) -> + do_bucket_op(Bucket, OwnerId, [delete_policy], delete_policy, Pbc). + +%% Get the proper bucket name for either the MOSS object +%% bucket or the data block bucket. +-spec to_bucket_name(objects | blocks, binary()) -> binary(). +to_bucket_name(Type, Bucket) -> + case Type of + objects -> + Prefix = ?OBJECT_BUCKET_PREFIX; + blocks -> + Prefix = ?BLOCK_BUCKET_PREFIX + end, + BucketHash = md5(Bucket), + <>. + + +sha_mac(KeyData, STS) -> + crypto:mac(hmac, sha, KeyData, STS). + +md5(Bin) -> + crypto:hash(md5, Bin). + +%% =================================================================== +%% Internal functions +%% =================================================================== + +%% @doc Check if a bucket is empty +-spec bucket_empty(binary(), pid()) -> boolean(). +bucket_empty(Bucket, Pbc) -> + ManifestBucket = to_bucket_name(objects, Bucket), + %% @TODO Use `stream_list_keys' instead and + ListKeysResult = list_keys(ManifestBucket, Pbc), + bucket_empty_handle_list_keys(Pbc, + Bucket, + ListKeysResult). + +bucket_empty_handle_list_keys(Pbc, Bucket, {ok, Keys}) -> + %% `lists:any/2' will break out early as soon + %% as something returns `true' + not lists:any( + fun (Key) -> key_exists(Pbc, Bucket, Key) end, + Keys); +bucket_empty_handle_list_keys(_RiakcPid, _Bucket, _Error) -> + false. + +key_exists(Pbc, Bucket, Key) -> + key_exists_handle_get_manifests( + get_manifests(Pbc, Bucket, Key, ?LFS_DEFAULT_OBJECT_VERSION)). + +key_exists_handle_get_manifests({ok, _Object, Manifests}) -> + active_to_bool(active_manifest_from_response({ok, Manifests})); +key_exists_handle_get_manifests(Error) -> + active_to_bool(active_manifest_from_response(Error)). + +active_to_bool({ok, _Active}) -> + true; +active_to_bool({error, notfound}) -> + false. + +active_manifest_from_response({ok, Manifests}) -> + handle_active_manifests( + rcs_common_manifest_utils:active_manifest(Manifests)); +active_manifest_from_response({error, notfound}=NotFound) -> + NotFound. + +handle_active_manifests({ok, _Active}=ActiveReply) -> + ActiveReply; +handle_active_manifests({error, no_active_manifest}) -> + {error, notfound}. + +bucket_available(Bucket, RequesterId, BucketOp, Pbc) -> + GetOptions = [{pr, all}], + case ?TURNAROUND_TIME(riakc_pb_socket:get(Pbc, ?BUCKETS_BUCKET, Bucket, GetOptions)) of + {{ok, BucketObj}, TAT} -> + stanchion_stats:update([riakc, get_cs_bucket], TAT), + OwnerId = riakc_obj:get_value(BucketObj), + case {OwnerId, BucketOp} of + {?FREE_BUCKET_MARKER, create} -> + is_bucket_ready_to_create(Bucket, Pbc, BucketObj); + {?FREE_BUCKET_MARKER, _} -> + {false, no_such_bucket}; + + {RequesterId, delete} -> + is_bucket_ready_to_delete(Bucket, Pbc, BucketObj); + {RequesterId, _} -> + {true, BucketObj}; + _ -> + {false, bucket_already_exists} + end; + + {{error, notfound}, TAT} -> + stanchion_stats:update([riakc, get_cs_bucket], TAT), + case BucketOp of + create -> + BucketObj = riakc_obj:new(?BUCKETS_BUCKET, Bucket, RequesterId), + {true, BucketObj}; + update_acl -> + {false, no_such_bucket}; + update_policy -> + {false, no_such_bucket}; + update_versioning -> + {false, no_such_bucket}; + delete -> + {false, no_such_bucket} + end; + + {{error, Reason}, TAT} -> + stanchion_stats:update([riakc, get_cs_bucket], TAT), + %% @TODO Maybe bubble up this error info + logger:warning("Error occurred trying to check if the bucket ~p exists. Reason: ~p", + [Bucket, Reason]), + {false, Reason} + end. + + +do_bucket_op(<<"riak-cs">>, _OwnerId, _Opts, _BucketOp, _Pbc) -> + {error, access_denied}; +do_bucket_op(Bucket, OwnerId, Opts, BucketOp, Pbc) -> + %% Buckets operations can only be completed if the bucket exists + %% and the requesting party owns the bucket. + case bucket_available(Bucket, OwnerId, BucketOp, Pbc) of + {true, BucketObj} -> + Value = case BucketOp of + create -> OwnerId; + update_acl -> OwnerId; + update_policy -> OwnerId; + delete_policy -> OwnerId; + update_versioning -> OwnerId; + delete -> ?FREE_BUCKET_MARKER + end, + put_bucket(BucketObj, Value, Opts, Pbc); + {false, Reason1} -> + {error, Reason1} + end. + +%% @doc Store a new bucket in Riak +%% though whole metadata itself is a dict, a metadata of ?MD_USERMETA is +%% proplists of {?MD_ACL, ACL::binary()}|{?MD_POLICY, PolicyBin::binary()}| +%% {?MD_BAG, BagId::binary()}, {?MD_VERSIONING, bucket_versioning_option()}}. +%% should preserve other metadata. ACL and Policy can be overwritten. +put_bucket(BucketObj, OwnerId, Opts, Pbc) -> + PutOptions = [{w, all}, {pw, all}], + UpdBucketObj0 = riakc_obj:update_value(BucketObj, OwnerId), + MD = case riakc_obj:get_metadatas(UpdBucketObj0) of + [] -> % create + dict:from_list([{?MD_USERMETA, []}]); + [MD0] -> MD0; + _E -> + MsgData = {siblings, riakc_obj:key(BucketObj)}, + logger:warning("bucket has siblings: ~p", [MsgData]), + throw(MsgData) % @TODO: data broken; handle this + end, + MetaData = make_new_metadata(MD, Opts), + UpdBucketObj = riakc_obj:update_metadata(UpdBucketObj0, MetaData), + {Result, TAT} = ?TURNAROUND_TIME(riakc_pb_socket:put(Pbc, UpdBucketObj, PutOptions)), + stanchion_stats:update([riakc, put_cs_bucket], TAT), + Result. + +make_new_metadata(MD, Opts) -> + MetaVals = dict:fetch(?MD_USERMETA, MD), + UserMetaData = make_new_user_metadata(MetaVals, Opts), + dict:store(?MD_USERMETA, UserMetaData, dict:erase(?MD_USERMETA, MD)). + +make_new_user_metadata(MetaVals, [])-> + MetaVals; +make_new_user_metadata(MetaVals, [{acl, Acl} | Opts]) -> + make_new_user_metadata(replace_meta(?MD_ACL, Acl, MetaVals), Opts); +make_new_user_metadata(MetaVals, [{policy, Policy} | Opts]) -> + make_new_user_metadata(replace_meta(?MD_POLICY, Policy, MetaVals), Opts); +make_new_user_metadata(MetaVals, [{bag, undefined} | Opts]) -> + make_new_user_metadata(MetaVals, Opts); +make_new_user_metadata(MetaVals, [{bag, BagId} | Opts]) -> + make_new_user_metadata(replace_meta(?MD_BAG, BagId, MetaVals), Opts); +make_new_user_metadata(MetaVals, [delete_policy | Opts]) -> + make_new_user_metadata(proplists:delete(?MD_POLICY, MetaVals), Opts); +make_new_user_metadata(MetaVals, [{versioning, VsnOpt} | Opts]) -> + make_new_user_metadata(replace_meta(?MD_VERSIONING, VsnOpt, MetaVals), Opts). + +replace_meta(Key, NewValue, MetaVals) -> + [{Key, term_to_binary(NewValue)} | proplists:delete(Key, MetaVals)]. + + +%% @doc bucket is ok to delete when bucket is empty. Ongoing multipart +%% uploads are all supposed to be automatically aborted by Riak CS. +%% If the bucket still has active objects, just fail. Else if the +%% bucket still has ongoing multipart, Stanchion returns error and +%% Riak CS retries some times, in case of concurrent multipart +%% initiation occuring. After a few retry Riak CS will eventually +%% returns error to the client (maybe 500?) Or fallback to heavy +%% abort-all-multipart and then deletes bucket? This will be a big +%% TODO. +is_bucket_ready_to_delete(Bucket, Pbc, BucketObj) -> + is_bucket_clean(Bucket, Pbc, BucketObj). + +%% @doc ensure there are no multipart uploads in creation time because +%% multipart uploads remains in deleted buckets in former versions +%% before 1.5.0 (or 1.4.6) where the bug (identified in riak_cs/#475). +is_bucket_ready_to_create(Bucket, Pbc, BucketObj) -> + is_bucket_clean(Bucket, Pbc, BucketObj). + +%% @doc here runs two list_keys, one in bucket_empty/2, another in +%% stanchion_multipart:check_no_multipart_uploads/2. If there are +%% bunch of pending_delete manifests this may slow (twice as before +%% #475 fix). If there are bunch of scheduled_delete manifests, this +%% may also slow, but wait for Garbage Collection to collect those +%% trashes may improve the speed. => TODO. +is_bucket_clean(Bucket, Pbc, BucketObj) -> + {ok, ManifestPbc} = manifest_connection(Pbc, BucketObj), + try + case bucket_empty(Bucket, ManifestPbc) of + false -> + {false, bucket_not_empty}; + true -> + case stanchion_multipart:check_no_multipart_uploads(Bucket, ManifestPbc) of + false -> + {false, remaining_multipart_upload}; + true -> + {true, BucketObj} + end + end + catch T:E:ST -> + logger:error("Could not check whether bucket was empty. Reason: ~p:~p - ~p", + [T, E, ST]), + error({T, E}) + after + close_manifest_connection(Pbc, ManifestPbc) + end. + +manifest_connection(Pbc, BucketObj) -> + case bag_id_from_bucket(BucketObj) of + undefined -> + {ok, Pbc}; + BagId -> + case conn_info_from_bag(BagId, application:get_env(riak_cs, bags)) of + %% No connection information for the bag. Mis-configuration. Stop processing. + undefined -> + {error, {no_bag, BagId}}; + {Address, Port} -> + riak_connection(Address, Port) + end + end. + + +conn_info_from_bag(_BagId, undefined) -> + undefined; +conn_info_from_bag(BagId, {ok, Bags}) -> + BagIdStr = binary_to_list(BagId), + case lists:keyfind(BagIdStr, 1, Bags) of + false -> + {error, no_bag}; + {BagIdStr, Address, Port} -> + {Address, Port} + end. + +bag_id_from_bucket(BucketObj) -> + Contents = riakc_obj:get_contents(BucketObj), + bag_id_from_contents(Contents). + +bag_id_from_contents([]) -> + undefined; +bag_id_from_contents([{MD, _} | Contents]) -> + case bag_id_from_meta(dict:fetch(?MD_USERMETA, MD)) of + undefined -> + bag_id_from_contents(Contents); + BagId -> + BagId + end. + +bag_id_from_meta([]) -> + undefined; +bag_id_from_meta([{?MD_BAG, Value} | _]) -> + binary_to_term(Value); +bag_id_from_meta([_MD | MDs]) -> + bag_id_from_meta(MDs). + +close_manifest_connection(Pbc, Pbc) -> + ok; +close_manifest_connection(_Pbc, ManifestPbc) -> + close_riak_connection(ManifestPbc). + +riak_connection(Host, Port) -> + %% We use start() here instead of start_link() because if we can't + %% connect to Host & Port for whatever reason (e.g. service down, + %% host down, host unreachable, ...), then we'll be crashed by the + %% newly-spawned-gen_server-proc's link to us. + %% + %% There is still a race condition if the PB socket proc's init() + %% is successful but then dies immediately *before* we call the + %% link() BIF. That's life in the big city. + case riakc_pb_socket:start(Host, Port) of + {ok, Pid} = Good -> + true = link(Pid), + Good; + {error, Else} -> + {error, {riak_connect_failed, {Else, Host, Port}}} + end. + +close_riak_connection(Pid) -> + riakc_pb_socket:stop(Pid). + + + +%% internal fun to retrieve the riak object +%% at a bucket/key +-spec get_manifests_raw(pid(), binary(), binary(), binary()) -> + {ok, riakc_obj:riakc_obj()} | {error, notfound}. +get_manifests_raw(Pbc, Bucket, Key, Vsn) -> + ManifestBucket = to_bucket_name(objects, Bucket), + {Res, TAT} = ?TURNAROUND_TIME( + riakc_pb_socket:get(Pbc, ManifestBucket, + rcs_common_manifest:make_versioned_key(Key, Vsn))), + stanchion_stats:update([riakc, get_manifest], TAT), + Res. + +save_user(User, Obj0, Pbc) -> + Meta = dict:store(?MD_INDEX, riak_cs_utils:object_indices(User), dict:new()), + Obj = riakc_obj:update_metadata( + riakc_obj:update_value(Obj0, term_to_binary(User)), + Meta), + {Res, TAT} = ?TURNAROUND_TIME(riakc_pb_socket:put(Pbc, Obj, ?CONSISTENT_WRITE_OPTIONS)), + stanchion_stats:update([riakc, put_cs_user], TAT), + normalize_ok(Res). + +normalize_ok(ok) -> ok; +normalize_ok({ok, _}) -> ok; +normalize_ok(Error) -> Error. + + + +update_user_buckets(add, User, Bucket) -> + Buckets = User?RCS_USER.buckets, + %% At this point any siblings from the read of the + %% user record have been resolved so the user bucket + %% list should have 0 or 1 buckets that share a name + %% with `Bucket'. + case [B || B <- Buckets, B?RCS_BUCKET.name =:= Bucket?RCS_BUCKET.name] of + [] -> + User?RCS_USER{buckets = [Bucket | Buckets]}; + [ExistingBucket] -> + UpdBuckets = [Bucket | lists:delete(ExistingBucket, Buckets)], + User?RCS_USER{buckets = UpdBuckets} + end; +update_user_buckets(delete, User, Bucket) -> + Buckets = User?RCS_USER.buckets, + case [B || B <- Buckets, B?RCS_BUCKET.name =:= Bucket?RCS_BUCKET.name] of + [] -> + logger:error("attempt to remove bucket ~s from user ~s who does not own it", + [Bucket?RCS_BUCKET.name, User?RCS_USER.name]), + User; + [ExistingBucket] -> + UpdBuckets = lists:delete(ExistingBucket, Buckets), + User?RCS_USER{buckets = UpdBuckets} + end. diff --git a/apps/riak_cs/src/stanchion_web.erl b/apps/riak_cs/src/stanchion_web.erl new file mode 100644 index 000000000..e9a6fb721 --- /dev/null +++ b/apps/riak_cs/src/stanchion_web.erl @@ -0,0 +1,50 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc Convenience functions for setting up the stanchion HTTP interface. + +-module(stanchion_web). + +-export([dispatch_table/0]). + +dispatch_table() -> + AuthBypass = + case application:get_env(riak_cs, auth_bypass) of + {ok, AuthBypass0} -> AuthBypass0; + undefined -> false + end, + [ + {["ping"], stanchion_wm_ping, [{auth_bypass, true}]}, + {["buckets"], stanchion_wm_buckets, [{auth_bypass, AuthBypass}]}, + {["buckets", bucket, "acl"], stanchion_wm_acl, [{auth_bypass, AuthBypass}]}, + {["buckets", bucket, "policy"], stanchion_wm_policy, [{auth_bypass, AuthBypass}]}, + {["buckets", bucket, "versioning"], stanchion_wm_versioning, [{auth_bypass, AuthBypass}]}, + {["buckets", bucket], stanchion_wm_bucket, [{auth_bypass, AuthBypass}]}, + {["users", key_id], stanchion_wm_user, [{auth_bypass, AuthBypass}]}, + {["users"], stanchion_wm_users, [{auth_bypass, AuthBypass}]}, + {["roles", arn], stanchion_wm_roles, [{auth_bypass, AuthBypass}]}, + {["roles"], stanchion_wm_roles, [{auth_bypass, AuthBypass}]}, + {["policies", arn], stanchion_wm_policies, [{auth_bypass, AuthBypass}]}, + {["policies"], stanchion_wm_policies, [{auth_bypass, AuthBypass}]}, + {["samlproviders"], stanchion_wm_samlprovider, [{auth_bypass, AuthBypass}]}, + {["samlproviders", arn], stanchion_wm_samlprovider, [{auth_bypass, AuthBypass}]}, + {["stats"], stanchion_wm_stats, [{auth_bypass, AuthBypass}]} + ]. diff --git a/apps/riak_cs/src/stanchion_wm_acl.erl b/apps/riak_cs/src/stanchion_wm_acl.erl new file mode 100644 index 000000000..f70ce94b0 --- /dev/null +++ b/apps/riak_cs/src/stanchion_wm_acl.erl @@ -0,0 +1,112 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_wm_acl). + +-export([init/1, + service_available/2, + is_authorized/2, + content_types_provided/2, + malformed_request/2, + to_json/2, + allowed_methods/2, + content_types_accepted/2, + accept_body/2 + ]). + +-ignore_xref([init/1, + service_available/2, + is_authorized/2, + content_types_provided/2, + malformed_request/2, + to_json/2, + allowed_methods/2, + content_types_accepted/2, + accept_body/2 + ]). + +-include("stanchion.hrl"). +-include_lib("webmachine/include/webmachine.hrl"). + +init(Config) -> + %% Check if authentication is disabled and + %% set that in the context. + AuthBypass = proplists:get_value(auth_bypass, Config), + {ok, #stanchion_context{auth_bypass=AuthBypass}}. + +-spec service_available(term(), term()) -> {true, term(), term()}. +service_available(RD, Ctx) -> + stanchion_wm_utils:service_available(RD, Ctx). + +-spec malformed_request(term(), term()) -> {false, term(), term()}. +malformed_request(RD, Ctx) -> + {false, RD, Ctx}. + +%% @doc Check that the request is from the admin user +is_authorized(RD, Ctx=#stanchion_context{auth_bypass=AuthBypass}) -> + AuthHeader = wrq:get_req_header("authorization", RD), + case stanchion_wm_utils:parse_auth_header(AuthHeader, AuthBypass) of + {ok, AuthMod, Args} -> + case AuthMod:authenticate(RD, Args) of + ok -> + %% Authentication succeeded + {true, RD, Ctx}; + {error, _Reason} -> + %% Authentication failed, deny access + stanchion_response:api_error(access_denied, RD, Ctx) + end + end. + +%% @doc Get the list of methods this resource supports. +-spec allowed_methods(term(), term()) -> {[atom()], term(), term()}. +allowed_methods(RD, Ctx) -> + {['GET', 'PUT'], RD, Ctx}. + +-spec content_types_provided(term(), term()) -> + {[{string(), atom()}], term(), term()}. +content_types_provided(RD, Ctx) -> + {[{"application/json", to_xml}], RD, Ctx}. + +-spec content_types_accepted(term(), term()) -> + {[{string(), atom()}], term(), term()}. +content_types_accepted(RD, Ctx) -> + {[{"application/json", accept_body}], RD, Ctx}. + +-spec to_json(term(), term()) -> + {{'halt', term()}, #wm_reqdata{}, term()}. +to_json(RD, Ctx) -> + Bucket = wrq:path_info(bucket, RD), + stanchion_response:list_buckets_response(Bucket, + RD, + Ctx). + +%% @doc Process the request body on `PUT'. +accept_body(RD, Ctx) -> + Bucket = list_to_binary(wrq:path_info(bucket, RD)), + Body = wrq:req_body(RD), + FF = #{acl := AclMap} = jsx:decode(Body, [{labels, atom}]), + case stanchion_server:set_bucket_acl( + Bucket, FF#{acl => riak_cs_acl:exprec_acl(AclMap)}) of + ok -> + {true, RD, Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. diff --git a/apps/riak_cs/src/stanchion_wm_bucket.erl b/apps/riak_cs/src/stanchion_wm_bucket.erl new file mode 100644 index 000000000..f3b43eb82 --- /dev/null +++ b/apps/riak_cs/src/stanchion_wm_bucket.erl @@ -0,0 +1,113 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_wm_bucket). + +-export([init/1, + service_available/2, + is_authorized/2, + content_types_provided/2, + malformed_request/2, + to_xml/2, + allowed_methods/2, + content_types_accepted/2, + %% accept_body/2, + delete_resource/2 + ]). +-ignore_xref([init/1, + service_available/2, + is_authorized/2, + content_types_provided/2, + malformed_request/2, + to_xml/2, + allowed_methods/2, + content_types_accepted/2, + %% accept_body/2, + delete_resource/2 + ]). + +-include("stanchion.hrl"). +-include_lib("webmachine/include/webmachine.hrl"). + + +init(Config) -> + AuthBypass = proplists:get_value(auth_bypass, Config), + {ok, #stanchion_context{auth_bypass=AuthBypass}}. + +-spec service_available(term(), term()) -> {true, term(), term()}. +service_available(RD, Ctx) -> + stanchion_wm_utils:service_available(RD, Ctx). + +-spec malformed_request(term(), term()) -> {false, term(), term()}. +malformed_request(RD, Ctx) -> + {false, RD, Ctx}. + +%% @doc Check that the request is from the admin user +is_authorized(RD, Ctx=#stanchion_context{auth_bypass=AuthBypass}) -> + AuthHeader = wrq:get_req_header("authorization", RD), + case stanchion_wm_utils:parse_auth_header(AuthHeader, AuthBypass) of + {ok, AuthMod, Args} -> + case AuthMod:authenticate(RD, Args) of + ok -> + %% Authentication succeeded + {true, RD, Ctx}; + {error, _Reason} -> + %% Authentication failed, deny access + stanchion_response:api_error(access_denied, RD, Ctx) + end + end. + +%% @doc Get the list of methods this resource supports. +-spec allowed_methods(term(), term()) -> {[atom()], term(), term()}. +allowed_methods(RD, Ctx) -> + {['GET', 'PUT', 'DELETE'], RD, Ctx}. + +-spec content_types_provided(term(), term()) -> + {[{string(), atom()}], term(), term()}. +content_types_provided(RD, Ctx) -> + %% @TODO Add JSON support + {[{"application/xml", to_xml}], RD, Ctx}. + +content_types_accepted(RD, Ctx) -> + case wrq:get_req_header("content-type", RD) of + undefined -> + {[{"application/octet-stream", accept_body}], RD, Ctx}; + CType -> + {[{CType, accept_body}], RD, Ctx} + end. + + +-spec to_xml(term(), term()) -> + {{'halt', _}, #wm_reqdata{}, term()}. +to_xml(RD, Ctx) -> + Bucket = wrq:path_info(bucket, RD), + stanchion_response:list_buckets_response(Bucket, RD, Ctx). + +-spec delete_resource(term(), term()) -> {'true' | {'halt', term()}, #wm_reqdata{}, term()}. +delete_resource(RD, Ctx) -> + Bucket = list_to_binary(wrq:path_info(bucket, RD)), + RequesterId = list_to_binary(wrq:get_qs_value("requester", "", RD)), + case stanchion_server:delete_bucket(Bucket, RequesterId) of + ok -> + {true, RD, Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. diff --git a/apps/riak_cs/src/stanchion_wm_buckets.erl b/apps/riak_cs/src/stanchion_wm_buckets.erl new file mode 100644 index 000000000..4666d80b1 --- /dev/null +++ b/apps/riak_cs/src/stanchion_wm_buckets.erl @@ -0,0 +1,107 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_wm_buckets). + +-export([init/1, + service_available/2, + allowed_methods/2, + is_authorized/2, + content_types_accepted/2, + post_is_create/2, + create_path/2, + accept_body/2 + ]). + +-ignore_xref([init/1, + service_available/2, + allowed_methods/2, + is_authorized/2, + content_types_accepted/2, + post_is_create/2, + create_path/2, + accept_body/2 + ]). + +-include("stanchion.hrl"). +-include_lib("webmachine/include/webmachine.hrl"). +-include_lib("kernel/include/logger.hrl"). + + +init(Config) -> + %% Check if authentication is disabled and + %% set that in the context. + AuthBypass = proplists:get_value(auth_bypass, Config), + {ok, #stanchion_context{auth_bypass=AuthBypass}}. + +-spec service_available(#wm_reqdata{}, #stanchion_context{}) -> {true, #wm_reqdata{}, #stanchion_context{}}. +service_available(RD, Ctx) -> + stanchion_wm_utils:service_available(RD, Ctx). + +%% @doc Get the list of methods this resource supports. +-spec allowed_methods(term(), term()) -> {[atom()], term(), term()}. +allowed_methods(RD, Ctx) -> + {['POST'], RD, Ctx}. + +%% @doc Check that the request is from the admin user +-spec is_authorized(#wm_reqdata{}, #stanchion_context{}) -> {boolean(), #wm_reqdata{}, #stanchion_context{}}. +is_authorized(RD, Ctx=#stanchion_context{auth_bypass=AuthBypass}) -> + AuthHeader = wrq:get_req_header("authorization", RD), + case stanchion_wm_utils:parse_auth_header(AuthHeader, AuthBypass) of + {ok, AuthMod, Args} -> + case AuthMod:authenticate(RD, Args) of + ok -> + %% Authentication succeeded + {true, RD, Ctx}; + {error, _Reason} -> + %% Authentication failed, deny access + stanchion_response:api_error(access_denied, RD, Ctx) + end + end. + +-spec post_is_create(#wm_reqdata{}, #stanchion_context{}) -> {true, #wm_reqdata{}, #stanchion_context{}}. +post_is_create(_RD, _Ctx) -> + {true, _RD, _Ctx}. + +%% @doc Set the path for the new bucket resource and set +%% the Location header to generate a 201 Created response. +-spec create_path(#wm_reqdata{}, #stanchion_context{}) -> {string(), #wm_reqdata{}, #stanchion_context{}}. +create_path(RD, Ctx) -> + {wrq:disp_path(RD), RD, Ctx}. + +-spec content_types_accepted(#wm_reqdata{}, #stanchion_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #stanchion_context{}}. +content_types_accepted(RD, Ctx) -> + {[{"application/json", accept_body}], RD, Ctx}. + +%% @doc Create a bucket from a POST +-spec accept_body(#wm_reqdata{}, #stanchion_context{}) -> + {true | {halt, pos_integer()}, + #wm_reqdata{}, #stanchion_context{}}. +accept_body(RD, Ctx) -> + Body = wrq:req_body(RD), + case stanchion_server:create_bucket( + jsx:decode(Body, [{labels, atom}])) of + ok -> + {true, RD, Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. diff --git a/apps/riak_cs/src/stanchion_wm_error_handler.erl b/apps/riak_cs/src/stanchion_wm_error_handler.erl new file mode 100644 index 000000000..84c262683 --- /dev/null +++ b/apps/riak_cs/src/stanchion_wm_error_handler.erl @@ -0,0 +1,37 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_wm_error_handler). + +-export([render_error/3]). +-ignore_xref([render_error/3]). + +render_error(500, Req, Reason) -> + {ok, ReqState} = Req:add_response_header("Content-Type", "text/html"), + {Path,_} = Req:path(), + logger:error("webmachine error: path=~p. Reason: ~p", [Path, Reason]), + STString = io_lib:format("~p", [Reason]), + ErrorStart = "500 Internal Server Error

Internal Server Error

The server encountered an error while processing this request:
",
+    ErrorEnd = "

mochiweb+webmachine web server
", + ErrorIOList = [ErrorStart,STString,ErrorEnd], + {erlang:iolist_to_binary(ErrorIOList), ReqState}; +render_error(_Code, Req, _Reason) -> + Req:response_body(). diff --git a/src/riak_cs_wm_ping.erl b/apps/riak_cs/src/stanchion_wm_ping.erl similarity index 76% rename from src/riak_cs_wm_ping.erl rename to apps/riak_cs/src/stanchion_wm_ping.erl index 30644f508..8ea29e746 100644 --- a/src/riak_cs_wm_ping.erl +++ b/apps/riak_cs/src/stanchion_wm_ping.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2022, 2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,62 +19,62 @@ %% %% --------------------------------------------------------------------- --module(riak_cs_wm_ping). +-module(stanchion_wm_ping). -export([init/1, service_available/2, allowed_methods/2, to_html/2, - finish_request/2]). + finish_request/2 + ]). --include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). +-ignore_xref([init/1, + service_available/2, + allowed_methods/2, + to_html/2, + finish_request/2 + ]). --record(ping_context, {pool_pid=true :: boolean(), - riak_client :: undefined | riak_client()}). +-include("riak_cs_web.hrl"). + +-record(ping_context, {pool_pid = true :: boolean(), + riak_client :: undefined | pid()}). %% ------------------------------------------------------------------- %% Webmachine callbacks %% ------------------------------------------------------------------- init(_Config) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"init">>), {ok, #ping_context{}}. -spec service_available(#wm_reqdata{}, #ping_context{}) -> {boolean(), #wm_reqdata{}, #ping_context{}}. service_available(RD, Ctx) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"service_available">>), {Available, UpdCtx} = riak_ping(get_connection_pid(), Ctx), {Available, RD, UpdCtx}. -spec allowed_methods(term(), term()) -> {[atom()], term(), term()}. allowed_methods(RD, Ctx) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"allowed_methods">>), {['GET', 'HEAD'], RD, Ctx}. to_html(ReqData, Ctx) -> {"OK", ReqData, Ctx}. finish_request(RD, Ctx=#ping_context{riak_client=undefined}) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"finish_request">>, [0], []), {true, RD, Ctx}; finish_request(RD, Ctx=#ping_context{riak_client=RcPid, pool_pid=PoolPid}) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"finish_request">>, [1], []), case PoolPid of true -> riak_cs_riak_client:checkin(RcPid); false -> riak_cs_riak_client:stop(RcPid) end, - riak_cs_dtrace:dt_wm_return(?MODULE, <<"finish_request">>, [1], []), {true, RD, Ctx#ping_context{riak_client=undefined}}. %% ------------------------------------------------------------------- %% Internal functions %% ------------------------------------------------------------------- --spec get_connection_pid() -> {riak_client(), boolean()}. get_connection_pid() -> case pool_checkout() of full -> @@ -82,7 +83,6 @@ get_connection_pid() -> {RcPid, true} end. --spec pool_checkout() -> full | riak_client(). pool_checkout() -> case riak_cs_riak_client:checkout(request_pool) of {ok, RcPid} -> @@ -91,7 +91,6 @@ pool_checkout() -> full end. --spec non_pool_connection() -> {undefined | riak_client(), false}. non_pool_connection() -> case riak_cs_riak_client:start_link([]) of {ok, RcPid} -> @@ -100,7 +99,6 @@ non_pool_connection() -> {undefined, false} end. --spec riak_ping({riak_client(), boolean()}, #ping_context{}) -> {boolean(), #ping_context{}}. riak_ping({RcPid, PoolPid}, Ctx) -> {ok, MasterPbc} = riak_cs_riak_client:master_pbc(RcPid), Timeout = riak_cs_config:ping_timeout(), diff --git a/apps/riak_cs/src/stanchion_wm_policies.erl b/apps/riak_cs/src/stanchion_wm_policies.erl new file mode 100644 index 000000000..2ac73a4cc --- /dev/null +++ b/apps/riak_cs/src/stanchion_wm_policies.erl @@ -0,0 +1,128 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_wm_policies). + +-export([init/1, + service_available/2, + allowed_methods/2, + is_authorized/2, + create_path/2, + post_is_create/2, + content_types_accepted/2, + accept_json/2, + delete_resource/2 + ]). + +-ignore_xref([init/1, + service_available/2, + allowed_methods/2, + is_authorized/2, + create_path/2, + post_is_create/2, + content_types_accepted/2, + accept_json/2, + delete_resource/2 + ]). + +-include("stanchion.hrl"). +-include("aws_api.hrl"). +-include_lib("webmachine/include/webmachine.hrl"). +-include_lib("kernel/include/logger.hrl"). + +init(Config) -> + AuthBypass = proplists:get_value(auth_bypass, Config), + {ok, #stanchion_context{auth_bypass=AuthBypass}}. + +-spec service_available(#wm_reqdata{}, #stanchion_context{}) -> {true, #wm_reqdata{}, #stanchion_context{}}. +service_available(RD, Ctx) -> + stanchion_wm_utils:service_available(RD, Ctx). + +-spec allowed_methods(#wm_reqdata{}, #stanchion_context{}) -> {[atom()], #wm_reqdata{}, #stanchion_context{}}. +allowed_methods(RD, Ctx) -> + {['POST', 'PUT', 'DELETE'], RD, Ctx}. + +-spec is_authorized(#wm_reqdata{}, #stanchion_context{}) -> {boolean(), #wm_reqdata{}, #stanchion_context{}}. +is_authorized(RD, Ctx = #stanchion_context{auth_bypass=AuthBypass}) -> + AuthHeader = wrq:get_req_header("authorization", RD), + case stanchion_wm_utils:parse_auth_header(AuthHeader, AuthBypass) of + {ok, AuthMod, Args} -> + case AuthMod:authenticate(RD, Args) of + ok -> + %% Authentication succeeded + {true, RD, Ctx}; + {error, _Reason} -> + %% Authentication failed, deny access + stanchion_response:api_error(access_denied, RD, Ctx) + end + end. + +-spec post_is_create(#wm_reqdata{}, #stanchion_context{}) -> {true, #wm_reqdata{}, #stanchion_context{}}. +post_is_create(_RD, _Ctx) -> + {true, _RD, _Ctx}. + +-spec create_path(#wm_reqdata{}, #stanchion_context{}) -> {string(), #wm_reqdata{}, #stanchion_context{}}. +create_path(RD, Ctx) -> + {wrq:disp_path(RD), RD, Ctx}. + +-spec content_types_accepted(#wm_reqdata{}, #stanchion_context{}) -> + {[{string(), module()}], #wm_reqdata{}, #stanchion_context{}}. +content_types_accepted(RD, Ctx) -> + {[{"application/json", accept_json}], RD, Ctx}. + +-spec accept_json(#wm_reqdata{}, #stanchion_context{}) -> + {true | {halt, pos_integer()}, #wm_reqdata{}, #stanchion_context{}}. +accept_json(RD, Ctx) -> + case wrq:method(RD) of + 'POST' -> + do_create(RD, Ctx); + 'PUT' -> + do_update(RD, Ctx) + end. + +do_create(RD, Ctx) -> + FF = jsx:decode(wrq:req_body(RD), [{labels, atom}]), + case stanchion_server:create_policy(FF) of + {ok, Policy} -> + Doc = riak_cs_json:to_json(Policy), + {true, wrq:set_resp_body(Doc, RD), Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. + +do_update(RD, Ctx) -> + FF = jsx:decode(wrq:req_body(RD), [{labels, atom}]), + case stanchion_server:update_policy(FF) of + ok -> + {true, RD, Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. + +-spec delete_resource(#wm_reqdata{}, #stanchion_context{}) -> + {boolean() | {halt, term()}, #wm_reqdata{}, #stanchion_context{}}. +delete_resource(RD, Ctx = #stanchion_context{}) -> + Arn = mochiweb_util:unquote(wrq:path_info(arn, RD)), + case stanchion_server:delete_policy(Arn) of + ok -> + {true, RD, Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. diff --git a/apps/riak_cs/src/stanchion_wm_policy.erl b/apps/riak_cs/src/stanchion_wm_policy.erl new file mode 100644 index 000000000..f2f4e690e --- /dev/null +++ b/apps/riak_cs/src/stanchion_wm_policy.erl @@ -0,0 +1,128 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_wm_policy). + +-export([init/1, + service_available/2, + is_authorized/2, + content_types_provided/2, + malformed_request/2, + to_json/2, + allowed_methods/2, + content_types_accepted/2, + accept_body/2, + delete_resource/2 + ]). + +-ignore_xref([init/1, + service_available/2, + is_authorized/2, + content_types_provided/2, + malformed_request/2, + to_json/2, + allowed_methods/2, + content_types_accepted/2, + accept_body/2, + delete_resource/2 + ]). + +-include("stanchion.hrl"). +-include_lib("webmachine/include/webmachine.hrl"). +-include_lib("kernel/include/logger.hrl"). + +init(Config) -> + %% Check if authentication is disabled and + %% set that in the context. + AuthBypass = proplists:get_value(auth_bypass, Config), + % eval every atoms to make binary_to_existing_atom/2 work. + {ok, #stanchion_context{auth_bypass=AuthBypass}}. + +-spec service_available(term(), term()) -> {true, term(), term()}. +service_available(RD, Ctx) -> + stanchion_wm_utils:service_available(RD, Ctx). + +-spec malformed_request(term(), term()) -> {false, term(), term()}. +malformed_request(RD, Ctx) -> + {false, RD, Ctx}. + +%% @doc Check that the request is from the admin user +is_authorized(RD, Ctx=#stanchion_context{auth_bypass=AuthBypass}) -> + AuthHeader = wrq:get_req_header("authorization", RD), + case stanchion_wm_utils:parse_auth_header(AuthHeader, AuthBypass) of + {ok, AuthMod, Args} -> + case AuthMod:authenticate(RD, Args) of + ok -> + %% Authentication succeeded + {true, RD, Ctx}; + {error, _Reason} -> + %% Authentication failed, deny access + stanchion_response:api_error(access_denied, RD, Ctx) + end + end. + +%% @doc Get the list of methods this resource supports. +-spec allowed_methods(term(), term()) -> {[atom()], term(), term()}. +allowed_methods(RD, Ctx) -> + {['GET', 'PUT', 'DELETE'], RD, Ctx}. + +-spec content_types_provided(term(), term()) -> + {[{string(), atom()}], term(), term()}. +content_types_provided(RD, Ctx) -> + {[{"application/json", to_xml}], RD, Ctx}. + +-spec content_types_accepted(term(), term()) -> + {[{string(), atom()}], term(), term()}. +content_types_accepted(RD, Ctx) -> + {[{"application/json", accept_body}], RD, Ctx}. + +-spec to_json(term(), term()) -> + {{'halt', term()}, #wm_reqdata{}, term()}. +to_json(RD, Ctx) -> + Bucket = wrq:path_info(bucket, RD), + stanchion_response:list_buckets_response(Bucket, RD, Ctx). + +%% @doc Process the request body on `PUT'. +accept_body(RD, Ctx) -> + Bucket = list_to_binary(wrq:path_info(bucket, RD)), + Body = wrq:req_body(RD), + ParsedBody = jsx:decode(Body, [{labels, atom}]), + case stanchion_server:set_bucket_policy(Bucket, + ParsedBody) of + ok -> + {true, RD, Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. + +%% @doc Callback for deleting an object. +-spec delete_resource(#wm_reqdata{}, #stanchion_context{}) -> {{halt, 204}, #wm_reqdata{}, #stanchion_context{}} | {true, #wm_reqdata{}, #stanchion_context{}}. +delete_resource(RD, Ctx) -> + Bucket = list_to_binary(wrq:path_info(bucket, RD)), + RequesterId = list_to_binary(wrq:get_qs_value("requester", "", RD)), + + case stanchion_server:delete_bucket_policy(Bucket, RequesterId) of + ok -> + % @TODO: does 204 really good? how does s3 works? + {{halt, 204}, RD, Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. diff --git a/apps/riak_cs/src/stanchion_wm_roles.erl b/apps/riak_cs/src/stanchion_wm_roles.erl new file mode 100644 index 000000000..f412c1eb6 --- /dev/null +++ b/apps/riak_cs/src/stanchion_wm_roles.erl @@ -0,0 +1,129 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_wm_roles). + +-export([init/1, + service_available/2, + allowed_methods/2, + is_authorized/2, + create_path/2, + post_is_create/2, + content_types_accepted/2, + accept_json/2, + delete_resource/2 + ]). + +-ignore_xref([init/1, + service_available/2, + allowed_methods/2, + is_authorized/2, + create_path/2, + post_is_create/2, + content_types_accepted/2, + accept_json/2, + delete_resource/2 + ]). + +-include("stanchion.hrl"). +-include("aws_api.hrl"). +-include_lib("webmachine/include/webmachine.hrl"). +-include_lib("kernel/include/logger.hrl"). + +init(Config) -> + AuthBypass = proplists:get_value(auth_bypass, Config), + {ok, #stanchion_context{auth_bypass=AuthBypass}}. + +-spec service_available(#wm_reqdata{}, #stanchion_context{}) -> {true, #wm_reqdata{}, #stanchion_context{}}. +service_available(RD, Ctx) -> + stanchion_wm_utils:service_available(RD, Ctx). + +-spec allowed_methods(#wm_reqdata{}, #stanchion_context{}) -> {[atom()], #wm_reqdata{}, #stanchion_context{}}. +allowed_methods(RD, Ctx) -> + {['POST', 'PUT', 'DELETE'], RD, Ctx}. + +-spec is_authorized(#wm_reqdata{}, #stanchion_context{}) -> {boolean(), #wm_reqdata{}, #stanchion_context{}}. +is_authorized(RD, Ctx = #stanchion_context{auth_bypass = AuthBypass}) -> + AuthHeader = wrq:get_req_header("authorization", RD), + case stanchion_wm_utils:parse_auth_header(AuthHeader, AuthBypass) of + {ok, AuthMod, Args} -> + case AuthMod:authenticate(RD, Args) of + ok -> + %% Authentication succeeded + {true, RD, Ctx}; + {error, _Reason} -> + %% Authentication failed, deny access + stanchion_response:api_error(access_denied, RD, Ctx) + end + end. + +-spec post_is_create(#wm_reqdata{}, #stanchion_context{}) -> {true, #wm_reqdata{}, #stanchion_context{}}. +post_is_create(_RD, _Ctx) -> + {true, _RD, _Ctx}. + +-spec create_path(#wm_reqdata{}, #stanchion_context{}) -> {string(), #wm_reqdata{}, #stanchion_context{}}. +create_path(RD, Ctx) -> + {wrq:disp_path(RD), RD, Ctx}. + +-spec content_types_accepted(#wm_reqdata{}, #stanchion_context{}) -> + {[{string(), module()}], #wm_reqdata{}, #stanchion_context{}}. +content_types_accepted(RD, Ctx) -> + {[{"application/json", accept_json}], RD, Ctx}. + +-spec accept_json(#wm_reqdata{}, #stanchion_context{}) -> + {true | {halt, pos_integer()}, #wm_reqdata{}, #stanchion_context{}}. +accept_json(RD, Ctx) -> + case wrq:method(RD) of + 'POST' -> + do_create(RD, Ctx); + 'PUT' -> + do_update(RD, Ctx) + end. + +do_create(RD, Ctx) -> + FF = jsx:decode(wrq:req_body(RD), [{labels, atom}]), + case stanchion_server:create_role(FF) of + {ok, Role} -> + Doc = riak_cs_json:to_json(Role), + {true, wrq:set_resp_body(Doc, RD), Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. + +do_update(RD, Ctx) -> + FF = jsx:decode(wrq:req_body(RD), [{labels, atom}]), + case stanchion_server:update_role(FF) of + ok -> + {true, RD, Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. + +-spec delete_resource(#wm_reqdata{}, #stanchion_context{}) -> + {boolean() | {halt, term()}, #wm_reqdata{}, #stanchion_context{}}. +delete_resource(RD, Ctx = #stanchion_context{}) -> + Arn = mochiweb_util:unquote(wrq:path_info(arn, RD)), + case stanchion_server:delete_role(Arn) of + ok -> + {true, RD, Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. diff --git a/apps/riak_cs/src/stanchion_wm_samlprovider.erl b/apps/riak_cs/src/stanchion_wm_samlprovider.erl new file mode 100644 index 000000000..f3e360093 --- /dev/null +++ b/apps/riak_cs/src/stanchion_wm_samlprovider.erl @@ -0,0 +1,111 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_wm_samlprovider). + +-export([init/1, + service_available/2, + allowed_methods/2, + is_authorized/2, + create_path/2, + post_is_create/2, + content_types_accepted/2, + accept_json/2, + delete_resource/2 + ]). + +-ignore_xref([init/1, + service_available/2, + allowed_methods/2, + is_authorized/2, + create_path/2, + post_is_create/2, + content_types_accepted/2, + accept_json/2, + delete_resource/2 + ]). + +-include("stanchion.hrl"). +-include("aws_api.hrl"). +-include_lib("webmachine/include/webmachine.hrl"). +-include_lib("kernel/include/logger.hrl"). + +init(Config) -> + AuthBypass = proplists:get_value(auth_bypass, Config), + {ok, #stanchion_context{auth_bypass=AuthBypass}}. + +-spec service_available(#wm_reqdata{}, #stanchion_context{}) -> {true, #wm_reqdata{}, #stanchion_context{}}. +service_available(RD, Ctx) -> + stanchion_wm_utils:service_available(RD, Ctx). + +-spec allowed_methods(#wm_reqdata{}, #stanchion_context{}) -> {[atom()], #wm_reqdata{}, #stanchion_context{}}. +allowed_methods(RD, Ctx) -> + {['POST', 'DELETE'], RD, Ctx}. + +-spec is_authorized(#wm_reqdata{}, #stanchion_context{}) -> {boolean(), #wm_reqdata{}, #stanchion_context{}}. +is_authorized(RD, Ctx=#stanchion_context{auth_bypass=AuthBypass}) -> + AuthHeader = wrq:get_req_header("authorization", RD), + case stanchion_wm_utils:parse_auth_header(AuthHeader, AuthBypass) of + {ok, AuthMod, Args} -> + case AuthMod:authenticate(RD, Args) of + ok -> + %% Authentication succeeded + {true, RD, Ctx}; + {error, _Reason} -> + %% Authentication failed, deny access + stanchion_response:api_error(access_denied, RD, Ctx) + end + end. + +-spec post_is_create(#wm_reqdata{}, #stanchion_context{}) -> {true, #wm_reqdata{}, #stanchion_context{}}. +post_is_create(_RD, _Ctx) -> + {true, _RD, _Ctx}. + +-spec create_path(#wm_reqdata{}, #stanchion_context{}) -> {string(), #wm_reqdata{}, #stanchion_context{}}. +create_path(RD, Ctx) -> + {wrq:disp_path(RD), RD, Ctx}. + +-spec content_types_accepted(#wm_reqdata{}, #stanchion_context{}) -> + {[{string(), module()}], #wm_reqdata{}, #stanchion_context{}}. +content_types_accepted(RD, Ctx) -> + {[{"application/json", accept_json}], RD, Ctx}. + +-spec accept_json(#wm_reqdata{}, #stanchion_context{}) -> + {true | {halt, pos_integer()}, #wm_reqdata{}, #stanchion_context{}}. +accept_json(RD, Ctx) -> + FF = jsx:decode(wrq:req_body(RD), [{labels, atom}]), + case stanchion_server:create_saml_provider(FF) of + {ok, {Arn, Tags}} -> + Resp = jason:encode(#{arn => Arn, tags => Tags}, [{records, [{tag, record_info(fields, tag)}]}]), + {true, wrq:set_resp_body(Resp, RD), Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. + +-spec delete_resource(#wm_reqdata{}, #stanchion_context{}) -> + {boolean() | {halt, term()}, #wm_reqdata{}, #stanchion_context{}}. +delete_resource(RD, Ctx = #stanchion_context{}) -> + Arn = mochiweb_util:unquote(wrq:path_info(arn, RD)), + case stanchion_server:delete_saml_provider(Arn) of + ok -> + {true, RD, Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. diff --git a/apps/riak_cs/src/stanchion_wm_stats.erl b/apps/riak_cs/src/stanchion_wm_stats.erl new file mode 100644 index 000000000..a3f14c84c --- /dev/null +++ b/apps/riak_cs/src/stanchion_wm_stats.erl @@ -0,0 +1,84 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_wm_stats). + +-export([init/1, + service_available/2, + allowed_methods/2, + is_authorized/2, + content_types_provided/2, + produce_body/2 + ]). + +-ignore_xref([init/1, + service_available/2, + allowed_methods/2, + is_authorized/2, + content_types_provided/2, + produce_body/2 + ]). + +-include("stanchion.hrl"). +-include_lib("webmachine/include/webmachine.hrl"). + +init(Config) -> + AuthBypass = proplists:get_value(auth_bypass, Config), + {ok, #stanchion_context{auth_bypass=AuthBypass}}. + +-spec service_available(#wm_reqdata{}, #stanchion_context{}) -> + {true, #wm_reqdata{}, #stanchion_context{}}. +service_available(RD, Ctx) -> + stanchion_wm_utils:service_available(RD, Ctx). + +-spec allowed_methods(#wm_reqdata{}, #stanchion_context{}) -> + {[atom()], #wm_reqdata{}, #stanchion_context{}}. +allowed_methods(RD, Ctx) -> + {['GET'], RD, Ctx}. + +%% @doc Check that the request is from the admin user +is_authorized(RD, Ctx) -> + #stanchion_context{auth_bypass = AuthBypass} = Ctx, + AuthHeader = wrq:get_req_header("authorization", RD), + case stanchion_wm_utils:parse_auth_header(AuthHeader, AuthBypass) of + {ok, AuthMod, Args} -> + case AuthMod:authenticate(RD, Args) of + ok -> + %% Authentication succeeded + {true, RD, Ctx}; + {error, _Reason} -> + %% Authentication failed, deny access + stanchion_response:api_error(access_denied, RD, Ctx) + end + end. + +-spec content_types_provided(#wm_reqdata{}, #stanchion_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #stanchion_context{}}. +content_types_provided(RD, Ctx) -> + {[{"application/json", produce_body}], + RD, Ctx}. + +-spec produce_body(#wm_reqdata{}, #stanchion_context{}) -> + {binary(), #wm_reqdata{}, #stanchion_context{}}. +produce_body(RD, Ctx) -> + Stats = stanchion_stats:get_stats(), + JSON = jsx:encode(Stats), + {JSON, RD, Ctx}. diff --git a/apps/riak_cs/src/stanchion_wm_user.erl b/apps/riak_cs/src/stanchion_wm_user.erl new file mode 100644 index 000000000..87b8323dd --- /dev/null +++ b/apps/riak_cs/src/stanchion_wm_user.erl @@ -0,0 +1,113 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_wm_user). + +-export([init/1, + service_available/2, + allowed_methods/2, + is_authorized/2, + create_path/2, + content_types_accepted/2, + accept_body/2, + delete_resource/2 + ]). + +-ignore_xref([init/1, + service_available/2, + allowed_methods/2, + is_authorized/2, + create_path/2, + content_types_accepted/2, + accept_body/2, + delete_resource/2 + ]). + +-include("stanchion.hrl"). +-include_lib("webmachine/include/webmachine.hrl"). + +-spec init(proplists:proplist()) -> {ok, #stanchion_context{}}. +init(Config) -> + %% Check if authentication is disabled and + %% set that in the context. + AuthBypass = proplists:get_value(auth_bypass, Config), + {ok, #stanchion_context{auth_bypass = AuthBypass}}. + +-spec service_available(#wm_reqdata{}, #stanchion_context{}) -> {true, #wm_reqdata{}, #stanchion_context{}}. +service_available(RD, Ctx) -> + stanchion_wm_utils:service_available(RD, Ctx). + +-spec allowed_methods(#wm_reqdata{}, #stanchion_context{}) -> {[atom()], #wm_reqdata{}, #stanchion_context{}}. +allowed_methods(RD, Ctx) -> + {['PUT', 'DELETE'], RD, Ctx}. + +%% @doc Check that the request is from the admin user +-spec is_authorized(#wm_reqdata{}, #stanchion_context{}) -> {boolean(), #wm_reqdata{}, #stanchion_context{}}. +is_authorized(RD, Ctx=#stanchion_context{auth_bypass = AuthBypass}) -> + AuthHeader = wrq:get_req_header("authorization", RD), + case stanchion_wm_utils:parse_auth_header(AuthHeader, AuthBypass) of + {ok, AuthMod, Args} -> + case AuthMod:authenticate(RD, Args) of + ok -> + %% Authentication succeeded + {true, RD, Ctx}; + {error, _Reason} -> + %% Authentication failed, deny access + stanchion_response:api_error(access_denied, RD, Ctx) + end + end. + +%% @doc Set the path for the new user resource and set +%% the Location header to generate a 201 Created response. +-spec create_path(#wm_reqdata{}, #stanchion_context{}) -> {string(), #wm_reqdata{}, #stanchion_context{}}. +create_path(RD, Ctx) -> + {wrq:disp_path(RD), RD, Ctx}. + +-spec content_types_accepted(#wm_reqdata{}, #stanchion_context{}) -> + {[{string(), atom()}], #wm_reqdata{}, #stanchion_context{}}. +content_types_accepted(RD, Ctx) -> + {[{"application/json", accept_body}], RD, Ctx}. + +%% @doc Create a user from a POST +-spec accept_body(#wm_reqdata{}, #stanchion_context{}) -> + {true | {halt, pos_integer()}, + #wm_reqdata{}, #stanchion_context{}}. +accept_body(RD, Ctx) -> + Body = wrq:req_body(RD), + FF = jsx:decode(Body, [{labels, atom}]), + case stanchion_server:update_user(FF) of + {ok, User} -> + Doc = riak_cs_json:to_json(User), + {true, wrq:set_resp_body(Doc, RD), Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. + +-spec delete_resource(#wm_reqdata{}, #stanchion_context{}) -> + {boolean() | {halt, term()}, #wm_reqdata{}, #stanchion_context{}}. +delete_resource(RD, Ctx) -> + TransArn = mochiweb_util:unquote(wrq:path_info(key_id, RD)), + case stanchion_server:delete_user(TransArn) of + ok -> + {true, RD, Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. diff --git a/apps/riak_cs/src/stanchion_wm_users.erl b/apps/riak_cs/src/stanchion_wm_users.erl new file mode 100644 index 000000000..41350c50f --- /dev/null +++ b/apps/riak_cs/src/stanchion_wm_users.erl @@ -0,0 +1,106 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_wm_users). + +-export([init/1, + service_available/2, + allowed_methods/2, + is_authorized/2, + post_is_create/2, + create_path/2, + content_types_accepted/2, + accept_body/2 + ]). + +-ignore_xref([init/1, + service_available/2, + allowed_methods/2, + is_authorized/2, + post_is_create/2, + create_path/2, + content_types_accepted/2, + accept_body/2 + ]). + +-include("stanchion.hrl"). +-include_lib("webmachine/include/webmachine.hrl"). +-include_lib("kernel/include/logger.hrl"). + + +init(Config) -> + %% Check if authentication is disabled and + %% set that in the context. + AuthBypass = proplists:get_value(auth_bypass, Config), + {ok, #stanchion_context{auth_bypass=AuthBypass}}. + +-spec service_available(#wm_reqdata{}, #stanchion_context{}) -> {true, #wm_reqdata{}, #stanchion_context{}}. +service_available(RD, Ctx) -> + stanchion_wm_utils:service_available(RD, Ctx). + +-spec allowed_methods(term(), term()) -> {[atom()], term(), term()}. +allowed_methods(RD, Ctx) -> + {['POST'], RD, Ctx}. + +%% @doc Check that the request is from the admin user +-spec is_authorized(#wm_reqdata{}, #stanchion_context{}) -> {boolean(), #wm_reqdata{}, #stanchion_context{}}. +is_authorized(RD, Ctx = #stanchion_context{auth_bypass = AuthBypass}) -> + AuthHeader = wrq:get_req_header("authorization", RD), + case stanchion_wm_utils:parse_auth_header(AuthHeader, AuthBypass) of + {ok, AuthMod, Args} -> + case AuthMod:authenticate(RD, Args) of + ok -> + %% Authentication succeeded + {true, RD, Ctx}; + {error, _Reason} -> + %% Authentication failed, deny access + stanchion_response:api_error(access_denied, RD, Ctx) + end + end. + +-spec post_is_create(#wm_reqdata{}, #stanchion_context{}) -> {true, #wm_reqdata{}, #stanchion_context{}}. +post_is_create(RD, Ctx) -> + {true, RD, Ctx}. + +%% @doc Set the path for the new user resource and set +%% the Location header to generate a 201 Created response. +%% -spec create_path(term(), term()) -> {string(), term(), term()}. +create_path(RD, Ctx) -> + {wrq:disp_path(RD), RD, Ctx}. + +-spec content_types_accepted(term(), term()) -> + {[{string(), atom()}], term(), term()}. +content_types_accepted(RD, Ctx) -> + {[{"application/json", accept_body}], RD, Ctx}. + +%% @doc Create a user from a POST +-spec accept_body(#wm_reqdata{}, #stanchion_context{}) -> + {true | {halt, pos_integer()}, + #wm_reqdata{}, #stanchion_context{}}. +accept_body(RD, Ctx) -> + Body = wrq:req_body(RD), + case stanchion_server:create_user( + jsx:decode(Body, [{labels, atom}])) of + ok -> + {true, RD, Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. diff --git a/apps/riak_cs/src/stanchion_wm_utils.erl b/apps/riak_cs/src/stanchion_wm_utils.erl new file mode 100644 index 000000000..25c487c80 --- /dev/null +++ b/apps/riak_cs/src/stanchion_wm_utils.erl @@ -0,0 +1,62 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_wm_utils). + +-export([service_available/2, + parse_auth_header/2, + iso_8601_datetime/0 + ]). + +-ignore_xref([service_available/2, + parse_auth_header/2, + iso_8601_datetime/0 + ]). + +-include("stanchion.hrl"). + +service_available(RD, Ctx) -> + {true, RD, Ctx}. + +%% @doc Parse an authentication header string and determine +%% the appropriate module to use to authenticate the request. +%% The passthru auth can be used either with a KeyID or +%% anonymously by leving the header empty. +-spec parse_auth_header(string(), boolean()) -> {ok, atom(), [string()]} | {error, term()}. +parse_auth_header(_, true) -> + {ok, stanchion_passthru_auth, []}; +parse_auth_header("MOSS " ++ Key, _) -> + case string:tokens(Key, ":") of + [KeyId, KeyData] -> + {ok, stanchion_auth, [KeyId, KeyData]}; + _Other -> + {error, {bad_key, Key}} + end; +parse_auth_header(_, false) -> + {ok, stanchion_blockall_auth, ["unkown_auth_scheme"]}. + +%% @doc Get an ISO 8601 formatted timestamp representing +%% current time. +-spec iso_8601_datetime() -> [non_neg_integer()]. +iso_8601_datetime() -> + {{Year, Month, Day}, {Hour, Min, Sec}} = erlang:universaltime(), + io_lib:format("~4.10.0B-~2.10.0B-~2.10.0BT~2.10.0B:~2.10.0B:~2.10.0B.000Z", + [Year, Month, Day, Hour, Min, Sec]). diff --git a/apps/riak_cs/src/stanchion_wm_versioning.erl b/apps/riak_cs/src/stanchion_wm_versioning.erl new file mode 100644 index 000000000..885e6c028 --- /dev/null +++ b/apps/riak_cs/src/stanchion_wm_versioning.erl @@ -0,0 +1,92 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +-module(stanchion_wm_versioning). + +-export([init/1, + service_available/2, + is_authorized/2, + malformed_request/2, + allowed_methods/2, + content_types_accepted/2, + accept_body/2 + ]). + +-ignore_xref([init/1, + service_available/2, + is_authorized/2, + malformed_request/2, + allowed_methods/2, + content_types_accepted/2, + accept_body/2 + ]). + +-include("stanchion.hrl"). +-include_lib("kernel/include/logger.hrl"). + +init(Config) -> + %% Check if authentication is disabled and + %% set that in the context. + AuthBypass = proplists:get_value(auth_bypass, Config), + % eval every atoms to make binary_to_existing_atom/2 work. + {ok, #stanchion_context{auth_bypass=AuthBypass}}. + +-spec service_available(term(), term()) -> {true, term(), term()}. +service_available(RD, Ctx) -> + stanchion_wm_utils:service_available(RD, Ctx). + +-spec malformed_request(term(), term()) -> {false, term(), term()}. +malformed_request(RD, Ctx) -> + {false, RD, Ctx}. + +%% @doc Check that the request is from the admin user +is_authorized(RD, Ctx=#stanchion_context{auth_bypass=AuthBypass}) -> + AuthHeader = wrq:get_req_header("authorization", RD), + case stanchion_wm_utils:parse_auth_header(AuthHeader, AuthBypass) of + {ok, AuthMod, Args} -> + case AuthMod:authenticate(RD, Args) of + ok -> + {true, RD, Ctx}; + {error, _Reason} -> + stanchion_response:api_error(access_denied, RD, Ctx) + end + end. + +%% @doc Get the list of methods this resource supports. +-spec allowed_methods(term(), term()) -> {[atom()], term(), term()}. +allowed_methods(RD, Ctx) -> + {['PUT'], RD, Ctx}. + +-spec content_types_accepted(term(), term()) -> + {[{string(), atom()}], term(), term()}. +content_types_accepted(RD, Ctx) -> + {[{"application/json", accept_body}], RD, Ctx}. + +%% @doc Process the request body on `PUT'. +accept_body(RD, Ctx) -> + Bucket = list_to_binary(wrq:path_info(bucket, RD)), + Specs = jsx:decode(wrq:req_body(RD), [{labels, atom}]), + case stanchion_server:set_bucket_versioning(Bucket, Specs) of + ok -> + {true, RD, Ctx}; + {error, Reason} -> + stanchion_response:api_error(Reason, RD, Ctx) + end. diff --git a/apps/riak_cs/src/velvet.erl b/apps/riak_cs/src/velvet.erl new file mode 100644 index 000000000..42a744cdd --- /dev/null +++ b/apps/riak_cs/src/velvet.erl @@ -0,0 +1,608 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved,, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc Client module for interacting with `stanchion' application. + +-module(velvet). + +-export([create_bucket/3, + create_user/3, + delete_user/2, + update_user/4, + delete_bucket/3, + set_bucket_acl/4, + set_bucket_policy/4, + set_bucket_versioning/4, + delete_bucket_policy/3, + create_role/3, + delete_role/2, + update_role/4, + create_policy/3, + delete_policy/2, + update_policy/4, + create_saml_provider/3, + delete_saml_provider/2 + ]). + +-include("moss.hrl"). +-include("riak_cs.hrl"). +-include("aws_api.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-define(MAX_REQUEST_RETRIES, 3). + +%% =================================================================== +%% Public API +%% =================================================================== + +%% @doc Create a bucket for a requesting party. +-spec create_bucket(string(), binary(), proplists:proplist()) -> + ok | {error, reportable_error_reason()}. +create_bucket(ContentType, BucketDoc, Options) -> + AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), + Path = buckets_path(<<>>), + Headers0 = [{"Content-Md5", content_md5(BucketDoc)}, + {"Date", httpd_util:rfc1123_date()}], + case AuthCreds of + {_, _} -> + Headers = + [{"Authorization", auth_header('POST', + ContentType, + Headers0, + Path, + AuthCreds)} | + Headers0]; + no_auth_creds -> + Headers = Headers0 + end, + case request(post, Path, [201], ContentType, Headers, BucketDoc) of + {ok, {{_, 201, _}, _RespHeaders, _RespBody}} -> + ok; + {error, Error} -> + {error, Error} + end. + + + +%% @doc Create a bucket for a requesting party. +-spec create_user(string(), binary(), proplists:proplist()) -> + ok | {error, reportable_error_reason()}. +create_user(ContentType, UserDoc, Options) -> + AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), + Path = users_path([]), + Headers0 = [{"Content-Md5", content_md5(UserDoc)}, + {"Date", httpd_util:rfc1123_date()}], + case AuthCreds of + {_, _} -> + Headers = + [{"Authorization", auth_header('POST', + ContentType, + Headers0, + Path, + AuthCreds)} | + Headers0]; + no_auth_creds -> + Headers = Headers0 + end, + case request(post, Path, [201], ContentType, Headers, UserDoc) of + {ok, {{_, 201, _}, _RespHeaders, _RespBody}} -> + ok; + {error, Error} -> + {error, Error} + end. + +-spec delete_user(binary(), proplists:proplist()) -> + ok | {error, reportable_error_reason()}. +delete_user(TransKeyId, Options) -> + AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), + Path = users_path(binary_to_list(TransKeyId)), + Headers0 = [{"Date", httpd_util:rfc1123_date()}], + case AuthCreds of + {_, _} -> + Headers = + [{"Authorization", auth_header('DELETE', + "", + Headers0, + Path, + AuthCreds)} | + Headers0]; + no_auth_creds -> + Headers = Headers0 + end, + case request(delete, Path, [204], Headers) of + {ok, {{_, 204, _}, _RespHeaders, _}} -> + ok; + {error, Error} -> + {error, Error} + end. + +%% @doc Delete a bucket. The bucket must be owned by +%% the requesting party. +-spec delete_bucket(binary(), binary(), proplists:proplist()) -> + ok | {error, reportable_error_reason()}. +delete_bucket(Bucket, Requester, Options) -> + AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), + QS = requester_qs(Requester), + Path = buckets_path(Bucket), + Headers0 = [{"Date", httpd_util:rfc1123_date()}], + case AuthCreds of + {_, _} -> + Headers = + [{"Authorization", auth_header('DELETE', + [], + Headers0, + Path, + AuthCreds)} | + Headers0]; + no_auth_creds -> + Headers = Headers0 + end, + case request(delete, Path ++ QS, [204], Headers) of + {ok, {{_, 204, _}, _RespHeaders, _}} -> + ok; + {error, Error} -> + {error, Error} + end. + +-spec set_bucket_acl(binary(), string(), binary(), proplists:proplist()) -> + ok | {error, reportable_error_reason()}. +set_bucket_acl(Bucket, ContentType, AclDoc, Options) -> + Path = buckets_path(Bucket, acl), + update_bucket(Path, ContentType, AclDoc, Options, 204). + +-spec set_bucket_policy(binary(), string(), binary(), proplists:proplist()) -> + ok | {error, reportable_error_reason()}. +set_bucket_policy(Bucket, ContentType, PolicyDoc, Options) -> + Path = buckets_path(Bucket, policy), + update_bucket(Path, ContentType, PolicyDoc, Options, 204). + +-spec set_bucket_versioning(binary(), string(), binary(), proplists:proplist()) -> + ok | {error, reportable_error_reason()}. +set_bucket_versioning(Bucket, ContentType, Doc, Options) -> + Path = buckets_path(Bucket, versioning), + update_bucket(Path, ContentType, Doc, Options, 204). + +%% @doc Delete a bucket. The bucket must be owned by +%% the requesting party. +-spec delete_bucket_policy(binary(), binary(), proplists:proplist()) -> + ok | {error, reportable_error_reason()}. +delete_bucket_policy(Bucket, Requester, Options) -> + AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), + QS = requester_qs(Requester), + Path = buckets_path(Bucket, policy), + Headers0 = [{"Date", httpd_util:rfc1123_date()}], + case AuthCreds of + {_, _} -> + Headers = + [{"Authorization", auth_header('DELETE', + [], + Headers0, + Path, + AuthCreds)} | + Headers0]; + no_auth_creds -> + Headers = Headers0 + end, + case request(delete, Path ++ QS, [204], Headers) of + {ok, {{_, 204, _}, _RespHeaders, _}} -> + ok; + {error, Error} -> + {error, Error} + end. + +%% @doc Update a user record +-spec update_user(string(), binary(), binary(), proplists:proplist()) -> + {ok, rcs_user()} | {error, reportable_error_reason()}. +update_user(ContentType, KeyId, UserDoc, Options) -> + AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), + Path = users_path(KeyId), + Headers0 = [{"Content-Md5", content_md5(UserDoc)}, + {"Date", httpd_util:rfc1123_date()}], + case AuthCreds of + {_, _} -> + Headers = + [{"Authorization", auth_header('PUT', + ContentType, + Headers0, + Path, + AuthCreds)} | + Headers0]; + no_auth_creds -> + Headers = Headers0 + end, + case request(put, Path, [200, 204], ContentType, Headers, UserDoc) of + {ok, {_, _RespHeaders, RespBody}} -> + User = riak_cs_iam:exprec_user(jsx:decode(list_to_binary(RespBody), [{labels, atom}])), + {ok, User}; + {error, Error} -> + {error, Error} + end. + + +%% =================================================================== +%% Internal functions +%% =================================================================== + +% @doc send request to stanchion server +% @TODO merge with create_bucket, create_user, delete_bucket +-spec update_bucket(string(), string(), binary(), proplists:proplist(), pos_integer()) -> + ok | {error, reportable_error_reason()}. +update_bucket(Path, ContentType, Doc, Options, Expect) -> + AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), + Headers0 = [{"Content-Md5", content_md5(Doc)}, + {"Date", httpd_util:rfc1123_date()}], + case AuthCreds of + {_, _} -> + Headers = + [{"Authorization", auth_header('PUT', + ContentType, + Headers0, + Path, + AuthCreds)} | + Headers0]; + no_auth_creds -> + Headers = Headers0 + end, + case request(put, Path, [Expect], ContentType, Headers, Doc) of + {ok, {{_, Expect, _}, _RespHeaders, _RespBody}} -> + ok; + {error, Error} -> + {error, Error} + end. + + +%% @doc Assemble the path for a bucket request +buckets_path(Bucket) -> + stringy(["/buckets", + ["/" ++ binary_to_list(Bucket) || Bucket /= <<>>]]). + +%% @doc Assemble the path for a bucket request +buckets_path(Bucket, acl) -> + stringy([buckets_path(Bucket), "/acl"]); +buckets_path(Bucket, policy) -> + stringy([buckets_path(Bucket), "/policy"]); +buckets_path(Bucket, versioning) -> + stringy([buckets_path(Bucket), "/versioning"]). + + + +-spec create_role(string(), binary(), proplists:proplist()) -> + {ok, role()} | {error, reportable_error_reason()}. +create_role(ContentType, Doc, Options) -> + AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), + Path = roles_path([]), + Headers0 = [{"Content-Md5", content_md5(Doc)}, + {"Date", httpd_util:rfc1123_date()}], + case AuthCreds of + {_, _} -> + Headers = + [{"Authorization", auth_header('POST', + ContentType, + Headers0, + Path, + AuthCreds)} | + Headers0]; + no_auth_creds -> + Headers = Headers0 + end, + case request(post, Path, [201], ContentType, Headers, Doc) of + {ok, {{_, 201, _}, _RespHeaders, RespBody}} -> + Role_ = ?IAM_ROLE{assume_role_policy_document = A} = + riak_cs_iam:exprec_role(jsx:decode(list_to_binary(RespBody), [{labels, atom}])), + Role = Role_?IAM_ROLE{assume_role_policy_document = base64:decode(A)}, + {ok, Role}; + {error, Error} -> + {error, Error} + end. + +-spec delete_role(binary(), proplists:proplist()) -> + ok | {error, reportable_error_reason()}. +delete_role(Arn, Options) -> + AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), + Path = roles_path(binary_to_list(Arn)), + Headers0 = [{"Date", httpd_util:rfc1123_date()}], + case AuthCreds of + {_, _} -> + Headers = + [{"Authorization", auth_header('DELETE', + "", + Headers0, + Path, + AuthCreds)} | + Headers0]; + no_auth_creds -> + Headers = Headers0 + end, + case request(delete, Path, [204], Headers) of + {ok, {{_, 204, _}, _RespHeaders, _}} -> + ok; + {error, {ok, {{_, StatusCode, Reason}, _RespHeaders, RespBody}}} -> + {error, {error_status, StatusCode, Reason, RespBody}}; + {error, Error} -> + {error, Error} + end. + +-spec update_role(string(), binary(), binary(), proplists:proplist()) -> + ok | {error, reportable_error_reason()}. +update_role(ContentType, Arn, Doc, Options) -> + AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), + Path = roles_path(Arn), + Headers0 = [{"Content-Md5", content_md5(Doc)}, + {"Date", httpd_util:rfc1123_date()}], + case AuthCreds of + {_, _} -> + Headers = + [{"Authorization", auth_header('PUT', + ContentType, + Headers0, + Path, + AuthCreds)} | + Headers0]; + no_auth_creds -> + Headers = Headers0 + end, + case request(put, Path, [204], ContentType, Headers, Doc) of + {ok, {{_, 204, _}, _RespHeaders, _RespBody}} -> + ok; + {error, Error} -> + {error, Error} + end. + + +-spec create_policy(string(), binary(), proplists:proplist()) -> + {ok, iam_policy()} | {error, reportable_error_reason()}. +create_policy(ContentType, Doc, Options) -> + AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), + Path = policies_path([]), + Headers0 = [{"Content-Md5", content_md5(Doc)}, + {"Date", httpd_util:rfc1123_date()}], + case AuthCreds of + {_, _} -> + Headers = + [{"Authorization", auth_header('POST', + ContentType, + Headers0, + Path, + AuthCreds)} | + Headers0]; + no_auth_creds -> + Headers = Headers0 + end, + case request(post, Path, [201], ContentType, Headers, Doc) of + {ok, {{_, 201, _}, _RespHeaders, RespBody}} -> + Policy = riak_cs_iam:exprec_iam_policy(jsx:decode(list_to_binary(RespBody), [{labels, atom}])), + {ok, Policy}; + {error, Error} -> + {error, Error} + end. + +-spec delete_policy(binary(), proplists:proplist()) -> + ok | {error, reportable_error_reason()}. +delete_policy(Arn, Options) -> + AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), + Path = policies_path(binary_to_list(Arn)), + Headers0 = [{"Date", httpd_util:rfc1123_date()}], + case AuthCreds of + {_, _} -> + Headers = + [{"Authorization", auth_header('DELETE', + "", + Headers0, + Path, + AuthCreds)} | + Headers0]; + no_auth_creds -> + Headers = Headers0 + end, + case request(delete, Path, [204], Headers) of + {ok, {{_, 204, _}, _RespHeaders, _}} -> + ok; + {error, Error} -> + {error, Error} + end. + +-spec update_policy(string(), binary(), binary(), proplists:proplist()) -> + ok | {error, reportable_error_reason()}. +update_policy(ContentType, Arn, Doc, Options) -> + AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), + Path = policies_path(Arn), + Headers0 = [{"Content-Md5", content_md5(Doc)}, + {"Date", httpd_util:rfc1123_date()}], + case AuthCreds of + {_, _} -> + Headers = + [{"Authorization", auth_header('PUT', + ContentType, + Headers0, + Path, + AuthCreds)} | + Headers0]; + no_auth_creds -> + Headers = Headers0 + end, + case request(put, Path, [204], ContentType, Headers, Doc) of + {ok, {{_, 204, _}, _RespHeaders, _RespBody}} -> + ok; + {error, Error} -> + {error, Error} + end. + + +-spec create_saml_provider(string(), binary(), proplists:proplist()) -> + {ok, {binary(), [tag()]}} | {error, reportable_error_reason()}. +create_saml_provider(ContentType, Doc, Options) -> + AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), + Path = "/samlproviders", + Headers0 = [{"Content-Md5", content_md5(Doc)}, + {"Date", httpd_util:rfc1123_date()}], + case AuthCreds of + {_, _} -> + Headers = + [{"Authorization", auth_header('POST', + ContentType, + Headers0, + Path, + AuthCreds)} | + Headers0]; + no_auth_creds -> + Headers = Headers0 + end, + case request(post, Path, [201], ContentType, Headers, Doc) of + {ok, {{_, 201, _}, _RespHeaders, RespBody}} -> + #{arn := Arn, tags := Tags_} = jason:decode(RespBody, [{mode, map}, {binary, v}]), + {ok, {Arn, [#tag{key = K, value = V} || Tags_ /= <<>>, #{key := K, value := V} <- Tags_]}}; + {error, Error} -> + {error, Error} + end. + +-spec delete_saml_provider(binary(), proplists:proplist()) -> + ok | {error, reportable_error_reason()}. +delete_saml_provider(Arn, Options) -> + AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), + Path = saml_provider_path(binary_to_list(Arn)), + Headers0 = [{"Date", httpd_util:rfc1123_date()}], + case AuthCreds of + {_, _} -> + Headers = + [{"Authorization", auth_header('DELETE', + "", + Headers0, + Path, + AuthCreds)} | + Headers0]; + no_auth_creds -> + Headers = Headers0 + end, + case request(delete, Path, [204], Headers) of + {ok, {{_, 204, _}, _RespHeaders, _}} -> + ok; + {error, Error} -> + {error, Error} + end. + + + +%% ------------------------------------- +%% supporting functions +%% ------------------------------------- + +%% @doc send an HTTP request where `Expect' is a list +%% of expected HTTP status codes. +request(Method, Url, Expect, Headers) -> + request(Method, Url, Expect, [], Headers, []). + +%% @doc send an HTTP request where `Expect' is a list +%% of expected HTTP status codes. +request(Method, Path, Expect, ContentType, Headers, Body) -> + request(Method, Path, Expect, ContentType, Headers, Body, ?MAX_REQUEST_RETRIES). + +request(Method, Path, _Expect, _ContentType, _Headers, _Body, 0) -> + {Ip, Port, Ssl} = riak_cs_utils:stanchion_data(), + logger:warning("Giving up trying to send a ~s request to stanchion (~s)", + [Method, url(Ip, Port, Ssl, Path)]), + {error, stanchion_recovery_failure}; +request(Method, Path, Expect, ContentType, Headers, Body, Attempt) -> + stanchion_migration:validate_stanchion(), + {Ip, Port, Ssl} = riak_cs_utils:stanchion_data(), + Url = url(Ip, Port, Ssl, Path), + + case Method == put orelse + Method == post of + true -> + Request = {Url, Headers, ContentType, Body}; + false -> + Request = {Url, Headers} + end, + case httpc:request(Method, Request, [], []) of + Resp = {ok, {{_, Status, _}, _RespHeaders, RespBody}} -> + case lists:member(Status, Expect) of + true -> + Resp; + false -> + case catch jsx:decode(list_to_binary(RespBody), [{labels, atom}]) of + #{error_tag := Tag_, + resource := _Resource} -> + Tag = binary_to_term(base64:decode(Tag_)), + ?LOG_DEBUG("stanchion op non-success response, tag: ~p, resource: ~s", [Tag, _Resource]), + {error, Tag}; + {'EXIT', _} -> + logger:error("Unexpected response from stanchion (~p). Is it up?", [Status]), + ?LOG_DEBUG("Stanchion response body: ~p", [RespBody]), + {error, stanchion_recovery_failure} + end + end; + Error -> + if Attempt == ?MAX_REQUEST_RETRIES -> + %% first_call_failing_is_ok_on_startup + ok; + el/=se -> + logger:warning("Error contacting stanchion at ~s: ~p; retrying...", [Url, Error]) + end, + ok = stanchion_migration:adopt_stanchion(), + request(Method, Path, Expect, ContentType, Headers, Body, Attempt - 1) + end. + + +%% @doc Assemble the root URL for the given client +root_url(Ip, Port, true) -> + ["https://", Ip, ":", integer_to_list(Port)]; +root_url(Ip, Port, false) -> + ["http://", Ip, ":", integer_to_list(Port)]. + +url(Ip, Port, Ssl, Path) -> + lists:flatten( + [root_url(Ip, Port, Ssl), + Path + ]). + +%% @doc Calculate an MD5 hash of a request body. +content_md5(Body) -> + base64:encode_to_string(riak_cs_utils:md5(Body)). + +%% @doc Construct a MOSS authentication header +auth_header(HttpVerb, ContentType, Headers, Path, {AuthKey, AuthSecret}) -> + Signature = velvet_auth:request_signature(HttpVerb, + [{"content-type", ContentType} | + Headers], + Path, + binary_to_list(AuthSecret)), + lists:flatten(io_lib:format("MOSS ~s:~s", [AuthKey, Signature])). + +%% @doc Assemble a requester query string for +%% user in a bucket deletion request. +requester_qs(Requester) -> + "?requester=" ++ + mochiweb_util:quote_plus(Requester). + +users_path(A) -> + stringy(["/users", ["/" ++ mochiweb_util:quote_plus(A) || A /= []]]). + +roles_path(A) -> + stringy(["/roles", ["/" ++ mochiweb_util:quote_plus(A) || A /= []]]). + +policies_path(A) -> + stringy(["/policies", ["/" ++ mochiweb_util:quote_plus(A) || A /= []]]). + +saml_provider_path(A) -> + stringy(["/samlproviders", ["/" ++ mochiweb_util:quote_plus(A) || A /= []]]). + +stringy(A) -> + binary_to_list(iolist_to_binary(A)). diff --git a/src/velvet_auth.erl b/apps/riak_cs/src/velvet_auth.erl similarity index 97% rename from src/velvet_auth.erl rename to apps/riak_cs/src/velvet_auth.erl index 0007cac57..2e464c122 100644 --- a/src/velvet_auth.erl +++ b/apps/riak_cs/src/velvet_auth.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/test/cs782_regression_test.erl b/apps/riak_cs/test/cs782_regression_test.erl similarity index 96% rename from test/cs782_regression_test.erl rename to apps/riak_cs/test/cs782_regression_test.erl index d99f8db6d..7a4b85298 100644 --- a/test/cs782_regression_test.erl +++ b/apps/riak_cs/test/cs782_regression_test.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -20,8 +21,6 @@ -module(cs782_regression_test). --compile(export_all). - -include("riak_cs.hrl"). -include_lib("eunit/include/eunit.hrl"). diff --git a/test/riak_cs_acl_utils_eqc.erl b/apps/riak_cs/test/prop_riak_cs_acl_utils.erl similarity index 65% rename from test/riak_cs_acl_utils_eqc.erl rename to apps/riak_cs/test/prop_riak_cs_acl_utils.erl index bfce0119c..fb1f82419 100644 --- a/test/riak_cs_acl_utils_eqc.erl +++ b/apps/riak_cs/test/prop_riak_cs_acl_utils.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,15 +19,13 @@ %% %% --------------------------------------------------------------------- -%% @doc Quickcheck test module for `riak_cs_acl_utils'. +%% @doc PropErtest module for `riak_cs_acl_utils'. --module(riak_cs_acl_utils_eqc). --compile(export_all). +-module(prop_riak_cs_acl_utils). -include("riak_cs.hrl"). --ifdef(EQC). --include_lib("eqc/include/eqc.hrl"). +-include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). %% eqc property @@ -37,7 +36,7 @@ test/1]). -define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> + proper:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). -define(TEST_ITERATIONS, 1000). @@ -48,8 +47,8 @@ eqc_test_() -> {spawn, [ - {timeout, 60, ?_assertEqual(true, quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_add_grant_idempotent()))))}, - {timeout, 60, ?_assertEqual(true, quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_grant_gives_permission()))))} + {timeout, 60, ?_assertEqual(true, proper:quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_add_grant_idempotent()))))}, + {timeout, 60, ?_assertEqual(true, proper:quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_grant_gives_permission()))))} ] }. @@ -72,8 +71,13 @@ prop_grant_gives_permission() -> {Grants, elements(Grants)}), begin CombinedGrants = lists:foldl(fun riak_cs_acl_utils:add_grant/2, [], Grants), - {{_DisplayName, CanonicalID}, [RequestedAccess]} = RandomGrant, - riak_cs_acl:has_permission(CombinedGrants, RequestedAccess, CanonicalID) + case RandomGrant of + ?ACL_GRANT{grantee = #{canonical_id := CanonicalID}, + perms = [RequestedAccess]} -> + riak_cs_acl:has_permission(CombinedGrants, RequestedAccess, CanonicalID); + ?ACL_GRANT{perms = [RequestedAccess]} -> + riak_cs_acl:has_group_permission(CombinedGrants, RequestedAccess) + end end). @@ -82,13 +86,20 @@ prop_grant_gives_permission() -> %%==================================================================== grantee() -> - elements([{"a", 1}, {"b", 2}, {"c", 3}, {"d", 4}]). + oneof([owner(), group_grant()]). +owner() -> + #{display_name => riak_cs_aws_utils:make_id(4), + canonical_id => riak_cs_aws_utils:make_id(8), + email => iolist_to_binary([riak_cs_aws_utils:make_id(2), $@, riak_cs_aws_utils:make_id(4), ".com"])}. +group_grant() -> + oneof(['AllUsers', 'AuthUsers']). permission() -> elements(['READ', 'WRITE', 'READ_ACP', 'WRITE_ACP', 'FULL_CONTROL']). grant() -> - {grantee(), [permission()]}. + ?ACL_GRANT{grantee = grantee(), + perms = [permission()]}. %%==================================================================== %% Helpers @@ -98,7 +109,5 @@ test() -> test(?TEST_ITERATIONS). test(Iterations) -> - eqc:quickcheck(eqc:numtests(Iterations, prop_add_grant_idempotent())), - eqc:quickcheck(eqc:numtests(Iterations, prop_grant_gives_permission())). - --endif. + proper:quickcheck(proper:numtests(Iterations, prop_add_grant_idempotent())), + proper:quickcheck(proper:numtests(Iterations, prop_grant_gives_permission())). diff --git a/test/riak_cs_s3_auth_eqc.erl b/apps/riak_cs/test/prop_riak_cs_aws_auth.erl similarity index 84% rename from test/riak_cs_s3_auth_eqc.erl rename to apps/riak_cs/test/prop_riak_cs_aws_auth.erl index 1c19a3e05..6795e4663 100644 --- a/test/riak_cs_s3_auth_eqc.erl +++ b/apps/riak_cs/test/prop_riak_cs_aws_auth.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,28 +19,23 @@ %% %% --------------------------------------------------------------------- --module(riak_cs_s3_auth_eqc). - --compile(export_all). - --ifdef(EQC). +-module(prop_riak_cs_aws_auth). -include("riak_cs.hrl"). --include_lib("eqc/include/eqc.hrl"). +-include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("erlcloud/include/erlcloud_aws.hrl"). -define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> - io:format(user, Str, Args) end, P)). + on_output(fun(Str, Args) -> + io:format(user, Str, Args) end, P)). -define(TESTING_TIME, 20). -auth_v2_eqc_test_() -> +auth_v2_proper_test_() -> Tests = [{timeout, ?TESTING_TIME*2, - ?_assertEqual(true, quickcheck(eqc:testing_time(?TESTING_TIME, - ?QC_OUT(prop_v2_auth())))) - }], + ?_assertEqual(true, proper:quickcheck(?QC_OUT(prop_v2_auth()))) + }], [{inparallel, Tests}]. @@ -49,8 +45,8 @@ prop_v2_auth() -> ?FORALL(Request, gen_request(RootHost), begin {KeyData, KeySecret, RD} = Request, - SignedString = riak_cs_s3_auth:calculate_signature_v2(KeySecret, RD), - CSAuthHeader = ["AWS ", KeyData, $:, list_to_binary(SignedString)], + SignedString = riak_cs_aws_auth:calculate_signature_v2(KeySecret, RD), + CSAuthHeader = ["AWS ", KeyData, $:, SignedString], ErlCloudAuthHeader = erlcloud_hdr(KeyData, KeySecret, RD), CSAuthHeader =:= ErlCloudAuthHeader @@ -76,7 +72,7 @@ gen_request(RootHost) -> {"Date", Date}, {"Content-MD5", ContentMD5}, {"Content-Type", ContentType}]), - {Headers, Path} = riak_cs_s3_rewrite:rewrite(Verb, https, Version, Headers0, OrigPath), + {Headers, Path} = riak_cs_aws_s3_rewrite:rewrite(Verb, https, Version, Headers0, OrigPath), RD = wrq:create(Verb, Version, Path, Headers), {KeyData, KeySecret, RD} end). @@ -108,7 +104,7 @@ make_authorization(Config, Method, ContentMD5, ContentType, Date, AmzHeaders, format_subresources(Subresources) ], - Signature = base64:encode(crypto:hmac(sha, Config#aws_config.secret_access_key, StringToSign)), + Signature = base64:encode(crypto:mac(hmac, sha, Config#aws_config.secret_access_key, StringToSign)), ["AWS ", Config#aws_config.access_key_id, $:, Signature]. format_subresources([]) -> @@ -124,4 +120,3 @@ format_subresource({Subresource, Value}) when is_integer(Value) -> Subresource ++ "=" ++ integer_to_list(Value); format_subresource(Subresource) -> Subresource. --endif. diff --git a/test/riak_cs_gc_manager_eqc.erl b/apps/riak_cs/test/prop_riak_cs_gc_manager.erl similarity index 79% rename from test/riak_cs_gc_manager_eqc.erl rename to apps/riak_cs/test/prop_riak_cs_gc_manager.erl index 826f9db5b..2b3adab72 100644 --- a/test/riak_cs_gc_manager_eqc.erl +++ b/apps/riak_cs/test/prop_riak_cs_gc_manager.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,18 +19,18 @@ %% %% --------------------------------------------------------------------- -%% @doc Quickcheck test module for `riak_cs_gc_manager'. +%% @doc PropEr test module for `riak_cs_gc_manager'. --module(riak_cs_gc_manager_eqc). +-module(prop_riak_cs_gc_manager). + +-compile([{nowarn_deprecated_function, [{gen_fsm, sync_send_all_state_event, 2}]}]). -include("include/riak_cs_gc.hrl"). --ifdef(EQC). --include_lib("eqc/include/eqc.hrl"). --include_lib("eqc/include/eqc_fsm.hrl"). +-include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). -%% eqc properties +%% proper properties -export([prop_set_interval/0, prop_manual_commands/0]). @@ -37,7 +38,7 @@ -export([idle/1, running/1]). -%% eqc_fsm callbacks +%% proper_fsm callbacks -export([initial_state/0, initial_state_data/0, next_state_data/5, @@ -45,8 +46,8 @@ postcondition/5]). -define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> - io:format(user, Str, Args) end, P)). + on_output(fun(Str, Args) -> + io:format(user, Str, Args) end, P)). -define(TEST_ITERATIONS, 500). @@ -64,7 +65,7 @@ %% Eunit tests %%==================================================================== -eqc_test_() -> +proper_test_() -> {spawn, [{foreach, fun() -> @@ -77,8 +78,8 @@ eqc_test_() -> meck:unload() end, [ - {timeout, 20, ?_assertEqual(true, eqc:quickcheck(eqc:testing_time(10, ?QC_OUT(prop_set_interval()))))}, - {timeout, 60, ?_assertEqual(true, eqc:quickcheck(eqc:testing_time(30, ?QC_OUT(prop_manual_commands()))))} + {timeout, 20, ?_assertEqual(true, proper:quickcheck(?QC_OUT(prop_set_interval())))}, + {timeout, 60, ?_assertEqual(true, proper:quickcheck(?QC_OUT(prop_manual_commands())))} ] }]}. @@ -108,19 +109,19 @@ prop_set_interval() -> prop_manual_commands() -> ?FORALL(Cmds, - commands(?MODULE), + proper_fsm:commands(?MODULE), begin {ok, Pid} = riak_cs_gc_manager:test_link(), try - {H, {_F, _S}, Res} = run_commands(?MODULE, Cmds), - aggregate(zip(state_names(H), command_names(Cmds)), + {H, _, Res} = proper_fsm:run_commands(?MODULE, Cmds), + aggregate(zip(proper_fsm:state_names(H), command_names(Cmds)), ?WHENFAIL( begin - eqc:format("Cmds: ~p~n~n", - [zip(state_names(H), - command_names(Cmds))]), - eqc:format("Result: ~p~n~n", [Res]), - eqc:format("History: ~p~n~n", [H]) + io:format("Cmds: ~p~n~n", + [zip(proper_fsm:state_names(H), + command_names(Cmds))]), + io:format("Result: ~p~n~n", [Res]), + io:format("History: ~p~n~n", [H]) end, equals(ok, Res))) after @@ -129,7 +130,7 @@ prop_manual_commands() -> end). %%==================================================================== -%% eqc_fsm callbacks +%% proper_fsm callbacks %%==================================================================== idle(_S) -> @@ -160,7 +161,7 @@ next_state_data(_From, _To, S, _R, _C) -> precondition(_From, _To, _S, _C) -> true. -postcondition(From, To, S , {call, _M, ManualCommad, _A}=C, R) -> +postcondition(From, To, S, {call, _M, ManualCommad, _A}=C, R) -> {ok, {Actual, _, _}} = riak_cs_gc_manager:status(), ?assertEqual(To, Actual), ExpectedRes = expected_result(From, To, ManualCommad), @@ -168,9 +169,9 @@ postcondition(From, To, S , {call, _M, ManualCommad, _A}=C, R) -> ExpectedRes when ManualCommad =/= status -> true; {ok, {S, _}} -> true; _ -> - eqc:format("Result: ~p~n", [R]), - eqc:format("Expected: ~p~n", [ExpectedRes]), - eqc:format("when {From, To, S, C}: ~p <- ~p~n", [{From, To, S, C}, ManualCommad]), + io:format("Result: ~p~n", [R]), + io:format("Expected: ~p~n", [ExpectedRes]), + io:format("when {From, To, S, C}: ~p <- ~p~n", [{From, To, S, C}, ManualCommad]), false end. @@ -210,5 +211,3 @@ hold_until_unregisterd(RegName, N) -> timer:sleep(1), hold_until_unregisterd(RegName, N - 1) end. - --endif. diff --git a/test/riak_cs_gc_single_run_eqc.erl b/apps/riak_cs/test/prop_riak_cs_gc_single_run.erl similarity index 92% rename from test/riak_cs_gc_single_run_eqc.erl rename to apps/riak_cs/test/prop_riak_cs_gc_single_run.erl index 40f77491a..4c5ef0a60 100644 --- a/test/riak_cs_gc_single_run_eqc.erl +++ b/apps/riak_cs/test/prop_riak_cs_gc_single_run.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,19 +19,22 @@ %% %% --------------------------------------------------------------------- -%% @doc EQC test module for single gc run. +%% @doc PropEr test module for single gc run. %% Test targets is a combination of `riak_cs_gc_batch' and `riak_cs_gc_worker'. %% All calls to riak, 2i/GET/DELETE, are mocked away by `meck'. --module(riak_cs_gc_single_run_eqc). +-module(prop_riak_cs_gc_single_run). + +-export([prop_epochspec/0, + prop_gc_batch/1]). + +-export([]). -include("riak_cs_gc.hrl"). --ifdef(EQC). --include_lib("eqc/include/eqc.hrl"). +-include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). --compile(export_all). -type fileset_keys_input() :: {num_fileset_keys(), no_error}. -type num_fileset_keys() :: non_neg_integer(). @@ -54,14 +58,14 @@ -define(BLOCK_NUM_IN_MANIFEST, 20). -define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). + proper:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). -define(TESTING_TIME, 30). %%==================================================================== %% Eunit tests %%==================================================================== -eqc_test_() -> +proper_test_() -> {foreach, fun() -> application:set_env(lager, handlers, []), @@ -86,20 +90,17 @@ eqc_test_() -> end, [ {timeout, ?TESTING_TIME*2, - ?_assert(quickcheck(eqc:testing_time(?TESTING_TIME, - ?QC_OUT(prop_epochspec()))))}, + ?_assert(proper:quickcheck(?QC_OUT(prop_epochspec())))}, {timeout, ?TESTING_TIME*2, - ?_assert(quickcheck(eqc:testing_time(?TESTING_TIME, - ?QC_OUT(prop_gc_batch(no_error)))))}, + ?_assert(proper:quickcheck(?QC_OUT(prop_gc_batch(no_error))))}, {timeout, ?TESTING_TIME*2, - ?_assert(quickcheck(eqc:testing_time(?TESTING_TIME, - ?QC_OUT(prop_gc_batch(with_errors)))))} + ?_assert(proper:quickcheck(?QC_OUT(prop_gc_batch(with_errors))))} ]}. prop_epochspec() -> ?FORALL({N0, N1, N2, Leeway, BatchSize, MaxWorkers}, {nat2(), nat2(), nat2(), oneof([nat(), nat2()]), - pos_integer(), pos_integer()}, + small_pos_integer(), small_pos_integer()}, begin %[StartKey, EndKey, BatchStart] = [N+1000000000||N<-lists:sort([N0,N1,N2])], [StartKey, EndKey, BatchStart] = lists:sort([N0,N1,N2]), @@ -116,7 +117,7 @@ prop_epochspec() -> nat2() -> low_bounded_int(1000000000). -pos_integer() -> +small_pos_integer() -> low_bounded_int(1). low_bounded_int(LB) -> @@ -135,7 +136,7 @@ prop_gc_batch(ErrorOrNot) -> ?FORALL({ListOfFilesetKeysInput, BatchSize, MaxWorkers}, {non_empty(list(fileset_keys_input(ErrorOrNot))), - pos_integer(), pos_integer()}, + small_pos_integer(), small_pos_integer()}, begin Self = self(), meck:expect(riak_cs_gc_manager, finished, @@ -150,7 +151,7 @@ prop_gc_batch(ErrorOrNot) -> stop_and_wait_for_gc_batch(), ?WHENFAIL( begin - eqc:format("ListOfFilesetKeysInput: ~p~n", + proper:format("ListOfFilesetKeysInput: ~p~n", [ListOfFilesetKeysInput]) end, conjunction([{batch_count, equals(ExpectedBatchCount, element(1, Res))}, @@ -175,7 +176,7 @@ wait_for_stop(Pid) -> ok end. --spec gc_batch([fileset_keys_input()], pos_integer(), pos_integer()) -> eqc:property(). +-spec gc_batch([fileset_keys_input()], pos_integer(), pos_integer()) -> proper:property(). gc_batch(ListOfFilesetKeysInput, BatchSize, MaxWorkers) -> %% For `riak-cs-gc' 2i query, use a process to hold `ListOfFilesetKeysInput'. %% ?debugVal(ListOfFilesetKeysInput), @@ -184,7 +185,7 @@ gc_batch(ListOfFilesetKeysInput, BatchSize, MaxWorkers) -> %% SortedKeys = lists:sort(ListOfFilesetKeysInput), %% {StartKey, _} = hd(SortedKeys), %% {EndKey, _} = lists:last(SortedKeys), - BatchStart = riak_cs_gc:timestamp(), + BatchStart = os:system_time(millisecond), %% ?debugVal({StartKey, EndKey, BatchStart}), {ok, _} = riak_cs_gc_batch:start_link(#gc_batch_state{ batch_start=BatchStart, @@ -200,7 +201,7 @@ gc_batch(ListOfFilesetKeysInput, BatchSize, MaxWorkers) -> block_count=BlockCount} = _State} -> {BatchCount, BatchSkips, ManifCount, BlockCount}; OtherMsg -> - eqc:format("OtherMsg: ~p~n", [OtherMsg]), + proper:format("OtherMsg: ~p~n", [OtherMsg]), {error, error, error, error} end. @@ -231,7 +232,7 @@ expectations(ListOfFilesetKeysInput) -> %% Generator of numbers of fileset keys included in a single object %% of the `riak-cs-gc' bucket, with information of error injection. -spec fileset_keys_input(no_error | with_errors) -> - eqc_gen:gen({non_neg_integer(), error_or_not()}). + proper_gen:gen({non_neg_integer(), error_or_not()}). fileset_keys_input(no_error) -> {num_fileset_keys(), no_error}; fileset_keys_input(with_errors) -> @@ -240,7 +241,7 @@ fileset_keys_input(with_errors) -> {1, {num_fileset_keys(), {error, in_fileset_delete}}}, {1, {num_fileset_keys(), {error, in_block_delete}}}]). --spec num_fileset_keys() -> eqc_gen:gen(Positive::integer()). +-spec num_fileset_keys() -> proper_gen:gen(Positive::integer()). num_fileset_keys() -> ?LET(N, nat(), N+1). @@ -329,7 +330,7 @@ dummy_start_delete_fsm(_Node, [_RcPid, {_UUID, ?MANIFEST{bkey={_, K}}=_Manifest} {match, _} -> 0 end, DummyDeleteFsmPid = - spawn(fun() -> FinishFun({self(), {ok, {b, k, uuid, NumDeleted, TotalBlocks}}}) end), + spawn(fun() -> FinishFun({self(), {ok, {b, k, vsn, uuid, NumDeleted, TotalBlocks}}}) end), {ok, DummyDeleteFsmPid}. %% ==================================================================== @@ -408,12 +409,10 @@ meck_pool_worker() -> dummy_pbc() -> receive stop -> ok; - M -> eqc:format("dummy_worker received M: ~p~n", [M]), + M -> proper:format("dummy_worker received M: ~p~n", [M]), dummy_pbc() end. -spec i2b(integer()) -> binary(). i2b(Integer) -> list_to_binary(integer_to_list(Integer)). - --endif. diff --git a/test/riak_cs_get_fsm_eqc.erl b/apps/riak_cs/test/prop_riak_cs_get_fsm.erl similarity index 68% rename from test/riak_cs_get_fsm_eqc.erl rename to apps/riak_cs/test/prop_riak_cs_get_fsm.erl index dde09faf8..500f5f378 100644 --- a/test/riak_cs_get_fsm_eqc.erl +++ b/apps/riak_cs/test/prop_riak_cs_get_fsm.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,37 +19,37 @@ %% %% --------------------------------------------------------------------- -%% @doc Quickcheck test module for `riak_cs_get_fsm'. +%% @doc PropEr test module for `riak_cs_get_fsm'. --module(riak_cs_get_fsm_eqc). +-module(prop_riak_cs_get_fsm). --ifdef(EQC). - --include_lib("eqc/include/eqc.hrl"). --include_lib("eqc/include/eqc_fsm.hrl"). +-include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). %% Public API --compile(export_all). -export([test/0, test/1]). -%% eqc_fsm callbacks +%% proper_fsm callbacks -export([initial_state/0, next_state_data/5, precondition/4, - postcondition/5]). + postcondition/5, + start_fsm/2, + stop_fsm/1]). -%% eqc property +%% proper property -export([prop_get_fsm/0]). %% States -export([start/1, waiting_chunk/1, - stop/1]). + stop/1, + nop/0, + get_chunk/1]). -define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> - io:format(user, Str, Args) end, P)). + on_output(fun(Str, Args) -> + io:format(user, Str, Args) end, P)). -define(TEST_ITERATIONS, 500). -record(state, {fsm_pid :: pid(), %% real pid of riak_cs_get_fsm} @@ -60,15 +61,15 @@ %% Eunit tests %%==================================================================== -eqc_test_() -> +proper_test_() -> {spawn, [ {setup, fun setup/0, fun cleanup/1, [%% Run the quickcheck tests - {timeout, 30, - ?_assertEqual(true, eqc:quickcheck(eqc:testing_time(15, ?QC_OUT(prop_get_fsm()))))} + {timeout, 60 * 10, + ?_assertEqual(true, proper:quickcheck(?QC_OUT(prop_get_fsm())))} ] } ] @@ -88,34 +89,34 @@ test() -> test(?TEST_ITERATIONS). test(Iterations) -> - eqc:quickcheck(eqc:numtests(Iterations, prop_get_fsm())). + proper:quickcheck(numtests(Iterations, prop_get_fsm())). %% ==================================================================== -%% eqc property +%% proper property %% ==================================================================== prop_get_fsm() -> application:set_env(riak_cs, lfs_block_size, 1048576), - ?FORALL(State, #state{content_length=?LET(X, riak_cs_gen:bounded_content_length(), X * 10)}, - ?FORALL(Cmds, eqc_statem:more_commands(10, commands(?MODULE, {start, State})), - begin - {H, {_F, FinalState}, Res} = run_commands(?MODULE, Cmds), - #state{fsm_pid=FsmPid, content_length=ContentLength, - total_blocks=TotalBlocks, counter=Counter} = FinalState, - stop_fsm(FsmPid), - %% Collect stats how much percentages of blocks are consumed. - ConsumedPercentage = - case TotalBlocks of - undefined -> no_consumption; - _ -> min(100, trunc(100*(Counter)/TotalBlocks)) - end, - collect(with_title(consumed_percentage), ConsumedPercentage, - collect(with_title(content_length_mm), ContentLength/1000000, - collect(with_title(command_length), length(Cmds), - ?WHENFAIL(io:format("history is ~p ~n", [[S || {S, _Rs} <- H]]), - equals(ok, Res))))) - end)). + ?FORALL(State, #state{content_length = ?LET(X, riak_cs_gen:bounded_content_length(), X * 10)}, + ?FORALL(Cmds, proper_statem:more_commands(10, proper_fsm:commands(?MODULE, {start, State})), + begin + {H, {_F, FinalState}, Res} = proper_fsm:run_commands(?MODULE, Cmds), + #state{fsm_pid=FsmPid, content_length=ContentLength, + total_blocks=TotalBlocks, counter=Counter} = FinalState, + stop_fsm(FsmPid), + %% Collect stats how much percentages of blocks are consumed. + ConsumedPercentage = + case TotalBlocks of + undefined -> no_consumption; + _ -> min(100, trunc(100*(Counter)/TotalBlocks)) + end, + collect(with_title(consumed_percentage), ConsumedPercentage, + collect(with_title(content_length_mm), ContentLength/1000000, + collect(with_title(command_length), length(Cmds), + ?WHENFAIL(io:format("history is ~p ~n", [[S || {S, _Rs} <- H]]), + equals(ok, Res))))) + end)). %%==================================================================== %% Generators @@ -143,7 +144,7 @@ stop_fsm(_FsmPid) -> ok. %%==================================================================== -%% eqc_fsm callbacks +%% proper_fsm callbacks %%==================================================================== initial_state() -> @@ -155,8 +156,11 @@ next_state_data(start, waiting_chunk, #state{content_length=ContentLength}=S, R, next_state_data(waiting_chunk, waiting_chunk, #state{counter=Counter}=S, _R, _C) -> S#state{counter=Counter+1}; next_state_data(waiting_chunk, stop, S, _R, _C) -> + S; +next_state_data(stop, stop, S, _R, _C) -> S. + start(#state{content_length=ContentLength}) -> [{waiting_chunk, {call, ?MODULE, start_fsm, [ContentLength, block_size()]}}]. @@ -165,7 +169,9 @@ waiting_chunk(#state{fsm_pid=Pid}) -> {stop, {call, ?MODULE, stop_fsm, [Pid]}}]. stop(_S) -> - []. + [{history, {call, ?MODULE, nop, []}}]. +nop() -> + true. precondition(start, waiting_chunk, _S, _C) -> true; @@ -174,14 +180,20 @@ precondition(waiting_chunk, waiting_chunk, Counter =< TotalBlocks; precondition(waiting_chunk, stop, #state{counter=Counter, total_blocks=TotalBlocks}, _C) -> - Counter =:= TotalBlocks + 1. + Counter =:= TotalBlocks + 1; +precondition(stop, stop, _S, _C) -> + true. + postcondition(start, waiting_chunk, _State, _C, _R) -> true; postcondition(waiting_chunk, waiting_chunk, #state{counter=Counter}, _C, Response) -> validate_waiting_chunk_response(Counter, Response); postcondition(waiting_chunk, stop, #state{counter=Counter, total_blocks=TotalBlocks}, _C, _R) -> - Counter =:= TotalBlocks + 1. + Counter =:= TotalBlocks + 1; +postcondition(stop, stop, _, _, _) -> + true. + %%==================================================================== %% Helpers @@ -195,5 +207,3 @@ validate_waiting_chunk_response(_Counter, {done, Chunk}) -> block_size() -> riak_cs_lfs_utils:block_size(). - --endif. diff --git a/test/riak_cs_lfs_utils_eqc.erl b/apps/riak_cs/test/prop_riak_cs_lfs_utils.erl similarity index 80% rename from test/riak_cs_lfs_utils_eqc.erl rename to apps/riak_cs/test/prop_riak_cs_lfs_utils.erl index 23d248b4f..cadd7250c 100644 --- a/test/riak_cs_lfs_utils_eqc.erl +++ b/apps/riak_cs/test/prop_riak_cs_lfs_utils.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,17 +19,16 @@ %% %% --------------------------------------------------------------------- -%% @doc Quickcheck test module for `riak_cs_lfs_utils'. +%% @doc PropEr test module for `riak_cs_lfs_utils'. --module(riak_cs_lfs_utils_eqc). +-module(prop_riak_cs_lfs_utils). -include("riak_cs.hrl"). --ifdef(EQC). --include_lib("eqc/include/eqc.hrl"). +-include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). -%% eqc property +%% proper property -export([prop_block_count/0]). %% Helpers @@ -36,19 +36,19 @@ test/1]). -define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> - io:format(user, Str, Args) end, P)). + proper:on_output(fun(Str, Args) -> + io:format(user, Str, Args) end, P)). -define(TEST_ITERATIONS, 500). %%==================================================================== %% Eunit tests %%==================================================================== -eqc_test_() -> +proper_test_() -> {spawn, [ - {timeout, 20, ?_assertEqual(true, quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_block_count()))))}, - {timeout, 60, ?_assertEqual(true, quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_manifest_manipulation()))))} + {timeout, 20, ?_assertEqual(true, proper:quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_block_count()))))}, + {timeout, 60, ?_assertEqual(true, proper:quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_manifest_manipulation()))))} ] }. @@ -72,9 +72,10 @@ prop_block_count() -> %% all of the blocks calculated by `initial_blocks` %% have been removed from the manifest prop_manifest_manipulation() -> - ?FORALL({Bucket, FileName, UUID, CLength, Md5, MD}, + ?FORALL({Bucket, FileName, Vsn, UUID, CLength, Md5, MD}, {riak_cs_gen:bucket(), riak_cs_gen:file_name(), + riak_cs_gen:vsn(), riak_cs_gen:uuid(), riak_cs_gen:content_length(), riak_cs_gen:md5(), @@ -85,6 +86,7 @@ prop_manifest_manipulation() -> Manifest = riak_cs_lfs_utils:new_manifest( Bucket, FileName, + Vsn, UUID, CLength, <<"ctype">>, @@ -113,7 +115,5 @@ test() -> test(500). test(Iterations) -> - eqc:quickcheck(eqc:numtests(Iterations, prop_block_count())), - eqc:quickcheck(eqc:numtests(Iterations, prop_manifest_manipulation())). - --endif. + proper:quickcheck(proper:numtests(Iterations, prop_block_count())), + proper:quickcheck(proper:numtests(Iterations, prop_manifest_manipulation())). diff --git a/test/riak_cs_list_objects_fsm_v2_eqc.erl b/apps/riak_cs/test/prop_riak_cs_list_objects_fsm_v2.erl similarity index 86% rename from test/riak_cs_list_objects_fsm_v2_eqc.erl rename to apps/riak_cs/test/prop_riak_cs_list_objects_fsm_v2.erl index 7aca78a54..357bc3b24 100644 --- a/test/riak_cs_list_objects_fsm_v2_eqc.erl +++ b/apps/riak_cs/test/prop_riak_cs_list_objects_fsm_v2.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,18 +19,14 @@ %% %% --------------------------------------------------------------------- --module(riak_cs_list_objects_fsm_v2_eqc). +-module(prop_riak_cs_list_objects_fsm_v2). --ifdef(EQC). - --compile(export_all). --include_lib("eqc/include/eqc.hrl"). +-include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("riak_cs.hrl"). --include("list_objects.hrl"). -%% eqc properties +%% properties -export([prop_skip_past_prefix_and_delimiter/0, prop_prefix_must_be_in_between/0, prop_list_all_active_keys_without_delimiter/0, @@ -41,36 +38,38 @@ -define(TEST_ITERATIONS, 1000). -define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). + on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). + +-define(LOGFILE, "riak_cs_list_objects_fsm_v2_proper.log"). %%==================================================================== %% Eunit tests %%==================================================================== -eqc_test_() -> +proper_test_() -> {spawn, [ {setup, fun setup/0, fun cleanup/1, [ - ?_assert(quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_skip_past_prefix_and_delimiter())))), - ?_assert(quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_prefix_must_be_in_between())))), + ?_assert(proper:quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_skip_past_prefix_and_delimiter())))), + ?_assert(proper:quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_prefix_must_be_in_between())))), {timeout, 10*60, % 10min. - ?_assert(quickcheck(numtests( - ?TEST_ITERATIONS, - ?QC_OUT(prop_list_all_active_keys_without_delimiter()))))}, + ?_assert(proper:quickcheck(numtests( + ?TEST_ITERATIONS, + ?QC_OUT(prop_list_all_active_keys_without_delimiter()))))}, {timeout, 10*60, % 10min. - ?_assert(quickcheck(numtests( - ?TEST_ITERATIONS, - ?QC_OUT(prop_list_all_active_keys_with_delimiter()))))} + ?_assert(proper:quickcheck(numtests( + ?TEST_ITERATIONS, + ?QC_OUT(prop_list_all_active_keys_with_delimiter()))))} ] } ]}. setup() -> error_logger:tty(false), - error_logger:logfile({open, "riak_cs_list_objects_fsm_v2_eqc.log"}), + error_logger:logfile({open, ?LOGFILE}), application:set_env(lager, handlers, []), exometer:start(), riak_cs_stats:init(), @@ -78,6 +77,7 @@ setup() -> cleanup(_) -> exometer:stop(), + file:delete(?LOGFILE), ok. %% ==================================================================== @@ -231,13 +231,10 @@ test() -> test(?TEST_ITERATIONS). test(Iterations) -> - eqc:quickcheck(eqc:numtests(Iterations, prop_skip_past_prefix_and_delimiter())), - eqc:quickcheck(eqc:numtests(Iterations, prop_prefix_must_be_in_between())), - eqc:quickcheck(eqc:numtests(Iterations, prop_list_all_active_keys_without_delimiter())), - eqc:quickcheck(eqc:numtests(Iterations, prop_list_all_active_keys_with_delimiter())). - -test(Iterations, Prop) -> - eqc:quickcheck(eqc:numtests(Iterations, ?MODULE:Prop())). + proper:quickcheck(numtests(Iterations, prop_skip_past_prefix_and_delimiter())), + proper:quickcheck(numtests(Iterations, prop_prefix_must_be_in_between())), + proper:quickcheck(numtests(Iterations, prop_list_all_active_keys_without_delimiter())), + proper:quickcheck(numtests(Iterations, prop_list_all_active_keys_with_delimiter())). %% TODO: Common prefix, more randomness @@ -254,7 +251,9 @@ raw_manifest(Key, State) -> state=State, content_md5 = <<"Content-MD5">>, content_length=100, - acl=?ACL{owner={"display-name", "canonical-id", "key-id"}}}. + acl=?ACL{owner=#{display_name => "display-name", + canonical_id => "canonical-id", + key_id =>"key-id"}}}. bin_key({no_prefix, Rest}) -> Rest; @@ -264,17 +263,18 @@ bin_key(Key) -> Key. process_manifest(Manifest=?MANIFEST{state=State}) -> + TS = os:system_time(millisecond), case State of writing -> - Manifest?MANIFEST{last_block_written_time=os:timestamp()}; + Manifest?MANIFEST{last_block_written_time = TS}; active -> %% this clause isn't needed but it makes things more clear imho - Manifest?MANIFEST{last_block_deleted_time=os:timestamp()}; + Manifest?MANIFEST{last_block_deleted_time = TS}; pending_delete -> - Manifest?MANIFEST{last_block_deleted_time=os:timestamp()}; + Manifest?MANIFEST{last_block_deleted_time = TS}; scheduled_delete -> - Manifest?MANIFEST{last_block_deleted_time=os:timestamp(), - scheduled_delete_time=os:timestamp()} + Manifest?MANIFEST{last_block_deleted_time = TS, + scheduled_delete_time = TS} end. sort_manifests(Manifests) -> @@ -302,7 +302,7 @@ active_manifest_keys(KeysAndStates, <<$/>>=_Delimiter, undefined=_Prefix) -> {lists:usort(CommonPrefixes), lists:usort(Keys)}. keys_in_list({CPrefixes, Contents}) -> - {CPrefixes, [Key || #list_objects_key_content_v1{key=Key} <- Contents]}. + {CPrefixes, [Key || #list_objects_key_content{key=Key} <- Contents]}. list_manifests(Manifests, Opts, UserPage, BatchSize) -> {ok, DummyRc} = riak_cs_dummy_riak_client_list_objects_v2:start_link([Manifests]), @@ -312,7 +312,8 @@ list_manifests_to_the_end(DummyRc, Opts, UserPage, BatchSize, CPrefixesAcc, Cont Bucket = <<"bucket">>, %% TODO: Generator? %% delimeter, marker and prefix should be generated? - ListKeysRequest = riak_cs_list_objects:new_request(Bucket, + ListKeysRequest = riak_cs_list_objects:new_request(objects, + Bucket, UserPage, Opts), {ok, FsmPid} = riak_cs_list_objects_fsm_v2:start_link(DummyRc, ListKeysRequest, BatchSize), @@ -338,7 +339,7 @@ list_manifests_to_the_end(DummyRc, Opts, UserPage, BatchSize, CPrefixesAcc, Cont create_marker(?LORESP{next_marker=undefined, contents=Contents}) -> LastEntry = lists:last(Contents), - LastEntry#list_objects_key_content_v1.key; + LastEntry#list_objects_key_content.key; create_marker(?LORESP{next_marker=NextMarker}) -> NextMarker. @@ -366,7 +367,7 @@ format_diff({NumKeys, StateFlavor, PrefixFlavor}, ok. output_entries(Manifests) -> - FileName = <<".riak_cs_list_objects_fsm_v2_eqc.txt">>, + FileName = <<".riak_cs_list_objects_fsm_v2_proper.txt">>, io:format("Write states and keys to file: ~s ...", [filename:absname(FileName)]), {ok, File} = file:open(FileName, [write, raw]), output_entries(File, Manifests, 1), @@ -423,5 +424,3 @@ print_key_and_manifest(Key, Label, [M | Manifests]) -> _ -> print_key_and_manifest(Key, Label, Manifests) end. - --endif. diff --git a/test/riak_cs_manifest_resolution_eqc.erl b/apps/riak_cs/test/prop_riak_cs_manifest_resolution.erl similarity index 63% rename from test/riak_cs_manifest_resolution_eqc.erl rename to apps/riak_cs/test/prop_riak_cs_manifest_resolution.erl index 5599c032c..8d80919a3 100644 --- a/test/riak_cs_manifest_resolution_eqc.erl +++ b/apps/riak_cs/test/prop_riak_cs_manifest_resolution.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,18 +19,14 @@ %% %% --------------------------------------------------------------------- --module(riak_cs_manifest_resolution_eqc). +-module(prop_riak_cs_manifest_resolution). --ifdef(EQC). - --include_lib("eqc/include/eqc.hrl"). +-include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("riak_cs.hrl"). --compile(export_all). - -%% eqc property +%% proper property -export([prop_resolution_commutative/0]). %% helpers @@ -37,21 +34,21 @@ -define(TEST_ITERATIONS, 500). -define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> - io:format(user, Str, Args) end, P)). + on_output(fun(Str, Args) -> + io:format(user, Str, Args) end, P)). %%==================================================================== %% Eunit tests %%==================================================================== -eqc_test_() -> +proper_test_() -> {spawn, [{setup, fun setup/0, fun cleanup/1, [%% Run the quickcheck tests {timeout, 300, - ?_assertEqual(true, quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT((prop_resolution_commutative())))))} + ?_assertEqual(true, proper:quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT((prop_resolution_commutative())))))} ] } ] @@ -65,19 +62,19 @@ cleanup(_) -> %% ==================================================================== -%% eqc property +%% PropEr property %% ==================================================================== prop_resolution_commutative() -> - ?FORALL(Manifests, eqc_gen:resize(50, manifests()), + ?FORALL(Manifests, resize(50, manifests()), begin MapFun = fun(Mani) -> - riak_cs_manifest_utils:new_dict(Mani?MANIFEST.uuid, Mani) + rcs_common_manifest_utils:new_dict(Mani?MANIFEST.uuid, Mani) end, Filtered = only_one_active(Manifests), WrappedManifests = lists:map(MapFun, Filtered), - Resolved = riak_cs_manifest_resolution:resolve(WrappedManifests), - ReversedResolved = riak_cs_manifest_resolution:resolve(lists:reverse(WrappedManifests)), + Resolved = rcs_common_manifest_resolution:resolve(WrappedManifests), + ReversedResolved = rcs_common_manifest_resolution:resolve(lists:reverse(WrappedManifests)), Resolved == ReversedResolved end). @@ -86,9 +83,10 @@ prop_resolution_commutative() -> %%==================================================================== raw_manifest() -> - ?MANIFEST{uuid=riak_cs_gen:bounded_uuid(), - bkey={<<"bucket">>, <<"key">>}, - state=riak_cs_gen:manifest_state()}. + ?MANIFEST{uuid = riak_cs_gen:bounded_uuid(), + bkey = {<<"bucket">>, <<"key">>}, + vsn = <<"a">>, + state = riak_cs_gen:manifest_state()}. manifest() -> ?LET(Manifest, raw_manifest(), process_manifest(Manifest)). @@ -96,28 +94,28 @@ manifest() -> process_manifest(Manifest=?MANIFEST{state=State}) -> case State of writing -> - Manifest?MANIFEST{last_block_written_time=erlang:now(), - write_blocks_remaining=blocks_set()}; + Manifest?MANIFEST{last_block_written_time = erlang:timestamp(), + write_blocks_remaining = blocks_set()}; active -> %% this clause isn't %% needed but it makes %% things more clear imho - Manifest?MANIFEST{last_block_deleted_time=erlang:now()}; + Manifest?MANIFEST{last_block_deleted_time = erlang:timestamp()}; pending_delete -> - Manifest?MANIFEST{last_block_deleted_time=erlang:now(), - delete_blocks_remaining=blocks_set()}; + Manifest?MANIFEST{last_block_deleted_time = erlang:timestamp(), + delete_blocks_remaining = blocks_set()}; scheduled_delete -> - Manifest?MANIFEST{last_block_deleted_time=erlang:now(), - delete_blocks_remaining=blocks_set()}; + Manifest?MANIFEST{last_block_deleted_time = erlang:timestamp(), + delete_blocks_remaining = blocks_set()}; deleted -> - Manifest?MANIFEST{last_block_deleted_time=erlang:now()} + Manifest?MANIFEST{last_block_deleted_time = erlang:timestamp()} end. manifests() -> - eqc_gen:list(manifest()). + list(manifest()). blocks_set() -> - ?LET(L, eqc_gen:list(int()), ordsets:from_list(L)). + ?LET(L, list(int()), ordsets:from_list(L)). %%==================================================================== %% Helpers @@ -127,7 +125,7 @@ test() -> test(100). test(Iterations) -> - eqc:quickcheck(eqc:numtests(Iterations, prop_resolution_commutative())). + proper:quickcheck(numtests(Iterations, prop_resolution_commutative())). only_one_active(Manifests) -> {_, FilteredManifests} = lists:foldl(fun only_one_active_helper/2, {not_found, []}, Manifests), @@ -137,9 +135,7 @@ only_one_active_helper(?MANIFEST{state=active}, {found, List}) -> {found, List}; only_one_active_helper(Manifest, {found, List}) -> {found, [Manifest | List]}; -only_one_active_helper(Manifest=?MANIFEST{state=active}, {not_found, List}) -> +only_one_active_helper(Manifest=?MANIFEST{state = active}, {not_found, List}) -> {found, [Manifest | List]}; only_one_active_helper(Manifest, {not_found, List}) -> {not_found, [Manifest | List]}. - --endif. %EQC diff --git a/test/riak_cs_manifest_utils_eqc.erl b/apps/riak_cs/test/prop_riak_cs_manifest_utils.erl similarity index 69% rename from test/riak_cs_manifest_utils_eqc.erl rename to apps/riak_cs/test/prop_riak_cs_manifest_utils.erl index caf00c6b7..e7b98061a 100644 --- a/test/riak_cs_manifest_utils_eqc.erl +++ b/apps/riak_cs/test/prop_riak_cs_manifest_utils.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,18 +19,14 @@ %% %% --------------------------------------------------------------------- --module(riak_cs_manifest_utils_eqc). +-module(prop_riak_cs_manifest_utils). --ifdef(EQC). - --include_lib("eqc/include/eqc.hrl"). +-include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("riak_cs.hrl"). --compile(export_all). - -%% eqc property +%% proper property -export([prop_active_manifests/0]). %% helpers @@ -37,23 +34,23 @@ -define(TEST_ITERATIONS, 100). -define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> - io:format(user, Str, Args) end, P)). + on_output(fun(Str, Args) -> + io:format(user, Str, Args) end, P)). %%==================================================================== %% Eunit tests %%==================================================================== -eqc_test_() -> +proper_test_() -> {spawn, [{setup, fun setup/0, fun cleanup/1, [%% Run the quickcheck tests {timeout, 300, - ?_assertEqual(true, quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT((prop_active_manifests())))))}, + ?_assertEqual(true, proper:quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT((prop_active_manifests())))))}, {timeout, 300, - ?_assertEqual(true, quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT((prop_prune_manifests())))))} + ?_assertEqual(true, proper:quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT((prop_prune_manifests())))))} ] } ] @@ -71,12 +68,12 @@ cleanup(_) -> %% ==================================================================== prop_active_manifests() -> - ?FORALL(Manifests, eqc_gen:resize(50, manifests()), + ?FORALL(Manifests, resize(50, manifests()), begin - AlteredManifests = lists:map(fun(M) -> M?MANIFEST{uuid=druuid:v4()} end, Manifests), + AlteredManifests = lists:map(fun(M) -> M?MANIFEST{uuid = uuid:get_v4()} end, Manifests), AsDict = orddict:from_list([{M?MANIFEST.uuid, M} || M <- AlteredManifests]), - ToGcUUIDs = lists:sort(riak_cs_manifest_utils:deleted_while_writing(AsDict)), - Active = riak_cs_manifest_utils:active_manifest(AsDict), + ToGcUUIDs = lists:sort(rcs_common_manifest_utils:deleted_while_writing(AsDict)), + Active = rcs_common_manifest_utils:active_manifest(AsDict), case Active of {error, no_active_manifest} -> %% If no manifest is returned then there should @@ -84,7 +81,7 @@ prop_active_manifests() -> %% the manifest has a write_start_time < delete_marked_time of %% some other deleted manifest's delete_marked_time. ActiveManis = lists:filter(fun(?MANIFEST{state=State}) -> - State == active + State == active end, AlteredManifests), ActiveUUIDs = lists:sort([M?MANIFEST.uuid || M <- ActiveManis]), ActiveUUIDs =:= ToGcUUIDs; @@ -97,30 +94,32 @@ prop_active_manifests() -> end end). +-define(PRUNE_LEEWAY_SECS, 5). + prop_prune_manifests() -> ?FORALL({Manifests, MaxCount}, - {eqc_gen:resize(50, manifests()), frequency([{9, nat()}, {1, 'unlimited'}])}, + {resize(50, manifests()), frequency([{9, nat()}, {1, 'unlimited'}])}, begin - AlteredManifests = lists:map(fun(M) -> M?MANIFEST{uuid=druuid:v4()} end, Manifests), + AlteredManifests = lists:map(fun(M) -> M?MANIFEST{uuid = uuid:get_v4()} end, Manifests), AsDict = orddict:from_list([{M?MANIFEST.uuid, M} || M <- AlteredManifests]), - NowTime = {-1, -1, -1}, + NowTime = -1, case MaxCount of 'unlimited' -> %% We should not prune any manifests if the prune %% count is set to `unlimited'. - AsDict =:= riak_cs_manifest_utils:prune(AsDict, NowTime, MaxCount); + AsDict =:= rcs_common_manifest_utils:prune(AsDict, NowTime, MaxCount, ?PRUNE_LEEWAY_SECS); _ -> prune_helper(AsDict, NowTime, MaxCount) end end). prune_helper(AsDict, NowTime, MaxCount) -> - Pruned = riak_cs_manifest_utils:prune(AsDict, NowTime, MaxCount), - RemainingScheduledDelete = riak_cs_manifest_utils:filter_manifests_by_state(Pruned, [scheduled_delete]), + Pruned = rcs_common_manifest_utils:prune(AsDict, NowTime, MaxCount, ?PRUNE_LEEWAY_SECS), + RemainingScheduledDelete = rcs_common_manifest_utils:filter_manifests_by_state(Pruned, [scheduled_delete]), RemainingScheduledDeleteUUIDs = [UUID || {UUID, _Mani} <- RemainingScheduledDelete], RemainingScheduledDeleteTimes = [M?MANIFEST.scheduled_delete_time || {_UUID, M} <- RemainingScheduledDelete], - AllScheduledDelete = riak_cs_manifest_utils:filter_manifests_by_state(AsDict, [scheduled_delete]), + AllScheduledDelete = rcs_common_manifest_utils:filter_manifests_by_state(AsDict, [scheduled_delete]), DroppedScheduledDelete = orddict:filter(fun (UUID, _) -> not lists:member(UUID, RemainingScheduledDeleteUUIDs) end, AllScheduledDelete), DroppedScheduledDeleteTimes = [M?MANIFEST.scheduled_delete_time || {_UUID, M} <- DroppedScheduledDelete], @@ -146,32 +145,33 @@ manifest() -> ?LET(Manifest, raw_manifest(), process_manifest(Manifest)). process_manifest(Manifest=?MANIFEST{state=State}) -> + TS = os:system_time(millisecond), case State of writing -> - Manifest?MANIFEST{last_block_written_time=erlang:now(), - write_blocks_remaining=blocks_set()}; + Manifest?MANIFEST{last_block_written_time = TS, + write_blocks_remaining = blocks_set()}; active -> %% this clause isn't %% needed but it makes %% things more clear imho - Manifest?MANIFEST{last_block_deleted_time=erlang:now(), - write_start_time=riak_cs_gen:timestamp()}; + Manifest?MANIFEST{last_block_deleted_time = TS, + write_start_time = TS}; pending_delete -> - Manifest?MANIFEST{last_block_deleted_time=erlang:now(), - delete_blocks_remaining=blocks_set(), - delete_marked_time=riak_cs_gen:timestamp(), - props=riak_cs_gen:props()}; + Manifest?MANIFEST{last_block_deleted_time = TS, + delete_blocks_remaining = blocks_set(), + delete_marked_time = TS, + props = riak_cs_gen:props()}; scheduled_delete -> - Manifest?MANIFEST{delete_marked_time=riak_cs_gen:timestamp(), - scheduled_delete_time=riak_cs_gen:timestamp(), - props=riak_cs_gen:props()} + Manifest?MANIFEST{delete_marked_time = TS, + scheduled_delete_time = TS, + props = riak_cs_gen:props()} end. manifests() -> - eqc_gen:list(manifest()). + list(manifest()). blocks_set() -> - ?LET(L, eqc_gen:list(int()), ordsets:from_list(L)). + ?LET(L, list(int()), ordsets:from_list(L)). %%==================================================================== %% Helpers @@ -181,7 +181,5 @@ test() -> test(100). test(Iterations) -> - [eqc:quickcheck(eqc:numtests(Iterations, prop_active_manifests())), - eqc:quickcheck(eqc:numtests(Iterations, prop_prune_manifests()))]. - --endif. %EQC + [proper:quickcheck(numtests(Iterations, prop_active_manifests())), + proper:quickcheck(numtests(Iterations, prop_prune_manifests()))]. diff --git a/test/riak_cs_s3_policy_eqc.erl b/apps/riak_cs/test/prop_riak_cs_s3_policy.erl similarity index 70% rename from test/riak_cs_s3_policy_eqc.erl rename to apps/riak_cs/test/prop_riak_cs_s3_policy.erl index ed365c031..b51cb3d2c 100644 --- a/test/riak_cs_s3_policy_eqc.erl +++ b/apps/riak_cs/test/prop_riak_cs_s3_policy.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,62 +19,64 @@ %% %% --------------------------------------------------------------------- --module(riak_cs_s3_policy_eqc). +-module(prop_riak_cs_s3_policy). --compile(export_all). +-export([prop_ip_filter/0, + prop_secure_transport/0, + prop_eval/0, + prop_policy/0]). --ifdef(TEST). --ifdef(EQC). +-export([string_condition/0, + numeric_condition/0, + date_condition/0]). -include("riak_cs.hrl"). --include("s3_api.hrl"). --include_lib("eqc/include/eqc.hrl"). +-include("aws_api.hrl"). +-include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). --include_lib("webmachine/include/wm_reqdata.hrl"). - -define(TEST_ITERATIONS, 500). -define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). + on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). -define(TIMEOUT, 60). -eqc_test_()-> +proper_test_() -> {inparallel, [ {timeout, ?TIMEOUT, ?_assertEqual(true, - quickcheck(numtests(?TEST_ITERATIONS, - ?QC_OUT(prop_ip_filter()))))}, + proper:quickcheck(numtests(?TEST_ITERATIONS, + ?QC_OUT(prop_ip_filter()))))}, {timeout, ?TIMEOUT, ?_assertEqual(true, - quickcheck(numtests(?TEST_ITERATIONS, - ?QC_OUT(prop_secure_transport()))))}, + proper:quickcheck(numtests(?TEST_ITERATIONS, + ?QC_OUT(prop_secure_transport()))))}, {timeout, ?TIMEOUT, ?_assertEqual(true, - quickcheck(numtests(?TEST_ITERATIONS, - ?QC_OUT(prop_eval()))))}, + proper:quickcheck(numtests(?TEST_ITERATIONS, + ?QC_OUT(prop_eval()))))}, {timeout, ?TIMEOUT, ?_assertEqual(true, - quickcheck(numtests(?TEST_ITERATIONS, - ?QC_OUT(prop_policy_v1()))))} + proper:quickcheck(numtests(?TEST_ITERATIONS, + ?QC_OUT(prop_policy()))))} ]}. %% accept case of ip filtering %% TODO: reject case of ip filtering prop_ip_filter() -> ?FORALL({Policy0, Access0, IP, PrefixDigit}, - {policy_v1(), access_v1(), inet_ip_address_v4(), choose(0,32)}, + {policy(), access_v1(), inet_ip_address_v4(), choose(0,32)}, begin application:set_env(riak_cs, trust_x_forwarded_for, true), %% replace IP in the policy with prefix mask Statement0 = hd(Policy0?POLICY.statement), IPStr0 = lists:flatten(io_lib:format("~s/~p", [inet_parse:ntoa(IP), PrefixDigit])), - IPTuple = riak_cs_s3_policy:parse_ip(IPStr0), + IPTuple = riak_cs_aws_policy:parse_ip(IPStr0), Cond = {'IpAddress', [{'aws:SourceIp', IPTuple}]}, - Statement = Statement0#statement{condition_block=[Cond]}, - Policy = Policy0?POLICY{statement=[Statement]}, + Statement = Statement0#statement{condition_block = [Cond]}, + Policy = Policy0?POLICY{statement = [Statement]}, %% replace IP in the wm_reqdata to match the policy Peer = lists:flatten(io_lib:format("~s", [inet_parse:ntoa(IP)])), @@ -81,8 +84,8 @@ prop_ip_filter() -> Access = Access0#access_v1{req = ReqData0#wm_reqdata{peer=Peer}}, %% eval - JsonPolicy = riak_cs_s3_policy:policy_to_json_term(Policy), - Result = riak_cs_s3_policy:eval(Access, JsonPolicy), + JsonPolicy = riak_cs_aws_policy:policy_to_json_term(Policy), + Result = riak_cs_aws_policy:eval(Access, JsonPolicy), Effect = Statement#statement.effect, case {Result, Effect} of @@ -98,7 +101,7 @@ prop_ip_filter() -> prop_secure_transport() -> %% needs better name instead of Bool - ?FORALL({Policy0, Access, Bool}, {policy_v1(), access_v1(), bool()}, + ?FORALL({Policy0, Access, Bool}, {policy(), access_v1(), bool()}, begin %% inject SecureTransport policy Statement0 = hd(Policy0?POLICY.statement), @@ -111,8 +114,8 @@ prop_secure_transport() -> Scheme = ReqData#wm_reqdata.scheme, %% eval - JsonPolicy = riak_cs_s3_policy:policy_to_json_term(Policy), - Result = riak_cs_s3_policy:eval(Access, JsonPolicy), + JsonPolicy = riak_cs_aws_policy:policy_to_json_term(Policy), + Result = riak_cs_aws_policy:eval(Access, JsonPolicy), Effect = Statement#statement.effect, case {Result, {Scheme, Bool}} of @@ -126,15 +129,15 @@ prop_secure_transport() -> {_, {https, true}} -> (Effect =:= allow) =:= Result end end). - + %% checking not to throw or return unexpected result prop_eval() -> - ?FORALL({Policy, Access}, {policy_v1(), access_v1()}, + ?FORALL({Policy, Access}, {policy(), access_v1()}, begin application:set_env(riak_cs, trust_x_forwarded_for, true), - JsonPolicy = riak_cs_s3_policy:policy_to_json_term(Policy), - case riak_cs_s3_policy:eval(Access, JsonPolicy) of + JsonPolicy = riak_cs_aws_policy:policy_to_json_term(Policy), + case riak_cs_aws_policy:eval(Access, JsonPolicy) of true -> true; false -> true; undefined -> true @@ -142,29 +145,32 @@ prop_eval() -> end). %% policy conversion between JSON <==> record -prop_policy_v1()-> - ?FORALL(Policy, policy_v1(), +prop_policy() -> + ?FORALL(Policy, policy(), begin + %% ?debugVal(Policy), application:set_env(riak_cs, trust_x_forwarded_for, true), JsonPolicy = - riak_cs_s3_policy:policy_to_json_term(Policy), + riak_cs_aws_policy:policy_to_json_term(Policy), {ok, PolicyFromJson} = - riak_cs_s3_policy:policy_from_json(JsonPolicy), + riak_cs_aws_policy:policy_from_json(JsonPolicy), + %%?debugVal({Policy?POLICY.id, PolicyFromJson?POLICY.id}), (Policy?POLICY.id =:= PolicyFromJson?POLICY.id) andalso (Policy?POLICY.version =:= PolicyFromJson?POLICY.version) andalso lists:all(fun({LHS, RHS}) -> - riak_cs_s3_policy:statement_eq(LHS, RHS) + riak_cs_aws_policy:statement_eq(LHS, RHS) end, - lists:zip(Policy?POLICY.statement, - PolicyFromJson?POLICY.statement)) + lists:zip(lists:sort(Policy?POLICY.statement), + lists:sort(PolicyFromJson?POLICY.statement))) end). %% Generators -object_action() -> oneof(?SUPPORTED_OBJECT_ACTION). -bucket_action() -> oneof(?SUPPORTED_BUCKET_ACTION). +object_action() -> oneof(?SUPPORTED_OBJECT_ACTIONS). +bucket_action() -> oneof(?SUPPORTED_BUCKET_ACTIONS). +iam_action() -> oneof(?SUPPORTED_IAM_ACTIONS). string_condition() -> oneof(?STRING_CONDITION_ATOMS). numeric_condition() -> oneof(?NUMERIC_CONDITION_ATOMS). @@ -179,7 +185,7 @@ ip_with_mask() -> begin %% this code is to be moved to riak_cs_s3_policy String = lists:flatten(io_lib:format("~s/~p", [inet_parse:ntoa(IP), PrefixDigit])), - riak_cs_s3_policy:parse_ip(String) + riak_cs_aws_policy:parse_ip(String) end). condition_pair() -> @@ -197,24 +203,23 @@ one_or_more_ip_with_mask() -> %% TODO: FIXME: add a more various form of path path() -> - "test/*". + <<"test/*">>. arn_id() -> %% removing ":" which confuses parser - ?LET(String, list(oneof([choose(33,57), choose(59,127)])), - list_to_binary(String)). + nonempty_binary_char_string(). arn_v1() -> #arn_v1{ provider = aws, service = s3, - region = "cs-ap-e1", %% TODO: add generator + region = <<"cs-ap-e1">>, id = arn_id(), path = path() }. -principal() -> oneof(['*', {aws, '*'}]). +principal() -> oneof(['*', {aws, '*'}, {aws, nonempty_binary_char_string()}]). effect() -> oneof([allow, deny]). @@ -223,33 +228,28 @@ statement() -> sid = nonempty_binary_char_string(), effect = effect(), principal = principal(), - action = oneof([ object_action(), bucket_action(), '*' ]), + action = [action()], not_action = [], resource = oneof([arn_v1(), '*']), - condition_block = list(condition_pair()) + condition_block = [condition_pair()] }. statements() -> non_empty(list(statement())). -creation_time() -> - {nat(), choose(0, 1000000), choose(0, 1000000)}. - -string() -> list(choose(33,127)). +ustring() -> list(choose($a, $z)). binary_char_string() -> - ?LET(String, string(), list_to_binary(String)). + ?LET(String, ustring(), list_to_binary(String)). nonempty_binary_char_string() -> - ?LET({Char, BinString}, {choose(33,127), binary_char_string()}, + ?LET({Char, BinString}, {choose($a,$z), binary_char_string()}, <>). -policy_v1() -> - #policy_v1{ - version = <<"2008-10-17">>, - id = oneof([undefined, binary_char_string()]), - statement = statements(), - creation_time = creation_time() +policy() -> + #policy{ + id = oneof([undefined, riak_cs_aws_utils:make_id(11)]), + statement = statements() }. method() -> @@ -262,25 +262,29 @@ method_from_target(bucket_acl) -> method_from_target(bucket_location) -> 'GET'; method_from_target(bucket_policy) -> oneof(['PUT', 'GET', 'DELETE']); -%% method_from_target(bucket_version) -> 'GET'; +method_from_target(bucket_versioning) -> 'GET'; method_from_target(bucket_uploads) -> 'GET'; method_from_target(object) -> oneof(['PUT', 'GET', 'POST', 'DELETE', 'HEAD']); method_from_target(object_acl) -> oneof(['PUT', 'GET']). +action() -> + oneof([ object_action(), bucket_action(), iam_action(), <<"s3:*">>, <<"iam:*">>, <<"sts:*">>, <<"*">> ]). + access_v1() -> ?LET(Target, oneof([bucket, bucket_acl, bucket_location, - bucket_policy, bucket_uploads, %% bucket_version, + bucket_policy, bucket_uploads, bucket_versioning, bucket_uploads, object, object_acl]), ?LET(Method, method_from_target(Target), #access_v1{ method = Method, target = Target, - id = string(), + action = action(), + id = nonempty_binary_char_string(), bucket = nonempty_binary_char_string(), - key = oneof([undefined, binary_char_string()]), + key = oneof([undefined, nonempty_binary_char_string()]), req = wm_reqdata() })). @@ -303,8 +307,8 @@ wm_reqdata() -> response_code = oneof([undefined, http_response_code()]), max_recv_body = nat(), max_recv_hunk = nat(), - req_cookie = string(), - req_qs = string(), + req_cookie = ustring(), + req_qs = ustring(), req_headers = undefined, req_body = binary_char_string(), resp_redirect = bool(), @@ -315,6 +319,3 @@ wm_reqdata() -> port = choose(1,65535), notes = list(nonempty_binary_char_string()) %% any..? }). - --endif. --endif. diff --git a/test/riak_cs_s3_rewrite_eqc.erl b/apps/riak_cs/test/prop_riak_cs_s3_rewrite.erl similarity index 80% rename from test/riak_cs_s3_rewrite_eqc.erl rename to apps/riak_cs/test/prop_riak_cs_s3_rewrite.erl index 70b0478d6..47cf0ba5e 100644 --- a/test/riak_cs_s3_rewrite_eqc.erl +++ b/apps/riak_cs/test/prop_riak_cs_s3_rewrite.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,40 +19,36 @@ %% %% --------------------------------------------------------------------- -%% @doc Quickcheck test module for `riak_cs_s3_rewrite'. +%% @doc PropEr test module for `riak_cs_s3_rewrite'. --module(riak_cs_s3_rewrite_eqc). +-module(prop_riak_cs_s3_rewrite). -include("riak_cs.hrl"). --ifdef(EQC). --include_lib("eqc/include/eqc.hrl"). +-include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). -%% eqc property +%% PropEr property -export([prop_extract_bucket_from_host/0]). -define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> - io:format(user, Str, Args) end, P)). + on_output(fun(Str, Args) -> + io:format(user, Str, Args) end, P)). -define(TESTING_TIME, 20). %%==================================================================== %% Eunit tests %%==================================================================== -eqc_test_() -> +proper_test_() -> Tests = [ {timeout, ?TESTING_TIME*2, - ?_assertEqual(true, quickcheck(eqc:testing_time(?TESTING_TIME, - ?QC_OUT(prop_s3_rewrite(pathstyle)))))}, + ?_assertEqual(true, proper:quickcheck(?QC_OUT(prop_s3_rewrite(pathstyle))))}, {timeout, ?TESTING_TIME*2, - ?_assertEqual(true, quickcheck(eqc:testing_time(?TESTING_TIME, - ?QC_OUT(prop_s3_rewrite(hoststyle)))))}, + ?_assertEqual(true, proper:quickcheck(?QC_OUT(prop_s3_rewrite(hoststyle))))}, {timeout, ?TESTING_TIME*2, - ?_assertEqual(true, quickcheck(eqc:testing_time(?TESTING_TIME, - ?QC_OUT(prop_extract_bucket_from_host())))) + ?_assertEqual(true, proper:quickcheck(?QC_OUT(prop_extract_bucket_from_host()))) }], [{inparallel, Tests}]. @@ -62,9 +59,9 @@ eqc_test_() -> %% @doc This test verifies that the key for object manifest is exactly same as %% the key before URL encoding. This is also a regression test of riak_cs#1040. prop_s3_rewrite(Style) -> - RewriteModule = riak_cs_s3_rewrite, - RootHost = "example.com", - ok = application:set_env(riak_cs, cs_root_host, RootHost), + RewriteModule = riak_cs_aws_s3_rewrite, + RootHost = "s3.amazonaws.com", + ok = application:set_env(riak_cs, s3_root_host, RootHost), DispatchTable = riak_cs_web:object_api_dispatch_table(), ?FORALL({CSBucket, CSKey, Verb, Scheme, Version}, {riak_cs_gen:bucket(), riak_cs_gen:file_name(), @@ -93,8 +90,8 @@ prop_s3_rewrite(Style) -> %% Corresponding {Bucket, Key} for manifest is %% <<"0o:", hash(Bucket)/binary>> and Key - The key should be exactly %% same as the original one in the client-app before URL encoding. - Ctx = #context{local_context=#key_context{}}, - {ok, #context{local_context=LocalCtx}} = riak_cs_wm_utils:extract_key(RD, Ctx), + Ctx = #rcs_web_context{local_context = #key_context{}}, + {ok, #rcs_web_context{local_context = LocalCtx}} = riak_cs_wm_utils:extract_key(RD, Ctx), %% ?debugVal(CSKey), {CSBucket, CSKey} =:= @@ -111,7 +108,7 @@ prop_extract_bucket_from_host() -> Host = compose_host(BucketStr, BaseHost), ExpectedBucket = expected_bucket(BucketStr, BaseHost), ResultBucket = - riak_cs_s3_rewrite:bucket_from_host(Host, BaseHost), + riak_cs_aws_s3_rewrite:bucket_from_host(Host), equals(ExpectedBucket, ResultBucket) end)). @@ -131,7 +128,7 @@ build_original_path_info(pathstyle, CSBucket, CSKey, RootHost) -> {Encoded, RootHost}. base_host() -> - oneof(["s3.amazonaws.com", "riakcs.net", "snarf", "hah-hah", ""]). + oneof(["s3.amazonaws.com", "s3.out-west.amazonaws.com"]). compose_host([], BaseHost) -> BaseHost; @@ -146,5 +143,3 @@ expected_bucket(_Bucket, []) -> undefined; expected_bucket(Bucket, _) -> Bucket. - --endif. diff --git a/apps/riak_cs/test/prop_riak_cs_utils.erl b/apps/riak_cs/test/prop_riak_cs_utils.erl new file mode 100644 index 000000000..96ae3d947 --- /dev/null +++ b/apps/riak_cs/test/prop_riak_cs_utils.erl @@ -0,0 +1,59 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc PropEr test module for `riak_cs_utils'. + +-module(prop_riak_cs_utils). + +-include("riak_cs.hrl"). +-include_lib("proper/include/proper.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-export([prop_md5/0]). + +-define(QC_OUT(P), + on_output(fun(Str, Args) -> + io:format(user, Str, Args) end, P)). + +%%==================================================================== +%% Eunit tests +%%==================================================================== + +proper_test_() -> + Time = 8, + [ + {timeout, Time*4, ?_assertEqual(true, + proper:quickcheck(?QC_OUT(prop_md5())))} + ]. + +%% ==================================================================== +%% EQC Properties +%% ==================================================================== + +prop_md5() -> + _ = crypto:start(), + ?FORALL(Bin, gen_bin(), + crypto:hash(md5, Bin) == riak_cs_utils:md5(Bin)). + +gen_bin() -> + oneof([binary(), + ?LET({Size, Char}, {choose(5, 2*1024*1024 + 1024), choose(0, 255)}, + list_to_binary(lists:duplicate(Size, Char)))]). diff --git a/test/twop_set_eqc.erl b/apps/riak_cs/test/prop_twop_set.erl similarity index 74% rename from test/twop_set_eqc.erl rename to apps/riak_cs/test/prop_twop_set.erl index f86fde5f5..cc35dba50 100644 --- a/test/twop_set_eqc.erl +++ b/apps/riak_cs/test/prop_twop_set.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,17 +19,14 @@ %% %% --------------------------------------------------------------------- -%% @doc twop_set_eqc: Quickcheck testing for the `twop_set' module. +%% @doc prop_twop_set: PropEr testing for the `twop_set' module. --module(twop_set_eqc). +-module(prop_twop_set). --ifdef(EQC). - --include_lib("eqc/include/eqc.hrl"). --include_lib("eqc/include/eqc_fsm.hrl"). +-include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). -%% eqc properties +%% proper properties -export([prop_twop_set_api/0, prop_twop_set_resolution/0]). @@ -36,7 +34,7 @@ -export([stopped/1, running/1]). -%% eqc_fsm callbacks +%% proper_fsm callbacks -export([initial_state/0, initial_state_data/0, next_state_data/5, @@ -50,30 +48,26 @@ -define(TEST_ITERATIONS, 500). -define(SET_MODULE, twop_set). -define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). + proper:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). --ifdef(namespaced_types). -type stdlib_set() :: sets:set(). --else. --type stdlib_set() :: set(). --endif. --record(eqc_state, {adds=sets:new() :: stdlib_set(), - deletes=sets:new() :: stdlib_set(), - operation_count=0 :: non_neg_integer(), - operation_limit=500 :: pos_integer(), - set :: twop_set:twop_set(), - size=0 :: non_neg_integer()}). +-record(proper_state, {adds=sets:new() :: stdlib_set(), + deletes=sets:new() :: stdlib_set(), + operation_count=0 :: non_neg_integer(), + operation_limit=500 :: pos_integer(), + set :: twop_set:twop_set(), + size=0 :: non_neg_integer()}). %%==================================================================== %% Eunit tests %%==================================================================== -eqc_test_() -> +proper_test_() -> {spawn, [ - {timeout, 60, ?_assertEqual(true, quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_twop_set_api()))))}, - {timeout, 60, ?_assertEqual(true, quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_twop_set_resolution()))))} + {timeout, 60, ?_assertEqual(true, proper:quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_twop_set_api()))))}, + {timeout, 60, ?_assertEqual(true, proper:quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_twop_set_resolution()))))} ] }. @@ -83,24 +77,24 @@ eqc_test_() -> prop_twop_set_api() -> ?FORALL(Cmds, - commands(?MODULE), + proper_fsm:commands(?MODULE), begin - {H, {_F, S}, Res} = run_commands(?MODULE, Cmds), - aggregate(zip(state_names(H), command_names(Cmds)), + {H, {_F, S}, Res} = proper_fsm:run_commands(?MODULE, Cmds), + aggregate(zip(proper_fsm:state_names(H), command_names(Cmds)), ?WHENFAIL( begin ?debugFmt("Cmds: ~p~n", - [zip(state_names(H), + [zip(proper_fsm:state_names(H), command_names(Cmds))]), ?debugFmt("Result: ~p~n", [Res]), ?debugFmt("History: ~p~n", [H]), - ?debugFmt("Current expected size: ~p~n", [S#eqc_state.size]), - ?debugFmt("Current actual size: ~p~n", [twop_set:size(S#eqc_state.set)]), - ?debugFmt("Operation count: ~p~n", [S#eqc_state.operation_count]), - ?debugFmt("Operation limit: ~p~n", [S#eqc_state.operation_limit]), - ?debugFmt("Adds: ~p~n", [sets:to_list(S#eqc_state.adds)]), - ?debugFmt("Deletes: ~p~n", [sets:to_list(S#eqc_state.deletes)]), - ?debugFmt("Set: ~p~n", [twop_set:to_list(S#eqc_state.set)]) + ?debugFmt("Current expected size: ~p~n", [S#proper_state.size]), + ?debugFmt("Current actual size: ~p~n", [twop_set:size(S#proper_state.set)]), + ?debugFmt("Operation count: ~p~n", [S#proper_state.operation_count]), + ?debugFmt("Operation limit: ~p~n", [S#proper_state.operation_limit]), + ?debugFmt("Adds: ~p~n", [sets:to_list(S#proper_state.adds)]), + ?debugFmt("Deletes: ~p~n", [sets:to_list(S#proper_state.deletes)]), + ?debugFmt("Set: ~p~n", [twop_set:to_list(S#proper_state.set)]) end, equals(ok, Res))) end @@ -134,13 +128,13 @@ prop_twop_set_resolution() -> )). %%==================================================================== -%% eqc_fsm callbacks +%% proper_fsm callbacks %%==================================================================== stopped(_S) -> [{running, {call, ?SET_MODULE, new, []}}]. -running(#eqc_state{operation_count=OpCount, +running(#proper_state{operation_count=OpCount, operation_limit=OpLimit, set=Set}) -> [{stopped, {call, ?SET_MODULE, size, [Set]}} || OpCount > OpLimit] ++ @@ -155,21 +149,21 @@ initial_state() -> stopped. initial_state_data() -> - #eqc_state{}. + #proper_state{}. next_state_data(running, stopped, S, _R, _C) -> - S#eqc_state{adds=sets:new(), - deletes=sets:new(), - operation_count=0, - set=undefined, - size=0}; + S#proper_state{adds=sets:new(), + deletes=sets:new(), + operation_count=0, + set=undefined, + size=0}; next_state_data(stopped, running, S, R, {call, _M, new, _}) -> - S#eqc_state{set=R}; + S#proper_state{set=R}; next_state_data(_From, _To, S, R, {call, _M, add_element, [Element, _Set]}) -> - Adds = S#eqc_state.adds, - Dels = S#eqc_state.deletes, - Size = S#eqc_state.size, - OpCount = S#eqc_state.operation_count, + Adds = S#proper_state.adds, + Dels = S#proper_state.deletes, + Size = S#proper_state.size, + OpCount = S#proper_state.operation_count, case sets:is_element(Element, Adds) orelse sets:is_element(Element, Dels) of @@ -180,15 +174,15 @@ next_state_data(_From, _To, S, R, {call, _M, add_element, [Element, _Set]}) -> UpdAdds = sets:add_element(Element, Adds), UpdSize = Size + 1 end, - S#eqc_state{adds=UpdAdds, - operation_count=OpCount+1, - set=R, - size=UpdSize}; + S#proper_state{adds=UpdAdds, + operation_count=OpCount+1, + set=R, + size=UpdSize}; next_state_data(_From, _To, S, R, {call, _M, del_element, [Element, _Set]}) -> - Adds = S#eqc_state.adds, - Dels = S#eqc_state.deletes, - Size = S#eqc_state.size, - OpCount = S#eqc_state.operation_count, + Adds = S#proper_state.adds, + Dels = S#proper_state.deletes, + Size = S#proper_state.size, + OpCount = S#proper_state.operation_count, case sets:is_element(Element, Dels) of true -> UpdDels = Dels, @@ -204,32 +198,32 @@ next_state_data(_From, _To, S, R, {call, _M, del_element, [Element, _Set]}) -> UpdSize = Size end end, - S#eqc_state{deletes=UpdDels, - operation_count=OpCount+1, - set=R, - size=UpdSize}; + S#proper_state{deletes=UpdDels, + operation_count=OpCount+1, + set=R, + size=UpdSize}; next_state_data(_From, _To, S, _R, _C) -> - OpCount = S#eqc_state.operation_count, - S#eqc_state{operation_count=OpCount+1}. + OpCount = S#proper_state.operation_count, + S#proper_state{operation_count=OpCount+1}. precondition(_From, _To, _S, _C) -> true. -postcondition(_From, _To, #eqc_state{size=Size} ,{call, _M, size, _}, R) -> +postcondition(_From, _To, #proper_state{size=Size} ,{call, _M, size, _}, R) -> R =:= Size; postcondition(_From, _To, S, {call, _M, to_list, _}, R) -> - #eqc_state{adds=Adds, - deletes=Dels} = S, + #proper_state{adds=Adds, + deletes=Dels} = S, R =:= sets:to_list(sets:subtract(Adds, Dels)); postcondition(_From, _To, S, {call, _M, is_element, [Element, _Set]}, R) -> - #eqc_state{adds=Adds, - deletes=Dels} = S, + #proper_state{adds=Adds, + deletes=Dels} = S, (sets:is_element(Element, Adds) andalso not sets:is_element(Element, Dels)) =:= R; postcondition(_From, _To, S, {call, _M, add_element, [Element, Set]}, R) -> - #eqc_state{adds=Adds, - deletes=Dels} = S, + #proper_state{adds=Adds, + deletes=Dels} = S, ResultContainsElement = sets:is_element(Element, twop_set:adds(R)), ShouldContainElement = not sets:is_element(Element, Dels), case sets:is_element(Element, Adds) @@ -256,7 +250,7 @@ test() -> test(500). test(Iterations) -> - eqc:quickcheck(eqc:numtests(Iterations, prop_twop_set_api())). + proper:quickcheck(numtests(Iterations, prop_twop_set_api())). -spec sibling_sets([{[term()], [term()]}]) -> [twop_set:twop_set()]. sibling_sets(Siblings) -> @@ -284,5 +278,3 @@ expected_adds_and_dels(Siblings) -> siblings() -> list({list(int()), list(int())}). - --endif. diff --git a/test/riak_cs_s3_auth_v4_test.erl b/apps/riak_cs/test/riak_cs_aws_auth_v4_test.erl similarity index 84% rename from test/riak_cs_s3_auth_v4_test.erl rename to apps/riak_cs/test/riak_cs_aws_auth_v4_test.erl index b981f0edb..fcda615e4 100644 --- a/test/riak_cs_s3_auth_v4_test.erl +++ b/apps/riak_cs/test/riak_cs_aws_auth_v4_test.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,16 +19,17 @@ %% %% --------------------------------------------------------------------- --module(riak_cs_s3_auth_v4_test). +-module(riak_cs_aws_auth_v4_test). -compile(export_all). +-compile(nowarn_export_all). -include("riak_cs.hrl"). -include_lib("eunit/include/eunit.hrl"). %% Test cases at http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html --define(ACCESS_KEY_ID, "AKIAIOSFODNN7EXAMPLE"). --define(SECRET_ACCESS_KEY, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"). +-define(ACCESS_KEY_ID, <<"AKIAIOSFODNN7EXAMPLE">>). +-define(SECRET_ACCESS_KEY, <<"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY">>). -define(VERSION, {1, 1}). -define(BUCKET, "examplebucket"). @@ -72,10 +74,10 @@ auth_v4_GET_Object() -> RD = wrq:create(Method, ?VERSION, "rewritten/path", AllHeaders), Context = context_not_used, ?assertEqual({?ACCESS_KEY_ID, {v4, AuthAttrs}}, - riak_cs_s3_auth:identify(RD, Context)), + riak_cs_aws_auth:identify(RD, Context)), ?assertEqual(ok, - riak_cs_s3_auth:authenticate(?RCS_USER{key_secret=?SECRET_ACCESS_KEY}, - {v4, AuthAttrs}, RD, Context)). + riak_cs_aws_auth:authenticate(?RCS_USER{key_secret=?SECRET_ACCESS_KEY}, + {v4, AuthAttrs}, RD, Context)). auth_v4_GET_Object_with_extra_spaces() -> Method = 'GET', @@ -97,10 +99,10 @@ auth_v4_GET_Object_with_extra_spaces() -> RD = wrq:create(Method, ?VERSION, "rewritten/path", AllHeaders), Context = context_not_used, ?assertEqual({?ACCESS_KEY_ID, {v4, AuthAttrs}}, - riak_cs_s3_auth:identify(RD, Context)), + riak_cs_aws_auth:identify(RD, Context)), ?assertEqual(ok, - riak_cs_s3_auth:authenticate(?RCS_USER{key_secret=?SECRET_ACCESS_KEY}, - {v4, AuthAttrs}, RD, Context)). + riak_cs_aws_auth:authenticate(?RCS_USER{key_secret=?SECRET_ACCESS_KEY}, + {v4, AuthAttrs}, RD, Context)). auth_v4_PUT_Object() -> Method = 'PUT', @@ -120,10 +122,10 @@ auth_v4_PUT_Object() -> RD = wrq:create(Method, ?VERSION, "rewritten/path", AllHeaders), Context = context_not_used, ?assertEqual({?ACCESS_KEY_ID, {v4, AuthAttrs}}, - riak_cs_s3_auth:identify(RD, Context)), + riak_cs_aws_auth:identify(RD, Context)), ?assertEqual(ok, - riak_cs_s3_auth:authenticate(?RCS_USER{key_secret=?SECRET_ACCESS_KEY}, - {v4, AuthAttrs}, RD, Context)). + riak_cs_aws_auth:authenticate(?RCS_USER{key_secret=?SECRET_ACCESS_KEY}, + {v4, AuthAttrs}, RD, Context)). auth_v4_GET_Object_Lifecycle() -> Method = 'GET', @@ -142,10 +144,10 @@ auth_v4_GET_Object_Lifecycle() -> RD = wrq:create(Method, ?VERSION, "rewritten/path", AllHeaders), Context = context_not_used, ?assertEqual({?ACCESS_KEY_ID, {v4, AuthAttrs}}, - riak_cs_s3_auth:identify(RD, Context)), + riak_cs_aws_auth:identify(RD, Context)), ?assertEqual(ok, - riak_cs_s3_auth:authenticate(?RCS_USER{key_secret=?SECRET_ACCESS_KEY}, - {v4, AuthAttrs}, RD, Context)). + riak_cs_aws_auth:authenticate(?RCS_USER{key_secret=?SECRET_ACCESS_KEY}, + {v4, AuthAttrs}, RD, Context)). auth_v4_GET_Bucket() -> Method = 'GET', @@ -163,10 +165,10 @@ auth_v4_GET_Bucket() -> RD = wrq:create(Method, ?VERSION, "rewritten/path", AllHeaders), Context = context_not_used, ?assertEqual({?ACCESS_KEY_ID, {v4, AuthAttrs}}, - riak_cs_s3_auth:identify(RD, Context)), + riak_cs_aws_auth:identify(RD, Context)), ?assertEqual(ok, - riak_cs_s3_auth:authenticate(?RCS_USER{key_secret=?SECRET_ACCESS_KEY}, - {v4, AuthAttrs}, RD, Context)). + riak_cs_aws_auth:authenticate(?RCS_USER{key_secret=?SECRET_ACCESS_KEY}, + {v4, AuthAttrs}, RD, Context)). authorization_header(AuthAttrs) -> authorization_header(AuthAttrs, no_space). diff --git a/apps/riak_cs/test/riak_cs_aws_policy_test.erl b/apps/riak_cs/test/riak_cs_aws_policy_test.erl new file mode 100644 index 000000000..156717d62 --- /dev/null +++ b/apps/riak_cs/test/riak_cs_aws_policy_test.erl @@ -0,0 +1,267 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + +%% @doc ad-hoc policy tests + +-module(riak_cs_aws_policy_test). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("riak_cs.hrl"). +-include("aws_api.hrl"). + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +parse_ip_test_()-> + [ + ?_assertEqual({{192,0,0,1}, {255,0,0,0}}, + riak_cs_aws_policy:parse_ip(<<"192.0.0.1/8">>)), + ?_assertEqual({error, einval}, + riak_cs_aws_policy:parse_ip(<<"192.3.1/16">>)), + ?_assertEqual(<<"1.2.3.4">>, + riak_cs_aws_policy:print_ip(riak_cs_aws_policy:parse_ip(<<"1.2.3.4">>))), + ?_assertEqual(<<"1.2.3.4/13">>, + riak_cs_aws_policy:print_ip(riak_cs_aws_policy:parse_ip(<<"1.2.3.4/13">>))), + ?_assertEqual({error, einval}, + riak_cs_aws_policy:parse_ip(<<"0">>)), + ?_assertEqual({error, einval}, + riak_cs_aws_policy:parse_ip(<<"0/0">>)) + ]. + +empty_statement_conversion_test() -> + Policy = ?POLICY{id = <<"hello">>, statement = [#statement{}]}, + {error, missing_principal} = riak_cs_aws_policy:policy_from_json( + riak_cs_aws_policy:policy_to_json_term(Policy)). + +sample_plain_allow_policy()-> + <<"{" + "\"Id\":\"Policy1354069963875\"," + "\"Statement\":[" + "{" + " \"Sid\":\"Stmt1354069958376\"," + " \"Action\":[" + " \"s3:CreateBucket\"," + " \"s3:DeleteBucket\"," + " \"s3:DeleteBucketPolicy\"," + " \"s3:DeleteObject\"," + " \"s3:GetBucketAcl\"," + " \"s3:GetBucketPolicy\"," + " \"s3:GetObject\"," + " \"s3:GetObjectAcl\"," + " \"s3:ListAllMyBuckets\"," + " \"s3:ListBucket\"," + " \"s3:PutBucketAcl\"," + " \"s3:PutBucketPolicy\"," +% " \"s3:PutObject\"," + " \"s3:PutObjectAcl\"" + " ]," + " \"Condition\":{" + " \"IpAddress\": { \"aws:SourceIp\":[\"192.168.0.1/8\", \"192.168.0.2/17\"] }" + " }," + " \"Effect\": \"Allow\"," + " \"Resource\": \"arn:aws:s3:::test\"," + " \"Principal\": {" + " \"AWS\": \"*\"" + " }" + " }" + " ]" + "}" >>. + +sample_policy_check_test() -> + application:set_env(riak_cs, trust_x_forwarded_for, true), + JsonPolicy0 = sample_plain_allow_policy(), + {ok, Policy} = riak_cs_aws_policy:policy_from_json(JsonPolicy0), + Access = #access_v1{method = 'GET', + target = object, + id = <<"spam/ham/egg">>, + req = #wm_reqdata{peer = "192.168.0.1"}, + action = 'iam:CreateUser', + bucket = <<"test">>}, + Access2 = Access#access_v1{method='PUT', target=object}, + ?assertEqual(undefined, riak_cs_aws_policy:eval(Access2, Policy)), + Access3 = Access#access_v1{req = #wm_reqdata{peer = "1.1.1.1"}}, + ?assertEqual(undefined, riak_cs_aws_policy:eval(Access3, Policy)). + +sample_conversion_test() -> + JsonPolicy0 = sample_plain_allow_policy(), + {ok, Policy} = riak_cs_aws_policy:policy_from_json(JsonPolicy0), + {ok, PolicyFromJson} = riak_cs_aws_policy:policy_from_json( + riak_cs_aws_policy:policy_to_json_term(Policy)), + ?assertEqual(Policy?POLICY.id, PolicyFromJson?POLICY.id), + ?assert(lists:all(fun({LHS, RHS}) -> + riak_cs_aws_policy:statement_eq(LHS, RHS) + end, + lists:zip(Policy?POLICY.statement, + PolicyFromJson?POLICY.statement))), + ?assertEqual(Policy?POLICY.version, PolicyFromJson?POLICY.version). + + +eval_all_ip_addr_test() -> + ?assert(riak_cs_aws_policy:eval_all_ip_addr([{{192,168,0,1}, {255,255,255,255}}], {192,168,0,1})), + ?assert(not riak_cs_aws_policy:eval_all_ip_addr([{{192,168,0,1}, {255,255,255,255}}], {192,168,25,1})), + ?assert(riak_cs_aws_policy:eval_all_ip_addr([{{192,168,0,1}, {255,255,255,0}}], {192,168,0,23})). + +eval_ip_address_test() -> + ?assert(riak_cs_aws_policy:eval_ip_address(#wm_reqdata{peer = "23.23.23.23"}, + [garbage, {chiba, boo}, "saitama", + {'aws:SourceIp', {{23,23,0,0}, {255,255,0,0}}}, hage])). + +eval_ip_address_test_trust_x_forwarded_for_false_test() -> + application:set_env(riak_cs, trust_x_forwarded_for, false), + Conds = [garbage,{chiba, boo},"saitama", + {'aws:SourceIp', {{23,23,0,0},{255,255,0,0}}}, hage], + %% This test fails because it tries to use the socket from wm_reqstate to + %% get the peer address, but it's not a real wm request. + %% If trust_x_forwarded_for = true, it would just use the peer address and the call would + %% succeed + {'EXIT', {{badrecord, ThisorThatRecord}, _}} = + (catch riak_cs_aws_policy:eval_ip_address(#wm_reqdata{peer="23.23.23.23"}, Conds)), + ?assert(ThisorThatRecord == wm_reqstate orelse + ThisorThatRecord == defined_on_call), + %% Reset env for next test + application:set_env(riak_cs, trust_x_forwarded_for, true). + +eval_ip_addresses_test() -> + ?assert(riak_cs_aws_policy:eval_ip_address(#wm_reqdata{peer = "23.23.23.23"}, + [{'aws:SourceIp', {{1,1,1,1}, {255,255,255,0}}}, + {'aws:SourceIp', {{23,23,0,0}, {255,255,0,0}}}, hage])). + +eval_condition_test() -> + ?assert(riak_cs_aws_policy:eval_condition(#wm_reqdata{peer = "23.23.23.23"}, + {'IpAddress', [garbage,{chiba, boo},"saitama", + {'aws:SourceIp', {{23,23,0,0}, {255,255,0,0}}}, hage]})). + +eval_statement_test() -> + Access = #access_v1{method = 'GET', + action = 's3:GetObject', + target = object, + req = #wm_reqdata{peer = "23.23.23.23"}, + bucket= <<"testbokee">>}, + Statement = #statement{effect = allow, + condition_block = + [{'IpAddress', + [{'aws:SourceIp', {{23,23,0,0}, {255,255,0,0}}}]}], + action = ['s3:GetObject'], + resource = '*'}, + ?assert(riak_cs_aws_policy:eval_statement(Access, Statement)). + +my_split_test_()-> + [ + ?_assertEqual(["foo", "bar"], riak_cs_aws_policy:my_split($:, "foo:bar", [], [])), + ?_assertEqual(["foo", "", "", "bar"], riak_cs_aws_policy:my_split($:, "foo:::bar", [], [])), + ?_assertEqual(["arn", "aws", "s3", "", "", "hoge"], + riak_cs_aws_policy:my_split($:, "arn:aws:s3:::hoge", [], [])), + ?_assertEqual(["arn", "aws", "s3", "", "", "hoge/*"], + riak_cs_aws_policy:my_split($:, "arn:aws:s3:::hoge/*", [], [])) + ]. + +parse_arn_test()-> + List0 = [<<"arn:aws:s3:::hoge">>, <<"arn:aws:s3:::hoge/*">>], + {ok, ARNS0} = riak_cs_aws_policy:parse_arns(List0), + ?assertEqual(List0, riak_cs_aws_policy:print_arns(ARNS0)), + + List1 = [<<"arn:aws:s3:ap-northeast-1:000000:hoge">>, <<"arn:aws:s3:::hoge/*">>], + {ok, ARNS1} = riak_cs_aws_policy:parse_arns(List1), + ?assertEqual(List1, riak_cs_aws_policy:print_arns(ARNS1)), + + ?assertEqual({error, bad_arn}, riak_cs_aws_policy:parse_arns([<<"asdfiua;sfkjsd">>])), + + List2 = <<"*">>, + {ok, ARNS2} = riak_cs_aws_policy:parse_arns(List2), + ?assertEqual(List2, riak_cs_aws_policy:print_arns(ARNS2)). + +sample_securetransport_statement() -> + <<"{" + "\"Id\":\"Policy135406996387500\"," + "\"Statement\":[" + "{" + " \"Sid\":\"Stmt135406995deadbeef\"," + " \"Action\":[" + " \"s3:GetObject\"," + " \"s3:PutObject\"," + " \"s3:DeleteObject\"" + " ]," + " \"Condition\":{" + " \"Bool\": { \"aws:SecureTransport\":true }" + " }," + " \"Effect\": \"Allow\"," + " \"Resource\": \"arn:aws:s3:::test\"," + " \"Principal\": {" + " \"AWS\": \"*\"" + " }" + " }" + " ]" + "}" >>. + + +secure_transport_test() -> + application:set_env(riak_cs, trust_x_forwarded_for, true), + JsonPolicy0 = sample_securetransport_statement(), + {ok, Policy} = riak_cs_aws_policy:policy_from_json(JsonPolicy0), + Req = #wm_reqdata{peer = "192.168.0.1", scheme = https}, + Access = #access_v1{method ='GET', + target = object, + id = <<"spam/ham/egg">>, + action = 's3:GetObject', + req = Req, + bucket = <<"test">>}, + ?assert(riak_cs_aws_policy:eval(Access, Policy)), + % io:format(standard_error, "~w~n", [Policy]), + Access2 = Access#access_v1{req = Req#wm_reqdata{scheme = http}}, + ?assertEqual(undefined, riak_cs_aws_policy:eval(Access2, Policy)). + +%% "Bool": { "aws:SecureTransport" : true, +%% "aws:SecureTransport" : false } is recognized as false +%% +%% "Bool": { "aws:SecureTransport" : false, +%% "aws:SecureTransport" : true } is recognized as true + +malformed_json_statement()-> + <<"{" + "\"Id\":\"Policy135406996387500\"," + "\"Statement\":[" + "{" + " \"Sid\":\"Stmt135406995deadbeef\"," + " \"Action\":[" + " \"s3:GetObject\"," + " \"s3:PutObject\"," + " \"s3:DeleteObject\"" + " ]," + " \"Condition\":{" + " \"Bool\": { \"aws:SecureTransport\":tr }" + " }," + " \"Effect\": \"Allow\"," + " \"Resource\": \"arn:aws:s3:::test\"," + " \"Principal\": {" + " \"AWS\": \"*\"" + " }" + " }" + " ]" + "}" >>. + +malformed_policy_json_test()-> + JsonPolicy0 = malformed_json_statement(), + {error, malformed_policy_json} = riak_cs_aws_policy:policy_from_json(JsonPolicy0). + +-endif. diff --git a/test/riak_cs_bucket_name_test.erl b/apps/riak_cs/test/riak_cs_bucket_name_test.erl similarity index 94% rename from test/riak_cs_bucket_name_test.erl rename to apps/riak_cs/test/riak_cs_bucket_name_test.erl index 717d75ae4..8b33471ec 100644 --- a/test/riak_cs_bucket_name_test.erl +++ b/apps/riak_cs/test/riak_cs_bucket_name_test.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -21,6 +22,7 @@ -module(riak_cs_bucket_name_test). -compile(export_all). +-compile(nowarn_export_all). -ifdef(TEST). diff --git a/test/riak_cs_bucket_test.erl b/apps/riak_cs/test/riak_cs_bucket_test.erl similarity index 57% rename from test/riak_cs_bucket_test.erl rename to apps/riak_cs/test/riak_cs_bucket_test.erl index 8f77e9753..7cfd6b76d 100644 --- a/test/riak_cs_bucket_test.erl +++ b/apps/riak_cs/test/riak_cs_bucket_test.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -21,50 +22,34 @@ -module(riak_cs_bucket_test). -compile(export_all). +-compile(nowarn_export_all). -ifdef(TEST). -include("riak_cs.hrl"). -include_lib("eunit/include/eunit.hrl"). -handle_delete_response_test() -> - ErrorDoc = - "" - " MultipartUploadRemaining Multipart uploads still remaining." - "/buckets/riak-test-bucket", - %% which is defined at stanchion_response.erl - ?assertEqual({error, remaining_multipart_upload}, - riak_cs_bucket:handle_stanchion_response(409, ErrorDoc, delete, <<>>)), - ?assertThrow({remaining_multipart_upload_on_deleted_bucket, <<>>}, - riak_cs_bucket:handle_stanchion_response(409, ErrorDoc, create, <<>>)), - ErrorResponse = riak_cs_s3_response:error_response(ErrorDoc), - ?assertEqual(ErrorResponse, - riak_cs_bucket:handle_stanchion_response(503, ErrorDoc, delete, <<>>)), - ?assertEqual(ErrorResponse, - riak_cs_bucket:handle_stanchion_response(204, ErrorDoc, delete, <<>>)). - - bucket_resolution_test() -> %% @TODO Replace or augment this with eqc testing. - UserRecord = riak_cs_user:user_record("uncle fester", - "fester@tester.com", - "festersquest", - "wasthebest", - "cid"), - BucketList1 = [riak_cs_bucket:bucket_record(<<"bucket1">>, create), - riak_cs_bucket:bucket_record(<<"bucket2">>, create), - riak_cs_bucket:bucket_record(<<"bucket3">>, create)], - BucketList2 = [riak_cs_bucket:bucket_record(<<"bucket1">>, create), - riak_cs_bucket:bucket_record(<<"bucket1">>, create), - riak_cs_bucket:bucket_record(<<"bucket1">>, create)], - BucketList3 = [riak_cs_bucket:bucket_record(<<"bucket1">>, create), - riak_cs_bucket:bucket_record(<<"bucket1">>, delete), - riak_cs_bucket:bucket_record(<<"bucket1">>, create)], - BucketList4 = [riak_cs_bucket:bucket_record(<<"bucket1">>, create), - riak_cs_bucket:bucket_record(<<"bucket1">>, create), - riak_cs_bucket:bucket_record(<<"bucket1">>, delete)], - BucketList5 = [riak_cs_bucket:bucket_record(<<"bucket1">>, delete), - riak_cs_bucket:bucket_record(<<"bucket1">>, delete), - riak_cs_bucket:bucket_record(<<"bucket1">>, delete)], + UserRecord = ?RCS_USER{name = <<"uncle fester">>, + email = <<"fester@tester.com">>, + key_id = <<"festersquest">>, + key_secret = <<"wasthebest">>, + id = <<"cid">>}, + BucketList1 = [bucket_record(<<"bucket1">>, create), + bucket_record(<<"bucket2">>, create), + bucket_record(<<"bucket3">>, create)], + BucketList2 = [bucket_record(<<"bucket1">>, create), + bucket_record(<<"bucket1">>, create), + bucket_record(<<"bucket1">>, create)], + BucketList3 = [bucket_record(<<"bucket1">>, create), + bucket_record(<<"bucket1">>, delete), + bucket_record(<<"bucket1">>, create)], + BucketList4 = [bucket_record(<<"bucket1">>, create), + bucket_record(<<"bucket1">>, create), + bucket_record(<<"bucket1">>, delete)], + BucketList5 = [bucket_record(<<"bucket1">>, delete), + bucket_record(<<"bucket1">>, delete), + bucket_record(<<"bucket1">>, delete)], Obj1 = riakc_obj:new_obj(<<"bucket">>, <<"key">>, <<"value">>, @@ -106,5 +91,17 @@ bucket_resolution_test() -> ?assertEqual([hd(lists:reverse(BucketList4))], ResBuckets4), ?assertEqual([hd(BucketList5)], ResBuckets5). +%% This function existed in riak_cs_bucket, but was refactored out. +%% Return a bucket record for the specified bucket name. +bucket_record(Name, Operation) -> + Action = case Operation of + create -> created; + delete -> deleted; + _ -> undefined + end, + ?RCS_BUCKET{name=binary_to_list(Name), + last_action=Action, + creation_date=riak_cs_wm_utils:iso_8601_datetime(), + modification_time=os:timestamp()}. -endif. diff --git a/test/riak_cs_config_test.erl b/apps/riak_cs/test/riak_cs_config_test.erl similarity index 69% rename from test/riak_cs_config_test.erl rename to apps/riak_cs/test/riak_cs_config_test.erl index bf9821fe5..2b35f7ed3 100644 --- a/test/riak_cs_config_test.erl +++ b/apps/riak_cs/test/riak_cs_config_test.erl @@ -1,26 +1,48 @@ +%% --------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% --------------------------------------------------------------------- + -module(riak_cs_config_test). -compile(export_all). +-compile(nowarn_export_all). +-include("riak_cs.hrl"). -include_lib("eunit/include/eunit.hrl"). default_config_test() -> Config = cuttlefish_unit:generate_templated_config(schema_files(), [], context()), cuttlefish_unit:assert_config(Config, "riak_cs.listener", {"127.0.0.1", 8080}), cuttlefish_unit:assert_config(Config, "riak_cs.riak_host", {"127.0.0.1", 8087}), - cuttlefish_unit:assert_config(Config, "riak_cs.stanchion_host", {"127.0.0.1", 8085}), + cuttlefish_unit:assert_config(Config, "riak_cs.stanchion_port", 8085), cuttlefish_unit:assert_config(Config, "riak_cs.stanchion_ssl", false), cuttlefish_unit:assert_not_configured(Config, "riak_cs.ssl"), cuttlefish_unit:assert_config(Config, "riak_cs.anonymous_user_creation", false), - cuttlefish_unit:assert_config(Config, "riak_cs.admin_key", "admin-key"), + cuttlefish_unit:assert_config(Config, "riak_cs.admin_key", binary_to_list(?DEFAULT_ADMIN_KEY)), cuttlefish_unit:assert_not_configured(Config, "riak_cs.admin_secret"), cuttlefish_unit:assert_not_configured(Config, "riak_cs.admin_ip"), cuttlefish_unit:assert_not_configured(Config, "riak_cs.admin_port"), - cuttlefish_unit:assert_config(Config, "riak_cs.cs_root_host", "s3.amazonaws.com"), - cuttlefish_unit:assert_config(Config, "riak_cs.cs_version", 10300), + cuttlefish_unit:assert_config(Config, "riak_cs.s3_root_host", ?S3_ROOT_HOST), + cuttlefish_unit:assert_config(Config, "riak_cs.cs_version", ?RCS_VERSION), cuttlefish_unit:assert_config(Config, "riak_cs.proxy_get", false), cuttlefish_unit:assert_not_configured(Config, "riak_cs.rewrite_module"), cuttlefish_unit:assert_not_configured(Config, "riak_cs.auth_module"), - cuttlefish_unit:assert_config(Config, "riak_cs.fold_objects_for_list_keys", true), cuttlefish_unit:assert_config(Config, "riak_cs.max_buckets_per_user", 100), cuttlefish_unit:assert_config(Config, "riak_cs.max_key_length", 1024), cuttlefish_unit:assert_config(Config, "riak_cs.trust_x_forwarded_for", false), @@ -28,7 +50,6 @@ default_config_test() -> cuttlefish_unit:assert_config(Config, "riak_cs.max_scheduled_delete_manifests", 50), cuttlefish_unit:assert_config(Config, "riak_cs.gc_interval", 900), cuttlefish_unit:assert_config(Config, "riak_cs.gc_retry_interval", 21600), - cuttlefish_unit:assert_config(Config, "riak_cs.gc_paginated_indexes", true), cuttlefish_unit:assert_config(Config, "riak_cs.gc_max_workers", 2), cuttlefish_unit:assert_config(Config, "riak_cs.gc_batch_size", 1000), cuttlefish_unit:assert_config(Config, "riak_cs.active_delete_threshold", 0), @@ -41,7 +62,6 @@ default_config_test() -> cuttlefish_unit:assert_not_configured(Config, "riak_cs.storage_schedule"), cuttlefish_unit:assert_config(Config, "riak_cs.storage_archive_period", 86400), cuttlefish_unit:assert_config(Config, "riak_cs.usage_request_limit", 744), - cuttlefish_unit:assert_config(Config, "riak_cs.dtrace_support", false), cuttlefish_unit:assert_config(Config, "riak_cs.connection_pools", [{request_pool, {128, 0}}, {bucket_list_pool, {5, 0}}]), @@ -50,17 +70,6 @@ default_config_test() -> {riak_cs_access_log_handler, []}]), cuttlefish_unit:assert_config(Config, "webmachine.server_name", "Riak CS"), - {ok, [ConsoleLog, ErrorLog]} = cuttlefish_unit:path(cuttlefish_variable:tokenize("lager.handlers"), Config), - cuttlefish_unit:assert_config([ConsoleLog], "lager_file_backend", [{file, "./log/console.log"}, - {level, info}, - {size, 10485760}, - {date, "$D0"}, - {count, 5}]), - cuttlefish_unit:assert_config([ErrorLog], "lager_file_backend", [{file, "./log/error.log"}, - {level, error}, - {size, 10485760}, - {date, "$D0"}, - {count, 5}]), cuttlefish_unit:assert_not_configured(Config, "riak_cs.supercluster_members"), cuttlefish_unit:assert_config(Config, "riak_cs.supercluster_weight_refresh_interval", 900), %% cuttlefish_unit:assert_config(Config, "vm_args.+scl", false), @@ -103,7 +112,7 @@ gc_interval_infinity_test() -> cuttlefish_unit:assert_config(Config, "riak_cs.gc_interval", infinity), ok. -max_scheduled_delete_manifests_unlimited_test() -> +max_scheduled_delete_manifests_unlimited_test() -> Conf = [{["max_scheduled_delete_manifests"], unlimited}], Config = cuttlefish_unit:generate_templated_config(schema_files(), Conf, context()), cuttlefish_unit:assert_config(Config, "riak_cs.max_scheduled_delete_manifests", unlimited), @@ -115,52 +124,6 @@ active_delete_threshold_test() -> cuttlefish_unit:assert_config(Config, "riak_cs.active_delete_threshold", 10*1024*1024), ok. -lager_syslog_test() -> - Conf = [{["log", "syslog"], on}, - {["log", "syslog", "ident"], "ident-test"}, - {["log", "syslog", "facility"], local7}, - {["log", "syslog", "level"], debug} - ], - Config = cuttlefish_unit:generate_templated_config(schema_files(), Conf, context()), - cuttlefish_unit:assert_config(Config, "lager.handlers.lager_syslog_backend", ["ident-test", local7, debug]), - ok. - -lager_hander_test() -> - Conf = [ - {["log", "console", "file"], "./log/consolefile.log"}, - {["log", "console", "level"], "debug"}, - {["log", "console", "size"], "1MB"}, - {["log", "console", "rotation"], "$D5"}, - {["log", "console", "rotation", "keep"], "10"}, - {["log", "error", "file"], "./log/errorfile.log"}, - {["log", "error", "size"], "1KB"}, - {["log", "error", "rotation"], "$D10"}, - {["log", "error", "rotation", "keep"], "20"} - ], - Config = cuttlefish_unit:generate_templated_config(schema_files(), Conf, context()), - {ok, [ConsoleLog, ErrorLog]} = cuttlefish_unit:path(cuttlefish_variable:tokenize("lager.handlers"), Config), - cuttlefish_unit:assert_config([ConsoleLog], "lager_file_backend", [{file, "./log/consolefile.log"}, - {level, debug}, - {size, 1048576}, - {date, "$D5"}, - {count, 10}]), - cuttlefish_unit:assert_config([ErrorLog], "lager_file_backend", [{file, "./log/errorfile.log"}, - {level, error}, - {size, 1024}, - {date, "$D10"}, - {count, 20}]), - - CurrentConf1 = [{["log", "console", "rotation", "keep"], "current"}], - Config1 = cuttlefish_unit:generate_templated_config(schema_files(), CurrentConf1, context()), - {ok, [ConsoleLog1, _ErrorLog1]} = cuttlefish_unit:path(cuttlefish_variable:tokenize("lager.handlers"), Config1), - cuttlefish_unit:assert_config([ConsoleLog1], "lager_file_backend.count", 0), - - CurrentConf2 = [{["log", "error", "rotation", "keep"], "current"}], - Config2 = cuttlefish_unit:generate_templated_config(schema_files(), CurrentConf2, context()), - {ok, [_ConsoleLog2, ErrorLog2]} = cuttlefish_unit:path(cuttlefish_variable:tokenize("lager.handlers"), Config2), - cuttlefish_unit:assert_config([ErrorLog2], "lager_file_backend.count", 0), - ok. - max_buckets_per_user_test() -> DefConf = [{["max_buckets_per_user"], "100"}], DefConfig = cuttlefish_unit:generate_templated_config(schema_files(), DefConf, context()), @@ -205,7 +168,7 @@ wm_log_config_test_() -> end, fun(AssertAlog) -> [{"Default access log directory", - ?_test(AssertAlog([{["log", "access", "dir"], "$(platform_log_dir)"}], + ?_test(AssertAlog([{["log", "access", "dir"], "./log"}], ["./log"]))}, {"Customized access log directory", ?_test(AssertAlog([{["log", "access", "dir"], "/path/to/custom/dir/"}], @@ -214,7 +177,7 @@ wm_log_config_test_() -> ?_test(AssertAlog([], ["./log"]))}, {"Disable access log", - ?_test(AssertAlog([{["log", "access", "dir"], "$(platform_log_dir)"}, + ?_test(AssertAlog([{["log", "access", "dir"], "./log"}, {["log", "access"], "off"}], no_alog))} ] @@ -265,8 +228,8 @@ supercluster_weight_refresh_interval_test_() -> ]. schema_files() -> - ["../rel/files/riak_cs.schema"]. + ["apps/riak_cs/priv/riak_cs.schema"]. context() -> - {ok, Context} = file:consult("../rel/vars.config"), + {ok, Context} = file:consult("rel/vars.config"), Context. diff --git a/test/riak_cs_delete_deadlock.erl b/apps/riak_cs/test/riak_cs_delete_deadlock.erl similarity index 91% rename from test/riak_cs_delete_deadlock.erl rename to apps/riak_cs/test/riak_cs_delete_deadlock.erl index 7bb0afa15..f613b0082 100644 --- a/test/riak_cs_delete_deadlock.erl +++ b/apps/riak_cs/test/riak_cs_delete_deadlock.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved, +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -74,7 +75,8 @@ prop_delete_deadlock() -> BlockSize = riak_cs_lfs_utils:block_size(), Manifest = riak_cs_lfs_utils:new_manifest( <<"bucket">>, - "test_file", + <<"test_file">>, + ?LFS_DEFAULT_OBJECT_VERSION, UUID, ?CONTENT_LENGTH, <<"ctype">>, @@ -88,7 +90,7 @@ prop_delete_deadlock() -> undefined, undefined), MpM = ?MULTIPART_MANIFEST{parts = Parts}, - NewManifest = Manifest?MANIFEST{props = + NewManifest = Manifest?MANIFEST{props = riak_cs_mp_utils:replace_mp_manifest(MpM, Manifest?MANIFEST.props)}, OutputList = riak_cs_lfs_utils:block_sequences_for_manifest(NewManifest), @@ -111,21 +113,21 @@ assemble_test_list(ContentLength, BlockSize, Parts) -> part_manifests() -> not_empty(eqc_gen:list(part())). -raw_part() -> - ?PART_MANIFEST{bucket= <<"part_bucket">>, +raw_part() -> + ?PART_MANIFEST{bucket= <<"part_bucket">>, key = <<"part_key">>, start_time = os:timestamp(), part_id = g_uuid(), content_length = ?CONTENT_LENGTH, block_size=?BLOCK_SIZE}. -part() -> +part() -> ?LET(Part, raw_part(), process_part(Part)). process_part(Part) -> Part?PART_MANIFEST{part_number = choose(1,1000)}. g_uuid() -> - noshrink(eqc_gen:bind(eqc_gen:bool(), fun(_) -> druuid:v4_str() end)). + noshrink(eqc_gen:bind(eqc_gen:bool(), fun(_) -> uuid:get_v4() end)). not_empty(G) -> ?SUCHTHAT(X, G, X /= [] andalso X /= <<>>). diff --git a/test/riak_cs_dummy_reader.erl b/apps/riak_cs/test/riak_cs_dummy_reader.erl similarity index 84% rename from test/riak_cs_dummy_reader.erl rename to apps/riak_cs/test/riak_cs_dummy_reader.erl index c35668de7..f3d7f814c 100644 --- a/test/riak_cs_dummy_reader.erl +++ b/apps/riak_cs/test/riak_cs_dummy_reader.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -54,6 +55,7 @@ block_size :: integer(), bucket :: binary(), key :: binary(), + obj_vsn :: binary(), caller_pid :: pid()}). -type state() :: #state{}. @@ -79,28 +81,30 @@ get_manifest(Pid) -> %% =================================================================== %% @doc Initialize the server. --spec init([pid()] | {test, [pid()]}) -> {ok, state()} | {stop, term()}. -init([CallerPid, Bucket, Key, ContentLength, BlockSize]) -> +init([CallerPid, Bucket, Key, Vsn, ContentLength, BlockSize]) -> process_flag(trap_exit, true), %% Get a connection to riak - random:seed(now()), - {ok, #state{content_length=ContentLength, - remaining=ContentLength, - bucket=Bucket, - key=Key, - block_size=BlockSize, - caller_pid=CallerPid}}. + rand:seed(exsss, erlang:timestamp()), + {ok, #state{content_length = ContentLength, + remaining = ContentLength, + bucket = Bucket, + key = Key, + obj_vsn = Vsn, + block_size = BlockSize, + caller_pid = CallerPid}}. %% @doc Unused -spec handle_call(term(), {pid(), term()}, state()) -> {reply, ok, state()}. -handle_call(get_manifest, _From, #state{bucket=Bucket, - key=Key, - content_length=ContentLength}=State) -> +handle_call(get_manifest, _From, #state{bucket = Bucket, + key = Key, + obj_vsn = Vsn, + content_length = ContentLength} = State) -> Manifest = riak_cs_lfs_utils:new_manifest( Bucket, Key, - druuid:v4(), + Vsn, + uuid:get_v4(), ContentLength, "application/test", <<"md5">>, diff --git a/test/riak_cs_dummy_riak_client_list_objects_v2.erl b/apps/riak_cs/test/riak_cs_dummy_riak_client_list_objects_v2.erl similarity index 97% rename from test/riak_cs_dummy_riak_client_list_objects_v2.erl rename to apps/riak_cs/test/riak_cs_dummy_riak_client_list_objects_v2.erl index 61e6764e6..5a69cc290 100644 --- a/test/riak_cs_dummy_riak_client_list_objects_v2.erl +++ b/apps/riak_cs/test/riak_cs_dummy_riak_client_list_objects_v2.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved, +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -166,6 +167,6 @@ manifests_to_robjs(Manifests) -> %% TODO: Metadatas manifest_to_robj(?MANIFEST{bkey={Bucket, Key}, uuid=UUID}=M) -> - Dict = riak_cs_manifest_utils:new_dict(UUID, M), + Dict = rcs_common_manifest_utils:new_dict(UUID, M), ManifestBucket = riak_cs_utils:to_bucket_name(objects, Bucket), riakc_obj:new(ManifestBucket, Key, riak_cs_utils:encode_term(Dict)). diff --git a/test/riak_cs_gen.erl b/apps/riak_cs/test/riak_cs_gen.erl similarity index 92% rename from test/riak_cs_gen.erl rename to apps/riak_cs/test/riak_cs_gen.erl index d9561e219..cace9de8f 100644 --- a/test/riak_cs_gen.erl +++ b/apps/riak_cs/test/riak_cs_gen.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -18,13 +19,11 @@ %% %% --------------------------------------------------------------------- -%% @doc Common QuickCheck generators for Riak CS +%% @doc Common PropEr generators for Riak CS -module(riak_cs_gen). --ifdef(EQC). - --include_lib("eqc/include/eqc.hrl"). +-include_lib("proper/include/proper.hrl"). %% Generators -export([base64string/0, @@ -34,6 +33,7 @@ bucket/0, bucket_or_blank/0, file_name/0, + vsn/0, block_size/0, content_length/0, bounded_content_length/0, @@ -46,6 +46,7 @@ datetime/0, md5_chunk_size/0, timestamp/0, + email/0, props/0]). -export([non_blank_string/0]). @@ -75,6 +76,9 @@ bucket_or_blank() -> file_name() -> non_blank_string(). +vsn() -> + non_blank_string(). + block_size() -> elements([bs(El) || El <- [8, 16, 32]]). @@ -110,7 +114,7 @@ datetime() -> {choose(0, 23), choose(0, 59), choose(0, 59)}}. timestamp() -> - {choose(0, 5000), choose(0, 999999), choose(0, 999999)}. + nat(). md5_chunk_size() -> oneof([2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]). @@ -118,6 +122,9 @@ md5_chunk_size() -> props() -> oneof([[], [{deleted, true}]]). +email() -> + iolist_to_binary([riak_cs_aws_utils:make_id(4), $@, riak_cs_aws_utils:make_id(4), ".com"]). + %%==================================================================== %% Helpers %%==================================================================== @@ -176,6 +183,3 @@ char_mid() -> char_end() -> oneof(?ALPHABET ++ ?DIGIT). - - --endif. diff --git a/test/riak_cs_get_fsm_pulse.erl b/apps/riak_cs/test/riak_cs_get_fsm_pulse.erl similarity index 98% rename from test/riak_cs_get_fsm_pulse.erl rename to apps/riak_cs/test/riak_cs_get_fsm_pulse.erl index 95832995e..5ece6f22b 100644 --- a/test/riak_cs_get_fsm_pulse.erl +++ b/apps/riak_cs/test/riak_cs_get_fsm_pulse.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -20,7 +21,6 @@ -module(riak_cs_get_fsm_pulse). --ifdef(EQC). -ifdef(PULSE). -include_lib("eqc/include/eqc.hrl"). @@ -166,4 +166,3 @@ test(Iterations) -> eqc:quickcheck(eqc:numtests(Iterations, prop_blocks_in_order())). -endif. %% PULSE --endif. %% EQC diff --git a/test/riak_cs_lfs_utils_test.erl b/apps/riak_cs/test/riak_cs_lfs_utils_test.erl similarity index 95% rename from test/riak_cs_lfs_utils_test.erl rename to apps/riak_cs/test/riak_cs_lfs_utils_test.erl index c5d05f0db..f886b7098 100644 --- a/test/riak_cs_lfs_utils_test.erl +++ b/apps/riak_cs/test/riak_cs_lfs_utils_test.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/test/riak_cs_list_objects_utils_test.erl b/apps/riak_cs/test/riak_cs_list_objects_utils_test.erl similarity index 85% rename from test/riak_cs_list_objects_utils_test.erl rename to apps/riak_cs/test/riak_cs_list_objects_utils_test.erl index 166ad7fc1..5c198e552 100644 --- a/test/riak_cs_list_objects_utils_test.erl +++ b/apps/riak_cs/test/riak_cs_list_objects_utils_test.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -23,6 +24,7 @@ -ifdef(TEST). -compile(export_all). +-compile(nowarn_export_all). -include("riak_cs.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -32,24 +34,24 @@ filter_prefix_keys_test_() -> [ %% simple test - test_creator(riak_cs_list_objects:new_request(<<"bucket">>), + test_creator(riak_cs_list_objects:new_request(objects, <<"bucket">>), {manifests(), []}), %% simple prefix - test_creator(riak_cs_list_objects:new_request(<<"bucket">>, + test_creator(riak_cs_list_objects:new_request(objects, <<"bucket">>, 1000, [{prefix, <<"a">>}]), {manifests([<<"a">>]), []}), %% simple prefix 2 - test_creator(riak_cs_list_objects:new_request(<<"bucket">>, + test_creator(riak_cs_list_objects:new_request(objects, <<"bucket">>, 1000, [{prefix, <<"photos/">>}]), {lists:sublist(manifests(), 4, length(manifests())), []}), %% prefix and delimiter - test_creator(riak_cs_list_objects:new_request(<<"bucket">>, + test_creator(riak_cs_list_objects:new_request(objects, <<"bucket">>, 1000, [{prefix, <<"photos/">>}, {delimiter, <<"/">>}]), @@ -58,14 +60,14 @@ filter_prefix_keys_test_() -> %% prefix and delimiter 2 %% The only difference from the above test is %% in the `prefix', note the lack of `/' after `photos' - test_creator(riak_cs_list_objects:new_request(<<"bucket">>, + test_creator(riak_cs_list_objects:new_request(objects, <<"bucket">>, 1000, [{prefix, <<"photos">>}, {delimiter, <<"/">>}]), {[], [<<"photos/">>]}), %% prefix and delimiter - test_creator(riak_cs_list_objects:new_request(<<"bucket">>, + test_creator(riak_cs_list_objects:new_request(objects, <<"bucket">>, 1000, [{delimiter, <<"/">>}]), {manifests([<<"a">>, <<"b">>, <<"c">>]), diff --git a/test/riak_cs_s3_rewrite_test.erl b/apps/riak_cs/test/riak_cs_s3_rewrite_test.erl similarity index 74% rename from test/riak_cs_s3_rewrite_test.erl rename to apps/riak_cs/test/riak_cs_s3_rewrite_test.erl index ae30ac9e8..35fed1dd3 100644 --- a/test/riak_cs_s3_rewrite_test.erl +++ b/apps/riak_cs/test/riak_cs_s3_rewrite_test.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -21,32 +22,25 @@ -module(riak_cs_s3_rewrite_test). -include("riak_cs.hrl"). --include("s3_api.hrl"). +-include("aws_api.hrl"). -include_lib("eunit/include/eunit.hrl"). --define(RCS_REWRITE_HEADER, "x-rcs-rewrite-path"). --compile(export_all). +-compile([export_all, nowarn_export_all]). rewrite_path_test() -> - application:set_env(riak_cs, cs_root_host, ?ROOT_HOST), + application:set_env(riak_cs, cs_root_host, ?S3_ROOT_HOST), rstrt(), - rewrite_header(riak_cs_s3_rewrite), - rewrite_path(riak_cs_s3_rewrite), - strict_rewrite_path(riak_cs_s3_rewrite). - -legacy_rewrite_path_test() -> - application:set_env(riak_cs, cs_root_host, ?ROOT_HOST), - rewrite_header(riak_cs_s3_rewrite_legacy), - rewrite_path(riak_cs_s3_rewrite_legacy), - legacy_rewrite_path(riak_cs_s3_rewrite_legacy). + rewrite_header(riak_cs_aws_s3_rewrite), + rewrite_path(riak_cs_aws_s3_rewrite), + strict_rewrite_path(riak_cs_aws_s3_rewrite). rstrt() -> - ?assertEqual("foo." ++ ?ROOT_HOST, - riak_cs_s3_rewrite:bucket_from_host("foo." ++ ?ROOT_HOST ++ "." ++ ?ROOT_HOST, - ?ROOT_HOST)). + ?assertEqual("foo." ++ ?S3_ROOT_HOST, + riak_cs_aws_s3_rewrite:bucket_from_host( + "foo." ++ ?S3_ROOT_HOST ++ "." ++ ?S3_ROOT_HOST)). rewrite_path(Mod) -> - application:set_env(riak_cs, cs_root_host, ?ROOT_HOST), + application:set_env(riak_cs, cs_root_host, ?S3_ROOT_HOST), %% List Buckets URL equal_paths("/buckets", rewrite_with(Mod, headers([]), "/")), @@ -55,57 +49,57 @@ rewrite_path(Mod) -> rewrite_with(Mod, 'GET', headers([]), "/testbucket")), equal_paths("/buckets/testbucket/objects", - rewrite_with(Mod, 'GET', headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + rewrite_with(Mod, 'GET', headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/")), equal_paths("/buckets/testbucket/objects?max-keys=20&delimiter=%2F&prefix=123", rewrite_with(Mod, 'GET', headers([]), "/testbucket?prefix=123&delimiter=/&max-keys=20")), equal_paths("/buckets/testbucket/objects?max-keys=20&delimiter=%2F&prefix=123", - rewrite_with(Mod, 'GET', headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + rewrite_with(Mod, 'GET', headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/?prefix=123&delimiter=/&max-keys=20")), equal_paths("/buckets/testbucket", rewrite_with(Mod, 'HEAD', headers([]), "/testbucket")), equal_paths("/buckets/testbucket", - rewrite_with(Mod, 'HEAD', headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + rewrite_with(Mod, 'HEAD', headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/")), equal_paths("/buckets/testbucket", rewrite_with(Mod, 'PUT', headers([]), "/testbucket")), equal_paths("/buckets/testbucket", - rewrite_with(Mod, 'PUT', headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + rewrite_with(Mod, 'PUT', headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/")), equal_paths("/buckets/testbucket", rewrite_with(Mod, 'DELETE', headers([]), "/testbucket")), equal_paths("/buckets/testbucket", - rewrite_with(Mod, 'DELETE', headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + rewrite_with(Mod, 'DELETE', headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/")), equal_paths("/buckets/testbucket/acl", rewrite_with(Mod, headers([]), "/testbucket?acl")), equal_paths("/buckets/testbucket/acl", - rewrite_with(Mod, headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + rewrite_with(Mod, headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/?acl")), equal_paths("/buckets/testbucket/location", rewrite_with(Mod, headers([]), "/testbucket?location")), equal_paths("/buckets/testbucket/location", - rewrite_with(Mod, headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + rewrite_with(Mod, headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/?location")), equal_paths("/buckets/testbucket/versioning", rewrite_with(Mod, headers([]), "/testbucket?versioning")), equal_paths("/buckets/testbucket/versioning", - rewrite_with(Mod, headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + rewrite_with(Mod, headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/?versioning")), equal_paths("/buckets/testbucket/policy", rewrite_with(Mod, headers([]), "/testbucket?policy")), equal_paths("/buckets/testbucket/policy", - rewrite_with(Mod, headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + rewrite_with(Mod, headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/?policy")), equal_paths("/buckets/testbucket/uploads", rewrite_with(Mod, headers([]), "/testbucket?uploads")), equal_paths("/buckets/testbucket/uploads", - rewrite_with(Mod, headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + rewrite_with(Mod, headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/?uploads")), equal_paths("/buckets/testbucket/uploads?delimiter=D&prefix=ABC&max-uploads=10" "&key-marker=bob&upload-id-marker=blah", @@ -114,163 +108,144 @@ rewrite_path(Mod) -> "&max-uploads=10&prefix=ABC&delimiter=D")), equal_paths("/buckets/testbucket/uploads?delimiter=D&prefix=ABC&max-uploads=10" "&key-marker=bob&upload-id-marker=blah", - rewrite_with(Mod, headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + rewrite_with(Mod, headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/?uploads&upload-id-marker=blah&key-marker=bob" "&max-uploads=10&prefix=ABC&delimiter=D")), equal_paths("/buckets/testbucket/delete", rewrite_with(Mod, 'POST', headers([]), "/testbucket/?delete")), equal_paths("/buckets/testbucket/delete", - rewrite_with(Mod, 'POST', headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + rewrite_with(Mod, 'POST', headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/?delete")), %% Object Operations - equal_paths("/buckets/testbucket/objects/testobject", + equal_paths("/buckets/testbucket/objects/testobject/versions/null", rewrite_with(Mod, headers([]), "/testbucket/testobject")), - equal_paths("/buckets/testbucket/objects/testdir%2F", + equal_paths("/buckets/testbucket/objects/testdir%2F/versions/null", rewrite_with(Mod, headers([]), "/testbucket/testdir/")), - equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject", + equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject/versions/null", rewrite_with(Mod, headers([]), "/testbucket/testdir/testobject")), - equal_paths("/buckets/testbucket/objects/testobject", + equal_paths("/buckets/testbucket/objects/testobject/versions/null", rewrite_with(Mod, - headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/testobject")), - equal_paths("/buckets/testbucket/objects/testdir%2F", + equal_paths("/buckets/testbucket/objects/testdir%2F/versions/null", rewrite_with(Mod, - headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/testdir/")), - equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject", + equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject/versions/null", rewrite_with(Mod, - headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/testdir/testobject")), - equal_paths("/buckets/testbucket/objects/testobject/acl", + equal_paths("/buckets/testbucket/objects/testobject/versions/null/acl", rewrite_with(Mod, headers([]), "/testbucket/testobject?acl")), - equal_paths("/buckets/testbucket/objects/testdir%2F/acl", + equal_paths("/buckets/testbucket/objects/testdir%2F/versions/null/acl", rewrite_with(Mod, headers([]), "/testbucket/testdir/?acl")), - equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject/acl", + equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject/versions/null/acl", rewrite_with(Mod, headers([]), "/testbucket/testdir/testobject?acl")), - equal_paths("/buckets/testbucket/objects/testobject/acl", + equal_paths("/buckets/testbucket/objects/testobject/versions/null/acl", rewrite_with(Mod, - headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/testobject?acl")), - equal_paths("/buckets/testbucket/objects/testdir%2F/acl", + equal_paths("/buckets/testbucket/objects/testdir%2F/versions/null/acl", rewrite_with(Mod, - headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/testdir/?acl")), - equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject/acl", + equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject/versions/null/acl", rewrite_with(Mod, - headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/testdir/testobject?acl")), - equal_paths("/buckets/testbucket/objects/testobject/uploads", + equal_paths("/buckets/testbucket/objects/testobject/versions/null/uploads", rewrite_with(Mod, headers([]), "/testbucket/testobject?uploads")), - equal_paths("/buckets/testbucket/objects/testobject/uploads", + equal_paths("/buckets/testbucket/objects/testobject/versions/null/uploads", rewrite_with(Mod, - headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/testobject?uploads")), - equal_paths("/buckets/testbucket/objects/testobject/uploads/2", + equal_paths("/buckets/testbucket/objects/testobject/versions/null/uploads/2", rewrite_with(Mod, headers([]), "/testbucket/testobject?uploadId=2")), - equal_paths("/buckets/testbucket/objects/testobject/uploads/2", + equal_paths("/buckets/testbucket/objects/testobject/versions/null/uploads/2", rewrite_with(Mod, - headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/testobject?uploadId=2")), - equal_paths("/buckets/testbucket/objects/testobject/uploads/2?partNumber=1", + equal_paths("/buckets/testbucket/objects/testobject/versions/null/uploads/2?partNumber=1", rewrite_with(Mod, headers([]), "/testbucket/testobject?partNumber=1&uploadId=2")), - equal_paths("/buckets/testbucket/objects/testobject/uploads/2?partNumber=1", + equal_paths("/buckets/testbucket/objects/testobject/versions/null/uploads/2?partNumber=1", rewrite_with(Mod, - headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/testobject?partNumber=1&uploadId=2")), - equal_paths("/buckets/testbucket/objects/testobject/uploads/2?AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR" + equal_paths("/buckets/testbucket/objects/testobject/versions/null/uploads/2?AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR" "&Expires=1364406757&Signature=x%2B0vteNN1YillZNw4yDGVQWrT2s%3D", rewrite_with(Mod, headers([]), "/testbucket/testobject?Signature=x%2B0vteNN1YillZNw4yDGVQWrT2s%3D" "&Expires=1364406757&AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR&uploadId=2")), - equal_paths("/buckets/testbucket/objects/testobject/uploads/2?AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR" + equal_paths("/buckets/testbucket/objects/testobject/versions/null/uploads/2?AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR" "&Expires=1364406757&Signature=x%2B0vteNN1YillZNw4yDGVQWrT2s%3D", rewrite_with(Mod, - headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/testobject?Signature=x%2B0vteNN1YillZNw4yDGVQWrT2s%3D" "&Expires=1364406757&AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR&uploadId=2")), - equal_paths("/buckets/testbucket/objects/testobject/uploads/2?AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR" + equal_paths("/buckets/testbucket/objects/testobject/versions/null/uploads/2?AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR" "&Expires=1364406757&Signature=x%2B0vteNN1YillZNw4yDGVQWrT2s%3D&partNumber=1", rewrite_with(Mod, headers([]), "/testbucket/testobject?Signature=x%2B0vteNN1YillZNw4yDGVQWrT2s%3D" "&Expires=1364406757&AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR&partNumber=1&uploadId=2")), - equal_paths("/buckets/testbucket/objects/testobject/uploads/2?AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR" + equal_paths("/buckets/testbucket/objects/testobject/versions/null/uploads/2?AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR" "&Expires=1364406757&Signature=x%2B0vteNN1YillZNw4yDGVQWrT2s%3D&partNumber=1", rewrite_with(Mod, - headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/testobject?Signature=x%2B0vteNN1YillZNw4yDGVQWrT2s%3D" "&Expires=1364406757&AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR&partNumber=1&uploadId=2")), - equal_paths("/buckets/testbucket/objects/testobject?AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR" + equal_paths("/buckets/testbucket/objects/testobject/versions/null?AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR" "&Expires=1364406757&Signature=x%2B0vteNN1YillZNw4yDGVQWrT2s%3D", rewrite_with(Mod, headers([]), "/testbucket/testobject?Signature=x%2B0vteNN1YillZNw4yDGVQWrT2s%3D" "&Expires=1364406757&AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR")), - equal_paths("/buckets/testbucket/objects/testobject?AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR" + equal_paths("/buckets/testbucket/objects/testobject/versions/null?AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR" "&Expires=1364406757&Signature=x%2B0vteNN1YillZNw4yDGVQWrT2s%3D", rewrite_with(Mod, - headers([{"host", "testbucket." ++ ?ROOT_HOST}]), + headers([{"host", "testbucket." ++ ?S3_ROOT_HOST}]), "/testobject?Signature=x%2B0vteNN1YillZNw4yDGVQWrT2s%3D" "&Expires=1364406757&AWSAccessKeyId=BF_BI8XYKFJSIW-NNAIR")), %% Urlencoded path-style bucketname. - equal_paths("/buckets/testbucket/objects/testobject", + equal_paths("/buckets/testbucket/objects/testobject/versions/null", rewrite_with(Mod, headers([]), "/%74estbucket/testobject")), - equal_paths("/buckets/testbucket/objects/path%2Ftestobject", + equal_paths("/buckets/testbucket/objects/path%2Ftestobject/versions/null", rewrite_with(Mod, headers([]), "/%74estbucket/path/testobject")). strict_rewrite_path(Mod) -> - equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject%252Bplus", - rewrite_with(Mod, - headers([]), - "/testbucket/testdir/testobject%2Bplus")), - equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject%2Bplus", - rewrite_with(Mod, - headers([]), - "/testbucket/testdir/testobject+plus")), - equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject%25%2500", - rewrite_with(Mod, - headers([]), - "/testbucket/testdir/testobject%%00")). - -%% This should be buggy behaviour but also we have to preserve old behaviour -legacy_rewrite_path(Mod) -> - equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject%2Bplus", + equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject%252Bplus/versions/null", rewrite_with(Mod, headers([]), "/testbucket/testdir/testobject%2Bplus")), - equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject+plus", + equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject%2Bplus/versions/null", rewrite_with(Mod, headers([]), "/testbucket/testdir/testobject+plus")), - equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject%2Bplus", - rewrite_with(Mod, - headers([]), - "/testbucket/testdir/testobject%2Bplus")), - equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject%25%00", + equal_paths("/buckets/testbucket/objects/testdir%2Ftestobject%25%2500/versions/null", rewrite_with(Mod, headers([]), "/testbucket/testdir/testobject%%00")). diff --git a/test/riak_cs_storage_mr_test.erl b/apps/riak_cs/test/riak_cs_storage_mr_test.erl similarity index 98% rename from test/riak_cs_storage_mr_test.erl rename to apps/riak_cs/test/riak_cs_storage_mr_test.erl index 045b90fef..660b09e38 100644 --- a/test/riak_cs_storage_mr_test.erl +++ b/apps/riak_cs/test/riak_cs_storage_mr_test.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -23,11 +24,10 @@ -module(riak_cs_storage_mr_test). -compile(export_all). +-compile(nowarn_export_all). -include("riak_cs.hrl"). --ifdef(TEST). - -include_lib("eunit/include/eunit.hrl"). -define(M, (m())?MANIFEST). @@ -135,7 +135,7 @@ bucket_summary_map_test_() -> ]. m() -> - ?MANIFEST{uuid=druuid:v4(), + ?MANIFEST{uuid=uuid:get_v4(), content_length=0, write_start_time={1200, 0, 0}, block_size=riak_cs_lfs_utils:block_size()}. @@ -202,5 +202,3 @@ count_multipart_parts_test_() -> [{<<"pocketburgers">>, ?MANIFEST{props=[{multipart, ValidMPManifest}], state=writing}}])) ]. - --endif. diff --git a/test/riak_object.erl b/apps/riak_cs/test/riak_object.erl similarity index 94% rename from test/riak_object.erl rename to apps/riak_cs/test/riak_object.erl index fe9060cf3..437a0b451 100644 --- a/test/riak_object.erl +++ b/apps/riak_cs/test/riak_object.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved, +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/test/riak_socket_dummy.erl b/apps/riak_cs/test/riak_socket_dummy.erl similarity index 95% rename from test/riak_socket_dummy.erl rename to apps/riak_cs/test/riak_socket_dummy.erl index f948c4e42..1bfdce8d9 100644 --- a/test/riak_socket_dummy.erl +++ b/apps/riak_cs/test/riak_socket_dummy.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file diff --git a/test/riakc_pb_socket_fake.erl b/apps/riak_cs/test/riakc_pb_socket_fake.erl similarity index 98% rename from test/riakc_pb_socket_fake.erl rename to apps/riak_cs/test/riakc_pb_socket_fake.erl index 951cd403e..7c30e308a 100644 --- a/test/riakc_pb_socket_fake.erl +++ b/apps/riak_cs/test/riakc_pb_socket_fake.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved, +%% 2021 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -161,5 +162,3 @@ code_change(_OldVsn, State, _Extra) -> %%%=================================================================== %%% Internal functions %%%=================================================================== - - diff --git a/apps/riak_cs_multibag/src/riak_cs_multibag.app.src b/apps/riak_cs_multibag/src/riak_cs_multibag.app.src new file mode 100644 index 000000000..b4bb81c65 --- /dev/null +++ b/apps/riak_cs_multibag/src/riak_cs_multibag.app.src @@ -0,0 +1,17 @@ +%%-*- mode: erlang -*- +%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. + +{application, riak_cs_multibag, + [ + {description, "riak_cs_multibag"}, + {vsn, "3.2.5"}, + {modules, []}, + {registered, []}, + {applications, + [kernel, + stdlib, + inets, + crypto + ]}, + {env, []} + ]}. diff --git a/apps/riak_cs_multibag/src/riak_cs_multibag.erl b/apps/riak_cs_multibag/src/riak_cs_multibag.erl new file mode 100644 index 000000000..abe5d3ac3 --- /dev/null +++ b/apps/riak_cs_multibag/src/riak_cs_multibag.erl @@ -0,0 +1,127 @@ +%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. +%% @doc Support multi Riak clusters in single Riak CS system + +-module(riak_cs_multibag). + +-export([process_specs/0, choose_bag_id/2]). +-export([bags/0, cluster_id/2]). +-export([tab_name/0, tab_info/0]). +-export([bag_id_from_bucket/1]). + +-export_type([weight_info/0]). + +-define(ETS_TAB, ?MODULE). + +-record(bag, {bag_id :: bag_id(), + pool_name :: atom(), + address :: string(), + port :: non_neg_integer(), + cluster_id :: binary() | undefined}). + +-include("riak_cs_multibag.hrl"). +-include_lib("riak_pb/include/riak_pb_kv_codec.hrl"). + +-type weight_info() :: #weight_info{}. + +process_specs() -> + maybe_init(), + BagServer = #{id => riak_cs_multibag_server, + start => {riak_cs_multibag_server, start_link, []}}, + %% Pass connection open/close information not to "explicitly" depends on riak_cs + %% and to make unit test easier. + %% TODO: Pass these MF's by argument process_specs + WeightUpdaterArgs = [{conn_open_mf, {riak_cs_utils, riak_connection}}, + {conn_close_mf, {riak_cs_utils, close_riak_connection}}], + WeightUpdater = #{id => riak_cs_multibag_weight_updater, + start => {riak_cs_multibag_weight_updater, start_link, [WeightUpdaterArgs]}}, + [BagServer, WeightUpdater]. + +%% Choose bag ID for new bucket or new manifest +-spec choose_bag_id(manifet | block, term()) -> bag_id(). +choose_bag_id(AllocType, Seed) -> + {ok, BagId} = riak_cs_multibag_server:choose_bag(AllocType, Seed), + BagId. + +-spec bags() -> [{bag_id(), Address::string(), Port::non_neg_integer()}]. +bags() -> + maybe_init(), + [{BagId, Address, Port} || + #bag{bag_id=BagId, pool_name=_Name, address=Address, port=Port} <- + ets:tab2list(?ETS_TAB)]. + +-spec cluster_id(fun(), bag_id()) -> undefined | binary(). +cluster_id(GetClusterIdFun, undefined) -> + cluster_id(GetClusterIdFun, <<"master">>); +cluster_id(GetClusterIdFun, BagId) -> + case ets:lookup(?ETS_TAB, BagId) of + [#bag{cluster_id=ClusterId}] when is_binary(ClusterId) -> + ClusterId; + [Bag] -> + ClusterId = GetClusterIdFun(BagId), + true = ets:insert(?ETS_TAB, Bag#bag{cluster_id=ClusterId}), + ClusterId + end. + +%% Extract bag ID from `riakc_obj' in moss.buckets +-spec bag_id_from_bucket(riakc_obj:riakc_obj()) -> bag_id(). +bag_id_from_bucket(BucketObj) -> + Contents = riakc_obj:get_contents(BucketObj), + bag_id_from_contents(Contents). + +maybe_init() -> + case catch ets:info(?ETS_TAB) of + undefined -> init(); + _ -> ok + end. + +init() -> + _Tid = init_ets(), + {ok, Bags} = application:get_env(riak_cs_multibag, bags), + {MasterAddress, MasterPort} = riak_cs_config:riak_host_port(), + ok = store_pool_record({"master", MasterAddress, MasterPort}), + ok = store_pool_records(Bags), + ok. + +init_ets() -> + ets:new(?ETS_TAB, [{keypos, #bag.bag_id}, + named_table, public, {read_concurrency, true}]). + +store_pool_records([]) -> + ok; +store_pool_records([Bag | RestBags]) -> + ok = store_pool_record(Bag), + store_pool_records(RestBags). + +store_pool_record({BagIdStr, Address, Port}) -> + BagId = list_to_binary(BagIdStr), + Name = riak_cs_riak_client:pbc_pool_name(BagId), + true = ets:insert_new(?ETS_TAB, #bag{bag_id = BagId, + pool_name = Name, + address = Address, + port = Port}), + ok. + +bag_id_from_contents([]) -> + undefined; +bag_id_from_contents([{MD, _} | Contents]) -> + case bag_id_from_meta(dict:fetch(?MD_USERMETA, MD)) of + undefined -> + bag_id_from_contents(Contents); + BagId -> + BagId + end. + +bag_id_from_meta([]) -> + undefined; +bag_id_from_meta([{?MD_BAG, Value} | _]) -> + binary_to_term(Value); +bag_id_from_meta([_MD | MDs]) -> + bag_id_from_meta(MDs). + +%% For Debugging + +tab_name() -> + ?ETS_TAB. + +tab_info() -> + ets:tab2list(?ETS_TAB). diff --git a/apps/riak_cs_multibag/src/riak_cs_multibag.hrl b/apps/riak_cs_multibag/src/riak_cs_multibag.hrl new file mode 100644 index 000000000..aaff94865 --- /dev/null +++ b/apps/riak_cs_multibag/src/riak_cs_multibag.hrl @@ -0,0 +1,15 @@ +%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. + +%% Riak's bucket and key to store weight information +-define(WEIGHT_BUCKET, <<"riak-cs-multibag">>). +-define(WEIGHT_KEY, <<"weight">>). + +-record(weight_info, { + bag_id :: bag_id(), + weight :: non_neg_integer(), + opts = [] :: proplists:proplist() %% Not used + }). + +-define(MD_BAG, <<"X-Rcs-Bag">>). + +-type bag_id() :: undefined | binary(). diff --git a/apps/riak_cs_multibag/src/riak_cs_multibag_console.erl b/apps/riak_cs_multibag/src/riak_cs_multibag_console.erl new file mode 100644 index 000000000..c9f6280e2 --- /dev/null +++ b/apps/riak_cs_multibag/src/riak_cs_multibag_console.erl @@ -0,0 +1,148 @@ +%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. + +%% @doc These functions are used by the riak-cs-supercluster command line script. + +-module(riak_cs_multibag_console). + +-export(['list-members'/1, weight/1, 'weight-manifest'/1, 'weight-block'/1]). +-export([show_weights/1, show_weights_for_bag/2, refresh/1]). + +-include("riak_cs_multibag.hrl"). + +-define(SAFELY(Code, Description), + try + Code + catch + Type:Reason:ST -> + io:format("~s failed:~n ~p:~p\n ~p\n", + [Description, Type, Reason, ST]), + error + end). + +-define(SCRIPT_NAME, "riak-cs-multibag"). + +%%%=================================================================== +%%% Public API +%%%=================================================================== + +'list-members'(_Args) -> + ?SAFELY(list_bags(), "List all bags"). + +weight([]) -> + ?SAFELY(with_status(fun show_weights/1), "List all weights"); +weight([BagId]) -> + ?SAFELY(with_status(fun(Status) -> show_weights_for_bag(BagId, Status) end), + "List weights for the bag"); +weight([BagId, Weight]) -> + ?SAFELY(set_weight(BagId, Weight), "Set weight for the bag"); +weight(_) -> + io:format("Invalid arguments"), + error. + +'weight-manifest'(Args) -> + weight_by_type(manifest, Args). + +'weight-block'(Args) -> + weight_by_type(block, Args). + +refresh(_Opts) -> + ?SAFELY(handle_result(riak_cs_multibag_weight_updater:refresh()), + "Refresh weight information"). + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +weight_by_type(Type, []) -> + ?SAFELY(with_status(fun(Status) -> show_weights(Type, Status) end), + io_lib:format("List all ~s weights", [Type])); +weight_by_type(Type, [BagId]) -> + ?SAFELY(with_status(fun(Status) -> show_weights_for_bag(Type, BagId, Status) end), + io_lib:format("List ~s weights for the bag", [Type])); +weight_by_type(Type, [BagId, Weight]) -> + ?SAFELY(set_weight(Type, BagId, Weight), + io_lib:format("Set ~s weight for the bag", [Type])); +weight_by_type(_Type, _) -> + io:format("Invalid arguments"), + error. + +list_bags() -> + [print_bag(BagId, Address, Port) || + {BagId, Address, Port} <- riak_cs_multibag:bags()]. + +print_bag(BagId, Address, Port) -> + io:format("~s ~s:~B~n", [BagId, Address, Port]). + +show_weights(Status) -> + show_weights(manifest, Status), + show_weights(block, Status). + +show_weights(Type, Status) -> + WeightInfoList = proplists:get_value(Type, Status), + _ = [io:format("~s (~s): ~B~n", [BagId, Type, Weight]) || + #weight_info{bag_id=BagId, weight=Weight} <- WeightInfoList], + ok. + +show_weights_for_bag(BagId, Status) -> + show_weights_for_bag(manifest, BagId, Status), + show_weights_for_bag(block, BagId, Status). + +show_weights_for_bag(Type, InputBagIdStr, Status) -> + InputBagId = list_to_binary(InputBagIdStr), + WeightInfoList = proplists:get_value(Type, Status), + _ = [io:format("~s (~s): ~B~n", [BagId, Type, Weight]) || + #weight_info{bag_id=BagId, weight=Weight} <- WeightInfoList, + BagId =:= InputBagId], + ok. + +set_weight(BagIdStr, WeightStr) -> + BagId = list_to_binary(BagIdStr), + Weight = list_to_integer(WeightStr), + case lists:member(BagId, all_bag_ids()) of + false -> + io:format("Error: invalid bag ID~n"), + error; + _ -> + riak_cs_multibag_weight_updater:set_weight( + #weight_info{bag_id=BagId, weight=Weight}) + end. + +set_weight(Type, BagIdStr, WeightStr) -> + BagId = list_to_binary(BagIdStr), + Weight = list_to_integer(WeightStr), + case lists:member(BagId, all_bag_ids()) of + false -> + io:format("Error: invalid bag ID~n"), + error; + _ -> + riak_cs_multibag_weight_updater:set_weight_by_type( + Type, #weight_info{bag_id=BagId, weight=Weight}) + end. + +with_status(Fun) -> + case riak_cs_multibag_server:status() of + {error, Reason} -> + io:format("Error: ~p~n", [Reason]), + error; + {ok, Status} -> + case proplists:get_value(initialized, Status) of + false -> + io:format("Error: not initialized.~n"), + error; + _ -> + Fun(Status) + end + end. + +all_bag_ids() -> + [BagId || + {BagId, _Address, _Port} <- riak_cs_multibag:bags()]. + +handle_result(ok) -> + ok; +handle_result({ok, Result}) -> + io:format("~p~n", [Result]), + ok; +handle_result({error, Reason}) -> + io:format("Error: ~p~n", [Reason]), + ok. diff --git a/apps/riak_cs_multibag/src/riak_cs_multibag_riak_client.erl b/apps/riak_cs_multibag/src/riak_cs_multibag_riak_client.erl new file mode 100644 index 000000000..53c69f35f --- /dev/null +++ b/apps/riak_cs_multibag/src/riak_cs_multibag_riak_client.erl @@ -0,0 +1,245 @@ +%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. + +-module(riak_cs_multibag_riak_client). + +-behaviour(gen_server). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3, format_status/2]). + +-include_lib("riak_cs/include/riak_cs.hrl"). +-include_lib("riak_pb/include/riak_pb_kv_codec.hrl"). + +-define(SERVER, ?MODULE). +-define(DEFAULT_BAG, undefined). + +%% Bag ID `undefined' represents objects were stored in single bag +%% configuration, and we use the default bag for the objects. +%% (currently, "default bag" means "master bag".) +%% To avoid confusion between `undefined's in record attributes +%% and ones for "default bag", set fresh values to `uninitialized' +%% in this record. +-record(state, { + master_pbc = uninitialized :: uninitialized | pid(), + manifest_pbc = uninitialized :: uninitialized | pid(), + block_pbc = uninitialized :: uninitialized | pid(), + + manifest_bag = uninitialized :: uninitialized | ?DEFAULT_BAG | binary(), + block_bag = uninitialized :: uninitialized | ?DEFAULT_BAG | binary(), + + bucket_name = uninitialized :: uninitialized | binary(), + bucket_obj = uninitialized :: uninitialized | term(), % riakc_obj:riakc_obj() + manifest = uninitialized :: uninitialized | {binary(), term()} % UUID and lfs_manifest() + }). + +init([]) -> + {ok, fresh_state()}. + +handle_call(stop, _From, State) -> + _ = do_cleanup(State), + {stop, normal, ok, State}; +handle_call(cleanup, _From, State) -> + {reply, ok, do_cleanup(State)}; + +handle_call({get_bucket, BucketName}, _From, + #state{bucket_name=uninitialized} = State) -> + case do_get_bucket(State#state{bucket_name=BucketName}) of + {ok, #state{bucket_obj=BucketObj} = NewState} -> + {reply, {ok, BucketObj}, NewState}; + {error, Reason, NewState} -> + {reply, {error, Reason}, NewState} + end; +handle_call({get_bucket, BucketName}, _From, + #state{bucket_name=BucketName, bucket_obj=BucketObj} = State) -> + {reply, {ok, BucketObj}, State}; +handle_call({get_bucket, RequestedBucketName}, _From, + #state{bucket_name=BucketName} = State) -> + {reply, {error, {bucket_name_already_set, RequestedBucketName, BucketName}}, State}; + +handle_call({set_bucket_name, BucketName}, _From, + #state{bucket_name = uninitialized} = State) -> + case do_get_bucket(State#state{bucket_name=BucketName}) of + {ok, NewState} -> + {reply, ok, NewState}; + {error, Reason, NewState} -> + {reply, {error, Reason}, NewState} + end; +handle_call({set_bucket_name, BucketName}, _From, + #state{bucket_name = BucketName} = State) -> + {reply, ok, State}; +handle_call({set_bucket_name, RequestedBucketName}, _From, + #state{bucket_name = BucketName} = State) -> + {reply, {error, {bucket_name_already_set, RequestedBucketName, BucketName}}, State}; + +handle_call(master_pbc, _From, State) -> + case ensure_master_pbc(State) of + {ok, #state{master_pbc=MasterPbc} = NewState} -> + {reply, {ok, MasterPbc}, NewState}; + {error, Reason} -> + {reply, {error, Reason}, State} + end; +handle_call(manifest_pbc, _From, State) -> + case ensure_manifest_pbc(State) of + {ok, #state{manifest_pbc=ManifestPbc} = NewState} -> + {reply, {ok, ManifestPbc}, NewState}; + {error, Reason} -> + {reply, {error, Reason}, State} + end; + +handle_call({set_manifest_bag, ManifestBagId}, _From, + #state{manifest_bag=uninitialized} = State) + when ManifestBagId =:= ?DEFAULT_BAG orelse is_binary(ManifestBagId) -> + case ensure_manifest_pbc(State#state{manifest_bag=ManifestBagId}) of + {ok, NewState} -> + {reply, ok, NewState}; + {error, Reason} -> + {reply, {error, Reason}, State} + end; +handle_call({set_manifest_bag, ManifestBagId}, _From, + #state{manifest_bag=ManifestBagId} = State) -> + {reply, ok, State}; +handle_call({set_manifest_bag, RequestedBagId}, _From, + #state{manifest_bag=ManifestBagId} = State) -> + {reply, {error, {manifest_bag_already_set, RequestedBagId, ManifestBagId}}, State}; +handle_call(get_manifest_bag, _From, #state{manifest_bag=ManifestBagId} = State) -> + {reply, {ok, ManifestBagId}, State}; + +handle_call({set_manifest, {UUID, Manifest}}, _From, + #state{manifest=uninitialized} = State) -> + case ensure_block_pbc(State#state{manifest={UUID, Manifest}}) of + {ok, NewState} -> + {reply, ok, NewState}; + {error, Reason} -> + {reply, {error, Reason}, State} + end; +handle_call({set_manifest, {UUID, _ReqestedManifest}}, _From, + #state{manifest={UUID, _Manifest}} = State) -> + {reply, ok, State}; +handle_call({set_manifest, RequestedManifest}, _From, + #state{manifest=Manifest} = State) -> + {reply, {error, {manifest_already_set, RequestedManifest, Manifest}}, State}; + +handle_call(block_pbc, _From, State) -> + case ensure_block_pbc(State) of + {ok, #state{block_pbc=BlockPbc} = NewState} -> + {reply, {ok, BlockPbc}, NewState}; + {error, Reason} -> + {reply, {error, Reason}, State} + end; + +handle_call(Request, _From, State) -> + Reply = {error, {invalid_request, Request}}, + {reply, Reply, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +format_status(_Opt, [_PDict, Status]) -> + format_status(Status). + +format_status(Status) -> + Fields = record_info(fields, state), + [_Name | Values] = tuple_to_list(Status), + lists:zip(Fields, Values). + +%%% Internal functions + +fresh_state() -> + #state{}. + +do_cleanup(State) -> + stop_pbcs([{State#state.master_pbc, <<"master">>}, + {State#state.manifest_pbc, State#state.manifest_bag}, + {State#state.block_pbc, State#state.block_bag}]), + fresh_state(). + +stop_pbcs([]) -> + ok; +stop_pbcs([{uninitialized, _BagId} | Rest]) -> + stop_pbcs(Rest); +stop_pbcs([{Pbc, BagId} | Rest]) when is_pid(Pbc) -> + riak_cs_utils:close_riak_connection(pool_name(BagId), Pbc), + stop_pbcs(Rest). + +do_get_bucket(State) -> + case ensure_master_pbc(State) of + {ok, #state{master_pbc = MasterPbc, + bucket_name = BucketName} = NewState} -> + case riak_cs_riak_client:get_bucket_with_pbc(MasterPbc, BucketName) of + {ok, Obj} -> + {ok, NewState#state{bucket_obj = Obj}}; + {error, Reason} -> + {error, Reason, NewState} + end; + {error, Reason} -> + {error, Reason, State} + end. + +ensure_master_pbc(#state{master_pbc = MasterPbc} = State) + when is_pid(MasterPbc) -> + {ok, State}; +ensure_master_pbc(#state{} = State) -> + case riak_cs_utils:riak_connection(pool_name(master)) of + {ok, MasterPbc} -> {ok, State#state{master_pbc=MasterPbc}}; + {error, Reason} -> {error, Reason} + end. + +ensure_manifest_pbc(#state{manifest_pbc = ManifestPbc} = State) + when is_pid(ManifestPbc) -> + {ok, State}; +ensure_manifest_pbc(#state{manifest_bag = ?DEFAULT_BAG} = State) -> + case ensure_master_pbc(State) of + {ok, #state{master_pbc=MasterPbc} = NewState} -> + {ok, NewState#state{manifest_pbc=MasterPbc}}; + {error, Reason} -> + {error, Reason} + end; +ensure_manifest_pbc(#state{manifest_bag = BagId} = State) + when is_binary(BagId) -> + case riak_cs_utils:riak_connection(pool_name(BagId)) of + {ok, Pbc} -> + {ok, State#state{manifest_pbc = Pbc}}; + {error, Reason} -> + {error, Reason} + end; +ensure_manifest_pbc(#state{bucket_obj = BucketObj} = State) + when BucketObj =/= uninitialized -> + ManifestBagId = riak_cs_multibag:bag_id_from_bucket(BucketObj), + ensure_manifest_pbc(State#state{manifest_bag = ManifestBagId}). + +ensure_block_pbc(#state{block_pbc = BlockPbc} = State) + when is_pid(BlockPbc) -> + {ok, State}; +ensure_block_pbc(#state{block_bag = ?DEFAULT_BAG} = State) -> + case ensure_master_pbc(State) of + {ok, #state{master_pbc=MasterPbc} = NewState} -> + {ok, NewState#state{block_pbc=MasterPbc}}; + {error, Reason} -> + {error, Reason} + end; +ensure_block_pbc(#state{block_bag = BagId} = State) + when is_binary(BagId) -> + case riak_cs_utils:riak_connection(pool_name(BagId)) of + {ok, Pbc} -> + {ok, State#state{block_pbc = Pbc}}; + {error, Reason} -> + {error, Reason} + end; +ensure_block_pbc(#state{manifest={_UUID, Manifest}} = State) -> + BlockBagId = riak_cs_mb_helper:bag_id_from_manifest(Manifest), + ensure_block_pbc(State#state{block_bag=BlockBagId}). + +pool_name(?DEFAULT_BAG) -> + riak_cs_riak_client:pbc_pool_name(master); +pool_name(BagId) -> + riak_cs_riak_client:pbc_pool_name(BagId). diff --git a/apps/riak_cs_multibag/src/riak_cs_multibag_server.erl b/apps/riak_cs_multibag/src/riak_cs_multibag_server.erl new file mode 100644 index 000000000..e26892af5 --- /dev/null +++ b/apps/riak_cs_multibag/src/riak_cs_multibag_server.erl @@ -0,0 +1,162 @@ +%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. + +%% @doc Keep weight information and choose bag ID before allocating +%% for each new bucket or manifest. + +%% The argument of choose_bag_by_weight/1, `Type' is one of +%% - `manifest' for a new bucket +%% - `block' for a new manifest + +-module(riak_cs_multibag_server). + +-behavior(gen_server). + +-export([start_link/0]). +-export([choose_bag/2, status/0, new_weights/1]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-include("riak_cs_multibag.hrl"). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +-define(SERVER, ?MODULE). + +-record(state, { + initialized = false :: boolean(), + block = [] :: [riak_cs_multibag:weight_info()], + manifest = [] :: [riak_cs_multibag:weight_info()] + }). + +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +-spec choose_bag(manifest | block, term()) -> {ok, bag_id()} | {error, term()}. +choose_bag(Type, Seed) -> + gen_server:call(?SERVER, {choose_bag, Type, term_to_binary(Seed)}). + +new_weights(Weights) -> + gen_server:cast(?SERVER, {new_weights, Weights}). + +status() -> + gen_server:call(?SERVER, status). + +init([]) -> + %% Recieve weights as soon as possible after restart + riak_cs_multibag_weight_updater:maybe_refresh(), + {ok, #state{}}. + +handle_call({choose_bag, Type, Seed}, _From, #state{initialized = true} = State) + when Type =:= manifest orelse Type =:= block -> + Choice = case Type of + block -> choose_bag_by_weight(State#state.block, Seed); + manifest -> choose_bag_by_weight(State#state.manifest, Seed) + end, + case Choice of + {ok, BagId} -> {reply, {ok, BagId}, State}; + {error, no_bag} -> {reply, {error, no_bag}, State} + end; +handle_call({choose_bag, _Type, _Seed}, _From, #state{initialized = false} = State) -> + {reply, {error, not_initialized}, State}; +handle_call(status, _From, #state{initialized=Initialized, + block=BlockWeights, manifest=ManifestWeights} = State) -> + {reply, {ok, [{initialized, Initialized}, + {block, BlockWeights}, {manifest, ManifestWeights}]}, State}; +handle_call(Request, _From, State) -> + {reply, {error, {unknown_request, Request}}, State}. + +handle_cast({new_weights, Weights}, State) -> + NewState = update_weight_state(Weights, State), + %% TODO: write log only when weights are updated. + %% logger:info("new_weights: ~p~n", [NewState]), + {noreply, NewState}; +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%% Choose a bag, to which block/manifest will be stored randomly, regarding weights +%% bag weight cummulative-weight point (1..60) +%% bag1 20 20 1..20 +%% bag2 10 30 21..30 +%% bag3 0 30 N/A +%% bag4 30 60 31..60 +%% TODO: Make this function deterministic +-spec choose_bag_by_weight([riak_cs_multibag:weight_info()], binary()) -> + {ok, bag_id()} | {error, no_bag}. +choose_bag_by_weight([], _Seed) -> + {error, no_bag}; +choose_bag_by_weight(WeightInfoList, Seed) -> + %% TODO: SumOfWeights can be stored in state + SumOfWeights = lists:sum([Weight || #weight_info{weight = Weight} <- WeightInfoList]), + case SumOfWeights of + 0 -> + %% Zero is special for transition from single bag, see README + {ok, undefined}; + _ -> + <> = riak_cs_utils:sha(Seed), + Point = SHA rem SumOfWeights + 1, + choose_bag_by_weight1(Point, WeightInfoList) + end. + +%% Always "1 =< Point" holds, bag_id with weight=0 never selected. +choose_bag_by_weight1(Point, [#weight_info{bag_id = BagId, weight = Weight} | _WeightInfoList]) + when Point =< Weight -> + {ok, BagId}; +choose_bag_by_weight1(Point, [#weight_info{weight = Weight} | WeightInfoList]) -> + choose_bag_by_weight1(Point - Weight, WeightInfoList). + +update_weight_state([], State) -> + State#state{initialized = true}; +update_weight_state([{Type, WeightsForType} | Rest], State) -> + Sorted = lists:sort(WeightsForType), + NewState = case Type of + block -> + State#state{block = Sorted}; + manifest -> + State#state{manifest = Sorted} + end, + update_weight_state(Rest, NewState). + +%% =================================================================== +%% EUnit tests +%% =================================================================== +-ifdef(TEST). + +choose_bag_by_weight1_test() -> + %% Better to convert to quickcheck? + WeightInfoList = dummy_weights(), + ListOfPointAndBagId = [ + %% <<"bag-Z*">> are never selected + { 1, <<"bag-A">>}, + { 10, <<"bag-A">>}, + { 30, <<"bag-A">>}, + { 31, <<"bag-B">>}, + {100, <<"bag-B">>}, + {101, <<"bag-C">>}, + {110, <<"bag-C">>}, + {120, <<"bag-C">>}], + [?assertEqual({ok, BagId}, choose_bag_by_weight1(Point, WeightInfoList)) || + {Point, BagId} <- ListOfPointAndBagId]. + +dummy_weights() -> + [ + #weight_info{bag_id = <<"bag-Z1">>, weight= 0}, + #weight_info{bag_id = <<"bag-Z2">>, weight= 0}, + #weight_info{bag_id = <<"bag-A">>, weight=30}, + #weight_info{bag_id = <<"bag-B">>, weight=70}, + #weight_info{bag_id = <<"bag-Z3">>, weight= 0}, + #weight_info{bag_id = <<"bag-C">>, weight=20}, + #weight_info{bag_id = <<"bag-Z4">>, weight= 0} + ]. + +-endif. diff --git a/apps/riak_cs_multibag/src/riak_cs_multibag_weight_updater.erl b/apps/riak_cs_multibag/src/riak_cs_multibag_weight_updater.erl new file mode 100644 index 000000000..f2289e3d4 --- /dev/null +++ b/apps/riak_cs_multibag/src/riak_cs_multibag_weight_updater.erl @@ -0,0 +1,225 @@ +%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. + +%% @doc The worker process to update weight information +%% +%% Triggered periodically by timer or manually by commands. +%% Because GET/PUT operation for Riak may block, e.g. network +%% failure, updating and refreshing is done by this separate +%% process rather than doing by `riak_cs_multibag_server'. + +-module(riak_cs_multibag_weight_updater). + +-behavior(gen_server). + +-export([start_link/1]). +-export([status/0, set_weight/1, set_weight_by_type/2, refresh/0, weights/0]). +-export([maybe_refresh/0, refresh_interval/0, set_refresh_interval/1]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-include("riak_cs_multibag.hrl"). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +-record(state, { + timer_ref :: reference() | undefined, + %% Consecutive refresh failures + failed_count = 0 :: non_neg_integer(), + weights :: [{manifest | block, + [riak_cs_multibag:weight_info()]}], + conn_open_fun :: fun(), + conn_close_fun :: fun() + }). + +-define(SERVER, ?MODULE). +-define(REFRESH_INTERVAL, 900). % 900 sec = 15 min + +start_link(Args) -> + gen_server:start_link({local, ?SERVER}, ?MODULE, Args, []). + +status() -> + gen_server:call(?SERVER, status). + +refresh() -> + gen_server:call(?SERVER, refresh). + +maybe_refresh() -> + case whereis(?SERVER) of + undefined -> ok; + _ -> refresh() + end. + +set_weight(WeightInfo) -> + gen_server:call(?SERVER, {set_weight, WeightInfo}). + +set_weight_by_type(Type, WeightInfo) -> + gen_server:call(?SERVER, {set_weight_by_type, Type, WeightInfo}). + +weights() -> + gen_server:call(?SERVER, weights). + +refresh_interval() -> + case application:get_env(riak_cs_multibag, weight_refresh_interval) of + undefined -> ?REFRESH_INTERVAL; + {ok, Value} -> Value + end. + +set_refresh_interval(Interval) when is_integer(Interval) andalso Interval > 0 -> + application:set_env(riak_cs_multibag, weight_refresh_interval, Interval). + +init(Args) -> + {conn_open_mf, {OpenM, OpenF}} = lists:keyfind(conn_open_mf, 1, Args), + {conn_close_mf, {CloseM, CloseF}} = lists:keyfind(conn_close_mf, 1, Args), + {ok, #state{weights=[{block, []}, {manifest, []}], + conn_open_fun=fun OpenM:OpenF/0, conn_close_fun=fun CloseM:CloseF/1}, + 0}. + +handle_call(status, _From, #state{failed_count=FailedCount} = State) -> + {reply, {ok, [{interval, refresh_interval()}, {failed_count, FailedCount}]}, State}; +handle_call(weights, _From, #state{weights = Weights} = State) -> + {reply, {ok, Weights}, State}; +handle_call({set_weight, WeightInfo}, _From, State) -> + case set_weight(WeightInfo, State) of + {ok, NewWeights} -> + {reply, {ok, NewWeights}, State#state{weights=NewWeights}}; + {error, Reason} -> + {reply, {error, Reason}, State} + end; +handle_call({set_weight_by_type, Type, WeightInfo}, _From, State) -> + case set_weight(Type, WeightInfo, State) of + {ok, NewWeights} -> + {reply, {ok, NewWeights}, State#state{weights=NewWeights}}; + {error, Reason} -> + {reply, {error, Reason}, State} + end; +handle_call(refresh, _From, State) -> + case fetch_weights(State) of + {ok, _WeightInfoList, NewState} -> + {reply, ok, NewState}; + {error, Reason, NewState} -> + {reply, {error, Reason}, NewState} + end; +handle_call(Request, _From, State) -> + {reply, {error, {unknown_request, Request}}, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(Event, State) when Event =:= refresh_by_timer orelse Event =:= timeout -> + case refresh_by_timer(State) of + {ok, _WeightInfoList, NewState} -> + {noreply, NewState}; + {error, Reason, NewState} -> + logger:error("Refresh of cluster weight information failed. Reason: ~p", [Reason]), + {noreply, NewState} + end; +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +refresh_by_timer(State) -> + case fetch_weights(State) of + {ok, WeightInfoList, State1} -> + State2 = schedule(State1), + {ok, WeightInfoList, State2}; + {error, Reason, State1} -> + State2 = schedule(State1), + {error, Reason, State2} + end. + +%% Connect to master cluster and GET weight information +fetch_weights(#state{conn_open_fun=OpenFun, conn_close_fun=CloseFun} = State) -> + case OpenFun() of + {ok, Riakc} -> + Result = riakc_pb_socket:get(Riakc, ?WEIGHT_BUCKET, ?WEIGHT_KEY), + CloseFun(Riakc), + handle_weight_info_list(Result, State); + {error, _Reason} = E -> + handle_weight_info_list(E, State) + end. + +handle_weight_info_list({error, notfound}, State) -> + logger:debug("Bag weight information is not found"), + {ok, [], State#state{failed_count = 0}}; +handle_weight_info_list({error, Reason}, #state{failed_count = Count} = State) -> + logger:error("Retrieval of bag weight information failed. Reason: ~p", [Reason]), + {error, Reason, State#state{failed_count = Count + 1}}; +handle_weight_info_list({ok, Obj}, State) -> + %% TODO: How to handle siblings + [Value | _] = riakc_obj:get_values(Obj), + Weights = binary_to_term(Value), + riak_cs_multibag_server:new_weights(Weights), + {ok, Weights, State#state{failed_count = 0, weights = Weights}}. + +schedule(State) -> + IntervalMSec = refresh_interval() * 1000, + Ref = erlang:send_after(IntervalMSec, self(), refresh_by_timer), + State#state{timer_ref = Ref}. + +set_weight(WeightInfo, #state{weights = Weights} = State) -> + NewWeights = [{Type, update_or_add_weight(WeightInfo, WeighInfoList)} || + {Type, WeighInfoList} <- Weights], + update_weight_info(NewWeights, State). + +set_weight(Type, WeightInfo, #state{weights = Weights} = State) -> + NewWeights = + case lists:keytake(Type, 1, Weights) of + false -> + [{Type, [WeightInfo]} | Weights]; + {value, {Type, WeighInfoList}, OtherWeights} -> + [{Type, update_or_add_weight(WeightInfo, WeighInfoList)} | + OtherWeights] + end, + update_weight_info(NewWeights, State). + +update_or_add_weight(#weight_info{bag_id=BagId}=WeightInfo, + WeighInfoList) -> + OtherBags = lists:keydelete(BagId, #weight_info.bag_id, WeighInfoList), + [WeightInfo | OtherBags]. + +%% Connect to Riak cluster and overwrite weights at {riak-cs-bag, weight} +update_weight_info(WeightInfoList, #state{conn_open_fun=OpenFun, conn_close_fun=CloseFun}) -> + case OpenFun() of + {ok, Riakc} -> + try + update_weight_info1(Riakc, WeightInfoList) + after + CloseFun(Riakc) + end; + {error, _Reason} = E -> + E + end. + +update_weight_info1(Riakc, WeightInfoList) -> + Current = case riakc_pb_socket:get(Riakc, ?WEIGHT_BUCKET, ?WEIGHT_KEY) of + {error, notfound} -> + {ok, riakc_obj:new(?WEIGHT_BUCKET, ?WEIGHT_KEY)}; + {error, Reason} -> + {error, Reason}; + {ok, Obj} -> + {ok, Obj} + end, + put_weight_info(Riakc, WeightInfoList, Current). + +put_weight_info(_Riakc, _WeightInfoList, {error, Reason}) -> + logger:error("Retrieval of bag weight information failed. Reason: ~p", [Reason]), + {error, Reason}; +put_weight_info(Riakc, WeightInfoList, {ok, Obj}) -> + NewObj = riakc_obj:update_value( + riakc_obj:update_metadata(Obj, dict:new()), + term_to_binary(WeightInfoList)), + case riakc_pb_socket:put(Riakc, NewObj) of + ok -> + riak_cs_multibag_server:new_weights(WeightInfoList), + {ok, WeightInfoList}; + {error, Reason} -> + logger:error("Update of bag weight information failed. Reason: ~p", [Reason]), + {error, Reason} + end. diff --git a/bom.xml b/bom.xml new file mode 100644 index 000000000..4390c8346 --- /dev/null +++ b/bom.xml @@ -0,0 +1 @@ +ranch1.8.0Socket acceptor pool for TCP protocols.pkg:hex/ranch@1.8.049fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5cowlib2.12.1Support library for manipulating Web protocols.pkg:hex/cowlib@2.12.1163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0bear1.0.0A set of statistics functions for erlangApache-2.0pkg:hex/bear@1.0.0157b67901adf84ff0da6eae035ca1292a0ac18aa55148154d8c582b2c68959dbsetup2.1.0Generic setup application for Erlang-based systemsApache-2.0pkg:hex/setup@2.1.0efd072578f0cf85bea96caaffc7adb0992398272522660a136e10567377071c5riak_pb3.0.10+tiotRiak Protocol Buffers Messagespkg:github/ti-tokyo/riak_pb@a838173209f89b465de8a0f9248146da0ff0866cquickrand2.0.6Quick Random Number Generationpkg:hex/quickrand@2.0.686a03c7fc96b9c6b9fa6bef5fceb773e359c772ee3814be60b8eb44b54f99140hut1.3.0helper library for making Erlang libraries logging framework agnosticMITpkg:hex/hut@1.3.07e15d28555d8a1f2b5a3a931ec120af0753e4853a4c66053db354f35bf9ab563folsom1.0.0Erlang based metrics systemApache-2.0pkg:hex/folsom@1.0.0dd6ab97278e94f9e4cfc43e188224a7b8c7eaec0dd2e935007005177f3eebb0ecowboy2.10.0Small, fast, modern HTTP server.pkg:hex/cowboy@2.10.03afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6bwebmachine1.11.2webmachineApachepkg:github/ti-tokyo/webmachine@f173a243293054715e02c43f406cf2e8715f9655uuid_erl2.0.6Native UUID Generationpkg:hex/uuid_erl@2.0.61d54d5de4cc66317d882f62347a21655601e84ccce913e80aade202cdb3a8c65riakc3.0.13-tiotRiak Clientpkg:github/ti-tokyo/riak-erlang-client@83c0ffda78a3c02ab7cc019e6d025492584c4acdriak_repl_pb_api3.1.0Protobuffs API for riak MDCpkg:github/ti-tokyo/riak_repl_pb_api@3046e29d6e31246c8de9ebed2ee2790d19e0691apoolboy1.5.2A hunky Erlang worker pool factoryUnlicenseApache-2.0pkg:hex/poolboy@1.5.2dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3parse_trans3.4.1Parse transform libraryApache-2.0pkg:hex/parse_trans@3.4.1620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49amochiweb3.1.2MochiMedia Web ServerMITpkg:hex/mochiweb@3.1.2a8609035711f9264e8abe54312d1b1645374b9e08986ae06e979bb0ac8b99f9fjsx3.1.0a streaming, evented json parsing toolkitMITpkg:hex/jsx@3.1.00c5cc8fdc11b53cc25cf65ac6705ad39e54ecc56d1c22e4adb8f5a53fb9427f3jason_erl1.2.2JSON encode/decode library written in ErlangISCpkg:hex/jason_erl@1.2.23851b6150d231f203852822b77d0f60a069f89add1260e3033755345e0b604d4getopt1.0.2Command-line options parser for ErlangBSDpkg:hex/getopt@1.0.2a0029aea4322fb82a61f6876a6d9c66dc9878b6cb61faa13df3187384fd4ea26exometer_core1.6.2Code instrumentation and metrics collection package.MPL-2.0pkg:hex/exometer_core@1.6.2287f9ada56961933d8df913cc140bb0bdcff2545e33ab94d04b668e09626e297esaml4.5.0SAML Server Provider library for erlangBSDpkg:hex/esaml@4.5.04697e5cdd70c9ea0fd7ff8ff1c4049566bfa9202588b56924b181c3b2976d67dcluster_info2.1.0Cluster info/postmortem apppkg:github/basho/cluster_info@389d43af7ac1550b3c01cd55b8147bcc0e20022f \ No newline at end of file diff --git a/client_tests/README.md b/client_tests/README.md deleted file mode 100644 index eae0c63cb..000000000 --- a/client_tests/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Riak CS Client Library Tests - -Within each of the directories at the same level of this README are tests for -language specific S3 client libraries. Executing these tests should typically -be done from the root of the project using `make`. - -Example: - -```bash -$ make test-boto -``` - -If you are interested in using the tests to understand how to communicate with -Riak CS, each language sub-directory contains its own `README` with details on -how to get the tests running locally. diff --git a/client_tests/clojure/clj-s3/.gitignore b/client_tests/clojure/clj-s3/.gitignore deleted file mode 100644 index b3257aa1a..000000000 --- a/client_tests/clojure/clj-s3/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -/pom.xml -*jar -/lib -/classes -/native -/.lein-failures -/checkouts -/.lein-deps-sum -/target -.lein-repl-history -.nrepl-port diff --git a/client_tests/clojure/clj-s3/README.md b/client_tests/clojure/clj-s3/README.md deleted file mode 100644 index 7ee732d61..000000000 --- a/client_tests/clojure/clj-s3/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# Riak CS clj-s3 Tests - -## Overview - -These tests are written using a copy of -[clj-aws-s3](https://github.com/weavejester/clj-aws-s3) at tag 0.3.10, -for S3 API call. On the other hand, Riak CS specific administrative API -call is implemented in - -```bash -client_tests/clojure/clj-s3/src/java_s3_tests/user_creation.clj -``` - -New tests cases are added to: - -```bash -client_tests/clojure/clj-s3/test/java_s3_tests/test/client.clj -``` - -The tests are written using the Clojure testing library -[midje](https://github.com/marick/Midje), which has -[great documentation](https://github.com/marick/Midje/wiki). - -## Dependencies - -Install [Leiningen](http://leiningen.org/): - -```bash -$ curl -O https://raw.github.com/technomancy/leiningen/stable/bin/lein -$ mv lein ~/bin # Or some other directory in your $PATH -$ chmod 755 ~/bin/lein -``` - -Install dependencies: - -```bash -$ lein deps -``` - -## Configuration - -Ensure that the Riak CS `app.config` has `anonymous_user_creation` set to -`true`. If it was previously set to `false`, make sure that the `riak-cs` -service is restarted: - -```bash -$ riak-cs restart -``` - -## Execution - -```bash -$ lein midje -``` diff --git a/client_tests/clojure/clj-s3/project.clj b/client_tests/clojure/clj-s3/project.clj deleted file mode 100644 index db654dc1e..000000000 --- a/client_tests/clojure/clj-s3/project.clj +++ /dev/null @@ -1,16 +0,0 @@ -(defproject java-s3-tests/java-s3-tests "0.0.2" - :dependencies [[org.clojure/clojure "1.7.0"] - [org.clojure/tools.logging "0.3.1"] - ;; aws-sdk-java deps - [com.amazonaws/aws-java-sdk-s3 "1.10.8" :exclusions [joda-time]] - [joda-time "2.8.1"] - [com.fasterxml.jackson.core/jackson-core "2.5.3"] - [com.fasterxml.jackson.core/jackson-databind "2.5.3"] - ;; user_creation deps - [clj-http "2.0.0"] - [cheshire "5.1.0"]] - ;; :jvm-opts ["-verbose:class"] - :profiles {:dev {:dependencies [[midje "1.7.0"]]}} - :min-lein-version "2.0.0" - :plugins [[lein-midje "3.1.3"]] - :description "integration tests for Riak CS using the offical Java SDK") diff --git a/client_tests/clojure/clj-s3/src/aws/sdk/s3.clj b/client_tests/clojure/clj-s3/src/aws/sdk/s3.clj deleted file mode 100644 index bab0bad99..000000000 --- a/client_tests/clojure/clj-s3/src/aws/sdk/s3.clj +++ /dev/null @@ -1,661 +0,0 @@ -;; Copyright (c) 2014 James Reeves -;; Distributed under the Eclipse Public License, the same as Clojure. -;; -;; Original repository is https://github.com/weavejester/clj-aws-s3. - -(ns aws.sdk.s3 - "Functions to access the Amazon S3 storage service. - - Each function takes a map of credentials as its first argument. The - credentials map should contain an :access-key key and a :secret-key key, - optionally an :endpoint key to denote an AWS endpoint and optionally a :proxy - key to define a HTTP proxy to go through. - - The :proxy key must contain keys for :host and :port, and may contain keys - for :user, :password, :domain and :workstation." - (:require [clojure.string :as str] - [clj-time.core :as t] - [clj-time.coerce :as coerce] - [clojure.walk :as walk]) - (:import com.amazonaws.auth.BasicAWSCredentials - com.amazonaws.auth.BasicSessionCredentials - com.amazonaws.services.s3.AmazonS3Client - com.amazonaws.AmazonServiceException - com.amazonaws.ClientConfiguration - com.amazonaws.HttpMethod - com.amazonaws.services.s3.model.AccessControlList - com.amazonaws.services.s3.model.Bucket - com.amazonaws.services.s3.model.Grant - com.amazonaws.services.s3.model.CanonicalGrantee - com.amazonaws.services.s3.model.CreateBucketRequest - com.amazonaws.services.s3.model.CopyObjectResult - com.amazonaws.services.s3.model.EmailAddressGrantee - com.amazonaws.services.s3.model.GetObjectRequest - com.amazonaws.services.s3.model.GetObjectMetadataRequest - com.amazonaws.services.s3.model.Grant - com.amazonaws.services.s3.model.GroupGrantee - com.amazonaws.services.s3.model.ListObjectsRequest - com.amazonaws.services.s3.model.ListVersionsRequest - com.amazonaws.services.s3.model.Owner - com.amazonaws.services.s3.model.ObjectMetadata - com.amazonaws.services.s3.model.ObjectListing - com.amazonaws.services.s3.model.Permission - com.amazonaws.services.s3.model.PutObjectRequest - com.amazonaws.services.s3.model.S3Object - com.amazonaws.services.s3.model.S3ObjectSummary - com.amazonaws.services.s3.model.S3VersionSummary - com.amazonaws.services.s3.model.VersionListing - com.amazonaws.services.s3.model.InitiateMultipartUploadRequest - com.amazonaws.services.s3.model.AbortMultipartUploadRequest - com.amazonaws.services.s3.model.CompleteMultipartUploadRequest - com.amazonaws.services.s3.model.UploadPartRequest - java.util.concurrent.Executors - java.io.ByteArrayInputStream - java.io.File - java.io.InputStream - java.nio.charset.Charset)) - -(defn- s3-client* - [cred] - (let [client-configuration (ClientConfiguration.)] - (when-let [conn-timeout (:conn-timeout cred)] - (.setConnectionTimeout client-configuration conn-timeout)) - (when-let [socket-timeout (:socket-timeout cred)] - (.setSocketTimeout client-configuration socket-timeout)) - (when-let [max-retries (:max-retries cred)] - (.setMaxErrorRetry client-configuration max-retries)) - (when-let [max-conns (:max-conns cred)] - (.setMaxConnections client-configuration max-conns)) - (when-let [proxy-host (get-in cred [:proxy :host])] - (.setProxyHost client-configuration proxy-host)) - (when-let [proxy-port (get-in cred [:proxy :port])] - (.setProxyPort client-configuration proxy-port)) - (when-let [proxy-user (get-in cred [:proxy :user])] - (.setProxyUsername client-configuration proxy-user)) - (when-let [proxy-pass (get-in cred [:proxy :password])] - (.setProxyPassword client-configuration proxy-pass)) - (when-let [proxy-domain (get-in cred [:proxy :domain])] - (.setProxyDomain client-configuration proxy-domain)) - (when-let [proxy-workstation (get-in cred [:proxy :workstation])] - (.setProxyWorkstation client-configuration proxy-workstation)) - (let [aws-creds - (if (:token cred) - (BasicSessionCredentials. (:access-key cred) (:secret-key cred) (:token cred)) - (BasicAWSCredentials. (:access-key cred) (:secret-key cred))) - - client (AmazonS3Client. aws-creds client-configuration)] - (when-let [endpoint (:endpoint cred)] - (.setEndpoint client endpoint)) - client))) - -(def ^{:private true :tag AmazonS3Client} - s3-client - (memoize s3-client*)) - -(defprotocol ^{:no-doc true} Mappable - "Convert a value into a Clojure map." - (^{:no-doc true} to-map [x] "Return a map of the value.")) - -(extend-protocol Mappable - Bucket - (to-map [bucket] - {:name (.getName bucket) - :creation-date (.getCreationDate bucket) - :owner (to-map (.getOwner bucket))}) - Owner - (to-map [owner] - {:id (.getId owner) - :display-name (.getDisplayName owner)}) - nil - (to-map [_] nil)) - -(defn bucket-exists? - "Returns true if the supplied bucket name already exists in S3." - [cred name] - (.doesBucketExist (s3-client cred) name)) - -(defn- ^CreateBucketRequest ->CreateBucketRequest - "Create a PutBucketRequest instance from a bucket name." - [^String bucket ^String key request] - (CreateBucketRequest. bucket)) - -(declare create-acl) ; used by create-bucket and put-object - -(defn create-bucket - "Create a new S3 bucket with the supplied name." - [cred ^String name & [metadata & permissions]] - (let [req (CreateBucketRequest. name)] - (when permissions - (.setAccessControlList req (create-acl permissions))) - (.createBucket (s3-client cred) req))) - -(defn delete-bucket - "Delete the S3 bucket with the supplied name." - [cred ^String name] - (.deleteBucket (s3-client cred) name)) - -(defn list-buckets - "List all the S3 buckets for the supplied credentials. The buckets will be - returned as a seq of maps with the following keys: - :name - the bucket name - :creation-date - the date when the bucket was created - :owner - the owner of the bucket" - [cred] - (map to-map (.listBuckets (s3-client cred)))) - -(defprotocol ^{:no-doc true} ToPutRequest - "A protocol for constructing a map that represents an S3 put request." - (^{:no-doc true} put-request [x] "Convert a value into a put request.")) - -(extend-protocol ToPutRequest - InputStream - (put-request [is] {:input-stream is}) - File - (put-request [f] {:file f}) - String - (put-request [s] - (let [bytes (.getBytes s)] - {:input-stream (ByteArrayInputStream. bytes) - :content-length (count bytes) - :content-type (str "text/plain; charset=" (.name (Charset/defaultCharset)))}))) - -(defmacro set-attr - "Set an attribute on an object if not nil." - {:private true} - [object setter value] - `(if-let [v# ~value] - (~setter ~object v#))) - -(defn- maybe-int [x] - (if x (int x))) - -(defn- map->ObjectMetadata - "Convert a map of object metadata into a ObjectMetadata instance." - [metadata] - (doto (ObjectMetadata.) - (set-attr .setCacheControl (:cache-control metadata)) - (set-attr .setContentDisposition (:content-disposition metadata)) - (set-attr .setContentEncoding (:content-encoding metadata)) - (set-attr .setContentLength (:content-length metadata)) - (set-attr .setContentMD5 (:content-md5 metadata)) - (set-attr .setContentType (:content-type metadata)) - (set-attr .setServerSideEncryption (:server-side-encryption metadata)) - (set-attr .setUserMetadata - (walk/stringify-keys (dissoc metadata - :cache-control - :content-disposition - :content-encoding - :content-length - :content-md5 - :content-type - :server-side-encryption))))) - -(defn- ^PutObjectRequest ->PutObjectRequest - "Create a PutObjectRequest instance from a bucket name, key and put request - map." - [^String bucket ^String key request] - (cond - (:file request) - (let [put-obj-req (PutObjectRequest. bucket key ^java.io.File (:file request))] - (.setMetadata put-obj-req (map->ObjectMetadata (dissoc request :file))) - put-obj-req) - (:input-stream request) - (PutObjectRequest. - bucket key - (:input-stream request) - (map->ObjectMetadata (dissoc request :input-stream))))) - -(defn put-object - "Put a value into an S3 bucket at the specified key. The value can be - a String, InputStream or File (or anything that implements the ToPutRequest - protocol). - - An optional map of metadata may also be supplied that can include any of the - following keys: - :cache-control - the cache-control header (see RFC 2616) - :content-disposition - how the content should be downloaded by browsers - :content-encoding - the encoding of the content (e.g. gzip) - :content-length - the length of the content in bytes - :content-md5 - the MD5 sum of the content - :content-type - the mime type of the content - :server-side-encryption - set to AES256 if SSE is required - - An optional list of grant functions can be provided after metadata. - These functions will be applied to a clear ACL and the result will be - the ACL for the newly created object." - [cred bucket key value & [metadata & permissions]] - (let [req (->> (merge (put-request value) metadata) - (->PutObjectRequest bucket key))] - (when permissions - (.setAccessControlList req (create-acl permissions))) - (.putObject (s3-client cred) req))) - -(defn- initiate-multipart-upload - [cred bucket key] - (.getUploadId (.initiateMultipartUpload - (s3-client cred) - (InitiateMultipartUploadRequest. bucket key)))) - -(defn- abort-multipart-upload - [{cred :cred bucket :bucket key :key upload-id :upload-id}] - (.abortMultipartUpload - (s3-client cred) - (AbortMultipartUploadRequest. bucket key upload-id))) - -(defn- complete-multipart-upload - [{cred :cred bucket :bucket key :key upload-id :upload-id e-tags :e-tags}] - (.completeMultipartUpload - (s3-client cred) - (CompleteMultipartUploadRequest. bucket key upload-id e-tags))) - -(defn- upload-part - [{cred :cred bucket :bucket key :key upload-id :upload-id - part-size :part-size offset :offset ^java.io.File file :file}] - (.getPartETag - (.uploadPart - (s3-client cred) - (doto (UploadPartRequest.) - (.setBucketName bucket) - (.setKey key) - (.setUploadId upload-id) - (.setPartNumber (+ 1 (/ offset part-size))) - (.setFileOffset offset) - (.setPartSize ^long (min part-size (- (.length file) offset))) - (.setFile file))))) - -(defn put-multipart-object - "Do a multipart upload of a file into a S3 bucket at the specified key. - The value must be a java.io.File object. The entire file is uploaded - or not at all. If an exception happens at any time the upload is aborted - and the exception is rethrown. The size of the parts and the number of - threads uploading the parts can be configured in the last argument as a - map with the following keys: - :part-size - the size in bytes of each part of the file. Must be 5mb - or larger. Defaults to 5mb - :threads - the number of threads that will upload parts concurrently. - Defaults to 16." - [cred bucket key ^java.io.File file & [{:keys [part-size threads] - :or {part-size (* 5 1024 1024) threads 16}}]] - (let [upload-id (initiate-multipart-upload cred bucket key) - upload {:upload-id upload-id :cred cred :bucket bucket :key key :file file} - pool (Executors/newFixedThreadPool threads) - offsets (range 0 (.length file) part-size) - tasks (map #(fn [] (upload-part (assoc upload :offset % :part-size part-size))) - offsets)] - (try - (complete-multipart-upload - (assoc upload :e-tags (map #(.get ^java.util.concurrent.Future %) (.invokeAll pool tasks)))) - (catch Exception ex - (abort-multipart-upload upload) - (.shutdown pool) - (throw ex)) - (finally (.shutdown pool))))) - -(extend-protocol Mappable - S3Object - (to-map [object] - {:content (.getObjectContent object) - :metadata (to-map (.getObjectMetadata object)) - :bucket (.getBucketName object) - :key (.getKey object)}) - ObjectMetadata - (to-map [metadata] - {:cache-control (.getCacheControl metadata) - :content-disposition (.getContentDisposition metadata) - :content-encoding (.getContentEncoding metadata) - :content-length (.getContentLength metadata) - :content-md5 (.getContentMD5 metadata) - :content-type (.getContentType metadata) - :etag (.getETag metadata) - :last-modified (.getLastModified metadata) - :server-side-encryption (.getServerSideEncryption metadata) - :user (walk/keywordize-keys (into {} (.getUserMetadata metadata))) - :version-id (.getVersionId metadata)}) - ObjectListing - (to-map [listing] - {:bucket (.getBucketName listing) - :objects (map to-map (.getObjectSummaries listing)) - :prefix (.getPrefix listing) - :common-prefixes (seq (.getCommonPrefixes listing)) - :truncated? (.isTruncated listing) - :max-keys (.getMaxKeys listing) - :marker (.getMarker listing) - :next-marker (.getNextMarker listing)}) - S3ObjectSummary - (to-map [summary] - {:metadata {:content-length (.getSize summary) - :etag (.getETag summary) - :last-modified (.getLastModified summary)} - :bucket (.getBucketName summary) - :key (.getKey summary)}) - S3VersionSummary - (to-map [summary] - {:metadata {:content-length (.getSize summary) - :etag (.getETag summary) - :last-modified (.getLastModified summary)} - :version-id (.getVersionId summary) - :latest? (.isLatest summary) - :delete-marker? (.isDeleteMarker summary) - :bucket (.getBucketName summary) - :key (.getKey summary)}) - VersionListing - (to-map [listing] - {:bucket (.getBucketName listing) - :versions (map to-map (.getVersionSummaries listing)) - :prefix (.getPrefix listing) - :common-prefixes (seq (.getCommonPrefixes listing)) - :delimiter (.getDelimiter listing) - :truncated? (.isTruncated listing) - :max-results (maybe-int (.getMaxKeys listing)) ; AWS API is inconsistent, should be .getMaxResults - :key-marker (.getKeyMarker listing) - :next-key-marker (.getNextKeyMarker listing) - :next-version-id-marker (.getNextVersionIdMarker listing) - :version-id-marker (.getVersionIdMarker listing)}) - CopyObjectResult - (to-map [result] - {:etag (.getETag result) - :expiration-time (.getExpirationTime result) - :expiration-time-rule-id (.getExpirationTimeRuleId result) - :last-modified-date (.getLastModifiedDate result) - :server-side-encryption (.getServerSideEncryption result)})) - -(defn get-object - "Get an object from an S3 bucket. The object is returned as a map with the - following keys: - :content - an InputStream to the content - :metadata - a map of the object's metadata - :bucket - the name of the bucket - :key - the object's key - Be extremely careful when using this method; the :content value in the returned - map contains a direct stream of data from the HTTP connection. The underlying - HTTP connection cannot be closed until the user finishes reading the data and - closes the stream. - Therefore: - * Use the data from the :content input stream as soon as possible - * Close the :content input stream as soon as possible - If these rules are not followed, the client can run out of resources by - allocating too many open, but unused, HTTP connections." - ([cred ^String bucket ^String key] - (to-map (.getObject (s3-client cred) bucket key))) - ([cred ^String bucket ^String key ^String version-id] - (to-map (.getObject (s3-client cred) (GetObjectRequest. bucket key version-id))))) - -(defn- map->GetObjectMetadataRequest - "Create a ListObjectsRequest instance from a map of values." - [request] - (GetObjectMetadataRequest. (:bucket request) (:key request) (:version-id request))) - -(defn get-object-metadata - "Get an object's metadata from a bucket. A optional map of options may be supplied. - Available options are: - :version-id - the version of the object - The metadata is a map with the - following keys: - :cache-control - the CacheControl HTTP header - :content-disposition - the ContentDisposition HTTP header - :content-encoding - the character encoding of the content - :content-length - the length of the content in bytes - :content-md5 - the MD5 hash of the content - :content-type - the mime-type of the content - :etag - the HTTP ETag header - :last-modified - the last modified date - :server-side-encryption - the server-side encryption algorithm" - [cred bucket key & [options]] - (to-map - (.getObjectMetadata - (s3-client cred) - (map->GetObjectMetadataRequest (merge {:bucket bucket :key key} options))))) - -(defn- map->ListObjectsRequest - "Create a ListObjectsRequest instance from a map of values." - ^ListObjectsRequest - [request] - (doto (ListObjectsRequest.) - (set-attr .setBucketName (:bucket request)) - (set-attr .setDelimiter (:delimiter request)) - (set-attr .setMarker (:marker request)) - (set-attr .setMaxKeys (maybe-int (:max-keys request))) - (set-attr .setPrefix (:prefix request)))) - -(defn- http-method [method] - (-> method name str/upper-case HttpMethod/valueOf)) - -(defn generate-presigned-url - "Return a presigned URL for an S3 object. Accepts the following options: - :expires - the date at which the URL will expire (defaults to 1 day from now) - :http-method - the HTTP method for the URL (defaults to :get)" - [cred bucket key & [options]] - (.toString - (.generatePresignedUrl - (s3-client cred) - bucket - key - (coerce/to-date (:expires options (-> 1 t/days t/from-now))) - (http-method (:http-method options :get))))) - -(defn list-objects - "List the objects in an S3 bucket. A optional map of options may be supplied. - Available options are: - :delimiter - read only keys up to the next delimiter (such as a '/') - :marker - read objects after this key - :max-keys - read only this many objects - :prefix - read only objects with this prefix - - The object listing will be returned as a map containing the following keys: - :bucket - the name of the bucket - :prefix - the supplied prefix (or nil if none supplied) - :objects - a list of objects - :common-prefixes - the common prefixes of keys omitted by the delimiter - :max-keys - the maximum number of objects to be returned - :truncated? - true if the list of objects was truncated - :marker - the marker of the listing - :next-marker - the next marker of the listing" - [cred bucket & [options]] - (to-map - (.listObjects - (s3-client cred) - (map->ListObjectsRequest (merge {:bucket bucket} options))))) - -(defn delete-object - "Delete an object from an S3 bucket." - [cred bucket key] - (.deleteObject (s3-client cred) bucket key)) - -(defn object-exists? - "Returns true if an object exists in the supplied bucket and key." - [cred bucket key] - (try - (get-object-metadata cred bucket key) - true - (catch AmazonServiceException e - (if (= 404 (.getStatusCode e)) - false - (throw e))))) - -(defn copy-object - "Copy an existing S3 object to another key. Returns a map containing - the data returned from S3" - ([cred bucket src-key dest-key] - (copy-object cred bucket src-key bucket dest-key)) - ([cred src-bucket src-key dest-bucket dest-key] - (to-map (.copyObject (s3-client cred) src-bucket src-key dest-bucket dest-key)))) - -(defn- map->ListVersionsRequest - "Create a ListVersionsRequest instance from a map of values." - [request] - (doto (ListVersionsRequest.) - (set-attr .setBucketName (:bucket request)) - (set-attr .setDelimiter (:delimiter request)) - (set-attr .setKeyMarker (:key-marker request)) - (set-attr .setMaxResults (maybe-int (:max-results request))) - (set-attr .setPrefix (:prefix request)) - (set-attr .setVersionIdMarker (:version-id-marker request)))) - -(defn list-versions - "List the versions in an S3 bucket. A optional map of options may be supplied. - Available options are: - :delimiter - the delimiter used in prefix (such as a '/') - :key-marker - read versions from the sorted list of all versions starting - at this marker. - :max-results - read only this many versions - :prefix - read only versions with keys having this prefix - :version-id-marker - read objects after this version id - - The version listing will be returned as a map containing the following versions: - :bucket - the name of the bucket - :prefix - the supplied prefix (or nil if none supplied) - :versions - a sorted list of versions, newest first, each - version has: - :version-id - the unique version id - :latest? - is this the latest version for that key? - :delete-marker? - is this a delete-marker? - :common-prefixes - the common prefixes of keys omitted by the delimiter - :max-results - the maximum number of results to be returned - :truncated? - true if the results were truncated - :key-marker - the key marker of the listing - :next-version-id-marker - the version ID marker to use in the next listVersions - request in order to obtain the next page of results. - :version-id-marker - the version id marker of the listing" - [cred bucket & [options]] - (to-map - (.listVersions - (s3-client cred) - (map->ListVersionsRequest (merge {:bucket bucket} options))))) - -(defn delete-version - "Deletes a specific version of the specified object in the specified bucket." - [cred bucket key version-id] - (.deleteVersion (s3-client cred) bucket key version-id)) - -(defprotocol ^{:no-doc true} ToClojure - "Convert an object into an idiomatic Clojure value." - (^{:no-doc true} to-clojure [x] "Turn the object into a Clojure value.")) - -(extend-protocol ToClojure - CanonicalGrantee - (to-clojure [grantee] - {:id (.getIdentifier grantee) - :display-name (.getDisplayName grantee)}) - EmailAddressGrantee - (to-clojure [grantee] - {:email (.getIdentifier grantee)}) - GroupGrantee - (to-clojure [grantee] - (condp = grantee - GroupGrantee/AllUsers :all-users - GroupGrantee/AuthenticatedUsers :authenticated-users - GroupGrantee/LogDelivery :log-delivery)) - Permission - (to-clojure [permission] - (condp = permission - Permission/FullControl :full-control - Permission/Read :read - Permission/ReadAcp :read-acp - Permission/Write :write - Permission/WriteAcp :write-acp))) - -(extend-protocol Mappable - Grant - (to-map [grant] - {:grantee (to-clojure (.getGrantee grant)) - :permission (to-clojure (.getPermission grant))}) - AccessControlList - (to-map [acl] - {:grants (set (map to-map (.getGrants acl))) - :owner (to-map (.getOwner acl))})) - -(defn get-bucket-acl - "Get the access control list (ACL) for the supplied bucket. The ACL is a map - containing two keys: - :owner - the owner of the ACL - :grants - a set of access permissions granted - - The grants themselves are maps with keys: - :grantee - the individual or group being granted access - :permission - the type of permission (:read, :write, :read-acp, :write-acp or - :full-control)." - [cred ^String bucket] - (to-map (.getBucketAcl (s3-client cred) bucket))) - -(defn get-object-acl - "Get the access control list (ACL) for the supplied object. See get-bucket-acl - for a detailed description of the return value." - [cred bucket key] - (to-map (.getObjectAcl (s3-client cred) bucket key))) - -(defn- permission [perm] - (case perm - :full-control Permission/FullControl - :read Permission/Read - :read-acp Permission/ReadAcp - :write Permission/Write - :write-acp Permission/WriteAcp)) - -(defn- grantee [grantee] - (cond - (keyword? grantee) - (case grantee - :all-users GroupGrantee/AllUsers - :authenticated-users GroupGrantee/AuthenticatedUsers - :log-delivery GroupGrantee/LogDelivery) - (:id grantee) - (CanonicalGrantee. (:id grantee)) - (:email grantee) - (EmailAddressGrantee. (:email grantee)))) - -(defn- clear-acl [^AccessControlList acl] - (doseq [grantee (->> (.getGrants acl) - (map #(.getGrantee ^Grant %)) - (set))] - (.revokeAllPermissions acl grantee))) - -(defn- add-acl-grants [^AccessControlList acl grants] - (doseq [g grants] - (.grantPermission acl - (grantee (:grantee g)) - (permission (:permission g))))) - -(defn- update-acl [^AccessControlList acl funcs] - (let [grants (:grants (to-map acl)) - update (apply comp (reverse funcs))] - (clear-acl acl) - (add-acl-grants acl (update grants)))) - -(defn- create-acl [permissions] - (doto (AccessControlList.) - (update-acl permissions))) - -(defn update-bucket-acl - "Update the access control list (ACL) for the named bucket using functions - that update a set of grants (see get-bucket-acl). - - This function is often used with the grant and revoke functions, e.g. - - (update-bucket-acl cred bucket - (grant :all-users :read) - (grant {:email \"foo@example.com\"} :full-control) - (revoke {:email \"bar@example.com\"} :write))" - [cred ^String bucket & funcs] - (let [acl (.getBucketAcl (s3-client cred) bucket)] - (update-acl acl funcs) - (.setBucketAcl (s3-client cred) bucket acl))) - -(defn update-object-acl - "Updates the access control list (ACL) for the supplied object using functions - that update a set of grants (see update-bucket-acl for more details)." - [cred ^String bucket ^String key & funcs] - (let [acl (.getObjectAcl (s3-client cred) bucket key)] - (update-acl acl funcs) - (.setObjectAcl (s3-client cred) bucket key acl))) - -(defn grant - "Returns a function that adds a new grant map to a set of grants. - See update-bucket-acl." - [grantee permission] - #(conj % {:grantee grantee :permission permission})) - -(defn revoke - "Returns a function that removes a grant map from a set of grants. - See update-bucket-acl." - [grantee permission] - #(disj % {:grantee grantee :permission permission})) diff --git a/client_tests/clojure/clj-s3/src/java_s3_tests/core.clj b/client_tests/clojure/clj-s3/src/java_s3_tests/core.clj deleted file mode 100644 index e9b735e24..000000000 --- a/client_tests/clojure/clj-s3/src/java_s3_tests/core.clj +++ /dev/null @@ -1,17 +0,0 @@ -;; Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -;; -;; This file is provided to you under the Apache License, -;; Version 2.0 (the "License"); you may not use this file -;; except in compliance with the License. You may obtain -;; a copy of the License at -;; -;; http://www.apache.org/licenses/LICENSE-2.0 -;; -;; Unless required by applicable law or agreed to in writing, -;; software distributed under the License is distributed on an -;; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -;; KIND, either express or implied. See the License for the -;; specific language governing permissions and limitations -;; under the License. - -(ns java-s3-tests.core) diff --git a/client_tests/clojure/clj-s3/src/java_s3_tests/user_creation.clj b/client_tests/clojure/clj-s3/src/java_s3_tests/user_creation.clj deleted file mode 100644 index 55f8db1ef..000000000 --- a/client_tests/clojure/clj-s3/src/java_s3_tests/user_creation.clj +++ /dev/null @@ -1,58 +0,0 @@ -;; Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -;; -;; This file is provided to you under the Apache License, -;; Version 2.0 (the "License"); you may not use this file -;; except in compliance with the License. You may obtain -;; a copy of the License at -;; -;; http://www.apache.org/licenses/LICENSE-2.0 -;; -;; Unless required by applicable law or agreed to in writing, -;; software distributed under the License is distributed on an -;; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -;; KIND, either express or implied. See the License for the -;; specific language governing permissions and limitations -;; under the License. - -(ns java-s3-tests.user-creation - (:require [cheshire.core :as cheshire]) - (:require [clj-http.client :as http])) - -(defn ^:internal make-host-port-string [host port] - (str host ":" port)) - -(defn ^:internal make-url-with-resource [host-port-string resource] - (str host-port-string "/" resource)) - -(defn ^:internal make-body [user-name email] - (cheshire/generate-string - {"email" email "name" user-name})) - -(defn ^:internal make-user-url [host port] - (let [location (make-host-port-string host port)] - (make-url-with-resource location "/riak-cs/user"))) - -(defn ^:internal parse-response-body [string] - (cheshire/parse-string string true)) - -(defn ^:internal parse-response [response] - (parse-response-body (:body response))) - -(defn ^:internal string-uuid [] - (str (java.util.UUID/randomUUID))) - -(defn create-user - "create a new user from the /user - resource. Returns a map with keys - :key_id and :key_secret" - [host port user-name email] - (let [url (make-user-url host port) - body (make-body user-name email) - headers {"Content-Type" "application/json"}] - (parse-response (http/post url {:body body :headers headers})))) - -(defn create-random-user [host port] - (let [random-token (string-uuid) - user-name random-token - email (str random-token "@example.com")] - (create-user host port user-name email))) diff --git a/client_tests/clojure/clj-s3/test/java_s3_tests/test/client.clj b/client_tests/clojure/clj-s3/test/java_s3_tests/test/client.clj deleted file mode 100644 index 40f337058..000000000 --- a/client_tests/clojure/clj-s3/test/java_s3_tests/test/client.clj +++ /dev/null @@ -1,249 +0,0 @@ -;; Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -;; -;; This file is provided to you under the Apache License, -;; Version 2.0 (the "License"); you may not use this file -;; except in compliance with the License. You may obtain -;; a copy of the License at -;; -;; http://www.apache.org/licenses/LICENSE-2.0 -;; -;; Unless required by applicable law or agreed to in writing, -;; software distributed under the License is distributed on an -;; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -;; KIND, either express or implied. See the License for the -;; specific language governing permissions and limitations -;; under the License. - -(ns java-s3-tests.test.client - (:import java.security.MessageDigest - org.apache.commons.codec.binary.Hex - com.amazonaws.services.s3.model.AmazonS3Exception) - - (:require [aws.sdk.s3 :as s3]) - (:require [java-s3-tests.user-creation :as user-creation]) - (:require [clojure.tools.logging :as log]) - (:use midje.sweet)) - -(def ^:internal riak-cs-host "127.0.0.1") -(def ^:internal riak-cs-host-with-protocol (str "http://" riak-cs-host)) - -(defn get-riak-cs-port-str - "Try to get a TCP port number from the OS environment" - [] - (let [port-str (get (System/getenv) "CS_HTTP_PORT")] - (cond (nil? port-str) "8080" - :else port-str))) - -(defn get-riak-cs-port [] - (Integer/parseInt (get-riak-cs-port-str) 10)) - -(defn md5-byte-array [input-byte-array] - (let [instance (MessageDigest/getInstance "MD5")] - (.digest instance input-byte-array))) - -(defn md5-string - [input-byte-array] - (let [b (md5-byte-array input-byte-array)] - (String. (Hex/encodeHex b)))) - -(defn user-to-cred - [user] - {:endpoint "http://s3.amazonaws.com" - :access-key (:key_id user) - :secret-key (:key_secret user) - :proxy {:host riak-cs-host - :port (get-riak-cs-port)}}) - -(defn random-user - [] - (user-creation/create-random-user riak-cs-host-with-protocol (get-riak-cs-port))) - -(defmacro with-random-user - "Execute `body` in implicit do with a random-user bound to `var-name`" - [var-name & body] - `(let [~var-name (random-user)] - (do ~@body))) - -(defn random-cred - [] - (user-to-cred (random-user))) - -(defmacro with-random-cred - "Execute `body` in implicit do with a random-cred bound to `var-name`" - [var-name & body] - `(let [~var-name (random-cred)] - (do ~@body))) - -(defn random-string [] - (str (java.util.UUID/randomUUID))) - -(defn write-file [filename content] - (with-open [w (clojure.java.io/writer filename :append false)] - (.write w content))) - -(defn etag-suffix [etag] - (subs etag (- (count etag) 2))) - -(defn upload-file [cred bucket key file-name part-size] - (let [f (clojure.java.io/file file-name)] - (s3/put-multipart-object cred bucket key f {:part-size part-size :threads 2}) - (.delete f))) - -(fact "bogus creds raises an exception" - (let [bogus-user {:key_id "foo" - :key_secret "bar"}] - (s3/list-buckets (user-to-cred bogus-user))) - => (throws AmazonS3Exception)) - -(fact "new users have no buckets" - (with-random-cred c - (s3/list-buckets c)) - => []) - -(let [bucket-name (random-string)] - (fact "creating a bucket should list one bucket in list buckets" - (with-random-cred c - (s3/create-bucket c bucket-name) - (map :name (s3/list-buckets c)) - => [bucket-name]))) - -(let [bucket-name (random-string) - object-name (random-string)] - (fact "simple put works" - (with-random-cred cred - (s3/create-bucket cred bucket-name) - (s3/put-object cred bucket-name object-name - "contents")) - => truthy)) - -(let [bucket-name (random-string) - object-name (random-string) - value "this is the value!"] - (fact "the value received during GET is the same - as the object that was PUT" - (with-random-cred c - (s3/create-bucket c bucket-name) - (s3/put-object c bucket-name object-name value) - ((comp slurp :content) (s3/get-object c bucket-name object-name))) - => value)) - -(let [bucket-name (random-string) - object-name (random-string) - value "this is the value!" - as-bytes (.getBytes value "UTF-8") - md5-sum (md5-string as-bytes)] - (fact "check that the etag of the response is the same as the md5 - of the original object" - (with-random-cred c - (s3/create-bucket c bucket-name) - (s3/put-object c bucket-name object-name value) - ((comp :etag :metadata) - (s3/get-object - c bucket-name object-name))) - => md5-sum)) - -(let [bucket-name (random-string) - object-name (random-string) - value "aaaaaaaaaabbbbbbbbbb" - file-name "./clj-mp-test.txt"] - (fact "multipart upload works" - (with-random-cred c - (s3/create-bucket c bucket-name) - (write-file file-name value) - (upload-file c bucket-name object-name file-name 10) - (let [fetched-object (s3/get-object - c bucket-name object-name)] - ((comp slurp :content) fetched-object) - => value - ((comp etag-suffix :etag :metadata) fetched-object) - => "-2")))) - -(let [bucket-name (random-string) - object-name (random-string) - value "this is the real value" - wrong-md5 "2945d7de2f70de5b8c0cb3fbcba4fe92"] - (fact "Bad content md5 throws an exception" - (with-random-cred c - (s3/create-bucket c bucket-name) - (s3/put-object c bucket-name object-name value - {:content-md5 wrong-md5})) - => (throws AmazonS3Exception))) - -(def bad-canonical-id - "0f80b2d002a3d018faaa4a956ce8aa243332a30e878f5dc94f82749984ebb30b") - -(let [bucket-name (random-string) - object-name (random-string) - value-string "this is the real value"] - (fact "Nonexistent canonical-id grant header returns HTTP 400 on - a put object request (not just an ACL subresource request)" - (with-random-cred c - (s3/create-bucket c bucket-name) - (s3/put-object c bucket-name object-name value-string {} - (s3/grant {:id bad-canonical-id} :full-control))) - => (throws AmazonS3Exception))) - -(let [bucket-name (random-string) - object-name (random-string) - value-string "this is the real value" - public-read-grant {:grantee :all-users, :permission :read}] - (fact "Creating an object with an ACL returns the same ACL when you read - the ACL" - (with-random-cred c - (s3/create-bucket c bucket-name) - (s3/put-object c bucket-name object-name value-string {} - (s3/grant :all-users :read)) - (:grants (s3/get-object-acl c bucket-name object-name))) - => (contains (just public-read-grant)))) - -(let [bucket-name (random-string) - object-name (random-string) - value-string "this is the real value" - public-read-grant {:grantee :all-users, :permission :read}] - (fact "Creating an object with an ACL returns the same ACL when you read - the ACL" - (with-random-cred c - (s3/create-bucket c bucket-name) - (s3/put-object c bucket-name object-name value-string {} - (s3/grant :all-users :read)) - (:grants (s3/get-object-acl c bucket-name object-name))) - => (contains (just public-read-grant)))) - -(let [bucket-name (random-string) - object-name (random-string) - value-string "this is the real value"] - (with-random-cred c - (with-random-user u2 - (fact "Creating an object with an (non-canned) ACL returns the same ACL - when you read the ACL" - (s3/create-bucket c bucket-name) - (s3/put-object c bucket-name object-name value-string {} - (s3/grant {:id (:id u2)} :read)) - (:grants (s3/get-object-acl c bucket-name object-name)) - => (contains (just {:grantee {:id (:id u2), - :display-name (:display_name u2)}, - :permission :read})))))) - -(let [bucket-name (random-string) - object-name (random-string)] - (fact "Creating a bucket with an ACL returns the same ACL when you read - the ACL" - (with-random-cred c - (do - (s3/create-bucket c bucket-name {} - (s3/grant :all-users :read)) - (:grants (s3/get-bucket-acl c bucket-name)))) - => (contains (just {:grantee :all-users, :permission :read})))) - -(let [bucket-name (random-string) - object-name (random-string)] - (with-random-cred c - (with-random-user u2 - (fact "Creating a bucket with an (non-canned) ACL returns the same ACL - when you read the ACL" - (s3/create-bucket c bucket-name {} - (s3/grant {:id (:id u2)} :write)) - (:grants (s3/get-bucket-acl c bucket-name)) - => (contains (just {:grantee {:id (:id u2), - :display-name (:display_name u2)}, - :permission :write})))))) diff --git a/client_tests/erlang/erlcloud_eqc.erl b/client_tests/erlang/erlcloud_eqc.erl deleted file mode 100644 index 1e75bfc5f..000000000 --- a/client_tests/erlang/erlcloud_eqc.erl +++ /dev/null @@ -1,402 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc Quickcheck test module for `erlcloud' S3 API interaction with Riak CS. - --module(erlcloud_eqc). - --include("riak_cs.hrl"). - --include_lib("erlcloud/include/erlcloud_aws.hrl"). - --ifdef(EQC). --include_lib("eqc/include/eqc.hrl"). --include_lib("eqc/include/eqc_fsm.hrl"). --include_lib("eunit/include/eunit.hrl"). - --compile(export_all). - -%% eqc properties --export([prop_api_test/4 - %% prop_parallel_api_test/4 - ]). - -%% States --export([start/1, - user_created/1, - bucket_created/1, - bucket_deleted/1, - object_created/1, - object_deleted/1]). - -%% eqc_fsm callbacks --export([initial_state/0, - initial_state_data/0, - initial_state_data/4, - next_state_data/5, - precondition/4, - postcondition/5]). - -%% Helpers --export([test/0, - test/2, - test/5, - create_user/3, - user_name/0, - user_email/0]). - --define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> - io:format(user, Str, Args) end, P)). --define(TEST_ITERATIONS, 100). - --define(P(EXPR), PPP = (EXPR), case PPP of true -> ok; _ -> io:format(user, "PPP ~p at line ~p\n", [PPP, ?LINE]) end, PPP). - --define(S3_MODULE, erlcloud_s3). --define(DEFAULT_HOST, "s3.amazonaws.com"). --define(DEFAULT_PORT, 80). --define(DEFAULT_PROXY_HOST, "localhost"). --define(DEFAULT_PROXY_PORT, 8080). --define(VALUE, "test value"). - --record(api_state, {aws_config :: aws_config(), - bucket :: string(), - keys :: [string()]}). - -%%==================================================================== -%% Eunit tests -%%==================================================================== - -eqc_test_() -> - {spawn, - [ - {timeout, 600, ?_assertEqual(true, - quickcheck(numtests(?TEST_ITERATIONS, - ?QC_OUT(prop_api_test(?DEFAULT_HOST, - ?DEFAULT_PORT, - ?DEFAULT_PROXY_HOST, - cs_port())))))} - %% {timeout, 60, ?_assertEqual(true, quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_parallel_api_test(?DEFAULT_HOST, ?DEFAULT_PORT)))))} - ] - }. - -%% ==================================================================== -%% EQC Properties -%% ==================================================================== - -prop_api_test(Host, Port, ProxyHost, ProxyPort) -> - ?FORALL(Cmds, - eqc_gen:noshrink(commands(?MODULE, {start, initial_state_data(Host, Port, ProxyHost, ProxyPort)})), - begin - RunningApps = application:which_applications(), - case lists:keymember(erlcloud, 1, RunningApps) of - true -> - ok; - false -> - erlcloud:start(), - timer:sleep(200), - ok - end, - {H, {_F, _S}, Res} = run_commands(?MODULE, Cmds), - %% application:stop(erlcloud), - - aggregate(zip(state_names(H), command_names(Cmds)), - ?WHENFAIL( - begin - ?debugFmt("Cmds: ~p~n", - [zip(state_names(H), - command_names(Cmds))]), - ?debugFmt("Result: ~p~n", [Res]), - ?debugFmt("History: ~p~n", [H]) - end, - equals(ok, Res))) - end - ). - -%% prop_parallel_api_test(Host, Port, ProxyHost, ProxyPort) -> -%% ?FORALL(Cmds={Seq, Par}, -%% parallel_commands(?MODULE, {start, initial_state_data(Host, Port)}), -%% begin -%% erlcloud:start(), -%% {H, _ParH, Res} = run_parallel_commands(?MODULE, {Seq, Par}), -%% %% aggregate(zip(state_names(H), command_names(Cmds)), -%% aggregate(command_names(Cmds), -%% ?WHENFAIL( -%% begin -%% ?debugFmt("Cmds: ~p~n", -%% [zip(state_names(H), -%% command_names(Cmds))]), -%% ?debugFmt("Result: ~p~n", [Res]), -%% ?debugFmt("History: ~p~n", [H]) -%% end, -%% equals(ok, Res))) -%% end -%% ). - -%%==================================================================== -%% eqc_fsm callbacks -%%==================================================================== - -start(#api_state{aws_config=Config}) -> - [ - {user_created, {call, ?MODULE, create_user, [user_name(), user_email(), Config]}} - ]. - -user_created(#api_state{aws_config=Config}) -> - [ - {history, {call, ?S3_MODULE, list_buckets, [Config]}}, - {bucket_created, {call, ?S3_MODULE, create_bucket, [bucket(), Config]}} - ]. - -bucket_created(#api_state{aws_config=Config, - bucket=Bucket}) -> - [ - {history, {call, ?S3_MODULE, list_buckets, [Config]}}, - {history, {call, ?S3_MODULE, list_objects, [Bucket, Config]}}, - {object_created, {call, ?S3_MODULE, put_object, [Bucket, key(), ?VALUE, Config]}}, - {bucket_deleted, {call, ?S3_MODULE, delete_bucket, [Bucket, Config]}} - ]. - -bucket_deleted(#api_state{aws_config=Config}) -> - [ - {history, {call, ?S3_MODULE, list_buckets, [Config]}}, - {bucket_created, {call, ?S3_MODULE, create_bucket, [bucket(), Config]}} - ]. - -object_created(#api_state{aws_config=Config, - bucket=Bucket, - keys=[Key | _]}) -> - [ - {history, {call, ?MODULE, get_object, [Bucket, Key, Config]}}, - {history, {call, ?S3_MODULE, list_objects, [Bucket, Config]}}, - {object_deleted, {call, ?S3_MODULE, delete_object, [Bucket, Key, Config]}} - ]. - -object_deleted(#api_state{aws_config=Config, - bucket=Bucket, - keys=[Key | _]}) -> - [ - {history, {call, ?S3_MODULE, list_objects, [Bucket, Config]}}, - {bucket_created, {call, ?MODULE, get_object, [Bucket, Key, Config]}} - ]. - -initial_state() -> - start. - -initial_state_data() -> - #api_state{}. - -initial_state_data(Host, Port, ProxyHost, ProxyPort) -> - #api_state{aws_config=#aws_config{s3_host=Host, - s3_port=Port, - s3_prot="http", - http_options=[{proxy_host, ProxyHost}, - {proxy_port, ProxyPort}]}}. - -next_state_data(start, user_created, S, AwsConfig, _C) -> - S#api_state{aws_config=AwsConfig}; -next_state_data(user_created, bucket_created, S, _R, {call, ?S3_MODULE, create_bucket, [Bucket, _]}) -> - S#api_state{bucket=Bucket}; -next_state_data(bucket_created, bucket_deleted, S, _R, {call, ?S3_MODULE, delete_bucket, [_, _]}) -> - S#api_state{bucket=undefined}; -next_state_data(bucket_deleted, bucket_created, S, _R, {call, ?S3_MODULE, create_bucket, [Bucket, _]}) -> - S#api_state{bucket=Bucket}; -next_state_data(object_deleted, bucket_created, S, _R, {call, ?MODULE, get_object, [_, _, _]}) -> - [_ | RestKeys] = S#api_state.keys, - S#api_state{keys=RestKeys}; -next_state_data(bucket_created, object_created, S, _R, {call, ?S3_MODULE, put_object, [_, Key, _, _]}) -> - Keys = update_keys(Key, S#api_state.keys), - S#api_state{keys=Keys}; -next_state_data(object_created, object_deleted, S, _R, {call, ?S3_MODULE, delete_object, [_, _, _]}) -> - S; -next_state_data(_From, _To, S, _R, _C) -> - S. - -precondition(bucket_created, bucket_deleted, #api_state{keys=undefined}, _C) -> - true; -precondition(bucket_created, bucket_deleted, #api_state{keys=[]}, _C) -> - true; -precondition(bucket_created, bucket_deleted, _S, _C) -> - false; -precondition(_From, _To, _S, _C) -> - true. - -postcondition(start, user_created, _S, _C, {error, _}) -> - ?P(false); -postcondition(start, user_created, _S, _C, Config) -> - ?P(is_record(Config, aws_config)); -postcondition(user_created, user_created, _S, _C, [{buckets, []}]) -> - true; -postcondition(user_created, bucket_created, _S, _C, ok) -> - true; -postcondition(user_created, bucket_created, _S, _C, _) -> - ?P(false); -postcondition(bucket_created, bucket_created, #api_state{bucket=Bucket}, {call, _, list_buckets, _}, [{buckets, [Bucket]}]) -> - true; -postcondition(bucket_created, bucket_created, #api_state{bucket=Bucket}, {call, _, list_objects, _}, R) -> - ?P(is_empty_object_list(Bucket, R)); -postcondition(bucket_created, bucket_deleted, _S, _C, ok) -> - true; -postcondition(bucket_created, bucket_deleted, _S, _C, _) -> - ?P(false); -postcondition(bucket_created, object_created, _S, _C, [{version_id, _}]) -> - true; -postcondition(bucket_created, object_created, _S, _C, _) -> - ?P(false); -postcondition(object_created, object_created, _S, {call, _, get_object, _}, R) -> - ContentLength = proplists:get_value(content_length, R), - Content = proplists:get_value(content, R), - ContentLength =:= "10" andalso Content =:= <<"test value">>; -postcondition(object_created, object_created, #api_state{bucket=Bucket, keys=Keys}, {call, _, list_objects, _}, R) -> - Name = proplists:get_value(name, R), - Contents = proplists:get_value(contents, R), - ?P(Name =:= Bucket andalso verify_object_list_contents(Keys, Contents)); -postcondition(object_created, object_deleted, _S, _C, [{delete_marker, _}, {version_id, _}]) -> - true; -postcondition(object_deleted, object_deleted, #api_state{keys=[Key | _]}, _C, R) -> - Contents = proplists:get_value(contents, R, []), - ?P(not lists:member(Key, Contents)); -postcondition(object_created, object_deleted, _S, _C, _) -> - ?P(false); -%% Catch all -postcondition(_From, _To, _S, _C, _R) -> - true. - -%%==================================================================== -%% Helpers -%%==================================================================== - -test() -> - test(?DEFAULT_HOST, ?DEFAULT_PORT). - -test(Host, Port) -> - test(Host, Port, ?DEFAULT_PROXY_HOST, cs_port(), 500). - -test(Host, Port, ProxyHost, ProxyPort, Iterations) -> - eqc:quickcheck(eqc:numtests(Iterations, prop_api_test(Host, Port, ProxyHost, ProxyPort))). - %% eqc:quickcheck(eqc:numtests(Iterations, prop_parallel_api_test(Host, Port, ProxyHost, ProxyPort))). - -create_user(Name, Email, Config) -> - process_post( - post_user_request( - compose_url(Config), - compose_request(Name, Email)), - Config). - -get_object(Bucket, Key, Config) -> - try - ?S3_MODULE:get_object(Bucket, Key, Config) - catch _:_ -> - {error, not_found} - end. - -post_user_request(Url, RequestDoc) -> - Request = {Url, [], "application/json", RequestDoc}, - httpc:request(post, Request, [], []). - -process_post({ok, {{_, 201, _}, _RespHeaders, RespBody}}, Config) -> - User = json_to_user_record(mochijson2:decode(RespBody)), - Config#aws_config{access_key_id=User?RCS_USER.key_id, - secret_access_key=User?RCS_USER.key_secret}; -process_post(Error, _) -> - Error. - -json_to_user_record({struct, UserItems}) -> - lists:foldl(fun item_to_record_field/2, ?RCS_USER{}, UserItems); -json_to_user_record(_) -> - {error, received_invalid_json}. - -item_to_record_field({<<"email">>, Email}, User) -> - User?RCS_USER{email=binary_to_list(Email)}; -item_to_record_field({<<"name">>, Name}, User) -> - User?RCS_USER{name=binary_to_list(Name)}; -item_to_record_field({<<"display_name">>, DispName}, User) -> - User?RCS_USER{display_name=binary_to_list(DispName)}; -item_to_record_field({<<"id">>, Id}, User) -> - User?RCS_USER{canonical_id=binary_to_list(Id)}; -item_to_record_field({<<"key_id">>, KeyId}, User) -> - User?RCS_USER{key_id=binary_to_list(KeyId)}; -item_to_record_field({<<"key_secret">>, KeySecret}, User) -> - User?RCS_USER{key_secret=binary_to_list(KeySecret)}; -item_to_record_field(_, User) -> - User. - -compose_url(#aws_config{http_options=HTTPOptions}) -> - {_, Host} = lists:keyfind(proxy_host, 1, HTTPOptions), - {_, Port} = lists:keyfind(proxy_port, 1, HTTPOptions), - compose_url(Host, Port). - -compose_url(Host, Port) -> - lists:flatten(["http://", Host, ":", integer_to_list(Port), "/riak-cs/user"]). - -compose_request(Name, Email) -> - binary_to_list( - iolist_to_binary( - mochijson2:encode({struct, [{<<"email">>, list_to_binary(Email)}, - {<<"name">>, list_to_binary(Name)}]}))). - -user_name() -> - ?LET(X, timestamp(), X). - -user_email() -> - ?LET(X, timestamp(), lists:flatten([X, $@, "me.com"])). - -bucket() -> - ?LET(X, timestamp(), X). - -key() -> - ?LET(X, timestamp(), X). - -%% @doc Generator for strings that need to be unique within the VM -timestamp() -> - {MegaSecs, Secs, MicroSecs} = erlang:now(), - to_list(MegaSecs) ++ to_list(Secs) ++ to_list(MicroSecs). - -to_list(X) when X < 10 -> - lists:flatten([$0, X]); -to_list(X) -> - integer_to_list(X). - -update_keys(Key, undefined) -> - [Key]; -update_keys(Key, ExistingKeys) -> - [Key | ExistingKeys]. - -is_empty_object_list(Bucket, ObjectList) -> - Bucket =:= proplists:get_value(name, ObjectList) - andalso [] =:= proplists:get_value(contents, ObjectList). - -verify_object_list_contents([], []) -> - true; -verify_object_list_contents(_, []) -> - false; -verify_object_list_contents(ExpectedKeys, [HeadContent | RestContents]) -> - Key = proplists:get_value(key, HeadContent), - verify_object_list_contents(lists:delete(Key, ExpectedKeys), RestContents). - -cs_port() -> - case os:getenv("CS_HTTP_PORT") of - false -> - ?DEFAULT_PROXY_PORT; - Str -> - list_to_integer(Str) - end. - --endif. diff --git a/client_tests/go/.gitignore b/client_tests/go/.gitignore deleted file mode 100644 index d4f1ec763..000000000 --- a/client_tests/go/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -env/ -tmp/ diff --git a/client_tests/go/Makefile b/client_tests/go/Makefile deleted file mode 100644 index 542006784..000000000 --- a/client_tests/go/Makefile +++ /dev/null @@ -1,46 +0,0 @@ -.PHONY: all clean gof3r test-mp-put-get - -CS_HTTP_PORT ?= 8080 -HTTP_PROXY ?= 127.0.0.1:$(CS_HTTP_PORT) -CS_BUCKET ?= test -CS_KEY ?= $(shell date '+%s') - -TMPDIR := $(PWD)/tmp -RANDFILE := $(TMPDIR)/rand-50MB -GETFILE := $(TMPDIR)/rand-get -GOOPTS := --no-ssl - -export GOPATH := $(PWD)/env -export PATH := $(GOPATH)/bin:$(PATH) -export HTTP_PROXY - -all: test-mp-put-get - -#### TESTS - -## Test multipart upload completes without error -## See: https://github.com/basho/riak_cs/issues/855 -test-mp-put-get: gof3r $(RANDFILE) - gof3r put $(GOOPTS) -s 5242880 -p $(RANDFILE) -b $(CS_BUCKET) -k $(CS_KEY) - rm -f $(GETFILE) - gof3r get $(GOOPTS) -b $(CS_BUCKET) -k $(CS_KEY) - > $(GETFILE) - diff $(RANDFILE) $(GETFILE) - rm -f $(GETFILE) - -### Environment setup - -env: - mkdir $(GOPATH) - -gof3r: $(GOPATH)/bin/gof3r - -$(GOPATH)/bin/gof3r: env - go get github.com/rlmcpherson/s3gof3r/gof3r - -$(RANDFILE): - mkdir -p $(TMPDIR) - dd if=/dev/urandom of=$(RANDFILE) bs=1M count=50 - -clean: - rm -rf $(TMPDIR) - rm -rf $(GOPATH) diff --git a/client_tests/go/README.md b/client_tests/go/README.md deleted file mode 100644 index 80f876692..000000000 --- a/client_tests/go/README.md +++ /dev/null @@ -1,46 +0,0 @@ -# Riak CS Go Tests - -## Dependencies - -- Go Lang runtime installation, see http://golang.org/doc/ . - -## Configuration - -Before execution, settings below should be done for Riak CS: - -- Create an user and export its key and secret as environment - variables. -- Create an bucket owned by the user, export its name as an an - environment variable. -- Export proxy information to connect Riak CS. - -Sample bash script to export these settings: - -``` -export AWS_ACCESS_KEY_ID=8P4GB-NTTTWKDBP6AVLF -export AWS_SECRET_ACCESS_KEY=1xJYjxqtVzogYmo697ZzNOVp8r0dMvWbnPVfiQ== -export CS_HTTP_PORT=15018 -export CS_BUCKET=test-gof3r -export CS_KEY=complete-multipart-upload -``` - -`CS_HTTP_PORT` can be ommitted if `HTTP_PROXY` is set -(e.g. `HTTP_PROXY=127.0.0.1:15018`). -`CS_BUCKET` and `CS_KEY` are optional, the default values are `test` -and an epoch seconds each. - -## Execution - -Installation of s3gof3r and gof3r command line is automatically done -by `Makefile`. So all you should do is just `make`. - -``` -cd client_tests/go -make -``` - -This executes one test target `test-mp-put-get`. - -- First creates input file of 50MB at `./tmp`, -- then executes multipart upload with 5MB part size, -- finally gets the object and compares it with the input file. diff --git a/client_tests/php/.gitignore b/client_tests/php/.gitignore deleted file mode 100644 index 8d1c47b37..000000000 --- a/client_tests/php/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -phpunit -vendor diff --git a/client_tests/php/Makefile b/client_tests/php/Makefile deleted file mode 100644 index 78c28bb5b..000000000 --- a/client_tests/php/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -.PHONY: test all clean -.DEFAULT: all - -all: test - -phpunit: - @composer install --dev - -vendor composer.lock: composer.json - @composer install --dev - -test: phpunit - @./phpunit - -clean: - @rm -rf composer.lock vendor phpunit diff --git a/client_tests/php/README.md b/client_tests/php/README.md deleted file mode 100644 index 22e28c5dc..000000000 --- a/client_tests/php/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# Riak CS AWS PHP SDK Tests - -## Dependencies - -* PHP54 (5.4.x) - - on OSX with homebrew: - - ```bash - brew tap homebrew/dupes - brew tap josegonzalez/homebrew-php - brew install php54 - ``` - -* [Composer](https://getcomposer.org/): - - ```bash - $ curl -s https://getcomposer.org/installer | php && mv -v composer.phar /usr/local/bin/composer - ``` - -## Execution - -```bash -$ make -``` diff --git a/client_tests/php/composer.json b/client_tests/php/composer.json deleted file mode 100644 index 9f23adb53..000000000 --- a/client_tests/php/composer.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "config": { - "bin-dir": "." - }, - "require": { - "aws/aws-sdk-php": "2.3.*" - }, - "require-dev": { - "phpunit/phpunit": "3.7.*" - } -} diff --git a/client_tests/php/composer.lock b/client_tests/php/composer.lock deleted file mode 100644 index b880513d6..000000000 --- a/client_tests/php/composer.lock +++ /dev/null @@ -1,651 +0,0 @@ -{ - "_readme": [ - "This file locks the dependencies of your project to a known state", - "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file" - ], - "hash": "2bd51bb9e4409cf52d733e0ccf23d48c", - "packages": [ - { - "name": "aws/aws-sdk-php", - "version": "2.3.4", - "source": { - "type": "git", - "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "43b63c66bde4c6c35378a417a09ed1240ec9eecb" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/43b63c66bde4c6c35378a417a09ed1240ec9eecb", - "reference": "43b63c66bde4c6c35378a417a09ed1240ec9eecb", - "shasum": "" - }, - "require": { - "guzzle/guzzle": "~3.6.0", - "php": ">=5.3.3" - }, - "require-dev": { - "doctrine/cache": "~1.0", - "ext-apc": "*", - "ext-openssl": "*", - "monolog/monolog": "1.4.*", - "phpunit/phpunit": "3.7.*", - "symfony/class-loader": "2.*", - "symfony/yaml": "2.*" - }, - "suggest": { - "doctrine/cache": "Adds support for caching of credentials and responses", - "ext-apc": "Allows service description opcode caching, request and response caching, and credentials caching", - "ext-openssl": "Allows working with CloudFront private distributions and verifying received SNS messages", - "monolog/monolog": "Adds support for logging HTTP requests and responses", - "symfony/yaml": "Eases the ability to write manifests for creating jobs in AWS Import/Export" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "2.3.x-dev" - } - }, - "autoload": { - "psr-0": { - "Aws": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "Apache-2.0" - ], - "authors": [ - { - "name": "Amazon Web Services", - "homepage": "http://aws.amazon.com" - } - ], - "description": "AWS SDK for PHP", - "homepage": "http://aws.amazon.com/sdkforphp2", - "keywords": [ - "amazon", - "aws", - "dynamodb", - "ec2", - "s3", - "sdk" - ], - "time": "2013-05-30 16:30:25" - }, - { - "name": "guzzle/guzzle", - "version": "v3.6.0", - "source": { - "type": "git", - "url": "https://github.com/guzzle/guzzle.git", - "reference": "b550d534c9b668c767b6a532bd686d0942505f7a" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/guzzle/guzzle/zipball/b550d534c9b668c767b6a532bd686d0942505f7a", - "reference": "b550d534c9b668c767b6a532bd686d0942505f7a", - "shasum": "" - }, - "require": { - "ext-curl": "*", - "php": ">=5.3.2", - "symfony/event-dispatcher": ">=2.1" - }, - "replace": { - "guzzle/batch": "self.version", - "guzzle/cache": "self.version", - "guzzle/common": "self.version", - "guzzle/http": "self.version", - "guzzle/inflection": "self.version", - "guzzle/iterator": "self.version", - "guzzle/log": "self.version", - "guzzle/parser": "self.version", - "guzzle/plugin": "self.version", - "guzzle/plugin-async": "self.version", - "guzzle/plugin-backoff": "self.version", - "guzzle/plugin-cache": "self.version", - "guzzle/plugin-cookie": "self.version", - "guzzle/plugin-curlauth": "self.version", - "guzzle/plugin-error-response": "self.version", - "guzzle/plugin-history": "self.version", - "guzzle/plugin-log": "self.version", - "guzzle/plugin-md5": "self.version", - "guzzle/plugin-mock": "self.version", - "guzzle/plugin-oauth": "self.version", - "guzzle/service": "self.version", - "guzzle/stream": "self.version" - }, - "require-dev": { - "doctrine/cache": "*", - "monolog/monolog": "1.*", - "phpunit/phpunit": "3.7.*", - "psr/log": "1.0.*", - "symfony/class-loader": "*", - "zendframework/zend-cache": "2.0.*", - "zendframework/zend-log": "2.0.*" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "3.6-dev" - } - }, - "autoload": { - "psr-0": { - "Guzzle\\Tests": "tests/", - "Guzzle": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Michael Dowling", - "email": "mtdowling@gmail.com", - "homepage": "https://github.com/mtdowling" - }, - { - "name": "Guzzle Community", - "homepage": "https://github.com/guzzle/guzzle/contributors" - } - ], - "description": "Guzzle is a PHP HTTP client library and framework for building RESTful web service clients", - "homepage": "http://guzzlephp.org/", - "keywords": [ - "client", - "curl", - "framework", - "http", - "http client", - "rest", - "web service" - ], - "time": "2013-05-30 07:01:25" - }, - { - "name": "symfony/event-dispatcher", - "version": "v2.4.0", - "target-dir": "Symfony/Component/EventDispatcher", - "source": { - "type": "git", - "url": "https://github.com/symfony/EventDispatcher.git", - "reference": "acd1707236f6eb96fbb8d58f63d289b72ebc2f6e" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/acd1707236f6eb96fbb8d58f63d289b72ebc2f6e", - "reference": "acd1707236f6eb96fbb8d58f63d289b72ebc2f6e", - "shasum": "" - }, - "require": { - "php": ">=5.3.3" - }, - "require-dev": { - "symfony/dependency-injection": "~2.0" - }, - "suggest": { - "symfony/dependency-injection": "", - "symfony/http-kernel": "" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "2.4-dev" - } - }, - "autoload": { - "psr-0": { - "Symfony\\Component\\EventDispatcher\\": "" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Fabien Potencier", - "email": "fabien@symfony.com" - }, - { - "name": "Symfony Community", - "homepage": "http://symfony.com/contributors" - } - ], - "description": "Symfony EventDispatcher Component", - "homepage": "http://symfony.com", - "time": "2013-12-03 14:52:22" - } - ], - "packages-dev": [ - { - "name": "phpunit/php-code-coverage", - "version": "1.2.13", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "466e7cd2554b4e264c9e3f31216d25ac0e5f3d94" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/466e7cd2554b4e264c9e3f31216d25ac0e5f3d94", - "reference": "466e7cd2554b4e264c9e3f31216d25ac0e5f3d94", - "shasum": "" - }, - "require": { - "php": ">=5.3.3", - "phpunit/php-file-iterator": ">=1.3.0@stable", - "phpunit/php-text-template": ">=1.1.1@stable", - "phpunit/php-token-stream": ">=1.1.3@stable" - }, - "require-dev": { - "phpunit/phpunit": "3.7.*@dev" - }, - "suggest": { - "ext-dom": "*", - "ext-xdebug": ">=2.0.5" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.2.x-dev" - } - }, - "autoload": { - "classmap": [ - "PHP/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "include-path": [ - "" - ], - "license": [ - "BSD-3-Clause" - ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sb@sebastian-bergmann.de", - "role": "lead" - } - ], - "description": "Library that provides collection, processing, and rendering functionality for PHP code coverage information.", - "homepage": "https://github.com/sebastianbergmann/php-code-coverage", - "keywords": [ - "coverage", - "testing", - "xunit" - ], - "time": "2013-09-10 08:14:32" - }, - { - "name": "phpunit/php-file-iterator", - "version": "1.3.4", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/php-file-iterator.git", - "reference": "acd690379117b042d1c8af1fafd61bde001bf6bb" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/acd690379117b042d1c8af1fafd61bde001bf6bb", - "reference": "acd690379117b042d1c8af1fafd61bde001bf6bb", - "shasum": "" - }, - "require": { - "php": ">=5.3.3" - }, - "type": "library", - "autoload": { - "classmap": [ - "File/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "include-path": [ - "" - ], - "license": [ - "BSD-3-Clause" - ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sb@sebastian-bergmann.de", - "role": "lead" - } - ], - "description": "FilterIterator implementation that filters files based on a list of suffixes.", - "homepage": "https://github.com/sebastianbergmann/php-file-iterator/", - "keywords": [ - "filesystem", - "iterator" - ], - "time": "2013-10-10 15:34:57" - }, - { - "name": "phpunit/php-text-template", - "version": "1.1.4", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/php-text-template.git", - "reference": "5180896f51c5b3648ac946b05f9ec02be78a0b23" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/5180896f51c5b3648ac946b05f9ec02be78a0b23", - "reference": "5180896f51c5b3648ac946b05f9ec02be78a0b23", - "shasum": "" - }, - "require": { - "php": ">=5.3.3" - }, - "type": "library", - "autoload": { - "classmap": [ - "Text/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "include-path": [ - "" - ], - "license": [ - "BSD-3-Clause" - ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sb@sebastian-bergmann.de", - "role": "lead" - } - ], - "description": "Simple template engine.", - "homepage": "https://github.com/sebastianbergmann/php-text-template/", - "keywords": [ - "template" - ], - "time": "2012-10-31 18:15:28" - }, - { - "name": "phpunit/php-timer", - "version": "1.0.5", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/php-timer.git", - "reference": "19689d4354b295ee3d8c54b4f42c3efb69cbc17c" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/19689d4354b295ee3d8c54b4f42c3efb69cbc17c", - "reference": "19689d4354b295ee3d8c54b4f42c3efb69cbc17c", - "shasum": "" - }, - "require": { - "php": ">=5.3.3" - }, - "type": "library", - "autoload": { - "classmap": [ - "PHP/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "include-path": [ - "" - ], - "license": [ - "BSD-3-Clause" - ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sb@sebastian-bergmann.de", - "role": "lead" - } - ], - "description": "Utility class for timing", - "homepage": "https://github.com/sebastianbergmann/php-timer/", - "keywords": [ - "timer" - ], - "time": "2013-08-02 07:42:54" - }, - { - "name": "phpunit/php-token-stream", - "version": "1.2.1", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/php-token-stream.git", - "reference": "5220af2a7929aa35cf663d97c89ad3d50cf5fa3e" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-token-stream/zipball/5220af2a7929aa35cf663d97c89ad3d50cf5fa3e", - "reference": "5220af2a7929aa35cf663d97c89ad3d50cf5fa3e", - "shasum": "" - }, - "require": { - "ext-tokenizer": "*", - "php": ">=5.3.3" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.2-dev" - } - }, - "autoload": { - "classmap": [ - "PHP/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "include-path": [ - "" - ], - "license": [ - "BSD-3-Clause" - ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sb@sebastian-bergmann.de", - "role": "lead" - } - ], - "description": "Wrapper around PHP's tokenizer extension.", - "homepage": "https://github.com/sebastianbergmann/php-token-stream/", - "keywords": [ - "tokenizer" - ], - "time": "2013-09-13 04:58:23" - }, - { - "name": "phpunit/phpunit", - "version": "3.7.28", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "3b97c8492bcafbabe6b6fbd2ab35f2f04d932a8d" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/3b97c8492bcafbabe6b6fbd2ab35f2f04d932a8d", - "reference": "3b97c8492bcafbabe6b6fbd2ab35f2f04d932a8d", - "shasum": "" - }, - "require": { - "ext-dom": "*", - "ext-pcre": "*", - "ext-reflection": "*", - "ext-spl": "*", - "php": ">=5.3.3", - "phpunit/php-code-coverage": "~1.2.1", - "phpunit/php-file-iterator": ">=1.3.1", - "phpunit/php-text-template": ">=1.1.1", - "phpunit/php-timer": ">=1.0.4", - "phpunit/phpunit-mock-objects": "~1.2.0", - "symfony/yaml": "~2.0" - }, - "require-dev": { - "pear-pear/pear": "1.9.4" - }, - "suggest": { - "ext-json": "*", - "ext-simplexml": "*", - "ext-tokenizer": "*", - "phpunit/php-invoker": ">=1.1.0,<1.2.0" - }, - "bin": [ - "composer/bin/phpunit" - ], - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "3.7.x-dev" - } - }, - "autoload": { - "classmap": [ - "PHPUnit/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "include-path": [ - "", - "../../symfony/yaml/" - ], - "license": [ - "BSD-3-Clause" - ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sebastian@phpunit.de", - "role": "lead" - } - ], - "description": "The PHP Unit Testing framework.", - "homepage": "http://www.phpunit.de/", - "keywords": [ - "phpunit", - "testing", - "xunit" - ], - "time": "2013-10-17 07:27:40" - }, - { - "name": "phpunit/phpunit-mock-objects", - "version": "1.2.3", - "source": { - "type": "git", - "url": "https://github.com/sebastianbergmann/phpunit-mock-objects.git", - "reference": "5794e3c5c5ba0fb037b11d8151add2a07fa82875" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit-mock-objects/zipball/5794e3c5c5ba0fb037b11d8151add2a07fa82875", - "reference": "5794e3c5c5ba0fb037b11d8151add2a07fa82875", - "shasum": "" - }, - "require": { - "php": ">=5.3.3", - "phpunit/php-text-template": ">=1.1.1@stable" - }, - "suggest": { - "ext-soap": "*" - }, - "type": "library", - "autoload": { - "classmap": [ - "PHPUnit/" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "include-path": [ - "" - ], - "license": [ - "BSD-3-Clause" - ], - "authors": [ - { - "name": "Sebastian Bergmann", - "email": "sb@sebastian-bergmann.de", - "role": "lead" - } - ], - "description": "Mock Object library for PHPUnit", - "homepage": "https://github.com/sebastianbergmann/phpunit-mock-objects/", - "keywords": [ - "mock", - "xunit" - ], - "time": "2013-01-13 10:24:48" - }, - { - "name": "symfony/yaml", - "version": "v2.4.0", - "target-dir": "Symfony/Component/Yaml", - "source": { - "type": "git", - "url": "https://github.com/symfony/Yaml.git", - "reference": "1ae235a1b9d3ad3d9f3860ff20acc072df95b7f5" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/Yaml/zipball/1ae235a1b9d3ad3d9f3860ff20acc072df95b7f5", - "reference": "1ae235a1b9d3ad3d9f3860ff20acc072df95b7f5", - "shasum": "" - }, - "require": { - "php": ">=5.3.3" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "2.4-dev" - } - }, - "autoload": { - "psr-0": { - "Symfony\\Component\\Yaml\\": "" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Fabien Potencier", - "email": "fabien@symfony.com" - }, - { - "name": "Symfony Community", - "homepage": "http://symfony.com/contributors" - } - ], - "description": "Symfony Yaml Component", - "homepage": "http://symfony.com", - "time": "2013-11-26 16:40:27" - } - ], - "aliases": [ - - ], - "minimum-stability": "stable", - "stability-flags": [ - - ], - "platform": [ - - ], - "platform-dev": [ - - ] -} diff --git a/client_tests/php/phpunit.xml.dist b/client_tests/php/phpunit.xml.dist deleted file mode 100644 index 170298107..000000000 --- a/client_tests/php/phpunit.xml.dist +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - tests - - - diff --git a/client_tests/php/test_services.json b/client_tests/php/test_services.json deleted file mode 100644 index 5b7d52a57..000000000 --- a/client_tests/php/test_services.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "includes" : ["_aws"], - "services" : { - "default_settings" : { - "params" : { - "key" : "key_id", - "secret" : "key_secret", - "region" : "us-east-1", - "base_url" : "http://s3.amazonaws.com", - "curl.options": {"CURLOPT_PROXY" : "localhost:8080"}, - "scheme" : "http" - } - } - } -} diff --git a/client_tests/php/tests/S3ClientTest.php b/client_tests/php/tests/S3ClientTest.php deleted file mode 100644 index 8e177e830..000000000 --- a/client_tests/php/tests/S3ClientTest.php +++ /dev/null @@ -1,135 +0,0 @@ -client = $this->getServiceBuilder()->get('s3'); - $this->client->getEventDispatcher()->removeSubscriber(BackoffPlugin::getExponentialBackoff()); // disable retry - $this->bucket = randBucket(); - } - - public function tearDown() - { - if ( ! $this->client->doesBucketExist($this->bucket)) { return; } - - $objects = $this->client->getIterator('ListObjects', array('Bucket' => $this->bucket)); - foreach ($objects as $object) { - $this->client->deleteObject(array('Bucket' => $this->bucket, 'Key' => $object['Key'])); - } - $this->client->deleteBucket(array('Bucket' => $this->bucket)); - } - - public function testBucketNotExists() - { - $this->assertFalse($this->client->doesBucketExist($this->bucket)); - try { - $this->client->getBucketAcl(array('Bucket' => $this->bucket))['Grants']; - $this->fail(); - } catch (Aws\S3\Exception\S3Exception $e) { /* noop */ } - - try { - $this->client->getObject(array('Bucket' => $this->bucket, 'Key' => randKey())); - $this->fail(); - } catch (Aws\S3\Exception\S3Exception $e) { /* noop */ } - } - - public function testCreateDeleteBucket() - { - $this->client->createBucket(array('Bucket' => $this->bucket)); - $this->assertTrue($this->client->doesBucketExist($this->bucket)); - - $this->client->deleteBucket(array('Bucket' => $this->bucket)); - $this->assertFalse($this->client->doesBucketExist($this->bucket)); - } - - public function testPutGetDeleteObject() - { - $this->client->createBucket(array('Bucket' => $this->bucket)); - $key = randKey(); - - $this->client->putObject( - array( - 'Bucket' => $this->bucket, - 'Key' => $key, - 'Content-Type' => 'text/plain', - 'Body' => "This is a entity body.")); - $this->assertTrue($this->client->doesObjectExist($this->bucket, $key)); - - $object = $this->client->getObject(array('Bucket' => $this->bucket, 'Key' => $key)); - $this->assertEquals('This is a entity body.',(string)$object['Body']); - - $this->client->deleteObject(array('Bucket' => $this->bucket, 'Key' => $key)); - $this->assertFalse($this->client->doesObjectExist($this->bucket, $key)); - - } - - public function testBucketACL() - { - $this->client->createBucket(array('Bucket' => $this->bucket)); - $this->client->putBucketAcl(array('Bucket' => $this->bucket, 'ACL' => 'public-read')); - $grants = $this->client->getBucketAcl(array('Bucket' => $this->bucket))['Grants']; - - $filtered_grants = array_filter($grants, function($item){ - return array_key_exists('URI', $item['Grantee']) - && $item['Grantee']['URI'] == self::ALL_USER_URL - && $item['Permission'] == 'READ'; - }); - $this->assertGreaterThanOrEqual(1, sizeof($filtered_grants)); - } - - public function testObjectACL() - { - $this->client->createBucket(array('Bucket' => $this->bucket)); - $key = randKey(); - $this->client->putObject( - array( - 'Bucket' => $this->bucket, - 'Key' => $key, - 'Content-Type' => 'text/plain', - 'Body' => "This is a entity body.")); - $this->client->putObjectAcl(array('Bucket' => $this->bucket, 'Key' => $key,'ACL' => 'public-read')); - $grants = $this->client->getObjectAcl(array('Bucket' => $this->bucket, 'Key' => $key))['Grants']; - $filtered_grants = array_filter($grants, function($item){ - return array_key_exists('URI', $item['Grantee']) - && $item['Grantee']['URI'] == self::ALL_USER_URL - && $item['Permission'] == 'READ'; - }); - $this->assertGreaterThanOrEqual(1, sizeof($filtered_grants)); - } -} -function randBucket() -{ - return uniqid('aws-sdk-test-'); -} - -function randKey() -{ - return uniqid('key-'); -} -?> diff --git a/client_tests/php/tests/bootstrap.php b/client_tests/php/tests/bootstrap.php deleted file mode 100644 index 3143625e2..000000000 --- a/client_tests/php/tests/bootstrap.php +++ /dev/null @@ -1,51 +0,0 @@ - $user['key_id'], - 'secret' => $user['key_secret'], - 'curl.options' => array('CURLOPT_PROXY' => 'localhost:' . cs_port()) - )); -} - -function creat_user() { - $name = uniqid('riakcs-'); - $client = new Client('http://localhost:' . cs_port()); - $request = $client->put('/riak-cs/user', - array('Content-Type' => 'application/json'), - "{\"name\":\"{$name}\", \"email\":\"{$name}@example.com\"}"); - return $request->send()->json(); -} - -function cs_port() { - return getenv('CS_HTTP_PORT') ? getenv('CS_HTTP_PORT') : 8080; -} - - -Guzzle\Tests\GuzzleTestCase::setServiceBuilder(newServiceBuilder()); - -?> diff --git a/client_tests/python/.gitignore b/client_tests/python/.gitignore deleted file mode 100644 index e35672b97..000000000 --- a/client_tests/python/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.pyc -env diff --git a/client_tests/python/Makefile b/client_tests/python/Makefile deleted file mode 100644 index 0b2bd9679..000000000 --- a/client_tests/python/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -.PHONY: boto_tests ceph_tests - -all: - $(MAKE) -C ./boto_tests - $(MAKE) -C ./ceph_tests - -boto_tests: - $(MAKE) -C ./boto_tests - -ceph_tests: - $(MAKE) -C ./ceph_tests - -clean: - $(MAKE) -C ./boto_tests clean - $(MAKE) -C ./ceph_tests clean diff --git a/client_tests/python/README.md b/client_tests/python/README.md deleted file mode 100644 index d66ac744d..000000000 --- a/client_tests/python/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# Riak CS Python Tests: Boto based tests and ceph's s3-tests - -## Dependencies - -* [virtualenv](http://www.virtualenv.org/en/latest/#installation) (I'm using 1.9.1) -* Python (I'm using 2.7.2) -* `uuidgen` command (e.g. `apt-get install uuid-runtime` on Debian/Ubuntu) - -## Configuration - -Ensure that the Riak CS `advanced.config` has the following items: - -```erlang -{anonymous_user_creation, true}, -{enforce_multipart_part_size, false}, -{max_buckets_per_user, 300}, -{auth_v4_enabled, true}, -``` - -## Execution - -There is a `Makefile` that will set everything up for you, including all of the -dependencies. The `all` target will install everything and run the integration -tests: - -```bash -make -``` - -Take a look at the `Makefile` for more detail about how the test is set up. diff --git a/client_tests/python/boto_tests/Makefile b/client_tests/python/boto_tests/Makefile deleted file mode 100644 index abdcdfd02..000000000 --- a/client_tests/python/boto_tests/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -.PHONY: test test-auth-v2 test-auth-v4 clean all -.DEFAULT: all - -DEPS = env/lib/python2.7/site-packages -BIN = env/bin - -## overwrite auth.py because boto has bugs of auth v4 -## diff from 2.35.1: bb5e02b1c61b3cf03ba9cafc7a3f6c56adcf92ad.patch -PATCHED_AUTH_PY = auth.py.patched - -all: test - -env: - @virtualenv env - -$(DEPS) $(BIN): requirements.txt env - @env/bin/pip install -r requirements.txt - cp $(PATCHED_AUTH_PY) $(DEPS)/boto/auth.py - rm -f $(DEPS)/boto/auth.pyc - -test: test-auth-v2 test-auth-v4 - -test-auth-v2: $(DEPS) $(BIN) - env CS_AUTH=auth-v2 env/bin/python boto_test.py - -test-auth-v4: $(DEPS) $(BIN) - env CS_AUTH=auth-v4 env/bin/python boto_test.py - -clean: - @rm -rf env diff --git a/client_tests/python/boto_tests/auth.py.patched b/client_tests/python/boto_tests/auth.py.patched deleted file mode 100644 index eac147171..000000000 --- a/client_tests/python/boto_tests/auth.py.patched +++ /dev/null @@ -1,1041 +0,0 @@ -# Copyright 2010 Google Inc. -# Copyright (c) 2011 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2011, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - - -""" -Handles authentication required to AWS and GS -""" - -import base64 -import boto -import boto.auth_handler -import boto.exception -import boto.plugin -import boto.utils -from collections import OrderedDict -import copy -import datetime -from email.utils import formatdate -import hmac -import os -import posixpath - -from boto.compat import urllib, encodebytes -from boto.auth_handler import AuthHandler -from boto.exception import BotoClientError - -try: - from hashlib import sha1 as sha - from hashlib import sha256 as sha256 -except ImportError: - import sha - sha256 = None - - -# Region detection strings to determine if SigV4 should be used -# by default. -SIGV4_DETECT = [ - '.cn-', - # In eu-central we support both host styles for S3 - '.eu-central', - '-eu-central', -] - - -class HmacKeys(object): - """Key based Auth handler helper.""" - - def __init__(self, host, config, provider): - if provider.access_key is None or provider.secret_key is None: - raise boto.auth_handler.NotReadyToAuthenticate() - self.host = host - self.update_provider(provider) - - def update_provider(self, provider): - self._provider = provider - self._hmac = hmac.new(self._provider.secret_key.encode('utf-8'), - digestmod=sha) - if sha256: - self._hmac_256 = hmac.new(self._provider.secret_key.encode('utf-8'), - digestmod=sha256) - else: - self._hmac_256 = None - - def algorithm(self): - if self._hmac_256: - return 'HmacSHA256' - else: - return 'HmacSHA1' - - def _get_hmac(self): - if self._hmac_256: - digestmod = sha256 - else: - digestmod = sha - return hmac.new(self._provider.secret_key.encode('utf-8'), - digestmod=digestmod) - - def sign_string(self, string_to_sign): - new_hmac = self._get_hmac() - new_hmac.update(string_to_sign.encode('utf-8')) - return encodebytes(new_hmac.digest()).decode('utf-8').strip() - - def __getstate__(self): - pickled_dict = copy.copy(self.__dict__) - del pickled_dict['_hmac'] - del pickled_dict['_hmac_256'] - return pickled_dict - - def __setstate__(self, dct): - self.__dict__ = dct - self.update_provider(self._provider) - - -class AnonAuthHandler(AuthHandler, HmacKeys): - """ - Implements Anonymous requests. - """ - - capability = ['anon'] - - def __init__(self, host, config, provider): - super(AnonAuthHandler, self).__init__(host, config, provider) - - def add_auth(self, http_request, **kwargs): - pass - - -class HmacAuthV1Handler(AuthHandler, HmacKeys): - """ Implements the HMAC request signing used by S3 and GS.""" - - capability = ['hmac-v1', 's3'] - - def __init__(self, host, config, provider): - AuthHandler.__init__(self, host, config, provider) - HmacKeys.__init__(self, host, config, provider) - self._hmac_256 = None - - def update_provider(self, provider): - super(HmacAuthV1Handler, self).update_provider(provider) - self._hmac_256 = None - - def add_auth(self, http_request, **kwargs): - headers = http_request.headers - method = http_request.method - auth_path = http_request.auth_path - if 'Date' not in headers: - headers['Date'] = formatdate(usegmt=True) - - if self._provider.security_token: - key = self._provider.security_token_header - headers[key] = self._provider.security_token - string_to_sign = boto.utils.canonical_string(method, auth_path, - headers, None, - self._provider) - boto.log.debug('StringToSign:\n%s' % string_to_sign) - b64_hmac = self.sign_string(string_to_sign) - auth_hdr = self._provider.auth_header - auth = ("%s %s:%s" % (auth_hdr, self._provider.access_key, b64_hmac)) - boto.log.debug('Signature:\n%s' % auth) - headers['Authorization'] = auth - - -class HmacAuthV2Handler(AuthHandler, HmacKeys): - """ - Implements the simplified HMAC authorization used by CloudFront. - """ - capability = ['hmac-v2', 'cloudfront'] - - def __init__(self, host, config, provider): - AuthHandler.__init__(self, host, config, provider) - HmacKeys.__init__(self, host, config, provider) - self._hmac_256 = None - - def update_provider(self, provider): - super(HmacAuthV2Handler, self).update_provider(provider) - self._hmac_256 = None - - def add_auth(self, http_request, **kwargs): - headers = http_request.headers - if 'Date' not in headers: - headers['Date'] = formatdate(usegmt=True) - if self._provider.security_token: - key = self._provider.security_token_header - headers[key] = self._provider.security_token - - b64_hmac = self.sign_string(headers['Date']) - auth_hdr = self._provider.auth_header - headers['Authorization'] = ("%s %s:%s" % - (auth_hdr, - self._provider.access_key, b64_hmac)) - - -class HmacAuthV3Handler(AuthHandler, HmacKeys): - """Implements the new Version 3 HMAC authorization used by Route53.""" - - capability = ['hmac-v3', 'route53', 'ses'] - - def __init__(self, host, config, provider): - AuthHandler.__init__(self, host, config, provider) - HmacKeys.__init__(self, host, config, provider) - - def add_auth(self, http_request, **kwargs): - headers = http_request.headers - if 'Date' not in headers: - headers['Date'] = formatdate(usegmt=True) - - if self._provider.security_token: - key = self._provider.security_token_header - headers[key] = self._provider.security_token - - b64_hmac = self.sign_string(headers['Date']) - s = "AWS3-HTTPS AWSAccessKeyId=%s," % self._provider.access_key - s += "Algorithm=%s,Signature=%s" % (self.algorithm(), b64_hmac) - headers['X-Amzn-Authorization'] = s - - -class HmacAuthV3HTTPHandler(AuthHandler, HmacKeys): - """ - Implements the new Version 3 HMAC authorization used by DynamoDB. - """ - - capability = ['hmac-v3-http'] - - def __init__(self, host, config, provider): - AuthHandler.__init__(self, host, config, provider) - HmacKeys.__init__(self, host, config, provider) - - def headers_to_sign(self, http_request): - """ - Select the headers from the request that need to be included - in the StringToSign. - """ - headers_to_sign = {'Host': self.host} - for name, value in http_request.headers.items(): - lname = name.lower() - if lname.startswith('x-amz'): - headers_to_sign[name] = value - return headers_to_sign - - def canonical_headers(self, headers_to_sign): - """ - Return the headers that need to be included in the StringToSign - in their canonical form by converting all header keys to lower - case, sorting them in alphabetical order and then joining - them into a string, separated by newlines. - """ - l = sorted(['%s:%s' % (n.lower().strip(), - headers_to_sign[n].strip()) for n in headers_to_sign]) - return '\n'.join(l) - - def string_to_sign(self, http_request): - """ - Return the canonical StringToSign as well as a dict - containing the original version of all headers that - were included in the StringToSign. - """ - headers_to_sign = self.headers_to_sign(http_request) - canonical_headers = self.canonical_headers(headers_to_sign) - string_to_sign = '\n'.join([http_request.method, - http_request.auth_path, - '', - canonical_headers, - '', - http_request.body]) - return string_to_sign, headers_to_sign - - def add_auth(self, req, **kwargs): - """ - Add AWS3 authentication to a request. - - :type req: :class`boto.connection.HTTPRequest` - :param req: The HTTPRequest object. - """ - # This could be a retry. Make sure the previous - # authorization header is removed first. - if 'X-Amzn-Authorization' in req.headers: - del req.headers['X-Amzn-Authorization'] - req.headers['X-Amz-Date'] = formatdate(usegmt=True) - if self._provider.security_token: - req.headers['X-Amz-Security-Token'] = self._provider.security_token - string_to_sign, headers_to_sign = self.string_to_sign(req) - boto.log.debug('StringToSign:\n%s' % string_to_sign) - hash_value = sha256(string_to_sign.encode('utf-8')).digest() - b64_hmac = self.sign_string(hash_value) - s = "AWS3 AWSAccessKeyId=%s," % self._provider.access_key - s += "Algorithm=%s," % self.algorithm() - s += "SignedHeaders=%s," % ';'.join(headers_to_sign) - s += "Signature=%s" % b64_hmac - req.headers['X-Amzn-Authorization'] = s - - -class HmacAuthV4Handler(AuthHandler, HmacKeys): - """ - Implements the new Version 4 HMAC authorization. - """ - - capability = ['hmac-v4'] - - def __init__(self, host, config, provider, - service_name=None, region_name=None): - AuthHandler.__init__(self, host, config, provider) - HmacKeys.__init__(self, host, config, provider) - # You can set the service_name and region_name to override the - # values which would otherwise come from the endpoint, e.g. - # ..amazonaws.com. - self.service_name = service_name - self.region_name = region_name - - def _sign(self, key, msg, hex=False): - if not isinstance(key, bytes): - key = key.encode('utf-8') - - if hex: - sig = hmac.new(key, msg.encode('utf-8'), sha256).hexdigest() - else: - sig = hmac.new(key, msg.encode('utf-8'), sha256).digest() - return sig - - def headers_to_sign(self, http_request): - """ - Select the headers from the request that need to be included - in the StringToSign. - """ - host_header_value = self.host_header(self.host, http_request) - headers_to_sign = {'Host': host_header_value} - for name, value in http_request.headers.items(): - lname = name.lower() - if lname.startswith('x-amz'): - if isinstance(value, bytes): - value = value.decode('utf-8') - headers_to_sign[name] = value - return headers_to_sign - - def host_header(self, host, http_request): - port = http_request.port - secure = http_request.protocol == 'https' - if ((port == 80 and not secure) or (port == 443 and secure)): - return host - return '%s:%s' % (host, port) - - def query_string(self, http_request): - parameter_names = sorted(http_request.params.keys()) - pairs = [] - for pname in parameter_names: - pval = boto.utils.get_utf8_value(http_request.params[pname]) - pairs.append(urllib.parse.quote(pname, safe='') + '=' + - urllib.parse.quote(pval, safe='-_~')) - return '&'.join(pairs) - - def canonical_query_string(self, http_request): - # POST requests pass parameters in through the - # http_request.body field. - if http_request.method == 'POST': - return "" - l = [] - for param in sorted(http_request.params): - value = boto.utils.get_utf8_value(http_request.params[param]) - l.append('%s=%s' % (urllib.parse.quote(param, safe='-_.~'), - urllib.parse.quote(value, safe='-_.~'))) - return '&'.join(l) - - def canonical_headers(self, headers_to_sign): - """ - Return the headers that need to be included in the StringToSign - in their canonical form by converting all header keys to lower - case, sorting them in alphabetical order and then joining - them into a string, separated by newlines. - """ - canonical = [] - - for header in headers_to_sign: - c_name = header.lower().strip() - raw_value = str(headers_to_sign[header]) - if '"' in raw_value: - c_value = raw_value.strip() - else: - c_value = ' '.join(raw_value.strip().split()) - canonical.append('%s:%s' % (c_name, c_value)) - return '\n'.join(canonical) - - def signed_headers(self, headers_to_sign): - l = ['%s' % n.lower().strip() for n in headers_to_sign] - return ';'.join(l) - - def canonical_uri(self, http_request): - path = http_request.auth_path - # Normalize the path - # in windows normpath('/') will be '\\' so we chane it back to '/' - normalized = posixpath.normpath(path).replace('\\', '/') - # Then urlencode whatever's left. - encoded = urllib.parse.quote(normalized) - if len(path) > 1 and path.endswith('/'): - encoded += '/' - return encoded - - def payload(self, http_request): - body = http_request.body - # If the body is a file like object, we can use - # boto.utils.compute_hash, which will avoid reading - # the entire body into memory. - if hasattr(body, 'seek') and hasattr(body, 'read'): - return boto.utils.compute_hash(body, hash_algorithm=sha256)[0] - elif not isinstance(body, bytes): - body = body.encode('utf-8') - return sha256(body).hexdigest() - - def canonical_request(self, http_request): - cr = [http_request.method.upper()] - cr.append(self.canonical_uri(http_request)) - cr.append(self.canonical_query_string(http_request)) - headers_to_sign = self.headers_to_sign(http_request) - cr.append(self.canonical_headers(headers_to_sign) + '\n') - cr.append(self.signed_headers(headers_to_sign)) - cr.append(self.payload(http_request)) - return '\n'.join(cr) - - def scope(self, http_request): - scope = [self._provider.access_key] - scope.append(http_request.timestamp) - scope.append(http_request.region_name) - scope.append(http_request.service_name) - scope.append('aws4_request') - return '/'.join(scope) - - def split_host_parts(self, host): - return host.split('.') - - def determine_region_name(self, host): - parts = self.split_host_parts(host) - if self.region_name is not None: - region_name = self.region_name - elif len(parts) > 1: - if parts[1] == 'us-gov': - region_name = 'us-gov-west-1' - else: - if len(parts) == 3: - region_name = 'us-east-1' - else: - region_name = parts[1] - else: - region_name = parts[0] - - return region_name - - def determine_service_name(self, host): - parts = self.split_host_parts(host) - if self.service_name is not None: - service_name = self.service_name - else: - service_name = parts[0] - return service_name - - def credential_scope(self, http_request): - scope = [] - http_request.timestamp = http_request.headers['X-Amz-Date'][0:8] - scope.append(http_request.timestamp) - # The service_name and region_name either come from: - # * The service_name/region_name attrs or (if these values are None) - # * parsed from the endpoint ..amazonaws.com. - region_name = self.determine_region_name(http_request.host) - service_name = self.determine_service_name(http_request.host) - http_request.service_name = service_name - http_request.region_name = region_name - - scope.append(http_request.region_name) - scope.append(http_request.service_name) - scope.append('aws4_request') - return '/'.join(scope) - - def string_to_sign(self, http_request, canonical_request): - """ - Return the canonical StringToSign as well as a dict - containing the original version of all headers that - were included in the StringToSign. - """ - sts = ['AWS4-HMAC-SHA256'] - sts.append(http_request.headers['X-Amz-Date']) - sts.append(self.credential_scope(http_request)) - sts.append(sha256(canonical_request.encode('utf-8')).hexdigest()) - return '\n'.join(sts) - - def signature(self, http_request, string_to_sign): - key = self._provider.secret_key - k_date = self._sign(('AWS4' + key).encode('utf-8'), - http_request.timestamp) - k_region = self._sign(k_date, http_request.region_name) - k_service = self._sign(k_region, http_request.service_name) - k_signing = self._sign(k_service, 'aws4_request') - return self._sign(k_signing, string_to_sign, hex=True) - - def add_auth(self, req, **kwargs): - """ - Add AWS4 authentication to a request. - - :type req: :class`boto.connection.HTTPRequest` - :param req: The HTTPRequest object. - """ - # This could be a retry. Make sure the previous - # authorization header is removed first. - if 'X-Amzn-Authorization' in req.headers: - del req.headers['X-Amzn-Authorization'] - now = datetime.datetime.utcnow() - req.headers['X-Amz-Date'] = now.strftime('%Y%m%dT%H%M%SZ') - if self._provider.security_token: - req.headers['X-Amz-Security-Token'] = self._provider.security_token - qs = self.query_string(req) - - qs_to_post = qs - - # We do not want to include any params that were mangled into - # the params if performing s3-sigv4 since it does not - # belong in the body of a post for some requests. Mangled - # refers to items in the query string URL being added to the - # http response params. However, these params get added to - # the body of the request, but the query string URL does not - # belong in the body of the request. ``unmangled_resp`` is the - # response that happened prior to the mangling. This ``unmangled_req`` - # kwarg will only appear for s3-sigv4. - if 'unmangled_req' in kwargs: - qs_to_post = self.query_string(kwargs['unmangled_req']) - - if qs_to_post and req.method == 'POST': - # Stash request parameters into post body - # before we generate the signature. - req.body = qs_to_post - req.headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8' - req.headers['Content-Length'] = str(len(req.body)) - else: - # Safe to modify req.path here since - # the signature will use req.auth_path. - req.path = req.path.split('?')[0] - - if qs: - # Don't insert the '?' unless there's actually a query string - req.path = req.path + '?' + qs - canonical_request = self.canonical_request(req) - boto.log.debug('CanonicalRequest:\n%s' % canonical_request) - string_to_sign = self.string_to_sign(req, canonical_request) - boto.log.debug('StringToSign:\n%s' % string_to_sign) - signature = self.signature(req, string_to_sign) - boto.log.debug('Signature:\n%s' % signature) - headers_to_sign = self.headers_to_sign(req) - l = ['AWS4-HMAC-SHA256 Credential=%s' % self.scope(req)] - l.append('SignedHeaders=%s' % self.signed_headers(headers_to_sign)) - l.append('Signature=%s' % signature) - req.headers['Authorization'] = ','.join(l) - - -class S3HmacAuthV4Handler(HmacAuthV4Handler, AuthHandler): - """ - Implements a variant of Version 4 HMAC authorization specific to S3. - """ - capability = ['hmac-v4-s3'] - - def __init__(self, *args, **kwargs): - super(S3HmacAuthV4Handler, self).__init__(*args, **kwargs) - - if self.region_name: - self.region_name = self.clean_region_name(self.region_name) - - def clean_region_name(self, region_name): - if region_name.startswith('s3-'): - return region_name[3:] - - return region_name - - def canonical_uri(self, http_request): - # S3 does **NOT** do path normalization that SigV4 typically does. - # Urlencode the path, **NOT** ``auth_path`` (because vhosting). - path = urllib.parse.urlparse(http_request.path) - # Because some quoting may have already been applied, let's back it out. - # unquoted = urllib.parse.unquote(path.path) - unquoted = urllib.parse.unquote(path.path.encode('utf-8')) - # Requote, this time addressing all characters. - encoded = urllib.parse.quote(unquoted) - return encoded - - def canonical_query_string(self, http_request): - # Note that we just do not return an empty string for - # POST request. Query strings in url are included in canonical - # query string. - l = [] - for param in sorted(http_request.params): - value = boto.utils.get_utf8_value(http_request.params[param]) - l.append('%s=%s' % (urllib.parse.quote(param, safe='-_.~'), - urllib.parse.quote(value, safe='-_.~'))) - return '&'.join(l) - - def host_header(self, host, http_request): - port = http_request.port - secure = http_request.protocol == 'https' - if ((port == 80 and not secure) or (port == 443 and secure)): - return http_request.host - return '%s:%s' % (http_request.host, port) - - def headers_to_sign(self, http_request): - """ - Select the headers from the request that need to be included - in the StringToSign. - """ - host_header_value = self.host_header(self.host, http_request) - headers_to_sign = {'host': host_header_value} - for name, value in http_request.headers.items(): - lname = name.lower() - # Hooray for the only difference! The main SigV4 signer only does - # ``Host`` + ``x-amz-*``. But S3 wants pretty much everything - # signed, except for authorization itself. - if lname not in ['authorization']: - headers_to_sign[lname] = value - ordered_headeres = OrderedDict(sorted(headers_to_sign.items(), - key=lambda t: t[0])) - return ordered_headeres - - def determine_region_name(self, host): - # S3's different format(s) of representing region/service from the - # rest of AWS makes this hurt too. - # - # Possible domain formats: - # - s3.amazonaws.com (Classic) - # - s3-us-west-2.amazonaws.com (Specific region) - # - bukkit.s3.amazonaws.com (Vhosted Classic) - # - bukkit.s3-ap-northeast-1.amazonaws.com (Vhosted specific region) - # - s3.cn-north-1.amazonaws.com.cn - (Beijing region) - # - bukkit.s3.cn-north-1.amazonaws.com.cn - (Vhosted Beijing region) - parts = self.split_host_parts(host) - - if self.region_name is not None: - region_name = self.region_name - else: - # Classic URLs - s3-us-west-2.amazonaws.com - if len(parts) == 3: - region_name = self.clean_region_name(parts[0]) - - # Special-case for Classic. - if region_name == 's3': - region_name = 'us-east-1' - else: - # Iterate over the parts in reverse order. - for offset, part in enumerate(reversed(parts)): - part = part.lower() - - # Look for the first thing starting with 's3'. - # Until there's a ``.s3`` TLD, we should be OK. :P - if part == 's3': - # If it's by itself, the region is the previous part. - region_name = parts[-offset] - - # Unless it's Vhosted classic - if region_name == 'amazonaws': - region_name = 'us-east-1' - - break - elif part.startswith('s3-'): - region_name = self.clean_region_name(part) - break - - return region_name - - def determine_service_name(self, host): - # Should this signing mechanism ever be used for anything else, this - # will fail. Consider utilizing the logic from the parent class should - # you find yourself here. - return 's3' - - def mangle_path_and_params(self, req): - """ - Returns a copy of the request object with fixed ``auth_path/params`` - attributes from the original. - """ - modified_req = copy.copy(req) - - # Unlike the most other services, in S3, ``req.params`` isn't the only - # source of query string parameters. - # Because of the ``query_args``, we may already have a query string - # **ON** the ``path/auth_path``. - # Rip them apart, so the ``auth_path/params`` can be signed - # appropriately. - parsed_path = urllib.parse.urlparse(modified_req.auth_path) - modified_req.auth_path = parsed_path.path - - if modified_req.params is None: - modified_req.params = {} - else: - # To keep the original request object untouched. We must make - # a copy of the params dictionary. Because the copy of the - # original request directly refers to the params dictionary - # of the original request. - copy_params = req.params.copy() - modified_req.params = copy_params - - raw_qs = parsed_path.query - existing_qs = urllib.parse.parse_qs( - raw_qs, - keep_blank_values=True - ) - - # ``parse_qs`` will return lists. Don't do that unless there's a real, - # live list provided. - for key, value in existing_qs.items(): - if isinstance(value, (list, tuple)): - if len(value) == 1: - existing_qs[key] = value[0] - - modified_req.params.update(existing_qs) - return modified_req - - def payload(self, http_request): - if http_request.headers.get('x-amz-content-sha256'): - return http_request.headers['x-amz-content-sha256'] - - return super(S3HmacAuthV4Handler, self).payload(http_request) - - def add_auth(self, req, **kwargs): - if 'x-amz-content-sha256' not in req.headers: - if '_sha256' in req.headers: - req.headers['x-amz-content-sha256'] = req.headers.pop('_sha256') - else: - req.headers['x-amz-content-sha256'] = self.payload(req) - updated_req = self.mangle_path_and_params(req) - return super(S3HmacAuthV4Handler, self).add_auth(updated_req, - unmangled_req=req, - **kwargs) - - def presign(self, req, expires, iso_date=None): - """ - Presign a request using SigV4 query params. Takes in an HTTP request - and an expiration time in seconds and returns a URL. - - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html - """ - if iso_date is None: - iso_date = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') - - region = self.determine_region_name(req.host) - service = self.determine_service_name(req.host) - - params = { - 'X-Amz-Algorithm': 'AWS4-HMAC-SHA256', - 'X-Amz-Credential': '%s/%s/%s/%s/aws4_request' % ( - self._provider.access_key, - iso_date[:8], - region, - service - ), - 'X-Amz-Date': iso_date, - 'X-Amz-Expires': expires, - 'X-Amz-SignedHeaders': 'host' - } - - if self._provider.security_token: - params['X-Amz-Security-Token'] = self._provider.security_token - - headers_to_sign = self.headers_to_sign(req) - l = sorted(['%s' % n.lower().strip() for n in headers_to_sign]) - params['X-Amz-SignedHeaders'] = ';'.join(l) - - req.params.update(params) - - cr = self.canonical_request(req) - - # We need to replace the payload SHA with a constant - cr = '\n'.join(cr.split('\n')[:-1]) + '\nUNSIGNED-PAYLOAD' - - # Date header is expected for string_to_sign, but unused otherwise - req.headers['X-Amz-Date'] = iso_date - - sts = self.string_to_sign(req, cr) - signature = self.signature(req, sts) - - # Add signature to params now that we have it - req.params['X-Amz-Signature'] = signature - - return 'https://%s%s?%s' % (req.host, req.path, - urllib.parse.urlencode(req.params)) - - -class STSAnonHandler(AuthHandler): - """ - Provides pure query construction (no actual signing). - - Used for making anonymous STS request for operations like - ``assume_role_with_web_identity``. - """ - - capability = ['sts-anon'] - - def _escape_value(self, value): - # This is changed from a previous version because this string is - # being passed to the query string and query strings must - # be url encoded. In particular STS requires the saml_response to - # be urlencoded when calling assume_role_with_saml. - return urllib.parse.quote(value) - - def _build_query_string(self, params): - keys = list(params.keys()) - keys.sort(key=lambda x: x.lower()) - pairs = [] - for key in keys: - val = boto.utils.get_utf8_value(params[key]) - pairs.append(key + '=' + self._escape_value(val.decode('utf-8'))) - return '&'.join(pairs) - - def add_auth(self, http_request, **kwargs): - headers = http_request.headers - qs = self._build_query_string( - http_request.params - ) - boto.log.debug('query_string in body: %s' % qs) - headers['Content-Type'] = 'application/x-www-form-urlencoded' - # This will be a POST so the query string should go into the body - # as opposed to being in the uri - http_request.body = qs - - -class QuerySignatureHelper(HmacKeys): - """ - Helper for Query signature based Auth handler. - - Concrete sub class need to implement _calc_sigature method. - """ - - def add_auth(self, http_request, **kwargs): - headers = http_request.headers - params = http_request.params - params['AWSAccessKeyId'] = self._provider.access_key - params['SignatureVersion'] = self.SignatureVersion - params['Timestamp'] = boto.utils.get_ts() - qs, signature = self._calc_signature( - http_request.params, http_request.method, - http_request.auth_path, http_request.host) - boto.log.debug('query_string: %s Signature: %s' % (qs, signature)) - if http_request.method == 'POST': - headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8' - http_request.body = qs + '&Signature=' + urllib.parse.quote_plus(signature) - http_request.headers['Content-Length'] = str(len(http_request.body)) - else: - http_request.body = '' - # if this is a retried request, the qs from the previous try will - # already be there, we need to get rid of that and rebuild it - http_request.path = http_request.path.split('?')[0] - http_request.path = (http_request.path + '?' + qs + - '&Signature=' + urllib.parse.quote_plus(signature)) - - -class QuerySignatureV0AuthHandler(QuerySignatureHelper, AuthHandler): - """Provides Signature V0 Signing""" - - SignatureVersion = 0 - capability = ['sign-v0'] - - def _calc_signature(self, params, *args): - boto.log.debug('using _calc_signature_0') - hmac = self._get_hmac() - s = params['Action'] + params['Timestamp'] - hmac.update(s.encode('utf-8')) - keys = params.keys() - keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower())) - pairs = [] - for key in keys: - val = boto.utils.get_utf8_value(params[key]) - pairs.append(key + '=' + urllib.parse.quote(val)) - qs = '&'.join(pairs) - return (qs, base64.b64encode(hmac.digest())) - - -class QuerySignatureV1AuthHandler(QuerySignatureHelper, AuthHandler): - """ - Provides Query Signature V1 Authentication. - """ - - SignatureVersion = 1 - capability = ['sign-v1', 'mturk'] - - def __init__(self, *args, **kw): - QuerySignatureHelper.__init__(self, *args, **kw) - AuthHandler.__init__(self, *args, **kw) - self._hmac_256 = None - - def _calc_signature(self, params, *args): - boto.log.debug('using _calc_signature_1') - hmac = self._get_hmac() - keys = params.keys() - keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower())) - pairs = [] - for key in keys: - hmac.update(key.encode('utf-8')) - val = boto.utils.get_utf8_value(params[key]) - hmac.update(val) - pairs.append(key + '=' + urllib.parse.quote(val)) - qs = '&'.join(pairs) - return (qs, base64.b64encode(hmac.digest())) - - -class QuerySignatureV2AuthHandler(QuerySignatureHelper, AuthHandler): - """Provides Query Signature V2 Authentication.""" - - SignatureVersion = 2 - capability = ['sign-v2', 'ec2', 'ec2', 'emr', 'fps', 'ecs', - 'sdb', 'iam', 'rds', 'sns', 'sqs', 'cloudformation'] - - def _calc_signature(self, params, verb, path, server_name): - boto.log.debug('using _calc_signature_2') - string_to_sign = '%s\n%s\n%s\n' % (verb, server_name.lower(), path) - hmac = self._get_hmac() - params['SignatureMethod'] = self.algorithm() - if self._provider.security_token: - params['SecurityToken'] = self._provider.security_token - keys = sorted(params.keys()) - pairs = [] - for key in keys: - val = boto.utils.get_utf8_value(params[key]) - pairs.append(urllib.parse.quote(key, safe='') + '=' + - urllib.parse.quote(val, safe='-_~')) - qs = '&'.join(pairs) - boto.log.debug('query string: %s' % qs) - string_to_sign += qs - boto.log.debug('string_to_sign: %s' % string_to_sign) - hmac.update(string_to_sign.encode('utf-8')) - b64 = base64.b64encode(hmac.digest()) - boto.log.debug('len(b64)=%d' % len(b64)) - boto.log.debug('base64 encoded digest: %s' % b64) - return (qs, b64) - - -class POSTPathQSV2AuthHandler(QuerySignatureV2AuthHandler, AuthHandler): - """ - Query Signature V2 Authentication relocating signed query - into the path and allowing POST requests with Content-Types. - """ - - capability = ['mws'] - - def add_auth(self, req, **kwargs): - req.params['AWSAccessKeyId'] = self._provider.access_key - req.params['SignatureVersion'] = self.SignatureVersion - req.params['Timestamp'] = boto.utils.get_ts() - qs, signature = self._calc_signature(req.params, req.method, - req.auth_path, req.host) - boto.log.debug('query_string: %s Signature: %s' % (qs, signature)) - if req.method == 'POST': - req.headers['Content-Length'] = str(len(req.body)) - req.headers['Content-Type'] = req.headers.get('Content-Type', - 'text/plain') - else: - req.body = '' - # if this is a retried req, the qs from the previous try will - # already be there, we need to get rid of that and rebuild it - req.path = req.path.split('?')[0] - req.path = (req.path + '?' + qs + - '&Signature=' + urllib.parse.quote_plus(signature)) - - -def get_auth_handler(host, config, provider, requested_capability=None): - """Finds an AuthHandler that is ready to authenticate. - - Lists through all the registered AuthHandlers to find one that is willing - to handle for the requested capabilities, config and provider. - - :type host: string - :param host: The name of the host - - :type config: - :param config: - - :type provider: - :param provider: - - Returns: - An implementation of AuthHandler. - - Raises: - boto.exception.NoAuthHandlerFound - """ - ready_handlers = [] - auth_handlers = boto.plugin.get_plugin(AuthHandler, requested_capability) - for handler in auth_handlers: - try: - ready_handlers.append(handler(host, config, provider)) - except boto.auth_handler.NotReadyToAuthenticate: - pass - - if not ready_handlers: - checked_handlers = auth_handlers - names = [handler.__name__ for handler in checked_handlers] - raise boto.exception.NoAuthHandlerFound( - 'No handler was ready to authenticate. %d handlers were checked.' - ' %s ' - 'Check your credentials' % (len(names), str(names))) - - # We select the last ready auth handler that was loaded, to allow users to - # customize how auth works in environments where there are shared boto - # config files (e.g., /etc/boto.cfg and ~/.boto): The more general, - # system-wide shared configs should be loaded first, and the user's - # customizations loaded last. That way, for example, the system-wide - # config might include a plugin_directory that includes a service account - # auth plugin shared by all users of a Google Compute Engine instance - # (allowing sharing of non-user data between various services), and the - # user could override this with a .boto config that includes user-specific - # credentials (for access to user data). - return ready_handlers[-1] - - -def detect_potential_sigv4(func): - def _wrapper(self): - if os.environ.get('EC2_USE_SIGV4', False): - return ['hmac-v4'] - - if boto.config.get('ec2', 'use-sigv4', False): - return ['hmac-v4'] - - if hasattr(self, 'region'): - # If you're making changes here, you should also check - # ``boto/iam/connection.py``, as several things there are also - # endpoint-related. - if getattr(self.region, 'endpoint', ''): - for test in SIGV4_DETECT: - if test in self.region.endpoint: - return ['hmac-v4'] - - return func(self) - return _wrapper - - -def detect_potential_s3sigv4(func): - def _wrapper(self): - if os.environ.get('S3_USE_SIGV4', False): - return ['hmac-v4-s3'] - - if boto.config.get('s3', 'use-sigv4', False): - return ['hmac-v4-s3'] - - if hasattr(self, 'host'): - # If you're making changes here, you should also check - # ``boto/iam/connection.py``, as several things there are also - # endpoint-related. - for test in SIGV4_DETECT: - if test in self.host: - return ['hmac-v4-s3'] - - return func(self) - return _wrapper diff --git a/client_tests/python/boto_tests/bb5e02b1c61b3cf03ba9cafc7a3f6c56adcf92ad.patch b/client_tests/python/boto_tests/bb5e02b1c61b3cf03ba9cafc7a3f6c56adcf92ad.patch deleted file mode 100644 index 1a46648d7..000000000 --- a/client_tests/python/boto_tests/bb5e02b1c61b3cf03ba9cafc7a3f6c56adcf92ad.patch +++ /dev/null @@ -1,71 +0,0 @@ -From bb5e02b1c61b3cf03ba9cafc7a3f6c56adcf92ad Mon Sep 17 00:00:00 2001 -From: Shunichi Shinohara -Date: Wed, 28 Jan 2015 12:14:29 +0900 -Subject: [PATCH] Fix S3 Version 4 HMAC auth bugs - -- Canonical headers must be sorted by *key* for canonical requests. -- Header names in canonical headers must be lowercase even it is - capitalized in HTTP request header. -- Unicode bug in URL path normalzation, path should be treated as - just a binary. This bug manifests when an object key includes - Unicode outside of ASCII and also it is uploaded by Multipart. ---- - boto/auth.py | 15 +++++++++------ - 1 file changed, 9 insertions(+), 6 deletions(-) - -diff --git a/boto/auth.py b/boto/auth.py -index f769472..eac1471 100644 ---- a/boto/auth.py -+++ b/boto/auth.py -@@ -32,6 +32,7 @@ - import boto.exception - import boto.plugin - import boto.utils -+from collections import OrderedDict - import copy - import datetime - from email.utils import formatdate -@@ -375,11 +376,10 @@ def canonical_headers(self, headers_to_sign): - else: - c_value = ' '.join(raw_value.strip().split()) - canonical.append('%s:%s' % (c_name, c_value)) -- return '\n'.join(sorted(canonical)) -+ return '\n'.join(canonical) - - def signed_headers(self, headers_to_sign): - l = ['%s' % n.lower().strip() for n in headers_to_sign] -- l = sorted(l) - return ';'.join(l) - - def canonical_uri(self, http_request): -@@ -569,7 +569,8 @@ def canonical_uri(self, http_request): - # Urlencode the path, **NOT** ``auth_path`` (because vhosting). - path = urllib.parse.urlparse(http_request.path) - # Because some quoting may have already been applied, let's back it out. -- unquoted = urllib.parse.unquote(path.path) -+ # unquoted = urllib.parse.unquote(path.path) -+ unquoted = urllib.parse.unquote(path.path.encode('utf-8')) - # Requote, this time addressing all characters. - encoded = urllib.parse.quote(unquoted) - return encoded -@@ -598,15 +599,17 @@ def headers_to_sign(self, http_request): - in the StringToSign. - """ - host_header_value = self.host_header(self.host, http_request) -- headers_to_sign = {'Host': host_header_value} -+ headers_to_sign = {'host': host_header_value} - for name, value in http_request.headers.items(): - lname = name.lower() - # Hooray for the only difference! The main SigV4 signer only does - # ``Host`` + ``x-amz-*``. But S3 wants pretty much everything - # signed, except for authorization itself. - if lname not in ['authorization']: -- headers_to_sign[name] = value -- return headers_to_sign -+ headers_to_sign[lname] = value -+ ordered_headeres = OrderedDict(sorted(headers_to_sign.items(), -+ key=lambda t: t[0])) -+ return ordered_headeres - - def determine_region_name(self, host): - # S3's different format(s) of representing region/service from the diff --git a/client_tests/python/boto_tests/boto_test.py b/client_tests/python/boto_tests/boto_test.py deleted file mode 100755 index 5f4ad6b96..000000000 --- a/client_tests/python/boto_tests/boto_test.py +++ /dev/null @@ -1,883 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -## --------------------------------------------------------------------- -## -## Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -## -## This file is provided to you under the Apache License, -## Version 2.0 (the "License"); you may not use this file -## except in compliance with the License. You may obtain -## a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, -## software distributed under the License is distributed on an -## "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -## KIND, either express or implied. See the License for the -## specific language governing permissions and limitations -## under the License. -## -## --------------------------------------------------------------------- -import os, httplib, json, unittest, uuid, md5 -from cStringIO import StringIO - -from file_generator import FileGenerator - -from boto.exception import S3ResponseError -from boto.s3.connection import S3Connection, OrdinaryCallingFormat -from boto.s3.key import Key -from boto.utils import compute_md5 -import boto - - -def setup_auth_scheme(): - auth_mech=os.environ.get('CS_AUTH', 'auth-v2') - if not boto.config.has_section('s3'): - boto.config.add_section('s3') - if auth_mech == 'auth-v4': - setup_auth_v4() - else: - setup_auth_v2() - -def setup_auth_v4(): - print('Use AWS Version 4 authentication') - if not boto.config.get('s3', 'use-sigv4'): - boto.config.set('s3', 'use-sigv4', 'True') - -def setup_auth_v2(): - print('Use AWS Version 2 authentication') - if not boto.config.get('s3', 'use-sigv4'): - boto.config.set('s3', 'use-sigv4', '') - -setup_auth_scheme() - -def create_user(host, port, name, email): - url = '/riak-cs/user' - body = json.dumps({"email": email, "name": name}) - conn = httplib.HTTPConnection(host, port) - headers = {"Content-Type": "application/json"} - conn.request("POST", url, body, headers) - response = conn.getresponse() - data = response.read() - conn.close() - return json.loads(data) - -# Take a boto 'Key' and returns a hex -# digest of the md5 (calculated by actually -# retrieving the bytes and calculating the md5) -def md5_from_key(boto_key): - m = md5.new() - for byte in boto_key: - m.update(byte) - return m.hexdigest() - -# `parts_list` should be a list of file-like objects -def upload_multipart(bucket, key_name, parts_list, metadata={}, policy=None): - upload = bucket.initiate_multipart_upload(key_name, metadata=metadata, - policy=policy) - for index, val in enumerate(parts_list): - upload.upload_part_from_file(val, index + 1) - result = upload.complete_upload() - return upload, result - - -class S3ApiVerificationTestBase(unittest.TestCase): - host="127.0.0.1" - try: - port=int(os.environ['CS_HTTP_PORT']) - except KeyError: - port=8080 - - user1 = None - user2 = None - - SimpleAcl = "" + \ - "" + \ - "%s" + \ - "%s" + \ - "" + \ - "" + \ - "" + \ - "" + \ - "%s" + \ - "%s" + \ - "" + \ - "%s" + \ - "" + \ - "" + \ - "" - PublicReadAcl = "" + \ - "" + \ - "%s" + \ - "%s" + \ - "" + \ - "" + \ - "" + \ - "" + \ - "http://acs.amazonaws.com/groups/global/AllUsers" + \ - "" + \ - "READ" + \ - "" + \ - "" + \ - "" + \ - "%s" + \ - "%s" + \ - "" + \ - "%s" + \ - "" + \ - "" + \ - "" - - def make_connection(self, user): - return S3Connection(user['key_id'], user['key_secret'], is_secure=False, - host="s3.amazonaws.com", debug=False, - proxy="127.0.0.1", proxy_port=self.port, - calling_format=OrdinaryCallingFormat() ) - - @classmethod - def setUpClass(cls): - # Create test user so credentials don't have to be updated - # for each test setup. - # TODO: Once changes are in place so users can be deleted, use - # userX@example.me for email addresses and clean up at the end of - # the test run. - cls.maxDiff = 10000000000 - cls.user1 = create_user(cls.host, cls.port, "user1", str(uuid.uuid4()) + "@example.me") - cls.user2 = create_user(cls.host, cls.port, "user2", str(uuid.uuid4()) + "@example.me") - cls.bucket_name = str(uuid.uuid4()) - cls.key_name = str(uuid.uuid4()) - cls.data = file("/dev/urandom").read(1024) - - def defaultAcl(self, user): - return self.SimpleAcl % (user['id'], user['display_name'], user['id'], user['display_name'], 'FULL_CONTROL') - - def prAcl(self, user): - return self.PublicReadAcl % (user['id'], user['display_name'], user['id'], user['display_name'], 'FULL_CONTROL') - - def setUp(self): - self.conn = self.make_connection(self.user1) - -class BasicTests(S3ApiVerificationTestBase): - def test_auth(self): - bad_user = json.loads('{"email":"baduser@example.me","display_name":"baduser","name":"user1","key_id":"bad_key","key_secret":"BadSecret","id":"bad_canonical_id"}') - conn = self.make_connection(bad_user) - self.assertRaises(S3ResponseError, conn.get_canonical_user_id) - - def test_auth_weird_query_param(self): - bucket = self.conn.create_bucket(self.bucket_name) - query_args = 'foo=bar%20baz' - response = bucket.connection.make_request('GET', bucket.name, "notfound", - query_args=query_args) - body = response.read() - boto.log.debug(body) - if response.status != 404: - raise bucket.connection.provider.storage_response_error( - response.status, response.reason, body) - - def test_create_bucket(self): - self.conn.create_bucket(self.bucket_name) - self.assertIn(self.bucket_name, - [b.name for b in self.conn.get_all_buckets()]) - - def test_put_object(self): - bucket = self.conn.create_bucket(self.bucket_name) - k = Key(bucket) - k.key = self.key_name - k.set_contents_from_string(self.data) - self.assertEqual(k.get_contents_as_string(), self.data) - self.assertIn(self.key_name, - [k.key for k in bucket.get_all_keys()]) - - def test_put_object_with_trailing_slash(self): - bucket = self.conn.create_bucket(self.bucket_name) - key_name_with_slash = self.key_name + "/" - k = Key(bucket) - k.key = key_name_with_slash - k.set_contents_from_string(self.data) - self.assertEqual(k.get_contents_as_string(), self.data) - self.assertIn(key_name_with_slash, - [k.key for k in bucket.get_all_keys()]) - - def test_delete_object(self): - bucket = self.conn.create_bucket(self.bucket_name) - k = Key(bucket) - k.key = self.key_name - k.delete() - self.assertNotIn(self.key_name, - [k.key for k in bucket.get_all_keys()]) - - def test_delete_objects(self): - bucket = self.conn.create_bucket(self.bucket_name) - keys = ['0', '1', u'Unicodeあいうえお', '2', 'multiple spaces'] - keys.sort() - for key in keys: - k = Key(bucket) - k.key = key - k.set_contents_from_string(key) - - all_keys = [k.key for k in bucket.get_all_keys()] - all_keys.sort() - self.assertEqual(keys, all_keys) - result = bucket.delete_keys(keys) - - self.assertEqual(keys, [k.key for k in result.deleted]) - self.assertEqual([], result.errors) - result = bucket.delete_keys(['nosuchkeys']) - self.assertEqual([], result.errors) - self.assertEqual(['nosuchkeys'], [k.key for k in result.deleted]) - all_keys = [k.key for k in bucket.get_all_keys()] - self.assertEqual([], all_keys) - - def test_delete_bucket(self): - bucket = self.conn.get_bucket(self.bucket_name) - bucket.delete() - self.assertNotIn(self.bucket_name, - [b.name for b in self.conn.get_all_buckets()]) - - def test_get_bucket_acl(self): - bucket = self.conn.create_bucket(self.bucket_name) - self.assertEqual(bucket.get_acl().to_xml(), self.defaultAcl(self.user1)) - - def test_set_bucket_acl(self): - bucket = self.conn.get_bucket(self.bucket_name) - bucket.set_canned_acl('public-read') - self.assertEqual(bucket.get_acl().to_xml(), self.prAcl(self.user1)) - - def test_get_object_acl(self): - bucket = self.conn.create_bucket(self.bucket_name) - k = Key(bucket) - k.key = self.key_name - k.set_contents_from_string(self.data) - self.assertEqual(k.get_contents_as_string(), self.data) - self.assertEqual(k.get_acl().to_xml(), self.defaultAcl(self.user1)) - - def test_set_object_acl(self): - bucket = self.conn.create_bucket(self.bucket_name) - k = Key(bucket) - k.key = self.key_name - k.set_canned_acl('public-read') - self.assertEqual(k.get_acl().to_xml(), self.prAcl(self.user1)) - -class MultiPartUploadTests(S3ApiVerificationTestBase): - def multipart_md5_helper(self, parts, key_suffix=u''): - key_name = unicode(str(uuid.uuid4())) + key_suffix - stringio_parts = [StringIO(p) for p in parts] - expected_md5 = md5.new(''.join(parts)).hexdigest() - bucket = self.conn.create_bucket(self.bucket_name) - upload, result = upload_multipart(bucket, key_name, stringio_parts) - key = Key(bucket, key_name) - actual_md5 = md5_from_key(key) - self.assertEqual(expected_md5, actual_md5) - self.assertEqual(key_name, result.key_name) - return upload, result - - def test_small_strings_upload_1(self): - parts = ['this is part one', 'part two is just a rewording', - 'surprise that part three is pretty much the same', - 'and the last part is number four'] - self.multipart_md5_helper(parts) - - def test_small_strings_upload_2(self): - parts = ['just one lonely part'] - self.multipart_md5_helper(parts) - - def test_small_strings_upload_3(self): - parts = [str(uuid.uuid4()) for _ in xrange(100)] - self.multipart_md5_helper(parts) - - def test_small_strings_upload_4(self): - parts = [str(uuid.uuid4()) for _ in xrange(20)] - self.multipart_md5_helper(parts) - - def test_acl_is_set(self): - parts = [str(uuid.uuid4()) for _ in xrange(5)] - key_name = str(uuid.uuid4()) - stringio_parts = [StringIO(p) for p in parts] - expected_md5 = md5.new(''.join(parts)).hexdigest() - bucket = self.conn.create_bucket(self.bucket_name) - upload = upload_multipart(bucket, key_name, stringio_parts, - policy='public-read') - key = Key(bucket, key_name) - actual_md5 = md5_from_key(key) - self.assertEqual(expected_md5, actual_md5) - self.assertEqual(key.get_acl().to_xml(), self.prAcl(self.user1)) - - def test_standard_storage_class(self): - # Test for bug reported in - # https://github.com/basho/riak_cs/pull/575 - bucket = self.conn.create_bucket(self.bucket_name) - key_name = 'test_standard_storage_class' - _never_finished_upload = bucket.initiate_multipart_upload(key_name) - uploads = list(bucket.list_multipart_uploads()) - for u in uploads: - self.assertEqual(u.storage_class, 'STANDARD') - self.assertTrue(True) - - def test_upload_japanese_key(self): - parts = ['this is part one', 'part two is just a rewording', - 'surprise that part three is pretty much the same', - 'and the last part is number four'] - self.multipart_md5_helper(parts, key_suffix=u'日本語サフィックス') - - def test_list_japanese_key(self): - bucket = self.conn.create_bucket(self.bucket_name) - key_name = u'test_日本語キーのリスト' - _never_finished_upload = bucket.initiate_multipart_upload(key_name) - uploads = list(bucket.list_multipart_uploads()) - for u in uploads: - self.assertEqual(u.key_name, key_name) - -def one_kb_string(): - "Return a 1KB string of all a's" - return ''.join(['a' for _ in xrange(1024)]) - -def kb_gen_fn(num_kilobytes): - s = one_kb_string() - def fn(): - return (s for _ in xrange(num_kilobytes)) - return fn - -def kb_file_gen(num_kilobytes): - gen_fn = kb_gen_fn(num_kilobytes) - return FileGenerator(gen_fn, num_kilobytes * 1024) - -def mb_file_gen(num_megabytes): - return kb_file_gen(num_megabytes * 1024) - -def md5_from_file(file_object): - m = md5.new() - update_md5_from_file(m, file_object) - return m.hexdigest() - -def md5_from_files(file_objects): - "note the plural" - m = md5.new() - for f in file_objects: - update_md5_from_file(m, f) - return m.hexdigest() - -def update_md5_from_file(md5_object, file_object): - "Helper function for calculating the hex md5 of a file-like object" - go = True - while go: - byte = file_object.read(8196) - if byte: - md5_object.update(byte) - else: - go = False - return md5_object - -def remove_double_quotes(string): - "remove double quote from a string" - return string.replace('"', '') - -class FileGenTest(unittest.TestCase): - def test_read_twice(self): - """ Read 2KB file and reset (seek to the head) and re-read 2KB """ - num_kb = 2 - f = kb_file_gen(num_kb) - - first1 = f.read(1024) - self.assertEqual(1024, len(first1)) - first2 = f.read(1024) - self.assertEqual(1024, len(first2)) - self.assertEqual(2048, f.pos) - self.assertEqual('', f.read(1)) - self.assertEqual('', f.read(1)) - self.assertEqual(2048, f.pos) - - f.seek(0) - self.assertEqual(0, f.pos) - second1 = f.read(1024) - self.assertEqual(1024, len(first1)) - second2 = f.read(1024) - self.assertEqual(1024, len(second2)) - self.assertEqual(2048, f.pos) - self.assertEqual('', f.read(1)) - self.assertEqual('', f.read(1)) - -class LargerFileUploadTest(S3ApiVerificationTestBase): - "Larger, regular key uploads" - - def upload_helper(self, num_kilobytes): - key_name = str(uuid.uuid4()) - bucket = self.conn.create_bucket(self.bucket_name) - md5_expected = md5_from_file(kb_file_gen(num_kilobytes)) - file_obj = kb_file_gen(num_kilobytes) - key = Key(bucket, key_name) - key.set_contents_from_file(file_obj, - md5=key.get_md5_from_hexdigest(md5_expected)) - self.assertEqual(md5_expected, remove_double_quotes(key.etag)) - - def test_1kb(self): - return self.upload_helper(1) - - def test_2kb(self): - return self.upload_helper(2) - - def test_256kb(self): - return self.upload_helper(256) - - def test_512kb(self): - return self.upload_helper(512) - - def test_1mb(self): - return self.upload_helper(1 * 1024) - - def test_4mb(self): - return self.upload_helper(4 * 1024) - - def test_8mb(self): - return self.upload_helper(8 * 1024) - - def test_16mb(self): - return self.upload_helper(16 * 1024) - - def test_32mb(self): - return self.upload_helper(32 * 1024) - -class LargerMultipartFileUploadTest(S3ApiVerificationTestBase): - """ - Larger, multipart file uploads - to pass this test, - requires '{enforce_multipart_part_size, false},' entry at riak_cs's app.config - """ - - def upload_parts_helper(self, zipped_parts_and_md5s, expected_md5): - key_name = str(uuid.uuid4()) - bucket = self.conn.create_bucket(self.bucket_name) - upload = bucket.initiate_multipart_upload(key_name) - key = Key(bucket, key_name) - for idx, (part, md5_of_part) in enumerate(zipped_parts_and_md5s): - upload.upload_part_from_file(part, idx + 1, - md5=key.get_md5_from_hexdigest(md5_of_part)) - upload.complete_upload() - actual_md5 = md5_from_key(key) - self.assertEqual(expected_md5, actual_md5) - - def from_mb_list(self, mb_list): - md5_list = [md5_from_file(mb_file_gen(m)) for m in mb_list] - expected_md5 = md5_from_files([mb_file_gen(m) for m in mb_list]) - parts = [mb_file_gen(m) for m in mb_list] - self.upload_parts_helper(zip(parts, md5_list), expected_md5) - - def test_upload_1(self): - mb_list = [5, 6, 5, 7, 8, 9] - self.from_mb_list(mb_list) - - def test_upload_2(self): - mb_list = [10, 11, 5, 7, 9, 14, 12] - self.from_mb_list(mb_list) - - def test_upload_3(self): - mb_list = [15, 14, 13, 12, 11, 10] - self.from_mb_list(mb_list) - -class UnicodeNamedObjectTest(S3ApiVerificationTestBase): - ''' test to check unicode object name works ''' - utf8_key_name = u"utf8ファイル名.txt" - # ^^^^^^^^^ filename in Japanese - - def test_unicode_object(self): - bucket = self.conn.create_bucket(self.bucket_name) - k = Key(bucket) - k.key = UnicodeNamedObjectTest.utf8_key_name - k.set_contents_from_string(self.data) - self.assertEqual(k.get_contents_as_string(), self.data) - self.assertIn(UnicodeNamedObjectTest.utf8_key_name, - [obj.key for obj in bucket.list()]) - - def test_delete_object(self): - bucket = self.conn.create_bucket(self.bucket_name) - k = Key(bucket) - k.key = UnicodeNamedObjectTest.utf8_key_name - k.delete() - self.assertNotIn(UnicodeNamedObjectTest.utf8_key_name, - [obj.key for obj in bucket.list()]) - - -class BucketPolicyTest(S3ApiVerificationTestBase): - "test bucket policy" - - def test_no_policy(self): - bucket = self.conn.create_bucket(self.bucket_name) - bucket.delete_policy() - try: bucket.get_policy() - except S3ResponseError: pass - else: self.fail() - - def create_bucket_and_set_policy(self, policy_template): - bucket = self.conn.create_bucket(self.bucket_name) - bucket.delete_policy() - policy = policy_template % bucket.name - bucket.set_policy(policy, headers={'content-type':'application/json'}) - return bucket - - def test_put_policy_invalid_ip(self): - policy_template = ''' -{"Version":"2008-10-17","Statement":[{"Sid":"Stmtaaa","Effect":"Allow","Principal":"*","Action":["s3:GetObjectAcl","s3:GetObject"],"Resource":"arn:aws:s3:::%s/*","Condition":{"IpAddress":{"aws:SourceIp":"0"}}}]} -''' - try: - self.create_bucket_and_set_policy(policy_template) - except S3ResponseError as e: - self.assertEqual(e.status, 400) - self.assertEqual(e.reason, 'Bad Request') - - def test_put_policy(self): - ### old version name - policy_template = ''' -{"Version":"2008-10-17","Statement":[{"Sid":"Stmtaaa","Effect":"Allow","Principal":"*","Action":["s3:GetObjectAcl","s3:GetObject"],"Resource":"arn:aws:s3:::%s/*","Condition":{"IpAddress":{"aws:SourceIp":"127.0.0.1/32"}}}]} -''' - bucket = self.create_bucket_and_set_policy(policy_template) - got_policy = bucket.get_policy() - self.assertEqual(policy_template % bucket.name , got_policy) - - def test_put_policy_2(self): - ### new version name, also regression of #911 - policy_template = ''' -{"Version":"2012-10-17","Statement":[{"Sid":"Stmtaaa","Effect":"Allow","Principal":"*","Action":["s3:GetObjectAcl","s3:GetObject"],"Resource":"arn:aws:s3:::%s/*","Condition":{"IpAddress":{"aws:SourceIp":"127.0.0.1/32"}}}]} -''' - bucket = self.create_bucket_and_set_policy(policy_template) - got_policy = bucket.get_policy() - self.assertEqual(policy_template % bucket.name, got_policy) - - def test_put_policy_3(self): - policy_template = ''' -{"Version":"somebadversion","Statement":[{"Sid":"Stmtaaa","Effect":"Allow","Principal":"*","Action":["s3:GetObjectAcl","s3:GetObject"],"Resource":"arn:aws:s3:::%s/*","Condition":{"IpAddress":{"aws:SourceIp":"127.0.0.1/32"}}}]} -''' - try: - self.create_bucket_and_set_policy(policy_template) - except S3ResponseError as e: - self.assertEqual(e.status, 400) - self.assertEqual(e.reason, 'Bad Request') - - - def test_ip_addr_policy(self): - policy_template = ''' -{"Version":"2008-10-17","Statement":[{"Sid":"Stmtaaa","Effect":"Deny","Principal":"*","Action":["s3:GetObject"],"Resource":"arn:aws:s3:::%s/*","Condition":{"IpAddress":{"aws:SourceIp":"%s"}}}]} -''' % ('%s', self.host) - bucket = self.create_bucket_and_set_policy(policy_template) - - key_name = str(uuid.uuid4()) - key = Key(bucket, key_name) - - key.set_contents_from_string(self.data) - bucket.list() - try: - key.get_contents_as_string() - self.fail() - except S3ResponseError as e: - self.assertEqual(e.status, 404) - self.assertEqual(e.reason, 'Not Found') - - policy = ''' -{"Version":"2008-10-17","Statement":[{"Sid":"Stmtaaa","Effect":"Allow","Principal":"*","Action":["s3:GetObject"],"Resource":"arn:aws:s3:::%s/*","Condition":{"IpAddress":{"aws:SourceIp":"%s"}}}]} -''' % (bucket.name, self.host) - self.assertTrue(bucket.set_policy(policy, headers={'content-type':'application/json'})) - key.get_contents_as_string() ## throws nothing - - - def test_invalid_transport_addr_policy(self): - bucket = self.conn.create_bucket(self.bucket_name) - key_name = str(uuid.uuid4()) - key = Key(bucket, key_name) - key.set_contents_from_string(self.data) - - ## anyone may GET this object - policy = ''' -{"Version":"2008-10-17","Statement":[{"Sid":"Stmtaaa0","Effect":"Allow","Principal":"*","Action":["s3:GetObject"],"Resource":"arn:aws:s3:::%s/*","Condition":{"Bool":{"aws:SecureTransport":wat}}}]} -''' % bucket.name - try: - bucket.set_policy(policy, headers={'content-type':'application/json'}) - except S3ResponseError as e: - self.assertEqual(e.status, 400) - self.assertEqual(e.reason, 'Bad Request') - - def test_transport_addr_policy(self): - bucket = self.conn.create_bucket(self.bucket_name) - key_name = str(uuid.uuid4()) - key = Key(bucket, key_name) - key.set_contents_from_string(self.data) - - ## anyone may GET this object - policy = ''' -{"Version":"2008-10-17","Statement":[{"Sid":"Stmtaaa0","Effect":"Allow","Principal":"*","Action":["s3:GetObject"],"Resource":"arn:aws:s3:::%s/*","Condition":{"Bool":{"aws:SecureTransport":false}}}]} -''' % bucket.name - self.assertTrue(bucket.set_policy(policy, headers={'content-type':'application/json'})) - key.get_contents_as_string() - - ## policy accepts anyone who comes with http - conn = httplib.HTTPConnection(self.host, self.port) - headers = { "Host" : "%s.s3.amazonaws.com" % bucket.name } - conn.request('GET', ("/%s" % key_name) , None, headers) - response = conn.getresponse() - self.assertEqual(response.status, 200) - self.assertEqual(response.read(), key.get_contents_as_string()) - - ## anyone without https may not do any operation - policy = ''' -{"Version":"2008-10-17","Statement":[{"Sid":"Stmtaaa0","Effect":"Deny","Principal":"*","Action":"*","Resource":"arn:aws:s3:::%s/*","Condition":{"Bool":{"aws:SecureTransport":false}}}]} -''' % bucket.name - self.assertTrue(bucket.set_policy(policy, headers={'content-type':'application/json'})) - - ## policy accepts anyone who comes with http - conn = httplib.HTTPConnection(self.host, self.port) - headers = { "Host" : "%s.s3.amazonaws.com" % bucket.name } - conn.request('GET', ("/%s" % key_name) , None, headers) - response = conn.getresponse() - self.assertEqual(response.status, 403) - self.assertEqual(response.reason, 'Forbidden') - - -class MultipartUploadTestsUnderPolicy(S3ApiVerificationTestBase): - - def test_small_strings_upload_1(self): - bucket = self.conn.create_bucket(self.bucket_name) - parts = ['this is part one', 'part two is just a rewording', - 'surprise that part three is pretty much the same', - 'and the last part is number four'] - stringio_parts = [StringIO(p) for p in parts] - expected_md5 = md5.new(''.join(parts)).hexdigest() - - key_name = str(uuid.uuid4()) - key = Key(bucket, key_name) - - ## anyone may PUT this object - policy = ''' -{"Version":"2008-10-17","Statement":[{"Sid":"Stmtaaa0","Effect":"Allow","Principal":"*","Action":["s3:PutObject"],"Resource":"arn:aws:s3:::%s/*","Condition":{"Bool":{"aws:SecureTransport":false}}}]} -''' % bucket.name - self.assertTrue(bucket.set_policy(policy, headers={'content-type':'application/json'})) - - upload = upload_multipart(bucket, key_name, stringio_parts) - actual_md5 = md5_from_key(key) - self.assertEqual(expected_md5, actual_md5) - - ## anyone without https may not do any operation - policy = ''' -{"Version":"2008-10-17","Statement":[{"Sid":"Stmtaaa0","Effect":"Deny","Principal":"*","Action":["s3:PutObject"],"Resource":"arn:aws:s3:::%s/*","Condition":{"Bool":{"aws:SecureTransport":false}}}]} -''' % bucket.name - self.assertTrue(bucket.set_policy(policy, headers={'content-type':'application/json'})) - - try: - upload = upload_multipart(bucket, key_name, stringio_parts) - self.fail() - except S3ResponseError as e: - self.assertEqual(e.status, 403) - self.assertEqual(e.reason, 'Forbidden') - -class ObjectMetadataTest(S3ApiVerificationTestBase): - "Test object metadata, e.g. Content-Encoding, x-amz-meta-*, for PUT/GET" - - metadata = { - "Content-Disposition": 'attachment; filename="metaname.txt"', - "Content-Encoding": 'identity', - "Cache-Control": "max-age=3600", - "Expires": "Tue, 19 Jan 2038 03:14:07 GMT", - "mtime": "1364742057", - "UID": "0", - "with-hypen": "1", - "space-in-value": "abc xyz"} - - updated_metadata = { - "Content-Disposition": 'attachment; filename="newname.txt"', - "Cache-Control": "private", - "Expires": "Tue, 19 Jan 2038 03:14:07 GMT", - "mtime": "2222222222", - "uid": "0", - "space-in-value": "ABC XYZ", - "new-entry": "NEW"} - - def test_normal_object_metadata(self): - key_name = str(uuid.uuid4()) - bucket = self.conn.create_bucket(self.bucket_name) - key = Key(bucket, key_name) - for k,v in self.metadata.items(): - key.set_metadata(k, v) - key.set_contents_from_string("test_normal_object_metadata") - self.assert_metadata(bucket, key_name) - self.change_metadata(bucket, key_name) - self.assert_updated_metadata(bucket, key_name) - - def test_mp_object_metadata(self): - key_name = str(uuid.uuid4()) - bucket = self.conn.create_bucket(self.bucket_name) - upload = upload_multipart(bucket, key_name, [StringIO("part1")], - metadata=self.metadata) - self.assert_metadata(bucket, key_name) - self.change_metadata(bucket, key_name) - self.assert_updated_metadata(bucket, key_name) - - def assert_metadata(self, bucket, key_name): - key = Key(bucket, key_name) - key.get_contents_as_string() - - self.assertEqual(key.content_disposition, - 'attachment; filename="metaname.txt"') - self.assertEqual(key.content_encoding, "identity") - self.assertEqual(key.cache_control, "max-age=3600") - # TODO: Expires header can be accessed by boto? - # self.assertEqual(key.expires, "Tue, 19 Jan 2038 03:14:07 GMT") - self.assertEqual(key.get_metadata("mtime"), "1364742057") - self.assertEqual(key.get_metadata("uid"), "0") - self.assertEqual(key.get_metadata("with-hypen"), "1") - self.assertEqual(key.get_metadata("space-in-value"), "abc xyz") - # x-amz-meta-* headers should be normalized to lowercase - self.assertEqual(key.get_metadata("Mtime"), None) - self.assertEqual(key.get_metadata("MTIME"), None) - self.assertEqual(key.get_metadata("Uid"), None) - self.assertEqual(key.get_metadata("UID"), None) - self.assertEqual(key.get_metadata("With-Hypen"), None) - self.assertEqual(key.get_metadata("Space-In-Value"), None) - - def change_metadata(self, bucket, key_name): - key = Key(bucket, key_name) - key.copy(bucket.name, key_name, self.updated_metadata) - - def assert_updated_metadata(self, bucket, key_name): - key = Key(bucket, key_name) - key.get_contents_as_string() - - # unchanged - self.assertEqual(key.get_metadata("uid"), "0") - # updated - self.assertEqual(key.content_disposition, - 'attachment; filename="newname.txt"') - self.assertEqual(key.cache_control, "private") - self.assertEqual(key.get_metadata("mtime"), "2222222222") - self.assertEqual(key.get_metadata("space-in-value"), "ABC XYZ") - # removed - self.assertEqual(key.content_encoding, None) - self.assertEqual(key.get_metadata("with-hypen"), None) - # inserted - self.assertEqual(key.get_metadata("new-entry"), "NEW") - # TODO: Expires header can be accessed by boto? - # self.assertEqual(key.expires, "Tue, 19 Jan 2038 03:14:07 GMT") - -class ContentMd5Test(S3ApiVerificationTestBase): - def test_catches_bad_md5(self): - '''Make sure Riak CS catches a bad content-md5 header''' - key_name = str(uuid.uuid4()) - bucket = self.conn.create_bucket(self.bucket_name) - key = Key(bucket, key_name) - s = StringIO('not the real content') - x = compute_md5(s) - with self.assertRaises(S3ResponseError): - key.set_contents_from_string('this is different from the md5 we calculated', md5=x) - - def test_bad_md5_leaves_old_object_alone(self): - '''Github #705 Regression test: - Make sure that overwriting an object using a bad md5 - simply leaves the old version in place.''' - key_name = str(uuid.uuid4()) - bucket = self.conn.create_bucket(self.bucket_name) - value = 'good value' - - good_key = Key(bucket, key_name) - good_key.set_contents_from_string(value) - - bad_key = Key(bucket, key_name) - s = StringIO('not the real content') - x = compute_md5(s) - try: - bad_key.set_contents_from_string('this is different from the md5 we calculated', md5=x) - except S3ResponseError: - pass - self.assertEqual(good_key.get_contents_as_string(), value) - -class SimpleCopyTest(S3ApiVerificationTestBase): - - def create_test_object(self): - bucket = self.conn.create_bucket(self.bucket_name) - k = Key(bucket) - k.key = self.key_name - k.set_contents_from_string(self.data) - self.assertEqual(k.get_contents_as_string(), self.data) - self.assertIn(self.key_name, - [k.key for k in bucket.get_all_keys()]) - return k - - def test_put_copy_object(self): - k = self.create_test_object() - - target_bucket_name = str(uuid.uuid4()) - target_key_name = str(uuid.uuid4()) - target_bucket = self.conn.create_bucket(target_bucket_name) - - target_bucket.copy_key(target_key_name, self.bucket_name, self.key_name) - - target_key = Key(target_bucket) - target_key.key = target_key_name - self.assertEqual(target_key.get_contents_as_string(), self.data) - self.assertIn(target_key_name, - [k.key for k in target_bucket.get_all_keys()]) - - def test_put_copy_object_from_mp(self): - bucket = self.conn.create_bucket(self.bucket_name) - (upload, result) = upload_multipart(bucket, self.key_name, [StringIO(self.data)]) - - target_bucket_name = str(uuid.uuid4()) - target_key_name = str(uuid.uuid4()) - target_bucket = self.conn.create_bucket(target_bucket_name) - - target_bucket.copy_key(target_key_name, self.bucket_name, self.key_name) - - target_key = Key(target_bucket) - target_key.key = target_key_name - self.assertEqual(target_key.get_contents_as_string(), self.data) - self.assertIn(target_key_name, - [k.key for k in target_bucket.get_all_keys()]) - - def test_upload_part_from_non_mp(self): - k = self.create_test_object() - - target_bucket_name = str(uuid.uuid4()) - target_key_name = str(uuid.uuid4()) - target_bucket = self.conn.create_bucket(target_bucket_name) - start_offset=0 - end_offset=9 - target_bucket = self.conn.create_bucket(target_bucket_name) - upload = target_bucket.initiate_multipart_upload(target_key_name) - upload.copy_part_from_key(self.bucket_name, self.key_name, part_num=1, - start=start_offset, end=end_offset) - upload.complete_upload() - print([k.key for k in target_bucket.get_all_keys()]) - - target_key = Key(target_bucket) - target_key.key = target_key_name - self.assertEqual(self.data[start_offset:(end_offset+1)], - target_key.get_contents_as_string()) - - def test_upload_part_from_mp(self): - bucket = self.conn.create_bucket(self.bucket_name) - key_name = str(uuid.uuid4()) - (upload, result) = upload_multipart(bucket, key_name, [StringIO(self.data)]) - - target_bucket_name = str(uuid.uuid4()) - target_bucket = self.conn.create_bucket(target_bucket_name) - start_offset=0 - end_offset=9 - upload2 = target_bucket.initiate_multipart_upload(key_name) - upload2.copy_part_from_key(self.bucket_name, self.key_name, part_num=1, - start=start_offset, end=end_offset) - upload2.complete_upload() - - target_key = Key(target_bucket, key_name) - self.assertEqual(self.data[start_offset:(end_offset+1)], - target_key.get_contents_as_string()) - - def test_put_copy_from_non_existing_key_404(self): - bucket = self.conn.create_bucket(self.bucket_name) - - target_bucket_name = str(uuid.uuid4()) - target_key_name = str(uuid.uuid4()) - target_bucket = self.conn.create_bucket(target_bucket_name) - try: - target_bucket.copy_key(target_key_name, self.bucket_name, 'not_existing') - self.fail() - except S3ResponseError as e: - print e - self.assertEqual(e.status, 404) - self.assertEqual(e.reason, 'Not Found') - -if __name__ == "__main__": - unittest.main(verbosity=2) diff --git a/client_tests/python/boto_tests/file_generator.py b/client_tests/python/boto_tests/file_generator.py deleted file mode 100755 index 65c11adf8..000000000 --- a/client_tests/python/boto_tests/file_generator.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python -## --------------------------------------------------------------------- -## -## Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -## -## This file is provided to you under the Apache License, -## Version 2.0 (the "License"); you may not use this file -## except in compliance with the License. You may obtain -## a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, -## software distributed under the License is distributed on an -## "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -## KIND, either express or implied. See the License for the -## specific language governing permissions and limitations -## under the License. -## -## --------------------------------------------------------------------- - -import os - -class FileGenerator(object): - def __init__(self, gen_fn, size=None): - """ - - Create a read-only file-like object that is driven by a generator. - The file can only be read once or sought to the head. The file also - supports a limited usage of `tell`, mostly just to appease boto's use of - those methods to determine file-size. The optional `size` parameter - _must_ be used if you intend to use this object with boto. - To support seek to the head, generators are created from `gen_fn` - once per seek to the head. The `gen_fn` must return the same - generators for every call. Currently, the total length of the generated - values from generators must agree with `size`, no automatic truncation - will be performed for you. - - Here are some example uses: - - ``` - # create a 1kb string first, to be used by our - # file generator - s = ''.join('a' for _ in xrange(1024)) - # the generator to drive the file, 1MB (1KB * 1024) - def fn(): - return (s for _ in xrange(1024)) - fg = FileGenerator(fn, 1024 ** 2) - - # now remember, each FileGenerator instance can only - # be used once, so pretend a new one is created for each - # of the following example: - - # copy it to 'real' file - import shutil - shutil.copyfileobj(fg, open('destination-file', 'w')) - - # compute the md5 - import md5 - - m = md5.new() - go = True - while go: - byte = fg.read(8196) - if byte: - m.update(byte) - else: - go = False - print m.hexdigest() - """ - self.gen_fn = gen_fn - self.gen = gen_fn() - self.size = size - self.pos = 0 - - self.closed = False - self.buf = None - - def close(self): - self.closed = True - return None - - def tell(self): - return self.pos - - def seek(self, offset, whence=None): - if offset != 0: - raise ValueError('offset must be ZERO') - - if whence == os.SEEK_END and offset == 0: - self.pos = self.size - else: - self.pos = 0 - self.gen = self.gen_fn() - return None - - def read(self, max_size=None): - if not max_size: - res = self.buf + ''.join(list(self.gen)) - self.pos = len(res) - self.buf = '' - return res - else: - if self.buf: - res = self.buf[:max_size] - self.buf = self.buf[max_size:] - self.pos += len(res) - return res - else: - try: - data = self.gen.next() - res = data[:max_size] - self.buf = data[max_size:] - self.pos += len(res) - return res - except StopIteration: - return '' - diff --git a/client_tests/python/boto_tests/requirements.txt b/client_tests/python/boto_tests/requirements.txt deleted file mode 100644 index 1aa451715..000000000 --- a/client_tests/python/boto_tests/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -boto==2.35.1 - diff --git a/client_tests/python/ceph_tests/Makefile b/client_tests/python/ceph_tests/Makefile deleted file mode 100644 index 9d022ab7c..000000000 --- a/client_tests/python/ceph_tests/Makefile +++ /dev/null @@ -1,34 +0,0 @@ -.PHONY: test clean env all -.DEFAULT: all - -DEPS = env/lib/python2.7/site-packages -BIN = env/bin -NOSE_ARGS = s3tests.functional.test_s3 -A -NOSE_ATTR = (method != "post") -NOSE_ATTR += and (resource != "bucket.log") -NOSE_ATTR += and (not versioning) -NOSE_ATTR += and (not cors) -NOSE_ATTR += and (not fails_on_aws) -NOSE_ATTR += and (not fails_with_subdomain) -NOSE_ATTR += and (not fails_on_rcs) - -all: test - -s3-tests: - @git clone --quiet https://github.com/basho/s3-tests \ - -b riakcs-2.1 - -env: - @cd s3-tests && virtualenv env - -$(DEPS) $(BIN): s3-tests env - @cd s3-tests && env/bin/pip install -r requirements.txt - -test: $(DEPS) $(BIN) - @echo $(CS_HTTP_PORT) - @./s3_conf.sh > s3-tests/s3test.conf - cd s3-tests && \ - S3TEST_CONF=s3test.conf env/bin/nosetests $(NOSE_ARGS) '$(NOSE_ATTR)' - -clean: - @rm -rf s3-tests diff --git a/client_tests/python/ceph_tests/s3_conf.sh b/client_tests/python/ceph_tests/s3_conf.sh deleted file mode 100755 index 05de70390..000000000 --- a/client_tests/python/ceph_tests/s3_conf.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/sh - -usage() -{ - echo "$0 [HOST PORT]" -} - -if [ -z "$1" ] -then - HOST="localhost" -else - HOST=$1 -fi - -if [ -z "$2" ] -then - PORT=${CS_HTTP_PORT:=8080} -else - PORT=$2 -fi - -PARSECREDS="import json -import sys -user_input = sys.stdin.read() -user_json = json.loads(user_input) -sys.stdout.write(user_json['key_id'] + ' ' + user_json['key_secret'] + ' ' + user_json['id']) -sys.stdout.flush()" - -NAME1=`uuidgen` -EMAIL1="$NAME1@s3-test.basho" -CREDS1=`curl -s --data "{\"email\":\"$EMAIL1\", \"name\":\"$NAME1\"}" -H 'Content-Type: application/json' "http://$HOST:$PORT/riak-cs/user" | python -c "$PARSECREDS"` -KEYID1=`echo $CREDS1 | awk '{print $1}'` -KEYSECRET1=`echo $CREDS1 | awk '{print $2}'` -CANONICALID1=`echo $CREDS1 | awk '{print $3}'` - -NAME2=`uuidgen` -EMAIL2="$NAME2@s3-test.basho" -CREDS2=`curl -s --data "{\"email\":\"$EMAIL2\", \"name\":\"$NAME2\"}" -H 'Content-Type: application/json' "http://$HOST:$PORT/riak-cs/user" | python -c "$PARSECREDS"` -KEYID2=`echo $CREDS2 | awk '{print $1}'` -KEYSECRET2=`echo $CREDS2 | awk '{print $2}'` -CANONICALID2=`echo $CREDS2 | awk '{print $3}'` - -CONFIG="[DEFAULT] -## this section is just used as default for all the \"s3 *\" -## sections, you can place these variables also directly there - -## replace with e.g. \"localhost\" to run against local software -host = s3.amazonaws.com - -## uncomment the port to use something other than 80 -port = 80 - -proxy = $HOST -proxy_port = $PORT - -## say \"no\" to disable TLS -is_secure = no - -api_name = us-east-1 - -[fixtures] -## all the buckets created will start with this prefix; -## {random} will be filled with random characters to pad -## the prefix to 30 characters long, and avoid collisions -bucket prefix = cs-s3-tests-{random}- - -[s3 main] -## the tests assume two accounts are defined, \"main\" and \"alt\". - -## user_id is a 64-character hexstring -user_id = $CANONICALID1 - -## display name typically looks more like a unix login, \"jdoe\" etc -display_name = $NAME1 - -## replace these with your access keys -access_key = $KEYID1 -secret_key = $KEYSECRET1 - -[s3 alt] -## another user account, used for ACL-related tests -user_id = $CANONICALID2 -display_name = $NAME2 -## the \"alt\" user needs to have email set, too -email = $EMAIL2 -access_key = $KEYID2 -secret_key = $KEYSECRET2" - -echo "$CONFIG" diff --git a/client_tests/ruby/.gitignore b/client_tests/ruby/.gitignore deleted file mode 100644 index 00cbe9d10..000000000 --- a/client_tests/ruby/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -Gemfile.lock -conf/s3.yml -vendor/ -.bundle/ diff --git a/client_tests/ruby/Gemfile b/client_tests/ruby/Gemfile deleted file mode 100644 index d1f7226f4..000000000 --- a/client_tests/ruby/Gemfile +++ /dev/null @@ -1,10 +0,0 @@ -source "https://rubygems.org" - -gem "aws-sdk", "~>1.8.5" -gem "uuid", "~>2.3.6" -gem "rake", "~>10.0.2" -gem "httparty", "~>0.10.2" - -group :test do - gem "rspec", "~>2.11.0" -end diff --git a/client_tests/ruby/README.md b/client_tests/ruby/README.md deleted file mode 100644 index c266679d5..000000000 --- a/client_tests/ruby/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Riak CS AWS Ruby SDK Tests - -## Dependencies - -Install [Bundler](http://gembundler.com/): - -```bash -$ gem install bundler -``` - -Install dependencies: - -```bash -$ bundle --gemfile client_tests/ruby/Gemfile --path vendor -``` - -## Execution - -```bash -$ cd client_tests/ruby && bundle exec rake spec -``` diff --git a/client_tests/ruby/Rakefile b/client_tests/ruby/Rakefile deleted file mode 100644 index 2abe082e3..000000000 --- a/client_tests/ruby/Rakefile +++ /dev/null @@ -1,11 +0,0 @@ -# Rakefile -require 'rspec/core/rake_task' - -RSpec::Core::RakeTask.new(:spec) do |config| - config.rspec_opts = '--color' -end - -task :default => :spec - -# spec/spec_helper.rb -require 'rspec/autorun' diff --git a/client_tests/ruby/spec/helper.rb b/client_tests/ruby/spec/helper.rb deleted file mode 100644 index 4aa2b954b..000000000 --- a/client_tests/ruby/spec/helper.rb +++ /dev/null @@ -1,41 +0,0 @@ -require 'httparty' -require 'json' -require 'uuid' -require 'tempfile' - -def create_user(name) - response = HTTParty.put("#{cs_uri}/riak-cs/user", - :body => { - :name => name, - :email => "#{name}@example.com"}.to_json, - :headers => { - "Content-Type" => "application/json"}) - json_body = JSON.parse(response.body) - return json_body['key_id'], json_body['key_secret'] -end - -def s3_conf - key_id, key_secret = create_user(UUID::generate) - { - access_key_id: key_id, - secret_access_key: key_secret, - proxy_uri: cs_uri, - use_ssl: false, - http_read_timeout: 2000, - max_retries: 0 - } -end - -def cs_uri - "http://localhost:#{cs_port}" -end - -def cs_port - ENV['CS_HTTP_PORT'] || 8080 -end - -def new_mb_temp_file(size) - temp = Tempfile.new 'riakcs-test' - (size*1024*1024).times {|i| temp.write 0} - temp -end diff --git a/client_tests/ruby/spec/s3_spec.rb b/client_tests/ruby/spec/s3_spec.rb deleted file mode 100644 index d46825897..000000000 --- a/client_tests/ruby/spec/s3_spec.rb +++ /dev/null @@ -1,119 +0,0 @@ -## --------------------------------------------------------------------- -## -## Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -## -## This file is provided to you under the Apache License, -## Version 2.0 (the "License"); you may not use this file -## except in compliance with the License. You may obtain -## a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, -## software distributed under the License is distributed on an -## "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -## KIND, either express or implied. See the License for the -## specific language governing permissions and limitations -## under the License. -## -## --------------------------------------------------------------------- - -require 'aws-sdk' -require 'uuid' -require 'yaml' -require 'helper' - -class AWS::S3::AccessControlList::Grant - def ==(other) - @grantee.to_s == other.grantee.to_s and @permission.to_s == other.permission.to_s - end -end - -describe AWS::S3 do - let(:s3) { AWS::S3.new( s3_conf ) } - let(:bucket_name) { "aws-sdk-test-" + UUID::generate } - let(:object_name) { "key-" + UUID::generate } - let(:grant_public_read) { - acl = AWS::S3::AccessControlList.new - acl.grant(:read).to(:uri => 'http://acs.amazonaws.com/groups/global/AllUsers') - acl.grants.first - } - - after :each do - begin - if s3.buckets[bucket_name].exists? - if s3.buckets[bucket_name].objects.count > 0 - s3.buckets[bucket_name].objects.each &:delete - end - s3.buckets[bucket_name].delete - end - rescue Exception => e - puts e - end - end - - describe "when there is no bucket" do - it "should not find the bucket." do - s3.buckets[bucket_name].exists?.should == false - end - - it "should fail on delete operation." do - lambda{s3.buckets[bucket_name].delete}.should raise_error - end - - it "should fail on get acl operation" do - lambda{s3.buckets[bucket_name].acl}.should raise_error - end - end - - describe "when there is a bucket" do - it "should be able to create and delete a bucket" do - s3.buckets.create(bucket_name).should be_kind_of(AWS::S3::Bucket) - s3.buckets[bucket_name].exists?.should == true - - lambda{s3.buckets[bucket_name].delete}.should_not raise_error - s3.buckets[bucket_name].exists?.should == false - end - - it "should be able to list buckets" do - s3.buckets.create(bucket_name).should be_kind_of(AWS::S3::Bucket) - s3.buckets.count.should > 0 - end - - it "should be able to put, get and delete object" do - s3.buckets.create(bucket_name).should be_kind_of(AWS::S3::Bucket) - - s3.buckets[bucket_name].objects[object_name].write('Rakefile') - s3.buckets[bucket_name].objects[object_name].exists?.should == true - - s3.buckets[bucket_name].objects[object_name].read.should == 'Rakefile' - - s3.buckets[bucket_name].objects[object_name].delete - s3.buckets[bucket_name].objects[object_name].exists?.should == false - end - - it "should be able to put and get bucket ACL" do - s3.buckets.create(bucket_name, :acl => :public_read).should be_kind_of(AWS::S3::Bucket) - s3.buckets[bucket_name].acl.grants.include?(grant_public_read).should == true - end - - it "should be able to put and get object ACL" do - s3.buckets.create(bucket_name).should be_kind_of(AWS::S3::Bucket) - s3.buckets[bucket_name].objects[object_name].write('Rakefile', :acl => :public_read) - - s3.buckets[bucket_name].objects[object_name].acl.grants.include?(grant_public_read).should == true - end - - it "should be able to put object using multipart upload" do - s3.buckets.create(bucket_name).should be_kind_of(AWS::S3::Bucket) - - temp = new_mb_temp_file 6 # making 6MB file - s3.buckets[bucket_name].objects[object_name].write( - :file => temp.path, - :multipart_threshold => 1024 * 1024, - ) - s3.buckets[bucket_name].objects[object_name].exists?.should == true - temp.close - end - end -end diff --git a/config/sys.config.defaults b/config/sys.config.defaults new file mode 100644 index 000000000..b302a17c3 --- /dev/null +++ b/config/sys.config.defaults @@ -0,0 +1,46 @@ +PLATFORM_LOG_DIR="./log" +PLATFORM_LIB_DIR="./lib" +PLATFORM_ETC_DIR="./etc" +PLATFORM_BIN_DIR="./bin" +ADMIN_KEY="admin-key" +RIAK_HOST="127.0.0.1" +RIAK_PORT=8087 +RCS_LISTENER_IP="0.0.0.0" +RCS_LISTENER_PORT=8080 +STANCHION_HOSTING_MODE=auto +STANCHION_LISTENER_IP="0.0.0.0" +STANCHION_LISTENER_PORT=8085 +STANCHION_PORT=8085 +STANCHION_SSL=false +STANCHION_SUBNET="192.168.0.0" +STANCHION_NETMASK="255.255.255.255" +TUSSLE_VOSS_RIAK_HOST=auto +AUTH_V4_ENABLED=true +AUTH_BYPASS=false + +SUPERCLUSTER_WEIGHT_REFRESH_INTERVAL=900 +PROXY_GET=false +USAGE_REQUEST_LIMIT=744 +STORAGE_ARCHIVE_PERIOD=86400 +ACCESS_ARCHIVER_MAX_WORKERS=2 +ACCESS_ARCHIVER_MAX_BACKLOG=2 +ACCESS_ARCHIVE_PERIOD=3600 +ACCESS_LOG_FLUSH_SIZE=1000000 +ACCESS_LOG_FLUSH_FACTOR=1 +FAST_USER_GET=false +ACTIVE_DELETE_THRESHOLD=0 +GC_BATCH_SIZE=1000 +GC_MAX_WORKERS=2 +GC_PAGINATED_INDEXES=true +GC_RETRY_INTERVAL=21600 +GC_INTERVAL=900 +LEEWAY_SECONDS=86400 +MAX_SCHEDULED_DELETE_MANIFESTS=50 +TRUST_X_FORWARDED_FOR=false +MAX_KEY_LENGTH=1024 +MAX_BUCKETS_PER_USER=100 +S3_ROOT_HOST="s3.amazonaws.com" +ANONYMOUS_USER_CREATION=false +IAM_CREATE_USER_DEFAULT_EMAIL_HOST="my-riak-cs-megacorp.com" + +KERNEL_LOGGER_LEVEL=info diff --git a/config/sys.docker.config.src b/config/sys.docker.config.src new file mode 100644 index 000000000..253757d2b --- /dev/null +++ b/config/sys.docker.config.src @@ -0,0 +1,78 @@ +[{riak_cs, + [{supercluster_weight_refresh_interval, ${SUPERCLUSTER_WEIGHT_REFRESH_INTERVAL}}, + {platform_log_dir, ${PLATFORM_LOG_DIR}}, + {platform_lib_dir, ${PLATFORM_LIB_DIR}}, + {platform_etc_dir, ${PLATFORM_ETC_DIR}}, + {platform_bin_dir, ${PLATFORM_BIN_DIR}}, + {dtrace_support, false}, + {proxy_get, ${PROXY_GET}}, + {cs_version,030205}, %% to match values in include/riak_cs.hrl and rel/vars.config and others in rel/pkg/ + {usage_request_limit, ${USAGE_REQUEST_LIMIT}}, + {storage_archive_period, ${STORAGE_ARCHIVE_PERIOD}}, + {access_archiver_max_workers, ${ACCESS_ARCHIVER_MAX_WORKERS}}, + {access_archiver_max_backlog, ${ACCESS_ARCHIVER_MAX_BACKLOG}}, + {access_archive_period, ${ACCESS_ARCHIVE_PERIOD}}, + {access_log_flush_size, ${ACCESS_LOG_FLUSH_SIZE}}, + {access_log_flush_factor, ${ACCESS_LOG_FLUSH_FACTOR}}, + {fast_user_get, ${FAST_USER_GET}}, + {active_delete_threshold, ${ACTIVE_DELETE_THRESHOLD}}, + {gc_batch_size, ${GC_BATCH_SIZE}}, + {gc_max_workers, ${GC_MAX_WORKERS}}, + {gc_paginated_indexes, ${GC_PAGINATED_INDEXES}}, + {gc_retry_interval, ${GC_RETRY_INTERVAL}}, + {gc_interval, ${GC_INTERVAL}}, + {leeway_seconds, ${LEEWAY_SECONDS}}, + {max_scheduled_delete_manifests, ${MAX_SCHEDULED_DELETE_MANIFESTS}}, + {trust_x_forwarded_for, ${TRUST_X_FORWARDED_FOR}}, + {max_key_length, ${MAX_KEY_LENGTH}}, + {max_buckets_per_user, ${MAX_BUCKETS_PER_USER}}, + {s3_root_host, ${S3_ROOT_HOST}}, + {auth_v4_enabled, ${AUTH_V4_ENABLED}}, + {auth_bypass, ${AUTH_BYPASS}}, + {admin_key, ${ADMIN_KEY}}, + {anonymous_user_creation, ${ANONYMOUS_USER_CREATION}}, + {riak_host, {${RIAK_HOST}, ${RIAK_PORT}}}, + {listener, {${RCS_LISTENER_IP}, ${RCS_LISTENER_PORT}}}, + {stanchion_hosting_mode, ${STANCHION_HOSTING_MODE}}, + {stanchion_ssl, ${STANCHION_SSL}}, + {stanchion_ssl_certfile, ${STANCHION_SSL_CERTFILE}}, + {stanchion_ssl_keyfile, ${STANCHION_SSL_KEYFILE}}, + {stanchion_subnet, ${STANCHION_SUBNET}, + {stanchion_netmask, ${STANCHION_NETMASK}, + {tussle_voss_riak_host, ${TUSSLE_VOSS_RIAK_HOST}}, + {iam_create_user_default_email_host, ${IAM_CREATE_USER_DEFAULT_EMAIL_HOST}}, + {connection_pools,[{request_pool,{128,0}},{bucket_list_pool,{5,0}}]}]}, + {sasl,[{sasl_error_logger,false}]}, + {webmachine, + [{server_name,"Riak CS"}, + {log_handlers, + [{webmachine_access_log_handler,[${PLATFORM_LOG_DIR}"/log"]}, + {riak_cs_access_log_handler,[]}]}]}, + {kernel, + [{logger_level, ${KERNEL_LOGGER_LEVEL}}, + {logger, + [{handler,default,logger_std_h, + #{config => + #{type => standard_io}, + filters => + [{no_sasl, + {fun logger_filters:domain/2, + {stop,super,[otp,sasl]}}}], + formatter => + {logger_formatter, + #{template => + [time," [",level,"] ",pid,"@",mfa,":",line," ", + msg,"\n"]}}}}, + {handler,sasl,logger_std_h, + #{config => + #{type => standard_error}, + filter_default => stop, + filters => + [{sasl_here, + {fun logger_filters:domain/2, + {log,equal,[otp,sasl]}}}], + formatter => + {logger_formatter, + #{legacy_header => true,single_line => false}}}}]}]}, + {syslogger,[{log_opts,[pid]},{ident,"riak_cs"}]} +]. diff --git a/config/vm.args b/config/vm.args new file mode 100644 index 000000000..809ddd7f8 --- /dev/null +++ b/config/vm.args @@ -0,0 +1,2 @@ +-sname riak-cs +-setcookie riak diff --git a/dialyzer.ignore-warnings.ee b/dialyzer.ignore-warnings.ee deleted file mode 100644 index 776dcf372..000000000 --- a/dialyzer.ignore-warnings.ee +++ /dev/null @@ -1,24 +0,0 @@ -# Errors -riak_cs_block_server.erl:324: The pattern Success = {'ok', _} can never match the type {'error',_} -riak_cs_block_server.erl:361: The pattern {'ok', RiakObject} can never match the type {'error',_} -riak_cs_pbc.erl:58: The variable _ can never match since previous clauses completely covered the type 'pong' -riak_cs_pbc.erl:175: The pattern {'ok', ClusterID} can never match the type {'error',_} -# Warnings -Unknown functions: - app_helper:get_prop_or_env/3 - app_helper:get_prop_or_env/4 - dtrace:init/0 - riak:stop/1 - riak_core_bucket:get_bucket/1 - riak_cs_dummy_reader:get_manifest/1 - riak_cs_dummy_reader:start_link/1 - riak_object:bucket/1 - riak_object:get_contents/1 - riak_object:key/1 - riakc_pb_socket_fake:start_link/0 -Unknown types: - riak_kv_backend:fold_buckets_fun/0 - riak_kv_backend:fold_keys_fun/0 - riak_kv_backend:fold_objects_fun/0 - riak_object:bucket/0 - riak_object:key/0 diff --git a/dialyzer.ignore-warnings.oss b/dialyzer.ignore-warnings.oss deleted file mode 100644 index 6437a1c85..000000000 --- a/dialyzer.ignore-warnings.oss +++ /dev/null @@ -1,10 +0,0 @@ -# Warnings -Unknown functions: - riak_cs_multibag:choose_bag_id/1 - riak_cs_multibag:list_pool/0 - riak_cs_multibag:list_pool/1 - riak_cs_multibag:pool_name_for_bag/2 - riak_cs_multibag:pool_specs/1 - riak_cs_multibag:process_specs/0 - riak_repl_pb_api:get/5 - riak_repl_pb_api:get_clusterid/2 diff --git a/include/list_objects.hrl b/include/list_objects.hrl deleted file mode 100644 index 9c5ff7dac..000000000 --- a/include/list_objects.hrl +++ /dev/null @@ -1,104 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% see also: http://docs.amazonwebservices.com/AmazonS3/latest/API/RESTBucketGET.html -%% non mandatory keys have `| undefined' as a -%% type option. -%% -%% This record actually does not need to be versioned, -%% as it's never persisted. --record(list_objects_request_v1, { - %% the name of the bucket - name :: binary(), - - %% how many keys to return in the response - max_keys :: non_neg_integer(), - - %% a 'starts-with' parameter - prefix :: binary() | undefined, - - %% a binary to group keys by - delimiter :: binary() | undefined, - - %% the key to start with - marker :: binary() | undefined}). --type list_object_request() :: #list_objects_request_v1{}. --define(LOREQ, #list_objects_request_v1). - --type next_marker() :: 'undefined' | binary(). - -%% This record actually does not need to be versioned, -%% as it's never persisted. --record(list_objects_response_v1, { - %% Params just echoed back from the request -------------------------- - - %% the name of the bucket - name :: binary(), - - %% how many keys were requested to be - %% returned in the response - max_keys :: non_neg_integer(), - - %% a 'starts-with' parameter - prefix :: binary() | undefined, - - %% a binary to group keys by - delimiter :: binary() | undefined, - - %% the marker used in the _request_ - marker :: binary() | undefined, - - %% the (optional) marker to use for pagination - %% in the _next_ request - next_marker :: next_marker(), - - %% The actual response ----------------------------------------------- - is_truncated :: boolean(), - - contents :: list(list_objects_key_content()), - - common_prefixes :: list_objects_common_prefixes()}). --type list_object_response() :: #list_objects_response_v1{}. --define(LORESP, #list_objects_response_v1). - --record(list_objects_key_content_v1, { - key :: binary(), - last_modified :: term(), - etag :: binary(), - size :: non_neg_integer(), - owner :: list_objects_owner(), - storage_class :: binary()}). --type list_objects_key_content() :: #list_objects_key_content_v1{}. --define(LOKC, #list_objects_key_content_v1). - --record(list_objects_owner_v1, { - id :: binary(), - display_name :: binary()}). --type list_objects_owner() :: #list_objects_owner_v1{}. - --type list_objects_common_prefixes() :: list(binary()). - --define(LIST_OBJECTS_CACHE, list_objects_cache). --define(ENABLE_CACHE, true). --define(CACHE_TIMEOUT, timer:minutes(15)). --define(MIN_KEYS_TO_CACHE, 2000). --define(MAX_CACHE_BYTES, 104857600). % 100MB --define(KEY_LIST_MULTIPLIER, 1.1). --define(FOLD_OBJECTS_FOR_LIST_KEYS, true). diff --git a/include/riak_cs.hrl b/include/riak_cs.hrl deleted file mode 100644 index b112b7216..000000000 --- a/include/riak_cs.hrl +++ /dev/null @@ -1,526 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --define(MANIFEST, #lfs_manifest_v3). - --define(ACL, #acl_v2). --define(RCS_BUCKET, #moss_bucket_v1). --define(MOSS_USER, #rcs_user_v2). --define(RCS_USER, #rcs_user_v2). --define(MULTIPART_MANIFEST, #multipart_manifest_v1). --define(MULTIPART_MANIFEST_RECNAME, multipart_manifest_v1). --define(PART_MANIFEST, #part_manifest_v1). --define(PART_MANIFEST_RECNAME, part_manifest_v1). --define(MULTIPART_DESCR, #multipart_descr_v1). --define(COMPRESS_TERMS, false). --define(PART_DESCR, #part_descr_v1). - --record(moss_user, { - name :: string(), - key_id :: string(), - key_secret :: string(), - buckets = []}). - --record(moss_user_v1, { - name :: string(), - display_name :: string(), - email :: string(), - key_id :: string(), - key_secret :: string(), - canonical_id :: string(), - buckets=[] :: [cs_bucket()]}). - --record(rcs_user_v2, { - name :: string(), - display_name :: string(), - email :: string(), - key_id :: string(), - key_secret :: string(), - canonical_id :: string(), - buckets=[] :: [cs_bucket()], - status=enabled :: enabled | disabled}). --type moss_user() :: #rcs_user_v2{} | #moss_user_v1{}. --type rcs_user() :: #rcs_user_v2{} | #moss_user_v1{}. - --record(moss_bucket, { - name :: string(), - creation_date :: term(), - acl :: acl()}). - --record(moss_bucket_v1, { - name :: string() | binary(), - last_action :: created | deleted, - creation_date :: string(), - modification_time :: erlang:timestamp(), - acl :: acl()}). - --type cs_bucket() :: #moss_bucket_v1{}. --type bucket_operation() :: create | delete | update_acl | update_policy - | delete_policy. --type bucket_action() :: created | deleted. - --record(context, {start_time :: erlang:timestamp(), - auth_bypass :: atom(), - user :: undefined | moss_user(), - user_object :: riakc_obj:riakc_obj(), - bucket :: binary(), - acl :: 'undefined' | acl(), - requested_perm :: acl_perm(), - riak_client :: riak_client(), - rc_pool :: atom(), % pool name which riak_client belongs to - auto_rc_close = true :: boolean(), - submodule :: atom(), - exports_fun :: function(), - auth_module :: atom(), - response_module :: atom(), - policy_module :: atom(), - %% Key for API rate and latency stats. - %% If `stats_prefix' or `stats_key' is `no_stats', no stats - %% will be gathered by riak_cs_wm_common. - %% The prefix is defined by `stats_prefix()' callback of sub-module. - %% If sub-module provides only `stats_prefix' (almost the case), - %% stats key is [Prefix, HttpMethod]. Otherwise, sum-module - %% can set specific `stats_key' by any callback that returns - %% this context. - stats_prefix = no_stats :: atom(), - stats_key=prefix_and_method :: prefix_and_method | - no_stats | - riak_cs_stats:key(), - local_context :: term(), - api :: atom() - }). - --record(key_context, {manifest :: 'notfound' | lfs_manifest(), - upload_id :: 'undefined' | binary(), - part_number :: 'undefined' | integer(), - part_uuid :: 'undefined' | binary(), - get_fsm_pid :: pid(), - putctype :: string(), - bucket :: binary(), - bucket_object :: undefined | notfound | riakc_obj:riakc_obj(), - key :: list(), - owner :: 'undefined' | string(), - size :: non_neg_integer(), - content_md5 :: 'undefined' | binary(), - update_metadata=false :: boolean()}). - --type acl_perm() :: 'READ' | 'WRITE' | 'READ_ACP' | 'WRITE_ACP' | 'FULL_CONTROL'. --type acl_perms() :: [acl_perm()]. --type group_grant() :: 'AllUsers' | 'AuthUsers'. --type acl_grantee() :: {DisplayName :: string(), - CanonicalID :: string()} | - group_grant(). --type acl_grant() :: {acl_grantee(), acl_perms()}. -%% acl_v1 owner fields: {DisplayName, CanonicalId} --type acl_owner2() :: {string(), string()}. -%% acl_owner3: {display name, canonical id, key id} --type acl_owner3() :: {string(), string(), string()}. --type acl_owner() :: acl_owner2() | acl_owner3(). --record(acl_v1, {owner={"", ""} :: acl_owner(), - grants=[] :: [acl_grant()], - creation_time=now() :: erlang:timestamp()}). -%% acl_v2 owner fields: {DisplayName, CanonicalId, KeyId} --record(acl_v2, {owner={"", "", ""} :: acl_owner(), - grants=[] :: [acl_grant()], - creation_time=now() :: erlang:timestamp()}). --type acl() :: #acl_v1{} | #acl_v2{}. - --type cluster_id() :: undefined | binary(). %% flattened string as binary --type cs_uuid() :: binary(). --type bag_id() :: undefined | binary(). --type riak_client() :: pid(). - --record(lfs_manifest_v2, { - version=2 :: integer(), - block_size :: integer(), - bkey :: {binary(), binary()}, - metadata :: orddict:orddict(), - created=riak_cs_wm_utils:iso_8601_datetime(), - uuid :: cs_uuid(), - content_length :: non_neg_integer(), - content_type :: binary(), - content_md5 :: term(), - state=undefined :: undefined | writing | active | - pending_delete | scheduled_delete | deleted, - write_start_time :: term(), %% immutable - last_block_written_time :: term(), - write_blocks_remaining :: ordsets:ordset(integer()), - delete_marked_time :: term(), - last_block_deleted_time :: term(), - delete_blocks_remaining :: ordsets:ordset(integer()), - acl :: acl(), - props = [] :: proplists:proplist(), - cluster_id :: cluster_id() - }). - --record(lfs_manifest_v3, { - %% "global" properties - %% ----------------------------------------------------------------- - - %% this isn't as important anymore - %% since we're naming the record - %% to include the version number, - %% but I figured it's worth keeping - %% in case we change serialization - %% formats in the future. - version=3 :: integer(), - - %% the block_size setting when this manifest - %% was written. Needed if the user - %% ever changes the block size after writing - %% data - block_size :: integer(), - - %% identifying properties - %% ----------------------------------------------------------------- - bkey :: {binary(), binary()}, - - %% user metadata that would normally - %% be placed on the riak_object. We avoid - %% putting it on the riak_object so that - %% we can use that metadata ourselves - metadata :: orddict:orddict(), - - %% the date the manifest was created. - %% not sure if we need both this and - %% write_start_time. My thought was that - %% write_start_time would have millisecond - %% resolution, but I suppose there's no - %% reason we can't change created - %% to have millisecond as well. - created=riak_cs_wm_utils:iso_8601_datetime(), - uuid :: cs_uuid(), - - %% content properties - %% ----------------------------------------------------------------- - content_length :: non_neg_integer(), - content_type :: binary(), - content_md5 :: term(), - - %% state properties - %% ----------------------------------------------------------------- - state=undefined :: undefined | writing | active | - pending_delete | scheduled_delete | deleted, - - %% writing/active state - %% ----------------------------------------------------------------- - write_start_time :: term(), %% immutable - - %% used for two purposes - %% 1. to mark when a file has finished uploading - %% 2. to decide if a write crashed before completing - %% and needs to be garbage collected - last_block_written_time :: term(), - - %% a shrink-only (during resolution) - %% set to denote which blocks still - %% need to be written. We use a shrinking - %% (rather than growing) set to that the - %% set is empty after the write has completed, - %% which should be most of the lifespan on disk - write_blocks_remaining :: ordsets:ordset(integer()), - - %% pending_delete/deleted state - %% ----------------------------------------------------------------- - %% set to the current time - %% when a manifest is marked as deleted - %% and enters the pending_delete state - delete_marked_time :: term(), - - %% the timestamp serves a similar - %% purpose to last_block_written_time, - %% in that it's used for figuring out - %% when delete processes have died - %% and garbage collection needs to - %% pick up where they left off. - last_block_deleted_time :: term(), - - %% a shrink-only (during resolution) - %% set to denote which blocks - %% still need to be deleted. - %% See write_blocks_remaining for - %% an explanation of why we chose - %% a shrinking set - delete_blocks_remaining :: ordsets:ordset(integer()), - - %% the time the manifest was put - %% into the scheduled_delete - %% state - scheduled_delete_time :: term(), - - %% The ACL for the version of the object represented - %% by this manifest. - acl :: acl() | no_acl_yet, - - %% There are a couple of cases where we want to add record - %% member'ish data without adding new members to the record, - %% e.g. - %% 1. Data for which the common value is 'undefined' or not - %% used/set for this particular manifest - %% 2. Cases where we do want to change the structure of the - %% record but don't want to go through the full code - %% refactoring and backward-compatibility tap dance - %% until sometime later. - %% 'undefined' is for backward compatibility with v3 manifests - %% written with Riak CS 1.2.2 or earlier. - props = [] :: 'undefined' | proplists:proplist(), - - %% cluster_id: A couple of uses, both short- and longer-term - %% possibilities: - %% - %% 1. We don't have a good story in early 2012 for how to - %% build a stable 2,000 node Riak cluster. If MOSS can - %% talk to multiple Riak clusters, then each individual - %% cluster can be a size that we're comfortable - %% supporting. - %% - %% 2. We may soon have Riak EE's replication have full - %% plumbing to make it feasible to forward arbitrary - %% traffic between clusters. Then if a slave cluster is - %% missing a data block, and read-repair cannot - %% automagically fix the 'not_found' problem, then perhaps - %% forwarding a get request to the source Riak cluster can - %% fetch us the missing data. - cluster_id :: cluster_id() - }). --type lfs_manifest() :: #lfs_manifest_v3{}. - --type cs_uuid_and_manifest() :: {cs_uuid(), lfs_manifest()}. - --record(part_manifest_v1, { - bucket :: binary(), - key :: binary(), - - %% used to judge races between concurrent uploads - %% of the same part_number - start_time :: erlang:timestamp(), - - %% one-of 1-10000, inclusive - part_number :: integer(), - - %% a UUID to prevent conflicts with concurrent - %% uploads of the same {upload_id, part_number}. - part_id :: binary(), - - %% each individual part upload always has a content-length - %% content_md5 is used for the part ETag, alas. - content_length :: integer(), - content_md5 :: 'undefined' | binary(), - - %% block size just like in `lfs_manifest_v2'. Concievably, - %% parts for the same upload id could have different block_sizes. - block_size :: integer() -}). --type part_manifest() :: #part_manifest_v1{}. - --record(multipart_manifest_v1, { - upload_id :: binary(), - owner :: acl_owner3(), - - %% since we don't have any point of strong - %% consistency (other than stanchion), we - %% can get concurrent `complete' and `abort' - %% requests. There are still some details to - %% work out, but what we observe here will - %% affect whether we accept future `complete' - %% or `abort' requests. - - %% Stores references to all of the parts uploaded - %% with this `upload_id' so far. A part - %% can be uploaded more than once with the same - %% part number. type = #part_manifest_vX - parts = ordsets:new() :: ordsets:ordset(?PART_MANIFEST{}), - %% List of UUIDs for parts that are done uploading. - %% The part number is redundant, so we only store - %% {UUID::binary(), PartETag::binary()} here. - done_parts = ordsets:new() :: ordsets:ordset({binary(), binary()}), - %% type = #part_manifest_vX - cleanup_parts = ordsets:new() :: ordsets:ordset(?PART_MANIFEST{}), - - %% a place to stuff future information - %% without having to change - %% the record format - props = [] :: proplists:proplist() -}). --type multipart_manifest() :: #multipart_manifest_v1{}. - -%% Basis of list multipart uploads output --record(multipart_descr_v1, { - %% Object key for the multipart upload - key :: binary(), - - %% UUID of the multipart upload - upload_id :: binary(), - - %% User that initiated the upload - owner_display :: string(), - owner_key_id :: string(), - - %% storage class: no real options here - storage_class = standard, - - %% Time that the upload was initiated - initiated :: string() %% conflict of func vs. type: riak_cs_wm_utils:iso_8601_datetime() -}). - -%% Basis of multipart list parts output --record(part_descr_v1, { - part_number :: integer(), - last_modified :: string(), % TODO ?? - etag :: binary(), - size :: integer() -}). - --define(USER_BUCKET, <<"moss.users">>). --define(ACCESS_BUCKET, <<"moss.access">>). --define(STORAGE_BUCKET, <<"moss.storage">>). --define(BUCKETS_BUCKET, <<"moss.buckets">>). --define(GC_BUCKET, <<"riak-cs-gc">>). --define(FREE_BUCKET_MARKER, <<"0">>). --define(DEFAULT_MAX_BUCKETS_PER_USER, 100). --define(DEFAULT_MAX_CONTENT_LENGTH, 5368709120). %% 5 GB --define(DEFAULT_LFS_BLOCK_SIZE, 1048576).%% 1 MB --define(XML_PROLOG, ""). --define(S3_XMLNS, 'http://s3.amazonaws.com/doc/2006-03-01/'). --define(DEFAULT_STANCHION_IP, "127.0.0.1"). --define(DEFAULT_STANCHION_PORT, 8085). --define(DEFAULT_STANCHION_SSL, true). --define(MD_BAG, <<"X-Rcs-Bag">>). --define(MD_ACL, <<"X-Moss-Acl">>). --define(MD_POLICY, <<"X-Rcs-Policy">>). --define(EMAIL_INDEX, <<"email_bin">>). --define(ID_INDEX, <<"c_id_bin">>). --define(KEY_INDEX, <<"$key">>). --define(AUTH_USERS_GROUP, "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"). --define(ALL_USERS_GROUP, "http://acs.amazonaws.com/groups/global/AllUsers"). --define(LOG_DELIVERY_GROUP, "http://acs.amazonaws.com/groups/s3/LogDelivery"). --define(DEFAULT_FETCH_CONCURRENCY, 1). --define(DEFAULT_PUT_CONCURRENCY, 1). --define(DEFAULT_DELETE_CONCURRENCY, 1). -%% A number to multiplied with the block size -%% to determine the PUT buffer size. -%% ex. 2 would mean BlockSize * 2 --define(DEFAULT_PUT_BUFFER_FACTOR, 1). -%% Similar to above, but for fetching -%% This is also max ram per fetch request --define(DEFAULT_FETCH_BUFFER_FACTOR, 32). --define(N_VAL_1_GET_REQUESTS, true). --define(DEFAULT_PING_TIMEOUT, 5000). --define(JSON_TYPE, "application/json"). --define(XML_TYPE, "application/xml"). --define(S3_API_MOD, riak_cs_s3_rewrite). --define(S3_LEGACY_API_MOD, riak_cs_s3_rewrite_legacy). --define(OOS_API_MOD, riak_cs_oos_rewrite). --define(S3_RESPONSE_MOD, riak_cs_s3_response). --define(OOS_RESPONSE_MOD, riak_cs_oos_response). - -%% Major categories of Erlang-triggered DTrace probes -%% -%% The main R15B01 USDT probe that can be triggered by Erlang code is defined -%% like this: -%% -%% /** -%% * Multi-purpose probe: up to 4 NUL-terminated strings and 4 -%% * 64-bit integer arguments. -%% * -%% * @param proc, the PID (string form) of the sending process -%% * @param user_tag, the user tag of the sender -%% * @param i1, integer -%% * @param i2, integer -%% * @param i3, integer -%% * @param i4, integer -%% * @param s1, string/iolist. D's arg6 is NULL if not given by Erlang -%% * @param s2, string/iolist. D's arg7 is NULL if not given by Erlang -%% * @param s3, string/iolist. D's arg8 is NULL if not given by Erlang -%% * @param s4, string/iolist. D's arg9 is NULL if not given by Erlang -%% */ -%% probe user_trace__i4s4(char *proc, char *user_tag, -%% int i1, int i2, int i3, int i4, -%% char *s1, char *s2, char *s3, char *s4); -%% -%% The convention that we'll use of these probes is: -%% param D arg name use -%% ----- ---------- --- -%% i1 arg2 Application category (see below) -%% i2 arg3 1 = function entry, 2 = function return -%% NOTE! Not all function entry probes have a return probe -%% i3-i4 arg4-arg5 Varies, zero usually means unused (but not always!) -%% s1 arg6 Module name -%% s2 arg7 Function name -%% s3-4 arg8-arg9 Varies, NULL means unused -%% --define(DT_BLOCK_OP, 700). --define(DT_SERVICE_OP, 701). --define(DT_BUCKET_OP, 702). --define(DT_OBJECT_OP, 703). -%% perhaps add later? -define(DT_AUTH_OP, 704). --define(DT_WM_OP, 705). - --define(USER_BUCKETS_PRUNE_TIME, 86400). %% one-day in seconds --define(DEFAULT_CLUSTER_ID_TIMEOUT,5000). --define(DEFAULT_AUTH_MODULE, riak_cs_s3_auth). --define(DEFAULT_LIST_OBJECTS_MAX_KEYS, 1000). --define(DEFAULT_MD5_CHUNK_SIZE, 1048576). %% 1 MB --define(DEFAULT_MANIFEST_WARN_SIBLINGS, 20). --define(DEFAULT_MANIFEST_WARN_BYTES, 5*1024*1024). %% 5MB --define(DEFAULT_MANIFEST_WARN_HISTORY, 30). --define(DEFAULT_MAX_PART_NUMBER, 10000). - -%% timeout hitting Riak PB API --define(DEFAULT_RIAK_TIMEOUT, 60000). - -%% General system info --define(WORD_SIZE, erlang:system_info(wordsize)). - --define(DEFAULT_POLICY_MODULE, riak_cs_s3_policy). - --record(access_v1, { - method :: 'PUT' | 'GET' | 'POST' | 'DELETE' | 'HEAD', - target :: atom(), % object | object_acl | .... - id :: string(), - bucket :: binary(), - key = <<>> :: undefined | binary(), - req %:: #wm_reqdata{} % request of webmachine - }). - --type access() :: #access_v1{}. - --type policy() :: riak_cs_s3_policy:policy1(). - -%% just to persuade dyalizer --type crypto_context() :: {'md4' | 'md5' | 'ripemd160' | 'sha' | - 'sha224' | 'sha256' | 'sha384' | 'sha512', - binary()}. --type digest() :: binary(). - --define(USERMETA_BUCKET, "RCS-bucket"). --define(USERMETA_KEY, "RCS-key"). --define(USERMETA_BCSUM, "RCS-bcsum"). - --define(OBJECT_BUCKET_PREFIX, <<"0o:">>). % Version # = 0 --define(BLOCK_BUCKET_PREFIX, <<"0b:">>). % Version # = 0 - --define(MAX_S3_KEY_LENGTH, 1024). - --ifdef(namespaced_types). --type mochiweb_headers() :: gb_trees:tree(). --else. --type mochiweb_headers() :: gb_tree(). --endif. diff --git a/include/s3_api.hrl b/include/s3_api.hrl deleted file mode 100644 index f61c83af2..000000000 --- a/include/s3_api.hrl +++ /dev/null @@ -1,147 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --define(ROOT_HOST, "s3.amazonaws.com"). --define(SUBRESOURCES, ["acl", "location", "logging", "notification", "partNumber", - "policy", "requestPayment", "torrent", "uploadId", "uploads", - "versionId", "versioning", "versions", "website", - "delete", "lifecycle"]). - -% type and record definitions for S3 policy API --type s3_object_action() :: 's3:GetObject' | 's3:GetObjectVersion' - | 's3:GetObjectAcl' | 's3:GetObjectVersionAcl' - | 's3:PutObject' | 's3:PutObjectAcl' - | 's3:PutObjectVersionAcl' - | 's3:DeleteObject' | 's3:DeleteObjectVersion' - | 's3:ListMultipartUploadParts' - | 's3:AbortMultipartUpload' - %| 's3:GetObjectTorrent' we never do this - %| 's3:GetObjectVersionTorrent' we never do this - | 's3:RestoreObject'. - --define(SUPPORTED_OBJECT_ACTION, - [ 's3:GetObject', 's3:GetObjectAcl', 's3:PutObject', 's3:PutObjectAcl', - 's3:DeleteObject', - 's3:ListMultipartUploadParts', 's3:AbortMultipartUpload' ]). - --type s3_bucket_action() :: 's3:CreateBucket' - | 's3:DeleteBucket' - | 's3:ListBucket' - | 's3:ListBucketVersions' - | 's3:ListAllMyBuckets' - | 's3:ListBucketMultipartUploads' - | 's3:GetBucketAcl' | 's3:PutBucketAcl' - | 's3:GetBucketVersioning' | 's3:PutBucketVersioning' - | 's3:GetBucketRequestPayment' | 's3:PutBucketRequestPayment' - | 's3:GetBucketLocation' - | 's3:GetBucketPolicy' | 's3:DeleteBucketPolicy' | 's3:PutBucketPolicy' - | 's3:GetBucketNotification' | 's3:PutBucketNotification' - | 's3:GetBucketLogging' | 's3:PutBucketLogging' - | 's3:GetBucketWebsite' | 's3:PutBucketWebsite' | 's3:DeleteBucketWebsite' - | 's3:GetLifecycleConfiguration' | 's3:PutLifecycleConfiguration'. - --define(SUPPORTED_BUCKET_ACTION, - [ 's3:CreateBucket', 's3:DeleteBucket', 's3:ListBucket', 's3:ListAllMyBuckets', - 's3:GetBucketAcl', 's3:PutBucketAcl', - 's3:GetBucketPolicy', 's3:DeleteBucketPolicy', 's3:PutBucketPolicy', - 's3:ListBucketMultipartUploads']). - -% one of string, numeric, date&time, boolean, IP address, ARN and existence of condition keys --type string_condition_type() :: 'StringEquals' | streq | 'StringNotEquals' | strneq - | 'StringEqualsIgnoreCase' | streqi | 'StringNotEqualsIgnoreCase' | strneqi - | 'StringLike' | strl | 'StringNotLike' | strnl. - --define(STRING_CONDITION_ATOMS, - [ 'StringEquals' , streq, 'StringNotEquals', strneq, - 'StringEqualsIgnoreCase', streqi, 'StringNotEqualsIgnoreCase', strneqi, - 'StringLike', strl, 'StringNotLike' , strnl]). - --type numeric_condition_type() :: 'NumericEquals' | numeq | 'NumericNotEquals' | numneq - | 'NumericLessThan' | numlt | 'NumericLessThanEquals' | numlteq - | 'NumericGreaterThan' | numgt | 'NumericGreaterThanEquals' | numgteq. - --define(NUMERIC_CONDITION_ATOMS, - [ 'NumericEquals', numeq, 'NumericNotEquals', numneq, - 'NumericLessThan' , numlt, 'NumericLessThanEquals', numlteq, - 'NumericGreaterThan', numgt, 'NumericGreaterThanEquals', numgteq]). - --type date_condition_type() :: 'DateEquals' | dateeq - | 'DateNotEquals' | dateneq - | 'DateLessThan' | datelt - | 'DateLessThanEquals' | datelteq - | 'DateGreaterThan' | dategt - | 'DateGreaterThanEquals' | dategteq. - --define(DATE_CONDITION_ATOMS, - [ 'DateEquals', dateeq, - 'DateNotEquals', dateneq, - 'DateLessThan', datelt, - 'DateLessThanEquals', datelteq, - 'DateGreaterThan', dategt, - 'DateGreaterThanEquals', dategteq]). - - --type ip_addr_condition_type() :: 'IpAddress' | 'NotIpAddress'. - --define(IP_ADDR_CONDITION_ATOMS, - ['IpAddress', 'NotIpAddress']). - --type condition_pair() :: {date_condition_type(), [{'aws:CurrentTime', binary()}]} - | {numeric_condition_type(), [{'aws:EpochTime', non_neg_integer()}]} - | {boolean(), 'aws:SecureTransport'} - | {ip_addr_condition_type(), [{'aws:SourceIp', {IP::inet:ip_address(), inet:ip_address()}}]} - | {string_condition_type(), [{'aws:UserAgent', binary()}]} - | {string_condition_type(), [{'aws:Referer', binary()}]}. - --record(arn_v1, { - provider = aws :: aws, - service = s3 :: s3, - region :: string(), - id :: binary(), - path :: string() - }). - --type arn() :: #arn_v1{}. - --type principal() :: '*' - | [{canonical_id, string()}|{aws, '*'}]. - --record(statement, { - sid = undefined :: undefined | binary(), % had better use uuid: should be UNIQUE - effect = deny :: allow | deny, - principal = [] :: principal(), - action = [] :: [ s3_object_action() | s3_bucket_action() ] | '*', - not_action = [] :: [ s3_object_action() | s3_bucket_action() ] | '*', - resource = [] :: [ arn() ] | '*', - condition_block = [] :: [ condition_pair() ] - }). - --record(policy_v1, { - version = <<"2008-10-17">> :: binary(), % no other value is allowed than default - id = undefined :: undefined | binary(), % had better use uuid: should be UNIQUE - statement = [] :: [#statement{}], - creation_time=os:timestamp() :: erlang:timestamp() - }). - - --define(POLICY, #policy_v1). --define(ARN, #arn_v1). - --define(DEFAULT_REGION, "us-east-1"). diff --git a/misc/mapdeps.erl b/misc/mapdeps.erl index d7ab9ee61..28381c9c5 100755 --- a/misc/mapdeps.erl +++ b/misc/mapdeps.erl @@ -74,8 +74,7 @@ map_dir(BaseDir) -> map_rebar(BaseDir, Path, Acc) -> case file:consult(Path) of {ok, Opts} -> - Deps = proplists:get_value(deps, Opts, []) ++ - proplists:get_value(deps_ee, Opts, []), + Deps = proplists:get_value(deps, Opts, []), lists:foldl( fun({DepName, _, _}, A) -> From = app_name(Path), diff --git a/misc/prepare-riak-for-cs b/misc/prepare-riak-for-cs new file mode 100755 index 000000000..d4a8e16ec --- /dev/null +++ b/misc/prepare-riak-for-cs @@ -0,0 +1,105 @@ +#!/bin/bash + +# This is a convenience script to help set up a Riak instance to which +# your Riak CS node is configured to connect. It does the following: +# +# 1. Modifies riak.conf and advanced.config (enables milti-backend, +# with leveldb and bitcask for object data and manifests, resp.). +# +# 2. If Riak is installed on the same host as Riak CS, its ebin path +# will be added to riak/advanced.config; otherwise, beams will be +# rsync'ed to where Riak can load them (if the remote host is not +# accessible, files will need to be copied in some other way). +# +# Relevant environment variables (with defaults): +# * RCS_CONFIG_DIR, "/etc/riak-cs". +# * RIAK_CONFIG_DIR, "/etc/riak". +# * RCS_LIB, "/usr/lib/riak-cs/lib" if it exists, else "/usr/lib64/riak-cs/lib". +# * RIAK_EBIN, "/usr/lib/riak/lib" +# +# For a devrel, assuming you are in the riak dir where you did `make +# devrel`, and have riak and riak_cs as sister dirs in .., you might +# use: +# +# RCS_CONFIG_DIR=../riak_cs/dev/dev1/riak-cs/etc \ +# RIAK_CONFIG_DIR=dev/dev1/riak/etc \ +# RCS_LIB=../riak_cs/dev/dev1/riak-cs/lib/ \ +# RIAK_EBIN=dev/dev1/riak/lib/riak_kv-riak_kv-3.0.9+build.3962.ref1d4ee470/ebin \ +# ../riak_cs/misc/prepare-riak-for-cs + +set -e + +# 0. get riak host address +RCS_CONFIG_DIR=${RCS_CONFIG_DIR:-"/etc/riak-cs"} +RIAK_HOST=`sed -n 's/^riak_host = \(.*\):.*/\1/p' $RCS_CONFIG_DIR/riak-cs.conf` +RIAK_EBIN=${RIAK_EBIN:-/usr/lib/riak/lib} + +# 1. Copy modules that are to be loaded on riak VM + +if [ -d /usr/lib/riak-cs ]; then + RCS_LIB_PKG_INST=/usr/lib/riak-cs/lib +elif [ -d /usr/lib64/riak-cs ]; then + RCS_LIB_PKG_INST=/usr/lib64/riak-cs/lib +fi + +RCS_LIB=${RCS_LIB:-$RCS_LIB_PKG_INST} + +mods_to_add=`find -L $RCS_LIB \ + -name riak_cs_kv_multi_backend.beam \ +-o -name riak_cs_riak_mapred.beam \ +-o -name riak_cs_utils.beam \ +-o -name rcs_common_manifest_utils.beam \ +-o -name rcs_common_manifest_resolution.beam \ +-o -name getopt.beam` + +if [ $RIAK_HOST = "127.0.0.1" ]; then + riak_dest= +else + riak_dest=$RIAK_HOST: +fi + +echo "* Copying CS modules to $riak_dest$RIAK_EBIN" +rsync $mods_to_add $riak_dest$RIAK_EBIN/ + + +# 2. Modify riak.conf and advanced.config + +RIAK_CONFIG_DIR=${RIAK_CONFIG_DIR:-"/etc/riak"} + +echo "* Pulling riak.conf and advanced.config" + +d=/tmp/$RIAK_HOST-riak-configs +mkdir -p $d +rsync $riak_dest$RIAK_CONFIG_DIR/* $d/ + +echo "* Modifying riak.conf" +sed -i \ + -e "s|storage_backend = bitcask|storage_backend = multi|" \ + $d/riak.conf + +echo " +buckets.default.merge_strategy = 2" \ + >>$d/riak.conf + +echo "* Modifying advanced.config" + +sed -i -e \ + "s|]\\.|,\n\ + {riak_kv, [\n\ + $paths_option\ + {multi_backend,\n\ + [{be_default,riak_kv_eleveldb_backend,\n\ + [{max_open_files,20}]},\n\ + {be_blocks,riak_kv_bitcask_backend,\n\ + []}]},\n\ + {multi_backend_default,be_default},\n\ + {multi_backend_prefix_list,[{<<\"0b:\">>,be_blocks}]},\n\ + {storage_backend,riak_kv_multi_backend}\n\ + ]}\n\ + ].\n|" \ + $d/advanced.config + +echo "* Pushing riak.conf and advanced.config" +rsync $d/* $riak_dest$RIAK_CONFIG_DIR + +rm -rf $d diff --git a/misc/smoke-test.freebsd b/misc/smoke-test.freebsd new file mode 100755 index 000000000..29984d328 --- /dev/null +++ b/misc/smoke-test.freebsd @@ -0,0 +1,59 @@ +#!/bin/bash + +# A minimal smoke test for Riak CS Suite, intended for a quick sanity +# check of Riak CS components. After installing riak and riak-cs from +# packages, make sure you run ./prepare-riak-for-cs. + +set -e + +echo "Smoke testing Riak CS Suite" + +echo +echo "1. using service" +service riak start +riak admin wait-for-service riak_kv +riak admin ringready +riak admin test +#riak admin status + +# allow all three apps to coexist +chmod 777 /tmp/erl_pipes + +service riak-cs start + +while [[ x`riak-cs ping` != x"pong" ]]; do + sleep 2 +done + +riak-cs admin test + +riak-cs admin status + +for s in riak-cs riak; do + echo "Stopping $s" + service $s stop +done + +echo "2. using cmd start/stop" +riak start +riak pid +riak-admin wait-for-service riak_kv +riak-admin test + +for s in riak-cs; do + $s start + $s pid +done +riak-cs-admin test + +rm -f /usr/local/lib/riak/*.tar.gz +riak-debug -v +rm -f /usr/local/lib/riak-cs/*.tar.gz +riak-cs-debug -v + +for s in riak-cs riak; do + echo "Stopping $s" + $s stop +done + +echo "All good." diff --git a/misc/smoke-test.linux b/misc/smoke-test.linux new file mode 100755 index 000000000..f159da460 --- /dev/null +++ b/misc/smoke-test.linux @@ -0,0 +1,56 @@ +#!/bin/bash + +# A minimal smoke test for Riak CS Suite, intended for a quick sanity +# check of Riak CS components. After installing riak and riak-cs from +# packages, make sure you run ./prepare-riak-for-cs. + +set -e + +echo "Smoke testing Riak CS Suite" + +echo +echo "1. using systemctl" +systemctl start riak +riak admin wait-for-service riak_kv +riak admin ringready +riak admin test +#riak admin status + +# allow all three apps to coexist +chmod 777 /tmp/erl_pipes + +systemctl start riak-cs + +while [[ x`riak-cs ping` != x"pong" ]]; do + sleep 2 +done + +riak-cs admin test +riak-cs admin status + +echo "Stopping $s" +systemctl stop riak-cs riak + +echo "2. using cmd start/stop" +riak start +riak pid +riak admin wait-for-service riak_kv +riak admin test + +for s in riak-cs; do + $s start + $s pid +done +riak-cs admin test + +rm -f /usr/lib{,64}/riak/*.tar.gz +riak-debug -v +rm -f /usr/lib{,64}/riak-cs/*.tar.gz +riak-cs-debug -v + +for s in riak-cs riak; do + echo "Stopping $s" + $s stop +done + +echo "All good." diff --git a/rebar b/rebar deleted file mode 100755 index 60410665a..000000000 Binary files a/rebar and /dev/null differ diff --git a/rebar.config b/rebar.config index 5174654b5..5435dc8cf 100644 --- a/rebar.config +++ b/rebar.config @@ -1,60 +1,266 @@ -{sub_dirs, ["rel"]}. - -{require_otp_vsn, "R16|17"}. - -{cover_enabled, false}. - -%% EDoc options -{edoc_opts, [preprocess]}. - -{lib_dirs, ["deps", "apps"]}. - -{erl_opts, [debug_info, - warnings_as_errors, - {parse_transform, lager_transform}, - {platform_define, "^[0-9]+", namespaced_types}]}. - -{xref_checks, []}. -{xref_queries, - [{"(XC - UC) || (XU - X - B - \"(^riak$|^riak_cs_dummy_reader$|^riak_core_bucket$|^app_helper$|^riakc_pb_socket_fake$|^riak_object$|^riak_repl_pb_api$|^riak_cs_multibag$)\" : Mod)", []}]}. -{xref_queries_ee, - [{"(XC - UC) || (XU - X - B - \"(^riak$|^riak_cs_dummy_reader$|^riak_core_bucket$|^app_helper$|^riakc_pb_socket_fake$|^riak_object$)\" : Mod)", []}]}. - -{reset_after_eunit, true}. - -{plugin_dir, ".plugins"}. -{plugins, [rebar_test_plugin, rebar_lock_deps_plugin]}. - -{client_test, [ - {test_paths, ["client_tests/erlang"]}, - {test_output, ".client_test"} -]}. -{riak_test, [ - {test_paths, ["riak_test/tests", "riak_test/src", - "deps/riak_cs_multibag/riak_test/tests", - "deps/riak_cs_multibag/riak_test/src"]}, - {test_output, "riak_test/ebin"} -]}. - -{deps, [ - {lager, ".*", {git, "git://github.com/basho/lager", {tag, "2.2.0"}}}, - {lager_syslog, ".*", {git, "git://github.com/basho/lager_syslog", {tag, "2.1.1"}}}, - {cuttlefish, ".*", {git, "git://github.com/basho/cuttlefish.git", {tag, "2.0.4"}}}, - {node_package, ".*", {git, "git://github.com/basho/node_package", {tag, "2.0.3"}}}, - {getopt, ".*", {git, "git://github.com/jcomellas/getopt.git", {tag, "v0.8.2"}}}, - {webmachine, ".*", {git, "git://github.com/basho/webmachine", {tag, "1.10.8"}}}, - {riakc, ".*", {git, "git://github.com/basho/riak-erlang-client", {tag, "2.1.1"}}}, - {eper, ".*", {git, "git://github.com/basho/eper.git", "0.92-basho1"}}, - {druuid, ".*", {git, "git://github.com/kellymclaughlin/druuid.git", {tag, "0.2"}}}, - {poolboy, "0.8.*", {git, "git://github.com/basho/poolboy", "0.8.1p3"}}, - {exometer_core, ".*", {git, "git://github.com/Feuerlabs/exometer_core", {tag, "1.2"}}}, - {cluster_info, ".*", {git, "git://github.com/basho/cluster_info", {tag, "2.0.3"}}}, - {xmerl, ".*", {git, "git://github.com/shino/xmerl", "1b016a05473e086abadbb3c12f63d167fe96c00f"}}, - {erlcloud, ".*", {git, "git://github.com/basho/erlcloud.git", {tag, "0.4.6"}}}, - {rebar_lock_deps_plugin, ".*", {git, "git://github.com/seth/rebar_lock_deps_plugin.git", {tag, "3.1.0"}}} - ]}. - -{deps_ee, [ - {riak_repl_pb_api,".*",{git,"git@github.com:basho/riak_repl_pb_api.git", {tag, "2.1.1"}}}, - {riak_cs_multibag,".*",{git,"git@github.com:basho/riak_cs_multibag.git", {tag, "2.1.0p1"}}} - ]}. +%% -*- erlang -*- +{minimum_otp_vsn, "22"}. + +{erl_opts, [ debug_info + , warnings_as_errors + , {nowarn_deprecated_function, [ {gen_fsm, start_link, 3} + , {gen_fsm, start_link, 4} + , {gen_fsm, send_event, 2} + , {gen_fsm, send_all_state_event, 2} + , {gen_fsm, sync_send_event, 3} + , {gen_fsm, sync_send_all_state_event, 3} + , {gen_fsm, reply, 2} + ]} + ] +}. + +{deps, [ {getopt, "1.0.3"} + , {parse_trans, "3.4.2"} + , {uuid, "2.0.7", {pkg, uuid_erl}} + , {jsx, "3.1.0"} + , {jason, "1.2.2", {pkg, jason_erl}} + , {poolboy, "1.5.2"} + , {exometer_core, "1.6.1"} + , {esaml, "4.5.0"} + , {webmachine, {git, "https://github.com/TI-Tokyo/webmachine.git", {tag, "1.11.3"}}} + , {riakc, {git, "https://github.com/TI-Tokyo/riak-erlang-client", {tag, "3.0.13-tiot2"}}} + , {cluster_info, {git, "https://github.com/basho/cluster_info", {tag, "2.1.0"}}} + , {riak_repl_pb_api, {git, "https://github.com/TI-Tokyo/riak_repl_pb_api.git", {tag, "3.1.0"}}} + ] +}. + + +{project_plugins, [ {rebar3_cuttlefish, {git, "https://github.com/TI-Tokyo/rebar3_cuttlefish", {tag, "0.2.2"}}} + , rebar3_proper + , rebar3_sbom + ] +}. + +{relx, [ {release, {'riak-cs', "3.2.5"}, + [ kernel + , stdlib + , riak_cs + , riak_cs_multibag + ] + } + + , {dev_mode, false} + , {include_erts, true} + + , {overlay, [ {template, "rel/files/advanced.config", "etc/advanced.config"} + , {template, "rel/files/riak-cs", "usr/bin/riak-cs"} + , {template, "rel/files/riak-cs-admin", "bin/riak-cs-admin"} + , {template, "rel/files/riak-cs-debug", "bin/riak-cs-debug"} + , {template, "rel/files/riak-cs-chkconfig", "bin/riak-cs-chkconfig"} + , {template, "rel/files/riak-cs-supercluster", "bin/riak-cs-supercluster"} + , {template, "rel/files/lib.sh", "lib/lib.sh"} + , {copy, "rel/files/app_epath.escript", "lib/app_epath.escript"} + , {copy, "rel/files/cert.pem", "etc/cert.pem"} + , {copy, "rel/files/key.pem", "etc/key.pem"} + , {mkdir, "priv/tools/internal"} + , {mkdir, "log"} + , {copy, "misc/prepare-riak-for-cs", "bin/prepare-riak-for-cs"} + , {copy, "tools/repair_gc_bucket.erl", "priv/tools/repair_gc_bucket.erl"} + , {copy, "tools/internal/README.md", "priv/tools/internal/README.md"} + , {copy, "tools/internal/block_audit.erl", "priv/tools/internal/block_audit.erl"} + , {copy, "tools/internal/ensure_orphan_blocks.erl", "priv/tools/internal/ensure_orphan_blocks.erl"} + , {copy, "tools/internal/offline_delete.erl", "priv/tools/internal/offline_delete.erl"} + , {copy, "tools/internal/riak_cs_inspector.erl", "priv/tools/internal/riak_cs_inspector.erl"} + , {copy, "tools/internal/select_gc_bucket.erl", "priv/tools/internal/select_gc_bucket.erl"} + , {copy, "rel/files/hooks/erl_codeloadingmode","bin/hooks/erl_codeloadingmode"} + , {template, "rel/files/hooks/riak_not_running", "bin/hooks/riak_not_running"} + , {copy, "rel/files/hooks/ok", "bin/hooks/ok"} + ] + } + , {generate_start_script, true} + , {extended_start_script, true} + , {extended_start_script_extensions, + [ {admin, "riak-cs-admin"} + , {supercluster, "riak-cs-supercluster"} + , {debug, "riak-cs-debug"} + , {chkconfig, "riak-cs-chkconfig"} + ] + } + ] +}. + +{profiles, + [ {rel, + [ {relx, + [ {overlay_vars, "rel/vars.config"} + , {extended_start_script_hooks, + [ {post_start, + [{wait_for_process, riak_cs_get_fsm_sup}] + } + , {post_stop, [{custom, "hooks/ok"}]} + ] + } + ] + } + ] + } + + , {dev, + [ {relx, + [ {dev_mode, true} + , {extended_start_script_hooks, + [ {post_start, + [{wait_for_process, riak_cs_get_fsm_sup}] + } + , {post_stop, [{custom, "hooks/ok"}]} + ] + } + ] + } + ] + } + + , {test, + [ {relx, + [ {dev_mode, true} + ] + } + , {deps, + [ {proper, "1.4.0"} + , {erlcloud, "3.7.2"} + , {meck, "0.9.2"} + ] + } + ] + } + + , {rpm, + [ {relx, + [ {overlay_vars, "rel/pkg/rpm/vars.config"} + , {extended_start_script_hooks, + [ {pre_start, + [ {custom, "hooks/erl_codeloadingmode"} + ] + } + , {post_start, + [{pid, "/run/riak-cs/riak-cs.pid"}, + {wait_for_process, riak_cs_get_fsm_sup}] + } + , {post_stop, [{custom, "hooks/ok"}]} + ] + } + ] + } + ] + } + + , {deb, + [ {relx, + [ {overlay_vars, "rel/pkg/deb/vars.config"} + , {extended_start_script_hooks, + [ {pre_start, + [ {custom, "hooks/erl_codeloadingmode"} + ] + } + , {post_start, + [ {pid, "/run/riak-cs/riak-cs.pid"} + , {wait_for_process, riak_cs_get_fsm_sup} + ] + } + , {post_stop, [{custom, "hooks/ok"}]} + ] + } + ] + } + ] + } + + , {fbsdng, + [ {relx, + [ {overlay_vars, "rel/pkg/fbsdng/vars.config"} + , {extended_start_script_hooks, + [ {pre_start, + [ {custom, "hooks/erl_codeloadingmode"} + ] + } + , {post_start, + [{pid, "/var/run/riak-cs/riak-cs.pid"}, + {wait_for_process, riak_cs_sup}] + } + , {post_stop, [{custom, "hooks/ok"}]} + ] + } + ] + } + ] + } + + , {alpine, + [ {relx, + [ {overlay_vars, "rel/pkg/alpine/vars.config"} + , {overlay, + [ {template, "rel/pkg/alpine/riak-cs.nosu", "usr/bin/riak-cs.nosu"} + %% to be renamed to riak-cs in Makefile. We would rather + %% have a special version of riak-cs in pkg/alpine dir, + %% but relx seems to give precedence to the template + %% entry in the common section. + ]} + , {extended_start_script_hooks, + [ {post_start, + [{pid, "/var/run/riak-cs/riak-cs.pid"}, + {wait_for_process, riak_cs_get_fsm_sup}] + } + , {post_stop, [{custom, "hooks/ok"}]} + ] + } + ] + } + ] + } + + %% it's essentially `make rel` tarred + , {osx, + [ {relx, + [ {overlay_vars, "rel/pkg/osx/vars.config"} + , {extended_start_script_hooks, + [ {post_start, + [{wait_for_process, riak_cs_get_fsm_sup}] + } + , {post_stop, [{custom, "hooks/ok"}]} + ] + } + ] + } + ] + } + ] +}. + + +{cuttlefish, [ {file_name, "riak-cs.conf"} + , {disable_bin_scripts, true} + , {schema_discovery, true} + , {schema_order, [ riak_cs + , erlang_vm + ]} + ] +}. + + +{xref_checks, [ undefined_function_calls + , undefined_functions + , locals_not_used + ] +}. +%% skip this module outright as it's supposed to be loaded in riak +{xref_ignores, [ riak_cs_kv_multi_backend +%% additionally, special provision for these functions, used in +%% riak_cs_kv_multi_backend, which xref still scans and complains +%% about despite the module from which they are called being +%% blacklisted: + , {app_helper, get_prop_or_env, 3} + , {app_helper, get_prop_or_env, 4} + , {riak_core_bucket, get_bucket, 1} + , {riak_object, bucket, 1} + , {riak_object, key, 1} + , {riak_object, get_contents, 1} + , {riak_object, get_values, 1} + ] +}. + +{eunit_opts, [verbose, {print_depth, 5}]}. diff --git a/rebar.config.script b/rebar.config.script deleted file mode 100644 index 74706a568..000000000 --- a/rebar.config.script +++ /dev/null @@ -1,96 +0,0 @@ -{IsEE, Package} = case os:getenv("RIAK_CS_EE_DEPS") of - false -> {false, "riak-cs"}; - _ -> {true, "riak-cs-ee"} - end, -io:format("Building ~s~n", [Package]), - -DepsUpdated = - case IsEE of - false -> - CONFIG; - true -> - {value, {deps, Deps}} = lists:keysearch(deps, 1, CONFIG), - {value, {deps_ee, DepsEE}} = lists:keysearch(deps_ee, 1, CONFIG), - lists:keyreplace(deps, 1, CONFIG, {deps, Deps ++ DepsEE}) - end, - -XrefQueryUpdated = - case IsEE of - false -> - DepsUpdated; - true -> - {value, {xref_queries_ee, QueriesEE}} = - lists:keysearch(xref_queries_ee, 1, DepsUpdated), - lists:keyreplace(xref_queries, 1, DepsUpdated, {xref_queries, QueriesEE}) - end, - -FinalConfig = XrefQueryUpdated, - -%% Write dialyzer.ignore_warnings - -{ok, _} = file:copy("dialyzer.ignore-warnings.ee", "dialyzer.ignore-warnings"), -case IsEE of - false -> - {ok, _} = file:copy("dialyzer.ignore-warnings.oss", - {"dialyzer.ignore-warnings", [append]}); - true -> ok -end, - -%% Write pkg.vars.config -case IsEE of - false -> - Bytes = "%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ts=4 sw=4 et - -%% -%% Packaging -%% -{package_name, \"riak-cs\"}. -{package_install_name, \"riak-cs\"}. -{package_install_user, \"riakcs\"}. -{package_install_group, \"riak\"}. -{package_install_user_desc, \"Riak CS user\"}. -{package_commands, {list, [[{name, \"riak-cs\"}], [{name, \"riak-cs-access\"}], [{name, \"riak-cs-gc\"}], [{name, \"riak-cs-storage\"}], [{name, \"riak-cs-stanchion\"}], [{name, \"riak-cs-debug\"}], [{name, \"riak-cs-admin\"}]]}}. -{package_shortdesc, \"Riak CS\"}. -{package_patch_dir, \"basho-patches\"}. -{package_desc, \"Riak CS\"}. -{bin_or_sbin, \"sbin\"}. -{license_type, \"Apache License, Version 2.0\"}. -{copyright, \"2013 Basho Technologies, Inc\"}. -{vendor_name, \"Basho Technologies, Inc\"}. -{vendor_url, \"http://basho.com\"}. -{vendor_contact_name, \"Basho Package Maintainer\"}. -{vendor_contact_email, \"packaging@basho.com\"}. -{license_full_text, \"This software is provided under license from Basho Technologies.\"}. -{solaris_pkgname, \"BASHOriak-cs\"}.", - file:write_file("pkg.vars.config", Bytes); - true -> - Bytes = "%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ts=4 sw=4 et - -%% -%% Packaging -%% -{package_name, \"riak-cs-ee\"}. -{package_install_name, \"riak-cs\"}. -{package_install_user, \"riakcs\"}. -{package_install_group, \"riak\"}. -{package_install_user_desc, \"Riak CS user\"}. -{package_commands, {list, [[{name, \"riak-cs\"}], [{name, \"riak-cs-access\"}], [{name, \"riak-cs-gc\"}], [{name, \"riak-cs-storage\"}], [{name, \"riak-cs-stanchion\"}], [{name, \"riak-cs-debug\"}], [{name, \"riak-cs-admin\"}], [{name, \"riak-cs-supercluster\"}], [{name, \"riak-cs-multibag\"}]]}}. -{package_shortdesc, \"Riak CS\"}. -{package_patch_dir, \"basho-patches\"}. -{package_desc, \"Riak CS\"}. -{bin_or_sbin, \"sbin\"}. -{license_type, \"Proprietary\"}. -{copyright, \"2013 Basho Technologies, Inc\"}. -{vendor_name, \"Basho Technologies, Inc\"}. -{vendor_url, \"http://basho.com\"}. -{vendor_contact_name, \"Basho Package Maintainer\"}. -{vendor_contact_email, \"packaging@basho.com\"}. -{license_full_text, \"This software is provided under license from Basho Technologies.\"}. -{solaris_pkgname, \"BASHOriak-cs-ee\"}. -{debuild_extra_options, \"-e RIAK_CS_EE_DEPS=true\"}.", - file:write_file("pkg.vars.config", Bytes) -end, - -FinalConfig. diff --git a/rebar.docker.config b/rebar.docker.config new file mode 100644 index 000000000..bad8bdd88 --- /dev/null +++ b/rebar.docker.config @@ -0,0 +1,65 @@ +%% -*- erlang -*- +{minimum_otp_vsn, "22"}. + +{erl_opts, [ debug_info + , warnings_as_errors + , {nowarn_deprecated_function, [ {gen_fsm, start_link, 3} + , {gen_fsm, start_link, 4} + , {gen_fsm, send_event, 2} + , {gen_fsm, send_all_state_event, 2} + , {gen_fsm, sync_send_event, 3} + , {gen_fsm, sync_send_all_state_event, 3} + , {gen_fsm, reply, 2} + ]} + ] +}. + +{deps, [ {getopt, "1.0.3"} + , {parse_trans, "3.4.2"} + , {mochiweb, "3.2.1"} + , {uuid, "2.0.7", {pkg, uuid_erl}} + , {jsx, "3.1.0"} + , {jason, "1.2.2", {pkg, jason_erl}} + , {poolboy, "1.5.2"} + , {exometer_core, "1.6.1"} + , {esaml, "4.5.0"} + , {webmachine, {git, "https://github.com/TI-Tokyo/webmachine.git", {tag, "1.11.2"}}} + , {riakc, {git, "https://github.com/TI-Tokyo/riak-erlang-client", {tag, "3.0.13-tiot"}}} + , {cluster_info, {git, "https://github.com/basho/cluster_info", {tag, "2.1.0"}}} + , {riak_repl_pb_api, {git, "https://github.com/TI-Tokyo/riak_repl_pb_api.git", {tag, "3.1.0"}}} + ] +}. + + +{relx, [ {release, {'riak-cs', "3.2.5"}, + [ sasl + , riak_cs + ] + } + , {dev_mode, false} + , {mode, prod} + , {overlay, [ {template, "rel/files/riak-cs", "usr/sbin/riak-cs"} + , {template, "rel/files/riak-cs-admin", "usr/sbin/riak-cs-admin"} + , {template, "rel/files/riak-cs-debug", "usr/sbin/riak-cs-debug"} + , {template, "rel/files/riak-cs-supercluster", "usr/sbin/riak-cs-supercluster"} + , {template, "rel/files/riak-cs-chkconfig", "usr/sbin/riak-cs-chkconfig"} + , {copy, "rel/files/cert.pem", "etc/cert.pem"} + , {copy, "rel/files/key.pem", "etc/key.pem"} + , {copy, "rel/files/app_epath.escript", "lib/app_epath.escript"} + , {template, "rel/files/lib.sh", "lib/lib.sh"} + , {mkdir, "priv/tools/internal"} + , {mkdir, "log"} + , {copy, "tools/create-admin", "priv/tools/create-admin"} + , {copy, "rel/tools/repair_gc_bucket.erl", "tools/repair_gc_bucket.erl"} + , {copy, "rel/tools/internal/block_audit.erl", "tools/internal/block_audit.erl"} + , {copy, "rel/tools/internal/ensure_orphan_blocks.erl", "tools/internal/ensure_orphan_blocks.erl"} + , {copy, "rel/tools/internal/offline_delete.erl", "tools/internal/offline_delete.erl"} + , {copy, "rel/tools/internal/riak_cs_inspector.erl", "tools/internal/riak_cs_inspector.erl"} + , {copy, "rel/tools/internal/select_gc_bucket.erl", "tools/internal/select_gc_bucket.erl"} + ] + } + , {generate_start_script, true} + , {extended_start_script, true} + , {sys_config_src, "config/sys.docker.config.src"} + ] +}. diff --git a/rebar.lock b/rebar.lock new file mode 100644 index 000000000..bf7d5af6c --- /dev/null +++ b/rebar.lock @@ -0,0 +1,76 @@ +{"1.2.0", +[{<<"bear">>,{pkg,<<"bear">>,<<"1.0.0">>},2}, + {<<"cluster_info">>, + {git,"https://github.com/basho/cluster_info", + {ref,"389d43af7ac1550b3c01cd55b8147bcc0e20022f"}}, + 0}, + {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.12.0">>},1}, + {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.13.0">>},2}, + {<<"esaml">>,{pkg,<<"esaml">>,<<"4.5.0">>},0}, + {<<"exometer_core">>,{pkg,<<"exometer_core">>,<<"1.6.1">>},0}, + {<<"folsom">>,{pkg,<<"folsom">>,<<"1.0.0">>},1}, + {<<"getopt">>,{pkg,<<"getopt">>,<<"1.0.3">>},0}, + {<<"hut">>,{pkg,<<"hut">>,<<"1.3.0">>},1}, + {<<"jason">>,{pkg,<<"jason_erl">>,<<"1.2.2">>},0}, + {<<"jsx">>,{pkg,<<"jsx">>,<<"3.1.0">>},0}, + {<<"mochiweb">>,{pkg,<<"mochiweb">>,<<"3.2.1">>},1}, + {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.4.2">>},0}, + {<<"poolboy">>,{pkg,<<"poolboy">>,<<"1.5.2">>},0}, + {<<"quickrand">>,{pkg,<<"quickrand">>,<<"2.0.7">>},1}, + {<<"ranch">>,{pkg,<<"ranch">>,<<"1.8.0">>},2}, + {<<"riak_pb">>, + {git,"https://github.com/TI-Tokyo/riak_pb.git", + {ref,"a838173209f89b465de8a0f9248146da0ff0866c"}}, + 1}, + {<<"riak_repl_pb_api">>, + {git,"https://github.com/TI-Tokyo/riak_repl_pb_api.git", + {ref,"3046e29d6e31246c8de9ebed2ee2790d19e0691a"}}, + 0}, + {<<"riakc">>, + {git,"https://github.com/TI-Tokyo/riak-erlang-client", + {ref,"8a3805397b707f43794be41e3f4f02977813658a"}}, + 0}, + {<<"setup">>,{pkg,<<"setup">>,<<"2.1.0">>},1}, + {<<"uuid">>,{pkg,<<"uuid_erl">>,<<"2.0.7">>},0}, + {<<"webmachine">>, + {git,"https://github.com/TI-Tokyo/webmachine.git", + {ref,"c674f9d09a010a9db4ae821cb5739f973d931aa0"}}, + 0}]}. +[ +{pkg_hash,[ + {<<"bear">>, <<"430419C1126B477686CDE843E88BA0F2C7DC5CDF0881C677500074F704339A99">>}, + {<<"cowboy">>, <<"F276D521A1FF88B2B9B4C54D0E753DA6C66DD7BE6C9FCA3D9418B561828A3731">>}, + {<<"cowlib">>, <<"DB8F7505D8332D98EF50A3EF34B34C1AFDDEC7506E4EE4DD4A3A266285D282CA">>}, + {<<"esaml">>, <<"4C79A47511B212DEAB235816B10F89CF884A7DA662EF50956176BFF8BCA1F8E8">>}, + {<<"exometer_core">>, <<"742ED00E1F10F8BEFF61A0F43E105C78C2F982E10F1C4E740D59DF89924C65EA">>}, + {<<"folsom">>, <<"50ECC998D2149939F1D5E0AA3E32788F8ED16A58E390D81B5C0BE4CC4EF25589">>}, + {<<"getopt">>, <<"4F3320C1F6F26B2BEC0F6C6446B943EB927A1E6428EA279A1C6C534906EE79F1">>}, + {<<"hut">>, <<"71F2F054E657C03F959CF1ACC43F436EA87580696528CA2A55C8AFB1B06C85E7">>}, + {<<"jason">>, <<"AD3CE50A3F14D9E1717FB1F69CF35363A66DF59FF7A6BC5A3BCCB3343601C216">>}, + {<<"jsx">>, <<"D12516BAA0BB23A59BB35DCCAF02A1BD08243FCBB9EFE24F2D9D056CCFF71268">>}, + {<<"mochiweb">>, <<"FF287E1EC653A0828F226CD5A009D52BE74537DC3FC274B765525A77CE01F8EC">>}, + {<<"parse_trans">>, <<"C352DDC1A0D5E54F9B1654D45F9C432EEF76F9CEA371C55DDFF769EF688FDB74">>}, + {<<"poolboy">>, <<"392B007A1693A64540CEAD79830443ABF5762F5D30CF50BC95CB2C1AAAFA006B">>}, + {<<"quickrand">>, <<"D2BD76676A446E6A058D678444B7FDA1387B813710D1AF6D6E29BB92186C8820">>}, + {<<"ranch">>, <<"8C7A100A139FD57F17327B6413E4167AC559FBC04CA7448E9BE9057311597A1D">>}, + {<<"setup">>, <<"05F69185A5EB71474C9BC6BA892565651EC7507791F85632B7B914DBFE130510">>}, + {<<"uuid">>, <<"B2078D2CC814F53AFA52D36C91E08962C7E7373585C623F4C0EA6DFB04B2AF94">>}]}, +{pkg_hash_ext,[ + {<<"bear">>, <<"157B67901ADF84FF0DA6EAE035CA1292A0AC18AA55148154D8C582B2C68959DB">>}, + {<<"cowboy">>, <<"8A7ABE6D183372CEB21CAA2709BEC928AB2B72E18A3911AA1771639BEF82651E">>}, + {<<"cowlib">>, <<"E1E1284DC3FC030A64B1AD0D8382AE7E99DA46C3246B815318A4B848873800A4">>}, + {<<"esaml">>, <<"4697E5CDD70C9EA0FD7FF8FF1C4049566BFA9202588B56924B181C3B2976D67D">>}, + {<<"exometer_core">>, <<"EC1AF0A961F29B3854F61B594C79454D26AC7B1F6CB89BBF0688BFF65F6A6D6B">>}, + {<<"folsom">>, <<"DD6AB97278E94F9E4CFC43E188224A7B8C7EAEC0DD2E935007005177F3EEBB0E">>}, + {<<"getopt">>, <<"7E01DE90AC540F21494FF72792B1E3162D399966EBBFC674B4CE52CB8F49324F">>}, + {<<"hut">>, <<"7E15D28555D8A1F2B5A3A931EC120AF0753E4853A4C66053DB354F35BF9AB563">>}, + {<<"jason">>, <<"3851B6150D231F203852822B77D0F60A069F89ADD1260E3033755345E0B604D4">>}, + {<<"jsx">>, <<"0C5CC8FDC11B53CC25CF65AC6705AD39E54ECC56D1C22E4ADB8F5A53FB9427F3">>}, + {<<"mochiweb">>, <<"975466D335403A78CD58186636B8E960E3C84C4D9C1A85EB7FE53B6A5DD54DE7">>}, + {<<"parse_trans">>, <<"4C25347DE3B7C35732D32E69AB43D1CEEE0BEAE3F3B3ADE1B59CBD3DD224D9CA">>}, + {<<"poolboy">>, <<"DAD79704CE5440F3D5A3681C8590B9DC25D1A561E8F5A9C995281012860901E3">>}, + {<<"quickrand">>, <<"B8ACBF89A224BC217C3070CA8BEBC6EB236DBE7F9767993B274084EA044D35F0">>}, + {<<"ranch">>, <<"49FBCFD3682FAB1F5D109351B61257676DA1A2FDBE295904176D5E521A2DDFE5">>}, + {<<"setup">>, <<"EFD072578F0CF85BEA96CAAFFC7ADB0992398272522660A136E10567377071C5">>}, + {<<"uuid">>, <<"4E4C5CA3461DC47C5E157ED42AA3981A053B7A186792AF972A27B14A9489324E">>}]} +]. diff --git a/rebar3 b/rebar3 new file mode 100755 index 000000000..043b35212 Binary files /dev/null and b/rebar3 differ diff --git a/rel/.gitignore b/rel/.gitignore new file mode 100644 index 000000000..2af97318d --- /dev/null +++ b/rel/.gitignore @@ -0,0 +1,2 @@ +vars/*_vars.config +riak-cs/ diff --git a/rel/files/advanced.config b/rel/files/advanced.config index 9f23a5e7e..7172f7aba 100644 --- a/rel/files/advanced.config +++ b/rel/files/advanced.config @@ -1,5 +1,11 @@ [ {riak_cs, [ - ]} + ] + }, + {setup, + [{data_dir, "{{platform_data_dir}}"}, + {log_dir, "{{platform_log_dir}}"} + ] + } ]. diff --git a/rel/files/app_epath.escript b/rel/files/app_epath.escript new file mode 100644 index 000000000..f84608c9a --- /dev/null +++ b/rel/files/app_epath.escript @@ -0,0 +1,9 @@ +-module(app_epath). +-export([main/1]). + +main([AdvancedConfig]) -> + {ok, [AA]} = file:consult(AdvancedConfig), + [p(A) || A <- AA]. + +p({App, CC}) -> + [io:format("~s ~s ~9999p\n", [App, K, V]) || {K, V} <- CC]. diff --git a/rel/files/erl b/rel/files/erl deleted file mode 100755 index d97c51482..000000000 --- a/rel/files/erl +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -## This script replaces the default "erl" in erts-VSN/bin. This is necessary -## as escript depends on erl and in turn, erl depends on having access to a -## bootscript (start.boot). Note that this script is ONLY invoked as a side-effect -## of running escript -- the embedded node bypasses erl and uses erlexec directly -## (as it should). -## -## Note that this script makes the assumption that there is a start_clean.boot -## file available in $ROOTDIR/release/VSN. - -# Determine the abspath of where this script is executing from. -ERTS_BIN_DIR=$(cd ${0%/*} && pwd) - -# Now determine the root directory -- this script runs from erts-VSN/bin, -# so we simply need to strip off two dirs from the end of the ERTS_BIN_DIR -# path. -ROOTDIR=${ERTS_BIN_DIR%/*/*} - -# Parse out release and erts info -START_ERL=`cat $ROOTDIR/releases/start_erl.data` -ERTS_VSN=${START_ERL% *} -APP_VSN=${START_ERL#* } - -BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin -EMU=beam -PROGNAME=`echo $0 | sed 's/.*\///'` -CMD="$BINDIR/erlexec" -export EMU -export ROOTDIR -export BINDIR -export PROGNAME - -exec $CMD -boot $ROOTDIR/releases/$APP_VSN/start_clean ${1+"$@"} \ No newline at end of file diff --git a/rel/files/hooks/erl_codeloadingmode b/rel/files/hooks/erl_codeloadingmode new file mode 100755 index 000000000..b6c8c6716 --- /dev/null +++ b/rel/files/hooks/erl_codeloadingmode @@ -0,0 +1,3 @@ +#!/bin/sh +CODE_LOADING_MODE="${CODE_LOADING_MODE:-interactive}" +export CODE_LOADING_MODE diff --git a/rel/files/hooks/ok b/rel/files/hooks/ok new file mode 100755 index 000000000..569f245f0 --- /dev/null +++ b/rel/files/hooks/ok @@ -0,0 +1,3 @@ +#!/bin/sh + +echo "ok" diff --git a/rel/files/hooks/riak_not_running b/rel/files/hooks/riak_not_running new file mode 100755 index 000000000..bc71fedff --- /dev/null +++ b/rel/files/hooks/riak_not_running @@ -0,0 +1,8 @@ +#!/bin/sh + +RIAK="{{platform_bin_dir}}/riak-cs" + +if $RIAK "ping" > /dev/null; then + echo "Node is already running!" + exit 1 +fi diff --git a/rel/files/lib.sh b/rel/files/lib.sh new file mode 100644 index 000000000..275399a11 --- /dev/null +++ b/rel/files/lib.sh @@ -0,0 +1,95 @@ +if [ "$OTP_VER" \< "23" ]; then + opening_bracket="[[" + closing_bracket="]]" +else + opening_bracket="[" + closing_bracket="]" +fi + +fmt() { + printf "$opening_bracket" + local n=$# + if [ $n -ne 0 ]; then + printf "\"$1\"" + shift + n=$((n-1)) + fi + while [ $n -ne 0 ]; do + printf ", \"$1\"" + shift + n=$((n-1)) + done + printf "$closing_bracket\n" +} + +rpc() { + local mod=$1 + local fun=$2 + shift 2 + if [ $# -gt 0 ]; then + "${PLATFORM_BIN_DIR}/riak-cs" rpc $mod $fun `fmt "$@"` + else + "${PLATFORM_BIN_DIR}/riak-cs" rpc $mod $fun "[[]]" + fi +} + + +## Example usage: +# +# #!/bin/sh +# +# # Load the functions +# . path/to/app_epath.sh +# +# # Build the path info +# epaths=`make_app_epaths path/to/app.config` +# +# # View all of the settings. Quotes are important. +# echo "$epaths" +# +# # Grep around in the paths for various items. +# echo "$epaths" | grep 'riak_core ring_creation_size' +# echo "$epaths" | grep "lager handlers lager_file_backend" | grep info +# +# # Use the epath function to get directly at settings +# epath 'riak_core ring_creation_size' "$epaths" +# epath 'riak_core platform_bin_dir' "$epaths" +# epath 'riak_kv storage_backend' "$epaths" +# +# # Use epath to view all of the riak_core settings +# epath riak_core +# +## End example + +## Here is a command you can put in your path for general cli use. +# +# #!/bin/sh +# +# # $1 quoted eterm path for search: 'riak_core ring_creation_size' +# # $2 path/to/app.config +# +# . path/to/app_epath.sh +# +# epaths=`make_app_epaths "$2"` +# epath "$1" "$epaths" +# +## End epath command + +# make_app_epaths takes a path to an app.config file as its argument and +# and returns (prints) a flattened text structure of configuration settings. + +make_app_epaths () { + ERTS_VER=$(cd ${PLATFORM_BASE_DIR} && ls -d erts-*) + ERTS_PATH="${PLATFORM_BASE_DIR}/$ERTS_VER/bin" + $ERTS_PATH/escript $cs_lib_dir/app_epath.escript "$1" +} + +epath () { + # arg1 - a pattern to search for + # arg2 - output of make_app_epaths, passed in quoted + # output - search of arg2 for arg1, trimming arg1 from the beginning + # Note: there may be multiple lines of output. + pat=$1 + shift + echo "$*" | grep "$pat " | sed "s/^${pat} *//" +} diff --git a/rel/files/riak-cs b/rel/files/riak-cs new file mode 100755 index 000000000..a36fe88ea --- /dev/null +++ b/rel/files/riak-cs @@ -0,0 +1,82 @@ +#!/bin/bash + +## ------------------------------------------------------------------- +## +## riak-cs: Riak CS launcher +## +## Copyright (c) 2021 TI Tokyo. All Rights Reserved. +## +## This file is provided to you under the Apache License, +## Version 2.0 (the "License"); you may not use this file +## except in compliance with the License. You may obtain +## a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, +## software distributed under the License is distributed on an +## "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +## KIND, either express or implied. See the License for the +## specific language governing permissions and limitations +## under the License. +## +## ------------------------------------------------------------------- + +RUNNER_GEN_DIR={{platform_gen_dir}} +RELX_RIAK={{platform_bin_dir}}/riak-cs +export PID_DIR={{pid_dir}} +export RUNNER_LOG_DIR={{platform_log_dir}} + +mkdir -p $PID_DIR +chown riak_cs:riak_cs $PID_DIR + +# cuttlefish should be doing this, but it doesn't: +VMARGS_PATH=`ls -1 ${RUNNER_GEN_DIR}/generated.conf/vm.*.args 2>/dev/null | tail -1` +if [ ! -r "$VMARGS_PATH" ]; then + VMARGS_PATH="{{platform_base_dir}}/releases/{{rel_vsn}}/vm.args" +fi +export VMARGS_PATH + +# centos7-based distros have a su that contacts pam and prints the "Last logged in" message +if [ "`cat /etc/redhat-release 2>&1`" = "CentOS Stream release 7" ] || + [ "`cat /etc/system-release 2>&1`" = "Amazon Linux release 2 (Karoo)" ]; then + COPTION="--session-command" +else + COPTION="-c" +fi + +function maybe_su { + if [[ $EUID -ne 0 ]]; then + $* + else + # if we are executing an admin command that spins up a + # (hidden) node to then execute custom erlang code via -eval, + # we need to cd to a dir containing the erlang cookie + # (previously implicitly done by su -, which option we have + # removed in order to allow any env vars to be available for + # the ultimate invocation of riak/riak-cs/stanchion) + cd "{{platform_base_dir}}" + # freebsd su is fairly limited, so: + mkdir -p "$RUNNER_GEN_DIR" + chown riak_cs:riak_cs "$RUNNER_GEN_DIR" + f=`mktemp "$RUNNER_GEN_DIR"/su_piggy-XXXXXXX` + cat >"$f" <" instead of this. -############################################################################ -EOS - -{{runner_script_dir}}/riak-cs-admin access "$@" diff --git a/rel/files/riak-cs-admin b/rel/files/riak-cs-admin index d1252b3f1..5045f416d 100755 --- a/rel/files/riak-cs-admin +++ b/rel/files/riak-cs-admin @@ -1,38 +1,91 @@ -#!/bin/sh -# -*- tab-width:4;indent-tabs-mode:nil -*- -# ex: ts=4 sw=4 et +#!/bin/bash -# Pull environment for this install -. "{{runner_base_dir}}/lib/env.sh" +## ------------------------------------------------------------------- +## +## riak-cs-admin: Administrative tasks on Riak CS +## +## Copyright (c) 2014 Basho Technologies, Inc., +## 2021-2023 TI Tokyo. All Rights Reserved. +## +## This file is provided to you under the Apache License, +## Version 2.0 (the "License"); you may not use this file +## except in compliance with the License. You may obtain +## a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, +## software distributed under the License is distributed on an +## "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +## KIND, either express or implied. See the License for the +## specific language governing permissions and limitations +## under the License. +## +## ------------------------------------------------------------------- -# Make sure the user running this script is the owner and/or su to that user -check_user "$@" +SCRIPT="riak-cs-admin" -# Make sure CWD is set to runner run dir -cd $RUNNER_BASE_DIR +PLATFORM_BASE_DIR={{platform_base_dir}} +PLATFORM_BASE_DIR=${PLATFORM_BASE_DIR:-$(cd $(dirname "$0")/.. && pwd -P)} -# Identify the script name -SCRIPT=`basename $0` +PLATFORM_BIN_DIR={{platform_bin_dir}} +if [ "$PLATFORM_BIN_DIR" = "${PLATFORM_BIN_DIR#/}" ]; then + PLATFORM_BIN_DIR=$PLATFORM_BASE_DIR/$PLATFORM_BIN_DIR +fi + +PLATFORM_LIB_DIR={{platform_lib_dir}} +if [ "$PLATFORM_LIB_DIR" = "${PLATFORM_LIB_DIR#/}" ]; then + PLATFORM_LIB_DIR=$PLATFORM_BASE_DIR/$PLATFORM_LIB_DIR +fi +. $PLATFORM_LIB_DIR/lib.sh + +export USE_NODETOOL=1 # {use_nodetool, true} is not clear enough usage() { - echo "Usage: $SCRIPT { status | gc | access | storage | stanchion | cluster-info | cleanup-orphan-multipart | audit-bucket-ownership}" + echo "Usage: $SCRIPT { access | audit-bucket-ownership | cleanup-orphan-multipart |" + echo " cluster-info | create-admin-user | gc | stanchion | status |" + echo " storage | supps | test | version }" } # Check the first argument for instructions case "$1" in + version) + shift + rpc riak_cs_console version + ;; status) shift - node_up_check - $NODETOOL rpc riak_cs_console status + rpc riak_cs_console status + ;; + supps) + shift + rpc riak_cs_console supps $@ + ;; + create[_-]admin[_-]user) + shift + case "$1" in + "--dont-touch-riak-cs-conf"|"-n"|"--terse"|"-t"|"") + output=`rpc riak_cs_console create_admin $@` + echo "$output" + key=`echo "$output" | sed -ne 's/ *KeyId: \(.*\)/\1/p'` + if [ -z "$1" ] && [ -n "$key" ]; then + read -p "Do you want to paste this key into your riak-cs.conf now? " a + if [ x$a = x"y" ] || [ x$a = x"Y" ]; then + sed -i "s/admin\\.key = .*/admin.key = $key/" /etc/riak-cs/riak-cs.conf + fi + fi + ;; + *) + echo "Usage: $SCRIPT create-admin-user [--terse|-t] [--dont-touch-riak-cs-conf|-n]" + exit 1 + ;; + esac ;; gc) shift case "$1" in batch|status|pause|resume|cancel|set-interval|set-leeway|earliest-keys) - # Make sure the local node IS running - node_up_check - - $NODETOOL rpc riak_cs_gc_console "$@" + rpc riak_cs_gc_console "$@" ;; *) echo "Usage: $SCRIPT gc { batch [|--help] | status | pause | resume | cancel |" @@ -46,10 +99,7 @@ case "$1" in shift case "$1" in batch|status|pause|resume|cancel) - # Make sure the local node IS running - node_up_check - - $NODETOOL rpc riak_cs_storage_console "$@" + rpc riak_cs_storage_console "$@" ;; *) echo "Usage: $SCRIPT storage $1 { batch | status | pause | resume | cancel }" @@ -61,10 +111,7 @@ case "$1" in shift case "$1" in flush) - # Make sure the local node IS running - node_up_check - - $NODETOOL rpc riak_cs_access_console flush "$@" + rpc riak_cs_access_console flush "$@" ;; *) echo "Usage: $SCRIPT access $1 { flush }" @@ -75,18 +122,19 @@ case "$1" in stanchion) shift case "$1" in - switch|show) - # Make sure the local node IS running - node_up_check - - $NODETOOL rpc riak_cs_stanchion_console "$@" + show) + rpc riak_cs_stanchion_console "$@" ;; *) - echo "Usage: $SCRIPT stanchion $1 { switch HOST PORT | show }" + echo "Usage: $SCRIPT stanchion $1 show" exit 1 ;; esac ;; + 'test') + shift + rpc riak_cs_console test + ;; cluster[_-]info) if [ $# -lt 2 ]; then echo "Usage: $SCRIPT $1 " @@ -94,26 +142,22 @@ case "$1" in fi shift - # Make sure the local node is running - node_up_check - - $NODETOOL rpc_infinity riak_cs_console cluster_info "$@" + rpc riak_cs_console cluster_info "$@" ;; cleanup[_-]orphan[_-]multipart) shift - # Make sure the local node IS running - node_up_check - - $NODETOOL rpc riak_cs_console cleanup_orphan_multipart "$@" + rpc riak_cs_console cleanup_orphan_multipart "$@" ;; audit[_-]bucket[_-]ownership) shift - node_up_check - $NODETOOL rpc riak_cs_console audit_bucket_ownership "$@" + rpc riak_cs_console audit_bucket_ownership "$@" ;; *) usage exit 1 ;; esac + +# No explicit exit from within script or nodetool, assumed to have succeeded. +exit 0 diff --git a/rel/files/riak-cs-chkconfig b/rel/files/riak-cs-chkconfig new file mode 100755 index 000000000..8671714a8 --- /dev/null +++ b/rel/files/riak-cs-chkconfig @@ -0,0 +1,30 @@ +#!/bin/sh + +PLATFORM_BASE_DIR={{platform_base_dir}} +PLATFORM_BASE_DIR=${PLATFORM_BASE_DIR:-$(cd $(dirname "$0")/.. && pwd -P)} +RUNNER_GEN_DIR={{platform_gen_dir}} + +RELEASE_ROOT_DIR=$PLATFORM_BASE_DIR + +ERTS_VER=$(cd ${PLATFORM_BASE_DIR} && ls -d erts-*) +ERTS_DIR="${PLATFORM_BASE_DIR}/$ERTS_VER" + +. $PLATFORM_BASE_DIR/bin/cf_config + +BOOT_FILE="${PLATFORM_BASE_DIR}/releases/{{release_version}}/start_clean" + +CODE=" try + {ok, _} = file:consult(\"$CONFIG_PATH\"), + io:format(\"config is OK\\n\"), + halt(0) + catch + _:_ -> + io:format(\"Error reading ~p\\n\", [\"$CONFIG_PATH\"]), + halt(1) + end." + +$ERTS_DIR/bin/erl -noshell -noinput \ + -boot $BOOT_FILE \ + -eval "$CODE" + +echo $CUTTLE_CONF diff --git a/rel/files/riak-cs-debug b/rel/files/riak-cs-debug index 8af20aad8..af64eb062 100755 --- a/rel/files/riak-cs-debug +++ b/rel/files/riak-cs-debug @@ -4,7 +4,8 @@ ## ## riak-cs-debug: Gather info from a node for troubleshooting. ## -## Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. +## Copyright (c) 2014 Basho Technologies, Inc., +## 2021 TI Tokyo. All Rights Reserved. ## ## This file is provided to you under the Apache License, ## Version 2.0 (the "License"); you may not use this file @@ -22,6 +23,12 @@ ## ## ------------------------------------------------------------------- +set +e +# this is to revert the effect of `set -e`, set by relx-generated code +# when we are invoked via `riak-cs debug` (otherwise, non-zero exits, +# which occur normally, will silently cause riak-cs-debug to +# terminate) + # /bin/sh on Solaris is not a POSIX compatible shell, but /usr/bin/ksh is. if [ `uname -s` = 'SunOS' -a "${POSIX_SHELL}" != "true" ]; then POSIX_SHELL="true" @@ -33,10 +40,55 @@ if [ `uname -s` = 'SunOS' -a "${POSIX_SHELL}" != "true" ]; then fi unset POSIX_SHELL # clear it so if we invoke other scripts, they run as ksh as well +### +### Set up variables +### + +is_relative() { + if [ "${1#/}" = "$1" ]; then + return 0 + else + return 1 + fi +} + +PLATFORM_BASE_DIR={{platform_base_dir}} +PLATFORM_BASE_DIR=${PLATFORM_BASE_DIR:-$(cd $(dirname "$0")/.. && pwd -P)} +cs_base_dir=$PLATFORM_BASE_DIR +if is_relative "{{platform_bin_dir}}"; then + cs_bin_dir="$PLATFORM_BASE_DIR/{{platform_bin_dir}}" +else + cs_bin_dir="{{platform_bin_dir}}" +fi +if is_relative "{{platform_etc_dir}}"; then + cs_etc_dir="$PLATFORM_BASE_DIR/{{platform_etc_dir}}" +else + cs_etc_dir="{{platform_etc_dir}}" +fi +if is_relative "{{platform_log_dir}}"; then + cs_log_dir="$PLATFORM_BASE_DIR/{{platform_log_dir}}" +else + cs_log_dir="{{platform_log_dir}}" +fi +if is_relative "{{platform_lib_dir}}"; then + cs_lib_dir="$PLATFORM_BASE_DIR/{{platform_lib_dir}}" +else + cs_lib_dir="{{platform_lib_dir}}" +fi +if is_relative "{{platform_gen_dir}}"; then + cs_gen_dir="$PLATFORM_BASE_DIR/{{platform_gen_dir}}" +else + cs_gen_dir="{{platform_gen_dir}}" +fi + +cuttlefish_conf_dir=${cs_gen_dir}/generated.conf + ### ### Function declarations ### +. $cs_lib_dir/lib.sh + echoerr () { echo "$@" 1>&2; } mkdir_or_die () { @@ -127,23 +179,28 @@ EOF exit } -### -### Set up variables -### +curdir=`pwd` +cd "$PLATFORM_BASE_DIR" -# These paths may be overridden with environment variables. -RUNNER_BASE_DIR={{runner_base_dir}} -cs_base_dir={{runner_base_dir}} -cs_bin_dir={{runner_script_dir}} -cs_etc_dir={{runner_etc_dir}} -cs_log_dir={{runner_log_dir}} +gen_result=`"{{platform_bin_dir}}"/riak-cs-chkconfig | grep -v "OK"` +app_config=`echo $gen_result | cut -d' ' -f 2` -if [ -d {{platform_data_dir}}/generated.configs ]; then - cuttlefish_conf_dir={{platform_data_dir}}/generated.configs -else - cuttlefish_conf_dir={{runner_base_dir}}/{{platform_data_dir}}/generated.configs +# make a flat, searchable version of the app.config +if [ -z "$app_config" ]; then + echoerr "App configs missing in {{platform_base_dir}}/generated.conf. Aborting." + exit 1 +fi +riak_epaths=`make_app_epaths "${app_config}"` + +if [ -d "${cuttlefish_conf_dir}" ]; then + vmargs=`ls -t ${cuttlefish_conf_dir}/vm.*.args 2>/dev/null | head -1` fi + +### +### Set defaults for items to gather in the report +### + get_cfgs=0 get_ssl_certs=0 get_logs=0 @@ -249,24 +306,6 @@ if [ 0 -eq $get_cfgs -a 1 -eq $get_ssl_certs ]; then fi if [ 0 -ne $(( $get_cfgs + $get_logs + $get_riakcmds + $get_extracmds )) ]; then - # Information specific to Riak requested. Need app_epath.sh and must have - # valid base and etc paths defined. - - # app_epath.sh provides make_app_epaths and epath functions. - # Allow overriding with APP_EPATH - if [ -n "$APP_EPATH" ]; then - epath_file="$APP_EPATH" - else - epath_file="${cs_base_dir}/lib/app_epath.sh" - fi - - if [ ! -f "$epath_file" ]; then - echoerr "Required file app_epath.sh not found. Expected here:" - echoerr "$epath_file" - echoerr "See 'riak-cs-debug -h' and manpage for help." - exit 1 - fi - . "$epath_file" # Allow overriding cs_base_dir and cs_etc_dir from the environment if [ -n "$cs_base_dir" ]; then @@ -300,13 +339,6 @@ if [ 0 -ne $(( $get_cfgs + $get_logs + $get_riakcmds + $get_extracmds )) ]; then fi fi -#prefer vm.args if it exists -if [ -f "${cs_etc_dir}/vm.args" ]; then - vmargs="${cs_etc_dir}/vm.args" -elif [ -d "${cuttlefish_conf_dir}" ]; then - vmargs=`ls -t ${cuttlefish_conf_dir}/vm.*.args | head -1` -fi - if [ -f "${vmargs}" ]; then node_name="`egrep '^\-s?name' "${vmargs}" 2>/dev/null | cut -d ' ' -f 2`" @@ -336,7 +368,7 @@ fi if [ -z "$outfile" ]; then # If output file not specified, output to the default - outfile="`pwd`"/"${debug_dir}".tar.gz + outfile="${curdir}"/"${debug_dir}".tar.gz fi if [ '-' != "$outfile" ] && [ -f "$outfile" ]; then @@ -390,7 +422,7 @@ if [ 1 -eq $get_syscmds ]; then BLOCKDEV=/sbin/blockdev if [ -e $BLOCKDEV ]; then - for mount_point in `mount | egrep '^/' | awk '{print $1}'`; do + for mount_point in `mount | grep -E '^/' | grep -v /dev/shm | grep -v /var/lib/snapd | awk '{print $1}'`; do flat_point=`echo $mount_point | sed 's:/:_:g'` dump blockdev.$flat_point $BLOCKDEV --getra $mount_point done @@ -469,14 +501,14 @@ if [ 1 -eq $get_riakcmds ]; then mkdir_or_die "${start_dir}"/"${debug_dir}"/commands/.info cd "${start_dir}"/"${debug_dir}"/commands - dump riak_cs_ping "$cs_bin_dir"/riak-cs ping - dump riak_cs_version "$cs_bin_dir"/riak-cs version - dump riak_cs_status "$cs_bin_dir"/riak-cs-admin status - dump riak_cs_gc_status "$cs_bin_dir"/riak-cs-admin gc status - dump riak_cs_storage_status "$cs_bin_dir"/riak-cs-admin storage status - if [ -f "$cs_bin_dir/riak-cs-multibag" ]; then - dump riak_cs_multibag_list_bags "$cs_bin_dir"/riak-cs-multibag list-bags - dump riak_cs_multibag_weight "$cs_bin_dir"/riak-cs-multibag weight + dump riak_cs_ping riak-cs ping + dump riak_cs_version riak-cs admin version + dump riak_cs_status riak-cs admin status + dump riak_cs_gc_status riak-cs admin gc status + dump riak_cs_storage_status riak-cs admin storage status + if grep -q supercluster $cs_etc_dir/riak-cs.conf; then + dump riak_cs_multibag_list_bags riak-cs supercluster list-members + dump riak_cs_multibag_weight riak-cs supercluster weight fi CI=`pwd`/cluster-info.html @@ -494,25 +526,13 @@ if [ 1 -eq $get_logs ]; then mkdir_or_die "${start_dir}"/"${debug_dir}"/logs/.info cd "${start_dir}"/"${debug_dir}"/logs - # if not already made, make a flat, searchable version of the app.config - if [ -z "$riak_epaths" ]; then - #prefer app.config if it exists - if [ -f "${cs_etc_dir}/app.config" ]; then - appconfig="${cs_etc_dir}/app.config" - riak_epaths=`make_app_epaths "${appconfig}"` - elif [ -d "${cuttlefish_conf_dir}" ]; then - appconfig=`ls -t ${cuttlefish_conf_dir}/app.*.config | head -1` - riak_epaths=`make_app_epaths "${appconfig}"` - fi - fi - # grab a listing of the log directory to show if there are crash dumps dump ls_log_dir ls -lhR $cs_log_dir # Get any logs in the platform_log_dir if [ -d "${cs_log_dir}" ]; then # check to ensure there is at least something to get - ls -1 "${cs_log_dir}" | grep -q "log" + ls -1 "${cs_log_dir}" 2>/dev/null | grep -q "log" if [ 0 -eq $? ]; then mkdir_or_die "${start_dir}"/"${debug_dir}"/logs/platform_log_dir @@ -529,38 +549,6 @@ if [ 1 -eq $get_logs ]; then fi fi fi - - # Lager info and error files - new_format_lager_files="`epath 'lager handlers lager_file_backend file' "$riak_epaths" | sed -e 's/^"//' -e 's/".*$//'`" - if [ -z "$riak_epaths" ]; then - lager_files="" - elif [ -z "$new_format_lager_files" ]; then - lager_files="`epath 'lager handlers lager_file_backend' "$riak_epaths" | cut -d' ' -f 1 | sed -e 's/^"//' -e 's/".*$//'`" - else - lager_files=$new_format_lager_files - fi - lager_num=0 - for lager_path in $lager_files; do - # Get lager logs if they weren't in platform_log_dir - if [ -n "$lager_path" ]; then - if [ '/' != `echo "$lager_path" | cut -c1` ]; then - # relative path. prepend base dir - lager_path="$cs_base_dir"/"$lager_path" - fi - - lager_file="`echo $lager_path | awk -F/ '{print $NF}'`" - lager_dir="`echo $lager_path | awk -F/$lager_file '{print $1}'|sed 's|/\./|/|'`" - if [ "$cs_log_dir" != "$lager_dir" ]; then - ls -1 "${lager_dir}" | grep -q "${lager_file}" - if [ 0 -eq $? ]; then - mkdir_or_die "${start_dir}"/"${debug_dir}"/logs/lager_${lager_num} - find "${lager_dir}" -mtime -$log_mtime -name "${lager_file}*" \ - -exec cp -p {} "${start_dir}"/"${debug_dir}"/logs/lager_"${lager_num}"/ \; - fi - fi - lager_num=`expr $lager_num + 1` - fi - done fi ### @@ -591,12 +579,12 @@ if [ 1 -eq $get_cfgs ]; then if [ -z "$exclude" ]; then run="find . -type f -exec sh -c ' mkdir -p \"\$0/\${1%/*}\"; - cp \"\$1\" \"\$0/\$1\" + cp -a \"\$1\" \"\$0/\$1\" ' \"\${start_dir}\"/\"\${debug_dir}\"/config {} \;" else run="find . -type f \\( $exclude \\) -exec sh -c ' mkdir -p \"\$0/\${1%/*}\"; - cp \"\$1\" \"\$0/\$1\" + cp -a \"\$1\" \"\$0/\$1\" ' \"\${start_dir}\"/\"\${debug_dir}\"/config {} \;" fi eval $run diff --git a/rel/files/riak-cs-gc b/rel/files/riak-cs-gc deleted file mode 100755 index 89367cabd..000000000 --- a/rel/files/riak-cs-gc +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -cat<" instead of this. -############################################################################ -EOS - -{{runner_script_dir}}/riak-cs-admin gc "$@" diff --git a/rel/files/riak-cs-multibag b/rel/files/riak-cs-multibag deleted file mode 100755 index 7d795904b..000000000 --- a/rel/files/riak-cs-multibag +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh - -cat<" instead of this. -############################################################################ -EOS - -# Pull environment for this install -. "{{runner_base_dir}}/lib/env.sh" - -# Make sure the user running this script is the owner and/or su to that user -check_user $@ - -# Make sure CWD is set to runner run dir -cd $RUNNER_BASE_DIR - -# Identify the script name -SCRIPT=`basename $0` - -# Check the first argument for instructions -case "$1" in - list-members|weight|weight-manifest|weight-block|refresh) - # Make sure the local node IS running - node_up_check - - $NODETOOL rpc riak_cs_multibag_console "$@" - ;; - list-bags) - node_up_check - $NODETOOL rpc riak_cs_multibag_console list-members - ;; - *) - echo "Usage: $SCRIPT { list-members | weight | weight-manifest | weight-block | refresh | list-bags }" - exit 1 - ;; -esac diff --git a/rel/files/riak-cs-stanchion b/rel/files/riak-cs-stanchion deleted file mode 100755 index b3181df1a..000000000 --- a/rel/files/riak-cs-stanchion +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -cat<" instead of this. -############################################################################ -EOS - -{{runner_script_dir}}/riak-cs-admin stanchion "$@" diff --git a/rel/files/riak-cs-storage b/rel/files/riak-cs-storage deleted file mode 100755 index 8d5c0423c..000000000 --- a/rel/files/riak-cs-storage +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -cat<" instead of this. -############################################################################ -EOS - -{{runner_script_dir}}/riak-cs-admin storage "$@" diff --git a/rel/files/riak-cs-supercluster b/rel/files/riak-cs-supercluster index e9f4c5cf4..08f4df590 100755 --- a/rel/files/riak-cs-supercluster +++ b/rel/files/riak-cs-supercluster @@ -1,27 +1,53 @@ #!/bin/sh +## ------------------------------------------------------------------- +## +## riak-cs-supercluster: Riak CS supercluster management +## +## Copyright (c) 2014 Basho Technologies, Inc., +## 2021-2023 TI Tokyo. All Rights Reserved. +## +## This file is provided to you under the Apache License, +## Version 2.0 (the "License"); you may not use this file +## except in compliance with the License. You may obtain +## a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, +## software distributed under the License is distributed on an +## "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +## KIND, either express or implied. See the License for the +## specific language governing permissions and limitations +## under the License. +## +## ------------------------------------------------------------------- -# Pull environment for this install -. "{{runner_base_dir}}/lib/env.sh" +SCRIPT="riak-cs-supercluster" +# if we are called as `riak admin`, PLATFORM_BASE_DIR will have been set +PLATFORM_BASE_DIR={{platform_base_dir}} +PLATFORM_BASE_DIR=${PLATFORM_BASE_DIR:-$(cd $(dirname "$0")/.. && pwd -P)} -# Make sure the user running this script is the owner and/or su to that user -check_user $@ +PLATFORM_BIN_DIR={{platform_bin_dir}} +if [ "$PLATFORM_BIN_DIR" = "${PLATFORM_BIN_DIR#/}" ]; then + PLATFORM_BIN_DIR=$PLATFORM_BASE_DIR/$PLATFORM_BIN_DIR +fi -# Make sure CWD is set to runner run dir -cd $RUNNER_BASE_DIR +PLATFORM_LIB_DIR={{platform_lib_dir}} +if [ "$PLATFORM_LIB_DIR" = "${PLATFORM_LIB_DIR#/}" ]; then + PLATFORM_LIB_DIR=$PLATFORM_BASE_DIR/$PLATFORM_LIB_DIR +fi +. $PLATFORM_LIB_DIR/lib.sh -# Identify the script name -SCRIPT=`basename $0` +export USE_NODETOOL=1 # {use_nodetool, true} is not clear enough # Check the first argument for instructions case "$1" in list-members|weight|weight-manifest|weight-block|refresh) - # Make sure the local node IS running - node_up_check - - $NODETOOL rpc riak_cs_multibag_console "$@" + rpc riak_cs_multibag_console "$@" ;; *) - echo "Usage: $SCRIPT { list-members | weight | weight-manifest | weight-block | refresh }" + echo "Usage: $SCRIPT { list-members | weight | weight-manifest | weight-block | refresh }" exit 1 ;; esac +exit 0 diff --git a/rel/gen_dev b/rel/gen_dev index c661bdbfb..b6f581900 100755 --- a/rel/gen_dev +++ b/rel/gen_dev @@ -11,15 +11,23 @@ VARFILE=$3 NODE="rcs-$NAME@127.0.0.1" +SCRIPT_DIR="`dirname $0`/.." +BUILD_ROOT="`cd "$SCRIPT_DIR" && pwd -P`" + NUMBER=${NAME##dev} RIAKBASE=$((10000 + 10 * $NUMBER)) +FIRSTRIAKPBPORT=$((10000 + 10 * 1 + 7)) CSBASE=$((15000 + 10 * $NUMBER)) RIAKPBPORT=$(($RIAKBASE + 7)) CSPORT=$(($CSBASE + 8)) ADMINPORT=$(($CSBASE + 9)) +PLATFORM_BASE_DIR="$BUILD_ROOT/dev/$NAME/riak-cs" + echo "Generating $NAME - node='$NODE' riakpbport=$RIAKPBPORT csport=$CSPORT adminport=$ADMINPORT" -sed -e "s/@NODE@/$NODE/" \ +sed -e "s|@PLATFORM_BASE_DIR@|$PLATFORM_BASE_DIR|" \ + -e "s/@NODE@/$NODE/" \ -e "s/@RIAKPBPORT@/$RIAKPBPORT/" \ + -e "s/@FIRSTRIAKPBPORT@/$FIRSTRIAKPBPORT/" \ -e "s/@CSPORT@/$CSPORT/" \ -e "s/@ADMINPORT@/$ADMINPORT/" < $TEMPLATE > $VARFILE diff --git a/rel/pkg/.gitignore b/rel/pkg/.gitignore new file mode 100644 index 000000000..89f9ac04a --- /dev/null +++ b/rel/pkg/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/rel/pkg/Makefile b/rel/pkg/Makefile new file mode 100644 index 000000000..f637f31c4 --- /dev/null +++ b/rel/pkg/Makefile @@ -0,0 +1,75 @@ +## +## Export all variables to sub-invocation +## +export + +OS = $(shell uname -s) +ERLANG_BIN ?= $(shell dirname $(shell which erl)) +DEPS_DIR ?= deps + +## +## Support RPM and Debian based linux systems +## +ifeq ($(OS),Linux) +ARCH = $(shell uname -m) +ISAMZ = $(shell cat /etc/system-release 2> /dev/null) +ISRPM = $(shell cat /etc/redhat-release 2> /dev/null) +ISAMZ = $(shell cat /etc/system-release 2> /dev/null) +ISDEB = $(shell cat /etc/debian_version 2> /dev/null) +ISSLES = $(shell cat /etc/SuSE-release 2> /dev/null) +ifneq ($(ISAMZ),) +OSNAME = Amazon +PKGERDIR = rpm +BUILDDIR = rpmbuild +else +ifneq ($(ISRPM),) +OSNAME = RedHat +PKGERDIR = rpm +BUILDDIR = rpmbuild +else +ifneq ($(ISDEB),) +OSNAME = Debian +PKGERDIR = deb +BUILDDIR = debuild +else +ifneq ($(ISSLES),) +OSNAME = SLES +PKGERDIR = rpm +BUILDDIR = rpmbuild +endif # SLES +endif # deb +endif # rpm +endif # amazon +endif # linux + +ifeq ($(OS),Darwin) # OSX +OSNAME = OSX +ARCH = $(shell file `which erlc` | grep -c x86_64 2> /dev/null | awk \ + '{if ($$1 == "0") {print "i386"} else {print "x86_64"}}') +PKGERDIR = osx +BUILDDIR = osxbuild +endif + +ifeq ($(OS),FreeBSD) +OSNAME = FreeBSD +ARCH = $(shell uname -m) +BUILDDIR = fbsdbuild +PKGERDIR = fbsdng +endif + +DATE = $(shell date +%Y-%m-%d) + +# Default the package build version to 1 if not already set +PKG_BUILD ?= 1 + +.PHONY: ostype varcheck + +## Call platform dependent makefile +ostype: varcheck + $(if $(PKGERDIR),,$(error "Operating system '$(OS)' not supported by node_package")) + $(MAKE) -C $(BASE_DIR)/rel/pkg/out/riak_cs-$(PKG_ID) -f $(BASE_DIR)/rel/pkg/$(PKGERDIR)/Makefile + +## Check required settings before continuing +varcheck: + $(if $(PKG_VERSION),,$(error "Variable PKG_VERSION must be set and exported, see basho/node_package readme")) + $(if $(PKG_ID),,$(error "Variable PKG_ID must be set and exported, see basho/node_package readme")) diff --git a/rel/pkg/alpine/README.md b/rel/pkg/alpine/README.md new file mode 100644 index 000000000..c0134d1ee --- /dev/null +++ b/rel/pkg/alpine/README.md @@ -0,0 +1,11 @@ +# Packaging Riak CS for Alpine Linux + +Alpine Linux is a minimalistic, Gentoo-inspired distribution. + +Packaging instructions for Alpine cannot be placed in +rel/pkg/alpine/Makefile without bending too many rules and +conventions. + +Instead, TI Tokyo [builds](https://github.com/TI-Tokyo/alpine-builds) +apks for x86_64 and aarch64 and maintains an external Alpine +repository at https://files.tiot.jp/alpine/. diff --git a/rel/pkg/alpine/riak-cs.nosu b/rel/pkg/alpine/riak-cs.nosu new file mode 100755 index 000000000..a67aecd60 --- /dev/null +++ b/rel/pkg/alpine/riak-cs.nosu @@ -0,0 +1,34 @@ +#!/bin/bash + +RUNNER_GEN_DIR={{platform_gen_dir}} +RELX_RIAK={{platform_bin_dir}}/riak-cs +export PID_DIR={{pid_dir}} +export RUNNER_LOG_DIR={{platform_log_dir}} + +PID_FILE=$PID_DIR/riak-cs.pid + +mkdir -p $PID_DIR + +# cuttlefish should be doing this, but it doesn't: +VMARGS_PATH=`ls -1 ${RUNNER_GEN_DIR}/generated.conf/vm.*.args 2>/dev/null | tail -1` +if [ ! -r "$VMARGS_PATH" ]; then + VMARGS_PATH="{{platform_base_dir}}/releases/{{rel_vsn}}/vm.args" +fi +export VMARGS_PATH + +case "$1" in + start) + $RELX_RIAK $* -pa {{platform_patch_dir}} + test -r $PID_FILE && exit 0 + ;; + console|foreground) + $RELX_RIAK $* -pa {{platform_patch_dir}} + ;; + stop) + $RELX_RIAK $* \ + && rm -f $PID_FILE + ;; + *) + $RELX_RIAK $* + ;; +esac diff --git a/rel/pkg/alpine/vars.config b/rel/pkg/alpine/vars.config new file mode 100644 index 000000000..3dca5691f --- /dev/null +++ b/rel/pkg/alpine/vars.config @@ -0,0 +1,37 @@ +%% -*- mode: erlang -*- + +{rel_vsn, "{{release_version}}"}. + +{platform_base_dir, "/usr/lib/riak-cs"}. +{platform_bin_dir, "/usr/lib/riak-cs/bin"}. +{platform_data_dir, "/var/lib/riak-cs"}. +{platform_etc_dir, "/etc/riak-cs"}. +{platform_lib_dir, "/usr/lib/riak-cs/lib"}. +{platform_log_dir, "/var/log/riak-cs"}. +{platform_gen_dir, "{{platform_data_dir}}"}. +{platform_patch_dir, "{{platform_lib_dir}}/patches"}. + +{cs_ip, "127.0.0.1"}. +{cs_port, 8080}. +{admin_ip, "127.0.0.1"}. +{admin_port, 8000}. +{riak_ip, "127.0.0.1"}. +{riak_pb_port, 8087}. +{auth_bypass, false}. +{admin_key, "admin-key"}. +{stanchion_port, 8085}. +{stanchion_ssl, off}. +{cs_version, 030300}. +{stanchion_hosting_mode, auto}. +{tussle_voss_riak_host, auto}. + +{node, "riak-cs@127.0.0.1"}. +{crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. + +{pid_dir, "/run/riak-cs"}. + +{log_level, info}. +{logger_sasl_enabled, false}. + +{cuttlefish, "on"}. +{cuttlefish_conf, "riak-cs.conf"}. diff --git a/rel/pkg/deb/Makefile b/rel/pkg/deb/Makefile new file mode 100644 index 000000000..5dae6987b --- /dev/null +++ b/rel/pkg/deb/Makefile @@ -0,0 +1,29 @@ +export + +TAR_VERSION = $(shell git describe --tags | sed -e 's/\([0-9.]*\-[0-9]*\)-.*/\1/') + +DEBEMAIL = $(shell git config user.email) + +default: + mkdir -p $(BASE_DIR)/rel/pkg/out/$(PKG_ID)/debian + cp -R $(BASE_DIR)/rel/pkg/deb/debian/* $(BASE_DIR)/rel/pkg/out/$(PKG_ID)/debian + + tar -xf $(BASE_DIR)/rel/pkg/out/$(PKG_ID).tar.gz -C $(BASE_DIR)/rel/pkg/out/$(PKG_ID)/ + ln -sf $(BASE_DIR)/rel/pkg/out/$(PKG_ID).tar.gz $(BASE_DIR)/rel/pkg/out/$(TAR_VERSION).orig.tar.gz + + cd $(BASE_DIR)/rel/pkg/out/$(PKG_ID)/; \ + (mkdir -p _build/default && cd _build/default && for d in lib; do ln -fs $(BASE_DIR)/_build/default/$$d; done); \ + dch --create --package riak-cs -v "$(PKG_VERSION)" \ + "Build from $(PKG_ID)";\ + debuild --prepend-path=$(ERLANG_BIN) \ + -e REVISION=$(PKG_VERSION) \ + -e RELEASE=$(PKG_BUILD) \ + -e REBAR=$(REBAR) \ + -i -uc -us -b + + mkdir -p $(BASE_DIR)/rel/pkg/out/packages + cd $(BASE_DIR)/rel/pkg/out && mv *.deb ../out/packages + cd $(BASE_DIR)/rel/pkg/out/packages && \ + for debfile in *.deb; do \ + sha256sum $${debfile} > $${debfile}.sha \ + ; done diff --git a/rel/pkg/deb/debian/compat b/rel/pkg/deb/debian/compat new file mode 100644 index 000000000..f599e28b8 --- /dev/null +++ b/rel/pkg/deb/debian/compat @@ -0,0 +1 @@ +10 diff --git a/rel/pkg/deb/debian/control b/rel/pkg/deb/debian/control new file mode 100644 index 000000000..f54daae3f --- /dev/null +++ b/rel/pkg/deb/debian/control @@ -0,0 +1,13 @@ +Source: riak-cs +Section: net +Priority: optional +Maintainer: Andrei Zavada +Build-Depends: debhelper (>= 9.20), libssl-dev +Standards-Version: 4.6.0.1 +Homepage: https://tiot.jp + +Package: riak-cs +Architecture: any +Depends: ${misc:Depends}, ${shlibs:Depends}, adduser, openssl +Homepage: https://tiot.jp +Description: Riak CS Database diff --git a/rel/pkg/deb/debian/copyright b/rel/pkg/deb/debian/copyright new file mode 100644 index 000000000..747043a40 --- /dev/null +++ b/rel/pkg/deb/debian/copyright @@ -0,0 +1,7 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: riak-cs +Upstream-Contact: Andrei Zavada + +Files: * +Copyright: 2012-16 Basho Technologies, Inc; 2021 TI Tokyo +License: Apache-2.0 diff --git a/rel/pkg/deb/debian/dirs b/rel/pkg/deb/debian/dirs new file mode 100644 index 000000000..e801954c3 --- /dev/null +++ b/rel/pkg/deb/debian/dirs @@ -0,0 +1,3 @@ +etc/logrotate.d +var/lib/riak-cs +var/log/riak-cs diff --git a/rel/pkg/deb/debian/install b/rel/pkg/deb/debian/install new file mode 100644 index 000000000..a18e37ab3 --- /dev/null +++ b/rel/pkg/deb/debian/install @@ -0,0 +1,8 @@ +rel/riak-cs/lib usr/lib/riak-cs +rel/riak-cs/share usr/lib/riak-cs +rel/riak-cs/erts* usr/lib/riak-cs +rel/riak-cs/releases usr/lib/riak-cs +rel/riak-cs/bin/riak-cs-* usr/sbin +rel/riak-cs/usr/bin/riak-cs usr/sbin +rel/riak-cs/bin usr/lib/riak-cs +rel/riak-cs/etc/* etc/riak-cs diff --git a/rel/pkg/deb/debian/postinst b/rel/pkg/deb/debian/postinst new file mode 100755 index 000000000..272674ad0 --- /dev/null +++ b/rel/pkg/deb/debian/postinst @@ -0,0 +1,43 @@ +#!/bin/sh +# postinst script for riak-cs +# +# see: dh_installdeb(1) + +set -e + +# create group +if ! getent group riak_cs >/dev/null; then + addgroup --system riak_cs +fi + +# create user +if ! getent passwd riak_cs >/dev/null; then + adduser --ingroup riak_cs \ + --home /var/lib/riak-cs \ + --disabled-password \ + --system --shell /bin/bash --no-create-home \ + --gecos "Riak CS Database" riak_cs +fi + +chown riak_cs:riak_cs /var/log/riak-cs /etc/riak-cs /var/lib/riak-cs +chmod 750 /var/log/riak-cs /etc/riak-cs /var/lib/riak-cs + +case "$1" in + configure) + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/rel/pkg/deb/debian/postrm b/rel/pkg/deb/debian/postrm new file mode 100755 index 000000000..be1c5bbaa --- /dev/null +++ b/rel/pkg/deb/debian/postrm @@ -0,0 +1,64 @@ +#!/bin/sh +# postrm script for riak_cs +# +# see: dh_installdeb(1) + + +# summary of how this script can be called: +# * `remove' +# * `purge' +# * `upgrade' +# * `failed-upgrade' +# * `abort-install' +# * `abort-install' +# * `abort-upgrade' +# * `disappear' +# +# for details, see http://www.debian.org/doc/debian-policy/ or +# the debian-policy package + +set -e + +case "$1" in + purge) + rm -f /etc/default/riak-cs + if [ -d /var/log/riak-cs ]; then + rm -r /var/log/riak-cs + fi + if [ -d /var/lib/riak-cs ]; then + rm -r /var/lib/riak-cs + fi + if [ -d /run/riak-cs ]; then + rm -r /var/run/riak-cs + fi + if [ -d /etc/riak-cs ]; then + rm -r /etc/riak-cs + fi + if [ -e /etc/init.d/riak-cs ]; then + rm /etc/init.d/riak-cs + fi + # Remove User & Group, killing any process owned by them + if getent passwd riak_cs >/dev/null; then + pkill -u riak_cs || true + deluser --quiet --system riak_cs + fi + if getent group riak_cs >/dev/null; then + delgroup --quiet --system --only-if-empty riak_cs || true + fi + ;; + + remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + ;; + + *) + echo "postrm called with unknown argument \`$1\`" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/rel/pkg/deb/debian/riak-cs.manpages b/rel/pkg/deb/debian/riak-cs.manpages new file mode 100644 index 000000000..e3db37c1b --- /dev/null +++ b/rel/pkg/deb/debian/riak-cs.manpages @@ -0,0 +1 @@ +doc/man/man1/*.1.gz \ No newline at end of file diff --git a/rel/pkg/deb/debian/riak-cs.service b/rel/pkg/deb/debian/riak-cs.service new file mode 100644 index 000000000..9b722eb35 --- /dev/null +++ b/rel/pkg/deb/debian/riak-cs.service @@ -0,0 +1,14 @@ +[Unit] +Description=Riak CS Database + +[Service] +User=riak_cs +ExecStart=/usr/sbin/riak-cs start +ExecStop=/usr/sbin/riak-cs stop +Type=simple +PIDFile=/var/run/riak-cs/riak-cs.pid +EnvironmentFile=-/etc/default/riak-cs +RuntimeDirectory=riak-cs + +[Install] +WantedBy=multi-user.target diff --git a/rel/pkg/deb/debian/rules b/rel/pkg/deb/debian/rules new file mode 100755 index 000000000..4942ff722 --- /dev/null +++ b/rel/pkg/deb/debian/rules @@ -0,0 +1,55 @@ +#!/usr/bin/make -f +# -*- makefile -*- +# Sample debian/rules that uses debhelper. +# This file was originally written by Joey Hess and Craig Small. +# As a special exception, when this file is copied by dh-make into a +# dh-make output file, you may use that output file without restriction. +# This special exception was added by Craig Small in version 0.37 of dh-make. + +# modified for node_package by dizzyd@basho.com and jared@basho.com + +# Uncomment this to turn on verbose mode. +export DH_VERBOSE=1 + +ROOTDIR := debian/riak-cs + +## Clear variables that may confound our build of sub-projects; also +## note that it is necessary to use overlay_vars relative to .. as +## the generate command EXECUTES in rel/ +build: + make rel-deb + rm -rf rel/riak-cs/lib/*/c_src rel/riak-cs/lib/*/src + touch build + +clean: + dh_clean + rm -f build + make clean + +## dh_shlibdeps was added to figure out the dependencies on shared libraries +## and will populate the ${shlibs:Depends} callout in the control file +install: LIBDIR := $(ROOTDIR)/usr/lib64/riak-cs +install: build + dh_testdir + dh_testroot + dh_prep + dh_installdirs + dh_install + dh_installman + dh_fixperms + dh_shlibdeps + dh_systemd_enable + +# We have nothing to do by default. +binary-indep: install build-stamp +build-stamp: + +# Build architecture-dependent files here. +binary-arch: install + dh_strip -a + dh_compress -a + dh_installdeb + dh_gencontrol + dh_builddeb + +binary: binary-indep binary-arch diff --git a/rel/pkg/deb/vars.config b/rel/pkg/deb/vars.config new file mode 100644 index 000000000..c3c9e43b6 --- /dev/null +++ b/rel/pkg/deb/vars.config @@ -0,0 +1,37 @@ +%% -*- mode: erlang -*- + +{rel_vsn, "{{release_version}}"}. + +{platform_base_dir, "/usr/lib/riak-cs"}. +{platform_bin_dir, "/usr/lib/riak-cs/bin"}. +{platform_data_dir, "/var/lib/riak-cs"}. +{platform_etc_dir, "/etc/riak-cs"}. +{platform_lib_dir, "/usr/lib/riak-cs/lib"}. +{platform_log_dir, "/var/log/riak-cs"}. +{platform_gen_dir, "{{platform_data_dir}}"}. +{platform_patch_dir, "{{platform_lib_dir}}/patches"}. + +{cs_ip, "127.0.0.1"}. +{cs_port, 8080}. +{admin_ip, "127.0.0.1"}. +{admin_port, 8000}. +{riak_ip, "127.0.0.1"}. +{riak_pb_port, 8087}. +{auth_bypass, false}. +{admin_key, "admin-key"}. +{stanchion_port, 8085}. +{stanchion_ssl, off}. +{cs_version, 030205}. +{stanchion_hosting_mode, auto}. +{tussle_voss_riak_host, auto}. + +{node, "riak-cs@127.0.0.1"}. +{crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. + +{pid_dir, "/var/run/riak-cs"}. + +{log_level, info}. +{logger_sasl_enabled, false}. + +{cuttlefish, "on"}. +{cuttlefish_conf, "riak-cs.conf"}. diff --git a/rel/pkg/fbsdng/+MANIFEST b/rel/pkg/fbsdng/+MANIFEST new file mode 100644 index 000000000..97f92c04e --- /dev/null +++ b/rel/pkg/fbsdng/+MANIFEST @@ -0,0 +1,18 @@ +name: "riak_cs" +origin: "databases" +comment: "Riak CS" +licenses: ["Apache 2"] +licenselogic: "single" +arch: "freebsd:13:x86:64" +www: "tiot.jp" +maintainer: "packaging@tiot.jp" +users: ["riak_cs"] +groups: ["riak_cs"] +prefix: "/usr/local" +categories: ["databases"] +desc: "Riak CS is a Riak-backed S3 drop-in replacement" +scripts: { + pre-install: "if ! pw groupshow riak_cs 2>/dev/null; then pw groupadd riak_cs; fi \n if ! pw usershow riak_cs 2>/dev/null; then pw useradd riak_cs -g riak_cs -h - -d /usr/local/var/lib/riak-cs -s /bin/sh -c \"Riak CS user\"; fi \n if [ ! -d /var/log/riak-cs ]; then mkdir /var/log/riak-cs && chown riak_cs:riak_cs /var/log/riak-cs; fi" + post-install: "chown riak_cs:riak_cs /usr/local/etc/riak-cs; (cd /usr/local/lib/riak-cs; ln -s erts-* erts)" + pre-deinstall: "rm -f /usr/local/lib/riak-cs/erts" +} diff --git a/rel/pkg/fbsdng/Makefile b/rel/pkg/fbsdng/Makefile new file mode 100644 index 000000000..b7e557407 --- /dev/null +++ b/rel/pkg/fbsdng/Makefile @@ -0,0 +1,109 @@ +export + +BUILDDIR = $(shell pwd) +BUILD_STAGE_DIR = $(BUILDDIR)/riak-cs + +# Where we install things (based on vars.config) +# /usr/local based dirs +PMAN_DIR = $(BUILD_STAGE_DIR)/usr/local/man +PBIN_DIR = $(BUILD_STAGE_DIR)/usr/local/lib/riak-cs/bin +PETC_DIR = $(BUILD_STAGE_DIR)/usr/local/etc/riak-cs +PLIB_DIR = $(BUILD_STAGE_DIR)/usr/local/lib/riak-cs +PUSRBIN_DIR = $(BUILD_STAGE_DIR)/usr/local/bin +# /var based dirs +PDATA_DIR = $(BUILD_STAGE_DIR)/usr/local/var/lib/riak-cs +PLOG_DIR = $(BUILD_STAGE_DIR)/var/log/riak-cs + +PKGNAME = riak_cs-$(PKG_VERSION)-$(OSNAME)-$(ARCH).tbz + + +# Recursive assignment of ERTS version +# We will know this after building the rel +ERTS_PATH = $(shell ls $(BUILDDIR)/rel/riak-cs | egrep -o "erts-.*") + +build: packing_list_files + @echo "Building package $(PKGNAME)" + + cd $(BUILD_STAGE_DIR) && \ + mkdir ../../../out/packages && \ + pkg create -m . -r . -o ../../../out/packages + + cd ../../out/packages && \ + for f in *.pkg; do \ + shasum -a 256 $${f} > $${f}.sha \ + ; done + +packing_list_files: $(BUILD_STAGE_DIR) + @mv ${BUILDDIR}/rel/pkg/fbsdng/+MANIFEST ${BUILD_STAGE_DIR} + sed -e "s/%ERTS_PATH%/${ERTS_PATH}/" < \ + ${BUILDDIR}/rel/pkg/fbsdng/rc.d > ${BUILD_STAGE_DIR}/usr/local/etc/rc.d/riak-cs + chmod -w ${BUILD_STAGE_DIR}/usr/local/etc/rc.d/riak-cs + chmod +x ${BUILD_STAGE_DIR}/usr/local/etc/rc.d/riak-cs + @cd $(BUILD_STAGE_DIR) && \ + echo "version: \"${PKG_VERSION}\"" >> +MANIFEST && \ + echo "files: {" >> +MANIFEST + + @echo "Copying Man pages to staging directory" + @cd $(BUILDDIR) && \ + if [ -d doc/man/man1 ]; then \ + mkdir -p $(PMAN_DIR) && \ + cp -R doc/man/man1 $(PMAN_DIR); fi + + @echo "Packaging /usr/local files" + @cd $(BUILD_STAGE_DIR) && \ + find usr -type f | while read file ; do \ + mode=$$(stat -f%p "$$file" | cut -c 3-) && \ + sum=$$(sha256 -q $$file) && \ + echo " \"/$$file\": { sum: \"$$sum\", perm: \"$$mode\", uname: \"root\", gname: \"wheel\" }," >> +MANIFEST; done && \ + sed -i .bak '$$s/,$$//' +MANIFEST && \ + rm -- +MANIFEST.bak && \ + echo " }" >> +MANIFEST + + @cd $(BUILD_STAGE_DIR) && \ + echo "directories: {" >> +MANIFEST && \ + echo " /usr/local/lib/riak-cs: \"y\"," >> +MANIFEST && \ + echo " /usr/local/var/lib/riak-cs: {uname: \"riak_cs\", gname: \"riak_cs\", perm: \"0700\" }," >> +MANIFEST && \ + echo " /usr/local/etc/riak-cs: \"y\"" >> +MANIFEST && \ + echo " }" >> +MANIFEST + +# Copy the app rel directory to the staging directory to build our +# package structure and move the directories into the right place +# for the package, see the vars.config file for destination +# directories +$(BUILD_STAGE_DIR): buildrel + @echo "Copying rel directory to staging directory" + mkdir -p $@ + mkdir -p $(PBIN_DIR) $(PUSRBIN_DIR) + for f in riak-cs-admin riak-cs-debug riak-cs-supercluster riak-cs-chkconfig; do \ + cp -R rel/riak-cs/bin/$$f $(PUSRBIN_DIR); \ + done + cp -R rel/riak-cs/usr/bin/riak-cs $(PUSRBIN_DIR) + cp -R rel/riak-cs/bin $(PLIB_DIR); + mkdir -p $(PETC_DIR) + cp -R rel/riak-cs/etc/* $(PETC_DIR) + mkdir -p $(PLIB_DIR) + cp -R rel/riak-cs/lib $(PLIB_DIR) + cp -R rel/riak-cs/share $(PLIB_DIR) + cp -R rel/riak-cs/erts-* $(PLIB_DIR) + (cd $(PLIB_DIR) && ln -s erts-* erts) + cp -R rel/riak-cs/releases $(PLIB_DIR) + mkdir -p $(PDATA_DIR) +# cp -R rel/riak-cs/data/* $(PDATA_DIR) || true # as of 3.0.0, there is nothing in data + mkdir -p ${BUILD_STAGE_DIR}/usr/local/etc/rc.d + +# Build the release we need to package +# * Ensure all binaries are executable +# * copy the vars.config over for build config +buildrel: + tar -xf $(BASE_DIR)/rel/pkg/out/$(PKG_ID).tar.gz -C $(BASE_DIR)/rel/pkg/out/riak_cs-$(PKG_ID) + cd $(BASE_DIR)/rel/pkg/out/$(PKG_ID); \ + (mkdir -p _build/default && cd _build/default && for d in lib; do ln -fs $(BASE_DIR)/_build/default/$$d; done); \ + $(MAKE) -C $(BASE_DIR)/rel/pkg/out/riak_cs-$(PKG_ID) rel-fbsdng + rm -rf rel/riak-cs/lib/*/src + chmod 0755 rel/riak-cs/bin/* rel/riak-cs/erts-*/bin/* + +$(BUILDDIR): + mkdir -p $@ + +$(PKGERDIR)/pkgclean: + rm -rf $(BUILD_STAGE_DIR) $(BUILDDIR) diff --git a/rel/pkg/fbsdng/rc.d b/rel/pkg/fbsdng/rc.d new file mode 100644 index 000000000..c1d63b430 --- /dev/null +++ b/rel/pkg/fbsdng/rc.d @@ -0,0 +1,19 @@ +#!/bin/sh + +# $FreeBSD$ +# +# PROVIDE: riak_cs +# REQUIRE: LOGIN +# KEYWORD: shutdown + +. /etc/rc.subr + +name=riak_cs +command=/usr/local/lib/riak-cs/%ERTS_PATH%/bin/beam.smp +rcvar=riak_cs_enable +start_cmd="/usr/local/bin/riak-cs start" +stop_cmd="/usr/local/bin/riak-cs stop" +pidfile="/var/run/riak-cs/riak-cs.pid" + +load_rc_config $name +run_rc_command "$1" diff --git a/rel/pkg/fbsdng/vars.config b/rel/pkg/fbsdng/vars.config new file mode 100644 index 000000000..795ad6096 --- /dev/null +++ b/rel/pkg/fbsdng/vars.config @@ -0,0 +1,37 @@ +%% -*- mode: erlang -*- + +{rel_vsn, "{{release_version}}"}. + +{platform_base_dir, "/usr/local/lib/riak-cs"}. +{platform_bin_dir, "/usr/local/lib/riak-cs/bin"}. +{platform_data_dir, "/usr/local/var/lib/riak-cs"}. +{platform_etc_dir, "/usr/local/etc/riak-cs"}. +{platform_lib_dir, "/usr/local/lib/riak-cs/lib"}. +{platform_log_dir, "/var/log/riak-cs"}. +{platform_gen_dir, "{{platform_data_dir}}"}. +{platform_patch_dir, "{{platform_lib_dir}}/patches"}. + +{cs_ip, "127.0.0.1"}. +{cs_port, 8080}. +{admin_ip, "127.0.0.1"}. +{admin_port, 8000}. +{riak_ip, "127.0.0.1"}. +{riak_pb_port, 8087}. +{auth_bypass, false}. +{admin_key, "admin-key"}. +{stanchion_port, 8085}. +{stanchion_ssl, off}. +{cs_version, 030205}. +{stanchion_hosting_mode, auto}. +{tussle_voss_riak_host, auto}. + +{node, "riak-cs@127.0.0.1"}. +{crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. + +{pid_dir, "/var/run/riak-cs"}. + +{log_level, info}. +{logger_sasl_enabled, false}. + +{cuttlefish, "on"}. +{cuttlefish_conf, "riak-cs.conf"}. diff --git a/rel/pkg/osx/Makefile b/rel/pkg/osx/Makefile new file mode 100644 index 000000000..e1318a5b6 --- /dev/null +++ b/rel/pkg/osx/Makefile @@ -0,0 +1,15 @@ +export + +BUILDDIR = $(shell pwd) +BUILD_STAGE_DIR = $(BUILDDIR)/riak_cs +PKGNAME = riak_cs-$(PKG_ID) + +default: buildrel + mkdir ../packages + tar -czf ../packages/$(PKGNAME).tar.gz -C rel riak-cs && \ + (cd ../packages && shasum -a 256 $(PKGNAME).tar.gz > $(PKGNAME).tar.gz.sha) + +buildrel: + tar -xf $(BASE_DIR)/rel/pkg/out/$(PKG_ID).tar.gz -C $(BASE_DIR)/rel/pkg/out/$(PKGNAME) + $(MAKE) -C $(BASE_DIR)/rel/pkg/out/$(PKGNAME) rel-osx + chmod 0755 rel/riak-cs/bin/* rel/riak-cs/erts-*/bin/* diff --git a/rel/pkg/osx/vars.config b/rel/pkg/osx/vars.config new file mode 120000 index 000000000..0a676b8ba --- /dev/null +++ b/rel/pkg/osx/vars.config @@ -0,0 +1 @@ +../../vars.config \ No newline at end of file diff --git a/rel/pkg/rpm/Makefile b/rel/pkg/rpm/Makefile new file mode 100644 index 000000000..7017204fc --- /dev/null +++ b/rel/pkg/rpm/Makefile @@ -0,0 +1,22 @@ +PWD = $(shell pwd) + +# No hyphens are allowed in the _version field in RPM +PKG_VERSION_NO_H ?= $(shell echo $(PKG_VERSION) | tr - .) + +default: + rpmbuild --define "_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}$(DISTRO).%%{ARCH}.rpm" \ + --define '_topdir $(BASE_DIR)/rel/pkg/out/' \ + --define '_sourcedir $(BASE_DIR)/rel/pkg/out/' \ + --define '_specdir $(BASE_DIR)/rel/pkg/out/' \ + --define '_rpmdir $(BASE_DIR)/rel/pkg/out/packages' \ + --define '_srcrpmdir $(BASE_DIR)/rel/pkg/out/packages' \ + --define "_revision $(PKG_VERSION)" \ + --define "_version $(PKG_VERSION_NO_H)" \ + --define "_release $(PKG_BUILD)" \ + --define "_tarname $(PKG_ID).tar.gz" \ + --define "_tarname_base $(PKG_ID)" \ + -ba $(BASE_DIR)/rel/pkg/rpm/specfile + cd $(BASE_DIR)/rel/pkg/out/packages && \ + for rpmfile in *.rpm; do \ + sha256sum $${rpmfile} > $${rpmfile}.sha \ + ; done diff --git a/rel/pkg/rpm/riak-cs.service b/rel/pkg/rpm/riak-cs.service new file mode 100644 index 000000000..9b722eb35 --- /dev/null +++ b/rel/pkg/rpm/riak-cs.service @@ -0,0 +1,14 @@ +[Unit] +Description=Riak CS Database + +[Service] +User=riak_cs +ExecStart=/usr/sbin/riak-cs start +ExecStop=/usr/sbin/riak-cs stop +Type=simple +PIDFile=/var/run/riak-cs/riak-cs.pid +EnvironmentFile=-/etc/default/riak-cs +RuntimeDirectory=riak-cs + +[Install] +WantedBy=multi-user.target diff --git a/rel/pkg/rpm/specfile b/rel/pkg/rpm/specfile new file mode 100644 index 000000000..8c61df690 --- /dev/null +++ b/rel/pkg/rpm/specfile @@ -0,0 +1,170 @@ +## ------------------------------------------------------------------- +## +## Copyright (c) 2014 Basho Technologies, Inc.; 2021-2023 TI Tokyo +## +## This file is provided to you under the Apache License, +## Version 2.0 (the "License"); you may not use this file +## except in compliance with the License. You may obtain +## a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, +## software distributed under the License is distributed on an +## "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +## KIND, either express or implied. See the License for the +## specific language governing permissions and limitations +## under the License. +## +## ------------------------------------------------------------------- + +Name: riak-cs +Version: %{_version} +Release: %{_release}%{?dist} +License: "Apache 2.0" +Group: Development/Libraries +Source: %{_tarname} +URL: "https://tiot.jp" +Vendor: "TI Tokyo" +Packager: "Riak CS Package Maintainer" <"andriy.zavada@tiot.jp"> +BuildRoot: %{_tmppath}/%{name}-%{_revision}-%{release}-root +Summary: "Riak CS is a Riak-backed S3 drop-in replacement" +BuildRequires: systemd +Requires(post): systemd +Requires(preun): systemd +Requires(postun): systemd + +%description +"Riak CS is a Riak-backed AWS S3 drop-in replacement" + +%define debug_package %{nil} +%define __prelink_undo_cmd /bin/cat prelink library + +# disable shebang mangling +%undefine __brp_mangle_shebangs + +%define platform_etc_dir %{_sysconfdir}/riak-cs +%define platform_base_dir %{_libdir}/riak-cs +%define platform_bin_dir %{platform_base_dir}/bin +%define platform_lib_dir %{platform_base_dir}/lib +%define platform_log_dir %{_localstatedir}/log/riak-cs +# no data as such; it's only used for cuttlefish-generated configs +%define platform_data_dir %{_localstatedir}/lib/riak-cs + +%prep +%setup -q -n %{_tarname_base} -c %{_tarname_base} + +# Setup vars.config like other platforms, but do it inside of spec file +cp rel/pkg/rpm/vars.config.part rel/pkg/rpm/vars.config +cat >>rel/pkg/rpm/vars.config </dev/null 2>&1; then + groupadd -r riak_cs +fi + +if getent passwd riak_cs >/dev/null 2>&1; then + usermod -d %{_localstatedir}/lib/riak-cs riak_cs || true +else + useradd -r -g riak_cs \ + --home %{_localstatedir}/lib/riak-cs \ + --comment "Riak CS User" \ + --shell /bin/bash \ + riak_cs +fi + +%post +# For distros with SELinux (RHEL/Fedora) +if [ `which selinuxenabled > /dev/null 2>&1` ] ; then + # Fixup perms for SELinux (if it is enabled) + selinuxenabled && find %{_localstatedir}/lib/riak-cs -name "*.so" -exec chcon -t textrel_shlib_t {} \; +fi + +%systemd_post %{name}.service + +%preun +%systemd_preun %{name}.service + +%postun +%systemd_postun %{name}.service +rm -f %{_sysconfdir}/riak-cs/riak-cs.conf.new +rm -rf %{_localstatedir}/lib/riak-cs +rm -rf %{_localstatedir}/log/riak-cs + +%files +%defattr(-,root,root) +%{_libdir}/* +%{_unitdir}/%{name}.service +%{_sbindir}/* + +%defattr(-,riak_cs,riak_cs) +%{_sysconfdir}/riak-cs +%{_localstatedir}/lib/riak-cs +%{_localstatedir}/log/riak-cs + +%config(noreplace) %{_sysconfdir}/riak-cs/* + +%clean +rm -rf %{buildroot} diff --git a/rel/pkg/rpm/vars.config.part b/rel/pkg/rpm/vars.config.part new file mode 100644 index 000000000..d0ae94137 --- /dev/null +++ b/rel/pkg/rpm/vars.config.part @@ -0,0 +1,27 @@ +%% -*- mode: erlang -*- + +{rel_vsn, "{{release_version}}"}. + +{cs_ip, "127.0.0.1"}. +{cs_port, 8080}. +{admin_ip, "127.0.0.1"}. +{admin_port, 8000}. +{riak_ip, "127.0.0.1"}. +{riak_pb_port, 8087}. +{auth_bypass, false}. +{admin_key, "admin-key"}. +{stanchion_port, 8085}. +{stanchion_ssl, off}. +{cs_version, 030205}. +{stanchion_hosting_mode, auto}. +{tussle_voss_riak_host, auto}. + +{node, "riak-cs@127.0.0.1"}. + +{pid_dir, "/run/riak-cs"}. + +{log_level, info}. +{logger_sasl_enabled, false}. + +{cuttlefish, "on"}. +{cuttlefish_conf, "riak-cs.conf"}. diff --git a/rel/rebar.config b/rel/rebar.config deleted file mode 100644 index 1c0080b52..000000000 --- a/rel/rebar.config +++ /dev/null @@ -1,4 +0,0 @@ -{lib_dirs, ["../deps"]}. -{plugin_dir, "../deps/cuttlefish/src"}. -{plugins, [cuttlefish_rebar_plugin]}. -{cuttlefish_filename, "riak-cs.conf"}. diff --git a/rel/reltool.config b/rel/reltool.config deleted file mode 100644 index add2b1520..000000000 --- a/rel/reltool.config +++ /dev/null @@ -1,86 +0,0 @@ -%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ts=4 sw=4 et -{sys, [ - {lib_dirs, ["../deps", "../apps"]}, - {rel, "riak-cs", "2.1.0", - [ - kernel, - stdlib, - sasl, - public_key, - ssl, - os_mon, - crypto, - runtime_tools, - mochiweb, - webmachine, - lager, - lager_syslog, - poolboy, - exometer_core, - cluster_info, - riak_cs - ]}, - {rel, "start_clean", "", - [ - kernel, - stdlib - ]}, - {boot_rel, "riak-cs"}, - {profile, embedded}, - {excl_sys_filters, ["^bin/.*", - "^erts.*/bin/(dialyzer|typer)"]}, - {excl_archive_filters, [".*"]}, - {app, sasl, [{incl_cond, include}]}, - {app, cuttlefish, [{incl_cond, include}]}, - {app, lager, [{incl_cond, include}]}, - {app, poolboy, [{incl_cond, include}]}, - {app, folsom, [{incl_cond, include}]}, - {app, eper, [{incl_cond, include}]}, - {app, riak_cs, [{incl_cond, include}]} - ]}. - - -{target_dir, "riak-cs"}. - -{overlay_vars, "vars.config"}. - -{overlay, [ - %% Setup basic dirs that packaging requires - {mkdir, "log"}, - {mkdir, "data/riak-cs"}, - - %% Copy base files for starting and interacting w/ node - {copy, "../deps/node_package/priv/base/erl", - "{{erts_vsn}}/bin/erl"}, - {copy, "../deps/node_package/priv/base/nodetool", - "{{erts_vsn}}/bin/nodetool"}, - {copy, "../deps/cuttlefish/cuttlefish", - "{{erts_vsn}}/bin/cuttlefish"}, - {template, "../deps/node_package/priv/base/runner", - "bin/riak-cs"}, - {template, "../deps/node_package/priv/base/env.sh", - "lib/env.sh"}, - {template, "../deps/node_package/priv/base/app_epath.sh", - "lib/app_epath.sh"}, - - %% Copy config files - {template, "files/riak_cs.schema", "lib/10-riak_cs.schema"}, - {template, "../deps/cuttlefish/priv/erlang_vm.schema", "lib/11-erlang_vm.schema"}, - {template, "files/advanced.config", "etc/advanced.config"}, - - %% Copy SSL Certs - {template, "files/cert.pem", "etc/cert.pem"}, - {template, "files/key.pem", "etc/key.pem"}, - - %% Copy additional bin scripts - {template, "files/riak-cs-access", "bin/riak-cs-access"}, - {template, "files/riak-cs-storage", "bin/riak-cs-storage"}, - {template, "files/riak-cs-gc", "bin/riak-cs-gc"}, - {template, "files/riak-cs-stanchion", "bin/riak-cs-stanchion"}, - {template, "files/riak-cs-debug", "bin/riak-cs-debug"}, - {template, "files/riak-cs-admin", "bin/riak-cs-admin"}, - {template, "files/riak-cs-supercluster", "bin/riak-cs-supercluster"}, - {template, "files/riak-cs-multibag", "bin/riak-cs-multibag"}, - {mkdir, "lib/basho-patches"} - ]}. diff --git a/rel/vars.config b/rel/vars.config index 898698c54..43229aaeb 100644 --- a/rel/vars.config +++ b/rel/vars.config @@ -1,20 +1,16 @@ -%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ts=4 sw=4 et +%% -*- mode: erlang -*- -%% Platform-specific installation paths -{platform_bin_dir, "./bin"}. -{platform_data_dir, "./data"}. -{platform_etc_dir, "./etc"}. -{platform_lib_dir, "./lib"}. -{platform_log_dir, "./log"}. +{rel_vsn, "{{release_version}}"}. -%% lager -{console_log_default, file}. +{platform_base_dir, "${RCS_PATH:-$RELEASE_ROOT_DIR}"}. +{platform_bin_dir, "./bin"}. +{platform_data_dir, "./data"}. +{platform_etc_dir, "./etc"}. +{platform_lib_dir, "./lib"}. +{platform_log_dir, "./log"}. +{platform_gen_dir, "."}. +{platform_patch_dir, "./lib/patches"}. - -%% -%% riak_cs.conf -%% {cs_ip, "127.0.0.1"}. {cs_port, 8080}. {admin_ip, "127.0.0.1"}. @@ -23,35 +19,19 @@ {riak_pb_port, 8087}. {auth_bypass, false}. {admin_key, "admin-key"}. -{stanchion_ip, "127.0.0.1"}. {stanchion_port, 8085}. {stanchion_ssl, off}. -{cs_version, 010300}. -{rewrite_module, riak_cs_s3_rewrite}. -{auth_module, riak_cs_s3_auth}. +{cs_version, 030205}. %% to match values in config/sys.docker.config, include/riak_cs.hrl +{rewrite_module, riak_cs_aws_rewrite}. +{auth_module, riak_cs_aws_auth}. +{stanchion_hosting_mode, auto}. +{tussle_voss_riak_host, auto}. -%% -%% etc/vm.args -%% {node, "riak-cs@127.0.0.1"}. {crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. -%% -%% bin/riak_cs -%% -{data_dir, "{{target_dir}}/data"}. -{runner_script_dir, "\`cd \\`dirname $0\\` && /bin/pwd\`"}. -{runner_base_dir, "{{runner_script_dir}}/.."}. -{runner_etc_dir, "$RUNNER_BASE_DIR/etc"}. -{runner_log_dir, "$RUNNER_BASE_DIR/log"}. -{runner_lib_dir, "$RUNNER_BASE_DIR/lib"}. -{runner_patch_dir, "$RUNNER_BASE_DIR/lib/basho-patches"}. -{pipe_dir, "/tmp/$RUNNER_BASE_DIR/"}. -{runner_user, ""}. -{runner_wait_process, "riak_cs_put_fsm_sup"}. +{log_level, info}. +{logger_sasl_enabled, false}. -%% -%% cuttlefish -%% {cuttlefish, "on"}. {cuttlefish_conf, "riak-cs.conf"}. diff --git a/rel/vars/dev_vars.config.src b/rel/vars/dev_vars.config.src index 071bd0797..4308c9d12 100644 --- a/rel/vars/dev_vars.config.src +++ b/rel/vars/dev_vars.config.src @@ -1,18 +1,17 @@ -%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ts=4 sw=4 et +%% -*- mode: erlang; -*- -{devrel, true}. +{rel_vsn, "{{release_version}}"}. %% Platform-specific installation paths -{platform_bin_dir, "./bin"}. -{platform_data_dir, "./data"}. -{platform_etc_dir, "./etc"}. -{platform_lib_dir, "./lib"}. -{platform_log_dir, "./log"}. +{platform_base_dir, "@PLATFORM_BASE_DIR@"}. +{platform_bin_dir, "{{platform_base_dir}}/bin"}. +{platform_data_dir, "{{platform_base_dir}}/data"}. +{platform_etc_dir, "{{platform_base_dir}}/etc"}. +{platform_lib_dir, "{{platform_base_dir}}/lib"}. +{platform_log_dir, "{{platform_base_dir}}/log"}. +{platform_gen_dir, "{{platform_base_dir}}"}. +{platform_patch_dir, "{{platform_lib_dir}}/patches"}. -%% -%% etc/app.config -%% {cs_ip, "127.0.0.1"}. {cs_port, @CSPORT@}. {admin_ip, "127.0.0.1"}. @@ -21,35 +20,21 @@ {riak_pb_port, @RIAKPBPORT@}. {auth_bypass, false}. {admin_key, "admin-key"}. -{stanchion_ip, "127.0.0.1"}. {stanchion_port, 8085}. {stanchion_ssl, off}. -{cs_version, 010300}. -{rewrite_module, riak_cs_s3_rewrite}. -{auth_module, riak_cs_s3_auth}. +{cs_version, 030205}. +{rewrite_module, riak_cs_aws_rewrite}. +{auth_module, riak_cs_aws_auth}. +{stanchion_hosting_mode, auto}. +{tussle_voss_riak_host, auto}. -%% -%% etc/vm.args -%% {node, "@NODE@"}. {crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. -%% -%% bin/riak_cs -%% -{data_dir, "{{target_dir}}/data"}. -{runner_script_dir, "\`cd \\`dirname $0\\` && /bin/pwd\`"}. -{runner_base_dir, "{{runner_script_dir}}/.."}. -{runner_etc_dir, "$RUNNER_BASE_DIR/etc"}. -{runner_log_dir, "$RUNNER_BASE_DIR/log"}. -{runner_lib_dir, "$RUNNER_BASE_DIR/lib"}. -{runner_patch_dir, "$RUNNER_BASE_DIR/lib/basho-patches"}. -{pipe_dir, "/tmp/$RUNNER_BASE_DIR/"}. -{runner_user, ""}. -{runner_wait_process, "riak_cs_put_fsm_sup"}. +{log_level, debug}. +{logger_sasl_enabled, true}. -%% -%% cuttlefish -%% {cuttlefish, "on"}. {cuttlefish_conf, "riak-cs.conf"}. + +{devrel, true}. diff --git a/releasenotes/riak-cs.1.ja.md b/releasenotes/riak-cs.1.ja.md deleted file mode 100644 index 23fed7906..000000000 --- a/releasenotes/riak-cs.1.ja.md +++ /dev/null @@ -1,647 +0,0 @@ -# Riak CS 1.5.4 リリースノート - -## 修正されたバグ - -- バックプレッシャーのスリープ発動後に取得済みのRiakオブジェクトを破棄 - [riak_cs/#1041](https://github.com/basho/riak_cs/pull/1041)。 - これは次の場合に起こり得る Sibling の増加を防ぎます。 - (a) 高い同時実行アップロードによるバックプレッシャーが起動しており、かつ - (b) バックプレッシャーによるスリープ中にアップロードがインターリーブするとき。 - この問題はマルチパートアップロードへは影響しません。 -- 不要なURLデコードを引き起こす S3 API における不正確なURLパスの rewrite 処理。 - [riak_cs/#1040](https://github.com/basho/riak_cs/pull/1040). - URLエンコード・デコードが不正確な事により、 - `%[0-9a-fA-F][0-9a-fA-F]` (正規表現) や `+` を含むオブジェクト名は - 誤ったデコードが実施されていました。この結果、前者は異なるバイナリへ、 - 後者は ` ` (空白) へと置き換わり、どちらの場合でも暗黙的にデータを - 上書きする可能性があります。例えば後者のケースでは、 キー に `+` を含む - オブジェクト(例：`foo+bar`) は、`+` が ` ` に置き換わっただけの、 - ほぼ同じ名前のオブジェクト(`foo bar`)に上書きされます。逆も起こり得ます。 - この修正は次の問題にも関連します： - [riak_cs/#910](https://github.com/basho/riak_cs/pull/910) - [riak_cs/#977](https://github.com/basho/riak_cs/pull/977). - -## アップグレード時の注意 - -Riak CS 1.5.4 へアップグレードすると、デフォルト設定のままでは、 -キーに `%[0-9a-fA-F][0-9a-fA-F]` や `+` を含むオブジェクトは見えなくなり、 -違うオブジェクト名で見えるようになります。 -前者は余分にデコードされたオブジェクトとして参照され、 -後者は ` ` を `+` で置き換えたキー(例： `foo bar`)で参照されるようになります。 - -下記の表はアップグレードの前後で -`%[0-9a-fA-F][0-9a-fA-F]` を含むURLがどう振る舞うかの例です。 - - - | アップグレード前 | アップグレード後 | -:------------|:-------------------------|:------------------| - 書き込み時 | `a%2Fkey` | - | - 読み込み時 | `a%2Fkey` または `a/key` | `a/key` | -リスト表示時 | `a/key` | `a/key` | - -`+` や ` ` を含むオブジェクトのアップグレード前後の例： - - | アップグレード前 | アップグレード後 | -:------------|:-------------------------|:------------------| - 書き込み時 | `a+key` | - | - 読み込み時 | `a+key` または `a key` | `a key` | -リスト表示時 | `a key` | `a key` | - - | アップグレード前 | アップグレード後 | -:------------|:-------------------------|:------------------| - 書き込み時 | `a key` | - | - 読み込み時 | `a+key` または `a key` | `a key` | -リスト表示時 | `a key` | `a key` | - -またこの修正によりアクセスログのフォーマットも単一のURLエンコードから二重エンコードスタイルへ変わります。 -下記は変更前の例です： - -``` -127.0.0.1 - - [07/Jan/2015:08:27:07 +0000] "PUT /buckets/test/objects/path1%2Fpath2%2Fte%2Bst.txt HTTP/1.1" 200 0 "" "" -``` - -そしてこちらが新しいフォーマットです。 - -``` -127.0.0.1 - - [07/Jan/2015:08:27:07 +0000] "PUT /buckets/test/objects/path1%2Fpath2%2Fte%252Bst.txt HTTP/1.1" 200 0 "" "" -``` - -この例から分かるように、オブジェクトのパスが `path1%2Fpath2%2Fte%252Bst.txt` -から `path1%2Fpath2%2Fte%2Bst.txt` へ変わることに注意して下さい。 - -もし Riak CS を利用するアプリケーション側の都合で -以前の挙動のままにしたい場合、アップグレード時に -Riak CSの設定を変更すればそれが可能です。 -この場合、`rewrite_module` 設定を下記のように変更してください： - -```erlang -{riak_cs, [ - %% Other settings - {rewrite_module, riak_cs_s3_rewrite_legacy}, - %% Other settings -]} -``` - -**注意**: 以前の挙動は技術的に不適切であり、 -前述したように暗黙的なデータの上書きが起こり得ます。 -注意の上でご利用下さい。 - -# Riak CS 1.5.3 リリースノート - -## 新規追加 - -- read_before_last_manifest_writeオプションの追加。 - 一部のkeyへの高頻度かつ多並列でのアクセスによるSibling explosionの回避に有効。 - [riak_cs/#1011](https://github.com/basho/riak_cs/pull/1011) -- タイムアウト設定の追加。Riak - Riak CS 間の全アクセスに対してタイムアウトを設定可能にし、運用に柔軟性を提供。 - [riak_cs/#1021](https://github.com/basho/riak_cs/pull/1021) - -## 修正されたバグ - -- ストレージ統計の集計結果に削除済バケットのデータが含まれ得る問題を修正。 - [riak_cs/#996](https://github.com/basho/riak_cs/pull/996) - -# Riak CS 1.5.2 リリースノート - -## 新規追加 - -- Riakに対する接続失敗に関するロギングの改善 - [riak_cs/#987](https://github.com/basho/riak_cs/pull/987). -- Riakに対してアクセス統計情報の保存に失敗した際のログを追加 - [riak_cs/#988](https://github.com/basho/riak_cs/pull/988). - これは一時的な Riak - Riak CS 間の接続エラーによるアクセス統計ログの消失を防ぎます。 - アクセスログは `console.log` へ `warning` レベルで保存されます。 -- 不正なガベージコレクション manifest の修復スクリプトの追加 - [riak_cs/#983](https://github.com/basho/riak_cs/pull/983)。 - active manifest が GCバケットへ保存される際に - [既知の問題](https://github.com/basho/riak_cs/issues/827) があります。 - このスクリプトは不正な状態を正常な状態へ変更します。 - -## 修正されたバグ - -- プロトコルバッファのコネクションプール (`pbc_pool_master`) のリークを修正 - [riak_cs/#986](https://github.com/basho/riak_cs/pull/986) 。 - 存在しないバケットに対する認証ヘッダ無しのリクエストや、ユーザ一覧のリクエストが - コネクションプールのリークを引き起こし、プールは結果的に空になります。このバグは1.5.0から含まれます。 - -# Riak CS 1.5.1 リリースノート - -## 新規追加 - -- Sibling Explosionを避けるために sleep-after-update を追加 [riak_cs/#959](https://github.com/basho/riak_cs/pull/959) -- `riak-cs-debug` の multibag サポート [riak_cs/#930](https://github.com/basho/riak_cs/pull/930) -- Riak CS におけるバケット数に上限を追加 [riak_cs/#950](https://github.com/basho/riak_cs/pull/950) -- バケットの衝突解決を効率化 [riak_cs/#951](https://github.com/basho/riak_cs/pull/951) - -## 修正されたバグ - -- `riak_cs_delete_fsm` のデッドロックによるGCの停止 [riak_cs/#949](https://github.com/basho/riak_cs/pull/949) -- `riak-cs-debug` がログを収集するディレクトリのパスを修正 [riak_cs/#953](https://github.com/basho/riak_cs/pull/953) -- DST-awareなローカルタイムからGMTへの変換を回避 [riak_cs/#954](https://github.com/basho/riak_cs/pull/954) -- Secretの代わりに UUID をカノニカルID生成時のシードに利用 [riak_cs/#956](https://github.com/basho/riak_cs/pull/956) -- マルチパートアップロードにおけるパート数の上限を追加 [riak_cs/#957](https://github.com/basho/riak_cs/pull/957) -- タイムアウトをデフォルトの 5000ms から無限に設定 [riak_cs/#963](https://github.com/basho/riak_cs/pull/963) -- GC バケット内の無効な状態のマニフェストをスキップ [riak_cs/#964](https://github.com/basho/riak_cs/pull/964) - -## アップグレード時の注意点 - -### ユーザー毎のバケット数 - -Riak CS 1.5.1 を使うと、ユーザーが作ることのできるバケット数を制限することができます。 -デフォルトでこの最大値は 100 です。この制限はユーザーの新たなバケット作成を禁止しますが、 -既に制限数を超えているユーザーが実施する、バケット削除を含む他の操作へは影響しません。 -デフォルトの制限を変更するには `app.config` の `riak_cs` セクションで次の箇所を変更してください: - -```erlang -{riak_cs, [ - %% ... - {max_buckets_per_user, 5000}, - %% ... - ]} -``` - -この制限を利用しない場合は `max_buckets_per_user` を `unlimited` に設定してください。 - -# Riak CS 1.5.0 リリースノート - -## 新規追加 - -* `cluster-info` 取得を含む新規コマンド `riak-cs-debug` を追加 [riak_cs/#769](https://github.com/basho/riak_cs/pull/769), [riak_cs/#832](https://github.com/basho/riak_cs/pull/832) -* 既存コマンド群を新規コマンド `riak-cs-admin` へ統合 [riak_cs/#839](https://github.com/basho/riak_cs/pull/839) -* Stanchion の IP、ポートを変更する新規コマンド `riak-cs-admin stanchion` を追加 [riak_cs/#657](https://github.com/basho/riak_cs/pull/657) -* 並行 GC によるガベージコレクション性能の向上 [riak_cs/#830](https://github.com/basho/riak_cs/pull/830) -* Iterator refresh [riak_cs/#805](https://github.com/basho/riak_cs/pull/805) -* `fold_objects_for_list_keys` 設定をデフォルト有効に変更 [riak_cs/#737](https://github.com/basho/riak_cs/pull/737), [riak_cs/#785](https://github.com/basho/riak_cs/pull/785) -* Cache-Control ヘッダーのサポートを追加 [riak_cs/#821](https://github.com/basho/riak_cs/pull/821) -* 猶予期間(`leeway_seconds`)内でもオブジェクトをガベージコレクション可能にする変更 [riak_cs/#470](https://github.com/basho/riak_cs/pull/470) -* オブジェクト、マルチパートともに PUT Copy API を追加 [riak_cs/#548](https://github.com/basho/riak_cs/pull/548) -* lager 2.0.3 へ更新 -* R16B0x をビルド環境に追加 (リリースは R15B01 でビルド) -* `gc_paginated_index` 設定をデフォルト有効に変更 [riak_cs/#881](https://github.com/basho/riak_cs/issues/881) -* 新規 API: Delete Multiple Objects の追加[riak_cs/#728](https://github.com/basho/riak_cs/pull/728) -* マニフェストに対して siblings, バイト、履歴の肥大化を警告するログ追加 [riak_cs/#915](https://github.com/basho/riak_cs/pull/915) - -## 修正されたバグ - -* `ERL_MAX_PORTS` を Riak のデフォルトに合わせ 64000 へ変更 [riak_cs/#636](https://github.com/basho/riak_cs/pull/636) -* Riak CS 管理リソースを OpenStack API でも利用可能にする修正 [riak_cs/#666](https://github.com/basho/riak_cs/pull/666) -* Solaris でのソースビルドのバグ修正のため、パス代入コードの変更 [riak_cs/#733](https://github.com/basho/riak_cs/pull/733) -* `riakc_pb_socket` エラー時の `sanity_check(true,false)` バグを修正 [riak_cs/#683](https://github.com/basho/riak_cs/pull/683) -* Riak-CS-GC のスケジューラタイムスタンプが 2013 ではなく 0043 になるバグを修正 [riak_cs/#713](https://github.com/basho/riak_cs/pull/713) fixed by [riak_cs/#676](https://github.com/basho/riak_cs/pull/676) -* OTP code_server プロセスを過剰に呼び出すバグを修正 [riak_cs/#675](https://github.com/basho/riak_cs/pull/675) -* content-md5 が一致しない場合に HTTP 400 を返すよう修正 [riak_cs/#596](https://github.com/basho/riak_cs/pull/596) -* `/riak-cs/stats` が `admin_auth_enabled=false` の時に動作しないバグを修正. [riak_cs/#719](https://github.com/basho/riak_cs/pull/719) -* ストレージ計算で tombstone および undefined の manifest.props を処理できないバグを修正 [riak_cs/#849](https://github.com/basho/riak_cs/pull/849) -* 未完了のマルチパートオブジェクトが、バケットの削除、作成後にも残るバグを修正 [riak_cs/#857](https://github.com/basho/riak_cs/pull/857) and [stanchion/#78](https://github.com/basho/stanchion/pull/78) -* list multipart upload の空クエリパラメータの扱いを修正 [riak_cs/#843](https://github.com/basho/riak_cs/pull/843) -* PUT Object 時にヘッダ指定の ACL が設定されないバグを修正 [riak_cs/#631](https://github.com/basho/riak_cs/pull/631) -* ping リクエストの poolboy タイムアウト処理を改善 [riak_cs/#763](https://github.com/basho/riak_cs/pull/763) -* 匿名アクセス時の不要なログを削除 [riak_cs/#876](https://github.com/basho/riak_cs/issues/876) -* マルチパートでアップロードされたオブジェクトの ETag 不正を修正 [riak_cs/#855](https://github.com/basho/riak_cs/issues/855) -* PUT Bucket Policy のポリシーバージョン確認の不具合を修正[riak_cs/#911](https://github.com/basho/riak_cs/issues/911) -* コマンド成功時に終了コード 0 を返すよう修正 [riak_cs/#908](https://github.com/basho/riak_cs/issues/908) -* `{error, disconnected}` が内部で notfound に書き換えられる問題を修正 [riak_cs/#929](https://github.com/basho/riak_cs/issues/929) - -## アップグレードに関する注意事項 - -### Riak Version - -このリリースは Riak 1.4.10 上でテストされました。 -[互換性マトリクス](http://docs.basho.com/riakcs/latest/cookbooks/Version-Compatibility/) -を参考に、正しいバージョンを使用していることを確認してください。 - -### 未完了のマルチパートアップロード - -[riak_cs/#475](https://github.com/basho/riak_cs/issues/475) はセキュリティ -に関する問題で、以前に作られた同名のバケットに -対する未完了のマルチパートアップロードが、新しく作成されたバケットに -含まれてしまう可能性があります。これは次のように修正されました。 - -- バケット作成時には、有効なマルチパートが存在するかを確認し、 - 存在する場合には 500 エラーをクライアントに返します。 - -- バケット削除時には、まず存在する有効なマルチパートの削除を試みた後に、 - 有効なマルチパートが存在するかを(Stanchion 上で)再度確認します。 - 存在する場合には 409 エラーをクライアントに返します。 - -1.4.x (またはそれより前のバージョン)から 1.5.0 へのアップグレード後には -いくつかの操作が必要です。 - -- すべてのバケットを正常な状態にするため、 `riak-cs-admin - cleanup-orphan-multipart` を実行します。マルチパートアップロードとバ - ケット削除が競合したときに発生しうるコーナーケースを避けるために、こ - のコマンドは `2014-07-30T11:09:30.000Z`のような、 ISO 8601 形式の日付 - を引数として指定することができます。この引数があるとき、バケットのク - リーンアップ操作はそのタイムスタンプよりも新しいマルチパートアップロー - ドを削除しません。もしこれを指定する場合は、全てのCSノードのアップグ - レードが終わって以降の時間がよいでしょう。 - -- 上記操作が終了するまでの期間は、削除済みのバケットで、未完了のマルチ - パートアップロードを含むバケットは再作成が出来ない場合があります。こ - のような再作成の失敗は [critical] ログ (`"Multipart upload remains - in deleted bucket "`) で確認可能です。 - -### ガベージコレクションの猶予期間(Leeway seconds)とディスク空き容量 - -[riak_cs/#470](https://github.com/basho/riak_cs/pull/470) は、 -オブジェクト削除とガベージコレクションの振る舞いを次のように変更します。 -これまで、ガベージコレクションバケットのタイムスタンプはオブジェクトが -回収される将来の時刻でしたが、削除された時刻そのものへと変わります。 -同時に、ガベージコレクターは現在の時刻までのタイムスタンプを回収していましたが、 -猶予期間(`leeway_seconds`)だけ過去のタイムスタンプまでだけを回収するようになります。 - -以前(- 1.4.x): - -``` - t1 t2 ------------+--------------------------+-------------------> - DELETE object: GC 実行: - "t1 + leeway" "t2" までの - とマークされる オブジェクトを回収 -``` - -今後(1.5.0-): - -``` - t1 t2 ------------+--------------------------+-------------------> - DELETE object: GC 実行: - "t1" "t2 - leeway" までの - とマークされる オブジェクトを回収 -``` - -これにより、1.5.0 へのアップグレード直後(仮に`t0`とします）にはオブジェ -クトが回収されない期間ができます。つまり `t0` から `t0 + leeway` までの -期間です。そして `t0` 直前に削除されたオブジェクトは `t0 + 2 * leeway` -時点で回収可能になります。 - -ローリングアップグレードに際しては、GC を実行している CS ノードを -**最初に** アップグレードする必要があります。 -GC を実行しない CS ノードは、猶予期間が正しく動作するために、その後から -アップグレードして下さい。 -また、`riak-cs-admin gc set-interval infinity` コマンドを実行して -ガベージコレクションを無効にしておくと、ノードの順序を -気にすることなくアップグレードが可能です。 - -マルチデータセンター構成のクラスタは、より慎重になる必要があります。 -ガベージコレクションを確実に無効化してからアップグレードしてください。 - -## 既知の問題と制限事項 - -* コピーを実行中にクライアントが次のリクエストを送信するとコピーは中断 - されます。これはクライアントの切断を検出してコピーを中止する機構の副 - 作用です。詳しくは [#932](https://github.com/basho/riak_cs/pull/932) - をご覧ください。 - -* OOSインターフェースでのコピーはサポートされていません。 - -* Multibag はオブジェクトのマニフェストとブロックを複数の異なるクラスタ - に分けて格納する機能です。これは Riak CS Enterprise の機能として追加 - されましたが、技術プレビューの段階にあります。クラスタ間レプリケーショ - ンによる `proxy_get` はサポートされておりません。Multibagは今のところ、 - ひとつのDCでのみ動作するように設計されています。 - -# Riak CS 1.4.5 リリースノート - -## 修正されたバグ - -* list objects v2 fsm のいくつかのデータが「見えない」バグを修正 [riak_cs/788](https://github.com/basho/riak_cs/pull/788) -* HEADリクエスト時にアクセス集計していた問題を修正 [riak_cs/791](https://github.com/basho/riak_cs/pull/791) -* POST/PUTリクエスト時のXML中の空白文字の対処 [riak_cs/795](https://github.com/basho/riak_cs/pull/795) -* ストレージ使用量計算時の誤ったバケット名を修正 [riak_cs/800](https://github.com/basho/riak_cs/pull/800) - Riak CS 1.4.4 で混入したバグにより、そのバージョンを使用している期間の - ストレージ計算はバケット名が文字列 "struct" に置き換わった結果となっていました。 - 本バージョン 1.4.5 でこのバグ自体は修正されましたが、すでに計算済みの古い結果を - さかのぼって修正することは不可能です。バケット名が "struct" に置き換わってしまった - 計算結果では、個別バケットの使用量を知ることはできませんが、その場合であっても - 個々のユーザに関して所有バケットにわたる合計は正しい数字を示します。 -* Unicodeのユーザ名とXMLの対応 [riak_cs/807](https://github.com/basho/riak_cs/pull/807) -* ストレージ使用量で必要なXMLフィールドを追加 [riak_cs/808](https://github.com/basho/riak_cs/pull/808) -* オブジェクトのfoldのタイムアウトを揃えた [riak_cs/811](https://github.com/basho/riak_cs/pull/811) -* 削除されたバケットをユーザーのレコードから削除 [riak_cs/812](https://github.com/basho/riak_cs/pull/812) - -## 新規追加 - -* オブジェクト一覧表示のv2 FSMでプレフィクスを使用する最適化を追加 [riak_cs/804](https://github.com/basho/riak_cs/pull/804) - -# Riak CS 1.4.4 リリースノート - -これはバグフィックスのためのリリースです。統計計算の修正が含まれています。 - -## 修正されたバグ - -* basho-patches ディレクトリが作成されなかった問題を修正 [riak_cs/775](https://github.com/basho/riak_cs/issues/775) . - -* `sum_bucket` のタイムアウトが全ての容量計算をクラッシュさせていた問題を修正 [riak_cs/759](https://github.com/basho/riak_cs/issues/759) . - -* アクセス集計のスロットリング失敗を修正 [riak_cs/758](https://github.com/basho/riak_cs/issues/758) . - -* アクセス集計のクラッシュを修正 [riak_cs/747](https://github.com/basho/riak_cs/issues/747) . - - -# Riak CS 1.4.3 リリースノート - -## 修正された問題 - -- schedule_delete状態のマニフェストがpending_deleteやactive状態へ復帰するバグを修正。 -- 上書きによって既に削除されたマニフェストをカウントしない。 -- 誤ったmd5による上書き操作で、既存バージョンのオブジェクトを削除しない。 - -## 新規追加 - -- マニフェストプルーニングのパフォーマンス改善。 -- GCにおける2iのページングオプションを追加。GC対象データ収集時のタイムアウト対策。 -- ブロック取得処理における接続断のハンドリングを改善。 -- lager 2.0.1へのアップデート。 -- 時刻によるマニフェストプルーニングに個数オプションを追加。 -- 複数アクセスアーカイブプロセスの並行実行を許可。 - -# Riak CS 1.4.2 リリースノート - -## 修正された問題 - -- Debian Linux 上の Enterprise 版ビルドの問題を修正。 -- ソース tarball ビルドの問題を修正。 -- アクセス統計において、正常アクセスがエラーと扱われてしまうバグを修正。 -- Riak バージョン 1.4 以前とあわせて動作するよう、バケットリスト - map フェーズのログを lager バージョンに依存しないよう変更。 -- Riak CS 1.3.0 以前で保存されたマニフェストについて、 `props` フィールド - の `undefined` を正しく扱うよう修正。 - -## 新規追加 - -- 最初のガベージコレクションの遅延を設定する `initial_gc_delay` オプションを追加。 -- ガベージコレクションバケットのキーにランダムなサフィックスを追加し、 - ホットキーの回避と削除の性能を向上。 -- マニフェストに cluster id が指定されていない場合に用いる - `default_proxy_cluster_id` オプションを追加。OSS 版から Enterprise 版への - 移行が容易になる。 - -# Riak CS 1.4.1 リリースノート - -## 修正された問題 - -- 最初の1002個のキーがpending delete状態だったときにlist objectsがクラッシュ - する問題を修正 -- GCデーモンがクラッシュする問題を解決 -- node_packageをアップデートしパッケージ作成の問題を解決 - -# Riak CS 1.4.0 リリースノート - -## 修正された問題 - -- GCバケットで使われていないキーを削除 -- マルチパートアップロードのクエリ文字での認証を修正 -- マルチパートでアップロードされたオブジェクトのストレージクラスを修正 -- マルチパートアップロードされたオブジェクトのetagsを修正 -- Riak CSのマルチバックエンドのインデックス修正をサポート -- GETリクエストの際、通信が遅い場合のメモリ増大を修正 -- アクセス統計処理のメモリ使用量を削減 -- オブジェクトのACL HEADリクエストの際の500を修正 -- マルチパートでアップロードされたオブジェクトの並列アップロードや削除の際の競合の問題を解決 -- Content-md5のヘッダがあった場合に整合性をチェックするように修正 -- Riakとのコネクションが切れた際のハンドリングを修正 - -## 新規追加 - -- Swift APIとKeystone認証のサポートを試験的に追加 -- Riak 1.4.0以降と併用された場合のオブジェクト一覧取得のパフォーマンスを改善 -- ユーザーアカウント名とメールアドレスは変更可能に -- データセンタ間レプリケーションv3のサポートを追加 -- Riakとのコネクションタイムアウトを変更可能に -- Lagerのsyslogサポートを追加 -- データブロックへのリクエスト時は1つのvnodeへアクセス - -# Riak CS 1.3.1 Release Notes - -## Bugs Fixed - -- Fix bug in handling of active object manifests in the case of - overwrite or delete that could lead to old object versions being - resurrected. -- Fix improper capitalization of user metadata header names. -- Fix issue where the S3 rewrite module omits any query parameters - that are not S3 subresources. Also correct handling of query - parameters so that parameter values are not URL decoded twice. This - primarily affects pre-signed URLs because the access key and request - signature are included as query parameters. -- Fix for issue with init script stop. - -# Riak CS 1.3.0 Release Notes - -## Bugs Fixed - -- Fix handling of cases where buckets have siblings. Previously this - resulted in 500 errors returned to the client. -- Reduce likelihood of sibling creation when creating a bucket. -- Return a 404 instead of a 403 when accessing a deleted object. -- Unquote URLs to accommodate clients that URL encode `/` characters - in URLs. -- Deny anonymous service-level requests to avoid unnecessary error - messages trying to list the buckets owned by an undefined user. - -## Additions - -- Support for multipart file uploads. Parts must be in the range of - 5MB-5GB. -- Support for bucket policies using a restricted set of principals and - conditions. -- Support for returning bytes ranges of a file using the Range header. -- Administrative commands may be segrated onto a separate interface. -- Authentication for administrative commands may be disabled. -- Performance and stability improvements for listing the contents of - buckets. -- Support for the prefix, delimiter, and marker options when listing - the contents of a bucket. -- Support for using Webmachine's access logging features in - conjunction with the Riak CS internal access logging mechanism. -- Moved all administrative resources under /riak-cs. -- Riak CS now supports packaging for FreeBSD, SmartOS, and Solaris. - -# Riak CS 1.2.2 Release Notes - -## Bugs Fixed - -- Fix problem where objects with utf-8 unicode key cannot be listed - nor fetched. -- Speed up bucket_empty check and fix process leak. This bug was - originally found when a user was having trouble with `s3cmd - rb s3://foo --recursive`. The operation first tries to delete the - (potentially large) bucket, which triggers our bucket empty - check. If the bucket has more than 32k items, we run out of - processes unless +P is set higher (because of the leak). - -## Additions - -- Full support for MDC replication - -# Riak CS 1.2.1 Release Notes - -## Bugs Fixed - -- Return 403 instead of 404 when a user attempts to list contents of - nonexistent bucket. -- Do not do bucket list for HEAD or ?versioning or ?location request. - -## Additions - -- Add reduce phase for listing bucket contents to provide backpressure - when executing the MapReduce job. -- Use prereduce during storage calculations. -- Return 403 instead of 404 when a user attempts to list contents of - nonexistent bucket. - -# Riak CS 1.2.0 Release Notes - -## Bugs Fixed - -- Do not expose stack traces to users on 500 errors -- Fix issue with sibling creation on user record updates -- Fix crash in terminate state when fsm state is not fully populated -- Script fixes and updates in response to node_package updates - -## Additions - -- Add preliminary support for MDC replication -- Quickcheck test to exercise the erlcloud library against Riak CS -- Basic support for riak_test integration - -# Riak CS 1.1.0 Release Notes - -## Bugs Fixed - -- Check for timeout when checking out a connection from poolboy. -- PUT object now returns 200 instead of 204. -- Fixes for Dialyzer errors and warnings. -- Return readable error message with 500 errors instead of large webmachine backtraces. - -## Additions - -- Update user creation to accept a JSON or XML document for user - creation instead of URL encoded text string. -- Configuration option to allow anonymous users to create accounts. In - the default mode, only the administrator is allowed to create - accounts. -- Ping resource for health checks. -- Support for user-specified metadata headers. -- User accounts may be disabled by the administrator. -- A new key_secret can be issued for a user by the administrator. -- Administrator can now list all system users and optionally filter by - enabled or disabled account status. -- Garbage collection for deleted and overwritten objects. -- Separate connection pool for object listings with a default of 5 - connections. -- Improved performance for listing all objects in a bucket. -- Statistics collection and querying. -- DTrace probing. - -# Riak CS 1.0.2 Release Notes - -## Additions - -- Support query parameter authentication as specified in [[http://docs.amazonwebservices.com/AmazonS3/latest/dev/RESTAuthentication.html][Signing and Authenticating REST Requests]]. - -# Riak CS 1.0.1 Release Notes - -## Bugs Fixed - -- Default content-type is not passed into function to handle PUT - request body -- Requests hang when a node in the Riak cluster is unavailable -- Correct inappropriate use of riak_moss_utils:get_user by - riak_moss_acl_utils:get_owner_data - -# Riak CS 1.0.0 Release Notes - -## Bugs Fixed - -- Fix PUTs for zero-byte files -- Fix fsm initialization race conditions -- Canonicalize the entire path if there is no host header, but there are - tokens -- Fix process and socket leaks in get fsm - -## Other Additions - -- Subsystem for calculating user access and storage usage -- Fixed-size connection pool of Riak connections -- Use a single Riak connection per request to avoid deadlock conditions -- Object ACLs -- Management for multiple versions of a file manifest -- Configurable block size and max content length -- Support specifying non-default ACL at bucket creation time - -# Riak CS 0.1.2 Release Notes - -## Bugs Fixed - -- Return 403 instead of 503 for invalid anonymous or signed requests. -- Properly clean up processes and connections on object requests. - -# Riak CS 0.1.1 Release Notes - -## Bugs Fixed - -- HEAD requests always result in a `403 Forbidden`. -- `s3cmd info` on a bucket object results in an error due to missing - ACL document. -- Incorrect atom specified in `riak_moss_wm_utils:parse_auth_header`. -- Bad match condition used in `riak_moss_acl:has_permission/2`. - -# Riak CS 0.1.0 Release Notes - -## Bugs Fixed - -- `s3cmd info` fails due to missing `'last-modified` key in return document. -- `s3cmd get` of 0 byte file fails. -- Bucket creation fails with status code `415` using the AWS Java SDK. - -## Other Additions - -- Bucket-level access control lists -- User records have been modified so that an system-wide unique email - address is required to create a user. -- User creation requests are serialized through `stanchion` to be - certain the email address is unique. -- Bucket creation and deletion requests are serialized through - `stanchion` to ensure bucket names are unique in the system. -- The `stanchion` serialization service is now required to be installed - and running for the system to be fully operational. -- The concept of an administrative user has been added to the system. The credentials of the - administrative user must be added to the app.config files for `moss` and `stanchion`. -- User credentials are now created using a url-safe base64 encoding module. - -## Known Issues - -- Object-level access control lists have not yet been implemented. - -# Riak CS 0.0.3 Release Notes - -## Bugs Fixed - -- URL decode keys on put so they are represented correctly. This - eliminates confusion when objects with spaces in their names are - listed and when attempting to access them. -- Properly handle zero-byte files -- Reap all processes during file puts - -## Other Additions - -- Support for the s3cmd subcommands sync, du, and rb - - - Return valid size and checksum for each object when listing bucket objects. - - Changes so that a bucket may be deleted if it is empty. - -- Changes so a subdirectory path can be specified when storing or retrieving files. -- Make buckets private by default -- Support the prefix query parameter -- Enhance process dependencies for improved failure handling - -## Known Issues - -- Buckets are marked as /private/ by default, but globally-unique - bucket names are not enforced. This means that two users may - create the same bucket and this could result in unauthorized - access and unintentional overwriting of files. This will be - addressed in a future release by ensuring that bucket names are - unique across the system. diff --git a/releasenotes/riak-cs.1.md b/releasenotes/riak-cs.1.md deleted file mode 100644 index 332bdea1d..000000000 --- a/releasenotes/riak-cs.1.md +++ /dev/null @@ -1,666 +0,0 @@ -# Riak CS 1.5.4 Release Notes - -## Bugs Fixed - -- Disable previous Riak object after backpressure sleep is triggered - [riak_cs/#1041](https://github.com/basho/riak_cs/pull/1041). This - change prevents unnecessary siblings growth in cases where (a) - backpressure is triggered under high upload concurrency and (b) - uploads are interleaved during backpressure sleep. This issue does not - affect multipart uploads. -- Fix an incorrect path rewrite in the S3 API caused by unnecessary URL - decoding - [riak_cs/#1040](https://github.com/basho/riak_cs/pull/1040). Due to - the incorrect handling of URL encoding/decoding, object keys including - `%[0-9a-fA-F][0-9a-fA-F]` (as a regular expression) or `+` had been - mistakenly decoded. As a consequence, the former case was decoded to - some other binary and for the latter case (`+`) was replaced with ` ` - (space). In both cases, there was a possibility of an implicit data - overwrite. For the latter case, an overwrite occurs for an object - including `+` in its key (e.g. `foo+bar`) by a different object with a - name that is largely similar but replaced with ` ` (space, e.g. `foo - bar`), and vice versa. This fix also addresses - [riak_cs/#910](https://github.com/basho/riak_cs/pull/910) and - [riak_cs/#977](https://github.com/basho/riak_cs/pull/977). - -## Notes on upgrading - -After upgrading to Riak CS 1.5.4, objects including -`%[0-9a-fA-F][0-9a-fA-F]` or `+` in their key (e.g. `foo+bar`) become -invisible and can be seen as objects with a different name. For the -former case, objects will be referred as unnecessary decoded key. For -the latter case, those objects will be referred as keys `+` replaced -with ` ` (e.g. `foo bar`) by default. - -The table below provides examples for URLs including -`%[0-9a-fA-F][0-9a-fA-F]` and how they will work before and after the -upgrade. - - | before upgrade | after upgrade | -:-----------|:-------------------|:--------------| - written as | `a%2Fkey` | - | - read as | `a%2Fkey`or`a/key` | `a/key` | - listed as | `a/key` | `a/key` | - -Examples on unique objects including `+` or ` ` through upgrade: - - | before upgrade | after upgrade | -:-----------|------------------|---------------| - written as | `a+key` | - | - read as | `a+key`or`a key` | `a key` | - listed as | `a key` | `a key` | - - | before upgrade | after upgrade | -------------|------------------|---------------| - written as | `a key` | - | - read as | `a+key`or`a key` | `a key` | - listed as | `a key` | `a key` | - -This fix also changes the path format in access logs from the single -URL-encoded style to the doubly-encoded URL style. Below is an example -of the old style: - -``` -127.0.0.1 - - [07/Jan/2015:08:27:07 +0000] "PUT /buckets/test/objects/path1%2Fpath2%2Fte%2Bst.txt HTTP/1.1" 200 0 "" "" -``` - -Below is an example of the new style: - -``` -127.0.0.1 - - [07/Jan/2015:08:27:07 +0000] "PUT /buckets/test/objects/path1%2Fpath2%2Fte%252Bst.txt HTTP/1.1" 200 0 "" "" -``` - -Note that the object path has changed from -`path1%2Fpath2%2Fte%2Bst.txt` to `path1%2Fpath2%2Fte%252Bst.txt` between -the two examples shown. - -If the old behavior is preferred, perhaps because -applications using Riak CS have been written to use it, you can retain -that behavior by modifying your Riak CS configuration upon upgrade. -Change the `rewrite_module` setting as follows: - -```erlang -{riak_cs, [ - %% Other settings - {rewrite_module, riak_cs_s3_rewrite_legacy}, - %% Other settings -]} -``` - -**Note**: The old behavior is technically incorrect and implicitly -overwrites data in the ways described above, so please retain the old -behavior with caution. - -# Riak CS 1.5.3 Release Notes - -## Additions - -- Add read_before_last_manifest_write option to help avoid sibling - explosion for use cases involving high churn and concurrency on a - fixed set of keys. [riak_cs/#1011](https://github.com/basho/riak_cs/pull/1011) -- Add configurable timeouts for all Riak CS interactions with Riak to - provide more flexibility in operational - situations. [riak_cs/#1021](https://github.com/basho/riak_cs/pull/1021) - -## Bugs Fixed - -- Fix storage usage calculation bug where data for deleted buckets - would be included in the calculation - results. [riak_cs/#996](https://github.com/basho/riak_cs/pull/996) - -# Riak CS 1.5.2 Release Notes - -## Additions - -- Improved logging around connection failures with Riak - [riak_cs/#987](https://github.com/basho/riak_cs/pull/987). -- Add amendment log output when storing access stats into Riak failed - [riak_cs/#988](https://github.com/basho/riak_cs/pull/988). This - prevents losing access stats logs in cases of temporary connection - failure between Riak and Riak CS. Access logs are stored in - `console.log` at the `warning` level. -- Add script to repair invalid garbage collection manifests - [riak_cs/#983](https://github.com/basho/riak_cs/pull/983). There is - a [known issue](https://github.com/basho/riak_cs/issues/827) where - an active manifest would be stored in the GC bucket. This script - changes invalid state to valid state. - -## Bugs Fixed - -- Fix Protocol Buffer connection pool (`pbc_pool_master`) leak - [riak_cs/#986](https://github.com/basho/riak_cs/pull/986) . Requests - for non-existent buckets without an authorization header and - requests asking for listing users make connections leak from the - pool, causing the pool to eventually go empty. This bug was introduced - in release 1.5.0. - -# Riak CS 1.5.1 Release Notes - -## Additions - -- Add sleep-after-update manifests to avoid sibling explosion [riak_cs/#959](https://github.com/basho/riak_cs/pull/959) -- Multibag support on `riak-cs-debug` [riak_cs/#930](https://github.com/basho/riak_cs/pull/930) -- Add bucket number limit check in Riak CS process [riak_cs/#950](https://github.com/basho/riak_cs/pull/950) -- More efficient bucket resolution [riak_cs/#951](https://github.com/basho/riak_cs/pull/951) - -## Bugs Fixed - -- GC may stall due to `riak_cs_delete_fsm` deadlock [riak_cs/#949](https://github.com/basho/riak_cs/pull/949) -- Fix wrong log directory for gathering logs on `riak-cs-debug` [riak_cs/#953](https://github.com/basho/riak_cs/pull/953) -- Avoid DST-aware translation from local time to GMT [riak_cs/#954](https://github.com/basho/riak_cs/pull/954) -- Use new UUID for seed of canonical ID instead of secret [riak_cs/#956](https://github.com/basho/riak_cs/pull/956) -- Add max part number limitation [riak_cs/#957](https://github.com/basho/riak_cs/pull/957) -- Set timeout as infinity to replace the default of 5000ms [riak_cs/#963](https://github.com/basho/riak_cs/pull/963) -- Skip invalid state manifests in GC bucket [riak_cs/#964](https://github.com/basho/riak_cs/pull/964) - -## Notes on Upgrading - -### Bucket number per user - -Beginning with Riak CS 1.5.1, you can limit the number of buckets that -can be created per user. The default maximum number is 100. While this -limitation prohibits the creation of new buckets by users, users that -exceed the limit can still perform other operations, including bucket -deletion. To change the default limit, add the following line to the -`riak_cs` section of `app.config`: - - -```erlang -{riak_cs, [ - %% ... - {max_buckets_per_user, 5000}, - %% ... - ]} -``` - -To avoid having a limit, set `max_buckets_per_user` to `unlimited`. - -# Riak CS 1.5.0 Release Notes - -## Additions - -* A new command `riak-cs-debug` including `cluster-info` [riak_cs/#769](https://github.com/basho/riak_cs/pull/769), [riak_cs/#832](https://github.com/basho/riak_cs/pull/832) -* Tie up all existing commands into a new command `riak-cs-admin` [riak_cs/#839](https://github.com/basho/riak_cs/pull/839) -* Add a command `riak-cs-admin stanchion` to switch Stanchion IP and port manually [riak_cs/#657](https://github.com/basho/riak_cs/pull/657) -* Performance of garbage collection has been improved via Concurrent GC [riak_cs/#830](https://github.com/basho/riak_cs/pull/830) -* Iterator refresh [riak_cs/#805](https://github.com/basho/riak_cs/pull/805) -* `fold_objects_for_list_keys` made default in Riak CS [riak_cs/#737](https://github.com/basho/riak_cs/pull/737), [riak_cs/#785](https://github.com/basho/riak_cs/pull/785) -* Add support for Cache-Control header [riak_cs/#821](https://github.com/basho/riak_cs/pull/821) -* Allow objects to be reaped sooner than leeway interval. [riak_cs/#470](https://github.com/basho/riak_cs/pull/470) -* PUT Copy on both objects and upload parts [riak_cs/#548](https://github.com/basho/riak_cs/pull/548) -* Update to lager 2.0.3 -* Compiles with R16B0x (Releases still by R15B01) -* Change default value of `gc_paginated_index` to `true` [riak_cs/#881](https://github.com/basho/riak_cs/issues/881) -* Add new API: Delete Multiple Objects [riak_cs/#728](https://github.com/basho/riak_cs/pull/728) -* Add warning logs for manifests, siblings, bytes and history [riak_cs/#915](https://github.com/basho/riak_cs/pull/915) - -## Bugs Fixed - -* Align `ERL_MAX_PORTS` with Riak default: 64000 [riak_cs/#636](https://github.com/basho/riak_cs/pull/636) -* Allow Riak CS admin resources to be used with OpenStack API [riak_cs/#666](https://github.com/basho/riak_cs/pull/666) -* Fix path substitution code to fix Solaris source builds [riak_cs/#733](https://github.com/basho/riak_cs/pull/733) -* `sanity_check(true,false)` logs invalid error on `riakc_pb_socket` error [riak_cs/#683](https://github.com/basho/riak_cs/pull/683) -* Riak-CS-GC timestamp for scheduler is in the year 0043, not 2013. [riak_cs/#713](https://github.com/basho/riak_cs/pull/713) fixed by [riak_cs/#676](https://github.com/basho/riak_cs/pull/676) -* Excessive calls to OTP code_server process #669 fixed by [riak_cs/#675](https://github.com/basho/riak_cs/pull/675) -* Return HTTP 400 if content-md5 does not match [riak_cs/#596](https://github.com/basho/riak_cs/pull/596) -* `/riak-cs/stats` and `admin_auth_enabled=false` don't work together correctly. [riak_cs/#719](https://github.com/basho/riak_cs/pull/719) -* Storage calculation doesn't handle tombstones, nor handle undefined manifest.props [riak_cs/#849](https://github.com/basho/riak_cs/pull/849) -* MP initiated objects remains after delete/create buckets #475 fixed by [riak_cs/#857](https://github.com/basho/riak_cs/pull/857) and [stanchion/#78](https://github.com/basho/stanchion/pull/78) -* handling empty query string on list multipart upload [riak_cs/#843](https://github.com/basho/riak_cs/pull/843) -* Setting ACLs via headers at PUT Object creation [riak_cs/#631](https://github.com/basho/riak_cs/pull/631) -* Improve handling of poolboy timeouts during ping requests [riak_cs/#763](https://github.com/basho/riak_cs/pull/763) -* Remove unnecessary log message on anonymous access [riak_cs/#876](https://github.com/basho/riak_cs/issues/876) -* Fix inconsistent ETag on objects uploaded by multipart [riak_cs/#855](https://github.com/basho/riak_cs/issues/855) -* Fix policy version validation in PUT Bucket Policy [riak_cs/#911](https://github.com/basho/riak_cs/issues/911) -* Fix return code of several commands, to return 0 for success [riak_cs/#908](https://github.com/basho/riak_cs/issues/908) -* Fix `{error, disconnected}` repainted with notfound [riak_cs/#929](https://github.com/basho/riak_cs/issues/929) - -## Notes on Upgrading - -### Riak Version - -This release of Riak CS was tested with Riak 1.4.10. Be sure to -consult the -[Compatibility Matrix](http://docs.basho.com/riakcs/latest/cookbooks/Version-Compatibility/) -to ensure that you are using the correct version. - -### Incomplete multipart uploads - -[riak_cs/#475](https://github.com/basho/riak_cs/issues/475) was a -security issue where a newly created bucket may include unaborted or -incomplete multipart uploads which was created in previous epoch of -the bucket with same name. This was fixed by: - -- on creating buckets; checking if live multipart exists and if - exists, return 500 failure to client. - -- on deleting buckets; trying to clean up all live multipart remains, - and checking if live multipart remains (in stanchion). if exists, - return 409 failure to client. - -Note that a few operations are needed after upgrading from 1.4.x (or -former) to 1.5.0. - -- run `riak-cs-admin cleanup-orphan-multipart` to cleanup all - buckets. To avoid some corner cases where multipart uploads can - conflict with bucket deletion, this command can also be run with a - timestamp with ISO 8601 format such as `2014-07-30T11:09:30.000Z` as - an argument. When this argument is provided, the cleanup operation - will not clean up multipart uploads that are newer than the provided - timestamp. If used, this should be set to a time when you expect - your upgrade to be completed. - -- there might be a time period until above cleanup finished, where no - client can create bucket if unfinished multipart upload remains - under deleted bucket. You can find [critical] log (`"Multipart - upload remains in deleted bucket "`) if such bucket - creation is attempted. - -### Leeway seconds and disk space - -[riak_cs/#470](https://github.com/basho/riak_cs/pull/470) changed the -behaviour of object deletion and garbage collection. The timestamps in -garbage collection bucket were changed from the future time when the -object is to be deleted, to the current time when the object is -deleted, Garbage collector was also changed to collect objects until -'now - leeway seconds', from collecting objects until 'now'. - -Before (-1.4.x): - -``` - t1 t2 ------------+--------------------------+-------------------> - DELETE object: GC triggered: - marked as collects objects - "t1+leeway" marked as "t2" -``` - -After (1.5.0-): - -``` - t1 t2 ------------+--------------------------+-------------------> - DELETE object: GC triggered: - marked as "t1" collects objects - in GC bucket marked as "t2 - leeway" -``` - -This means that there will be a period where no objects are collected -immediately following an upgrade to 1.5.0. If your cluster is upgraded -at `t0`, no objects will be cleaned up until `t0 + leeway` . Objects -deleted just before `t0` won't be collected until `t0 + 2*leeway` . - -Also, all CS nodes which run GC should be upgraded *first.* CS nodes -which do not run GC should be upgraded later, to ensure the leeway -setting is intiated properly. Alternatively, you may stop GC while -upgrading, by running `riak-cs-admin gc set-interval infinity` . - -Multi data center cluster should be upgraded more carefully, as to -make sure GC is not running while upgrading. - -## Known Issues and Limitations - -* If a second client request is made using the same connection while a - copy operation is in progress, the copy will be aborted. This is a - side effect of the way Riak CS currently handles client disconnect - detection. See [#932](https://github.com/basho/riak_cs/pull/932) for - further information. - -* Copying objects in OOS interface is not yet implemented. - -* Multibag, the ability to store object manifests and blocks in - separate clusters or groups of clusters, has been added as - Enterprise feature, but it is in early preview status. `proxy_get` - has not yet been implemented for this preview feature. Multibag is - intended for single DC only at this time. - -# Riak CS 1.4.5 Release Notes - -## Bugs Fixed - -* Fix several 'data hiding' bugs with the v2 list objects FSM [riak_cs/788](https://github.com/basho/riak_cs/pull/788) -* Don't treat HEAD requests toward BytesOut in access statistics [riak_cs/791](https://github.com/basho/riak_cs/pull/791) -* Handle whitespace in POST/PUT XML documents [riak_cs/795](https://github.com/basho/riak_cs/pull/795) -* Fix bad bucketname in storage usage [riak_cs/800](https://github.com/basho/riak_cs/pull/800) - Riak CS 1.4.4 introduced a bug where storage calculations made while running - that version would have the bucket-name replaced by the string "struct". This - version fixes the bug, but can't go back and retroactively fix the old - storage calculations. Aggregations on an entire user-account should still - be accurate, but you won't be able to break-down storage by bucket, as they - will all share the name "struct". -* Handle unicode user-names and XML [riak_cs/807](https://github.com/basho/riak_cs/pull/807) -* Fix missing XML fields on storage usage [riak_cs/808](https://github.com/basho/riak_cs/pull/808) -* Adjust fold-objects timeout [riak_cs/811](https://github.com/basho/riak_cs/pull/811) -* Prune deleted buckets from user record [riak_cs/812](https://github.com/basho/riak_cs/pull/812) - -## Additions - -* Optimize the list objects v2 FSM for prefix requests [riak_cs/804](https://github.com/basho/riak_cs/pull/804) - -# Riak CS 1.4.4 Release Notes - -This is a bugfix release. The major fixes are to the storage calculation. - -## Bugs Fixed - -* Create basho-patches directory [riak_cs/775](https://github.com/basho/riak_cs/issues/775) . - -* `sum_bucket` timeout crashes all storage calculation is fixed by [riak_cs/759](https://github.com/basho/riak_cs/issues/759) . - -* Failure to throttle access archiver is fixed by [riak_cs/758](https://github.com/basho/riak_cs/issues/758) . - -* Access archiver crash is fixed by [riak_cs/747](https://github.com/basho/riak_cs/issues/747) . - - -# Riak CS 1.4.3 Release Notes - -## Bugs Fixed - -- Fix bug that reverted manifests in the scheduled_delete state to the - pending_delete or active state. -- Don't count already deleted manifests as overwritten -- Don't delete current object version on overwrite with incorrect md5 - -## Additions - -- Improve performance of manifest pruning -- Optionally use paginated 2i for the GC daemon. This is to help prevent - timeouts when collecting data that can be garbage collected. -- Improve handling of Riak disconnects on block fetches -- Update to lager 2.0.1 -- Optionally prune manifests based on count, in addition to time -- Allow multiple access archiver processes to run concurrently - -# Riak CS 1.4.2 Release Notes - -## Bugs Fixed - -- Fix issue with Enterprise build on Debian Linux distributions. -- Fix source tarball build. -- Fix access statistics bug that caused all accesses to be treated as - errors. -- Make logging in bucket listing map phase function lager version - agnostic to avoid issues when using versions of Riak older than 1.4. -- Handle undefined `props` field in manifests to fix issue accessing - objects written with a version of Riak CS older than 1.3.0. - -## Additions - -- Add option to delay initial GC sweep on a node using the - initial_gc_delay configuration option. -- Append random suffix to GC bucket keys to avoid hot keys and improve - performance during periods of frequent deletion. -- Add default_proxy_cluster_id option to provide a way to specify a - default cluster id to be used when the cluster id is undefined. This is - to facilitate migration from the OSS version to the - Enterprise version. - -# Riak CS 1.4.1 Release Notes - -## Bugs Fixed - -- Fix list objects crash when more than the first 1001 keys are in - the pending delete state -- Fix crash in garbage collection daemon -- Fix packaging bug by updating node_package dependency - -# Riak CS 1.4.0 Release Notes - -## Bugs Fixed - -- Remove unnecessary keys in GC bucket -- Fix query-string authentication for multi-part uploads -- Fix Storage Class for multi-part uploaded objects -- Fix etags for multi-part uploads -- Support reformat indexes in the Riak CS multi-backend -- Fix unbounded memory-growth on GET requests with a slow connection -- Reduce access-archiver memory use -- Fix 500 on object ACL HEAD request -- Fix semantics for concurrent upload and delete of the same key with a - multi-part upload -- Verify content-md5 header if supplied -- Handle transient Riak connection failures - -## Additions - -- Add preliminary support for the Swift API and Keystone authentication -- Improve performance of object listing when using Riak 1.4.0 or greater -- Add ability to edit user account name and email address -- Add support for v3 multi-data-center replication -- Add configurable Riak connection timeouts -- Add syslog support via Lager -- Only contact one vnode for immutable block requests - -# Riak CS 1.3.1 Release Notes - -## Bugs Fixed -- Fix bug in handling of active object manifests in the case of - overwrite or delete that could lead to old object versions being - resurrected. -- Fix improper capitalization of user metadata header names. -- Fix issue where the S3 rewrite module omits any query parameters - that are not S3 subresources. Also correct handling of query - parameters so that parameter values are not URL decoded twice. This - primarily affects pre-signed URLs because the access key and request - signature are included as query parameters. -- Fix for issue with init script stop. - -# Riak CS 1.3.0 Release Notes - -## Bugs Fixed - -- Fix handling of cases where buckets have siblings. Previously this - resulted in 500 errors returned to the client. -- Reduce likelihood of sibling creation when creating a bucket. -- Return a 404 instead of a 403 when accessing a deleted object. -- Unquote URLs to accommodate clients that URL encode `/` characters - in URLs. -- Deny anonymous service-level requests to avoid unnecessary error - messages trying to list the buckets owned by an undefined user. - -## Additions - -- Support for multipart file uploads. Parts must be in the range of - 5MB-5GB. -- Support for bucket policies using a restricted set of principals and - conditions. -- Support for returning bytes ranges of a file using the Range header. -- Administrative commands may be segrated onto a separate interface. -- Authentication for administrative commands may be disabled. -- Performance and stability improvements for listing the contents of - buckets. -- Support for the prefix, delimiter, and marker options when listing - the contents of a bucket. -- Support for using Webmachine's access logging features in - conjunction with the Riak CS internal access logging mechanism. -- Moved all administrative resources under /riak-cs. -- Riak CS now supports packaging for FreeBSD, SmartOS, and Solaris. - -# Riak CS 1.2.2 Release Notes - -## Bugs Fixed - -- Fix problem where objects with utf-8 unicode key cannot be listed - nor fetched. -- Speed up bucket_empty check and fix process leak. This bug was - originally found when a user was having trouble with `s3cmd - rb s3://foo --recursive`. The operation first tries to delete the - (potentially large) bucket, which triggers our bucket empty - check. If the bucket has more than 32k items, we run out of - processes unless +P is set higher (because of the leak). - -## Additions - -- Full support for MDC replication - -# Riak CS 1.2.1 Release Notes - -## Bugs Fixed - -- Return 403 instead of 404 when a user attempts to list contents of - nonexistent bucket. -- Do not do bucket list for HEAD or ?versioning or ?location request. - -## Additions - -- Add reduce phase for listing bucket contents to provide backpressure - when executing the MapReduce job. -- Use prereduce during storage calculations. -- Return 403 instead of 404 when a user attempts to list contents of - nonexistent bucket. - -# Riak CS 1.2.0 Release Notes - -## Bugs Fixed - -- Do not expose stack traces to users on 500 errors -- Fix issue with sibling creation on user record updates -- Fix crash in terminate state when fsm state is not fully populated -- Script fixes and updates in response to node_package updates - -## Additions - -- Add preliminary support for MDC replication -- Quickcheck test to exercise the erlcloud library against Riak CS -- Basic support for riak_test integration - -# Riak CS 1.1.0 Release Notes - -## Bugs Fixed - -- Check for timeout when checking out a connection from poolboy. -- PUT object now returns 200 instead of 204. -- Fixes for Dialyzer errors and warnings. -- Return readable error message with 500 errors instead of large webmachine backtraces. - -## Additions - -- Update user creation to accept a JSON or XML document for user - creation instead of URL encoded text string. -- Configuration option to allow anonymous users to create accounts. In - the default mode, only the administrator is allowed to create - accounts. -- Ping resource for health checks. -- Support for user-specified metadata headers. -- User accounts may be disabled by the administrator. -- A new key_secret can be issued for a user by the administrator. -- Administrator can now list all system users and optionally filter by - enabled or disabled account status. -- Garbage collection for deleted and overwritten objects. -- Separate connection pool for object listings with a default of 5 - connections. -- Improved performance for listing all objects in a bucket. -- Statistics collection and querying. -- DTrace probing. - -# Riak CS 1.0.2 Release Notes - -## Additions - -- Support query parameter authentication as specified in [Signing and Authenticating REST Requests](http://docs.amazonwebservices.com/AmazonS3/latest/dev/RESTAuthentication.html) - -# Riak CS 1.0.1 Release Notes - -## Bugs Fixed - -- Default content-type is not passed into function to handle PUT - request body -- Requests hang when a node in the Riak cluster is unavailable -- Correct inappropriate use of riak_moss_utils:get_user by - riak_moss_acl_utils:get_owner_data - -# Riak CS 1.0.0 Release Notes - -## Bugs Fixed - -- Fix PUTs for zero-byte files -- Fix fsm initialization race conditions -- Canonicalize the entire path if there is no host header, but there are - tokens -- Fix process and socket leaks in get fsm - -## Other Additions - -- Subsystem for calculating user access and storage usage -- Fixed-size connection pool of Riak connections -- Use a single Riak connection per request to avoid deadlock conditions -- Object ACLs -- Management for multiple versions of a file manifest -- Configurable block size and max content length -- Support specifying non-default ACL at bucket creation time - -# Riak CS 0.1.2 Release Notes - -## Bugs Fixed - -- Return 403 instead of 503 for invalid anonymous or signed requests. -- Properly clean up processes and connections on object requests. - -# Riak CS 0.1.1 Release Notes - -## Bugs Fixed - -- HEAD requests always result in a `403 Forbidden`. -- `s3cmd info` on a bucket object results in an error due to missing - ACL document. -- Incorrect atom specified in `riak_moss_wm_utils:parse_auth_header`. -- Bad match condition used in `riak_moss_acl:has_permission/2`. - -# Riak CS 0.1.0 Release Notes - -## Bugs Fixed - -- `s3cmd info` fails due to missing `'last-modified` key in return document. -- `s3cmd get` of 0 byte file fails. -- Bucket creation fails with status code `415` using the AWS Java SDK. - -## Other Additions - -- Bucket-level access control lists -- User records have been modified so that an system-wide unique email - address is required to create a user. -- User creation requests are serialized through `stanchion` to be - certain the email address is unique. -- Bucket creation and deletion requests are serialized through - `stanchion` to ensure bucket names are unique in the system. -- The `stanchion` serialization service is now required to be installed - and running for the system to be fully operational. -- The concept of an administrative user has been added to the system. The credentials of the - administrative user must be added to the app.config files for `moss` and `stanchion`. -- User credentials are now created using a url-safe base64 encoding module. - -## Known Issues - -- Object-level access control lists have not yet been implemented. - -# Riak CS 0.0.3 Release Notes - -## Bugs Fixed - -- URL decode keys on put so they are represented correctly. This - eliminates confusion when objects with spaces in their names are - listed and when attempting to access them. -- Properly handle zero-byte files -- Reap all processes during file puts - -## Other Additions - -- Support for the s3cmd subcommands sync, du, and rb - - - Return valid size and checksum for each object when listing bucket objects. - - Changes so that a bucket may be deleted if it is empty. - -- Changes so a subdirectory path can be specified when storing or retrieving files. -- Make buckets private by default -- Support the prefix query parameter - -- Enhance process dependencies for improved failure handling - -## Known Issues - -- Buckets are marked as /private/ by default, but globally-unique - bucket names are not enforced. This means that two users may - create the same bucket and this could result in unauthorized - access and unintentional overwriting of files. This will be - addressed in a future release by ensuring that bucket names are - unique across the system. diff --git a/riak_test/README.md b/riak_test/README.md deleted file mode 100644 index ee5e5b22a..000000000 --- a/riak_test/README.md +++ /dev/null @@ -1,178 +0,0 @@ -# General instruction - -1. Make sure that your `riak_test` is the latest one for 2.0. - -1. Ensure that riak_test builds are in place for: - * Riak - * Riak EE - * Riak CS - * Stanchion - -Example to setup old and new CS: - -``` -$ mkdir ~/rt -$ cd ~/rt -$ cd path/to/repo/riak_cs -## Only for Enterprise build -$ export RIAK_CS_EE_DEPS=true -$ riak_test/bin/rtdev-build-releases.sh -$ riak_test/bin/rtdev-setup-releases.sh -## make sure runtime is Basho's patched R16B02-basho5 -$ make devrel && riak_test/bin/rtdev-current.sh -``` - -Example to setup old and new Stanchion: - -``` -$ cd path/to/repo/stanchion -$ riak_test/bin/rtdev-build-releases.sh -$ riak_test/bin/rtdev-setup-releases.sh -## make sure runtime is Basho's patched R16B02-basho5 -$ make devrel && riak_test/bin/rtdev-current.sh -``` - -Example to setup 1.4.x and 2.0 as old and new Riak (maybe same as Riak -OSS... while shell scripts are included in riak_test repo): - -``` -$ mkdir ~/rt/riak_ee -## make sure runtime is Basho's patched R16B02-basho5 -$ tar xzf riak-ee-2.0.1.tar.gz -$ cd riak-ee-2.0.1 && make devrel -$ riak_test/bin/rteedev-setup-releases.sh -$ riak_test/bin/rteedev-current.sh - -## change runtime to Basho's patched R15B01 -$ tar xzf riak-ee-1.4.10.tar.gz -$ cd riak-ee-1.4.10 && make devrel -$ mkdir ~/rt/riak_ee/riak-ee-1.4.10 -$ cp -r dev ~/rt/riak_ee/riak-ee-1.4.10 -$ cd ~/rt/riak_ee -$ git add riak-ee-1.4.10 -$ git commit -m "Add 1.4 series Riak EE" -``` - - -1. Setup a `~/.riak_test.config` file like this: - -```erlang - {default, [ - {rt_max_wait_time, 180000}, - {rt_retry_delay, 1000} - ]}. - - {rtdev, [ - {rt_deps, [ - "/Users/kelly/basho/riak_test_builds/riak/deps", - "deps" - ]}, - {rt_retry_delay, 500}, - {rt_harness, rtdev}, - {rtdev_path, [{root, "/Users/kelly/rt/riak"}, - {current, "/Users/kelly/rt/riak/current"}, - {ee_root, "/Users/kelly/rt/riak_ee"}, - {ee_current, "/Users/kelly/rt/riak_ee/current"} - ]} - ]}. - -{rtcs_dev, [ - {rt_project, "riak_cs"}, - {rt_deps, [ - "/home/kuenishi/cs-2.0/riak_cs/deps" - ]}, - {rt_retry_delay, 500}, - {rt_harness, rtcs_dev}, - {build_paths, [{root, "/home/kuenishi/rt/riak_ee"}, - {current, "/home/kuenishi/rt/riak_ee/current"}, - {ee_root, "/home/kuenishi/rt/riak_ee"}, - {ee_current, "/home/kuenishi/rt/riak_ee/current"}, - {ee_previous, "/home/kuenishi/rt/riak_ee/riak-ee-1.4.10"}, - {cs_root, "/home/kuenishi/rt/riak_cs"}, - {cs_current, "/home/kuenishi/rt/riak_cs/current"}, - {cs_previous, - "/home/kuenishi/rt/riak_cs/riak-cs-1.5.1"}, - {stanchion_root, "/home/kuenishi/rt/stanchion"}, - {stanchion_current, "/home/kuenishi/rt/stanchion/current"}, - {stanchion_previous, - "/home/kuenishi/rt/stanchion/stanchion-1.5.0"} - ]}, - {test_paths, ["/home/kuenishi/cs-2.0/riak_cs/riak_test/ebin"]}, - {src_paths, [{cs_src_root, "/home/kuenishi/cs-2.0/riak_cs"}]}, - {lager_level, debug}, - %%{build_type, oss}, - {build_type, ee}, - {flavor, basic}, - {backend, {multi_backend, bitcask}} -]}. -``` - -Running the RiakCS tests for `riak_test` use a different test harness -(`rtcs_dev`) than running the Riak tests and so requires a separate -configuration section. Notice the extra `riak_cs/deps` in the -`rt_deps` section. `RT_DEST_DIR` should be replaced by the path used -when setting up `riak_test` builds for Riak (by default -`$HOME/rt/riak`). The same should be done for `RTEE_DEST_DIR` (default -`$HOME/rt/riak_ee`), `RTCS_DEST_DIR` (default `$HOME/rt/riak_cs`) and -`RTSTANCHION_DEST_DIR` (default `$HOME/rt/stanchion`). - -The `build_type` option is used to differentiate between an -open-source (`oss`) build of RiakCS and the enterprise version (`ee`). -The default is `oss` and this option can be omitted when these tests -are used by open-source users. - -The `backend` option is used to indicate which Riak backend option -should be used. The valid options are `{multi_backend, bitcask}` and -`memory`. `{multi_backend, bitcask}` is the default option and -represents the default recommended backed for production use of -RiakCS. - -The `test_paths` option is a list of fully-qualified paths which -`riak_test` will use to find additional tests. Since the Riak CS tests -do not live inside the `riak_test` repository and escript, this should -point to the compiled tests in `riak_cs/riak_test/ebin`. - -The `flavor` option is used to vary environment setup. Some -`riak_test` modules use only S3 API and does not depend on details, -such as number of riak nodes, riak's backend, MDC or not, multibag or -not. By adding flavor setting to riak_test config, such generic test -cases can be utilized to verify Riak CS's behavior in various setups. -The scope of setup functions affected by flavors are `rtcs:setup/1` -and `rtcs:setup/2`. Other setup functions, for example -`rtcs:setup2x2` used by `repl_test`, does not change their behavior. -The valid option values are `basic` (default) and `{multibag, disjoint}`. -`{multibag, disjoint}` setup multibag environment with 3 bags, the master -bag with two riak nodes and two additional bags with one riak node each. - - -1. To build the `riak_test` files use the `compile-riak-test` Makefile - target or run `./rebar riak_test_compile`. - -1. The Riak client tests are now automated by the - `tests/external_client_tests.erl` test. There are several - prerequisites: - -* Your $PATH must have `erl` available. -* Your $PATH must have a version of Python available that also has - access to the Boto S3 libraries. -* Your $PATH must have Clojure's "lein" available. "lein" is the main - executable for the Leinigen tool. -* Your system must have libevent installed. If you see an error for a - missing 'event.h' file during test runs, this is because libevent is - not installed. -* Your system must have Ruby > 2.0 and PHP > 5.5 and PHP composer insalled. -* Your system must have Golang (go, $GOHOME, $GOROOT) correctly installed. - -1. Before running the Riak client tests, your -`~/.riak_test.config` file must contain an entry for `cs_src_root` in -the `src_paths` list, as shown above. The source in this directory -must be successfully compiled using the top level `make all` target. - -1. Before running the Riak client tests, you must first use the -commands `make clean-client-test` and then `make compile-client-test`. - -1. To execute a test, run the following from the `riak_test` repo: - - ```shell - ./riak_test -c rtcs_dev -t TEST_NAME - ``` diff --git a/riak_test/bin/rtdev-build-releases.sh b/riak_test/bin/rtdev-build-releases.sh deleted file mode 100755 index ebe34bcf7..000000000 --- a/riak_test/bin/rtdev-build-releases.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash - -# just bail out if things go south -set -e - -# You need to use this script once to build a set of devrels for prior -# releases of Riak (for mixed version / upgrade testing). You should -# create a directory and then run this script from within that directory. -# I have ~/test-releases that I created once, and then re-use for testing. -# -# See rtdev-setup-releases.sh as an example of setting up mixed version layout -# for testing. - -# Different versions of Riak were released using different Erlang versions, -# make sure to build with the appropriate version. - -# This is based on my usage of having multiple Erlang versions in different -# directories. If using kerl or whatever, modify to use kerl's activate logic. -# Or, alternatively, just substitute the paths to the kerl install paths as -# that should work too. - -R15B01=${R15B01:-$HOME/erlang/R15B01-64} -R16B02=${R16B02:-$HOME/erlang/R16B02-64} -: ${RTCS_DEST_DIR:="$HOME/rt/riak_cs"} - -checkbuild() -{ - ERLROOT=$1 - - if [ ! -d $ERLROOT ]; then - echo -n "$ERLROOT cannot be found, install kerl? [y|N]: " - read ans - if [[ $ans == n || $ans == N ]]; then - exit 1 - fi - fi -} - -kerl() -{ - RELEASE=$1 - BUILDNAME=$2 - - if [ ! -x kerl ]; then - curl -O https://raw.github.com/spawngrid/kerl/master/kerl; chmod a+x kerl - fi - - ./kerl build $RELEASE $BUILDNAME - ./kerl install $BUILDNAME $HOME/$BUILDNAME -} - -build() -{ - SRCDIR=$1 - ERLROOT=$2 - - if [ ! -d $ERLROOT ]; then - BUILDNAME=`basename $ERLROOT` - RELEASE=`echo $BUILDNAME | awk -F- '{ print $2 }'` - kerl $RELEASE $BUILDNAME - fi - - echo - echo "Building $SRCDIR" - cd $SRCDIR - - RUN="env PATH=$ERLROOT/bin:$ERLROOT/lib/erlang/bin:$PATH \ - C_INCLUDE_PATH=$ERLROOT/usr/include \ - LD_LIBRARY_PATH=$ERLROOT/usr/lib \ - DEVNODES=8" - echo $RUN - $RUN make -j 8 && $RUN make -j devrel - cd .. -} - -setup() -{ - SRCDIR=$1 - cd $SRCDIR - VERSION=$SRCDIR - echo " - Copying devrel to $RTCS_DEST_DIR/$VERSION " - mkdir -p $RTCS_DEST_DIR/$VERSION/ - cp -p -P -R dev $RTCS_DEST_DIR/$VERSION/ - ## echo " - Writing $RTCS_DEST_DIR/$VERSION/VERSION" - ## echo -n $VERSION > $RTCS_DEST_DIR/$VERSION/VERSION - cd $RTCS_DEST_DIR - echo " - Adding $VERSION to git state of $RTCS_DEST_DIR" - git add $VERSION - git commit -a -m "riak_test adding version $VERSION" ## > /dev/null 2>&1 -} - -download() -{ - URI=$1 - FILENAME=`echo $URI | awk -F/ '{ print $8 }'` - if [ ! -f $FILENAME ]; then - wget $URI - fi -} - -checkbuild $R15B01 -checkbuild $R16B02 - - -if env | grep -q 'RIAK_CS_EE_DEPS=' -then - echo "RIAK_CS_EE_DEPS is set to \"$RIAK_CS_EE_DEPS\"." - echo "This script if for OSS version." - echo "unset RIAK_CS_EE_DEPS or use script for ee build." - exit 1 -fi - -echo "Download and build OSS package..." -download http://s3.amazonaws.com/downloads.basho.com/riak-cs/1.5/1.5.4/riak-cs-1.5.4.tar.gz - -tar -xf riak-cs-1.5.4.tar.gz -build "riak-cs-1.5.4" $R15B01 diff --git a/riak_test/bin/rtdev-current.sh b/riak_test/bin/rtdev-current.sh deleted file mode 100755 index 3994e5914..000000000 --- a/riak_test/bin/rtdev-current.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# just bail out if things go south -set -e - -: ${RTCS_DEST_DIR:="$HOME/rt/riak_cs"} - -echo "Making $(pwd) the current release:" -cwd=$(pwd) -echo -n " - Determining version: " -if [ -f $cwd/dependency_manifest.git ]; then - VERSION=`cat $cwd/dependency_manifest.git | awk '/^-/ { print $NF }'` -else - VERSION="$(git describe --tags)-$(git branch | awk '/\*/ {print $2}')" -fi -echo $VERSION -cd $RTCS_DEST_DIR -echo " - Resetting existing $RTCS_DEST_DIR" -git reset HEAD --hard > /dev/null 2>&1 -git clean -fd > /dev/null 2>&1 -rm -rf $RTCS_DEST_DIR/current -mkdir $RTCS_DEST_DIR/current -cd $cwd -echo " - Copying devrel to $RTCS_DEST_DIR/current" -cp -p -P -R dev $RTCS_DEST_DIR/current -echo " - Writing $RTCS_DEST_DIR/current/VERSION" -echo -n $VERSION > $RTCS_DEST_DIR/current/VERSION -cd $RTCS_DEST_DIR -echo " - Reinitializing git state" -git add --all . -git commit -a -m "riak_test init" --amend > /dev/null diff --git a/riak_test/bin/rtdev-setup-releases.sh b/riak_test/bin/rtdev-setup-releases.sh deleted file mode 100755 index 74c82e780..000000000 --- a/riak_test/bin/rtdev-setup-releases.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# just bail out if things go south -set -e - -# Creates a mixed-version directory structure for running riak_test -# using rtdev-mixed.config settings. Should be run inside a directory -# that contains devrels for prior Riak CS releases. Easy way to create this -# is to use the rtdev-build-releases.sh script - -: ${RTCS_DEST_DIR:="$HOME/rt/riak_cs"} - -rm -rf $RTCS_DEST_DIR -mkdir $RTCS_DEST_DIR - -count=$(ls */dev 2> /dev/null | wc -l) -if [ "$count" -ne "0" ] -then - for rel in */dev; do - vsn=$(dirname "$rel") - echo " - Initializing $RTCS_DEST_DIR/$vsn" - mkdir -p "$RTCS_DEST_DIR/$vsn" - cp -p -P -R "$rel" "$RTCS_DEST_DIR/$vsn" - done -else - # This is useful when only testing with 'current' - # The repo still needs to be initialized for current - # and we don't want to bomb out if */dev doesn't exist - touch $RTCS_DEST_DIR/.current_init - echo "No devdirs found. Not copying any releases." -fi - -cd $RTCS_DEST_DIR -git init - -## Some versions of git and/or OS require these fields -git config user.name "Riak Test" -git config user.email "dev@basho.com" -git config --local core.excludesfile global_dot_gitignore_should_be_ignored - -git add . -git commit -a -m "riak_test init" > /dev/null -echo " - Successfully completed initial git commit of $RTCS_DEST_DIR" diff --git a/riak_test/intercepts/intercept.erl b/riak_test/intercepts/intercept.erl deleted file mode 100644 index 7d8bf119c..000000000 --- a/riak_test/intercepts/intercept.erl +++ /dev/null @@ -1,405 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- - --module(intercept). -%% Export explicit API but also send compile directive to export all -%% because some of these private functions are useful in their own -%% right. --export([add/3, add/4, clean/1]). --compile(export_all). - --type abstract_code() :: term(). --type form() :: term(). --type proplist(K, V) :: proplists:proplist(K, V). --type fun_name() :: atom(). --type fun_type() :: fun_name() | tuple(). --type target_fun() :: {fun_name(), arity()}. --type intercept_fun() :: fun_type(). --type mapping() :: proplist(target_fun(), intercept_fun()). --type form_mod() :: fun((form()) -> form()). --type code_mod() :: fun((form(), abstract_code()) -> abstract_code()). - -%% The "original" is the `Target' module with the suffix `_orig'. It -%% is where original code for the `Target' module resides after -%% intercepts are added. --define(ORIGINAL(Mod), list_to_atom(atom_to_list(Mod) ++ "_orig")). --define(FAKE_LINE_NO,1). - -%% @doc Add intercepts against the `Target' module. -%% -%% `Target' - The module on which to intercept calls. -%% E.g. `hashtree'. -%% -%% `Intercept' - The module containing intercept definitions. -%% E.g. `hashtree_intercepts' -%% -%% `Mapping' - The mapping from target functions to intercept -%% functions. -%% -%% E.g. `[{{update_perform,2}, sleep_update_perform}]' --spec add(module(), module(), mapping(), string()) -> ok. -add(Target, Intercept, Mapping, OutDir) -> - Original = ?ORIGINAL(Target), - TargetAC = get_abstract_code(Target), - - ProxyAC = make_proxy_abstract_code(Target, Intercept, Mapping, - Original, TargetAC), - OrigAC = make_orig_abstract_code(Target, Original, TargetAC), - - ok = compile_and_load(Original, OrigAC, OutDir), - ok = compile_and_load(Target, ProxyAC, OutDir). - --spec add(module(), module(), mapping()) -> ok. -add(Target, Intercept, Mapping) -> - add(Target, Intercept, Mapping, undefined). - -%% @doc Cleanup proxy and backuped original module --spec clean(module()) -> ok|{error, term()}. -clean(Target) -> - _ = code:purge(Target), - _ = code:purge(?ORIGINAL(Target)), - case code:load_file(Target) of - {module, Target} -> - ok; - {error, Reason} -> - {error, Reason} - end. - -%% @private -%% -%% @doc Compile the abstract code `AC' and load it into the code server. --spec compile_and_load(module(), abstract_code(), undefined | string()) -> ok. -compile_and_load(Module, AC, OutDir) -> - {ok, Module, Bin} = compile:forms(AC,[debug_info]), - ModStr = atom_to_list(Module), - _ = is_list(OutDir) andalso - file:write_file(filename:join(OutDir, ModStr ++ ".beam"), Bin), - {module, Module} = code:load_binary(Module, ModStr, Bin), - ok. - -%% @private --spec make_orig_abstract_code(module(), module(), abstract_code()) -> - abstract_code(). -make_orig_abstract_code(Target, OrigName, TargetAC) -> - export_all(move_all_funs(Target, change_module_name(OrigName, TargetAC))). - -%% @private -%% -%% @doc Make the abstract code for the proxy module. The proxy module -%% sits in place of the original module and decides whether to -%% forward to the `Intercept' or the `Original' depending on the -%% `Mapping'. --spec make_proxy_abstract_code(module(), module(), mapping(), - module(), abstract_code()) -> - abstract_code(). -make_proxy_abstract_code(Target, Intercept, Mapping, Original, TargetAC) -> - AC1 = forward_all(Original, TargetAC), - AC2 = export_all(change_module_name(Target, AC1)), - apply_intercepts(AC2, Intercept, Mapping). - - -%% @private -%% -%% @doc Apply intercepts to the abstract code `AC' based on `Mapping'. --spec apply_intercepts(abstract_code(), module(), mapping()) -> abstract_code(). -apply_intercepts(AC, Intercept, Mapping) -> - apply_to_funs(mapping_fun(Intercept, Mapping), AC). - -%% @private -%% -%% @doc Return a form modifier function that uses `Mapping' to -%% determine if a function should be modified to forward to the -%% `Intercept' module. --spec mapping_fun(module(), proplists:proplist()) -> form_mod(). -mapping_fun(Intercept, Mapping) -> - fun(Form) -> - Key = {fun_name(Form), fun_arity(Form)}, - case proplists:get_value(Key, Mapping, '$none') of - '$none' -> - Form; - InterceptFun -> - forward(Intercept, InterceptFun, Form) - end - end. - -%% @private -%% -%% @doc Modify the abstract code `AC' to forward all function calls to -%% `Module' and move the original definitions under -%% `_orig'. --spec move_all_funs(module(), abstract_code()) -> abstract_code(). -move_all_funs(Module, AC) -> - lists:reverse(lists:foldl(move_all_funs(Module), [], AC)). - -%% @private -%% -%% @doc Return a function which folds over the abstract code of a -%% module, represented by `Form'. Every function is modified to -%% forward to `ModuleName' and it's original definition is stored -%% under `_orig'. --spec move_all_funs(module()) -> code_mod(). -move_all_funs(ModuleName) -> - fun(Form, NewAC) -> - case is_fun(Form) of - false -> - [Form|NewAC]; - true -> - %% Move current function code under different name - Name = fun_name(Form), - OrigForm = setelement(3, Form, ?ORIGINAL(Name)), - - %% Modify original function to forward to `ModuleName' - FwdForm = forward(ModuleName, Name, Form), - [FwdForm,OrigForm|NewAC] - end - end. - -%% @private -%% -%% @doc Modify all function definitions in the abstract code `AC' to -%% forward to `Module:FunName_orig'. --spec forward_all(module(), abstract_code()) -> abstract_code(). -forward_all(Module, AC) -> - F = fun(Form) -> - forward(Module, ?ORIGINAL(fun_name(Form)), Form) - end, - apply_to_funs(F, AC). - -%% @private -%% -%% @doc Modify the function `Form' to forward to `Module:Fun'. --spec forward(module(), fun_type(), form()) -> form(). -forward(Module, Fun, Form) -> - Clause = hd(fun_clauses(Form)), - Args = clause_args(Clause), - NumArgs = length(Args), - GenArgs = [{var,?FAKE_LINE_NO,list_to_atom("Arg" ++ integer_to_list(I))} - || I <- lists:seq(1,NumArgs)], - Clause2 = clause_set_args(Clause, GenArgs), - Clause3 = clause_clear_guards(Clause2), - Body = [{call, 1, - case Fun of - Fun when is_atom(Fun) -> - {remote,1,{atom,1,Module},{atom,1,Fun}}; - %% If Fun is a tuple, it's a pair comprising a list of - %% local variables to capture and an anonymous function - %% that's already in the abstract format. The anonymous - %% function uses the local variables. - {FreeVars, AnonFun} -> - generate_fun_wrapper(FreeVars, AnonFun, NumArgs) - end, GenArgs}], - Clause4 = clause_set_body(Clause3, Body), - fun_set_clauses(Form, [Clause4]). - -change_module_name(NewName, AC) -> - lists:keyreplace(module, 3, AC, {attribute,1,module,NewName}). - -%% @private -%% -%% @doc Generate an anonymous function wrapper that sets up calls for an -%% anonymous function interceptor. -%% -%% This function returns the abstract code equivalent of the following -%% code. If you change this code, please update this comment. -%% -%% fun(__A0_, __A1_, ...) -> -%% __Bindings0_ = lists:foldl(fun({__Bn_,__Bv_},__Acc_) -> -%% erl_eval:add_binding(__Bn_,__Bv_,__Acc_) -%% end, -%% erl_eval:new_bindings(), -%% ), -%% __Bindings = lists:foldl(fun({{var,_,__Vn_},__V_},__Acc) -> -%% erl_eval:add_binding(__Vn_,__V_,__Acc_) -%% end, -%% __Bindings0_, -%% <__A0_ etc. args from generate_freevars>), -%% erl_eval:expr(, -%% __Bindings_, none, none, value). -%% -generate_fun_wrapper(FreeVars, AnonFun, NumArgs) -> - L = ?FAKE_LINE_NO, - Args = [{var,L,list_to_atom(lists:flatten(["__A",Var+$0],"_"))} || - Var <- lists:seq(1, NumArgs)], - {'fun',L, - {clauses, - [{clause,L,Args,[], - [{match,L+1, - {var,L+1,'__Bindings0_'}, - {call,L+1, - {remote,L+1,{atom,L+1,lists},{atom,L+1,foldl}}, - [{'fun',L+1, - {clauses, - [{clause,L+1, - [{tuple,L+1,[{var,L+1,'__Bn_'},{var,L+1,'__Bv_'}]}, - {var,L+1,'__Acc_'}], - [], - [{call,L+2, - {remote,L+2, - {atom,L+2,erl_eval}, - {atom,L+2,add_binding}}, - [{var,L+2,'__Bn_'},{var,L+2,'__Bv_'},{var,L+2,'__Acc_'}]}] - }]}}, - {call,L+3, - {remote,L+3,{atom,L+3,erl_eval},{atom,L+3,new_bindings}},[]}, - generate_freevars(FreeVars,L+3)]}}, - {match,L+4, - {var,L+4,'__Bindings_'}, - {call,L+4, - {remote,L+4,{atom,L+4,lists},{atom,L+4,foldl}}, - [{'fun',L+4, - {clauses, - [{clause,L+4, - [{tuple,L+4,[{tuple,L+4,[{atom,L+4,var},{var,L+4,'_'}, - {var,L+4,'__Vn_'}]},{var,L+4,'__V_'}]}, - {var,L+4,'__Acc_'}], - [], - [{call,L+5, - {remote,L+5, - {atom,L+5,erl_eval}, - {atom,L+5,add_binding}}, - [{var,L+5,'__Vn_'},{var,L+5,'__V_'},{var,L+5,'__Acc_'}]}] - }]}}, - {var,L+6,'__Bindings0_'}, - lists:foldl(fun(V,Acc) -> - AV = erl_parse:abstract(V), - {cons,L+6,{tuple,L+6,[AV,V]},Acc} - end,{nil,L+6},Args)]}}, - {call,L+7, - {remote,L+7, - {atom,L+7,erl_eval}, - {atom,L+7,expr}}, - [erl_parse:abstract({call,L+7,AnonFun, - [{var,L+7,V} || {var,_,V} <- Args]},L+7), - {var,L+7,'__Bindings_'}, - {atom,L+7,none}, - {atom,L+7,none}, - {atom,L+7,value}]}]}]}}. - -%% @private -%% -%% @doc Convert generate_fun_wrapper freevars to abstract code -generate_freevars([], L) -> - {nil,L}; -generate_freevars([FreeVar|FreeVars], L) -> - {cons,L, - generate_freevar(FreeVar,L), - generate_freevars(FreeVars,L)}. - -%% @private -%% -%% @doc Convert one freevar to abstract code -%% -%% This returns an abstract format tuple representing a freevar as -%% {VarName, VarValue}. For function values we check their env for their -%% own freevars, but if no env is available, we raise an error. Pids, -%% ports, and references have no abstract format, so they are first -%% converted to binaries and the abstract format of the binary is used -%% instead. Their abstract format values generated here convert them back -%% from binaries to terms when accessed. -generate_freevar({Name,Var},L) when is_function(Var) -> - {env, Env} = erlang:fun_info(Var, env), - case Env of - [] -> - error({badarg, Var}); - [FreeVars,_,_,Clauses] -> - {arity, Arity} = erlang:fun_info(Var, arity), - AnonFun = {'fun',L,{clauses,Clauses}}, - {tuple,L, - [{atom,L,Name}, - generate_fun_wrapper(FreeVars, AnonFun, Arity)]} - end; -generate_freevar({Name,Var}, L) - when is_pid(Var); is_port(Var); is_reference(Var) -> - NVar = term_to_binary(Var), - {tuple,L, - [{atom,L,Name}, - {call,L, - {remote,L,{atom,L,erlang},{atom,L,binary_to_term}}, - [erl_parse:abstract(NVar)]}]}; -generate_freevar(NameVar, L) -> - erl_parse:abstract(NameVar,L). - -%% @private -%% -%% @doc Add the `export_all' compile directive to the abstract code `AC'. -export_all(AC) -> - [A,B|Rest] = AC, - [A,B,{attribute,2,compile,export_all}|Rest]. - -%% @private -%% -%% @doc Apply the form modify `F' to all forms in `AC' that are -%% function definitions. --spec apply_to_funs(form_mod(), abstract_code()) -> abstract_code(). -apply_to_funs(F, AC) -> - F2 = apply_if_fun_def(F), - lists:map(F2, AC). - -%% @private -%% -%% @doc Get the abstract code for `Module'. This function assumes -%% code is compiled with `debug_info'. --spec get_abstract_code(module()) -> abstract_code(). -get_abstract_code(Module) -> - {_, Bin, _} = code:get_object_code(Module), - {ok,{_,[{abstract_code,{_,AC}}]}} = beam_lib:chunks(Bin, [abstract_code]), - AC. - -%% @private -apply_if_fun_def(Fun) -> - fun(Form) when element(1, Form) == function -> Fun(Form); - (Form) -> Form - end. - -%% @private -is_fun(Form) -> - element(1, Form) == function. - -%% @private -clause_args(Form) -> - element(3, Form). - -%% @private -clause_set_args(Form, Args) -> - setelement(3, Form, Args). - -%% @private -clause_clear_guards(Form) -> - setelement(4, Form, []). - -%% @private -clause_set_body(Form, Body) -> - setelement(5, Form, Body). - -%% @private -fun_arity(Form) -> - element(4, Form). - -%% @private -fun_clauses(Form) -> - element(5, Form). - -%% @private -fun_set_clauses(Form, Clauses) -> - setelement(5, Form, Clauses). - -%% @private -fun_name(Form) -> - element(3, Form). diff --git a/riak_test/intercepts/intercept.hrl b/riak_test/intercepts/intercept.hrl deleted file mode 100644 index d292e0ba1..000000000 --- a/riak_test/intercepts/intercept.hrl +++ /dev/null @@ -1,23 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- - --define(I_TAG(S), "INTERCEPT: " ++ S). --define(I_INFO(Msg), error_logger:info_msg(?I_TAG(Msg))). --define(I_INFO(Msg, Args), error_logger:info_msg(?I_TAG(Msg), Args)). diff --git a/riak_test/intercepts/riak_cs_block_server_intercepts.erl b/riak_test/intercepts/riak_cs_block_server_intercepts.erl deleted file mode 100644 index f072d2223..000000000 --- a/riak_test/intercepts/riak_cs_block_server_intercepts.erl +++ /dev/null @@ -1,37 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- - --module(riak_cs_block_server_intercepts). --compile(export_all). --include("intercept.hrl"). --define(M, riak_cs_block_server_orig). - -get_block_local_timeout(_RcPid, _FullBucket, _FullKey, _GetOptions, _Timeout, _StatsKey) -> - {error, timeout}. - -get_block_local_insufficient_vnode_at_nval1(RcPid, FullBucket, FullKey, GetOptions, Timeout, StatsKey) -> - case proplists:get_value(n_val, GetOptions) of - 1 -> - ?I_INFO("riak_cs_block_server:get_block_local/6 returns insufficient_vnodes"), - {error, <<"{insufficient_vnodes,0,need,1}">>}; - N -> - ?I_INFO("riak_cs_block_server:get_block_local/6 forwards original code with n_val=~p", [N]), - ?M:get_block_local_orig(RcPid, FullBucket, FullKey, GetOptions, Timeout, StatsKey) - end. diff --git a/riak_test/intercepts/riak_cs_riak_client_intercepts.erl b/riak_test/intercepts/riak_cs_riak_client_intercepts.erl deleted file mode 100644 index be843e195..000000000 --- a/riak_test/intercepts/riak_cs_riak_client_intercepts.erl +++ /dev/null @@ -1,26 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- - --module(riak_cs_riak_client_intercepts). --compile(export_all). --define(M, riak_cs_riak_client_orig). - -get_user_timeout(_MasterPbc, _Key) -> - {error, timeout}. diff --git a/riak_test/intercepts/riakc_pb_socket_intercepts.erl b/riak_test/intercepts/riakc_pb_socket_intercepts.erl deleted file mode 100644 index 38674a061..000000000 --- a/riak_test/intercepts/riakc_pb_socket_intercepts.erl +++ /dev/null @@ -1,32 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%%------------------------------------------------------------------- - --module(riakc_pb_socket_intercepts). --compile(export_all). --define(M, riakc_pb_socket_orig). - -get_timeout(_Pid, _Bucket, _Key, _Options, _Timeout) -> - {error, timeout}. - -put_timeout(_Pid, _Obj, _Options, _Timeout) -> - {error, timeout}. - -get_overload(_Pid, _Bucket, _Key, _Options, _Timeout) -> - {error, <<"overload">>}. diff --git a/riak_test/src/cs_suites.erl b/riak_test/src/cs_suites.erl deleted file mode 100644 index a3ed6bea6..000000000 --- a/riak_test/src/cs_suites.erl +++ /dev/null @@ -1,577 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% CS test suites - -%% TODO: Can this be rt/rtcs-independent? - -%% A suite is composed of several circles and single cleanup. -%% -%% Single circle run includes: -%% - creation of 5 buckets and 5 objects for each -%% - deletion of one bucket including objects in it -%% - PUT/GET/List/DELETE operations for pre-created buckets -%% - Access stats flush and storage stat calculation -%% - GC batch completion -%% -%% Cleanup includes: -%% - deletion of all objects and buckets ever created -%% - GC batch completion -%% - bitcask merge & delete -%% - confirmation that all data files become small enough - --module(cs_suites). - --export([new/1, fold_with_state/2, run/2, cleanup/1]). --export([nodes_of/2, - set_node1_version/2, - admin_credential/1]). --export([ops/0, reduced_ops/0]). - --export_type([state/0, op/0, tag/0]). --type op() :: atom(). --type tag() :: string(). - --include_lib("kernel/include/file.hrl"). --include_lib("eunit/include/eunit.hrl"). --include_lib("erlcloud/include/erlcloud_aws.hrl"). - --record(state, - { - begin_at :: string(), - bucket_count = 5 :: pos_integer(), - key_count = 5 :: pos_integer(), - ops = ops() :: [atom()], - node1_cs_vsn = current :: previous | current, - riak_vsn = current :: current, - riak_nodes :: list(), - cs_nodes :: list(), - stanchion_nodes :: list(), - admin_config :: term(), - prefix = "t-" :: string(), - circles = [] :: [circle()] - }). --opaque state() :: #state{}. - --record(bucket, - {name :: string(), - count = 0 :: non_neg_integer()}). --type bucket() :: #bucket{}. - --record(circle, - {tag :: string(), - user_config :: term(), - buckets = [] :: [bucket()]}). --type circle() :: #circle{}. - --define(GC_LEEWAY, 1). - --spec ops() -> [op()]. -ops() -> [put_buckets, - put_objects, - put_objects_old, - get_objects, - get_objects_old, - list_objects, - list_objects_old, - delete_bucket, - delete_bucket_old, - stats_access, - stats_storage, - gc, - end_of_op]. - --spec reduced_ops() -> [op()]. -reduced_ops() -> [put_buckets, - put_objects, - end_of_op]. - --spec cleanup_ops() -> [op()]. -cleanup_ops() -> - [delete_all, - gc, - merge, - end_of_cleanup_op]. - -%% Create configuration state for subsequennt runs. --spec new(term()) -> {ok, state()}. -new({AdminConfig, {RiakNodes, CSNodes, StanchionNode}}) -> - new({AdminConfig, {RiakNodes, CSNodes, StanchionNode}}, ops()). - --spec new(term(), [op()]) -> {ok, state()}. -new({AdminConfig, {RiakNodes, CSNodes, StanchionNode}}, Ops) -> - rt:setup_log_capture(hd(CSNodes)), - rtcs_exec:gc(1, "set-interval infinity"), - Begin = rtcs:datetime(), - %% FIXME: workaround for riak_cs#766 - timer:sleep(timer:seconds(1)), - {ok, #state{begin_at = Begin, - ops = Ops, - riak_nodes = RiakNodes, - cs_nodes = CSNodes, - stanchion_nodes = [StanchionNode], - admin_config = AdminConfig}}. - -%% Utility functions to avoid many `StateN''s by appplying MFA lists sequentially --spec fold_with_state(state(), [{module(), atom(), [term()]}]) -> {ok, state()}. -fold_with_state(State, []) -> - {ok, State}; -fold_with_state(State, [{M, F, A} | Rest]) -> - {ok, NewState} = apply(M, F, A ++ [State]), - fold_with_state(NewState, Rest). - --spec run(tag(), state()) -> {ok, state()}. -run(Tag, #state{ops=Ops} = State) -> - run(Tag, Ops, State). - --spec cleanup(state()) -> {ok, state()}. -cleanup(State) -> - Tag = "cleanup", - run(Tag, cleanup_ops(), State). - -run(Tag, Ops, State) -> - lager:info("[~s] BEGIN", [Tag]), - Circle = init_circle(Tag, State), - {USec, {ok, NewState}} = - timer:tc(fun() -> apply_operations(Circle, State, Ops) end), - lager:info("[~s] END: ~B [msec]", [Tag, USec div 1000]), - {ok, NewState}. - -set_node1_version(Vsn, State) when Vsn =:= previous orelse Vsn =:= current -> - {ok, State#state{node1_cs_vsn=Vsn}}. - -admin_credential(#state{admin_config=AdminConfig}) -> - {AdminConfig#aws_config.access_key_id, - AdminConfig#aws_config.secret_access_key}. - --spec nodes_of(riak | stanchion | riak, state()) -> [node()]. -nodes_of(riak, State) -> State#state.riak_nodes; -nodes_of(stanchion, State) -> State#state.stanchion_nodes; -nodes_of(cs, State) -> State#state.cs_nodes. - -init_circle(Tag, #state{admin_config=AdminConfig, riak_nodes = [RiakNode|_]} = _State) -> - Port = rtcs_config:cs_port(RiakNode), - Name = concat("user-", Tag), - Email = concat(Name, "@example.com"), - {UserConfig, _Id} = - rtcs_admin:create_user(Port, AdminConfig, Email, Name), - #circle{tag = Tag, user_config = UserConfig}. - --spec apply_operations(circle(), state(), [op()]) -> {ok, state()}. -apply_operations(_Circle, State, []) -> - {ok, State}; -apply_operations(#circle{tag=Tag} = Circle, State, [Op | Rest]) -> - lager:info("[~s] Applying operation ~w ...", [Tag, Op]), - {USec, {ok, NewCircle, NewState}} = - timer:tc(fun() -> apply_operation(Op, Circle, State) end), - lager:info("[~s] Finished operation ~w in ~B [msec]", [Tag, Op, USec div 1000]), - apply_operations(NewCircle, NewState, Rest). - --spec apply_operation(op(), circle(), state()) -> {ok, circle(), state()}. -apply_operation(put_buckets, #circle{tag=Tag, user_config=UserConfig} = Circle, - #state{bucket_count = Count, prefix=Prefix} = State) -> - Buckets = [#bucket{name = concat(Prefix, Tag, to_str(Suffix))} - || Suffix <- lists:seq(1, Count)], - [?assertEqual(ok, erlcloud_s3:create_bucket(B#bucket.name, UserConfig)) || - B <- Buckets], - {ok, Circle#circle{buckets = Buckets}, State}; -apply_operation(put_objects, Circle, - #state{key_count=KeyCount} = State) -> - NewCircle = put_objects_to_every_bucket(KeyCount, Circle), - {ok, NewCircle, State}; -apply_operation(put_objects_old, CurrentCircle, - #state{key_count=KeyCount, circles=Circles} = State) -> - NewCircles = [put_objects_to_every_bucket(KeyCount, Circle) || Circle <- Circles], - {ok, CurrentCircle, State#state{circles=NewCircles}}; -apply_operation(get_objects, Circle, State) -> - NewCircle = get_objects_from_every_bucket(Circle), - {ok, NewCircle, State}; -apply_operation(get_objects_old, CurrentCircle, - #state{circles=Circles} = State) -> - NewCircles = [get_objects_from_every_bucket(Circle) || Circle <- Circles], - {ok, CurrentCircle, State#state{circles=NewCircles}}; -apply_operation(list_objects, Circle, State) -> - NewCircle = list_objects_from_every_bucket(Circle), - {ok, NewCircle, State}; -apply_operation(list_objects_old, CurrentCircle, #state{circles=Circles} = State) -> - NewCircles = [list_objects_from_every_bucket(Circle) || Circle <- Circles], - {ok, CurrentCircle, State#state{circles=NewCircles}}; -apply_operation(delete_bucket, Circle, State) -> - NewCircle = delete_first_bucket(Circle), - {ok, NewCircle, State}; -apply_operation(delete_bucket_old, CurrentCircle, #state{circles=Circles} = State) -> - NewCircles = [delete_first_bucket(Circle) || Circle <- Circles], - {ok, CurrentCircle, State#state{circles=NewCircles}}; -apply_operation(stats_access, Circle, State) -> - Res = rtcs_exec:flush_access(1), - lager:info("riak-cs-access flush result: ~s", [Res]), - ExpectRegexp = "All access logs were flushed.\n$", - ?assertMatch({match, _}, re:run(Res, ExpectRegexp)), - %% TODO - %% - Get access stats and assert them with real access generated so far - {ok, Circle, State}; -apply_operation(stats_storage, CurrentCircle, - #state{admin_config=AdminConfig, begin_at=Begin, - cs_nodes=[CSNode|_], circles=Circles} = State) -> - Res = rtcs_exec:calculate_storage(1), - lager:info("riak-cs-storage batch result: ~s", [Res]), - ExpectRegexp = "Batch storage calculation started.\n$", - ?assertMatch({match, _}, re:run(Res, ExpectRegexp)), - true = rt:expect_in_log(CSNode, "Finished storage calculation"), - %% FIXME: workaround for riak_cs#766 - timer:sleep(timer:seconds(2)), - End = rtcs:datetime(), - [get_storage_stats(AdminConfig, Begin, End, Circle) || Circle <- [CurrentCircle|Circles]], - {ok, CurrentCircle, State}; -apply_operation(gc, Circle, #state{cs_nodes=[CSNode|_]} = State) -> - timer:sleep(timer:seconds(?GC_LEEWAY + 1)), - rtcs_exec:gc(1, "batch 1"), - ok = rt:wait_until( - CSNode, - fun(_N) -> - Res = rtcs_exec:gc(1, "status"), - ExpectSubstr = "There is no garbage collection in progress", - case string:str(Res, ExpectSubstr) of - 0 -> - lager:debug("riak-cs-gc status: ~s", [Res]), - false; - _ -> - lager:debug("GC completed"), - true - end - end), - %% TODO: Calculate manif_count and block_count and assert them specifically - %% true = rt:expect_in_log(CSNode, - %% "Finished garbage collection: \\d+ seconds, " - %% "\\d+ batch_count, \\d+ batch_skips, " - %% "\\d+ manif_count, \\d+ block_count"), - {ok, Circle, State}; -apply_operation(end_of_op, Circle, - #state{node1_cs_vsn=Vsn, circles=SoFar} = State) -> - rtcs:assert_error_log_empty(Vsn, 1), - {ok, Circle, State#state{circles=[Circle|SoFar]}}; - -apply_operation(delete_all, CurrentCircle, #state{circles=Circles} = State) -> - NewCircles = [delete_all_buckets(Circle) || Circle <- Circles], - {ok, CurrentCircle, State#state{circles=NewCircles}}; -apply_operation(merge, CurrentCircle, State) -> - merge_all_bitcask(State), - {ok, CurrentCircle, State}; -apply_operation(end_of_cleanup_op, CurrentCircle, State) -> - {ok, CurrentCircle, State}. - --spec put_objects_to_every_bucket(non_neg_integer(), circle()) -> circle(). -put_objects_to_every_bucket(KeyCount, - #circle{user_config=UserConfig, - buckets=Buckets} = Circle) -> - NewBuckets = [put_objects(KeyCount, UserConfig, B) || B <- Buckets], - Circle#circle{buckets=NewBuckets}. - --spec put_objects(non_neg_integer(), #aws_config{}, bucket()) -> bucket(). -put_objects(KeyCount, UserConfig, #bucket{name=B, count=Before} = Bucket) -> - [case bk_to_body(B, K) of - Body when is_binary(Body) -> - erlcloud_s3:put_object(B, to_str(K), Body, UserConfig); - Parts when is_list(Parts) -> - multipart_upload(B, to_str(K), Parts, UserConfig); - {copy, {SrcB, SrcK}} -> - ?assertEqual([{copy_source_version_id, "false"}, - {version_id, "null"}], - erlcloud_s3:copy_object(B, to_str(K), SrcB, to_str(SrcK), UserConfig)) - end || - K <- lists:seq(Before + 1, Before + KeyCount) ], - Bucket#bucket{count = Before + KeyCount}. - -multipart_upload(Bucket, Key, Parts, Config) -> - InitRes = erlcloud_s3_multipart:initiate_upload( - Bucket, Key, "text/plain", [], Config), - UploadId = erlcloud_xml:get_text( - "/InitiateMultipartUploadResult/UploadId", InitRes), - upload_parts(Bucket, Key, UploadId, Config, 1, Parts, []). - -upload_parts(Bucket, Key, UploadId, UserConfig, _PartCount, [], PartEtags) -> - ?assertEqual(ok, erlcloud_s3_multipart:complete_upload( - Bucket, Key, UploadId, lists:reverse(PartEtags), UserConfig)), - ok; -upload_parts(Bucket, Key, UploadId, UserConfig, PartCount, [Part | Parts], PartEtags) -> - {RespHeaders, _UploadRes} = erlcloud_s3_multipart:upload_part( - Bucket, Key, UploadId, PartCount, Part, UserConfig), - PartEtag = proplists:get_value("ETag", RespHeaders), - upload_parts(Bucket, Key, UploadId, UserConfig, PartCount + 1, - Parts, [{PartCount, PartEtag} | PartEtags]). - - --spec get_objects_from_every_bucket(circle()) -> circle(). -get_objects_from_every_bucket(#circle{user_config=UserConfig, - buckets=Buckets} = Circle) -> - NewBuckets = [get_objects(UserConfig, Bucket)|| Bucket <- Buckets], - Circle#circle{buckets=NewBuckets}. - -%% TODO: More variants, e.g. range, conditional --spec get_objects(#aws_config{}, bucket()) -> bucket(). -get_objects(UserConfig, #bucket{name=B, count=KeyCount} = Bucket) -> - [begin - GetResponse = erlcloud_s3:get_object(B, to_str(K), UserConfig), - Expected = case bk_to_body(B, K) of - Body when is_binary(Body) -> Body; - Parts when is_list(Parts) -> to_bin(Parts); - {copy, {SrcB, SrcK}} -> bk_to_body(SrcB, SrcK) - end, - case proplists:get_value(content, GetResponse) of - Expected -> ok; - Other -> - lager:error("Unexpected contents for bucket=~s, key=~s", [B, K]), - lager:error("First 100 bytes of expected content~n~s", - [first_n_bytes(Expected, 100)]), - lager:error("First 100 bytes of actual content~n~s", - [first_n_bytes(Other, 100)]), - error({content_unmatched, B, K}) - end - end || - K <- lists:seq(1, KeyCount)], - Bucket. - -%% TODO: repeating GET Bucket calls by small (<5) `max-keys' param -list_objects_from_every_bucket(#circle{user_config=UserConfig, - buckets=Buckets} = Circle) -> - Opts = [], - [begin - KeyCount = Bucket#bucket.count, - Response = erlcloud_s3:list_objects(Bucket#bucket.name, Opts, UserConfig), - Result = proplists:get_value(contents, Response), - ?assertEqual(KeyCount, length(Result)) - end || - Bucket <- Buckets], - Circle. - -%% Delete objects in the first bucket if exists, leave untouched the rest. --spec delete_first_bucket(circle()) -> circle(). -delete_first_bucket(#circle{buckets=[]} = Circle) -> - Circle; -delete_first_bucket(#circle{user_config=UserConfig, - buckets=[Bucket|UntouchedBucket]} = Circle) -> - delete_bucket(UserConfig, Bucket), - Circle#circle{buckets=UntouchedBucket}. - -delete_all_buckets(#circle{buckets=[]} = Circle) -> - Circle; -delete_all_buckets(#circle{user_config=UserConfig, - buckets=[Bucket|RestBucket]} = Circle) -> - delete_bucket(UserConfig, Bucket), - delete_all_buckets(Circle#circle{buckets=RestBucket}). - -delete_bucket(UserConfig, Bucket) -> - B = Bucket#bucket.name, - KeyCount = Bucket#bucket.count, - [erlcloud_s3:delete_object(B, to_str(K), UserConfig) || - K <- lists:seq(1, KeyCount)], - ListResponse = erlcloud_s3:list_objects(B, [], UserConfig), - Contents = proplists:get_value(contents, ListResponse), - ?assertEqual(0, length(Contents)), - ?assertEqual(ok, erlcloud_s3:delete_bucket(B, UserConfig)). - -get_storage_stats(AdminConfig, Begin, End, - #circle{user_config=UserConfig, buckets=Buckets} = Circle) -> - lager:debug("storage stats for user ~s , ~s/~s", - [UserConfig#aws_config.access_key_id, Begin, End]), - Expected = lists:sort( - [{B, {Count, bucket_bytes(B, Count)}} || - #bucket{name=B, count=Count} <- Buckets]), - Actual = rtcs_admin:storage_stats_json_request(AdminConfig, UserConfig, - Begin, End), - ?assertEqual(Expected, Actual), - Circle. - -first_n_bytes(Binary, ByteSize) -> - binary:part(Binary, 0, math:max(byte_size(Binary), ByteSize)). - --spec bk_to_body(string(), integer()) -> binary() | [binary()]. -bk_to_body(B, K) -> - case os:getenv("CS_RT_DEBUG") of - "true" -> - %% Trick to make duration/stacktrace smaller for debugging (/ω・＼) - bk_to_body_debug(B, K); - _ -> - %% This branch should be used normally - bk_to_body_actual(B, K) - end. - -%% Generate object body by modulo 5 of `K::integer()': -%% 1 : multiple blocks (3 MB) -%% 2 : Multipart Upload (2 parts * 5 MB/part) -%% 3 : Put Copy of `1' (3 MB) -%% 4, 0 : small objects (~ 10 KB) --spec bk_to_body_actual(string(), integer()) -> - binary() | [binary()] | {copy, {string(), integer()}}. -bk_to_body_actual(_B, K) when (K rem 5) =:= 1 -> binary:copy(<<"A">>, mb(3)); -bk_to_body_actual(_B, K) when (K rem 5) =:= 2 -> lists:duplicate(2, binary:copy(<<"A">>, mb(5))); -bk_to_body_actual(B, K) when (K rem 5) =:= 3 -> {copy, {B, 1}}; -bk_to_body_actual(B, K) -> binary:copy(lead(B, K), 1024). - --spec bk_to_body_debug(string(), integer()) -> binary() | [binary()]. -bk_to_body_debug(B, K) -> - lead(B, K). - -lead(B, K) -> - to_bin([B, $:, to_str(K), $\n]). - -%% Calculate total byte size for the bucket --spec bucket_bytes(string(), non_neg_integer()) -> non_neg_integer(). -bucket_bytes(B, Count) -> - lists:sum([obj_bytes(B, K) || K <- lists:seq(1, Count)]). - -%% Returns byte size of the binary that will be returned by -%% `bk_to_body(B, K)'. --spec obj_bytes(string(), integer()) -> non_neg_integer(). -obj_bytes(B, K) -> - case os:getenv("CS_RT_DEBUG") of - "true" -> - %% Use OS env var to restrict object body to small binary - %% for debugging. no multi-MB obj, no Multipart, no PUT Copy. - byte_size(bk_to_body_debug(B, K)); - _ -> - obj_bytes_actual(B, K) - end. - -obj_bytes_actual(_B, K) when (K rem 5) =:= 1 -> mb(3); -obj_bytes_actual(_B, K) when (K rem 5) =:= 2 -> 2 * mb(5); -obj_bytes_actual(B, K) when (K rem 5) =:= 3 -> obj_bytes_actual(B, 1); -obj_bytes_actual(B, K) -> kb(byte_size(lead(B, K))). - -kb(KB) -> KB * 1024. - -mb(MB) -> MB * 1024 * 1024. - -merge_all_bitcask(#state{riak_nodes=Nodes, riak_vsn=Vsn} = _State) -> - wait_until_merge_worker_idle(Nodes), - [trigger_bitcask_merge(N, Vsn) || N <- Nodes], - [ok = rt:wait_until(N, - fun(Node) -> - check_data_files_are_small_enough(Node, Vsn) - end) || N <- Nodes], - ok. - -wait_until_merge_worker_idle(Nodes) -> - [ok = rt:wait_until( - N, - fun(Node) -> - Status = rpc:call(Node, bitcask_merge_worker, status, []), - lager:debug("Wait util bitcask_merge_worker to finish on ~p:" - " status=~p~n", [Node, Status]), - Status =:= {0, undefined} - end) || N <- Nodes], - ok. - -trigger_bitcask_merge(Node, Vsn) -> - lager:debug("Trigger bitcask merger for node ~s (version: ~s)~n", [Node, Vsn]), - VnodeNames = bitcask_vnode_names(Node, Vsn), - [begin - VnodeDir = filename:join(bitcask_data_root(), VnodeName), - DataFiles = bitcask_data_files(Node, Vsn, VnodeName, rel), - ok = rpc:call(Node, bitcask_merge_worker, merge, - [VnodeDir, [], {DataFiles, []}]) - end || - VnodeName <- VnodeNames], - ok. - -bitcask_data_root() -> "./data/bitcask". - -%% Returns vnode dir name, -%% For example: "548063113999088594326381812268606132370974703616" --spec bitcask_vnode_names(atom(), previous | current) -> [string()]. -bitcask_vnode_names(Node, Vsn) -> - Prefix = rtcs_config:devpath(riak, Vsn), - BitcaskAbsRoot = rtcs_config:riak_bitcaskroot(Prefix, rtcs_dev:node_id(Node)), - {ok, VnodeDirNames} = file:list_dir(BitcaskAbsRoot), - VnodeDirNames. - -%% Data file names should start bitcask's `data_root' for `bitcask_merge_worker'. -%% This function returns a list proper for the aim, e.g. -%% ["./data/bitcask/548063113999088594326381812268606132370974703616/5.bitcask.data", -%% "./data/bitcask/548063113999088594326381812268606132370974703616/6.bitcask.data"] --spec bitcask_data_files(node(), previous | current, string(), abs | rel) -> - [string()]. -bitcask_data_files(Node, Vsn, VnodeName, AbsOrRel) -> - Prefix = rtcs_config:devpath(riak, Vsn), - BitcaskAbsRoot = rtcs_config:riak_bitcaskroot(Prefix, rtcs_dev:node_id(Node)), - VnodeAbsPath = filename:join(BitcaskAbsRoot, VnodeName), - {ok, Fs0} = file:list_dir(VnodeAbsPath), - [case AbsOrRel of - rel -> filename:join([bitcask_data_root(), VnodeName, F]); - abs -> filename:join([BitcaskAbsRoot, VnodeName, F]) - end || - F <- Fs0, filename:extension(F) =:= ".data"]. - -%% Assert bitcask data is "small" -%% 1) The number of *.data files should be =< 2 -%% 2) Each *.data file size should be < 32 KiB -check_data_files_are_small_enough(Node, Vsn) -> - VnodeNames = bitcask_vnode_names(Node, Vsn), - check_data_files_are_small_enough(Node, Vsn, VnodeNames). - -check_data_files_are_small_enough(_Node, _Vsn, []) -> - true; -check_data_files_are_small_enough(Node, Vsn, [VnodeName|Rest]) -> - DataFiles = bitcask_data_files(Node, Vsn, VnodeName, abs), - FileSizes = [case file:read_file_info(F) of - {ok, #file_info{size=S}} -> {F, S}; - {error, enoent} -> {F, 0} % already deleted - end || F <- DataFiles], - lager:debug("FileSizes (~p): ~p~n", [{Node, VnodeName}, FileSizes]), - TotalSize = lists:sum([S || {_, S} <- FileSizes]), - case {length(FileSizes) =< 2, TotalSize < 32*1024} of - {true, true} -> - lager:info("bitcask data file check OK for ~p ~p", - [Node, VnodeName]), - check_data_files_are_small_enough(Node, Vsn, Rest); - {false, _} -> - lager:info("bitcask data file check failed, count(files)=~p for ~p ~p", - [length(FileSizes), Node, VnodeName]), - false; - {_, false} -> - lager:info("bitcask data file check failed, sum(file size)=~p for ~p ~p", - [TotalSize, Node, VnodeName]), - false - end. - -%% Misc utilities - -concat(IoList1, IoList2) -> - concat([IoList1, IoList2]). - -concat(IoList1, IoList2, IoList3) -> - concat([IoList1, IoList2, IoList3]). - -concat(IoList) -> - lists:flatten(IoList). - -to_str(Int) when is_integer(Int) -> - integer_to_list(Int); -to_str(String) when is_list(String) -> - String; -to_str(Bin) when is_binary(Bin) -> - binary_to_list(Bin). - -to_bin(Int) when is_integer(Int) -> - to_bin(integer_to_list(Int)); -to_bin(String) when is_list(String) -> - iolist_to_binary(String); -to_bin(Bin) when is_binary(Bin) -> - Bin. diff --git a/riak_test/src/list_objects_test_helper.erl b/riak_test/src/list_objects_test_helper.erl deleted file mode 100644 index c537c737f..000000000 --- a/riak_test/src/list_objects_test_helper.erl +++ /dev/null @@ -1,174 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(list_objects_test_helper). - --compile(export_all). --include_lib("eunit/include/eunit.hrl"). - --define(TEST_BUCKET, "riak-test-bucket"). - -test(UserConfig) -> - lager:info("User is valid on the cluster, and has no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(UserConfig)), - - ?assertError({aws_error, {http_error, 404, _, _}}, erlcloud_s3:list_objects(?TEST_BUCKET, UserConfig)), - - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - - ?assertMatch([{buckets, [[{name, ?TEST_BUCKET}, _]]}], - erlcloud_s3:list_buckets(UserConfig)), - - %% Put 10 objects in the bucket - Count1 = 10, - load_objects(?TEST_BUCKET, Count1, UserConfig), - - %% Successively list the buckets, verify the output, and delete an - %% object from the bucket until the bucket is empty again. - delete_and_verify_objects(?TEST_BUCKET, Count1, UserConfig), - - %% Put 200 objects in the bucket - Count2 = 200, - load_objects(?TEST_BUCKET, Count2, UserConfig), - - %% Successively list the buckets, verify the output, and delete an - %% object from the bucket until the bucket is empty again. - delete_and_verify_objects(?TEST_BUCKET, Count2, UserConfig), - - %% Put 30 objects in the bucket - Count3 = 30, - load_objects(?TEST_BUCKET, Count3, UserConfig), - - %% Use `max-keys' to restrict object list to 20 results - Options1 = [{max_keys, 20}], - ObjList1 = erlcloud_s3:list_objects(?TEST_BUCKET, Options1, UserConfig), - verify_object_list(ObjList1, 20, 30), - - %% Use `marker' to request remainder of results - Options2 = [{marker, "27"}], - verify_object_list(erlcloud_s3:list_objects(?TEST_BUCKET, Options2, UserConfig), 10, 30, 21), - - %% Put 2 sets of 4 objects with keys that have a common subdirectory - Prefix1 = "0/prefix1/", - Prefix2 = "0/prefix2/", - load_objects(?TEST_BUCKET, 4, Prefix1, UserConfig), - load_objects(?TEST_BUCKET, 4, Prefix2, UserConfig), - - %% Use `prefix' and `delimiter' to get the key groups under - %% the `prefix'. Should get 2 common prefix results back. - Options3 = [{prefix, "0/"}, {delimiter, "/"}], - ObjList2 = erlcloud_s3:list_objects(?TEST_BUCKET, Options3, UserConfig), - CommonPrefixes1 = proplists:get_value(common_prefixes, ObjList2), - ?assert(lists:member([{prefix, Prefix1}], CommonPrefixes1)), - ?assert(lists:member([{prefix, Prefix2}], CommonPrefixes1)), - ?assertEqual([], proplists:get_value(contents, ObjList2)), - - %% Use `prefix' option to restrict results to only keys that - %% begin with that prefix. Without the `delimiter' option the keys - %% are returned in the contents instead of as common prefixes. - Options4 = [{prefix, "0/"}], - ObjList3 = erlcloud_s3:list_objects(?TEST_BUCKET, Options4, UserConfig), - CommonPrefixes2 = proplists:get_value(common_prefixes, ObjList3), - ?assertEqual([], CommonPrefixes2), - ExpObjList1 = [Prefix1 ++ integer_to_list(X) || X <- lists:seq(1,4)] ++ - [Prefix2 ++ integer_to_list(Y) || Y <- lists:seq(1,4)], - ?assertEqual(ExpObjList1, [proplists:get_value(key, O) || - O <- proplists:get_value(contents, ObjList3)]), - - %% Request remainder of results - Options5 = [{marker, "7"}], - verify_object_list(erlcloud_s3:list_objects(?TEST_BUCKET, Options5, UserConfig), 2, 30, 29), - - %% Use `delimiter' and verify results include a single common - %% prefixe and 30 keys in the contents - Options6 = [{delimiter, "/"}], - ObjList4 = erlcloud_s3:list_objects(?TEST_BUCKET, Options6, UserConfig), - CommonPrefixes3 = proplists:get_value(common_prefixes, ObjList4), - ?assert(lists:member([{prefix, "0/"}], CommonPrefixes3)), - verify_object_list(ObjList4, 30), - - %% Don't fail even if Prefix is longer than 1024, even if keys are - %% restricted to be shorter than it. That's S3. - TooLongKey = "0/" ++ binary_to_list(binary:copy(<<"b">>, 1025)) ++ "/", - OptionsX = [{prefix, TooLongKey}], - ObjListX = erlcloud_s3:list_objects(?TEST_BUCKET, OptionsX, UserConfig), - ?assertEqual([], proplists:get_value(contents, ObjListX)), - - delete_objects(?TEST_BUCKET, Count3, [], UserConfig), - delete_objects(?TEST_BUCKET, 4, Prefix1, UserConfig), - delete_objects(?TEST_BUCKET, 4, Prefix2, UserConfig), - - Options7 = [{max_keys, "invalid"}], - ?assertError({aws_error, {http_error, 400, _, _}}, - erlcloud_s3:list_objects(?TEST_BUCKET, Options7, UserConfig)), - - lager:info("deleting bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:delete_bucket(?TEST_BUCKET, UserConfig)), - - ?assertError({aws_error, {http_error, 404, _, _}}, erlcloud_s3:list_objects(?TEST_BUCKET, UserConfig)), - rtcs:pass(). - -load_objects(Bucket, Count, Config) -> - load_objects(Bucket, Count, [], Config). - -load_objects(Bucket, Count, KeyPrefix, Config) -> - [erlcloud_s3:put_object(Bucket, - KeyPrefix ++ integer_to_list(X), - crypto:rand_bytes(100), - Config) || X <- lists:seq(1,Count)]. - -verify_object_list(ObjList, ExpectedCount) -> - verify_object_list(ObjList, ExpectedCount, ExpectedCount, 1). - -verify_object_list(ObjList, ExpectedCount, TotalCount) -> - verify_object_list(ObjList, ExpectedCount, TotalCount, 1). - -verify_object_list(ObjList, ExpectedCount, TotalCount, Offset) -> - verify_object_list(ObjList, ExpectedCount, TotalCount, Offset, []). - -verify_object_list(ObjList, ExpectedCount, TotalCount, 1, KeyPrefix) - when ExpectedCount =:= TotalCount -> - ?assertEqual(lists:sort([KeyPrefix ++ integer_to_list(X) - || X <- lists:seq(1, ExpectedCount)]), - [proplists:get_value(key, O) || - O <- proplists:get_value(contents, ObjList)]); -verify_object_list(ObjList, ExpectedCount, TotalCount, Offset, KeyPrefix) -> - ?assertEqual(lists:sublist( - lists:sort([KeyPrefix ++ integer_to_list(X) - || X <- lists:seq(1, TotalCount)]), - Offset, - ExpectedCount), - [proplists:get_value(key, O) || - O <- proplists:get_value(contents, ObjList)]). - -delete_and_verify_objects(Bucket, 0, Config) -> - verify_object_list(erlcloud_s3:list_objects(Bucket, Config), 0), - ok; -delete_and_verify_objects(Bucket, Count, Config) -> - verify_object_list(erlcloud_s3:list_objects(Bucket, Config), Count), - erlcloud_s3:delete_object(Bucket, integer_to_list(Count), Config), - delete_and_verify_objects(Bucket, Count-1, Config). - -delete_objects(Bucket, Count, Prefix, Config) -> - [erlcloud_s3:delete_object(Bucket, - Prefix ++ integer_to_list(X), - Config) - || X <- lists:seq(1, Count)]. diff --git a/riak_test/src/rc_helper.erl b/riak_test/src/rc_helper.erl deleted file mode 100644 index de48e4e80..000000000 --- a/riak_test/src/rc_helper.erl +++ /dev/null @@ -1,96 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(rc_helper). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). - -to_riak_bucket(objects, CSBucket) -> - %% or make version switch here. - <<"0o:", (md5(CSBucket))/binary>>; -to_riak_bucket(blocks, CSBucket) -> - %% or make version switch here. - <<"0b:", (md5(CSBucket))/binary>>; -to_riak_bucket(_, CSBucket) -> - CSBucket. - --spec md5(iodata()) -> binary(). -md5(IOData) -> - crypto:hash(md5, IOData). - -to_riak_key(objects, CsKey) when is_binary(CsKey) -> - CsKey; -to_riak_key(objects, CsKey) when is_list(CsKey) -> - list_to_binary(CsKey); -to_riak_key(blocks, {UUID, Seq}) -> - <>; -to_riak_key(Kind, _) -> - error({not_yet_implemented, Kind}). - --spec get_riakc_obj([term()], objects | blocks, binary(), term()) -> term(). -get_riakc_obj(RiakNodes, Kind, CsBucket, Opts) -> - {Pbc, Key} = case Kind of - objects -> - {rtcs:pbc(RiakNodes, Kind, CsBucket), Opts}; - blocks -> - {CsKey, UUID, Seq} = Opts, - {rtcs:pbc(RiakNodes, Kind, {CsBucket, CsKey, UUID}), - {UUID, Seq}} - end, - RiakBucket = to_riak_bucket(Kind, CsBucket), - RiakKey = to_riak_key(Kind, Key), - Result = riakc_pb_socket:get(Pbc, RiakBucket, RiakKey), - riakc_pb_socket:stop(Pbc), - Result. - --spec update_riakc_obj([term()], objects | blocks, binary(), term(), riakc_obj:riakc_obj()) -> term(). -update_riakc_obj(RiakNodes, ObjectKind, CsBucket, CsKey, NewObj) -> - Pbc = rtcs:pbc(RiakNodes, ObjectKind, CsBucket), - RiakBucket = to_riak_bucket(ObjectKind, CsBucket), - RiakKey = to_riak_key(ObjectKind, CsKey), - OldObj = case riakc_pb_socket:get(Pbc, RiakBucket, RiakKey, [deletedvclock]) of - {ok, Obj} -> Obj; - {error, notfound} -> riakc_obj:new(RiakBucket, RiakKey); - {error, notfound, OldVclock} -> - riakc_obj:set_vclock(riakc_obj:new(RiakBucket, RiakKey), - OldVclock) - end, - NewMD = riakc_obj:get_metadata(NewObj), - NewValue = riakc_obj:get_value(NewObj), - Updated = riakc_obj:update_value( - riakc_obj:update_metadata(OldObj, NewMD), NewValue), - Result = riakc_pb_socket:put(Pbc, Updated), - riakc_pb_socket:stop(Pbc), - Result. - -delete_riakc_obj(RiakNodes, Kind, CsBucket, Opts) -> - {Pbc, Key} = case Kind of - objects -> - {rtcs:pbc(RiakNodes, Kind, CsBucket), Opts}; - blocks -> - {CsKey, UUID, Seq} = Opts, - {rtcs:pbc(RiakNodes, Kind, {CsBucket, CsKey, UUID}), - {UUID, Seq}} - end, - RiakBucket = to_riak_bucket(Kind, CsBucket), - RiakKey = to_riak_key(Kind, Key), - Result = riakc_pb_socket:delete(Pbc, RiakBucket, RiakKey), - riakc_pb_socket:stop(Pbc), - Result. diff --git a/riak_test/src/repl_helpers.erl b/riak_test/src/repl_helpers.erl deleted file mode 100644 index 47a122430..000000000 --- a/riak_test/src/repl_helpers.erl +++ /dev/null @@ -1,377 +0,0 @@ --module(repl_helpers). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). - -verify_sites_balanced(NumSites, BNodes0) -> - rt:wait_until(fun() -> - L = rpc:call(hd(BNodes0), riak_repl_leader, leader_node, []), - L =/= undefined - end), - Leader = rpc:call(hd(BNodes0), riak_repl_leader, leader_node, []), - case node_has_version(Leader, "1.2.0") of - true -> - BNodes = nodes_with_version(BNodes0, "1.2.0") -- [Leader], - NumNodes = length(BNodes), - case NumNodes of - 0 -> - %% only leader is upgraded, runs clients locally - ?assertEqual(NumSites, client_count(Leader)); - _ -> - NodeCounts = [{Node, client_count(Node)} || Node <- BNodes], - lager:notice("nodecounts ~p", [NodeCounts]), - lager:notice("leader ~p", [Leader]), - Min = NumSites div NumNodes, - [?assert(Count >= Min) || {_Node, Count} <- NodeCounts] - end; - false -> - ok - end. - -%% does the node meet the version requirement? -node_has_version(Node, Version) -> - NodeVersion = rtdev:node_version(rtdev:node_id(Node)), - case NodeVersion of - current -> - %% current always satisfies any version check - true; - _ -> - NodeVersion >= Version - end. - -nodes_with_version(Nodes, Version) -> - [Node || Node <- Nodes, node_has_version(Node, Version)]. - -client_count(Node) -> - Clients = rpc:call(Node, supervisor, which_children, [riak_repl_client_sup]), - length(Clients). - -add_site(Node, {IP, Port, Name}) -> - lager:info("Add site ~p ~p:~p at node ~p", [Name, IP, Port, Node]), - Args = [IP, integer_to_list(Port), Name], - Res = rpc:call(Node, riak_repl_console, add_site, [Args]), - ?assertEqual(ok, Res). - -del_site(Node, Name) -> - lager:info("Del site ~p at ~p", [Name, Node]), - Res = rpc:call(Node, riak_repl_console, del_site, [[Name]]), - ?assertEqual(ok, Res). - -verify_listeners(Listeners) -> - Strs = [IP ++ ":" ++ integer_to_list(Port) || {IP, Port, _} <- Listeners], - [?assertEqual(ok, verify_listener(Node, Strs)) || {_, _, Node} <- Listeners]. - -verify_listener(Node, Strs) -> - lager:info("Verify listeners ~p ~p", [Node, Strs]), - rt:wait_until(Node, - fun(_) -> - Status = rpc:call(Node, riak_repl_console, status, [quiet]), - lists:all(fun(Str) -> - lists:keymember(Str, 2, Status) - end, Strs) - end). - -add_listeners(Nodes=[FirstNode|_]) -> - Ports = gen_ports(9010, length(Nodes)), - IPs = lists:duplicate(length(Nodes), "127.0.0.1"), - PN = lists:zip3(IPs, Ports, Nodes), - [add_listener(FirstNode, Node, IP, Port) || {IP, Port, Node} <- PN], - PN. - -add_listener(N, Node, IP, Port) -> - lager:info("Adding repl listener to ~p ~s:~p", [Node, IP, Port]), - Args = [[atom_to_list(Node), IP, integer_to_list(Port)]], - Res = rpc:call(N, riak_repl_console, add_listener, Args), - ?assertEqual(ok, Res). - -gen_ports(Start, Len) -> - lists:seq(Start, Start + Len - 1). - -verify_site_ips(Leader, Site, Listeners) -> - rt:wait_until( - fun() -> Status = rpc:call(Leader, riak_repl_console, status, [quiet]), - Key = lists:flatten([Site, "_ips"]), - IPStr = proplists:get_value(Key, Status), - IPs = lists:sort(re:split(IPStr, ", ")), - ExpectedIPs = lists:sort( - [list_to_binary([IP, ":", integer_to_list(Port)]) || - {IP, Port, _Node} <- Listeners]), - ExpectedIPs =:= IPs - end). - -start_and_wait_until_fullsync_complete(Node) -> - Status0 = rpc:call(Node, riak_repl_console, status, [quiet]), - Count = proplists:get_value(server_fullsyncs, Status0) + 1, - lager:info("waiting for fullsync count to be ~p", [Count]), - - lager:info("Starting fullsync on ~p (~p)", [Node, - rtdev:node_version(rtdev:node_id(Node))]), - rpc:call(Node, riak_repl_console, start_fullsync, [[]]), - %% sleep because of the old bug where stats will crash if you call it too - %% soon after starting a fullsync - timer:sleep(500), - - Res = rt:wait_until(Node, - fun(_) -> - Status = rpc:call(Node, riak_repl_console, status, [quiet]), - case proplists:get_value(server_fullsyncs, Status) of - C when C >= Count -> - true; - _ -> - false - end - end), - case node_has_version(Node, "1.2.0") of - true -> - ?assertEqual(ok, Res); - _ -> - case Res of - ok -> - ok; - _ -> - ?assertEqual(ok, wait_until_connection(Node)), - lager:warning("Pre 1.2.0 node failed to fullsync, retrying"), - start_and_wait_until_fullsync_complete(Node) - end - end, - - lager:info("Fullsync on ~p complete", [Node]). - - -wait_until_leader(Node) -> - Res = rt:wait_until(Node, - fun(_) -> - Status = rpc:call(Node, riak_repl_console, status, [quiet]), - case Status of - {badrpc, _} -> - false; - _ -> - case proplists:get_value(leader, Status) of - undefined -> - false; - _ -> - true - end - end - end), - ?assertEqual(ok, Res). - - -wait_until_13_leader(Node) -> - wait_until_new_leader(Node, undefined). - -%% taken from -%% https://github.com/basho/riak_test/blob/master/tests/repl_util.erl -wait_until_new_leader(Node, OldLeader) -> - Res = rt:wait_until(Node, - fun(_) -> - Status = rpc:call(Node, riak_core_cluster_mgr, get_leader, []), - case Status of - {badrpc, _} -> - false; - undefined -> - false; - OldLeader -> - false; - _Other -> - true - end - end), - ?assertEqual(ok, Res). - -wait_until_leader_converge([Node|_] = Nodes) -> - rt:wait_until(Node, - fun(_) -> - length(lists:usort([begin - case rpc:call(N, riak_core_cluster_mgr, get_leader, []) of - undefined -> - false; - L -> - %lager:info("Leader for ~p is ~p", - %[N,L]), - L - end - end || N <- Nodes])) == 1 - end). - -wait_until_connection(Node) -> - rt:wait_until(Node, - fun(_) -> - Status = rpc:call(Node, riak_repl_console, status, [quiet]), - case proplists:get_value(server_stats, Status) of - [] -> - false; - [_C] -> - true; - Conns -> - lager:warning("multiple connections detected: ~p", - [Conns]), - true - end - end, 80, 500). %% 40 seconds is enough for repl - -%% The functions below are for 1.3 repl (aka Advanced Mode MDC) -connect_cluster(Node, IP, Port) -> - Res = rpc:call(Node, riak_repl_console, connect, - [[IP, integer_to_list(Port)]]), - ?assertEqual(ok, Res). - -disconnect_cluster(Node, Name) -> - Res = rpc:call(Node, riak_repl_console, disconnect, - [[Name]]), - ?assertEqual(ok, Res). - -wait_for_connection(Node, Name) -> - rt:wait_until(Node, - fun(_) -> - {ok, Connections} = rpc:call(Node, riak_core_cluster_mgr, - get_connections, []), - lists:any(fun({{cluster_by_name, N}, _}) when N == Name -> true; - (_) -> false - end, Connections) - end). - -wait_until_no_connection(Node) -> - rt:wait_until(Node, - fun(_) -> - Status = rpc:call(Node, riak_repl_console, status, [quiet]), - case proplists:get_value(connected_clusters, Status) of - [] -> - true; - _ -> - false - end - end). %% 40 seconds is enough for repl - -enable_realtime(Node, Cluster) -> - Res = rpc:call(Node, riak_repl_console, realtime, [["enable", Cluster]]), - ?assertEqual(ok, Res). - -disable_realtime(Node, Cluster) -> - Res = rpc:call(Node, riak_repl_console, realtime, [["disable", Cluster]]), - ?assertEqual(ok, Res). - -enable_fullsync(Node, Cluster) -> - Res = rpc:call(Node, riak_repl_console, fullsync, [["enable", Cluster]]), - ?assertEqual(ok, Res). - -start_realtime(Node, Cluster) -> - Res = rpc:call(Node, riak_repl_console, realtime, [["start", Cluster]]), - ?assertEqual(ok, Res). - -stop_realtime(Node, Cluster) -> - Res = rpc:call(Node, riak_repl_console, realtime, [["stop", Cluster]]), - ?assertEqual(ok, Res). - -name_cluster(Node, Name) -> - lager:info("Naming cluster ~p",[Name]), - Res = rpc:call(Node, riak_repl_console, clustername, [[Name]]), - ?assertEqual(ok, Res). - -connect_clusters13(LeaderA, ANodes, BPort, Name) -> - lager:info("Connecting to ~p", [Name]), - connect_cluster13(LeaderA, "127.0.0.1", BPort), - ?assertEqual(ok, wait_for_connection13(LeaderA, Name)), - repl_util:enable_realtime(LeaderA, Name), - rt:wait_until_ring_converged(ANodes), - repl_util:start_realtime(LeaderA, Name), - rt:wait_until_ring_converged(ANodes), - repl_util:enable_fullsync(LeaderA, Name), - rt:wait_until_ring_converged(ANodes), - enable_pg13(LeaderA, Name), - rt:wait_until_ring_converged(ANodes), - ?assertEqual(ok, wait_for_connection13(LeaderA, Name)), - rt:wait_until_ring_converged(ANodes). - -disconnect_clusters13(LeaderA, ANodes, Name) -> - lager:info("Disconnecting from ~p", [Name]), - disconnect_cluster13(LeaderA, Name), - repl_util:disable_realtime(LeaderA, Name), - rt:wait_until_ring_converged(ANodes), - repl_util:stop_realtime(LeaderA, Name), - rt:wait_until_ring_converged(ANodes), - disable_pg13(LeaderA, Name), - rt:wait_until_ring_converged(ANodes), - ?assertEqual(ok, wait_until_no_connection13(LeaderA)), - rt:wait_until_ring_converged(ANodes). - -start_and_wait_until_fullsync_complete13(Node) -> - Status0 = rpc:call(Node, riak_repl_console, status, [quiet]), - Count = proplists:get_value(server_fullsyncs, Status0) + 1, - lager:info("waiting for fullsync count to be ~p", [Count]), - - lager:info("Starting fullsync on ~p (~p)", [Node, - rtdev:node_version(rtdev:node_id(Node))]), - rpc:call(Node, riak_repl_console, fullsync, [["start"]]), - %% sleep because of the old bug where stats will crash if you call it too - %% soon after starting a fullsync - timer:sleep(500), - - Res = rt:wait_until(Node, - fun(_) -> - Status = rpc:call(Node, riak_repl_console, status, [quiet]), - case proplists:get_value(server_fullsyncs, Status) of - C when C >= Count -> - true; - _ -> - false - end - end), - ?assertEqual(ok, Res), - - lager:info("Fullsync on ~p complete", [Node]). - -wait_for_connection13(Node, Name) -> - rt:wait_until(Node, - fun(_) -> - {ok, Connections} = rpc:call(Node, riak_core_cluster_mgr, - get_connections, []), - lists:any(fun({{cluster_by_name, N}, _}) when N == Name -> true; - (_) -> false - end, Connections) - end). - -wait_until_no_connection13(Node) -> - rt:wait_until(Node, - fun(_) -> - Status = rpc:call(Node, riak_repl_console, status, [quiet]), - case proplists:get_value(connected_clusters, Status) of - [] -> - true; - _ -> - false - end - end). %% 40 seconds is enough for repl - -wait_until_realtime_sync_complete(Nodes) -> - [wait_until_rtq_drained(Node)||Node <- Nodes]. - -wait_until_rtq_drained(Node) -> - rt:wait_until(Node, - fun(_) -> - case rpc:call(Node, riak_repl2_rtq, dumpq, []) of - [] -> - true; - _ -> - false - end - end), - lager:info("Realtime sync on ~p complete", [Node]). - -connect_cluster13(Node, IP, Port) -> - Res = rpc:call(Node, riak_repl_console, connect, - [[IP, integer_to_list(Port)]]), - ?assertEqual(ok, Res). - -disconnect_cluster13(Node, Name) -> - Res = rpc:call(Node, riak_repl_console, disconnect, - [[Name]]), - ?assertEqual(ok, Res). - -enable_pg13(Node, Cluster) -> - Res = rpc:call(Node, riak_repl_console, proxy_get, [["enable", Cluster]]), - ?assertEqual(ok, Res). - -disable_pg13(Node, Cluster) -> - Res = rpc:call(Node, riak_repl_console, proxy_get, [["disable", Cluster]]), - ?assertEqual(ok, Res). - diff --git a/riak_test/src/rtcs.erl b/riak_test/src/rtcs.erl deleted file mode 100644 index ed44731cc..000000000 --- a/riak_test/src/rtcs.erl +++ /dev/null @@ -1,397 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- --module(rtcs). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). --include_lib("erlcloud/include/erlcloud_aws.hrl"). --include_lib("xmerl/include/xmerl.hrl"). - --import(rt, [join/2, - wait_until_nodes_ready/1, - wait_until_no_pending_changes/1]). - --define(DEVS(N), lists:concat(["dev", N, "@127.0.0.1"])). --define(DEV(N), list_to_atom(?DEVS(N))). --define(CSDEVS(N), lists:concat(["rcs-dev", N, "@127.0.0.1"])). --define(CSDEV(N), list_to_atom(?CSDEVS(N))). - -setup(NumNodes) -> - setup(NumNodes, [], current). - -setup(NumNodes, Configs) -> - setup(NumNodes, Configs, current). - -setup(NumNodes, Configs, Vsn) -> - Flavor = rt_config:get(flavor, basic), - lager:info("Flavor : ~p", [Flavor]), - flavored_setup(NumNodes, Flavor, Configs, Vsn). - -setup2x2() -> - setup2x2([]). - -setup2x2(Configs) -> - JoinFun = fun(Nodes) -> - [A,B,C,D] = Nodes, - join(B,A), - join(D,C) - end, - setup_clusters(Configs, JoinFun, 4, current). - -%% 1 cluster with N nodes + M cluster with 1 node -setupNxMsingles(N, M) -> - setupNxMsingles(N, M, [], current). - -setupNxMsingles(N, M, Configs, Vsn) - when Vsn =:= current orelse Vsn =:= previous -> - JoinFun = fun(Nodes) -> - [Target | Joiners] = lists:sublist(Nodes, N), - [join(J, Target) || J <- Joiners] - end, - setup_clusters(Configs, JoinFun, N + M, Vsn). - -flavored_setup(NumNodes, basic, Configs, Vsn) -> - JoinFun = fun(Nodes) -> - [First|Rest] = Nodes, - [join(Node, First) || Node <- Rest] - end, - setup_clusters(Configs, JoinFun, NumNodes, Vsn); -flavored_setup(NumNodes, {multibag, _} = Flavor, Configs, Vsn) - when Vsn =:= current orelse Vsn =:= previous -> - rtcs_bag:flavored_setup(NumNodes, Flavor, Configs, Vsn). - -setup_clusters(Configs, JoinFun, NumNodes, Vsn) -> - %% Start the erlcloud app - erlcloud:start(), - - %% STFU sasl - application:load(sasl), - application:set_env(sasl, sasl_error_logger, false), - - {RiakNodes, _CSNodes, _Stanchion} = Nodes = - deploy_nodes(NumNodes, rtcs_config:configs(Configs, Vsn), Vsn), - rt:wait_until_nodes_ready(RiakNodes), - lager:info("Make cluster"), - JoinFun(RiakNodes), - ?assertEqual(ok, wait_until_nodes_ready(RiakNodes)), - ?assertEqual(ok, wait_until_no_pending_changes(RiakNodes)), - rt:wait_until_ring_converged(RiakNodes), - AdminConfig = setup_admin_user(NumNodes, Vsn), - {AdminConfig, Nodes}. - - -pass() -> - teardown(), - pass. - -teardown() -> - %% catch application:stop(sasl), - catch application:stop(erlcloud), - catch application:stop(ibrowse). - -%% Return Riak node IDs, one per cluster. -%% For example, in basic single cluster case, just return [1]. --spec riak_id_per_cluster(pos_integer()) -> [pos_integer()]. -riak_id_per_cluster(NumNodes) -> - case rt_config:get(flavor, basic) of - basic -> [1]; - {multibag, _} = Flavor -> rtcs_bag:riak_id_per_cluster(NumNodes, Flavor) - end. - --spec deploy_nodes(list(), list(), current|previous) -> any(). -deploy_nodes(NumNodes, InitialConfig, Vsn) - when Vsn =:= current orelse Vsn =:= previous -> - lager:info("Initial Config: ~p", [InitialConfig]), - {RiakNodes, CSNodes, StanchionNode} = Nodes = {riak_nodes(NumNodes), - cs_nodes(NumNodes), - stanchion_node()}, - - NodeMap = orddict:from_list(lists:zip(RiakNodes, lists:seq(1, NumNodes))), - rt_config:set(rt_nodes, NodeMap), - CSNodeMap = orddict:from_list(lists:zip(CSNodes, lists:seq(1, NumNodes))), - rt_config:set(rt_cs_nodes, CSNodeMap), - - {_RiakRoot, RiakVsn} = rtcs_dev:riak_root_and_vsn(Vsn, rt_config:get(build_type, oss)), - lager:debug("setting rt_versions> ~p =>", [Vsn]), - - VersionMap = lists:zip(lists:seq(1, NumNodes), lists:duplicate(NumNodes, RiakVsn)), - rt_config:set(rt_versions, VersionMap), - - rtcs_exec:stop_all_nodes(node_list(NumNodes), Vsn), - - rtcs_dev:create_dirs(RiakNodes), - - %% Set initial config - rtcs_config:set_configs(NumNodes, - InitialConfig, - Vsn), - rtcs_exec:start_all_nodes(node_list(NumNodes), Vsn), - - [ok = rt:wait_until_pingable(N) || N <- RiakNodes ++ CSNodes ++ [StanchionNode]], - [ok = rt:check_singleton_node(N) || N <- RiakNodes], - rt:wait_until_nodes_ready(RiakNodes), - - lager:info("NodeMap: ~p", [ NodeMap ]), - lager:info("VersionMap: ~p", [VersionMap]), - lager:info("Deployed nodes: ~p", [Nodes]), - - Nodes. - -node_id(Node) -> - NodeMap = rt_config:get(rt_cs_nodes), - orddict:fetch(Node, NodeMap). - -setup_admin_user(NumNodes, Vsn) - when Vsn =:= current orelse Vsn =:= previous -> - - %% Create admin user and set in cs and stanchion configs - AdminCreds = rtcs_admin:create_admin_user(1), - #aws_config{access_key_id=KeyID, - secret_access_key=KeySecret} = AdminCreds, - - AdminConf = [{admin_key, KeyID}] - ++ case Vsn of - current -> []; - previous -> [{admin_secret, KeySecret}] - end, - rt:pmap(fun(N) -> - rtcs:set_advanced_conf({cs, Vsn, N}, [{riak_cs, AdminConf}]) - end, lists:seq(1, NumNodes)), - rtcs:set_advanced_conf({stanchion, Vsn}, [{stanchion, AdminConf}]), - - UpdateFun = fun({Node, App}) -> - ok = rpc:call(Node, application, set_env, - [App, admin_key, KeyID]), - ok = rpc:call(Node, application, set_env, - [App, admin_secret, KeySecret]) - end, - ZippedNodes = [{stanchion_node(), stanchion} | - [{CSNode, riak_cs} || CSNode <- cs_nodes(NumNodes) ]], - lists:foreach(UpdateFun, ZippedNodes), - - lager:info("AdminCreds: ~p", [AdminCreds]), - AdminCreds. - --spec set_conf(atom() | {atom(), atom()} | string(), [{string(), string()}]) -> ok. -set_conf(all, NameValuePairs) -> - lager:info("rtcs:set_conf(all, ~p)", [NameValuePairs]), - [ set_conf(DevPath, NameValuePairs) || DevPath <- rtcs_dev:devpaths()], - ok; -set_conf(Name, NameValuePairs) when Name =:= riak orelse - Name =:= cs orelse - Name =:= stanchion -> - set_conf({Name, current}, NameValuePairs), - ok; -set_conf({Name, Vsn}, NameValuePairs) -> - lager:info("rtcs:set_conf({~p, ~p}, ~p)", [Name, Vsn, NameValuePairs]), - set_conf(rtcs_dev:devpath(Name, Vsn), NameValuePairs), - ok; -set_conf({Name, Vsn, N}, NameValuePairs) -> - lager:info("rtcs:set_conf({~p, ~p, ~p}, ~p)", [Name, Vsn, N, NameValuePairs]), - rtdev:append_to_conf_file(rtcs_dev:get_conf(rtcs_dev:devpath(Name, Vsn), N), NameValuePairs), - ok; -set_conf(Node, NameValuePairs) when is_atom(Node) -> - rtdev:append_to_conf_file(rtcs_dev:get_conf(Node), NameValuePairs), - ok; -set_conf(DevPath, NameValuePairs) -> - lager:info("rtcs:set_conf(~p, ~p)", [DevPath, NameValuePairs]), - [rtdev:append_to_conf_file(RiakConf, NameValuePairs) || RiakConf <- rtcs_dev:all_the_files(DevPath, "etc/*.conf")], - ok. - --spec set_advanced_conf(atom() | {atom(), atom()} | string(), [{string(), string()}]) -> ok. -set_advanced_conf(all, NameValuePairs) -> - lager:info("rtcs:set_advanced_conf(all, ~p)", [NameValuePairs]), - [ set_advanced_conf(DevPath, NameValuePairs) || DevPath <- rtcs_dev:devpaths()], - ok; -set_advanced_conf(Name, NameValuePairs) when Name =:= riak orelse - Name =:= cs orelse - Name =:= stanchion -> - set_advanced_conf({Name, current}, NameValuePairs), - ok; -set_advanced_conf({Name, Vsn}, NameValuePairs) -> - lager:info("rtcs:set_advanced_conf({~p, ~p}, ~p)", [Name, Vsn, NameValuePairs]), - set_advanced_conf(rtcs_dev:devpath(Name, Vsn), NameValuePairs), - ok; -set_advanced_conf({Name, Vsn, N}, NameValuePairs) -> - lager:info("rtcs:set_advanced_conf({~p, ~p, ~p}, ~p)", [Name, Vsn, N, NameValuePairs]), - rtcs_dev:update_app_config_file(rtcs_dev:get_app_config(rtcs_dev:devpath(Name, Vsn), N), NameValuePairs), - ok; -set_advanced_conf(Node, NameValuePairs) when is_atom(Node) -> - rtcs_dev:update_app_config_file(rtcs_dev:get_app_config(Node), NameValuePairs), - ok; -set_advanced_conf(DevPath, NameValuePairs) -> - AdvancedConfs = case rtcs_dev:all_the_files(DevPath, "etc/a*.config") of - [] -> - %% no advanced conf? But we _need_ them, so make 'em - rtdev:make_advanced_confs(DevPath); - Confs -> - Confs - end, - lager:info("AdvancedConfs = ~p~n", [AdvancedConfs]), - [rtcs_dev:update_app_config_file(RiakConf, NameValuePairs) || RiakConf <- AdvancedConfs], - ok. - -assert_error_log_empty(N) -> - assert_error_log_empty(current, N). - -assert_error_log_empty(Vsn, N) -> - ErrorLog = rtcs_config:riakcs_logpath(rtcs_config:devpath(cs, Vsn), N, "error.log"), - case file:read_file(ErrorLog) of - {error, enoent} -> ok; - {ok, <<>>} -> ok; - {ok, Errors} -> - lager:warning("Not empty error.log (~s): the first few lines are...~n~s", - [ErrorLog, - lists:map( - fun(L) -> io_lib:format("cs dev~p error.log: ~s\n", [N, L]) end, - lists:sublist(binary:split(Errors, <<"\n">>, [global]), 3))]), - error(not_empty_error_log) - end. - -truncate_error_log(N) -> - Cmd = os:find_executable("rm"), - ErrorLog = rtcs_config:riakcs_logpath(rt_config:get(rtcs_config:cs_current()), N, "error.log"), - ok = rtcs_exec:cmd(Cmd, [{args, ["-f", ErrorLog]}]). - -wait_until(_, _, 0, _) -> - fail; -wait_until(Fun, Condition, Retries, Delay) -> - Result = Fun(), - case Condition(Result) of - true -> - Result; - false -> - timer:sleep(Delay), - wait_until(Fun, Condition, Retries-1, Delay) - end. - -%% Kind = objects | blocks | users | buckets ... -pbc(RiakNodes, ObjectKind, Opts) -> - pbc(rt_config:get(flavor, basic), ObjectKind, RiakNodes, Opts). - -pbc(basic, _ObjectKind, RiakNodes, _Opts) -> - rt:pbc(hd(RiakNodes)); -pbc({multibag, _} = Flavor, ObjectKind, RiakNodes, Opts) -> - rtcs_bag:pbc(Flavor, ObjectKind, RiakNodes, Opts). - -sha_mac(Key,STS) -> crypto:hmac(sha, Key,STS). -sha(Bin) -> crypto:hash(sha, Bin). -md5(Bin) -> crypto:hash(md5, Bin). - -datetime() -> - {{YYYY,MM,DD}, {H,M,S}} = calendar:universal_time(), - lists:flatten(io_lib:format("~4..0B~2..0B~2..0BT~2..0B~2..0B~2..0BZ", [YYYY, MM, DD, H, M, S])). - - - -json_get(Key, Json) when is_binary(Key) -> - json_get([Key], Json); -json_get([], Json) -> - Json; -json_get([Key | Keys], {struct, JsonProps}) -> - case lists:keyfind(Key, 1, JsonProps) of - false -> - notfound; - {Key, Value} -> - json_get(Keys, Value) - end. - -check_no_such_bucket(Response, Resource) -> - check_error_response(Response, - 404, - "NoSuchBucket", - "The specified bucket does not exist.", - Resource). - -check_error_response({_, Status, _, RespStr} = _Response, - Status, - Code, Message, Resource) -> - {RespXml, _} = xmerl_scan:string(RespStr), - lists:all(error_child_element_verifier(Code, Message, Resource), - RespXml#xmlElement.content). - -error_child_element_verifier(Code, Message, Resource) -> - fun(#xmlElement{name='Code', content=[Content]}) -> - Content#xmlText.value =:= Code; - (#xmlElement{name='Message', content=[Content]}) -> - Content#xmlText.value =:= Message; - (#xmlElement{name='Resource', content=[Content]}) -> - Content#xmlText.value =:= Resource; - (_) -> - true - end. - -assert_versions(App, Nodes, Regexp) -> - [begin - {ok, Vsn} = rpc:call(N, application, get_key, [App, vsn]), - lager:debug("~s's vsn at ~s: ~s", [App, N, Vsn]), - {match, _} = re:run(Vsn, Regexp) - end || - N <- Nodes]. - - -%% Copy from rts:iso8601/1 -iso8601(Timestamp) when is_integer(Timestamp) -> - GregSec = Timestamp + 719528 * 86400, - Datetime = calendar:gregorian_seconds_to_datetime(GregSec), - {{Y,M,D},{H,I,S}} = Datetime, - io_lib:format("~4..0b~2..0b~2..0bT~2..0b~2..0b~2..0bZ", - [Y, M, D, H, I, S]). - -reset_log(Node) -> - {ok, _Logs} = rpc:call(Node, gen_event, delete_handler, - [lager_event, riak_test_lager_backend, normal]), - ok = rpc:call(Node, gen_event, add_handler, - [lager_event, riak_test_lager_backend, - [rt_config:get(lager_level, info), false]]). - -riak_node(N) -> - ?DEV(N). - -cs_node(N) -> - ?CSDEV(N). - -stanchion_node() -> - 'stanchion@127.0.0.1'. - -maybe_load_intercepts(Node) -> - case rt_intercept:are_intercepts_loaded(Node, [intercept_path()]) of - false -> - ok = rt_intercept:load_intercepts([Node], [intercept_path()]); - true -> - ok - end. - -%% private - -riak_nodes(NumNodes) -> - [?DEV(N) || N <- lists:seq(1, NumNodes)]. - -cs_nodes(NumNodes) -> - [?CSDEV(N) || N <- lists:seq(1, NumNodes)]. - -node_list(NumNodes) -> - NL0 = lists:zip(cs_nodes(NumNodes), - riak_nodes(NumNodes)), - {CS1, R1} = hd(NL0), - [{CS1, R1, stanchion_node()} | tl(NL0)]. - -intercept_path() -> - filename:join([rtcs_dev:srcpath(cs_src_root), - "riak_test", "intercepts", "*.erl"]). diff --git a/riak_test/src/rtcs_admin.erl b/riak_test/src/rtcs_admin.erl deleted file mode 100644 index b35c0ae06..000000000 --- a/riak_test/src/rtcs_admin.erl +++ /dev/null @@ -1,228 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(rtcs_admin). - --export([storage_stats_json_request/4, - create_user/2, - create_user/3, - create_user/4, - create_admin_user/1, - update_user/5, - get_user/4, - list_users/4, - make_authorization/5, - make_authorization/6, - make_authorization/7, - aws_config/2, - aws_config/3]). - --include_lib("eunit/include/eunit.hrl"). --include_lib("erlcloud/include/erlcloud_aws.hrl"). --include_lib("xmerl/include/xmerl.hrl"). - --define(S3_HOST, "s3.amazonaws.com"). --define(DEFAULT_PROTO, "http"). --define(PROXY_HOST, "localhost"). - --spec storage_stats_json_request(#aws_config{}, #aws_config{}, string(), string()) -> - [{string(), {non_neg_integer(), non_neg_integer()}}]. -storage_stats_json_request(AdminConfig, UserConfig, Begin, End) -> - Samples = samples_from_json_request(AdminConfig, UserConfig, {Begin, End}), - lager:debug("Storage samples[json]: ~p", [Samples]), - {struct, Slice} = latest(Samples, undefined), - by_bucket_list(Slice, []). - --spec create_admin_user(atom()) -> #aws_config{}. -create_admin_user(Node) -> - User = "admin", - Email = "admin@me.com", - {UserConfig, Id} = create_user(rtcs_config:cs_port(Node), Email, User), - lager:info("Riak CS Admin account created with ~p",[Email]), - lager:info("KeyId = ~p",[UserConfig#aws_config.access_key_id]), - lager:info("KeySecret = ~p",[UserConfig#aws_config.secret_access_key]), - lager:info("Id = ~p",[Id]), - UserConfig. - --spec create_user(atom(), non_neg_integer()) -> #aws_config{}. -create_user(Node, UserIndex) -> - {A, B, C} = erlang:now(), - User = "Test User" ++ integer_to_list(UserIndex), - Email = lists:flatten(io_lib:format("~p~p~p@basho.com", [A, B, C])), - {UserConfig, _Id} = create_user(rtcs_config:cs_port(Node), Email, User), - lager:info("Created user ~p with keys ~p ~p", [Email, - UserConfig#aws_config.access_key_id, - UserConfig#aws_config.secret_access_key]), - UserConfig. - --spec create_user(non_neg_integer(), string(), string()) -> {#aws_config{}, string()}. -create_user(Port, EmailAddr, Name) -> - %% create_user(Port, undefined, EmailAddr, Name). - create_user(Port, aws_config("admin-key", "admin-secret", Port), EmailAddr, Name). - --spec create_user(non_neg_integer(), string(), string(), string()) -> {#aws_config{}, string()}. -create_user(Port, UserConfig, EmailAddr, Name) -> - lager:debug("Trying to create user ~p", [EmailAddr]), - Resource = "/riak-cs/user", - ReqBody = "{\"email\":\"" ++ EmailAddr ++ "\", \"name\":\"" ++ Name ++"\"}", - Delay = rt_config:get(rt_retry_delay), - Retries = rt_config:get(rt_max_wait_time) div Delay, - OutputFun = fun() -> catch erlcloud_s3:s3_request( - UserConfig, post, "", Resource, [], "", - {ReqBody, "application/json"}, []) - end, - Condition = fun({'EXIT', Res}) -> - lager:debug("create_user failing, Res: ~p", [Res]), - false; - ({_ResHeader, _ResBody}) -> - true - end, - {_ResHeader, ResBody} = rtcs:wait_until(OutputFun, Condition, Retries, Delay), - lager:debug("ResBody: ~s", [ResBody]), - JsonData = mochijson2:decode(ResBody), - [KeyId, KeySecret, Id] = [binary_to_list(rtcs:json_get([K], JsonData)) || - K <- [<<"key_id">>, <<"key_secret">>, <<"id">>]], - {aws_config(KeyId, KeySecret, Port), Id}. - --spec update_user(#aws_config{}, non_neg_integer(), string(), string(), string()) -> string(). -update_user(UserConfig, _Port, Resource, ContentType, UpdateDoc) -> - {_ResHeader, ResBody} = erlcloud_s3:s3_request( - UserConfig, put, "", Resource, [], "", - {UpdateDoc, ContentType}, []), - lager:debug("ResBody: ~s", [ResBody]), - ResBody. - --spec get_user(#aws_config{}, non_neg_integer(), string(), string()) -> string(). -get_user(UserConfig, _Port, Resource, AcceptContentType) -> - lager:debug("Retreiving user record"), - Headers = [{"Accept", AcceptContentType}], - {_ResHeader, ResBody} = erlcloud_s3:s3_request( - UserConfig, get, "", Resource, [], "", "", Headers), - lager:debug("ResBody: ~s", [ResBody]), - ResBody. - --spec list_users(#aws_config{}, non_neg_integer(), string(), string()) -> string(). -list_users(UserConfig, _Port, Resource, AcceptContentType) -> - Headers = [{"Accept", AcceptContentType}], - {_ResHeader, ResBody} = erlcloud_s3:s3_request( - UserConfig, get, "", Resource, [], "", "", Headers), - ResBody. - --spec(make_authorization(string(), string(), string(), #aws_config{}, string()) -> string()). -make_authorization(Method, Resource, ContentType, Config, Date) -> - make_authorization(Method, Resource, ContentType, Config, Date, []). - --spec(make_authorization(string(), string(), string(), #aws_config{}, string(), [{string(), string()}]) -> string()). -make_authorization(Method, Resource, ContentType, Config, Date, AmzHeaders) -> - make_authorization(s3, Method, Resource, ContentType, Config, Date, AmzHeaders). - --spec(make_authorization(atom(), string(), string(), string(), #aws_config{}, string(), [{string(), string()}]) -> string()). -make_authorization(Type, Method, Resource, ContentType, Config, Date, AmzHeaders) -> - Prefix = case Type of - s3 -> "AWS"; - velvet -> "MOSS" - end, - StsAmzHeaderPart = [[K, $:, V, $\n] || {K, V} <- AmzHeaders], - StringToSign = [Method, $\n, [], $\n, ContentType, $\n, Date, $\n, - StsAmzHeaderPart, Resource], - lager:debug("StringToSign~n~s~n", [StringToSign]), - Signature = - base64:encode_to_string(rtcs:sha_mac(Config#aws_config.secret_access_key, StringToSign)), - lists:flatten([Prefix, " ", Config#aws_config.access_key_id, $:, Signature]). - --spec aws_config(string(), string(), non_neg_integer()) -> #aws_config{}. -aws_config(Key, Secret, Port) -> - erlcloud_s3:new(Key, - Secret, - ?S3_HOST, - Port, % inets issue precludes using ?S3_PORT - ?DEFAULT_PROTO, - ?PROXY_HOST, - Port, - []). - --spec aws_config(#aws_config{}, [{atom(), term()}]) -> #aws_config{}. -aws_config(UserConfig, []) -> - UserConfig; -aws_config(UserConfig, [{port, Port}|Props]) -> - UpdConfig = erlcloud_s3:new(UserConfig#aws_config.access_key_id, - UserConfig#aws_config.secret_access_key, - ?S3_HOST, - Port, % inets issue precludes using ?S3_PORT - ?DEFAULT_PROTO, - ?PROXY_HOST, - Port, - []), - aws_config(UpdConfig, Props); -aws_config(UserConfig, [{key, KeyId}|Props]) -> - UpdConfig = erlcloud_s3:new(KeyId, - UserConfig#aws_config.secret_access_key, - ?S3_HOST, - UserConfig#aws_config.s3_port, % inets issue precludes using ?S3_PORT - ?DEFAULT_PROTO, - ?PROXY_HOST, - UserConfig#aws_config.s3_port, - []), - aws_config(UpdConfig, Props); -aws_config(UserConfig, [{secret, Secret}|Props]) -> - UpdConfig = erlcloud_s3:new(UserConfig#aws_config.access_key_id, - Secret, - ?S3_HOST, - UserConfig#aws_config.s3_port, % inets issue precludes using ?S3_PORT - ?DEFAULT_PROTO, - ?PROXY_HOST, - UserConfig#aws_config.s3_port, - []), - aws_config(UpdConfig, Props). - -%% private - - -latest([], {_, Candidate}) -> - Candidate; -latest([Sample | Rest], undefined) -> - StartTime = rtcs:json_get([<<"StartTime">>], Sample), - latest(Rest, {StartTime, Sample}); -latest([Sample | Rest], {CandidateStartTime, Candidate}) -> - StartTime = rtcs:json_get([<<"StartTime">>], Sample), - NewCandidate = case StartTime < CandidateStartTime of - true -> {CandidateStartTime, Candidate}; - _ -> {StartTime, Sample} - end, - latest(Rest, NewCandidate). - -by_bucket_list([], Acc) -> - lists:sort(Acc); -by_bucket_list([{<<"StartTime">>, _} | Rest], Acc) -> - by_bucket_list(Rest, Acc); -by_bucket_list([{<<"EndTime">>, _} | Rest], Acc) -> - by_bucket_list(Rest, Acc); -by_bucket_list([{BucketBin, {struct,[{<<"Objects">>, Objs}, - {<<"Bytes">>, Bytes}]}} | Rest], - Acc) -> - by_bucket_list(Rest, [{binary_to_list(BucketBin), {Objs, Bytes}}|Acc]). - -samples_from_json_request(AdminConfig, UserConfig, {Begin, End}) -> - KeyId = UserConfig#aws_config.access_key_id, - StatsKey = string:join(["usage", KeyId, "bj", Begin, End], "/"), - GetResult = erlcloud_s3:get_object("riak-cs", StatsKey, AdminConfig), - Usage = mochijson2:decode(proplists:get_value(content, GetResult)), - rtcs:json_get([<<"Storage">>, <<"Samples">>], Usage). - diff --git a/riak_test/src/rtcs_config.erl b/riak_test/src/rtcs_config.erl deleted file mode 100644 index 7f5ac3fec..000000000 --- a/riak_test/src/rtcs_config.erl +++ /dev/null @@ -1,536 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- --module(rtcs_config). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). - --define(RIAK_CURRENT, <<"build_paths.current">>). --define(RIAK_PREVIOUS, <<"build_paths.previous">>). --define(EE_CURRENT, <<"build_paths.ee_current">>). --define(EE_PREVIOUS, <<"build_paths.ee_previous">>). --define(CS_CURRENT, <<"build_paths.cs_current">>). --define(CS_PREVIOUS, <<"build_paths.cs_previous">>). --define(STANCHION_CURRENT, <<"build_paths.stanchion_current">>). --define(STANCHION_PREVIOUS, <<"build_paths.stanchion_previous">>). - --define(S3_PORT, 80). - --define(REQUEST_POOL_SIZE, 8). --define(BUCKET_LIST_POOL_SIZE, 2). - -request_pool_size() -> - ?REQUEST_POOL_SIZE. - -bucket_list_pool_size() -> - ?BUCKET_LIST_POOL_SIZE. - -configs(CustomConfigs) -> - configs(CustomConfigs, current). - -configs(CustomConfigs, current) -> - merge(default_configs(), CustomConfigs); -configs(CustomConfigs, previous) -> - merge(previous_default_configs(), CustomConfigs). - -previous_configs() -> - previous_configs([]). - -previous_configs(CustomConfigs) -> - merge(previous_default_configs(), CustomConfigs). - -default_configs() -> - [{riak, riak_config()}, - {stanchion, stanchion_config()}, - {cs, cs_config()}]. - -previous_default_configs() -> - [{riak, previous_riak_config()}, - {stanchion, previous_stanchion_config()}, - {cs, previous_cs_config()}]. - -pb_port(N) when is_integer(N) -> - 10000 + (N * 10) + 7; -pb_port(Node) -> - pb_port(rtcs_dev:node_id(Node)). - -cs_port(N) when is_integer(N) -> - 15008 + 10 * N; -cs_port(Node) -> - cs_port(rtcs_dev:node_id(Node)). - -stanchion_port() -> 9095. - -riak_conf() -> - [{"ring_size", "8"}, - {"buckets.default.allow_mult", "true"}, - {"buckets.default.merge_strategy", "2"}]. - -riak_config(CustomConfig) -> - orddict:merge(fun(_, LHS, RHS) -> LHS ++ RHS end, - orddict:from_list(lists:sort(CustomConfig)), - orddict:from_list(lists:sort(riak_config()))). - -riak_config() -> - riak_config( - current, - ?CS_CURRENT, - rt_config:get(build_type, oss), - rt_config:get(backend, {multi_backend, bitcask})). - -riak_config(Vsn, CsVsn, oss, Backend) -> - riak_oss_config(Vsn, CsVsn, Backend); -riak_config(Vsn, CsVsn, ee, Backend) -> - riak_ee_config(Vsn, CsVsn, Backend). - -riak_oss_config(Vsn, CsVsn, Backend) -> - CSPath = rt_config:get(CsVsn), - AddPaths = filelib:wildcard(CSPath ++ "/dev/dev1/lib/riak_cs*/ebin"), - [ - lager_config(), - riak_core_config(Vsn), - {riak_api, - [{pb_backlog, 256}]}, - {riak_kv, - [{add_paths, AddPaths}] ++ - backend_config(CsVsn, Backend) - } - ]. - -riak_core_config(current) -> - {riak_core, []}; -riak_core_config(previous) -> - {riak_core, - [{default_bucket_props, [{allow_mult, true}]}, - {ring_creation_size, 8}] - }. - -backend_config(_CsVsn, memory) -> - [{storage_backend, riak_kv_memory_backend}]; -backend_config(_CsVsn, {multi_backend, BlocksBackend}) -> - [ - {storage_backend, riak_cs_kv_multi_backend}, - {multi_backend_prefix_list, [{<<"0b:">>, be_blocks}]}, - {multi_backend_default, be_default}, - {multi_backend, - [{be_default, riak_kv_eleveldb_backend, - [ - {max_open_files, 20}, - {data_root, "./data/leveldb"} - ]}, - blocks_backend_config(BlocksBackend) - ]} - ]; -backend_config(?CS_CURRENT, prefix_multi) -> - [ - {storage_backend, riak_kv_multi_prefix_backend}, - {riak_cs_version, 20000} - ]; -backend_config(OlderCsVsn, prefix_multi) -> - backend_config(OlderCsVsn, {multi_backend, bitcask}). - -blocks_backend_config(fs) -> - {be_blocks, riak_kv_fs2_backend, [{data_root, "./data/fs2"}, - {block_size, 1050000}]}; -blocks_backend_config(_) -> - {be_blocks, riak_kv_bitcask_backend, [{data_root, "./data/bitcask"}]}. - -riak_ee_config(Vsn, CsVsn, Backend) -> - [repl_config() | riak_oss_config(Vsn, CsVsn, Backend)]. - -repl_config() -> - {riak_repl, - [ - {fullsync_on_connect, false}, - {fullsync_interval, disabled}, - {proxy_get, enabled} - ]}. - -previous_riak_config() -> - riak_config( - previous, - ?CS_PREVIOUS, - rt_config:get(build_type, oss), - rt_config:get(backend, {multi_backend, bitcask})). - -previous_riak_config(CustomConfig) -> - orddict:merge(fun(_, LHS, RHS) -> LHS ++ RHS end, - orddict:from_list(lists:sort(CustomConfig)), - orddict:from_list(lists:sort(previous_riak_config()))). - -previous_cs_config() -> - previous_cs_config([], []). - -previous_cs_config(UserExtra) -> - previous_cs_config(UserExtra, []). - -previous_cs_config(UserExtra, OtherApps) -> - [ - lager_config(), - {riak_cs, - UserExtra ++ - [ - {connection_pools, - [ - {request_pool, {request_pool_size(), 0} }, - {bucket_list_pool, {bucket_list_pool_size(), 0} } - ]}, - {block_get_max_retries, 1}, - {proxy_get, enabled}, - {anonymous_user_creation, true}, - {riak_pb_port, 10017}, - {stanchion_port, stanchion_port()}, - {cs_version, 010300} - ] - }] ++ OtherApps. - -cs_config() -> - cs_config([], []). - -cs_config(UserExtra) -> - cs_config(UserExtra, []). - -cs_config(UserExtra, OtherApps) -> - [ - lager_config(), - {riak_cs, - UserExtra ++ - [ - {connection_pools, - [ - {request_pool, {request_pool_size(), 0} }, - {bucket_list_pool, {bucket_list_pool_size(), 0} } - ]}, - {block_get_max_retries, 1}, - {proxy_get, enabled}, - {anonymous_user_creation, true}, - {stanchion_host, {"127.0.0.1", stanchion_port()}}, - {riak_host, {"127.0.0.1", 10017}}, - {cs_version, 010300} - ] - }] ++ OtherApps. - -replace(Key, Value, Config0) -> - Config1 = proplists:delete(Key, Config0), - [proplists:property(Key, Value)|Config1]. - -previous_stanchion_config() -> - [ - lager_config(), - {stanchion, - [ - {stanchion_port, stanchion_port()}, - {riak_pb_port, 10017} - ] - }]. - -stanchion_config() -> - [ - lager_config(), - {stanchion, - [ - {host, {"127.0.0.1", stanchion_port()}}, - {riak_host, {"127.0.0.1", 10017}} - ] - }]. - -lager_config() -> - {lager, - [ - {handlers, - [ - {lager_file_backend, - [ - {"./log/error.log", error, 10485760, "$D0",5}, - {"./log/console.log", rt_config:get(console_log_level, debug), - 10485760, "$D0", 5} - ]} - ]} - ]}. - -riak_bitcaskroot(Prefix, N) -> - io_lib:format("~s/dev/dev~b/data/bitcask", [Prefix, N]). - -riak_binpath(Prefix, N) -> - io_lib:format("~s/dev/dev~b/bin/riak", [Prefix, N]). - -riakcmd(Path, N, Cmd) -> - lists:flatten(io_lib:format("~s ~s", [riak_binpath(Path, N), Cmd])). - -riakcs_home(Prefix, N) -> - io_lib:format("~s/dev/dev~b/", [Prefix, N]). - -riakcs_binpath(Prefix, N) -> - io_lib:format("~s/dev/dev~b/bin/riak-cs", [Prefix, N]). - -riakcs_etcpath(Prefix, N) -> - io_lib:format("~s/dev/dev~b/etc", [Prefix, N]). - -riakcs_libpath(Prefix, N) -> - io_lib:format("~s/dev/dev~b/lib", [Prefix, N]). - -riakcs_logpath(Prefix, N, File) -> - io_lib:format("~s/dev/dev~b/log/~s", [Prefix, N, File]). - -riakcscmd(Path, N, Cmd) -> - lists:flatten(io_lib:format("~s ~s", [riakcs_binpath(Path, N), Cmd])). - -riakcs_statuscmd(Path, N) -> - lists:flatten(io_lib:format("~s-admin status", [riakcs_binpath(Path, N)])). - -riakcs_switchcmd(Path, N, Cmd) -> - lists:flatten(io_lib:format("~s-admin stanchion ~s", [riakcs_binpath(Path, N), Cmd])). - -riakcs_gccmd(Path, N, Cmd) -> - lists:flatten(io_lib:format("~s-admin gc ~s", [riakcs_binpath(Path, N), Cmd])). - -riakcs_accesscmd(Path, N, Cmd) -> - lists:flatten(io_lib:format("~s-admin access ~s", [riakcs_binpath(Path, N), Cmd])). - -riakcs_storagecmd(Path, N, Cmd) -> - lists:flatten(io_lib:format("~s-admin storage ~s", [riakcs_binpath(Path, N), Cmd])). - -stanchion_binpath(Prefix) -> - io_lib:format("~s/dev/stanchion/bin/stanchion", [Prefix]). - -stanchion_etcpath(Prefix) -> - io_lib:format("~s/dev/stanchion/etc", [Prefix]). - -stanchioncmd(Path, Cmd) -> - lists:flatten(io_lib:format("~s ~s", [stanchion_binpath(Path), Cmd])). - -stanchion_statuscmd(Path) -> - lists:flatten(io_lib:format("~s-admin status", [stanchion_binpath(Path)])). - -cs_current() -> - ?CS_CURRENT. - -stanchion_current() -> - ?STANCHION_CURRENT. - -devpath(riak, current) -> - case rt_config:get(build_type, oss) of - oss -> rt_config:get(?RIAK_CURRENT); - ee -> rt_config:get(?EE_CURRENT) - end; -devpath(riak, previous) -> - case rt_config:get(build_type, oss) of - oss -> rt_config:get(?RIAK_PREVIOUS); - ee -> rt_config:get(?EE_PREVIOUS) - end; -devpath(cs, current) -> rt_config:get(?CS_CURRENT); -devpath(cs, previous) -> rt_config:get(?CS_PREVIOUS); -devpath(stanchion, current) -> rt_config:get(?STANCHION_CURRENT); -devpath(stanchion, previous) -> rt_config:get(?STANCHION_PREVIOUS). - -set_configs(NumNodes, Config, Vsn) -> - rtcs:set_conf({riak, Vsn}, riak_conf()), - rt:pmap(fun(N) -> - rtcs_dev:update_app_config(rtcs:riak_node(N), - proplists:get_value(riak, Config)), - update_cs_config(devpath(cs, Vsn), N, - proplists:get_value(cs, Config)) - end, - lists:seq(1, NumNodes)), - update_stanchion_config(devpath(stanchion, Vsn), - proplists:get_value(stanchion, Config)), - enable_zdbbl(Vsn). - -read_config(Vsn, N, Who) -> - Prefix = devpath(Who, Vsn), - EtcPath = case Who of - cs -> riakcs_etcpath(Prefix, N); - stanchion -> stanchion_etcpath(Prefix) - end, - case file:consult(EtcPath ++ "/advanced.config") of - {ok, [Config]} -> - Config; - {error, enoent}-> - {ok, [Config]} = file:consult(EtcPath ++ "/app.config"), - Config - end. - -update_cs_config(Prefix, N, Config, {AdminKey, _AdminSecret}) -> - CSSection = proplists:get_value(riak_cs, Config), - UpdConfig = [{riak_cs, update_admin_creds(CSSection, AdminKey)} | - proplists:delete(riak_cs, Config)], - update_cs_config(Prefix, N, UpdConfig). - -update_cs_config(Prefix, N, Config) -> - CSSection = proplists:get_value(riak_cs, Config), - UpdConfig = [{riak_cs, update_cs_port(CSSection, N)} | - proplists:delete(riak_cs, Config)], - update_app_config(riakcs_etcpath(Prefix, N), UpdConfig). - -update_admin_creds(Config, AdminKey) -> - [{admin_key, AdminKey}| - proplists:delete(admin_key, Config)]. - -update_cs_port(Config, N) -> - Config2 = [{riak_host, {"127.0.0.1", pb_port(N)}} | proplists:delete(riak_host, Config)], - [{listener, {"127.0.0.1", cs_port(N)}} | proplists:delete(listener, Config2)]. - -update_stanchion_config(Prefix, Config, {AdminKey, _AdminSecret}) -> - StanchionSection = proplists:get_value(stanchion, Config), - UpdConfig = [{stanchion, update_admin_creds(StanchionSection, AdminKey)} | - proplists:delete(stanchion, Config)], - update_stanchion_config(Prefix, UpdConfig). - -update_stanchion_config(Prefix, Config) -> - update_app_config(stanchion_etcpath(Prefix), Config). - -update_app_config(Path, Config) -> - lager:debug("rtcs:update_app_config(~s,~p)", [Path, Config]), - FileFormatString = "~s/~s.config", - AppConfigFile = io_lib:format(FileFormatString, [Path, "app"]), - AdvConfigFile = io_lib:format(FileFormatString, [Path, "advanced"]), - %% If there's an app.config, do it old style - %% if not, use cuttlefish's adavnced.config - case filelib:is_file(AppConfigFile) of - true -> - rtcs_dev:update_app_config_file(AppConfigFile, Config); - _ -> - rtcs_dev:update_app_config_file(AdvConfigFile, Config) - end. - -enable_zdbbl(Vsn) -> - Fs = filelib:wildcard(filename:join([devpath(riak, Vsn), - "dev", "dev*", "etc", "vm.args"])), - lager:info("rtcs:enable_zdbbl for vm.args : ~p~n", [Fs]), - [os:cmd("sed -i -e 's/##+zdbbl /+zdbbl /g' " ++ F) || F <- Fs], - ok. - -merge(BaseConfig, undefined) -> - BaseConfig; -merge(BaseConfig, Config) -> - lager:debug("Merging Config: BaseConfig=~p", [BaseConfig]), - lager:debug("Merging Config: Config=~p", [Config]), - MergeA = orddict:from_list(Config), - MergeB = orddict:from_list(BaseConfig), - MergedConfig = orddict:merge(fun internal_merge/3, MergeA, MergeB), - lager:debug("Merged config: ~p", [MergedConfig]), - MergedConfig. - -internal_merge(_Key, [{_, _}|_] = VarsA, [{_, _}|_] = VarsB) -> - MergeC = orddict:from_list(VarsA), - MergeD = orddict:from_list(VarsB), - orddict:merge(fun internal_merge/3, MergeC, MergeD); -internal_merge(_Key, VarsA, _VarsB) -> - VarsA. - -%% @doc update current app.config, assuming CS is already stopped -upgrade_cs(N, AdminCreds) -> - migrate_cs(previous, current, N, AdminCreds). - -%% @doc update config file from `From' to `To' version. -migrate_cs(From, To, N, AdminCreds) -> - migrate(From, To, N, AdminCreds, cs). - -migrate(From, To, N, AdminCreds, Who) when - (From =:= current andalso To =:= previous) - orelse ( From =:= previous andalso To =:= current) -> - Config0 = read_config(From, N, Who), - Config1 = migrate_config(From, To, Config0, Who), - Prefix = devpath(Who, To), - lager:debug("migrating ~s => ~s", [devpath(Who, From), Prefix]), - case Who of - cs -> update_cs_config(Prefix, N, Config1, AdminCreds); - stanchion -> update_stanchion_config(Prefix, Config1, AdminCreds) - end. - -migrate_stanchion(From, To, AdminCreds) -> - migrate(From, To, -1, AdminCreds, stanchion). - -migrate_config(previous, current, Conf, stanchion) -> - {AddList, RemoveList} = diff_config(stanchion_config(), - previous_stanchion_config()), - migrate_config(Conf, AddList, RemoveList); -migrate_config(current, previous, Conf, stanchion) -> - {AddList, RemoveList} = diff_config(previous_stanchion_config(), - stanchion_config()), - migrate_config(Conf, AddList, RemoveList); -migrate_config(previous, current, Conf, cs) -> - {AddList, RemoveList} = diff_config(cs_config([{anonymous_user_creation, false}]), - previous_cs_config()), - migrate_config(Conf, AddList, RemoveList); -migrate_config(current, previous, Conf, cs) -> - {AddList, RemoveList} = diff_config(previous_cs_config(), cs_config()), - migrate_config(Conf, AddList, RemoveList). - -migrate_config(Conf0, AddList, RemoveList) -> - RemoveFun = fun(Key, Config) -> - InnerConf0 = proplists:get_value(Key, Config), - InnerRemoveList = proplists:get_value(Key, RemoveList), - InnerConf1 = lists:foldl(fun proplists:delete/2, - InnerConf0, - proplists:get_keys(InnerRemoveList)), - replace(Key, InnerConf1, Config) - end, - Conf1 = lists:foldl(RemoveFun, Conf0, proplists:get_keys(RemoveList)), - - AddFun = fun(Key, Config) -> - InnerConf = proplists:get_value(Key, Config) - ++ proplists:get_value(Key, AddList), - replace(Key, InnerConf, Config) - end, - lists:foldl(AddFun, Conf1, proplists:get_keys(AddList)). - -diff_config(Conf, BaseConf)-> - Keys = lists:umerge(proplists:get_keys(Conf), - proplists:get_keys(BaseConf)), - - Fun = fun(Key, {AddList, RemoveList}) -> - {Add, Remove} = diff_props(proplists:get_value(Key,Conf), - proplists:get_value(Key, BaseConf)), - case {Add, Remove} of - {[], []} -> - {AddList, RemoveList}; - {{}, Remove} -> - {AddList, RemoveList++[{Key, Remove}]}; - {Add, []} -> - {AddList++[{Key, Add}], RemoveList}; - {Add, Remove} -> - {AddList++[{Key, Add}], RemoveList++[{Key, Remove}]} - end - end, - lists:foldl(Fun, {[], []}, Keys). - -diff_props(undefined, BaseProps) -> - {[], BaseProps}; -diff_props(Props, undefined) -> - {Props, []}; -diff_props(Props, BaseProps) -> - Keys = lists:umerge(proplists:get_keys(Props), - proplists:get_keys(BaseProps)), - Fun = fun(Key, {Add, Remove}) -> - Values = {proplists:get_value(Key, Props), - proplists:get_value(Key, BaseProps)}, - case Values of - {undefined, V2} -> - {Add, Remove++[{Key, V2}]}; - {V1, undefined} -> - {Add++[{Key, V1}], Remove}; - {V, V} -> - {Add, Remove}; - {V1, V2} -> - {Add++[{Key, V1}], Remove++[{Key, V2}]} - end - end, - lists:foldl(Fun, {[], []}, Keys). - diff --git a/riak_test/src/rtcs_dev.erl b/riak_test/src/rtcs_dev.erl deleted file mode 100644 index aa4c5c3e9..000000000 --- a/riak_test/src/rtcs_dev.erl +++ /dev/null @@ -1,519 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2013 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% @private --module(rtcs_dev). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). - --define(BUILD_PATHS, (rt_config:get(build_paths))). --define(SRC_PATHS, (rt_config:get(src_paths))). - --define(RIAK_ROOT, <<"build_paths.root">>). --define(EE_ROOT, <<"build_paths.ee_root">>). --define(CS_ROOT, <<"build_paths.cs_root">>). --define(STANCHION_ROOT, <<"build_paths.stanchion_root">>). - -get_deps() -> - lists:flatten(io_lib:format("~s/dev/dev1/lib", [relpath(current)])). - -setup_harness(_Test, _Args) -> - confirm_build_type(rt_config:get(build_type, oss)), - %% Stop all discoverable nodes, not just nodes we'll be using for this test. - rt:pmap(fun(X) -> stop_all(X ++ "/dev") end, devpaths()), - - %% Reset nodes to base state - reset_cluster(), - - lager:info("Cleaning up lingering pipe directories"), - rt:pmap(fun(Dir) -> - %% when joining two absolute paths, filename:join intentionally - %% throws away the first one. ++ gets us around that, while - %% keeping some of the security of filename:join. - %% the extra slashes will be pruned by filename:join, but this - %% ensures that there will be at least one between "/tmp" and Dir - PipeDir = filename:join(["/tmp//" ++ Dir, "dev"]), - %% when using filelib:wildcard/2, there must be a wildchar char - %% before the first '/'. - Files = filelib:wildcard("dev?/*.{r,w}", PipeDir), - [ file:delete(filename:join(PipeDir, File)) || File <- Files], - file:del_dir(PipeDir) - end, devpaths()), - ok. - -confirm_build_type(BuildType) -> - [ok = confirm_build_type(BuildType, Vsn) || Vsn <- [cs_current, cs_previous]]. - -confirm_build_type(BuildType, Vsn) -> - ReplPB = filename:join([relpath(Vsn), "dev/dev1/lib/riak_repl_pb_api*"]), - case {BuildType, filelib:wildcard(ReplPB)} of - {oss, []} -> ok; - {ee, [_|_]} -> ok; - _ -> - lager:error("Build type of ~p is not ~p", - [Vsn, BuildType]), - {error, {build_type_mismatch, Vsn}} - end. - -relpath(Vsn) -> - Path = ?BUILD_PATHS, - path(Vsn, Path). - -srcpath(Vsn) -> - Path = ?SRC_PATHS, - path(Vsn, Path). - -path(Vsn, Paths=[{_,_}|_]) -> - orddict:fetch(Vsn, orddict:from_list(Paths)); -path(current, Path) -> - Path; -path(root, Path) -> - Path; -path(_, _) -> - throw("Version requested but only one path provided"). - -upgrade(Node, NewVersion) -> - N = node_id(Node), - Version = node_version(N), - lager:info("Upgrading ~p : ~p -> ~p", [Node, Version, NewVersion]), - catch stop(Node), - rt:wait_until_unpingable(Node), - OldPath = relpath(Version), - NewPath = relpath(NewVersion), - - Commands = [ - io_lib:format("cp -p -P -R \"~s/dev/dev~b/data\" \"~s/dev/dev~b\"", - [OldPath, N, NewPath, N]), - io_lib:format("rm -rf ~s/dev/dev~b/data/*", - [OldPath, N]), - io_lib:format("cp -p -P -R \"~s/dev/dev~b/etc\" \"~s/dev/dev~b\"", - [OldPath, N, NewPath, N]) - ], - [ begin - lager:info("Running: ~s", [Cmd]), - os:cmd(Cmd) - end || Cmd <- Commands], - VersionMap = orddict:store(N, NewVersion, rt_config:get(rt_versions)), - rt_config:set(rt_versions, VersionMap), - start(Node), - rt:wait_until_pingable(Node), - ok. - --spec riak_root_and_vsn(atom(), atom()) -> {binary(), atom()}. -riak_root_and_vsn(current, oss) -> {?RIAK_ROOT, current}; -riak_root_and_vsn(current, ee) -> {?EE_ROOT, ee_current}; -riak_root_and_vsn(previous, oss) -> {?RIAK_ROOT, previous}; -riak_root_and_vsn(previous, ee) -> {?EE_ROOT, ee_previous}. - --spec reset_cluster() -> ok. -reset_cluster() -> - [reset_nodes(Project, Path) || - {Project, Path} <- [{riak, rt_config:get(?RIAK_ROOT)}, - {riak_ee, rt_config:get(?EE_ROOT)}, - {riak_cs, rt_config:get(?CS_ROOT)}, - {stanchion, rt_config:get(?STANCHION_ROOT)}]], - ok. - --spec reset_nodes(atom(), string()) -> ok. -reset_nodes(Project, Path) -> - %% Reset nodes to base state - lager:info("Resetting ~p nodes to fresh state", [Project]), - lager:debug("Project path for reset: ~p", [Path]), - rtdev:run_git(Path, "reset HEAD --hard"), - rtdev:run_git(Path, "clean -fd"), - ok. - -get_conf(Node) -> - N = node_id(Node), - Path = relpath(node_version(N)), - get_conf(Path, N). - -get_conf(DevPath, N) -> - WildCard = io_lib:format("~s/dev/dev~b/etc/*.conf", [DevPath, N]), - [Conf] = filelib:wildcard(WildCard), - Conf. - -get_app_config(Node) -> - N = node_id(Node), - Path = relpath(node_version(N)), - get_conf(Path, N). - -get_app_config(DevPath, N) -> - WildCard = io_lib:format("~s/dev/dev~b/etc/a*.config", [DevPath, N]), - [Conf] = filelib:wildcard(WildCard), - Conf. - -all_the_app_configs(DevPath) -> - lager:error("The dev path is ~p", [DevPath]), - case filelib:is_dir(DevPath) of - true -> - Devs = filelib:wildcard(DevPath ++ "/dev/dev*"), - [ Dev ++ "/etc/app.config" || Dev <- Devs]; - _ -> - lager:debug("~s is not a directory.", [DevPath]), - [] - end. - -update_app_config(all, Config) -> - lager:info("rtdev:update_app_config(all, ~p)", [Config]), - [ update_app_config(DevPath, Config) || DevPath <- devpaths()]; -update_app_config(Node, Config) when is_atom(Node) -> - N = node_id(Node), - Path = relpath(node_version(N)), - FileFormatString = "~s/dev/dev~b/etc/~s.config", - - AppConfigFile = io_lib:format(FileFormatString, [Path, N, "app"]), - AdvConfigFile = io_lib:format(FileFormatString, [Path, N, "advanced"]), - %% If there's an app.config, do it old style - %% if not, use cuttlefish's adavnced.config - case filelib:is_file(AppConfigFile) of - true -> - update_app_config_file(AppConfigFile, Config); - _ -> - update_app_config_file(AdvConfigFile, Config) - end; -update_app_config(DevPath, Config) -> - [update_app_config_file(AppConfig, Config) || AppConfig <- all_the_app_configs(DevPath)]. - -update_app_config_file(ConfigFile, Config) -> - lager:info("rtdev:update_app_config_file(~s, ~p)", [ConfigFile, Config]), - - BaseConfig = case file:consult(ConfigFile) of - {ok, [ValidConfig]} -> - ValidConfig; - {error, enoent} -> - [] - end, - MergeA = orddict:from_list(Config), - MergeB = orddict:from_list(BaseConfig), - NewConfig = - orddict:merge(fun(_, VarsA, VarsB) -> - MergeC = orddict:from_list(VarsA), - MergeD = orddict:from_list(VarsB), - orddict:merge(fun(_, ValA, _ValB) -> - ValA - end, MergeC, MergeD) - end, MergeA, MergeB), - NewConfigOut = io_lib:format("~p.", [NewConfig]), - ?assertEqual(ok, file:write_file(ConfigFile, NewConfigOut)), - ok. - -%% Appropriate backend will be set by rtcs later. -get_backends() -> - cs_multi_backend. - -node_path(Node) -> - N = node_id(Node), - Path = relpath(node_version(N)), - lists:flatten(io_lib:format("~s/dev/dev~b", [Path, N])). - -create_dirs(Nodes) -> - Snmp = [node_path(Node) ++ "/data/snmp/agent/db" || Node <- Nodes], - [?assertCmd("mkdir -p " ++ Dir) || Dir <- Snmp]. - -clean_data_dir(Nodes, SubDir) when is_list(Nodes) -> - DataDirs = [node_path(Node) ++ "/data/" ++ SubDir || Node <- Nodes], - lists:foreach(fun rm_dir/1, DataDirs). - -rm_dir(Dir) -> - lager:info("Removing directory ~s", [Dir]), - ?assertCmd("rm -rf " ++ Dir), - ?assertEqual(false, filelib:is_dir(Dir)). - -add_default_node_config(Nodes) -> - case rt_config:get(rt_default_config, undefined) of - undefined -> ok; - Defaults when is_list(Defaults) -> - rt:pmap(fun(Node) -> - update_app_config(Node, Defaults) - end, Nodes), - ok; - BadValue -> - lager:error("Invalid value for rt_default_config : ~p", [BadValue]), - throw({invalid_config, {rt_default_config, BadValue}}) - end. - -stop_all(DevPath) -> - case filelib:is_dir(DevPath) of - true -> - Devs = filelib:wildcard(DevPath ++ "/{dev,stanchion}*"), - - %% Works, but I'd like it to brag a little more about it. - Stop = fun(C) -> - Cmd = stop_command(C), - [Output | _Tail] = string:tokens(os:cmd(Cmd), "\n"), - Status = case Output of - "ok" -> "ok"; - _ -> "wasn't running" - end, - lager:info("Stopping Node... ~s ~~ ~s.", [Cmd, Status]) - end, - [Stop(D) || D <- Devs]; - _ -> lager:info("~s is not a directory.", [DevPath]) - end, - ok. - -stop_command(C) -> - IsRiakCS = string:str(C, "riak_cs"), - IsStanchion = string:str(C, "stanchion"), - if - IsRiakCS > 0 -> - C ++ "/bin/riak-cs stop"; - IsStanchion > 0 -> - C ++ "/bin/stanchion stop"; - true -> - C ++ "/bin/riak stop" - end. - -stop(Node) -> - RiakPid = rpc:call(Node, os, getpid, []), - N = node_id(Node), - rtdev:run_riak(N, relpath(node_version(N)), "stop"), - F = fun(_N) -> - os:cmd("kill -0 " ++ RiakPid) =/= [] - end, - ?assertEqual(ok, rt:wait_until(Node, F)), - ok. - -start(Node) -> - N = node_id(Node), - rtdev:run_riak(N, relpath(node_version(N)), "start"), - ok. - -attach(Node, Expected) -> - interactive(Node, "attach", Expected). - -attach_direct(Node, Expected) -> - interactive(Node, "attach-direct", Expected). - -console(Node, Expected) -> - interactive(Node, "console", Expected). - -interactive(Node, Command, Exp) -> - N = node_id(Node), - Path = relpath(node_version(N)), - Cmd = rtdev:riakcmd(Path, N, Command), - lager:info("Opening a port for riak ~s.", [Command]), - lager:debug("Calling open_port with cmd ~s", [binary_to_list(iolist_to_binary(Cmd))]), - P = open_port({spawn, binary_to_list(iolist_to_binary(Cmd))}, - [stream, use_stdio, exit_status, binary, stderr_to_stdout]), - interactive_loop(P, Exp). - -interactive_loop(Port, Expected) -> - receive - {Port, {data, Data}} -> - %% We've gotten some data, so the port isn't done executing - %% Let's break it up by newline and display it. - Tokens = string:tokens(binary_to_list(Data), "\n"), - [lager:debug("~s", [Text]) || Text <- Tokens], - - %% Now we're going to take hd(Expected) which is either {expect, X} - %% or {send, X}. If it's {expect, X}, we foldl through the Tokenized - %% data looking for a partial match via rt:str/2. If we find one, - %% we pop hd off the stack and continue iterating through the list - %% with the next hd until we run out of input. Once hd is a tuple - %% {send, X}, we send that test to the port. The assumption is that - %% once we send data, anything else we still have in the buffer is - %% meaningless, so we skip it. That's what that {sent, sent} thing - %% is about. If there were a way to abort mid-foldl, I'd have done - %% that. {sent, _} -> is just a pass through to get out of the fold. - - NewExpected = lists:foldl(fun(X, Expect) -> - [{Type, Text}|RemainingExpect] = case Expect of - [] -> [{done, "done"}|[]]; - E -> E - end, - case {Type, rt:str(X, Text)} of - {expect, true} -> - RemainingExpect; - {expect, false} -> - [{Type, Text}|RemainingExpect]; - {send, _} -> - port_command(Port, list_to_binary(Text ++ "\n")), - [{sent, "sent"}|RemainingExpect]; - {sent, _} -> - Expect; - {done, _} -> - [] - end - end, Expected, Tokens), - %% Now that the fold is over, we should remove {sent, sent} if it's there. - %% The fold might have ended not matching anything or not sending anything - %% so it's possible we don't have to remove {sent, sent}. This will be passed - %% to interactive_loop's next iteration. - NewerExpected = case NewExpected of - [{sent, "sent"}|E] -> E; - E -> E - end, - %% If NewerExpected is empty, we've met all expected criteria and in order to boot - %% Otherwise, loop. - case NewerExpected of - [] -> ?assert(true); - _ -> interactive_loop(Port, NewerExpected) - end; - {Port, {exit_status,_}} -> - %% This port has exited. Maybe the last thing we did was {send, [4]} which - %% as Ctrl-D would have exited the console. If Expected is empty, then - %% We've met every expectation. Yay! If not, it means we've exited before - %% something expected happened. - ?assertEqual([], Expected) - after rt_config:get(rt_max_wait_time) -> - %% interactive_loop is going to wait until it matches expected behavior - %% If it doesn't, the test should fail; however, without a timeout it - %% will just hang forever in search of expected behavior. See also: Parenting - ?assertEqual([], Expected) - end. - -admin(Node, Args, Options) -> - N = node_id(Node), - Path = relpath(node_version(N)), - Cmd = rtdev:riak_admin_cmd(Path, N, Args), - lager:info("Running: ~s", [Cmd]), - Result = execute_admin_cmd(Cmd, Options), - lager:info("~s", [Result]), - {ok, Result}. - -execute_admin_cmd(Cmd, Options) -> - {_ExitCode, Result} = FullResult = wait_for_cmd(spawn_cmd(Cmd)), - case lists:member(return_exit_code, Options) of - true -> - FullResult; - false -> - Result - end. - -riak(Node, Args) -> - N = node_id(Node), - Path = relpath(node_version(N)), - Result = rtdev:run_riak(N, Path, Args), - lager:info("~s", [Result]), - {ok, Result}. - -node_id(Node) -> - NodeMap = rt_config:get(rt_nodes), - orddict:fetch(Node, NodeMap). - -node_version(N) -> - VersionMap = rt_config:get(rt_versions), - orddict:fetch(N, VersionMap). - -spawn_cmd(Cmd) -> - spawn_cmd(Cmd, []). -spawn_cmd(Cmd, Opts) -> - Port = open_port({spawn, Cmd}, [stream, in, exit_status, stderr_to_stdout] ++ Opts), - Port. - -wait_for_cmd(Port) -> - rt:wait_until(node(), - fun(_) -> - receive - {Port, Msg={data, _}} -> - self() ! {Port, Msg}, - false; - {Port, Msg={exit_status, _}} -> - catch port_close(Port), - self() ! {Port, Msg}, - true - after 0 -> - false - end - end), - get_cmd_result(Port, []). - -cmd(Cmd) -> - cmd(Cmd, []). - -cmd(Cmd, Opts) -> - wait_for_cmd(spawn_cmd(Cmd, Opts)). - -get_cmd_result(Port, Acc) -> - receive - {Port, {data, Bytes}} -> - get_cmd_result(Port, [Bytes|Acc]); - {Port, {exit_status, Status}} -> - Output = lists:flatten(lists:reverse(Acc)), - {Status, Output} - after 0 -> - timeout - end. - -check_node({_N, Version}) -> - case proplists:is_defined(Version, rt_config:get(build_paths)) of - true -> ok; - _ -> - lager:error("You don't have Riak ~s installed or configured", [Version]), - erlang:error("You don't have Riak " ++ atom_to_list(Version) ++ " installed or configured") - end. - -set_backend(Backend) -> - lager:info("rtdev:set_backend(~p)", [Backend]), - update_app_config(all, [{riak_kv, [{storage_backend, Backend}]}]), - get_backends(). - -get_version() -> - case file:read_file(relpath(cs_current) ++ "/VERSION") of - {error, enoent} -> unknown; - {ok, Version} -> Version - end. - -teardown() -> - %% Stop all discoverable nodes, not just nodes we'll be using for this test. - rt:pmap(fun(X) -> stop_all(X ++ "/dev") end, - devpaths()). - -whats_up() -> - io:format("Here's what's running...~n"), - - Up = [rpc:call(Node, os, cmd, ["pwd"]) || Node <- nodes()], - [io:format(" ~s~n",[string:substr(Dir, 1, length(Dir)-1)]) || Dir <- Up]. - -devpaths() -> - lists:usort([ DevPath || {Name, DevPath} <- rt_config:get(build_paths), - not lists:member(Name, [root, ee_root, cs_root, stanchion_root]) - ]). - -all_the_files(DevPath, File) -> - case filelib:is_dir(DevPath) of - true -> - Wildcard = io_lib:format("~s/dev/*/~s", [DevPath, File]), - filelib:wildcard(Wildcard); - _ -> - lager:debug("~s is not a directory.", [DevPath]), - [] - end. - -devpath(Name, Vsn) -> - rtcs_config:devpath(Name, Vsn). - -versions() -> - proplists:get_keys(rt_config:get(build_paths)) -- [root]. - -get_node_logs() -> - lists:flatmap(fun get_node_logs/1, [root, ee_root, cs_root, stanchion_root]). - -get_node_logs(Base) -> - Root = filename:absname(proplists:get_value(Base, ?BUILD_PATHS)), - %% Unlike Riak, Riak CS has multiple things in the root and so we need - %% to distinguish between them. - RootLen = length(filename:dirname(Root)) + 1, %% Remove the leading slash - [ begin - {ok, Port} = file:open(Filename, [read, binary]), - {lists:nthtail(RootLen, Filename), Port} - end || Filename <- filelib:wildcard(Root ++ "/*/dev/dev*/log/*") ]. diff --git a/riak_test/src/rtcs_exec.erl b/riak_test/src/rtcs_exec.erl deleted file mode 100644 index b0467a407..000000000 --- a/riak_test/src/rtcs_exec.erl +++ /dev/null @@ -1,277 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- --module(rtcs_exec). --compile(export_all). - -start_cs_and_stanchion_nodes(NodeList, Vsn) -> - rt:pmap(fun({_CSNode, RiakNode, _Stanchion}) -> - N = rtcs_dev:node_id(RiakNode), - start_stanchion(Vsn), - start_cs(N, Vsn); - ({_CSNode, RiakNode}) -> - N = rtcs_dev:node_id(RiakNode), - start_cs(N, Vsn) - end, NodeList). - -stop_cs_and_stanchion_nodes(NodeList, Vsn) -> - rt:pmap(fun({CSNode, RiakNode, Stanchion}) -> - N = rtcs_dev:node_id(RiakNode), - stop_cs(N, Vsn), - stop_stanchion(Vsn), - rt:wait_until_unpingable(CSNode), - rt:wait_until_unpingable(Stanchion); - ({CSNode, RiakNode}) -> - N = rtcs_dev:node_id(RiakNode), - stop_cs(N, Vsn), - rt:wait_until_unpingable(CSNode) - end, NodeList). - -start_all_nodes(NodeList, Vsn) -> - rt:pmap(fun({_CSNode, RiakNode, _Stanchion}) -> - N = rtcs_dev:node_id(RiakNode), - NodeVersion = rtcs_dev:node_version(N), - lager:debug("starting riak #~p > ~p => ~p", - [N, NodeVersion, - rtcs_dev:relpath(NodeVersion)]), - rtdev:run_riak(N, rtcs_dev:relpath(NodeVersion), "start"), - rt:wait_for_service(RiakNode, riak_kv), - spawn(fun() -> start_stanchion(Vsn) end), - spawn(fun() -> start_cs(N, Vsn) end); - ({_CSNode, RiakNode}) -> - N = rtcs_dev:node_id(RiakNode), - rtdev:run_riak(N, rtcs_dev:relpath(rtcs_dev:node_version(N)), "start"), - rt:wait_for_service(RiakNode, riak_kv), - spawn(fun() -> start_cs(N, Vsn) end) - end, NodeList). - -stop_all_nodes(NodeList, Vsn) -> - rt:pmap(fun({CSNode, RiakNode, Stanchion}) -> - N = rtcs_dev:node_id(RiakNode), - stop_cs(N, Vsn), - stop_stanchion(Vsn), - rtdev:run_riak(N, rtcs_dev:relpath(rtcs_dev:node_version(N)), "stop"), - rt:wait_until_unpingable(CSNode), - rt:wait_until_unpingable(Stanchion), - rt:wait_until_unpingable(RiakNode); - ({CSNode, RiakNode}) -> - N = rtcs_dev:node_id(RiakNode), - stop_cs(N, Vsn), - rtdev:run_riak(N, rtcs_dev:relpath(rtcs_dev:node_version(N)), "stop"), - rt:wait_until_unpingable(CSNode), - rt:wait_until_unpingable(RiakNode) - end, NodeList). - -start_cs(N) -> start_cs(N, current). - -start_cs(N, Vsn) -> - NodePath = rtcs_config:devpath(cs, Vsn), - Cmd = riakcscmd(NodePath, N, "start"), - lager:info("Running ~p", [Cmd]), - R = os:cmd(Cmd), - rtcs:maybe_load_intercepts(rtcs:cs_node(N)), - R. - -stop_cs(N) -> stop_cs(N, current). - -stop_cs(N, Vsn) -> - Cmd = riakcscmd(rtcs_config:devpath(cs, Vsn), N, "stop"), - lager:info("Running ~p", [Cmd]), - os:cmd(Cmd). - -riakcmd(Path, N, Cmd) -> - lists:flatten(io_lib:format("~s ~s", [riak_binpath(Path, N), Cmd])). - -riakcscmd(Path, N, Cmd) -> - lists:flatten(io_lib:format("~s ~s", [riakcs_binpath(Path, N), Cmd])). - -riakcs_statuscmd(Path, N) -> - lists:flatten(io_lib:format("~s-admin status", [riakcs_binpath(Path, N)])). - -riakcs_switchcmd(Path, N, Cmd) -> - lists:flatten(io_lib:format("~s-admin stanchion ~s", [riakcs_binpath(Path, N), Cmd])). - -riakcs_gccmd(Path, N, Cmd) -> - lists:flatten(io_lib:format("~s-admin gc ~s", [riakcs_binpath(Path, N), Cmd])). - -riakcs_accesscmd(Path, N, Cmd) -> - lists:flatten(io_lib:format("~s-admin access ~s", [riakcs_binpath(Path, N), Cmd])). - -riakcs_storagecmd(Path, N, Cmd) -> - lists:flatten(io_lib:format("~s-admin storage ~s", [riakcs_binpath(Path, N), Cmd])). - -riakcs_debugcmd(Path, N, Cmd) -> - lists:flatten(io_lib:format("~s-debug ~s", [riakcs_binpath(Path, N), Cmd])). - -stanchioncmd(Path, Cmd) -> - lists:flatten(io_lib:format("~s ~s", [stanchion_binpath(Path), Cmd])). - -stanchion_statuscmd(Path) -> - lists:flatten(io_lib:format("~s-admin status", [stanchion_binpath(Path)])). - -riak_bitcaskroot(Prefix, N) -> - io_lib:format("~s/dev/dev~b/data/bitcask", [Prefix, N]). - -riak_binpath(Prefix, N) -> - io_lib:format("~s/dev/dev~b/bin/riak", [Prefix, N]). - -riakcs_home(Prefix, N) -> - io_lib:format("~s/dev/dev~b/", [Prefix, N]). - -riakcs_binpath(Prefix, N) -> - io_lib:format("~s/dev/dev~b/bin/riak-cs", [Prefix, N]). - -riakcs_etcpath(Prefix, N) -> - io_lib:format("~s/dev/dev~b/etc", [Prefix, N]). - -riakcs_libpath(Prefix, N) -> - io_lib:format("~s/dev/dev~b/lib", [Prefix, N]). - -riakcs_logpath(Prefix, N, File) -> - io_lib:format("~s/dev/dev~b/log/~s", [Prefix, N, File]). - -stanchion_binpath(Prefix) -> - io_lib:format("~s/dev/stanchion/bin/stanchion", [Prefix]). - -stanchion_etcpath(Prefix) -> - io_lib:format("~s/dev/stanchion/etc", [Prefix]). - -repair_gc_bucket(N, Options) -> repair_gc_bucket(N, Options, current). - -repair_gc_bucket(N, Options, Vsn) -> - Prefix = rtcs_config:devpath(cs, Vsn), - RepairScriptWild = string:join([riakcs_libpath(Prefix, N), "riak_cs*", - "priv/tools/repair_gc_bucket.erl"] , "/"), - [RepairScript] = filelib:wildcard(RepairScriptWild), - Cmd = riakcscmd(Prefix, N, "escript " ++ RepairScript ++ - " " ++ Options), - lager:info("Running ~p", [Cmd]), - os:cmd(Cmd). - -exec_priv_escript(N, Command, Options) -> - exec_priv_escript(N, Command, Options, cs). - -exec_priv_escript(N, Command, Options, ByWhom) -> - CsPrefix = rtcs_config:devpath(cs, current), - ExecuterPrefix = rtcs_config:devpath(ByWhom, current), - ScriptWild = string:join([riakcs_libpath(CsPrefix, N), "riak_cs*", - "priv/tools/"] , "/"), - [ToolsDir] = filelib:wildcard(ScriptWild), - Cmd = case ByWhom of - cs -> - riakcscmd(ExecuterPrefix, N, "escript " ++ ToolsDir ++ - "/" ++ Command ++ - " " ++ Options); - riak -> - riakcmd(ExecuterPrefix, N, "escript " ++ ToolsDir ++ - "/" ++ Command ++ - " " ++ Options) - end, - lager:info("Running ~p", [Cmd]), - os:cmd(Cmd). - -switch_stanchion_cs(N, Host, Port) -> switch_stanchion_cs(N, Host, Port, current). - -switch_stanchion_cs(N, Host, Port, Vsn) -> - SubCmd = io_lib:format("switch ~s ~p", [Host, Port]), - Cmd = riakcs_switchcmd(rtcs_config:devpath(cs, Vsn), N, SubCmd), - lager:info("Running ~p", [Cmd]), - os:cmd(Cmd). - -show_stanchion_cs(N) -> show_stanchion_cs(N, current). - -show_stanchion_cs(N, Vsn) -> - Cmd = riakcs_switchcmd(rtcs_config:devpath(cs, Vsn), N, "show"), - lager:info("Running ~p", [Cmd]), - os:cmd(Cmd). - -start_stanchion() -> start_stanchion(current). - -start_stanchion(Vsn) -> - Cmd = stanchioncmd(rtcs_config:devpath(stanchion, Vsn), "start"), - lager:info("Running ~p", [Cmd]), - R = os:cmd(Cmd), - rtcs:maybe_load_intercepts(rtcs:stanchion_node()), - R. - -stop_stanchion() -> stop_stanchion(current). - -stop_stanchion(Vsn) -> - Cmd = stanchioncmd(rtcs_config:devpath(stanchion, Vsn), "stop"), - lager:info("Running ~p", [Cmd]), - os:cmd(Cmd). - -flush_access(N) -> flush_access(N, current). - -flush_access(N, Vsn) -> - Cmd = riakcs_accesscmd(rtcs_config:devpath(cs, Vsn), N, "flush"), - lager:info("Running ~p", [Cmd]), - os:cmd(Cmd). - -gc(N, SubCmd) -> gc(N, SubCmd, current). - -gc(N, SubCmd, Vsn) -> - Cmd = riakcs_gccmd(rtcs_config:devpath(cs, Vsn), N, SubCmd), - lager:info("Running ~p", [Cmd]), - os:cmd(Cmd). - -calculate_storage(N) -> calculate_storage(N, current). - -calculate_storage(N, Vsn) -> - Cmd = riakcs_storagecmd(rtcs_config:devpath(cs, Vsn), N, "batch -r"), - lager:info("Running ~p", [Cmd]), - os:cmd(Cmd). - -enable_proxy_get(SrcN, Vsn, SinkCluster) -> - rtdev:run_riak_repl(SrcN, rtcs_config:devpath(riak, Vsn), - "proxy_get enable " ++ SinkCluster). - -disable_proxy_get(SrcN, Vsn, SinkCluster) -> - rtdev:run_riak_repl(SrcN, rtcs_config:devpath(riak, Vsn), - "proxy_get disable " ++ SinkCluster). - -%% TODO: this is added as riak-1.4 branch of riak_test/src/rtcs_dev.erl -%% throws out the return value. Let's get rid of these functions when -%% we entered to Riak CS 2.0 dev, updating to riak_test master branch -cmd(Cmd, Opts) -> - cmd(Cmd, Opts, rt_config:get(rt_max_wait_time)). - -cmd(Cmd, Opts, WaitTime) -> - lager:info("Command: ~s", [Cmd]), - lager:info("Options: ~p", [Opts]), - Port = open_port({spawn_executable, Cmd}, - [in, exit_status, binary, - stream, stderr_to_stdout,{line, 200} | Opts]), - get_cmd_result(Port, WaitTime). - -get_cmd_result(Port, WaitTime) -> - receive - {Port, {data, {Flag, Line}}} when Flag =:= eol orelse Flag =:= noeol -> - lager:info(Line), - get_cmd_result(Port, WaitTime); - {Port, {exit_status, 0}} -> - ok; - {Port, {exit_status, Status}} -> - {error, {exit_status, Status}}; - {Port, Other} -> - lager:warning("Other data from port: ~p", [Other]), - get_cmd_result(Port, WaitTime) - after WaitTime -> - {error, timeout} - end. diff --git a/riak_test/src/rtcs_multipart.erl b/riak_test/src/rtcs_multipart.erl deleted file mode 100644 index d64cd9277..000000000 --- a/riak_test/src/rtcs_multipart.erl +++ /dev/null @@ -1,88 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(rtcs_multipart). - --compile(export_all). --include_lib("eunit/include/eunit.hrl"). - -%% Upload object by multipart and return generetad (=expected) content -multipart_upload(Bucket, Key, Sizes, Config) -> - InitRes = erlcloud_s3_multipart:initiate_upload( - Bucket, Key, "text/plain", [], Config), - UploadId = erlcloud_xml:get_text( - "/InitiateMultipartUploadResult/UploadId", InitRes), - Content = upload_parts(Bucket, Key, UploadId, Config, 1, Sizes, [], []), - Content. - -upload_parts(Bucket, Key, UploadId, Config, _PartCount, [], Contents, Parts) -> - ?assertEqual(ok, erlcloud_s3_multipart:complete_upload( - Bucket, Key, UploadId, lists:reverse(Parts), Config)), - iolist_to_binary(lists:reverse(Contents)); -upload_parts(Bucket, Key, UploadId, Config, PartCount, [Size | Sizes], Contents, Parts) -> - Content = crypto:rand_bytes(Size), - {RespHeaders, _UploadRes} = erlcloud_s3_multipart:upload_part( - Bucket, Key, UploadId, PartCount, Content, Config), - PartEtag = proplists:get_value("ETag", RespHeaders), - lager:debug("UploadId: ~p~n", [UploadId]), - lager:debug("PartEtag: ~p~n", [PartEtag]), - upload_parts(Bucket, Key, UploadId, Config, PartCount + 1, - Sizes, [Content | Contents], [{PartCount, PartEtag} | Parts]). - -upload_part_copy(BucketName, Key, UploadId, PartNum, SrcBucket, SrcKey, Config) -> - upload_part_copy(BucketName, Key, UploadId, PartNum, SrcBucket, SrcKey, undefined, Config). - -upload_part_copy(BucketName, Key, UploadId, PartNum, SrcBucket, SrcKey, SrcRange, Config) -> - Url = "/" ++ Key, - Source = filename:join([SrcBucket, SrcKey]), - Subresources = [{"partNumber", integer_to_list(PartNum)}, - {"uploadId", UploadId}], - Headers = [%%{"content-length", byte_size(PartData)}, - {"x-amz-copy-source", Source} | - source_range(SrcRange)], - erlcloud_s3:s3_request(Config, put, BucketName, Url, - Subresources, [], {<<>>, []}, Headers). - -source_range(undefined) -> []; -source_range({First, Last}) -> - [{"x-amz-copy-source-range", - lists:flatten(io_lib:format("bytes=~b-~b", [First, Last]))}]. - -upload_and_assert_part(Bucket, Key, UploadId, PartNum, PartData, Config) -> - {RespHeaders, _UploadRes} = erlcloud_s3_multipart:upload_part(Bucket, Key, UploadId, PartNum, PartData, Config), - assert_part(Bucket, Key, UploadId, PartNum, Config, RespHeaders). - - -assert_part(Bucket, Key, UploadId, PartNum, Config, RespHeaders) -> - PartEtag = proplists:get_value("ETag", RespHeaders), - PartsTerm = erlcloud_s3_multipart:parts_to_term( - erlcloud_s3_multipart:list_parts(Bucket, Key, UploadId, [], Config)), - %% lager:debug("~p", [PartsTerm]), - Parts = proplists:get_value(parts, PartsTerm), - ?assertEqual(Bucket, proplists:get_value(bucket, PartsTerm)), - ?assertEqual(Key, proplists:get_value(key, PartsTerm)), - ?assertEqual(UploadId, proplists:get_value(upload_id, PartsTerm)), - verify_part(PartEtag, proplists:get_value(PartNum, Parts)), - PartEtag. - -verify_part(_, undefined) -> - ?assert(false); -verify_part(ExpectedEtag, PartInfo) -> - ?assertEqual(ExpectedEtag, proplists:get_value(etag, PartInfo)). diff --git a/riak_test/src/rtcs_object.erl b/riak_test/src/rtcs_object.erl deleted file mode 100644 index b2128bae4..000000000 --- a/riak_test/src/rtcs_object.erl +++ /dev/null @@ -1,98 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2015 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(rtcs_object). - --compile(export_all). --include_lib("eunit/include/eunit.hrl"). --include_lib("erlcloud/include/erlcloud_aws.hrl"). - -upload(UserConfig, normal, B, K) -> - Content = crypto:rand_bytes(mb(4)), - erlcloud_s3:put_object(B, K, Content, UserConfig), - {B, K, Content}; -upload(UserConfig, multipart, B, K) -> - Content = rtcs_multipart:multipart_upload(B, K, [mb(10), 400], UserConfig), - {B, K, Content}; -upload(UserConfig, {normal_partial, CL, Actual}, B, K) when is_list(K), - CL >= Actual -> - %% Dumbest handmade S3 PUT Client - %% Send partial payload to the socket and suddenly close - Host = io_lib:format("~s.s3.amazonaws.com", [B]), - Date = httpd_util:rfc1123_date(erlang:localtime()), - %% Fake checksum, this request should fail if all payloads were sent - MD5 = "1B2M2Y8AsgTpgAmY7PhCfg==", - ToSign = ["PUT", $\n, MD5, $\n, "application/octet-stream", $\n, - Date, $\n, [], $/, B, $/, K, []], - lager:debug("String to Sign: ~s", [ToSign]), - Sig = base64:encode_to_string(crypto:hmac( - sha, - UserConfig#aws_config.secret_access_key, - ToSign)), - Auth = io_lib:format("Authorization: AWS ~s:~s", - [UserConfig#aws_config.access_key_id, Sig]), - {ok, Sock} = gen_tcp:connect("127.0.0.1", 15018, [{active, false}]), - FirstLine = io_lib:format("PUT /~s HTTP/1.1", [K]), - Binary = binary:copy(<<"*">>, Actual), - ReqHdr = [FirstLine, $\n, "Host: ", Host, $\n, Auth, $\n, - "Content-Length: ", integer_to_list(CL), $\n, - "Content-Md5: ", MD5, $\n, - "Content-Type: application/octet-stream", $\n, - "Date: ", Date, $\n], - lager:info("~s", [iolist_to_binary(ReqHdr)]), - case gen_tcp:send(Sock, [ReqHdr, $\n, Binary]) of - ok -> - %% Let caller handle the socket call, either close or continue - {ok, Sock}; - Error -> - Error - end. - -upload(UserConfig, normal_copy, B, DstK, SrcK) -> - ?assertEqual([{copy_source_version_id, "false"}, {version_id, "null"}], - erlcloud_s3:copy_object(B, DstK, B, SrcK, UserConfig)); -upload(UserConfig, multipart_copy, B, DstK, SrcK) -> - InitUploadRes = erlcloud_s3_multipart:initiate_upload(B, DstK, "text/plain", [], UserConfig), - UploadId = erlcloud_s3_multipart:upload_id(InitUploadRes), - - {RespHeaders1, _} = rtcs_multipart:upload_part_copy( - B, DstK, UploadId, 1, B, SrcK, {0, mb(5)-1}, UserConfig), - Etag1 = rtcs_multipart:assert_part(B, DstK, UploadId, 1, UserConfig, RespHeaders1), - {RespHeaders2, _} = rtcs_multipart:upload_part_copy( - B, DstK, UploadId, 2, B, SrcK, {mb(5), mb(10)+400-1}, UserConfig), - Etag2 = rtcs_multipart:assert_part(B, DstK, UploadId, 2, UserConfig, RespHeaders2), - - EtagList = [ {1, Etag1}, {2, Etag2} ], - ?assertEqual(ok, erlcloud_s3_multipart:complete_upload( - B, DstK, UploadId, EtagList, UserConfig)). - -mb(MegaBytes) -> - MegaBytes * 1024 * 1024. - -assert_whole_content(UserConfig, Bucket, Key, ExpectedContent) -> - Obj = erlcloud_s3:get_object(Bucket, Key, UserConfig), - assert_whole_content(ExpectedContent, Obj). - -assert_whole_content(ExpectedContent, ResultObj) -> - Content = proplists:get_value(content, ResultObj), - ContentLength = proplists:get_value(content_length, ResultObj), - ?assertEqual(byte_size(ExpectedContent), list_to_integer(ContentLength)), - ?assertEqual(byte_size(ExpectedContent), byte_size(Content)), - ?assertEqual(ExpectedContent, Content). diff --git a/riak_test/src/tools_helper.erl b/riak_test/src/tools_helper.erl deleted file mode 100644 index 391daadc9..000000000 --- a/riak_test/src/tools_helper.erl +++ /dev/null @@ -1,103 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(tools_helper). --export([offline_delete/2]). --include_lib("eunit/include/eunit.hrl"). - -%% @doc execute `offline_delete.erl` scripts with assertion at -%% before / after of it. -%% - Assert all blocks in `BlockKeysFileList` exist before execution -%% - Stop all nodes -%% - Execute `offline_delete.erl` -%% - Start all nodes -%% - Assert no blocks in `BlockKeysFileList` exist after execution -offline_delete({RiakNodes, CSNodes, Stanchion}, BlockKeysFileList) -> - lager:info("Assert all blocks exist before deletion"), - [assert_all_blocks_exists(RiakNodes, BlockKeysFile) || - BlockKeysFile <- BlockKeysFileList], - - lager:info("Stop nodes and execute offline_delete script..."), - NL0 = lists:zip(CSNodes, RiakNodes), - {CS1, R1} = hd(NL0), - NodeList = [{CS1, R1, Stanchion} | tl(NL0)], - rtcs_exec:stop_all_nodes(NodeList, current), - - [begin - Res = rtcs_exec:exec_priv_escript( - 1, "internal/offline_delete.erl", - "-r 8 --yes " ++ - rtcs_config:riak_bitcaskroot(rtcs_config:devpath(riak, current), 1) ++ - " " ++ BlockKeysFile, - riak), - lager:debug("offline_delete.erl log:\n~s", [Res]), - lager:debug("offline_delete.erl log:============= END") - end || BlockKeysFile <- BlockKeysFileList], - - lager:info("Assert all blocks are non-existent now"), - rtcs_exec:start_all_nodes(NodeList, current), - [assert_any_blocks_not_exists(RiakNodes, BlockKeysFile) || - BlockKeysFile <- BlockKeysFileList], - lager:info("All cleaned up!"), - ok. - -assert_all_blocks_exists(RiakNodes, BlocksListFile) -> - BlockKeys = block_keys(BlocksListFile), - lager:info("Assert all blocks still exist."), - [assert_block_exists(RiakNodes, BlockKey) || - BlockKey <- BlockKeys], - ok. - -assert_any_blocks_not_exists(RiakNodes, BlocksListFile) -> - BlockKeys = block_keys(BlocksListFile), - lager:info("Assert all blocks still exist."), - [assert_block_not_exists(RiakNodes, BlockKey) || - BlockKey <- BlockKeys], - ok. - -block_keys(FileName) -> - {ok, Bin} = file:read_file(FileName), - Lines = binary:split(Bin, <<"\n">>, [global]), - [begin - [_BHex, _KHex, CsBucket, CsKey, UUIDHex, SeqStr] = - binary:split(L, [<<"\t">>, <<" ">>], [global]), - {CsBucket, - mochihex:to_bin(binary_to_list(CsKey)), - mochihex:to_bin(binary_to_list(UUIDHex)), - list_to_integer(binary_to_list(SeqStr))} - end || L <- Lines, L =/= <<>>]. - -assert_block_exists(RiakNodes, {CsBucket, CsKey, UUID, Seq}) -> - ok = case rc_helper:get_riakc_obj(RiakNodes, blocks, CsBucket, {CsKey, UUID, Seq}) of - {ok, _Obj} -> ok; - Other -> - lager:error("block not found: ~p for ~p~n", - [Other, {CsBucket, CsKey, UUID, Seq}]), - {error, block_notfound} - end. - -assert_block_not_exists(RiakNodes, {CsBucket, CsKey, UUID, Seq}) -> - ok = case rc_helper:get_riakc_obj(RiakNodes, blocks, - CsBucket, {CsKey, UUID, Seq}) of - {error, notfound} -> ok; - {ok, _Obj} -> - lager:error("block found: ~p", [{CsBucket, CsKey, UUID, Seq}]), - {error, block_found} - end. diff --git a/riak_test/tests/access_stats_test.erl b/riak_test/tests/access_stats_test.erl deleted file mode 100644 index f8f54bb9e..000000000 --- a/riak_test/tests/access_stats_test.erl +++ /dev/null @@ -1,161 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(access_stats_test). -%% @doc Integration test for access statistics. -%% TODO: Only several kinds of stats are covered - --compile(export_all). --export([confirm/0]). - --include_lib("erlcloud/include/erlcloud_aws.hrl"). --include_lib("xmerl/include/xmerl.hrl"). --include_lib("eunit/include/eunit.hrl"). - --define(BUCKET, "access-stats-test-1"). --define(KEY, "a"). - -confirm() -> - {UserConfig, {RiakNodes, CSNodes, _Stanchion}} = rtcs:setup(2), - rt:setup_log_capture(hd(CSNodes)), - - {Begin, End} = generate_some_accesses(UserConfig), - flush_access_stats(), - assert_access_stats(json, UserConfig, {Begin, End}), - assert_access_stats(xml, UserConfig, {Begin, End}), - verify_stats_lost_logging(UserConfig, RiakNodes, CSNodes), - - rtcs:pass(). - -generate_some_accesses(UserConfig) -> - Begin = rtcs:datetime(), - lager:info("creating bucket ~p", [?BUCKET]), - %% Create bucket - ?assertEqual(ok, erlcloud_s3:create_bucket(?BUCKET, UserConfig)), - %% Put 100-byte object, twice - Block = crypto:rand_bytes(100), - _ = erlcloud_s3:put_object(?BUCKET, ?KEY, Block, UserConfig), - _ = erlcloud_s3:put_object(?BUCKET, ?KEY, Block, UserConfig), - %% Get 100-byte object, once - _ = erlcloud_s3:get_object(?BUCKET, ?KEY, UserConfig), - %% Head Object - _ = erlcloud_s3:head_object(?BUCKET, ?KEY, UserConfig), - %% List objects (GET bucket) - _ = erlcloud_s3:list_objects(?BUCKET, UserConfig), - %% Delete object - _ = erlcloud_s3:delete_object(?BUCKET, ?KEY, UserConfig), - %% Delete bucket - ?assertEqual(ok, erlcloud_s3:delete_bucket(?BUCKET, UserConfig)), - %% Illegal URL such that riak_cs_access_log_handler:handle_event/2 gets {log_access, #wm_log_data{notes=undefined}} - ?assertError({aws_error, {http_error, 404, _, _}}, erlcloud_s3:get_object("", "//a", UserConfig)), %% Path-style access - ?assertError({aws_error, {http_error, 404, _, _}}, erlcloud_s3:get_object("riak-cs", "pong", UserConfig)), - End = rtcs:datetime(), - {Begin, End}. - -flush_access_stats() -> - Res = rtcs_exec:flush_access(1), - lager:info("riak-cs-access flush result: ~s", [Res]), - ExpectRegexp = "All access logs were flushed.\n$", - ?assertMatch({match, _}, re:run(Res, ExpectRegexp)). - -assert_access_stats(Format, UserConfig, {Begin, End}) -> - KeyId = UserConfig#aws_config.access_key_id, - FormatInstruction = case Format of - json -> "j"; - xml -> "x" - end, - StatsKey = lists:flatten(["usage/", KeyId, "/a", FormatInstruction, "/", - Begin, "/", End, "/"]), - GetResult = erlcloud_s3:get_object("riak-cs", StatsKey, UserConfig), - lager:debug("GET Access stats response: ~p", [GetResult]), - Content = proplists:get_value(content, GetResult), - Samples = node_samples_from_content(Format, "rcs-dev1@127.0.0.1", Content), - lager:debug("Access samples (~s): ~p", [Format, Samples]), - - ?assertEqual( 1, sum_samples(Format, "BucketCreate", "Count", Samples)), - ?assertEqual( 2, sum_samples(Format, "KeyWrite", "Count", Samples)), - ?assertEqual(200, sum_samples(Format, "KeyWrite", "BytesIn", Samples)), - ?assertEqual( 0, sum_samples(Format, "KeyWrite", "BytesOut", Samples)), - ?assertEqual( 1, sum_samples(Format, "KeyRead", "Count", Samples)), - ?assertEqual( 0, sum_samples(Format, "KeyRead", "BytesIn", Samples)), - ?assertEqual(100, sum_samples(Format, "KeyRead", "BytesOut", Samples)), - ?assertEqual( 1, sum_samples(Format, "KeyStat", "Count", Samples)), - ?assertEqual( 0, sum_samples(Format, "KeyStat", "BytesOut", Samples)), - ?assertEqual( 1, sum_samples(Format, "BucketRead", "Count", Samples)), - ?assertEqual( 1, sum_samples(Format, "KeyDelete", "Count", Samples)), - ?assertEqual( 1, sum_samples(Format, "BucketDelete", "Count", Samples)), - pass. - -verify_stats_lost_logging(UserConfig, RiakNodes, CSNodes) -> - KeyId = UserConfig#aws_config.access_key_id, - {_Begin, _End} = generate_some_accesses(UserConfig), - %% kill riak - [ rt:brutal_kill(Node) || Node <- RiakNodes ], - %% force archive - flush_access_stats(), - %% check logs, at same node with flush_access_stats - CSNode = hd(CSNodes), - lager:info("Checking log in ~p", [CSNode]), - ExpectLine = io_lib:format("lost access stat: User=~s, Slice=", [KeyId]), - lager:debug("expected log line: ~s", [ExpectLine]), - true = rt:expect_in_log(CSNode, ExpectLine), - pass. - - -node_samples_from_content(json, Node, Content) -> - Usage = mochijson2:decode(Content), - ListOfNodeStats = rtcs:json_get([<<"Access">>, <<"Nodes">>], Usage), - lager:debug("ListOfNodeStats: ~p", [ListOfNodeStats]), - NodeBin = list_to_binary(Node), - [NodeStats | _] = lists:dropwhile( - fun(NodeStats) -> - rtcs:json_get(<<"Node">>, NodeStats) =/= NodeBin - end, ListOfNodeStats), - rtcs:json_get(<<"Samples">>, NodeStats); -node_samples_from_content(xml, Node, Content) -> - {Usage, _Rest} = xmerl_scan:string(unicode:characters_to_list(Content, utf8)), - xmerl_xpath:string("/Usage/Access/Nodes/Node[@name='" ++ Node ++ "']/Sample", Usage). - -sum_samples(json, OperationType, StatsKey, Data) -> - sum_samples_json([list_to_binary(OperationType), list_to_binary(StatsKey)], Data); -sum_samples(xml, OperationType, StatsKey, Data) -> - sum_samples_xml(OperationType, StatsKey, Data). - -%% Sum up statistics entries in possibly multiple samples -sum_samples_json(Keys, Samples) -> - sum_samples_json(Keys, Samples, 0). -sum_samples_json(_Keys, [], Sum) -> - Sum; -sum_samples_json(Keys, [Sample | Samples], Sum) -> - InSample = case rtcs:json_get(Keys, Sample) of - notfound -> - 0; - Value when is_integer(Value) -> - Value - end, - sum_samples_json(Keys, Samples, Sum + InSample). - -sum_samples_xml(OperationType, StatsKey, Samples) -> - lists:sum([list_to_integer(T#xmlText.value) || - Sample <- Samples, - T <- xmerl_xpath:string( - "/Sample/Operation[@type='" ++ OperationType ++ "']/" ++ - StatsKey ++ " /text()", - Sample)]). diff --git a/riak_test/tests/active_delete_test.erl b/riak_test/tests/active_delete_test.erl deleted file mode 100644 index e859c2fe8..000000000 --- a/riak_test/tests/active_delete_test.erl +++ /dev/null @@ -1,40 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(active_delete_test). - -%% @doc `riak_test' module for testing active delete situation behavior. - --export([confirm/0]). -config() -> - [{riak_cs, [{active_delete_threshold, 10000000}]}]. - -confirm() -> - %% 10MB threshold, for 3MB objects are used in cs_suites:run/2 - rtcs:set_advanced_conf(cs, config()), - Setup = rtcs:setup(1), - - %% Just do verify on typical normal case - History = [{cs_suites, run, ["run-1"]}], - {ok, InitialState} = cs_suites:new(Setup), - {ok, EvolvedState} = cs_suites:fold_with_state(InitialState, History), - {ok, _FinalState} = cs_suites:cleanup(EvolvedState), - - rtcs:pass(). diff --git a/riak_test/tests/auth_bypass_test.erl b/riak_test/tests/auth_bypass_test.erl deleted file mode 100644 index b88b31a64..000000000 --- a/riak_test/tests/auth_bypass_test.erl +++ /dev/null @@ -1,110 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(auth_bypass_test). - --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). --include_lib("erlcloud/include/erlcloud_aws.hrl"). -config() -> - [{riak_cs, [{admin_auth_enabled, false}]}]. - -confirm() -> - rtcs:set_advanced_conf(cs, config()), - {UserConfig, {RiakNodes, _CSNodes, _Stanchion}} = rtcs:setup(1), - KeyId = UserConfig#aws_config.access_key_id, - Port = rtcs_config:cs_port(hd(RiakNodes)), - - confirm_auth_bypass_for_stats("riak-cs", "stats", UserConfig, Port), - confirm_auth_bypass("riak-cs", "users", UserConfig, Port), - confirm_auth_bypass("riak-cs", "user/" ++ KeyId, UserConfig, Port), - confirm_auth_bypass("riak-cs", "usage/" ++ KeyId ++ "/ab/" ++ - rtcs:datetime() ++ "/" ++ rtcs:datetime(), - UserConfig, Port), - rtcs:pass(). - -confirm_auth_bypass_for_stats(Bucket, Key, UserConfig, Port) -> - {S3Content, CurlContent} = get_both_contents(Bucket, Key, UserConfig, Port), - S3Json = drop_volatile_stats_keys(mochijson2:decode(S3Content)), - CurlJson = drop_volatile_stats_keys(mochijson2:decode(CurlContent)), - ?assertEqual([], remove_volatile(S3Json -- CurlJson)), - ?assertEqual([], remove_volatile(CurlJson -- S3Json)). - -remove_volatile(Lists) -> - lists:filter(fun({<<"memory_", _/binary>>, _V}) -> false; - (_) -> true - end, Lists). - -confirm_auth_bypass(Bucket, Key, UserConfig, Port) -> - {S3Content, CurlContent} = get_both_contents(Bucket, Key, UserConfig, Port), - ?assertEqual(S3Content, CurlContent). - -get_both_contents(Bucket, Key, UserConfig, Port) -> - S3Result = erlcloud_s3:get_object(Bucket, Key, UserConfig), - S3Content = extract_contents(proplists:get_value(content, S3Result)), - lager:debug("erlcloud output: ~p~n", [S3Content]), - - CurlContent = extract_contents(curl_request(Bucket, Key, Port)), - lager:debug("curl output: ~p~n", [CurlContent]), - {S3Content, CurlContent}. - -curl_request(Bucket, Key, Port) -> - Cmd = "curl -s http://localhost:" ++ integer_to_list(Port) - ++ "/" ++ Bucket ++ "/" ++ Key, - lager:debug("cmd: ~p", [Cmd]), - os:cmd(Cmd). - -extract_contents(Output) when is_binary(Output) -> - extract_contents(binary_to_list(Output)); -extract_contents(Output) -> - [MaybeBoundary | Tokens] = string:tokens(Output, "\r\n"), - extract_contents(Tokens, MaybeBoundary, []). - -extract_contents([], NonMultipartContent, []) -> - lager:debug("extracted contents: ~p~n", [NonMultipartContent]), - NonMultipartContent; -extract_contents([], _Boundary, Contents) -> - lager:debug("extracted contents: ~p~n", [Contents]), - Contents; -extract_contents(["Content-Type: application/xml", Content | Tokens], - Boundary, Contents) -> - extract_contents(Tokens, Boundary, Contents ++ [Content]); -extract_contents([Boundary | Tokens], Boundary, Contents) -> - extract_contents(Tokens, Boundary, Contents); -extract_contents([_ | Tokens], Boundary, Contents) -> - extract_contents(Tokens, Boundary, Contents). - -drop_volatile_stats_keys({struct, KVs}) -> - [{K, V} || {K, V} <- KVs, not lists:member(K, volatile_stats_keys())]. - -volatile_stats_keys() -> - [<<"pbc_pool_master_workers">>, - <<"pbc_pool_master_size">>, - <<"object_web_active_sockets">>, - <<"memory_total">>, - <<"memory_processes">>, - <<"memory_processes_used">>, - <<"memory_system">>, - <<"memory_atom_used">>, - <<"memory_binary">>, - <<"memory_ets">>, - <<"sys_monitor_count">>, - <<"sys_port_count">>, - <<"sys_process_count">>]. diff --git a/riak_test/tests/block_audit_test.erl b/riak_test/tests/block_audit_test.erl deleted file mode 100644 index 52d779286..000000000 --- a/riak_test/tests/block_audit_test.erl +++ /dev/null @@ -1,128 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(block_audit_test). - -%% @doc `riak_test' module for testing block audit scripts - --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). - --define(BUCKET1, "rt-bucket1"). --define(BUCKET2, "rt-bucket2"). - --define(KEY_ALIVE, "alive"). --define(KEY_ORPHANED, "orphaned"). --define(KEY_FALSE_ORPHANED, "false-orphaned"). --define(KEY_ALIVE_MP, "alive-mp"). --define(KEY_ORPHANED_MP, "orphaned-mp"). --define(KEY_FALSE_ORPHANED_MP, "false-orphaned-mp"). - -confirm() -> - case rt_config:get(flavor, basic) of - {multibag, _} -> - lager:info("Block audit script does not supprt multibag env."), - lager:info("Skip the test."), - rtcs:pass(); - _ -> confirm1() - end. - -confirm1() -> - {UserConfig, {RiakNodes, CSNodes, Stanchion}} = rtcs:setup(1), - ?assertEqual(ok, erlcloud_s3:create_bucket(?BUCKET1, UserConfig)), - ?assertEqual(ok, erlcloud_s3:create_bucket(?BUCKET2, UserConfig)), - FalseOrphans1 = - [setup_objects(RiakNodes, UserConfig, Bucket, normal, - ?KEY_ALIVE, ?KEY_ORPHANED, ?KEY_FALSE_ORPHANED) || - Bucket <- [?BUCKET1, ?BUCKET2]], - FalseOrphans2 = - [setup_objects(RiakNodes, UserConfig, Bucket, mp, - ?KEY_ALIVE_MP, ?KEY_ORPHANED_MP, ?KEY_FALSE_ORPHANED_MP) || - Bucket <- [?BUCKET1, ?BUCKET2]], - Home = rtcs_config:riakcs_home(rtcs_config:devpath(cs, current), 1), - os:cmd("rm -rf " ++ filename:join([Home, "maybe-orphaned-blocks"])), - os:cmd("rm -rf " ++ filename:join([Home, "actual-orphaned-blocks"])), - Res1 = rtcs_exec:exec_priv_escript(1, "internal/block_audit.erl", - "-h 127.0.0.1 -p 10017 -dd"), - lager:debug("block_audit.erl log:\n~s", [Res1]), - lager:debug("block_audit.erl log:============= END"), - fake_false_orphans(RiakNodes, FalseOrphans1 ++ FalseOrphans2), - Res2 = rtcs_exec:exec_priv_escript(1, "internal/ensure_orphan_blocks.erl", - "-h 127.0.0.1 -p 10017 -dd"), - lager:debug("ensure_orphan_blocks.erl log:\n~s", [Res2]), - lager:debug("ensure_orphan_blocks.erl log:============= END"), - assert_result(?BUCKET1), - assert_result(?BUCKET2), - - BlockKeysFileList = [filename:join([Home, "actual-orphaned-blocks", B]) || - B <- [?BUCKET1, ?BUCKET2]], - tools_helper:offline_delete({RiakNodes, CSNodes, Stanchion}, BlockKeysFileList), - rtcs:pass(). - -setup_objects(RiakNodes, UserConfig, Bucket, Type, - KeyAlive, KeyOrphaned, KeyFalseOrphaned) -> - case Type of - normal -> - SingleBlock = crypto:rand_bytes(400), - [erlcloud_s3:put_object(Bucket, Key, SingleBlock, UserConfig) || - Key <- [KeyAlive, KeyOrphaned, KeyFalseOrphaned]]; - mp -> - ok = rc_helper:delete_riakc_obj(RiakNodes, objects, Bucket, KeyOrphaned), - [rtcs_multipart:multipart_upload(Bucket, Key, - [mb(10), mb(5)], UserConfig) || - Key <- [KeyAlive, KeyOrphaned, KeyFalseOrphaned]] - end, - ok = rc_helper:delete_riakc_obj(RiakNodes, objects, Bucket, KeyOrphaned), - lager:info("To fake deficit replicas for ~p, delete objects and restore it " - "between block_audit.er and block_audit2.erl runs", - [{Bucket, KeyFalseOrphaned}]), - {ok, FalseOrphanedRObj} = rc_helper:get_riakc_obj(RiakNodes, objects, - Bucket, KeyFalseOrphaned), - ok = rc_helper:delete_riakc_obj(RiakNodes, objects, Bucket, KeyFalseOrphaned), - {Bucket, KeyFalseOrphaned, FalseOrphanedRObj}. - -fake_false_orphans(RiakNodes, FalseOrphans) -> - [begin - R = rc_helper:update_riakc_obj(RiakNodes, objects, B, K, O), - lager:debug("fake_false_orphans ~p: ~p", [{B, K}, R]) - end || - {B, K, O} <- FalseOrphans]. - -assert_result(Bucket) -> - Home = rtcs_config:riakcs_home(rtcs_config:devpath(cs, current), 1), - OutFile1 = filename:join([Home, "actual-orphaned-blocks", Bucket]), - {ok, Bin} = file:read_file(OutFile1), - KeySeqs = [begin - [_RiakBucketHex, _RiakKeyHex, - _CSBucket, CSKeyHex, _UUIDHex, SeqStr] = - binary:split(Line, [<<$\t>>], [global, trim]), - {binary_to_list(mochihex:to_bin(binary_to_list(CSKeyHex))), - list_to_integer(binary_to_list(SeqStr))} - end || Line <- binary:split(Bin, [<<$\n>>], [global, trim])], - ?assertEqual([?KEY_ORPHANED, ?KEY_ORPHANED_MP], - lists:sort(proplists:get_keys(KeySeqs))), - ?assertEqual([0], proplists:get_all_values(?KEY_ORPHANED, KeySeqs)), - ?assertEqual([0,0,1,1,2,2,3,3,4,4,5,6,7,8,9], - lists:sort(proplists:get_all_values(?KEY_ORPHANED_MP, KeySeqs))), - ok. - -mb(MegaBytes) -> - MegaBytes * 1024 * 1024. - diff --git a/riak_test/tests/buckets_test.erl b/riak_test/tests/buckets_test.erl deleted file mode 100644 index b598603b7..000000000 --- a/riak_test/tests/buckets_test.erl +++ /dev/null @@ -1,220 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(buckets_test). - -%% @doc `riak_test' module for testing object get behavior. - --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). - -%% keys for non-multipart objects --define(TEST_BUCKET, "riak-test-bucket"). --define(KEY_SINGLE_BLOCK, "riak_test_key1"). --define(REGION, "boom-boom-tokyo-42"). - -%% keys for multipart uploaded objects --define(KEY_MP, "riak_test_mp"). % single part, single block -config() -> - [{riak_cs, [{region, ?REGION}]}]. - -confirm() -> - rtcs:set_advanced_conf(cs, config()), - {UserConfig, {RiakNodes, CSNodes, _Stanchion}} = rtcs:setup(1), - - - %% User 1, Cluster 1 config - UserConfig1 = rtcs_admin:create_user(hd(RiakNodes), 1), - - ok = verify_create_delete(UserConfig), - - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - - ok = verify_bucket_location(UserConfig), - - ok = verify_bucket_delete_fails(UserConfig), - - ok = verify_bucket_mpcleanup(UserConfig), - - ok = verify_bucket_mpcleanup_racecond_and_fix(UserConfig, UserConfig1, - RiakNodes, hd(CSNodes)), - - ok = verify_cleanup_orphan_mp(UserConfig, UserConfig1, RiakNodes, hd(CSNodes)), - - ok = verify_max_buckets_per_user(UserConfig), - - rtcs:pass(). - - -verify_create_delete(UserConfig) -> - lager:info("User is valid on the cluster, and has no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(UserConfig)), - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - - lager:info("deleting bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:delete_bucket(?TEST_BUCKET, UserConfig)), - lager:info("User is valid on the cluster, and has no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(UserConfig)), - ok. - -verify_bucket_delete_fails(UserConfig) -> - %% setup objects - SingleBlock = crypto:rand_bytes(400), - erlcloud_s3:put_object(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, SingleBlock, UserConfig), - - %% verify bucket deletion fails if any objects exist - lager:info("deleting bucket ~p (to fail)", [?TEST_BUCKET]), - ?assertError({aws_error, {http_error, _, _, _}}, - erlcloud_s3:delete_bucket(?TEST_BUCKET, UserConfig)), - - %% cleanup object - erlcloud_s3:delete_object(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, UserConfig), - ok. - - -verify_bucket_mpcleanup(UserConfig) -> - Bucket = ?TEST_BUCKET, - Key = ?KEY_SINGLE_BLOCK, - InitUploadRes = erlcloud_s3_multipart:initiate_upload(Bucket, Key, [], [], UserConfig), - lager:info("InitUploadRes = ~p", [InitUploadRes]), - UploadId = erlcloud_s3_multipart:upload_id(InitUploadRes), - lager:info("UploadId = ~p", [UploadId]), - - %% make sure that mp uploads created - UploadsList1 = erlcloud_s3_multipart:list_uploads(Bucket, [], UserConfig), - Uploads1 = proplists:get_value(uploads, UploadsList1, []), - ?assertEqual(Bucket, proplists:get_value(bucket, UploadsList1)), - ?assert(mp_upload_test:upload_id_present(UploadId, Uploads1)), - - lager:info("deleting bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:delete_bucket(?TEST_BUCKET, UserConfig)), - - %% check that writing mp uploads never resurrect - %% after bucket delete - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - UploadsList2 = erlcloud_s3_multipart:list_uploads(Bucket, [], UserConfig), - Uploads2 = proplists:get_value(uploads, UploadsList2, []), - ?assertEqual([], Uploads2), - ?assertEqual(Bucket, proplists:get_value(bucket, UploadsList2)), - ?assertNot(mp_upload_test:upload_id_present(UploadId, Uploads2)), - ok. - -%% @doc in race condition: on delete_bucket -verify_bucket_mpcleanup_racecond_and_fix(UserConfig, UserConfig1, - RiakNodes, CSNode) -> - Key = ?KEY_MP, - Bucket = ?TEST_BUCKET, - prepare_bucket_with_orphan_mp(Bucket, Key, UserConfig, RiakNodes), - - %% then fail on creation - %%TODO: check fail fail fail => 500 - ?assertError({aws_error, {http_error, 500, [], _}}, - erlcloud_s3:create_bucket(Bucket, UserConfig)), - - ?assertError({aws_error, {http_error, 500, [], _}}, - erlcloud_s3:create_bucket(Bucket, UserConfig1)), - - %% but we have a cleanup script, for existing system with 1.4.x or earlier - %% DO cleanup here - case rpc:call(CSNode, riak_cs_console, cleanup_orphan_multipart, []) of - {badrpc, Error} -> - lager:error("cleanup_orphan_multipart error: ~p~n", [Error]), - throw(Error); - Res -> - lager:info("Result of cleanup_orphan_multipart: ~p~n", [Res]) - end, - - %% list_keys here? wait for GC? - - %% and Okay, it's clear, another user creates same bucket - ?assertEqual(ok, erlcloud_s3:create_bucket(Bucket, UserConfig1)), - - %% Nothing found - UploadsList2 = erlcloud_s3_multipart:list_uploads(Bucket, [], UserConfig1), - Uploads2 = proplists:get_value(uploads, UploadsList2, []), - ?assertEqual([], Uploads2), - ?assertEqual(Bucket, proplists:get_value(bucket, UploadsList2)), - ok. - -%% @doc cleanup orphan multipart for 30 buckets (> pool size) -verify_cleanup_orphan_mp(UserConfig, UserConfig1, RiakNodes, CSNode) -> - [begin - Suffix = integer_to_list(Index), - Bucket = ?TEST_BUCKET ++ Suffix, - Key = ?KEY_MP ++ Suffix, - prepare_bucket_with_orphan_mp(Bucket, Key, UserConfig, RiakNodes) - end || Index <- lists:seq(1, 30)], - - %% but we have a cleanup script, for existing system with 1.4.x or earlier - %% DO cleanup here - case rpc:call(CSNode, riak_cs_console, cleanup_orphan_multipart, []) of - {badrpc, Error} -> - lager:error("cleanup_orphan_multipart error: ~p~n", [Error]), - throw(Error); - Res -> - lager:info("Result of cleanup_orphan_multipart: ~p~n", [Res]) - end, - - %% and Okay, it's clear, another user creates same bucket - Bucket1 = ?TEST_BUCKET ++ "1", - ?assertEqual(ok, erlcloud_s3:create_bucket(Bucket1, UserConfig1)), - - %% Nothing found - UploadsList = erlcloud_s3_multipart:list_uploads(Bucket1, [], UserConfig1), - Uploads = proplists:get_value(uploads, UploadsList, []), - ?assertEqual([], Uploads), - ?assertEqual(Bucket1, proplists:get_value(bucket, UploadsList)), - ok. - -prepare_bucket_with_orphan_mp(BucketName, Key, UserConfig, RiakNodes) -> - ?assertEqual(ok, erlcloud_s3:create_bucket(BucketName, UserConfig)), - _InitUploadRes = erlcloud_s3_multipart:initiate_upload(BucketName, Key, [], [], UserConfig), - - %% Reserve riak object to emulate prior 1.4.5 behavior afterwards - {ok, ManiObj} = rc_helper:get_riakc_obj(RiakNodes, objects, BucketName, Key), - - ?assertEqual(ok, erlcloud_s3:delete_bucket(BucketName, UserConfig)), - - %% emulate a race condition, during the deletion MP initiate happened - ok = rc_helper:update_riakc_obj(RiakNodes, objects, BucketName, Key, ManiObj). - - -verify_max_buckets_per_user(UserConfig) -> - [{buckets, Buckets}] = erlcloud_s3:list_buckets(UserConfig), - lager:debug("existing buckets: ~p", [Buckets]), - BucketNameBase = "toomanybuckets", - [begin - BucketName = BucketNameBase++integer_to_list(N), - lager:debug("creating bucket ~p", [BucketName]), - ?assertEqual(ok, - erlcloud_s3:create_bucket(BucketName, UserConfig)) - end - || N <- lists:seq(1,100-length(Buckets))], - lager:debug("100 buckets created", []), - BucketName1 = BucketNameBase ++ "101", - ?assertError({aws_error, {http_error, 400, [], _}}, - erlcloud_s3:create_bucket(BucketName1, UserConfig)), - ok. - -verify_bucket_location(UserConfig) -> - ?assertEqual(?REGION, - erlcloud_s3:get_bucket_attribute(?TEST_BUCKET, location, UserConfig)). diff --git a/riak_test/tests/cs743_regression_test.erl b/riak_test/tests/cs743_regression_test.erl deleted file mode 100644 index d10672180..000000000 --- a/riak_test/tests/cs743_regression_test.erl +++ /dev/null @@ -1,100 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(cs743_regression_test). - -%% @doc Regression test for `riak_cs' -%% issue 286. - --export([confirm/0]). - --include_lib("erlcloud/include/erlcloud_aws.hrl"). --include_lib("eunit/include/eunit.hrl"). - --define(TEST_BUCKET, "riak-test-bucket"). - -confirm() -> - rtcs:set_conf(cs, [{"stats.storage.archive_period", "1s"}]), - rtcs:set_advanced_conf(cs, [{riak_cs, [{storage_calc_timeout, 1}]}]), - {UserConfig, {_RiakNodes, CSNodes, _Stanchion}} = rtcs:setup(2), - - Begin = rtcs:datetime(), - run_storage_batch(hd(CSNodes)), - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - - N = 1024, - lager:info("creating ~p objects in ~p", [N, ?TEST_BUCKET]), - ok = etoomanyobjects(N, UserConfig), - timer:sleep(1000), - - run_storage_batch(hd(CSNodes)), - timer:sleep(1000), - End = rtcs:datetime(), - - assert_storage_stats(UserConfig, Begin, End), - rtcs:pass(). - -assert_storage_stats(UserConfig, Begin, End) -> - KeyId = UserConfig#aws_config.access_key_id, - StatsKey = lists:flatten(["usage/", KeyId, "/bj/", Begin, "/", End, "/"]), - GetResult = erlcloud_s3:get_object("riak-cs", StatsKey, UserConfig), - lager:info("Storage stats response: ~p", [GetResult]), - Usage = mochijson2:decode(proplists:get_value(content, GetResult)), - lager:info("Storage Usage: ~p", [Usage]), - Samples = rtcs:json_get([<<"Storage">>, <<"Samples">>], Usage), - - ?assert(lists:any( - fun(Sample) -> - case rtcs:json_get(list_to_binary(?TEST_BUCKET), Sample) of - notfound -> false; - ResultStr -> - ?assert(not is_integer(ResultStr)), - ?assertEqual(<<"{error,{timeout,[]}}">>, ResultStr), - true - end - end, - Samples)). - %% supposed to be "{error, timeout}" - -run_storage_batch(CSNode) -> - {ok, Status0} = rpc:call(CSNode, riak_cs_storage_d, status, []), - lager:info("~p", [Status0]), - ok = rpc:call(CSNode, riak_cs_storage_d, start_batch, [[{recalc,true}]]), - {ok, Status1} = rpc:call(CSNode, riak_cs_storage_d, status, []), - lager:info("~p", [Status1]), - %%{ok, - %% {calculating,[{schedule,[]},{last,undefined},{current,{{2013,12,26},{3,55,29}}}, - %% {next,undefined},{elapsed,0},{users_done,1},{users_skipped,0},{users_left,0}]}} - - {_Status, Result} = Status1, - 1 = proplists:get_value(users_done,Result), - 0 = proplists:get_value(users_skipped,Result), - 0 = proplists:get_value(users_left,Result). - -etoomanyobjects(N, UserConfig) -> - SingleBlock = crypto:rand_bytes(400), - lists:map(fun(I) -> - R = erlcloud_s3:put_object(?TEST_BUCKET, integer_to_list(I), - SingleBlock, UserConfig), - [{version_id,"null"}] = R - end, - lists:seq(1,N)), - ok. diff --git a/riak_test/tests/error_response_test.erl b/riak_test/tests/error_response_test.erl deleted file mode 100644 index bbeae8f91..000000000 --- a/riak_test/tests/error_response_test.erl +++ /dev/null @@ -1,73 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(error_response_test). - --compile(export_all). --export([confirm/0]). - --include_lib("eunit/include/eunit.hrl"). - --define(BUCKET, "error-response-test"). --define(BUCKET2, "error-response-test2"). --define(KEY, "a"). --define(ErrNodeId, 2). - -confirm() -> - {UserConfig, {RiakNodes, CSNodes, Stanchion}} = rtcs:setup(2), - ErrCSNode = lists:nth(?ErrNodeId, CSNodes), - ErrNode = lists:nth(?ErrNodeId, RiakNodes), - ErrConfig = rtcs_admin:aws_config(UserConfig, [{port, rtcs_config:cs_port(ErrNode)}]), - - %% setup initial data - ?assertEqual(ok, erlcloud_s3:create_bucket(?BUCKET, UserConfig)), - SingleBlock = crypto:rand_bytes(400), - erlcloud_s3:put_object(?BUCKET, ?KEY, SingleBlock, UserConfig), - - %% vefity response for timeout during getting a user. - %% FIXME: This should be http_error 503 - rt_intercept:add(ErrCSNode, {riak_cs_riak_client, [{{get_user, 2}, get_user_timeout}]}), - ?assertError({aws_error, {http_error, 403, [], _}}, - erlcloud_s3:get_object(?BUCKET, ?KEY, ErrConfig)), - rt_intercept:clean(ErrCSNode, riak_cs_riak_client), - - %% vefity response for timeout during getting block. - %% FIXME: This should be http_error 503 - rt_intercept:add(ErrCSNode, {riak_cs_block_server, [{{get_block_local, 6}, get_block_local_timeout}]}), - ?assertError({aws_error, {socket_error, retry_later}}, erlcloud_s3:get_object(?BUCKET, ?KEY, ErrConfig)), - rt_intercept:clean(ErrCSNode, riak_cs_block_server), - - - %% vefity response for timeout during get a bucket on stanchion. - %% FIXME: This should be http_error 503 - rt_intercept:add(Stanchion, {riakc_pb_socket, [{{get, 5}, get_timeout}]}), - ?assertError({aws_error, {http_error, 500, [], _}}, - erlcloud_s3:create_bucket(?BUCKET2, ErrConfig)), - rt_intercept:clean(Stanchion, riakc_pb_socket), - - %% vefity response for timeout during put a bucket on stanchion. - %% FIXME: This should be http_error 503 - rt_intercept:add(Stanchion, {riakc_pb_socket, [{{put, 4}, put_timeout}]}), - ?assertError({aws_error, {http_error, 500, [], _}}, - erlcloud_s3:create_bucket(?BUCKET2, ErrConfig)), - rt_intercept:clean(Stanchion, riakc_pb_socket), - - rtcs:pass(). - diff --git a/riak_test/tests/external_client_tests.erl b/riak_test/tests/external_client_tests.erl deleted file mode 100644 index 32506ac3e..000000000 --- a/riak_test/tests/external_client_tests.erl +++ /dev/null @@ -1,42 +0,0 @@ --module(external_client_tests). - --export([confirm/0]). --include_lib("erlcloud/include/erlcloud_aws.hrl"). --include_lib("eunit/include/eunit.hrl"). - --define(TEST_BUCKET, "external-client-test"). - -confirm() -> - %% NOTE: This 'cs_src_root' path must appear in - %% ~/.riak_test.config in the 'rtcs_dev' section, 'src_paths' - %% subsection. - CsSrcDir = rtcs_dev:srcpath(cs_src_root), - lager:debug("cs_src_root = ~p", [CsSrcDir]), - - {UserConfig, {RiakNodes, _CSNodes, _Stanchion}} = rtcs:setup(2, [{cs, cs_config()}]), - ok = erlcloud_s3:create_bucket("external-client-test", UserConfig), - CsPortStr = integer_to_list(rtcs_config:cs_port(hd(RiakNodes))), - - Cmd = os:find_executable("make"), - Args = ["-j", "8", "test-client"], - Env = [{"CS_HTTP_PORT", CsPortStr}, - {"AWS_ACCESS_KEY_ID", UserConfig#aws_config.access_key_id}, - {"AWS_SECRET_ACCESS_KEY", UserConfig#aws_config.secret_access_key}, - {"CS_BUCKET", ?TEST_BUCKET}], - WaitTime = 5 * rt_config:get(rt_max_wait_time), - case rtcs_exec:cmd(Cmd, [{cd, CsSrcDir}, {env, Env}, {args, Args}], WaitTime) of - ok -> - rtcs:pass(); - {error, Reason} -> - lager:error("Error : ~p", [Reason]), - error({external_client_tests, Reason}) - end. - -cs_config() -> - [{riak_cs, - [{connection_pools, [{request_pool, {32, 0}}]}, - {enforce_multipart_part_size, false}, - {max_buckets_per_user, 300}, - {auth_v4_enabled, true} - ] - }]. diff --git a/riak_test/tests/gc_tests.erl b/riak_test/tests/gc_tests.erl deleted file mode 100644 index bdb93f650..000000000 --- a/riak_test/tests/gc_tests.erl +++ /dev/null @@ -1,278 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(gc_tests). - -%% @doc `riak_test' module for testing garbage collection - --export([confirm/0]). - --include_lib("eunit/include/eunit.hrl"). --include("riak_cs.hrl"). - -%% keys for non-multipart objects --define(TEST_BUCKET, "riak-test-bucket"). --define(TEST_KEY, "riak_test_key"). --define(TEST_KEY_MP, "riak_test_mp"). --define(TEST_KEY_BAD_STATE, "riak_test_key_bad_state"). --define(TIMESLICES, 5). - -confirm() -> - NumNodes = 1, - {UserConfig, {RiakNodes, CSNodes, _Stanchion}} = rtcs:setup(NumNodes), - %% Set up to grep logs to verify messages - rt:setup_log_capture(hd(CSNodes)), - - rtcs_exec:gc(1, "set-interval infinity"), - rtcs_exec:gc(1, "set-leeway 1"), - rtcs_exec:gc(1, "cancel"), - - lager:info("Test GC run under an invalid state manifest..."), - {GCKey, {BKey, UUID}} = setup_obj(RiakNodes, UserConfig), - - %% Ensure the leeway has expired - timer:sleep(2000), - - Result = rtcs_exec:gc(1, "earliest-keys"), - lager:debug("~p", [Result]), - ?assert(string:str(Result, "GC keys in") > 0), - - ok = verify_gc_run(hd(CSNodes), GCKey), - ok = verify_riak_object_remaining_for_bad_key(RiakNodes, GCKey, {BKey, UUID}), - - lager:info("Test repair script (repair_gc_bucket.erl) with more invlaid states..."), - ok = put_more_bad_keys(RiakNodes, UserConfig), - %% Ensure the leeway has expired - timer:sleep(2000), - RiakIDs = rtcs:riak_id_per_cluster(NumNodes), - [repair_gc_bucket(ID) || ID <- RiakIDs], - ok = verify_gc_run2(hd(CSNodes)), - - %% Determinisitc GC test - - %% Create keys not to be deleted - setup_normal_obj([{"spam", 42}, {"ham", 65536}, {"egg", 7}], UserConfig), - timer:sleep(1000), %% Next timestamp... - - %% Create keys to be deleted - Start = os:timestamp(), - [begin - setup_normal_obj([{"hop", 42}, {"step", 65536}, {"jump", 7}], UserConfig), - timer:sleep(2000) - end || _ <- lists:seq(0,?TIMESLICES) ], - End = os:timestamp(), - - timer:sleep(1000), %% Next timestamp... - setup_normal_obj([{"spam", 42}, {"ham", 65536}, {"egg", 7}], UserConfig), - - verify_partial_gc_run(hd(CSNodes), RiakNodes, Start, End), - rtcs:pass(). - -setup_normal_obj(ObjSpecs, UserConfig) -> - %% Put and delete some objects - [begin - Block = crypto:rand_bytes(Size), - Key = ?TEST_KEY ++ Suffix, - erlcloud_s3:put_object(?TEST_BUCKET, Key, Block, UserConfig), - erlcloud_s3:delete_object(?TEST_BUCKET, Key, UserConfig) - end || {Suffix, Size} <- ObjSpecs]. - -setup_obj(RiakNodes, UserConfig) -> - %% Setup bucket - lager:info("User is valid on the cluster, and has no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(UserConfig)), - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - - setup_normal_obj([{"1", 100}, {"2", 200}, {"3", 0}], UserConfig), - - %% Put and delete, but modified to pretend it is in wrong state - SingleBlock = crypto:rand_bytes(400), - erlcloud_s3:put_object(?TEST_BUCKET, ?TEST_KEY_BAD_STATE, SingleBlock, UserConfig), - erlcloud_s3:delete_object(?TEST_BUCKET, ?TEST_KEY_BAD_STATE, UserConfig), - %% Change the state in the manifest in gc bucket to active. - %% See https://github.com/basho/riak_cs/issues/827#issuecomment-54567839 - GCPbc = rtcs:pbc(RiakNodes, objects, ?TEST_BUCKET), - {ok, GCKeys} = riakc_pb_socket:list_keys(GCPbc, ?GC_BUCKET), - BKey = {list_to_binary(?TEST_BUCKET), list_to_binary(?TEST_KEY_BAD_STATE)}, - lager:info("Changing state to active ~p, ~p", [?TEST_BUCKET, ?TEST_KEY_BAD_STATE]), - {ok, GCKey, UUID} = change_state_to_active(GCPbc, BKey, GCKeys), - - %% Put and delete some more objects - setup_normal_obj([{"Z", 0}, {"Y", 150}, {"X", 1}], UserConfig), - - riakc_pb_socket:stop(GCPbc), - {GCKey, {BKey, UUID}}. - -change_state_to_active(_Pbc, TargetBKey, []) -> - lager:warning("Target BKey ~p not found in GC bucket", [TargetBKey]), - {error, notfound}; -change_state_to_active(Pbc, TargetBKey, [GCKey|Rest]) -> - {ok, Obj0} = riakc_pb_socket:get(Pbc, ?GC_BUCKET, GCKey), - Manifests = twop_set:to_list(binary_to_term(riakc_obj:get_value(Obj0))), - case [{UUID, M?MANIFEST{state=active}} || - {UUID, M} <- Manifests, - M?MANIFEST.bkey =:= TargetBKey] of - [] -> - change_state_to_active(Pbc, TargetBKey, Rest); - [{TargetUUID, TargetManifest}] -> - lager:info("Target BKey ~p found in GC bucket ~p", [TargetBKey, GCKey]), - NewManifestSet = - lists:foldl(fun twop_set:add_element/2, twop_set:new(), - [{TargetUUID, - TargetManifest?MANIFEST{ - state = active, - delete_marked_time=undefined, - delete_blocks_remaining=undefined}} | - lists:keydelete(TargetUUID, 1, Manifests)]), - UpdObj = riakc_obj:update_value(Obj0, term_to_binary(NewManifestSet)), - ok = riakc_pb_socket:put(Pbc, UpdObj), - lager:info("Bad state manifests have been put at ~p: ~p~n", - [GCKey, twop_set:to_list(NewManifestSet)]), - {ok, GCKey, TargetUUID} - end. - -put_more_bad_keys(RiakNodes, UserConfig) -> - %% Put and delete some objects - [begin - Block = crypto:rand_bytes(10), - Key = ?TEST_KEY ++ integer_to_list(Suffix), - erlcloud_s3:put_object(?TEST_BUCKET, Key, Block, UserConfig), - erlcloud_s3:delete_object(?TEST_BUCKET, Key, UserConfig) - end || Suffix <- lists:seq(100, 199)], - GCPbc = rtcs:pbc(RiakNodes, objects, ?TEST_BUCKET), - {ok, GCKeys} = riakc_pb_socket:list_keys(GCPbc, ?GC_BUCKET), - BadGCKeys = put_more_bad_keys(GCPbc, GCKeys, []), - lager:info("Bad state manifests have been put at ~p", [BadGCKeys]), - ok. - -put_more_bad_keys(_Pbc, [], BadGCKeys) -> - BadGCKeys; -put_more_bad_keys(Pbc, [GCKey|Rest], BadGCKeys) -> - case riakc_pb_socket:get(Pbc, ?GC_BUCKET, GCKey) of - {error, notfound} -> - put_more_bad_keys(Pbc, Rest, BadGCKeys); - {ok, Obj0} -> - Manifests = twop_set:to_list(binary_to_term(riakc_obj:get_value(Obj0))), - NewManifests = [{UUID, M?MANIFEST{state = active, - delete_marked_time=undefined, - delete_blocks_remaining=undefined}} || - {UUID, M} <- Manifests], - NewManifestSet = - lists:foldl(fun twop_set:add_element/2, twop_set:new(), NewManifests), - UpdObj = riakc_obj:update_value(Obj0, term_to_binary(NewManifestSet)), - ok = riakc_pb_socket:put(Pbc, UpdObj), - put_more_bad_keys(Pbc, Rest, [GCKey | BadGCKeys]) - end. - -repair_gc_bucket(RiakNodeID) -> - PbPort = integer_to_list(rtcs_config:pb_port(RiakNodeID)), - Res = rtcs_exec:repair_gc_bucket(1, "--host 127.0.0.1 --port " ++ PbPort ++ - " --leeway-seconds 1 --page-size 5 --debug"), - Lines = binary:split(list_to_binary(Res), [<<"\n">>], [global]), - lager:info("Repair script result: ==== BEGIN", []), - [lager:info("~s", [L]) || L <- Lines], - lager:info("Repair script result: ==== END", []), - ok. - -verify_gc_run(Node, GCKey) -> - rtcs_exec:gc(1, "batch 1"), - lager:info("Check log, warning for invalid state and info for GC finish"), - true = rt:expect_in_log(Node, - "Invalid state manifest in GC bucket at <<\"" - ++ binary_to_list(GCKey) ++ "\">>, " - ++ "bucket=<<\"" ++ ?TEST_BUCKET ++ "\">> " - ++ "key=<<\"" ++ ?TEST_KEY_BAD_STATE ++ "\">>: "), - true = rt:expect_in_log(Node, - "Finished garbage collection: \\d+ seconds, " - "\\d batch_count, 0 batch_skips, " - "7 manif_count, 4 block_count"), - ok. - -verify_gc_run2(Node) -> - rtcs_exec:gc(1, "batch 1"), - lager:info("Check collected count =:= 101, 1 from setup_obj, " - "100 from put_more_bad_keys."), - true = rt:expect_in_log(Node, - "Finished garbage collection: \\d+ seconds, " - "\\d+ batch_count, 0 batch_skips, " - "101 manif_count, 101 block_count"), - ok. - -%% Verify riak objects in gc buckets, manifest, block are all remaining. -verify_riak_object_remaining_for_bad_key(RiakNodes, GCKey, {{Bucket, Key}, UUID}) -> - {ok, _BlockObj} = rc_helper:get_riakc_obj(RiakNodes, blocks, Bucket, {Key, UUID, 0}), - {ok, _ManifestObj} = rc_helper:get_riakc_obj(RiakNodes, objects, Bucket, Key), - - GCPbc = rtcs:pbc(RiakNodes, objects, Bucket), - {ok, FileSetObj} = riakc_pb_socket:get(GCPbc, ?GC_BUCKET, GCKey), - Manifests = twop_set:to_list(binary_to_term(riakc_obj:get_value(FileSetObj))), - {UUID, Manifest} = lists:keyfind(UUID, 1, Manifests), - riakc_pb_socket:stop(GCPbc), - lager:info("As expected, BAD manifest in GC bucket remains," - " stand off orphan manfiests/blocks: ~p", [Manifest]), - ok. - -verify_partial_gc_run(CSNode, RiakNodes, - {MegaSec0, Sec0, _}, - {MegaSec1, Sec1, _}) -> - Start0 = MegaSec0 * 1000000 + Sec0, - End0 = MegaSec1 * 1000000 + Sec1, - Interval = erlang:max(1, (End0 - Start0) div ?TIMESLICES), - Starts = [ {Start0 + N * Interval, Start0 + (N+1) * Interval} - || N <- lists:seq(0, ?TIMESLICES-1) ] ++ - [{Start0 + ?TIMESLICES * Interval, End0}], - - [begin - %% We have to clear log as the message 'Finished garbage - %% col...' has been output many times before, during this - %% test. - rtcs:reset_log(CSNode), - - lager:debug("GC: (start, end) = (~p, ~p)", [S0, E0]), - S = rtcs:iso8601(S0), - E = rtcs:iso8601(E0), - BatchCmd = "batch -s " ++ S ++ " -e " ++ E, - rtcs_exec:gc(1, BatchCmd), - - true = rt:expect_in_log(CSNode, - "Finished garbage collection: \\d+ seconds, " - "\\d+ batch_count, 0 batch_skips, " - "\\d+ manif_count, \\d+ block_count") - end || {S0, E0} <- Starts], - lager:info("GC target period: (~p, ~p)", [Start0, End0]), - %% Reap! - timer:sleep(3000), - GCPbc = rtcs:pbc(RiakNodes, objects, ?TEST_BUCKET), - {ok, Keys} = riakc_pb_socket:list_keys(GCPbc, ?GC_BUCKET), - lager:debug("Keys: ~p", [Keys]), - StartKey = list_to_binary(integer_to_list(Start0)), - EndKey = list_to_binary(integer_to_list(End0)), - EndKeyHPF = fun(Key) -> EndKey < Key end, - StartKeyLPF = fun(Key) -> Key < StartKey end, - BPF = fun(Key) -> StartKey < Key andalso Key < EndKey end, - - lager:debug("Remaining Keys: ~p", [Keys]), - lager:debug("HPF result: ~p", [lists:filter(EndKeyHPF, Keys)]), - lager:debug("LPF result: ~p", [lists:filter(StartKeyLPF, Keys)]), - ?assertEqual(3, length(lists:filter(EndKeyHPF, Keys))), - ?assertEqual(3, length(lists:filter(StartKeyLPF, Keys))), - ?assertEqual([], lists:filter(BPF, Keys)), - ok. diff --git a/riak_test/tests/legacy_s3_rewrite_test.erl b/riak_test/tests/legacy_s3_rewrite_test.erl deleted file mode 100644 index c7436b6c2..000000000 --- a/riak_test/tests/legacy_s3_rewrite_test.erl +++ /dev/null @@ -1,64 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(legacy_s3_rewrite_test). - --export([confirm/0]). --include_lib("erlcloud/include/erlcloud_aws.hrl"). --include_lib("eunit/include/eunit.hrl"). - --define(TEST_BUCKET, "legacy-s3-rewrite-test"). - -confirm() -> - %% NOTE: This 'cs_src_root' path must appear in - %% ~/.riak_test.config in the 'rtcs_dev' section, 'src_paths' - %% subsection. - CsSrcDir = rtcs_dev:srcpath(cs_src_root), - lager:debug("cs_src_root = ~p", [CsSrcDir]), - - rtcs:set_advanced_conf(cs, cs_config()), - {UserConfig, {RiakNodes, _CSNodes, _Stanchion}} = rtcs:setup(1), - ok = erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig), - CsPortStr = integer_to_list(rtcs_config:cs_port(hd(RiakNodes))), - - Cmd = os:find_executable("make"), - Args = ["-C", "client_tests/python/boto_tests", "test-auth-v2"], - Env = [{"CS_HTTP_PORT", CsPortStr}, - {"AWS_ACCESS_KEY_ID", UserConfig#aws_config.access_key_id}, - {"AWS_SECRET_ACCESS_KEY", UserConfig#aws_config.secret_access_key}, - {"CS_BUCKET", ?TEST_BUCKET}], - WaitTime = 2 * rt_config:get(rt_max_wait_time), - case rtcs_exec:cmd(Cmd, [{cd, CsSrcDir}, {env, Env}, {args, Args}], WaitTime) of - ok -> - rtcs:pass(); - {error, Reason} -> - lager:error("Error : ~p", [Reason]), - error({?MODULE, Reason}) - end. - -cs_config() -> - [ - {riak_cs, - [ - {enforce_multipart_part_size, false}, - {max_buckets_per_user, 150}, - {rewrite_module, riak_cs_s3_rewrite_legacy} - ] - }]. diff --git a/riak_test/tests/list_objects_v2_test.erl b/riak_test/tests/list_objects_v2_test.erl deleted file mode 100644 index 5389399e9..000000000 --- a/riak_test/tests/list_objects_v2_test.erl +++ /dev/null @@ -1,57 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(list_objects_v2_test). - -%% @doc Integration test for list the contents of a bucket - --export([confirm/0]). - --include_lib("eunit/include/eunit.hrl"). - -confirm() -> - {UserConfig, {RiakNodes, CSNodes, _Stanchion}} = rtcs:setup(2), - assert_v2_is_default(CSNodes), - pass = list_objects_test_helper:test(UserConfig), - - ok = list_to_non_existent_bucket_many_times(RiakNodes), - rtcs:pass(). - -assert_v2_is_default(CSNodes) -> - true = rpc:call(hd(CSNodes), riak_cs_list_objects_utils, fold_objects_for_list_keys, []), - ok. - -list_to_non_existent_bucket_many_times(RiakNodes) -> - [?assertEqual({0, "404"}, - list_objects_by_anonymous( - RiakNodes, - "non-existent-bucket-" ++ integer_to_list(I))) || - I <- lists:seq(1, 30)], - ok. - -list_objects_by_anonymous(RiakNodes, Bucket) -> - Port = rtcs_config:cs_port(hd(RiakNodes)), - %% --write-out '%{http_code}': output http response status code to stdout - Cmd = "curl -s --write-out '%{http_code}' -o /dev/null http://localhost:" ++ - integer_to_list(Port) ++ "/" ++ Bucket, - rt:cmd(Cmd). - - - diff --git a/riak_test/tests/migration_15_to_20_test.erl b/riak_test/tests/migration_15_to_20_test.erl deleted file mode 100644 index 39b6ca750..000000000 --- a/riak_test/tests/migration_15_to_20_test.erl +++ /dev/null @@ -1,166 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% Upgrading test case from {Riak, CS} = {1.4, 1.5} to {2.0, 2.0} -%% -%% Scenario: -%% - Setup cluster by Riak 1.4.x and CS/Stanchion 1.5.x (i.e. previous) -%% - Execute some S3 API requests and background admin jobs, 1st time -%% - Upgrade Stanchion to 2.x -%% - Upgrade partial pairs of Riak and Riak CS to 2.x -%% - Execute some S3 API requests and background admin jobs, 2nd time -%% - Upgrade remaining pairs of Riak and Riak CS to 2.x -%% - Execute some S3 API requests and background admin jobs, 3rd time -%% - Delete all S3 objects -%% - Trigger bitcask merge and wait for merge & delete finished -%% - Assert bitcask data files are shrinked to small ones - --module(migration_15_to_20_test). - --export([confirm/0]). --export([upgrade_stanchion/1, upgrade_partially/2, upgrade_remaining/2]). - --include_lib("eunit/include/eunit.hrl"). - -%% TODO: More than 1 is better --define(UPGRADE_FIRST, 1). - -confirm() -> - rt_config:set(console_log_level, info), - confirm(upgrade_with_full_ops). - -%% `upgrade_with_reduced_ops' and `no_upgrade_with_reduced_ops' are only for -%% debugging of this module and/or `cs_suites'. --spec confirm(Profile::atom()) -> pass | no_return(). -confirm(upgrade_with_full_ops) -> - SetupRes = setup_previous(), - {ok, InitialState} = cs_suites:new(SetupRes), - {ok, EvolvedState} = cs_suites:fold_with_state(InitialState, upgrade_history()), - {ok, _FinalState} = cs_suites:cleanup(EvolvedState), - rtcs:pass(); -confirm(upgrade_with_reduced_ops) -> - SetupRes = setup_previous(), - {ok, InitialState} = cs_suites:new(SetupRes, rtcs:reduced_ops()), - {ok, EvolvedState} = cs_suites:fold_with_state(InitialState, upgrade_history()), - {ok, _FinalState} = cs_suites:cleanup(EvolvedState), - rtcs:pass(); -confirm(no_upgrade_with_reduced_ops) -> - SetupRes = rtcs:setup(2, rtcs_config:configs(custom_configs(current))), - {ok, InitialState} = cs_suites:new(SetupRes, cs_suites:reduced_ops()), - {ok, EvolvedState} = cs_suites:fold_with_state(InitialState, no_upgrade_history()), - {ok, _FinalState} = cs_suites:cleanup(EvolvedState), - rtcs:pass(). - -setup_previous() -> - PrevConfigs = rtcs_config:previous_configs(custom_configs(previous)), - SetupRes = rtcs:setup(2, PrevConfigs, previous), - lager:info("rt_nodes> ~p", [rt_config:get(rt_nodes)]), - lager:info("rt_versions> ~p", [rt_config:get(rt_versions)]), - {_AdminConfig, {RiakNodes, CSNodes, StanchionNode}} = SetupRes, - rtcs:assert_versions(riak_kv, RiakNodes, "^1\.4\."), - rtcs:assert_versions(stanchion, [StanchionNode], "^1\.5\."), - rtcs:assert_versions(riak_cs, CSNodes, "^1\.5\."), - SetupRes. - -%% Custom configurations for Riak and Riak CS -%% Make data file size tiny and leeway period tight in order to confirm -%% only small data will remain in bitcask directories after deleting all -%% CS objects, run GC and merge+delete of bitcask. -custom_configs(previous) -> - [{riak, - rtcs_config:previous_riak_config([{bitcask, [{max_file_size, 4*1024*1024}]}])}, - {cs, - rtcs_config:previous_cs_config([{leeway_seconds, 1}])}]; -custom_configs(current) -> - %% This branch is only for debugging this module - [{riak, - rtcs_config:riak_config([{bitcask, [{max_file_size, 4*1024*1024}]}])}, - {cs, - rtcs_config:cs_config([{leeway_seconds, 1}])}]. - -upgrade_history() -> - [ - {cs_suites, set_node1_version, [previous]}, - {cs_suites, run, ["1-pre"]}, - {?MODULE , upgrade_stanchion, []}, - {?MODULE , upgrade_partially, [?UPGRADE_FIRST]}, - {cs_suites, run, ["2-mix"]}, - {?MODULE , upgrade_remaining, [?UPGRADE_FIRST]}, - {cs_suites, run, ["3-fin"]} - ]. - -no_upgrade_history() -> - [ - {cs_suites, set_node1_version, [current]}, - {cs_suites, run,["1st"]}, - {cs_suites, run,["2nd"]} - ]. - -upgrade_stanchion(State) -> - rtcs_exec:stop_stanchion(previous), - rtcs_config:migrate_stanchion(previous, current, cs_suites:admin_credential(State)), - rtcs_exec:start_stanchion(current), - rtcs:assert_versions(stanchion, cs_suites:nodes_of(stanchion, State), "^2\."), - {ok, State}. - -upgrade_partially(UpgradeFirstCount, State) -> - RiakNodes = cs_suites:nodes_of(riak, State), - CsNodes = cs_suites:nodes_of(cs, State), - AdminCreds = cs_suites:admin_credential(State), - {RiakUpgrades, RiakRemainings} = lists:split(UpgradeFirstCount, RiakNodes), - {CsUpgrades, CsRemainings} = lists:split(UpgradeFirstCount, CsNodes), - upgrade_nodes(AdminCreds, RiakUpgrades), - rtcs:assert_versions(riak_kv, RiakUpgrades, "^2\."), - rtcs:assert_versions(riak_cs, CsUpgrades, "^2\."), - rtcs:assert_versions(riak_kv, RiakRemainings, "^1\.4\."), - rtcs:assert_versions(riak_cs, CsRemainings, "^1\.5\."), - rt:setup_log_capture(hd(cs_suites:nodes_of(cs, State))), - {ok, NewState} = cs_suites:set_node1_version(current, State), - {ok, NewState}. - -upgrade_remaining(UpgradeFirstCount, State) -> - RiakNodes = cs_suites:nodes_of(riak, State), - CsNodes = cs_suites:nodes_of(cs, State), - AdminCreds = cs_suites:admin_credential(State), - {_RiakUpgraded, RiakRemainings} = lists:split(UpgradeFirstCount, RiakNodes), - {_CsUpgraded, CsRemainings} = lists:split(UpgradeFirstCount, CsNodes), - upgrade_nodes(AdminCreds, RiakRemainings), - rtcs:assert_versions(riak_kv, RiakRemainings, "^2\."), - rtcs:assert_versions(riak_cs, CsRemainings, "^2\."), - {ok, State}. - -%% Upgrade Riak and Riak CS pairs of nodes -upgrade_nodes(AdminCreds, RiakNodes) -> - {_, RiakCurrentVsn} = - rtcs_dev:riak_root_and_vsn(current, rt_config:get(build_type, oss)), - [begin - N = rtcs_dev:node_id(RiakNode), - rtcs_exec:stop_cs(N, previous), - ok = rt:upgrade(RiakNode, RiakCurrentVsn), - rt:wait_for_service(RiakNode, riak_kv), - ok = rtcs_config:upgrade_cs(N, AdminCreds), - rtcs:set_advanced_conf({cs, current, N}, - [{riak_cs, - [{riak_host, {"127.0.0.1", rtcs_config:pb_port(1)}}]}]), - rtcs_exec:start_cs(N, current) - end - || RiakNode <- RiakNodes], - ok = rt:wait_until_ring_converged(RiakNodes), - ok. diff --git a/riak_test/tests/migration_mixed_test.erl b/riak_test/tests/migration_mixed_test.erl deleted file mode 100644 index 189daf4f7..000000000 --- a/riak_test/tests/migration_mixed_test.erl +++ /dev/null @@ -1,141 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% Upgrading test case from {Riak, CS} = {2.0, 1.5} and {1.4, 2.0} - -%% Scenario: -%% | step | riak | stanchion | cs | -%% | 1 | 1.4 | 1.5 | 1.5 | -%% | 2 | 1.4 | 2.0 | 2.0 | -%% | 3 | 2.0 | 2.0 | 1.5 | - --module(migration_mixed_test). - --export([confirm/0]). --export([upgrade_stanchion/1, - transition_to_cs20_with_kv14/1, - transition_to_cs15_with_kv20/1]). - --include_lib("eunit/include/eunit.hrl"). - -confirm() -> - rt_config:set(console_log_level, info), - SetupRes = setup_previous(), - {ok, InitialState} = cs_suites:new(SetupRes), - {ok, EvolvedState} = cs_suites:fold_with_state(InitialState, history()), - {ok, _FinalState} = cs_suites:cleanup(EvolvedState), - rtcs:pass(). - -setup_previous() -> - PrevConfigs = rtcs_config:previous_configs(custom_configs(previous)), - SetupRes = rtcs:setup(2, PrevConfigs, previous), - lager:info("rt_nodes> ~p", [rt_config:get(rt_nodes)]), - lager:info("rt_versions> ~p", [rt_config:get(rt_versions)]), - {_AdminConfig, {RiakNodes, CSNodes, StanchionNode}} = SetupRes, - rtcs:assert_versions(riak_kv, RiakNodes, "^1\.4\."), - rtcs:assert_versions(stanchion, [StanchionNode], "^1\.5\."), - rtcs:assert_versions(riak_cs, CSNodes, "^1\.5\."), - SetupRes. - -%% Custom configurations for Riak and Riak CS -%% Make data file size tiny and leeway period tight in order to confirm -%% only small data will remain in bitcask directories after deleting all -%% CS objects, run GC and merge+delete of bitcask. -custom_configs(previous) -> - [{riak, - rtcs_config:previous_riak_config([{bitcask, [{max_file_size, 4*1024*1024}]}])}, - {cs, - rtcs_config:previous_cs_config([{leeway_seconds, 1}])}]; -custom_configs(current) -> - %% This branch is only for debugging this module - [{riak, - rtcs_config:riak_config([{bitcask, [{max_file_size, 4*1024*1024}]}])}, - {cs, - rtcs_config:cs_config([{leeway_seconds, 1}])}]. - -history() -> - [ - {cs_suites, set_node1_version, [previous]}, - {cs_suites, run, ["15-14"]}, - {?MODULE , upgrade_stanchion, []}, - {?MODULE , transition_to_cs20_with_kv14, []}, - {cs_suites, run, ["20-14"]}, - {?MODULE , transition_to_cs15_with_kv20, []}, - {cs_suites, run, ["15-20"]} - ]. - -upgrade_stanchion(State) -> - rtcs_exec:stop_stanchion(previous), - rtcs_config:migrate_stanchion(previous, current, cs_suites:admin_credential(State)), - rtcs_exec:start_stanchion(current), - rtcs:assert_versions(stanchion, cs_suites:nodes_of(stanchion, State), "^2\."), - {ok, State}. - -transition_to_cs20_with_kv14(State) -> - RiakNodes = cs_suites:nodes_of(riak, State), - CsNodes = cs_suites:nodes_of(cs, State), - AdminCreds = cs_suites:admin_credential(State), - migrate_nodes_to_cs20_with_kv14(AdminCreds, RiakNodes), - rtcs:assert_versions(riak_kv, RiakNodes, "^1\.4\."), - rtcs:assert_versions(riak_cs, CsNodes, "^2\."), - rt:setup_log_capture(hd(cs_suites:nodes_of(cs, State))), - {ok, NewState} = cs_suites:set_node1_version(current, State), - {ok, NewState}. - -migrate_nodes_to_cs20_with_kv14(AdminCreds, RiakNodes) -> - [begin - N = rtcs_dev:node_id(RiakNode), - rtcs_exec:stop_cs(N, previous), - ok = rtcs_config:upgrade_cs(N, AdminCreds), - %% actually after CS 2.1.1 - rtcs:set_advanced_conf({cs, current, N}, - [{riak_cs, - [{riak_host, {"127.0.0.1", rtcs_config:pb_port(1)}}]}]), - rtcs_exec:start_cs(N, current) - end - || RiakNode <- RiakNodes], - ok. - -transition_to_cs15_with_kv20(State) -> - RiakNodes = cs_suites:nodes_of(riak, State), - CsNodes = cs_suites:nodes_of(cs, State), - AdminCreds = cs_suites:admin_credential(State), - migrate_nodes_to_cs15_with_kv20(AdminCreds, RiakNodes), - rtcs:assert_versions(riak_kv, RiakNodes, "^2\."), - rtcs:assert_versions(riak_cs, CsNodes, "^1\.5\."), - rt:setup_log_capture(hd(cs_suites:nodes_of(cs, State))), - {ok, State}. - -migrate_nodes_to_cs15_with_kv20(AdminCreds, RiakNodes) -> - {_, RiakCurrentVsn} = - rtcs_dev:riak_root_and_vsn(current, rt_config:get(build_type, oss)), - [begin - N = rtcs_dev:node_id(RiakNode), - rtcs_exec:stop_cs(N, current), - %% to check error log emptyness afterwards, truncate it here. - rtcs:truncate_error_log(N), - ok = rt:upgrade(RiakNode, RiakCurrentVsn), - rt:wait_for_service(RiakNode, riak_kv), - ok = rtcs_config:migrate_cs(current, previous, N, AdminCreds), - rtcs_exec:start_cs(N, previous) - end - || RiakNode <- RiakNodes], - rt:wait_until_ring_converged(RiakNodes), - ok. diff --git a/riak_test/tests/mp_upload_test.erl b/riak_test/tests/mp_upload_test.erl deleted file mode 100644 index f3c6b18e7..000000000 --- a/riak_test/tests/mp_upload_test.erl +++ /dev/null @@ -1,363 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(mp_upload_test). - -%% @doc `riak_test' module for testing multipart upload behavior. - --export([confirm/0, upload_id_present/2]). --include_lib("eunit/include/eunit.hrl"). --include("riak_cs.hrl"). - --define(TEST_BUCKET, "riak-test-bucket"). --define(TEST_KEY1, "riak_test_key1"). --define(TEST_KEY2, "riak_test_key2"). --define(PART_COUNT, 5). --define(GOOD_PART_SIZE, 5*1024*1024). --define(BAD_PART_SIZE, 2*1024*1024). - -confirm() -> - {UserConfig, {_RiakNodes, _CSNodes, _Stanchion}} = rtcs:setup(1), - - lager:info("User is valid on the cluster, and has no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(UserConfig)), - - ?assertError({aws_error, {http_error, 404, _, _}}, erlcloud_s3:list_objects(?TEST_BUCKET, UserConfig)), - - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - - ?assertMatch([{buckets, [[{name, ?TEST_BUCKET}, _]]}], - erlcloud_s3:list_buckets(UserConfig)), - - %% Test cases - basic_upload_test_case(?TEST_BUCKET, ?TEST_KEY1, UserConfig), - ok = parts_too_small_test_case(?TEST_BUCKET, ?TEST_KEY1, UserConfig), - aborted_upload_test_case(?TEST_BUCKET, ?TEST_KEY2, UserConfig), - nonexistent_bucket_listing_test_case("fake-bucket", UserConfig), - invalid_part_number_test_case(?TEST_BUCKET, ?TEST_KEY1, UserConfig), - - %% Start 10 uploads for 10 different keys - Count1 = 10, - initiate_uploads(?TEST_BUCKET, Count1, UserConfig), - - %% Successively list the in-progress uploads, verify the output, - %% and abort an upload until all uploads are aborted - abort_and_verify_uploads(?TEST_BUCKET, Count1, UserConfig), - - %% Start 100 uploads for 100 different keys - Count2 = 100, - initiate_uploads(?TEST_BUCKET, Count2, UserConfig), - - %% List uploads and verify all 100 are returned - UploadList1 = erlcloud_s3_multipart:list_uploads(?TEST_BUCKET, [], UserConfig), - verify_upload_list(UploadList1, Count2), - - %% List uploads and verify all 100 are returned with empty options. - %% Some s3 tools send empty parameters. - EmptyOptions = [{delimiter, ""}, {prefix, ""}], - UploadList1 = erlcloud_s3_multipart:list_uploads(?TEST_BUCKET, EmptyOptions, UserConfig), - verify_upload_list(UploadList1, Count2), - - %% @TODO Use max-uploads option to request first 50 results - %% Options1 = [{max_uploads, 50}], - %% UploadList2 = erlcloud_s3_multipart:list_uploads(?TEST_BUCKET, Options1, UserConfig), - %% verify_upload_list(ObjList1, 50, 100), - - %% Initiate uploads for 2 sets of 4 objects with keys that have - %% a common subdirectory - Prefix1 = "0/prefix1/", - Prefix2 = "0/prefix2/", - initiate_uploads(?TEST_BUCKET, 4, Prefix1, UserConfig), - initiate_uploads(?TEST_BUCKET, 4, Prefix2, UserConfig), - - %% @TODO Uncomment this block once support for `max-uploads' is done. - %% Use `max-uploads', `prefix' and `delimiter' to get first 50 - %% results back and verify results are truncated and 2 common - %% prefixes are returned. - %% Options2 = [{max_uploads, 50}, {prefix, "0/"}, {delimiter, "/"}], - %% UploadList3 = erlcloud_s3_multipart:list_uploads(?TEST_BUCKET, Options2, UserConfig), - %% CommonPrefixes = proplists:get_value(common_prefixes, UploadList3), - %% ?assert(lists:member([{prefix, Prefix1}], CommonPrefixes)), - %% ?assert(lists:member([{prefix, Prefix2}], CommonPrefixes)), - %% verify_upload_list(UploadList3, 48, 100), - - %% @TODO Replace this with the commented-out code blocks above and - %% below this one once the support for `max-uploads' is in place. - %% Use `prefix' and `delimiter' to get the active uploads back and - %% verify that 2 common prefixes are returned. - Options2 = [{prefix, "0/"}, {delimiter, "/"}], - UploadList3 = erlcloud_s3_multipart:list_uploads(?TEST_BUCKET, Options2, UserConfig), - CommonPrefixes1 = proplists:get_value(common_prefixes, UploadList3), - ?assert(lists:member([{prefix, Prefix1}], CommonPrefixes1)), - ?assert(lists:member([{prefix, Prefix2}], CommonPrefixes1)), - ?assertEqual([], proplists:get_value(uploads, UploadList3)), - - %% Use `delimiter' to get the active uploads back and - %% verify that 2 common prefixes are returned. - Options3 = [{delimiter, "/"}], - UploadList4 = erlcloud_s3_multipart:list_uploads(?TEST_BUCKET, Options3, UserConfig), - CommonPrefixes2 = proplists:get_value(common_prefixes, UploadList4), - ?assert(lists:member([{prefix, "0/"}], CommonPrefixes2)), - verify_upload_list(UploadList4, Count2), - - %% @TODO Uncomment this block once support for `max-uploads' is done. - %% Use `key-marker' and `upload-id-marker' to request - %% remainder of in-progress upload results - %% Options3 = [{key_marker, "48"}, {upload_id_marker, "X"}], - %% UploadList4 = erlcloud_s3_multipart:list_uploads(?TEST_BUCKET, Options3, UserConfig), - %% verify_upload_list(UploadList4, 52, 100, 49), - - %% Abort all uploads for the bucket - abort_uploads(?TEST_BUCKET, UserConfig), - - lager:info("deleting bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:delete_bucket(?TEST_BUCKET, UserConfig)), - - ?assertError({aws_error, {http_error, 404, _, _}}, erlcloud_s3:list_objects(?TEST_BUCKET, UserConfig)), - rtcs:pass(). - -upload_and_assert_parts(Bucket, Key, UploadId, PartCount, Size, Config) -> - [{X, rtcs_multipart:upload_and_assert_part(Bucket, - Key, - UploadId, - X, - generate_part_data(X, Size), - Config)} - || X <- lists:seq(1, PartCount)]. - - -generate_part_data(X, Size) - when 0 =< X, X =< 255 -> - list_to_binary( - [X || _ <- lists:seq(1, Size)]). - -aborted_upload_test_case(Bucket, Key, Config) -> - %% Initiate a multipart upload - lager:info("Initiating multipart upload"), - InitUploadRes = erlcloud_s3_multipart:initiate_upload(Bucket, Key, [], [], Config), - UploadId = erlcloud_s3_multipart:upload_id(InitUploadRes), - - %% Verify the upload id is in list_uploads results and - %% that the bucket information is correct - UploadsList1 = erlcloud_s3_multipart:list_uploads(Bucket, [], Config), - Uploads1 = proplists:get_value(uploads, UploadsList1, []), - ?assertEqual(Bucket, proplists:get_value(bucket, UploadsList1)), - ?assert(upload_id_present(UploadId, Uploads1)), - - lager:info("Uploading parts"), - _EtagList = upload_and_assert_parts(Bucket, - Key, - UploadId, - ?PART_COUNT, - ?GOOD_PART_SIZE, - Config), - - %% List bucket contents and verify empty - ObjList1= erlcloud_s3:list_objects(Bucket, Config), - ?assertEqual([], proplists:get_value(contents, ObjList1)), - - %% Abort upload - lager:info("Aborting multipart upload"), - ?assertEqual(ok, erlcloud_s3_multipart:abort_upload(Bucket, - Key, - UploadId, - Config)), - - %% List uploads and verify upload id is no longer present - UploadsList2 = erlcloud_s3_multipart:list_uploads(Bucket, [], Config), - Uploads2 = proplists:get_value(uploads, UploadsList2, []), - ?assertNot(upload_id_present(UploadId, Uploads2)), - - %% List bucket contents and verify key is still not listed - ObjList2 = erlcloud_s3:list_objects(Bucket, Config), - ?assertEqual([], proplists:get_value(contents, ObjList2)). - -nonexistent_bucket_listing_test_case(Bucket, Config) -> - ?assertError({aws_error, {http_error, 404, _, _}}, erlcloud_s3_multipart:list_uploads(Bucket, [], Config)). - -invalid_part_number_test_case(Bucket, Key, Config) -> - InitUploadRes = erlcloud_s3_multipart:initiate_upload(Bucket, Key, [], [], Config), - UploadId = erlcloud_s3_multipart:upload_id(InitUploadRes), - InvalidPartNumber = ?DEFAULT_MAX_PART_NUMBER + 1, - {'EXIT', {{aws_error, {http_error, 400, _, Body}}, _Backtrace}} = - (catch erlcloud_s3_multipart:upload_part(Bucket, - Key, - UploadId, - InvalidPartNumber, - generate_part_data(0, ?GOOD_PART_SIZE), - Config)), - ErrorPattern = - "InvalidArgument" - "Part number must be an integer between 1 and 10000, inclusive", - ?assertMatch({match, _}, re:run(Body, ErrorPattern, [multiline])), - abort_uploads(Bucket, Config). - - -basic_upload_test_case(Bucket, Key, Config) -> - %% Initiate a multipart upload - lager:info("Initiating multipart upload"), - InitUploadRes = erlcloud_s3_multipart:initiate_upload(Bucket, Key, [], [], Config), - UploadId = erlcloud_s3_multipart:upload_id(InitUploadRes), - - %% Verify the upload id is in list_uploads results and - %% that the bucket information is correct - UploadsList1 = erlcloud_s3_multipart:list_uploads(Bucket, [], Config), - Uploads1 = proplists:get_value(uploads, UploadsList1, []), - ?assertEqual(Bucket, proplists:get_value(bucket, UploadsList1)), - ?assert(upload_id_present(UploadId, Uploads1)), - - lager:info("Uploading parts"), - EtagList = upload_and_assert_parts(Bucket, - Key, - UploadId, - ?PART_COUNT, - ?GOOD_PART_SIZE, - Config), - - %% List bucket contents and verify empty - ObjList1= erlcloud_s3:list_objects(Bucket, Config), - ?assertEqual([], proplists:get_value(contents, ObjList1)), - - %% Complete upload - lager:info("Completing multipart upload"), - - ?assertEqual(ok, erlcloud_s3_multipart:complete_upload(Bucket, - Key, - UploadId, - EtagList, - Config)), - - %% List uploads and verify upload id is no longer present - UploadsList2 = erlcloud_s3_multipart:list_uploads(Bucket, [], Config), - Uploads2 = proplists:get_value(uploads, UploadsList2, []), - ?assertNot(upload_id_present(UploadId, Uploads2)), - - %% List bucket contents and verify key is now listed - ObjList2 = erlcloud_s3:list_objects(Bucket, Config), - ?assertEqual([Key], - [proplists:get_value(key, O) || - O <- proplists:get_value(contents, ObjList2)]), - - %% Get the object: it better be what we expect - ExpectedObj = list_to_binary([generate_part_data(X, ?GOOD_PART_SIZE) || - X <- lists:seq(1, ?PART_COUNT)]), - GetRes = erlcloud_s3:get_object(Bucket, Key, Config), - ?assertEqual(ExpectedObj, proplists:get_value(content, GetRes)), - - %% Delete uploaded object - erlcloud_s3:delete_object(Bucket, Key, Config), - - %% List bucket contents and verify empty - ObjList3 = erlcloud_s3:list_objects(Bucket, Config), - ?assertEqual([], proplists:get_value(contents, ObjList3)). - -parts_too_small_test_case(Bucket, Key, Config) -> - %% Initiate a multipart upload - lager:info("Initiating multipart upload (bad)"), - InitUploadRes = erlcloud_s3_multipart:initiate_upload(Bucket, Key, [], [], Config), - UploadId = erlcloud_s3_multipart:upload_id(InitUploadRes), - - lager:info("Uploading parts (bad)"), - EtagList = upload_and_assert_parts(Bucket, - Key, - UploadId, - ?PART_COUNT, - ?BAD_PART_SIZE, - Config), - - %% Complete upload - lager:info("Completing multipart upload (bad)"), - - {'EXIT', {{aws_error, {http_error, 400, _, Body}}, _Backtrace}} = - (catch erlcloud_s3_multipart:complete_upload(Bucket, - Key, - UploadId, - EtagList, - Config)), - ?assertMatch({match, _}, - re:run(Body, "EntityTooSmall", [multiline])), - - Abort = fun() -> erlcloud_s3_multipart:abort_upload(Bucket, - Key, - UploadId, - Config) - end, - ?assertEqual(ok, Abort()), - ?assertError({aws_error, {http_error, 404, _, _}}, Abort()), - ok. - -initiate_uploads(Bucket, Count, Config) -> - initiate_uploads(Bucket, Count, [], Config). - -initiate_uploads(Bucket, Count, KeyPrefix, Config) -> - [erlcloud_s3_multipart:initiate_upload(Bucket, - KeyPrefix ++ integer_to_list(X), - "text/plain", - [], - Config) || X <- lists:seq(1, Count)]. - -verify_upload_list(UploadList, ExpectedCount) -> - verify_upload_list(UploadList, ExpectedCount, ExpectedCount, 1). - -%% verify_upload_list(UploadList, ExpectedCount, TotalCount) -> -%% verify_upload_list(UploadList, ExpectedCount, TotalCount, 1). - -verify_upload_list(UploadList, ExpectedCount, TotalCount, 1) - when ExpectedCount =:= TotalCount -> - ?assertEqual(lists:sort([integer_to_list(X) || X <- lists:seq(1, ExpectedCount)]), - [proplists:get_value(key, O) || - O <- proplists:get_value(uploads, UploadList)]); -verify_upload_list(UploadList, ExpectedCount, TotalCount, Offset) -> - ?assertEqual(lists:sublist( - lists:sort([integer_to_list(X) || X <- lists:seq(1, TotalCount)]), - Offset, - ExpectedCount), - [proplists:get_value(key, O) || - O <- proplists:get_value(uploads, UploadList)]). - -abort_and_verify_uploads(Bucket, 0, Config) -> - verify_upload_list(erlcloud_s3_multipart:list_uploads(Bucket, [], Config), 0), - ok; -abort_and_verify_uploads(Bucket, Count, Config) -> - UploadList = erlcloud_s3_multipart:list_uploads(Bucket, [], Config), - verify_upload_list(UploadList, Count), - Key = integer_to_list(Count), - UploadId = upload_id_for_key(Key, UploadList), - erlcloud_s3_multipart:abort_upload(Bucket, Key, UploadId, Config), - abort_and_verify_uploads(Bucket, Count-1, Config). - -upload_id_present(UploadId, UploadList) -> - [] /= [UploadData || UploadData <- UploadList, - proplists:get_value(upload_id, UploadData) =:= UploadId]. - -upload_id_for_key(Key, UploadList) -> - Uploads = proplists:get_value(uploads, UploadList), - [KeyUpload] = [UploadData || UploadData <- Uploads, - proplists:get_value(key, UploadData) =:= Key], - proplists:get_value(upload_id, KeyUpload). - -abort_uploads(Bucket, Config) -> - UploadList = erlcloud_s3_multipart:list_uploads(Bucket, [], Config), - [begin - Key = proplists:get_value(key, Upload), - UploadId = proplists:get_value(upload_id, Upload), - erlcloud_s3_multipart:abort_upload(Bucket, Key, UploadId, Config) - end || Upload <- proplists:get_value(uploads, UploadList)]. diff --git a/riak_test/tests/object_get_conditional_test.erl b/riak_test/tests/object_get_conditional_test.erl deleted file mode 100644 index aed9b645e..000000000 --- a/riak_test/tests/object_get_conditional_test.erl +++ /dev/null @@ -1,121 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(object_get_conditional_test). - -%% @doc `riak_test' module for testing conditional object get behavior. - --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). - -%% keys for non-multipart objects --define(TEST_BUCKET, "riak-test-bucket"). --define(TEST_KEY, "riak_test_key1"). --define(ETAG_NOTEXIST, "\"NoTeXiSt\""). - -confirm() -> - {UserConfig, {_RiakNodes, _CSNodes, _Stanchion}} = rtcs:setup(1), - - lager:info("User is valid on the cluster, and has no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(UserConfig)), - - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - - ?assertMatch([{buckets, [[{name, ?TEST_BUCKET}, _]]}], - erlcloud_s3:list_buckets(UserConfig)), - - {Content, Etag, ThreeDates} = - setup_object(?TEST_BUCKET, ?TEST_KEY, UserConfig), - lager:debug("Etag: ~p~n", [Etag]), - lager:debug("{Before, LastModified, After}: ~p~n", [ThreeDates]), - - last_modified_condition_test_cases(?TEST_BUCKET, ?TEST_KEY, - Content, ThreeDates, UserConfig), - match_condition_test_cases(?TEST_BUCKET, ?TEST_KEY, - Content, Etag, UserConfig), - rtcs:pass(). - -setup_object(Bucket, Key, UserConfig) -> - Content = crypto:rand_bytes(400), - erlcloud_s3:put_object(Bucket, Key, Content, UserConfig), - Obj = erlcloud_s3:get_object(Bucket, Key, UserConfig), - ?assertEqual(Content, proplists:get_value(content, Obj)), - Etag = proplists:get_value(etag, Obj), - {Before, LastModified, After} = before_and_after_of_last_modified(Obj), - {Content, Etag, {Before, LastModified, After}}. - -before_and_after_of_last_modified(Obj) -> - Headers = proplists:get_value(headers, Obj), - LastModified = proplists:get_value("Last-Modified", Headers), - LastModifiedErlDate = httpd_util:convert_request_date(LastModified), - LastModifiedSec = calendar:datetime_to_gregorian_seconds(LastModifiedErlDate), - Before = rfc1123_date(LastModifiedSec - 1), - After = rfc1123_date(LastModifiedSec + 1), - %% Sleep 1 sec because webmachine ignores if-modified-since header - %% if it is future date. - timer:sleep(1000), - {Before, LastModified, After}. - -rfc1123_date(GregorianSecs) -> - ErlDate = calendar:gregorian_seconds_to_datetime(GregorianSecs), - riak_cs_wm_utils:iso_8601_to_rfc_1123(riak_cs_wm_utils:iso_8601_datetime(ErlDate)). - -last_modified_condition_test_cases(Bucket, Key, ExpectedContent, - {Before, LastModified, After}, UserConfig) -> - normal_get_case(Bucket, Key, ExpectedContent, - [{if_modified_since, Before}], UserConfig), - not_modified_case(Bucket, Key, - [{if_modified_since, LastModified}], UserConfig), - not_modified_case(Bucket, Key, - [{if_modified_since, After}], UserConfig), - - normal_get_case(Bucket, Key, ExpectedContent, - [{if_unmodified_since, After}], UserConfig), - precondition_failed_case(Bucket, Key, - [{if_unmodified_since, Before}], UserConfig). - -match_condition_test_cases(Bucket, Key, ExpectedContent, - Etag, UserConfig) -> - normal_get_case(Bucket, Key, ExpectedContent, - [{if_match, Etag}], UserConfig), - normal_get_case(Bucket, Key, ExpectedContent, - [{if_match, Etag ++ ", " ++ ?ETAG_NOTEXIST}], UserConfig), - precondition_failed_case(Bucket, Key, - [{if_match, ?ETAG_NOTEXIST}], UserConfig), - - normal_get_case(Bucket, Key, ExpectedContent, - [{if_none_match, ?ETAG_NOTEXIST}], UserConfig), - not_modified_case(Bucket, Key, - [{if_none_match, Etag}], UserConfig), - not_modified_case(Bucket, Key, - [{if_none_match, Etag ++ ", " ++ ?ETAG_NOTEXIST}], UserConfig). - -normal_get_case(Bucket, Key, ExpectedContent, Options, UserConfig) -> - Obj = erlcloud_s3:get_object(Bucket, Key, Options, UserConfig), - ?assertEqual(ExpectedContent, proplists:get_value(content, Obj)). - -not_modified_case(Bucket, Key, Options, UserConfig) -> - ?assertError({aws_error, {http_error, 304, _, _Body}}, - erlcloud_s3:get_object(Bucket, Key, Options, UserConfig)). - -precondition_failed_case(Bucket, Key, Options, UserConfig) -> - ?assertError({aws_error, {http_error, 412, _, _Body}}, - erlcloud_s3:get_object(Bucket, Key, Options, UserConfig)). diff --git a/riak_test/tests/object_get_test.erl b/riak_test/tests/object_get_test.erl deleted file mode 100644 index 06324bcdc..000000000 --- a/riak_test/tests/object_get_test.erl +++ /dev/null @@ -1,283 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(object_get_test). - -%% @doc `riak_test' module for testing object get behavior. - --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). --include_lib("xmerl/include/xmerl.hrl"). - -%% keys for non-multipart objects --define(TEST_BUCKET, "riak-test-bucket"). --define(KEY_SINGLE_BLOCK, "riak_test_key1"). --define(KEY_MULTIPLE_BLOCK, "riak_test_key2"). - -%% keys for multipart uploaded objects --define(KEY_MP_TINY, "riak_test_mp_tiny"). % single part, single block --define(KEY_MP_SMALL, "riak_test_mp_small"). % single part, multiple blocks --define(KEY_MP_LARGE, "riak_test_mp_large"). % multiple parts - - -confirm() -> - {UserConfig, {_RiakNodes, _CSNodes, _Stanchion}} = rtcs:setup(1), - - lager:info("User is valid on the cluster, and has no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(UserConfig)), - - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - - ?assertMatch([{buckets, [[{name, ?TEST_BUCKET}, _]]}], - erlcloud_s3:list_buckets(UserConfig)), - - non_mp_get_cases(UserConfig), - mp_get_cases(UserConfig), - timestamp_skew_cases(UserConfig), - long_key_cases(UserConfig), - rtcs:pass(). - -non_mp_get_cases(UserConfig) -> - %% setup objects - SingleBlock = crypto:rand_bytes(400), - erlcloud_s3:put_object(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, SingleBlock, UserConfig), - MultipleBlock = crypto:rand_bytes(4000000), % not aligned to block boundary - erlcloud_s3:put_object(?TEST_BUCKET, ?KEY_MULTIPLE_BLOCK, MultipleBlock, UserConfig), - - %% basic GET test cases - basic_get_test_case(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, SingleBlock, UserConfig), - basic_get_test_case(?TEST_BUCKET, ?KEY_MULTIPLE_BLOCK, MultipleBlock, UserConfig), - - %% GET after nval=1 GET failure - rt_intercept:add(rtcs:cs_node(1), {riak_cs_block_server, [{{get_block_local, 6}, get_block_local_insufficient_vnode_at_nval1}]}), - Res = erlcloud_s3:get_object(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, UserConfig), - ?assertEqual(SingleBlock, proplists:get_value(content, Res)), - rt_intercept:clean(rtcs:cs_node(1), riak_cs_block_server), - - %% Range GET for single-block object test cases - [range_get_test_case(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, SingleBlock, - Range, UserConfig) - || Range <- [{10, 20}, - {0, none}, {none, 10}, - {0, 0}, {0, 99}, {0, 1000000}]], - - %% Range GET for multiple-block object test cases - [range_get_test_case(?TEST_BUCKET, ?KEY_MULTIPLE_BLOCK, MultipleBlock, - {0, End}, UserConfig) - || End <- [mb(1)-2, mb(1)-1, mb(1), mb(1)+1]], - [range_get_test_case(?TEST_BUCKET, ?KEY_MULTIPLE_BLOCK, MultipleBlock, - {Start, mb(2)}, UserConfig) - || Start <- [mb(1)-2, mb(1)-1, mb(1), mb(1)+1]], - range_get_test_case(?TEST_BUCKET, ?KEY_MULTIPLE_BLOCK, MultipleBlock, - {100, mb(3)}, UserConfig), - range_get_test_case(?TEST_BUCKET, ?KEY_MULTIPLE_BLOCK, MultipleBlock, - {0, none}, UserConfig), - - %% Multiple ranges, CS returns whole resources. - multiple_range_get_test_case(?TEST_BUCKET, ?KEY_MULTIPLE_BLOCK, MultipleBlock, - [{10, 20}, {30, 50}], UserConfig), - multiple_range_get_test_case(?TEST_BUCKET, ?KEY_MULTIPLE_BLOCK, MultipleBlock, - [{10, 50}, {20, 50}], UserConfig), - - %% Invalid ranges - LastPosExceeded = byte_size(MultipleBlock), - invalid_range_get_test_case(?TEST_BUCKET, ?KEY_MULTIPLE_BLOCK, MultipleBlock, - [{LastPosExceeded, LastPosExceeded + 10}], UserConfig), - invalid_range_get_test_case(?TEST_BUCKET, ?KEY_MULTIPLE_BLOCK, MultipleBlock, - [{20, 10}], UserConfig), - ok. - -mp_get_cases(UserConfig) -> - TinyContent = multipart_upload(?TEST_BUCKET, ?KEY_MP_TINY, - [400], UserConfig), - SmallContent = multipart_upload(?TEST_BUCKET, ?KEY_MP_SMALL, - [mb(3)], UserConfig), - LargeContent = multipart_upload(?TEST_BUCKET, ?KEY_MP_LARGE, - [mb(10), mb(5), mb(9) + 123, mb(6), 400], % 30MB + 523 B - UserConfig), - - %% Range GET for single part / single block - [range_get_test_case(?TEST_BUCKET, ?KEY_MP_TINY, TinyContent, - Range, UserConfig) - || Range <- [{10, 20}, - {0, none}, {none, 10}, - {0, 0}, {0, 99}, {0, 1000000}]], - - %% Range GET for single part / multiple blocks - [range_get_test_case(?TEST_BUCKET, ?KEY_MP_SMALL, SmallContent, - {Start, End}, UserConfig) - || Start <- [100, mb(1)+100], - End <- [mb(1)+100, mb(3)-1, mb(4)]], - - %% Range GET for multiple parts / multiple blocks - [range_get_test_case(?TEST_BUCKET, ?KEY_MP_LARGE, LargeContent, - {Start, End}, UserConfig) - || Start <- [mb(1), mb(16)], - End <- [mb(16), mb(30), mb(30) + 500, mb(1000)]], - ok. - -timestamp_skew_cases(UserConfig) -> - BucketName = "timestamp-skew-cases", - KeyName = "timestamp-skew-cases", - Data = <<"bark! bark! bark!!!">>, - ?assertEqual(ok, erlcloud_s3:create_bucket(BucketName, UserConfig)), - erlcloud_s3:put_object(BucketName, KeyName, Data, UserConfig), - - meck:new(httpd_util, [passthrough]), - %% To emulate clock skew, override erlang:localtime/0 to - %% enable random walk time, as long as erlcloud_s3 uses - %% httpd_util:rfc1123_date/1 for generating timestamp of - %% HTTP request header. - %% `Date = httpd_util:rfc1123_date(erlang:localtime()),` - meck:expect(httpd_util, rfc1123_date, - fun(Localtime) -> - Seconds = calendar:datetime_to_gregorian_seconds(Localtime), - SkewedTime = calendar:gregorian_seconds_to_datetime(Seconds - 987), - Date = meck:passthrough([SkewedTime]), - lager:info("Clock skew: ~p => ~p => ~p", [Localtime, SkewedTime, Date]), - Date - end), - try - erlcloud_s3:get_object(BucketName, KeyName, UserConfig) - catch - error:{aws_error, {http_error, 403, _, Body0}} -> - Body = unicode:characters_to_list(Body0), - #xmlElement{name = 'Error'} = XML = element(1,xmerl_scan:string(Body)), - ?assertEqual("RequestTimeTooSkewed", erlcloud_xml:get_text("/Error/Code", XML)), - ErrMsg = "The difference between the request time and the current time is too large.", - ?assertEqual(ErrMsg, erlcloud_xml:get_text("/Error/Message", XML)); - E:R -> - lager:error("~p:~p", [E, R]), - ?assert(false) - after - meck:unload(httpd_util) - end. - -long_key_cases(UserConfig) -> - LongKey = binary_to_list(binary:copy(<<"a">>, 1024)), - TooLongKey = binary_to_list(binary:copy(<<"b">>, 1025)), - Data = <<"pocketburger">>, - ?assertEqual([{version_id,"null"}], - erlcloud_s3:put_object(?TEST_BUCKET, LongKey, Data, UserConfig)), - ErrorString = "" - "KeyTooLongErrorYour key is too long1025" - "1024", - ?assertError({aws_error, {http_error, 400, [], ErrorString}}, - erlcloud_s3:put_object(?TEST_BUCKET, TooLongKey, Data, UserConfig)). - -mb(MegaBytes) -> - MegaBytes * 1024 * 1024. - -basic_get_test_case(Bucket, Key, ExpectedContent, Config) -> - Obj = erlcloud_s3:get_object(Bucket, Key, Config), - assert_whole_content(ExpectedContent, Obj). - -assert_whole_content(ExpectedContent, ResultObj) -> - Content = proplists:get_value(content, ResultObj), - ContentLength = proplists:get_value(content_length, ResultObj), - ?assertEqual(byte_size(ExpectedContent), list_to_integer(ContentLength)), - ?assertEqual(byte_size(ExpectedContent), byte_size(Content)), - ?assertEqual(ExpectedContent, Content). - -range_get_test_case(Bucket, Key, WholeContent, {Start, End}, Config) -> - Range = format_ranges([{Start, End}]), - Obj = erlcloud_s3:get_object(Bucket, Key, [{range, Range}], Config), - Content = proplists:get_value(content, Obj), - ContentLength = proplists:get_value(content_length, Obj), - WholeSize = byte_size(WholeContent), - {Skip, Length} = range_skip_length({Start, End}, WholeSize), - ?assertEqual(Length, list_to_integer(ContentLength)), - ?assertEqual(Length, byte_size(Content)), - assert_content_range(Skip, Length, WholeSize, Obj), - ExpectedContent = binary:part(WholeContent, Skip, Length), - ?assertEqual(ExpectedContent, Content). - -multiple_range_get_test_case(Bucket, Key, WholeContent, Ranges, Config) -> - RangeValue = format_ranges(Ranges), - Obj = erlcloud_s3:get_object(Bucket, Key, [{range, RangeValue}], Config), - assert_whole_content(WholeContent, Obj). - -invalid_range_get_test_case(Bucket, Key, _WholeContent, Ranges, Config) -> - RangeValue = format_ranges(Ranges), - {'EXIT', {{aws_error, {http_error, 416, _, Body}}, _Backtrace}} = - (catch erlcloud_s3:get_object(Bucket, Key, [{range, RangeValue}], Config)), - ?assertMatch({match, _}, - re:run(Body, "InvalidRange", [multiline])). - -format_ranges(Ranges) -> - Formatted = [format_range(Range) || Range <- Ranges], - io_lib:format("bytes=~s", [string:join(Formatted, ",")]). - -format_range(Range) -> - RangeStr = case Range of - {none, End} -> - io_lib:format("-~B", [End]); - {Start, none} -> - io_lib:format("~B-", [Start]); - {Start, End} -> - io_lib:format("~B-~B", [Start, End]) - end, - lists:flatten(RangeStr). - -assert_content_range(Skip, Length, Size, Obj) -> - Expected = lists:flatten( - io_lib:format("bytes ~B-~B/~B", [Skip, Skip + Length - 1, Size])), - Headers = proplists:get_value(headers, Obj), - ContentRange = proplists:get_value("Content-Range", Headers), - ?assertEqual(Expected, ContentRange). - -%% TODO: riak_test includes its own mochiweb by escriptizing. -%% End position which is lager than size is fixed on the branch 1.5 of basho/mochweb: -%% https://github.com/basho/mochiweb/commit/38992be7822ddc1b8e6f318ba8e73fc8c0b7fd22 -%% Accept range end position which exceededs the resource size -%% After mochiweb is tagged, change riakhttpc and webmachine's deps to the tag. -%% So this function should be removed and replaced by mochiweb_http:range_skip_length/2. -range_skip_length(Spec, Size) -> - case Spec of - {Start, End} when is_integer(Start), is_integer(End), - 0 =< Start, Start < Size, Size =< End -> - {Start, Size - Start}; - _ -> - mochiweb_http:range_skip_length(Spec, Size) - end. - -multipart_upload(Bucket, Key, Sizes, Config) -> - InitRes = erlcloud_s3_multipart:initiate_upload( - Bucket, Key, "text/plain", [], Config), - UploadId = erlcloud_xml:get_text( - "/InitiateMultipartUploadResult/UploadId", InitRes), - Content = upload_parts(Bucket, Key, UploadId, Config, 1, Sizes, [], []), - basic_get_test_case(Bucket, Key, Content, Config), - Content. - -upload_parts(Bucket, Key, UploadId, Config, _PartCount, [], Contents, Parts) -> - ?assertEqual(ok, erlcloud_s3_multipart:complete_upload( - Bucket, Key, UploadId, lists:reverse(Parts), Config)), - iolist_to_binary(lists:reverse(Contents)); -upload_parts(Bucket, Key, UploadId, Config, PartCount, [Size | Sizes], Contents, Parts) -> - Content = crypto:rand_bytes(Size), - {RespHeaders, _UploadRes} = erlcloud_s3_multipart:upload_part( - Bucket, Key, UploadId, PartCount, Content, Config), - PartEtag = proplists:get_value("ETag", RespHeaders), - lager:debug("UploadId: ~p~n", [UploadId]), - lager:debug("PartEtag: ~p~n", [PartEtag]), - upload_parts(Bucket, Key, UploadId, Config, PartCount + 1, - Sizes, [Content | Contents], [{PartCount, PartEtag} | Parts]). diff --git a/riak_test/tests/put_copy_test.erl b/riak_test/tests/put_copy_test.erl deleted file mode 100644 index 96799701a..000000000 --- a/riak_test/tests/put_copy_test.erl +++ /dev/null @@ -1,355 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- --module(put_copy_test). - --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). - --include_lib("erlcloud/include/erlcloud_aws.hrl"). - --define(assert403(X), - ?assertError({aws_error, {http_error, 403, _, _}}, (X))). --define(assertHeader(Key, Expected, Props), - ?assertEqual(Expected, - proplists:get_value(Key, proplists:get_value(headers, Props)))). - --define(BUCKET, "put-copy-bucket-test"). --define(KEY, "pocket"). - --define(DATA0, <<"pocket">>). - --define(BUCKET2, "put-target-bucket"). --define(KEY2, "sidepocket"). --define(KEY3, "superpocket"). --define(BUCKET3, "the-other-put-target-bucket"). --define(BUCKET4, "no-cl-bucket"). --define(SRC_KEY, "source"). --define(TGT_KEY, "target"). --define(MP_TGT_KEY, "mp-target"). --define(REPLACE_KEY, "replace-target"). - -confirm() -> - {UserConfig, {RiakNodes, _CSNodes, _}} = rtcs:setup(1), - ?assertEqual(ok, erlcloud_s3:create_bucket(?BUCKET, UserConfig)), - Data = ?DATA0, - ?assertEqual([{version_id, "null"}], - erlcloud_s3:put_object(?BUCKET, ?KEY, Data, UserConfig)), - ?assertEqual([{version_id, "null"}], - erlcloud_s3:put_object(?BUCKET, ?KEY2, Data, UserConfig)), - - RiakNode = hd(RiakNodes), - UserConfig2 = rtcs_admin:create_user(RiakNode, 1), - UserConfig3 = rtcs_admin:create_user(RiakNode, 1), - - ?assertEqual(ok, erlcloud_s3:create_bucket(?BUCKET2, UserConfig)), - ?assertEqual(ok, erlcloud_s3:create_bucket(?BUCKET3, UserConfig2)), - - ok = verify_simple_copy(UserConfig), - ok = verify_others_copy(UserConfig, UserConfig2), - ok = verify_multipart_copy(UserConfig), - ok = verify_security(UserConfig, UserConfig2, UserConfig3), - ok = verify_source_not_found(UserConfig), - ok = verify_replace_usermeta(UserConfig), - ok = verify_without_cl_header(UserConfig), - - ?assertEqual([{delete_marker, false}, {version_id, "null"}], - erlcloud_s3:delete_object(?BUCKET, ?KEY, UserConfig)), - ?assertEqual([{delete_marker, false}, {version_id, "null"}], - erlcloud_s3:delete_object(?BUCKET, ?KEY2, UserConfig)), - ?assertEqual([{delete_marker, false}, {version_id, "null"}], - erlcloud_s3:delete_object(?BUCKET2, ?KEY2, UserConfig)), - ?assertEqual([{delete_marker, false}, {version_id, "null"}], - erlcloud_s3:delete_object(?BUCKET3, ?KEY, UserConfig2)), - rtcs:pass(). - - -verify_simple_copy(UserConfig) -> - - ?assertEqual([{copy_source_version_id, "false"}, {version_id, "null"}], - erlcloud_s3:copy_object(?BUCKET2, ?KEY2, ?BUCKET, ?KEY, UserConfig)), - - Props = erlcloud_s3:get_object(?BUCKET2, ?KEY2, UserConfig), - lager:debug("copied object: ~p", [Props]), - ?assertEqual(?DATA0, proplists:get_value(content, Props)), - - ok. - - -verify_others_copy(UserConfig, OtherUserConfig) -> - %% try copy to fail, because no permission - ?assert403(erlcloud_s3:copy_object(?BUCKET3, ?KEY, ?BUCKET, ?KEY, OtherUserConfig)), - - %% set key public - Acl = [{acl, public_read}], - ?assertEqual([{version_id,"null"}], - erlcloud_s3:put_object(?BUCKET, ?KEY, ?DATA0, - Acl, [], UserConfig)), - - %% make sure observable from Other - Props = erlcloud_s3:get_object(?BUCKET, ?KEY, OtherUserConfig), - ?assertEqual(?DATA0, proplists:get_value(content, Props)), - - %% try copy - ?assertEqual([{copy_source_version_id, "false"}, {version_id, "null"}], - erlcloud_s3:copy_object(?BUCKET3, ?KEY, ?BUCKET, ?KEY, OtherUserConfig)), - - Props2 = erlcloud_s3:get_object(?BUCKET3, ?KEY, OtherUserConfig), - lager:debug("copied object: ~p", [Props2]), - ?assertEqual(?DATA0, proplists:get_value(content, Props2)), - ok. - -verify_multipart_copy(UserConfig) -> - %% ~6MB iolist - MillionPockets = binary:copy(<<"pocket">>, 1000000), - MillionBurgers = binary:copy(<<"burger">>, 1000000), - - ?assertEqual([{version_id,"null"}], - erlcloud_s3:put_object(?BUCKET, ?KEY, MillionPockets, UserConfig)), - ?assertEqual([{version_id,"null"}], - erlcloud_s3:put_object(?BUCKET, ?KEY2, MillionBurgers, UserConfig)), - - InitUploadRes=erlcloud_s3_multipart:initiate_upload(?BUCKET2, ?KEY3, "text/plain", [], UserConfig), - UploadId = erlcloud_s3_multipart:upload_id(InitUploadRes), - lager:info("~p ~p", [InitUploadRes, UploadId]), - - {RespHeaders1, _} = rtcs_multipart:upload_part_copy(?BUCKET2, ?KEY3, UploadId, 1, - ?BUCKET, ?KEY, UserConfig), - lager:debug("RespHeaders1: ~p", [RespHeaders1]), - Etag1 = rtcs_multipart:assert_part(?BUCKET2, ?KEY3, UploadId, 1, UserConfig, RespHeaders1), - - {RespHeaders2, _} = rtcs_multipart:upload_part_copy(?BUCKET2, ?KEY3, UploadId, 2, - ?BUCKET, ?KEY2, UserConfig), - lager:debug("RespHeaders2: ~p", [RespHeaders2]), - Etag2 = rtcs_multipart:assert_part(?BUCKET2, ?KEY3, UploadId, 2, UserConfig, RespHeaders2), - - List = erlcloud_s3_multipart:list_uploads(?BUCKET2, [], UserConfig), - lager:debug("List: ~p", [List]), - - EtagList = [ {1, Etag1}, {2, Etag2} ], - ?assertEqual(ok, erlcloud_s3_multipart:complete_upload(?BUCKET2, - ?KEY3, - UploadId, - EtagList, - UserConfig)), - - UploadsList2 = erlcloud_s3_multipart:list_uploads(?BUCKET2, [], UserConfig), - Uploads2 = proplists:get_value(uploads, UploadsList2, []), - ?assertNot(mp_upload_test:upload_id_present(UploadId, Uploads2)), - - MillionPocketBurgers = iolist_to_binary([MillionPockets, MillionBurgers]), - Props = erlcloud_s3:get_object(?BUCKET2, ?KEY3, UserConfig), - %% lager:debug("~p> Props => ~p", [?LINE, Props]), - ?assertEqual(MillionPocketBurgers, proplists:get_value(content, Props)), - ok. - -verify_security(Alice, Bob, Charlie) -> - AlicesBucket = "alice", - AlicesPublicBucket = "alice-public", - AlicesObject = "alices-secret-note", - AlicesPublicObject = "alices-public-note", - - BobsBucket = "bob", - BobsObject = "bobs-secret-note", - - CharliesBucket = "charlie", - - %% setup Alice's data - ?assertEqual(ok, erlcloud_s3:create_bucket(AlicesBucket, Alice)), - ?assertEqual(ok, erlcloud_s3:create_bucket(AlicesPublicBucket, public_read_write, Alice)), - - ?assertEqual([{version_id, "null"}], - erlcloud_s3:put_object(AlicesBucket, AlicesObject, - <<"I'm here!!">>, Alice)), - ?assertEqual([{version_id, "null"}], - erlcloud_s3:put_object(AlicesBucket, AlicesPublicObject, - <<"deadbeef">>, [{acl, public_read}], Alice)), - ?assertEqual([{version_id, "null"}], - erlcloud_s3:put_object(AlicesPublicBucket, AlicesObject, - <<"deadly public beef">>, Alice)), - - %% setup Bob's box - ?assertEqual(ok, erlcloud_s3:create_bucket(BobsBucket, Bob)), - ?assertEqual([{version_id, "null"}], - erlcloud_s3:put_object(BobsBucket, BobsObject, - <<"bobfat">>, Bob)), - - %% >> setup Charlie's box - ?assertEqual(ok, erlcloud_s3:create_bucket(CharliesBucket, Charlie)), - - %% >> Bob can do it right - %% Bring Alice's objects to Bob's bucket - ?assert403(erlcloud_s3:copy_object(BobsBucket, AlicesObject, - AlicesBucket, AlicesObject, Bob)), - - ?assertEqual([{copy_source_version_id, "false"}, {version_id, "null"}], - erlcloud_s3:copy_object(BobsBucket, AlicesPublicObject, - AlicesBucket, AlicesPublicObject, Bob)), - - %% TODO: put to public bucket is degrated for now - %% ?assertEqual([{copy_source_version_id, "false"}, {version_id, "null"}], - %% erlcloud_s3:copy_object(BobsBucket, AlicesObject, - %% AlicesPublicBucket, AlicesObject, Bob)), - - %% Bring Bob's object to Alice's bucket - ?assertEqual([{copy_source_version_id, "false"}, {version_id, "null"}], - erlcloud_s3:copy_object(AlicesPublicBucket, BobsObject, - BobsBucket, BobsObject, Bob)), - %% Cleanup Bob's - ?assertEqual([{delete_marker, false}, {version_id, "null"}], - erlcloud_s3:delete_object(BobsBucket, AlicesPublicObject, Bob)), - ?assertEqual([{delete_marker, false}, {version_id, "null"}], - erlcloud_s3:delete_object(BobsBucket, AlicesObject, Bob)), - %% ?assertEqual([{delete_marker, false}, {version_id, "null"}], - %% erlcloud_s3:delete_object(AlicesPublicObject, BobsObject, Bob)), - - %% >> Charlie can't do it - %% try copying Alice's private object to Charlie's - ?assert403(erlcloud_s3:copy_object(CharliesBucket, AlicesObject, - AlicesBucket, AlicesObject, Charlie)), - - ?assert403(erlcloud_s3:copy_object(AlicesPublicBucket, AlicesObject, - AlicesBucket, AlicesObject, Charlie)), - - %% try copy Alice's public object to Bob's - ?assert403(erlcloud_s3:copy_object(BobsBucket, AlicesPublicObject, - AlicesBucket, AlicesPublicObject, Charlie)), - ?assert403(erlcloud_s3:copy_object(BobsBucket, AlicesObject, - AlicesPublicBucket, AlicesObject, Charlie)), - - %% charlie tries to copy anonymously, which should fail in 403 - CSPort = Charlie#aws_config.s3_port, - URL = lists:flatten(io_lib:format("http://~s.~s:~p/~s", - [AlicesPublicBucket, Charlie#aws_config.s3_host, - CSPort, AlicesObject])), - Headers = [{"x-amz-copy-source", string:join([AlicesBucket, AlicesObject], "/")}, - {"Content-Length", 0}], - {ok, Status, Hdr, _Msg} = ibrowse:send_req(URL, Headers, put, [], - Charlie#aws_config.http_options), - lager:debug("request ~p ~p => ~p ~p", [URL, Headers, Status, Hdr]), - ?assertEqual("403", Status), - - ok. - -verify_source_not_found(UserConfig) -> - NonExistingKey = "non-existent-source", - {'EXIT', {{aws_error, {http_error, 404, _, ErrorXml}}, _Stack}} = - (catch erlcloud_s3:copy_object(?BUCKET2, ?KEY2, - ?BUCKET, NonExistingKey, UserConfig)), - lager:debug("ErrorXml: ~s", [ErrorXml]), - ?assert(string:str(ErrorXml, - "/" ++ ?BUCKET ++ - "/" ++ NonExistingKey ++ "") > 0). - -verify_replace_usermeta(UserConfig) -> - lager:info("Verify replacing usermeta using Put Copy"), - - %% Put Initial Object - Headers0 = [{"Content-Type", "text/plain"}], - Options0 = [{meta, [{"key1", "val1"}, {"key2", "val2"}]}], - ?assertEqual([{version_id, "null"}], - erlcloud_s3:put_object(?BUCKET, ?REPLACE_KEY, ?DATA0, Options0, Headers0, UserConfig)), - Props0 = erlcloud_s3:get_object(?BUCKET, ?REPLACE_KEY, UserConfig), - lager:debug("Initial Obj: ~p", [Props0]), - ?assertEqual("text/plain", proplists:get_value(content_type, Props0)), - ?assertHeader("x-amz-meta-key1", "val1", Props0), - ?assertHeader("x-amz-meta-key2", "val2", Props0), - - %% Replace usermeta using Put Copy - Headers1 = [{"Content-Type", "application/octet-stream"}], - Options1 = [{metadata_directive, "REPLACE"}, - {meta, [{"key3", "val3"}, {"key4", "val4"}]}], - ?assertEqual([{copy_source_version_id, "false"}, {version_id, "null"}], - erlcloud_s3:copy_object(?BUCKET, ?REPLACE_KEY, ?BUCKET, ?REPLACE_KEY, - Options1, Headers1, UserConfig)), - Props1 = erlcloud_s3:get_object(?BUCKET, ?REPLACE_KEY, UserConfig), - lager:debug("Updated Obj: ~p", [Props1]), - ?assertEqual(?DATA0, proplists:get_value(content, Props1)), - ?assertEqual("application/octet-stream", proplists:get_value(content_type, Props1)), - ?assertHeader("x-amz-meta-key1", undefined, Props1), - ?assertHeader("x-amz-meta-key2", undefined, Props1), - ?assertHeader("x-amz-meta-key3", "val3", Props1), - ?assertHeader("x-amz-meta-key4", "val4", Props1), - ok. - - -%% Verify reuqests without Content-Length header, they should succeed. -%% To avoid automatic Content-Length header addition by HTTP client library, -%% this test uses `curl' command line utitlity, intended. -verify_without_cl_header(UserConfig) -> - ?assertEqual(ok, erlcloud_s3:create_bucket(?BUCKET4, UserConfig)), - Data = ?DATA0, - ?assertEqual([{version_id, "null"}], - erlcloud_s3:put_object(?BUCKET4, ?SRC_KEY, Data, UserConfig)), - verify_without_cl_header(UserConfig, normal, Data), - verify_without_cl_header(UserConfig, mp, Data), - ok. - -verify_without_cl_header(UserConfig, normal, Data) -> - lager:info("Verify basic (non-MP) PUT copy without Content-Length header"), - Target = fmt("/~s/~s", [?BUCKET4, ?TGT_KEY]), - Source = fmt("/~s/~s", [?BUCKET4, ?SRC_KEY]), - _Res = exec_curl(UserConfig, "PUT", Target, [{"x-amz-copy-source", Source}]), - - Props = erlcloud_s3:get_object(?BUCKET4, ?TGT_KEY, UserConfig), - ?assertEqual(Data, proplists:get_value(content, Props)), - ok; -verify_without_cl_header(UserConfig, mp, Data) -> - lager:info("Verify Multipart upload copy without Content-Length header"), - InitUploadRes = erlcloud_s3_multipart:initiate_upload( - ?BUCKET4, ?MP_TGT_KEY, "application/octet-stream", - [], UserConfig), - UploadId = erlcloud_s3_multipart:upload_id(InitUploadRes), - lager:info("~p ~p", [InitUploadRes, UploadId]), - Source = fmt("/~s/~s", [?BUCKET4, ?SRC_KEY]), - MpTarget = fmt("/~s/~s?partNumber=1&uploadId=~s", [?BUCKET4, ?MP_TGT_KEY, UploadId]), - _Res = exec_curl(UserConfig, "PUT", MpTarget, - [{"x-amz-copy-source", Source}, - {"x-amz-copy-source-range", "bytes=1-2"}]), - - ListPartsXml = erlcloud_s3_multipart:list_parts(?BUCKET4, ?MP_TGT_KEY, UploadId, [], UserConfig), - lager:debug("ListParts: ~p", [ListPartsXml]), - ListPartsRes = erlcloud_s3_multipart:parts_to_term(ListPartsXml), - Parts = proplists:get_value(parts, ListPartsRes), - EtagList = [{PartNum, Etag} || {PartNum, [{etag, Etag}, {size, _Size}]} <- Parts], - lager:debug("EtagList: ~p", [EtagList]), - ?assertEqual(ok, erlcloud_s3_multipart:complete_upload( - ?BUCKET4, ?MP_TGT_KEY, UploadId, EtagList, UserConfig)), - Props = erlcloud_s3:get_object(?BUCKET4, ?MP_TGT_KEY, UserConfig), - ExpectedBody = binary:part(Data, 1, 2), - ?assertEqual(ExpectedBody, proplists:get_value(content, Props)), - ok. - -exec_curl(#aws_config{s3_port=Port} = UserConfig, Method, Resource, AmzHeaders) -> - ContentType = "application/octet-stream", - Date = httpd_util:rfc1123_date(), - Auth = rtcs_admin:make_authorization(Method, Resource, ContentType, UserConfig, Date, - AmzHeaders), - HeaderArgs = [fmt("-H '~s: ~s' ", [K, V]) || - {K, V} <- [{"Date", Date}, {"Authorization", Auth}, - {"Content-Type", ContentType} | AmzHeaders]], - Cmd="curl -X " ++ Method ++ " -v -s " ++ HeaderArgs ++ - "'http://127.0.0.1:" ++ integer_to_list(Port) ++ Resource ++ "'", - lager:debug("Curl command line: ~s", [Cmd]), - Res = os:cmd(Cmd), - lager:debug("Curl result: ~s", [Res]), - Res. - -fmt(Fmt, Args) -> - lists:flatten(io_lib:format(Fmt, Args)). diff --git a/riak_test/tests/regression_tests.erl b/riak_test/tests/regression_tests.erl deleted file mode 100644 index 55dbcc653..000000000 --- a/riak_test/tests/regression_tests.erl +++ /dev/null @@ -1,234 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(regression_tests). - -%% @doc this module gathers various regression tests which can be -%% separate easily. Regression tests which needs configuration change -%% can be written as different module. In case of rtcs:setup(1) with -%% vanilla CS setup used. Otherwise feel free to create an independent -%% module like cs743_regression_test. - --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). --include("riak_cs.hrl"). - --define(TEST_BUCKET_CS347, "test-bucket-cs347"). - -confirm() -> - {UserConfig, _} = SetupInfo = rtcs:setup(1), - - ok = verify_cs296(SetupInfo, "test-bucket-cs296"), - ok = verify_cs347(SetupInfo, "test-bucket-cs347"), - ok = verify_cs436(SetupInfo, "test-bucket-cs436"), - ok = verify_cs512(UserConfig, "test-bucket-cs512"), - ok = verify_cs770(SetupInfo, "test-bucket-cs770"), - - %% Append your next regression tests here - - rtcs:pass(). - -%% @doc Regression test for `riak_cs' -%% issue 296. The issue description is: 403 instead of 404 returned when -%% trying to list nonexistent bucket. -verify_cs296(_SetupInfo = {UserConfig, {_RiakNodes, _CSNodes, _Stanchion}}, BucketName) -> - lager:info("CS296: User is valid on the cluster, and has no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(UserConfig)), - - ?assertError({aws_error, {http_error, 404, _, _}}, erlcloud_s3:list_objects(BucketName, UserConfig)), - - lager:info("creating bucket ~p", [BucketName]), - ?assertEqual(ok, erlcloud_s3:create_bucket(BucketName, UserConfig)), - - ?assertMatch([{buckets, [[{name, BucketName}, _]]}], - erlcloud_s3:list_buckets(UserConfig)), - - lager:info("deleting bucket ~p", [BucketName]), - ?assertEqual(ok, erlcloud_s3:delete_bucket(BucketName, UserConfig)), - - ?assertError({aws_error, {http_error, 404, _, _}}, erlcloud_s3:list_objects(BucketName, UserConfig)), - ok. - -%% @doc Regression test for `riak_cs' -%% issue 347. The issue description is: No response body in 404 to the -%% bucket that have never been created once. -verify_cs347(_SetupInfo = {UserConfig, {_RiakNodes, _CSNodes, _Stanchion}}, BucketName) -> - - lager:info("CS347: User is valid on the cluster, and has no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(UserConfig)), - - ListObjectRes1 = - case catch erlcloud_s3:list_objects(BucketName, UserConfig) of - {'EXIT', {{aws_error, Error}, _}} -> - Error; - Result -> - Result - end, - ?assert(rtcs:check_no_such_bucket(ListObjectRes1, "/" ++ ?TEST_BUCKET_CS347 ++ "/")), - - lager:info("creating bucket ~p", [BucketName]), - ?assertEqual(ok, erlcloud_s3:create_bucket(BucketName, UserConfig)), - - ?assertMatch([{buckets, [[{name, BucketName}, _]]}], - erlcloud_s3:list_buckets(UserConfig)), - - lager:info("deleting bucket ~p", [BucketName]), - ?assertEqual(ok, erlcloud_s3:delete_bucket(BucketName, UserConfig)), - - ListObjectRes2 = - case catch erlcloud_s3:list_objects(BucketName, UserConfig) of - {'EXIT', {{aws_error, Error2}, _}} -> - Error2; - Result2 -> - Result2 - end, - ?assert(rtcs:check_no_such_bucket(ListObjectRes2, "/" ++ ?TEST_BUCKET_CS347 ++ "/")), - ok. - - -%% @doc Regression test for `riak_cs' -%% issue 436. The issue description is: A 500 is returned instead of a 404 when -%% trying to put to a nonexistent bucket. -verify_cs436(_SetupInfo = {UserConfig, {_RiakNodes, _CSNodes, _Stanchion}}, BucketName) -> - lager:info("CS436: User is valid on the cluster, and has no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(UserConfig)), - - ?assertError({aws_error, {http_error, 404, _, _}}, - erlcloud_s3:put_object(BucketName, - "somekey", - crypto:rand_bytes(100), - UserConfig)), - - %% Create and delete test bucket - lager:info("creating bucket ~p", [BucketName]), - ?assertEqual(ok, erlcloud_s3:create_bucket(BucketName, UserConfig)), - - ?assertMatch([{buckets, [[{name, BucketName}, _]]}], - erlcloud_s3:list_buckets(UserConfig)), - - lager:info("deleting bucket ~p", [BucketName]), - ?assertEqual(ok, erlcloud_s3:delete_bucket(BucketName, UserConfig)), - - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(UserConfig)), - - %% Attempt to put object again and ensure result is still 404 - ?assertError({aws_error, {http_error, 404, _, _}}, - erlcloud_s3:put_object(BucketName, - "somekey", - crypto:rand_bytes(100), - UserConfig)), - ok. - --define(KEY, "cs512-key"). - -verify_cs512(UserConfig, BucketName) -> - %% {ok, UserConfig} = setup(), - ?assertEqual(ok, erlcloud_s3:create_bucket(BucketName, UserConfig)), - put_and_get(UserConfig, BucketName, <<"OLD">>), - put_and_get(UserConfig, BucketName, <<"NEW">>), - delete(UserConfig, BucketName), - assert_notfound(UserConfig,BucketName), - ok. - -verify_cs770({UserConfig, {RiakNodes, _, _}}, BucketName) -> - %% put object and cancel it; - ?assertEqual(ok, erlcloud_s3:create_bucket(BucketName, UserConfig)), - Key = "foobar", - lager:debug("starting cs770 verification: ~s ~s", [BucketName, Key]), - - {ok, Socket} = rtcs_object:upload(UserConfig, - {normal_partial, 3*1024*1024, 1024*1024}, - BucketName, Key), - - [[{UUID, M}]] = get_manifests(RiakNodes, BucketName, Key), - - %% Even if CS is smart enough to remove canceled upload, at this - %% time the socket will be still alive, so no cancellation logic - %% shouldn't be triggerred. - ?assertEqual(writing, M?MANIFEST.state), - lager:debug("UUID of ~s ~s: ~p", [BucketName, Key, UUID]), - - %% Emulate socket error with {error, closed} at server - ok = gen_tcp:close(Socket), - %% This wait is just for convenience - timer:sleep(1000), - rt:wait_until(fun() -> - [[{UUID, Mx}]] = get_manifests(RiakNodes, BucketName, Key), - scheduled_delete =:= Mx?MANIFEST.state - end, 8, 4096), - - Pbc = rtcs:pbc(RiakNodes, objects, BucketName), - - %% verify that object is also stored in latest GC bucket - Ms = all_manifests_in_gc_bucket(Pbc), - lager:info("Retrieved ~p manifets from GC bucket", [length(Ms)]), - ?assertMatch( - [{UUID, _}], - lists:filter(fun({UUID0, M1}) when UUID0 =:= UUID -> - ?assertEqual(pending_delete, M1?MANIFEST.state), - true; - ({UUID0, _}) -> - lager:debug("UUID=~p / ~p", - [mochihex:to_hex(UUID0), mochihex:to_hex(UUID)]), - false; - (_Other) -> - lager:error("Unexpected: ~p", [_Other]), - false - end, Ms)), - - lager:info("cs770 verification ok", []), - ?assertEqual(ok, erlcloud_s3:delete_bucket(BucketName, UserConfig)), - ok. - -all_manifests_in_gc_bucket(Pbc) -> - {ok, Keys} = riakc_pb_socket:list_keys(Pbc, ?GC_BUCKET), - Ms = rt:pmap(fun(K) -> - {ok, O} = riakc_pb_socket:get(Pbc, <<"riak-cs-gc">>, K), - Some = [binary_to_term(V) || {_, V} <- riakc_obj:get_contents(O), - V =/= <<>>], - twop_set:to_list(twop_set:resolve(Some)) - end, Keys), - %% lager:debug("All manifests in GC buckets: ~p", [Ms]), - lists:flatten(Ms). - -get_manifests(RiakNodes, BucketName, Key) -> - rt:wait_until(fun() -> - case rc_helper:get_riakc_obj(RiakNodes, objects, BucketName, Key) of - {ok, _} -> true; - Error -> Error - end - end, 8, 500), - {ok, Obj} = rc_helper:get_riakc_obj(RiakNodes, objects, BucketName, Key), - %% Assuming no tombstone; - [binary_to_term(V) || {_, V} <- riakc_obj:get_contents(Obj), - V =/= <<>>]. - -put_and_get(UserConfig, BucketName, Data) -> - erlcloud_s3:put_object(BucketName, ?KEY, Data, UserConfig), - Props = erlcloud_s3:get_object(BucketName, ?KEY, UserConfig), - ?assertEqual(proplists:get_value(content, Props), Data). - -delete(UserConfig, BucketName) -> - erlcloud_s3:delete_object(BucketName, ?KEY, UserConfig). - -assert_notfound(UserConfig, BucketName) -> - ?assertException(_, - {aws_error, {http_error, 404, _, _}}, - erlcloud_s3:get_object(BucketName, ?KEY, UserConfig)). diff --git a/riak_test/tests/regression_tests_2.erl b/riak_test/tests/regression_tests_2.erl deleted file mode 100644 index ed3da0bbb..000000000 --- a/riak_test/tests/regression_tests_2.erl +++ /dev/null @@ -1,221 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc regression_tests running with two node cluster, while -%% regression_tests.erl is for single node cluster In case of -%% rtcs:setup(2) with vanilla CS setup used. Otherwise feel free to -%% create an independent module like cs743_regression_test. - - --module(regression_tests_2). - --compile(export_all). --include_lib("eunit/include/eunit.hrl"). - -confirm() -> - {UserConfig, {_RiakNodes, _CSNodes, _Stanchion}} = rtcs:setup(2), - - ok = verify_cs631(UserConfig, "cs-631-test-bukcet"), - ok = verify_cs654(UserConfig), - ok = verify_cs781(UserConfig, "cs-781-test-bucket"), - ok = verify_cs1255(UserConfig, "cs-1255-test-bucket"), - - %% Append your next regression tests here - - rtcs:pass(). - - -%% @doc Integration test for [https://github.com/basho/riak_cs/issues/631] -verify_cs631(UserConfig, BucketName) -> - ?assertEqual(ok, erlcloud_s3:create_bucket(BucketName, UserConfig)), - test_unknown_canonical_id_grant_returns_400(UserConfig, BucketName), - test_canned_acl_and_grants_returns_400(UserConfig, BucketName), - ok. - --define(KEY_1, "key-1"). --define(KEY_2, "key-2"). --define(VALUE, <<"632-test-value">>). - -test_canned_acl_and_grants_returns_400(UserConfig, BucketName) -> - Acl = [{acl, public_read}], - Headers = [{"x-amz-grant-write", "email=\"doesnmatter@example.com\""}], - ?assertError({aws_error, {http_error, 400, _, _}}, - erlcloud_s3:put_object(BucketName, ?KEY_1, ?VALUE, - Acl, Headers, UserConfig)). - -test_unknown_canonical_id_grant_returns_400(UserConfig, BucketName) -> - Acl = [], - Headers = [{"x-amz-grant-write", "id=\"badbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbad9badbad\""}], - ?assertError({aws_error, {http_error, 400, _, _}}, - erlcloud_s3:put_object(BucketName, ?KEY_2, ?VALUE, - Acl, Headers, UserConfig)). - -%% @doc Integration test for [https://github.com/basho/riak_cs/issues/654] -verify_cs654(UserConfig) -> - run_test_empty_common_prefixes(UserConfig), - run_test_no_duplicate_key(UserConfig), - run_test_no_infinite_loop(UserConfig). - -%% Test for the original issue found in Github #654: -%% ``` -%% $ s3cmd mb s3://test/ -%% $ mkdir a -%% $ for i in {0001..1002}; do echo ${i} > a/${i}.txt; done # in zsh -%% $ s3cmd put --recursive a s3://test -%% $ s3cmd del --recursive --force s3://test -%% $ s3cmd ls s3://test # !!FAIL!! -%% ''' - --define(TEST_BUCKET_1, "cs-654-test-bucket-1"). --define(KEY_PREFIX_1, "a/"). - -run_test_empty_common_prefixes(UserConfig) -> - lager:info("creating bucket ~p", [?TEST_BUCKET_1]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET_1, UserConfig)), - Count = 1002, - list_objects_test_helper:load_objects(?TEST_BUCKET_1, Count, ?KEY_PREFIX_1, UserConfig), - list_objects_test_helper:delete_objects(?TEST_BUCKET_1, Count, ?KEY_PREFIX_1, UserConfig), - ListObjectsOptions = [{delimiter, "/"}], - ?assertEqual([], - proplists:get_value(contents, - erlcloud_s3:list_objects(?TEST_BUCKET_1, - ListObjectsOptions, - UserConfig))), - ok. - -%% Test for issue in comment -%% [https://github.com/basho/riak_cs/pull/655#issuecomment-23309088] -%% The comment is reproduced here: -%% -%% When there are both some common prefixes and non-prefixed keys, -%% next start key is "rewinded" by skip_past_prefix_and_delimiter. -%% -%% A situation is like this: -%% -%% 100 active objects under 0/ -%% 1 active object whose name is 1.txt -%% 1000 pending_delete objects under 2/ -%% ls reports duplicated 1.txt. -%% -%% ``` -%% $ s3cmd ls s3://test -%% DIR s3://test/0/ -%% 2013-08-27 02:09 9 s3://test/1.txt -%% 2013-08-27 02:09 9 s3://test/1.txt -%% ''' - --define(TEST_BUCKET_2, "cs-654-test-bucket-2"). --define(ACTIVE_PREFIX, "0/"). --define(SINGLE_OBJECT, "1.txt"). --define(PENDING_DELETE_PREFIX, "2/"). - -run_test_no_duplicate_key(UserConfig) -> - lager:info("creating bucket ~p", [?TEST_BUCKET_2]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET_2, UserConfig)), - - list_objects_test_helper:load_objects(?TEST_BUCKET_2, 100, ?ACTIVE_PREFIX, - UserConfig), - - erlcloud_s3:put_object(?TEST_BUCKET_2, ?SINGLE_OBJECT, - crypto:rand_bytes(100), UserConfig), - - list_objects_test_helper:load_objects(?TEST_BUCKET_2, 1000, - ?PENDING_DELETE_PREFIX, - UserConfig), - list_objects_test_helper:delete_objects(?TEST_BUCKET_2, 1000, - ?PENDING_DELETE_PREFIX, - UserConfig), - - ListObjectsOptions = [{delimiter, "/"}], - Response = erlcloud_s3:list_objects(?TEST_BUCKET_2, - ListObjectsOptions, - UserConfig), - [SingleResult] = proplists:get_value(contents, Response), - ?assertEqual("1.txt", proplists:get_value(key, SingleResult)), - ok. - -%% Test for issue in comment -%% [https://github.com/basho/riak_cs/pull/655#issuecomment-23390742] -%% The comment is reproduced here: -%% Found one more issue. -%% -%% Infinite loop happens for list objects request with prefix without delimiter. -%% -%% Assume test bucket has 1100 active keys under prefix 0/. -%% s3cmd -c s3cfg.dev1.alice ls s3://test/0 (no slash at the end) does not respond -%% and CPU is used constantly even after killing s3cmd. - --define(TEST_BUCKET_3, "cs-654-test-bucket-3"). --define(ACTIVE_PREFIX_2, "0/"). - -run_test_no_infinite_loop(UserConfig) -> - lager:info("creating bucket ~p", [?TEST_BUCKET_3]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET_3, UserConfig)), - - list_objects_test_helper:load_objects(?TEST_BUCKET_3, 1100, ?ACTIVE_PREFIX, - UserConfig), - - ListObjectsOptions = [{delimiter, "/"}, {prefix, "0"}], - Response = erlcloud_s3:list_objects(?TEST_BUCKET_2, - ListObjectsOptions, - UserConfig), - [SingleResult] = proplists:get_value(common_prefixes, Response), - ?assertEqual("0/", proplists:get_value(prefix, SingleResult)), - ok. - - - -format_int(Int) -> - binary_to_list(iolist_to_binary(io_lib:format("~4..0B", [Int]))). - -%% @doc Integration test for [https://github.com/basho/riak_cs/issues/781] -verify_cs781(UserConfig, BucketName) -> - ?assertEqual(ok, erlcloud_s3:create_bucket(BucketName, UserConfig)), - Count = 1003, - [erlcloud_s3:put_object(BucketName, - format_int(X), - crypto:rand_bytes(100), - UserConfig) || X <- lists:seq(1, Count)], - erlcloud_s3:delete_object(BucketName, format_int(1), UserConfig), - erlcloud_s3:delete_object(BucketName, format_int(2), UserConfig), - ?assertEqual(true, - proplists:get_value(is_truncated, - erlcloud_s3:list_objects(BucketName, - [], - UserConfig))), - ok. - -%% Test for [https://github.com/basho/riak_cs/pull/1255] -verify_cs1255(UserConfig, BucketName) -> - ?assertEqual(ok, erlcloud_s3:create_bucket(BucketName, UserConfig)), - POSTData = {iolist_to_binary(crypto:rand_bytes(100)), "application/octet-stream"}, - - %% put objects using a binary key - erlcloud_s3:s3_request(UserConfig, put, BucketName, <<"/", 00>>, [], [], POSTData, []), - erlcloud_s3:s3_request(UserConfig, put, BucketName, <<"/", 01>>, [], [], POSTData, []), - erlcloud_s3:s3_request(UserConfig, put, BucketName, <<"/valid_key">>, [], [], POSTData, []), - - %% list objects without xmerl which throws error when parsing invalid charactor as XML 1.0 - {_Header, Body} = erlcloud_s3:s3_request(UserConfig, get, BucketName, "/", [], [], <<>>, []), - ?assertMatch({_, _}, binary:match(list_to_binary(Body), <<"", 00, "">>)), - ?assertMatch({_, _}, binary:match(list_to_binary(Body), <<"", 01, "">>)), - ?assertMatch({_, _}, binary:match(list_to_binary(Body), <<"valid_key">>)), - - ok. diff --git a/riak_test/tests/repl_v3_test.erl b/riak_test/tests/repl_v3_test.erl deleted file mode 100644 index 3d5ead831..000000000 --- a/riak_test/tests/repl_v3_test.erl +++ /dev/null @@ -1,295 +0,0 @@ --module(repl_v3_test). - --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). - --define(TEST_BUCKET, "riak-test-bucket"). - -confirm() -> - {UserConfig, {RiakNodes, _CSNodes, _Stanchion}} = rtcs:setup2x2(), - lager:info("UserConfig = ~p", [UserConfig]), - [A,B,C,D] = RiakNodes, - - ANodes = [A,B], - BNodes = [C,D], - - AFirst = hd(ANodes), - BFirst = hd(BNodes), - - %% User 1, Cluster 1 config - U1C1Config = rtcs_admin:create_user(AFirst, 1), - %% User 1, Cluster 2 config - U1C2Config = rtcs_admin:aws_config(U1C1Config, [{port, rtcs_config:cs_port(BFirst)}]), - - %% User 2, Cluster 2 config - U2C2Config = rtcs_admin:create_user(BFirst, 2), - %% User 2, Cluster 1 config - U2C1Config = rtcs_admin:aws_config(U2C2Config, [{port, rtcs_config:cs_port(AFirst)}]), - - lager:info("User 1 IS valid on the primary cluster, and has no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(U1C1Config)), - - lager:info("User 2 IS valid on the primary cluster, and has no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(U2C1Config)), - - lager:info("User 2 is NOT valid on the secondary cluster"), - ?assertError({aws_error, _}, erlcloud_s3:list_buckets(U2C2Config)), - - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, U1C1Config)), - - ?assertMatch([{buckets, [[{name, ?TEST_BUCKET}, _]]}], - erlcloud_s3:list_buckets(U1C1Config)), - - ObjList1= erlcloud_s3:list_objects(?TEST_BUCKET, U1C1Config), - ?assertEqual([], proplists:get_value(contents, ObjList1)), - - Object1 = crypto:rand_bytes(4194304), - - erlcloud_s3:put_object(?TEST_BUCKET, "object_one", Object1, U1C1Config), - - ObjList2 = erlcloud_s3:list_objects(?TEST_BUCKET, U1C1Config), - ?assertEqual(["object_one"], - [proplists:get_value(key, O) || - O <- proplists:get_value(contents, ObjList2)]), - - Obj = erlcloud_s3:get_object(?TEST_BUCKET, "object_one", U1C1Config), - ?assertEqual(Object1, proplists:get_value(content, Obj)), - - lager:info("set up v3 replication between clusters"), - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% Name and connect Riak Enterprise V3 replication between A and B - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - repl_helpers:name_cluster(AFirst,"A"), - repl_helpers:name_cluster(BFirst,"B"), - - - rt:wait_until_ring_converged(ANodes), - rt:wait_until_ring_converged(BNodes), - - repl_helpers:wait_until_13_leader(AFirst), - LeaderA = rpc:call(AFirst, riak_core_cluster_mgr, get_leader, []), - LeaderB = rpc:call(BFirst, riak_core_cluster_mgr, get_leader, []), - - ModeResA = rpc:call(LeaderA, riak_repl_console, modes, [["mode_repl13"]]), - ModeResB = rpc:call(LeaderB, riak_repl_console, modes, [["mode_repl13"]]), - lager:info("Replication Modes = ~p", [ModeResA]), - lager:info("Replication Modes = ~p", [ModeResB]), - - - {ok, {_IP, BPort}} = rpc:call(BFirst, application, get_env, - [riak_core, cluster_mgr]), - repl_helpers:connect_clusters13(LeaderA, ANodes, BPort, "B"), - - ?assertEqual(ok, repl_helpers:wait_for_connection13(LeaderA, "B")), - rt:wait_until_ring_converged(ANodes), - - repl_helpers:enable_realtime(LeaderA, "B"), - repl_helpers:start_realtime(LeaderA, "B"), - - PGEnableResult = rpc:call(LeaderA, riak_repl_console, proxy_get, [["enable","B"]]), - lager:info("Enabled pg: ~p", [PGEnableResult]), - Status = rpc:call(LeaderA, riak_repl_console, status, [quiet]), - - case proplists:get_value(proxy_get_enabled, Status) of - undefined -> ?assert(false); - EnabledFor -> lager:info("PG enabled for cluster ~p",[EnabledFor]) - end, - rt:wait_until_ring_converged(ANodes), - rt:wait_until_ring_converged(BNodes), - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% Done connection replication - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - %% run an initial fullsync - repl_helpers:start_and_wait_until_fullsync_complete13(LeaderA), - - lager:info("User 2 is valid on secondary cluster after fullsync," - " still no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(U2C2Config)), - - lager:info("User 1 has the test bucket on the secondary cluster now"), - ?assertMatch([{buckets, [[{name, ?TEST_BUCKET}, _]]}], - erlcloud_s3:list_buckets(U1C2Config)), - - lager:info("Object written on primary cluster is readable from secondary " - "cluster"), - Obj2 = erlcloud_s3:get_object(?TEST_BUCKET, "object_one", U1C2Config), - ?assertEqual(Object1, proplists:get_value(content, Obj2)), - - lager:info("write 2 more objects to the primary cluster"), - - Object2 = crypto:rand_bytes(4194304), - - erlcloud_s3:put_object(?TEST_BUCKET, "object_two", Object2, U1C1Config), - - Object3 = crypto:rand_bytes(4194304), - - erlcloud_s3:put_object(?TEST_BUCKET, "object_three", Object3, U1C1Config), - - lager:info("disable proxy_get"), - disable_pg(LeaderA, "B", ANodes, BNodes, BPort), - - lager:info("check we can still read the fullsynced object"), - Obj3 = erlcloud_s3:get_object(?TEST_BUCKET, "object_one", U1C2Config), - ?assertEqual(Object1, proplists:get_value(content,Obj3)), - - lager:info("check all 3 objects are listed on the sink cluster"), - ?assertEqual(["object_one", "object_three", "object_two"], - [proplists:get_value(key, O) || O <- proplists:get_value(contents, - erlcloud_s3:list_objects(?TEST_BUCKET, U1C2Config))]), - - lager:info("check that the 2 other objects can't be read"), - %% We expect errors here since proxy_get will fail due to the - %% clusters being disconnected. - ?assertError({aws_error, _}, erlcloud_s3:get_object(?TEST_BUCKET, - "object_two", U1C2Config)), - ?assertError({aws_error, _}, erlcloud_s3:get_object(?TEST_BUCKET, - "object_three", U1C2Config)), - - lager:info("enable proxy_get"), - enable_pg(LeaderA, "B", ANodes, BNodes, BPort), - - lager:info("check we can read object_two via proxy get"), - Obj6 = erlcloud_s3:get_object(?TEST_BUCKET, "object_two", U1C2Config), - ?assertEqual(Object2, proplists:get_value(content, Obj6)), - - lager:info("disable proxy_get again"), - disable_pg(LeaderA, "B", ANodes, BNodes, BPort), - - lager:info("check we still can't read object_three"), - ?assertError({aws_error, _}, erlcloud_s3:get_object(?TEST_BUCKET, - "object_three", U1C2Config)), - - lager:info("check that proxy getting object_two wrote it locally, so we" - " can read it"), - Obj8 = erlcloud_s3:get_object(?TEST_BUCKET, "object_two", U1C2Config), - ?assertEqual(Object2, proplists:get_value(content, Obj8)), - - lager:info("delete object_one while clusters are disconnected"), - erlcloud_s3:delete_object(?TEST_BUCKET, "object_one", U1C1Config), - - lager:info("enable proxy_get"), - enable_pg(LeaderA, "B", ANodes, BNodes, BPort), - - lager:info("delete object_two while clusters are connected"), - erlcloud_s3:delete_object(?TEST_BUCKET, "object_two", U1C1Config), - - lager:info("object_one is still visible on secondary cluster"), - Obj9 = erlcloud_s3:get_object(?TEST_BUCKET, "object_one", U1C2Config), - ?assertEqual(Object1, proplists:get_value(content, Obj9)), - - lager:info("object_two is deleted"), - ?assertError({aws_error, _}, - erlcloud_s3:get_object(?TEST_BUCKET, "object_two", U1C2Config)), - - repl_helpers:start_and_wait_until_fullsync_complete13(LeaderA), - - lager:info("object_one is deleted after fullsync"), - ?assertError({aws_error, _}, - erlcloud_s3:get_object(?TEST_BUCKET, "object_one", U1C2Config)), - - lager:info("disable proxy_get again"), - disable_pg(LeaderA, "B", ANodes, BNodes, BPort), - - Object3A = crypto:rand_bytes(4194304), - ?assert(Object3 /= Object3A), - - lager:info("write a new version of object_three"), - - erlcloud_s3:put_object(?TEST_BUCKET, "object_three", Object3A, U1C1Config), - - lager:info("Independently write different object_four and object_five to bolth clusters"), - - Object4A = crypto:rand_bytes(4194304), - Object4B = crypto:rand_bytes(4194304), - - Object5A = crypto:rand_bytes(4194304), - Object5B = crypto:rand_bytes(4194304), - - erlcloud_s3:put_object(?TEST_BUCKET, "object_four", Object4A, U1C1Config), - - erlcloud_s3:put_object(?TEST_BUCKET, "object_four", Object4B, U1C2Config), - erlcloud_s3:put_object(?TEST_BUCKET, "object_five", Object5B, U1C2Config), - - lager:info("delay writing object 5 on primary cluster 1 second after " - "writing to secondary cluster"), - timer:sleep(1000), - erlcloud_s3:put_object(?TEST_BUCKET, "object_five", Object5A, U1C1Config), - - lager:info("enable proxy_get"), - enable_pg(LeaderA, "B", ANodes, BNodes, BPort), - - lager:info("secondary cluster has old version of object three"), - Obj10 = erlcloud_s3:get_object(?TEST_BUCKET, "object_three", U1C2Config), - ?assertEqual(Object3, proplists:get_value(content, Obj10)), - - lager:info("secondary cluster has 'B' version of object four"), - Obj11 = erlcloud_s3:get_object(?TEST_BUCKET, "object_four", U1C2Config), - ?assertEqual(Object4B, proplists:get_value(content, Obj11)), - - repl_helpers:start_and_wait_until_fullsync_complete13(LeaderA), - - lager:info("secondary cluster has new version of object three"), - Obj12 = erlcloud_s3:get_object(?TEST_BUCKET, "object_three", U1C2Config), - ?assertEqual(Object3A, proplists:get_value(content, Obj12)), - - lager:info("secondary cluster has 'B' version of object four"), - Obj13 = erlcloud_s3:get_object(?TEST_BUCKET, "object_four", U1C2Config), - ?assertEqual(Object4B, proplists:get_value(content, Obj13)), - - lager:info("secondary cluster has 'A' version of object five, because it " - "was written later"), - Obj14 = erlcloud_s3:get_object(?TEST_BUCKET, "object_five", U1C2Config), - ?assertEqual(Object5A, proplists:get_value(content, Obj14)), - - lager:info("write 'A' version of object four again on primary cluster"), - - erlcloud_s3:put_object(?TEST_BUCKET, "object_four", Object4A, U1C1Config), - - lager:info("secondary cluster now has 'A' version of object four"), - - Obj15 = erlcloud_s3:get_object(?TEST_BUCKET, "object_four", U1C2Config), - ?assertEqual(Object4A, proplists:get_value(content,Obj15)), - - lager:info("Disable proxy_get (not disconnect) " - "and enable realtime block replication"), - set_proxy_get(LeaderA, "disable", "B", ANodes, BNodes), - set_block_rt(RiakNodes), - - Object6 = crypto:rand_bytes(4194304), - erlcloud_s3:put_object(?TEST_BUCKET, "object_six", Object6, U1C1Config), - repl_helpers:wait_until_realtime_sync_complete(ANodes), - lager:info("The object can be downloaded from sink cluster"), - Obj16 = erlcloud_s3:get_object(?TEST_BUCKET, "object_six", U1C2Config), - ?assertEqual(Object6, proplists:get_value(content, Obj16)), - - rtcs:pass(). - -enable_pg(SourceLeader, SinkName, ANodes, BNodes, BPort) -> - repl_helpers:connect_clusters13(SourceLeader, ANodes, BPort, SinkName), - set_proxy_get(SourceLeader, "enable", SinkName, ANodes, BNodes). - -disable_pg(SourceLeader, SinkName, ANodes, BNodes, _BPort) -> - set_proxy_get(SourceLeader, "disable", SinkName, ANodes, BNodes), - repl_helpers:disconnect_clusters13(SourceLeader, ANodes, SinkName). - -set_proxy_get(SourceLeader, EnableOrDisable, SinkName, ANodes, BNodes) -> - PGEnableResult = rpc:call(SourceLeader, riak_repl_console, proxy_get, - [[EnableOrDisable,SinkName]]), - - lager:info("Enabled pg: ~p", [PGEnableResult]), - Status = rpc:call(SourceLeader, riak_repl_console, status, [quiet]), - - case proplists:get_value(proxy_get_enabled, Status) of - undefined -> ?assert(false); - EnabledFor -> lager:info("PG enabled for cluster ~p",[EnabledFor]) - end, - rt:wait_until_ring_converged(ANodes), - rt:wait_until_ring_converged(BNodes), - ok. - -set_block_rt(RiakNodes) -> - rpc:multicall(RiakNodes, application, set_env, - [riak_repl, replicate_cs_blocks_realtime, true]). diff --git a/riak_test/tests/riak_cs_debug_test.erl b/riak_test/tests/riak_cs_debug_test.erl deleted file mode 100644 index 241d4b9fe..000000000 --- a/riak_test/tests/riak_cs_debug_test.erl +++ /dev/null @@ -1,154 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(riak_cs_debug_test). - --compile(export_all). --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). - --define(assertContainsAll(ExpectedList,ActualList), - lists:foreach( - fun(X) -> ?assert(lists:member(X, ActualList)) end, - ExpectedList)). - --define(assertMatchAny(Pattern, ActualList), - ?assert( - lists:any( - fun(X) -> - case re:run(X, Pattern) of - {match, _} -> true; - nomatch -> false - end - end, ActualList))). - --define(assertNotMatchAny(Pattern, ActualList), - ?assert( - lists:all( - fun(X) -> - case re:run(X, Pattern) of - {match, _} -> false; - nomatch -> true - end - end, ActualList))). - -confirm() -> - %% Run riak-cs-debug before cuttlefish generates configs. - TarGz1 = exec_cs_debug(), - List1 = trim_dir_prefix(list_files(TarGz1)), - ?assertContainsAll(minimum_necessary_files(), List1), - - _ = rtcs:setup(1), - - %% Run riak-cs-debug after cuttlefish generates configs. - TarGz2 = exec_cs_debug(), - List2 = trim_dir_prefix(list_files(TarGz2)), - ?assertContainsAll(minimum_necessary_files_after_boot(), List2), - ?assertMatchAny("^logs/platform_log_dir/access.log.*", List2), - ?assertMatchAny("^config/generated.configs/app.*.config", List2), - ?assertMatchAny("^config/generated.configs/vm.*.args", List2), - ?assertNotMatchAny("^config/*.pem$", List2), - ok = file:delete(TarGz2), - - %% Run riak-cs-debug with app.config and vm.args. - move_generated_configs_as_appconfigs(), - restart_cs_node(), - TarGz3 = exec_cs_debug(), - List3 = trim_dir_prefix(list_files(TarGz3)), - ?assertContainsAll(minimum_necessary_files_after_boot() - ++ ["config/app.config", "config/vm.args"], - List3), - ?assertNotMatchAny("^config/generated.configs/app.*.config", List3), - ?assertNotMatchAny("^config/generated.configs/vm.*.args", List3), - - rtcs:pass(). - -restart_cs_node() -> - rtcs_exec:stop_cs(1), - rt:wait_until_unpingable(rtcs:cs_node(1)), - rtcs_exec:start_cs(1), - ok. - -move_generated_configs_as_appconfigs() -> - DevPath = rtcs_config:devpath(cs, current), - GenConfPath = DevPath ++ "/dev/dev1/data/generated.configs/", - AppConfig = filelib:wildcard([GenConfPath ++ "app.*.config"]), - VmArgs = filelib:wildcard([GenConfPath ++ "vm.*.args"]), - - ConfPath = DevPath ++ "/dev/dev1/etc/", - ok = file:rename(AppConfig, ConfPath ++ "app.config"), - ok = file:rename(VmArgs, ConfPath ++ "vm.args"), - ok. - -exec_cs_debug() -> - DevPath = rtcs_config:devpath(cs, current), - Cmd = rtcs_exec:riakcs_debugcmd(DevPath, 1, []), - Output = os:cmd("cd " ++ DevPath ++ " && " ++ Cmd), - [_Results, File] = string:tokens(Output, " \n"), - File. - -list_files(TarGz) -> - Output = os:cmd("tar tf "++TarGz), - string:tokens(Output, " \n"). - -trim_dir_prefix(Files) -> - lists:map(fun(File) -> - [_Prefix|List] = string:tokens(File, "/"), - string:join(List, "/") - end - ,Files). - -minimum_necessary_files() -> - [ - "config/advanced.config", - "config/riak-cs.conf", - "commands/cluster-info", - "commands/cluster-info.html", - "commands/date", - "commands/df", - "commands/df_i", - "commands/dmesg", - "commands/hostname", - "commands/ifconfig", - "commands/last", - "commands/mount", - "commands/netstat_an", - "commands/netstat_i", - "commands/netstat_rn", - "commands/ps", - "commands/riak_cs_gc_status", - "commands/riak_cs_ping", - "commands/riak_cs_status", - "commands/riak_cs_storage_status", - "commands/riak_cs_version", - "commands/sysctl", - "commands/uname", - "commands/w" - ]. - -minimum_necessary_files_after_boot() -> - minimum_necessary_files() ++ - [ - "logs/platform_log_dir/console.log", - "logs/platform_log_dir/run_erl.log", - "logs/platform_log_dir/erlang.log.1", - "logs/platform_log_dir/crash.log", - "logs/platform_log_dir/error.log" - ]. diff --git a/riak_test/tests/select_gc_bucket_test.erl b/riak_test/tests/select_gc_bucket_test.erl deleted file mode 100644 index ee138db6b..000000000 --- a/riak_test/tests/select_gc_bucket_test.erl +++ /dev/null @@ -1,85 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(select_gc_bucket_test). - -%% @doc `riak_test' module for testing select_gc_bucket script - --export([confirm/0]). - --include_lib("eunit/include/eunit.hrl"). - --define(BUCKET, "rt-bucket"). - --define(KEY_ALIVE, "alive"). --define(KEY_DELETED_S, "deleted-S"). --define(KEY_ALIVE_MP, "alive-mp"). --define(KEY_DELETED_L1, "deleted-L1"). --define(KEY_DELETED_L2, "deleted-L2"). - -confirm() -> - case rt_config:get(flavor, basic) of - {multibag, _} -> - lager:info("select_gc_bucket script does not supprt multibag env."), - lager:info("Skip the test."), - rtcs:pass(); - _ -> confirm1() - end. - -confirm1() -> - {UserConfig, {RiakNodes, CSNodes, Stanchion}} = rtcs:setup(1), - - BlockKeysFile = "/tmp/select_gc.txt", - os:cmd("rm -f " ++ BlockKeysFile), - rtcs_exec:gc(1, "set-interval infinity"), - rtcs_exec:gc(1, "cancel"), - - ?assertEqual(ok, erlcloud_s3:create_bucket(?BUCKET, UserConfig)), - [upload_object(UserConfig, ?BUCKET, normal, K) || - K <- [?KEY_ALIVE, ?KEY_DELETED_S]], - [upload_object(UserConfig, ?BUCKET, mp, K) || - K <- [?KEY_ALIVE_MP, ?KEY_DELETED_L1, ?KEY_DELETED_L2]], - [delete_object(UserConfig, ?BUCKET, K) || - K <- [?KEY_DELETED_S, ?KEY_DELETED_L1, ?KEY_DELETED_L2]], - - timer:sleep(1000), - Res1 = rtcs_exec:exec_priv_escript(1, "internal/select_gc_bucket.erl", - "-h 127.0.0.1 -p 10017 -e today " - "-o " ++ BlockKeysFile), - lager:debug("select_gc_bucket.erl log:\n~s", [Res1]), - lager:debug("select_gc_bucket.erl log:============= END"), - - tools_helper:offline_delete({RiakNodes, CSNodes, Stanchion}, [BlockKeysFile]), - rtcs:pass(). - -upload_object(UserConfig, Bucket, normal, Key) -> - SingleBlock = crypto:rand_bytes(400), - erlcloud_s3:put_object(Bucket, Key, SingleBlock, UserConfig); -upload_object(UserConfig, Bucket, mp, Key) -> - rtcs_multipart:multipart_upload(Bucket, Key, - [mb(5), mb(1)], UserConfig). - -delete_object(UserConfig, Bucket, Key) -> - ?assertEqual([{delete_marker, false}, {version_id, "null"}], - erlcloud_s3:delete_object(Bucket, Key, UserConfig)). - -mb(MegaBytes) -> - MegaBytes * 1024 * 1024. - diff --git a/riak_test/tests/sibling_benchmark.erl b/riak_test/tests/sibling_benchmark.erl deleted file mode 100644 index fe354633d..000000000 --- a/riak_test/tests/sibling_benchmark.erl +++ /dev/null @@ -1,256 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(sibling_benchmark). - -%% @doc Microbenchmark of sibling convergence. Verifies the number of -%% siblings are in the same magnitude with number of update -%% concurrency. To test with previous version, add a configuration -%% like this to your harness:: -%% -%% {sibling_benchmark, -%% [{write_concurrency, 2}, -%% {duration_sec, 30}, -%% {leave_and_join, 0}, %% number of cluster membership change during the benchmark -%% {dvv_enabled, true}, -%% {version, previous}] -%% %%{version, current}] -%% }, - - --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). - -%% keys for non-multipart objects --define(TEST_BUCKET, "riak-test-bucket"). --define(KEY_SINGLE_BLOCK, "riak_test_key1"). - -confirm() -> - RTConfig = rt_config:get(sibling_benchmark, []), - Concurrency = proplists:get_value(write_concurrency, RTConfig, 4), - ?assert(is_integer(Concurrency) andalso Concurrency >= 0), - %% msec - Interval = proplists:get_value(write_interval, RTConfig, 100), - ?assert(is_integer(Interval) andalso Interval >= 0), - - Duration = proplists:get_value(duration_sec, RTConfig, 16), - ?assert(is_integer(Duration) andalso Duration >= 0), - - DVVEnabled = proplists:get_value(dvv_enabled, RTConfig, true), - ?assert(is_boolean(DVVEnabled)), - - Config = [{cs, - [{riak_cs, - [{leeway_seconds, 5}, - {gc_interval, 10}, - {connection_pools, [{request_pool, {Concurrency*2 + 5, 0}}]} - ]}]}, - {riak, - [{riak_core, - [{default_bucket_props, - [{allow_mult, true}, {dvv_enabled, DVVEnabled}]}]} - ]}], - {UserConfig, {RiakNodes, CSNodes, _Stanchion}} = - case proplists:get_value(version, RTConfig, current) of - current -> - put(version, current), - rtcs:setup(4, Config); - previous -> - put(version, previous), - rtcs:setup(4, Config, previous) - end, - - %% setting up the stage - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(UserConfig)), - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - %% The first object - erlcloud_s3:put_object(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, <<"boom!">>, UserConfig), - - get_counts(RiakNodes, ?TEST_BUCKET, ?KEY_SINGLE_BLOCK), - lager:info("====================== run benchmark ====================="), - {ok, Pid} = start_stats_checker(RiakNodes), - - lager:info("write_concurrency: ~p, write_interval: ~p msec, DVV: ~p", - [Concurrency, Interval, DVVEnabled]), - - Writers = [start_object_writer(UserConfig, Interval) || _ <- lists:seq(1, Concurrency)], - {ok, Reader} = start_object_reader(UserConfig), - - %% Do your madness or evil here - timer:sleep(Duration * 1000), - LeaveAndJoin = proplists:get_value(leave_and_join, RTConfig, 0), - leave_and_join_node(RiakNodes, LeaveAndJoin), - - ok = stop_object_reader(Reader), - [stop_object_writer(Writer) || {ok, Writer} <- Writers], - lager:info("====================== benchmark done ===================="), - MaxSib = stop_stats_checker(Pid), - %% Max number of siblings should not exceed number of upload concurrency - %% according to DVVset implementation - lager:info("MaxSib:Concurrency = ~p:~p", [MaxSib, Concurrency]), - %% Real concurrency is, added by GC workers - ?assert(MaxSib =< Concurrency + 5), - - get_counts(RiakNodes, ?TEST_BUCKET, ?KEY_SINGLE_BLOCK), - - %% Just test sibling resolver, for a major case of manifests - RawManifestBucket = rc_helper:to_riak_bucket(objects, ?TEST_BUCKET), - RawKey = ?KEY_SINGLE_BLOCK, - case rpc:call(hd(CSNodes), riak_cs_console, resolve_siblings, - [RawManifestBucket, RawKey]) of - ok -> ok; - {error, not_supported} -> - _ = lager:info("resolve_siblings does not suport multibag yet, skip it"), - ok - end, - - %% tearing down the stage - erlcloud_s3:delete_object(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, UserConfig), - lager:info("deleting bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:delete_bucket(?TEST_BUCKET, UserConfig)), - lager:info("User is valid on the cluster, and has no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(UserConfig)), - rtcs:pass(). - -start_object_reader(UserConfig) -> - Pid = spawn_link(fun() -> object_reader(UserConfig, 1) end), - {ok, Pid}. - -stop_object_reader(Pid) -> - Pid ! {stop, self()}, - receive Reply -> Reply end. - -object_reader(UserConfig, IntervalSec)-> - erlcloud_s3:get_object(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, UserConfig), - receive - {stop, From} -> From ! ok - after IntervalSec * 1000 -> - object_reader(UserConfig, IntervalSec) - end. - -start_object_writer(UserConfig, IntervalMilliSec) -> - timer:sleep(IntervalMilliSec), - Pid = spawn_link(fun() -> object_writer(UserConfig, IntervalMilliSec) end), - {ok, Pid}. - -stop_object_writer(Pid) -> - Pid ! {stop, self()}, - receive Reply -> Reply end. - -object_writer(UserConfig, IntervalMilliSec)-> - SingleBlock = crypto:rand_bytes(400), - erlcloud_s3:put_object(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, SingleBlock, UserConfig), - receive - {stop, From} -> From ! ok - after IntervalMilliSec -> - object_writer(UserConfig, IntervalMilliSec) - end. - -start_stats_checker(Nodes) -> - Pid = spawn_link(fun() -> stats_checker(Nodes, 5, -1) end), - {ok, Pid}. - -stop_stats_checker(Pid) -> - Pid ! {stop, self()}, - receive Reply -> Reply end. - -stats_checker(Nodes, IntervalSec, MaxSib0) -> - NewMaxSib = check_stats(Nodes, MaxSib0), - receive - {stop, From} -> From ! check_stats(Nodes, NewMaxSib) - after IntervalSec * 1000 -> - stats_checker(Nodes, IntervalSec, NewMaxSib) - end. - -check_stats(Nodes, MaxSib0) -> - Stats = collect_stats(Nodes), - MaxSib = pp_siblings(Stats), - pp_objsize(Stats), - pp_time(Stats), - get_counts(Nodes, ?TEST_BUCKET, ?KEY_SINGLE_BLOCK), - case MaxSib of - _ when is_integer(MaxSib) andalso MaxSib0 < MaxSib -> - MaxSib; - _ -> - MaxSib0 - end. - -collect_stats(Nodes) -> - RiakNodes = Nodes, - {Stats, _} = rpc:multicall(RiakNodes, riak_kv_stat, get_stats, []), - Stats. - -pp_siblings(Stats) -> pp(siblings, Stats). - -pp_objsize(Stats) -> pp(objsize, Stats). - -pp_time(Stats) -> pp(time, Stats). - -pp(Target, Stats) -> - AtomMeans = list_to_atom(lists:flatten(["node_get_fsm_", atom_to_list(Target), "_mean"])), - AtomMaxs = list_to_atom(lists:flatten(["node_get_fsm_", atom_to_list(Target), "_100"])), - Means = [ safe_get_value(AtomMeans, Stat) || Stat <- Stats ], - Maxs = [ safe_get_value(AtomMaxs, Stat) || Stat <- Stats ], - MeansStr = [ "\t" ++ safe_integer_to_list(Mean) || Mean <- Means ], - MaxsStr = [ "\t" ++ safe_integer_to_list(Max) || Max <- Maxs ], - lager:info("~s Mean: ~s", [Target, MeansStr]), - lager:info("~s Max: ~s", [Target, MaxsStr]), - Max = lists:foldl(fun erlang:max/2, 0, - lists:filter(fun is_integer/1, Maxs)), - %% lager:debug("Max ~p: ~p <= ~p", [Target, Max, Maxs]), - Max. - -safe_get_value(_AtomKey, {badrpc, _}) -> undefined; -safe_get_value(AtomKey, Stat) when is_list(Stat) -> proplists:get_value(AtomKey, Stat); -safe_get_value(_, _) -> undefined. - -safe_integer_to_list(I) when is_integer(I) -> - integer_to_list(I); -safe_integer_to_list(_) -> - " - ". - --spec get_counts(list(), string(), string()) -> {ok, non_neg_integer(), [non_neg_integer()]}. -get_counts(RiakNodes, Bucket, Key) -> - {ok, RiakObj} = rc_helper:get_riakc_obj(RiakNodes, objects, Bucket, Key), - SiblingCount = riakc_obj:value_count(RiakObj), - Histories = [ binary_to_term(V) || - V <- riakc_obj:get_values(RiakObj), - V /= <<>>], - %% [lager:info("History Length: ~p", [length(H)]) || H <- Histories], - HistoryCounts = [ length(H) || H <- Histories ], - lager:info("SiblingCount: ~p, HistoryCounts: ~w", [SiblingCount, HistoryCounts]), - {ok, SiblingCount, HistoryCounts}. - -leave_and_join_node(_RiakNodes, 0) -> ok; -leave_and_join_node(RiakNodes, N) -> - lager:info("leaving node2 (~p)", [N]), - Node2 = hd(tl(RiakNodes)), - rt:leave(Node2), - ?assertEqual(ok, rt:wait_until_unpingable(Node2)), - timer:sleep(1000), - - lager:info("joining node2 again (~p)", [N]), - Node1 = hd(RiakNodes), - rt:start_and_wait(Node2), - rt:staged_join(Node2, Node1), - rt:plan_and_commit(Node2), - timer:sleep(1000), - leave_and_join_node(RiakNodes, N-1). diff --git a/riak_test/tests/stanchion_switch_test.erl b/riak_test/tests/stanchion_switch_test.erl deleted file mode 100644 index 5571e1818..000000000 --- a/riak_test/tests/stanchion_switch_test.erl +++ /dev/null @@ -1,79 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(stanchion_switch_test). - -%% @doc `riak_test' module for testing riak-cs-stanchion switch command. - --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). - --define(TEST_BUCKET, "riak-test-bucket"). - --define(BACKUP_PORT, 9096). - -confirm() -> - {UserConfig, {RiakNodes, _CSNodes, Stanchion}} = rtcs:setup(1), - - lists:foreach(fun(RiakNode) -> - N = rtcs_dev:node_id(RiakNode), - ?assertEqual("Current Stanchion Adderss: http://127.0.0.1:9095\n", - rtcs_exec:show_stanchion_cs(N)) - end, RiakNodes), - - %% stanchion ops ok - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - lager:info("deleting bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:delete_bucket(?TEST_BUCKET, UserConfig)), - - %% stop stanchion to check ops fails - _ = rtcs_exec:stop_stanchion(), - rt:wait_until_unpingable(Stanchion), - - %% stanchion ops ng; we get 500 here for sure. - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertException(error, {aws_error, {http_error, 500, _, _}}, - erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - - rtcs:set_advanced_conf(stanchion, [{stanchion, [{host, {"127.0.0.1", ?BACKUP_PORT}}]}]), - _ = rtcs_exec:start_stanchion(), - rt:wait_until_pingable(Stanchion), - - %% stanchion ops ng; we get 500 here for sure. - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertException(error, {aws_error, {http_error, 500, _, _}}, - erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - - %% switch stanchion here, for all CS nodes - lists:foreach(fun(RiakNode) -> - N = rtcs_dev:node_id(RiakNode), - rtcs_exec:switch_stanchion_cs(N, "127.0.0.1", ?BACKUP_PORT), - ?assertEqual("Current Stanchion Adderss: http://127.0.0.1:9096\n", - rtcs_exec:show_stanchion_cs(N)) - end, RiakNodes), - - %% stanchion ops ok again - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - lager:info("deleting bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:delete_bucket(?TEST_BUCKET, UserConfig)), - rtcs:pass(). - diff --git a/riak_test/tests/stats_test.erl b/riak_test/tests/stats_test.erl deleted file mode 100644 index 1de0e81d3..000000000 --- a/riak_test/tests/stats_test.erl +++ /dev/null @@ -1,167 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(stats_test). - --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). --include_lib("erlcloud/include/erlcloud_aws.hrl"). - -confirm() -> - {UserConfig, {RiakNodes, _CSNodes, _Stanchion}} = rtcs:setup(1), - - lager:info("Confirming initial stats"), - confirm_initial_stats(cs, UserConfig, rtcs_config:cs_port(hd(RiakNodes))), - confirm_initial_stats(stanchion, UserConfig, rtcs_config:stanchion_port()), - - do_some_api_calls(UserConfig, "bucket1", "bucket2"), - - lager:info("Confirming stats after some operations"), - confirm_stats(cs, UserConfig, rtcs_config:cs_port(hd(RiakNodes))), - confirm_stats(stanchion, UserConfig, rtcs_config:stanchion_port()), - rtcs:pass(). - -confirm_initial_stats(cs, UserConfig, Port) -> - StatData = query_stats(cs, UserConfig, Port), - lager:debug("length(StatData) = ~p", [length(StatData)]), - ?assert(1125 < length(StatData)), - [begin - StatKey = list_to_binary(StatType ++ "_out_one"), - lager:debug("StatKey: ~p~n", [StatKey]), - ?assert(proplists:is_defined(StatKey, StatData)), - Value = proplists:get_value(StatKey, StatData), - ?assertEqual(0, Value) - end || StatType <- ["service_get", - "list_objects_get", - "bucket_put", - "bucket_delete", - "bucket_acl_get", - "bucket_acl_put", - "object_get", - "object_put", - "object_head", - "object_delete", - "object_acl_get", - "object_acl_put", - "riakc_get_block_n_one", - "riakc_put_block", - "riakc_delete_block_constrained" - ]], - ?assertEqual(1, proplists:get_value(<<"velvet_create_user_in_one">>, StatData)), - ?assertEqual(rtcs_config:request_pool_size() - 1, - proplists:get_value(<<"request_pool_workers">>, StatData)), - ?assertEqual(rtcs_config:bucket_list_pool_size(), - proplists:get_value(<<"bucket_list_pool_workers">>, StatData)); - -confirm_initial_stats(stanchion, UserConfig, Port) -> - Stats = query_stats(stanchion, UserConfig, Port), - lager:debug("length(Stats) = ~p", [length(Stats)]), - ?assert(130 < length(Stats)), - [begin - StatKey = list_to_binary(StatType ++ "_one"), - lager:debug("StatKey: ~p~n", [StatKey]), - ?assert(proplists:is_defined(StatKey, Stats)), - Value = proplists:get_value(StatKey, Stats), - ?assertEqual(0, Value) - end || StatType <- ["bucket_create", - "bucket_delete", - "bucket_put_acl", - "riakc_get_cs_bucket", - "riakc_put_cs_bucket", - "riakc_get_cs_user_strong", - "riakc_list_all_manifest_keys" - ]], - confirm_stat_count(Stats, <<"user_create_one">>, 1), - confirm_stat_count(Stats, <<"riakc_put_cs_user_one">>, 1), - - ?assert(proplists:is_defined(<<"waiting_time_mean">>, Stats)), - ?assert(proplists:is_defined(<<"waiting_time_median">>, Stats)), - ?assert(proplists:is_defined(<<"waiting_time_95">>, Stats)), - ?assert(proplists:is_defined(<<"waiting_time_99">>, Stats)), - ?assert(proplists:is_defined(<<"waiting_time_100">>, Stats)), - ?assert(proplists:is_defined(<<"sys_process_count">>, Stats)), - ?assert(proplists:is_defined(<<"webmachine_mochiweb_active_sockets">>, Stats)), - ok. - -confirm_stats(cs, UserConfig, Port) -> - confirm_status_cmd(cs, "service_get_in_one"), - Stats = query_stats(cs, UserConfig, Port), - confirm_stat_count(Stats, <<"service_get_out_one">>, 2), - confirm_stat_count(Stats, <<"object_get_out_one">>, 1), - confirm_stat_count(Stats, <<"object_put_out_one">>, 1), - confirm_stat_count(Stats, <<"object_delete_out_one">>, 1); -confirm_stats(stanchion, UserConfig, Port) -> - confirm_status_cmd(stanchion, "bucket_create_one"), - Stats = query_stats(stanchion, UserConfig, Port), - confirm_stat_count(Stats, <<"user_create_one">>, 1), - confirm_stat_count(Stats, <<"bucket_create_one">>, 2), - confirm_stat_count(Stats, <<"bucket_delete_one">>, 1), - confirm_stat_count(Stats, <<"riakc_put_cs_user_one">>, 1), - confirm_stat_count(Stats, <<"riakc_put_cs_bucket_one">>, 3), - %% this heavy list/gets can be reduced to ONE per delete-bucket (/-o-)/ ⌒ ┤ - confirm_stat_count(Stats, <<"riakc_list_all_manifest_keys_one">>, 2), - confirm_stat_count(Stats, <<"riakc_get_user_by_index_one">>, 1). - -do_some_api_calls(UserConfig, Bucket1, Bucket2) -> - ?assertEqual(ok, erlcloud_s3:create_bucket(Bucket1, UserConfig)), - - ?assertMatch([{buckets, [[{name, Bucket1}, _]]}], - erlcloud_s3:list_buckets(UserConfig)), - - Object = crypto:rand_bytes(500), - erlcloud_s3:put_object(Bucket1, "object_one", Object, UserConfig), - erlcloud_s3:get_object(Bucket1, "object_one", UserConfig), - erlcloud_s3:delete_object(Bucket1, "object_one", UserConfig), - erlcloud_s3:list_buckets(UserConfig), - - ?assertEqual(ok, erlcloud_s3:create_bucket(Bucket2, UserConfig)), - ?assertEqual(ok, erlcloud_s3:delete_bucket(Bucket2, UserConfig)), - ok. - -query_stats(Type, UserConfig, Port) -> - lager:debug("Querying stats to ~p", [Type]), - Date = httpd_util:rfc1123_date(), - {Resource, SignType} = case Type of - cs -> {"/riak-cs/stats", s3}; - stanchion -> {"/stats", velvet} - end, - Cmd="curl -s -H 'Date: " ++ Date ++ "' -H 'Authorization: " ++ - rtcs_admin:make_authorization(SignType, "GET", Resource, [], UserConfig, Date, []) ++ - "' http://localhost:" ++ - integer_to_list(Port) ++ Resource, - lager:info("Stats query cmd: ~p", [Cmd]), - Output = os:cmd(Cmd), - lager:debug("Stats output=~p~n",[Output]), - {struct, JsonData} = mochijson2:decode(Output), - JsonData. - -confirm_stat_count(StatData, StatType, ExpectedCount) -> - lager:debug("confirm_stat_count for ~p", [StatType]), - ?assertEqual(ExpectedCount, proplists:get_value(StatType, StatData)). - -confirm_status_cmd(Type, ExpectedToken) -> - Cmd = case Type of - cs -> - rtcs_exec:riakcs_statuscmd(rtcs_config:devpath(cs, current), 1); - stanchion -> - rtcs_exec:stanchion_statuscmd(rtcs_config:devpath(stanchion, current)) - end, - Res = os:cmd(Cmd), - ?assert(string:str(Res, ExpectedToken) > 0). diff --git a/riak_test/tests/storage_stats_detailed_test.erl b/riak_test/tests/storage_stats_detailed_test.erl deleted file mode 100644 index 33a22238a..000000000 --- a/riak_test/tests/storage_stats_detailed_test.erl +++ /dev/null @@ -1,155 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(storage_stats_detailed_test). -%% @doc Integration test for storage statistics. - --compile(export_all). --export([confirm/0]). - --include_lib("erlcloud/include/erlcloud_aws.hrl"). --include_lib("xmerl/include/xmerl.hrl"). --include_lib("eunit/include/eunit.hrl"). - --include("riak_cs.hrl"). - --define(BUCKET, "storage-stats-detailed"). - --define(KEY1, "1"). --define(KEY2, "2"). --define(KEY3, "3"). - -confirm() -> - rtcs:set_advanced_conf(cs, [{riak_cs, [{detailed_storage_calc, true}]}]), - SetupRes = rtcs:setup(1), - {AdminConfig, {RiakNodes, CSNodes, _Stanchion}} = SetupRes, - RiakNode = hd(RiakNodes), - UserConfig = rtcs_admin:create_user(RiakNode, 1), - - ?assertEqual(ok, erlcloud_s3:create_bucket(?BUCKET, UserConfig)), - lager:info("Investigating stats for this empty bucket..."), - assert_results_for_empty_bucket(AdminConfig, UserConfig, hd(CSNodes), ?BUCKET), - - setup_objects(UserConfig, ?BUCKET), - lager:info("Investigating stats for non empty bucket..."), - assert_results_for_non_empty_bucket(AdminConfig, UserConfig, hd(CSNodes), ?BUCKET), - - storage_stats_test:confirm_2(SetupRes), - rtcs:pass(). - -assert_results_for_empty_bucket(AdminConfig, UserConfig, CSNode, Bucket) -> - rt:setup_log_capture(CSNode), - {Begin, End} = storage_stats_test:calc_storage_stats(CSNode), - {JsonStat, XmlStat} = storage_stats_test:storage_stats_request( - AdminConfig, UserConfig, Begin, End), - rtcs:reset_log(CSNode), - lists:foreach(fun(K) -> - assert_storage_json_stats(Bucket, K, 0, JsonStat), - assert_storage_xml_stats(Bucket, K, 0, XmlStat) - end, - ["Objects", - "Bytes", - "Blocks", - "WritingMultipartObjects", - "WritingMultipartBytes", - "WritingMultipartBlocks", - "ScheduledDeleteNewObjects", - "ScheduledDeleteNewBytes", - "ScheduledDeleteNewBlocks"]), - ok. - -setup_objects(UserConfig, Bucket) -> - Block1 = crypto:rand_bytes(100), - ?assertEqual([{version_id, "null"}], - erlcloud_s3:put_object(Bucket, ?KEY1, Block1, UserConfig)), - Block1Overwrite = crypto:rand_bytes(300), - ?assertEqual([{version_id, "null"}], - erlcloud_s3:put_object(Bucket, ?KEY1, Block1Overwrite, UserConfig)), - Block2 = crypto:rand_bytes(200), - ?assertEqual([{version_id, "null"}], - erlcloud_s3:put_object(Bucket, ?KEY2, Block2, UserConfig)), - ?assertEqual([{delete_marker, false}, {version_id, "null"}], - erlcloud_s3:delete_object(Bucket, ?KEY2, UserConfig)), - - InitRes = erlcloud_s3_multipart:initiate_upload( - Bucket, ?KEY3, "text/plain", [], UserConfig), - UploadId = erlcloud_xml:get_text( - "/InitiateMultipartUploadResult/UploadId", InitRes), - MPBlocks = crypto:rand_bytes(2*1024*1024), - {_RespHeaders1, _UploadRes} = erlcloud_s3_multipart:upload_part( - Bucket, ?KEY3, UploadId, 1, MPBlocks, UserConfig), - {_RespHeaders2, _UploadRes} = erlcloud_s3_multipart:upload_part( - Bucket, ?KEY3, UploadId, 2, MPBlocks, UserConfig), - ok. - -assert_results_for_non_empty_bucket(AdminConfig, UserConfig, CSNode, Bucket) -> - rt:setup_log_capture(CSNode), - - {Begin, End} = storage_stats_test:calc_storage_stats(CSNode), - lager:info("Admin user will get every fields..."), - {JsonStat, XmlStat} = storage_stats_test:storage_stats_request( - AdminConfig, UserConfig, Begin, End), - - ?assert(rtcs:json_get([<<"StartTime">>], JsonStat) =/= notfound), - ?assert(rtcs:json_get([<<"EndTime">>], JsonStat) =/= notfound), - ?assert(proplists:get_value('StartTime', XmlStat) =/= notfound), - ?assert(proplists:get_value('EndTime', XmlStat) =/= notfound), - lists:foreach(fun({K, V}) -> - assert_storage_json_stats(Bucket, K, V, JsonStat), - assert_storage_xml_stats(Bucket, K, V, XmlStat) - end, - [{"Objects", 1 + 2}, - {"Bytes", 300 + 2 * 2*1024*1024}, - {"Blocks", 1 + 4}, - {"WritingMultipartObjects", 2}, - {"WritingMultipartBytes", 2 * 2*1024*1024}, - {"WritingMultipartBlocks", 2 * 2}, - {"ScheduledDeleteNewObjects", 2}, - {"ScheduledDeleteNewBytes", 100 + 200}, - {"ScheduledDeleteNewBlocks", 2}]), - - lager:info("Non-admin user will get only Objects and Bytes..."), - {JsonStat2, XmlStat2} = storage_stats_test:storage_stats_request( - UserConfig, UserConfig, Begin, End), - lists:foreach(fun({K, V}) -> - assert_storage_json_stats(Bucket, K, V, JsonStat2), - assert_storage_xml_stats(Bucket, K, V, XmlStat2) - end, - [{"Objects", 1 + 2}, - {"Bytes", 300 + 2 * 2*1024*1024}, - {"Blocks", notfound}, - {"WritingMultipartObjects", notfound}, - {"WritingMultipartBytes", notfound}, - {"WritingMultipartBlocks", notfound}, - {"ScheduledDeleteNewObjects", notfound}, - {"ScheduledDeleteNewBytes", notfound}, - {"ScheduledDeleteNewBlocks", notfound}]), - ok. - -assert_storage_json_stats(Bucket, K, V, Sample) -> - lager:debug("assert json: ~p", [{K, V}]), - ?assertEqual(V, rtcs:json_get([list_to_binary(Bucket), list_to_binary(K)], - Sample)). - -assert_storage_xml_stats(Bucket, K, V, Sample) -> - lager:debug("assert xml: ~p", [{K, V}]), - ?assertEqual(V, proplists:get_value(list_to_atom(K), - proplists:get_value(Bucket, Sample), - notfound)). diff --git a/riak_test/tests/storage_stats_test.erl b/riak_test/tests/storage_stats_test.erl deleted file mode 100644 index fe51a589d..000000000 --- a/riak_test/tests/storage_stats_test.erl +++ /dev/null @@ -1,324 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(storage_stats_test). -%% @doc Integration test for storage statistics. - --compile(export_all). --export([confirm/0]). - --include_lib("erlcloud/include/erlcloud_aws.hrl"). --include_lib("xmerl/include/xmerl.hrl"). --include_lib("eunit/include/eunit.hrl"). - --include("riak_cs.hrl"). - --define(BUCKET1, "storage-stats-test-1"). --define(BUCKET2, "storage-stats-test-2"). --define(BUCKET3, "storage-stats-test-3"). - --define(BUCKET4, "storage-stats-test-4"). --define(BUCKET5, "storage-stats-test-5"). --define(BUCKET6, "storage-stats-test-6"). --define(BUCKET7, "storage-stats-test-7"). --define(BUCKET8, "storage-stats-test-8"). - --define(BUCKET9, "storage-stats-test-9"). - --define(KEY, "1"). - --define(HIDDEN_KEY, "5=pockets"). - -confirm() -> - confirm_1(false). - -confirm_1(Use2iForStorageCalc) when is_boolean(Use2iForStorageCalc) -> - rtcs:set_advanced_conf(riak, [{riak_kv, [{delete_mode, keep}]}]), - rtcs:set_advanced_conf(cs, [{riak_cs, - [{use_2i_for_storage_calc, Use2iForStorageCalc}]}]), - SetupRes = rtcs:setup(1), - confirm_2(SetupRes). - -confirm_2({UserConfig, {RiakNodes, CSNodes, _Stanchion}}) -> - UserConfig2 = rtcs_admin:create_user(hd(RiakNodes), 1), - - TestSpecs = [store_object(?BUCKET1, UserConfig), - delete_object(?BUCKET2, UserConfig), - store_objects(?BUCKET3, UserConfig), - - %% for CS #840 regression - store_object(?BUCKET4, UserConfig), - store_object(?BUCKET5, UserConfig), - store_object(?BUCKET6, UserConfig), - store_object(?BUCKET7, UserConfig), - store_object(?BUCKET8, UserConfig), - give_over_bucket(?BUCKET9, UserConfig, UserConfig2) - ], - - verify_cs840_regression(UserConfig, RiakNodes), - - %% Set up to grep logs to verify messages - rt:setup_log_capture(hd(CSNodes)), - - {Begin, End} = calc_storage_stats(hd(CSNodes)), - {JsonStat, XmlStat} = storage_stats_request(UserConfig, Begin, End), - lists:foreach(fun(Spec) -> - assert_storage_json_stats(Spec, JsonStat), - assert_storage_xml_stats(Spec, XmlStat) - end, TestSpecs), - rtcs:pass(). - -%% @doc garbage data to check #840 regression, -%% due to this garbages, following tests may fail -%% makes manifest in BUCKET(4,5,6,7,8) to garbage, which can -%% be generated from former versions of riak cs than 1.4.5 -verify_cs840_regression(UserConfig, RiakNodes) -> - - %% None of thes objects should not be calculated effective in storage - ok = mess_with_writing_various_props( - RiakNodes, UserConfig, - [%% state=writing, .props=undefined - {?BUCKET4, ?KEY, writing, undefined}, - %% badly created ongoing multipart uploads (not really) - {?BUCKET5, ?KEY, writing, [{multipart, undefined}]}, - {?BUCKET6, ?KEY, writing, [{multipart, pocketburgerking}]}]), - - %% state=active, .props=undefined in {?BUCKET7, ?KEY} - ok = mess_with_active_undefined(RiakNodes), - %% tombstone in siblings in {?BUCKET8, ?KEY} - ok = mess_with_tombstone(RiakNodes, UserConfig), - ok. - -mess_with_writing_various_props(RiakNodes, UserConfig, VariousProps) -> - F = fun({CSBucket, CSKey, NewState, Props}) -> - Bucket = <<"0o:", (rtcs:md5(list_to_binary(CSBucket)))/binary>>, - Pid = rtcs:pbc(RiakNodes, objects, CSBucket), - {ok, RiakObject0} = riakc_pb_socket:get(Pid, Bucket, list_to_binary(CSKey)), - [{UUID, Manifest0}|_] = hd([binary_to_term(V) || V <- riakc_obj:get_values(RiakObject0)]), - Manifest1 = Manifest0?MANIFEST{state=NewState, props=Props}, - RiakObject = riakc_obj:update_value(RiakObject0, - term_to_binary([{UUID, Manifest1}])), - lager:info("~p", [Manifest1?MANIFEST.props]), - - Block = crypto:rand_bytes(100), - ?assertEqual([{version_id, "null"}], erlcloud_s3:put_object(CSBucket, CSKey, - Block, UserConfig)), - ok = riakc_pb_socket:put(Pid, RiakObject), - assure_num_siblings(Pid, Bucket, list_to_binary(CSKey), 2), - ok = riakc_pb_socket:stop(Pid) - end, - lists:foreach(F, VariousProps). - - -mess_with_active_undefined(RiakNodes) -> - CSBucket = ?BUCKET7, CSKey = ?KEY, - Pid = rtcs:pbc(RiakNodes, objects, CSBucket), - Bucket = <<"0o:", (rtcs:md5(list_to_binary(CSBucket)))/binary>>, - {ok, RiakObject0} = riakc_pb_socket:get(Pid, Bucket, list_to_binary(CSKey)), - [{UUID, Manifest0}|_] = hd([binary_to_term(V) || V <- riakc_obj:get_values(RiakObject0)]), - Manifest1 = Manifest0?MANIFEST{props=undefined}, - RiakObject = riakc_obj:update_value(RiakObject0, - term_to_binary([{UUID, Manifest1}])), - ok = riakc_pb_socket:put(Pid, RiakObject), - ok = riakc_pb_socket:stop(Pid). - -%% @doc messing with tombstone (see above adding {delete_mode, keep} to riak_kv) -mess_with_tombstone(RiakNodes, UserConfig) -> - CSBucket = ?BUCKET8, - CSKey = ?KEY, - Pid = rtcs:pbc(RiakNodes, objects, CSBucket), - Block = crypto:rand_bytes(100), - ?assertEqual([{version_id, "null"}], erlcloud_s3:put_object(CSBucket, CSKey, - Block, UserConfig)), - Bucket = <<"0o:", (rtcs:md5(list_to_binary(?BUCKET8)))/binary>>, - - %% %% This leaves a tombstone which messes up the storage calc - ok = riakc_pb_socket:delete(Pid, Bucket, list_to_binary(CSKey)), - %% lager:info("listkeys: ~p", [riakc_pb_socket:list_keys(Pid, Bucket)]), - - ?assertEqual([{version_id, "null"}], erlcloud_s3:put_object(?BUCKET8, CSKey, - Block, UserConfig)), - - {ok, RiakObject0} = riakc_pb_socket:get(Pid, Bucket, list_to_binary(CSKey)), - assure_num_siblings(Pid, Bucket, list_to_binary(CSKey), 1), - - Block2 = crypto:rand_bytes(100), - ?assertEqual([{version_id, "null"}], erlcloud_s3:put_object(?BUCKET8, CSKey, - Block2, UserConfig)), - - ok = riakc_pb_socket:delete_vclock(Pid, Bucket, list_to_binary(CSKey), - riakc_obj:vclock(RiakObject0)), - - %% Two siblings, alive object and new tombstone - assure_num_siblings(Pid, Bucket, list_to_binary(CSKey), 2), - - %% Here at last, ?BUCKET8 should have ?KEY alive and counted, but - %% #840 causes, ?KEY won't be counted in usage calc - Obj = erlcloud_s3:get_object(?BUCKET8, CSKey, UserConfig), - ?assertEqual(byte_size(Block2), list_to_integer(proplists:get_value(content_length, Obj))), - ?assertEqual(Block2, proplists:get_value(content, Obj)), - ok = riakc_pb_socket:stop(Pid). - -assure_num_siblings(Pid, Bucket, Key, Num) -> - {ok, RiakObject0} = riakc_pb_socket:get(Pid, Bucket, Key), - Contents = riakc_obj:get_values(RiakObject0), - ?assertEqual(Num, length(Contents)). - - -store_object(Bucket, UserConfig) -> - lager:info("creating bucket ~p", [Bucket]), - %% Create bucket - ?assertEqual(ok, erlcloud_s3:create_bucket(Bucket, UserConfig)), - %% Put 100-byte object - Block = crypto:rand_bytes(100), - ?assertEqual([{version_id, "null"}], erlcloud_s3:put_object(Bucket, ?KEY, Block, UserConfig)), - ExpectedObjects = 1, - ExpectedBytes = 100, - {Bucket, ExpectedObjects, ExpectedBytes}. - -delete_object(Bucket, UserConfig) -> - lager:info("creating bucket ~p", [Bucket]), - %% Create bucket - ?assertEqual(ok, erlcloud_s3:create_bucket(Bucket, UserConfig)), - %% Put 100-byte object - Block = crypto:rand_bytes(100), - ?assertEqual([{version_id, "null"}], erlcloud_s3:put_object(Bucket, ?KEY, Block, UserConfig)), - ?assertEqual([{delete_marker, false}, {version_id, "null"}], erlcloud_s3:delete_object(Bucket, ?KEY, UserConfig)), - ExpectedObjects = 0, - ExpectedBytes = 0, - {Bucket, ExpectedObjects, ExpectedBytes}. - -store_objects(Bucket, UserConfig) -> - lager:info("creating bucket ~p", [Bucket]), - %% Create bucket - ?assertEqual(ok, erlcloud_s3:create_bucket(Bucket, UserConfig)), - %% Put 100-byte object 10 times - Block = crypto:rand_bytes(100), - [?assertEqual([{version_id, "null"}], - erlcloud_s3:put_object(Bucket, integer_to_list(Key), Block, UserConfig)) - || Key <- lists:seq(1, 10)], - ExpectedObjects = 10, - ExpectedBytes = 1000, - {Bucket, ExpectedObjects, ExpectedBytes}. - -give_over_bucket(Bucket, UserConfig, AnotherUser) -> - %% Create bucket, put/delete object, delete bucket finally - ?assertEqual(ok, erlcloud_s3:create_bucket(Bucket, UserConfig)), - Block = crypto:rand_bytes(100), - ?assertEqual([{version_id, "null"}], erlcloud_s3:put_object(Bucket, ?KEY, Block, UserConfig)), - ?assertEqual([{delete_marker, false}, {version_id, "null"}], erlcloud_s3:delete_object(Bucket, ?KEY, UserConfig)), - ?assertEqual(ok, erlcloud_s3:delete_bucket(Bucket, UserConfig)), - - %% Another user re-create the bucket and put an object into it. - ?assertEqual(ok, erlcloud_s3:create_bucket(Bucket, AnotherUser)), - Block2 = crypto:rand_bytes(100), - ?assertEqual([{version_id, "null"}], - erlcloud_s3:put_object(Bucket, ?KEY, Block2, AnotherUser)), - {Bucket, undefined, undefined}. - -calc_storage_stats(CSNode) -> - Begin = rtcs:datetime(), - %% FIXME: workaround for #766 - timer:sleep(1000), - Res = rtcs_exec:calculate_storage(1), - lager:info("riak-cs-storage batch result: ~s", [Res]), - ExpectRegexp = "Batch storage calculation started.\n$", - ?assertMatch({match, _}, re:run(Res, ExpectRegexp)), - true = rt:expect_in_log(CSNode, "Finished storage calculation"), - %% FIXME: workaround for #766 - timer:sleep(1000), - End = rtcs:datetime(), - {Begin, End}. - -assert_storage_json_stats({Bucket, undefined, undefined}, Sample) -> - ?assertEqual(notfound, rtcs:json_get([list_to_binary(Bucket)], Sample)); -assert_storage_json_stats({Bucket, ExpectedObjects, ExpectedBytes}, Sample) -> - ?assertEqual(ExpectedObjects, rtcs:json_get([list_to_binary(Bucket), <<"Objects">>], Sample)), - ?assertEqual(ExpectedBytes, rtcs:json_get([list_to_binary(Bucket), <<"Bytes">>], Sample)), - ?assert(rtcs:json_get([<<"StartTime">>], Sample) =/= notfound), - ?assert(rtcs:json_get([<<"EndTime">>], Sample) =/= notfound), - ok. - -assert_storage_xml_stats({Bucket, undefined, undefined}, Sample) -> - ?assertEqual(undefined, proplists:get_value(Bucket, Sample)); -assert_storage_xml_stats({Bucket, ExpectedObjects, ExpectedBytes}, Sample) -> - ?assertEqual(ExpectedObjects, proplists:get_value('Objects', proplists:get_value(Bucket, Sample))), - ?assertEqual(ExpectedBytes, proplists:get_value('Bytes', proplists:get_value(Bucket, Sample))), - ?assert(proplists:get_value('StartTime', Sample) =/= notfound), - ?assert(proplists:get_value('EndTime', Sample) =/= notfound), - ok. - -storage_stats_request(UserConfig, Begin, End) -> - storage_stats_request(UserConfig, UserConfig, Begin, End). - -storage_stats_request(SignUserConfig, UserConfig, Begin, End) -> - {storage_stats_json_request(SignUserConfig, UserConfig, Begin, End), - storage_stats_xml_request(SignUserConfig, UserConfig, Begin, End)}. - -storage_stats_json_request(SignUserConfig, UserConfig, Begin, End) -> - Samples = samples_from_json_request(SignUserConfig, UserConfig, {Begin, End}), - lager:debug("Storage samples[json]: ~p", [Samples]), - ?assertEqual(1, length(Samples)), - [Sample] = Samples, - lager:info("Storage sample[json]: ~p", [Sample]), - Sample. - -storage_stats_xml_request(SignUserConfig, UserConfig, Begin, End) -> - Samples = samples_from_xml_request(SignUserConfig, UserConfig, {Begin, End}), - lager:debug("Storage samples[xml]: ~p", [Samples]), - ?assertEqual(1, length(Samples)), - [Sample] = Samples, - ParsedSample = to_proplist_stats(Sample), - lager:info("Storage sample[xml]: ~p", [ParsedSample]), - ParsedSample. - -samples_from_json_request(SignUserConfig, UserConfig, {Begin, End}) -> - KeyId = UserConfig#aws_config.access_key_id, - StatsKey = string:join(["usage", KeyId, "bj", Begin, End], "/"), - GetResult = erlcloud_s3:get_object("riak-cs", StatsKey, SignUserConfig), - lager:debug("GET Storage stats response[json]: ~p", [GetResult]), - Usage = mochijson2:decode(proplists:get_value(content, GetResult)), - lager:debug("Usage Response[json]: ~p", [Usage]), - rtcs:json_get([<<"Storage">>, <<"Samples">>], Usage). - -samples_from_xml_request(SignUserConfig, UserConfig, {Begin, End}) -> - KeyId = UserConfig#aws_config.access_key_id, - StatsKey = string:join(["usage", KeyId, "bx", Begin, End], "/"), - GetResult = erlcloud_s3:get_object("riak-cs", StatsKey, SignUserConfig), - lager:debug("GET Storage stats response[xml]: ~p", [GetResult]), - {Usage, _Rest} = xmerl_scan:string(binary_to_list(proplists:get_value(content, GetResult))), - lager:debug("Usage Response[xml]: ~p", [Usage]), - xmerl_xpath:string("//Storage/Samples/Sample",Usage). - -to_proplist_stats(Sample) -> - lists:foldl(fun extract_bucket/2, [], Sample#xmlElement.content) - ++ lists:foldl(fun extract_slice/2, [], Sample#xmlElement.attributes). - -extract_bucket(#xmlElement{name='Bucket', attributes=[#xmlAttribute{value=Bucket}], content=Content}, Acc) -> - [{Bucket, lists:foldl(fun extract_usage/2,[], Content)}|Acc]. - -extract_slice(#xmlAttribute{name=Name, value=Value}, Acc) -> - [{Name, Value}|Acc]. - -extract_usage(#xmlElement{name=Name, content=[Content]}, Acc) -> - [{Name, extract_value(Content)}|Acc]. - -extract_value(#xmlText{value=Content}) -> - list_to_integer(Content). diff --git a/riak_test/tests/too_large_entity_test.erl b/riak_test/tests/too_large_entity_test.erl deleted file mode 100644 index aabb92597..000000000 --- a/riak_test/tests/too_large_entity_test.erl +++ /dev/null @@ -1,117 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(too_large_entity_test). - -%% @doc `riak_test' module for testing the behavior in dealing with -%% entities that violate the specified object size restrictions - --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). - --define(TEST_BUCKET, "riak-test-bucket"). --define(TEST_KEY1, "riak_test_key1"). --define(TEST_KEY2, "riak_test_key2"). --define(PART_COUNT, 5). --define(GOOD_PART_SIZE, 5*1024*1024). --define(BAD_PART_SIZE, 2*1024*1024). - -confirm() -> - rtcs:set_advanced_conf(cs, cs_config()), - {UserConfig, {_RiakNodes, _CSNodes, _Stanchion}} = rtcs:setup(1), - - lager:info("User is valid on the cluster, and has no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(UserConfig)), - - ?assertError({aws_error, {http_error, 404, _, _}}, erlcloud_s3:list_objects(?TEST_BUCKET, UserConfig)), - - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - - ?assertMatch([{buckets, [[{name, ?TEST_BUCKET}, _]]}], - erlcloud_s3:list_buckets(UserConfig)), - - %% Test cases - too_large_upload_part_test_case(?TEST_BUCKET, ?TEST_KEY1, UserConfig), - too_large_object_put_test_case(?TEST_BUCKET, ?TEST_KEY2, UserConfig), - - lager:info("deleting bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:delete_bucket(?TEST_BUCKET, UserConfig)), - - ?assertError({aws_error, {http_error, 404, _, _}}, erlcloud_s3:list_objects(?TEST_BUCKET, UserConfig)), - rtcs:pass(). - -generate_part_data(X, Size) - when 0 =< X, X =< 255 -> - list_to_binary( - [X || _ <- lists:seq(1, Size)]). - -too_large_upload_part_test_case(Bucket, Key, Config) -> - %% Initiate a multipart upload - lager:info("Initiating multipart upload"), - InitUploadRes = erlcloud_s3_multipart:initiate_upload(Bucket, Key, [], [], Config), - UploadId = erlcloud_s3_multipart:upload_id(InitUploadRes), - - %% Verify the upload id is in list_uploads results and - %% that the bucket information is correct - UploadsList1 = erlcloud_s3_multipart:list_uploads(Bucket, [], Config), - Uploads1 = proplists:get_value(uploads, UploadsList1, []), - ?assertEqual(Bucket, proplists:get_value(bucket, UploadsList1)), - ?assert(upload_id_present(UploadId, Uploads1)), - - lager:info("Uploading an oversize part"), - ?assertError({aws_error, {http_error, 400, _, _}}, - erlcloud_s3_multipart:upload_part(Bucket, - Key, - UploadId, - 1, - generate_part_data(61, 2000), - Config)). - -too_large_object_put_test_case(Bucket, Key, Config) -> - Object1 = crypto:rand_bytes(1001), - Object2 = crypto:rand_bytes(1000), - - ?assertError({aws_error, {http_error, 400, _, _}}, - erlcloud_s3:put_object(Bucket, Key, Object1, Config)), - - erlcloud_s3:put_object(Bucket, Key, Object2, Config), - - ObjList1 = erlcloud_s3:list_objects(Bucket, Config), - ?assertEqual([Key], - [proplists:get_value(key, O) || - O <- proplists:get_value(contents, ObjList1)]), - - erlcloud_s3:delete_object(Bucket, Key, Config), - - ObjList2 = erlcloud_s3:list_objects(Bucket, Config), - ?assertEqual([], proplists:get_value(contents, ObjList2)). - -upload_id_present(UploadId, UploadList) -> - [] /= [UploadData || UploadData <- UploadList, - proplists:get_value(upload_id, UploadData) =:= UploadId]. - -cs_config() -> - [{riak_cs, - [ - {max_content_length, 1000}, - {enforce_multipart_part_size, false} - ] - }]. diff --git a/riak_test/tests/upgrade_downgrade_test.erl b/riak_test/tests/upgrade_downgrade_test.erl deleted file mode 100644 index cff44d5a7..000000000 --- a/riak_test/tests/upgrade_downgrade_test.erl +++ /dev/null @@ -1,156 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(upgrade_downgrade_test). --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). --include_lib("erlcloud/include/erlcloud_aws.hrl"). - --define(TEST_BUCKET, "riak-test-bucket-foobar"). --define(KEY_SINGLE_BLOCK, "riak_test_key1"). --define(KEY_MULTIPLE_BLOCK, "riak_test_key2"). - -confirm() -> - PrevConfig = rtcs_config:previous_configs(), - {UserConfig, {RiakNodes, _CSNodes, _Stanchion}} = - rtcs:setup(2, PrevConfig, previous), - - lager:info("nodes> ~p", [rt_config:get(rt_nodes)]), - lager:info("versions> ~p", [rt_config:get(rt_versions)]), - - {ok, Data} = prepare_all_data(UserConfig), - ok = verify_all_data(UserConfig, Data), - - AdminCreds = {UserConfig#aws_config.access_key_id, - UserConfig#aws_config.secret_access_key}, - {_, RiakCurrentVsn} = - rtcs_dev:riak_root_and_vsn(current, rt_config:get(build_type, oss)), - - %% Upgrade!!! - [begin - N = rtcs_dev:node_id(RiakNode), - lager:debug("upgrading ~p", [N]), - rtcs_exec:stop_cs(N, previous), - ok = rt:upgrade(RiakNode, RiakCurrentVsn), - rt:wait_for_service(RiakNode, riak_kv), - ok = rtcs_config:upgrade_cs(N, AdminCreds), - rtcs:set_advanced_conf({cs, current, N}, - [{riak_cs, - [{riak_host, {"127.0.0.1", rtcs_config:pb_port(1)}}]}]), - rtcs_exec:start_cs(N, current) - end - || RiakNode <- RiakNodes], - rt:wait_until_ring_converged(RiakNodes), - rtcs_exec:stop_stanchion(previous), - rtcs_config:migrate_stanchion(previous, current, AdminCreds), - rtcs_exec:start_stanchion(current), - - ok = verify_all_data(UserConfig, Data), - ok = cleanup_all_data(UserConfig), - lager:info("Upgrading to current successfully done"), - - {ok, Data2} = prepare_all_data(UserConfig), - - {_, RiakPrevVsn} = - rtcs_dev:riak_root_and_vsn(previous, rt_config:get(build_type, oss)), - - - %% Downgrade!! - rtcs_exec:stop_stanchion(current), - rtcs_config:migrate_stanchion(current, previous, AdminCreds), - rtcs_exec:start_stanchion(previous), - [begin - N = rtcs_dev:node_id(RiakNode), - lager:debug("downgrading ~p", [N]), - rtcs_exec:stop_cs(N, current), - rt:stop(RiakNode), - rt:wait_until_unpingable(RiakNode), - - %% get the bitcask directory - BitcaskDataDir = filename:join([rtcs_dev:node_path(RiakNode), "data", "bitcask"]), - lager:info("downgrading Bitcask datadir ~s...", [BitcaskDataDir]), - %% and run the downgrade script: - %% Downgrading from 2.0 does not work... - %% https://github.com/basho/bitcask/issues/178 - %% And here's the downgrade script, which is downloaded at `make compile-riak-test`. - %% https://github.com/basho/bitcask/pull/184 - Result = downgrade_bitcask:main([BitcaskDataDir]), - lager:info("downgrade script done: ~p", [Result]), - - ok = rt:upgrade(RiakNode, RiakPrevVsn), - rt:wait_for_service(RiakNode, riak_kv), - ok = rtcs_config:migrate_cs(current, previous, N, AdminCreds), - rtcs_exec:start_cs(N, previous) - end - || RiakNode <- RiakNodes], - rt:wait_until_ring_converged(RiakNodes), - - ok = verify_all_data(UserConfig, Data2), - lager:info("Downgrading to previous successfully done"), - - rtcs:pass(). - -%% TODO: add more data and test cases -prepare_all_data(UserConfig) -> - lager:info("User is valid on the cluster, and has no buckets"), - ?assertEqual([{buckets, []}], erlcloud_s3:list_buckets(UserConfig)), - - lager:info("creating bucket ~p", [?TEST_BUCKET]), - ?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)), - - ?assertMatch([{buckets, [[{name, ?TEST_BUCKET}, _]]}], - erlcloud_s3:list_buckets(UserConfig)), - - %% setup objects - SingleBlock = crypto:rand_bytes(400), - erlcloud_s3:put_object(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, SingleBlock, UserConfig), - MultipleBlock = crypto:rand_bytes(4000000), % not aligned to block boundary - erlcloud_s3:put_object(?TEST_BUCKET, ?KEY_MULTIPLE_BLOCK, MultipleBlock, UserConfig), - - {ok, [{single_block, SingleBlock}, - {multiple_block, MultipleBlock}]}. - -%% TODO: add more data and test cases -verify_all_data(UserConfig, Data) -> - SingleBlock = proplists:get_value(single_block, Data), - MultipleBlock = proplists:get_value(multiple_block, Data), - - %% basic GET test cases - basic_get_test_case(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, SingleBlock, UserConfig), - basic_get_test_case(?TEST_BUCKET, ?KEY_MULTIPLE_BLOCK, MultipleBlock, UserConfig), - - ok. - -cleanup_all_data(UserConfig) -> - erlcloud_s3:delete_object(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, UserConfig), - erlcloud_s3:delete_object(?TEST_BUCKET, ?KEY_MULTIPLE_BLOCK, UserConfig), - erlcloud_s3:delete_bucket(?TEST_BUCKET, UserConfig), - ok. - -basic_get_test_case(Bucket, Key, ExpectedContent, Config) -> - Obj = erlcloud_s3:get_object(Bucket, Key, Config), - assert_whole_content(ExpectedContent, Obj). - -assert_whole_content(ExpectedContent, ResultObj) -> - Content = proplists:get_value(content, ResultObj), - ContentLength = proplists:get_value(content_length, ResultObj), - ?assertEqual(byte_size(ExpectedContent), list_to_integer(ContentLength)), - ?assertEqual(byte_size(ExpectedContent), byte_size(Content)), - ?assertEqual(ExpectedContent, Content). diff --git a/riak_test/tests/user_test.erl b/riak_test/tests/user_test.erl deleted file mode 100644 index 249718c50..000000000 --- a/riak_test/tests/user_test.erl +++ /dev/null @@ -1,382 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(user_test). - --compile(export_all). --export([confirm/0]). --include_lib("eunit/include/eunit.hrl"). --include_lib("erlcloud/include/erlcloud_aws.hrl"). --include_lib("xmerl/include/xmerl.hrl"). - --define(TEST_BUCKET, "riak-test-bucket"). --define(JSON, "application/json"). --define(XML, "application/xml"). - -confirm() -> - {AdminUserConfig, {RiakNodes, _CSNodes, _Stanchion}} = rtcs:setup(1), - - HeadRiakNode = hd(RiakNodes), - AdminUser = {"admin@me.com", "admin", - AdminUserConfig#aws_config.access_key_id, - AdminUserConfig#aws_config.secret_access_key, - "enabled"}, - user_listing_json_test_case([AdminUser], AdminUserConfig, HeadRiakNode), - user_listing_xml_test_case([AdminUser], AdminUserConfig, HeadRiakNode), - - %% Create other 1003 users and re-run user listing test cases - Port = rtcs_config:cs_port(HeadRiakNode), - Users1 = [AdminUser | - create_users(Port, [{"bart@simpsons.com", "bart"}, - {"homer@simpsons.com", "homer"}, - {"taro@example.co.jp", japanese_aiueo()}], [])], - Users2 = Users1 ++ create_200_users(Port), - ?assertEqual(1 + 3 + 200, length(Users2)), - - user_listing_json_test_case(Users2, AdminUserConfig, HeadRiakNode), - user_listing_xml_test_case(Users2, AdminUserConfig, HeadRiakNode), - user_listing_many_times(Users2, AdminUserConfig, HeadRiakNode), - update_user_json_test_case(AdminUserConfig, HeadRiakNode), - update_user_xml_test_case(AdminUserConfig, HeadRiakNode), - rtcs:pass(). - -japanese_aiueo() -> - %% To avoid dependency on source code encoding, create list from chars. - %% These five numbers represents "あいうえお" (A-I-U-E-O in Japanese). - %% unicode:characters_to_binary([12354,12356,12358,12360,12362]). - Chars = [12354,12356,12358,12360,12362], - binary_to_list(unicode:characters_to_binary(Chars)). - -create_200_users(Port) -> - From = self(), - Processes = 5, - PerProcess = 40, - [spawn(fun() -> - Users = create_users( - Port, - [begin - Name = "zzz-" ++ integer_to_list(I * PerProcess + J), - {Name ++ "@thousand.example.com", Name} - end || J <- lists:seq(1, PerProcess)], - []), - From ! Users - end) || - I <- lists:seq(1, Processes)], - collect_users(Processes, []). - -collect_users(0, Acc) -> - lists:usort(lists:flatten(Acc)); -collect_users(N, Acc) -> - receive - Users -> collect_users(N-1, [Users | Acc]) - end. - -create_users(_Port, [], Acc) -> - ordsets:from_list(Acc); -create_users(Port, [{Email, Name} | Users], Acc) -> - {UserConfig, _Id} = rtcs_admin:create_user(Port, Email, Name), - create_users(Port, Users, [{Email, - Name, - UserConfig#aws_config.access_key_id, - UserConfig#aws_config.secret_access_key, - "enabled"} | Acc]). - -user_listing_json_test_case(Users, UserConfig, Node) -> - user_listing_test(Users, UserConfig, Node, ?JSON). - -user_listing_xml_test_case(Users, UserConfig, Node) -> - user_listing_test(Users, UserConfig, Node, ?XML). - -user_listing_many_times(Users, UserConfig, Node) -> - [user_listing_test(Users, UserConfig, Node, ?JSON) || - _I <- lists:seq(1, 15)], - ok. - -user_listing_test(ExpectedUsers, UserConfig, Node, ContentType) -> - Resource = "/riak-cs/users", - Port = rtcs_config:cs_port(Node), - Users = parse_user_info( - rtcs_admin:list_users(UserConfig, Port, Resource, ContentType)), - ?assertEqual(ExpectedUsers, Users). - -update_user_json_test_case(AdminConfig, Node) -> - Users = [{"fergus@brave.sco", "Fergus"}, - {"merida@brave.sco", "Merida"}, - {"seamus@brave.sco", "Seamus"}], - update_user_test(AdminConfig, Node, ?JSON, Users). - -update_user_xml_test_case(AdminConfig, Node) -> - Users = [{"gru@despicable.me", "Gru"}, - {"minion@despicable.me", "Minion Minion"}, - {"dr.nefario@despicable.me", "DrNefario"}], - update_user_test(AdminConfig, Node, ?XML, Users). - -update_user_test(AdminConfig, Node, ContentType, Users) -> - [{Email1, User1}, {Email2, User2}, {Email3, User3}]= Users, - Port = rtcs_config:cs_port(Node), - {UserConfig, _} = rtcs_admin:create_user(Port, Email1, User1), - {BadUserConfig, _} = rtcs_admin:create_user(Port, Email3, User3), - - #aws_config{access_key_id=Key, secret_access_key=Secret} = UserConfig, - - UserResource = "/riak-cs/user", - AdminResource = UserResource ++ "/" ++ Key, - - %% Fetch the user record using the user's own credentials - UserResult1 = parse_user_record( - rtcs_admin:get_user(UserConfig, Port, UserResource, ContentType), - ContentType), - %% Fetch the user record using the admin credentials - UserResult2 = parse_user_record( - rtcs_admin:get_user(AdminConfig, Port, AdminResource, ContentType), - ContentType), - ?assertMatch({Email1, User1, _, Secret, "enabled"}, UserResult1), - ?assertMatch({Email1, User1, _, Secret, "enabled"}, UserResult2), - - %% Attempt to update the user's email to be the same as the admin - %% user and verify that the update attempt returns an error. - Resource = "/riak-cs/user/" ++ Key, - InvalidUpdateDoc = update_email_and_name_doc(ContentType, "admin@me.com", "admin"), - - ErrorResult = parse_error_code( - catch rtcs_admin:update_user(UserConfig, - Port, - Resource, - ContentType, - InvalidUpdateDoc)), - ?assertEqual({409, "UserAlreadyExists"}, ErrorResult), - - %% Test updating the user's name and email - UpdateDoc = update_email_and_name_doc(ContentType, Email2, User2), - _ = rtcs_admin:update_user(UserConfig, Port, Resource, ContentType, UpdateDoc), - - %% Fetch the user record using the user's own credentials - UserResult3 = parse_user_record( - rtcs_admin:get_user(UserConfig, Port, UserResource, ContentType), - ContentType), - %% Fetch the user record using the admin credentials - UserResult4 = parse_user_record( - rtcs_admin:get_user(AdminConfig, Port, AdminResource, ContentType), - ContentType), - ?assertMatch({Email2, User2, _, Secret, "enabled"}, UserResult3), - ?assertMatch({Email2, User2, _, Secret, "enabled"}, UserResult4), - - %% Test that attempting to update another user's status with a - %% non-admin account is disallowed - UpdateDoc2 = update_status_doc(ContentType, "disabled"), - Resource = "/riak-cs/user/" ++ Key, - ErrorResult2 = parse_error_code( - catch rtcs_admin:update_user(BadUserConfig, - Port, - Resource, - ContentType, - UpdateDoc2)), - ?assertEqual({403, "AccessDenied"}, ErrorResult2), - - %% Test updating a user's own status - Resource = "/riak-cs/user/" ++ Key, - _ = rtcs_admin:update_user(UserConfig, Port, Resource, ContentType, UpdateDoc2), - - %% Fetch the user record using the user's own credentials. Since - %% the user is now disabled this should return an error. - UserResult5 = parse_error_code( - catch rtcs_admin:get_user(UserConfig, Port, - UserResource, ContentType)), - - %% Fetch the user record using the admin credentials. The user is - %% not able to retrieve their own account information now that the - %% account is disabled. - UserResult6 = parse_user_record( - rtcs_admin:get_user(AdminConfig, Port, AdminResource, ContentType), - ContentType), - ?assertEqual({403, "AccessDenied"}, UserResult5), - ?assertMatch({Email2, User2, _, Secret, "disabled"}, UserResult6), - - %% Re-enable the user - UpdateDoc3 = update_status_doc(ContentType, "enabled"), - Resource = "/riak-cs/user/" ++ Key, - _ = rtcs_admin:update_user(AdminConfig, Port, Resource, ContentType, UpdateDoc3), - - %% Test issuing a new key_secret - UpdateDoc4 = new_key_secret_doc(ContentType), - Resource = "/riak-cs/user/" ++ Key, - UpdateResult = rtcs_admin:update_user(AdminConfig, - Port, - Resource, - ContentType, - UpdateDoc4), - {_, _, _, UpdSecret1, _} = parse_user_record(UpdateResult, ContentType), - - %% Generate an updated user config with the new secret - UserConfig2 = rtcs_admin:aws_config(UserConfig, [{secret, UpdSecret1}]), - - %% Fetch the user record using the user's own credentials - UserResult7 = parse_user_record( - rtcs_admin:get_user(UserConfig2, Port, UserResource, ContentType), - ContentType), - %% Fetch the user record using the admin credentials - UserResult8 = parse_user_record( - rtcs_admin:get_user(AdminConfig, Port, AdminResource, ContentType), - ContentType), - ?assertMatch({_, _, _, UpdSecret1, _}, UserResult7), - ?assertMatch({_, _, _, UpdSecret1, _}, UserResult8), - ?assertMatch({Email2, User2, _, _, "enabled"}, UserResult7), - ?assertMatch({Email2, User2, _, _, "enabled"}, UserResult8). - -new_key_secret_doc(?JSON) -> - "{\"new_key_secret\": true}"; -new_key_secret_doc(?XML) -> - "true". - -update_status_doc(?JSON, Status) -> - "{\"status\":\"" ++ Status ++ "\"}"; -update_status_doc(?XML, Status) -> - "" ++ Status ++ "". - -update_email_and_name_doc(?JSON, Email, Name) -> - "{\"email\":\"" ++ Email ++ "\", \"name\":\"" ++ Name ++"\"}"; -update_email_and_name_doc(?XML, Email, Name) -> - "" ++ Email ++ - "" ++ Name ++ "enabled". - -parse_user_record(Output, ?JSON) -> - {struct, JsonData} = mochijson2:decode(Output), - Email = binary_to_list(proplists:get_value(<<"email">>, JsonData)), - Name = binary_to_list(proplists:get_value(<<"name">>, JsonData)), - KeyId = binary_to_list(proplists:get_value(<<"key_id">>, JsonData)), - KeySecret = binary_to_list(proplists:get_value(<<"key_secret">>, JsonData)), - Status = binary_to_list(proplists:get_value(<<"status">>, JsonData)), - {Email, Name, KeyId, KeySecret, Status}; -parse_user_record(Output, ?XML) -> - {ParsedData, _Rest} = xmerl_scan:string(Output, []), - lists:foldl(fun user_fields_from_xml/2, - {[], [], [], [], []}, - ParsedData#xmlElement.content). - -parse_user_records(Output, ?JSON) -> - JsonData = mochijson2:decode(Output), - [begin - Email = binary_to_list(proplists:get_value(<<"email">>, UserJson)), - Name = binary_to_list(proplists:get_value(<<"name">>, UserJson)), - KeyId = binary_to_list(proplists:get_value(<<"key_id">>, UserJson)), - KeySecret = binary_to_list(proplists:get_value(<<"key_secret">>, UserJson)), - Status = binary_to_list(proplists:get_value(<<"status">>, UserJson)), - {Email, Name, KeyId, KeySecret, Status} - end || {struct, UserJson} <- JsonData]; -parse_user_records(Output, ?XML) -> - {ParsedData, _Rest} = xmerl_scan:string(Output, []), - [lists:foldl(fun user_fields_from_xml/2, - {[], [], [], [], []}, - UserXml#xmlElement.content) - || UserXml <- ParsedData#xmlElement.content]. - --spec user_fields_from_xml(#xmlText{} | #xmlElement{}, tuple()) -> tuple(). -user_fields_from_xml(#xmlText{}, Acc) -> - Acc; -user_fields_from_xml(Element, {Email, Name, KeyId, Secret, Status}=Acc) -> - case Element#xmlElement.name of - 'Email' -> - [Content | _] = Element#xmlElement.content, - case is_record(Content, xmlText) of - true -> - {xml_text_value(Content), Name, KeyId, Secret, Status}; - false -> - Acc - end; - 'Name' -> - [Content | _] = Element#xmlElement.content, - case is_record(Content, xmlText) of - true -> - {Email, xml_text_value(Content), KeyId, Secret, Status}; - false -> - Acc - end; - 'KeyId' -> - [Content | _] = Element#xmlElement.content, - case is_record(Content, xmlText) of - true -> - {Email, Name, Content#xmlText.value, Secret, Status}; - false -> - Acc - end; - 'KeySecret' -> - [Content | _] = Element#xmlElement.content, - case is_record(Content, xmlText) of - true -> - {Email, Name, KeyId, Content#xmlText.value, Status}; - false -> - Acc - end; - 'Status' -> - [Content | _] = Element#xmlElement.content, - case is_record(Content, xmlText) of - true -> - {Email, Name, KeyId, Secret, Content#xmlText.value}; - false -> - Acc - end; - _ -> - Acc - end. - -xml_text_value(XmlText) -> - %% xmerl return list of UTF-8 characters, each element of it represent - %% one character (or codepoint), not one byte. - binary_to_list(unicode:characters_to_binary(XmlText#xmlText.value)). - -parse_error_code(Output) -> - {'EXIT', {{aws_error, {http_error, Status, _, Body}}, _Backtrace}} = Output, - {ParsedData, _Rest} = xmerl_scan:string(Body, []), - {Status, lists:foldl(fun error_code_from_xml/2, - undefined, - ParsedData#xmlElement.content)}. - -error_code_from_xml(#xmlText{}, Acc) -> - Acc; -error_code_from_xml(Element, Acc) -> - case Element#xmlElement.name of - 'Code' -> - [Content | _] = Element#xmlElement.content, - case is_record(Content, xmlText) of - true -> - Content#xmlText.value; - false -> - Acc - end; - _ -> - Acc - end. - -parse_user_info(Output) -> - [Boundary | Tokens] = string:tokens(Output, "\r\n"), - parse_user_info(Tokens, Boundary, []). - -parse_user_info([_LastToken], _, Users) -> - ordsets:from_list(Users); -parse_user_info(["Content-Type: application/xml", RawXml | RestTokens], - Boundary, Users) -> - UpdUsers = parse_user_records(RawXml, ?XML) ++ Users, - parse_user_info(RestTokens, Boundary, UpdUsers); -parse_user_info(["Content-Type: application/json", RawJson | RestTokens], - Boundary, Users) -> - UpdUsers = parse_user_records(RawJson, ?JSON) ++ Users, - parse_user_info(RestTokens, Boundary, UpdUsers); -parse_user_info([_ | RestTokens], Boundary, Users) -> - parse_user_info(RestTokens, Boundary, Users). diff --git a/src/base64url.erl b/src/base64url.erl deleted file mode 100644 index 86f4277ca..000000000 --- a/src/base64url.erl +++ /dev/null @@ -1,71 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc base64url is a wrapper around the base64 module to produce -%% base64-compatible encodings that are URL safe. -%% The / character in normal base64 encoding is replaced with -%% the _ character, and + is replaced with -. -%% This replacement scheme is named "base64url" by -%% http://en.wikipedia.org/wiki/Base64 - --module(base64url). - --export([decode/1, - decode_to_string/1, - encode/1, - encode_to_string/1, - mime_decode/1, - mime_decode_to_string/1]). - -decode(Base64url) -> - base64:decode(urldecode(Base64url)). - -decode_to_string(Base64url) -> - base64:decode_to_string(urldecode(Base64url)). - -mime_decode(Base64url) -> - base64:mime_decode(urldecode(Base64url)). - -mime_decode_to_string(Base64url) -> - base64:mime_decode_to_string(urldecode(Base64url)). - -encode(Data) -> - urlencode(base64:encode(Data)). - -encode_to_string(Data) -> - urlencode(base64:encode_to_string(Data)). - -urlencode(Base64) when is_list(Base64) -> - [urlencode_digit(D) || D <- Base64]; -urlencode(Base64) when is_binary(Base64) -> - << << (urlencode_digit(D)) >> || <> <= Base64 >>. - -urldecode(Base64url) when is_list(Base64url) -> - [urldecode_digit(D) || D <- Base64url ]; -urldecode(Base64url) when is_binary(Base64url) -> - << << (urldecode_digit(D)) >> || <> <= Base64url >>. - -urlencode_digit($/) -> $_; -urlencode_digit($+) -> $-; -urlencode_digit(D) -> D. - -urldecode_digit($_) -> $/; -urldecode_digit($-) -> $+; -urldecode_digit(D) -> D. diff --git a/src/riak_cs.app.src b/src/riak_cs.app.src deleted file mode 100644 index eb7b7a776..000000000 --- a/src/riak_cs.app.src +++ /dev/null @@ -1,30 +0,0 @@ -%%-*- mode: erlang -*- -{application, riak_cs, - [ - {description, "riak_cs"}, - {vsn, git}, - {modules, []}, - {registered, []}, - {applications, [ - kernel, - stdlib, - inets, - crypto, - mochiweb, - webmachine, - poolboy, - lager, - cluster_info, - exometer_core - ]}, - {mod, { riak_cs_app, []}}, - {env, [ - {put_fsm_buffer_size_max, 10485760}, - {access_archive_period, 3600}, - {access_log_flush_factor, 1}, - {access_log_flush_size, 1000000}, - {access_archiver_max_backlog, 2}, - {storage_archive_period, 86400}, - {usage_request_limit, 744} - ]} - ]}. diff --git a/src/riak_cs_acl_utils.erl b/src/riak_cs_acl_utils.erl deleted file mode 100644 index 1d16ca3c7..000000000 --- a/src/riak_cs_acl_utils.erl +++ /dev/null @@ -1,1162 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc ACL utility functions - --module(riak_cs_acl_utils). - --include("riak_cs.hrl"). --include_lib("xmerl/include/xmerl.hrl"). - --ifdef(TEST). - --compile(export_all). - --include_lib("eunit/include/eunit.hrl"). - --endif. - -%% Public API --export([acl/4, - default_acl/3, - canned_acl/3, - specific_acl_grant/3, - acl_from_xml/3, - empty_acl_xml/0, - requested_access/2, - check_grants/4, - check_grants/5, - validate_acl/2 - ]). - -%% Public API --export([ - acl_to_json_term/1 - ]). - -%% =================================================================== -%% Public API -%% =================================================================== - -%% @doc Construct an acl. The structure is the same for buckets -%% and objects. --spec acl(string(), string(), string(), [acl_grant()]) -> #acl_v2{}. -acl(DisplayName, CanonicalId, KeyId, Grants) -> - OwnerData = {DisplayName, CanonicalId, KeyId}, - ?ACL{owner=OwnerData, - grants=Grants}. - -%% @doc Construct a default acl. The structure is the same for buckets -%% and objects. --spec default_acl(string(), string(), string()) -> #acl_v2{}. -default_acl(DisplayName, CanonicalId, KeyId) -> - acl(DisplayName, - CanonicalId, - KeyId, - [{{DisplayName, CanonicalId}, ['FULL_CONTROL']}]). - -%% @doc Map a x-amz-acl header value to an -%% internal acl representation. --spec canned_acl(undefined | string(), - acl_owner(), - undefined | acl_owner()) -> #acl_v2{}. -canned_acl(undefined, {Name, CanonicalId, KeyId}, _) -> - default_acl(Name, CanonicalId, KeyId); -canned_acl(HeaderVal, Owner, BucketOwner) -> - {Name, CanonicalId, KeyId} = Owner, - acl(Name, CanonicalId, KeyId, canned_acl_grants(HeaderVal, - Owner, - BucketOwner)). - -%% @doc Turn a list of header-name, value pairs into an ACL. If the header -%% values don't parse correctly, return `{error, invalid_argument}'. If -%% the header references an email address that cannot be mapped to a -%% canonical id, return `{error, unresolved_grant_email}'. Otherwise -%% return an acl record. --spec specific_acl_grant(Owner :: acl_owner(), - [{HeaderName :: acl_perm(), - HeaderValue :: string()}], - riak_client()) -> - {ok, #acl_v2{}} | - {error, 'invalid_argument'} | - {error, 'unresolved_grant_email'}. -specific_acl_grant(Owner, Headers, RcPid) -> - %% TODO: this function is getting a bit long and confusing - Grants = [{HeaderName, parse_grant_header_value(GrantString)} || - {HeaderName, GrantString} <- Headers], - case promote_failure([Grant || {_HeaderName, Grant} <- Grants]) of - {error, invalid_argument}=E -> - E; - {ok, _GoodGrants} -> - EmailsTranslated = [{HeaderName, emails_to_ids(Grant, RcPid)} || - {HeaderName, {ok, Grant}} <- Grants], - case promote_failure([EmailOk || {_HeaderName, EmailOk} <- EmailsTranslated]) of - {error, unresolved_grant_email}=E -> - E; - {ok, _GoodEmails} -> - case valid_headers_to_grants([{HeaderName, Val} || {HeaderName, {ok, Val}} <- EmailsTranslated], - RcPid) of - {ok, AclGrants} -> - {DisplayName, CanonicalId, KeyId} = Owner, - {ok, acl(DisplayName, CanonicalId, KeyId, AclGrants)}; - {error, invalid_argument}=E -> - E - end - end - end. - -%% @doc Attempt to parse a list of ACL headers into a list -%% of `acl_grant()'s. --spec valid_headers_to_grants(list(), riak_client()) -> - {ok, list(acl_grant())} | {error, invalid_argument}. -valid_headers_to_grants(Pairs, RcPid) -> - MaybeGrants = [header_to_acl_grants(HeaderName, Grants, RcPid) || - {HeaderName, Grants} <- Pairs], - case promote_failure(MaybeGrants) of - {ok, Grants} -> - {ok, lists:foldl(fun add_grant/2, [], lists:flatten(Grants))}; - {error, invalid_argument}=E -> - E - end. - -%% @doc Attempt to turn a `acl_perm()' and list of grants -%% into a list of `acl_grant()'s. At this point, email -%% addresses have already been resolved, and headers parsed. --spec header_to_acl_grants(acl_perm(), list(), riak_client()) -> - {ok, list(acl_grant())} | {error, invalid_argument}. -header_to_acl_grants(HeaderName, Grants, RcPid) -> - MaybeGrantList = lists:map(fun (Identifier) -> - header_to_grant(HeaderName, Identifier, RcPid) end, Grants), - case promote_failure(MaybeGrantList) of - {ok, GrantList} -> - {ok, lists:foldl(fun add_grant/2, [], GrantList)}; - {error, invalid_argument}=E -> - E - end. - -%% Attempt to turn an `acl_perm()' and `grant_user_identifier()' -%% into an `acl_grant()'. If the `grant_user_identifier()' uses an -%% id, and the name can't be found, returns `{error, invalid_argument}'. --spec header_to_grant(acl_perm(), {grant_user_identifier(), string()}, riak_client()) -> - {ok, acl_grant()} | {error, invalid_argument}. -header_to_grant(Permission, {id, ID}, RcPid) -> - case name_for_canonical(ID, RcPid) of - {ok, DisplayName} -> - {ok, {{DisplayName, ID}, [Permission]}}; - {error, invalid_argument}=E -> - E - end; -header_to_grant(Permission, {uri, URI}, _RcPid) -> - case URI of - ?ALL_USERS_GROUP -> - {ok, {'AllUsers', [Permission]}}; - ?AUTH_USERS_GROUP -> - {ok, {'AuthUsers', [Permission]}} - end. - -%% @doc Attempt to parse a header into -%% a list of grant identifiers and strings. --type grant_user_identifier() :: 'emailAddress' | 'id' | 'uri'. --spec parse_grant_header_value(string()) -> - {ok, [{grant_user_identifier(), string()}]} | - {error, invalid_argument} | - {error, unresolved_grant_email}. -parse_grant_header_value(HeaderValue) -> - Mappings = split_header_values_and_strip(HeaderValue), - promote_failure(lists:map(fun parse_mapping/1, Mappings)). - -%% @doc split a string like: -%% `"emailAddress=\"xyz@amazon.com\", emailAddress=\"abc@amazon.com\""' -%% into: -%% `["emailAddress=\"xyz@amazon.com\"", -%% "emailAddress=\"abc@amazon.com\""]' --spec split_header_values_and_strip(string()) -> [string()]. -split_header_values_and_strip(Value) -> - [string:strip(V) || V <- string:tokens(Value, ",")]. - -%% @doc Attempt to parse a single grant, like: -%% `"emailAddress=\"name@example.com\""' -%% If the value can't be parsed, return -%% `{error, invalid_argument}'. --spec parse_mapping(string()) -> - {ok, {grant_user_identifier(), Value :: string()}} | - {error, invalid_argument}. -parse_mapping("emailAddress=" ++ QuotedEmail) -> - wrap('emailAddress', remove_quotes(QuotedEmail)); -parse_mapping("id=" ++ QuotedID) -> - wrap('id', remove_quotes(QuotedID)); -parse_mapping("uri=" ++ QuotedURI) -> - case remove_quotes(QuotedURI) of - {ok, NoQuote}=OK -> - case valid_uri(NoQuote) of - true -> - wrap('uri', OK); - false -> - {error, 'invalid_argument'} - end; - {error, invalid_argument}=E -> - E - end; -parse_mapping(_Else) -> - {error, invalid_argument}. - -%% @doc Return true if `URI' is a valid group grant URI. --spec valid_uri(string()) -> boolean(). -valid_uri(URI) -> - %% log delivery is not yet a supported option - lists:member(URI, [?ALL_USERS_GROUP, ?AUTH_USERS_GROUP]). - -%% @doc Combine the first and second argument, if the second -%% is wrapped in `ok'. Otherwise return the second argugment. --spec wrap(atom(), {'ok', term()} | {'error', atom()}) -> - {'error', atom()} | {ok, {atom(), term()}}. -wrap(_Atom, {error, invalid_argument}=E) -> - E; -wrap(Atom, {ok, Value}) -> - {ok, {Atom, Value}}. - -%% If `String' is enclosed in quotation marks, remove them. Otherwise -%% return an error. --spec remove_quotes(string()) -> {error, invalid_argument} | {ok, string()}. -remove_quotes(String) -> - case starts_and_ends_with_quotes(String) of - false -> - {error, invalid_argument}; - true -> - {ok, string:sub_string(String, 2, length(String) - 1)} - end. - -%% @doc Return true if `String' is enclosed in quotation -%% marks. The enclosed string must also be non-empty. --spec starts_and_ends_with_quotes(string()) -> boolean(). -starts_and_ends_with_quotes(String) -> - length(String) > 2 andalso - hd(String) =:= 34 andalso - lists:last(String) =:= 34. - -%% @doc Attempt to turn a list of grants that use email addresses -%% into a list of grants that only use canonical ids. Returns an error -%% if any of the emails cannot be turned into canonical ids. --spec emails_to_ids(list(), riak_client()) -> {ok, list()} | {error, unresolved_grant_email}. -emails_to_ids(Grants, RcPid) -> - {EmailGrants, RestGrants} = lists:partition(fun email_grant/1, Grants), - Ids = [canonical_for_email(EmailAddress, RcPid) || - {emailAddress, EmailAddress} <- EmailGrants], - case promote_failure(Ids) of - {error, unresolved_grant_email}=E -> - E; - {ok, AllIds} -> - {ok, RestGrants ++ [{id, ID} || ID <- AllIds]} - end. - --spec email_grant({atom(), term}) -> boolean(). -email_grant({Atom, _Val}) -> - Atom =:= 'emailAddress'. - -%% @doc Turn a list of ok-values or errors into either -%% an ok of list, or an error. Returns the latter is any -%% of the values in the input list are an error. --spec promote_failure(list({ok, A} | {error, term()})) -> - {ok, list(A)} | {'error', term()}. -promote_failure(List) -> - %% this will reverse the list, but we don't care - %% about order - case lists:foldl(fun fail_either/2, {ok, []}, List) of - {{error, _Reason}=E, _Acc} -> - E; - {ok, _Acc}=Ok -> - Ok - end. - -%% @doc Return an error if either argument is an error. Otherwise, -%% cons the value from the first argument onto the accumulator -%% in the second. --spec fail_either({ok, term()} | {error, term()}, - {{error, term()} | 'ok', list()}) -> - {ok | {error, term()}, list()}. -fail_either(_Elem, {{error, _Reason}=E, Acc}) -> - {E, Acc}; -fail_either(E={error, _Reason}, {_OkOrError, Acc}) -> - %% don't cons the error onto the acc - {E, Acc}; -fail_either({ok, Val}, {_OkOrError, Acc}) -> - {ok, [Val | Acc]}. - -%% @doc Convert an XML document representing an ACL into -%% an internal representation. --spec acl_from_xml(string(), string(), riak_client()) -> {ok, #acl_v2{}} | - {error, 'invalid_argument'} | - {error, 'unresolved_grant_email'} | - {error, 'malformed_acl_error'}. -acl_from_xml(Xml, KeyId, RcPid) -> - case riak_cs_xml:scan(Xml) of - {error, malformed_xml} -> {error, malformed_acl_error}; - {ok, ParsedData} -> - BareAcl = ?ACL{owner={[], [], KeyId}}, - process_acl_contents(ParsedData#xmlElement.content, BareAcl, RcPid) - end. - -%% @doc Convert an internal representation of an ACL -%% into XML. --spec empty_acl_xml() -> binary(). -empty_acl_xml() -> - XmlDoc = [{'AccessControlPolicy',[]}], - unicode:characters_to_binary( - xmerl:export_simple(XmlDoc, xmerl_xml, [{prolog, ?XML_PROLOG}])). - -%% @doc Map a request type to the type of ACL permissions needed -%% to complete the request. --type request_method() :: 'GET' | 'HEAD' | 'PUT' | 'POST' | - 'DELETE' | 'Dialyzer happiness'. --spec requested_access(request_method(), boolean()) -> acl_perm(). -requested_access(Method, AclRequest) -> - if - Method == 'GET' - andalso - AclRequest == true-> - 'READ_ACP'; - (Method == 'GET' - orelse - Method == 'HEAD') -> - 'READ'; - Method == 'PUT' - andalso - AclRequest == true-> - 'WRITE_ACP'; - (Method == 'POST' - orelse - Method == 'DELETE') - andalso - AclRequest == true-> - undefined; - Method == 'PUT' - orelse - Method == 'POST' - orelse - Method == 'DELETE' -> - 'WRITE'; - Method == 'Dialyzer happiness' -> - 'FULL_CONTROL'; - true -> - undefined - end. - --spec check_grants(undefined | rcs_user(), binary(), atom(), riak_client()) -> - boolean() | {true, string()}. -check_grants(User, Bucket, RequestedAccess, RcPid) -> - check_grants(User, Bucket, RequestedAccess, RcPid, undefined). - --spec check_grants(undefined | rcs_user(), binary(), atom(), riak_client(), acl()|undefined) -> - boolean() | {true, string()}. -check_grants(undefined, Bucket, RequestedAccess, RcPid, BucketAcl) -> - riak_cs_acl:anonymous_bucket_access(Bucket, RequestedAccess, RcPid, BucketAcl); -check_grants(User, Bucket, RequestedAccess, RcPid, BucketAcl) -> - riak_cs_acl:bucket_access(Bucket, - RequestedAccess, - User?RCS_USER.canonical_id, - RcPid, - BucketAcl). - --spec validate_acl({ok, acl()} | {error, term()}, string()) -> - {ok, acl()} | {error, access_denied}. -validate_acl({ok, Acl=?ACL{owner={_, Id, _}}}, Id) -> - {ok, Acl}; -validate_acl({ok, _}, _) -> - {error, access_denied}; -validate_acl({error, _}=Error, _) -> - Error. - -%% @doc Convert an internal representation of an ACL into -%% erlang terms that can be encoded using `mochijson2:encode'. --spec acl_to_json_term(acl()) -> term(). -acl_to_json_term(?ACL{owner={DisplayName, CanonicalId, KeyId}, - grants=Grants, - creation_time=CreationTime}) -> - {<<"acl">>, - {struct, [{<<"version">>, 1}, - owner_to_json_term(DisplayName, CanonicalId, KeyId), - grants_to_json_term(Grants, []), - erlang_time_to_json_term(CreationTime)]} - }. - - -%% =================================================================== -%% Internal functions -%% =================================================================== - -%% @doc Update the permissions for a grant in the provided -%% list of grants if an entry exists with matching grantee -%% data or add a grant to a list of grants. --spec add_grant(acl_grant(), [acl_grant()]) -> [acl_grant()]. -add_grant(NewGrant, Grants) -> - {NewGrantee, NewPerms} = NewGrant, - SplitFun = fun(G) -> - {Grantee, _} = G, - Grantee =:= NewGrantee - end, - {GranteeGrants, OtherGrants} = lists:partition(SplitFun, Grants), - case GranteeGrants of - [] -> - [NewGrant | Grants]; - _ -> - %% `GranteeGrants' will nearly always be a single - %% item list, but use a fold just in case. - %% The combined list of perms should be small so - %% using usort should not be too expensive. - FoldFun = fun({_, Perms}, Acc) -> - lists:usort(Perms ++ Acc) - end, - UpdPerms = lists:foldl(FoldFun, NewPerms, GranteeGrants), - [{NewGrantee, UpdPerms} | OtherGrants] - end. - -%% @doc Get the list of grants for a canned ACL --spec canned_acl_grants(string(), - acl_owner(), - undefined | acl_owner()) -> [acl_grant()]. -canned_acl_grants("public-read", Owner, _) -> - [{owner_grant(Owner), ['FULL_CONTROL']}, - {'AllUsers', ['READ']}]; -canned_acl_grants("public-read-write", Owner, _) -> - [{owner_grant(Owner), ['FULL_CONTROL']}, - {'AllUsers', ['READ', 'WRITE']}]; -canned_acl_grants("authenticated-read", Owner, _) -> - [{owner_grant(Owner), ['FULL_CONTROL']}, - {'AuthUsers', ['READ']}]; -canned_acl_grants("bucket-owner-read", Owner, undefined) -> - canned_acl_grants("private", Owner, undefined); -canned_acl_grants("bucket-owner-read", Owner, Owner) -> - [{owner_grant(Owner), ['FULL_CONTROL']}]; -canned_acl_grants("bucket-owner-read", Owner, BucketOwner) -> - [{owner_grant(Owner), ['FULL_CONTROL']}, - {owner_grant(BucketOwner), ['READ']}]; -canned_acl_grants("bucket-owner-full-control", Owner, undefined) -> - canned_acl_grants("private", Owner, undefined); -canned_acl_grants("bucket-owner-full-control", Owner, Owner) -> - [{owner_grant(Owner), ['FULL_CONTROL']}]; -canned_acl_grants("bucket-owner-full-control", Owner, BucketOwner) -> - [{owner_grant(Owner), ['FULL_CONTROL']}, - {owner_grant(BucketOwner), ['FULL_CONTROL']}]; -canned_acl_grants(_, Owner, _) -> - [{owner_grant(Owner), ['FULL_CONTROL']}]. - --spec owner_grant({string(), string(), string()}) -> {string(), string()}. -owner_grant({Name, CanonicalId, _}) -> - {Name, CanonicalId}. - -%% @doc Get the canonical id of the user associated with -%% a given email address. --spec canonical_for_email(string(), riak_client()) -> {ok, string()} | - {error, unresolved_grant_email} . -canonical_for_email(Email, RcPid) -> - case riak_cs_user:get_user_by_index(?EMAIL_INDEX, - list_to_binary(Email), - RcPid) of - {ok, {User, _}} -> - {ok, User?RCS_USER.canonical_id}; - {error, Reason} -> - _ = lager:debug("Failed to retrieve canonical id for ~p. Reason: ~p", [Email, Reason]), - {error, unresolved_grant_email} - end. - -%% @doc Get the display name of the user associated with -%% a given canonical id. --spec name_for_canonical(string(), riak_client()) -> {ok, string()} | - {error, 'invalid_argument'}. -name_for_canonical(CanonicalId, RcPid) -> - case riak_cs_user:get_user_by_index(?ID_INDEX, - list_to_binary(CanonicalId), - RcPid) of - {ok, {User, _}} -> - {ok, User?RCS_USER.display_name}; - {error, _} -> - {error, invalid_argument} - end. - -%% @doc Process the top-level elements of the --spec process_acl_contents([riak_cs_xml:xmlElement()], acl(), riak_client()) -> - {ok, #acl_v2{}} | - {error, invalid_argument} | - {error, unresolved_grant_email}. -process_acl_contents([], Acl, _) -> - {ok, Acl}; -process_acl_contents([#xmlElement{content=Content, - name=ElementName} - | RestElements], Acl, RcPid) -> - _ = lager:debug("Element name: ~p", [ElementName]), - UpdAclRes = - case ElementName of - 'Owner' -> - process_owner(Content, Acl, RcPid); - 'AccessControlList' -> - process_grants(Content, Acl, RcPid); - _ -> - _ = lager:debug("Encountered unexpected element: ~p", [ElementName]), - Acl - end, - case UpdAclRes of - {ok, UpdAcl} -> - process_acl_contents(RestElements, UpdAcl, RcPid); - {error, _}=Error -> - Error - end; -process_acl_contents([#xmlComment{} | RestElements], Acl, RcPid) -> - process_acl_contents(RestElements, Acl, RcPid); -process_acl_contents([#xmlText{} | RestElements], Acl, RcPid) -> - %% skip normalized space - process_acl_contents(RestElements, Acl, RcPid). - -%% @doc Process an XML element containing acl owner information. --spec process_owner([riak_cs_xml:xmlNode()], acl(), riak_client()) -> {ok, #acl_v2{}}. -process_owner([], Acl=?ACL{owner={[], CanonicalId, KeyId}}, RcPid) -> - case name_for_canonical(CanonicalId, RcPid) of - {ok, DisplayName} -> - {ok, Acl?ACL{owner={DisplayName, CanonicalId, KeyId}}}; - {error, _}=Error -> - Error - end; -process_owner([], Acl, _) -> - {ok, Acl}; -process_owner([#xmlElement{content=[Content], - name=ElementName} | - RestElements], Acl, RcPid) -> - Owner = Acl?ACL.owner, - case Content of - #xmlText{value=Value} -> - UpdOwner = - case ElementName of - 'ID' -> - _ = lager:debug("Owner ID value: ~p", [Value]), - {OwnerName, _, OwnerKeyId} = Owner, - {OwnerName, Value, OwnerKeyId}; - 'DisplayName' -> - _ = lager:debug("Owner Name content: ~p", [Value]), - {_, OwnerId, OwnerKeyId} = Owner, - {Value, OwnerId, OwnerKeyId}; - _ -> - _ = lager:debug("Encountered unexpected element: ~p", [ElementName]), - Owner - end, - process_owner(RestElements, Acl?ACL{owner=UpdOwner}, RcPid); - _ -> - process_owner(RestElements, Acl, RcPid) - end; -process_owner([_ | RestElements], Acl, RcPid) -> - %% this pattern matches with text, comment, etc.. - process_owner(RestElements, Acl, RcPid). - -%% @doc Process an XML element containing the grants for the acl. --spec process_grants([riak_cs_xml:xmlNode()], acl(), riak_client()) -> - {ok, #acl_v2{}} | - {error, invalid_argument} | - {error, unresolved_grant_email}. -process_grants([], Acl, _) -> - {ok, Acl}; -process_grants([#xmlElement{content=Content, - name=ElementName} | - RestElements], Acl, RcPid) -> - UpdAcl = - case ElementName of - 'Grant' -> - Grant = process_grant(Content, {{"", ""}, []}, Acl?ACL.owner, RcPid), - case Grant of - {error, _} -> - Grant; - _ -> - Acl?ACL{grants=add_grant(Grant, Acl?ACL.grants)} - end; - _ -> - _ = lager:debug("Encountered unexpected grants element: ~p", [ElementName]), - Acl - end, - case UpdAcl of - {error, _} -> UpdAcl; - _ -> process_grants(RestElements, UpdAcl, RcPid) - end; -process_grants([ #xmlComment{} | RestElements], Acl, RcPid) -> - process_grants(RestElements, Acl, RcPid); -process_grants([ #xmlText{} | RestElements], Acl, RcPid) -> - process_grants(RestElements, Acl, RcPid). - -%% @doc Process an XML element containing the grants for the acl. --spec process_grant([riak_cs_xml:xmlElement()], acl_grant(), acl_owner(), riak_client()) -> - acl_grant() | {error, atom()}. -process_grant([], Grant, _, _) -> - Grant; -process_grant([#xmlElement{content=Content, - name=ElementName} | - RestElements], Grant, AclOwner, RcPid) -> - _ = lager:debug("ElementName: ~p", [ElementName]), - _ = lager:debug("Content: ~p", [Content]), - UpdGrant = - case ElementName of - 'Grantee' -> - process_grantee(Content, Grant, AclOwner, RcPid); - 'Permission' -> - process_permission(Content, Grant); - _ -> - _ = lager:debug("Encountered unexpected grant element: ~p", [ElementName]), - Grant - end, - case UpdGrant of - {error, _}=Error -> - Error; - _ -> - process_grant(RestElements, UpdGrant, AclOwner, RcPid) - end; -process_grant([#xmlComment{}|RestElements], Grant, Owner, RcPid) -> - process_grant(RestElements, Grant, Owner, RcPid); -process_grant([#xmlText{}|RestElements], Grant, Owner, RcPid) -> - process_grant(RestElements, Grant, Owner, RcPid). - -%% @doc Process an XML element containing information about -%% an ACL permission grantee. --spec process_grantee([riak_cs_xml:xmlElement()], acl_grant(), acl_owner(), riak_client()) -> - acl_grant() | - {error, invalid_argument} | - {error, unresolved_grant_email}. -process_grantee([], {{[], CanonicalId}, _Perms}, {DisplayName, CanonicalId, _}, _) -> - {{DisplayName, CanonicalId}, _Perms}; -process_grantee([], {{[], CanonicalId}, _Perms}, _, RcPid) -> - %% Lookup the display name for the user with the - %% canonical id of `CanonicalId'. - case name_for_canonical(CanonicalId, RcPid) of - {ok, DisplayName} -> - {{DisplayName, CanonicalId}, _Perms}; - {error, _}=Error -> - Error - end; -process_grantee([], Grant, _, _) -> - Grant; -process_grantee([#xmlElement{content=[Content], - name=ElementName} | - RestElements], Grant, AclOwner, RcPid) -> - Value = Content#xmlText.value, - case ElementName of - 'ID' -> - _ = lager:debug("ID value: ~p", [Value]), - {{Name, _}, Perms} = Grant, - UpdGrant = {{Name, Value}, Perms}; - 'EmailAddress' -> - _ = lager:debug("Email value: ~p", [Value]), - UpdGrant = - case canonical_for_email(Value, RcPid) of - {ok, Id} -> - %% Get the canonical id for a given email address - _ = lager:debug("ID value: ~p", [Id]), - {{Name, _}, Perms} = Grant, - {{Name, Id}, Perms}; - {error, _}=Error -> - Error - end; - 'URI' -> - {_, Perms} = Grant, - case Value of - ?AUTH_USERS_GROUP -> - UpdGrant = {'AuthUsers', Perms}; - ?ALL_USERS_GROUP -> - UpdGrant = {'AllUsers', Perms}; - _ -> - %% Not yet supporting log delivery group - UpdGrant = Grant - end; - _ -> - UpdGrant = Grant - end, - case UpdGrant of - {error, _} -> - UpdGrant; - _ -> - process_grantee(RestElements, UpdGrant, AclOwner, RcPid) - end; -process_grantee([#xmlText{}|RestElements], Grant, Owner, RcPid) -> - process_grantee(RestElements, Grant, Owner, RcPid); -process_grantee([#xmlComment{}|RestElements], Grant, Owner, RcPid) -> - process_grantee(RestElements, Grant, Owner, RcPid). - -%% @doc Process an XML element containing information about -%% an ACL permission. --spec process_permission([riak_cs_xml:xmlText()], acl_grant()) -> acl_grant(). -process_permission([Content], Grant) -> - Value = list_to_existing_atom(Content#xmlText.value), - {Grantee, Perms} = Grant, - UpdPerms = case lists:member(Value, Perms) of - true -> Perms; - false -> [Value | Perms] - end, - {Grantee, UpdPerms}. - - -%% @doc Convert an information from an ACL into erlang -%% terms that can be encoded using `mochijson2:encode'. --spec erlang_time_to_json_term(erlang:timestamp()) -> term(). -erlang_time_to_json_term({MegaSecs, Secs, MicroSecs}) -> - {<<"creation_time">>, - {struct, [{<<"mega_seconds">>, MegaSecs}, - {<<"seconds">>, Secs}, - {<<"micro_seconds">>, MicroSecs}]} - }. - -%% @doc Convert grantee information from an ACL into erlang -%% terms that can be encoded using `mochijson2:encode'. --spec grantee_to_json_term(acl_grant()) -> term(). -grantee_to_json_term({Group, Perms}) when is_atom(Group) -> - {struct, [{<<"group">>, list_to_binary( - atom_to_list(Group))}, - {<<"permissions">>, permissions_to_json_term(Perms)}]}; -grantee_to_json_term({{DisplayName, CanonicalId}, Perms}) -> - {struct, [{<<"display_name">>, list_to_binary(DisplayName)}, - {<<"canonical_id">>, list_to_binary(CanonicalId)}, - {<<"permissions">>, permissions_to_json_term(Perms)}]}. - -%% @doc Convert owner information from an ACL into erlang -%% terms that can be encoded using `mochijson2:encode'. --spec grants_to_json_term([acl_grant()], [term()]) -> term(). -grants_to_json_term([], GrantTerms) -> - {<<"grants">>, GrantTerms}; -grants_to_json_term([HeadGrant | RestGrants], GrantTerms) -> - grants_to_json_term(RestGrants, - [grantee_to_json_term(HeadGrant) | GrantTerms]). - -%% @doc Convert owner information from an ACL into erlang -%% terms that can be encoded using `mochijson2:encode'. --spec owner_to_json_term(string(), string(), string()) -> term(). -owner_to_json_term(DisplayName, CanonicalId, KeyId) -> - {<<"owner">>, - {struct, [{<<"display_name">>, list_to_binary(DisplayName)}, - {<<"canonical_id">>, list_to_binary(CanonicalId)}, - {<<"key_id">>, list_to_binary(KeyId)}]} - }. - -%% @doc Convert a list of permissions into binaries -%% that can be encoded using `mochijson2:encode'. --spec permissions_to_json_term(acl_perms()) -> term(). -permissions_to_json_term(Perms) -> - [list_to_binary(atom_to_list(Perm)) || Perm <- Perms]. - - -%% =================================================================== -%% Eunit tests -%% =================================================================== - --ifdef(TEST). - - -%% @doc Construct an acl. The structure is the same for buckets -%% and objects. --spec acl(string(), string(), string(), [acl_grant()], erlang:timestamp()) -> acl(). -acl(DisplayName, CanonicalId, KeyId, Grants, CreationTime) -> - OwnerData = {DisplayName, CanonicalId, KeyId}, - ?ACL{owner=OwnerData, - grants=Grants, - creation_time=CreationTime}. - - -%% @doc Convert a set of JSON terms representing an ACL into -%% an internal representation. --spec acl_from_json(term()) -> acl(). -acl_from_json({struct, Json}) -> - process_acl_contents(Json, ?ACL{}); -acl_from_json(Json) -> - process_acl_contents(Json, ?ACL{}). - -%% @doc Process the top-level elements of the --spec process_acl_contents([term()], acl()) -> acl(). -process_acl_contents([], Acl) -> - Acl; -process_acl_contents([{Name, Value} | RestObjects], Acl) -> - _ = lager:debug("Object name: ~p", [Name]), - case Name of - <<"owner">> -> - {struct, OwnerData} = Value, - UpdAcl = process_owner(OwnerData, Acl); - <<"grants">> -> - UpdAcl = process_grants(Value, Acl); - <<"creation_time">> -> - {struct, TimeData} = Value, - CreationTime = process_creation_time(TimeData, {1,1,1}), - UpdAcl = Acl?ACL{creation_time=CreationTime}; - _ -> - UpdAcl = Acl - end, - process_acl_contents(RestObjects, UpdAcl). - -%% @doc Process an JSON element containing acl owner information. --spec process_owner([term()], acl()) -> acl(). -process_owner([], Acl) -> - Acl; -process_owner([{Name, Value} | RestObjects], Acl) -> - Owner = Acl?ACL.owner, - case Name of - <<"key_id">> -> - _ = lager:debug("Owner Key ID value: ~p", [Value]), - {OwnerName, OwnerCID, _} = Owner, - UpdOwner = {OwnerName, OwnerCID, binary_to_list(Value)}; - <<"canonical_id">> -> - _ = lager:debug("Owner ID value: ~p", [Value]), - {OwnerName, _, OwnerId} = Owner, - UpdOwner = {OwnerName, binary_to_list(Value), OwnerId}; - <<"display_name">> -> - _ = lager:debug("Owner Name content: ~p", [Value]), - {_, OwnerCID, OwnerId} = Owner, - UpdOwner = {binary_to_list(Value), OwnerCID, OwnerId}; - _ -> - _ = lager:debug("Encountered unexpected element: ~p", [Name]), - UpdOwner = Owner - end, - process_owner(RestObjects, Acl?ACL{owner=UpdOwner}). - -%% @doc Process an JSON element containing the grants for the acl. --spec process_grants([term()], acl()) -> acl(). -process_grants([], Acl) -> - Acl; -process_grants([{_, Value} | RestObjects], Acl) -> - Grant = process_grant(Value, {{"", ""}, []}), - UpdAcl = Acl?ACL{grants=[Grant | Acl?ACL.grants]}, - process_grants(RestObjects, UpdAcl). - -%% @doc Process an JSON element containing information about -%% an ACL permission grants. --spec process_grant([term()], acl_grant()) -> acl_grant(). -process_grant([], Grant) -> - Grant; -process_grant([{Name, Value} | RestObjects], Grant) -> - case Name of - <<"canonical_id">> -> - _ = lager:debug("ID value: ~p", [Value]), - {{DispName, _}, Perms} = Grant, - UpdGrant = {{DispName, binary_to_list(Value)}, Perms}; - <<"display_name">> -> - _ = lager:debug("Name value: ~p", [Value]), - {{_, Id}, Perms} = Grant, - UpdGrant = {{binary_to_list(Value), Id}, Perms}; - <<"group">> -> - _ = lager:debug("Group value: ~p", [Value]), - {_, Perms} = Grant, - UpdGrant = {list_to_atom( - binary_to_list(Value)), Perms}; - <<"permissions">> -> - {Grantee, _} = Grant, - Perms = process_permissions(Value), - _ = lager:debug("Perms value: ~p", [Value]), - UpdGrant = {Grantee, Perms}; - _ -> - UpdGrant = Grant - end, - process_grant(RestObjects, UpdGrant). - -%% @doc Process a list of JSON elements containing -%% ACL permissions. --spec process_permissions([binary()]) -> acl_perms(). -process_permissions(Perms) -> - lists:usort( - lists:filter(fun(X) -> X /= undefined end, - [binary_perm_to_atom(Perm) || Perm <- Perms])). - -%% @doc Convert a binary permission type to a -%% corresponding atom or return `undefined' if -%% the permission is invalid. --spec binary_perm_to_atom(binary()) -> atom(). -binary_perm_to_atom(Perm) -> - case Perm of - <<"FULL_CONTROL">> -> - 'FULL_CONTROL'; - <<"READ">> -> - 'READ'; - <<"READ_ACP">> -> - 'READ_ACP'; - <<"WRITE">> -> - 'WRITE'; - <<"WRITE_ACP">> -> - 'WRITE_ACP'; - _ -> - undefined - end. - -%% @doc Process the JSON element containing creation time -%% data for an ACL. --spec process_creation_time([term()], erlang:timestamp()) -> erlang:timestamp(). -process_creation_time([], CreationTime) -> - CreationTime; -process_creation_time([{Name, Value} | RestObjects], CreationTime) -> - case Name of - <<"mega_seconds">> -> - {_, Secs, MicroSecs} = CreationTime, - UpdCreationTime = {Value, Secs, MicroSecs}; - <<"seconds">> -> - {MegaSecs, _, MicroSecs} = CreationTime, - UpdCreationTime = {MegaSecs, Value, MicroSecs}; - <<"micro_seconds">> -> - {MegaSecs, Secs, _} = CreationTime, - UpdCreationTime = {MegaSecs, Secs, Value} - end, - process_creation_time(RestObjects, UpdCreationTime). - -%% @TODO Use eqc to do some more interesting case explorations. - -default_acl_test() -> - ExpectedXml = <<"TESTID1tester1TESTID1tester1FULL_CONTROL">>, - DefaultAcl = default_acl("tester1", "TESTID1", "TESTKEYID1"), - ?assertMatch({acl_v2,{"tester1","TESTID1", "TESTKEYID1"}, - [{{"tester1","TESTID1"},['FULL_CONTROL']}], _}, DefaultAcl), - ?assertEqual(ExpectedXml, riak_cs_xml:to_xml(DefaultAcl)). - -acl_from_xml_test() -> - Xml = "TESTID1tester1TESTID1tester1FULL_CONTROL", - DefaultAcl = default_acl("tester1", "TESTID1", "TESTKEYID1"), - {ok, Acl} = acl_from_xml(Xml, "TESTKEYID1", undefined), - {ExpectedOwnerName, ExpectedOwnerId, _} = DefaultAcl?ACL.owner, - {ActualOwnerName, ActualOwnerId, _} = Acl?ACL.owner, - ?assertEqual(DefaultAcl?ACL.grants, Acl?ACL.grants), - ?assertEqual(ExpectedOwnerName, ActualOwnerName), - ?assertEqual(ExpectedOwnerId, ActualOwnerId). - -roundtrip_test() -> - Xml1 = "TESTID1tester1TESTID1tester1FULL_CONTROL", - Xml2 = "TESTID1tester1TESTID1tester1FULL_CONTROLhttp://acs.amazonaws.com/groups/global/AuthenticatedUsersREAD", - {ok, AclFromXml1} = acl_from_xml(Xml1, "TESTKEYID1", undefined), - {ok, AclFromXml2} = acl_from_xml(Xml2, "TESTKEYID2", undefined), - ?assertEqual(Xml1, binary_to_list(riak_cs_xml:to_xml(AclFromXml1))), - ?assertEqual(Xml2, binary_to_list(riak_cs_xml:to_xml(AclFromXml2))). - -requested_access_test() -> - ?assertEqual('READ', requested_access('GET', false)), - ?assertEqual('READ_ACP', requested_access('GET', true)), - ?assertEqual('WRITE', requested_access('PUT', false)), - ?assertEqual('WRITE_ACP', requested_access('PUT', true)), - ?assertEqual('WRITE', requested_access('POST', false)), - ?assertEqual('WRITE', requested_access('DELETE', false)), - ?assertEqual(undefined, requested_access('POST', true)), - ?assertEqual(undefined, requested_access('DELETE', true)), - ?assertEqual(undefined, requested_access('GARBAGE', false)), - ?assertEqual(undefined, requested_access('GARBAGE', true)). - -canned_acl_test() -> - Owner = {"tester1", "TESTID1", "TESTKEYID1"}, - BucketOwner = {"owner1", "OWNERID1", "OWNERKEYID1"}, - DefaultAcl = canned_acl(undefined, Owner, undefined), - PrivateAcl = canned_acl("private", Owner, undefined), - PublicReadAcl = canned_acl("public-read", Owner, undefined), - PublicRWAcl = canned_acl("public-read-write", Owner, undefined), - AuthReadAcl = canned_acl("authenticated-read", Owner, undefined), - BucketOwnerReadAcl1 = canned_acl("bucket-owner-read", Owner, undefined), - BucketOwnerReadAcl2 = canned_acl("bucket-owner-read", Owner, BucketOwner), - BucketOwnerReadAcl3 = canned_acl("bucket-owner-read", Owner, Owner), - BucketOwnerFCAcl1 = canned_acl("bucket-owner-full-control", Owner, undefined), - BucketOwnerFCAcl2 = canned_acl("bucket-owner-full-control", Owner, BucketOwner), - BucketOwnerFCAcl3 = canned_acl("bucket-owner-full-control", Owner, Owner), - - ?assertMatch({acl_v2,{"tester1","TESTID1","TESTKEYID1"}, - [{{"tester1","TESTID1"},['FULL_CONTROL']}], _}, DefaultAcl), - ?assertMatch({acl_v2,{"tester1","TESTID1","TESTKEYID1"}, - [{{"tester1","TESTID1"},['FULL_CONTROL']}], _}, PrivateAcl), - ?assertMatch({acl_v2,{"tester1","TESTID1","TESTKEYID1"}, - [{{"tester1","TESTID1"},['FULL_CONTROL']}, - {'AllUsers', ['READ']}], _}, PublicReadAcl), - ?assertMatch({acl_v2,{"tester1","TESTID1","TESTKEYID1"}, - [{{"tester1","TESTID1"},['FULL_CONTROL']}, - {'AllUsers', ['READ', 'WRITE']}], _}, PublicRWAcl), - ?assertMatch({acl_v2,{"tester1","TESTID1","TESTKEYID1"}, - [{{"tester1","TESTID1"},['FULL_CONTROL']}, - {'AuthUsers', ['READ']}], _}, AuthReadAcl), - ?assertMatch({acl_v2,{"tester1","TESTID1","TESTKEYID1"}, - [{{"tester1","TESTID1"},['FULL_CONTROL']}], _}, BucketOwnerReadAcl1), - ?assertMatch({acl_v2,{"tester1","TESTID1","TESTKEYID1"}, - [{{"tester1","TESTID1"},['FULL_CONTROL']}, - {{"owner1", "OWNERID1"}, ['READ']}], _}, BucketOwnerReadAcl2), - ?assertMatch({acl_v2,{"tester1","TESTID1","TESTKEYID1"}, - [{{"tester1","TESTID1"},['FULL_CONTROL']}], _}, BucketOwnerReadAcl3), - ?assertMatch({acl_v2,{"tester1","TESTID1","TESTKEYID1"}, - [{{"tester1","TESTID1"},['FULL_CONTROL']}], _}, BucketOwnerFCAcl1), - ?assertMatch({acl_v2,{"tester1","TESTID1","TESTKEYID1"}, - [{{"tester1","TESTID1"},['FULL_CONTROL']}, - {{"owner1", "OWNERID1"}, ['FULL_CONTROL']}], _}, BucketOwnerFCAcl2), - ?assertMatch({acl_v2,{"tester1","TESTID1","TESTKEYID1"}, - [{{"tester1","TESTID1"},['FULL_CONTROL']}], _}, BucketOwnerFCAcl3). - - -indented_xml_with_comments() -> - Xml=" " - " " - " eb874c6afce06925157eda682f1b3c6eb0f3b983bbee3673ae62f41cce21f6b1" - " admin " - " " - " " - " " - " eb874c6afce06925157eda682f1b3c6eb0f3b983bbee3673ae62f41cce21f6b1 " - " admin " - " " - " FULL_CONTROL " - " " - " " - " ", - Xml. - -comment_space_test() -> - Xml = indented_xml_with_comments(), - %% if cs782 alive, error:{badrecord,xmlElement} thrown here. - {ok, ?ACL{} = Acl} = riak_cs_acl_utils:acl_from_xml(Xml, boom, foo), - %% Compare the result with the one from XML without comments and extra spaces - StrippedXml0 = re:replace(Xml, "", "", [global]), - StrippedXml1 = re:replace(StrippedXml0, " *<", "<", [global]), - StrippedXml = binary_to_list(iolist_to_binary(re:replace(StrippedXml1, " *$", "", [global]))), - {ok, ?ACL{} = AclFromStripped} = riak_cs_acl_utils:acl_from_xml(StrippedXml, boom, foo), - ?assertEqual(AclFromStripped?ACL{creation_time=Acl?ACL.creation_time}, - Acl), - ok. - -acl_from_json_test() -> - CreationTime = erlang:now(), - {AclMegaSecs, AclSecs, AclMicroSecs} = CreationTime, - JsonTerm = [{<<"version">>,1}, - {<<"owner">>, - {struct, - [{<<"display_name">>,<<"tester1">>}, - {<<"canonical_id">>,<<"TESTID1">>}, - {<<"key_id">>,<<"TESTKEYID1">>}]}}, - {<<"grants">>, - [{struct, - [{<<"group">>,<<"AllUsers">>}, - {<<"permissions">>,[<<"WRITE_ACP">>]}]}, - {struct, - [{<<"display_name">>,<<"tester2">>}, - {<<"canonical_id">>,<<"TESTID2">>}, - {<<"permissions">>,[<<"WRITE">>]}]}, - {struct, - [{<<"display_name">>,<<"tester1">>}, - {<<"canonical_id">>,<<"TESTID1">>}, - {<<"permissions">>,[<<"READ">>]}]}]}, - {<<"creation_time">>, - {struct, - [{<<"mega_seconds">>, AclMegaSecs}, - {<<"seconds">>, AclSecs}, - {<<"micro_seconds">>, AclMicroSecs}]}}], - Acl = acl_from_json(JsonTerm), - ExpectedAcl = acl("tester1", - "TESTID1", - "TESTKEYID1", - [{{"tester1", "TESTID1"}, ['READ']}, - {{"tester2", "TESTID2"}, ['WRITE']}, - {'AllUsers', ['WRITE_ACP']}], - CreationTime), - ?assertEqual(ExpectedAcl, Acl). - -acl_to_json_term_test() -> - CreationTime = erlang:now(), - Acl = acl("tester1", - "TESTID1", - "TESTKEYID1", - [{{"tester1", "TESTID1"}, ['READ']}, - {{"tester2", "TESTID2"}, ['WRITE']}], - CreationTime), - JsonTerm = acl_to_json_term(Acl), - {AclMegaSecs, AclSecs, AclMicroSecs} = CreationTime, - ExpectedTerm = {<<"acl">>, - {struct, - [{<<"version">>,1}, - {<<"owner">>, - {struct, - [{<<"display_name">>,<<"tester1">>}, - {<<"canonical_id">>,<<"TESTID1">>}, - {<<"key_id">>,<<"TESTKEYID1">>}]}}, - {<<"grants">>, - [{struct, - [{<<"display_name">>,<<"tester2">>}, - {<<"canonical_id">>,<<"TESTID2">>}, - {<<"permissions">>,[<<"WRITE">>]}]}, - {struct, - [{<<"display_name">>,<<"tester1">>}, - {<<"canonical_id">>,<<"TESTID1">>}, - {<<"permissions">>,[<<"READ">>]}]}]}, - {<<"creation_time">>, - {struct, - [{<<"mega_seconds">>, AclMegaSecs}, - {<<"seconds">>, AclSecs}, - {<<"micro_seconds">>, AclMicroSecs}]}}]}}, - ?assertEqual(ExpectedTerm, JsonTerm). - -owner_to_json_term_test() -> - JsonTerm = owner_to_json_term("name", "cid123", "keyid123"), - ExpectedTerm = {<<"owner">>, - {struct, [{<<"display_name">>, <<"name">>}, - {<<"canonical_id">>, <<"cid123">>}, - {<<"key_id">>, <<"keyid123">>}]} - }, - ?assertEqual(ExpectedTerm, JsonTerm). - -grants_to_json_term_test() -> - CreationTime = erlang:now(), - Acl = acl("tester1", - "TESTID1", - "TESTKEYID1", - [{{"tester1", "TESTID1"}, ['READ']}, - {{"tester2", "TESTID2"}, ['WRITE']}], - CreationTime), - JsonTerm = grants_to_json_term(Acl?ACL.grants, []), - ExpectedTerm = - {<<"grants">>, [{struct, - [{<<"display_name">>,<<"tester2">>}, - {<<"canonical_id">>,<<"TESTID2">>}, - {<<"permissions">>,[<<"WRITE">>]}]}, - {struct, - [{<<"display_name">>,<<"tester1">>}, - {<<"canonical_id">>,<<"TESTID1">>}, - {<<"permissions">>,[<<"READ">>]}]}]}, - - ?assertEqual(ExpectedTerm, JsonTerm). - -grantee_to_json_term_test() -> - JsonTerm1 = grantee_to_json_term({{"tester1", "TESTID1"}, ['READ']}), - JsonTerm2 = grantee_to_json_term({'AllUsers', ['WRITE']}), - ExpectedTerm1 = {struct, - [{<<"display_name">>,<<"tester1">>}, - {<<"canonical_id">>,<<"TESTID1">>}, - {<<"permissions">>,[<<"READ">>]}]}, - ExpectedTerm2 = {struct, - [{<<"group">>,<<"AllUsers">>}, - {<<"permissions">>,[<<"WRITE">>]}]}, - ?assertEqual(ExpectedTerm1, JsonTerm1), - ?assertEqual(ExpectedTerm2, JsonTerm2). - -permissions_to_json_term_test() -> - JsonTerm = permissions_to_json_term(['READ', - 'WRITE', - 'READ_ACP', - 'WRITE_ACP', - 'FULL_CONTROL']), - ExpectedTerm = [<<"READ">>, - <<"WRITE">>, - <<"READ_ACP">>, - <<"WRITE_ACP">>, - <<"FULL_CONTROL">>], - ?assertEqual(ExpectedTerm, JsonTerm). - -erlang_time_to_json_term_test() -> - JsonTerm = erlang_time_to_json_term({1000, 100, 10}), - ExpectedTerm = {<<"creation_time">>, - {struct, - [{<<"mega_seconds">>, 1000}, - {<<"seconds">>, 100}, - {<<"micro_seconds">>, 10}]}}, - ?assertEqual(ExpectedTerm, JsonTerm). - - --endif. diff --git a/src/riak_cs_api.erl b/src/riak_cs_api.erl deleted file mode 100644 index 7b4a4c530..000000000 --- a/src/riak_cs_api.erl +++ /dev/null @@ -1,63 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(riak_cs_api). - --export([list_buckets/1, - list_objects/5]). - --include("riak_cs.hrl"). --include("riak_cs_api.hrl"). --include("list_objects.hrl"). - -%% @doc Return a user's buckets. --spec list_buckets(rcs_user()) -> ?LBRESP{}. -list_buckets(User=?RCS_USER{buckets=Buckets}) -> - ?LBRESP{user=User, - buckets=[Bucket || Bucket <- Buckets, - Bucket?RCS_BUCKET.last_action /= deleted]}. - --type options() :: [{atom(), 'undefined' | binary()}]. --spec list_objects([string()], binary(), non_neg_integer(), options(), riak_client()) -> - {ok, ?LORESP{}} | {error, term()}. -list_objects([], _, _, _, _) -> - {error, no_such_bucket}; -list_objects(_UserBuckets, _Bucket, {error, _}=Error, _Options, _RcPid) -> - Error; -list_objects(_UserBuckets, Bucket, MaxKeys, Options, RcPid) -> - ListKeysRequest = riak_cs_list_objects:new_request(Bucket, - MaxKeys, - Options), - BinPid = riak_cs_utils:pid_to_binary(self()), - CacheKey = << BinPid/binary, <<":">>/binary, Bucket/binary >>, - UseCache = riak_cs_list_objects_ets_cache:cache_enabled(), - list_objects(RcPid, ListKeysRequest, CacheKey, UseCache). - -list_objects(RcPid, ListKeysRequest, CacheKey, UseCache) -> - case riak_cs_list_objects_utils:start_link(RcPid, - self(), - ListKeysRequest, - CacheKey, - UseCache) of - {ok, ListFSMPid} -> - riak_cs_list_objects_utils:get_object_list(ListFSMPid); - {error, _}=Error -> - Error - end. diff --git a/src/riak_cs_app.erl b/src/riak_cs_app.erl deleted file mode 100644 index 7e24f171d..000000000 --- a/src/riak_cs_app.erl +++ /dev/null @@ -1,203 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc Callbacks for the riak_cs application. - --module(riak_cs_app). - --behaviour(application). - -%% application API --export([start/2, - stop/1, - check_bucket_props/2, - atoms_for_check_bucket_props/0]). - --include("riak_cs.hrl"). - --type start_type() :: normal | {takeover, node()} | {failover, node()}. --type start_args() :: term(). - -%% =================================================================== -%% Public API -%% =================================================================== - -%% @doc application start callback for riak_cs. --spec start(start_type(), start_args()) -> {ok, pid()} | - {error, term()}. -start(_Type, _StartArgs) -> - riak_cs_config:warnings(), - sanity_check(is_config_valid(), - check_admin_creds(), - check_bucket_props()). - -%% @doc application stop callback for riak_cs. --spec stop(term()) -> ok. -stop(_State) -> - ok. - --spec check_admin_creds() -> ok | {error, term()}. -check_admin_creds() -> - case riak_cs_config:admin_creds() of - {ok, {"admin-key", _}} -> - %% The default key - lager:warning("admin.key is defined as default. Please create" - " admin user and configure it.", []), - application:set_env(riak_cs, admin_secret, "admin-secret"), - ok; - {ok, {undefined, _}} -> - _ = lager:warning("The admin user's key id has not been specified."), - {error, admin_key_undefined}; - {ok, {[], _}} -> - _ = lager:warning("The admin user's key id has not been specified."), - {error, admin_key_undefined}; - {ok, {Key, undefined}} -> - fetch_and_cache_admin_creds(Key); - {ok, {Key, []}} -> - fetch_and_cache_admin_creds(Key); - {ok, {Key, _}} -> - _ = lager:warning("The admin user's secret is specified. Ignoring."), - fetch_and_cache_admin_creds(Key) - end. - -fetch_and_cache_admin_creds(Key) -> - %% Not using as the master pool might not be initialized - {ok, MasterPbc} = riak_connection(), - try - %% Do we count this into stats?; This is a startup query and - %% system latency is expected to be low. So get timeout can be - %% low like 10% of configuration value. - case riak_cs_pbc:get_sans_stats(MasterPbc, ?USER_BUCKET, iolist_to_binary(Key), - [{notfound_ok, false}], - riak_cs_config:get_user_timeout() div 10) of - {ok, Obj} -> - User = riak_cs_user:from_riakc_obj(Obj, false), - Secret = User?RCS_USER.key_secret, - lager:info("setting admin secret as ~s", [Secret]), - application:set_env(riak_cs, admin_secret, Secret); - Error -> - _ = lager:error("Couldn't get admin user (~s) record: ~p", - [Key, Error]), - Error - end - catch T:E -> - _ = lager:error("Couldn't get admin user (~s) record: ~p", - [Key, {T, E}]), - {error, {T, E}} - after - riakc_pb_socket:stop(MasterPbc) - end. - --spec sanity_check(boolean(), - ok | {error, term()}, - {ok, boolean()} | {error, term()}) -> - {ok, pid()} | {error, term()}. -sanity_check(true, ok, {ok, true}) -> - riak_cs_sup:start_link(); -sanity_check(false, _, _) -> - _ = lager:error("You must update your Riak CS app.config. Please see the" - "release notes for more information on updating you" - "configuration."), - {error, bad_config}; -sanity_check(true, _, {ok, false}) -> - _ = lager:error("Invalid Riak bucket properties detected. Please " - "verify that allow_mult is set to true for all " - "buckets."), - {error, invalid_bucket_props}; -sanity_check(true, _, {error, Reason}) -> - _ = lager:error("Could not verify bucket properties. Error was" - " ~p.", [Reason]), - {error, error_verifying_props}; -sanity_check(_, {error, Reason}, _) -> - _ = lager:error("Admin credentials are not properly set: ~p.", - [Reason]), - {error, Reason}. - --spec is_config_valid() -> boolean(). -is_config_valid() -> - get_env_response_to_bool(application:get_env(riak_cs, connection_pools)). - --spec get_env_response_to_bool(term()) -> boolean(). -get_env_response_to_bool({ok, _}) -> - true; -get_env_response_to_bool(_) -> - false. - --spec check_bucket_props() -> {ok, boolean()} | {error, term()}. -check_bucket_props() -> - Buckets = [?USER_BUCKET, - ?ACCESS_BUCKET, - ?STORAGE_BUCKET, - ?BUCKETS_BUCKET], - {ok, MasterPbc} = riak_connection(), - try - Results = [check_bucket_props(Bucket, MasterPbc) || - Bucket <- Buckets], - lists:foldl(fun promote_errors/2, {ok, true}, Results) - after - riakc_pb_socket:stop(MasterPbc) - end. - -%% Put atoms into atom table to suppress warning logs in `check_bucket_props' -atoms_for_check_bucket_props() -> - [riak_core_util, chash_std_keyfun, - riak_kv_wm_link_walker, mapreduce_linkfun]. - -promote_errors(_Elem, {error, _Reason}=E) -> - E; -promote_errors({error, _Reason}=E, _Acc) -> - E; -promote_errors({ok, false}=F, _Acc) -> - F; -promote_errors({ok, true}, Acc) -> - Acc. - --spec check_bucket_props(binary(), pid()) -> {ok, boolean()} | {error, term()}. -check_bucket_props(Bucket, MasterPbc) -> - case catch riakc_pb_socket:get_bucket(MasterPbc, Bucket) of - {ok, Props} -> - case lists:keyfind(allow_mult, 1, Props) of - {allow_mult, true} -> - _ = lager:debug("~s bucket was" - " already configured correctly.", - [Bucket]), - {ok, true}; - _ -> - _ = lager:warning("~p is misconfigured", [Bucket]), - {ok, false} - end; - {error, Reason}=E -> - _ = lager:warning( - "Unable to verify ~s bucket settings (~p).", - [Bucket, Reason]), - E - end. - -riak_connection() -> - {Host, Port} = riak_cs_config:riak_host_port(), - Timeout = case application:get_env(riak_cs, riakc_connect_timeout) of - {ok, ConfigValue} -> - ConfigValue; - undefined -> - 10000 - end, - StartOptions = [{connect_timeout, Timeout}, - {auto_reconnect, true}], - riakc_pb_socket:start_link(Host, Port, StartOptions). diff --git a/src/riak_cs_dtrace.erl b/src/riak_cs_dtrace.erl deleted file mode 100644 index fa234c670..000000000 --- a/src/riak_cs_dtrace.erl +++ /dev/null @@ -1,176 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(riak_cs_dtrace). - --export([dtrace/1, dtrace/3, dtrace/4, dtrace/6]). --include("riak_cs.hrl"). --export([dt_wm_entry/2, - dt_wm_entry/4, - dt_service_entry/2, - dt_service_entry/4, - dt_bucket_entry/4, - dt_object_entry/4, - dt_wm_return/2, - dt_wm_return/4, - dt_wm_return_bool/3, - dt_wm_return_bool_with_default/4, - dt_service_return/4, - dt_bucket_return/4, - dt_object_return/4]). --export([t/1, t/2, tt/3]). % debugging use only - - --define(MAGIC, '**DTRACE*SUPPORT**'). - -dtrace(ArgList) -> - case get(?MAGIC) of - undefined -> - case application:get_env(riak_cs, dtrace_support) of - {ok, true} -> - case string:to_float(erlang:system_info(version)) of - {5.8, _} -> - %% R14B04 - put(?MAGIC, dtrace), - dtrace(ArgList); - {Num, _} when Num > 5.8 -> - %% R15B or higher, though dyntrace option - %% was first available in R15B01. - put(?MAGIC, dyntrace), - dtrace(ArgList); - _ -> - put(?MAGIC, unsupported), - false - end; - _ -> - put(?MAGIC, unsupported), - false - end; - dyntrace -> - erlang:apply(dyntrace, p, ArgList); - dtrace -> - erlang:apply(dtrace, p, ArgList); - _ -> - false - end. - -dtrace(Int0, Ints, Strings) when is_integer(Int0) -> - case get(?MAGIC) of - unsupported -> - false; - _ -> - dtrace([Int0] ++ Ints ++ Strings) - end. - -dtrace(Int0, Ints, String0, Strings) when is_integer(Int0) -> - case get(?MAGIC) of - unsupported -> - false; - _ -> - dtrace([Int0] ++ Ints ++ [String0] ++ Strings) - end. - -%% NOTE: Due to use of ?MODULE, we may have cases where the type -%% of String0 is an atom and not a string/iodata. - -dtrace(Int0, Int1, Ints, String0, String1, Strings) - when is_integer(Int0), is_integer(Int1) -> - case get(?MAGIC) of - unsupported -> - false; - _ -> - S0 = if is_atom(String0) -> erlang:atom_to_binary(String0, latin1); - true -> String0 - end, - dtrace([Int0, Int1] ++ Ints ++ [S0, String1] ++ Strings) - end. - -t(L) -> % debugging/micro-performance - dtrace(L). - -t(Ints, Strings) -> % debugging/micro-performance - dtrace([77] ++ Ints ++ ["entry"] ++ Strings). - -tt(Int0, Ints, Strings) -> % debugging/micro-performance - case get(?MAGIC) of - X when X == dyntrace; X == dtrace -> - dtrace([Int0] ++ Ints ++ Strings); - _ -> - false - end. - -dt_wm_entry(Mod, Func) -> - dt_wm_entry(Mod, Func, [], []). - -dt_wm_entry({Mod, SubMod}, Func, Ints, Strings) when is_atom(Mod), is_atom(SubMod)-> - dt_wm_entry(common_submod_to_bin(Mod, SubMod), Func, Ints, Strings); -dt_wm_entry(Mod, Func, Ints, Strings) -> - riak_cs_dtrace:dtrace(?DT_WM_OP, 1, Ints, Mod, Func, Strings). - -dt_service_entry(Mod, Func) -> - dt_service_entry(Mod, Func, [], []). -dt_service_entry(Mod, Func, Ints, Strings) -> - riak_cs_dtrace:dtrace(?DT_SERVICE_OP, 1, Ints, Mod, Func, Strings). - -dt_bucket_entry(Mod, Func, Ints, Strings) -> - riak_cs_dtrace:dtrace(?DT_BUCKET_OP, 1, Ints, Mod, Func, Strings). - -dt_object_entry(Mod, Func, Ints, Strings) -> - riak_cs_dtrace:dtrace(?DT_OBJECT_OP, 1, Ints, Mod, Func, Strings). - -dt_wm_return_bool(Mod, Func, true) -> - dt_wm_return(Mod, Func, [1], []); -dt_wm_return_bool(Mod, Func, false) -> - dt_wm_return(Mod, Func, [0], []). - -%% Like `dt_wm_return_bool', but uses a default -%% boolean value from `Default' if the 3rd argument is -%% a `{halt, integer()}' tuple -dt_wm_return_bool_with_default(Mod, Func, Bool, _Default) when is_boolean(Bool) -> - dt_wm_return_bool(Mod, Func, Bool); -dt_wm_return_bool_with_default(Mod, Func, {halt, _Code}, Default) when is_integer(_Code) -> - dt_wm_return_bool(Mod, Func, Default). - -dt_wm_return(Mod, Func) -> - dt_wm_return(Mod, Func, [], []). - -dt_wm_return({Mod, SubMod}, Func, Ints, Strings) when is_atom(Mod), is_atom(SubMod)-> - dt_wm_return(common_submod_to_bin(Mod, SubMod), Func, Ints, Strings); -dt_wm_return(Mod, Func, Ints, Strings) -> - riak_cs_dtrace:dtrace(?DT_WM_OP, 2, Ints, Mod, Func, Strings). - -dt_service_return(Mod, Func, Ints, Strings) -> - riak_cs_dtrace:dtrace(?DT_SERVICE_OP, 2, Ints, Mod, Func, Strings). - -dt_bucket_return(Mod, Func, Ints, Strings) -> - riak_cs_dtrace:dtrace(?DT_BUCKET_OP, 2, Ints, Mod, Func, Strings). - -dt_object_return(Mod, Func, Ints, Strings) -> - riak_cs_dtrace:dtrace(?DT_OBJECT_OP, 2, Ints, Mod, Func, Strings). - - -%% =================================================================== -%% Internal Functions -%% =================================================================== - -common_submod_to_bin(Mod, SubMod) -> - <<(atom_to_binary(Mod, latin1))/binary, - "/", - (atom_to_binary(SubMod, latin1))/binary>>. diff --git a/src/riak_cs_list_objects.erl b/src/riak_cs_list_objects.erl deleted file mode 100644 index f53e3d998..000000000 --- a/src/riak_cs_list_objects.erl +++ /dev/null @@ -1,129 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc - --module(riak_cs_list_objects). - --include("riak_cs.hrl"). --include("list_objects.hrl"). - -%% API --export([new_request/1, - new_request/3, - new_response/5, - manifest_to_keycontent/1]). - -%%%=================================================================== -%%% API -%%%=================================================================== - -%% Request -%%-------------------------------------------------------------------- - --spec new_request(binary()) -> list_object_request(). -new_request(Name) -> - new_request(Name, 1000, []). - --spec new_request(binary(), pos_integer(), list()) -> list_object_request(). -new_request(Name, MaxKeys, Options) -> - process_options(#list_objects_request_v1{name=Name, - max_keys=MaxKeys}, - Options). - -%% @private --spec process_options(list_object_request(), list()) -> - list_object_request(). -process_options(Request, Options) -> - lists:foldl(fun process_options_helper/2, - Request, - Options). - -process_options_helper({prefix, Val}, Req) -> - Req#list_objects_request_v1{prefix=Val}; -process_options_helper({delimiter, Val}, Req) -> - Req#list_objects_request_v1{delimiter=Val}; -process_options_helper({marker, Val}, Req) -> - Req#list_objects_request_v1{marker=Val}. - -%% Response -%%-------------------------------------------------------------------- - --spec new_response(list_object_request(), - IsTruncated :: boolean(), - NextMarker :: next_marker(), - CommonPrefixes :: list(list_objects_common_prefixes()), - ObjectContents :: list(list_objects_key_content())) -> - list_object_response(). -new_response(?LOREQ{name=Name, - max_keys=MaxKeys, - prefix=Prefix, - delimiter=Delimiter, - marker=Marker}, - IsTruncated, NextMarker, CommonPrefixes, ObjectContents) -> - ?LORESP{name=Name, - max_keys=MaxKeys, - prefix=Prefix, - delimiter=Delimiter, - marker=Marker, - next_marker=NextMarker, - is_truncated=IsTruncated, - contents=ObjectContents, - common_prefixes=CommonPrefixes}. - -%% Rest -%%-------------------------------------------------------------------- - --spec manifest_to_keycontent(lfs_manifest()) -> list_objects_key_content(). -manifest_to_keycontent(?MANIFEST{bkey={_Bucket, Key}, - created=Created, - content_md5=ContentMd5, - content_length=ContentLength, - acl=ACL}) -> - - LastModified = list_to_binary(riak_cs_wm_utils:to_iso_8601(Created)), - - %% Etag - ETagString = riak_cs_utils:etag_from_binary(ContentMd5), - Etag = list_to_binary(ETagString), - - Size = ContentLength, - Owner = acl_to_owner(ACL), - %% hardcoded since we don't support reduced redundancy or glacier - StorageClass = <<"STANDARD">>, - - #list_objects_key_content_v1{key=Key, - last_modified=LastModified, - etag=Etag, - size=Size, - owner=Owner, - storage_class=StorageClass}. - -%% ==================================================================== -%% Internal functions -%% ==================================================================== - --spec acl_to_owner(acl()) -> list_objects_owner(). -acl_to_owner(?ACL{owner=Owner}) -> - {DisplayName, CanonicalId, _KeyId} = Owner, - CanonicalIdBinary = list_to_binary(CanonicalId), - DisplayNameBinary = list_to_binary(DisplayName), - #list_objects_owner_v1{id=CanonicalIdBinary, - display_name=DisplayNameBinary}. diff --git a/src/riak_cs_list_objects_ets_cache.erl b/src/riak_cs_list_objects_ets_cache.erl deleted file mode 100644 index b91ed48cd..000000000 --- a/src/riak_cs_list_objects_ets_cache.erl +++ /dev/null @@ -1,308 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc - --module(riak_cs_list_objects_ets_cache). - --behaviour(gen_server). --include("riak_cs.hrl"). --include("list_objects.hrl"). - -%% API --export([start_link/0, - lookup/1, - can_write/3, - can_write/4, - write/2, - bytes_used/0, - bytes_used/1]). - -%% gen_server callbacks --export([init/1, - handle_call/3, - handle_cast/2, - handle_info/2, - terminate/2, - code_change/3]). - -%% Config getters --export([default_ets_table/0, - cache_enabled/0, - cache_timeout/0, - min_keys_to_cache/0, - max_cache_size/0]). - --define(DICTMODULE, dict). - --ifdef(namespaced_types). --type dictionary() :: dict:dict(). --else. --type dictionary() :: dict(). --endif. - --record(state, {tid :: ets:tid(), - monitor_to_timer = ?DICTMODULE:new() :: dictionary(), - key_to_monitor = ?DICTMODULE:new() :: dictionary()}). - --type state() :: #state{}. --type cache_lookup_result() :: {true, [binary()]} | false. - -%%%=================================================================== -%%% API -%%%=================================================================== - -start_link() -> - %% named proc - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - --spec lookup(binary()) -> cache_lookup_result(). -lookup(Key) -> - try - _ = lager:debug("Reading info for ~p from cache", [Key]), - format_lookup_result(ets:lookup(default_ets_table(), Key)) - catch - _:Reason -> - _ = lager:warning("List objects cache lookup failed. Reason: ~p", [Reason]), - false - end. - --spec can_write(CacheKey :: binary(), - MonitorPid :: pid(), - NumKeys :: non_neg_integer()) -> boolean(). -can_write(CacheKey, MonitorPid, NumKeys) -> - can_write(?MODULE, CacheKey, MonitorPid, NumKeys). - --spec can_write(Pid :: pid() | atom(), - CacheKey :: binary(), - MonitorPid :: pid(), - NumKeys :: non_neg_integer()) -> boolean(). -can_write(ServerPid, CacheKey, MonitorPid, NumKeys) -> - Message = {can_write, {CacheKey, MonitorPid, NumKeys}}, - gen_server:call(ServerPid, Message, infinity). - --spec write(binary(), term()) -> ok. -write(Key, Value) -> - try - unsafe_write(Key, Value) - catch - _:Reason -> - _ = lager:warning("List objects cache write failed. Reason: ~p", [Reason]), - ok - end. - --spec bytes_used() -> non_neg_integer(). -bytes_used() -> - bytes_used(default_ets_table()). - --spec bytes_used(ets:tab()) -> non_neg_integer(). -bytes_used(TableID) -> - WordsUsed = ets:info(TableID, memory), - words_to_bytes(WordsUsed). - -%%%=================================================================== -%%% gen_server callbacks -%%%=================================================================== - --spec init(list()) -> {ok, state()}. -init([]) -> - Tid = new_table(), - NewState = #state{tid=Tid}, - {ok, NewState}. - -handle_call(get_tid, _From, State=#state{tid=Tid}) -> - {reply, Tid, State}; -handle_call({can_write, {CacheKey, MonitorPid, NumKeys}}, - _From, State) -> - Bool = should_write(NumKeys), - NewState = case Bool of - true -> - update_state_with_refs(CacheKey, MonitorPid, State); - false -> - State - end, - {reply, Bool, NewState}; -handle_call(Msg, _From, State) -> - _ = lager:debug("got unknown message: ~p", [Msg]), - {reply, ok, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info({cache_expiry, ExpiredKey}, State) -> - NewState = handle_cache_expiry(ExpiredKey, State), - {noreply, NewState}; -handle_info({'DOWN', MonitorRef, process, _Pid, _Info}, State) -> - NewState = handle_down(MonitorRef, State), - {noreply, NewState}; -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%%=================================================================== -%%% Config getters -%%%=================================================================== - -%% @private -get_with_default(EnvKey, Default) -> - case application:get_env(riak_cs, EnvKey) of - {ok, Val} -> - Val; - undefined -> - Default - end. - -default_ets_table() -> - get_with_default(list_objects_ets_cache_table_name, ?LIST_OBJECTS_CACHE). - -cache_enabled() -> - get_with_default(list_objects_ets_cache_enabled, ?ENABLE_CACHE). - -cache_timeout() -> - get_with_default(list_objects_ets_cache_timeout, ?CACHE_TIMEOUT). - -min_keys_to_cache() -> - get_with_default(list_objects_ets_cache_min_keys, ?MIN_KEYS_TO_CACHE). - -max_cache_size() -> - get_with_default(list_objects_ets_cache_max_bytes, ?MAX_CACHE_BYTES). - -%%%=================================================================== -%%% Internal functions -%%%=================================================================== - --spec should_write(non_neg_integer()) -> boolean(). -should_write(NumKeys) -> - enough_keys_to_cache(NumKeys) andalso space_in_cache(NumKeys). - --spec enough_keys_to_cache(non_neg_integer()) -> boolean(). -enough_keys_to_cache(NumKeys) -> - NumKeys >= min_keys_to_cache(). - -%% @doc Based on current ETS usage and some estimate, decide whether -%% or not we can fit these keys in the cache -%% -%% Current estimate is ~ 64 bytes per key. --spec space_in_cache(non_neg_integer()) -> boolean(). -space_in_cache(NumKeys) -> - space_in_cache(default_ets_table(), NumKeys). - -space_in_cache(TableID, NumKeys) -> - BytesUsed = bytes_used(TableID), - space_available(BytesUsed, NumKeys). - -words_to_bytes(Words) -> - Words * ?WORD_SIZE. - -space_available(BytesUsed, NumKeys) -> - space_available(BytesUsed, num_keys_to_bytes(NumKeys), max_cache_size()). - -space_available(BytesUsed, ProspectiveAdd, MaxCacheSize) - when (BytesUsed + ProspectiveAdd) < MaxCacheSize -> - true; -space_available(_BytesUsed, _ProspectiveAdd, _MaxCacheSize) -> - false. - -num_keys_to_bytes(NumKeys) -> - NumKeys * 64. - - -unsafe_write(Key, Value) -> - TS = riak_cs_utils:second_resolution_timestamp(os:timestamp()), - _ = lager:debug("Writing entry for ~p to LO Cache", [Key]), - ets:insert(default_ets_table(), {Key, Value, TS}), - ok. - --spec update_state_with_refs(binary(), pid(), state()) -> state(). -update_state_with_refs(CacheKey, MonitorPid, State) -> - MonitorRef = erlang:monitor(process, MonitorPid), - TimerRef = erlang:send_after(cache_timeout(), ?MODULE, - {cache_expiry, CacheKey}), - update_state_with_refs_helper(MonitorRef, TimerRef, CacheKey, State). - --spec update_state_with_refs_helper(reference(), reference(), binary(), state()) -> - state(). -update_state_with_refs_helper(MonitorRef, TimerRef, CacheKey, - State=#state{monitor_to_timer=MonToTimer, - key_to_monitor=KeyToMon}) -> - NewMonToTimer = ?DICTMODULE:store(MonitorRef, {TimerRef, CacheKey}, MonToTimer), - NewKeyToMon = ?DICTMODULE:store(TimerRef, MonitorRef, KeyToMon), - State#state{monitor_to_timer=NewMonToTimer, - key_to_monitor=NewKeyToMon}. - --spec handle_cache_expiry(binary(), state()) -> state(). -handle_cache_expiry(ExpiredKey, State=#state{key_to_monitor=KeyToMon}) -> - ets:delete(default_ets_table(), ExpiredKey), - NewKeyToMon = remove_monitor(ExpiredKey, KeyToMon), - State#state{key_to_monitor=NewKeyToMon}. - --spec handle_down(reference(), state()) -> state(). -handle_down(MonitorRef, State=#state{monitor_to_timer=MonToTimer}) -> - NewMonToTimer = remove_timer(MonitorRef, MonToTimer), - State#state{monitor_to_timer=NewMonToTimer}. - --spec remove_monitor(binary(), dictionary()) -> dictionary(). -remove_monitor(ExpiredKey, KeyToMon) -> - RefResult = safe_fetch(ExpiredKey, KeyToMon), - case RefResult of - {ok, Ref} -> - true = erlang:demonitor(Ref); - {error, _Reason} -> - true - end, - ?DICTMODULE:erase(ExpiredKey, KeyToMon). - --spec remove_timer(reference(), dictionary()) -> dictionary(). -remove_timer(MonitorRef, MonToTimer) -> - RefResult = safe_fetch(MonitorRef, MonToTimer), - _ = case RefResult of - {ok, {TimerRef, CacheKey}} -> - %% can be true | false - _ = ets:delete(default_ets_table(), CacheKey), - erlang:cancel_timer(TimerRef); - {error, _Reason} -> - true - end, - ?DICTMODULE:erase(MonitorRef, MonToTimer). - --spec safe_fetch(Key :: term(), Dict :: dictionary()) -> - {ok, term()} | {error, term()}. -safe_fetch(Key, Dict) -> - try - {ok, ?DICTMODULE:fetch(Key, Dict)} - catch error:Reason -> - {error, Reason} - end. - --spec new_table() -> ets:tid(). -new_table() -> - TableSpec = [public, set, named_table, {write_concurrency, true}], - ets:new(default_ets_table(), TableSpec). - --spec format_lookup_result([{binary(), term(), integer()}]) -> cache_lookup_result(). -format_lookup_result([]) -> - false; -format_lookup_result([{_, Value, _}]) -> - {true, Value}. diff --git a/src/riak_cs_list_objects_ets_cache_sup.erl b/src/riak_cs_list_objects_ets_cache_sup.erl deleted file mode 100644 index b9a28dec4..000000000 --- a/src/riak_cs_list_objects_ets_cache_sup.erl +++ /dev/null @@ -1,59 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc --module(riak_cs_list_objects_ets_cache_sup). - --behaviour(supervisor). - -%% API --export([start_link/0]). - -%% Supervisor callbacks --export([init/1]). - -%%%=================================================================== -%%% API functions -%%%=================================================================== - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -%%%=================================================================== -%%% Supervisor callbacks -%%%=================================================================== - -init([]) -> - RestartStrategy = one_for_one, - MaxRestarts = 1000, - MaxSecondsBetweenRestarts = 3600, - - SupFlags = {RestartStrategy, MaxRestarts, MaxSecondsBetweenRestarts}, - - ETSCache = {riak_cs_list_objects_ets_cache, - {riak_cs_list_objects_ets_cache, start_link, []}, - permanent, 5000, worker, - [riak_cs_list_objects_ets_cache]}, - - {ok, {SupFlags, [ETSCache]}}. - -%%%=================================================================== -%%% Internal functions -%%%=================================================================== diff --git a/src/riak_cs_list_objects_fsm.erl b/src/riak_cs_list_objects_fsm.erl deleted file mode 100644 index 1d1409beb..000000000 --- a/src/riak_cs_list_objects_fsm.erl +++ /dev/null @@ -1,691 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc - -%% TODO: -%% 1. optimize for when list keys returns [], we shouldn't -%% even have to do map-reduce then - --module(riak_cs_list_objects_fsm). - --behaviour(gen_fsm). - --ifdef(TEST). --compile([export_all]). --include_lib("eunit/include/eunit.hrl"). --endif. - --include("riak_cs.hrl"). --include("list_objects.hrl"). - -%% API --export([start_link/3, - start_link/5]). - -%% gen_fsm callbacks --export([init/1, - prepare/2, - waiting_list_keys/2, - waiting_map_reduce/2, - handle_event/3, - handle_sync_event/4, - handle_info/3, - terminate/3, - code_change/4]). - --record(profiling, { - %% floating point secs - list_keys_start_time :: float(), - list_keys_end_time :: float(), - list_keys_num_results :: non_neg_integer(), - - temp_mr_req :: {Request :: {StartIdx :: non_neg_integer(), - EndIdx :: non_neg_integer()}, - NumKeysRequested :: non_neg_integer(), - StartTime :: float()}, - mr_requests=[] :: [{Request :: {StartIdx :: non_neg_integer(), - EndIdx :: non_neg_integer()}, - NumKeysRequested :: non_neg_integer(), - Timing :: {StartTime :: float(), - EndTime :: float()}}]}). --type profiling() :: #profiling{}. - - --record(state, {riak_client :: riak_client(), - caller_pid :: pid(), - req :: list_object_request(), - - reply_ref :: undefined | {pid(), any()}, - - %% list keys ---- - list_keys_req_id :: undefined | non_neg_integer(), - key_buffer=[] :: undefined | list(), - keys=[] :: undefined | list(), - %% we cache the number of keys because - %% `length/1' is linear time - num_keys :: undefined | non_neg_integer(), - - filtered_keys :: undefined | list(), - - %% The number of keys that could be used - %% for a map-reduce request. This accounts - %% the `marker' that may be in the request. - %% This number should always be =< `num_keys'. - %% This field is used to help determine whether - %% we have enough results yet from map-reduce - %% to fullfill our query. - num_considerable_keys :: undefined | non_neg_integer(), - - %% We issue a map reduce request for _more_ - %% keys than the user asks for because some - %% of the keys returned from list keys - %% may be tombstoned or marked as deleted. - %% We assume this will happen to some percentage - %% of keys, so we account for it in every request - %% by multiplying the number of objects we need - %% by some constant. - key_multiplier :: float(), - - %% map reduce ---- - %% this field will change, it represents - %% the current outstanding m/r request - map_red_req_id :: undefined | non_neg_integer(), - mr_requests=[] :: [{StartIdx :: non_neg_integer(), - EndIdx :: non_neg_integer()}], - object_buffer=[] :: list(), - - response :: undefined | list_object_response(), - - req_profiles=#profiling{} :: profiling(), - - %% whether or not to bypass the cache entirely - use_cache :: boolean(), - %% Key to use to check for cached results from key listing - cache_key :: term(), - common_prefixes=ordsets:new() :: list_objects_common_prefixes()}). - -%% some useful shared types - --type state() :: #state{}. - --type fsm_state_return() :: {next_state, atom(), state()} | - {next_state, atom(), state(), non_neg_integer()} | - {stop, term(), state()}. - --type streaming_req_response() :: {ok, ReqID :: non_neg_integer()} | - {error, term()}. - --type list_keys_event() :: {ReqID :: non_neg_integer(), done} | - {ReqID :: non_neg_integer(), {keys, list()}} | - {ReqID :: non_neg_integer(), {error, term()}}. - --type mapred_event() :: {ReqID :: non_neg_integer(), done} | - {ReqID :: non_neg_integer(), - {mapred, Phase :: non_neg_integer(), list()}} | - {ReqID :: non_neg_integer, {error, term()}}. - -%%%=================================================================== -%%% API -%%%=================================================================== - --spec start_link(riak_client(), list_object_request(), term()) -> - {ok, pid()} | {error, term()}. -start_link(RcPid, ListKeysRequest, CacheKey) -> - start_link(RcPid, self(), ListKeysRequest, CacheKey, true). - --spec start_link(riak_client(), pid(), list_object_request(), term(), UseCache :: boolean()) -> - {ok, pid()} | {error, term()}. -start_link(RcPid, CallerPid, ListKeysRequest, CacheKey, UseCache) -> - gen_fsm:start_link(?MODULE, [RcPid, CallerPid, ListKeysRequest, - CacheKey, UseCache], []). - -%%%=================================================================== -%%% gen_fsm callbacks -%%%=================================================================== - --spec init(list()) -> {ok, prepare, state(), 0}. -init([RcPid, CallerPid, Request, CacheKey, UseCache]) -> - %% TODO: should we be linking or monitoring - %% the proc that called us? - - %% TODO: this should not be hardcoded. Maybe there should - %% be two `start_link' arities, and one will use a default - %% val from app.config and the other will explicitly - %% take a val - KeyMultiplier = riak_cs_config:key_list_multiplier(), - - State = #state{riak_client=RcPid, - caller_pid=CallerPid, - key_multiplier=KeyMultiplier, - req=Request, - use_cache=UseCache, - cache_key=CacheKey}, - {ok, prepare, State, 0}. - --spec prepare(timeout, state()) -> fsm_state_return(). -prepare(timeout, State=#state{riak_client=RcPid, - req=Request}) -> - maybe_fetch_key_list(RcPid, Request, State). - --spec waiting_list_keys(list_keys_event(), state()) -> fsm_state_return(). -waiting_list_keys({ReqID, done}, State=#state{list_keys_req_id=ReqID}) -> - handle_keys_done(State); -waiting_list_keys({ReqID, {keys, Keys}}, - State=#state{list_keys_req_id=ReqID}) -> - handle_keys_received(Keys, State); -waiting_list_keys({ReqID, {error, Reason}}, - State=#state{list_keys_req_id=ReqID}) -> - %% if we get an error while we're in this state, - %% we still have the option to return `HTTP 500' - %% to the client, since we haven't written - %% any bytes yet - {stop, Reason, State}. - --spec waiting_map_reduce(mapred_event(), state()) -> fsm_state_return(). -waiting_map_reduce({ReqID, done}, State=#state{map_red_req_id=ReqID}) -> - %% depending on the result of this, we'll either - %% make another m/r request, or be done - State2 = update_state_with_mr_end_profiling(State), - maybe_map_reduce(State2); -waiting_map_reduce({ReqID, {mapred, _Phase, Results}}, - State=#state{map_red_req_id=ReqID}) -> - handle_mapred_results(Results, State); -waiting_map_reduce({ReqID, {error, Reason}}, - State=#state{map_red_req_id=ReqID}) -> - {stop, Reason, State}. - -handle_event(_Event, StateName, State) -> - {next_state, StateName, State}. - -handle_sync_event(get_object_list, _From, done, State=#state{response=Resp}) -> - Reply = {ok, Resp}, - {stop, normal, Reply, State}; -handle_sync_event(get_object_list, From, StateName, State) -> - NewStateData = State#state{reply_ref=From}, - {next_state, StateName, NewStateData}; -handle_sync_event(get_internal_state, _From, StateName, State) -> - Reply = {StateName, State}, - {reply, Reply, StateName, State}; -handle_sync_event(Event, _From, StateName, State) -> - _ = lager:debug("got unknown event ~p in state ~p", [Event, StateName]), - Reply = ok, - {reply, Reply, StateName, State}. - -%% the responses from `riakc_pb_socket:stream_list_keys' -%% come back as regular messages, so just pass -%% them along as if they were gen_server events. -handle_info(Info, waiting_list_keys, State) -> - waiting_list_keys(Info, State); -handle_info(Info, waiting_map_reduce, State) -> - waiting_map_reduce(Info, State). - -terminate(normal, _StateName, #state{req_profiles=Profilings}) -> - _ = print_profiling(Profilings), - ok; -terminate(_Reason, _StateName, _State) -> - ok. - -code_change(_OldVsn, StateName, State, _Extra) -> - {ok, StateName, State}. - -%%-------------------------------------------------------------------- -%% Internal helpers -%%-------------------------------------------------------------------- - -%% List Keys stuff -%%-------------------------------------------------------------------- - --spec maybe_fetch_key_list(riak_client(), list_object_request(), state()) -> - fsm_state_return(). -maybe_fetch_key_list(RcPid, Request, State=#state{use_cache=true, - cache_key=CacheKey}) -> - CacheResult = riak_cs_list_objects_ets_cache:lookup(CacheKey), - fetch_key_list(RcPid, Request, State, CacheResult); -maybe_fetch_key_list(RcPid, Request, State=#state{use_cache=false}) -> - fetch_key_list(RcPid, Request, State, false). - --spec maybe_write_to_cache(state(), list()) -> ok. -maybe_write_to_cache(#state{use_cache=false}, _ListofListofKeys) -> - ok; -maybe_write_to_cache(#state{cache_key=CacheKey, - caller_pid=CallerPid}, ListofListofKeys) -> - ListOfListOfKeysLength = lists:flatlength(ListofListofKeys), - case riak_cs_list_objects_ets_cache:can_write(CacheKey, - CallerPid, - ListOfListOfKeysLength) of - true -> - _ = lager:debug("writing to the cache"), - riak_cs_list_objects_ets_cache:write(CacheKey, ListofListofKeys); - false -> - _ = lager:debug("not writing to the cache"), - ok - end. - -%% @doc Either proceed using the cached key list or make the request -%% to start a key listing. --type cache_lookup_result() :: {true, [binary()]} | false. --spec fetch_key_list(riak_client(), list_object_request(), state(), - cache_lookup_result()) -> fsm_state_return(). -fetch_key_list(_, _, State, {true, Value}) -> - _ = lager:debug("Using cached key list"), - NewState = prepare_state_for_first_mapred(Value, - State#state{key_buffer=Value}), - maybe_map_reduce(NewState); -fetch_key_list(RcPid, Request, State, false) -> - _ = lager:debug("Requesting fresh key list"), - handle_streaming_list_keys_call( - make_list_keys_request(RcPid, Request), - State). - -%% function to create a list keys request -%% TODO: -%% could this also be a phase-less map-reduce request -%% with key filters? --spec make_list_keys_request(riak_client(), list_object_request()) -> - streaming_req_response(). -make_list_keys_request(RcPid, ?LOREQ{name=BucketName}) -> - ServerTimeout = riak_cs_config:list_keys_list_objects_timeout(), - ManifestBucket = riak_cs_utils:to_bucket_name(objects, BucketName), - {ok, ManifestPbc} = riak_cs_riak_client:manifest_pbc(RcPid), - riakc_pb_socket:stream_list_keys(ManifestPbc, - ManifestBucket, - ServerTimeout). - --spec handle_streaming_list_keys_call(streaming_req_response(), state()) -> - fsm_state_return(). -handle_streaming_list_keys_call({ok, ReqID}, State) -> - ListKeysStartTime = riak_cs_utils:timestamp_to_seconds(os:timestamp()), - Profiling2 = #profiling{list_keys_start_time=ListKeysStartTime}, - {next_state, waiting_list_keys, State#state{list_keys_req_id=ReqID, - req_profiles=Profiling2}}; -handle_streaming_list_keys_call({error, Reason}, State) -> - {stop, Reason, State}. - --spec handle_keys_done(state()) -> fsm_state_return(). -handle_keys_done(State=#state{key_buffer=ListofListofKeys}) -> - %% TODO: this could potentially be pretty expensive - %% and memory instensive. More reason to think about starting - %% to only keep a smaller buffer. See comment in - %% `handle_keys_received' - SortedFlattenedKeys = lists:sort(lists:flatten(ListofListofKeys)), - ok = maybe_write_to_cache(State, SortedFlattenedKeys), - NewState = prepare_state_for_first_mapred(SortedFlattenedKeys, State), - maybe_map_reduce(NewState). - --spec prepare_state_for_first_mapred(list(), state()) -> state(). -prepare_state_for_first_mapred(KeyList, State=#state{req=Request, - req_profiles=Profiling}) -> - NumKeys = length(KeyList), - - %% profiling info - EndTime = riak_cs_utils:timestamp_to_seconds(os:timestamp()), - Profiling2 = Profiling#profiling{list_keys_num_results=NumKeys, - list_keys_end_time=EndTime}, - - FilteredKeys = filtered_keys_from_request(Request, - KeyList, - NumKeys), - TotalCandidateKeys = length(FilteredKeys), - State#state{keys=undefined, - num_keys=NumKeys, - num_considerable_keys=TotalCandidateKeys, - filtered_keys=FilteredKeys, - %% free up the space - key_buffer=undefined, - req_profiles=Profiling2}. - -handle_keys_received(Keys, State=#state{key_buffer=PrevKeyBuffer}) -> - %% TODO: - %% this is where we might eventually do a 'top-k' keys - %% kind of thing, like - %% `lists:sublist(lists:sort([Keys | PrevKeyBuffer]), BufferSize)' - NewState = State#state{key_buffer=[lists:sort(Keys) | PrevKeyBuffer]}, - {next_state, waiting_list_keys, NewState}. - -%% Map Reduce stuff -%%-------------------------------------------------------------------- - --spec maybe_map_reduce(state()) -> fsm_state_return(). -maybe_map_reduce(State=#state{object_buffer=ObjectBuffer, - common_prefixes=CommonPrefixes, - req=Request, - num_considerable_keys=TotalCandidateKeys}) -> - ManifestsAndPrefixes = {ObjectBuffer, CommonPrefixes}, - Enough = enough_results(ManifestsAndPrefixes, Request, TotalCandidateKeys), - handle_enough_results(Enough, State). - -handle_enough_results(true, State) -> - have_enough_results(State); -handle_enough_results(false, - State=#state{num_considerable_keys=TotalCandidateKeys, - mr_requests=MapRRequests}) -> - MoreQuery = could_query_more_mapreduce(MapRRequests, TotalCandidateKeys), - handle_could_query_more_map_reduce(MoreQuery, State). - --spec handle_could_query_more_map_reduce(boolean(), state()) -> - fsm_state_return(). -handle_could_query_more_map_reduce(true, - State=#state{req=Request, - riak_client=RcPid}) -> - NewStateData = prepare_state_for_mapred(State), - KeysToQuery = next_keys_from_state(NewStateData), - BucketName = Request?LOREQ.name, - ManifestBucketName = riak_cs_utils:to_bucket_name(objects, BucketName), - MapReduceRequestResult = make_map_reduce_request(RcPid, - ManifestBucketName, - KeysToQuery), - handle_map_reduce_call(MapReduceRequestResult, NewStateData); -handle_could_query_more_map_reduce(false, State) -> - have_enough_results(State). - -prepare_state_for_mapred(State=#state{req=Request, - key_multiplier=KeyMultiplier, - mr_requests=PrevRequests, - object_buffer=ObjectBuffer}) -> - TotalNeeded = Request?LOREQ.max_keys, - NewReq = next_mr_query_spec(PrevRequests, - TotalNeeded, - length(ObjectBuffer), - KeyMultiplier), - State#state{mr_requests=PrevRequests ++ [NewReq]}. - --spec make_response(list_object_request(), list(), list()) -> - list_object_response(). -make_response(Request=?LOREQ{max_keys=NumKeysRequested}, - ObjectBuffer, CommonPrefixes) -> - ObjectPrefixTuple = {ObjectBuffer, CommonPrefixes}, - NumObjects = - riak_cs_list_objects_utils:manifests_and_prefix_length(ObjectPrefixTuple), - IsTruncated = NumObjects > NumKeysRequested andalso NumKeysRequested > 0, - SlicedTaggedItems = - riak_cs_list_objects_utils:manifests_and_prefix_slice(ObjectPrefixTuple, - NumKeysRequested), - {NewManis, NewPrefixes} = - riak_cs_list_objects_utils:untagged_manifest_and_prefix(SlicedTaggedItems), - KeyContents = lists:map(fun riak_cs_list_objects:manifest_to_keycontent/1, - NewManis), - riak_cs_list_objects:new_response(Request, IsTruncated, undefined, NewPrefixes, - KeyContents). - --spec next_mr_query_spec(list(), - non_neg_integer(), - non_neg_integer(), - float()) -> - {integer(), integer()}. -next_mr_query_spec([], TotalNeeded, _NumHaveSoFar, KeyMultiplier) -> - StartIdx = 1, - EndIdx = round_up(TotalNeeded * KeyMultiplier), - {StartIdx, EndIdx}; -next_mr_query_spec(PrevRequests, TotalNeeded, NumHaveSoFar, KeyMultiplier) -> - MoreNeeded = TotalNeeded - NumHaveSoFar, - StartIdx = element(2, lists:last(PrevRequests)) + 1, - EndIdx = round_up((StartIdx + MoreNeeded) * KeyMultiplier), - {StartIdx, EndIdx}. - -next_keys_from_state(#state{mr_requests=Requests, - filtered_keys=FilteredKeys}) -> - next_keys(lists:last(Requests), FilteredKeys). - -next_keys({StartIdx, EndIdx}, Keys) -> - %% the last arg to sublist is _not_ an index, but - %% a length, hence the diff of `EndIdx' and `StartIdx' - Length = (EndIdx - StartIdx) + 1, - lists:sublist(Keys, StartIdx, Length). - - --spec handle_mapred_results(list(), state()) -> - fsm_state_return(). -handle_mapred_results(Results, State=#state{object_buffer=Buffer, - common_prefixes=OldPrefixes, - req=Request}) -> - CleanedResults = lists:map(fun clean_manifest/1, Results), - {NoPrefix, Prefixes} = - riak_cs_list_objects_utils:filter_prefix_keys({CleanedResults, OldPrefixes}, - Request), - NewBuffer = update_buffer(NoPrefix, Buffer), - NewState = State#state{object_buffer=NewBuffer, - common_prefixes=Prefixes}, - {next_state, waiting_map_reduce, NewState}. - -%% @doc Results come back (for historical reasons...?) like this -%% from the map/reduce call. Rather than change the m/r code -%% (which would make rolling upgrades more difficult), we just -%% transform the values of the list here. --spec clean_manifest({binary(), {ok, lfs_manifest()}}) -> - lfs_manifest(). -clean_manifest({_Key, {ok, Manifest}}) -> - Manifest. - --spec update_buffer(list(), list()) -> list(). -update_buffer(Results, Buffer) -> - %% TODO: is this the fastest way to do this? - lists:merge(lists:sort(Results), Buffer). - --spec make_map_reduce_request(riak_client(), binary(), list()) -> - streaming_req_response(). -make_map_reduce_request(RcPid, ManifestBucketName, Keys) -> - BKeyTuples = make_bkeys(ManifestBucketName, Keys), - send_map_reduce_request(RcPid, BKeyTuples). - --spec make_bkeys(binary(), list()) -> list(). -make_bkeys(ManifestBucketName, Keys) -> - [{ManifestBucketName, Key} || Key <- Keys]. - --spec send_map_reduce_request(riak_client(), list()) -> streaming_req_response(). -send_map_reduce_request(RcPid, BKeyTuples) -> - {ok, ManifestPbc} = riak_cs_riak_client:manifest_pbc(RcPid), - Timeout = riak_cs_config:list_objects_timeout(), - riakc_pb_socket:mapred_stream(ManifestPbc, - BKeyTuples, - mapred_query(), - self(), - Timeout, - infinity). - --spec mapred_query() -> list(). -mapred_query() -> - [{map, {modfun, riak_cs_utils, map_keys_and_manifests}, - undefined, false}, - {reduce, {modfun, riak_cs_utils, reduce_keys_and_manifests}, - undefined, true}]. - --spec handle_map_reduce_call(streaming_req_response(), state()) -> - fsm_state_return(). -handle_map_reduce_call({ok, ReqID}, State) -> - State2 = State#state{map_red_req_id=ReqID}, - State3 = update_state_with_mr_start_profiling(State2), - {next_state, waiting_map_reduce, State3}; -handle_map_reduce_call({error, Reason}, State) -> - {stop, Reason, State}. - --spec enough_results(riak_cs_list_objects_utils:manifests_and_prefixes(), - list_object_request(), - non_neg_integer()) -> - boolean(). -enough_results(ManifestsAndPrefixes, - ?LOREQ{max_keys=MaxKeysRequested}, TotalCandidateKeys) -> - ResultsLength = - riak_cs_list_objects_utils:manifests_and_prefix_length(ManifestsAndPrefixes), - %% we have enough results if one of two things is true: - %% 1. we have more results than requested - %% 2. there are less keys than were requested even possible - %% (where 'possible' includes filtering for things like - %% `marker' and `prefix' - - %% add 1 to `MaxKeysRequested' because we need to know if there - %% are more active manifests after this key, so we can - %% correctly return `isTruncated' - ResultsLength >= erlang:min(MaxKeysRequested + 1, TotalCandidateKeys). - --spec could_query_more_mapreduce(list(), non_neg_integer()) -> boolean(). -could_query_more_mapreduce(_Requests, 0) -> - false; -could_query_more_mapreduce([], _TotalKeys) -> - true; -could_query_more_mapreduce(Requests, TotalKeys) -> - more_results_possible(lists:last(Requests), TotalKeys). - --spec more_results_possible({non_neg_integer(), non_neg_integer()}, - non_neg_integer()) -> boolean(). -more_results_possible({_StartIdx, EndIdx}, TotalKeys) - when EndIdx < TotalKeys -> - true; -more_results_possible(_Request, _TotalKeys) -> - false. - --spec have_enough_results(state()) -> fsm_state_return(). -have_enough_results(State=#state{reply_ref=undefined, - req=Request, - object_buffer=ObjectBuffer, - common_prefixes=CommonPrefixes}) -> - Response = make_response(Request, ObjectBuffer, CommonPrefixes), - NewStateData = State#state{response=Response}, - {next_state, done, NewStateData, timer:seconds(60)}; -have_enough_results(State=#state{reply_ref=ReplyPid, - req=Request, - object_buffer=ObjectBuffer, - common_prefixes=CommonPrefixes}) -> - Response = make_response(Request, ObjectBuffer, CommonPrefixes), - _ = gen_fsm:reply(ReplyPid, {ok, Response}), - NewStateData = State#state{response=Response}, - {stop, normal, NewStateData}. - --spec filtered_keys_from_request(list_object_request(), - list(binary()), - non_neg_integer()) -> - list(binary()). -filtered_keys_from_request(?LOREQ{marker=Marker, - prefix=Prefix}, KeyList, KeyListLength) -> - AfterMarker = maybe_filter_marker(Marker, KeyList, KeyListLength), - maybe_filter_prefix(Prefix, AfterMarker). - --spec maybe_filter_marker(undefined | binary(), - list(binary()), - non_neg_integer()) -> - list(binary()). -maybe_filter_marker(undefined, KeyList, _KeyListLength) -> - KeyList; -maybe_filter_marker(Marker, KeyList, KeyListLength) -> - filter_marker(Marker, KeyList, KeyListLength). - --spec maybe_filter_prefix(undefined | binary(), - list(binary())) -> - list(binary()). -maybe_filter_prefix(undefined, KeyList) -> - KeyList; -maybe_filter_prefix(Prefix, KeyList) -> - filter_prefix(Prefix, KeyList). - -filter_marker(Marker, KeyList, KeyListLength) -> - MarkerIndex = index_of_first_greater_element(KeyList, Marker), - lists:sublist(KeyList, MarkerIndex, KeyListLength). - -filter_prefix(Prefix, KeyList) -> - PrefixFun = build_prefix_fun(Prefix), - lists:filter(PrefixFun, KeyList). - -build_prefix_fun(Prefix) -> - PrefixLen = byte_size(Prefix), - fun(Elem) -> - case Elem of - <> -> - true; - _Else -> - false - end - end. - -%% only works for positive numbers --spec round_up(float()) -> integer(). -round_up(X) -> - erlang:round(X + 0.5). - -%% @doc Return the index (1-based) where -%% all list members are > than `Element'. -%% If `List' is empty, `1' -%% is returned. If `Element' is greater than all elements -%% in `List', then `length(List) + 1' is returned. -%% `List' must be sorted and contain only unique elements. --spec index_of_first_greater_element(list(non_neg_integer()), term()) -> - pos_integer(). -index_of_first_greater_element(List, Element) -> - index_of_first_greater_element_helper(List, Element, 1). - -index_of_first_greater_element_helper([], _Element, 1) -> - 1; -index_of_first_greater_element_helper([Fst], Element, Index) - when Element < Fst -> - Index; -index_of_first_greater_element_helper([_Fst], _Element, Index) -> - Index + 1; -index_of_first_greater_element_helper([Head | _Rest], Element, Index) - when Element < Head -> - Index; -index_of_first_greater_element_helper([_Head | Rest], Element, Index) -> - index_of_first_greater_element_helper(Rest, Element, Index + 1). - -%% Profiling helper functions -%%-------------------------------------------------------------------- - -update_state_with_mr_start_profiling(State=#state{req_profiles=Profiling, - mr_requests=MRRequests}) -> - {StartIdx, EndIdx}=LastReq = lists:last(MRRequests), - Start = riak_cs_utils:timestamp_to_seconds(os:timestamp()), - NumKeysRequested = EndIdx - StartIdx, - TempReq = {LastReq, NumKeysRequested, Start}, - Profiling2 = Profiling#profiling{temp_mr_req=TempReq}, - State#state{req_profiles=Profiling2}. - -update_state_with_mr_end_profiling(State=#state{req_profiles=Profiling}) -> - PrevMrRequests = Profiling#profiling.mr_requests, - {Req, NumKeys, Start} = Profiling#profiling.temp_mr_req, - EndTime = riak_cs_utils:timestamp_to_seconds(os:timestamp()), - CompletedProfile = {Req, NumKeys, {Start, EndTime}}, - NewMrRequests = PrevMrRequests ++ [CompletedProfile], - Profiling2 = Profiling#profiling{temp_mr_req=undefined, - mr_requests=NewMrRequests}, - State#state{req_profiles=Profiling2}. - -print_profiling(Profiling) -> - _ = lager:debug(format_list_keys_profile(Profiling)), - _ = lager:debug(format_map_reduce_profile(Profiling)). - -format_list_keys_profile(#profiling{list_keys_start_time=undefined, - list_keys_num_results=NumResults}) -> - io_lib:format("A cached list keys result of ~p keys was used", [NumResults]); -format_list_keys_profile(#profiling{list_keys_start_time=Start, - list_keys_end_time=End, - list_keys_num_results=NumResults}) -> - SecondsDiff = End - Start, - io_lib:format("List keys returned ~p keys in ~6.2f seconds", [NumResults, - SecondsDiff]). - -format_map_reduce_profile(#profiling{mr_requests=MRRequests}) -> - string:concat("Map Reduce timings: ", - format_map_reduce_profile_helper(MRRequests)). - -format_map_reduce_profile_helper(MRRequests) -> - string:join(lists:map(fun format_single_mr_profile/1, MRRequests), - "~n"). - -format_single_mr_profile({_Request, NumKeysRequested, {Start, End}}) -> - TimeDiff = End - Start, - io_lib:format("~p keys in ~6.2f seconds", [NumKeysRequested, TimeDiff]). diff --git a/src/riak_cs_manifest_utils.erl b/src/riak_cs_manifest_utils.erl deleted file mode 100644 index 4cd9bc302..000000000 --- a/src/riak_cs_manifest_utils.erl +++ /dev/null @@ -1,521 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc Module for choosing and manipulating lists (well, orddict) of manifests - --module(riak_cs_manifest_utils). - --include("riak_cs.hrl"). --ifdef(TEST). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). --endif. - -%% export Public API --export([new_dict/2, - active_manifest/1, - active_manifests/1, - active_and_writing_manifests/1, - overwritten_UUIDs/1, - deleted_while_writing/1, - mark_pending_delete/2, - mark_deleted/2, - mark_scheduled_delete/2, - manifests_to_gc/2, - prune/1, - prune/3, - upgrade_wrapped_manifests/1, - upgrade_manifest/1]). - -%%%=================================================================== -%%% API -%%%=================================================================== - -%% @doc Return a new orddict of manifest (only -%% one in this case). Used when storing something -%% in Riak when the previous GET returned notfound, -%% so we're (maybe) creating a new object. --spec new_dict(binary(), lfs_manifest()) -> orddict:orddict(). -new_dict(UUID, Manifest) -> - orddict:store(UUID, Manifest, orddict:new()). - -%% @doc Return the current active manifest -%% from an orddict of manifests. --spec active_manifest(orddict:orddict()) -> {ok, lfs_manifest()} | {error, no_active_manifest}. -active_manifest(Dict) -> - live_manifest(lists:foldl(fun most_recent_active_manifest/2, - {no_active_manifest, undefined}, orddict_values(Dict))). - -%% @doc Ensure the manifest hasn't been deleted during upload. --spec live_manifest(tuple()) -> {ok, lfs_manifest()} | {error, no_active_manifest}. -live_manifest({no_active_manifest, _}) -> - {error, no_active_manifest}; -live_manifest({Manifest, undefined}) -> - {ok, Manifest}; -live_manifest({Manifest, DeleteTime}) -> - case DeleteTime > Manifest?MANIFEST.write_start_time of - true -> - {error, no_active_manifest}; - false -> - {ok, Manifest} - end. - -%% @doc Return all active manifests --spec active_manifests(orddict:orddict()) -> [lfs_manifest()] | []. -active_manifests(Dict) -> - lists:filter(fun manifest_is_active/1, orddict_values(Dict)). - -%% @doc Return a list of all manifests in the -%% `active' or `writing' state --spec active_and_writing_manifests(orddict:orddict()) -> [{binary(), lfs_manifest()}]. -active_and_writing_manifests(Dict) -> - orddict:to_list(filter_manifests_by_state(Dict, [active, writing])). - -%% @doc Extract all manifests that are not "the most active" -%% and not actively writing (within the leeway period). --spec overwritten_UUIDs(orddict:orddict()) -> term(). -overwritten_UUIDs(Dict) -> - case active_manifest(Dict) of - {error, no_active_manifest} -> - []; - {ok, Active} -> - lists:foldl(overwritten_UUIDs_active_fold_helper(Active), - [], - orddict:to_list(Dict)) - end. - --spec overwritten_UUIDs_active_fold_helper(lfs_manifest()) -> - fun(({binary(), lfs_manifest()}, [binary()]) -> [binary()]). -overwritten_UUIDs_active_fold_helper(Active) -> - fun({UUID, Manifest}, Acc) -> - update_acc(UUID, Manifest, Acc, Active =:= Manifest) - end. - --spec update_acc(binary(), lfs_manifest(), [binary()], boolean()) -> - [binary()]. -update_acc(_UUID, _Manifest, Acc, true) -> - Acc; -update_acc(UUID, ?MANIFEST{state=active}, Acc, false) -> - [UUID | Acc]; -update_acc(UUID, Manifest=?MANIFEST{state=writing}, Acc, _) -> - LBWT = Manifest?MANIFEST.last_block_written_time, - WST = Manifest?MANIFEST.write_start_time, - acc_leeway_helper(UUID, Acc, LBWT, WST); -update_acc(_, _, Acc, _) -> - Acc. - --spec acc_leeway_helper(binary(), [binary()], - undefined | erlang:timestamp(), - undefined | erlang:timestamp()) -> - [binary()]. -acc_leeway_helper(UUID, Acc, undefined, WST) -> - acc_leeway_helper(UUID, Acc, WST); -acc_leeway_helper(UUID, Acc, LBWT, _) -> - acc_leeway_helper(UUID, Acc, LBWT). - --spec acc_leeway_helper(binary(), [binary()], undefined | erlang:timestamp()) -> - [binary()]. -acc_leeway_helper(UUID, Acc, Time) -> - handle_leeway_elaped_time(leeway_elapsed(Time), UUID, Acc). - --spec handle_leeway_elaped_time(boolean(), binary(), [binary()]) -> - [binary()]. -handle_leeway_elaped_time(true, UUID, Acc) -> - [UUID | Acc]; -handle_leeway_elaped_time(false, _UUID, Acc) -> - Acc. - -%% @doc Return `Dict' with the manifests in -%% `UUIDsToMark' with their state changed to -%% `pending_delete' and {deleted, true} added to props. --spec mark_deleted(orddict:orddict(), list(binary())) -> - orddict:orddict(). -mark_deleted(Dict, UUIDsToMark) -> - MapFun = fun(K, V) -> - case lists:member(K, UUIDsToMark) of - true -> - V?MANIFEST{state=pending_delete, - delete_marked_time=os:timestamp(), - props=[{deleted, true} | V?MANIFEST.props]}; - false -> - V - end - end, - orddict:map(MapFun, Dict). - -%% @doc Return `Dict' with the manifests in -%% `UUIDsToMark' with their state changed to -%% `pending_delete' --spec mark_pending_delete(orddict:orddict(), list(binary())) -> - orddict:orddict(). -mark_pending_delete(Dict, UUIDsToMark) -> - MapFun = fun(K, V) -> - case lists:member(K, UUIDsToMark) of - true -> - V?MANIFEST{state=pending_delete, - delete_marked_time=os:timestamp()}; - false -> - V - end - end, - orddict:map(MapFun, Dict). - -%% @doc Return `Dict' with the manifests in -%% `UUIDsToMark' with their state changed to -%% `scheduled_delete' --spec mark_scheduled_delete(orddict:orddict(), list(cs_uuid())) -> - orddict:orddict(). -mark_scheduled_delete(Dict, UUIDsToMark) -> - MapFun = fun(K, V) -> - case lists:member(K, UUIDsToMark) of - true -> - V?MANIFEST{state=scheduled_delete, - scheduled_delete_time=os:timestamp()}; - false -> - V - end - end, - orddict:map(MapFun, Dict). - -%% @doc Return a list of manifests that are either -%% in `PendingDeleteUUIDs' or are in the `pending_delete' -%% state and have been there for longer than the retry -%% interval. --spec manifests_to_gc([cs_uuid()], orddict:orddict()) -> [cs_uuid_and_manifest()]. -manifests_to_gc(PendingDeleteUUIDs, Manifests) -> - FilterFun = pending_delete_helper(PendingDeleteUUIDs), - orddict:to_list(orddict:filter(FilterFun, Manifests)). - -%% @private -%% Return a function for use in `orddict:filter/2' -%% that will return true if the manifest key is -%% in `UUIDs' or the manifest should be retried -%% moving to the GC bucket --spec pending_delete_helper([binary()]) -> - fun((binary(), lfs_manifest()) -> boolean()). -pending_delete_helper(UUIDs) -> - fun(Key, Manifest) -> - lists:member(Key, UUIDs) orelse retry_manifest(Manifest) - end. - -%% @private -%% Return true if this manifest should be retried -%% moving to the GC bucket --spec retry_manifest(lfs_manifest()) -> boolean(). -retry_manifest(?MANIFEST{state=pending_delete, - delete_marked_time=MarkedTime}) -> - retry_from_marked_time(MarkedTime, os:timestamp()); -retry_manifest(_Manifest) -> - false. - -%% @private -%% Return true if the time elapsed between -%% `MarkedTime' and `Now' is greater than -%% `riak_cs_gc:gc_retry_interval()'. --spec retry_from_marked_time(erlang:timestamp(), erlang:timestamp()) -> - boolean(). -retry_from_marked_time(MarkedTime, Now) -> - NowSeconds = riak_cs_utils:second_resolution_timestamp(Now), - MarkedTimeSeconds = riak_cs_utils:second_resolution_timestamp(MarkedTime), - NowSeconds > (MarkedTimeSeconds + riak_cs_gc:gc_retry_interval()). - -%% @doc Remove all manifests that require pruning, -%% see needs_pruning() for definition of needing pruning. --spec prune(orddict:orddict()) -> orddict:orddict(). -prune(Dict) -> - MaxCount = riak_cs_gc:max_scheduled_delete_manifests(), - prune(Dict, erlang:now(), MaxCount). - --spec prune(orddict:orddict(), - erlang:timestamp(), - unlimited | non_neg_integer()) -> orddict:orddict(). -prune(Dict, Time, MaxCount) -> - Filtered = orddict:filter(fun (_Key, Value) -> not needs_pruning(Value, Time) end, - Dict), - prune_count(Filtered, MaxCount). - --spec prune_count(orddict:orddict(), unlimited | non_neg_integer()) -> - orddict:orddict(). -prune_count(Manifests, unlimited) -> - Manifests; -prune_count(Manifests, MaxCount) -> - ScheduledDelete = filter_manifests_by_state(Manifests, [scheduled_delete]), - UUIDAndTime = [{M?MANIFEST.uuid, M?MANIFEST.scheduled_delete_time} || - {_UUID, M} <- ScheduledDelete], - case length(UUIDAndTime) > MaxCount of - true -> - SortedByTimeRecentFirst = lists:keysort(2, UUIDAndTime), - UUIDsToPrune = sets:from_list([UUID || {UUID, _ScheduledDeleteTime} <- - lists:nthtail(MaxCount, SortedByTimeRecentFirst)]), - orddict:filter(fun (UUID, _Value) -> not sets:is_element(UUID, UUIDsToPrune) end, - Manifests); - false -> - Manifests - end. - --spec upgrade_wrapped_manifests([orddict:orddict()]) -> [orddict:orddict()]. -upgrade_wrapped_manifests(ListofOrdDicts) -> - DictMapFun = fun(_Key, Value) -> upgrade_manifest(Value) end, - MapFun = fun(Value) -> orddict:map(DictMapFun, Value) end, - lists:map(MapFun, ListofOrdDicts). - -%% @doc Upgrade the manifest to the most recent -%% version of the manifest record. This is so that -%% _most_ of the codebase only has to deal with -%% the most recent version of the record. --spec upgrade_manifest(lfs_manifest() | #lfs_manifest_v2{}) -> lfs_manifest(). -upgrade_manifest(#lfs_manifest_v2{block_size=BlockSize, - bkey=Bkey, - metadata=Metadata, - created=Created, - uuid=UUID, - content_length=ContentLength, - content_type=ContentType, - content_md5=ContentMd5, - state=State, - write_start_time=WriteStartTime, - last_block_written_time=LastBlockWrittenTime, - write_blocks_remaining=WriteBlocksRemaining, - delete_marked_time=DeleteMarkedTime, - last_block_deleted_time=LastBlockDeletedTime, - delete_blocks_remaining=DeleteBlocksRemaining, - acl=Acl, - props=Properties, - cluster_id=ClusterID}) -> - - upgrade_manifest(?MANIFEST{block_size=BlockSize, - bkey=Bkey, - metadata=Metadata, - created=Created, - uuid=UUID, - content_length=ContentLength, - content_type=ContentType, - content_md5=ContentMd5, - state=State, - write_start_time=WriteStartTime, - last_block_written_time=LastBlockWrittenTime, - write_blocks_remaining=WriteBlocksRemaining, - delete_marked_time=DeleteMarkedTime, - last_block_deleted_time=LastBlockDeletedTime, - delete_blocks_remaining=DeleteBlocksRemaining, - acl=Acl, - props=Properties, - cluster_id=ClusterID}); - -upgrade_manifest(?MANIFEST{props=Props}=M) -> - M?MANIFEST{props=fixup_props(Props)}. - --spec fixup_props(undefined | list()) -> list(). -fixup_props(undefined) -> - []; -fixup_props(Props) when is_list(Props) -> - Props. - -%%%=================================================================== -%%% Internal functions -%%%=================================================================== - -%% @doc Filter an orddict manifests and accept only manifests whose -%% current state is specified in the `AcceptedStates' list. --spec filter_manifests_by_state(orddict:orddict(), [atom()]) -> orddict:orddict(). -filter_manifests_by_state(Dict, AcceptedStates) -> - AcceptManifest = - fun(_, ?MANIFEST{state=State}) -> - lists:member(State, AcceptedStates) - end, - orddict:filter(AcceptManifest, Dict). - --spec leeway_elapsed(undefined | erlang:timestamp()) -> boolean(). -leeway_elapsed(undefined) -> - false; -leeway_elapsed(Timestamp) -> - Now = riak_cs_utils:second_resolution_timestamp(os:timestamp()), - Now > (riak_cs_utils:second_resolution_timestamp(Timestamp) + riak_cs_gc:leeway_seconds()). - -orddict_values(OrdDict) -> - [V || {_K, V} <- orddict:to_list(OrdDict)]. - -manifest_is_active(?MANIFEST{state=active}) -> true; -manifest_is_active(_Manifest) -> false. - --spec delete_time(lfs_manifest()) -> erlang:timestamp() | undefined. -delete_time(Manifest) -> - case proplists:is_defined(deleted, Manifest?MANIFEST.props) of - true -> - Manifest?MANIFEST.delete_marked_time; - false -> - undefined - end. - -%% @doc Return all active manifests that have timestamps before the latest deletion -%% This happens when a manifest is still uploading while it is deleted. The upload -%% is allowed to complete, but is not visible afterwards. --spec deleted_while_writing(orddict:orddict()) -> [binary()]. -deleted_while_writing(Manifests) -> - ManifestList = orddict_values(Manifests), - DeleteTime = latest_delete_time(ManifestList), - find_deleted_active_manifests(ManifestList, DeleteTime). - --spec find_deleted_active_manifests([lfs_manifest()], term()) -> [cs_uuid()]. -find_deleted_active_manifests(_Manifests, undefined) -> - []; -find_deleted_active_manifests(Manifests, DeleteTime) -> - [M?MANIFEST.uuid || M <- Manifests, M?MANIFEST.state =:= active, - M?MANIFEST.write_start_time < DeleteTime]. - --spec latest_delete_time([lfs_manifest()]) -> term() | undefined. -latest_delete_time(Manifests) -> - lists:foldl(fun(M, Acc) -> - DeleteTime = delete_time(M), - later(DeleteTime, Acc) - end, undefined, Manifests). - -%% @doc Return the later of two times, accounting for undefined --spec later(undefined | erlang:timestamp(), undefined | erlang:timestamp()) -> - undefined | erlang:timestamp(). -later(undefined, undefined) -> - undefined; -later(undefined, DeleteTime2) -> - DeleteTime2; -later(DeleteTime1, undefined) -> - DeleteTime1; -later(DeleteTime1, DeleteTime2) -> - case DeleteTime1 > DeleteTime2 of - true -> - DeleteTime1; - false -> - DeleteTime2 - end. - -%% NOTE: This is a foldl function, initial acc = {no_active_manifest, undefined} -%% Return the most recent active manifest as well as the most recent manifest delete time --spec most_recent_active_manifest(lfs_manifest(), { - no_active_manifest | lfs_manifest(), undefined | erlang:timestamp()}) -> - {no_active_manifest | lfs_manifest(), erlang:timestamp() | undefined}. -most_recent_active_manifest(Manifest=?MANIFEST{state=scheduled_delete}, - {MostRecent, undefined}) -> - {MostRecent, delete_time(Manifest)}; -most_recent_active_manifest(Manifest=?MANIFEST{state=scheduled_delete}, - {MostRecent, DeleteTime}) -> - Dt=delete_time(Manifest), - {MostRecent, later(Dt, DeleteTime)}; -most_recent_active_manifest(Manifest=?MANIFEST{state=pending_delete}, - {MostRecent, undefined}) -> - {MostRecent, delete_time(Manifest)}; -most_recent_active_manifest(Manifest=?MANIFEST{state=pending_delete}, - {MostRecent, DeleteTime}) -> - Dt=delete_time(Manifest), - {MostRecent, later(Dt, DeleteTime)}; -most_recent_active_manifest(Manifest=?MANIFEST{state=active}, - {no_active_manifest, undefined}) -> - {Manifest, undefined}; -most_recent_active_manifest(Man1=?MANIFEST{state=active}, - {Man2=?MANIFEST{state=active}, DeleteTime}) - when Man1?MANIFEST.write_start_time > Man2?MANIFEST.write_start_time -> - {Man1, DeleteTime}; -most_recent_active_manifest(_Man1=?MANIFEST{state=active}, - {Man2=?MANIFEST{state=active}, DeleteTime}) -> - {Man2, DeleteTime}; -most_recent_active_manifest(Man1=?MANIFEST{state=active}, - {no_active_manifest, DeleteTime}) -> - {Man1, DeleteTime}; -most_recent_active_manifest(_Man1, {Man2=?MANIFEST{state=active}, DeleteTime}) -> - {Man2, DeleteTime}; -most_recent_active_manifest(_Manifest, {MostRecent, DeleteTime}) -> - {MostRecent, DeleteTime}. - --spec needs_pruning(lfs_manifest(), erlang:timestamp()) -> boolean(). -needs_pruning(?MANIFEST{state=scheduled_delete, - scheduled_delete_time=ScheduledDeleteTime}, Time) -> - seconds_diff(Time, ScheduledDeleteTime) > riak_cs_gc:leeway_seconds(); -needs_pruning(_Manifest, _Time) -> - false. - -seconds_diff(T2, T1) -> - TimeDiffMicrosends = timer:now_diff(T2, T1), - SecondsTime = TimeDiffMicrosends / (1000 * 1000), - erlang:trunc(SecondsTime). - -%% =================================================================== -%% EUnit tests -%% =================================================================== --ifdef(TEST). - -new_mani_helper() -> - riak_cs_lfs_utils:new_manifest(<<"bucket">>, - <<"key">>, - <<"uuid">>, - 100, %% content-length - <<"ctype">>, - undefined, %% md5 - orddict:new(), - 10, - undefined, - [], - undefined, - undefined). - -manifest_test_() -> - {setup, - fun setup/0, - fun cleanup/1, - [fun wrong_state_for_pruning/0, - fun wrong_state_for_pruning_2/0, - fun does_need_pruning/0, - fun not_old_enough_for_pruning/0] - }. - -setup() -> - ok. - -cleanup(_Ctx) -> - ok. - -wrong_state_for_pruning() -> - Mani = new_mani_helper(), - Mani2 = Mani?MANIFEST{state=active}, - ?assert(not needs_pruning(Mani2, erlang:now())). - -wrong_state_for_pruning_2() -> - Mani = new_mani_helper(), - Mani2 = Mani?MANIFEST{state=pending_delete}, - ?assert(not needs_pruning(Mani2, erlang:now())). - -does_need_pruning() -> - application:set_env(riak_cs, leeway_seconds, 1), - %% 1000000 second diff - ScheduledDeleteTime = {1333,985708,445136}, - Now = {1334,985708,445136}, - Mani = new_mani_helper(), - Mani2 = Mani?MANIFEST{state=scheduled_delete, - scheduled_delete_time=ScheduledDeleteTime}, - ?assert(needs_pruning(Mani2, Now)). - -not_old_enough_for_pruning() -> - application:set_env(riak_cs, leeway_seconds, 2), - %$ 1 second diff - ScheduledDeleteTime = {1333,985708,445136}, - Now = {1333,985709,445136}, - Mani = new_mani_helper(), - Mani2 = Mani?MANIFEST{state=scheduled_delete, - scheduled_delete_time=ScheduledDeleteTime}, - ?assert(not needs_pruning(Mani2, Now)). - --endif. diff --git a/src/riak_cs_s3_response.erl b/src/riak_cs_s3_response.erl deleted file mode 100644 index c4455d63f..000000000 --- a/src/riak_cs_s3_response.erl +++ /dev/null @@ -1,414 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(riak_cs_s3_response). --export([api_error/3, - status_code/1, - respond/3, - respond/4, - error_message/1, - error_code/1, - error_response/1, - copy_object_response/3, - copy_part_response/3, - no_such_upload_response/3, - invalid_digest_response/3, - error_code_to_atom/1, - xml_error_code/1]). - --include("riak_cs.hrl"). --include("riak_cs_api.hrl"). --include("list_objects.hrl"). --include_lib("webmachine/include/webmachine.hrl"). --include_lib("xmerl/include/xmerl.hrl"). - --type xmlElement() :: #xmlElement{}. - --type error_reason() :: atom() - | {'riak_connect_failed', term()} - | {'malformed_policy_version', string()} - | {'invalid_argument', string()} - | {'key_too_long', pos_integer()}. --spec error_message(error_reason()) -> string(). -error_message(invalid_access_key_id) -> - "The AWS Access Key Id you provided does not exist in our records."; -error_message(invalid_email_address) -> - "The email address you provided is not a valid."; -error_message(access_denied) -> - "Access Denied"; -error_message(copy_source_access_denied) -> - "Access Denied"; -error_message(reqtime_tooskewed) -> - "The difference between the request time and the current time is too large."; -error_message(bucket_not_empty) -> - "The bucket you tried to delete is not empty."; -error_message(bucket_already_exists) -> - "The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again."; -error_message(toomanybuckets) -> - "You have attempted to create more buckets than allowed"; -error_message({key_too_long, _}) -> - "Your key is too long"; -error_message(user_already_exists) -> - "The specified email address has already been registered. Email addresses must be unique among all users of the system. Please try again with a different email address."; -error_message(entity_too_large) -> - "Your proposed upload exceeds the maximum allowed object size."; -error_message(entity_too_small) -> - "Your proposed upload is smaller than the minimum allowed object size. Each part must be at least 5 MB in size, except the last part."; -error_message(invalid_user_update) -> - "The user update you requested was invalid."; -error_message(no_such_bucket) -> - "The specified bucket does not exist."; -error_message({riak_connect_failed, Reason}) -> - io_lib:format("Unable to establish connection to Riak. Reason: ~p", [Reason]); -error_message(admin_key_undefined) -> "Please reduce your request rate."; -error_message(admin_secret_undefined) -> "Please reduce your request rate."; -error_message(bucket_owner_unavailable) -> "The user record for the bucket owner was unavailable. Try again later."; -error_message(econnrefused) -> "Please reduce your request rate."; -error_message(malformed_policy_json) -> "JSON parsing error"; -error_message({malformed_policy_version, Version}) -> - io_lib:format("Document is invalid: Invalid Version ~s", [Version]); -error_message({auth_not_supported, AuthType}) -> - io_lib:format("The authorization mechanism you have provided (~s) is not supported.", [AuthType]); -error_message(malformed_policy_missing) -> "Policy is missing required element"; -error_message(malformed_policy_resource) -> "Policy has invalid resource"; -error_message(malformed_policy_principal) -> "Invalid principal in policy"; -error_message(malformed_policy_action) -> "Policy has invalid action"; -error_message(malformed_policy_condition) -> "Policy has invalid condition"; -error_message(no_such_key) -> "The specified key does not exist."; -error_message(no_copy_source_key) -> "The specified key does not exist."; -error_message(no_such_bucket_policy) -> "The specified bucket does not have a bucket policy."; -error_message(no_such_upload) -> - "The specified upload does not exist. The upload ID may be invalid, " - "or the upload may have been aborted or completed."; -error_message(invalid_digest) -> - "The Content-MD5 you specified was invalid."; -error_message(bad_request) -> "Bad Request"; -error_message(invalid_argument) -> "Invalid Argument"; -error_message({invalid_argument, "x-amz-metadata-directive"}) -> - "Unknown metadata directive."; -error_message(unresolved_grant_email) -> "The e-mail address you provided does not match any account on record."; -error_message(invalid_range) -> "The requested range is not satisfiable"; -error_message(invalid_bucket_name) -> "The specified bucket is not valid."; -error_message(invalid_part_number) -> "Part number must be an integer between 1 and 10000, inclusive"; -error_message(unexpected_content) -> "This request does not support content"; -error_message(canned_acl_and_header_grant) -> "Specifying both Canned ACLs and Header Grants is not allowed"; -error_message(malformed_xml) -> "The XML you provided was not well-formed or did not validate against our published schema"; -error_message(remaining_multipart_upload) -> "Concurrent multipart upload initiation detected. Please stop it to delete bucket."; -error_message(not_implemented) -> "A request you provided implies functionality that is not implemented"; -error_message(ErrorName) -> - _ = lager:debug("Unknown error: ~p", [ErrorName]), - "Please reduce your request rate.". - --spec error_code(error_reason()) -> string(). -error_code(invalid_access_key_id) -> "InvalidAccessKeyId"; -error_code(access_denied) -> "AccessDenied"; -error_code(copy_source_access_denied) -> "AccessDenied"; -error_code(reqtime_tooskewed) -> "RequestTimeTooSkewed"; -error_code(bucket_not_empty) -> "BucketNotEmpty"; -error_code(bucket_already_exists) -> "BucketAlreadyExists"; -error_code(toomanybuckets) -> "TooManyBuckets"; -error_code({key_too_long, _}) -> "KeyTooLongError"; -error_code(user_already_exists) -> "UserAlreadyExists"; -error_code(entity_too_large) -> "EntityTooLarge"; -error_code(entity_too_small) -> "EntityTooSmall"; -error_code(bad_etag) -> "InvalidPart"; -error_code(bad_etag_order) -> "InvalidPartOrder"; -error_code(invalid_user_update) -> "InvalidUserUpdate"; -error_code(no_such_bucket) -> "NoSuchBucket"; -error_code(no_such_key) -> "NoSuchKey"; -error_code(no_copy_source_key) -> "NoSuchKey"; -error_code({riak_connect_failed, _}) -> "RiakConnectFailed"; -error_code(admin_key_undefined) -> "ServiceUnavailable"; -error_code(admin_secret_undefined) -> "ServiceUnavailable"; -error_code(bucket_owner_unavailable) -> "ServiceUnavailable"; -error_code(econnrefused) -> "ServiceUnavailable"; -error_code(malformed_policy_json) -> "MalformedPolicy"; -error_code(malformed_policy_missing) -> "MalformedPolicy"; -error_code({malformed_policy_version, _}) -> "MalformedPolicy"; -error_code({auth_not_supported, _}) -> "InvalidRequest"; -error_code(malformed_policy_resource) -> "MalformedPolicy"; -error_code(malformed_policy_principal) -> "MalformedPolicy"; -error_code(malformed_policy_action) -> "MalformedPolicy"; -error_code(malformed_policy_condition) -> "MalformedPolicy"; -error_code(no_such_bucket_policy) -> "NoSuchBucketPolicy"; -error_code(no_such_upload) -> "NoSuchUpload"; -error_code(invalid_digest) -> "InvalidDigest"; -error_code(bad_request) -> "BadRequest"; -error_code(invalid_argument) -> "InvalidArgument"; -error_code(invalid_range) -> "InvalidRange"; -error_code(invalid_bucket_name) -> "InvalidBucketName"; -error_code(invalid_part_number) -> "InvalidArgument"; -error_code(unresolved_grant_email) -> "UnresolvableGrantByEmailAddress"; -error_code(unexpected_content) -> "UnexpectedContent"; -error_code(canned_acl_and_header_grant) -> "InvalidRequest"; -error_code(malformed_acl_error) -> "MalformedACLError"; -error_code(malformed_xml) -> "MalformedXML"; -error_code(remaining_multipart_upload) -> "MultipartUploadRemaining"; -error_code(not_implemented) -> "NotImplemented"; -error_code(ErrorName) -> - _ = lager:debug("Unknown error: ~p", [ErrorName]), - "ServiceUnavailable". - -%% These should match: -%% http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html - --spec status_code(error_reason()) -> pos_integer(). -status_code(invalid_access_key_id) -> 403; -status_code(invalid_email_address) -> 400; -status_code(access_denied) -> 403; -status_code(copy_source_access_denied) -> 403; -status_code(reqtime_tooskewed) -> 403; -status_code(bucket_not_empty) -> 409; -status_code(bucket_already_exists) -> 409; -status_code(user_already_exists) -> 409; -status_code(toomanybuckets) -> 400; -status_code({key_too_long, _}) -> 400; -%% yes, 400, really, not 413 -status_code(entity_too_large) -> 400; -status_code(entity_too_small) -> 400; -status_code(bad_etag) -> 400; -status_code(bad_etag_order) -> 400; -status_code(invalid_user_update) -> 400; -status_code(no_such_bucket) -> 404; -status_code(no_such_key) -> 404; -status_code(no_copy_source_key) -> 404; -status_code({riak_connect_failed, _}) -> 503; -status_code(admin_key_undefined) -> 503; -status_code(admin_secret_undefined) -> 503; -status_code(bucket_owner_unavailable) -> 503; -status_code(multiple_bucket_owners) -> 503; -status_code(econnrefused) -> 503; -status_code(unsatisfied_constraint) -> 503; -status_code(malformed_policy_json) -> 400; -status_code({malformed_policy_version, _}) -> 400; -status_code(malformed_policy_missing) -> 400; -status_code(malformed_policy_resource) -> 400; -status_code(malformed_policy_principal) -> 400; -status_code(malformed_policy_action) -> 400; -status_code(malformed_policy_condition) -> 400; -status_code({auth_not_supported, _}) -> 400; -status_code(no_such_bucket_policy) -> 404; -status_code(no_such_upload) -> 404; -status_code(invalid_digest) -> 400; -status_code(bad_request) -> 400; -status_code(invalid_argument) -> 400; -status_code(unresolved_grant_email) -> 400; -status_code(invalid_range) -> 416; -status_code(invalid_bucket_name) -> 400; -status_code(invalid_part_number) -> 400; -status_code(unexpected_content) -> 400; -status_code(canned_acl_and_header_grant) -> 400; -status_code(malformed_acl_error) -> 400; -status_code(malformed_xml) -> 400; -status_code(remaining_multipart_upload) -> 409; -status_code(not_implemented) -> 501; -status_code(ErrorName) -> - _ = lager:debug("Unknown error: ~p", [ErrorName]), - 503. - --spec respond(term(), #wm_reqdata{}, #context{}) -> - {binary(), #wm_reqdata{}, #context{}}. -respond(?LBRESP{}=Response, RD, Ctx) -> - {riak_cs_xml:to_xml(Response), RD, Ctx}; -respond({ok, ?LORESP{}=Response}, RD, Ctx) -> - {riak_cs_xml:to_xml(Response), RD, Ctx}; -respond({error, _}=Error, RD, Ctx) -> - api_error(Error, RD, Ctx). - -respond(404 = _StatusCode, Body, ReqData, Ctx) -> - respond({404, "Not Found"}, Body, ReqData, Ctx); -respond(StatusCode, Body, ReqData, Ctx) -> - UpdReqData = wrq:set_resp_body(Body, - wrq:set_resp_header("Content-Type", - ?XML_TYPE, - ReqData)), - {{halt, StatusCode}, UpdReqData, Ctx}. - -api_error(Error, RD, Ctx) when is_atom(Error) -> - error_response(Error, - status_code(Error), - error_code(Error), - error_message(Error), - RD, - Ctx); -api_error({Tag, _}=Error, RD, Ctx) - when Tag =:= riak_connect_failed orelse - Tag =:= malformed_policy_version orelse - Tag =:= auth_not_supported -> - error_response(Tag, - status_code(Error), - error_code(Error), - error_message(Error), - RD, - Ctx); -api_error({toomanybuckets, Current, BucketLimit}, RD, Ctx) -> - toomanybuckets_response(Current, BucketLimit, RD, Ctx); -api_error({invalid_argument, Name, Value}, RD, Ctx) -> - invalid_argument_response(Name, Value, RD, Ctx); -api_error({key_too_long, Len}, RD, Ctx) -> - key_too_long(Len, RD, Ctx); -api_error({error, Reason}, RD, Ctx) -> - api_error(Reason, RD, Ctx). - -error_response(ErrorDoc) when length(ErrorDoc) =:= 0 -> - {error, error_code_to_atom("BadRequest")}; -error_response(ErrorDoc) -> - {error, error_code_to_atom(xml_error_code(ErrorDoc))}. - -error_response(ErrorTag, StatusCode, Code, Message, RD, Ctx) -> - XmlDoc = [{'Error', [{'Code', [Code]}, - {'Message', [Message]}, - {'Resource', [error_resource(ErrorTag, RD)]}, - {'RequestId', [""]}]}], - respond(StatusCode, riak_cs_xml:to_xml(XmlDoc), RD, Ctx). - --spec error_resource(atom(), #wm_reqdata{}) -> iodata(). -error_resource(Tag, RD) - when Tag =:= no_copy_source_key; - Tag =:= copy_source_access_denied-> - {B, K} = riak_cs_copy_object:get_copy_source(RD), - <<$/, B/binary, $/, K/binary>>; -error_resource(_Tag, RD) -> - {OrigResource, _} = riak_cs_s3_rewrite:original_resource(RD), - OrigResource. - -toomanybuckets_response(Current, BucketLimit, RD, Ctx) -> - XmlDoc = {'Error', - [ - {'Code', [error_code(toomanybuckets)]}, - {'Message', [error_message(toomanybuckets)]}, - {'CurrentNumberOfBuckets', [Current]}, - {'AllowedNumberOfBuckets', [BucketLimit]} - ]}, - Body = riak_cs_xml:to_xml([XmlDoc]), - respond(status_code(toomanybuckets), Body, RD, Ctx). - -invalid_argument_response(Name, Value, RD, Ctx) -> - XmlDoc = {'Error', - [ - {'Code', [error_code(invalid_argument)]}, - {'Message', [error_message({invalid_argument, Name})]}, - {'ArgumentName', [Name]}, - {'ArgumentValue', [Value]}, - {'RequestId', [""]} - ]}, - Body = riak_cs_xml:to_xml([XmlDoc]), - respond(status_code(invalid_argument), Body, RD, Ctx). - -key_too_long(Len, RD, Ctx) -> - XmlDoc = {'Error', - [ - {'Code', [error_code({key_too_long, Len})]}, - {'Message', [error_message({key_too_long, Len})]}, - {'Size', [Len]}, - {'MaxSizeAllowed', [riak_cs_config:max_key_length()]}, - {'RequestId', [""]} - ]}, - Body = riak_cs_xml:to_xml([XmlDoc]), - respond(status_code(invalid_argument), Body, RD, Ctx). - -copy_object_response(Manifest, RD, Ctx) -> - copy_response(Manifest, 'CopyObjectResult', RD, Ctx). - -copy_part_response(Manifest, RD, Ctx) -> - copy_response(Manifest, 'CopyPartResult', RD, Ctx). - -copy_response(Manifest, TagName, RD, Ctx) -> - LastModified = riak_cs_wm_utils:to_iso_8601(Manifest?MANIFEST.created), - ETag = riak_cs_manifest:etag(Manifest), - XmlDoc = [{TagName, - [{'LastModified', [LastModified]}, - {'ETag', [ETag]}]}], - respond(200, riak_cs_xml:to_xml(XmlDoc), RD, Ctx). - - -no_such_upload_response(InternalUploadId, RD, Ctx) -> - UploadId = case InternalUploadId of - {raw, ReqUploadId} -> ReqUploadId; - _ -> base64url:encode(InternalUploadId) - end, - XmlDoc = {'Error', - [ - {'Code', [error_code(no_such_upload)]}, - {'Message', [error_message(no_such_upload)]}, - {'UploadId', [UploadId]}, - {'HostId', ["host-id"]} - ]}, - Body = riak_cs_xml:to_xml([XmlDoc]), - respond(status_code(no_such_upload), Body, RD, Ctx). - -invalid_digest_response(ContentMd5, RD, Ctx) -> - XmlDoc = {'Error', - [ - {'Code', [error_code(invalid_digest)]}, - {'Message', [error_message(invalid_digest)]}, - {'Content-MD5', [ContentMd5]}, - {'HostId', ["host-id"]} - ]}, - Body = riak_cs_xml:to_xml([XmlDoc]), - respond(status_code(invalid_digest), Body, RD, Ctx). - -%% @doc Convert an error code string into its corresponding atom --spec error_code_to_atom(string()) -> atom(). -error_code_to_atom(ErrorCode) -> - case ErrorCode of - "BadRequest" -> - bad_request; - "InvalidAccessKeyId" -> - invalid_access_key_id; - "AccessDenied" -> - access_denied; - "BucketNotEmpty" -> - bucket_not_empty; - "BucketAlreadyExists" -> - bucket_already_exists; - "UserAlreadyExists" -> - user_already_exists; - "NoSuchBucket" -> - no_such_bucket; - _ -> - unknown - end. - -%% @doc Get the value of the `Code' element from -%% and XML document. --spec xml_error_code(string()) -> string(). -xml_error_code(Xml) -> - %% here comes response from velvet (Stanchion), - %% this scan should match, otherwise bug. - {ok, ParsedData} = riak_cs_xml:scan(Xml), - process_xml_error(ParsedData#xmlElement.content). - -%% @doc Process the top-level elements of the --spec process_xml_error([xmlElement()]) -> string(). -process_xml_error([]) -> - []; -process_xml_error([#xmlText{value=" "}|Rest]) -> - process_xml_error(Rest); -process_xml_error([HeadElement | RestElements]) -> - _ = lager:debug("Element name: ~p", [HeadElement#xmlElement.name]), - ElementName = HeadElement#xmlElement.name, - case ElementName of - 'Code' -> - [Content] = HeadElement#xmlElement.content, - Content#xmlText.value; - _ -> - process_xml_error(RestElements) - end. diff --git a/src/riak_cs_s3_rewrite_legacy.erl b/src/riak_cs_s3_rewrite_legacy.erl deleted file mode 100644 index e239e772c..000000000 --- a/src/riak_cs_s3_rewrite_legacy.erl +++ /dev/null @@ -1,39 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(riak_cs_s3_rewrite_legacy). - --export([rewrite/5]). - --include("riak_cs.hrl"). - -%% @doc Function to rewrite headers prior to processing by webmachine. --spec rewrite(atom(), atom(), {integer(), integer()}, mochiweb_headers(), string()) -> - {mochiweb_headers(), string()}. -rewrite(Method, _Scheme, _Vsn, Headers, Url) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"rewrite">>), - %% Unquote the path to accomodate some naughty client libs (looking - %% at you Fog) - {Path, QueryString, _} = mochiweb_util:urlsplit_path(Url), - riak_cs_s3_rewrite:rewrite_path_and_headers(Method, - Headers, - Url, - mochiweb_util:unquote(Path), - QueryString). diff --git a/src/riak_cs_sup.erl b/src/riak_cs_sup.erl deleted file mode 100644 index 4b6bd8d91..000000000 --- a/src/riak_cs_sup.erl +++ /dev/null @@ -1,231 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc Supervisor for the riak_cs application. - --module(riak_cs_sup). - --behaviour(supervisor). - -%% Public API --export([start_link/0]). - -%% supervisor callbacks --export([init/1]). - --include("riak_cs.hrl"). - --define(OPTIONS, [connection_pools, - {listener, {"0.0.0.0", 80}}, - admin_listener, - ssl, - admin_ssl, - {rewrite_module, ?S3_API_MOD}]). - --type startlink_err() :: {'already_started', pid()} | 'shutdown' | term(). --type startlink_ret() :: {'ok', pid()} | 'ignore' | {'error', startlink_err()}. --type proplist() :: proplists:proplist(). - -%% =================================================================== -%% Public API -%% =================================================================== - -%% @doc API for starting the supervisor. --spec start_link() -> startlink_ret(). -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -%% @doc supervisor callback. --spec init([]) -> {ok, {{supervisor:strategy(), - non_neg_integer(), - non_neg_integer()}, - [supervisor:child_spec()]}}. -init([]) -> - catch dyntrace:p(), % NIF load trigger (R15B01+) - riak_cs_stats:init(), - Options = [get_option_val(Option) || Option <- ?OPTIONS], - PoolSpecs = pool_specs(Options), - {ok, { {one_for_one, 10, 10}, PoolSpecs ++ - process_specs() ++ - web_specs(Options)}}. - --spec process_specs() -> [supervisor:child_spec()]. -process_specs() -> - BagProcessSpecs = riak_cs_mb_helper:process_specs(), - Archiver = {riak_cs_access_archiver_manager, - {riak_cs_access_archiver_manager, start_link, []}, - permanent, 5000, worker, - [riak_cs_access_archiver_manager]}, - Storage = {riak_cs_storage_d, - {riak_cs_storage_d, start_link, []}, - permanent, 5000, worker, [riak_cs_storage_d]}, - GC = {riak_cs_gc_manager, - {riak_cs_gc_manager, start_link, []}, - permanent, 5000, worker, [riak_cs_gc_manager]}, - DeleteFsmSup = {riak_cs_delete_fsm_sup, - {riak_cs_delete_fsm_sup, start_link, []}, - permanent, 5000, worker, dynamic}, - ListObjectsETSCacheSup = {riak_cs_list_objects_ets_cache_sup, - {riak_cs_list_objects_ets_cache_sup, start_link, []}, - permanent, 5000, supervisor, dynamic}, - GetFsmSup = {riak_cs_get_fsm_sup, - {riak_cs_get_fsm_sup, start_link, []}, - permanent, 5000, worker, dynamic}, - PutFsmSup = {riak_cs_put_fsm_sup, - {riak_cs_put_fsm_sup, start_link, []}, - permanent, 5000, worker, dynamic}, - DiagsSup = {riak_cs_diags, {riak_cs_diags, start_link, []}, - permanent, 5000, worker, dynamic}, - QuotaSup = {riak_cs_quota_sup, - {riak_cs_quota_sup, start_link, []}, - permanent, 5000, supervisor, dynamic}, - BagProcessSpecs ++ - [Archiver, - Storage, - GC, - ListObjectsETSCacheSup, - DeleteFsmSup, - GetFsmSup, - PutFsmSup, - DiagsSup, - QuotaSup]. - --spec get_option_val({atom(), term()} | atom()) -> {atom(), term()}. -get_option_val({Option, Default}) -> - handle_get_env_result(Option, get_env(Option), Default); -get_option_val(Option) -> - get_option_val({Option, undefined}). - --spec get_env(atom()) -> 'undefined' | {ok, term()}. -get_env(Key) -> - application:get_env(riak_cs, Key). - --spec handle_get_env_result(atom(), {ok, term()} | 'undefined', term()) -> {atom(), term()}. -handle_get_env_result(Option, {ok, Value}, _) -> - {Option, Value}; -handle_get_env_result(Option, undefined, Default) -> - {Option, Default}. - --spec web_specs(proplist()) -> [supervisor:child_spec()]. -web_specs(Options) -> - WebConfigs = - case single_web_interface(proplists:get_value(admin_listener, Options)) of - true -> - [{object_web, add_admin_dispatch_table(object_web_config(Options))}]; - false -> - [{admin_web, admin_web_config(Options)}, - {object_web, object_web_config(Options)}] - end, - [web_spec(Name, Config) || {Name, Config} <- WebConfigs]. - --spec pool_specs(proplist()) -> [supervisor:child_spec()]. -pool_specs(Options) -> - rc_pool_specs(Options) ++ - pbc_pool_specs(Options). - -rc_pool_specs(Options) -> - WorkerStop = fun(Worker) -> riak_cs_riak_client:stop(Worker) end, - MasterPools = proplists:get_value(connection_pools, Options), - [{Name, - {poolboy, start_link, [[{name, {local, Name}}, - {worker_module, riak_cs_riak_client}, - {size, Workers}, - {max_overflow, Overflow}, - {stop_fun, WorkerStop}], - []]}, - permanent, 5000, worker, [poolboy]} || - {Name, {Workers, Overflow}} <- MasterPools]. - -pbc_pool_specs(Options) -> - WorkerStop = fun(Worker) -> riak_cs_riakc_pool_worker:stop(Worker) end, - %% Use sums of fixed/overflow for pbc pool - MasterPools = proplists:get_value(connection_pools, Options), - {FixedSum, OverflowSum} = lists:foldl(fun({_, {Fixed, Overflow}}, {FAcc, OAcc}) -> - {Fixed + FAcc, Overflow + OAcc} - end, - {0, 0}, MasterPools), - riak_cs_config:set_multibag_appenv(), - Bags = riak_cs_mb_helper:bags(), - [pbc_pool_spec(BagId, FixedSum, OverflowSum, Address, Port, WorkerStop) - || {BagId, Address, Port} <- Bags]. - --spec pbc_pool_spec(bag_id(), non_neg_integer(), non_neg_integer(), - string(), non_neg_integer(), function()) -> - supervisor:child_spec(). -pbc_pool_spec(BagId, Fixed, Overflow, Address, Port, WorkerStop) -> - Name = riak_cs_riak_client:pbc_pool_name(BagId), - {Name, - {poolboy, start_link, [[{name, {local, Name}}, - {worker_module, riak_cs_riakc_pool_worker}, - {size, Fixed}, - {max_overflow, Overflow}, - {stop_fun, WorkerStop}], - [{address, Address}, {port, Port}]]}, - permanent, 5000, worker, [poolboy]}. - --spec web_spec(atom(), proplist()) -> supervisor:child_spec(). -web_spec(Name, Config) -> - {Name, - {webmachine_mochiweb, start, [Config]}, - permanent, 5000, worker, dynamic}. - --spec object_web_config(proplist()) -> proplist(). -object_web_config(Options) -> - {IP, Port} = proplists:get_value(listener, Options), - [{dispatch, riak_cs_web:object_api_dispatch_table()}, - {name, object_web}, - {dispatch_group, object_web}, - {ip, IP}, - {port, Port}, - {nodelay, true}, - {rewrite_module, proplists:get_value(rewrite_module, Options)}, - {error_handler, riak_cs_wm_error_handler}, - {resource_module_option, submodule}] ++ - maybe_add_ssl_opts(proplists:get_value(ssl, Options)). - --spec admin_web_config(proplist()) -> proplist(). -admin_web_config(Options) -> - {IP, Port} = proplists:get_value(admin_listener, - Options, {"127.0.0.1", 8000}), - [{dispatch, riak_cs_web:admin_api_dispatch_table()}, - {name, admin_web}, - {dispatch_group, admin_web}, - {ip, IP}, {port, Port}, - {nodelay, true}, - {error_handler, riak_cs_wm_error_handler}] ++ - maybe_add_ssl_opts(proplists:get_value(admin_ssl, Options)). - --spec single_web_interface('undefined' | term()) -> boolean(). -single_web_interface(undefined) -> - true; -single_web_interface(_) -> - false. - --spec maybe_add_ssl_opts('undefined' | proplist()) -> proplist(). -maybe_add_ssl_opts(undefined) -> - []; -maybe_add_ssl_opts(SSLOpts) -> - [{ssl, true}, {ssl_opts, SSLOpts}]. - --spec add_admin_dispatch_table(proplist()) -> proplist(). -add_admin_dispatch_table(Config) -> - UpdDispatchTable = proplists:get_value(dispatch, Config) ++ - riak_cs_web:admin_api_dispatch_table(), - [{dispatch, UpdDispatchTable} | proplists:delete(dispatch, Config)]. diff --git a/src/riak_cs_user.erl b/src/riak_cs_user.erl deleted file mode 100644 index 9f68cd9a3..000000000 --- a/src/riak_cs_user.erl +++ /dev/null @@ -1,341 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc riak_cs user related functions - --module(riak_cs_user). - -%% Public API --export([ - create_user/2, - create_user/4, - display_name/1, - is_admin/1, - get_user/2, - get_user_by_index/3, - from_riakc_obj/2, - to_3tuple/1, - save_user/3, - update_key_secret/1, - update_user/3, - key_id/1, - fetch_user_keys/1 - ]). - --include("riak_cs.hrl"). --include_lib("riakc/include/riakc.hrl"). - --ifdef(TEST). --compile(export_all). --endif. - -%% =================================================================== -%% Public API -%% =================================================================== - -%% @doc Create a new Riak CS user --spec create_user(string(), string()) -> {ok, rcs_user()} | {error, term()}. -create_user(Name, Email) -> - {KeyId, Secret} = generate_access_creds(Email), - create_user(Name, Email, KeyId, Secret). - -%% @doc Create a new Riak CS user --spec create_user(string(), string(), string(), string()) -> {ok, rcs_user()} | {error, term()}. -create_user(Name, Email, KeyId, Secret) -> - case validate_email(Email) of - ok -> - CanonicalId = generate_canonical_id(KeyId), - User = user_record(Name, Email, KeyId, Secret, CanonicalId), - create_credentialed_user(riak_cs_config:admin_creds(), User); - {error, _Reason}=Error -> - Error - end. - --spec create_credentialed_user({ok, {term(), term()}}, rcs_user()) -> - {ok, rcs_user()} | {error, term()}. -create_credentialed_user({ok, AdminCreds}, User) -> - {StIp, StPort, StSSL} = riak_cs_utils:stanchion_data(), - %% Make a call to the user request serialization service. - StatsKey = [velvet, create_user], - _ = riak_cs_stats:inflow(StatsKey), - StartTime = os:timestamp(), - Result = velvet:create_user(StIp, - StPort, - "application/json", - binary_to_list(riak_cs_json:to_json(User)), - [{ssl, StSSL}, {auth_creds, AdminCreds}]), - _ = riak_cs_stats:update_with_start(StatsKey, StartTime, Result), - handle_create_user(Result, User). - -handle_create_user(ok, User) -> - {ok, User}; -handle_create_user({error, {error_status, _, _, ErrorDoc}}, _User) -> - case riak_cs_config:api() of - s3 -> - riak_cs_s3_response:error_response(ErrorDoc); - oos -> - {error, ErrorDoc} - end; -handle_create_user({error, _}=Error, _User) -> - Error. - -handle_update_user(ok, User, UserObj, RcPid) -> - _ = save_user(User, UserObj, RcPid), - {ok, User}; -handle_update_user({error, {error_status, _, _, ErrorDoc}}, _User, _, _) -> - case riak_cs_config:api() of - s3 -> - riak_cs_s3_response:error_response(ErrorDoc); - oos -> - {error, ErrorDoc} - end; -handle_update_user({error, _}=Error, _User, _, _) -> - Error. - -%% @doc Update a Riak CS user record --spec update_user(rcs_user(), riakc_obj:riakc_obj(), riak_client()) -> - {ok, rcs_user()} | {error, term()}. -update_user(User, UserObj, RcPid) -> - {StIp, StPort, StSSL} = riak_cs_utils:stanchion_data(), - {ok, AdminCreds} = riak_cs_config:admin_creds(), - Options = [{ssl, StSSL}, {auth_creds, AdminCreds}], - StatsKey = [velvet, update_user], - _ = riak_cs_stats:inflow(StatsKey), - StartTime = os:timestamp(), - %% Make a call to the user request serialization service. - Result = velvet:update_user(StIp, - StPort, - "application/json", - User?RCS_USER.key_id, - binary_to_list(riak_cs_json:to_json(User)), - Options), - _ = riak_cs_stats:update_with_start(StatsKey, StartTime, Result), - handle_update_user(Result, User, UserObj, RcPid). - -%% @doc Retrieve a Riak CS user's information based on their id string. --spec get_user('undefined' | list(), riak_client()) -> {ok, {rcs_user(), riakc_obj:riakc_obj()}} | {error, term()}. -get_user(undefined, _RcPid) -> - {error, no_user_key}; -get_user(KeyId, RcPid) -> - %% Check for and resolve siblings to get a - %% coherent view of the bucket ownership. - BinKey = list_to_binary(KeyId), - case riak_cs_riak_client:get_user(RcPid, BinKey) of - {ok, {Obj, KeepDeletedBuckets}} -> - {ok, {from_riakc_obj(Obj, KeepDeletedBuckets), Obj}}; - Error -> - Error - end. - --spec from_riakc_obj(riakc_obj:riakc_obj(), boolean()) -> rcs_user(). -from_riakc_obj(Obj, KeepDeletedBuckets) -> - case riakc_obj:value_count(Obj) of - 1 -> - Value = binary_to_term(riakc_obj:get_value(Obj)), - User = update_user_record(Value), - Buckets = riak_cs_bucket:resolve_buckets([Value], [], KeepDeletedBuckets), - User?RCS_USER{buckets=Buckets}; - 0 -> - error(no_value); - N -> - Values = [binary_to_term(Value) || - Value <- riakc_obj:get_values(Obj), - Value /= <<>> % tombstone - ], - User = update_user_record(hd(Values)), - - KeyId = User?RCS_USER.key_id, - _ = lager:warning("User object of '~s' has ~p siblings", [KeyId, N]), - - Buckets = riak_cs_bucket:resolve_buckets(Values, [], KeepDeletedBuckets), - User?RCS_USER{buckets=Buckets} - end. - -%% @doc Retrieve a Riak CS user's information based on their -%% canonical id string or email. -%% @TODO May want to use mapreduce job for this. --spec get_user_by_index(binary(), binary(), riak_client()) -> - {ok, {rcs_user(), term()}} | - {error, term()}. -get_user_by_index(Index, Value, RcPid) -> - case get_user_index(Index, Value, RcPid) of - {ok, KeyId} -> - get_user(KeyId, RcPid); - {error, _}=Error1 -> - Error1 - end. - -%% @doc Query `Index' for `Value' in the users bucket. --spec get_user_index(binary(), binary(), riak_client()) -> {ok, string()} | {error, term()}. -get_user_index(Index, Value, RcPid) -> - {ok, MasterPbc} = riak_cs_riak_client:master_pbc(RcPid), - %% TODO: Does adding max_results=1 help latency or load to riak cluster? - case riak_cs_pbc:get_index_eq(MasterPbc, ?USER_BUCKET, Index, Value, - [riakc, get_user_by_index]) of - {ok, ?INDEX_RESULTS{keys=[]}} -> - {error, notfound}; - {ok, ?INDEX_RESULTS{keys=[Key | _]}} -> - {ok, binary_to_list(Key)}; - {error, Reason}=Error -> - _ = lager:warning("Error occurred trying to query ~p in user" - "index ~p. Reason: ~p", - [Value, Index, Reason]), - Error - end. - -%% @doc Determine if the specified user account is a system admin. --spec is_admin(rcs_user()) -> boolean(). -is_admin(User) -> - is_admin(User, riak_cs_config:admin_creds()). - --spec to_3tuple(rcs_user()) -> acl_owner(). -to_3tuple(U) -> - %% acl_owner3: {display name, canonical id, key id} - {U?RCS_USER.display_name, U?RCS_USER.canonical_id, - U?RCS_USER.key_id}. - -%% @doc Save information about a Riak CS user --spec save_user(rcs_user(), riakc_obj:riakc_obj(), riak_client()) -> ok | {error, term()}. -save_user(User, UserObj, RcPid) -> - riak_cs_riak_client:save_user(RcPid, User, UserObj). - - -%% @doc Generate a new `key_secret' for a user record. --spec update_key_secret(rcs_user()) -> rcs_user(). -update_key_secret(User=?RCS_USER{email=Email, - key_id=KeyId}) -> - EmailBin = list_to_binary(Email), - User?RCS_USER{key_secret=generate_secret(EmailBin, KeyId)}. - -%% @doc Strip off the user name portion of an email address --spec display_name(string()) -> string(). -display_name(Email) -> - Index = string:chr(Email, $@), - string:sub_string(Email, 1, Index-1). - -%% @doc Grab the whole list of Riak CS user keys. --spec fetch_user_keys(riak_client()) -> {ok, [binary()]} | {error, term()}. -fetch_user_keys(RcPid) -> - {ok, MasterPbc} = riak_cs_riak_client:master_pbc(RcPid), - Timeout = riak_cs_config:list_keys_list_users_timeout(), - riak_cs_pbc:list_keys(MasterPbc, ?USER_BUCKET, Timeout, - [riakc, list_all_user_keys]). - -%% =================================================================== -%% Internal functions -%% =================================================================== - -%% @doc Generate a new set of access credentials for user. --spec generate_access_creds(string()) -> {iodata(), iodata()}. -generate_access_creds(UserId) -> - UserBin = list_to_binary(UserId), - KeyId = generate_key(UserBin), - Secret = generate_secret(UserBin, KeyId), - {KeyId, Secret}. - -%% @doc Generate the canonical id for a user. --spec generate_canonical_id(string()) -> string(). -generate_canonical_id(KeyID) -> - Bytes = 16, - Id1 = riak_cs_utils:md5(KeyID), - Id2 = riak_cs_utils:md5(druuid:v4()), - riak_cs_utils:binary_to_hexlist( - iolist_to_binary(<< Id1:Bytes/binary, - Id2:Bytes/binary >>)). - -%% @doc Generate an access key for a user --spec generate_key(binary()) -> [byte()]. -generate_key(UserName) -> - Ctx = crypto:hmac_init(sha, UserName), - Ctx1 = crypto:hmac_update(Ctx, druuid:v4()), - Key = crypto:hmac_final_n(Ctx1, 15), - string:to_upper(base64url:encode_to_string(Key)). - -%% @doc Generate a secret access token for a user --spec generate_secret(binary(), string()) -> iodata(). -generate_secret(UserName, Key) -> - Bytes = 14, - Ctx = crypto:hmac_init(sha, UserName), - Ctx1 = crypto:hmac_update(Ctx, list_to_binary(Key)), - SecretPart1 = crypto:hmac_final_n(Ctx1, Bytes), - Ctx2 = crypto:hmac_init(sha, UserName), - Ctx3 = crypto:hmac_update(Ctx2, druuid:v4()), - SecretPart2 = crypto:hmac_final_n(Ctx3, Bytes), - base64url:encode_to_string( - iolist_to_binary(<< SecretPart1:Bytes/binary, - SecretPart2:Bytes/binary >>)). - -%% @doc Determine if the specified user account is a system admin. --spec is_admin(rcs_user(), {ok, {string(), string()}} | - {error, term()}) -> boolean(). -is_admin(?RCS_USER{key_id=KeyId, key_secret=KeySecret}, - {ok, {KeyId, KeySecret}}) -> - true; -is_admin(_, _) -> - false. - -%% @doc Validate an email address. --spec validate_email(string()) -> ok | {error, term()}. -validate_email(EmailAddr) -> - %% @TODO More robust email address validation - case string:chr(EmailAddr, $@) of - 0 -> - {error, invalid_email_address}; - _ -> - ok - end. - -%% @doc Update a user record from a previous version if necessary. --spec update_user_record(rcs_user()) -> rcs_user(). -update_user_record(User=?RCS_USER{buckets=Buckets}) -> - User?RCS_USER{buckets=[riak_cs_bucket:update_bucket_record(Bucket) || - Bucket <- Buckets]}; -update_user_record(User=#moss_user_v1{}) -> - ?RCS_USER{name=User#moss_user_v1.name, - display_name=User#moss_user_v1.display_name, - email=User#moss_user_v1.email, - key_id=User#moss_user_v1.key_id, - key_secret=User#moss_user_v1.key_secret, - canonical_id=User#moss_user_v1.canonical_id, - buckets=[riak_cs_bucket:update_bucket_record(Bucket) || - Bucket <- User#moss_user_v1.buckets]}. - -%% @doc Return a user record for the specified user name and -%% email address. --spec user_record(string(), string(), string(), string(), string()) -> rcs_user(). -user_record(Name, Email, KeyId, Secret, CanonicalId) -> - user_record(Name, Email, KeyId, Secret, CanonicalId, []). - -%% @doc Return a user record for the specified user name and -%% email address. --spec user_record(string(), string(), string(), string(), string(), [cs_bucket()]) -> - rcs_user(). -user_record(Name, Email, KeyId, Secret, CanonicalId, Buckets) -> - DisplayName = display_name(Email), - ?RCS_USER{name=Name, - display_name=DisplayName, - email=Email, - key_id=KeyId, - key_secret=Secret, - canonical_id=CanonicalId, - buckets=Buckets}. - -key_id(?RCS_USER{key_id=KeyId}) -> - KeyId. diff --git a/src/riak_cs_wm_bucket.erl b/src/riak_cs_wm_bucket.erl deleted file mode 100644 index 76433997e..000000000 --- a/src/riak_cs_wm_bucket.erl +++ /dev/null @@ -1,170 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(riak_cs_wm_bucket). - --export([stats_prefix/0, - content_types_provided/2, - to_xml/2, - allowed_methods/0, - malformed_request/2, - content_types_accepted/2, - accept_body/2, - delete_resource/2, - authorize/2]). - --include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). - --spec stats_prefix() -> bucket. -stats_prefix() -> bucket. - -%% @doc Get the list of methods this resource supports. --spec allowed_methods() -> [atom()]. -allowed_methods() -> - ['HEAD', 'PUT', 'DELETE']. - --spec malformed_request(#wm_reqdata{}, #context{}) -> {false, #wm_reqdata{}, #context{}}. -malformed_request(RD, Ctx) -> - case riak_cs_wm_utils:has_canned_acl_and_header_grant(RD) of - true -> - riak_cs_s3_response:api_error(canned_acl_and_header_grant, - RD, Ctx); - false -> - {false, RD, Ctx} - end. - --spec content_types_provided(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_provided(RD, Ctx) -> - {[{"application/xml", to_xml}], RD, Ctx}. - --spec content_types_accepted(#wm_reqdata{}, #context{}) -> - {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_accepted(RD, Ctx) -> - content_types_accepted(wrq:get_req_header("content-type", RD), RD, Ctx). - --spec content_types_accepted(undefined | string(), #wm_reqdata{}, #context{}) -> - {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_accepted(CT, RD, Ctx) when CT =:= undefined; - CT =:= [] -> - content_types_accepted("application/octet-stream", RD, Ctx); -content_types_accepted(CT, RD, Ctx) -> - {Media, _Params} = mochiweb_util:parse_header(CT), - {[{Media, add_acl_to_context_then_accept}], RD, Ctx}. - --spec authorize(#wm_reqdata{}, #context{}) -> {boolean(), #wm_reqdata{}, #context{}}. -authorize(RD, #context{user=User}=Ctx) -> - Method = wrq:method(RD), - RequestedAccess = - riak_cs_acl_utils:requested_access(Method, false), - Bucket = list_to_binary(wrq:path_info(bucket, RD)), - PermCtx = Ctx#context{bucket=Bucket, - requested_perm=RequestedAccess}, - - case {Method, RequestedAccess} of - {_, 'WRITE'} when User == undefined -> - %% unauthed users may neither create nor delete buckets - riak_cs_wm_utils:deny_access(RD, PermCtx); - {'PUT', 'WRITE'} -> - %% authed users are always allowed to attempt bucket creation - AccessRD = riak_cs_access_log_handler:set_user(User, RD), - {false, AccessRD, PermCtx}; - _ -> - riak_cs_wm_utils:bucket_access_authorize_helper(bucket, true, RD, Ctx) - end. - - --spec to_xml(#wm_reqdata{}, #context{}) -> - {binary() | {'halt', term()}, #wm_reqdata{}, #context{}}. -to_xml(RD, Ctx) -> - handle_read_request(RD, Ctx). - -%% @private -handle_read_request(RD, Ctx=#context{user=User, - bucket=Bucket}) -> - riak_cs_dtrace:dt_bucket_entry(?MODULE, <<"bucket_head">>, - [], [riak_cs_wm_utils:extract_name(User), Bucket]), - %% override the content-type on HEAD - HeadRD = wrq:set_resp_header("content-type", "text/html", RD), - StrBucket = binary_to_list(Bucket), - case [B || B <- riak_cs_bucket:get_buckets(User), - B?RCS_BUCKET.name =:= StrBucket] of - [] -> - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_head">>, - [404], [riak_cs_wm_utils:extract_name(User), Bucket]), - {{halt, 404}, HeadRD, Ctx}; - [_BucketRecord] -> - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_head">>, - [200], [riak_cs_wm_utils:extract_name(User), Bucket]), - {{halt, 200}, HeadRD, Ctx} - end. - -%% @doc Process request body on `PUT' request. --spec accept_body(#wm_reqdata{}, #context{}) -> {{halt, integer()}, #wm_reqdata{}, #context{}}. -accept_body(RD, Ctx=#context{user=User, - acl=ACL, - user_object=UserObj, - bucket=Bucket, - response_module=ResponseMod, - riak_client=RcPid}) -> - riak_cs_dtrace:dt_bucket_entry(?MODULE, <<"bucket_put">>, - [], [riak_cs_wm_utils:extract_name(User), Bucket]), - BagId = riak_cs_mb_helper:choose_bag_id(manifest, Bucket), - case riak_cs_bucket:create_bucket(User, - UserObj, - Bucket, - BagId, - ACL, - RcPid) of - ok -> - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_put">>, - [200], [riak_cs_wm_utils:extract_name(User), Bucket]), - {{halt, 200}, RD, Ctx}; - {error, Reason} -> - Code = ResponseMod:status_code(Reason), - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_put">>, - [Code], [riak_cs_wm_utils:extract_name(User), Bucket]), - ResponseMod:api_error(Reason, RD, Ctx) - end. - -%% @doc Callback for deleting a bucket. --spec delete_resource(#wm_reqdata{}, #context{}) -> - {boolean() | {'halt', term()}, #wm_reqdata{}, #context{}}. -delete_resource(RD, Ctx=#context{user=User, - user_object=UserObj, - response_module=ResponseMod, - bucket=Bucket, - riak_client=RcPid}) -> - riak_cs_dtrace:dt_bucket_entry(?MODULE, <<"bucket_delete">>, - [], [riak_cs_wm_utils:extract_name(User), Bucket]), - case riak_cs_bucket:delete_bucket(User, - UserObj, - Bucket, - RcPid) of - ok -> - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_delete">>, - [200], [riak_cs_wm_utils:extract_name(User), Bucket]), - {true, RD, Ctx}; - {error, Reason} -> - Code = ResponseMod:status_code(Reason), - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_delete">>, - [Code], [riak_cs_wm_utils:extract_name(User), Bucket]), - ResponseMod:api_error(Reason, RD, Ctx) - end. diff --git a/src/riak_cs_wm_bucket_acl.erl b/src/riak_cs_wm_bucket_acl.erl deleted file mode 100644 index 2e52d80c1..000000000 --- a/src/riak_cs_wm_bucket_acl.erl +++ /dev/null @@ -1,141 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(riak_cs_wm_bucket_acl). - --export([stats_prefix/0, - content_types_provided/2, - to_xml/2, - allowed_methods/0, - malformed_request/2, - content_types_accepted/2, - accept_body/2]). - --export([authorize/2]). - -%% TODO: DELETE? - --include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). - - --spec stats_prefix() -> bucket_acl. -stats_prefix() -> bucket_acl. - -%% @doc Get the list of methods this resource supports. --spec allowed_methods() -> [atom()]. -allowed_methods() -> - ['GET', 'PUT']. - --spec malformed_request(#wm_reqdata{}, #context{}) -> {false, #wm_reqdata{}, #context{}}. -malformed_request(RD, Ctx) -> - case riak_cs_wm_utils:has_acl_header_and_body(RD) of - true -> - riak_cs_s3_response:api_error(unexpected_content, - RD, Ctx); - false -> - {false, RD, Ctx} - end. - --spec content_types_provided(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_provided(RD, Ctx) -> - {[{"application/xml", to_xml}], RD, Ctx}. - --spec content_types_accepted(#wm_reqdata{}, #context{}) -> - {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_accepted(RD, Ctx) -> - case wrq:get_req_header("content-type", RD) of - undefined -> - {[{"application/octet-stream", add_acl_to_context_then_accept}], RD, Ctx}; - CType -> - {Media, _Params} = mochiweb_util:parse_header(CType), - {[{Media, add_acl_to_context_then_accept}], RD, Ctx} - end. - --spec authorize(#wm_reqdata{}, #context{}) -> {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #context{}}. -authorize(RD, Ctx) -> - riak_cs_wm_utils:bucket_access_authorize_helper(bucket_acl, true, RD, Ctx). - - --spec to_xml(#wm_reqdata{}, #context{}) -> - {binary() | {'halt', non_neg_integer()}, #wm_reqdata{}, #context{}}. -to_xml(RD, Ctx=#context{user=User, - bucket=Bucket, - riak_client=RcPid}) -> - riak_cs_dtrace:dt_bucket_entry(?MODULE, <<"bucket_get_acl">>, - [], [riak_cs_wm_utils:extract_name(User), Bucket]), - case riak_cs_acl:fetch_bucket_acl(Bucket, RcPid) of - {ok, Acl} -> - X = {riak_cs_xml:to_xml(Acl), RD, Ctx}, - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_acl_get">>, - [200], [riak_cs_wm_utils:extract_name(User), Bucket]), - X; - {error, Reason} -> - Code = riak_cs_s3_response:status_code(Reason), - X = riak_cs_s3_response:api_error(Reason, RD, Ctx), - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_acl">>, - [Code], [riak_cs_wm_utils:extract_name(User), Bucket]), - X - end. - -%% @doc Process request body on `PUT' request. --spec accept_body(#wm_reqdata{}, #context{}) -> {{halt, non_neg_integer()}, #wm_reqdata{}, #context{}}. -accept_body(RD, Ctx=#context{user=User, - user_object=UserObj, - acl=AclFromHeadersOrDefault, - bucket=Bucket, - riak_client=RcPid}) -> - riak_cs_dtrace:dt_bucket_entry(?MODULE, <<"bucket_put_acl">>, - [], [riak_cs_wm_utils:extract_name(User), Bucket]), - Body = binary_to_list(wrq:req_body(RD)), - AclRes = - case Body of - [] -> - {ok, AclFromHeadersOrDefault}; - _ -> - riak_cs_acl_utils:validate_acl( - riak_cs_acl_utils:acl_from_xml(Body, - User?RCS_USER.key_id, - RcPid), - User?RCS_USER.canonical_id) - end, - case AclRes of - {ok, ACL} -> - case riak_cs_bucket:set_bucket_acl(User, - UserObj, - Bucket, - ACL, - RcPid) of - ok -> - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_put_acl">>, - [200], [riak_cs_wm_utils:extract_name(User), Bucket]), - {{halt, 200}, RD, Ctx}; - {error, Reason} -> - Code = riak_cs_s3_response:status_code(Reason), - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_put_acl">>, - [Code], [riak_cs_wm_utils:extract_name(User), Bucket]), - riak_cs_s3_response:api_error(Reason, RD, Ctx) - end; - {error, Reason2} -> - Code = riak_cs_s3_response:status_code(Reason2), - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_put_acl">>, - [Code], [riak_cs_wm_utils:extract_name(User), Bucket]), - riak_cs_s3_response:api_error(Reason2, RD, Ctx) - end. diff --git a/src/riak_cs_wm_bucket_delete.erl b/src/riak_cs_wm_bucket_delete.erl deleted file mode 100644 index 2594b86aa..000000000 --- a/src/riak_cs_wm_bucket_delete.erl +++ /dev/null @@ -1,187 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc WM resouce for Delete Multiple Objects - --module(riak_cs_wm_bucket_delete). - --export([init/1, - stats_prefix/0, - allowed_methods/0, - post_is_create/2, - process_post/2 - ]). - --export([authorize/2]). - --include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). --include_lib("xmerl/include/xmerl.hrl"). - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). --endif. - --define(RIAKCPOOL, request_pool). - --spec init(#context{}) -> {ok, #context{}}. -init(Ctx) -> - {ok, Ctx#context{rc_pool=?RIAKCPOOL}}. - --spec stats_prefix() -> multiple_delete. -stats_prefix() -> multiple_delete. - --spec allowed_methods() -> [atom()]. -allowed_methods() -> - %% POST is for Delete Multiple Objects - ['POST']. - -%% TODO: change to authorize/spec/cleanup unneeded cases -%% TODO: requires update for multi-delete --spec authorize(#wm_reqdata{}, #context{}) -> {boolean(), #wm_reqdata{}, #context{}}. -authorize(RD, Ctx) -> - Bucket = list_to_binary(wrq:path_info(bucket, RD)), - {false, RD, Ctx#context{bucket=Bucket}}. - -post_is_create(RD, Ctx) -> - {false, RD, Ctx}. - --spec process_post(#wm_reqdata{}, #context{}) -> {term(), #wm_reqdata{}, #context{}}. -process_post(RD, Ctx=#context{bucket=Bucket, - riak_client=RcPid, user=User}) -> - UserName = riak_cs_wm_utils:extract_name(User), - riak_cs_dtrace:dt_bucket_entry(?MODULE, <<"multiple_delete">>, [], [UserName, Bucket]), - - - handle_with_bucket_obj(riak_cs_bucket:fetch_bucket_object(Bucket, RcPid), RD, Ctx). - -handle_with_bucket_obj({error, notfound}, RD, - #context{response_module=ResponseMod} = Ctx) -> - ResponseMod:api_error(no_such_bucket, RD, Ctx); - -handle_with_bucket_obj({error, _} = Error, RD, - #context{response_module=ResponseMod} = Ctx) -> - _ = lager:debug("bucket error: ~p", [Error]), - ResponseMod:api_error(Error, RD, Ctx); - -handle_with_bucket_obj({ok, BucketObj}, - RD, Ctx=#context{bucket=Bucket, - riak_client=RcPid, user=User, - response_module=ResponseMod, - policy_module=PolicyMod}) -> - - case parse_body(binary_to_list(wrq:req_body(RD))) of - {error, _} = Error -> - ResponseMod:api_error(Error, RD, Ctx); - {ok, BinKeys} when length(BinKeys) > 1000 -> - %% Delete Multiple Objects accepts a request to delete up to 1000 Objects. - ResponseMod:api_error(malformed_xml, RD, Ctx); - {ok, BinKeys} -> - lager:debug("deleting keys at ~p: ~p", [Bucket, BinKeys]), - - Policy = riak_cs_wm_utils:translate_bucket_policy(PolicyMod, BucketObj), - CanonicalId = riak_cs_wm_utils:extract_canonical_id(User), - Access0 = PolicyMod:reqdata_to_access(RD, object, CanonicalId), - - %% map: keys => delete_results => xmlElements - Results = - lists:map(fun(BinKey) -> - handle_key(RcPid, Bucket, BinKey, - check_permission( - RcPid, Bucket, BinKey, - Access0, CanonicalId, Policy, PolicyMod, BucketObj)) - end, BinKeys), - - %% xmlDoc => return body. - Xml = riak_cs_xml:to_xml([{'DeleteResult', [{'xmlns', ?S3_XMLNS}], Results}]), - - RD2 = wrq:set_resp_body(Xml, RD), - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"multiple_delete">>, [200], []), - {true, RD2, Ctx} - end. - --spec check_permission(riak_client(), binary(), binary(), - access(), string(), policy()|undefined, atom(), riakc_obj:riakc_obj()) -> - ok | {error, access_denied|notfound|no_active_manifest}. -check_permission(RcPid, Bucket, Key, - Access0, CanonicalId, Policy, PolicyMod, BucketObj) -> - case riak_cs_manifest:fetch(RcPid, Bucket, Key) of - {ok, Manifest} -> - ObjectAcl = riak_cs_manifest:object_acl(Manifest), - Access = Access0#access_v1{key=Key, method='DELETE', target=object}, - - case riak_cs_wm_utils:check_object_authorization(Access, false, ObjectAcl, - Policy, CanonicalId, PolicyMod, - RcPid, BucketObj) of - {ok, _} -> ok; - {error, _} -> {error, access_denied} - end; - E -> - E - end. - -%% bucket/key => delete => xml indicating each result --spec handle_key(riak_client(), binary(), binary(), - ok | {error, access_denied|notfound|no_active_manifest}) -> - {'Deleted', list(tuple())} | {'Error', list(tuple())}. -handle_key(_RcPid, _Bucket, Key, {error, notfound}) -> - %% delete is RESTful, thus this is success - {'Deleted', [{'Key', [Key]}]}; -handle_key(_RcPid, _Bucket, Key, {error, no_active_manifest}) -> - %% delete is RESTful, thus this is success - {'Deleted', [{'Key', [Key]}]}; -handle_key(_RcPid, _Bucket, Key, {error, Error}) -> - {'Error', - [{'Key', [Key]}, - {'Code', [riak_cs_s3_response:error_code(Error)]}, - {'Message', [riak_cs_s3_response:error_message(Error)]}]}; -handle_key(RcPid, Bucket, Key, ok) -> - case riak_cs_utils:delete_object(Bucket, Key, RcPid) of - {ok, _UUIDsMarkedforDelete} -> - {'Deleted', [{'Key', [Key]}]}; - Error -> - handle_key(RcPid, Bucket, Key, Error) - end. - --spec parse_body(string()) -> {ok, [binary()]} | {error, malformed_xml}. -parse_body(Body) -> - case riak_cs_xml:scan(Body) of - {ok, #xmlElement{name='Delete'} = ParsedData} -> - Keys = [ unicode:characters_to_binary( - [ T#xmlText.value || T <- xmerl_xpath:string("//Key/text()", Node)] - ) || Node <- xmerl_xpath:string("//Delete/Object/node()", ParsedData), - is_record(Node, xmlElement) ], - %% TODO: handle version id - %% VersionIds = [riak_cs_utils:hexlist_to_binary(string:strip(T#xmlText.value, both, $")) || - %% T <- xmerl_xpath:string("//Delete/Object/VersionId/text()", ParsedData)], - {ok, Keys}; - {ok, _ParsedData} -> - {error, malformed_xml}; - Error -> - Error - end. - --ifdef(TEST). - -parse_body_test() -> - Body = " </Key> ", - ?assertEqual({ok, [<<"">>]}, parse_body(Body)). - --endif. diff --git a/src/riak_cs_wm_bucket_policy.erl b/src/riak_cs_wm_bucket_policy.erl deleted file mode 100644 index d45b24f81..000000000 --- a/src/riak_cs_wm_bucket_policy.erl +++ /dev/null @@ -1,147 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(riak_cs_wm_bucket_policy). - --export([stats_prefix/0, - content_types_provided/2, - to_json/2, - allowed_methods/0, - content_types_accepted/2, - accept_body/2, - delete_resource/2]). - --export([authorize/2]). - -%% TODO: DELETE? - --include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). --include_lib("riak_pb/include/riak_pb_kv_codec.hrl"). - - --spec stats_prefix() -> bucket_policy. -stats_prefix() -> bucket_policy. - -%% @doc Get the list of methods this resource supports. --spec allowed_methods() -> [atom()]. -allowed_methods() -> - ['GET', 'PUT', 'DELETE']. - --spec content_types_provided(#wm_reqdata{}, #context{}) -> - {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_provided(RD, Ctx) -> - {[{"application/json", to_json}], RD, Ctx}. - --spec content_types_accepted(#wm_reqdata{}, #context{}) -> - {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_accepted(RD, Ctx) -> - case wrq:get_req_header("content-type", RD) of - undefined -> - {[{"application/json", accept_body}], RD, Ctx}; - "application/json" -> - {[{"application/json", accept_body}], RD, Ctx}; - _ -> - {false, RD, Ctx} - end. - --spec authorize(#wm_reqdata{}, #context{}) -> {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #context{}}. -authorize(RD, Ctx) -> - riak_cs_wm_utils:bucket_access_authorize_helper(bucket_policy, true, RD, Ctx). - - --spec to_json(#wm_reqdata{}, #context{}) -> - {binary() | {'halt', non_neg_integer()}, #wm_reqdata{}, #context{}}. -to_json(RD, Ctx=#context{start_time=_StartTime, - user=User, - bucket=Bucket, - riak_client=RcPid}) -> - riak_cs_dtrace:dt_bucket_entry(?MODULE, <<"bucket_get_policy">>, - [], [riak_cs_wm_utils:extract_name(User), Bucket]), - - case riak_cs_s3_policy:fetch_bucket_policy(Bucket, RcPid) of - {ok, PolicyJson} -> - {PolicyJson, RD, Ctx}; - {error, policy_undefined} -> - % S3 error: 404 (NoSuchBucketPolicy): The bucket policy does not exist - riak_cs_s3_response:api_error(no_such_bucket_policy, RD, Ctx); - {error, Reason} -> - Code = riak_cs_s3_response:status_code(Reason), - X = riak_cs_s3_response:api_error(Reason, RD, Ctx), - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_get_policy">>, - [Code], [riak_cs_wm_utils:extract_name(User), Bucket]), - X - end. - -%% @doc Process request body on `PUT' request. --spec accept_body(#wm_reqdata{}, #context{}) -> {{halt, non_neg_integer()}, #wm_reqdata{}, #context{}}. -accept_body(RD, Ctx=#context{user=User, - user_object=UserObj, - bucket=Bucket, - policy_module=PolicyMod, - riak_client=RcPid}) -> - riak_cs_dtrace:dt_bucket_entry(?MODULE, <<"bucket_put_policy">>, - [], [riak_cs_wm_utils:extract_name(User), Bucket]), - - PolicyJson = wrq:req_body(RD), - case PolicyMod:policy_from_json(PolicyJson) of - {ok, Policy} -> - Access = PolicyMod:reqdata_to_access(RD, bucket_policy, User#rcs_user_v2.canonical_id), - case PolicyMod:check_policy(Access, Policy) of - ok -> - case riak_cs_bucket:set_bucket_policy(User, UserObj, Bucket, PolicyJson, RcPid) of - ok -> - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_put_policy">>, - [200], [riak_cs_wm_utils:extract_name(User), Bucket]), - {{halt, 200}, RD, Ctx}; - {error, Reason} -> - Code = riak_cs_s3_response:status_code(Reason), - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_put_policy">>, - [Code], [riak_cs_wm_utils:extract_name(User), Bucket]), - riak_cs_s3_response:api_error(Reason, RD, Ctx) - end; - {error, Reason} -> %% good JSON, but bad as IAM policy - riak_cs_s3_response:api_error(Reason, RD, Ctx) - end; - {error, Reason} -> %% Broken as JSON - riak_cs_s3_response:api_error(Reason, RD, Ctx) - end. - -%% @doc Callback for deleting policy. --spec delete_resource(#wm_reqdata{}, #context{}) -> {true, #wm_reqdata{}, #context{}} | - {{halt, 200}, #wm_reqdata{}, #context{}}. -delete_resource(RD, Ctx=#context{user=User, - user_object=UserObj, - bucket=Bucket, - riak_client=RcPid}) -> - riak_cs_dtrace:dt_object_entry(?MODULE, <<"bucket_policy_delete">>, - [], [RD, Ctx, RcPid]), - - case riak_cs_bucket:delete_bucket_policy(User, UserObj, Bucket, RcPid) of - ok -> - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_put_policy">>, - [200], [riak_cs_wm_utils:extract_name(User), Bucket]), - {{halt, 200}, RD, Ctx}; - {error, Reason} -> - Code = riak_cs_s3_response:status_code(Reason), - riak_cs_dtrace:dt_bucket_return(?MODULE, <<"bucket_put_policy">>, - [Code], [riak_cs_wm_utils:extract_name(User), Bucket]), - riak_cs_s3_response:api_error(Reason, RD, Ctx) - end. diff --git a/src/riak_cs_wm_bucket_versioning.erl b/src/riak_cs_wm_bucket_versioning.erl deleted file mode 100644 index 7cafe5d09..000000000 --- a/src/riak_cs_wm_bucket_versioning.erl +++ /dev/null @@ -1,65 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(riak_cs_wm_bucket_versioning). - --export([stats_prefix/0, - content_types_provided/2, - to_xml/2, - allowed_methods/0]). - --export([authorize/2]). - --include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). - --spec stats_prefix() -> bucket_versioning. -stats_prefix() -> bucket_versioning. - -%% @doc Get the list of methods this resource supports. --spec allowed_methods() -> [atom()]. -allowed_methods() -> - %% TODO: PUT? - ['GET']. - --spec content_types_provided(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_provided(RD, Ctx) -> - {[{"application/xml", to_xml}], RD, Ctx}. - --spec authorize(#wm_reqdata{}, #context{}) -> - {boolean() | {halt, term()}, #wm_reqdata{}, #context{}}. -authorize(RD, Ctx) -> - riak_cs_wm_utils:bucket_access_authorize_helper(bucket_version, false, RD, Ctx). - - --spec to_xml(#wm_reqdata{}, #context{}) -> - {binary() | {halt, term()}, #wm_reqdata{}, #context{}}. -to_xml(RD, Ctx=#context{user=User,bucket=Bucket}) -> - StrBucket = binary_to_list(Bucket), - case [B || B <- riak_cs_bucket:get_buckets(User), - B?RCS_BUCKET.name =:= StrBucket] of - [] -> - riak_cs_s3_response:api_error(no_such_bucket, RD, Ctx); - [_BucketRecord] -> - {<<"">>, - RD, Ctx} - end. - - diff --git a/src/riak_cs_wm_common.erl b/src/riak_cs_wm_common.erl deleted file mode 100644 index de0550ad9..000000000 --- a/src/riak_cs_wm_common.erl +++ /dev/null @@ -1,649 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(riak_cs_wm_common). - --export([init/1, - service_available/2, - forbidden/2, - content_types_accepted/2, - content_types_provided/2, - generate_etag/2, - last_modified/2, - valid_entity_length/2, - validate_content_checksum/2, - malformed_request/2, - to_xml/2, - to_json/2, - post_is_create/2, - create_path/2, - process_post/2, - resp_body/2, - multiple_choices/2, - add_acl_to_context_then_accept/2, - accept_body/2, - produce_body/2, - allowed_methods/2, - delete_resource/2, - finish_request/2]). - --export([default_allowed_methods/0, - default_stats_prefix/0, - default_content_types_accepted/2, - default_content_types_provided/2, - default_generate_etag/2, - default_last_modified/2, - default_finish_request/2, - default_init/1, - default_authorize/2, - default_malformed_request/2, - default_valid_entity_length/2, - default_validate_content_checksum/2, - default_delete_resource/2, - default_anon_ok/0, - default_produce_body/2, - default_multiple_choices/2]). - --include("riak_cs.hrl"). --include("oos_api.hrl"). --include_lib("webmachine/include/webmachine.hrl"). - -%% =================================================================== -%% Webmachine callbacks -%% =================================================================== - --spec init([{atom(),term()}]) -> {ok, #context{}}. -init(Config) -> - _ = dyntrace:put_tag(pid_to_list(self())), - Mod = proplists:get_value(submodule, Config), - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"init">>), - %% Check if authentication is disabled and set that in the context. - AuthBypass = proplists:get_value(auth_bypass, Config), - AuthModule = proplists:get_value(auth_module, Config), - Api = riak_cs_config:api(), - RespModule = riak_cs_config:response_module(Api), - PolicyModule = proplists:get_value(policy_module, Config), - Exports = orddict:from_list(Mod:module_info(exports)), - ExportsFun = exports_fun(Exports), - StatsPrefix = resource_call(Mod, stats_prefix, [], ExportsFun), - Ctx = #context{auth_bypass=AuthBypass, - auth_module=AuthModule, - response_module=RespModule, - policy_module=PolicyModule, - exports_fun=ExportsFun, - stats_prefix=StatsPrefix, - start_time=os:timestamp(), - submodule=Mod, - api=Api}, - resource_call(Mod, init, [Ctx], ExportsFun). - --spec service_available(#wm_reqdata{}, #context{}) -> {boolean(), #wm_reqdata{}, #context{}}. -service_available(RD, Ctx=#context{rc_pool=undefined}) -> - service_available(RD, Ctx#context{rc_pool=request_pool}); -service_available(RD, Ctx=#context{submodule=Mod, rc_pool=Pool}) -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"service_available">>), - case riak_cs_riak_client:checkout(Pool) of - {ok, RcPid} -> - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, <<"service_available">>, [1], []), - {true, RD, Ctx#context{riak_client=RcPid}}; - {error, _Reason} -> - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, <<"service_available">>, [0], []), - {false, RD, Ctx} - end. - --spec malformed_request(#wm_reqdata{}, #context{}) -> {boolean(), #wm_reqdata{}, #context{}}. -malformed_request(RD, Ctx=#context{submodule=Mod, - exports_fun=ExportsFun, - stats_prefix=StatsPrefix}) -> - %% Methoid is used in stats keys, updating inflow should be *after* - %% allowed_methods assertion. - _ = update_stats_inflow(RD, StatsPrefix), - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"malformed_request">>), - {Malformed, _, _} = R = resource_call(Mod, - malformed_request, - [RD, Ctx], - ExportsFun), - riak_cs_dtrace:dt_wm_return_bool_with_default({?MODULE, Mod}, - <<"malformed_request">>, - Malformed, - false), - R. - - --spec valid_entity_length(#wm_reqdata{}, #context{}) -> {boolean(), #wm_reqdata{}, #context{}}. -valid_entity_length(RD, Ctx=#context{submodule=Mod, exports_fun=ExportsFun}) -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"valid_entity_length">>), - {Valid, _, _} = R = resource_call(Mod, - valid_entity_length, - [RD, Ctx], - ExportsFun), - riak_cs_dtrace:dt_wm_return_bool_with_default({?MODULE, Mod}, - <<"valid_entity_length">>, - Valid, - true), - R. - --type validate_checksum_response() :: {error, term()} | - {halt, pos_integer()} | - boolean(). --spec validate_content_checksum(#wm_reqdata{}, #context{}) -> - {validate_checksum_response(), #wm_reqdata{}, #context{}}. -validate_content_checksum(RD, Ctx=#context{submodule=Mod, exports_fun=ExportsFun}) -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"validate_content_checksum">>), - {Valid, _, _} = R = resource_call(Mod, - validate_content_checksum, - [RD, Ctx], - ExportsFun), - riak_cs_dtrace:dt_wm_return_bool_with_default({?MODULE, Mod}, - <<"validate_content_checksum">>, - Valid, - true), - R. - --spec forbidden(#wm_reqdata{}, #context{}) -> {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #context{}}. -forbidden(RD, Ctx=#context{auth_module=AuthMod, - submodule=Mod, - riak_client=RcPid, - exports_fun=ExportsFun}) -> - {AuthResult, AnonOk} = - case AuthMod:identify(RD, Ctx) of - failed -> - %% Identification failed, deny access - {{error, no_such_key}, false}; - {failed, Reason} -> - {{error, Reason}, false}; - {UserKey, AuthData} -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, - <<"forbidden">>, - [], - [riak_cs_wm_utils:extract_name(UserKey)]), - UserLookupResult = maybe_create_user( - riak_cs_user:get_user(UserKey, RcPid), - UserKey, - Ctx#context.api, - Ctx#context.auth_module, - AuthData, - RcPid), - {authenticate(UserLookupResult, RD, Ctx, AuthData), - resource_call(Mod, anon_ok, [], ExportsFun)} - end, - post_authentication(AuthResult, RD, Ctx, AnonOk). - -maybe_create_user({ok, {_, _}}=UserResult, _, _, _, _, _) -> - UserResult; -maybe_create_user({error, NE}, KeyId, oos, _, {UserData, _}, RcPid) - when NE =:= not_found; - NE =:= notfound; - NE =:= no_user_key -> - {Name, Email, UserId} = UserData, - {_, Secret} = riak_cs_oos_utils:user_ec2_creds(UserId, KeyId), - %% Attempt to create a Riak CS user to represent the OS tenant - _ = riak_cs_user:create_user(Name, Email, KeyId, Secret), - riak_cs_user:get_user(KeyId, RcPid); -maybe_create_user({error, NE}, KeyId, s3, riak_cs_keystone_auth, {UserData, _}, RcPid) - when NE =:= not_found; - NE =:= notfound; - NE =:= no_user_key -> - {Name, Email, UserId} = UserData, - {_, Secret} = riak_cs_oos_utils:user_ec2_creds(UserId, KeyId), - %% Attempt to create a Riak CS user to represent the OS tenant - _ = riak_cs_user:create_user(Name, Email, KeyId, Secret), - riak_cs_user:get_user(KeyId, RcPid); -maybe_create_user({error, no_user_key}=Error, _, _, _, _, _) -> - %% Anonymous access may be authorized by ACL or policy afterwards, - %% no logging here. - Error; -maybe_create_user({error, disconnected}=Error, _, _, _, _, RcPid) -> - {ok, MasterPid} = riak_cs_riak_client:master_pbc(RcPid), - riak_cs_pbc:check_connection_status(MasterPid, maybe_create_user), - Error; -maybe_create_user({error, Reason}=Error, _, Api, _, _, _) -> - _ = lager:error("Retrieval of user record for ~p failed. Reason: ~p", - [Api, Reason]), - Error. - -%% @doc Get the list of methods a resource supports. --spec allowed_methods(#wm_reqdata{}, #context{}) -> {[atom()], #wm_reqdata{}, #context{}}. -allowed_methods(RD, Ctx=#context{submodule=Mod, - exports_fun=ExportsFun}) -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"allowed_methods">>), - Methods = resource_call(Mod, - allowed_methods, - [], - ExportsFun), - {Methods, RD, Ctx}. - --spec content_types_accepted(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_accepted(RD, Ctx=#context{submodule=Mod, - exports_fun=ExportsFun}) -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"content_types_accepted">>), - resource_call(Mod, - content_types_accepted, - [RD,Ctx], - ExportsFun). - --spec content_types_provided(#wm_reqdata{}, #context{}) -> {[{string(), atom()}], #wm_reqdata{}, #context{}}. -content_types_provided(RD, Ctx=#context{submodule=Mod, - exports_fun=ExportsFun}) -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"content_types_provided">>), - resource_call(Mod, - content_types_provided, - [RD,Ctx], - ExportsFun). - --spec generate_etag(#wm_reqdata{}, #context{}) -> {string(), #wm_reqdata{}, #context{}}. -generate_etag(RD, Ctx=#context{submodule=Mod, - exports_fun=ExportsFun}) -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"generate_etag">>), - resource_call(Mod, - generate_etag, - [RD,Ctx], - ExportsFun). - --spec last_modified(#wm_reqdata{}, #context{}) -> {calendar:datetime(), #wm_reqdata{}, #context{}}. -last_modified(RD, Ctx=#context{submodule=Mod, - exports_fun=ExportsFun}) -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"last_modified">>), - resource_call(Mod, - last_modified, - [RD,Ctx], - ExportsFun). - --spec delete_resource(#wm_reqdata{}, #context{}) -> {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #context{}}. -delete_resource(RD, Ctx=#context{submodule=Mod,exports_fun=ExportsFun}) -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"delete_resource">>), - %% TODO: add dt_wm_return from subresource? - resource_call(Mod, - delete_resource, - [RD,Ctx], - ExportsFun). - --spec to_xml(#wm_reqdata{}, #context{}) -> - {binary() | {'halt', non_neg_integer()}, #wm_reqdata{}, #context{}}. -to_xml(RD, Ctx=#context{user=User, - submodule=Mod, - exports_fun=ExportsFun}) -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"to_xml">>), - Res = resource_call(Mod, - to_xml, - [RD, Ctx], - ExportsFun), - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, <<"to_xml">>, [], [riak_cs_wm_utils:extract_name(User)]), - Res. - --spec to_json(#wm_reqdata{}, #context{}) -> - {binary() | {'halt', non_neg_integer()}, #wm_reqdata{}, #context{}}. -to_json(RD, Ctx=#context{user=User, - submodule=Mod, - exports_fun=ExportsFun}) -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"to_json">>), - Res = resource_call(Mod, - to_json, - [RD, Ctx], - ExportsFun(to_json)), - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, <<"to_json">>, [], [riak_cs_wm_utils:extract_name(User)]), - Res. - -post_is_create(RD, Ctx=#context{submodule=Mod, - exports_fun=ExportsFun}) -> - resource_call(Mod, post_is_create, [RD, Ctx], ExportsFun). - -create_path(RD, Ctx=#context{submodule=Mod, - exports_fun=ExportsFun}) -> - resource_call(Mod, create_path, [RD, Ctx], ExportsFun). - -process_post(RD, Ctx=#context{submodule=Mod, - exports_fun=ExportsFun}) -> - resource_call(Mod, process_post, [RD, Ctx], ExportsFun). - -resp_body(RD, Ctx=#context{submodule=Mod, - exports_fun=ExportsFun}) -> - resource_call(Mod, resp_body, [RD, Ctx], ExportsFun). - -multiple_choices(RD, Ctx=#context{submodule=Mod, - exports_fun=ExportsFun}) -> - try - resource_call(Mod, multiple_choices, [RD, Ctx], ExportsFun) - catch _:_ -> - {false, RD, Ctx} - end. - -%% @doc Add an ACL (or default ACL) to the context, parsed from headers. If -%% parsing the headers fails, halt the request. -add_acl_to_context_then_accept(RD, Ctx) -> - case riak_cs_wm_utils:maybe_update_context_with_acl_from_headers(RD, Ctx) of - {ok, ContextWithAcl} -> - accept_body(RD, ContextWithAcl); - {error, HaltResponse} -> - HaltResponse - end. - --spec accept_body(#wm_reqdata{}, #context{}) -> - {boolean() | {'halt', non_neg_integer()}, #wm_reqdata{}, #context{}}. -accept_body(RD, Ctx=#context{submodule=Mod,exports_fun=ExportsFun,user=User}) -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"accept_body">>), - Res = resource_call(Mod, - accept_body, - [RD, Ctx], - ExportsFun), - %% TODO: extract response code and add to ints field - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, <<"accept_body">>, [], [riak_cs_wm_utils:extract_name(User)]), - Res. - --spec produce_body(#wm_reqdata{}, #context{}) -> - {iolist()|binary(), #wm_reqdata{}, #context{}} | - {{known_length_stream, non_neg_integer(), {<<>>, function()}}, #wm_reqdata{}, #context{}}. -produce_body(RD, Ctx=#context{user=User, - submodule=Mod, - exports_fun=ExportsFun}) -> - %% TODO: add dt_wm_return w/ content length - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"produce_body">>), - Res = resource_call(Mod, - produce_body, - [RD, Ctx], - ExportsFun), - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, <<"produce_body">>, [], [riak_cs_wm_utils:extract_name(User)]), - Res. - --spec finish_request(#wm_reqdata{}, #context{}) -> {boolean(), #wm_reqdata{}, #context{}}. -finish_request(RD, Ctx=#context{riak_client=RcPid, - auto_rc_close=AutoRcClose, - submodule=Mod, - exports_fun=ExportsFun}) - when RcPid =:= undefined orelse AutoRcClose =:= false -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"finish_request">>, [0], []), - Res = resource_call(Mod, - finish_request, - [RD, Ctx], - ExportsFun), - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, <<"finish_request">>, [0], []), - update_stats(RD, Ctx), - Res; -finish_request(RD, Ctx0=#context{riak_client=RcPid, - rc_pool=Pool, - submodule=Mod, - exports_fun=ExportsFun}) -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"finish_request">>, [1], []), - riak_cs_riak_client:checkin(Pool, RcPid), - Ctx = Ctx0#context{riak_client=undefined}, - Res = resource_call(Mod, - finish_request, - [RD, Ctx], - ExportsFun), - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, <<"finish_request">>, [1], []), - update_stats(RD, Ctx), - Res. - -%% =================================================================== -%% Helper functions -%% =================================================================== - --spec authorize(#wm_reqdata{}, #context{}) -> {boolean() | {halt, non_neg_integer()}, #wm_reqdata{}, #context{}}. -authorize(RD,Ctx=#context{submodule=Mod, exports_fun=ExportsFun}) -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"authorize">>), - {Success, _, _} = R = resource_call(Mod, authorize, [RD,Ctx], ExportsFun), - case Success of - {halt, Code} -> - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, <<"authorize">>, [Code], []); - false -> %% not forbidden, e.g. success - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, <<"authorize">>); - true -> %% forbidden - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, <<"authorize">>, [403], []) - end, - R. - --type user_lookup_result() :: {ok, {rcs_user(), riakc_obj:riakc_obj()}} | {error, term()}. --spec authenticate(user_lookup_result(), term(), term(), term()) -> - {ok, rcs_user(), riakc_obj:riakc_obj()} | {error, term()}. -authenticate({ok, {User, UserObj}}, RD, Ctx=#context{auth_module=AuthMod, submodule=Mod}, AuthData) - when User?RCS_USER.status =:= enabled -> - riak_cs_dtrace:dt_wm_entry({?MODULE, Mod}, <<"authenticate">>, [], [atom_to_binary(AuthMod, latin1)]), - case AuthMod:authenticate(User, AuthData, RD, Ctx) of - ok -> - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, <<"authenticate">>, [2], [atom_to_binary(AuthMod, latin1)]), - {ok, User, UserObj}; - {error, reqtime_tooskewed} -> - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, <<"authenticate">>, [1], [atom_to_binary(AuthMod, latin1)]), - {error, reqtime_tooskewed}; - {error, _Reason} -> - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, <<"authenticate">>, [0], [atom_to_binary(AuthMod, latin1)]), - {error, bad_auth} - end; -authenticate({ok, {User, _UserObj}}, _RD, _Ctx, _AuthData) - when User?RCS_USER.status =/= enabled -> - %% {ok, _} -> %% disabled account, we are going to 403 - {error, bad_auth}; -authenticate({error, _}=Error, _RD, _Ctx, _AuthData) -> - Error. - --spec exports_fun(orddict:orddict()) -> function(). -exports_fun(Exports) -> - fun(Function) -> - orddict:is_key(Function, Exports) - end. - - -resource_call(Mod, Fun, Args, true) -> - erlang:apply(Mod, Fun, Args); -resource_call(_Mod, Fun, Args, false) -> - erlang:apply(?MODULE, default(Fun), Args); -resource_call(Mod, Fun, Args, ExportsFun) -> - resource_call(Mod, Fun, Args, ExportsFun(Fun)). - - -post_authentication(AuthResult, RD, Ctx = #context{submodule=Mod}, AnonOk) -> - case post_authentication(AuthResult, RD, Ctx, fun authorize/2, AnonOk) of - {false, _RD2, Ctx2} = FalseRet -> - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, - <<"forbidden">>, [], - [riak_cs_wm_utils:extract_name(Ctx2#context.user), - <<"false">>]), - FalseRet; - {Rsn, _RD2, Ctx2} = Ret -> - Reason = - case Rsn of - {halt, Code} -> Code; - _ -> -1 - end, - riak_cs_dtrace:dt_wm_return({?MODULE, Mod}, - <<"forbidden">>, [Reason], - [riak_cs_wm_utils:extract_name(Ctx2#context.user), - <<"true">>]), - Ret - end. - -post_authentication({ok, User, UserObj}, RD, Ctx, Authorize, _) -> - %% given keyid and signature matched, proceed - Authorize(RD, Ctx#context{user=User, - user_object=UserObj}); -post_authentication({error, no_user_key}, RD, Ctx, Authorize, true) -> - %% no keyid was given, proceed anonymously - _ = lager:debug("No user key"), - Authorize(RD, Ctx); -post_authentication({error, no_user_key}, RD, Ctx, _, false) -> - %% no keyid was given, deny access - _ = lager:debug("No user key, deny"), - riak_cs_wm_utils:deny_access(RD, Ctx); -post_authentication({error, bad_auth}, RD, Ctx, _, _) -> - %% given keyid was found, but signature didn't match - _ = lager:debug("bad_auth"), - riak_cs_wm_utils:deny_access(RD, Ctx); -post_authentication({error, reqtime_tooskewed} = Error, RD, - #context{response_module = ResponseMod} = Ctx, _, _) -> - _ = lager:debug("reqtime_tooskewed"), - ResponseMod:api_error(Error, RD, Ctx); -post_authentication({error, {auth_not_supported, AuthType}}, RD, - #context{response_module=ResponseMod} = Ctx, _, _) -> - _ = lager:debug("auth_not_supported: ~s", [AuthType]), - ResponseMod:api_error({auth_not_supported, AuthType}, RD, Ctx); -post_authentication({error, notfound}, RD, Ctx, _, _) -> - %% This is rubbish. We need to differentiate between - %% no key_id being presented and the key_id lookup - %% failing in some better way. - _ = lager:debug("key_id not present or not found"), - riak_cs_wm_utils:deny_access(RD, Ctx); -post_authentication({error, Reason}, RD, Ctx, _, _) -> - %% no matching keyid was found, or lookup failed - _ = lager:debug("Authentication error: ~p", [Reason]), - riak_cs_wm_utils:deny_invalid_key(RD, Ctx). - -update_stats_inflow(_RD, undefined = _StatsPrefix) -> - ok; -update_stats_inflow(_RD, no_stats = _StatsPrefix) -> - ok; -update_stats_inflow(RD, StatsPrefix) -> - Method = riak_cs_wm_utils:lower_case_method(wrq:method(RD)), - Key = [StatsPrefix, Method], - riak_cs_stats:inflow(Key). - -update_stats(_RD, #context{stats_key=no_stats}) -> - ok; -update_stats(_RD, #context{stats_prefix=no_stats}) -> - ok; -update_stats(RD, #context{start_time=StartTime, - stats_prefix=StatsPrefix, stats_key=StatsKey}) -> - catch update_stats(StartTime, - wrq:response_code(RD), - StatsPrefix, - riak_cs_wm_utils:lower_case_method(wrq:method(RD)), - StatsKey). - -update_stats(StartTime, Code, StatsPrefix, Method, StatsKey0) -> - StatsKey = case StatsKey0 of - prefix_and_method -> [StatsPrefix, Method]; - _ -> StatsKey0 - end, - case Code of - 405 -> - %% Method Not Allowed: don't update stats because unallowed - %% mothod may lead to notfound warning in updating stats - ok; - Success when is_integer(Success) andalso Success < 400 -> - riak_cs_stats:update_with_start(StatsKey, StartTime); - _Error -> - riak_cs_stats:update_error_with_start(StatsKey, StartTime) - end. - -%% =================================================================== -%% Resource function defaults -%% =================================================================== - -default(init) -> - default_init; -default(stats_prefix) -> - default_stats_prefix; -default(allowed_methods) -> - default_allowed_methods; -default(content_types_accepted) -> - default_content_types_accepted; -default(content_types_provided) -> - default_content_types_provided; -default(generate_etag) -> - default_generate_etag; -default(last_modified) -> - default_last_modified; -default(malformed_request) -> - default_malformed_request; -default(valid_entity_length) -> - default_valid_entity_length; -default(validate_content_checksum) -> - default_validate_content_checksum; -default(delete_resource) -> - default_delete_resource; -default(authorize) -> - default_authorize; -default(finish_request) -> - default_finish_request; -default(anon_ok) -> - default_anon_ok; -default(produce_body) -> - default_produce_body; -default(multiple_choices) -> - default_multiple_choices; -default(_) -> - undefined. - -default_init(Ctx) -> - {ok, Ctx}. - -default_stats_prefix() -> - no_stats. - -default_malformed_request(RD, Ctx) -> - {false, RD, Ctx}. - -default_valid_entity_length(RD, Ctx) -> - {true, RD, Ctx}. - -default_validate_content_checksum(RD, Ctx) -> - {true, RD, Ctx}. - -default_content_types_accepted(RD, Ctx) -> - {[], RD, Ctx}. - --spec default_content_types_provided(#wm_reqdata{}, #context{}) -> - {[{string(), atom()}], - #wm_reqdata{}, - #context{}}. -default_content_types_provided(RD, Ctx=#context{api=oos}) -> - {[{"text/plain", produce_body}], RD, Ctx}; -default_content_types_provided(RD, Ctx) -> - {[{"application/xml", produce_body}], RD, Ctx}. - -default_generate_etag(RD, Ctx) -> - {undefined, RD, Ctx}. - -default_last_modified(RD, Ctx) -> - {undefined, RD, Ctx}. - -default_delete_resource(RD, Ctx) -> - {false, RD, Ctx}. - -default_allowed_methods() -> - []. - -default_finish_request(RD, Ctx) -> - {true, RD, Ctx}. - -default_anon_ok() -> - true. - -default_produce_body(RD, Ctx=#context{submodule=Mod, - response_module=ResponseMod, - exports_fun=ExportsFun}) -> - try - ResponseMod:respond( - resource_call(Mod, api_request, [RD, Ctx], ExportsFun), - RD, - Ctx) - catch error:{badmatch, {error, Reason}} -> - ResponseMod:api_error(Reason, RD, Ctx) - end. - -%% @doc this function will be called by `post_authenticate/2' if the user successfully -%% authenticates and the submodule does not provide an implementation -%% of authorize/2. The default implementation does not perform any authorization -%% and simply returns false to signify the request is not fobidden --spec default_authorize(term(), term()) -> {false, term(), term()}. -default_authorize(RD, Ctx) -> - {false, RD, Ctx}. - -default_multiple_choices(RD, Ctx) -> - {false, RD, Ctx}. diff --git a/src/riak_cs_wm_user.erl b/src/riak_cs_wm_user.erl deleted file mode 100644 index 89a0ec622..000000000 --- a/src/riak_cs_wm_user.erl +++ /dev/null @@ -1,406 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(riak_cs_wm_user). - --export([init/1, - service_available/2, - forbidden/2, - content_types_provided/2, - content_types_accepted/2, - accept_json/2, - accept_xml/2, - allowed_methods/2, - post_is_create/2, - create_path/2, - produce_json/2, - produce_xml/2, - finish_request/2]). - --include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). --include_lib("xmerl/include/xmerl.hrl"). - -%% ------------------------------------------------------------------- -%% Webmachine callbacks -%% ------------------------------------------------------------------- - -init(Config) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"init">>), - %% Check if authentication is disabled and - %% set that in the context. - AuthBypass = not proplists:get_value(admin_auth_enabled, Config), - Api = riak_cs_config:api(), - RespModule = riak_cs_config:response_module(Api), - {ok, #context{auth_bypass=AuthBypass, - api=Api, - response_module=RespModule}}. - --spec service_available(term(), term()) -> {true, term(), term()}. -service_available(RD, Ctx) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"service_available">>), - riak_cs_wm_utils:service_available(RD, Ctx). - --spec allowed_methods(term(), term()) -> {[atom()], term(), term()}. -allowed_methods(RD, Ctx) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"allowed_methods">>), - {['GET', 'HEAD', 'POST', 'PUT'], RD, Ctx}. - -forbidden(RD, Ctx=#context{auth_bypass=AuthBypass}) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"forbidden">>), - Method = wrq:method(RD), - AnonOk = ((Method =:= 'PUT' orelse Method =:= 'POST') andalso - riak_cs_config:anonymous_user_creation()) - orelse AuthBypass, - Next = fun(NewRD, NewCtx=#context{user=User}) -> - forbidden(wrq:method(RD), - NewRD, - NewCtx, - User, - user_key(RD), - AnonOk) - end, - UserAuthResponse = riak_cs_wm_utils:find_and_auth_user(RD, Ctx, Next), - handle_user_auth_response(UserAuthResponse). - -handle_user_auth_response({false, _RD, Ctx} = Ret) -> - riak_cs_dtrace:dt_wm_return(?MODULE, <<"forbidden">>, - [], [riak_cs_wm_utils:extract_name(Ctx#context.user), <<"false">>]), - Ret; -handle_user_auth_response({{halt, Code}, _RD, Ctx} = Ret) -> - riak_cs_dtrace:dt_wm_return(?MODULE, <<"forbidden">>, - [Code], [riak_cs_wm_utils:extract_name(Ctx#context.user), <<"true">>]), - Ret; -handle_user_auth_response({_Reason, _RD, Ctx} = Ret) -> - riak_cs_dtrace:dt_wm_return(?MODULE, <<"forbidden">>, - [-1], [riak_cs_wm_utils:extract_name(Ctx#context.user), <<"true">>]), - Ret. - --spec content_types_accepted(term(), term()) -> - {[{string(), atom()}], term(), term()}. -content_types_accepted(RD, Ctx) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"content_types_accepted">>), - {[{?XML_TYPE, accept_xml}, {?JSON_TYPE, accept_json}], RD, Ctx}. - -content_types_provided(RD, Ctx) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"content_types_provided">>), - {[{?XML_TYPE, produce_xml}, {?JSON_TYPE, produce_json}], RD, Ctx}. - -post_is_create(RD, Ctx) -> {true, RD, Ctx}. - -create_path(RD, Ctx) -> {"/riak-cs/user", RD, Ctx}. - --spec accept_json(#wm_reqdata{}, #context{}) -> - {boolean() | {halt, term()}, term(), term()}. -accept_json(RD, Ctx=#context{user=undefined}) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"accept_json">>), - Body = riak_cs_json:from_json(wrq:req_body(RD)), - {UserName, Email} = - riak_cs_json:value_or_default( - riak_cs_json:get(Body, [{<<"name">>, <<"email">>}]), - {<<>>, <<>>}), - user_response(riak_cs_user:create_user(binary_to_list(UserName), - binary_to_list(Email)), - ?JSON_TYPE, - RD, - Ctx); -accept_json(RD, Ctx) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"accept_json">>), - Body = wrq:req_body(RD), - case catch mochijson2:decode(Body) of - {struct, UserItems} -> - UpdateItems = lists:foldl(fun user_json_filter/2, [], UserItems), - user_response(update_user(UpdateItems, RD, Ctx), - ?JSON_TYPE, - RD, - Ctx); - {'EXIT', _} -> - riak_cs_s3_response:api_error(invalid_user_update, RD, Ctx) - end. - --spec accept_xml(term(), term()) -> - {boolean() | {halt, term()}, term(), term()}. -accept_xml(RD, Ctx=#context{user=undefined}) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"accept_xml">>), - Body = binary_to_list(wrq:req_body(RD)), - case riak_cs_xml:scan(Body) of - {error, malformed_xml} -> - riak_cs_s3_response:api_error(invalid_user_update, RD, Ctx); - {ok, ParsedData} -> - ValidItems = lists:foldl(fun user_xml_filter/2, - [], - ParsedData#xmlElement.content), - UserName = proplists:get_value(name, ValidItems, ""), - Email= proplists:get_value(email, ValidItems, ""), - user_response(riak_cs_user:create_user(UserName, Email), - ?XML_TYPE, - RD, - Ctx) - end; -accept_xml(RD, Ctx) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"accept_xml">>), - Body = binary_to_list(wrq:req_body(RD)), - case riak_cs_xml:scan(Body) of - {error, malformed_xml} -> - riak_cs_s3_response:api_error(invalid_user_update, RD, Ctx); - {ok, ParsedData} -> - UpdateItems = lists:foldl(fun user_xml_filter/2, - [], - ParsedData#xmlElement.content), - user_response(update_user(UpdateItems, RD, Ctx), - ?XML_TYPE, - RD, - Ctx) - - end. - -produce_json(RD, #context{user=User}=Ctx) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"produce_json">>), - Body = riak_cs_json:to_json(User), - Etag = etag(Body), - RD2 = wrq:set_resp_header("ETag", Etag, RD), - {Body, RD2, Ctx}. - -produce_xml(RD, #context{user=User}=Ctx) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"produce_xml">>), - Body = riak_cs_xml:to_xml(User), - Etag = etag(Body), - RD2 = wrq:set_resp_header("ETag", Etag, RD), - {Body, RD2, Ctx}. - -finish_request(RD, Ctx=#context{riak_client=undefined}) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"finish_request">>, [0], []), - {true, RD, Ctx}; -finish_request(RD, Ctx=#context{riak_client=RcPid}) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"finish_request">>, [1], []), - riak_cs_riak_client:checkin(RcPid), - riak_cs_dtrace:dt_wm_return(?MODULE, <<"finish_request">>, [1], []), - {true, RD, Ctx#context{riak_client=undefined}}. - -%% ------------------------------------------------------------------- -%% Internal functions -%% ------------------------------------------------------------------- - --spec admin_check(boolean(), term(), term()) -> {boolean(), term(), term()}. -admin_check(true, RD, Ctx) -> - {false, RD, Ctx#context{user=undefined}}; -admin_check(false, RD, Ctx) -> - riak_cs_wm_utils:deny_access(RD, Ctx). - -%% @doc Calculate the etag of a response body -etag(Body) -> - riak_cs_utils:etag_from_binary(riak_cs_utils:md5(Body)). - --spec forbidden(atom(), - term(), - term(), - undefined | rcs_user(), - string(), - boolean()) -> - {boolean() | {halt, term()}, term(), term()}. -forbidden(_Method, RD, Ctx, undefined, _UserPathKey, false) -> - %% anonymous access disallowed - riak_cs_wm_utils:deny_access(RD, Ctx); -forbidden(_, _RD, _Ctx, undefined, [], true) -> - {false, _RD, _Ctx}; -forbidden(_, RD, Ctx, undefined, UserPathKey, true) -> - get_user({false, RD, Ctx}, UserPathKey); -forbidden('POST', RD, Ctx, User, [], _) -> - %% Admin is creating a new user - admin_check(riak_cs_user:is_admin(User), RD, Ctx); -forbidden('PUT', RD, Ctx, User, [], _) -> - admin_check(riak_cs_user:is_admin(User), RD, Ctx); -forbidden(_Method, RD, Ctx, User, UserPathKey, _) when - UserPathKey =:= User?RCS_USER.key_id; - UserPathKey =:= [] -> - %% User is accessing own account - AccessRD = riak_cs_access_log_handler:set_user(User, RD), - {false, AccessRD, Ctx}; -forbidden(_Method, RD, Ctx, User, UserPathKey, _) -> - AdminCheckResult = admin_check(riak_cs_user:is_admin(User), RD, Ctx), - get_user(AdminCheckResult, UserPathKey). - --spec get_user({boolean() | {halt, term()}, term(), term()}, string()) -> - {boolean() | {halt, term()}, term(), term()}. -get_user({false, RD, Ctx}, UserPathKey) -> - handle_get_user_result( - riak_cs_user:get_user(UserPathKey, Ctx#context.riak_client), - RD, - Ctx); -get_user(AdminCheckResult, _) -> - AdminCheckResult. - --spec handle_get_user_result({ok, {rcs_user(), term()}} | {error, term()}, - term(), - term()) -> - {boolean() | {halt, term()}, term(), term()}. - -handle_get_user_result({ok, {User, UserObj}}, RD, Ctx) -> - {false, RD, Ctx#context{user=User, user_object=UserObj}}; -handle_get_user_result({error, Reason}, RD, Ctx) -> - _ = lager:warning("Failed to fetch user record. KeyId: ~p" - " Reason: ~p", [user_key(RD), Reason]), - riak_cs_s3_response:api_error(invalid_access_key_id, RD, Ctx). - --spec update_user([{atom(), term()}], #wm_reqdata{}, #context{}) -> - {ok, rcs_user()} | {halt, term()} | {error, term()}. -update_user(UpdateItems, RD, Ctx=#context{user=User}) -> - riak_cs_dtrace:dt_wm_entry(?MODULE, <<"update_user">>), - UpdateUserResult = update_user_record(User, UpdateItems, false), - handle_update_result(UpdateUserResult, RD, Ctx). - --spec update_user_record('undefined' | rcs_user(), [{atom(), term()}], boolean()) - -> {boolean(), rcs_user()}. -update_user_record(_User, [], RecordUpdated) -> - {RecordUpdated, _User}; -update_user_record(User=?RCS_USER{status=Status}, - [{status, Status} | RestUpdates], - _RecordUpdated) -> - update_user_record(User, RestUpdates, _RecordUpdated); -update_user_record(User, [{status, Status} | RestUpdates], _RecordUpdated) -> - update_user_record(User?RCS_USER{status=Status}, RestUpdates, true); -update_user_record(User, [{name, Name} | RestUpdates], _RecordUpdated) -> - update_user_record(User?RCS_USER{name=Name}, RestUpdates, true); -update_user_record(User, [{email, Email} | RestUpdates], _RecordUpdated) -> - DisplayName = riak_cs_user:display_name(Email), - update_user_record(User?RCS_USER{email=Email, display_name=DisplayName}, - RestUpdates, - true); -update_user_record(User=?RCS_USER{}, [{new_key_secret, true} | RestUpdates], _) -> - update_user_record(riak_cs_user:update_key_secret(User), RestUpdates, true); -update_user_record(_User, [_ | RestUpdates], _RecordUpdated) -> - update_user_record(_User, RestUpdates, _RecordUpdated). - --spec handle_update_result({boolean(), rcs_user()}, term(), term()) -> - {ok, rcs_user()} | {halt, term()} | {error, term()}. -handle_update_result({false, _User}, _RD, _Ctx) -> - {halt, 200}; -handle_update_result({true, User}, _RD, Ctx) -> - #context{user_object=UserObj, - riak_client=RcPid} = Ctx, - riak_cs_user:update_user(User, UserObj, RcPid). - --spec set_resp_data(string(), term(), term()) -> term(). -set_resp_data(ContentType, RD, #context{user=User}) -> - UserDoc = format_user_record(User, ContentType), - wrq:set_resp_body(UserDoc, RD). - --spec user_json_filter({binary(), binary()}, [{atom(), term()}]) -> [{atom(), term()}]. -user_json_filter({ItemKey, ItemValue}, Acc) -> - case ItemKey of - <<"email">> -> - [{email, binary_to_list(ItemValue)} | Acc]; - <<"name">> -> - [{name, binary_to_list(ItemValue)} | Acc]; - <<"status">> -> - case ItemValue of - <<"enabled">> -> - [{status, enabled} | Acc]; - <<"disabled">> -> - [{status, disabled} | Acc]; - _ -> - Acc - end; - <<"new_key_secret">> -> - [{new_key_secret, ItemValue} | Acc]; - _ -> - Acc - end. - -user_key(RD) -> - case wrq:path_tokens(RD) of - [KeyId|_] -> mochiweb_util:unquote(KeyId); - _ -> [] - end. - --spec user_xml_filter(#xmlText{} | #xmlElement{}, [{atom(), term()}]) -> [{atom(), term()}]. -user_xml_filter(#xmlText{}, Acc) -> - Acc; -user_xml_filter(Element, Acc) -> - case Element#xmlElement.name of - 'Email' -> - [Content | _] = Element#xmlElement.content, - case is_record(Content, xmlText) of - true -> - [{email, Content#xmlText.value} | Acc]; - false -> - Acc - end; - 'Name' -> - [Content | _] = Element#xmlElement.content, - case is_record(Content, xmlText) of - true -> - [{name, Content#xmlText.value} | Acc]; - false -> - Acc - end; - 'Status' -> - [Content | _] = Element#xmlElement.content, - case is_record(Content, xmlText) of - true -> - case Content#xmlText.value of - "enabled" -> - [{status, enabled} | Acc]; - "disabled" -> - [{status, disabled} | Acc]; - _ -> - Acc - end; - false -> - Acc - end; - 'NewKeySecret' -> - [Content | _] = Element#xmlElement.content, - case is_record(Content, xmlText) of - true -> - case Content#xmlText.value of - "true" -> - [{new_key_secret, true} | Acc]; - "false" -> - [{new_key_secret, false} | Acc]; - _ -> - Acc - end; - false -> - Acc - end; - _ -> - Acc - end. - --spec user_response({ok, rcs_user()} | {halt, term()} | {error, term()}, - string(), #wm_reqdata{}, #context{}) -> - {true | {halt, non_neg_integer()}, #wm_reqdata{}, #context{}}. -user_response({ok, User}, ContentType, RD, Ctx) -> - UserDoc = format_user_record(User, ContentType), - WrittenRD = - wrq:set_resp_body(UserDoc, - wrq:set_resp_header("Content-Type", ContentType, RD)), - {true, WrittenRD, Ctx}; -user_response({halt, 200}, ContentType, RD, Ctx) -> - {{halt, 200}, set_resp_data(ContentType, RD, Ctx), Ctx}; -user_response({error, Reason}, _, RD, Ctx) -> - riak_cs_s3_response:api_error(Reason, RD, Ctx). - --spec format_user_record(rcs_user(), string()) -> binary(). -format_user_record(User, ?JSON_TYPE) -> - riak_cs_json:to_json(User); -format_user_record(User, ?XML_TYPE) -> - riak_cs_xml:to_xml(User). diff --git a/src/riak_cs_xml.erl b/src/riak_cs_xml.erl deleted file mode 100644 index f8a461230..000000000 --- a/src/riak_cs_xml.erl +++ /dev/null @@ -1,366 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc A collection functions for going to or from XML to an erlang -%% record type. - --module(riak_cs_xml). - --include("riak_cs.hrl"). --include("riak_cs_api.hrl"). --include("list_objects.hrl"). - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). --endif. --include_lib("xmerl/include/xmerl.hrl"). - -%% Public API --export([scan/1, - to_xml/1]). - --define(XML_SCHEMA_INSTANCE, "http://www.w3.org/2001/XMLSchema-instance"). - --type attributes() :: [{atom(), string()}]. --type external_node() :: {atom(), [string()]}. --type internal_node() :: {atom(), [internal_node() | external_node()]} | - {atom(), attributes(), [internal_node() | external_node()]}. - -%% these types should be defined in xmerl.hrl or xmerl.erl -%% for now they're defined here for convenience. --type xmlElement() :: #xmlElement{}. --type xmlText() :: #xmlText{}. --type xmlComment() :: #xmlComment{}. --type xmlPI() :: #xmlPI{}. --type xmlDocument() :: #xmlDocument{}. --type xmlNode() :: xmlElement() | xmlText() | xmlComment() | - xmlPI() | xmlDocument(). --export_type([xmlNode/0, xmlElement/0, xmlText/0]). - -%% Types for simple forms --type tag() :: atom(). --type content() :: [element()]. --type element() :: {tag(), attributes(), content()} | - {tag(), content()} | - tag() | - iodata() | - integer() | - float(). % Really Needed? --type simple_form() :: content(). - -%% =================================================================== -%% Public API -%% =================================================================== - - -%% @doc parse XML and produce xmlElement (other comments and else are bad) -%% in R15B03 (and maybe later version), xmerl_scan:string/2 may return any -%% xml nodes, such as defined as xmlNode() above. It it unsafe because -%% `String' is the Body sent from client, which can be anything. --spec scan(string()) -> {ok, xmlElement()} | {error, malformed_xml}. -scan(String) -> - case catch xmerl_scan:string(String) of - {'EXIT', _E} -> {error, malformed_xml}; - { #xmlElement{} = ParsedData, _Rest} -> {ok, ParsedData}; - _E -> {error, malformed_xml} - end. - --spec to_xml(term()) -> binary(). -to_xml(undefined) -> - []; -to_xml(SimpleForm) when is_list(SimpleForm) -> - simple_form_to_xml(SimpleForm); -to_xml(?ACL{}=Acl) -> - acl_to_xml(Acl); -to_xml(#acl_v1{}=Acl) -> - acl_to_xml(Acl); -to_xml(?LBRESP{}=ListBucketsResp) -> - list_buckets_response_to_xml(ListBucketsResp); -to_xml(?LORESP{}=ListObjsResp) -> - SimpleForm = list_objects_response_to_simple_form(ListObjsResp), - to_xml(SimpleForm); -to_xml(?RCS_USER{}=User) -> - user_record_to_xml(User); -to_xml({users, Users}) -> - user_records_to_xml(Users). - -%% =================================================================== -%% Internal functions -%% =================================================================== - -export_xml(XmlDoc) -> - list_to_binary(xmerl:export_simple(XmlDoc, xmerl_xml, [{prolog, ?XML_PROLOG}])). - -%% @doc Convert simple form into XML. --spec simple_form_to_xml(simple_form()) -> iodata(). -simple_form_to_xml(Elements) -> - XmlDoc = format_elements(Elements), - export_xml(XmlDoc). - -format_elements(Elements) -> - [format_element(E) || E <- Elements]. - -format_element({Tag, Elements}) -> - {Tag, format_elements(Elements)}; -format_element({Tag, Attrs, Elements}) -> - {Tag, Attrs, format_elements(Elements)}; -format_element(Value) -> - format_value(Value). - -%% @doc Convert an internal representation of an ACL into XML. --spec acl_to_xml(acl()) -> binary(). -acl_to_xml(Acl) -> - Content = [make_internal_node('Owner', owner_content(acl_owner(Acl))), - make_internal_node('AccessControlList', make_grants(acl_grants(Acl)))], - XmlDoc = [make_internal_node('AccessControlPolicy', Content)], - export_xml(XmlDoc). - --spec acl_grants(?ACL{} | #acl_v1{}) -> [acl_grant()]. -acl_grants(?ACL{grants=Grants}) -> - Grants; -acl_grants(#acl_v1{grants=Grants}) -> - Grants. - --spec acl_owner(?ACL{} | #acl_v1{}) -> {string(), string()}. -acl_owner(?ACL{owner=Owner}) -> - {OwnerName, OwnerId, _} = Owner, - {OwnerName, OwnerId}; -acl_owner(#acl_v1{owner=Owner}) -> - Owner. - --spec owner_content({string(), string()}) -> [external_node()]. -owner_content({OwnerName, OwnerId}) -> - [make_external_node('ID', OwnerId), - make_external_node('DisplayName', OwnerName)]. - -list_objects_response_to_simple_form(Resp) -> - KeyContents = [{'Contents', key_content_to_simple_form(Content)} || - Content <- (Resp?LORESP.contents)], - CommonPrefixes = [{'CommonPrefixes', [{'Prefix', [CommonPrefix]}]} || - CommonPrefix <- Resp?LORESP.common_prefixes], - Contents = [{'Name', [Resp?LORESP.name]}, - {'Prefix', [Resp?LORESP.prefix]}, - {'Marker', [Resp?LORESP.marker]}] ++ - %% use a list-comprehension trick to only include - %% the `NextMarker' element if it's not `undefined' - [{'NextMarker', [NextMarker]} || - NextMarker <- [Resp?LORESP.next_marker], - NextMarker =/= undefined, - Resp?LORESP.is_truncated] ++ - [{'MaxKeys', [Resp?LORESP.max_keys]}, - {'Delimiter', [Resp?LORESP.delimiter]}, - {'IsTruncated', [Resp?LORESP.is_truncated]}] ++ - KeyContents ++ CommonPrefixes, - [{'ListBucketResult', [{'xmlns', ?S3_XMLNS}], Contents}]. - -key_content_to_simple_form(KeyContent) -> - #list_objects_owner_v1{id=Id, display_name=Name} = KeyContent?LOKC.owner, - [{'Key', [KeyContent?LOKC.key]}, - {'LastModified', [KeyContent?LOKC.last_modified]}, - {'ETag', [KeyContent?LOKC.etag]}, - {'Size', [KeyContent?LOKC.size]}, - {'StorageClass', [KeyContent?LOKC.storage_class]}, - {'Owner', [{'ID', [Id]}, - {'DisplayName', [Name]}]}]. - -list_buckets_response_to_xml(Resp) -> - BucketsContent = - make_internal_node('Buckets', - [bucket_to_xml(B?RCS_BUCKET.name, - B?RCS_BUCKET.creation_date) || - B <- Resp?LBRESP.buckets]), - UserContent = user_to_xml_owner(Resp?LBRESP.user), - Contents = [UserContent] ++ [BucketsContent], - export_xml([make_internal_node('ListAllMyBucketsResult', - [{'xmlns', ?S3_XMLNS}], - Contents)]). - -bucket_to_xml(Name, CreationDate) when is_binary(Name) -> - bucket_to_xml(binary_to_list(Name), CreationDate); -bucket_to_xml(Name, CreationDate) -> - make_internal_node('Bucket', - [make_external_node('Name', Name), - make_external_node('CreationDate', CreationDate)]). - -user_to_xml_owner(?RCS_USER{canonical_id=CanonicalId, display_name=Name}) -> - make_internal_node('Owner', [make_external_node('ID', [CanonicalId]), - make_external_node('DisplayName', [Name])]). - --spec make_internal_node(atom(), term()) -> internal_node(). -make_internal_node(Name, Content) -> - {Name, Content}. - --spec make_internal_node(atom(), attributes(), term()) -> internal_node(). -make_internal_node(Name, Attributes, Content) -> - {Name, Attributes, Content}. - --spec make_external_node(atom(), term()) -> external_node(). -make_external_node(Name, Content) -> - {Name, [format_value(Content)]}. - -%% @doc Assemble the xml for the set of grantees for an acl. --spec make_grants([acl_grant()]) -> [internal_node()]. -make_grants(Grantees) -> - make_grants(Grantees, []). - -%% @doc Assemble the xml for the set of grantees for an acl. --spec make_grants([acl_grant()], [[internal_node()]]) -> [internal_node()]. -make_grants([], Acc) -> - lists:flatten(Acc); -make_grants([{{GranteeName, GranteeId}, Perms} | RestGrantees], Acc) -> - Grantee = [make_grant(GranteeName, GranteeId, Perm) || Perm <- Perms], - make_grants(RestGrantees, [Grantee | Acc]); -make_grants([{Group, Perms} | RestGrantees], Acc) -> - Grantee = [make_grant(Group, Perm) || Perm <- Perms], - make_grants(RestGrantees, [Grantee | Acc]). - -%% @doc Assemble the xml for a group grantee for an acl. --spec make_grant(atom(), acl_perm()) -> internal_node(). -make_grant(Group, Permission) -> - Attributes = [{'xmlns:xsi', ?XML_SCHEMA_INSTANCE}, - {'xsi:type', "Group"}], - GranteeContent = [make_external_node('URI', uri_for_group(Group))], - GrantContent = - [make_internal_node('Grantee', Attributes, GranteeContent), - make_external_node('Permission', Permission)], - make_internal_node('Grant', GrantContent). - -%% @doc Assemble the xml for a single grantee for an acl. --spec make_grant(string(), string(), acl_perm()) -> internal_node(). -make_grant(DisplayName, CanonicalId, Permission) -> - Attributes = [{'xmlns:xsi', ?XML_SCHEMA_INSTANCE}, - {'xsi:type', "CanonicalUser"}], - GranteeContent = [make_external_node('ID', CanonicalId), - make_external_node('DisplayName', DisplayName)], - GrantContent = - [make_internal_node('Grantee', Attributes, GranteeContent), - make_external_node('Permission', Permission)], - make_internal_node('Grant', GrantContent). - --spec format_value(atom() | integer() | binary() | list()) -> string(). -%% @doc Convert value depending on its type into strings -format_value(undefined) -> - []; -format_value(Val) when is_atom(Val) -> - atom_to_list(Val); -format_value(Val) when is_binary(Val) -> - binary_to_list(Val); -format_value(Val) when is_integer(Val) -> - integer_to_list(Val); -format_value(Val) when is_list(Val) -> - Val; -format_value(Val) when is_float(Val) -> - io_lib:format("~p", [Val]). - -%% @doc Map a ACL group atom to its corresponding URI. --spec uri_for_group(atom()) -> string(). -uri_for_group('AllUsers') -> - ?ALL_USERS_GROUP; -uri_for_group('AuthUsers') -> - ?AUTH_USERS_GROUP. - -%% @doc Convert a Riak CS user record to XML --spec user_record_to_xml(rcs_user()) -> binary(). -user_record_to_xml(User) -> - export_xml([user_node(User)]). - -%% @doc Convert a set of Riak CS user records to XML --spec user_records_to_xml([rcs_user()]) -> binary(). -user_records_to_xml(Users) -> - UserNodes = [user_node(User) || User <- Users], - export_xml([make_internal_node('Users', UserNodes)]). - -user_node(?RCS_USER{email=Email, - display_name=DisplayName, - name=Name, - key_id=KeyID, - key_secret=KeySecret, - canonical_id=CanonicalID, - status=Status}) -> - StatusStr = case Status of - enabled -> - "enabled"; - _ -> - "disabled" - end, - Content = [make_external_node('Email', Email), - make_external_node('DisplayName', DisplayName), - make_external_node('Name', Name), - make_external_node('KeyId', KeyID), - make_external_node('KeySecret', KeySecret), - make_external_node('Id', CanonicalID), - make_external_node('Status', StatusStr)], - make_internal_node('User', Content). - -%% =================================================================== -%% Eunit tests -%% =================================================================== - --ifdef(TEST). - -acl_to_xml_test() -> - Xml = <<"TESTID1tester1TESTID2tester2WRITETESTID1tester1READ">>, - Grants1 = [{{"tester1", "TESTID1"}, ['READ']}, - {{"tester2", "TESTID2"}, ['WRITE']}], - Grants2 = [{{"tester2", "TESTID1"}, ['READ']}, - {{"tester1", "TESTID2"}, ['WRITE']}], - Acl1 = riak_cs_acl_utils:acl("tester1", "TESTID1", "TESTKEYID1", Grants1), - Acl2 = riak_cs_acl_utils:acl("tester1", "TESTID1", "TESTKEYID1", Grants2), - ?assertEqual(Xml, riak_cs_xml:to_xml(Acl1)), - ?assertNotEqual(Xml, riak_cs_xml:to_xml(Acl2)). - -list_objects_response_to_xml_test() -> - Xml = <<"bucket1000falsetestkey12012-11-29T17:50:30.000Z\"fba9dede6af29711d7271245a35813428\"12345STANDARDTESTID1tester1testkey22012-11-29T17:52:30.000Z\"43433281b2f27731ccf53597645a3985\"54321STANDARDTESTID2tester2">>, - Owner1 = #list_objects_owner_v1{id = <<"TESTID1">>, display_name = <<"tester1">>}, - Owner2 = #list_objects_owner_v1{id = <<"TESTID2">>, display_name = <<"tester2">>}, - Content1 = ?LOKC{key = <<"testkey1">>, - last_modified = riak_cs_wm_utils:to_iso_8601("Thu, 29 Nov 2012 17:50:30 GMT"), - etag = <<"\"fba9dede6af29711d7271245a35813428\"">>, - size = 12345, - owner = Owner1, - storage_class = <<"STANDARD">>}, - Content2 = ?LOKC{key = <<"testkey2">>, - last_modified = riak_cs_wm_utils:to_iso_8601("Thu, 29 Nov 2012 17:52:30 GMT"), - etag = <<"\"43433281b2f27731ccf53597645a3985\"">>, - size = 54321, - owner = Owner2, - storage_class = <<"STANDARD">>}, - ListObjectsResponse = ?LORESP{name = <<"bucket">>, - max_keys = 1000, - prefix = undefined, - delimiter = undefined, - marker = undefined, - is_truncated = false, - contents = [Content1, Content2], - common_prefixes = []}, - ?assertEqual(Xml, riak_cs_xml:to_xml(ListObjectsResponse)). - -user_record_to_xml_test() -> - Xml = <<"barf@spaceballs.combarfbarfolomewbarf_keysecretsauce1234enabled">>, - User = ?RCS_USER{name="barfolomew", - display_name="barf", - email="barf@spaceballs.com", - key_id="barf_key", - key_secret="secretsauce", - canonical_id="1234", - status=enabled}, - ?assertEqual(Xml, riak_cs_xml:to_xml(User)). - --endif. diff --git a/src/riak_kv_fs2_backend.erl b/src/riak_kv_fs2_backend.erl deleted file mode 100644 index a7c657cbc..000000000 --- a/src/riak_kv_fs2_backend.erl +++ /dev/null @@ -1,480 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% riak_fs2_backend: storage engine based on basic filesystem access -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -% @doc riak_kv_fs2_backend is filesystem storage system, Mark III - --module(riak_kv_fs2_backend). -%% -behavior(riak_kv_backend). % Not building within riak_kv - -%% KV Backend API --export([api_version/0, - capabilities/1, - capabilities/2, - start/2, - stop/1, - get/3, - put/5, - delete/4, - drop/1, - fold_buckets/4, - fold_keys/4, - fold_objects/4, - is_empty/1, - status/1, - callback/3]). - --define(API_VERSION, 1). --define(CAPABILITIES, [async_fold]). - -%% Borrowed from bitcask.hrl --define(VALSIZEFIELD, 32). --define(CRCSIZEFIELD, 32). --define(HEADER_SIZE, 8). % Differs from bitcask.hrl! --define(MAXVALSIZE, 2#11111111111111111111111111111111). - --ifdef(TEST). --ifdef(TEST_IN_RIAK_KV). --include_lib("eunit/include/eunit.hrl"). --endif. --endif. - --record(state, {dir}). --type state() :: #state{}. --type config() :: [{atom(), term()}]. - -%% =================================================================== -%% Public API -%% =================================================================== - -%% @doc Return the major version of the -%% current API. --spec api_version() -> {ok, integer()}. -api_version() -> - {ok, ?API_VERSION}. - -%% @doc Return the capabilities of the backend. --spec capabilities(state()) -> {ok, [atom()]}. -capabilities(_) -> - {ok, ?CAPABILITIES}. - -%% @doc Return the capabilities of the backend. --spec capabilities(riak_object:bucket(), state()) -> {ok, [atom()]}. -capabilities(_, _) -> - {ok, ?CAPABILITIES}. - -%% @doc Start this backend. 'riak_kv_fs_backend_root' must be set in -%% Riak's application environment. It must be set to a string -%% representing the base directory where this backend should -%% store its files. --spec start(integer(), config()) -> {ok, state()}. -start(Partition, Config) -> - PartitionName = integer_to_list(Partition), - ConfigRoot = app_helper:get_prop_or_env(fs2_backend_data_root, Config, riak_kv), - if - ConfigRoot =:= undefined -> - riak:stop("fs_backend_data_root unset, failing"); - true -> - ok - end, - Dir = filename:join([ConfigRoot,PartitionName]), - {ok = filelib:ensure_dir(Dir), #state{dir=Dir}}. - -%% @doc Stop the backend --spec stop(state()) -> ok. -stop(_State) -> ok. - -%% @doc Get the object stored at the given bucket/key pair --spec get(riak_object:bucket(), riak_object:key(), state()) -> - {ok, any(), state()} | - {ok, not_found, state()} | - {error, term(), state()}. -get(Bucket, Key, State) -> - File = location(State, {Bucket, Key}), - case filelib:is_file(File) of - false -> - {error, not_found, State}; - true -> - {ok, Bin} = file:read_file(File), - case unpack_ondisk(Bin) of - bad_crc -> - %% TODO logging? - {error, not_found, State}; - Val -> - {ok, Val, State} - end - end. - -%% @doc Store Val under Bkey --type index_spec() :: {add, Index, SecondaryKey} | {remove, Index, SecondaryKey}. --spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> - {ok, state()} | - {error, term(), state()}. -put(Bucket, PrimaryKey, _IndexSpecs, Val, State) -> - File = location(State, {Bucket, PrimaryKey}), - case filelib:ensure_dir(File) of - ok -> {atomic_write(File, Val), State}; - {error, X} -> {error, X, State} - end. - -%% @doc Delete the object stored at BKey --spec delete(riak_object:bucket(), riak_object:key(), [index_spec()], state()) -> - {ok, state()} | - {error, term(), state()}. -delete(Bucket, Key, _IndexSpecs, State) -> - File = location(State, {Bucket, Key}), - case file:delete(File) of - ok -> {ok, State}; - {error, enoent} -> {ok, State}; - {error, Err} -> {error, Err, State} - end. - -%% @doc Fold over all the buckets. --spec fold_buckets(riak_kv_backend:fold_buckets_fun(), - any(), - [], - state()) -> {ok, any()} | {async, fun()}. -fold_buckets(FoldBucketsFun, Acc, Opts, State) -> - FoldFun = fold_buckets_fun(FoldBucketsFun), - BucketFolder = - fun() -> - {FoldResult, _} = - lists:foldl(FoldFun, {Acc, sets:new()}, list(State)), - FoldResult - end, - case lists:member(async_fold, Opts) of - true -> - {async, BucketFolder}; - false -> - {ok, BucketFolder()} - end. - -%% @doc Fold over all the keys for one or all buckets. --spec fold_keys(riak_kv_backend:fold_keys_fun(), - any(), - [{atom(), term()}], - state()) -> {ok, term()} | {async, fun()}. -fold_keys(FoldKeysFun, Acc, Opts, State) -> - Bucket = proplists:get_value(bucket, Opts), - FoldFun = fold_keys_fun(FoldKeysFun, Bucket), - KeyFolder = - fun() -> - lists:foldl(FoldFun, Acc, list(State)) - end, - case lists:member(async_fold, Opts) of - true -> - {async, KeyFolder}; - false -> - {ok, KeyFolder()} - end. - -%% @doc Fold over all the objects for one or all buckets. --spec fold_objects(riak_kv_backend:fold_objects_fun(), - any(), - [{atom(), term()}], - state()) -> {ok, any()} | {async, fun()}. -fold_objects(FoldObjectsFun, Acc, Opts, State) -> - %% Warning: This ain't pretty. Hold your nose. - Bucket = proplists:get_value(bucket, Opts), - FoldFun = fold_objects_fun(FoldObjectsFun, Bucket), - ObjectFolder = - fun() -> - fold(State, FoldFun, Acc) - end, - case lists:member(async_fold, Opts) of - true -> - {async, ObjectFolder}; - false -> - {ok, ObjectFolder()} - end. - -%% @doc Delete all objects from this backend -%% and return a fresh reference. --spec drop(state()) -> {ok, state()}. -drop(State=#state{dir=Dir}) -> - _ = [file:delete(location(State, BK)) || BK <- list(State)], - Cmd = io_lib:format("rm -Rf ~s", [Dir]), - _ = os:cmd(Cmd), - ok = filelib:ensure_dir(Dir), - {ok, State}. - -%% @doc Returns true if this backend contains any -%% non-tombstone values; otherwise returns false. --spec is_empty(state()) -> boolean(). -is_empty(S) -> - list(S) == []. - -%% @doc Get the status information for this fs backend --spec status(state()) -> [no_status_sorry | {atom(), term()}]. -status(_S) -> - [no_status_sorry]. - -%% @doc Register an asynchronous callback --spec callback(reference(), any(), state()) -> {ok, state()}. -callback(_Ref, _Term, S) -> - {ok, S}. - -%% =================================================================== -%% Internal functions -%% =================================================================== - -%% @spec atomic_write(File :: string(), Val :: binary()) -> -%% ok | {error, Reason :: term()} -%% @doc store a atomic value to disk. Write to temp file and rename to -%% normal path. -atomic_write(File, Val) -> - FakeFile = File ++ ".tmpwrite", - case file:write_file(FakeFile, pack_ondisk(Val)) of - ok -> - file:rename(FakeFile, File); - X -> X - end. - -%% @private -%% Fold over the keys and objects on this backend -fold(State, Fun0, Acc) -> - Fun = fun(BKey, AccIn) -> - {Bucket, Key} = BKey, - case get(Bucket, Key, State) of - {ok, Bin, _} -> - Fun0(BKey, Bin, AccIn); - _ -> - AccIn - end - end, - lists:foldl(Fun, Acc, list(State)). - -%% @private -%% Return a function to fold over the buckets on this backend -fold_buckets_fun(FoldBucketsFun) -> - fun(BKey, {Acc, BucketSet}) -> - {Bucket, _} = BKey, - case sets:is_element(Bucket, BucketSet) of - true -> - {Acc, BucketSet}; - false -> - {FoldBucketsFun(Bucket, Acc), - sets:add_element(Bucket, BucketSet)} - end - end. - -%% @private -%% Return a function to fold over keys on this backend -fold_keys_fun(FoldKeysFun, undefined) -> - fun(BKey, Acc) -> - {Bucket, Key} = BKey, - FoldKeysFun(Bucket, Key, Acc) - end; -fold_keys_fun(FoldKeysFun, Bucket) -> - fun(BKey, Acc) -> - {B, Key} = BKey, - case B =:= Bucket of - true -> - FoldKeysFun(Bucket, Key, Acc); - false -> - Acc - end - end. - -%% @private -%% Return a function to fold over the objects on this backend -fold_objects_fun(FoldObjectsFun, undefined) -> - fun(BKey, Value, Acc) -> - {Bucket, Key} = BKey, - FoldObjectsFun(Bucket, Key, Value, Acc) - end; -fold_objects_fun(FoldObjectsFun, Bucket) -> - fun(BKey, Value, Acc) -> - {B, Key} = BKey, - case B =:= Bucket of - true -> - FoldObjectsFun(Bucket, Key, Value, Acc); - false -> - Acc - end - end. - -%% @spec list(state()) -> [{Bucket :: riak_object:bucket(), -%% Key :: riak_object:key()}] -%% @doc Get a list of all bucket/key pairs stored by this backend -list(#state{dir=Dir}) -> - % this is slow slow slow - % B,N,N,N,K - [location_to_bkey(X) || X <- filelib:wildcard("*/*/*/*/*", - Dir)]. - -%% @spec location(state(), {riak_object:bucket(), riak_object:key()}) -%% -> string() -%% @doc produce the file-path at which the object for the given Bucket -%% and Key should be stored -location(State, {Bucket, Key}) -> - B64 = encode_bucket(Bucket), - K64 = encode_key(Key), - [N1,N2,N3] = nest(K64), - filename:join([State#state.dir, B64, N1, N2, N3, K64]). - -%% @spec location_to_bkey(string()) -> -%% {riak_object:bucket(), riak_object:key()} -%% @doc reconstruct a Riak bucket/key pair, given the location at -%% which its object is stored on-disk -location_to_bkey(Path) -> - [B64,_,_,_,K64] = string:tokens(Path, "/"), - {decode_bucket(B64), decode_key(K64)}. - -%% @spec decode_bucket(string()) -> binary() -%% @doc reconstruct a Riak bucket, given a filename -%% @see encode_bucket/1 -decode_bucket(B64) -> - base64:decode(dirty(B64)). - -%% @spec decode_key(string()) -> binary() -%% @doc reconstruct a Riak object key, given a filename -%% @see encode_key/1 -decode_key(K64) -> - base64:decode(dirty(K64)). - -%% @spec dirty(string()) -> string() -%% @doc replace filename-troublesome base64 characters -%% @see clean/1 -dirty(Str64) -> - lists:map(fun($-) -> $=; - ($_) -> $+; - ($,) -> $/; - (C) -> C - end, - Str64). - -%% @spec encode_bucket(binary()) -> string() -%% @doc make a filename out of a Riak bucket -encode_bucket(Bucket) -> - clean(base64:encode_to_string(Bucket)). - -%% @spec encode_key(binary()) -> string() -%% @doc make a filename out of a Riak object key -encode_key(Key) -> - clean(base64:encode_to_string(Key)). - -%% @spec clean(string()) -> string() -%% @doc remove characters from base64 encoding, which may -%% cause trouble with filenames -clean(Str64) -> - lists:map(fun($=) -> $-; - ($+) -> $_; - ($/) -> $,; - (C) -> C - end, - Str64). - -%% @spec nest(string()) -> [string()] -%% @doc create a directory nesting, to keep the number of -%% files in a directory smaller -nest(Key) -> nest(lists:reverse(string:substr(Key, 1, 6)), 3, []). -nest(_, 0, Parts) -> Parts; -nest([Nb,Na|Rest],N,Acc) -> - nest(Rest, N-1, [[Na,Nb]|Acc]); -nest([Na],N,Acc) -> - nest([],N-1,[[Na]|Acc]); -nest([],N,Acc) -> - nest([],N-1,["0"|Acc]). - -%% Borrowed from bitcask_fileops.erl and then mangled --spec pack_ondisk(binary()) -> [binary()]. -pack_ondisk(Bin) -> - ValueSz = size(Bin), - true = (ValueSz =< ?MAXVALSIZE), - Bytes0 = [<>, Bin], - [<<(erlang:crc32(Bytes0)):?CRCSIZEFIELD>> | Bytes0]. - --spec unpack_ondisk(binary()) -> binary() | bad_crc. -unpack_ondisk(<>) -> - case erlang:crc32(Bytes) of - Crc32 -> - <> = Bytes, - Value; - _BadCrc -> - bad_crc - end. - -%% =================================================================== -%% EUnit tests -%% =================================================================== --ifdef(TEST). --ifdef(TEST_IN_RIAK_KV). - -%% Broken test: -simple_test_() -> - ?assertCmd("rm -rf test/fs-backend"), - Config = [{fs2_backend_data_root, "test/fs-backend"}], - riak_kv_backend:standard_test(?MODULE, Config). - -dirty_clean_test() -> - Dirty = "abc=+/def", - Clean = clean(Dirty), - [ ?assertNot(lists:member(C, Clean)) || C <- "=+/" ], - ?assertEqual(Dirty, dirty(Clean)). - -nest_test() -> - ?assertEqual(["ab","cd","ef"],nest("abcdefg")), - ?assertEqual(["ab","cd","ef"],nest("abcdef")), - ?assertEqual(["a","bc","de"], nest("abcde")), - ?assertEqual(["0","ab","cd"], nest("abcd")), - ?assertEqual(["0","a","bc"], nest("abc")), - ?assertEqual(["0","0","ab"], nest("ab")), - ?assertEqual(["0","0","a"], nest("a")), - ?assertEqual(["0","0","0"], nest([])). - --ifdef(EQC). -%% Broken test: -%% eqc_test() -> -%% Cleanup = fun(_State,_Olds) -> os:cmd("rm -rf test/fs-backend") end, -%% Config = [{riak_kv_fs_backend_root, "test/fs-backend"}], -%% ?assertCmd("rm -rf test/fs-backend"), -%% ?assertEqual(true, backend_eqc:test(?MODULE, false, Config, Cleanup)). - -eqc_test_() -> - {spawn, - [{inorder, - [{setup, - fun setup/0, - fun cleanup/1, - [ - {timeout, 60000, - [?_assertEqual(true, - backend_eqc:test(?MODULE, - false, - [{fs2_backend_data_root, - "test/fs-backend"}]))]} - ]}]}]}. - -setup() -> - application:load(sasl), - application:set_env(sasl, sasl_error_logger, - {file, "riak_kv_fs2_backend_eqc_sasl.log"}), - error_logger:tty(false), - error_logger:logfile({open, "riak_kv_fs2_backend_eqc.log"}), - ok. - -cleanup(_) -> - os:cmd("rm -rf test/fs-backend/*"). - --endif. % EQC --endif. % TEST_IN_RIAK_KV --endif. % TEST diff --git a/src/velvet.erl b/src/velvet.erl deleted file mode 100644 index 9a1242225..000000000 --- a/src/velvet.erl +++ /dev/null @@ -1,392 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc Client module for interacting with `stanchion' application. - --module(velvet). - --export([create_bucket/5, - create_user/5, - delete_bucket/5, - ping/3, - set_bucket_acl/6, - set_bucket_policy/6, - delete_bucket_policy/5, - update_user/6 - % @TODO: update_bucket/3 - ]). - -%% =================================================================== -%% Public API -%% =================================================================== - -%% @doc Create a bucket for a requesting party. --spec create_bucket(string(), - pos_integer(), - string(), - string(), - [{atom(), term()}]) -> ok | {error, term()}. -create_bucket(Ip, Port, ContentType, BucketDoc, Options) -> - Ssl = proplists:get_value(ssl, Options, true), - AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), - Path = buckets_path(<<>>), - Url = url(Ip, Port, Ssl, Path), - Headers0 = [{"Content-Md5", content_md5(BucketDoc)}, - {"Date", httpd_util:rfc1123_date()}], - case AuthCreds of - {_, _} -> - Headers = - [{"Authorization", auth_header('POST', - ContentType, - Headers0, - Path, - AuthCreds)} | - Headers0]; - no_auth_creds -> - Headers = Headers0 - end, - case request(post, Url, [201], ContentType, Headers, BucketDoc) of - {ok, {{_, 201, _}, _RespHeaders, _RespBody}} -> - ok; - {error, {ok, {{_, StatusCode, Reason}, _RespHeaders, RespBody}}} -> - {error, {error_status, StatusCode, Reason, RespBody}}; - {error, Error} -> - {error, Error} - end. - -%% @doc Create a bucket for a requesting party. --spec create_user(string(), - pos_integer(), - string(), - string(), - [{atom(), term()}]) -> ok | {error, term()}. -create_user(Ip, Port, ContentType, UserDoc, Options) -> - Ssl = proplists:get_value(ssl, Options, true), - AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), - Path = users_path([]), - Url = url(Ip, Port, Ssl, Path), - Headers0 = [{"Content-Md5", content_md5(UserDoc)}, - {"Date", httpd_util:rfc1123_date()}], - case AuthCreds of - {_, _} -> - Headers = - [{"Authorization", auth_header('POST', - ContentType, - Headers0, - Path, - AuthCreds)} | - Headers0]; - no_auth_creds -> - Headers = Headers0 - end, - case request(post, Url, [201], ContentType, Headers, UserDoc) of - {ok, {{_, 201, _}, _RespHeaders, _RespBody}} -> - ok; - {error, {ok, {{_, StatusCode, Reason}, _RespHeaders, RespBody}}} -> - {error, {error_status, StatusCode, Reason, RespBody}}; - {error, Error} -> - {error, Error} - end. - -%% @doc Delete a bucket. The bucket must be owned by -%% the requesting party. --spec delete_bucket(string(), - pos_integer(), - binary(), - string(), - [{atom(), term()}]) -> ok | {error, term()}. -delete_bucket(Ip, Port, Bucket, Requester, Options) -> - Ssl = proplists:get_value(ssl, Options, true), - AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), - QS = requester_qs(Requester), - Path = buckets_path(Bucket), - Url = url(Ip, Port, Ssl, stringy(Path ++ QS)), - Headers0 = [{"Date", httpd_util:rfc1123_date()}], - case AuthCreds of - {_, _} -> - Headers = - [{"Authorization", auth_header('DELETE', - [], - Headers0, - Path, - AuthCreds)} | - Headers0]; - no_auth_creds -> - Headers = Headers0 - end, - case request(delete, Url, [204], Headers) of - {ok, {{_, 204, _}, _RespHeaders, _}} -> - ok; - {error, {ok, {{_, StatusCode, Reason}, _RespHeaders, RespBody}}} -> - {error, {error_status, StatusCode, Reason, RespBody}}; - {error, Error} -> - {error, Error} - end. - -%% @doc Ping the server by requesting the "/ping" resource. --spec ping(string(), pos_integer(), boolean()) -> ok | {error, term()}. -ping(Ip, Port, Ssl) -> - Url = ping_url(Ip, Port, Ssl), - case request(get, Url, [200, 204]) of - {ok, {{_, _Status, _}, _Headers, _Body}} -> - ok; - {error, Error} -> - {error, Error} - end. - -%% @doc Create a bucket for a requesting party. --spec set_bucket_acl(string(), - inet:port_number(), - binary(), - string(), - string(), - [{atom(), term()}]) -> ok | {error, term()}. -set_bucket_acl(Ip, Port, Bucket, ContentType, AclDoc, Options) -> - Path = buckets_path(Bucket, acl), - update_bucket(Ip, Port, Path, ContentType, AclDoc, Options, 204). - -%% @doc Create a bucket for a requesting party. --spec set_bucket_policy(string(), - inet:port_number(), - binary(), - string(), - string(), - proplists:proplist()) -> ok | {error, term()}. -set_bucket_policy(Ip, Port, Bucket, ContentType, PolicyDoc, Options) -> - Path = buckets_path(Bucket, policy), - update_bucket(Ip, Port, Path, ContentType, PolicyDoc, Options, 204). - -%% @doc Delete a bucket. The bucket must be owned by -%% the requesting party. --spec delete_bucket_policy(string(), - pos_integer(), - binary(), - string(), - [{atom(), term()}]) -> ok | {error, term()}. -delete_bucket_policy(Ip, Port, Bucket, Requester, Options) -> - Ssl = proplists:get_value(ssl, Options, true), - AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), - QS = requester_qs(Requester), - Path = buckets_path(Bucket, policy), - Url = url(Ip, Port, Ssl, stringy(Path ++ QS)), - Headers0 = [{"Date", httpd_util:rfc1123_date()}], - case AuthCreds of - {_, _} -> - Headers = - [{"Authorization", auth_header('DELETE', - [], - Headers0, - Path, - AuthCreds)} | - Headers0]; - no_auth_creds -> - Headers = Headers0 - end, - case request(delete, Url, [204], Headers) of - {ok, {{_, 204, _}, _RespHeaders, _}} -> - ok; - {error, {ok, {{_, StatusCode, Reason}, _RespHeaders, RespBody}}} -> - {error, {error_status, StatusCode, Reason, RespBody}}; - {error, Error} -> - {error, Error} - end. - -%% @doc Update a user record --spec update_user(string(), - pos_integer(), - string(), - string(), - string(), - [{atom(), term()}]) -> ok | {error, term()}. -update_user(Ip, Port, ContentType, KeyId, UserDoc, Options) -> - Ssl = proplists:get_value(ssl, Options, true), - AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), - Path = users_path(KeyId), - Url = url(Ip, Port, Ssl, Path), - Headers0 = [{"Content-Md5", content_md5(UserDoc)}, - {"Date", httpd_util:rfc1123_date()}], - case AuthCreds of - {_, _} -> - Headers = - [{"Authorization", auth_header('PUT', - ContentType, - Headers0, - Path, - AuthCreds)} | - Headers0]; - no_auth_creds -> - Headers = Headers0 - end, - case request(put, Url, [204], ContentType, Headers, UserDoc) of - {ok, {{_, 204, _}, _RespHeaders, _RespBody}} -> - ok; - {error, {ok, {{_, StatusCode, Reason}, _RespHeaders, RespBody}}} -> - {error, {error_status, StatusCode, Reason, RespBody}}; - {error, Error} -> - {error, Error} - end. - -%% =================================================================== -%% Internal functions -%% =================================================================== - -% @doc send request to stanchion server -% @TODO merge with create_bucket, create_user, delete_bucket --spec update_bucket(string(), inet:port_number(), string(), - string(), string(), proplists:proplist(), - pos_integer()) -> - ok | {error, term()}. -update_bucket(Ip, Port, Path, ContentType, Doc, Options, Expect) -> - AuthCreds = proplists:get_value(auth_creds, Options, no_auth_creds), - Ssl = proplists:get_value(ssl, Options, true), - Url = url(Ip, Port, Ssl, Path), - Headers0 = [{"Content-Md5", content_md5(Doc)}, - {"Date", httpd_util:rfc1123_date()}], - case AuthCreds of - {_, _} -> - Headers = - [{"Authorization", auth_header('PUT', - ContentType, - Headers0, - Path, - AuthCreds)} | - Headers0]; - no_auth_creds -> - Headers = Headers0 - end, - case request(put, Url, [Expect], ContentType, Headers, Doc) of - {ok, {{_, Expect, _}, _RespHeaders, _RespBody}} -> - ok; - {error, {ok, {{_, StatusCode, Reason}, _RespHeaders, RespBody}}} -> - {error, {error_status, StatusCode, Reason, RespBody}}; - {error, Error} -> - {error, Error} - end. - -%% @doc Assemble the root URL for the given client --spec root_url(string(), pos_integer(), boolean()) -> [string()]. -root_url(Ip, Port, true) -> - ["https://", Ip, ":", integer_to_list(Port)]; -root_url(Ip, Port, false) -> - ["http://", Ip, ":", integer_to_list(Port)]. - -%% @doc Assemble the URL for the ping resource --spec ping_url(string(), pos_integer(), boolean()) -> string(). -ping_url(Ip, Port, Ssl) -> - lists:flatten([root_url(Ip, Port, Ssl), "ping/"]). - -%% @doc Assemble the path for a bucket request --spec buckets_path(binary()) -> string(). -buckets_path(Bucket) -> - stringy(["/buckets", - ["/" ++ binary_to_list(Bucket) || Bucket /= <<>>]]). - -%% @doc Assemble the path for a bucket request --spec buckets_path(binary(), acl|policy) -> string(). -buckets_path(Bucket, acl) -> - stringy([buckets_path(Bucket), "/acl"]); -buckets_path(Bucket, policy) -> - stringy([buckets_path(Bucket), "/policy"]). - -%% @doc Assemble the URL for a buckets request --spec url(string(), pos_integer(), boolean(), [string()]) -> - string(). -url(Ip, Port, Ssl, Path) -> - lists:flatten( - [root_url(Ip, Port, Ssl), - Path - ]). - -%% @doc send an HTTP request where `Expect' is a list -%% of expected HTTP status codes. --spec request(atom(), string(), [pos_integer()]) -> - {ok, {term(), term(), term()}} | {error, term()}. -request(Method, Url, Expect) -> - request(Method, Url, Expect, [], [], []). - -%% @doc send an HTTP request where `Expect' is a list -%% of expected HTTP status codes. --spec request(atom(), string(), [pos_integer()], [{string(), string()}]) -> - {ok, {term(), term(), term()}} | {error, term()}. -request(Method, Url, Expect, Headers) -> - request(Method, Url, Expect, [], Headers, []). - -%% @doc send an HTTP request where `Expect' is a list -%% of expected HTTP status codes. --spec request(atom(), - string(), - [pos_integer()], - string(), - [{string(), string()}], - string()) -> {ok, {term(), term(), term()}} | {error, term()}. -request(Method, Url, Expect, ContentType, Headers, Body) -> - case Method == put orelse - Method == post of - true -> - Request = {Url, Headers, ContentType, Body}; - false -> - Request = {Url, Headers} - end, - case httpc:request(Method, Request, [], []) of - Resp={ok, {{_, Status, _}, _RespHeaders, _RespBody}} -> - case lists:member(Status, Expect) of - true -> Resp; - false -> {error, Resp} - end; - Error -> - Error - end. - -%% @doc Calculate an MD5 hash of a request body. --spec content_md5(string()) -> string(). -content_md5(Body) -> - base64:encode_to_string(riak_cs_utils:md5(list_to_binary(Body))). - -%% @doc Construct a MOSS authentication header --spec auth_header(atom(), - string(), - [{string() | atom() | binary(), string()}], - string(), - {string(), iodata()}) -> nonempty_string(). -auth_header(HttpVerb, ContentType, Headers, Path, {AuthKey, AuthSecret}) -> - Signature = velvet_auth:request_signature(HttpVerb, - [{"content-type", ContentType} | - Headers], - Path, - AuthSecret), - "MOSS " ++ AuthKey ++ ":" ++ Signature. - -%% @doc Assemble a requester query string for -%% user in a bucket deletion request. --spec requester_qs(string()) -> string(). -requester_qs(Requester) -> - "?requester=" ++ - mochiweb_util:quote_plus(Requester). - -%% @doc Assemble the path for a users request --spec users_path(string()) -> string(). -users_path(User) -> - stringy(["/users", - ["/" ++ User || User /= []] - ]). - --spec stringy(string() | list(string())) -> string(). -stringy(List) -> - lists:flatten(List). diff --git a/test/riak_cs_list_objects_fsm_eqc.erl b/test/riak_cs_list_objects_fsm_eqc.erl deleted file mode 100644 index 035dc3588..000000000 --- a/test/riak_cs_list_objects_fsm_eqc.erl +++ /dev/null @@ -1,108 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(riak_cs_list_objects_fsm_eqc). - --ifdef(EQC). - --compile(export_all). --include_lib("eqc/include/eqc.hrl"). --include_lib("eqc/include/eqc_fsm.hrl"). --include_lib("eunit/include/eunit.hrl"). - -%% eqc properties --export([prop_index_of_first_greater_element/0]). - -%% Helpers --export([test/0, - test/1]). - --define(TEST_ITERATIONS, 500). --define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). - -%%==================================================================== -%% Eunit tests -%%==================================================================== - -eqc_test() -> - ?assert(quickcheck(numtests(?TEST_ITERATIONS, ?QC_OUT(prop_index_of_first_greater_element())))). - -%% ==================================================================== -%% EQC Properties -%% ==================================================================== - -prop_index_of_first_greater_element() -> - ?FORALL({L, E}, list_and_element(), - begin - Index = riak_cs_list_objects_fsm:index_of_first_greater_element(L, E), - index_of_first_greater_element_correct(L, Index, E) - end). - -index_of_first_greater_element_correct([], 1, _Elem) -> - true; -index_of_first_greater_element_correct([H], 1, Elem) -> - Elem =< H; -index_of_first_greater_element_correct([H], 2, Elem) -> - Elem >= H; -index_of_first_greater_element_correct([H | _Tail], 1, Elem) -> - Elem =< H; -index_of_first_greater_element_correct(List, Index, Elem) - when Index > length(List) -> - Elem >= lists:last(List); -index_of_first_greater_element_correct(List, Index, Elem) -> - Elem =< lists:nth(Index, List). - -%%==================================================================== -%% Helpers -%%==================================================================== - -test() -> - test(?TEST_ITERATIONS). - -test(Iterations) -> - eqc:quickcheck(eqc:numtests(Iterations, prop_index_of_first_greater_element())). - - -%%==================================================================== -%% Generators -%%==================================================================== - -%% Return a tuple of {A, B}. A is a sorted list of non-negative numbers, -%% where elements only appear once. B is either a member of that list, -%% or isn't, with some likelihood near 50%. -list_and_element() -> - ?LET(L, sorted_unique(), {L, element_or_not(L)}). - -%% Given a list, return a non-negative int if the list is empty, -%% otherwise, return an int that is either in the list or not, with some -%% likelihood. -element_or_not([]) -> - positive_int(); -element_or_not(List) -> - oneof([positive_int(), elements(List)]). - -positive_int() -> - ?LET(I, int(), abs(I)). - -sorted_unique() -> - ?LET(L, list(positive_int()), lists:usort(L)). - --endif. diff --git a/test/riak_cs_s3_policy_test.erl b/test/riak_cs_s3_policy_test.erl deleted file mode 100644 index 9f049c341..000000000 --- a/test/riak_cs_s3_policy_test.erl +++ /dev/null @@ -1,262 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - -%% @doc ad-hoc policy tests - --module(riak_cs_s3_policy_test). - --compile(export_all). - --include("riak_cs.hrl"). --include("s3_api.hrl"). --include_lib("webmachine/include/wm_reqdata.hrl"). - --ifdef(TEST). - --include_lib("eunit/include/eunit.hrl"). - -parse_ip_test_()-> - [ - ?_assertEqual({{192,0,0,1}, {255,0,0,0}}, - riak_cs_s3_policy:parse_ip(<<"192.0.0.1/8">>)), - ?_assertEqual({error, einval}, - riak_cs_s3_policy:parse_ip(<<"192.3.1/16">>)), - ?_assertEqual(<<"1.2.3.4">>, - riak_cs_s3_policy:print_ip(riak_cs_s3_policy:parse_ip(<<"1.2.3.4">>))), - ?_assertEqual(<<"1.2.3.4/13">>, - riak_cs_s3_policy:print_ip(riak_cs_s3_policy:parse_ip(<<"1.2.3.4/13">>))), - ?_assertEqual({error, einval}, - riak_cs_s3_policy:parse_ip(<<"0">>)), - ?_assertEqual({error, einval}, - riak_cs_s3_policy:parse_ip(<<"0/0">>)) - ]. - -empty_statement_conversion_test()-> - Policy = ?POLICY{id= <<"hello">>, statement=[#statement{}]}, - JsonPolicy = "{\"Version\":\"2008-10-17\",\"Statement\":[" - "{\"Sid\":\"undefined\",\"Effect\":\"Deny\",\"Principal\":[]," - "\"Action\":[],\"NotAction\":[],\"Resource\":[],\"Condition\":[]}" - "],\"Id\":\"hello\"}", - {struct, LHS} = mochijson2:decode(JsonPolicy), - {struct, RHS} = mochijson2:decode(riak_cs_s3_policy:policy_to_json_term(Policy)), - ?assertEqual(lists:sort(LHS), lists:sort(RHS)), - {ok, PolicyFromJson} = riak_cs_s3_policy:policy_from_json(list_to_binary(JsonPolicy)), - ?assertEqual(Policy?POLICY.id, PolicyFromJson?POLICY.id), - ?assertEqual(Policy?POLICY.version, PolicyFromJson?POLICY.version). - -sample_plain_allow_policy()-> - <<"{" - "\"Id\":\"Policy1354069963875\"," - "\"Statement\":[" - "{" - " \"Sid\":\"Stmt1354069958376\"," - " \"Action\":[" - " \"s3:CreateBucket\"," - " \"s3:DeleteBucket\"," - " \"s3:DeleteBucketPolicy\"," - " \"s3:DeleteObject\"," - " \"s3:GetBucketAcl\"," - " \"s3:GetBucketPolicy\"," - " \"s3:GetObject\"," - " \"s3:GetObjectAcl\"," - " \"s3:ListAllMyBuckets\"," - " \"s3:ListBucket\"," - " \"s3:PutBucketAcl\"," - " \"s3:PutBucketPolicy\"," -% " \"s3:PutObject\"," - " \"s3:PutObjectAcl\"" - " ]," - " \"Condition\":{" - " \"IpAddress\": { \"aws:SourceIp\":[\"192.168.0.1/8\", \"192.168.0.2/17\"] }" - " }," - " \"Effect\": \"Allow\"," - " \"Resource\": \"arn:aws:s3:::test\"," - " \"Principal\": {" - " \"AWS\": \"*\"" - " }" - " }" - " ]" - "}" >>. - -sample_policy_check_test()-> - application:set_env(riak_cs, trust_x_forwarded_for, true), - JsonPolicy0 = sample_plain_allow_policy(), - {ok, Policy} = riak_cs_s3_policy:policy_from_json(JsonPolicy0), - Access = #access_v1{method='GET', target=object, id="spam/ham/egg", - req = #wm_reqdata{peer="192.168.0.1"}, bucket= <<"test">>}, - ?assert(riak_cs_s3_policy:eval(Access, Policy)), - % io:format(standard_error, "~w~n", [Policy]), - Access2 = Access#access_v1{method='PUT', target=object}, - ?assertEqual(undefined, riak_cs_s3_policy:eval(Access2, Policy)), - Access3 = Access#access_v1{req=#wm_reqdata{peer="1.1.1.1"}}, - ?assertEqual(undefined, riak_cs_s3_policy:eval(Access3, Policy)). - -sample_conversion_test()-> - JsonPolicy0 = sample_plain_allow_policy(), - {ok, Policy} = riak_cs_s3_policy:policy_from_json(JsonPolicy0), - {ok, PolicyFromJson} = riak_cs_s3_policy:policy_from_json(riak_cs_s3_policy:policy_to_json_term(Policy)), - ?assertEqual(Policy?POLICY.id, PolicyFromJson?POLICY.id), - ?assert(lists:all(fun({LHS, RHS}) -> - riak_cs_s3_policy:statement_eq(LHS, RHS) - end, - lists:zip(Policy?POLICY.statement, - PolicyFromJson?POLICY.statement))), - ?assertEqual(Policy?POLICY.version, PolicyFromJson?POLICY.version). - - -eval_all_ip_addr_test() -> - ?assert(riak_cs_s3_policy:eval_all_ip_addr([{{192,168,0,1},{255,255,255,255}}], {192,168,0,1})), - ?assert(not riak_cs_s3_policy:eval_all_ip_addr([{{192,168,0,1},{255,255,255,255}}], {192,168,25,1})), - ?assert(riak_cs_s3_policy:eval_all_ip_addr([{{192,168,0,1},{255,255,255,0}}], {192,168,0,23})). - -eval_ip_address_test()-> - ?assert(riak_cs_s3_policy:eval_ip_address(#wm_reqdata{peer = "23.23.23.23"}, - [garbage,{chiba, boo},"saitama", - {'aws:SourceIp', {{23,23,0,0},{255,255,0,0}}}, hage])). - -eval_ip_address_test_trust_x_forwarded_for_false_test() -> - application:set_env(riak_cs, trust_x_forwarded_for, false), - Conds = [garbage,{chiba, boo},"saitama", - {'aws:SourceIp', {{23,23,0,0},{255,255,0,0}}}, hage], - %% This test fails because it tries to use the socket from wm_reqstate to - %% get the peer address, but it's not a real wm request. - %% If trust_x_forwarded_for = true, it would just use the peer address and the call would - %% succeed - ?assertError({badrecord, wm_reqstate}, - riak_cs_s3_policy:eval_ip_address(#wm_reqdata{peer="23.23.23.23"}, Conds)), - %% Reset env for next test - application:set_env(riak_cs, trust_x_forwarded_for, true). - -eval_ip_addresses_test()-> - ?assert(riak_cs_s3_policy:eval_ip_address(#wm_reqdata{peer = "23.23.23.23"}, - [{'aws:SourceIp', {{1,1,1,1}, {255,255,255,0}}}, - {'aws:SourceIp', {{23,23,0,0},{255,255,0,0}}}, hage])). - -eval_condition_test()-> - ?assert(riak_cs_s3_policy:eval_condition(#wm_reqdata{peer = "23.23.23.23"}, - {'IpAddress', [garbage,{chiba, boo},"saitama", - {'aws:SourceIp', {{23,23,0,0},{255,255,0,0}}}, hage]})). - -eval_statement_test()-> - Access = #access_v1{method='GET', target=object, - req=#wm_reqdata{peer="23.23.23.23"}, - bucket= <<"testbokee">>}, - Statement = #statement{effect=allow,condition_block= - [{'IpAddress', - [{'aws:SourceIp', {{23,23,0,0},{255,255,0,0}}}]}], - action=['s3:GetObject'], - resource='*'}, - ?assert(riak_cs_s3_policy:eval_statement(Access, Statement)). - -my_split_test_()-> - [ - ?_assertEqual(["foo", "bar"], riak_cs_s3_policy:my_split($:, "foo:bar", [], [])), - ?_assertEqual(["foo", "", "", "bar"], riak_cs_s3_policy:my_split($:, "foo:::bar", [], [])), - ?_assertEqual(["arn", "aws", "s3", "", "", "hoge"], - riak_cs_s3_policy:my_split($:, "arn:aws:s3:::hoge", [], [])), - ?_assertEqual(["arn", "aws", "s3", "", "", "hoge/*"], - riak_cs_s3_policy:my_split($:, "arn:aws:s3:::hoge/*", [], [])) - ]. - -parse_arn_test()-> - List0 = [<<"arn:aws:s3:::hoge">>, <<"arn:aws:s3:::hoge/*">>], - {ok, ARNS0} = riak_cs_s3_policy:parse_arns(List0), - ?assertEqual(List0, riak_cs_s3_policy:print_arns(ARNS0)), - - List1 = [<<"arn:aws:s3:ap-northeast-1:000000:hoge">>, <<"arn:aws:s3:::hoge/*">>], - {ok, ARNS1} = riak_cs_s3_policy:parse_arns(List1), - ?assertEqual(List1, riak_cs_s3_policy:print_arns(ARNS1)), - - ?assertEqual({error, bad_arn}, riak_cs_s3_policy:parse_arns([<<"asdfiua;sfkjsd">>])), - - List2 = <<"*">>, - {ok, ARNS2} = riak_cs_s3_policy:parse_arns(List2), - ?assertEqual(List2, riak_cs_s3_policy:print_arns(ARNS2)). - -sample_securetransport_statement()-> - <<"{" - "\"Id\":\"Policy135406996387500\"," - "\"Statement\":[" - "{" - " \"Sid\":\"Stmt135406995deadbeef\"," - " \"Action\":[" - " \"s3:GetObject\"," - " \"s3:PutObject\"," - " \"s3:DeleteObject\"" - " ]," - " \"Condition\":{" - " \"Bool\": { \"aws:SecureTransport\":true }" - " }," - " \"Effect\": \"Allow\"," - " \"Resource\": \"arn:aws:s3:::test\"," - " \"Principal\": {" - " \"AWS\": \"*\"" - " }" - " }" - " ]" - "}" >>. - - -secure_transport_test()-> - application:set_env(riak_cs, trust_x_forwarded_for, true), - JsonPolicy0 = sample_securetransport_statement(), - {ok, Policy} = riak_cs_s3_policy:policy_from_json(JsonPolicy0), - Req = #wm_reqdata{peer="192.168.0.1", scheme=https}, - Access = #access_v1{method='GET', target=object, id="spam/ham/egg", - req = Req, bucket= <<"test">>}, - ?assert(riak_cs_s3_policy:eval(Access, Policy)), - % io:format(standard_error, "~w~n", [Policy]), - Access2 = Access#access_v1{req=Req#wm_reqdata{scheme=http}}, - ?assertEqual(undefined, riak_cs_s3_policy:eval(Access2, Policy)). - -%% "Bool": { "aws:SecureTransport" : true, -%% "aws:SecureTransport" : false } is recognized as false -%% -%% "Bool": { "aws:SecureTransport" : false, -%% "aws:SecureTransport" : true } is recognized as true - -malformed_json_statement()-> - <<"{" - "\"Id\":\"Policy135406996387500\"," - "\"Statement\":[" - "{" - " \"Sid\":\"Stmt135406995deadbeef\"," - " \"Action\":[" - " \"s3:GetObject\"," - " \"s3:PutObject\"," - " \"s3:DeleteObject\"" - " ]," - " \"Condition\":{" - " \"Bool\": { \"aws:SecureTransport\":tr }" - " }," - " \"Effect\": \"Allow\"," - " \"Resource\": \"arn:aws:s3:::test\"," - " \"Principal\": {" - " \"AWS\": \"*\"" - " }" - " }" - " ]" - "}" >>. - -malformed_policy_json_test()-> - JsonPolicy0 = malformed_json_statement(), - {error, malformed_policy_json} = riak_cs_s3_policy:policy_from_json(JsonPolicy0). - --endif. diff --git a/test/riak_cs_utils_eqc.erl b/test/riak_cs_utils_eqc.erl deleted file mode 100644 index f43327a57..000000000 --- a/test/riak_cs_utils_eqc.erl +++ /dev/null @@ -1,52 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% ------------------------------------------------------------------- - -%% @doc Quickcheck test module for `riak_cs_utils'. - --module(riak_cs_utils_eqc). - --ifdef(EQC). - --include("riak_cs.hrl"). --include_lib("eqc/include/eqc.hrl"). --include_lib("eunit/include/eunit.hrl"). - --compile(export_all). - --define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> - io:format(user, Str, Args) end, P)). - -%%==================================================================== -%% Eunit tests -%%==================================================================== - -eqc_test_() -> - Time = 8, - [ - {timeout, Time*4, ?_assertEqual(true, - eqc:quickcheck(eqc:testing_time(Time,?QC_OUT(prop_md5()))))} - ]. - -%% ==================================================================== -%% EQC Properties -%% ==================================================================== - -prop_md5() -> - _ = crypto:start(), - ?FORALL(Bin, gen_bin(), - crypto:hash(md5, Bin) == riak_cs_utils:md5(Bin)). - -gen_bin() -> - oneof([binary(), - ?LET({Size, Char}, {choose(5, 2*1024*1024 + 1024), choose(0, 255)}, - list_to_binary(lists:duplicate(Size, Char)))]). - -%%==================================================================== -%% Helpers -%%==================================================================== - --endif. diff --git a/test/riak_cs_wm_service_test.erl b/test/riak_cs_wm_service_test.erl deleted file mode 100644 index 5dd8a3f09..000000000 --- a/test/riak_cs_wm_service_test.erl +++ /dev/null @@ -1,51 +0,0 @@ -%% --------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% --------------------------------------------------------------------- - --module(riak_cs_wm_service_test). - --compile(export_all). - --include("riak_cs.hrl"). --include_lib("webmachine/include/webmachine.hrl"). --include_lib("eunit/include/eunit.hrl"). - -service_test_() -> - {setup, - fun riak_cs_wm_test_utils:setup/0, - fun riak_cs_wm_test_utils:teardown/1, - [ - ]}. - -get_bucket_to_json() -> - %% XXX TODO: MAKE THESE ACTUALLY TEST SOMETHING - BucketNames = ["foo", "bar", "baz"], - UserName = "fooser", - Email = "fooser@fooser.com", - {ok, User} = riak_cs_user:create_user(UserName, Email), - KeyID = User?RCS_USER.key_id, - [riak_cs_utils:create_bucket(KeyID, Name) || Name <- BucketNames], - {ok, UpdatedUser} = riak_cs_user:get_user(User?RCS_USER.key_id), - CorrectJsonBucketNames = [list_to_binary(Name) || - Name <- lists:reverse(BucketNames)], - _EncodedCorrectNames = mochijson2:encode(CorrectJsonBucketNames), - _Context = #context{user=UpdatedUser}, - ?assert(true). - %%{ResultToTest, _, _} = riak_cs_wm_service:to_json(fake_rd, Context), - %%?assertEqual(EncodedCorrectNames, ResultToTest). diff --git a/tools.mk b/tools.mk deleted file mode 100644 index c3d61a822..000000000 --- a/tools.mk +++ /dev/null @@ -1,149 +0,0 @@ -# ------------------------------------------------------------------- -# -# Copyright (c) 2014 Basho Technologies, Inc. -# -# This file is provided to you under the Apache License, -# Version 2.0 (the "License"); you may not use this file -# except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# ------------------------------------------------------------------- - -# ------------------------------------------------------------------- -# NOTE: This file is is from https://github.com/basho/tools.mk. -# It should not be edited in a project. It should simply be updated -# wholesale when a new version of tools.mk is released. -# ------------------------------------------------------------------- - -REBAR ?= ./rebar -REVISION ?= $(shell git rev-parse --short HEAD) -PROJECT ?= $(shell basename `find src -name "*.app.src"` .app.src) - -.PHONY: compile-no-deps test docs xref dialyzer-run dialyzer-quick dialyzer \ - cleanplt upload-docs - -compile-no-deps: - ${REBAR} compile skip_deps=true - -test: compile - ${REBAR} eunit skip_deps=true - -upload-docs: docs - @if [ -z "${BUCKET}" -o -z "${PROJECT}" -o -z "${REVISION}" ]; then \ - echo "Set BUCKET, PROJECT, and REVISION env vars to upload docs"; \ - exit 1; fi - @cd doc; s3cmd put -P * "s3://${BUCKET}/${PROJECT}/${REVISION}/" > /dev/null - @echo "Docs built at: http://${BUCKET}.s3-website-us-east-1.amazonaws.com/${PROJECT}/${REVISION}" - -docs: - ${REBAR} doc skip_deps=true - -xref: compile - ${REBAR} xref skip_deps=true - -PLT ?= $(HOME)/.combo_dialyzer_plt -LOCAL_PLT = .local_dialyzer_plt -DIALYZER_FLAGS ?= -Wunmatched_returns - -${PLT}: compile - @if [ -f $(PLT) ]; then \ - dialyzer --check_plt --plt $(PLT) --apps $(DIALYZER_APPS) && \ - dialyzer --add_to_plt --plt $(PLT) --output_plt $(PLT) --apps $(DIALYZER_APPS) ; test $$? -ne 1; \ - else \ - dialyzer --build_plt --output_plt $(PLT) --apps $(DIALYZER_APPS); test $$? -ne 1; \ - fi - -${LOCAL_PLT}: compile - @if [ -d deps ]; then \ - if [ -f $(LOCAL_PLT) ]; then \ - dialyzer --check_plt --plt $(LOCAL_PLT) deps/*/ebin && \ - dialyzer --add_to_plt --plt $(LOCAL_PLT) --output_plt $(LOCAL_PLT) deps/*/ebin ; test $$? -ne 1; \ - else \ - dialyzer --build_plt --output_plt $(LOCAL_PLT) deps/*/ebin ; test $$? -ne 1; \ - fi \ - fi - -dialyzer-run: - @echo "==> $(shell basename $(shell pwd)) (dialyzer)" -# The bulk of the code below deals with the dialyzer.ignore-warnings file -# which contains strings to ignore if output by dialyzer. -# Typically the strings include line numbers. Using them exactly is hard -# to maintain as the code changes. This approach instead ignores the line -# numbers, but takes into account the number of times a string is listed -# for a given file. So if one string is listed once, for example, and it -# appears twice in the warnings, the user is alerted. It is possible but -# unlikely that this approach could mask a warning if one ignored warning -# is removed and two warnings of the same kind appear in the file, for -# example. But it is a trade-off that seems worth it. -# Details of the cryptic commands: -# - Remove line numbers from dialyzer.ignore-warnings -# - Pre-pend duplicate count to each warning with sort | uniq -c -# - Remove annoying white space around duplicate count -# - Save in dialyer.ignore-warnings.tmp -# - Do the same to dialyzer_warnings -# - Remove matches from dialyzer.ignore-warnings.tmp from output -# - Remove duplicate count -# - Escape regex special chars to use lines as regex patterns -# - Add pattern to match any line number (file.erl:\d+:) -# - Anchor to match the entire line (^entire line$) -# - Save in dialyzer_unhandled_warnings -# - Output matches for those patterns found in the original warnings - @if [ -f $(LOCAL_PLT) ]; then \ - PLTS="$(PLT) $(LOCAL_PLT)"; \ - else \ - PLTS=$(PLT); \ - fi; \ - if [ -f dialyzer.ignore-warnings ]; then \ - if [ $$(grep -cvE '[^[:space:]]' dialyzer.ignore-warnings) -ne 0 ]; then \ - echo "ERROR: dialyzer.ignore-warnings contains a blank/empty line, this will match all messages!"; \ - exit 1; \ - fi; \ - dialyzer $(DIALYZER_FLAGS) --plts $${PLTS} -c ebin > dialyzer_warnings ; \ - cat dialyzer.ignore-warnings \ - | sed -E 's/^([^:]+:)[^:]+:/\1/' \ - | sort \ - | uniq -c \ - | sed -E '/.*\.erl: /!s/^[[:space:]]*[0-9]+[[:space:]]*//' \ - > dialyzer.ignore-warnings.tmp ; \ - egrep -v "^[[:space:]]*(done|Checking|Proceeding|Compiling)" dialyzer_warnings \ - | sed -E 's/^([^:]+:)[^:]+:/\1/' \ - | sort \ - | uniq -c \ - | sed -E '/.*\.erl: /!s/^[[:space:]]*[0-9]+[[:space:]]*//' \ - | grep -F -f dialyzer.ignore-warnings.tmp -v \ - | sed -E 's/^[[:space:]]*[0-9]+[[:space:]]*//' \ - | sed -E 's/([]\^:+?|()*.$${}\[])/\\\1/g' \ - | sed -E 's/(\\\.erl\\\:)/\1[[:digit:]]+:/g' \ - | sed -E 's/^(.*)$$/^[[:space:]]*\1$$/g' \ - > dialyzer_unhandled_warnings ; \ - rm dialyzer.ignore-warnings.tmp; \ - if [ $$(cat dialyzer_unhandled_warnings | wc -l) -gt 0 ]; then \ - egrep -f dialyzer_unhandled_warnings dialyzer_warnings ; \ - found_warnings=1; \ - fi; \ - [ "$$found_warnings" != 1 ] ; \ - else \ - dialyzer $(DIALYZER_FLAGS) --plts $${PLTS} -c ebin; \ - fi - -dialyzer-quick: compile-no-deps dialyzer-run - -dialyzer: ${PLT} ${LOCAL_PLT} dialyzer-run - -cleanplt: - @echo - @echo "Are you sure? It takes several minutes to re-build." - @echo Deleting $(PLT) and $(LOCAL_PLT) in 5 seconds. - @echo - sleep 5 - rm $(PLT) - rm $(LOCAL_PLT) diff --git a/priv/tools/internal/README.md b/tools/internal/README.md similarity index 100% rename from priv/tools/internal/README.md rename to tools/internal/README.md diff --git a/priv/tools/internal/block_audit.erl b/tools/internal/block_audit.erl old mode 100755 new mode 100644 similarity index 96% rename from priv/tools/internal/block_audit.erl rename to tools/internal/block_audit.erl index 3a391e533..78334415c --- a/priv/tools/internal/block_audit.erl +++ b/tools/internal/block_audit.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved,. +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -27,22 +28,20 @@ -define(SLK_TIMEOUT, 360000000). %% 100 hours --include_lib("riak_cs/include/riak_cs.hrl"). +%%-include_lib("riak_cs/include/riak_cs.hrl"). +-define(BUCKETS_BUCKET, <<"moss.buckets">>). -record(buuid, {uuid :: binary(), seqs :: [non_neg_integer()] % sequence numbers }). main(Args) -> - _ = application:load(lager), - ok = application:set_env(lager, handlers, [{lager_console_backend, info}]), - ok = lager:start(), {ok, {Options, _PlainArgs}} = getopt:parse(option_spec(), Args), LogLevel = case proplists:get_value(debug, Options) of 0 -> info; _ -> - ok = lager:set_loglevel(lager_console_backend, debug), + ok = logger:set_primary_config(level, debug), debug end, debug("Log level is set to ~p", [LogLevel]), @@ -95,7 +94,7 @@ info(Format, Args) -> log(info, Format, Args). log(Level, Format, Args) -> - lager:log(Level, self(), Format, Args). + logger:log(Level, Format, Args). audit(Pid, Opts) -> Buckets = case proplists:get_all_values(bucket, Opts) of diff --git a/priv/tools/internal/ensure_orphan_blocks.erl b/tools/internal/ensure_orphan_blocks.erl old mode 100755 new mode 100644 similarity index 97% rename from priv/tools/internal/ensure_orphan_blocks.erl rename to tools/internal/ensure_orphan_blocks.erl index c59b7a716..ad5dc7fbf --- a/priv/tools/internal/ensure_orphan_blocks.erl +++ b/tools/internal/ensure_orphan_blocks.erl @@ -1,6 +1,7 @@ %% --------------------------------------------------------------------- %% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved,. +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -31,15 +32,12 @@ -include_lib("riak_cs/include/riak_cs.hrl"). main(Args) -> - _ = application:load(lager), - ok = application:set_env(lager, handlers, [{lager_console_backend, info}]), - ok = lager:start(), {ok, {Options, _PlainArgs}} = getopt:parse(option_spec(), Args), LogLevel = case proplists:get_value(debug, Options) of 0 -> info; _ -> - ok = lager:set_loglevel(lager_console_backend, debug), + ok = logger:set_primary_config(level, debug), debug end, debug("Log level is set to ~p", [LogLevel]), @@ -94,7 +92,7 @@ info(Format, Args) -> log(info, Format, Args). log(Level, Format, Args) -> - lager:log(Level, self(), Format, Args). + logger:log(Level, Format, Args). audit2(Pid, Opts) -> info("Filter actual orphaned blocks from maybe ones. This may take a while...", []), @@ -243,4 +241,3 @@ write_uuid(_Opts, Output, CSBucket, UUID, Seq, CSKey) -> mochihex:to_hex(CSKey), $\t, mochihex:to_hex(UUID), $\t, integer_to_list(Seq), $\n]). - diff --git a/priv/tools/internal/offline_delete.erl b/tools/internal/offline_delete.erl old mode 100755 new mode 100644 similarity index 97% rename from priv/tools/internal/offline_delete.erl rename to tools/internal/offline_delete.erl index 71027cf2e..c2ed44a46 --- a/priv/tools/internal/offline_delete.erl +++ b/tools/internal/offline_delete.erl @@ -1,8 +1,7 @@ -#!/usr/bin/env escript - %% --------------------------------------------------------------------- %% -%% Copyright (c) 2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2015 Basho Technologies, Inc. All Rights Reserved,. +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -27,10 +26,15 @@ %% %% Note: make sure you remove AAE tree after this script was run, and %% turn off AAE on other nodes that's running on the cluster. +%% +%% Note2: With riak-3.2.0, which does not include getopt, export +%% ERL_LIBS with a path to riak_cs/.../lib//getopt before running this +%% escript. -module(offline_delete). --compile(export_all). +-export([main/1]). + -mode(compile). options() -> diff --git a/priv/tools/internal/riak_cs_inspector.erl b/tools/internal/riak_cs_inspector.erl old mode 100755 new mode 100644 similarity index 99% rename from priv/tools/internal/riak_cs_inspector.erl rename to tools/internal/riak_cs_inspector.erl index 7dade03b9..d75e003c1 --- a/priv/tools/internal/riak_cs_inspector.erl +++ b/tools/internal/riak_cs_inspector.erl @@ -1,8 +1,7 @@ -#!/usr/bin/env escript - %% --------------------------------------------------------------------- %% -%% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved,. +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -1032,7 +1031,7 @@ print_manifest(V) -> -spec get_riak_object(pid(), binary(), binary()) -> [{SiblingNo::integer(), {tombstone, tombstone} | - {Metadata::dict(), Value::binary()}}]. + {Metadata::dict:dict(), Value::binary()}}]. get_riak_object(RiakcPid, RiakBucket, RiakKey) -> %% With option deletedvclock, tombstone is represented as Object with no contents case riakc_pb_socket:get(RiakcPid, RiakBucket, RiakKey, [deletedvclock]) of diff --git a/priv/tools/internal/select_gc_bucket.erl b/tools/internal/select_gc_bucket.erl similarity index 97% rename from priv/tools/internal/select_gc_bucket.erl rename to tools/internal/select_gc_bucket.erl index ca3c989a9..8ffc78f2a 100644 --- a/priv/tools/internal/select_gc_bucket.erl +++ b/tools/internal/select_gc_bucket.erl @@ -1,8 +1,7 @@ -%% #!/usr/bin/env escript - %% --------------------------------------------------------------------- %% -%% Copyright (c) 2015 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2015 Basho Technologies, Inc. All Rights Reserved,. +%% 2021, 2022 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -22,7 +21,8 @@ -module(select_gc_bucket). --compile(export_all). +-export([main/1]). + -mode(compile). -include_lib("riak_cs/include/riak_cs.hrl"). @@ -53,9 +53,9 @@ pgv(Key,Proplist) -> end. maybe_date("today") -> - list_to_binary(integer_to_list(riak_cs_gc:timestamp())); + list_to_binary(integer_to_list(os:system_time(millisecond))); maybe_date("yesterday") -> - list_to_binary(integer_to_list(riak_cs_gc:timestamp() - 86400)); + list_to_binary(integer_to_list(os:system_time(millisecond) - 86400000)); maybe_date([Y0,Y1,Y2,Y3,M0,M1,D0,D1]) -> DateTime = {{list_to_integer([Y0,Y1,Y2,Y3]), list_to_integer([M0,M1]), diff --git a/priv/tools/repair_gc_bucket.erl b/tools/repair_gc_bucket.erl old mode 100755 new mode 100644 similarity index 95% rename from priv/tools/repair_gc_bucket.erl rename to tools/repair_gc_bucket.erl index 381604dc1..395dd65b4 --- a/priv/tools/repair_gc_bucket.erl +++ b/tools/repair_gc_bucket.erl @@ -1,8 +1,7 @@ -#!/usr/bin/env escript - %% --------------------------------------------------------------------- %% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved,. +%% 2021-2023 TI Tokyo All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -53,15 +52,12 @@ -include_lib("riak_cs/include/riak_cs.hrl"). main(Args) -> - _ = application:load(lager), - ok = application:set_env(lager, handlers, [{lager_console_backend, info}]), - ok = lager:start(), {ok, {Options, _PlainArgs}} = getopt:parse(option_spec(), Args), LogLevel = case proplists:get_value(debug, Options) of 0 -> info; _ -> - ok = lager:set_loglevel(lager_console_backend, debug), + ok = logger:set_primary_config(level, debug), debug end, debug("Log level is set to ~p", [LogLevel]), @@ -112,7 +108,7 @@ info(Format, Args) -> log(info, Format, Args). log(Level, Format, Args) -> - lager:log(Level, self(), Format, Args). + logger:log(Level, Format, Args). -spec repair(pid(), proplists:proplist()) -> term(). repair(Pbc, Options) -> @@ -123,8 +119,8 @@ fetch_2i_keys(Pbc, Options, Continuation) -> debug("Fetching next ~p keys, Continuation=~p", [MaxResults, Continuation]), QueryOptions = [{max_results, MaxResults}, {continuation, Continuation}], - Now = riak_cs_gc:timestamp(), - Leeway = proplists:get_value(leeway_seconds, Options), + Now = os:system_time(millisecond), + Leeway = 1000 * proplists:get_value(leeway_seconds, Options), StartTime = riak_cs_gc:epoch_start(), EndTime = list_to_binary(integer_to_list(Now - Leeway)), debug("StartTime=~p, EndTime=~p", [StartTime, EndTime]), @@ -162,8 +158,7 @@ process_gc_keys(Pbc, Options, Continuation, [GCKey | Keys]) -> process_gc_keys(Pbc, Options, Continuation, Keys). -spec repair_manifests_for_gc_key(pid(), proplists:proplist(), binary()) -> - ok | - {error, term()}. + ok | {error, term()}. repair_manifests_for_gc_key(Pbc, Options, GCKey) -> Timeout = riak_cs_config:get_gckey_timeout(), case riakc_pb_socket:get(Pbc, ?GC_BUCKET, GCKey, [], Timeout) of @@ -274,7 +269,7 @@ get_actual_manifest_state(Pbc, Bucket, Key, UUID)-> case riakc_pb_socket:get(Pbc, RiakBucket, RiakKey, []) of {ok, RiakObj} -> ManifestDict = riak_cs_manifest:manifests_from_riak_object(RiakObj), - case riak_cs_manifest_utils:active_manifest(ManifestDict) of + case rcs_common_manifest_utils:active_manifest(ManifestDict) of {ok, ?MANIFEST{uuid=UUID}=M} -> {ok, M?MANIFEST.state}; {ok, ?MANIFEST{}} -> {ok, notfound}; {error, no_active_manifest} -> {ok, notfound}