From 86dc45f4446abff4fecb7ad7be0c84bbbbe64f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Thu, 26 Apr 2018 13:17:27 +0200 Subject: [PATCH 01/21] Add support for S3 through the REST API This is currently using the old S3 authentication (https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html), in order to support Exoscale Storage Refs #112 --- .travis.yml | 9 +- config.yml.example.s3 | 31 + ...ig.yml.example => config.yml.example.swift | 0 lib/remote_storage/rest_provider.rb | 492 ++++++++++ lib/remote_storage/s3_rest.rb | 110 +++ lib/remote_storage/swift.rb | 448 +-------- liquor-cabinet.rb | 3 + spec/s3/app_spec.rb | 871 ++++++++++++++++++ 8 files changed, 1519 insertions(+), 445 deletions(-) create mode 100644 config.yml.example.s3 rename config.yml.example => config.yml.example.swift (100%) create mode 100644 lib/remote_storage/rest_provider.rb create mode 100644 lib/remote_storage/s3_rest.rb create mode 100644 spec/s3/app_spec.rb diff --git a/.travis.yml b/.travis.yml index 0f38df9..6c1356e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,10 @@ rvm: - 2.4.1 services: - redis-server -before_install: - - gem install bundler before_script: - - cp config.yml.example config.yml + - cp config.yml.example.$BACKEND config.yml - mkdir -p tmp && echo "swifttoken" > tmp/swift_token.txt -script: ruby spec/swift/* +script: ruby spec/$BACKEND/* branches: only: - master @@ -21,3 +19,6 @@ notifications: - http://hook-juggler.herokuapp.com/hooks/travis on_success: always on_failure: always +env: + - BACKEND=s3 + - BACKEND=swift diff --git a/config.yml.example.s3 b/config.yml.example.s3 new file mode 100644 index 0000000..0acb9fd --- /dev/null +++ b/config.yml.example.s3 @@ -0,0 +1,31 @@ +development: &defaults + maintenance: false + # # uncomment this section + # s3: + # endpoint: "https://some-endpoint" + # region: "region" + # access_key_id: "" + # secret_key_id: "" + # bucket: "test-bucket" + # # Redis is needed for the swift backend + # redis: + # host: localhost + # port: 6379 + +test: + <<: *defaults + s3: + endpoint: "https://some-endpoint" + region: "region" + access_key_id: "" + secret_key_id: "" + bucket: "test-bucket" + redis: + host: localhost + port: 6379 + +staging: + <<: *defaults + +production: + <<: *defaults diff --git a/config.yml.example b/config.yml.example.swift similarity index 100% rename from config.yml.example rename to config.yml.example.swift diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb new file mode 100644 index 0000000..c353389 --- /dev/null +++ b/lib/remote_storage/rest_provider.rb @@ -0,0 +1,492 @@ +require "rest_client" +require "json" +require "cgi" +require "active_support/core_ext/time/conversions" +require "active_support/core_ext/numeric/time" +require "active_support/core_ext/hash" +require "redis" +require "digest/md5" + +module RemoteStorage + module RestProvider + + attr_accessor :settings, :server + + def initialize(settings, server) + @settings = settings + @server = server + end + + def authorize_request(user, directory, token, listing=false) + request_method = server.env["REQUEST_METHOD"] + + if directory.split("/").first == "public" + return true if ["GET", "HEAD"].include?(request_method) && !listing + end + + server.halt 401, "Unauthorized" if token.nil? || token.empty? + + authorizations = redis.smembers("authorizations:#{user}:#{token}") + permission = directory_permission(authorizations, directory) + + server.halt 401, "Unauthorized" unless permission + if ["PUT", "DELETE"].include? request_method + server.halt 401, "Unauthorized" unless permission == "rw" + end + end + + def get_head(user, directory, key) + url = url_for_key(user, directory, key) + + res = do_head_request(url) + + set_response_headers(res) + rescue RestClient::ResourceNotFound + server.halt 404 + end + + def get_data(user, directory, key) + url = url_for_key(user, directory, key) + + res = do_get_request(url) + + set_response_headers(res) + + none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") + .map(&:strip) + .map { |s| s.gsub(/^"?W\//, "") } + server.halt 304 if none_match.include? format_etag(res.headers[:etag]) + + return res.body + rescue RestClient::ResourceNotFound + server.halt 404, "Not Found" + end + + def get_head_directory_listing(user, directory) + get_directory_listing(user, directory) + + "" # just return empty body, headers are set by get_directory_listing + end + + def get_directory_listing(user, directory) + etag = redis.hget "rs:m:#{user}:#{directory}/", "e" + + server.headers["Content-Type"] = "application/ld+json" + + none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") + .map(&:strip) + .map { |s| s.gsub(/^"?W\//, "") } + + if etag + server.halt 304 if none_match.include? %Q("#{etag}") + + items = get_directory_listing_from_redis_via_lua(user, directory) + else + etag = etag_for(user, directory) + items = {} + + server.halt 304 if none_match.include? %Q("#{etag}") + end + + server.headers["ETag"] = %Q("#{etag}") + + listing = { + "@context" => "http://remotestorage.io/spec/folder-description", + "items" => items + } + + listing.to_json + end + + def put_data(user, directory, key, data, content_type) + server.halt 400 if server.env["HTTP_CONTENT_RANGE"] + server.halt 409, "Conflict" if has_name_collision?(user, directory, key) + + existing_metadata = redis.hgetall redis_metadata_object_key(user, directory, key) + url = url_for_key(user, directory, key) + + if required_match = server.env["HTTP_IF_MATCH"] + required_match = required_match.gsub(/^"?W\//, "") + unless required_match == %Q("#{existing_metadata["e"]}") + + # get actual metadata and compare in case redis metadata became out of sync + begin + head_res = do_head_request(url) + # The file doesn't exist, return 412 + rescue RestClient::ResourceNotFound + server.halt 412, "Precondition Failed" + end + + if required_match == format_etag(head_res.headers[:etag]) + # log previous size difference that was missed ealier because of redis failure + log_size_difference(user, existing_metadata["s"], head_res.headers[:content_length]) + else + server.halt 412, "Precondition Failed" + end + end + end + if server.env["HTTP_IF_NONE_MATCH"] == "*" + server.halt 412, "Precondition Failed" unless existing_metadata.empty? + end + + etag, timestamp = do_put_request_and_return_etag_and_last_modified(url, data, content_type) + + metadata = { + e: etag, + s: data.size, + t: content_type, + m: timestamp + } + + if update_metadata_object(user, directory, key, metadata) + if metadata_changed?(existing_metadata, metadata) + update_dir_objects(user, directory, timestamp, checksum_for(data)) + log_size_difference(user, existing_metadata["s"], metadata[:s]) + end + + server.headers["ETag"] = %Q("#{etag}") + server.halt existing_metadata.empty? ? 201 : 200 + else + server.halt 500 + end + end + + def delete_data(user, directory, key) + url = url_for_key(user, directory, key) + not_found = false + + existing_metadata = redis.hgetall "rs:m:#{user}:#{directory}/#{key}" + + if required_match = server.env["HTTP_IF_MATCH"] + unless required_match.gsub(/^"?W\//, "") == %Q("#{existing_metadata["e"]}") + server.halt 412, "Precondition Failed" + end + end + + not_found = !try_to_delete(url) + + log_size_difference(user, existing_metadata["s"], 0) + delete_metadata_objects(user, directory, key) + delete_dir_objects(user, directory) + + if not_found + server.halt 404, "Not Found" + else + server.headers["Etag"] = %Q("#{existing_metadata["e"]}") + server.halt 200 + end + end + + private + + # Implement this method in your class that includes this module. For example + # %Q("#{etag}") if the ETag does not already have quotes around it + def format_etag(etag) + NotImplementedError + end + + def base_url + NotImplementedError + end + + def container_url_for(user) + NotImplementedError + end + + def default_headers + raise NotImplementedError + end + + def set_response_headers(response) + server.headers["ETag"] = format_etag(response.headers[:etag]) + server.headers["Content-Type"] = response.headers[:content_type] + server.headers["Content-Length"] = response.headers[:content_length] + server.headers["Last-Modified"] = response.headers[:last_modified] + end + + def extract_category(directory) + if directory.match(/^public\//) + "public/#{directory.split('/')[1]}" + else + directory.split('/').first + end + end + + def directory_permission(authorizations, directory) + authorizations = authorizations.map do |auth| + auth.index(":") ? auth.split(":") : [auth, "rw"] + end + authorizations = Hash[*authorizations.flatten] + + permission = authorizations[""] + + authorizations.each do |key, value| + if directory.match(/^(public\/)?#{key}(\/|$)/) + if permission.nil? || permission == "r" + permission = value + end + return permission if permission == "rw" + end + end + + permission + end + + def has_name_collision?(user, directory, key) + lua_script = <<-EOF + local user = ARGV[1] + local directory = ARGV[2] + local key = ARGV[3] + + -- build table with parent directories from remaining arguments + local parent_dir_count = #ARGV - 3 + local parent_directories = {} + for i = 4, 4 + parent_dir_count do + table.insert(parent_directories, ARGV[i]) + end + + -- check for existing directory with the same name as the document + local redis_key = "rs:m:"..user..":" + if directory == "" then + redis_key = redis_key..key.."/" + else + redis_key = redis_key..directory.."/"..key.."/" + end + if redis.call("hget", redis_key, "e") then + return true + end + + for index, dir in pairs(parent_directories) do + if redis.call("hget", "rs:m:"..user..":"..dir.."/", "e") then + -- the directory already exists, no need to do further checks + return false + else + -- check for existing document with same name as directory + if redis.call("hget", "rs:m:"..user..":"..dir, "e") then + return true + end + end + end + + return false + EOF + + parent_directories = parent_directories_for(directory) + + redis.eval(lua_script, nil, [user, directory, key, *parent_directories]) + end + + def metadata_changed?(old_metadata, new_metadata) + # check metadata relevant to the directory listing + # ie. the timestamp (m) is not relevant, because it's not used in + # the listing + return old_metadata["e"] != new_metadata[:e] || + old_metadata["s"] != new_metadata[:s].to_s || + old_metadata["t"] != new_metadata[:t] + end + + def timestamp_for(date) + return DateTime.parse(date).strftime("%Q").to_i + end + + def log_size_difference(user, old_size, new_size) + delta = new_size.to_i - old_size.to_i + redis.incrby "rs:s:#{user}", delta + end + + def checksum_for(data) + Digest::MD5.hexdigest(data) + end + + def parent_directories_for(directory) + directories = directory.split("/") + parent_directories = [] + + while directories.any? + parent_directories << directories.join("/") + directories.pop + end + + parent_directories << "" # add empty string for the root directory + + parent_directories + end + + def top_directory(directory) + if directory.match(/\//) + directory.split("/").last + elsif directory != "" + return directory + end + end + + def parent_directory_for(directory) + if directory.match(/\//) + return directory[0..directory.rindex("/")] + elsif directory != "" + return "/" + end + end + + def update_metadata_object(user, directory, key, metadata) + redis_key = redis_metadata_object_key(user, directory, key) + redis.hmset(redis_key, *metadata) + redis.sadd "rs:m:#{user}:#{directory}/:items", key + + true + end + + def update_dir_objects(user, directory, timestamp, checksum) + parent_directories_for(directory).each do |dir| + etag = etag_for(dir, timestamp, checksum) + + key = "rs:m:#{user}:#{dir}/" + metadata = {e: etag, m: timestamp} + redis.hmset(key, *metadata) + redis.sadd "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" + end + end + + def delete_metadata_objects(user, directory, key) + redis.del redis_metadata_object_key(user, directory, key) + redis.srem "rs:m:#{user}:#{directory}/:items", key + end + + def delete_dir_objects(user, directory) + timestamp = (Time.now.to_f * 1000).to_i + + parent_directories_for(directory).each do |dir| + if dir_empty?(user, dir) + redis.del "rs:m:#{user}:#{dir}/" + redis.srem "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" + else + etag = etag_for(dir, timestamp) + + metadata = {e: etag, m: timestamp} + redis.hmset("rs:m:#{user}:#{dir}/", *metadata) + end + end + end + + def dir_empty?(user, dir) + redis.smembers("rs:m:#{user}:#{dir}/:items").empty? + end + + def redis_metadata_object_key(user, directory, key) + "rs:m:#{user}:#{[directory, key].delete_if(&:empty?).join("/")}" + end + + def url_for_key(user, directory, key) + File.join [container_url_for(user), escape(directory), escape(key)].compact + end + + def do_put_request(url, data, content_type) + deal_with_unauthorized_requests do + RestClient.put(url, data, default_headers.merge({content_type: content_type})) + end + end + + def do_put_request_and_return_etag_and_last_modified(url, data, content_type) + res = do_put_request(url, data, content_type) + + return [res.headers[:etag], timestamp_for(res.headers[:last_modified])] + end + + def do_get_request(url, &block) + deal_with_unauthorized_requests do + RestClient.get(url, default_headers, &block) + end + end + + def do_head_request(url, &block) + deal_with_unauthorized_requests do + RestClient.head(url, default_headers, &block) + end + end + + def do_delete_request(url) + deal_with_unauthorized_requests do + RestClient.delete(url, default_headers) + end + end + + def escape(url) + # We want spaces to turn into %20 and slashes to stay slashes + CGI::escape(url).gsub('+', '%20').gsub('%2F', '/') + end + + def redis + @redis ||= Redis.new(settings.redis.symbolize_keys) + end + + def etag_for(*args) + Digest::MD5.hexdigest args.join(":") + end + + def deal_with_unauthorized_requests(&block) + begin + block.call + rescue RestClient::Unauthorized => ex + Raven.capture_exception(ex) + server.halt 500 + end + end + + def try_to_delete(url) + found = true + + begin + do_delete_request(url) + rescue RestClient::ResourceNotFound + found = false + end + + found + end + + def get_directory_listing_from_redis_via_lua(user, directory) + lua_script = <<-EOF + local user = ARGV[1] + local directory = ARGV[2] + local items = redis.call("smembers", "rs:m:"..user..":"..directory.."/:items") + local listing = {} + + for index, name in pairs(items) do + local redis_key = "rs:m:"..user..":" + if directory == "" then + redis_key = redis_key..name + else + redis_key = redis_key..directory.."/"..name + end + + local metadata_values = redis.call("hgetall", redis_key) + local metadata = {} + + -- redis returns hashes as a single list of alternating keys and values + -- this collates it into a table + for idx = 1, #metadata_values, 2 do + metadata[metadata_values[idx]] = metadata_values[idx + 1] + end + + listing[name] = {["ETag"] = metadata["e"]} + if string.sub(name, -1) ~= "/" then + listing[name]["Content-Type"] = metadata["t"] + listing[name]["Content-Length"] = tonumber(metadata["s"]) + listing[name]["Last-Modified"] = tonumber(metadata["m"]) + end + end + + return cjson.encode(listing) + EOF + + items = JSON.parse(redis.eval(lua_script, nil, [user, directory])) + + items.reject{|k, _| k.end_with? "/"}.each do |_, v| + v["Last-Modified"] = Time.at(v["Last-Modified"]/1000).httpdate + end + + items + end + + end +end diff --git a/lib/remote_storage/s3_rest.rb b/lib/remote_storage/s3_rest.rb new file mode 100644 index 0000000..5f3db7d --- /dev/null +++ b/lib/remote_storage/s3_rest.rb @@ -0,0 +1,110 @@ +require "remote_storage/rest_provider" +require "digest" +require "base64" +require "openssl" +require "webrick/httputils" + +module RemoteStorage + class S3Rest + include RestProvider + + private + + # S3 already wraps the ETag around quotes + def format_etag(etag) + etag + end + + def do_put_request(url, data, content_type) + deal_with_unauthorized_requests do + md5 = Digest::MD5.base64digest(data) + authorization_headers = authorization_headers_for("PUT", md5, content_type, url) + RestClient.put(url, data, authorization_headers.merge({ "Content-Type" => content_type, "Content-Md5" => md5})) + end + end + + # S3 does not return a Last-Modified response header on PUTs + def do_put_request_and_return_etag_and_last_modified(url, data, content_type) + res = do_put_request(url, data, content_type) + head_res = do_head_request(url) + + return [res.headers[:etag].delete('"'), timestamp_for(head_res.headers[:last_modified])] + end + + def do_get_request(url, &block) + deal_with_unauthorized_requests do + authorization_headers = authorization_headers_for("GET", "", "", url) + RestClient.get(url, authorization_headers, &block) + end + end + + def do_head_request(url, &block) + deal_with_unauthorized_requests do + authorization_headers = authorization_headers_for("HEAD", "", "", url) + RestClient.head(url, authorization_headers, &block) + end + end + + def do_delete_request(url) + deal_with_unauthorized_requests do + authorization_headers = authorization_headers_for("DELETE", "", "", url) + RestClient.delete(url, authorization_headers) + end + end + + def try_to_delete(url) + found = true + + begin + do_head_request(url) + rescue RestClient::ResourceNotFound + found = false + end + + do_delete_request(url) if found + + return found + end + + # This is using the S3 authorizations, not the newer AW V4 Signatures + # (https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html) + def authorization_headers_for(http_verb, md5, content_type, url) + url = File.join("/", url.gsub(base_url, "")) + date = Time.now.httpdate + signed_data = signature(http_verb, md5, content_type, date, url) + { "Authorization" => "AWS #{credentials[:access_key_id]}:#{signed_data}", + "Date" => date} + end + + def credentials + @credentials ||= { access_key_id: settings.s3["access_key_id"], secret_key_id: settings.s3["secret_key_id"] } + end + + def digest(secret, string_to_sign) + Base64.encode64(hmac(secret, string_to_sign)).strip + end + + def hmac(key, value) + OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha1'), key, value) + end + + def uri_escape(s) + WEBrick::HTTPUtils.escape(s).gsub('%5B', '[').gsub('%5D', ']') + end + + def signature(http_verb, md5, content_type, date, url) + string_to_sign = [http_verb, md5, content_type, date, url].join "\n" + signature = digest(credentials[:secret_key_id], string_to_sign) + uri_escape(signature) + end + + def base_url + @base_url ||= settings.s3["endpoint"] + end + + def container_url_for(user) + "#{base_url}#{settings.s3["bucket"]}/#{user}" + end + end + +end diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index d104f11..cdef119 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -1,466 +1,32 @@ require "rest_client" -require "json" -require "cgi" require "active_support/core_ext/time/conversions" require "active_support/core_ext/numeric/time" require "active_support/core_ext/hash" -require "redis" -require "digest/md5" +require "remote_storage/rest_provider" module RemoteStorage class Swift - - attr_accessor :settings, :server - - def initialize(settings, server) - @settings = settings - @server = server - end - - def authorize_request(user, directory, token, listing=false) - request_method = server.env["REQUEST_METHOD"] - - if directory.split("/").first == "public" - return true if ["GET", "HEAD"].include?(request_method) && !listing - end - - server.halt 401, "Unauthorized" if token.nil? || token.empty? - - authorizations = redis.smembers("authorizations:#{user}:#{token}") - permission = directory_permission(authorizations, directory) - - server.halt 401, "Unauthorized" unless permission - if ["PUT", "DELETE"].include? request_method - server.halt 401, "Unauthorized" unless permission == "rw" - end - end - - def get_head(user, directory, key) - url = url_for_key(user, directory, key) - - res = do_head_request(url) - - set_response_headers(res) - rescue RestClient::ResourceNotFound - server.halt 404 - end - - def get_data(user, directory, key) - url = url_for_key(user, directory, key) - - res = do_get_request(url) - - set_response_headers(res) - - none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") - .map(&:strip) - .map { |s| s.gsub(/^"?W\//, "") } - server.halt 304 if none_match.include? %Q("#{res.headers[:etag]}") - - return res.body - rescue RestClient::ResourceNotFound - server.halt 404, "Not Found" - end - - def get_head_directory_listing(user, directory) - get_directory_listing(user, directory) - - "" # just return empty body, headers are set by get_directory_listing - end - - def get_directory_listing(user, directory) - etag = redis.hget "rs:m:#{user}:#{directory}/", "e" - - server.headers["Content-Type"] = "application/ld+json" - - none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") - .map(&:strip) - .map { |s| s.gsub(/^"?W\//, "") } - - if etag - server.halt 304 if none_match.include? %Q("#{etag}") - - items = get_directory_listing_from_redis_via_lua(user, directory) - else - etag = etag_for(user, directory) - items = {} - - server.halt 304 if none_match.include? %Q("#{etag}") - end - - server.headers["ETag"] = %Q("#{etag}") - - listing = { - "@context" => "http://remotestorage.io/spec/folder-description", - "items" => items - } - - listing.to_json - end - - def get_directory_listing_from_redis_via_lua(user, directory) - lua_script = <<-EOF - local user = ARGV[1] - local directory = ARGV[2] - local items = redis.call("smembers", "rs:m:"..user..":"..directory.."/:items") - local listing = {} - - for index, name in pairs(items) do - local redis_key = "rs:m:"..user..":" - if directory == "" then - redis_key = redis_key..name - else - redis_key = redis_key..directory.."/"..name - end - - local metadata_values = redis.call("hgetall", redis_key) - local metadata = {} - - -- redis returns hashes as a single list of alternating keys and values - -- this collates it into a table - for idx = 1, #metadata_values, 2 do - metadata[metadata_values[idx]] = metadata_values[idx + 1] - end - - listing[name] = {["ETag"] = metadata["e"]} - if string.sub(name, -1) ~= "/" then - listing[name]["Content-Type"] = metadata["t"] - listing[name]["Content-Length"] = tonumber(metadata["s"]) - listing[name]["Last-Modified"] = tonumber(metadata["m"]) - end - end - - return cjson.encode(listing) - EOF - - items = JSON.parse(redis.eval(lua_script, nil, [user, directory])) - - items.reject{|k, _| k.end_with? "/"}.each do |_, v| - v["Last-Modified"] = Time.at(v["Last-Modified"]/1000).httpdate - end - - items - end - - def put_data(user, directory, key, data, content_type) - server.halt 400 if server.env["HTTP_CONTENT_RANGE"] - server.halt 409, "Conflict" if has_name_collision?(user, directory, key) - - existing_metadata = redis.hgetall redis_metadata_object_key(user, directory, key) - url = url_for_key(user, directory, key) - - if required_match = server.env["HTTP_IF_MATCH"] - required_match = required_match.gsub(/^"?W\//, "") - unless required_match == %Q("#{existing_metadata["e"]}") - - # get actual metadata and compare in case redis metadata became out of sync - begin - head_res = do_head_request(url) - # The file doesn't exist in Orbit, return 412 - rescue RestClient::ResourceNotFound - server.halt 412, "Precondition Failed" - end - - if required_match == %Q("#{head_res.headers[:etag]}") - # log previous size difference that was missed ealier because of redis failure - log_size_difference(user, existing_metadata["s"], head_res.headers[:content_length]) - else - server.halt 412, "Precondition Failed" - end - end - end - if server.env["HTTP_IF_NONE_MATCH"] == "*" - server.halt 412, "Precondition Failed" unless existing_metadata.empty? - end - - res = do_put_request(url, data, content_type) - - timestamp = timestamp_for(res.headers[:last_modified]) - - metadata = { - e: res.headers[:etag], - s: data.size, - t: content_type, - m: timestamp - } - - if update_metadata_object(user, directory, key, metadata) - if metadata_changed?(existing_metadata, metadata) - update_dir_objects(user, directory, timestamp, checksum_for(data)) - log_size_difference(user, existing_metadata["s"], metadata[:s]) - end - - server.headers["ETag"] = %Q("#{res.headers[:etag]}") - server.halt existing_metadata.empty? ? 201 : 200 - else - server.halt 500 - end - end - - def log_size_difference(user, old_size, new_size) - delta = new_size.to_i - old_size.to_i - redis.incrby "rs:s:#{user}", delta - end - - def checksum_for(data) - Digest::MD5.hexdigest(data) - end - - def delete_data(user, directory, key) - url = url_for_key(user, directory, key) - not_found = false - - existing_metadata = redis.hgetall "rs:m:#{user}:#{directory}/#{key}" - - if required_match = server.env["HTTP_IF_MATCH"] - unless required_match.gsub(/^"?W\//, "") == %Q("#{existing_metadata["e"]}") - server.halt 412, "Precondition Failed" - end - end - - begin - do_delete_request(url) - rescue RestClient::ResourceNotFound - not_found = true - end - - log_size_difference(user, existing_metadata["s"], 0) - delete_metadata_objects(user, directory, key) - delete_dir_objects(user, directory) - - if not_found - server.halt 404, "Not Found" - else - server.headers["Etag"] = %Q("#{existing_metadata["e"]}") - server.halt 200 - end - end + include RestProvider private - def set_response_headers(response) - server.headers["ETag"] = %Q("#{response.headers[:etag]}") - server.headers["Content-Type"] = response.headers[:content_type] - server.headers["Content-Length"] = response.headers[:content_length] - server.headers["Last-Modified"] = response.headers[:last_modified] + # Add quotes around the ETag + def format_etag(etag) + %Q("#{etag}") end - def extract_category(directory) - if directory.match(/^public\//) - "public/#{directory.split('/')[1]}" - else - directory.split('/').first - end - end - - def directory_permission(authorizations, directory) - authorizations = authorizations.map do |auth| - auth.index(":") ? auth.split(":") : [auth, "rw"] - end - authorizations = Hash[*authorizations.flatten] - - permission = authorizations[""] - - authorizations.each do |key, value| - if directory.match(/^(public\/)?#{key}(\/|$)/) - if permission.nil? || permission == "r" - permission = value - end - return permission if permission == "rw" - end - end - - permission - end - - def has_name_collision?(user, directory, key) - lua_script = <<-EOF - local user = ARGV[1] - local directory = ARGV[2] - local key = ARGV[3] - - -- build table with parent directories from remaining arguments - local parent_dir_count = #ARGV - 3 - local parent_directories = {} - for i = 4, 4 + parent_dir_count do - table.insert(parent_directories, ARGV[i]) - end - - -- check for existing directory with the same name as the document - local redis_key = "rs:m:"..user..":" - if directory == "" then - redis_key = redis_key..key.."/" - else - redis_key = redis_key..directory.."/"..key.."/" - end - if redis.call("hget", redis_key, "e") then - return true - end - - for index, dir in pairs(parent_directories) do - if redis.call("hget", "rs:m:"..user..":"..dir.."/", "e") then - -- the directory already exists, no need to do further checks - return false - else - -- check for existing document with same name as directory - if redis.call("hget", "rs:m:"..user..":"..dir, "e") then - return true - end - end - end - - return false - EOF - - parent_directories = parent_directories_for(directory) - - redis.eval(lua_script, nil, [user, directory, key, *parent_directories]) - end - - def metadata_changed?(old_metadata, new_metadata) - # check metadata relevant to the directory listing - # ie. the timestamp (m) is not relevant, because it's not used in - # the listing - return old_metadata["e"] != new_metadata[:e] || - old_metadata["s"] != new_metadata[:s].to_s || - old_metadata["t"] != new_metadata[:t] - end - - def timestamp_for(date) - return DateTime.parse(date).strftime("%Q").to_i - end - - def parent_directories_for(directory) - directories = directory.split("/") - parent_directories = [] - - while directories.any? - parent_directories << directories.join("/") - directories.pop - end - - parent_directories << "" # add empty string for the root directory - - parent_directories - end - - def top_directory(directory) - if directory.match(/\//) - directory.split("/").last - elsif directory != "" - return directory - end - end - - def parent_directory_for(directory) - if directory.match(/\//) - return directory[0..directory.rindex("/")] - elsif directory != "" - return "/" - end - end - - def update_metadata_object(user, directory, key, metadata) - redis_key = redis_metadata_object_key(user, directory, key) - redis.hmset(redis_key, *metadata) - redis.sadd "rs:m:#{user}:#{directory}/:items", key - - true - end - - def update_dir_objects(user, directory, timestamp, checksum) - parent_directories_for(directory).each do |dir| - etag = etag_for(dir, timestamp, checksum) - - key = "rs:m:#{user}:#{dir}/" - metadata = {e: etag, m: timestamp} - redis.hmset(key, *metadata) - redis.sadd "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" - end - end - - def delete_metadata_objects(user, directory, key) - redis.del redis_metadata_object_key(user, directory, key) - redis.srem "rs:m:#{user}:#{directory}/:items", key - end - - def delete_dir_objects(user, directory) - timestamp = (Time.now.to_f * 1000).to_i - - parent_directories_for(directory).each do |dir| - if dir_empty?(user, dir) - redis.del "rs:m:#{user}:#{dir}/" - redis.srem "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" - else - etag = etag_for(dir, timestamp) - - metadata = {e: etag, m: timestamp} - redis.hmset("rs:m:#{user}:#{dir}/", *metadata) - end - end - end - - def dir_empty?(user, dir) - redis.smembers("rs:m:#{user}:#{dir}/:items").empty? - end - - def redis_metadata_object_key(user, directory, key) - "rs:m:#{user}:#{[directory, key].delete_if(&:empty?).join("/")}" + def base_url + @base_url ||= settings.swift["host"] end def container_url_for(user) "#{base_url}/rs:documents:#{settings.environment.to_s}/#{user}" end - def url_for_key(user, directory, key) - File.join [container_url_for(user), escape(directory), escape(key)].compact - end - - def base_url - @base_url ||= settings.swift["host"] - end - def default_headers {"x-auth-token" => swift_token} end - def do_put_request(url, data, content_type) - deal_with_unauthorized_requests do - RestClient.put(url, data, default_headers.merge({content_type: content_type})) - end - end - - def do_get_request(url, &block) - deal_with_unauthorized_requests do - RestClient.get(url, default_headers, &block) - end - end - - def do_head_request(url, &block) - deal_with_unauthorized_requests do - RestClient.head(url, default_headers, &block) - end - end - - def do_delete_request(url) - deal_with_unauthorized_requests do - RestClient.delete(url, default_headers) - end - end - - def escape(url) - # We want spaces to turn into %20 and slashes to stay slashes - CGI::escape(url).gsub('+', '%20').gsub('%2F', '/') - end - - def redis - @redis ||= Redis.new(settings.redis.symbolize_keys) - end - - def etag_for(*args) - Digest::MD5.hexdigest args.join(":") - end - def reload_swift_token server.logger.debug "Reloading swift token. Old token: #{settings.swift_token}" # Remove the line break from the token file. The line break that the diff --git a/liquor-cabinet.rb b/liquor-cabinet.rb index d9a41fb..2d21522 100644 --- a/liquor-cabinet.rb +++ b/liquor-cabinet.rb @@ -5,6 +5,7 @@ require 'sinatra/config_file' require "sinatra/reloader" require "remote_storage/swift" +require "remote_storage/s3_rest" class LiquorCabinet < Sinatra::Base @@ -130,6 +131,8 @@ def storage @storage ||= begin if settings.respond_to? :swift RemoteStorage::Swift.new(settings, self) + elsif settings.respond_to? :s3 + RemoteStorage::S3Rest.new(settings, self) else puts <<-EOF You need to set one storage backend in your config.yml file. diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb new file mode 100644 index 0000000..3842a3b --- /dev/null +++ b/spec/s3/app_spec.rb @@ -0,0 +1,871 @@ +require_relative "../spec_helper" + +describe "App" do + include Rack::Test::Methods + + def app + LiquorCabinet + end + + it "returns 404 on non-existing routes" do + get "/virginmargarita" + last_response.status.must_equal 404 + end + + describe "PUT requests" do + + before do + purge_redis + end + + context "authorized" do + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + end + + it "creates the metadata object in redis" do + put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + end + end + + metadata = redis.hgetall "rs:m:phil:food/aguacate" + metadata["s"].must_equal "2" + metadata["t"].must_equal "text/plain; charset=utf-8" + metadata["e"].must_equal "bla" + metadata["m"].length.must_equal 13 + end + + it "creates the directory objects metadata in redis" do + put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + get_stub = OpenStruct.new(body: "rootbody") + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + RestClient.stub :get, get_stub do + RemoteStorage::S3Rest.stub_any_instance :etag_for, "newetag" do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + end + end + end + end + + metadata = redis.hgetall "rs:m:phil:/" + metadata["e"].must_equal "newetag" + metadata["m"].length.must_equal 13 + + metadata = redis.hgetall "rs:m:phil:food/" + metadata["e"].must_equal "newetag" + metadata["m"].length.must_equal 13 + + food_items = redis.smembers "rs:m:phil:food/:items" + food_items.each do |food_item| + ["camaron", "aguacate"].must_include food_item + end + + root_items = redis.smembers "rs:m:phil:/:items" + root_items.must_equal ["food/"] + end + + context "response code" do + before do + @put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + end + + it "is 201 for newly created objects" do + RestClient.stub :put, @put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "muy deliciosa" + end + end + + last_response.status.must_equal 201 + end + + it "is 200 for updated objects" do + RestClient.stub :put, @put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "deliciosa" + put "/phil/food/aguacate", "muy deliciosa" + end + end + + last_response.status.must_equal 200 + end + end + + context "logging usage size" do + before do + @put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + @head_stub = OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + end + + it "logs the complete size when creating new objects" do + RestClient.stub :put, @put_stub do + RestClient.stub :head, @head_stub do + put "/phil/food/aguacate", "1234567890" + end + end + + size_log = redis.get "rs:s:phil" + size_log.must_equal "10" + end + + it "logs the size difference when updating existing objects" do + RestClient.stub :put, @put_stub do + RestClient.stub :head, @head_stub do + put "/phil/food/camaron", "1234567890" + put "/phil/food/aguacate", "1234567890" + put "/phil/food/aguacate", "123" + end + end + + size_log = redis.get "rs:s:phil" + size_log.must_equal "13" + end + end + + describe "objects in root dir" do + before do + put_stub = OpenStruct.new(headers: { + etag: '"bla"', + last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/bamboo.txt", "shir kan" + end + end + end + + it "are listed in the directory listing with all metadata" do + get "phil/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["items"]["bamboo.txt"].wont_be_nil + content["items"]["bamboo.txt"]["ETag"].must_equal "bla" + content["items"]["bamboo.txt"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["bamboo.txt"]["Content-Length"].must_equal 8 + content["items"]["bamboo.txt"]["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + end + + describe "name collision checks" do + it "is successful when there is no name collision" do + put_stub = OpenStruct.new(headers: { + etag: '"bla"', + last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" + }) + get_stub = OpenStruct.new(body: "rootbody") + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + RestClient.stub :get, get_stub do + RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do + put "/phil/food/aguacate", "si" + end + end + end + end + + last_response.status.must_equal 201 + + metadata = redis.hgetall "rs:m:phil:food/aguacate" + metadata["s"].must_equal "2" + end + + it "conflicts when there is a directory with same name as document" do + put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + put "/phil/food", "wontwork" + end + end + + last_response.status.must_equal 409 + last_response.body.must_equal "Conflict" + + metadata = redis.hgetall "rs:m:phil:food" + metadata.must_be_empty + end + + it "conflicts when there is a document with same name as directory" do + put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + put "/phil/food/aguacate/empanado", "wontwork" + end + end + + last_response.status.must_equal 409 + + metadata = redis.hgetall "rs:m:phil:food/aguacate/empanado" + metadata.must_be_empty + end + + it "returns 400 when a Content-Range header is sent" do + header "Content-Range", "bytes 0-3/3" + + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 400 + end + end + + describe "If-Match header" do + before do + put_stub = OpenStruct.new(headers: { + etag: '"oldetag"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + end + end + end + + it "allows the request if the header matches the current ETag" do + header "If-Match", "\"oldetag\"" + + put_stub = OpenStruct.new(headers: { + etag: '"newetag"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "aye" + end + end + + last_response.status.must_equal 200 + last_response.headers["Etag"].must_equal "\"newetag\"" + end + + it "allows the request if the header contains a weak ETAG matching the current ETag" do + header "If-Match", "W/\"oldetag\"" + + put_stub = OpenStruct.new(headers: { + etag: '"newetag"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "aye" + end + end + + last_response.status.must_equal 200 + last_response.headers["Etag"].must_equal "\"newetag\"" + end + + it "allows the request if the header contains a weak ETAG with leading quote matching the current ETag" do + header "If-Match", "\"W/\"oldetag\"" + + put_stub = OpenStruct.new(headers: { + etag: '"newetag"', + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "aye" + end + end + + last_response.status.must_equal 200 + last_response.headers["Etag"].must_equal "\"newetag\"" + end + + it "fails the request if the header does not match the current ETag" do + header "If-Match", "someotheretag" + + head_stub = OpenStruct.new(headers: { + etag: '"oldetag"', + last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", + content_type: "text/plain", + content_length: 23 + }) + + RestClient.stub :head, head_stub do + put "/phil/food/aguacate", "aye" + end + + last_response.status.must_equal 412 + last_response.body.must_equal "Precondition Failed" + end + + it "allows the request if redis metadata became out of sync" do + header "If-Match", "\"existingetag\"" + + head_stub = OpenStruct.new(headers: { + etag: '"existingetag"', + last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", + content_type: "text/plain", + content_length: 23 + }) + + put_stub = OpenStruct.new(headers: { + etag: '"newetag"' + }) + + RestClient.stub :head, head_stub do + RestClient.stub :put, put_stub do + put "/phil/food/aguacate", "aye" + end + end + + last_response.status.must_equal 200 + end + end + + describe "If-None-Match header set to '*'" do + it "succeeds when the document doesn't exist yet" do + put_stub = OpenStruct.new(headers: { + etag: '"someetag"' + }) + + header "If-None-Match", "*" + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + end + end + + last_response.status.must_equal 201 + end + + it "fails the request if the document already exists" do + put_stub = OpenStruct.new(headers: { + etag: '"someetag"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + end + end + + header "If-None-Match", "*" + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + end + end + + last_response.status.must_equal 412 + last_response.body.must_equal "Precondition Failed" + end + end + end + + end + + describe "DELETE requests" do + + before do + purge_redis + end + + context "not authorized" do + describe "with no token" do + it "says it's not authorized" do + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + describe "with empty token" do + it "says it's not authorized" do + header "Authorization", "Bearer " + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + describe "with wrong token" do + it "says it's not authorized" do + header "Authorization", "Bearer wrongtoken" + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + end + + context "authorized" do + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desayunos/bolon", "wow" + end + end + end + + it "decreases the size log by size of deleted object" do + RestClient.stub :delete, "" do + RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + delete "/phil/food/aguacate" + end + end + end + + size_log = redis.get "rs:s:phil" + size_log.must_equal "8" + end + + it "deletes the metadata object in redis" do + RestClient.stub :delete, "" do + RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + delete "/phil/food/aguacate" + end + end + end + + metadata = redis.hgetall "rs:m:phil:food/aguacate" + metadata.must_be_empty + end + + it "deletes the directory objects metadata in redis" do + old_metadata = redis.hgetall "rs:m:phil:food/" + + RestClient.stub :delete, "" do + RemoteStorage::S3Rest.stub_any_instance :etag_for, "newetag" do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + delete "/phil/food/aguacate" + end + end + end + + metadata = redis.hgetall "rs:m:phil:food/" + metadata["e"].must_equal "newetag" + metadata["m"].length.must_equal 13 + metadata["m"].wont_equal old_metadata["m"] + + food_items = redis.smembers "rs:m:phil:food/:items" + food_items.sort.must_equal ["camaron", "desayunos/"] + + root_items = redis.smembers "rs:m:phil:/:items" + root_items.must_equal ["food/"] + end + + it "deletes the parent directory objects metadata when deleting all items" do + RestClient.stub :delete, "" do + RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + delete "/phil/food/aguacate" + delete "/phil/food/camaron" + delete "/phil/food/desayunos/bolon" + end + end + end + + redis.smembers("rs:m:phil:food/desayunos:items").must_be_empty + redis.hgetall("rs:m:phil:food/desayunos/").must_be_empty + + redis.smembers("rs:m:phil:food/:items").must_be_empty + redis.hgetall("rs:m:phil:food/").must_be_empty + + redis.smembers("rs:m:phil:/:items").must_be_empty + end + + it "responds with the ETag of the deleted item in the header" do + RestClient.stub :delete, "" do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + delete "/phil/food/aguacate" + end + end + + last_response.headers["ETag"].must_equal "\"bla\"" + end + + context "when item doesn't exist" do + before do + purge_redis + + put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/steak", "si" + end + end + + raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } + RestClient.stub :head, raises_exception do + delete "/phil/food/steak" + end + end + + it "returns a 404" do + last_response.status.must_equal 404 + last_response.body.must_equal "Not Found" + end + + it "deletes any metadata that might still exist" do + raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } + RestClient.stub :head, raises_exception do + delete "/phil/food/steak" + end + + metadata = redis.hgetall "rs:m:phil:food/steak" + metadata.must_be_empty + + redis.smembers("rs:m:phil:food/:items").must_be_empty + redis.hgetall("rs:m:phil:food/").must_be_empty + + redis.smembers("rs:m:phil:/:items").must_be_empty + end + end + + describe "If-Match header" do + it "succeeds when the header matches the current ETag" do + header "If-Match", "\"bla\"" + + RestClient.stub :delete, "" do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + delete "/phil/food/aguacate" + end + end + + last_response.status.must_equal 200 + end + + it "succeeds when the header contains a weak ETAG matching the current ETag" do + header "If-Match", "W/\"bla\"" + + RestClient.stub :delete, "" do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + delete "/phil/food/aguacate" + end + end + + last_response.status.must_equal 200 + end + + it "fails the request if it does not match the current ETag" do + header "If-Match", "someotheretag" + + RestClient.stub :delete, "" do + RestClient.stub :head, OpenStruct.new(headers: { etag: '"someetag"' }) do + delete "/phil/food/aguacate" + end + end + + last_response.status.must_equal 412 + last_response.body.must_equal "Precondition Failed" + end + end + end + end + + describe "GET requests" do + + before do + purge_redis + end + + context "not authorized" do + + describe "without token" do + it "says it's not authorized" do + get "/phil/food/" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + describe "with wrong token" do + it "says it's not authorized" do + header "Authorization", "Bearer wrongtoken" + get "/phil/food/" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + end + + context "authorized" do + + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put_stub = OpenStruct.new(headers: { + etag: '"bla"' + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desayunos/bolon", "wow" + end + end + end + + describe "documents" do + + it "returns the required response headers" do + get_stub = OpenStruct.new(body: "si", headers: { + etag: '"0815etag"', + last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", + content_type: "text/plain; charset=utf-8", + content_length: 2 + }) + + RestClient.stub :get, get_stub do + get "/phil/food/aguacate" + end + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Cache-Control"].must_equal "no-cache" + last_response.headers["Content-Type"].must_equal "text/plain; charset=utf-8" + end + + it "returns a 404 when data doesn't exist" do + raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } + RestClient.stub :get, raises_exception do + get "/phil/food/steak" + end + + last_response.status.must_equal 404 + last_response.body.must_equal "Not Found" + end + + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"0815etag\"" + + get_stub = OpenStruct.new(body: "si", headers: { + etag: '"0815etag"', + last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", + content_type: "text/plain; charset=utf-8", + content_length: 2 + }) + + RestClient.stub :get, get_stub do + get "/phil/food/aguacate" + end + + last_response.status.must_equal 304 + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do + header "If-None-Match", "W/\"0815etag\"" + + get_stub = OpenStruct.new(body: "si", headers: { + etag: '"0815etag"', + last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", + content_type: "text/plain; charset=utf-8", + content_length: 2 + }) + + RestClient.stub :get, get_stub do + get "/phil/food/aguacate" + end + + last_response.status.must_equal 304 + end + + end + + describe "directory listings" do + + it "returns the correct ETag header" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + end + + it "returns a Cache-Control header with value 'no-cache'" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.headers["Cache-Control"].must_equal "no-cache" + end + + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + get "/phil/food/" + + last_response.status.must_equal 304 + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the ETag" do + header "If-None-Match", "W/\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + get "/phil/food/" + + last_response.status.must_equal 304 + end + + it "contains all items in the directory" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["@context"].must_equal "http://remotestorage.io/spec/folder-description" + content["items"]["aguacate"].wont_be_nil + content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["aguacate"]["Content-Length"].must_equal 2 + content["items"]["aguacate"]["ETag"].must_equal "bla" + content["items"]["camaron"].wont_be_nil + content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["camaron"]["Content-Length"].must_equal 5 + content["items"]["camaron"]["ETag"].must_equal "bla" + content["items"]["desayunos/"].wont_be_nil + content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d" + end + + it "contains all items in the root directory" do + get "phil/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["items"]["food/"].wont_be_nil + content["items"]["food/"]["ETag"].must_equal "f9f85fbf5aa1fa378fd79ac8aa0a457d" + end + + it "responds with an empty directory liting when directory doesn't exist" do + get "phil/some-non-existing-dir/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["items"].must_equal({}) + end + + end + end + + end + + describe "HEAD requests" do + + before do + purge_redis + end + + context "not authorized" do + + describe "without token" do + it "says it's not authorized" do + head "/phil/food/camarones" + + last_response.status.must_equal 401 + last_response.body.must_be_empty + end + end + + describe "with wrong token" do + it "says it's not authorized" do + header "Authorization", "Bearer wrongtoken" + head "/phil/food/camarones" + + last_response.status.must_equal 401 + last_response.body.must_be_empty + end + end + + end + + context "authorized" do + + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put_stub = OpenStruct.new(headers: { + etag: "bla" + }) + + RestClient.stub :put, put_stub do + RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desayunos/bolon", "wow" + end + end + end + + describe "directory listings" do + it "returns the correct header information" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + end + end + + describe "documents" do + it "returns a 404 when the document doesn't exist" do + raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } + RestClient.stub :head, raises_exception do + head "/phil/food/steak" + end + + last_response.status.must_equal 404 + last_response.body.must_be_empty + end + end + + end + + end + +end + From f083022e6db54778171e916f330971fb77fc7ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 16 Apr 2018 16:40:04 +0200 Subject: [PATCH 02/21] Get the ETag from Redis on a GET request This prevents doing a network request when we would return a 304 anyway --- lib/remote_storage/rest_provider.rb | 11 ++++++----- spec/s3/app_spec.rb | 28 +++++----------------------- spec/swift/app_spec.rb | 29 ++++++----------------------- 3 files changed, 17 insertions(+), 51 deletions(-) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index c353389..9034533 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -46,17 +46,18 @@ def get_head(user, directory, key) end def get_data(user, directory, key) + none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") + .map(&:strip) + .map { |s| s.gsub(/^"?W\//, "") } + etag = redis.hget redis_metadata_object_key(user, directory, key), "e" + server.halt 304 if none_match.include? %Q("#{etag}") + url = url_for_key(user, directory, key) res = do_get_request(url) set_response_headers(res) - none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") - .map(&:strip) - .map { |s| s.gsub(/^"?W\//, "") } - server.halt 304 if none_match.include? format_etag(res.headers[:etag]) - return res.body rescue RestClient::ResourceNotFound server.halt 404, "Not Found" diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 3842a3b..17188ed 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -639,7 +639,7 @@ def app header "Authorization", "Bearer amarillo" put_stub = OpenStruct.new(headers: { - etag: '"bla"' + etag: '"0815etag"' }) RestClient.stub :put, put_stub do @@ -684,16 +684,7 @@ def app it "responds with 304 when IF_NONE_MATCH header contains the ETag" do header "If-None-Match", "\"0815etag\"" - get_stub = OpenStruct.new(body: "si", headers: { - etag: '"0815etag"', - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain; charset=utf-8", - content_length: 2 - }) - - RestClient.stub :get, get_stub do - get "/phil/food/aguacate" - end + get "/phil/food/aguacate" last_response.status.must_equal 304 end @@ -701,16 +692,7 @@ def app it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do header "If-None-Match", "W/\"0815etag\"" - get_stub = OpenStruct.new(body: "si", headers: { - etag: '"0815etag"', - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain; charset=utf-8", - content_length: 2 - }) - - RestClient.stub :get, get_stub do - get "/phil/food/aguacate" - end + get "/phil/food/aguacate" last_response.status.must_equal 304 end @@ -758,11 +740,11 @@ def app content["items"]["aguacate"].wont_be_nil content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8" content["items"]["aguacate"]["Content-Length"].must_equal 2 - content["items"]["aguacate"]["ETag"].must_equal "bla" + content["items"]["aguacate"]["ETag"].must_equal "0815etag" content["items"]["camaron"].wont_be_nil content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8" content["items"]["camaron"]["Content-Length"].must_equal 5 - content["items"]["camaron"]["ETag"].must_equal "bla" + content["items"]["camaron"]["ETag"].must_equal "0815etag" content["items"]["desayunos/"].wont_be_nil content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d" end diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index ae207c7..1cb6aae 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -597,7 +597,7 @@ def app header "Authorization", "Bearer amarillo" put_stub = OpenStruct.new(headers: { - etag: "bla", + etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) @@ -639,18 +639,10 @@ def app end it "responds with 304 when IF_NONE_MATCH header contains the ETag" do - header "If-None-Match", "\"0815etag\"" - get_stub = OpenStruct.new(body: "si", headers: { - etag: "0815etag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain; charset=utf-8", - content_length: 2 - }) + header "If-None-Match", "\"0815etag\"" - RestClient.stub :get, get_stub do - get "/phil/food/aguacate" - end + get "/phil/food/aguacate" last_response.status.must_equal 304 end @@ -658,16 +650,7 @@ def app it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do header "If-None-Match", "W/\"0815etag\"" - get_stub = OpenStruct.new(body: "si", headers: { - etag: "0815etag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain; charset=utf-8", - content_length: 2 - }) - - RestClient.stub :get, get_stub do - get "/phil/food/aguacate" - end + get "/phil/food/aguacate" last_response.status.must_equal 304 end @@ -715,11 +698,11 @@ def app content["items"]["aguacate"].wont_be_nil content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8" content["items"]["aguacate"]["Content-Length"].must_equal 2 - content["items"]["aguacate"]["ETag"].must_equal "bla" + content["items"]["aguacate"]["ETag"].must_equal "0815etag" content["items"]["camaron"].wont_be_nil content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8" content["items"]["camaron"]["Content-Length"].must_equal 5 - content["items"]["camaron"]["ETag"].must_equal "bla" + content["items"]["camaron"]["ETag"].must_equal "0815etag" content["items"]["desayunos/"].wont_be_nil content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d" end From 7bd45543f6a0e51d9dc96ad9bbecb57e05dad8af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 16 Apr 2018 17:23:53 +0200 Subject: [PATCH 03/21] Set headers from the Redis metadata on a GET that results in a 304 Also add specs to check for the response headers --- lib/remote_storage/rest_provider.rb | 8 ++++++-- spec/s3/app_spec.rb | 4 ++++ spec/swift/app_spec.rb | 4 ++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index 9034533..37ebe10 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -49,8 +49,12 @@ def get_data(user, directory, key) none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") .map(&:strip) .map { |s| s.gsub(/^"?W\//, "") } - etag = redis.hget redis_metadata_object_key(user, directory, key), "e" - server.halt 304 if none_match.include? %Q("#{etag}") + existing_metadata = redis.hgetall redis_metadata_object_key(user, directory, key) + if none_match.include? %Q("#{existing_metadata["e"]}") + server.headers["ETag"] = %Q("#{existing_metadata["e"]}") + server.headers["Last-Modified"] = Time.at(existing_metadata["m"].to_i / 1000).httpdate + server.halt 304 + end url = url_for_key(user, directory, key) diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 17188ed..8f818f1 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -687,6 +687,8 @@ def app get "/phil/food/aguacate" last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" end it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do @@ -695,6 +697,8 @@ def app get "/phil/food/aguacate" last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" end end diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 1cb6aae..c400af4 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -645,6 +645,8 @@ def app get "/phil/food/aguacate" last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" end it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do @@ -653,6 +655,8 @@ def app get "/phil/food/aguacate" last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" end end From 21dad2aba7cb8b2e81ca6aacd55271dcabc643a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 16 Apr 2018 22:08:36 +0200 Subject: [PATCH 04/21] Get the metadata from Redis on a HEAD request No need to hit the storage backend, we can set the response headers from the data in Redis --- lib/remote_storage/rest_provider.rb | 25 ++++++++++------ spec/s3/app_spec.rb | 45 ++++++++++++++++++++++------- spec/swift/app_spec.rb | 44 ++++++++++++++++++++++------ 3 files changed, 86 insertions(+), 28 deletions(-) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index 37ebe10..0e95577 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -36,23 +36,30 @@ def authorize_request(user, directory, token, listing=false) end def get_head(user, directory, key) - url = url_for_key(user, directory, key) + none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") + .map(&:strip) + .map { |s| s.gsub(/^"?W\//, "") } + metadata = redis.hgetall redis_metadata_object_key(user, directory, key) - res = do_head_request(url) + server.halt 404 if metadata.empty? - set_response_headers(res) - rescue RestClient::ResourceNotFound - server.halt 404 + # Set the response headers for a 304 or 200 response + server.headers["ETag"] = %Q("#{metadata["e"]}") + server.headers["Last-Modified"] = Time.at(metadata["m"].to_i / 1000).httpdate + + if none_match.include? %Q("#{metadata["e"]}") + server.halt 304 + end end def get_data(user, directory, key) none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") .map(&:strip) .map { |s| s.gsub(/^"?W\//, "") } - existing_metadata = redis.hgetall redis_metadata_object_key(user, directory, key) - if none_match.include? %Q("#{existing_metadata["e"]}") - server.headers["ETag"] = %Q("#{existing_metadata["e"]}") - server.headers["Last-Modified"] = Time.at(existing_metadata["m"].to_i / 1000).httpdate + metadata = redis.hgetall redis_metadata_object_key(user, directory, key) + if none_match.include? %Q("#{metadata["e"]}") + server.headers["ETag"] = %Q("#{metadata["e"]}") + server.headers["Last-Modified"] = Time.at(metadata["m"].to_i / 1000).httpdate server.halt 304 end diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 8f818f1..6ab49ec 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -815,7 +815,7 @@ def app header "Authorization", "Bearer amarillo" put_stub = OpenStruct.new(headers: { - etag: "bla" + etag: "0815etag" }) RestClient.stub :put, put_stub do @@ -838,20 +838,45 @@ def app end describe "documents" do - it "returns a 404 when the document doesn't exist" do - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :head, raises_exception do + context "when the document doesn't exist" do + it "returns a 404" do head "/phil/food/steak" + + last_response.status.must_equal 404 + last_response.body.must_be_empty end + end - last_response.status.must_equal 404 - last_response.body.must_be_empty + context "when the document exists" do + it "returns the required response headers" do + head "/phil/food/aguacate" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Cache-Control"].must_equal "no-cache" + end + + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"0815etag\"" + + head "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do + header "If-None-Match", "W/\"0815etag\"" + + head "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end end end - end - end - end - diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index c400af4..16311d9 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -773,7 +773,7 @@ def app header "Authorization", "Bearer amarillo" put_stub = OpenStruct.new(headers: { - etag: "bla", + etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) @@ -795,20 +795,46 @@ def app end describe "documents" do - it "returns a 404 when the document doesn't exist" do - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :head, raises_exception do + context "when the document doesn't exist" do + it "returns a 404" do head "/phil/food/steak" + + last_response.status.must_equal 404 + last_response.body.must_be_empty + end + end + + context "when the document exists" do + it "returns the required response headers" do + head "/phil/food/aguacate" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Cache-Control"].must_equal "no-cache" end - last_response.status.must_equal 404 - last_response.body.must_be_empty + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"0815etag\"" + + head "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do + header "If-None-Match", "W/\"0815etag\"" + + head "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end end end - end - end - end From 21f3a9f60f974efefe9488a94e2d5c0a6078e724 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Thu, 19 Apr 2018 18:18:05 +0200 Subject: [PATCH 05/21] Remove all duplication the specs S3 and Swift now run the same specs. The only difference is the before block that defines the stubbed HTTP requests and the responses from the Swift and S3 servers --- Gemfile | 1 + Gemfile.lock | 12 + spec/s3/app_spec.rb | 918 ++-------------------------------------- spec/shared_examples.rb | 635 +++++++++++++++++++++++++++ spec/spec_helper.rb | 36 +- spec/swift/app_spec.rb | 872 ++------------------------------------ 6 files changed, 762 insertions(+), 1712 deletions(-) create mode 100644 spec/shared_examples.rb diff --git a/Gemfile b/Gemfile index 6f8b5eb..293f7e3 100644 --- a/Gemfile +++ b/Gemfile @@ -15,6 +15,7 @@ group :test do gem 'purdytest', :require => false gem 'm' gem 'minitest-stub_any_instance' + gem 'webmock' end group :staging, :production do diff --git a/Gemfile.lock b/Gemfile.lock index a3a4cca..e31610b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,12 +6,17 @@ GEM i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) + addressable (2.5.2) + public_suffix (>= 2.0.2, < 4.0) backports (3.11.2) concurrent-ruby (1.0.5) + crack (0.4.3) + safe_yaml (~> 1.0.0) domain_name (0.5.20170404) unf (>= 0.0.5, < 1.0.0) faraday (0.14.0) multipart-post (>= 1.2, < 3) + hashdiff (0.3.7) http-accept (1.7.0) http-cookie (1.0.3) domain_name (~> 0.5) @@ -31,6 +36,7 @@ GEM multipart-post (2.0.0) mustermann (1.0.2) netrc (0.11.0) + public_suffix (3.0.2) purdytest (2.0.0) minitest (~> 5.5) rack (2.0.4) @@ -50,6 +56,7 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) + safe_yaml (1.0.4) sentry-raven (2.7.2) faraday (>= 0.7.6, < 1.0) sinatra (2.0.1) @@ -74,6 +81,10 @@ GEM unicorn (5.4.0) kgio (~> 2.6) raindrops (~> 0.7) + webmock (3.3.0) + addressable (>= 2.3.6) + crack (>= 0.3.2) + hashdiff PLATFORMS ruby @@ -92,6 +103,7 @@ DEPENDENCIES sentry-raven sinatra sinatra-contrib + webmock BUNDLED WITH 1.16.0 diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 6ab49ec..9874edc 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -1,882 +1,56 @@ require_relative "../spec_helper" describe "App" do - include Rack::Test::Methods - - def app - LiquorCabinet - end - - it "returns 404 on non-existing routes" do - get "/virginmargarita" - last_response.status.must_equal 404 + def container_url_for(user) + "#{app.settings.s3["endpoint"]}#{app.settings.s3["bucket"]}/#{user}" end - describe "PUT requests" do - - before do - purge_redis - end - - context "authorized" do - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - end - - it "creates the metadata object in redis" do - put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - end - end - - metadata = redis.hgetall "rs:m:phil:food/aguacate" - metadata["s"].must_equal "2" - metadata["t"].must_equal "text/plain; charset=utf-8" - metadata["e"].must_equal "bla" - metadata["m"].length.must_equal 13 - end - - it "creates the directory objects metadata in redis" do - put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - get_stub = OpenStruct.new(body: "rootbody") - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - RestClient.stub :get, get_stub do - RemoteStorage::S3Rest.stub_any_instance :etag_for, "newetag" do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - end - end - end - end - - metadata = redis.hgetall "rs:m:phil:/" - metadata["e"].must_equal "newetag" - metadata["m"].length.must_equal 13 - - metadata = redis.hgetall "rs:m:phil:food/" - metadata["e"].must_equal "newetag" - metadata["m"].length.must_equal 13 - - food_items = redis.smembers "rs:m:phil:food/:items" - food_items.each do |food_item| - ["camaron", "aguacate"].must_include food_item - end - - root_items = redis.smembers "rs:m:phil:/:items" - root_items.must_equal ["food/"] - end - - context "response code" do - before do - @put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - end - - it "is 201 for newly created objects" do - RestClient.stub :put, @put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "muy deliciosa" - end - end - - last_response.status.must_equal 201 - end - - it "is 200 for updated objects" do - RestClient.stub :put, @put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "deliciosa" - put "/phil/food/aguacate", "muy deliciosa" - end - end - - last_response.status.must_equal 200 - end - end - - context "logging usage size" do - before do - @put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - @head_stub = OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) - end - - it "logs the complete size when creating new objects" do - RestClient.stub :put, @put_stub do - RestClient.stub :head, @head_stub do - put "/phil/food/aguacate", "1234567890" - end - end - - size_log = redis.get "rs:s:phil" - size_log.must_equal "10" - end - - it "logs the size difference when updating existing objects" do - RestClient.stub :put, @put_stub do - RestClient.stub :head, @head_stub do - put "/phil/food/camaron", "1234567890" - put "/phil/food/aguacate", "1234567890" - put "/phil/food/aguacate", "123" - end - end - - size_log = redis.get "rs:s:phil" - size_log.must_equal "13" - end - end - - describe "objects in root dir" do - before do - put_stub = OpenStruct.new(headers: { - etag: '"bla"', - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/bamboo.txt", "shir kan" - end - end - end - - it "are listed in the directory listing with all metadata" do - get "phil/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["items"]["bamboo.txt"].wont_be_nil - content["items"]["bamboo.txt"]["ETag"].must_equal "bla" - content["items"]["bamboo.txt"]["Content-Type"].must_equal "text/plain; charset=utf-8" - content["items"]["bamboo.txt"]["Content-Length"].must_equal 8 - content["items"]["bamboo.txt"]["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - end - - describe "name collision checks" do - it "is successful when there is no name collision" do - put_stub = OpenStruct.new(headers: { - etag: '"bla"', - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - get_stub = OpenStruct.new(body: "rootbody") - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - RestClient.stub :get, get_stub do - RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do - put "/phil/food/aguacate", "si" - end - end - end - end - - last_response.status.must_equal 201 - - metadata = redis.hgetall "rs:m:phil:food/aguacate" - metadata["s"].must_equal "2" - end - - it "conflicts when there is a directory with same name as document" do - put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - put "/phil/food", "wontwork" - end - end - - last_response.status.must_equal 409 - last_response.body.must_equal "Conflict" - - metadata = redis.hgetall "rs:m:phil:food" - metadata.must_be_empty - end - - it "conflicts when there is a document with same name as directory" do - put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - put "/phil/food/aguacate/empanado", "wontwork" - end - end - - last_response.status.must_equal 409 - - metadata = redis.hgetall "rs:m:phil:food/aguacate/empanado" - metadata.must_be_empty - end - - it "returns 400 when a Content-Range header is sent" do - header "Content-Range", "bytes 0-3/3" - - put "/phil/food/aguacate", "si" - - last_response.status.must_equal 400 - end - end - - describe "If-Match header" do - before do - put_stub = OpenStruct.new(headers: { - etag: '"oldetag"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - end - end - end - - it "allows the request if the header matches the current ETag" do - header "If-Match", "\"oldetag\"" - - put_stub = OpenStruct.new(headers: { - etag: '"newetag"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "aye" - end - end - - last_response.status.must_equal 200 - last_response.headers["Etag"].must_equal "\"newetag\"" - end - - it "allows the request if the header contains a weak ETAG matching the current ETag" do - header "If-Match", "W/\"oldetag\"" - - put_stub = OpenStruct.new(headers: { - etag: '"newetag"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "aye" - end - end - - last_response.status.must_equal 200 - last_response.headers["Etag"].must_equal "\"newetag\"" - end - - it "allows the request if the header contains a weak ETAG with leading quote matching the current ETag" do - header "If-Match", "\"W/\"oldetag\"" - - put_stub = OpenStruct.new(headers: { - etag: '"newetag"', - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "aye" - end - end - - last_response.status.must_equal 200 - last_response.headers["Etag"].must_equal "\"newetag\"" - end - - it "fails the request if the header does not match the current ETag" do - header "If-Match", "someotheretag" - - head_stub = OpenStruct.new(headers: { - etag: '"oldetag"', - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain", - content_length: 23 - }) - - RestClient.stub :head, head_stub do - put "/phil/food/aguacate", "aye" - end - - last_response.status.must_equal 412 - last_response.body.must_equal "Precondition Failed" - end - - it "allows the request if redis metadata became out of sync" do - header "If-Match", "\"existingetag\"" - - head_stub = OpenStruct.new(headers: { - etag: '"existingetag"', - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain", - content_length: 23 - }) - - put_stub = OpenStruct.new(headers: { - etag: '"newetag"' - }) - - RestClient.stub :head, head_stub do - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "aye" - end - end - - last_response.status.must_equal 200 - end - end - - describe "If-None-Match header set to '*'" do - it "succeeds when the document doesn't exist yet" do - put_stub = OpenStruct.new(headers: { - etag: '"someetag"' - }) - - header "If-None-Match", "*" - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - end - end - - last_response.status.must_equal 201 - end - - it "fails the request if the document already exists" do - put_stub = OpenStruct.new(headers: { - etag: '"someetag"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - end - end - - header "If-None-Match", "*" - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - end - end - - last_response.status.must_equal 412 - last_response.body.must_equal "Precondition Failed" - end - end - end - - end - - describe "DELETE requests" do - - before do - purge_redis - end - - context "not authorized" do - describe "with no token" do - it "says it's not authorized" do - delete "/phil/food/aguacate" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - describe "with empty token" do - it "says it's not authorized" do - header "Authorization", "Bearer " - delete "/phil/food/aguacate" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - describe "with wrong token" do - it "says it's not authorized" do - header "Authorization", "Bearer wrongtoken" - delete "/phil/food/aguacate" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - end - - context "authorized" do - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - - put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desayunos/bolon", "wow" - end - end - end - - it "decreases the size log by size of deleted object" do - RestClient.stub :delete, "" do - RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - delete "/phil/food/aguacate" - end - end - end - - size_log = redis.get "rs:s:phil" - size_log.must_equal "8" - end - - it "deletes the metadata object in redis" do - RestClient.stub :delete, "" do - RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - delete "/phil/food/aguacate" - end - end - end - - metadata = redis.hgetall "rs:m:phil:food/aguacate" - metadata.must_be_empty - end - - it "deletes the directory objects metadata in redis" do - old_metadata = redis.hgetall "rs:m:phil:food/" - - RestClient.stub :delete, "" do - RemoteStorage::S3Rest.stub_any_instance :etag_for, "newetag" do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - delete "/phil/food/aguacate" - end - end - end - - metadata = redis.hgetall "rs:m:phil:food/" - metadata["e"].must_equal "newetag" - metadata["m"].length.must_equal 13 - metadata["m"].wont_equal old_metadata["m"] - - food_items = redis.smembers "rs:m:phil:food/:items" - food_items.sort.must_equal ["camaron", "desayunos/"] - - root_items = redis.smembers "rs:m:phil:/:items" - root_items.must_equal ["food/"] - end - - it "deletes the parent directory objects metadata when deleting all items" do - RestClient.stub :delete, "" do - RemoteStorage::S3Rest.stub_any_instance :etag_for, "rootetag" do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - delete "/phil/food/aguacate" - delete "/phil/food/camaron" - delete "/phil/food/desayunos/bolon" - end - end - end - - redis.smembers("rs:m:phil:food/desayunos:items").must_be_empty - redis.hgetall("rs:m:phil:food/desayunos/").must_be_empty - - redis.smembers("rs:m:phil:food/:items").must_be_empty - redis.hgetall("rs:m:phil:food/").must_be_empty - - redis.smembers("rs:m:phil:/:items").must_be_empty - end - - it "responds with the ETag of the deleted item in the header" do - RestClient.stub :delete, "" do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - delete "/phil/food/aguacate" - end - end - - last_response.headers["ETag"].must_equal "\"bla\"" - end - - context "when item doesn't exist" do - before do - purge_redis - - put_stub = OpenStruct.new(headers: { - etag: '"bla"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/steak", "si" - end - end - - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :head, raises_exception do - delete "/phil/food/steak" - end - end - - it "returns a 404" do - last_response.status.must_equal 404 - last_response.body.must_equal "Not Found" - end - - it "deletes any metadata that might still exist" do - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :head, raises_exception do - delete "/phil/food/steak" - end - - metadata = redis.hgetall "rs:m:phil:food/steak" - metadata.must_be_empty - - redis.smembers("rs:m:phil:food/:items").must_be_empty - redis.hgetall("rs:m:phil:food/").must_be_empty - - redis.smembers("rs:m:phil:/:items").must_be_empty - end - end - - describe "If-Match header" do - it "succeeds when the header matches the current ETag" do - header "If-Match", "\"bla\"" - - RestClient.stub :delete, "" do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - delete "/phil/food/aguacate" - end - end - - last_response.status.must_equal 200 - end - - it "succeeds when the header contains a weak ETAG matching the current ETag" do - header "If-Match", "W/\"bla\"" - - RestClient.stub :delete, "" do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - delete "/phil/food/aguacate" - end - end - - last_response.status.must_equal 200 - end - - it "fails the request if it does not match the current ETag" do - header "If-Match", "someotheretag" - - RestClient.stub :delete, "" do - RestClient.stub :head, OpenStruct.new(headers: { etag: '"someetag"' }) do - delete "/phil/food/aguacate" - end - end - - last_response.status.must_equal 412 - last_response.body.must_equal "Precondition Failed" - end - end - end + def storage_class + RemoteStorage::S3Rest end - describe "GET requests" do - - before do - purge_redis - end - - context "not authorized" do - - describe "without token" do - it "says it's not authorized" do - get "/phil/food/" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - describe "with wrong token" do - it "says it's not authorized" do - header "Authorization", "Bearer wrongtoken" - get "/phil/food/" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - end - - context "authorized" do - - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - - put_stub = OpenStruct.new(headers: { - etag: '"0815etag"' - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desayunos/bolon", "wow" - end - end - end - - describe "documents" do - - it "returns the required response headers" do - get_stub = OpenStruct.new(body: "si", headers: { - etag: '"0815etag"', - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain; charset=utf-8", - content_length: 2 - }) - - RestClient.stub :get, get_stub do - get "/phil/food/aguacate" - end - - last_response.status.must_equal 200 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Cache-Control"].must_equal "no-cache" - last_response.headers["Content-Type"].must_equal "text/plain; charset=utf-8" - end - - it "returns a 404 when data doesn't exist" do - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :get, raises_exception do - get "/phil/food/steak" - end - - last_response.status.must_equal 404 - last_response.body.must_equal "Not Found" - end - - it "responds with 304 when IF_NONE_MATCH header contains the ETag" do - header "If-None-Match", "\"0815etag\"" - - get "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - - it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do - header "If-None-Match", "W/\"0815etag\"" - - get "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - - end - - describe "directory listings" do - - it "returns the correct ETag header" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - end - - it "returns a Cache-Control header with value 'no-cache'" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.headers["Cache-Control"].must_equal "no-cache" - end - - it "responds with 304 when IF_NONE_MATCH header contains the ETag" do - header "If-None-Match", "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - get "/phil/food/" - - last_response.status.must_equal 304 - end - - it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the ETag" do - header "If-None-Match", "W/\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - get "/phil/food/" - - last_response.status.must_equal 304 - end - - it "contains all items in the directory" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["@context"].must_equal "http://remotestorage.io/spec/folder-description" - content["items"]["aguacate"].wont_be_nil - content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8" - content["items"]["aguacate"]["Content-Length"].must_equal 2 - content["items"]["aguacate"]["ETag"].must_equal "0815etag" - content["items"]["camaron"].wont_be_nil - content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8" - content["items"]["camaron"]["Content-Length"].must_equal 5 - content["items"]["camaron"]["ETag"].must_equal "0815etag" - content["items"]["desayunos/"].wont_be_nil - content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d" - end - - it "contains all items in the root directory" do - get "phil/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["items"]["food/"].wont_be_nil - content["items"]["food/"]["ETag"].must_equal "f9f85fbf5aa1fa378fd79ac8aa0a457d" - end - - it "responds with an empty directory liting when directory doesn't exist" do - get "phil/some-non-existing-dir/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["items"].must_equal({}) - end - - end - end - + before do + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "si"). + to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "aye"). + to_return(status: 200, headers: { etag: '"0915etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "deliciosa"). + to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "muy deliciosa"). + to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/camaron"). + to_return(status: 200, headers: { etag: '"0816etag"' }) + stub_request(:head, "#{container_url_for("phil")}/food/camaron"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/camaron"). + to_return(status: 200, headers: { etag: '"0816etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/desayunos/bolon"). + to_return(status: 200, headers: { etag: '"0817etag"' }) + stub_request(:head, "#{container_url_for("phil")}/food/desayunos/bolon"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/desayunos/bolon"). + to_return(status: 200, headers: { etag: '"0817etag"' }) + stub_request(:get, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, body: "rootbody", headers: { etag: '"0817etag"', content_type: "text/plain; charset=utf-8" }) + stub_request(:put, "#{container_url_for("phil")}/bamboo.txt"). + to_return(status: 200, headers: { etag: '"0818etag"' }) + stub_request(:head, "#{container_url_for("phil")}/bamboo.txt"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:head, "#{container_url_for("phil")}/food/steak"). + to_return(status: 404) + stub_request(:get, "#{container_url_for("phil")}/food/steak"). + to_return(status: 404) end - describe "HEAD requests" do - - before do - purge_redis - end - - context "not authorized" do - - describe "without token" do - it "says it's not authorized" do - head "/phil/food/camarones" - - last_response.status.must_equal 401 - last_response.body.must_be_empty - end - end - - describe "with wrong token" do - it "says it's not authorized" do - header "Authorization", "Bearer wrongtoken" - head "/phil/food/camarones" - - last_response.status.must_equal 401 - last_response.body.must_be_empty - end - end - - end - - context "authorized" do - - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - - put_stub = OpenStruct.new(headers: { - etag: "0815etag" - }) - - RestClient.stub :put, put_stub do - RestClient.stub :head, OpenStruct.new(headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desayunos/bolon", "wow" - end - end - end - - describe "directory listings" do - it "returns the correct header information" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - end - end - - describe "documents" do - context "when the document doesn't exist" do - it "returns a 404" do - head "/phil/food/steak" - - last_response.status.must_equal 404 - last_response.body.must_be_empty - end - end - - context "when the document exists" do - it "returns the required response headers" do - head "/phil/food/aguacate" - - last_response.status.must_equal 200 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Cache-Control"].must_equal "no-cache" - end - - it "responds with 304 when IF_NONE_MATCH header contains the ETag" do - header "If-None-Match", "\"0815etag\"" - - head "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - - it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do - header "If-None-Match", "W/\"0815etag\"" - - head "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - end - end - end - end + it_behaves_like 'a REST adapter' end diff --git a/spec/shared_examples.rb b/spec/shared_examples.rb new file mode 100644 index 0000000..ec74e15 --- /dev/null +++ b/spec/shared_examples.rb @@ -0,0 +1,635 @@ +require_relative "./spec_helper" + +shared_examples_for 'a REST adapter' do + include Rack::Test::Methods + + def container_url_for(user) + raise NotImplementedError + end + + def storage_class + raise NotImplementedError + end + + def config_file + raise NotImplementedError + end + + it "returns 404 on non-existing routes" do + get "/virginmargarita" + last_response.status.must_equal 404 + end + + describe "PUT requests" do + + before do + purge_redis + end + + context "authorized" do + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + end + + it "creates the metadata object in redis" do + put "/phil/food/aguacate", "si" + + metadata = redis.hgetall "rs:m:phil:food/aguacate" + metadata["s"].must_equal "2" + metadata["t"].must_equal "text/plain; charset=utf-8" + metadata["e"].must_equal "0815etag" + metadata["m"].length.must_equal 13 + end + + it "creates the directory objects metadata in redis" do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + + metadata = redis.hgetall "rs:m:phil:/" + metadata["e"].must_equal "fe2976909daaf074660981ab563fe65d" + metadata["m"].length.must_equal 13 + + metadata = redis.hgetall "rs:m:phil:food/" + metadata["e"].must_equal "926f98ff820f2f9764fd3c60a22865ad" + metadata["m"].length.must_equal 13 + + food_items = redis.smembers "rs:m:phil:food/:items" + food_items.each do |food_item| + ["camaron", "aguacate"].must_include food_item + end + + root_items = redis.smembers "rs:m:phil:/:items" + root_items.must_equal ["food/"] + end + + context "response code" do + it "is 201 for newly created objects" do + put "/phil/food/aguacate", "ci" + + last_response.status.must_equal 201 + end + + it "is 200 for updated objects" do + put "/phil/food/aguacate", "deliciosa" + put "/phil/food/aguacate", "muy deliciosa" + + last_response.status.must_equal 200 + end + end + + context "logging usage size" do + it "logs the complete size when creating new objects" do + put "/phil/food/aguacate", "1234567890" + + size_log = redis.get "rs:s:phil" + size_log.must_equal "10" + end + + it "logs the size difference when updating existing objects" do + put "/phil/food/camaron", "1234567890" + put "/phil/food/aguacate", "1234567890" + put "/phil/food/aguacate", "123" + + size_log = redis.get "rs:s:phil" + size_log.must_equal "13" + end + end + + describe "objects in root dir" do + before do + put "/phil/bamboo.txt", "shir kan" + end + + it "are listed in the directory listing with all metadata" do + get "phil/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["items"]["bamboo.txt"].wont_be_nil + content["items"]["bamboo.txt"]["ETag"].must_equal "0818etag" + content["items"]["bamboo.txt"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["bamboo.txt"]["Content-Length"].must_equal 8 + content["items"]["bamboo.txt"]["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + end + + describe "name collision checks" do + it "is successful when there is no name collision" do + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 201 + + metadata = redis.hgetall "rs:m:phil:food/aguacate" + metadata["s"].must_equal "2" + end + + it "conflicts when there is a directory with same name as document" do + put "/phil/food/aguacate", "si" + put "/phil/food", "wontwork" + + last_response.status.must_equal 409 + last_response.body.must_equal "Conflict" + + metadata = redis.hgetall "rs:m:phil:food" + metadata.must_be_empty + end + + it "conflicts when there is a document with same name as directory" do + put "/phil/food/aguacate", "si" + put "/phil/food/aguacate/empanado", "wontwork" + + last_response.status.must_equal 409 + + metadata = redis.hgetall "rs:m:phil:food/aguacate/empanado" + metadata.must_be_empty + end + + it "returns 400 when a Content-Range header is sent" do + header "Content-Range", "bytes 0-3/3" + + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 400 + end + end + + describe "If-Match header" do + before do + put "/phil/food/aguacate", "si" + end + + it "allows the request if the header matches the current ETag" do + header "If-Match", "\"0815etag\"" + + put "/phil/food/aguacate", "aye" + + last_response.status.must_equal 200 + last_response.headers["Etag"].must_equal "\"0915etag\"" + end + + it "allows the request if the header contains a weak ETAG matching the current ETag" do + header "If-Match", "W/\"0815etag\"" + + put "/phil/food/aguacate", "aye" + + last_response.status.must_equal 200 + last_response.headers["Etag"].must_equal "\"0915etag\"" + end + + it "allows the request if the header contains a weak ETAG with leading quote matching the current ETag" do + header "If-Match", "\"W/\"0815etag\"" + + put "/phil/food/aguacate", "aye" + + last_response.status.must_equal 200 + last_response.headers["Etag"].must_equal "\"0915etag\"" + end + + it "fails the request if the header does not match the current ETag" do + header "If-Match", "someotheretag" + + put "/phil/food/aguacate", "aye" + + last_response.status.must_equal 412 + last_response.body.must_equal "Precondition Failed" + end + + it "allows the request if redis metadata became out of sync" do + header "If-Match", "\"0815etag\"" + + put "/phil/food/aguacate", "aye" + + last_response.status.must_equal 200 + end + end + + describe "If-None-Match header set to '*'" do + it "succeeds when the document doesn't exist yet" do + header "If-None-Match", "*" + + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 201 + end + + it "fails the request if the document already exists" do + put "/phil/food/aguacate", "si" + + header "If-None-Match", "*" + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 412 + last_response.body.must_equal "Precondition Failed" + end + end + end + + end + + describe "DELETE requests" do + + before do + purge_redis + end + + context "not authorized" do + describe "with no token" do + it "says it's not authorized" do + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + describe "with empty token" do + it "says it's not authorized" do + header "Authorization", "Bearer " + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + describe "with wrong token" do + it "says it's not authorized" do + header "Authorization", "Bearer wrongtoken" + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + end + + context "authorized" do + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desayunos/bolon", "wow" + end + + it "decreases the size log by size of deleted object" do + delete "/phil/food/aguacate" + + size_log = redis.get "rs:s:phil" + size_log.must_equal "8" + end + + it "deletes the metadata object in redis" do + delete "/phil/food/aguacate" + + metadata = redis.hgetall "rs:m:phil:food/aguacate" + metadata.must_be_empty + end + + it "deletes the directory objects metadata in redis" do + old_metadata = redis.hgetall "rs:m:phil:food/" + + storage_class.stub_any_instance :etag_for, "newetag" do + delete "/phil/food/aguacate" + end + + metadata = redis.hgetall "rs:m:phil:food/" + metadata["e"].must_equal "newetag" + metadata["m"].length.must_equal 13 + metadata["m"].wont_equal old_metadata["m"] + + food_items = redis.smembers "rs:m:phil:food/:items" + food_items.sort.must_equal ["camaron", "desayunos/"] + + root_items = redis.smembers "rs:m:phil:/:items" + root_items.must_equal ["food/"] + end + + it "deletes the parent directory objects metadata when deleting all items" do + delete "/phil/food/aguacate" + delete "/phil/food/camaron" + delete "/phil/food/desayunos/bolon" + + redis.smembers("rs:m:phil:food/desayunos:items").must_be_empty + redis.hgetall("rs:m:phil:food/desayunos/").must_be_empty + + redis.smembers("rs:m:phil:food/:items").must_be_empty + redis.hgetall("rs:m:phil:food/").must_be_empty + + redis.smembers("rs:m:phil:/:items").must_be_empty + end + + it "responds with the ETag of the deleted item in the header" do + delete "/phil/food/aguacate" + + last_response.headers["ETag"].must_equal "\"0815etag\"" + end + + context "when item doesn't exist" do + before do + purge_redis + + delete "/phil/food/steak" + end + + it "returns a 404" do + last_response.status.must_equal 404 + last_response.body.must_equal "Not Found" + end + + it "deletes any metadata that might still exist" do + delete "/phil/food/steak" + + metadata = redis.hgetall "rs:m:phil:food/steak" + metadata.must_be_empty + + redis.smembers("rs:m:phil:food/:items").must_be_empty + redis.hgetall("rs:m:phil:food/").must_be_empty + + redis.smembers("rs:m:phil:/:items").must_be_empty + end + end + + describe "If-Match header" do + it "succeeds when the header matches the current ETag" do + header "If-Match", "\"0815etag\"" + + delete "/phil/food/aguacate" + + last_response.status.must_equal 200 + end + + it "succeeds when the header contains a weak ETAG matching the current ETag" do + header "If-Match", "W/\"0815etag\"" + + delete "/phil/food/aguacate" + + last_response.status.must_equal 200 + end + + it "fails the request if it does not match the current ETag" do + header "If-Match", "someotheretag" + + delete "/phil/food/aguacate" + + last_response.status.must_equal 412 + last_response.body.must_equal "Precondition Failed" + end + end + end + end + + describe "GET requests" do + + before do + purge_redis + end + + context "not authorized" do + + describe "without token" do + it "says it's not authorized" do + get "/phil/food/" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + describe "with wrong token" do + it "says it's not authorized" do + header "Authorization", "Bearer wrongtoken" + get "/phil/food/" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + end + + context "authorized" do + + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desayunos/bolon", "wow" + end + + describe "documents" do + + it "returns the required response headers" do + get "/phil/food/aguacate" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"0817etag\"" + last_response.headers["Cache-Control"].must_equal "no-cache" + last_response.headers["Content-Type"].must_equal "text/plain; charset=utf-8" + end + + it "returns a 404 when data doesn't exist" do + get "/phil/food/steak" + + last_response.status.must_equal 404 + last_response.body.must_equal "Not Found" + end + + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"0815etag\"" + + get "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do + header "If-None-Match", "W/\"0815etag\"" + + get "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + + end + + describe "directory listings" do + + it "returns the correct ETag header" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + end + + it "returns a Cache-Control header with value 'no-cache'" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.headers["Cache-Control"].must_equal "no-cache" + end + + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + get "/phil/food/" + + last_response.status.must_equal 304 + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the ETag" do + header "If-None-Match", "W/\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + get "/phil/food/" + + last_response.status.must_equal 304 + end + + it "contains all items in the directory" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["@context"].must_equal "http://remotestorage.io/spec/folder-description" + content["items"]["aguacate"].wont_be_nil + content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["aguacate"]["Content-Length"].must_equal 2 + content["items"]["aguacate"]["ETag"].must_equal "0815etag" + content["items"]["camaron"].wont_be_nil + content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["camaron"]["Content-Length"].must_equal 5 + content["items"]["camaron"]["ETag"].must_equal "0816etag" + content["items"]["desayunos/"].wont_be_nil + content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d" + end + + it "contains all items in the root directory" do + get "phil/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["items"]["food/"].wont_be_nil + content["items"]["food/"]["ETag"].must_equal "f9f85fbf5aa1fa378fd79ac8aa0a457d" + end + + it "responds with an empty directory liting when directory doesn't exist" do + get "phil/some-non-existing-dir/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["items"].must_equal({}) + end + + end + end + + end + + describe "HEAD requests" do + + before do + purge_redis + end + + context "not authorized" do + + describe "without token" do + it "says it's not authorized" do + head "/phil/food/camarones" + + last_response.status.must_equal 401 + last_response.body.must_be_empty + end + end + + describe "with wrong token" do + it "says it's not authorized" do + header "Authorization", "Bearer wrongtoken" + head "/phil/food/camarones" + + last_response.status.must_equal 401 + last_response.body.must_be_empty + end + end + + end + + context "authorized" do + + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desayunos/bolon", "wow" + end + + describe "directory listings" do + it "returns the correct header information" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + end + end + + describe "documents" do + context "when the document doesn't exist" do + it "returns a 404" do + head "/phil/food/steak" + + last_response.status.must_equal 404 + last_response.body.must_be_empty + end + end + + context "when the document exists" do + it "returns the required response headers" do + head "/phil/food/aguacate" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Cache-Control"].must_equal "no-cache" + end + + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"0815etag\"" + + head "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do + header "If-None-Match", "W/\"0815etag\"" + + head "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + end + end + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index b08c6b5..16a4dad 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -6,12 +6,15 @@ require_relative '../liquor-cabinet' require 'minitest/autorun' +require "minitest/stub_any_instance" require 'rack/test' require 'purdytest' require "redis" require "rest_client" -require "minitest/stub_any_instance" require "ostruct" +require 'webmock/minitest' + +WebMock.disable_net_connect! def app LiquorCabinet @@ -19,17 +22,6 @@ def app app.set :environment, :test -def wait_a_second - now = Time.now.to_i - while Time.now.to_i == now; end -end - -def write_last_response_to_file(filename = "last_response.html") - File.open(filename, "w") do |f| - f.write last_response.body - end -end - alias context describe if app.settings.respond_to? :redis @@ -43,3 +35,23 @@ def purge_redis end end end + +MiniTest::Spec.class_eval do + def self.shared_examples + @shared_examples ||= {} + end +end + +module MiniTest::Spec::SharedExamples + def shared_examples_for(desc, &block) + MiniTest::Spec.shared_examples[desc] = block + end + + def it_behaves_like(desc) + self.instance_eval(&MiniTest::Spec.shared_examples[desc]) + end +end + +Object.class_eval { include(MiniTest::Spec::SharedExamples) } + +require_relative 'shared_examples' diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 16311d9..a90c2d7 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -1,840 +1,56 @@ require_relative "../spec_helper" describe "App" do - include Rack::Test::Methods - - def app - LiquorCabinet + def container_url_for(user) + "#{app.settings.swift["host"]}/rs:documents:test/#{user}" end - it "returns 404 on non-existing routes" do - get "/virginmargarita" - last_response.status.must_equal 404 + def storage_class + RemoteStorage::Swift end - describe "PUT requests" do - - before do - purge_redis - end - - context "authorized" do - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - end - - it "creates the metadata object in redis" do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end - - metadata = redis.hgetall "rs:m:phil:food/aguacate" - metadata["s"].must_equal "2" - metadata["t"].must_equal "text/plain; charset=utf-8" - metadata["e"].must_equal "bla" - metadata["m"].length.must_equal 13 - end - - it "creates the directory objects metadata in redis" do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - get_stub = OpenStruct.new(body: "rootbody") - - RestClient.stub :put, put_stub do - RestClient.stub :get, get_stub do - RemoteStorage::Swift.stub_any_instance :etag_for, "newetag" do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - end - end - end - - metadata = redis.hgetall "rs:m:phil:/" - metadata["e"].must_equal "newetag" - metadata["m"].length.must_equal 13 - - metadata = redis.hgetall "rs:m:phil:food/" - metadata["e"].must_equal "newetag" - metadata["m"].length.must_equal 13 - - food_items = redis.smembers "rs:m:phil:food/:items" - food_items.each do |food_item| - ["camaron", "aguacate"].must_include food_item - end - - root_items = redis.smembers "rs:m:phil:/:items" - root_items.must_equal ["food/"] - end - - context "response code" do - before do - @put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - end - - it "is 201 for newly created objects" do - RestClient.stub :put, @put_stub do - put "/phil/food/aguacate", "muy deliciosa" - end - - last_response.status.must_equal 201 - end - - it "is 200 for updated objects" do - RestClient.stub :put, @put_stub do - put "/phil/food/aguacate", "deliciosa" - put "/phil/food/aguacate", "muy deliciosa" - end - - last_response.status.must_equal 200 - end - end - - context "logging usage size" do - before do - @put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - end - - it "logs the complete size when creating new objects" do - RestClient.stub :put, @put_stub do - put "/phil/food/aguacate", "1234567890" - end - - size_log = redis.get "rs:s:phil" - size_log.must_equal "10" - end - - it "logs the size difference when updating existing objects" do - RestClient.stub :put, @put_stub do - put "/phil/food/camaron", "1234567890" - put "/phil/food/aguacate", "1234567890" - put "/phil/food/aguacate", "123" - end - - size_log = redis.get "rs:s:phil" - size_log.must_equal "13" - end - end - - describe "objects in root dir" do - before do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/bamboo.txt", "shir kan" - end - end - - it "are listed in the directory listing with all metadata" do - get "phil/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["items"]["bamboo.txt"].wont_be_nil - content["items"]["bamboo.txt"]["ETag"].must_equal "bla" - content["items"]["bamboo.txt"]["Content-Type"].must_equal "text/plain; charset=utf-8" - content["items"]["bamboo.txt"]["Content-Length"].must_equal 8 - content["items"]["bamboo.txt"]["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - end - - describe "name collision checks" do - it "is successful when there is no name collision" do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - get_stub = OpenStruct.new(body: "rootbody") - - RestClient.stub :put, put_stub do - RestClient.stub :get, get_stub do - RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do - put "/phil/food/aguacate", "si" - end - end - end - - last_response.status.must_equal 201 - - metadata = redis.hgetall "rs:m:phil:food/aguacate" - metadata["s"].must_equal "2" - end - - it "conflicts when there is a directory with same name as document" do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food", "wontwork" - end - - last_response.status.must_equal 409 - last_response.body.must_equal "Conflict" - - metadata = redis.hgetall "rs:m:phil:food" - metadata.must_be_empty - end - - it "conflicts when there is a document with same name as directory" do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/aguacate/empanado", "wontwork" - end - - last_response.status.must_equal 409 - - metadata = redis.hgetall "rs:m:phil:food/aguacate/empanado" - metadata.must_be_empty - end - - it "returns 400 when a Content-Range header is sent" do - header "Content-Range", "bytes 0-3/3" - - put "/phil/food/aguacate", "si" - - last_response.status.must_equal 400 - end - end - - describe "If-Match header" do - before do - put_stub = OpenStruct.new(headers: { - etag: "oldetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end - end - - it "allows the request if the header matches the current ETag" do - header "If-Match", "\"oldetag\"" - - put_stub = OpenStruct.new(headers: { - etag: "newetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "aye" - end - - last_response.status.must_equal 200 - last_response.headers["Etag"].must_equal "\"newetag\"" - end - - it "allows the request if the header contains a weak ETAG matching the current ETag" do - header "If-Match", "W/\"oldetag\"" - - put_stub = OpenStruct.new(headers: { - etag: "newetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "aye" - end - - last_response.status.must_equal 200 - last_response.headers["Etag"].must_equal "\"newetag\"" - end - - it "allows the request if the header contains a weak ETAG with leading quote matching the current ETag" do - header "If-Match", "\"W/\"oldetag\"" - - put_stub = OpenStruct.new(headers: { - etag: "newetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "aye" - end - - last_response.status.must_equal 200 - last_response.headers["Etag"].must_equal "\"newetag\"" - end - - it "fails the request if the header does not match the current ETag" do - header "If-Match", "someotheretag" - - head_stub = OpenStruct.new(headers: { - etag: "oldetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain", - content_length: 23 - }) - - RestClient.stub :head, head_stub do - put "/phil/food/aguacate", "aye" - end - - last_response.status.must_equal 412 - last_response.body.must_equal "Precondition Failed" - end - - it "allows the request if redis metadata became out of sync" do - header "If-Match", "\"existingetag\"" - - head_stub = OpenStruct.new(headers: { - etag: "existingetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain", - content_length: 23 - }) - - put_stub = OpenStruct.new(headers: { - etag: "newetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :head, head_stub do - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "aye" - end - end - - last_response.status.must_equal 200 - end - end - - describe "If-None-Match header set to '*'" do - it "succeeds when the document doesn't exist yet" do - put_stub = OpenStruct.new(headers: { - etag: "someetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - header "If-None-Match", "*" - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end - - last_response.status.must_equal 201 - end - - it "fails the request if the document already exists" do - put_stub = OpenStruct.new(headers: { - etag: "someetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end - - header "If-None-Match", "*" - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end - - last_response.status.must_equal 412 - last_response.body.must_equal "Precondition Failed" - end - end - end - - end - - describe "DELETE requests" do - - before do - purge_redis - end - - context "not authorized" do - describe "with no token" do - it "says it's not authorized" do - delete "/phil/food/aguacate" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - describe "with empty token" do - it "says it's not authorized" do - header "Authorization", "Bearer " - delete "/phil/food/aguacate" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - describe "with wrong token" do - it "says it's not authorized" do - header "Authorization", "Bearer wrongtoken" - delete "/phil/food/aguacate" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - end - - context "authorized" do - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desayunos/bolon", "wow" - end - end - - it "decreases the size log by size of deleted object" do - RestClient.stub :delete, "" do - RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do - delete "/phil/food/aguacate" - end - end - - size_log = redis.get "rs:s:phil" - size_log.must_equal "8" - end - - it "deletes the metadata object in redis" do - RestClient.stub :delete, "" do - RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do - delete "/phil/food/aguacate" - end - end - - metadata = redis.hgetall "rs:m:phil:food/aguacate" - metadata.must_be_empty - end - - it "deletes the directory objects metadata in redis" do - old_metadata = redis.hgetall "rs:m:phil:food/" - - RestClient.stub :delete, "" do - RemoteStorage::Swift.stub_any_instance :etag_for, "newetag" do - delete "/phil/food/aguacate" - end - end - - metadata = redis.hgetall "rs:m:phil:food/" - metadata["e"].must_equal "newetag" - metadata["m"].length.must_equal 13 - metadata["m"].wont_equal old_metadata["m"] - - food_items = redis.smembers "rs:m:phil:food/:items" - food_items.sort.must_equal ["camaron", "desayunos/"] - - root_items = redis.smembers "rs:m:phil:/:items" - root_items.must_equal ["food/"] - end - - it "deletes the parent directory objects metadata when deleting all items" do - RestClient.stub :delete, "" do - RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do - delete "/phil/food/aguacate" - delete "/phil/food/camaron" - delete "/phil/food/desayunos/bolon" - end - end - - redis.smembers("rs:m:phil:food/desayunos:items").must_be_empty - redis.hgetall("rs:m:phil:food/desayunos/").must_be_empty - - redis.smembers("rs:m:phil:food/:items").must_be_empty - redis.hgetall("rs:m:phil:food/").must_be_empty - - redis.smembers("rs:m:phil:/:items").must_be_empty - end - - it "responds with the ETag of the deleted item in the header" do - RestClient.stub :delete, "" do - delete "/phil/food/aguacate" - end - - last_response.headers["ETag"].must_equal "\"bla\"" - end - - context "when item doesn't exist" do - before do - purge_redis - - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/steak", "si" - end - - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :delete, raises_exception do - delete "/phil/food/steak" - end - end - - it "returns a 404" do - last_response.status.must_equal 404 - last_response.body.must_equal "Not Found" - end - - it "deletes any metadata that might still exist" do - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :delete, raises_exception do - delete "/phil/food/steak" - end - - metadata = redis.hgetall "rs:m:phil:food/steak" - metadata.must_be_empty - - redis.smembers("rs:m:phil:food/:items").must_be_empty - redis.hgetall("rs:m:phil:food/").must_be_empty - - redis.smembers("rs:m:phil:/:items").must_be_empty - end - end - - describe "If-Match header" do - it "succeeds when the header matches the current ETag" do - header "If-Match", "\"bla\"" - - RestClient.stub :delete, "" do - delete "/phil/food/aguacate" - end - - last_response.status.must_equal 200 - end - - it "succeeds when the header contains a weak ETAG matching the current ETag" do - header "If-Match", "W/\"bla\"" - - RestClient.stub :delete, "" do - delete "/phil/food/aguacate" - end - - last_response.status.must_equal 200 - end - - it "fails the request if it does not match the current ETag" do - header "If-Match", "someotheretag" - - delete "/phil/food/aguacate" - - last_response.status.must_equal 412 - last_response.body.must_equal "Precondition Failed" - end - end - end + def config_file + "config.yml.example.swift" end - describe "GET requests" do - - before do - purge_redis - end - - context "not authorized" do - - describe "without token" do - it "says it's not authorized" do - get "/phil/food/" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - describe "with wrong token" do - it "says it's not authorized" do - header "Authorization", "Bearer wrongtoken" - get "/phil/food/" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - end - - context "authorized" do - - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - - put_stub = OpenStruct.new(headers: { - etag: "0815etag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desayunos/bolon", "wow" - end - end - - describe "documents" do - - it "returns the required response headers" do - get_stub = OpenStruct.new(body: "si", headers: { - etag: "0815etag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain; charset=utf-8", - content_length: 2 - }) - - RestClient.stub :get, get_stub do - get "/phil/food/aguacate" - end - - last_response.status.must_equal 200 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Cache-Control"].must_equal "no-cache" - last_response.headers["Content-Type"].must_equal "text/plain; charset=utf-8" - end - - it "returns a 404 when data doesn't exist" do - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :get, raises_exception do - get "/phil/food/steak" - end - - last_response.status.must_equal 404 - last_response.body.must_equal "Not Found" - end - - it "responds with 304 when IF_NONE_MATCH header contains the ETag" do - - header "If-None-Match", "\"0815etag\"" - - get "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - - it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do - header "If-None-Match", "W/\"0815etag\"" - - get "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - - end - - describe "directory listings" do - - it "returns the correct ETag header" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - end - - it "returns a Cache-Control header with value 'no-cache'" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.headers["Cache-Control"].must_equal "no-cache" - end - - it "responds with 304 when IF_NONE_MATCH header contains the ETag" do - header "If-None-Match", "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - get "/phil/food/" - - last_response.status.must_equal 304 - end - - it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the ETag" do - header "If-None-Match", "W/\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - get "/phil/food/" - - last_response.status.must_equal 304 - end - - it "contains all items in the directory" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["@context"].must_equal "http://remotestorage.io/spec/folder-description" - content["items"]["aguacate"].wont_be_nil - content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8" - content["items"]["aguacate"]["Content-Length"].must_equal 2 - content["items"]["aguacate"]["ETag"].must_equal "0815etag" - content["items"]["camaron"].wont_be_nil - content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8" - content["items"]["camaron"]["Content-Length"].must_equal 5 - content["items"]["camaron"]["ETag"].must_equal "0815etag" - content["items"]["desayunos/"].wont_be_nil - content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d" - end - - it "contains all items in the root directory" do - get "phil/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["items"]["food/"].wont_be_nil - content["items"]["food/"]["ETag"].must_equal "f9f85fbf5aa1fa378fd79ac8aa0a457d" - end - - it "responds with an empty directory liting when directory doesn't exist" do - get "phil/some-non-existing-dir/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["items"].must_equal({}) - end - - end - end - + before do + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "si"). + to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "aye"). + to_return(status: 200, headers: { etag: "0915etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "deliciosa"). + to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "muy deliciosa"). + to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { etag: "0815etag" }) + stub_request(:put, "#{container_url_for("phil")}/food/camaron"). + to_return(status: 200, headers: { etag: "0816etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/camaron"). + to_return(status: 200, headers: { etag: "0816etag" }) + stub_request(:put, "#{container_url_for("phil")}/food/desayunos/bolon"). + to_return(status: 200, headers: { etag: "0817etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/desayunos/bolon"). + to_return(status: 200, headers: { etag: "0817etag" }) + stub_request(:get, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, body: "rootbody", headers: { etag: "0817etag", content_type: "text/plain; charset=utf-8" }) + stub_request(:put, "#{container_url_for("phil")}/bamboo.txt"). + to_return(status: 200, headers: { etag: "0818etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:head, "#{container_url_for("phil")}/food/steak"). + to_return(status: 404) + stub_request(:get, "#{container_url_for("phil")}/food/steak"). + to_return(status: 404) + stub_request(:delete, "#{container_url_for("phil")}/food/steak"). + to_return(status: 404) end - describe "HEAD requests" do - - before do - purge_redis - end - - context "not authorized" do - - describe "without token" do - it "says it's not authorized" do - head "/phil/food/camarones" - - last_response.status.must_equal 401 - last_response.body.must_be_empty - end - end - - describe "with wrong token" do - it "says it's not authorized" do - header "Authorization", "Bearer wrongtoken" - head "/phil/food/camarones" - - last_response.status.must_equal 401 - last_response.body.must_be_empty - end - end - - end - - context "authorized" do - - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - - put_stub = OpenStruct.new(headers: { - etag: "0815etag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desayunos/bolon", "wow" - end - end - - describe "directory listings" do - it "returns the correct header information" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - end - end - - describe "documents" do - context "when the document doesn't exist" do - it "returns a 404" do - head "/phil/food/steak" - - last_response.status.must_equal 404 - last_response.body.must_be_empty - end - end - - context "when the document exists" do - it "returns the required response headers" do - head "/phil/food/aguacate" - - last_response.status.must_equal 200 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Cache-Control"].must_equal "no-cache" - end - - it "responds with 304 when IF_NONE_MATCH header contains the ETag" do - header "If-None-Match", "\"0815etag\"" - - head "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - - it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do - header "If-None-Match", "W/\"0815etag\"" - - head "/phil/food/aguacate" - - last_response.status.must_equal 304 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - end - end - end - end + it_behaves_like 'a REST adapter' end - From b00fc5bee2a1e5df81d8018720bbe514bd549122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 15:10:55 +0200 Subject: [PATCH 06/21] Reverse the not found logic in the delete_data method to make it clearer --- lib/remote_storage/rest_provider.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index 0e95577..1b59601 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -165,7 +165,6 @@ def put_data(user, directory, key, data, content_type) def delete_data(user, directory, key) url = url_for_key(user, directory, key) - not_found = false existing_metadata = redis.hgetall "rs:m:#{user}:#{directory}/#{key}" @@ -175,17 +174,17 @@ def delete_data(user, directory, key) end end - not_found = !try_to_delete(url) + found = try_to_delete(url) log_size_difference(user, existing_metadata["s"], 0) delete_metadata_objects(user, directory, key) delete_dir_objects(user, directory) - if not_found - server.halt 404, "Not Found" - else + if found server.headers["Etag"] = %Q("#{existing_metadata["e"]}") server.halt 200 + else + server.halt 404, "Not Found" end end From 3b72b8d0bb5d139301773c631adbb470ff5c9df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 15:12:53 +0200 Subject: [PATCH 07/21] Pass the headers to the set_response_headers directly, not the response --- lib/remote_storage/rest_provider.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index 1b59601..c394689 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -67,7 +67,7 @@ def get_data(user, directory, key) res = do_get_request(url) - set_response_headers(res) + set_response_headers(res.headers) return res.body rescue RestClient::ResourceNotFound @@ -208,11 +208,11 @@ def default_headers raise NotImplementedError end - def set_response_headers(response) - server.headers["ETag"] = format_etag(response.headers[:etag]) - server.headers["Content-Type"] = response.headers[:content_type] - server.headers["Content-Length"] = response.headers[:content_length] - server.headers["Last-Modified"] = response.headers[:last_modified] + def set_response_headers(headers) + server.headers["ETag"] = format_etag(headers[:etag]) + server.headers["Content-Type"] = headers[:content_type] + server.headers["Content-Length"] = headers[:content_length] + server.headers["Last-Modified"] = headers[:last_modified] end def extract_category(directory) From 97cd5ec837ef7eed630a4f191e904a162c6a808d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 15:15:04 +0200 Subject: [PATCH 08/21] Move a comment to the relevant line --- lib/remote_storage/s3_rest.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/remote_storage/s3_rest.rb b/lib/remote_storage/s3_rest.rb index 5f3db7d..48b6cb4 100644 --- a/lib/remote_storage/s3_rest.rb +++ b/lib/remote_storage/s3_rest.rb @@ -23,9 +23,9 @@ def do_put_request(url, data, content_type) end end - # S3 does not return a Last-Modified response header on PUTs def do_put_request_and_return_etag_and_last_modified(url, data, content_type) res = do_put_request(url, data, content_type) + # S3 does not return a Last-Modified response header on PUTs head_res = do_head_request(url) return [res.headers[:etag].delete('"'), timestamp_for(head_res.headers[:last_modified])] From 1532a23d296f30fb66db4469e79c2eb48787f382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 15:30:16 +0200 Subject: [PATCH 09/21] Reorder the argument in authorization_headers_for Make content_type and md5 optional (set to nil by default) --- lib/remote_storage/s3_rest.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/remote_storage/s3_rest.rb b/lib/remote_storage/s3_rest.rb index 48b6cb4..fd9d62d 100644 --- a/lib/remote_storage/s3_rest.rb +++ b/lib/remote_storage/s3_rest.rb @@ -18,7 +18,7 @@ def format_etag(etag) def do_put_request(url, data, content_type) deal_with_unauthorized_requests do md5 = Digest::MD5.base64digest(data) - authorization_headers = authorization_headers_for("PUT", md5, content_type, url) + authorization_headers = authorization_headers_for("PUT", url, md5, content_type) RestClient.put(url, data, authorization_headers.merge({ "Content-Type" => content_type, "Content-Md5" => md5})) end end @@ -33,21 +33,21 @@ def do_put_request_and_return_etag_and_last_modified(url, data, content_type) def do_get_request(url, &block) deal_with_unauthorized_requests do - authorization_headers = authorization_headers_for("GET", "", "", url) + authorization_headers = authorization_headers_for("GET", url) RestClient.get(url, authorization_headers, &block) end end def do_head_request(url, &block) deal_with_unauthorized_requests do - authorization_headers = authorization_headers_for("HEAD", "", "", url) + authorization_headers = authorization_headers_for("HEAD", url) RestClient.head(url, authorization_headers, &block) end end def do_delete_request(url) deal_with_unauthorized_requests do - authorization_headers = authorization_headers_for("DELETE", "", "", url) + authorization_headers = authorization_headers_for("DELETE", url) RestClient.delete(url, authorization_headers) end end @@ -68,10 +68,10 @@ def try_to_delete(url) # This is using the S3 authorizations, not the newer AW V4 Signatures # (https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html) - def authorization_headers_for(http_verb, md5, content_type, url) + def authorization_headers_for(http_verb, url, md5 = nil, content_type = nil) url = File.join("/", url.gsub(base_url, "")) date = Time.now.httpdate - signed_data = signature(http_verb, md5, content_type, date, url) + signed_data = generate_s3_signature(http_verb, md5, content_type, date, url) { "Authorization" => "AWS #{credentials[:access_key_id]}:#{signed_data}", "Date" => date} end @@ -92,7 +92,7 @@ def uri_escape(s) WEBrick::HTTPUtils.escape(s).gsub('%5B', '[').gsub('%5D', ']') end - def signature(http_verb, md5, content_type, date, url) + def generate_s3_signature(http_verb, md5, content_type, date, url) string_to_sign = [http_verb, md5, content_type, date, url].join "\n" signature = digest(credentials[:secret_key_id], string_to_sign) uri_escape(signature) From 454f02dfafba15686770d39de6978ecf618bb402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 15:44:28 +0200 Subject: [PATCH 10/21] Rename the spec root description --- spec/s3/app_spec.rb | 2 +- spec/swift/app_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 9874edc..eb0bd39 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -1,6 +1,6 @@ require_relative "../spec_helper" -describe "App" do +describe "S3 provider" do def container_url_for(user) "#{app.settings.s3["endpoint"]}#{app.settings.s3["bucket"]}/#{user}" end diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index a90c2d7..ef31cb9 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -1,6 +1,6 @@ require_relative "../spec_helper" -describe "App" do +describe "Swift provider" do def container_url_for(user) "#{app.settings.swift["host"]}/rs:documents:test/#{user}" end From 5da0d0b5c345e2a2b199b08494435e630806bdf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 15:44:49 +0200 Subject: [PATCH 11/21] Delete unused methods --- spec/shared_examples.rb | 4 ---- spec/swift/app_spec.rb | 4 ---- 2 files changed, 8 deletions(-) diff --git a/spec/shared_examples.rb b/spec/shared_examples.rb index ec74e15..baa6900 100644 --- a/spec/shared_examples.rb +++ b/spec/shared_examples.rb @@ -11,10 +11,6 @@ def storage_class raise NotImplementedError end - def config_file - raise NotImplementedError - end - it "returns 404 on non-existing routes" do get "/virginmargarita" last_response.status.must_equal 404 diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index ef31cb9..41884f3 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -9,10 +9,6 @@ def storage_class RemoteStorage::Swift end - def config_file - "config.yml.example.swift" - end - before do stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) From ca0127d6a67c0702a6dfa13dc1f7a0012c4e0483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 15:46:18 +0200 Subject: [PATCH 12/21] Rename the S3 provider to just S3 --- lib/remote_storage/{s3_rest.rb => s3.rb} | 2 +- liquor-cabinet.rb | 4 ++-- spec/s3/app_spec.rb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename lib/remote_storage/{s3_rest.rb => s3.rb} (99%) diff --git a/lib/remote_storage/s3_rest.rb b/lib/remote_storage/s3.rb similarity index 99% rename from lib/remote_storage/s3_rest.rb rename to lib/remote_storage/s3.rb index fd9d62d..c195272 100644 --- a/lib/remote_storage/s3_rest.rb +++ b/lib/remote_storage/s3.rb @@ -5,7 +5,7 @@ require "webrick/httputils" module RemoteStorage - class S3Rest + class S3 include RestProvider private diff --git a/liquor-cabinet.rb b/liquor-cabinet.rb index 2d21522..6f6324f 100644 --- a/liquor-cabinet.rb +++ b/liquor-cabinet.rb @@ -5,7 +5,7 @@ require 'sinatra/config_file' require "sinatra/reloader" require "remote_storage/swift" -require "remote_storage/s3_rest" +require "remote_storage/s3" class LiquorCabinet < Sinatra::Base @@ -132,7 +132,7 @@ def storage if settings.respond_to? :swift RemoteStorage::Swift.new(settings, self) elsif settings.respond_to? :s3 - RemoteStorage::S3Rest.new(settings, self) + RemoteStorage::S3.new(settings, self) else puts <<-EOF You need to set one storage backend in your config.yml file. diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index eb0bd39..828c6f8 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -6,7 +6,7 @@ def container_url_for(user) end def storage_class - RemoteStorage::S3Rest + RemoteStorage::S3 end before do From f14ef4dc21e729da3f8d04f44fee382716f1ef1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 16:39:12 +0200 Subject: [PATCH 13/21] Consider that the metadata has changed when the Last-Modified changes Also adds a spec for it --- lib/remote_storage/rest_provider.rb | 3 +-- spec/s3/app_spec.rb | 8 ++++++++ spec/shared_examples.rb | 11 +++++++++++ spec/swift/app_spec.rb | 6 ++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index c394689..621d456 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -289,10 +289,9 @@ def has_name_collision?(user, directory, key) def metadata_changed?(old_metadata, new_metadata) # check metadata relevant to the directory listing - # ie. the timestamp (m) is not relevant, because it's not used in - # the listing return old_metadata["e"] != new_metadata[:e] || old_metadata["s"] != new_metadata[:s].to_s || + old_metadata["m"] != new_metadata[:m] || old_metadata["t"] != new_metadata[:t] end diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 828c6f8..6c922b2 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -24,6 +24,14 @@ def storage_class stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). with(body: "muy deliciosa"). to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/banano"). + with(body: "si"). + to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/banano"). + with(body: "oh, no"). + to_return(status: 200, headers: { etag: '"0817etag"' }) + stub_request(:head, "#{container_url_for("phil")}/food/banano"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:20 GMT" }) stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). diff --git a/spec/shared_examples.rb b/spec/shared_examples.rb index baa6900..fd5f742 100644 --- a/spec/shared_examples.rb +++ b/spec/shared_examples.rb @@ -38,6 +38,17 @@ def storage_class metadata["m"].length.must_equal 13 end + it "updates the metadata object in redis when it changes" do + put "/phil/food/banano", "si" + put "/phil/food/banano", "oh, no" + + metadata = redis.hgetall "rs:m:phil:food/banano" + metadata["s"].must_equal "6" + metadata["t"].must_equal "text/plain; charset=utf-8" + metadata["e"].must_equal "0817etag" + metadata["m"].must_equal "1457094020000" + end + it "creates the directory objects metadata in redis" do put "/phil/food/aguacate", "si" put "/phil/food/camaron", "yummi" diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 41884f3..2bc913d 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -24,6 +24,12 @@ def storage_class stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). with(body: "muy deliciosa"). to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:put, "#{container_url_for("phil")}/food/banano"). + with(body: "si"). + to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:put, "#{container_url_for("phil")}/food/banano"). + with(body: "oh, no"). + to_return(status: 200, headers: { etag: "0817etag", last_modified: "Fri, 04 Mar 2016 12:20:20 GMT" }) stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). From a92286703d56253d7e7a31c6060c32dfc64de96a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Mon, 30 Apr 2018 16:46:28 +0200 Subject: [PATCH 14/21] Uncomment the development config in the examples --- config.yml.example.s3 | 21 ++++++++++----------- config.yml.example.swift | 14 +++++++------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/config.yml.example.s3 b/config.yml.example.s3 index 0acb9fd..1ae5766 100644 --- a/config.yml.example.s3 +++ b/config.yml.example.s3 @@ -1,16 +1,15 @@ development: &defaults maintenance: false - # # uncomment this section - # s3: - # endpoint: "https://some-endpoint" - # region: "region" - # access_key_id: "" - # secret_key_id: "" - # bucket: "test-bucket" - # # Redis is needed for the swift backend - # redis: - # host: localhost - # port: 6379 + s3: + endpoint: "https://some-endpoint" + region: "region" + access_key_id: "" + secret_key_id: "" + bucket: "test-bucket" + # Redis is needed for the swift backend + redis: + host: localhost + port: 6379 test: <<: *defaults diff --git a/config.yml.example.swift b/config.yml.example.swift index b44c579..e7cad01 100644 --- a/config.yml.example.swift +++ b/config.yml.example.swift @@ -1,12 +1,12 @@ development: &defaults maintenance: false - # # uncomment this section - # swift: &swift_defaults - # host: "https://swift.example.com" - # # Redis is needed for the swift backend - # redis: - # host: localhost - # port: 6379 + # uncomment this section + swift: &swift_defaults + host: "https://swift.example.com" + # Redis is needed for the swift backend + redis: + host: localhost + port: 6379 test: <<: *defaults From d0a28c711129bc7f4a562d576902254b82e81c55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 9 May 2018 15:21:51 +0200 Subject: [PATCH 15/21] Remove useless comments and remove duplicate section in the example configs --- config.yml.example.s3 | 4 ---- config.yml.example.swift | 5 ----- 2 files changed, 9 deletions(-) diff --git a/config.yml.example.s3 b/config.yml.example.s3 index 1ae5766..a0e654e 100644 --- a/config.yml.example.s3 +++ b/config.yml.example.s3 @@ -6,7 +6,6 @@ development: &defaults access_key_id: "" secret_key_id: "" bucket: "test-bucket" - # Redis is needed for the swift backend redis: host: localhost port: 6379 @@ -19,9 +18,6 @@ test: access_key_id: "" secret_key_id: "" bucket: "test-bucket" - redis: - host: localhost - port: 6379 staging: <<: *defaults diff --git a/config.yml.example.swift b/config.yml.example.swift index e7cad01..40fa39f 100644 --- a/config.yml.example.swift +++ b/config.yml.example.swift @@ -1,9 +1,7 @@ development: &defaults maintenance: false - # uncomment this section swift: &swift_defaults host: "https://swift.example.com" - # Redis is needed for the swift backend redis: host: localhost port: 6379 @@ -12,9 +10,6 @@ test: <<: *defaults swift: host: "https://swift.example.com" - redis: - host: localhost - port: 6379 staging: <<: *defaults From 0ec76c8140bb6f28d5de8f018d5ad630a4a83de4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 9 May 2018 15:26:32 +0200 Subject: [PATCH 16/21] Fix coding style for a hash --- lib/remote_storage/s3.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/remote_storage/s3.rb b/lib/remote_storage/s3.rb index c195272..b3929a6 100644 --- a/lib/remote_storage/s3.rb +++ b/lib/remote_storage/s3.rb @@ -72,8 +72,10 @@ def authorization_headers_for(http_verb, url, md5 = nil, content_type = nil) url = File.join("/", url.gsub(base_url, "")) date = Time.now.httpdate signed_data = generate_s3_signature(http_verb, md5, content_type, date, url) - { "Authorization" => "AWS #{credentials[:access_key_id]}:#{signed_data}", - "Date" => date} + { + "Authorization" => "AWS #{credentials[:access_key_id]}:#{signed_data}", + "Date" => date + } end def credentials From 709f63555dbcb77a0bf3ef4343a7c7ba399d3af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 9 May 2018 15:55:00 +0200 Subject: [PATCH 17/21] Rewrite to avoid long lines --- lib/remote_storage/s3.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/remote_storage/s3.rb b/lib/remote_storage/s3.rb index b3929a6..f2346d4 100644 --- a/lib/remote_storage/s3.rb +++ b/lib/remote_storage/s3.rb @@ -18,8 +18,10 @@ def format_etag(etag) def do_put_request(url, data, content_type) deal_with_unauthorized_requests do md5 = Digest::MD5.base64digest(data) - authorization_headers = authorization_headers_for("PUT", url, md5, content_type) - RestClient.put(url, data, authorization_headers.merge({ "Content-Type" => content_type, "Content-Md5" => md5})) + authorization_headers = authorization_headers_for( + "PUT", url, md5, content_type + ).merge({ "Content-Type" => content_type, "Content-Md5" => md5 }) + RestClient.put(url, data, authorization_headers) end end From 639c3724f02a06310f2e5c1507e193b4091a71fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 9 May 2018 16:01:14 +0200 Subject: [PATCH 18/21] Refactor the put_request method to have a return value --- lib/remote_storage/rest_provider.rb | 15 +++++++-------- lib/remote_storage/s3.rb | 17 ++++++++--------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb index 621d456..3c5d38d 100644 --- a/lib/remote_storage/rest_provider.rb +++ b/lib/remote_storage/rest_provider.rb @@ -141,7 +141,7 @@ def put_data(user, directory, key, data, content_type) server.halt 412, "Precondition Failed" unless existing_metadata.empty? end - etag, timestamp = do_put_request_and_return_etag_and_last_modified(url, data, content_type) + etag, timestamp = do_put_request(url, data, content_type) metadata = { e: etag, @@ -392,14 +392,13 @@ def url_for_key(user, directory, key) def do_put_request(url, data, content_type) deal_with_unauthorized_requests do - RestClient.put(url, data, default_headers.merge({content_type: content_type})) - end - end + res = RestClient.put(url, data, default_headers.merge({content_type: content_type})) - def do_put_request_and_return_etag_and_last_modified(url, data, content_type) - res = do_put_request(url, data, content_type) - - return [res.headers[:etag], timestamp_for(res.headers[:last_modified])] + return [ + res.headers[:etag], + timestamp_for(res.headers[:last_modified]) + ] + end end def do_get_request(url, &block) diff --git a/lib/remote_storage/s3.rb b/lib/remote_storage/s3.rb index f2346d4..875f596 100644 --- a/lib/remote_storage/s3.rb +++ b/lib/remote_storage/s3.rb @@ -21,18 +21,17 @@ def do_put_request(url, data, content_type) authorization_headers = authorization_headers_for( "PUT", url, md5, content_type ).merge({ "Content-Type" => content_type, "Content-Md5" => md5 }) - RestClient.put(url, data, authorization_headers) + res = RestClient.put(url, data, authorization_headers) + # S3 does not return a Last-Modified response header on PUTs + head_res = do_head_request(url) + + return [ + res.headers[:etag].delete('"'), + timestamp_for(head_res.headers[:last_modified]) + ] end end - def do_put_request_and_return_etag_and_last_modified(url, data, content_type) - res = do_put_request(url, data, content_type) - # S3 does not return a Last-Modified response header on PUTs - head_res = do_head_request(url) - - return [res.headers[:etag].delete('"'), timestamp_for(head_res.headers[:last_modified])] - end - def do_get_request(url, &block) deal_with_unauthorized_requests do authorization_headers = authorization_headers_for("GET", url) From c0d88f1da6496a72dd7e5c98065ce1e8dbc68df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 9 May 2018 16:18:22 +0200 Subject: [PATCH 19/21] Run Travis builds on the Docker infrastructure --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6c1356e..02acefa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,3 +22,5 @@ notifications: env: - BACKEND=s3 - BACKEND=swift +# Run on Docker infrastructure +sudo: false From df65190df8fb2776ca5983342814b2f9c8db14a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 9 May 2018 18:42:04 +0200 Subject: [PATCH 20/21] Simply the request stubs, add comments Remove the stubs that are not required, making everything easier to understand --- spec/s3/app_spec.rb | 28 ++++++++++++++-------------- spec/swift/app_spec.rb | 30 +++++++++++++++--------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 6c922b2..8829ab9 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -12,18 +12,18 @@ def storage_class before do stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { etag: '"0815etag"' }) - stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). - with(body: "si"). - to_return(status: 200, headers: { etag: '"0815etag"' }) + # Write new content with an If-Match header (a new Etag is returned) stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). with(body: "aye"). to_return(status: 200, headers: { etag: '"0915etag"' }) - stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). - with(body: "deliciosa"). - to_return(status: 200, headers: { etag: '"0815etag"' }) - stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). - with(body: "muy deliciosa"). + stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:get, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, body: "rootbody", headers: { etag: '"0817etag"', content_type: "text/plain; charset=utf-8" }) + stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { etag: '"0815etag"' }) + + # PUT requests authorized updates the metadata object in redis when it changes stub_request(:put, "#{container_url_for("phil")}/food/banano"). with(body: "si"). to_return(status: 200, headers: { etag: '"0815etag"' }) @@ -32,28 +32,28 @@ def storage_class to_return(status: 200, headers: { etag: '"0817etag"' }) stub_request(:head, "#{container_url_for("phil")}/food/banano"). to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:20 GMT" }) - stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). - to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) - stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). - to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/camaron"). to_return(status: 200, headers: { etag: '"0816etag"' }) stub_request(:head, "#{container_url_for("phil")}/food/camaron"). to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) stub_request(:delete, "#{container_url_for("phil")}/food/camaron"). to_return(status: 200, headers: { etag: '"0816etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/desayunos/bolon"). to_return(status: 200, headers: { etag: '"0817etag"' }) stub_request(:head, "#{container_url_for("phil")}/food/desayunos/bolon"). to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) stub_request(:delete, "#{container_url_for("phil")}/food/desayunos/bolon"). to_return(status: 200, headers: { etag: '"0817etag"' }) - stub_request(:get, "#{container_url_for("phil")}/food/aguacate"). - to_return(status: 200, body: "rootbody", headers: { etag: '"0817etag"', content_type: "text/plain; charset=utf-8" }) + + # objects in root dir stub_request(:put, "#{container_url_for("phil")}/bamboo.txt"). to_return(status: 200, headers: { etag: '"0818etag"' }) stub_request(:head, "#{container_url_for("phil")}/bamboo.txt"). to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + + # 404 stub_request(:head, "#{container_url_for("phil")}/food/steak"). to_return(status: 404) stub_request(:get, "#{container_url_for("phil")}/food/steak"). diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 2bc913d..13e7145 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -12,40 +12,40 @@ def storage_class before do stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) - stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). - with(body: "si"). - to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + # Write new content with an If-Match header (a new Etag is returned) stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). with(body: "aye"). to_return(status: 200, headers: { etag: "0915etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) - stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). - with(body: "deliciosa"). - to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) - stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). - with(body: "muy deliciosa"). - to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:get, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, body: "rootbody", headers: { etag: "0817etag", content_type: "text/plain; charset=utf-8" }) + stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { etag: "0815etag" }) + + # PUT requests authorized updates the metadata object in redis when it changes stub_request(:put, "#{container_url_for("phil")}/food/banano"). with(body: "si"). to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) stub_request(:put, "#{container_url_for("phil")}/food/banano"). with(body: "oh, no"). to_return(status: 200, headers: { etag: "0817etag", last_modified: "Fri, 04 Mar 2016 12:20:20 GMT" }) - stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). - to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) - stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). - to_return(status: 200, headers: { etag: "0815etag" }) + stub_request(:put, "#{container_url_for("phil")}/food/camaron"). to_return(status: 200, headers: { etag: "0816etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) stub_request(:delete, "#{container_url_for("phil")}/food/camaron"). to_return(status: 200, headers: { etag: "0816etag" }) + stub_request(:put, "#{container_url_for("phil")}/food/desayunos/bolon"). to_return(status: 200, headers: { etag: "0817etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) stub_request(:delete, "#{container_url_for("phil")}/food/desayunos/bolon"). to_return(status: 200, headers: { etag: "0817etag" }) - stub_request(:get, "#{container_url_for("phil")}/food/aguacate"). - to_return(status: 200, body: "rootbody", headers: { etag: "0817etag", content_type: "text/plain; charset=utf-8" }) + + # objects in root dir stub_request(:put, "#{container_url_for("phil")}/bamboo.txt"). to_return(status: 200, headers: { etag: "0818etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + + # 404 stub_request(:head, "#{container_url_for("phil")}/food/steak"). to_return(status: 404) stub_request(:get, "#{container_url_for("phil")}/food/steak"). From be33b0e195850d7ceed1481e524b0a37f7fe4cd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Greg=20Kar=C3=A9kinian?= Date: Wed, 9 May 2018 19:08:32 +0200 Subject: [PATCH 21/21] Replace placeholder comment that I forgot to replace --- spec/s3/app_spec.rb | 2 +- spec/swift/app_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb index 8829ab9..6e60c41 100644 --- a/spec/s3/app_spec.rb +++ b/spec/s3/app_spec.rb @@ -23,7 +23,7 @@ def storage_class stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { etag: '"0815etag"' }) - # PUT requests authorized updates the metadata object in redis when it changes + # Write new content to check the metadata in Redis stub_request(:put, "#{container_url_for("phil")}/food/banano"). with(body: "si"). to_return(status: 200, headers: { etag: '"0815etag"' }) diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index 13e7145..eff8bd9 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -23,7 +23,7 @@ def storage_class stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). to_return(status: 200, headers: { etag: "0815etag" }) - # PUT requests authorized updates the metadata object in redis when it changes + # Write new content to check the metadata in Redis stub_request(:put, "#{container_url_for("phil")}/food/banano"). with(body: "si"). to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" })