diff --git a/.travis.yml b/.travis.yml index 0f38df9..02acefa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,10 @@ rvm: - 2.4.1 services: - redis-server -before_install: - - gem install bundler before_script: - - cp config.yml.example config.yml + - cp config.yml.example.$BACKEND config.yml - mkdir -p tmp && echo "swifttoken" > tmp/swift_token.txt -script: ruby spec/swift/* +script: ruby spec/$BACKEND/* branches: only: - master @@ -21,3 +19,8 @@ notifications: - http://hook-juggler.herokuapp.com/hooks/travis on_success: always on_failure: always +env: + - BACKEND=s3 + - BACKEND=swift +# Run on Docker infrastructure +sudo: false diff --git a/Gemfile b/Gemfile index 6f8b5eb..293f7e3 100644 --- a/Gemfile +++ b/Gemfile @@ -15,6 +15,7 @@ group :test do gem 'purdytest', :require => false gem 'm' gem 'minitest-stub_any_instance' + gem 'webmock' end group :staging, :production do diff --git a/Gemfile.lock b/Gemfile.lock index a3a4cca..e31610b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -6,12 +6,17 @@ GEM i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) + addressable (2.5.2) + public_suffix (>= 2.0.2, < 4.0) backports (3.11.2) concurrent-ruby (1.0.5) + crack (0.4.3) + safe_yaml (~> 1.0.0) domain_name (0.5.20170404) unf (>= 0.0.5, < 1.0.0) faraday (0.14.0) multipart-post (>= 1.2, < 3) + hashdiff (0.3.7) http-accept (1.7.0) http-cookie (1.0.3) domain_name (~> 0.5) @@ -31,6 +36,7 @@ GEM multipart-post (2.0.0) mustermann (1.0.2) netrc (0.11.0) + public_suffix (3.0.2) purdytest (2.0.0) minitest (~> 5.5) rack (2.0.4) @@ -50,6 +56,7 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) + safe_yaml (1.0.4) sentry-raven (2.7.2) faraday (>= 0.7.6, < 1.0) sinatra (2.0.1) @@ -74,6 +81,10 @@ GEM unicorn (5.4.0) kgio (~> 2.6) raindrops (~> 0.7) + webmock (3.3.0) + addressable (>= 2.3.6) + crack (>= 0.3.2) + hashdiff PLATFORMS ruby @@ -92,6 +103,7 @@ DEPENDENCIES sentry-raven sinatra sinatra-contrib + webmock BUNDLED WITH 1.16.0 diff --git a/config.yml.example.s3 b/config.yml.example.s3 new file mode 100644 index 0000000..a0e654e --- /dev/null +++ b/config.yml.example.s3 @@ -0,0 +1,26 @@ +development: &defaults + maintenance: false + s3: + endpoint: "https://some-endpoint" + region: "region" + access_key_id: "" + secret_key_id: "" + bucket: "test-bucket" + redis: + host: localhost + port: 6379 + +test: + <<: *defaults + s3: + endpoint: "https://some-endpoint" + region: "region" + access_key_id: "" + secret_key_id: "" + bucket: "test-bucket" + +staging: + <<: *defaults + +production: + <<: *defaults diff --git a/config.yml.example b/config.yml.example.swift similarity index 52% rename from config.yml.example rename to config.yml.example.swift index b44c579..40fa39f 100644 --- a/config.yml.example +++ b/config.yml.example.swift @@ -1,20 +1,15 @@ development: &defaults maintenance: false - # # uncomment this section - # swift: &swift_defaults - # host: "https://swift.example.com" - # # Redis is needed for the swift backend - # redis: - # host: localhost - # port: 6379 + swift: &swift_defaults + host: "https://swift.example.com" + redis: + host: localhost + port: 6379 test: <<: *defaults swift: host: "https://swift.example.com" - redis: - host: localhost - port: 6379 staging: <<: *defaults diff --git a/lib/remote_storage/rest_provider.rb b/lib/remote_storage/rest_provider.rb new file mode 100644 index 0000000..3c5d38d --- /dev/null +++ b/lib/remote_storage/rest_provider.rb @@ -0,0 +1,501 @@ +require "rest_client" +require "json" +require "cgi" +require "active_support/core_ext/time/conversions" +require "active_support/core_ext/numeric/time" +require "active_support/core_ext/hash" +require "redis" +require "digest/md5" + +module RemoteStorage + module RestProvider + + attr_accessor :settings, :server + + def initialize(settings, server) + @settings = settings + @server = server + end + + def authorize_request(user, directory, token, listing=false) + request_method = server.env["REQUEST_METHOD"] + + if directory.split("/").first == "public" + return true if ["GET", "HEAD"].include?(request_method) && !listing + end + + server.halt 401, "Unauthorized" if token.nil? || token.empty? + + authorizations = redis.smembers("authorizations:#{user}:#{token}") + permission = directory_permission(authorizations, directory) + + server.halt 401, "Unauthorized" unless permission + if ["PUT", "DELETE"].include? request_method + server.halt 401, "Unauthorized" unless permission == "rw" + end + end + + def get_head(user, directory, key) + none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") + .map(&:strip) + .map { |s| s.gsub(/^"?W\//, "") } + metadata = redis.hgetall redis_metadata_object_key(user, directory, key) + + server.halt 404 if metadata.empty? + + # Set the response headers for a 304 or 200 response + server.headers["ETag"] = %Q("#{metadata["e"]}") + server.headers["Last-Modified"] = Time.at(metadata["m"].to_i / 1000).httpdate + + if none_match.include? %Q("#{metadata["e"]}") + server.halt 304 + end + end + + def get_data(user, directory, key) + none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") + .map(&:strip) + .map { |s| s.gsub(/^"?W\//, "") } + metadata = redis.hgetall redis_metadata_object_key(user, directory, key) + if none_match.include? %Q("#{metadata["e"]}") + server.headers["ETag"] = %Q("#{metadata["e"]}") + server.headers["Last-Modified"] = Time.at(metadata["m"].to_i / 1000).httpdate + server.halt 304 + end + + url = url_for_key(user, directory, key) + + res = do_get_request(url) + + set_response_headers(res.headers) + + return res.body + rescue RestClient::ResourceNotFound + server.halt 404, "Not Found" + end + + def get_head_directory_listing(user, directory) + get_directory_listing(user, directory) + + "" # just return empty body, headers are set by get_directory_listing + end + + def get_directory_listing(user, directory) + etag = redis.hget "rs:m:#{user}:#{directory}/", "e" + + server.headers["Content-Type"] = "application/ld+json" + + none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") + .map(&:strip) + .map { |s| s.gsub(/^"?W\//, "") } + + if etag + server.halt 304 if none_match.include? %Q("#{etag}") + + items = get_directory_listing_from_redis_via_lua(user, directory) + else + etag = etag_for(user, directory) + items = {} + + server.halt 304 if none_match.include? %Q("#{etag}") + end + + server.headers["ETag"] = %Q("#{etag}") + + listing = { + "@context" => "http://remotestorage.io/spec/folder-description", + "items" => items + } + + listing.to_json + end + + def put_data(user, directory, key, data, content_type) + server.halt 400 if server.env["HTTP_CONTENT_RANGE"] + server.halt 409, "Conflict" if has_name_collision?(user, directory, key) + + existing_metadata = redis.hgetall redis_metadata_object_key(user, directory, key) + url = url_for_key(user, directory, key) + + if required_match = server.env["HTTP_IF_MATCH"] + required_match = required_match.gsub(/^"?W\//, "") + unless required_match == %Q("#{existing_metadata["e"]}") + + # get actual metadata and compare in case redis metadata became out of sync + begin + head_res = do_head_request(url) + # The file doesn't exist, return 412 + rescue RestClient::ResourceNotFound + server.halt 412, "Precondition Failed" + end + + if required_match == format_etag(head_res.headers[:etag]) + # log previous size difference that was missed ealier because of redis failure + log_size_difference(user, existing_metadata["s"], head_res.headers[:content_length]) + else + server.halt 412, "Precondition Failed" + end + end + end + if server.env["HTTP_IF_NONE_MATCH"] == "*" + server.halt 412, "Precondition Failed" unless existing_metadata.empty? + end + + etag, timestamp = do_put_request(url, data, content_type) + + metadata = { + e: etag, + s: data.size, + t: content_type, + m: timestamp + } + + if update_metadata_object(user, directory, key, metadata) + if metadata_changed?(existing_metadata, metadata) + update_dir_objects(user, directory, timestamp, checksum_for(data)) + log_size_difference(user, existing_metadata["s"], metadata[:s]) + end + + server.headers["ETag"] = %Q("#{etag}") + server.halt existing_metadata.empty? ? 201 : 200 + else + server.halt 500 + end + end + + def delete_data(user, directory, key) + url = url_for_key(user, directory, key) + + existing_metadata = redis.hgetall "rs:m:#{user}:#{directory}/#{key}" + + if required_match = server.env["HTTP_IF_MATCH"] + unless required_match.gsub(/^"?W\//, "") == %Q("#{existing_metadata["e"]}") + server.halt 412, "Precondition Failed" + end + end + + found = try_to_delete(url) + + log_size_difference(user, existing_metadata["s"], 0) + delete_metadata_objects(user, directory, key) + delete_dir_objects(user, directory) + + if found + server.headers["Etag"] = %Q("#{existing_metadata["e"]}") + server.halt 200 + else + server.halt 404, "Not Found" + end + end + + private + + # Implement this method in your class that includes this module. For example + # %Q("#{etag}") if the ETag does not already have quotes around it + def format_etag(etag) + NotImplementedError + end + + def base_url + NotImplementedError + end + + def container_url_for(user) + NotImplementedError + end + + def default_headers + raise NotImplementedError + end + + def set_response_headers(headers) + server.headers["ETag"] = format_etag(headers[:etag]) + server.headers["Content-Type"] = headers[:content_type] + server.headers["Content-Length"] = headers[:content_length] + server.headers["Last-Modified"] = headers[:last_modified] + end + + def extract_category(directory) + if directory.match(/^public\//) + "public/#{directory.split('/')[1]}" + else + directory.split('/').first + end + end + + def directory_permission(authorizations, directory) + authorizations = authorizations.map do |auth| + auth.index(":") ? auth.split(":") : [auth, "rw"] + end + authorizations = Hash[*authorizations.flatten] + + permission = authorizations[""] + + authorizations.each do |key, value| + if directory.match(/^(public\/)?#{key}(\/|$)/) + if permission.nil? || permission == "r" + permission = value + end + return permission if permission == "rw" + end + end + + permission + end + + def has_name_collision?(user, directory, key) + lua_script = <<-EOF + local user = ARGV[1] + local directory = ARGV[2] + local key = ARGV[3] + + -- build table with parent directories from remaining arguments + local parent_dir_count = #ARGV - 3 + local parent_directories = {} + for i = 4, 4 + parent_dir_count do + table.insert(parent_directories, ARGV[i]) + end + + -- check for existing directory with the same name as the document + local redis_key = "rs:m:"..user..":" + if directory == "" then + redis_key = redis_key..key.."/" + else + redis_key = redis_key..directory.."/"..key.."/" + end + if redis.call("hget", redis_key, "e") then + return true + end + + for index, dir in pairs(parent_directories) do + if redis.call("hget", "rs:m:"..user..":"..dir.."/", "e") then + -- the directory already exists, no need to do further checks + return false + else + -- check for existing document with same name as directory + if redis.call("hget", "rs:m:"..user..":"..dir, "e") then + return true + end + end + end + + return false + EOF + + parent_directories = parent_directories_for(directory) + + redis.eval(lua_script, nil, [user, directory, key, *parent_directories]) + end + + def metadata_changed?(old_metadata, new_metadata) + # check metadata relevant to the directory listing + return old_metadata["e"] != new_metadata[:e] || + old_metadata["s"] != new_metadata[:s].to_s || + old_metadata["m"] != new_metadata[:m] || + old_metadata["t"] != new_metadata[:t] + end + + def timestamp_for(date) + return DateTime.parse(date).strftime("%Q").to_i + end + + def log_size_difference(user, old_size, new_size) + delta = new_size.to_i - old_size.to_i + redis.incrby "rs:s:#{user}", delta + end + + def checksum_for(data) + Digest::MD5.hexdigest(data) + end + + def parent_directories_for(directory) + directories = directory.split("/") + parent_directories = [] + + while directories.any? + parent_directories << directories.join("/") + directories.pop + end + + parent_directories << "" # add empty string for the root directory + + parent_directories + end + + def top_directory(directory) + if directory.match(/\//) + directory.split("/").last + elsif directory != "" + return directory + end + end + + def parent_directory_for(directory) + if directory.match(/\//) + return directory[0..directory.rindex("/")] + elsif directory != "" + return "/" + end + end + + def update_metadata_object(user, directory, key, metadata) + redis_key = redis_metadata_object_key(user, directory, key) + redis.hmset(redis_key, *metadata) + redis.sadd "rs:m:#{user}:#{directory}/:items", key + + true + end + + def update_dir_objects(user, directory, timestamp, checksum) + parent_directories_for(directory).each do |dir| + etag = etag_for(dir, timestamp, checksum) + + key = "rs:m:#{user}:#{dir}/" + metadata = {e: etag, m: timestamp} + redis.hmset(key, *metadata) + redis.sadd "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" + end + end + + def delete_metadata_objects(user, directory, key) + redis.del redis_metadata_object_key(user, directory, key) + redis.srem "rs:m:#{user}:#{directory}/:items", key + end + + def delete_dir_objects(user, directory) + timestamp = (Time.now.to_f * 1000).to_i + + parent_directories_for(directory).each do |dir| + if dir_empty?(user, dir) + redis.del "rs:m:#{user}:#{dir}/" + redis.srem "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" + else + etag = etag_for(dir, timestamp) + + metadata = {e: etag, m: timestamp} + redis.hmset("rs:m:#{user}:#{dir}/", *metadata) + end + end + end + + def dir_empty?(user, dir) + redis.smembers("rs:m:#{user}:#{dir}/:items").empty? + end + + def redis_metadata_object_key(user, directory, key) + "rs:m:#{user}:#{[directory, key].delete_if(&:empty?).join("/")}" + end + + def url_for_key(user, directory, key) + File.join [container_url_for(user), escape(directory), escape(key)].compact + end + + def do_put_request(url, data, content_type) + deal_with_unauthorized_requests do + res = RestClient.put(url, data, default_headers.merge({content_type: content_type})) + + return [ + res.headers[:etag], + timestamp_for(res.headers[:last_modified]) + ] + end + end + + def do_get_request(url, &block) + deal_with_unauthorized_requests do + RestClient.get(url, default_headers, &block) + end + end + + def do_head_request(url, &block) + deal_with_unauthorized_requests do + RestClient.head(url, default_headers, &block) + end + end + + def do_delete_request(url) + deal_with_unauthorized_requests do + RestClient.delete(url, default_headers) + end + end + + def escape(url) + # We want spaces to turn into %20 and slashes to stay slashes + CGI::escape(url).gsub('+', '%20').gsub('%2F', '/') + end + + def redis + @redis ||= Redis.new(settings.redis.symbolize_keys) + end + + def etag_for(*args) + Digest::MD5.hexdigest args.join(":") + end + + def deal_with_unauthorized_requests(&block) + begin + block.call + rescue RestClient::Unauthorized => ex + Raven.capture_exception(ex) + server.halt 500 + end + end + + def try_to_delete(url) + found = true + + begin + do_delete_request(url) + rescue RestClient::ResourceNotFound + found = false + end + + found + end + + def get_directory_listing_from_redis_via_lua(user, directory) + lua_script = <<-EOF + local user = ARGV[1] + local directory = ARGV[2] + local items = redis.call("smembers", "rs:m:"..user..":"..directory.."/:items") + local listing = {} + + for index, name in pairs(items) do + local redis_key = "rs:m:"..user..":" + if directory == "" then + redis_key = redis_key..name + else + redis_key = redis_key..directory.."/"..name + end + + local metadata_values = redis.call("hgetall", redis_key) + local metadata = {} + + -- redis returns hashes as a single list of alternating keys and values + -- this collates it into a table + for idx = 1, #metadata_values, 2 do + metadata[metadata_values[idx]] = metadata_values[idx + 1] + end + + listing[name] = {["ETag"] = metadata["e"]} + if string.sub(name, -1) ~= "/" then + listing[name]["Content-Type"] = metadata["t"] + listing[name]["Content-Length"] = tonumber(metadata["s"]) + listing[name]["Last-Modified"] = tonumber(metadata["m"]) + end + end + + return cjson.encode(listing) + EOF + + items = JSON.parse(redis.eval(lua_script, nil, [user, directory])) + + items.reject{|k, _| k.end_with? "/"}.each do |_, v| + v["Last-Modified"] = Time.at(v["Last-Modified"]/1000).httpdate + end + + items + end + + end +end diff --git a/lib/remote_storage/s3.rb b/lib/remote_storage/s3.rb new file mode 100644 index 0000000..875f596 --- /dev/null +++ b/lib/remote_storage/s3.rb @@ -0,0 +1,113 @@ +require "remote_storage/rest_provider" +require "digest" +require "base64" +require "openssl" +require "webrick/httputils" + +module RemoteStorage + class S3 + include RestProvider + + private + + # S3 already wraps the ETag around quotes + def format_etag(etag) + etag + end + + def do_put_request(url, data, content_type) + deal_with_unauthorized_requests do + md5 = Digest::MD5.base64digest(data) + authorization_headers = authorization_headers_for( + "PUT", url, md5, content_type + ).merge({ "Content-Type" => content_type, "Content-Md5" => md5 }) + res = RestClient.put(url, data, authorization_headers) + # S3 does not return a Last-Modified response header on PUTs + head_res = do_head_request(url) + + return [ + res.headers[:etag].delete('"'), + timestamp_for(head_res.headers[:last_modified]) + ] + end + end + + def do_get_request(url, &block) + deal_with_unauthorized_requests do + authorization_headers = authorization_headers_for("GET", url) + RestClient.get(url, authorization_headers, &block) + end + end + + def do_head_request(url, &block) + deal_with_unauthorized_requests do + authorization_headers = authorization_headers_for("HEAD", url) + RestClient.head(url, authorization_headers, &block) + end + end + + def do_delete_request(url) + deal_with_unauthorized_requests do + authorization_headers = authorization_headers_for("DELETE", url) + RestClient.delete(url, authorization_headers) + end + end + + def try_to_delete(url) + found = true + + begin + do_head_request(url) + rescue RestClient::ResourceNotFound + found = false + end + + do_delete_request(url) if found + + return found + end + + # This is using the S3 authorizations, not the newer AW V4 Signatures + # (https://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html) + def authorization_headers_for(http_verb, url, md5 = nil, content_type = nil) + url = File.join("/", url.gsub(base_url, "")) + date = Time.now.httpdate + signed_data = generate_s3_signature(http_verb, md5, content_type, date, url) + { + "Authorization" => "AWS #{credentials[:access_key_id]}:#{signed_data}", + "Date" => date + } + end + + def credentials + @credentials ||= { access_key_id: settings.s3["access_key_id"], secret_key_id: settings.s3["secret_key_id"] } + end + + def digest(secret, string_to_sign) + Base64.encode64(hmac(secret, string_to_sign)).strip + end + + def hmac(key, value) + OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha1'), key, value) + end + + def uri_escape(s) + WEBrick::HTTPUtils.escape(s).gsub('%5B', '[').gsub('%5D', ']') + end + + def generate_s3_signature(http_verb, md5, content_type, date, url) + string_to_sign = [http_verb, md5, content_type, date, url].join "\n" + signature = digest(credentials[:secret_key_id], string_to_sign) + uri_escape(signature) + end + + def base_url + @base_url ||= settings.s3["endpoint"] + end + + def container_url_for(user) + "#{base_url}#{settings.s3["bucket"]}/#{user}" + end + end + +end diff --git a/lib/remote_storage/swift.rb b/lib/remote_storage/swift.rb index d104f11..cdef119 100644 --- a/lib/remote_storage/swift.rb +++ b/lib/remote_storage/swift.rb @@ -1,466 +1,32 @@ require "rest_client" -require "json" -require "cgi" require "active_support/core_ext/time/conversions" require "active_support/core_ext/numeric/time" require "active_support/core_ext/hash" -require "redis" -require "digest/md5" +require "remote_storage/rest_provider" module RemoteStorage class Swift - - attr_accessor :settings, :server - - def initialize(settings, server) - @settings = settings - @server = server - end - - def authorize_request(user, directory, token, listing=false) - request_method = server.env["REQUEST_METHOD"] - - if directory.split("/").first == "public" - return true if ["GET", "HEAD"].include?(request_method) && !listing - end - - server.halt 401, "Unauthorized" if token.nil? || token.empty? - - authorizations = redis.smembers("authorizations:#{user}:#{token}") - permission = directory_permission(authorizations, directory) - - server.halt 401, "Unauthorized" unless permission - if ["PUT", "DELETE"].include? request_method - server.halt 401, "Unauthorized" unless permission == "rw" - end - end - - def get_head(user, directory, key) - url = url_for_key(user, directory, key) - - res = do_head_request(url) - - set_response_headers(res) - rescue RestClient::ResourceNotFound - server.halt 404 - end - - def get_data(user, directory, key) - url = url_for_key(user, directory, key) - - res = do_get_request(url) - - set_response_headers(res) - - none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") - .map(&:strip) - .map { |s| s.gsub(/^"?W\//, "") } - server.halt 304 if none_match.include? %Q("#{res.headers[:etag]}") - - return res.body - rescue RestClient::ResourceNotFound - server.halt 404, "Not Found" - end - - def get_head_directory_listing(user, directory) - get_directory_listing(user, directory) - - "" # just return empty body, headers are set by get_directory_listing - end - - def get_directory_listing(user, directory) - etag = redis.hget "rs:m:#{user}:#{directory}/", "e" - - server.headers["Content-Type"] = "application/ld+json" - - none_match = (server.env["HTTP_IF_NONE_MATCH"] || "").split(",") - .map(&:strip) - .map { |s| s.gsub(/^"?W\//, "") } - - if etag - server.halt 304 if none_match.include? %Q("#{etag}") - - items = get_directory_listing_from_redis_via_lua(user, directory) - else - etag = etag_for(user, directory) - items = {} - - server.halt 304 if none_match.include? %Q("#{etag}") - end - - server.headers["ETag"] = %Q("#{etag}") - - listing = { - "@context" => "http://remotestorage.io/spec/folder-description", - "items" => items - } - - listing.to_json - end - - def get_directory_listing_from_redis_via_lua(user, directory) - lua_script = <<-EOF - local user = ARGV[1] - local directory = ARGV[2] - local items = redis.call("smembers", "rs:m:"..user..":"..directory.."/:items") - local listing = {} - - for index, name in pairs(items) do - local redis_key = "rs:m:"..user..":" - if directory == "" then - redis_key = redis_key..name - else - redis_key = redis_key..directory.."/"..name - end - - local metadata_values = redis.call("hgetall", redis_key) - local metadata = {} - - -- redis returns hashes as a single list of alternating keys and values - -- this collates it into a table - for idx = 1, #metadata_values, 2 do - metadata[metadata_values[idx]] = metadata_values[idx + 1] - end - - listing[name] = {["ETag"] = metadata["e"]} - if string.sub(name, -1) ~= "/" then - listing[name]["Content-Type"] = metadata["t"] - listing[name]["Content-Length"] = tonumber(metadata["s"]) - listing[name]["Last-Modified"] = tonumber(metadata["m"]) - end - end - - return cjson.encode(listing) - EOF - - items = JSON.parse(redis.eval(lua_script, nil, [user, directory])) - - items.reject{|k, _| k.end_with? "/"}.each do |_, v| - v["Last-Modified"] = Time.at(v["Last-Modified"]/1000).httpdate - end - - items - end - - def put_data(user, directory, key, data, content_type) - server.halt 400 if server.env["HTTP_CONTENT_RANGE"] - server.halt 409, "Conflict" if has_name_collision?(user, directory, key) - - existing_metadata = redis.hgetall redis_metadata_object_key(user, directory, key) - url = url_for_key(user, directory, key) - - if required_match = server.env["HTTP_IF_MATCH"] - required_match = required_match.gsub(/^"?W\//, "") - unless required_match == %Q("#{existing_metadata["e"]}") - - # get actual metadata and compare in case redis metadata became out of sync - begin - head_res = do_head_request(url) - # The file doesn't exist in Orbit, return 412 - rescue RestClient::ResourceNotFound - server.halt 412, "Precondition Failed" - end - - if required_match == %Q("#{head_res.headers[:etag]}") - # log previous size difference that was missed ealier because of redis failure - log_size_difference(user, existing_metadata["s"], head_res.headers[:content_length]) - else - server.halt 412, "Precondition Failed" - end - end - end - if server.env["HTTP_IF_NONE_MATCH"] == "*" - server.halt 412, "Precondition Failed" unless existing_metadata.empty? - end - - res = do_put_request(url, data, content_type) - - timestamp = timestamp_for(res.headers[:last_modified]) - - metadata = { - e: res.headers[:etag], - s: data.size, - t: content_type, - m: timestamp - } - - if update_metadata_object(user, directory, key, metadata) - if metadata_changed?(existing_metadata, metadata) - update_dir_objects(user, directory, timestamp, checksum_for(data)) - log_size_difference(user, existing_metadata["s"], metadata[:s]) - end - - server.headers["ETag"] = %Q("#{res.headers[:etag]}") - server.halt existing_metadata.empty? ? 201 : 200 - else - server.halt 500 - end - end - - def log_size_difference(user, old_size, new_size) - delta = new_size.to_i - old_size.to_i - redis.incrby "rs:s:#{user}", delta - end - - def checksum_for(data) - Digest::MD5.hexdigest(data) - end - - def delete_data(user, directory, key) - url = url_for_key(user, directory, key) - not_found = false - - existing_metadata = redis.hgetall "rs:m:#{user}:#{directory}/#{key}" - - if required_match = server.env["HTTP_IF_MATCH"] - unless required_match.gsub(/^"?W\//, "") == %Q("#{existing_metadata["e"]}") - server.halt 412, "Precondition Failed" - end - end - - begin - do_delete_request(url) - rescue RestClient::ResourceNotFound - not_found = true - end - - log_size_difference(user, existing_metadata["s"], 0) - delete_metadata_objects(user, directory, key) - delete_dir_objects(user, directory) - - if not_found - server.halt 404, "Not Found" - else - server.headers["Etag"] = %Q("#{existing_metadata["e"]}") - server.halt 200 - end - end + include RestProvider private - def set_response_headers(response) - server.headers["ETag"] = %Q("#{response.headers[:etag]}") - server.headers["Content-Type"] = response.headers[:content_type] - server.headers["Content-Length"] = response.headers[:content_length] - server.headers["Last-Modified"] = response.headers[:last_modified] + # Add quotes around the ETag + def format_etag(etag) + %Q("#{etag}") end - def extract_category(directory) - if directory.match(/^public\//) - "public/#{directory.split('/')[1]}" - else - directory.split('/').first - end - end - - def directory_permission(authorizations, directory) - authorizations = authorizations.map do |auth| - auth.index(":") ? auth.split(":") : [auth, "rw"] - end - authorizations = Hash[*authorizations.flatten] - - permission = authorizations[""] - - authorizations.each do |key, value| - if directory.match(/^(public\/)?#{key}(\/|$)/) - if permission.nil? || permission == "r" - permission = value - end - return permission if permission == "rw" - end - end - - permission - end - - def has_name_collision?(user, directory, key) - lua_script = <<-EOF - local user = ARGV[1] - local directory = ARGV[2] - local key = ARGV[3] - - -- build table with parent directories from remaining arguments - local parent_dir_count = #ARGV - 3 - local parent_directories = {} - for i = 4, 4 + parent_dir_count do - table.insert(parent_directories, ARGV[i]) - end - - -- check for existing directory with the same name as the document - local redis_key = "rs:m:"..user..":" - if directory == "" then - redis_key = redis_key..key.."/" - else - redis_key = redis_key..directory.."/"..key.."/" - end - if redis.call("hget", redis_key, "e") then - return true - end - - for index, dir in pairs(parent_directories) do - if redis.call("hget", "rs:m:"..user..":"..dir.."/", "e") then - -- the directory already exists, no need to do further checks - return false - else - -- check for existing document with same name as directory - if redis.call("hget", "rs:m:"..user..":"..dir, "e") then - return true - end - end - end - - return false - EOF - - parent_directories = parent_directories_for(directory) - - redis.eval(lua_script, nil, [user, directory, key, *parent_directories]) - end - - def metadata_changed?(old_metadata, new_metadata) - # check metadata relevant to the directory listing - # ie. the timestamp (m) is not relevant, because it's not used in - # the listing - return old_metadata["e"] != new_metadata[:e] || - old_metadata["s"] != new_metadata[:s].to_s || - old_metadata["t"] != new_metadata[:t] - end - - def timestamp_for(date) - return DateTime.parse(date).strftime("%Q").to_i - end - - def parent_directories_for(directory) - directories = directory.split("/") - parent_directories = [] - - while directories.any? - parent_directories << directories.join("/") - directories.pop - end - - parent_directories << "" # add empty string for the root directory - - parent_directories - end - - def top_directory(directory) - if directory.match(/\//) - directory.split("/").last - elsif directory != "" - return directory - end - end - - def parent_directory_for(directory) - if directory.match(/\//) - return directory[0..directory.rindex("/")] - elsif directory != "" - return "/" - end - end - - def update_metadata_object(user, directory, key, metadata) - redis_key = redis_metadata_object_key(user, directory, key) - redis.hmset(redis_key, *metadata) - redis.sadd "rs:m:#{user}:#{directory}/:items", key - - true - end - - def update_dir_objects(user, directory, timestamp, checksum) - parent_directories_for(directory).each do |dir| - etag = etag_for(dir, timestamp, checksum) - - key = "rs:m:#{user}:#{dir}/" - metadata = {e: etag, m: timestamp} - redis.hmset(key, *metadata) - redis.sadd "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" - end - end - - def delete_metadata_objects(user, directory, key) - redis.del redis_metadata_object_key(user, directory, key) - redis.srem "rs:m:#{user}:#{directory}/:items", key - end - - def delete_dir_objects(user, directory) - timestamp = (Time.now.to_f * 1000).to_i - - parent_directories_for(directory).each do |dir| - if dir_empty?(user, dir) - redis.del "rs:m:#{user}:#{dir}/" - redis.srem "rs:m:#{user}:#{parent_directory_for(dir)}:items", "#{top_directory(dir)}/" - else - etag = etag_for(dir, timestamp) - - metadata = {e: etag, m: timestamp} - redis.hmset("rs:m:#{user}:#{dir}/", *metadata) - end - end - end - - def dir_empty?(user, dir) - redis.smembers("rs:m:#{user}:#{dir}/:items").empty? - end - - def redis_metadata_object_key(user, directory, key) - "rs:m:#{user}:#{[directory, key].delete_if(&:empty?).join("/")}" + def base_url + @base_url ||= settings.swift["host"] end def container_url_for(user) "#{base_url}/rs:documents:#{settings.environment.to_s}/#{user}" end - def url_for_key(user, directory, key) - File.join [container_url_for(user), escape(directory), escape(key)].compact - end - - def base_url - @base_url ||= settings.swift["host"] - end - def default_headers {"x-auth-token" => swift_token} end - def do_put_request(url, data, content_type) - deal_with_unauthorized_requests do - RestClient.put(url, data, default_headers.merge({content_type: content_type})) - end - end - - def do_get_request(url, &block) - deal_with_unauthorized_requests do - RestClient.get(url, default_headers, &block) - end - end - - def do_head_request(url, &block) - deal_with_unauthorized_requests do - RestClient.head(url, default_headers, &block) - end - end - - def do_delete_request(url) - deal_with_unauthorized_requests do - RestClient.delete(url, default_headers) - end - end - - def escape(url) - # We want spaces to turn into %20 and slashes to stay slashes - CGI::escape(url).gsub('+', '%20').gsub('%2F', '/') - end - - def redis - @redis ||= Redis.new(settings.redis.symbolize_keys) - end - - def etag_for(*args) - Digest::MD5.hexdigest args.join(":") - end - def reload_swift_token server.logger.debug "Reloading swift token. Old token: #{settings.swift_token}" # Remove the line break from the token file. The line break that the diff --git a/liquor-cabinet.rb b/liquor-cabinet.rb index d9a41fb..6f6324f 100644 --- a/liquor-cabinet.rb +++ b/liquor-cabinet.rb @@ -5,6 +5,7 @@ require 'sinatra/config_file' require "sinatra/reloader" require "remote_storage/swift" +require "remote_storage/s3" class LiquorCabinet < Sinatra::Base @@ -130,6 +131,8 @@ def storage @storage ||= begin if settings.respond_to? :swift RemoteStorage::Swift.new(settings, self) + elsif settings.respond_to? :s3 + RemoteStorage::S3.new(settings, self) else puts <<-EOF You need to set one storage backend in your config.yml file. diff --git a/spec/s3/app_spec.rb b/spec/s3/app_spec.rb new file mode 100644 index 0000000..6e60c41 --- /dev/null +++ b/spec/s3/app_spec.rb @@ -0,0 +1,64 @@ +require_relative "../spec_helper" + +describe "S3 provider" do + def container_url_for(user) + "#{app.settings.s3["endpoint"]}#{app.settings.s3["bucket"]}/#{user}" + end + + def storage_class + RemoteStorage::S3 + end + + before do + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { etag: '"0815etag"' }) + # Write new content with an If-Match header (a new Etag is returned) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "aye"). + to_return(status: 200, headers: { etag: '"0915etag"' }) + stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:get, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, body: "rootbody", headers: { etag: '"0817etag"', content_type: "text/plain; charset=utf-8" }) + stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { etag: '"0815etag"' }) + + # Write new content to check the metadata in Redis + stub_request(:put, "#{container_url_for("phil")}/food/banano"). + with(body: "si"). + to_return(status: 200, headers: { etag: '"0815etag"' }) + stub_request(:put, "#{container_url_for("phil")}/food/banano"). + with(body: "oh, no"). + to_return(status: 200, headers: { etag: '"0817etag"' }) + stub_request(:head, "#{container_url_for("phil")}/food/banano"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:20 GMT" }) + + stub_request(:put, "#{container_url_for("phil")}/food/camaron"). + to_return(status: 200, headers: { etag: '"0816etag"' }) + stub_request(:head, "#{container_url_for("phil")}/food/camaron"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/camaron"). + to_return(status: 200, headers: { etag: '"0816etag"' }) + + stub_request(:put, "#{container_url_for("phil")}/food/desayunos/bolon"). + to_return(status: 200, headers: { etag: '"0817etag"' }) + stub_request(:head, "#{container_url_for("phil")}/food/desayunos/bolon"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/desayunos/bolon"). + to_return(status: 200, headers: { etag: '"0817etag"' }) + + # objects in root dir + stub_request(:put, "#{container_url_for("phil")}/bamboo.txt"). + to_return(status: 200, headers: { etag: '"0818etag"' }) + stub_request(:head, "#{container_url_for("phil")}/bamboo.txt"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + + # 404 + stub_request(:head, "#{container_url_for("phil")}/food/steak"). + to_return(status: 404) + stub_request(:get, "#{container_url_for("phil")}/food/steak"). + to_return(status: 404) + end + + it_behaves_like 'a REST adapter' +end diff --git a/spec/shared_examples.rb b/spec/shared_examples.rb new file mode 100644 index 0000000..fd5f742 --- /dev/null +++ b/spec/shared_examples.rb @@ -0,0 +1,642 @@ +require_relative "./spec_helper" + +shared_examples_for 'a REST adapter' do + include Rack::Test::Methods + + def container_url_for(user) + raise NotImplementedError + end + + def storage_class + raise NotImplementedError + end + + it "returns 404 on non-existing routes" do + get "/virginmargarita" + last_response.status.must_equal 404 + end + + describe "PUT requests" do + + before do + purge_redis + end + + context "authorized" do + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + end + + it "creates the metadata object in redis" do + put "/phil/food/aguacate", "si" + + metadata = redis.hgetall "rs:m:phil:food/aguacate" + metadata["s"].must_equal "2" + metadata["t"].must_equal "text/plain; charset=utf-8" + metadata["e"].must_equal "0815etag" + metadata["m"].length.must_equal 13 + end + + it "updates the metadata object in redis when it changes" do + put "/phil/food/banano", "si" + put "/phil/food/banano", "oh, no" + + metadata = redis.hgetall "rs:m:phil:food/banano" + metadata["s"].must_equal "6" + metadata["t"].must_equal "text/plain; charset=utf-8" + metadata["e"].must_equal "0817etag" + metadata["m"].must_equal "1457094020000" + end + + it "creates the directory objects metadata in redis" do + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + + metadata = redis.hgetall "rs:m:phil:/" + metadata["e"].must_equal "fe2976909daaf074660981ab563fe65d" + metadata["m"].length.must_equal 13 + + metadata = redis.hgetall "rs:m:phil:food/" + metadata["e"].must_equal "926f98ff820f2f9764fd3c60a22865ad" + metadata["m"].length.must_equal 13 + + food_items = redis.smembers "rs:m:phil:food/:items" + food_items.each do |food_item| + ["camaron", "aguacate"].must_include food_item + end + + root_items = redis.smembers "rs:m:phil:/:items" + root_items.must_equal ["food/"] + end + + context "response code" do + it "is 201 for newly created objects" do + put "/phil/food/aguacate", "ci" + + last_response.status.must_equal 201 + end + + it "is 200 for updated objects" do + put "/phil/food/aguacate", "deliciosa" + put "/phil/food/aguacate", "muy deliciosa" + + last_response.status.must_equal 200 + end + end + + context "logging usage size" do + it "logs the complete size when creating new objects" do + put "/phil/food/aguacate", "1234567890" + + size_log = redis.get "rs:s:phil" + size_log.must_equal "10" + end + + it "logs the size difference when updating existing objects" do + put "/phil/food/camaron", "1234567890" + put "/phil/food/aguacate", "1234567890" + put "/phil/food/aguacate", "123" + + size_log = redis.get "rs:s:phil" + size_log.must_equal "13" + end + end + + describe "objects in root dir" do + before do + put "/phil/bamboo.txt", "shir kan" + end + + it "are listed in the directory listing with all metadata" do + get "phil/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["items"]["bamboo.txt"].wont_be_nil + content["items"]["bamboo.txt"]["ETag"].must_equal "0818etag" + content["items"]["bamboo.txt"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["bamboo.txt"]["Content-Length"].must_equal 8 + content["items"]["bamboo.txt"]["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + end + + describe "name collision checks" do + it "is successful when there is no name collision" do + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 201 + + metadata = redis.hgetall "rs:m:phil:food/aguacate" + metadata["s"].must_equal "2" + end + + it "conflicts when there is a directory with same name as document" do + put "/phil/food/aguacate", "si" + put "/phil/food", "wontwork" + + last_response.status.must_equal 409 + last_response.body.must_equal "Conflict" + + metadata = redis.hgetall "rs:m:phil:food" + metadata.must_be_empty + end + + it "conflicts when there is a document with same name as directory" do + put "/phil/food/aguacate", "si" + put "/phil/food/aguacate/empanado", "wontwork" + + last_response.status.must_equal 409 + + metadata = redis.hgetall "rs:m:phil:food/aguacate/empanado" + metadata.must_be_empty + end + + it "returns 400 when a Content-Range header is sent" do + header "Content-Range", "bytes 0-3/3" + + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 400 + end + end + + describe "If-Match header" do + before do + put "/phil/food/aguacate", "si" + end + + it "allows the request if the header matches the current ETag" do + header "If-Match", "\"0815etag\"" + + put "/phil/food/aguacate", "aye" + + last_response.status.must_equal 200 + last_response.headers["Etag"].must_equal "\"0915etag\"" + end + + it "allows the request if the header contains a weak ETAG matching the current ETag" do + header "If-Match", "W/\"0815etag\"" + + put "/phil/food/aguacate", "aye" + + last_response.status.must_equal 200 + last_response.headers["Etag"].must_equal "\"0915etag\"" + end + + it "allows the request if the header contains a weak ETAG with leading quote matching the current ETag" do + header "If-Match", "\"W/\"0815etag\"" + + put "/phil/food/aguacate", "aye" + + last_response.status.must_equal 200 + last_response.headers["Etag"].must_equal "\"0915etag\"" + end + + it "fails the request if the header does not match the current ETag" do + header "If-Match", "someotheretag" + + put "/phil/food/aguacate", "aye" + + last_response.status.must_equal 412 + last_response.body.must_equal "Precondition Failed" + end + + it "allows the request if redis metadata became out of sync" do + header "If-Match", "\"0815etag\"" + + put "/phil/food/aguacate", "aye" + + last_response.status.must_equal 200 + end + end + + describe "If-None-Match header set to '*'" do + it "succeeds when the document doesn't exist yet" do + header "If-None-Match", "*" + + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 201 + end + + it "fails the request if the document already exists" do + put "/phil/food/aguacate", "si" + + header "If-None-Match", "*" + put "/phil/food/aguacate", "si" + + last_response.status.must_equal 412 + last_response.body.must_equal "Precondition Failed" + end + end + end + + end + + describe "DELETE requests" do + + before do + purge_redis + end + + context "not authorized" do + describe "with no token" do + it "says it's not authorized" do + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + describe "with empty token" do + it "says it's not authorized" do + header "Authorization", "Bearer " + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + describe "with wrong token" do + it "says it's not authorized" do + header "Authorization", "Bearer wrongtoken" + delete "/phil/food/aguacate" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + end + + context "authorized" do + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desayunos/bolon", "wow" + end + + it "decreases the size log by size of deleted object" do + delete "/phil/food/aguacate" + + size_log = redis.get "rs:s:phil" + size_log.must_equal "8" + end + + it "deletes the metadata object in redis" do + delete "/phil/food/aguacate" + + metadata = redis.hgetall "rs:m:phil:food/aguacate" + metadata.must_be_empty + end + + it "deletes the directory objects metadata in redis" do + old_metadata = redis.hgetall "rs:m:phil:food/" + + storage_class.stub_any_instance :etag_for, "newetag" do + delete "/phil/food/aguacate" + end + + metadata = redis.hgetall "rs:m:phil:food/" + metadata["e"].must_equal "newetag" + metadata["m"].length.must_equal 13 + metadata["m"].wont_equal old_metadata["m"] + + food_items = redis.smembers "rs:m:phil:food/:items" + food_items.sort.must_equal ["camaron", "desayunos/"] + + root_items = redis.smembers "rs:m:phil:/:items" + root_items.must_equal ["food/"] + end + + it "deletes the parent directory objects metadata when deleting all items" do + delete "/phil/food/aguacate" + delete "/phil/food/camaron" + delete "/phil/food/desayunos/bolon" + + redis.smembers("rs:m:phil:food/desayunos:items").must_be_empty + redis.hgetall("rs:m:phil:food/desayunos/").must_be_empty + + redis.smembers("rs:m:phil:food/:items").must_be_empty + redis.hgetall("rs:m:phil:food/").must_be_empty + + redis.smembers("rs:m:phil:/:items").must_be_empty + end + + it "responds with the ETag of the deleted item in the header" do + delete "/phil/food/aguacate" + + last_response.headers["ETag"].must_equal "\"0815etag\"" + end + + context "when item doesn't exist" do + before do + purge_redis + + delete "/phil/food/steak" + end + + it "returns a 404" do + last_response.status.must_equal 404 + last_response.body.must_equal "Not Found" + end + + it "deletes any metadata that might still exist" do + delete "/phil/food/steak" + + metadata = redis.hgetall "rs:m:phil:food/steak" + metadata.must_be_empty + + redis.smembers("rs:m:phil:food/:items").must_be_empty + redis.hgetall("rs:m:phil:food/").must_be_empty + + redis.smembers("rs:m:phil:/:items").must_be_empty + end + end + + describe "If-Match header" do + it "succeeds when the header matches the current ETag" do + header "If-Match", "\"0815etag\"" + + delete "/phil/food/aguacate" + + last_response.status.must_equal 200 + end + + it "succeeds when the header contains a weak ETAG matching the current ETag" do + header "If-Match", "W/\"0815etag\"" + + delete "/phil/food/aguacate" + + last_response.status.must_equal 200 + end + + it "fails the request if it does not match the current ETag" do + header "If-Match", "someotheretag" + + delete "/phil/food/aguacate" + + last_response.status.must_equal 412 + last_response.body.must_equal "Precondition Failed" + end + end + end + end + + describe "GET requests" do + + before do + purge_redis + end + + context "not authorized" do + + describe "without token" do + it "says it's not authorized" do + get "/phil/food/" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + describe "with wrong token" do + it "says it's not authorized" do + header "Authorization", "Bearer wrongtoken" + get "/phil/food/" + + last_response.status.must_equal 401 + last_response.body.must_equal "Unauthorized" + end + end + + end + + context "authorized" do + + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desayunos/bolon", "wow" + end + + describe "documents" do + + it "returns the required response headers" do + get "/phil/food/aguacate" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"0817etag\"" + last_response.headers["Cache-Control"].must_equal "no-cache" + last_response.headers["Content-Type"].must_equal "text/plain; charset=utf-8" + end + + it "returns a 404 when data doesn't exist" do + get "/phil/food/steak" + + last_response.status.must_equal 404 + last_response.body.must_equal "Not Found" + end + + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"0815etag\"" + + get "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do + header "If-None-Match", "W/\"0815etag\"" + + get "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + + end + + describe "directory listings" do + + it "returns the correct ETag header" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + end + + it "returns a Cache-Control header with value 'no-cache'" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.headers["Cache-Control"].must_equal "no-cache" + end + + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + get "/phil/food/" + + last_response.status.must_equal 304 + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the ETag" do + header "If-None-Match", "W/\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + get "/phil/food/" + + last_response.status.must_equal 304 + end + + it "contains all items in the directory" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["@context"].must_equal "http://remotestorage.io/spec/folder-description" + content["items"]["aguacate"].wont_be_nil + content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["aguacate"]["Content-Length"].must_equal 2 + content["items"]["aguacate"]["ETag"].must_equal "0815etag" + content["items"]["camaron"].wont_be_nil + content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8" + content["items"]["camaron"]["Content-Length"].must_equal 5 + content["items"]["camaron"]["ETag"].must_equal "0816etag" + content["items"]["desayunos/"].wont_be_nil + content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d" + end + + it "contains all items in the root directory" do + get "phil/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["items"]["food/"].wont_be_nil + content["items"]["food/"]["ETag"].must_equal "f9f85fbf5aa1fa378fd79ac8aa0a457d" + end + + it "responds with an empty directory liting when directory doesn't exist" do + get "phil/some-non-existing-dir/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + + content = JSON.parse(last_response.body) + content["items"].must_equal({}) + end + + end + end + + end + + describe "HEAD requests" do + + before do + purge_redis + end + + context "not authorized" do + + describe "without token" do + it "says it's not authorized" do + head "/phil/food/camarones" + + last_response.status.must_equal 401 + last_response.body.must_be_empty + end + end + + describe "with wrong token" do + it "says it's not authorized" do + header "Authorization", "Bearer wrongtoken" + head "/phil/food/camarones" + + last_response.status.must_equal 401 + last_response.body.must_be_empty + end + end + + end + + context "authorized" do + + before do + redis.sadd "authorizations:phil:amarillo", [":rw"] + header "Authorization", "Bearer amarillo" + + put "/phil/food/aguacate", "si" + put "/phil/food/camaron", "yummi" + put "/phil/food/desayunos/bolon", "wow" + end + + describe "directory listings" do + it "returns the correct header information" do + get "/phil/food/" + + last_response.status.must_equal 200 + last_response.content_type.must_equal "application/ld+json" + last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" + end + end + + describe "documents" do + context "when the document doesn't exist" do + it "returns a 404" do + head "/phil/food/steak" + + last_response.status.must_equal 404 + last_response.body.must_be_empty + end + end + + context "when the document exists" do + it "returns the required response headers" do + head "/phil/food/aguacate" + + last_response.status.must_equal 200 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Cache-Control"].must_equal "no-cache" + end + + it "responds with 304 when IF_NONE_MATCH header contains the ETag" do + header "If-None-Match", "\"0815etag\"" + + head "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + + it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do + header "If-None-Match", "W/\"0815etag\"" + + head "/phil/food/aguacate" + + last_response.status.must_equal 304 + last_response.headers["ETag"].must_equal "\"0815etag\"" + last_response.headers["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" + end + end + end + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index b08c6b5..16a4dad 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -6,12 +6,15 @@ require_relative '../liquor-cabinet' require 'minitest/autorun' +require "minitest/stub_any_instance" require 'rack/test' require 'purdytest' require "redis" require "rest_client" -require "minitest/stub_any_instance" require "ostruct" +require 'webmock/minitest' + +WebMock.disable_net_connect! def app LiquorCabinet @@ -19,17 +22,6 @@ def app app.set :environment, :test -def wait_a_second - now = Time.now.to_i - while Time.now.to_i == now; end -end - -def write_last_response_to_file(filename = "last_response.html") - File.open(filename, "w") do |f| - f.write last_response.body - end -end - alias context describe if app.settings.respond_to? :redis @@ -43,3 +35,23 @@ def purge_redis end end end + +MiniTest::Spec.class_eval do + def self.shared_examples + @shared_examples ||= {} + end +end + +module MiniTest::Spec::SharedExamples + def shared_examples_for(desc, &block) + MiniTest::Spec.shared_examples[desc] = block + end + + def it_behaves_like(desc) + self.instance_eval(&MiniTest::Spec.shared_examples[desc]) + end +end + +Object.class_eval { include(MiniTest::Spec::SharedExamples) } + +require_relative 'shared_examples' diff --git a/spec/swift/app_spec.rb b/spec/swift/app_spec.rb index ae207c7..eff8bd9 100644 --- a/spec/swift/app_spec.rb +++ b/spec/swift/app_spec.rb @@ -1,827 +1,58 @@ require_relative "../spec_helper" -describe "App" do - include Rack::Test::Methods - - def app - LiquorCabinet - end - - it "returns 404 on non-existing routes" do - get "/virginmargarita" - last_response.status.must_equal 404 - end - - describe "PUT requests" do - - before do - purge_redis - end - - context "authorized" do - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - end - - it "creates the metadata object in redis" do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end - - metadata = redis.hgetall "rs:m:phil:food/aguacate" - metadata["s"].must_equal "2" - metadata["t"].must_equal "text/plain; charset=utf-8" - metadata["e"].must_equal "bla" - metadata["m"].length.must_equal 13 - end - - it "creates the directory objects metadata in redis" do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - get_stub = OpenStruct.new(body: "rootbody") - - RestClient.stub :put, put_stub do - RestClient.stub :get, get_stub do - RemoteStorage::Swift.stub_any_instance :etag_for, "newetag" do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - end - end - end - - metadata = redis.hgetall "rs:m:phil:/" - metadata["e"].must_equal "newetag" - metadata["m"].length.must_equal 13 - - metadata = redis.hgetall "rs:m:phil:food/" - metadata["e"].must_equal "newetag" - metadata["m"].length.must_equal 13 - - food_items = redis.smembers "rs:m:phil:food/:items" - food_items.each do |food_item| - ["camaron", "aguacate"].must_include food_item - end - - root_items = redis.smembers "rs:m:phil:/:items" - root_items.must_equal ["food/"] - end - - context "response code" do - before do - @put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - end - - it "is 201 for newly created objects" do - RestClient.stub :put, @put_stub do - put "/phil/food/aguacate", "muy deliciosa" - end - - last_response.status.must_equal 201 - end - - it "is 200 for updated objects" do - RestClient.stub :put, @put_stub do - put "/phil/food/aguacate", "deliciosa" - put "/phil/food/aguacate", "muy deliciosa" - end - - last_response.status.must_equal 200 - end - end - - context "logging usage size" do - before do - @put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - end - - it "logs the complete size when creating new objects" do - RestClient.stub :put, @put_stub do - put "/phil/food/aguacate", "1234567890" - end - - size_log = redis.get "rs:s:phil" - size_log.must_equal "10" - end - - it "logs the size difference when updating existing objects" do - RestClient.stub :put, @put_stub do - put "/phil/food/camaron", "1234567890" - put "/phil/food/aguacate", "1234567890" - put "/phil/food/aguacate", "123" - end - - size_log = redis.get "rs:s:phil" - size_log.must_equal "13" - end - end - - describe "objects in root dir" do - before do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/bamboo.txt", "shir kan" - end - end - - it "are listed in the directory listing with all metadata" do - get "phil/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["items"]["bamboo.txt"].wont_be_nil - content["items"]["bamboo.txt"]["ETag"].must_equal "bla" - content["items"]["bamboo.txt"]["Content-Type"].must_equal "text/plain; charset=utf-8" - content["items"]["bamboo.txt"]["Content-Length"].must_equal 8 - content["items"]["bamboo.txt"]["Last-Modified"].must_equal "Fri, 04 Mar 2016 12:20:18 GMT" - end - end - - describe "name collision checks" do - it "is successful when there is no name collision" do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - get_stub = OpenStruct.new(body: "rootbody") - - RestClient.stub :put, put_stub do - RestClient.stub :get, get_stub do - RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do - put "/phil/food/aguacate", "si" - end - end - end - - last_response.status.must_equal 201 - - metadata = redis.hgetall "rs:m:phil:food/aguacate" - metadata["s"].must_equal "2" - end - - it "conflicts when there is a directory with same name as document" do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food", "wontwork" - end - - last_response.status.must_equal 409 - last_response.body.must_equal "Conflict" - - metadata = redis.hgetall "rs:m:phil:food" - metadata.must_be_empty - end - - it "conflicts when there is a document with same name as directory" do - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/aguacate/empanado", "wontwork" - end - - last_response.status.must_equal 409 - - metadata = redis.hgetall "rs:m:phil:food/aguacate/empanado" - metadata.must_be_empty - end - - it "returns 400 when a Content-Range header is sent" do - header "Content-Range", "bytes 0-3/3" - - put "/phil/food/aguacate", "si" - - last_response.status.must_equal 400 - end - end - - describe "If-Match header" do - before do - put_stub = OpenStruct.new(headers: { - etag: "oldetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end - end - - it "allows the request if the header matches the current ETag" do - header "If-Match", "\"oldetag\"" - - put_stub = OpenStruct.new(headers: { - etag: "newetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "aye" - end - - last_response.status.must_equal 200 - last_response.headers["Etag"].must_equal "\"newetag\"" - end - - it "allows the request if the header contains a weak ETAG matching the current ETag" do - header "If-Match", "W/\"oldetag\"" - - put_stub = OpenStruct.new(headers: { - etag: "newetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "aye" - end - - last_response.status.must_equal 200 - last_response.headers["Etag"].must_equal "\"newetag\"" - end - - it "allows the request if the header contains a weak ETAG with leading quote matching the current ETag" do - header "If-Match", "\"W/\"oldetag\"" - - put_stub = OpenStruct.new(headers: { - etag: "newetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "aye" - end - - last_response.status.must_equal 200 - last_response.headers["Etag"].must_equal "\"newetag\"" - end - - it "fails the request if the header does not match the current ETag" do - header "If-Match", "someotheretag" - - head_stub = OpenStruct.new(headers: { - etag: "oldetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain", - content_length: 23 - }) - - RestClient.stub :head, head_stub do - put "/phil/food/aguacate", "aye" - end - - last_response.status.must_equal 412 - last_response.body.must_equal "Precondition Failed" - end - - it "allows the request if redis metadata became out of sync" do - header "If-Match", "\"existingetag\"" - - head_stub = OpenStruct.new(headers: { - etag: "existingetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain", - content_length: 23 - }) - - put_stub = OpenStruct.new(headers: { - etag: "newetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :head, head_stub do - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "aye" - end - end - - last_response.status.must_equal 200 - end - end - - describe "If-None-Match header set to '*'" do - it "succeeds when the document doesn't exist yet" do - put_stub = OpenStruct.new(headers: { - etag: "someetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - header "If-None-Match", "*" - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end - - last_response.status.must_equal 201 - end - - it "fails the request if the document already exists" do - put_stub = OpenStruct.new(headers: { - etag: "someetag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end - - header "If-None-Match", "*" - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - end - - last_response.status.must_equal 412 - last_response.body.must_equal "Precondition Failed" - end - end - end - - end - - describe "DELETE requests" do - - before do - purge_redis - end - - context "not authorized" do - describe "with no token" do - it "says it's not authorized" do - delete "/phil/food/aguacate" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - describe "with empty token" do - it "says it's not authorized" do - header "Authorization", "Bearer " - delete "/phil/food/aguacate" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - describe "with wrong token" do - it "says it's not authorized" do - header "Authorization", "Bearer wrongtoken" - delete "/phil/food/aguacate" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - end - - context "authorized" do - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desayunos/bolon", "wow" - end - end - - it "decreases the size log by size of deleted object" do - RestClient.stub :delete, "" do - RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do - delete "/phil/food/aguacate" - end - end - - size_log = redis.get "rs:s:phil" - size_log.must_equal "8" - end - - it "deletes the metadata object in redis" do - RestClient.stub :delete, "" do - RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do - delete "/phil/food/aguacate" - end - end - - metadata = redis.hgetall "rs:m:phil:food/aguacate" - metadata.must_be_empty - end - - it "deletes the directory objects metadata in redis" do - old_metadata = redis.hgetall "rs:m:phil:food/" - - RestClient.stub :delete, "" do - RemoteStorage::Swift.stub_any_instance :etag_for, "newetag" do - delete "/phil/food/aguacate" - end - end - - metadata = redis.hgetall "rs:m:phil:food/" - metadata["e"].must_equal "newetag" - metadata["m"].length.must_equal 13 - metadata["m"].wont_equal old_metadata["m"] - - food_items = redis.smembers "rs:m:phil:food/:items" - food_items.sort.must_equal ["camaron", "desayunos/"] - - root_items = redis.smembers "rs:m:phil:/:items" - root_items.must_equal ["food/"] - end - - it "deletes the parent directory objects metadata when deleting all items" do - RestClient.stub :delete, "" do - RemoteStorage::Swift.stub_any_instance :etag_for, "rootetag" do - delete "/phil/food/aguacate" - delete "/phil/food/camaron" - delete "/phil/food/desayunos/bolon" - end - end - - redis.smembers("rs:m:phil:food/desayunos:items").must_be_empty - redis.hgetall("rs:m:phil:food/desayunos/").must_be_empty - - redis.smembers("rs:m:phil:food/:items").must_be_empty - redis.hgetall("rs:m:phil:food/").must_be_empty - - redis.smembers("rs:m:phil:/:items").must_be_empty - end - - it "responds with the ETag of the deleted item in the header" do - RestClient.stub :delete, "" do - delete "/phil/food/aguacate" - end - - last_response.headers["ETag"].must_equal "\"bla\"" - end - - context "when item doesn't exist" do - before do - purge_redis - - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/steak", "si" - end - - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :delete, raises_exception do - delete "/phil/food/steak" - end - end - - it "returns a 404" do - last_response.status.must_equal 404 - last_response.body.must_equal "Not Found" - end - - it "deletes any metadata that might still exist" do - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :delete, raises_exception do - delete "/phil/food/steak" - end - - metadata = redis.hgetall "rs:m:phil:food/steak" - metadata.must_be_empty - - redis.smembers("rs:m:phil:food/:items").must_be_empty - redis.hgetall("rs:m:phil:food/").must_be_empty - - redis.smembers("rs:m:phil:/:items").must_be_empty - end - end - - describe "If-Match header" do - it "succeeds when the header matches the current ETag" do - header "If-Match", "\"bla\"" - - RestClient.stub :delete, "" do - delete "/phil/food/aguacate" - end - - last_response.status.must_equal 200 - end - - it "succeeds when the header contains a weak ETAG matching the current ETag" do - header "If-Match", "W/\"bla\"" - - RestClient.stub :delete, "" do - delete "/phil/food/aguacate" - end - - last_response.status.must_equal 200 - end - - it "fails the request if it does not match the current ETag" do - header "If-Match", "someotheretag" - - delete "/phil/food/aguacate" - - last_response.status.must_equal 412 - last_response.body.must_equal "Precondition Failed" - end - end - end +describe "Swift provider" do + def container_url_for(user) + "#{app.settings.swift["host"]}/rs:documents:test/#{user}" end - describe "GET requests" do - - before do - purge_redis - end - - context "not authorized" do - - describe "without token" do - it "says it's not authorized" do - get "/phil/food/" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - describe "with wrong token" do - it "says it's not authorized" do - header "Authorization", "Bearer wrongtoken" - get "/phil/food/" - - last_response.status.must_equal 401 - last_response.body.must_equal "Unauthorized" - end - end - - end - - context "authorized" do - - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desayunos/bolon", "wow" - end - end - - describe "documents" do - - it "returns the required response headers" do - get_stub = OpenStruct.new(body: "si", headers: { - etag: "0815etag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain; charset=utf-8", - content_length: 2 - }) - - RestClient.stub :get, get_stub do - get "/phil/food/aguacate" - end - - last_response.status.must_equal 200 - last_response.headers["ETag"].must_equal "\"0815etag\"" - last_response.headers["Cache-Control"].must_equal "no-cache" - last_response.headers["Content-Type"].must_equal "text/plain; charset=utf-8" - end - - it "returns a 404 when data doesn't exist" do - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :get, raises_exception do - get "/phil/food/steak" - end - - last_response.status.must_equal 404 - last_response.body.must_equal "Not Found" - end - - it "responds with 304 when IF_NONE_MATCH header contains the ETag" do - header "If-None-Match", "\"0815etag\"" - - get_stub = OpenStruct.new(body: "si", headers: { - etag: "0815etag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain; charset=utf-8", - content_length: 2 - }) - - RestClient.stub :get, get_stub do - get "/phil/food/aguacate" - end - - last_response.status.must_equal 304 - end - - it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the current ETag" do - header "If-None-Match", "W/\"0815etag\"" - - get_stub = OpenStruct.new(body: "si", headers: { - etag: "0815etag", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT", - content_type: "text/plain; charset=utf-8", - content_length: 2 - }) - - RestClient.stub :get, get_stub do - get "/phil/food/aguacate" - end - - last_response.status.must_equal 304 - end - - end - - describe "directory listings" do - - it "returns the correct ETag header" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - end - - it "returns a Cache-Control header with value 'no-cache'" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.headers["Cache-Control"].must_equal "no-cache" - end - - it "responds with 304 when IF_NONE_MATCH header contains the ETag" do - header "If-None-Match", "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - get "/phil/food/" - - last_response.status.must_equal 304 - end - - it "responds with 304 when IF_NONE_MATCH header contains weak ETAG matching the ETag" do - header "If-None-Match", "W/\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - get "/phil/food/" - - last_response.status.must_equal 304 - end - - it "contains all items in the directory" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["@context"].must_equal "http://remotestorage.io/spec/folder-description" - content["items"]["aguacate"].wont_be_nil - content["items"]["aguacate"]["Content-Type"].must_equal "text/plain; charset=utf-8" - content["items"]["aguacate"]["Content-Length"].must_equal 2 - content["items"]["aguacate"]["ETag"].must_equal "bla" - content["items"]["camaron"].wont_be_nil - content["items"]["camaron"]["Content-Type"].must_equal "text/plain; charset=utf-8" - content["items"]["camaron"]["Content-Length"].must_equal 5 - content["items"]["camaron"]["ETag"].must_equal "bla" - content["items"]["desayunos/"].wont_be_nil - content["items"]["desayunos/"]["ETag"].must_equal "dd36e3cfe52b5f33421150b289a7d48d" - end - - it "contains all items in the root directory" do - get "phil/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["items"]["food/"].wont_be_nil - content["items"]["food/"]["ETag"].must_equal "f9f85fbf5aa1fa378fd79ac8aa0a457d" - end - - it "responds with an empty directory liting when directory doesn't exist" do - get "phil/some-non-existing-dir/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - - content = JSON.parse(last_response.body) - content["items"].must_equal({}) - end - - end - end - + def storage_class + RemoteStorage::Swift end - describe "HEAD requests" do - - before do - purge_redis - end - - context "not authorized" do - - describe "without token" do - it "says it's not authorized" do - head "/phil/food/camarones" - - last_response.status.must_equal 401 - last_response.body.must_be_empty - end - end - - describe "with wrong token" do - it "says it's not authorized" do - header "Authorization", "Bearer wrongtoken" - head "/phil/food/camarones" - - last_response.status.must_equal 401 - last_response.body.must_be_empty - end - end - - end - - context "authorized" do - - before do - redis.sadd "authorizations:phil:amarillo", [":rw"] - header "Authorization", "Bearer amarillo" - - put_stub = OpenStruct.new(headers: { - etag: "bla", - last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" - }) - - RestClient.stub :put, put_stub do - put "/phil/food/aguacate", "si" - put "/phil/food/camaron", "yummi" - put "/phil/food/desayunos/bolon", "wow" - end - end - - describe "directory listings" do - it "returns the correct header information" do - get "/phil/food/" - - last_response.status.must_equal 200 - last_response.content_type.must_equal "application/ld+json" - last_response.headers["ETag"].must_equal "\"f9f85fbf5aa1fa378fd79ac8aa0a457d\"" - end - end - - describe "documents" do - it "returns a 404 when the document doesn't exist" do - raises_exception = ->(url, headers) { raise RestClient::ResourceNotFound.new } - RestClient.stub :head, raises_exception do - head "/phil/food/steak" - end - - last_response.status.must_equal 404 - last_response.body.must_be_empty - end - end - - end - + before do + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + # Write new content with an If-Match header (a new Etag is returned) + stub_request(:put, "#{container_url_for("phil")}/food/aguacate"). + with(body: "aye"). + to_return(status: 200, headers: { etag: "0915etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:head, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:get, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, body: "rootbody", headers: { etag: "0817etag", content_type: "text/plain; charset=utf-8" }) + stub_request(:delete, "#{container_url_for("phil")}/food/aguacate"). + to_return(status: 200, headers: { etag: "0815etag" }) + + # Write new content to check the metadata in Redis + stub_request(:put, "#{container_url_for("phil")}/food/banano"). + with(body: "si"). + to_return(status: 200, headers: { etag: "0815etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:put, "#{container_url_for("phil")}/food/banano"). + with(body: "oh, no"). + to_return(status: 200, headers: { etag: "0817etag", last_modified: "Fri, 04 Mar 2016 12:20:20 GMT" }) + + stub_request(:put, "#{container_url_for("phil")}/food/camaron"). + to_return(status: 200, headers: { etag: "0816etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/camaron"). + to_return(status: 200, headers: { etag: "0816etag" }) + + stub_request(:put, "#{container_url_for("phil")}/food/desayunos/bolon"). + to_return(status: 200, headers: { etag: "0817etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + stub_request(:delete, "#{container_url_for("phil")}/food/desayunos/bolon"). + to_return(status: 200, headers: { etag: "0817etag" }) + + # objects in root dir + stub_request(:put, "#{container_url_for("phil")}/bamboo.txt"). + to_return(status: 200, headers: { etag: "0818etag", last_modified: "Fri, 04 Mar 2016 12:20:18 GMT" }) + + # 404 + stub_request(:head, "#{container_url_for("phil")}/food/steak"). + to_return(status: 404) + stub_request(:get, "#{container_url_for("phil")}/food/steak"). + to_return(status: 404) + stub_request(:delete, "#{container_url_for("phil")}/food/steak"). + to_return(status: 404) end + it_behaves_like 'a REST adapter' end -