From 7bdb4e52930b9bcd7aa0f44c575bc01fad95fb4b Mon Sep 17 00:00:00 2001 From: John Bellone Date: Tue, 28 Jan 2014 23:25:05 -0500 Subject: [PATCH 1/4] Add support for etcd registry. This is initial support for the etcd service to be used as a registry. --- Gemfile | 1 + lib/dcell/registries/etcd_adapter.rb | 84 ++++++++++++++++++++++ spec/dcell/registries/etcd_adapter_spec.rb | 7 ++ 3 files changed, 92 insertions(+) create mode 100644 lib/dcell/registries/etcd_adapter.rb create mode 100644 spec/dcell/registries/etcd_adapter_spec.rb diff --git a/Gemfile b/Gemfile index 6f45e72..08a7420 100644 --- a/Gemfile +++ b/Gemfile @@ -15,3 +15,4 @@ gem 'coveralls', require: false gemspec gem 'zk' +gem 'etcd' diff --git a/lib/dcell/registries/etcd_adapter.rb b/lib/dcell/registries/etcd_adapter.rb new file mode 100644 index 0000000..8e28ede --- /dev/null +++ b/lib/dcell/registries/etcd_adapter.rb @@ -0,0 +1,84 @@ +require 'etcd' + +module DCell + module Registry + class EtcdAdapter + PREFIX = '/dcell' + DEFAULT_PORT = 4001 + + # Create a new connect to Etc daemon. + # + # servers: List of Etcd servers to connect to. Each server + # has a host/port configuration. + def initialize(options={}) + options = options.inject({}) { |h,(k,v)| h[k.to_s] = v; h } + + @env = options['env'] || 'production' + @base_path = "#{PREFIX}/#{@env}" + + if options['server'] + servers = [ options['server'] ] + else + servers = options['servers'] + end + + # Sanity check. + raise 'no Etcd servers given' unless servers + + # Add default Etcd port unless specified. + servers.map! do |server| + if server[/:\d+$/] + server + else + "#{server}:#{DEFAULT_PORT}" + end + end + + server = server.first + @etcd = Etcd.client(host: server.host, port: server.port) + @node_registry = Registry.new(@etcd, @base_path, :nodes) + @global_registry = Registry.new(@etcd, @base_path, :globals) + end + + def get_node(node_id); @node_registry.get(node_id) end + def set_node(node_id, addr); @node_registry.set(node_id, addr) end + def nodes; @node_registry.all end + def clear_nodes; @node_registry.clear end + + def get_global(key); @global_registry.get(key) end + def set_global(key, value); @global_registry.set(key, value) end + def global_keys; @global_registry.all end + def clear_globals; @global_registry.clear end + + class Registry + def initialize(etcd, base_path, name) + @etcd = etcd + @base_path = File.join(base_path, name.to_s) + @etcd.create(@base_path, dir: true) + end + + def get(key) + result = @etcd.get(File.join(base_path, key)) + Marshal.load(result.value) + end + + def set(key, value) + path = File.join base_path, key + string = Marshal.dump value + @etcd.set(path, value: string) + rescue Etcd::NodeExist + @etcd.create(path, value: string) + end + + def all + @etcd.get(@base_path).children + end + + def clear + @etcd.delete(@base_path) + @etcd.create(@base_path, dir: true) + end + end + end + end +end diff --git a/spec/dcell/registries/etcd_adapter_spec.rb b/spec/dcell/registries/etcd_adapter_spec.rb new file mode 100644 index 0000000..7ef0dcb --- /dev/null +++ b/spec/dcell/registries/etcd_adapter_spec.rb @@ -0,0 +1,7 @@ +require 'spec_helper' +require 'dcell/registries/etcd_adapter' + +describe DCell::Registry::EtcdAdapter, :pending => ENV["CI"] && "no zookeeper" do + subject { DCell::Registry::EtcdAdapter.new :server => 'localhost', :env => 'test' } + it_behaves_like "a DCell registry" +end From 966cdb8b6113546c907f74487c06e024caa01f11 Mon Sep 17 00:00:00 2001 From: Hendrik Beskow Date: Sat, 1 Mar 2014 16:03:54 +0100 Subject: [PATCH 2/4] uses uris to configure etcd servers --- Gemfile | 3 ++- lib/dcell/celluloid_ext.rb | 3 +-- lib/dcell/registries/etcd_adapter.rb | 38 ++++++++++++++++++---------- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/Gemfile b/Gemfile index 08a7420..f50f898 100644 --- a/Gemfile +++ b/Gemfile @@ -4,6 +4,7 @@ gem 'celluloid', github: 'celluloid/celluloid', branch: 'master' gem 'celluloid-io', github: 'celluloid/celluloid-io', branch: 'master' gem 'celluloid-zmq', github: 'celluloid/celluloid-zmq', branch: 'master' gem 'celluloid-redis', github: 'celluloid/celluloid-redis', branch: 'master' +gem 'http', github: 'tarcieri/http', branch: 'master' gem 'reel', github: 'celluloid/reel', branch: 'master' #gem 'ffi-rzmq', github: 'chuckremes/ffi-rzmq' @@ -15,4 +16,4 @@ gem 'coveralls', require: false gemspec gem 'zk' -gem 'etcd' +gem 'etcd', '~> 0.2.0.beta.1' diff --git a/lib/dcell/celluloid_ext.rb b/lib/dcell/celluloid_ext.rb index 55909c3..3b37090 100644 --- a/lib/dcell/celluloid_ext.rb +++ b/lib/dcell/celluloid_ext.rb @@ -13,11 +13,10 @@ class ActorProxy # Marshal uses respond_to? to determine if this object supports _dump so # unfortunately we have to monkeypatch in _dump support as the proxy # itself normally jacks respond_to? and proxies to the actor - alias_method :__respond_to?, :respond_to? def respond_to?(meth, check_private = false) return false if meth == :marshal_dump return true if meth == :_dump - __respond_to?(meth, check_private) + method_missing(:respond_to?, meth, include_private) end # Dump an actor proxy via its mailbox diff --git a/lib/dcell/registries/etcd_adapter.rb b/lib/dcell/registries/etcd_adapter.rb index 8e28ede..d104833 100644 --- a/lib/dcell/registries/etcd_adapter.rb +++ b/lib/dcell/registries/etcd_adapter.rb @@ -1,18 +1,20 @@ +require 'uri' require 'etcd' module DCell module Registry class EtcdAdapter PREFIX = '/dcell' - DEFAULT_PORT = 4001 - + DEFAULT_SCHEME = 'http' + DEFAULT_PORT = 4001 + # Create a new connect to Etc daemon. # # servers: List of Etcd servers to connect to. Each server # has a host/port configuration. def initialize(options={}) options = options.inject({}) { |h,(k,v)| h[k.to_s] = v; h } - + @env = options['env'] || 'production' @base_path = "#{PREFIX}/#{@env}" @@ -27,15 +29,21 @@ def initialize(options={}) # Add default Etcd port unless specified. servers.map! do |server| - if server[/:\d+$/] + server = server.is_a?(URI) ? server : URI(server) + if server.scheme && server.host && server.port server else - "#{server}:#{DEFAULT_PORT}" + server = URI("#{DEFAULT_SCHEME}://#{server.to_s}:#{DEFAULT_PORT}") end end - - server = server.first - @etcd = Etcd.client(host: server.host, port: server.port) + + server = servers[rand(servers.count)] + @etcd = Etcd.client(host: server.host, + port: server.port, + use_ssl: server.scheme == 'https' ? true : false, + user_name: server.user, + password: server.password) + @node_registry = Registry.new(@etcd, @base_path, :nodes) @global_registry = Registry.new(@etcd, @base_path, :globals) end @@ -48,26 +56,28 @@ def clear_nodes; @node_registry.clear end def get_global(key); @global_registry.get(key) end def set_global(key, value); @global_registry.set(key, value) end def global_keys; @global_registry.all end - def clear_globals; @global_registry.clear end + def clear_globals; @global_registry.clear end class Registry def initialize(etcd, base_path, name) @etcd = etcd @base_path = File.join(base_path, name.to_s) - @etcd.create(@base_path, dir: true) + unless @etcd.exists?(@base_path) + @etcd.create(@base_path, dir: true) + end end def get(key) result = @etcd.get(File.join(base_path, key)) - Marshal.load(result.value) + Marshal.load(result.value) if result end def set(key, value) - path = File.join base_path, key - string = Marshal.dump value + path = File.join @base_path, key + string = Marshal.dump(value) @etcd.set(path, value: string) rescue Etcd::NodeExist - @etcd.create(path, value: string) + @etcd.update(path, value: string) end def all From 17ba070644a626e8344d2a8c493a65f2de28f1bd Mon Sep 17 00:00:00 2001 From: John Bellone Date: Tue, 28 Jan 2014 23:25:05 -0500 Subject: [PATCH 3/4] Add support for etcd registry. This is initial support for the etcd service to be used as a registry. --- Gemfile | 1 + lib/dcell/registries/etcd_adapter.rb | 84 ++++++++++++++++++++++ spec/dcell/registries/etcd_adapter_spec.rb | 7 ++ 3 files changed, 92 insertions(+) create mode 100644 lib/dcell/registries/etcd_adapter.rb create mode 100644 spec/dcell/registries/etcd_adapter_spec.rb diff --git a/Gemfile b/Gemfile index 6f45e72..08a7420 100644 --- a/Gemfile +++ b/Gemfile @@ -15,3 +15,4 @@ gem 'coveralls', require: false gemspec gem 'zk' +gem 'etcd' diff --git a/lib/dcell/registries/etcd_adapter.rb b/lib/dcell/registries/etcd_adapter.rb new file mode 100644 index 0000000..8e28ede --- /dev/null +++ b/lib/dcell/registries/etcd_adapter.rb @@ -0,0 +1,84 @@ +require 'etcd' + +module DCell + module Registry + class EtcdAdapter + PREFIX = '/dcell' + DEFAULT_PORT = 4001 + + # Create a new connect to Etc daemon. + # + # servers: List of Etcd servers to connect to. Each server + # has a host/port configuration. + def initialize(options={}) + options = options.inject({}) { |h,(k,v)| h[k.to_s] = v; h } + + @env = options['env'] || 'production' + @base_path = "#{PREFIX}/#{@env}" + + if options['server'] + servers = [ options['server'] ] + else + servers = options['servers'] + end + + # Sanity check. + raise 'no Etcd servers given' unless servers + + # Add default Etcd port unless specified. + servers.map! do |server| + if server[/:\d+$/] + server + else + "#{server}:#{DEFAULT_PORT}" + end + end + + server = server.first + @etcd = Etcd.client(host: server.host, port: server.port) + @node_registry = Registry.new(@etcd, @base_path, :nodes) + @global_registry = Registry.new(@etcd, @base_path, :globals) + end + + def get_node(node_id); @node_registry.get(node_id) end + def set_node(node_id, addr); @node_registry.set(node_id, addr) end + def nodes; @node_registry.all end + def clear_nodes; @node_registry.clear end + + def get_global(key); @global_registry.get(key) end + def set_global(key, value); @global_registry.set(key, value) end + def global_keys; @global_registry.all end + def clear_globals; @global_registry.clear end + + class Registry + def initialize(etcd, base_path, name) + @etcd = etcd + @base_path = File.join(base_path, name.to_s) + @etcd.create(@base_path, dir: true) + end + + def get(key) + result = @etcd.get(File.join(base_path, key)) + Marshal.load(result.value) + end + + def set(key, value) + path = File.join base_path, key + string = Marshal.dump value + @etcd.set(path, value: string) + rescue Etcd::NodeExist + @etcd.create(path, value: string) + end + + def all + @etcd.get(@base_path).children + end + + def clear + @etcd.delete(@base_path) + @etcd.create(@base_path, dir: true) + end + end + end + end +end diff --git a/spec/dcell/registries/etcd_adapter_spec.rb b/spec/dcell/registries/etcd_adapter_spec.rb new file mode 100644 index 0000000..745a0d3 --- /dev/null +++ b/spec/dcell/registries/etcd_adapter_spec.rb @@ -0,0 +1,7 @@ +require 'spec_helper' +require 'dcell/registries/etcd_adapter' + +describe DCell::Registry::EtcdAdapter, :pending => ENV["CI"] && "no etcd" do + subject { DCell::Registry::EtcdAdapter.new :server => 'localhost', :env => 'test' } + it_behaves_like "a DCell registry" +end From a0abff002d385ec6fc44ea3afa5e42d8c735f083 Mon Sep 17 00:00:00 2001 From: John Bellone Date: Sat, 1 Mar 2014 10:23:58 -0500 Subject: [PATCH 4/4] Fix update from merge. --- lib/dcell/registries/etcd_adapter.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/dcell/registries/etcd_adapter.rb b/lib/dcell/registries/etcd_adapter.rb index 3751d89..ce33bb9 100644 --- a/lib/dcell/registries/etcd_adapter.rb +++ b/lib/dcell/registries/etcd_adapter.rb @@ -88,8 +88,7 @@ def set(key, value) string = Marshal.dump value @etcd.set(path, value: string) rescue Etcd::NodeExist - @etcd.create(path, value: string) - Marshal.load(result.value) if result + @etcd.update(path, value: string) end def all