add rate limit protection

Author: dgiunta

Diff for: lib/codeship_api.rb

@@ -13,8 +13,8 @@
 module CodeshipApi
   ROOT = URI('https://api.codeship.com/v2/')
 
-  USERNAME = ENV.fetch('CODESHIP_API_USERNAME')
-  PASSWORD = ENV.fetch('CODESHIP_API_PASSWORD')
+  USERNAME = ENV.fetch('CODESHIP_API_USERNAME', nil)
+  PASSWORD = ENV.fetch('CODESHIP_API_PASSWORD', nil)
 
   class << self
     def client

Diff for: lib/codeship_api/build.rb

@@ -12,6 +12,25 @@ class Build < Base
       end
     end
 
+    def self.find_all_by_project(
+      project,
+      page: 1, builds: [],
+      filter_by: -> (build) { true },
+      stop_when: -> (new_builds) { new_builds.empty? }
+    )
+      new_builds = find_by_project(project, page: page)
+      builds += new_builds.select(&filter_by)
+
+      if stop_when.call(new_builds)
+        builds
+      else
+        find_all_by_project(
+          project, page: page + 1, builds: builds,
+          stop_when: stop_when, filter_by: filter_by
+        )
+      end
+    end
+
     def self.find_by_project(project, per_page: 50, page: 1)
       uri = "#{project.uri}/builds?per_page=#{per_page}&page=#{page}"
       CodeshipApi.client.get(uri)["builds"].map {|build| new(build) }

Diff for: lib/codeship_api/client.rb

@@ -1,5 +1,7 @@
 module CodeshipApi
   class Client
+    class RateLimitExceededError < StandardError; end
+
     def initialize(username=USERNAME, password=PASSWORD)
       @username = username
       @password = password
@@ -10,21 +12,27 @@ def authenticated?
     end
 
     def get(path)
-      uri = URI.join(ROOT, path.sub(/^\//, ""))
+      rate_limit_protected! do
+        uri = URI.join(ROOT, path.sub(/^\//, ""))
 
-      request = Net::HTTP::Get.new(uri, default_headers)
-      response = http.request(request)
+        request = Net::HTTP::Get.new(uri, default_headers)
+        response = http.request(request)
 
-      parsed_json_from(response)
+        parse_errors_from(response)
+        parsed_json_from(response)
+      end
     end
 
     def post(path)
-      uri = URI.join(ROOT, path.sub(/^\//, ""))
+      rate_limit_protected! do
+        uri = URI.join(ROOT, path.sub(/^\//, ""))
 
-      request = Net::HTTP::Post.new(uri, default_headers)
-      response = http.request(request)
+        request = Net::HTTP::Post.new(uri, default_headers)
+        response = http.request(request)
 
-      parsed_json_from(response)
+        parse_errors_from(response)
+        parsed_json_from(response)
+      end
     end
 
     def organizations
@@ -39,6 +47,20 @@ def projects
 
     attr_reader :username, :password
 
+    def rate_limit_protected!(&block)
+      time_start = Time.now
+      block.call.tap do
+        duration = Time.now - time_start
+        sleep 1.second - duration if duration < 1.second
+      end
+    rescue RateLimitExceededError => e
+      sleep_duration = 60
+      print "waiting #{sleep_duration}s for rate limit..."
+      sleep sleep_duration
+      puts " done."
+      rate_limit_protected!(&block)
+    end
+
     def token
       authenticate unless authenticated?
       authentication.access_token
@@ -49,18 +71,20 @@ def authentication
     end
 
     def authenticate
-      uri = URI.join(ROOT, 'auth')
-
-      request = Net::HTTP::Post.new(uri.request_uri, {
-        'Content-Type' => 'application/json',
-        'Accept' => 'application/json'
-      }).tap do |req|
-        req.basic_auth(username, password)
-      end
+      rate_limit_protected! do
+        uri = URI.join(ROOT, 'auth')
+
+        request = Net::HTTP::Post.new(uri.request_uri, {
+          'Content-Type' => 'application/json',
+          'Accept' => 'application/json'
+        }).tap do |req|
+          req.basic_auth(username, password)
+        end
 
-      response = http.request(request)
+        response = http.request(request)
 
-      @authentication = Authentication.new(JSON.parse(response.body))
+        @authentication = Authentication.new(JSON.parse(response.body))
+      end
     end
 
     def http
@@ -79,12 +103,16 @@ def default_headers
       }
     end
 
-    def parsed_json_from(response)
-      if response.body.length > 0
-        JSON.parse(response.body)
-      else
-        nil
+    def parse_errors_from(response)
+      if response.code.to_i == 403 && response.entity == "Rate Limit Exceeded"
+        raise RateLimitExceededError.new(response)
       end
     end
+
+    def parsed_json_from(response)
+      return nil unless response.body.length > 0
+
+      JSON.parse(response.body)
+    end
   end
 end

Diff for: test/codeship_api/client_test.rb

@@ -3,8 +3,8 @@
 describe CodeshipApi::Client do
   before do
     @client = CodeshipApi::Client.new
-    @fake_success_response = Struct.new(:body).new('{"status": "OK"}')
-    @fake_null_response = Struct.new(:body).new('')
+    @fake_success_response = Struct.new(:code, :body).new("200", '{"status": "OK"}')
+    @fake_null_response = Struct.new(:code, :body).new("200", '')
     @mock_http = Minitest::Mock.new
 
     def @client.token
@@ -18,7 +18,7 @@ def @client.token
       expires_at = Time.now + (10 * 60 * 60) # 10 minutes from now
       org = {uuid: "asdf-asdf-asdf-asdf", name: "fake_org", scopes: []}
 
-      auth_response = Struct.new(:body).new({
+      auth_response = Struct.new(:code, :body).new("200", {
         access_token: token,
         expires_at: expires_at.to_i,
         organizations: [org]