Merge branch 'release-0.5.0'

duracloud · Jun 7, 2017 · 9ce2c75 · 9ce2c75
2 parents aed79a9 + 8094364
commit 9ce2c75
Show file tree

Hide file tree

Showing 6 changed files with 128 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -276,6 +276,17 @@ D, [2016-05-19T13:37:39.831013 #28754] DEBUG -- : Duracloud::Client GET https://
  => ["space_id", "content_id", "md5"]
 ```
 
+*Added in v0.5.0: Support for asynchronous generation of manifest (Generate Manifest API)*
+
+```
+>> manifest = Duracloud::Manifest.new('ddr-validation')
+ => #<Duracloud::Manifest:0x007f8fd82fe328 @space_id="ddr-validation", @store_id=nil>
+>> manifest.generate
+ D, [2017-06-06T20:49:50.191301 #92029] DEBUG -- : Duracloud::Client POST https://duke.duracloud.org/durastore/manifest/ddr-validation 202 Accepted
+ I, [2017-06-06T20:49:50.191414 #92029]  INFO -- : We are processing your manifest generation request. To retrieve your file, please poll the URI in the Location header of this response: (https://duke.duracloud.org/durastore/x-duracloud-admin/generated-manifests/manifest-ddr-validation_amazon_s3_2017-06-07-00-49-50.txt.gz).
+ => "https://duke.duracloud.org/durastore/x-duracloud-admin/generated-manifests/manifest-ddr-validation_amazon_s3_2017-06-07-00-49-50.txt.gz"
+```  
+
 #### Bit Integrity Report
 
 ```

diff --git a/lib/duracloud/manifest.rb b/lib/duracloud/manifest.rb
@@ -1,10 +1,16 @@
+require 'tempfile'
+require 'zlib'
+
 module Duracloud
   class Manifest
     include TSV
 
     TSV_FORMAT = "TSV"
     BAGIT_FORMAT = "BAGIT"
 
+    MAX_TRIES = 120
+    RETRY_SLEEP = 10
+
     attr_reader :space_id, :store_id
 
     def initialize(space_id, store_id = nil)
@@ -30,14 +36,52 @@ def bagit(&block)
       download(BAGIT_FORMAT, &block)
     end
 
+    # Request the manifest for the space to be generated.
+    # @param format [Symbol, String] the format of the manifest.
+    #    Defaults to "TSV".
+    # @return [String] the URL of the generated manifest when available.
+    # @raise [Duracloud::NotFoundError]
+    # @note format parameter changed from positional to keyword argument
+    #   in v0.5.0.
+    def generate(format: TSV_FORMAT)
+      fmt = format.to_s.upcase
+      response = Client.generate_manifest(space_id, query(fmt))
+      response.header["Location"].first
+    end
+
+    # Downloads the generated manifest
+    # @yield [String] chunk of the manifest
+    # @param format [Symbol, String] the format of the manifest.
+    #   Defaults to "TSV".
+    # @param max_tries [Integer] max number of times to check the generated URL.
+    # @param retry_sleep [Integer] number of seconds between re-checks of the
+    #   generated URL.
+    # @raise [Duracloud::NotFoundError
+    def download_generated(format: TSV_FORMAT, max_tries: MAX_TRIES, retry_sleep: RETRY_SLEEP, &block)
+      url = generate(format: format)
+      check_generated(url, max_tries, retry_sleep)
+      Tempfile.open(["download", ".gz"], encoding: "ascii-8bit") do |gz_file|
+        client.execute(Request, :get, url) do |chunk|
+          gz_file.write(chunk)
+        end
+        gz_file.close
+        Zlib::GzipReader.open(gz_file.path) do |unzipped|
+          unzipped.each { |line| yield(line) }
+        end
+      end
+      url
+    end
+
     # Downloads the manifest
     # @yield [String] chunk of the manifest, if block given.
     # @param format [Symbol, String] the format of the manifest.
     #    Defaults to "TSV".
     # @return [Duracloud::Response, String] the response, if block
     #   given, or the manifest content, if no block.
     # @raise [Duracloud::NotFoundError]
-    def download(format = TSV_FORMAT, &block)
+    # @note format parameter changed from positional to keyword argument
+    #   in v0.5.0.
+    def download(format: TSV_FORMAT, &block)
       fmt = format.to_s.upcase
       if block_given?
         get_response(fmt, &block)
@@ -48,6 +92,27 @@ def download(format = TSV_FORMAT, &block)
 
     private
 
+    def client
+      @client ||= Client.new
+    end
+
+    def check_generated(url, max_tries, retry_sleep)
+      tries = 0
+      begin
+        tries += 1
+        client.logger.debug "Checking for generated manifest (try #{tries}/#{max_tries}) ... "
+        client.execute(Request, :head, url)
+      rescue NotFoundError => e
+        if tries < max_tries
+          client.logger.debug "Retrying in #{retry_sleep} seconds ..."
+          sleep(retry_sleep)
+          retry
+        else
+          raise
+        end
+      end
+    end
+
     def get_response(format, &block)
       Client.get_manifest(space_id, query(format), &block)
     end

diff --git a/lib/duracloud/rest_methods.rb b/lib/duracloud/rest_methods.rb
@@ -1,83 +1,108 @@
 module Duracloud
   module RestMethods
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetStores
     def get_stores
       durastore(:get, "stores")
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetSpaces
     def get_spaces(**query)
       durastore(:get, "spaces", **query)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetSpace
     def get_space(space_id, **query)
       durastore(:get, space_id, **query)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetSpaceProperties
     def get_space_properties(space_id, **query)
       durastore(:head, space_id, **query)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetSpaceACLs
     def get_space_acls(space_id, **query)
       durastore(:head, "acl/#{space_id}", **query)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-SetSpaceACLs
     def set_space_acls(space_id, **options)
       durastore(:post, "acl/#{space_id}", **options)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-CreateSpace
     def create_space(space_id, **query)
       durastore(:put, space_id, **query)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-DeleteSpace
     def delete_space(space_id, **query)
       durastore(:delete, space_id, **query)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetContent
     def get_content(space_id, content_id, **options, &block)
       durastore_content(:get, space_id, content_id, **options, &block)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetContentProperties
     def get_content_properties(space_id, content_id, **options)
       durastore_content(:head, space_id, content_id, **options)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-SetContentProperties
     def set_content_properties(space_id, content_id, **options)
       durastore_content(:post, space_id, content_id, **options)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-StoreContent
     def store_content(space_id, content_id, **options)
       durastore_content(:put, space_id, content_id, **options)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-CopyContent
     def copy_content(target_space_id, target_content_id, **options)
       durastore_content(:put, target_space_id, target_content_id, **options)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-DeleteContent
     def delete_content(space_id, content_id, **options)
       durastore_content(:delete, space_id, content_id, **options)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetAuditLog
     def get_audit_log(space_id, **query)
       durastore(:get, "audit/#{space_id}", **query)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetManifest
     def get_manifest(space_id, **query, &block)
       durastore(:get, "manifest/#{space_id}", **query, &block)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GenerateManifest
+    def generate_manifest(space_id, **query)
+      durastore(:post, "manifest/#{space_id}", **query)
+    end
+
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetBitIntegrityReport
     def get_bit_integrity_report(space_id, **query)
       durastore(:get, "bit-integrity/#{space_id}", **query)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetBitIntegrityReportProperties
     def get_bit_integrity_report_properties(space_id, **query)
       durastore(:head, "bit-integrity/#{space_id}", **query)
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetTasks
     def get_tasks(**query)
       raise NotImplementedError,
             "The API method 'Get Tasks' has not been implemented."
     end
 
+    # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-PerformTask
     def perform_task(task_name, **query)
       raise NotImplementedError,
             "The API method 'Perform Task' has not been implemented."

diff --git a/lib/duracloud/version.rb b/lib/duracloud/version.rb
@@ -1,3 +1,3 @@
 module Duracloud
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end
diff --git a/spec/unit/client_spec.rb b/spec/unit/client_spec.rb
@@ -275,6 +275,20 @@ module Duracloud
       }
     end
 
+    describe "generate_manifest" do
+      specify {
+        stub = stub_request(:post, "https://example.com/durastore/manifest/foo")
+        subject.generate_manifest("foo")
+        expect(stub).to have_been_requested
+      }
+      specify {
+        stub = stub_request(:post, "https://example.com/durastore/manifest/foo")
+               .with(query: {format: "BAGIT", storeID: 1})
+        subject.generate_manifest("foo", format: "BAGIT", storeID: 1)
+        expect(stub).to have_been_requested
+      }
+    end
+
     describe "get_bit_integrity_report" do
       specify {
         stub = stub_request(:get, "https://example.com/durastore/bit-integrity/foo")

diff --git a/spec/unit/manifest_spec.rb b/spec/unit/manifest_spec.rb
@@ -21,5 +21,16 @@ module Duracloud
       }
     end
 
+    describe "#generate" do
+      before do
+        stub_request(:post, "https://example.com/durastore/manifest/myspace?format=TSV")
+          .to_return(status: 202,
+                     headers: { "Location" => "https://example.com/durastore/x-duracloud-admin/generated-manifests/manifest-myspace_amazon_s3_2017-06-02-18-20-14.txt.gz" })
+      end
+      specify {
+        expect(subject.generate).to eq "https://example.com/durastore/x-duracloud-admin/generated-manifests/manifest-myspace_amazon_s3_2017-06-02-18-20-14.txt.gz"
+      }
+    end
+
   end
 end