Merge branch 'release-0.9.0'

README.md

@@ -140,12 +140,25 @@ Assumptions:
 Process:
 - The space manifest is downloaded from DuraCloud and converted to the expected input format for `md5deep` (two columns: md5 hash and file path, separated by two spaces).
 - `md5deep` is run against the content directory in "non-matching" mode (-X) with the converted manifest as the list of "known hashes".
-- Non-matching files from the `md5deep` run are re-checked individually by calling `Duracloud::Content.exist?`. This pass will account for content sync after `md5deep` started as well as files that have been chunked in DuraCloud.
+- Non-matching files from the `md5deep` run are re-checked individually by calling `Duracloud::Content.exist?`.
+This pass will account for content sync after `md5deep` started as well as files that have been chunked in DuraCloud.
+Results of this re-check are printed to STDOUT (or, if using the `:work_dir` option, below, to a file).
 
 ```ruby
 Duracloud::SyncValidation.call(space_id: 'foo', content_dir: '/var/foo/bar')
 ```
 
+*Added in version 0.9.0* - `:work_dir` option to specify existing directory in which to write validation files. The default behavior is to
+create a temporary directory which is deleted on completion of the process. If `:work_dir` is specified, no cleanup is performed.
+
+Files created in work directory:
+- `{SPACE_ID}-manifest.tsv` (DuraCloud manifest as downloaded)
+- `{SPACE_ID}-md5.txt` (Munged manifest for md5deep)
+- `{SPACE_ID}-audit.txt` (Output of md5deep, empty if all files match)
+- `{SPACE_ID}-recheck.txt` (Out of audit recheck, if necessary)
+
+*Added in version 0.9.0* - `Duracloud::FastSyncValidation`. This variant of sync validation does not compute local hashes but instead compares the local file list (generated with `find`) to the list of content IDs in the space manifest. Local misses are rechecked as in `SyncValidation` (but without MD5 comparison).
+
 ### Content
 
 #### Create a new content item and store it in DuraCloud

lib/duracloud.rb

@@ -8,13 +8,15 @@ module Duracloud
   autoload :ChunkedContent, "duracloud/chunked_content"
   autoload :Client, "duracloud/client"
   autoload :CLI, "duracloud/cli"
+  autoload :CommandOptions, "duracloud/command_options"
+  autoload :Commands, "duracloud/commands"
   autoload :Configuration, "duracloud/configuration"
   autoload :Connection, "duracloud/connection"
   autoload :Content, "duracloud/content"
   autoload :ContentManifest, "duracloud/content_manifest"
   autoload :DurastoreRequest, "duracloud/durastore_request"
   autoload :ErrorHandler, "duracloud/error_handler"
-  autoload :HasProperties, "duracloud/has_properties"
+  autoload :FastSyncValidation, "duracloud/fast_sync_validation"
   autoload :Manifest, "duracloud/manifest"
   autoload :Persistence, "duracloud/persistence"
   autoload :Properties, "duracloud/properties"
@@ -24,6 +26,8 @@ module Duracloud
   autoload :Space, "duracloud/space"
   autoload :SpaceAcls, "duracloud/space_acls"
   autoload :Store, "duracloud/store"
+  autoload :StorageReport, "duracloud/storage_report"
+  autoload :StorageReports, "duracloud/storage_reports"
   autoload :SyncValidation, "duracloud/sync_validation"
   autoload :TSV, "duracloud/tsv"
 end

lib/duracloud/cli.rb

@@ -4,8 +4,9 @@
 module Duracloud
   class CLI
     include ActiveModel::Model
+    include Commands
 
-    COMMANDS = %w( sync validate manifest properties )
+    COMMANDS = Commands.public_instance_methods.map(&:to_s)
 
     USAGE = <<-EOS
 Usage: duracloud [COMMAND] [options]
@@ -17,13 +18,28 @@ class CLI
 EOS
     HELP = "Type 'duracloud -h/--help' for usage."
 
-    attr_accessor :command, :user, :password, :host, :port,
-                  :space_id, :store_id, :content_id,
-                  :content_type, :md5,
-                  :content_dir, :format, :infile,
-                  :logging
-
-    validates_presence_of :space_id, message: "-s/--space-id option is required."
+    attr_accessor :all_spaces,
+                  :command,
+                  :content_dir,
+                  :content_id,
+                  :content_type,
+                  :fast,
+                  :format,
+                  :host,
+                  :infile,
+                  :logging,
+                  :md5,
+                  :missing,
+                  :password,
+                  :port,
+                  :prefix,
+                  :space_id,
+                  :store_id,
+                  :user,
+                  :work_dir
+
+    validates_presence_of :space_id, message: "-s/--space-id option is required.", unless: "command == 'get_storage_report'"
+    validates_inclusion_of :command, in: COMMANDS
 
     def self.error!(exception)
       $stderr.puts exception.message
@@ -34,112 +50,20 @@ def self.error!(exception)
     end
 
     def self.call(*args)
-      options = {}
-
-      parser = OptionParser.new do |opts|
-        opts.banner = USAGE
-
-        opts.on("-h", "--help",
-                "Prints help") do
-          puts opts
-          exit
-        end
-
-        opts.on("-H", "--host HOST",
-                "DuraCloud host") do |v|
-          options[:host] = v
-        end
-
-        opts.on("-P", "--port PORT",
-                "DuraCloud port") do |v|
-          options[:port] = v
-        end
-
-        opts.on("-u", "--user USER",
-                "DuraCloud user") do |v|
-          options[:user] = v
-        end
-
-        opts.on("-p", "--password PASSWORD",
-                "DuraCloud password") do |v|
-          options[:password] = v
-        end
-
-        opts.on("-l", "--[no-]logging",
-                "Enable/disable logging to STDERR") do |v|
-          options[:logging] = v
-        end
-
-        opts.on("-s", "--space-id SPACE_ID",
-                "DuraCloud space ID") do |v|
-          options[:space_id] = v
-        end
-
-        opts.on("-i", "--store-id STORE_ID",
-                "DuraCloud store ID") do |v|
-          options[:store_id] = v
-        end
-
-        opts.on("-c", "--content-id CONTENT_ID",
-                "DuraCloud content ID") do |v|
-          options[:content_id] = v
-        end
-
-        opts.on("-m", "--md5 MD5",
-                "MD5 digest of content to store or retrieve") do |v|
-          options[:md5] = v
-        end
-
-        opts.on("-b", "--bagit",
-                "Get manifest in BAGIT format (default is TSV)") do
-          options[:format] = Manifest::BAGIT_FORMAT
-        end
-
-        opts.on("-d", "--content-dir CONTENT_DIR",
-                "Local content directory") do |v|
-          options[:content_dir] = v
-        end
-
-        opts.on("-f", "--infile FILE",
-                "Input file") do |v|
-          options[:infile] = v
-        end
-      end
-
-      command = args.shift if COMMANDS.include?(args.first)
-      parser.parse!(args)
-
-      cli = new(options)
+      options = CommandOptions.new(*args)
+      cli = new(options) # .merge(command: command))
       if cli.invalid?
         message = cli.errors.map { |k, v| "ERROR: #{v}" }.join("\n")
         raise CommandError, message
       end
-      cli.execute(command)
+      cli.execute
     rescue => e
       error!(e)
     end
 
-    def execute(command)
+    def execute
       configure_client
-      send(command).call(self)
-    end
-
-    protected
-
-    def sync
-      Commands::Sync
-    end
-
-    def validate
-      Commands::Validate
-    end
-
-    def manifest
-      Commands::DownloadManifest
-    end
-
-    def properties
-      Commands::GetProperties
+      send(command, self)
     end
 
     private
@@ -157,5 +81,3 @@ def configure_client
 
   end
 end
-
-Dir[File.expand_path("../commands/*.rb", __FILE__)].each { |m| require m }

lib/duracloud/command_options.rb

@@ -0,0 +1,120 @@
+require 'optparse'
+require 'hashie'
+
+module Duracloud
+  class CommandOptions < Hashie::Mash
+
+    def initialize(*args)
+      super()
+      self.command = args.shift if CLI::COMMANDS.include?(args.first)
+      parser.parse!(args)
+    end
+
+    def print_version
+      puts "duracloud-client #{Duracloud::VERSION}"
+    end
+
+    def parser
+      OptionParser.new do |opts|
+        opts.banner = CLI::USAGE
+
+        opts.on("-h", "--help",
+                "Prints help") do
+          print_version
+          puts opts
+          exit
+        end
+
+        opts.on("-H", "--host HOST",
+                "DuraCloud host") do |v|
+          self.host = v
+        end
+
+        opts.on("-P", "--port PORT",
+                "DuraCloud port") do |v|
+          self.port = v
+        end
+
+        opts.on("-u", "--user USER",
+                "DuraCloud user") do |v|
+          self.user = v
+        end
+
+        opts.on("-p", "--password PASSWORD",
+                "DuraCloud password") do |v|
+          self.password = v
+        end
+
+        opts.on("-l", "--[no-]logging",
+                "Enable/disable logging to STDERR") do |v|
+          self.logging = v
+        end
+
+        opts.on("-s", "--space-id SPACE_ID",
+                "DuraCloud space ID") do |v|
+          self.space_id = v
+        end
+
+        opts.on("-i", "--store-id STORE_ID",
+                "DuraCloud store ID") do |v|
+          self.store_id = v
+        end
+
+        opts.on("-c", "--content-id CONTENT_ID",
+                "DuraCloud content ID") do |v|
+          self.content_id = v
+        end
+
+        opts.on("-m", "--md5 MD5",
+                "MD5 digest of content to store or retrieve") do |v|
+          self.md5 = v
+        end
+
+        opts.on("-b", "--bagit",
+                "Get manifest in BAGIT format (default is TSV)") do
+          self.format = Manifest::BAGIT_FORMAT
+        end
+
+        opts.on("-d", "--content-dir DIR",
+                "Local content directory") do |v|
+          self.content_dir = v
+        end
+
+        opts.on("-f", "--infile FILE",
+                "Input file") do |v|
+          self.infile = v
+        end
+
+        opts.on("-v", "--version",
+                "Print version and exit") do |v|
+          print_version
+          exit
+        end
+
+        opts.on("-w", "--work-dir DIR",
+                "Working directory") do |v|
+          self.work_dir = v
+        end
+
+        opts.on("-F", "--[no-]fast-audit",
+                "Use fast audit for sync validation") do |v|
+          self.fast = v
+        end
+
+        opts.on("-a", "--prefix PREFIX",
+                "Content prefix") do |v|
+          self.prefix = v
+        end
+
+        opts.on("-M", "--[no-]missing", "Find missing items") do |v|
+          self.missing = v
+        end
+
+        opts.on("--[no-]all-spaces", "Get report for all spaces") do |v|
+          self.all_spaces = v
+        end
+      end
+    end
+
+  end
+end

lib/duracloud/commands.rb

@@ -0,0 +1,40 @@
+module Duracloud
+  module Commands
+
+    def find(cli)
+      Find.call(cli)
+    end
+
+    def count(cli)
+      Count.call(cli)
+    end
+
+    def get_storage_report(cli)
+      GetStorageReport.call(cli)
+    end
+
+    def sync(cli)
+      Sync.call(cli)
+    end
+
+    def validate(cli)
+      Validate.call(cli)
+    end
+
+    def download_manifest(cli)
+      DownloadManifest.call(cli)
+    end
+
+    def list_content_ids(cli)
+      ListContentIds.call(cli)
+    end
+
+    def list_items(cli)
+      ListItems.call(cli)
+    end
+
+  end
+end
+
+require 'duracloud/commands/command'
+Dir[File.expand_path("../commands/*.rb", __FILE__)].each { |m| require m }