From 6209fe8d7e3ad8f1a0e237ead3088d11bf58ebe7 Mon Sep 17 00:00:00 2001 From: David Chandek-Stark Date: Wed, 26 Jul 2017 21:22:32 -0400 Subject: [PATCH 01/15] Bumped version to 0.9.0.pre --- lib/duracloud/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/duracloud/version.rb b/lib/duracloud/version.rb index b498b2c..a3cc6f6 100644 --- a/lib/duracloud/version.rb +++ b/lib/duracloud/version.rb @@ -1,3 +1,3 @@ module Duracloud - VERSION = "0.8.0" + VERSION = "0.9.0.pre" end From 478540827408996516f24518d29caf2871103e0c Mon Sep 17 00:00:00 2001 From: David Chandek-Stark Date: Thu, 27 Jul 2017 14:09:48 -0400 Subject: [PATCH 02/15] Adds -v/--version option to CLI (fixes #40). --- lib/duracloud/cli.rb | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/duracloud/cli.rb b/lib/duracloud/cli.rb index b7e4aef..3d5d036 100644 --- a/lib/duracloud/cli.rb +++ b/lib/duracloud/cli.rb @@ -25,6 +25,10 @@ class CLI validates_presence_of :space_id, message: "-s/--space-id option is required." + def self.print_version + puts "duracloud-client #{Duracloud::VERSION}" + end + def self.error!(exception) $stderr.puts exception.message if [ CommandError, OptionParser::ParseError ].include?(exception.class) @@ -41,6 +45,7 @@ def self.call(*args) opts.on("-h", "--help", "Prints help") do + print_version puts opts exit end @@ -104,6 +109,12 @@ def self.call(*args) "Input file") do |v| options[:infile] = v end + + opts.on("-v", "--version", + "Print version and exit") do |v| + print_version + exit + end end command = args.shift if COMMANDS.include?(args.first) From aa783ed94af0ed3290f3c1df9ee8d745d45a1dc7 Mon Sep 17 00:00:00 2001 From: David Chandek-Stark Date: Tue, 6 Jun 2017 21:53:34 -0400 Subject: [PATCH 03/15] Implements storage report APIs (closes #31). Adds `storage` command to bin/duracloud. This command gets the latest report by space (if space_id is given) or store (otherwise) and prints to stdout. --- lib/duracloud.rb | 2 + lib/duracloud/cli.rb | 15 ++++-- lib/duracloud/commands/get_storage_report.rb | 17 +++++++ lib/duracloud/rest_methods.rb | 15 ++++++ lib/duracloud/storage_report.rb | 33 +++++++++++++ lib/duracloud/storage_reports.rb | 52 ++++++++++++++++++++ spec/unit/cli_spec.rb | 28 ++++++----- spec/unit/client_spec.rb | 24 +++++++++ spec/unit/storage_report_spec.rb | 15 ++++++ spec/unit/storage_reports_spec.rb | 45 +++++++++++++++++ 10 files changed, 229 insertions(+), 17 deletions(-) create mode 100644 lib/duracloud/commands/get_storage_report.rb create mode 100644 lib/duracloud/storage_report.rb create mode 100644 lib/duracloud/storage_reports.rb create mode 100644 spec/unit/storage_report_spec.rb create mode 100644 spec/unit/storage_reports_spec.rb diff --git a/lib/duracloud.rb b/lib/duracloud.rb index d582516..b7a1e6f 100644 --- a/lib/duracloud.rb +++ b/lib/duracloud.rb @@ -24,6 +24,8 @@ module Duracloud autoload :Space, "duracloud/space" autoload :SpaceAcls, "duracloud/space_acls" autoload :Store, "duracloud/store" + autoload :StorageReport, "duracloud/storage_report" + autoload :StorageReports, "duracloud/storage_reports" autoload :SyncValidation, "duracloud/sync_validation" autoload :TSV, "duracloud/tsv" end diff --git a/lib/duracloud/cli.rb b/lib/duracloud/cli.rb index 3d5d036..1f0eb15 100644 --- a/lib/duracloud/cli.rb +++ b/lib/duracloud/cli.rb @@ -5,7 +5,7 @@ module Duracloud class CLI include ActiveModel::Model - COMMANDS = %w( sync validate manifest properties ) + COMMANDS = %w( sync validate manifest properties storage ) USAGE = <<-EOS Usage: duracloud [COMMAND] [options] @@ -23,7 +23,8 @@ class CLI :content_dir, :format, :infile, :logging - validates_presence_of :space_id, message: "-s/--space-id option is required." + validates_presence_of :space_id, message: "-s/--space-id option is required.", unless: "command == 'storage'" + validates_inclusion_of :command, in: COMMANDS def self.print_version puts "duracloud-client #{Duracloud::VERSION}" @@ -120,23 +121,27 @@ def self.call(*args) command = args.shift if COMMANDS.include?(args.first) parser.parse!(args) - cli = new(options) + cli = new(options.merge(command: command)) if cli.invalid? message = cli.errors.map { |k, v| "ERROR: #{v}" }.join("\n") raise CommandError, message end - cli.execute(command) + cli.execute rescue => e error!(e) end - def execute(command) + def execute configure_client send(command).call(self) end protected + def storage + Commands::GetStorageReport + end + def sync Commands::Sync end diff --git a/lib/duracloud/commands/get_storage_report.rb b/lib/duracloud/commands/get_storage_report.rb new file mode 100644 index 0000000..ba9c291 --- /dev/null +++ b/lib/duracloud/commands/get_storage_report.rb @@ -0,0 +1,17 @@ +require_relative "command" + +module Duracloud::Commands + class GetStorageReport < Command + + def call + reports = if space_id + Duracloud::StorageReports.by_space(space_id, store_id: store_id) + else + Duracloud::StorageReports.by_store(store_id: store_id) + end + report = reports.last + puts report.to_s + end + + end +end diff --git a/lib/duracloud/rest_methods.rb b/lib/duracloud/rest_methods.rb index a31b411..163d193 100644 --- a/lib/duracloud/rest_methods.rb +++ b/lib/duracloud/rest_methods.rb @@ -108,6 +108,21 @@ def perform_task(task_name, **query) "The API method 'Perform Task' has not been implemented." end + # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetStorageReportsbySpace + def get_storage_reports_by_space(space_id, **query) + durastore(:get, "report/space/#{space_id}", **query) + end + + # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetStorageReportsbyStore + def get_storage_reports_by_store(**query) + durastore(:get, "report/store", **query) + end + + # @see https://wiki.duraspace.org/display/DURACLOUDDOC/DuraCloud+REST+API#DuraCloudRESTAPI-GetStorageReportsforallSpacesinaStore(inasingleday) + def get_storage_reports_for_all_spaces_in_a_store(epoch_ms, **query) + durastore(:get, "report/store/#{epoch_ms}", **query) + end + private def durastore(*args, &block) diff --git a/lib/duracloud/storage_report.rb b/lib/duracloud/storage_report.rb new file mode 100644 index 0000000..1e66e7a --- /dev/null +++ b/lib/duracloud/storage_report.rb @@ -0,0 +1,33 @@ +require 'hashie' +require 'active_support' + +module Duracloud + class StorageReport < Hashie::Trash + + property "space_id", from: "spaceId" + property "store_id", from: "storeId" + property "byte_count", from: "byteCount" + property "object_count", from: "objectCount" + property "account_id", from: "accountId" + property "timestamp" + + def time + @time ||= Time.at(timestamp / 1000.0).utc + end + + def human_size + ActiveSupport::NumberHelper.number_to_human_size(byte_count, prefix: :si) + end + + def to_s + <<-EOS +Date: #{time} +Space ID: #{space_id || "(all)"} +Store ID: #{store_id} +Objects: #{object_count} +Total size: #{human_size} (#{byte_count} bytes) + EOS + end + + end +end diff --git a/lib/duracloud/storage_reports.rb b/lib/duracloud/storage_reports.rb new file mode 100644 index 0000000..94a19ec --- /dev/null +++ b/lib/duracloud/storage_reports.rb @@ -0,0 +1,52 @@ +require 'json' +require 'hashie' + +module Duracloud + class StorageReports + include Enumerable + extend Forwardable + + delegate :last => :to_a + + attr_reader :data + + def self.by_space(space_id, **query) + params = Params.new(query) + response = Client.get_storage_reports_by_space(space_id, **params) + new(response) + end + + def self.by_store(**query) + params = Params.new(query) + response = Client.get_storage_reports_by_store(**params) + new(response) + end + + def self.for_all_spaces_in_a_store(epoch_ms = nil, **query) + epoch_ms ||= Time.now.to_i * 1000 + params = Params.new(query) + response = Client.get_storage_reports_for_all_spaces_in_a_store(epoch_ms, **params) + new(response) + end + + def initialize(response) + @data = JSON.parse(response.body) + end + + def each + data.each do |report| + yield StorageReport.new(report) + end + end + + private + + class Params < Hashie::Trash + property :storeID, from: :store_id + property :groupBy, from: :group_by + property :start + property :end + end + + end +end diff --git a/spec/unit/cli_spec.rb b/spec/unit/cli_spec.rb index 6012b92..2604db0 100644 --- a/spec/unit/cli_spec.rb +++ b/spec/unit/cli_spec.rb @@ -4,38 +4,42 @@ module Duracloud subject { described_class.new(**opts) } describe "properties" do - let(:opts) { {space_id: "foo", content_id: "bar"} } - let(:command) { "properties" } + let(:opts) { {command: "properties", space_id: "foo", content_id: "bar"} } specify { expect(Commands::GetProperties).to receive(:call).with(subject) { nil } - subject.execute(command) + subject.execute } end describe "sync" do - let(:opts) { {space_id: "foo", content_id: "bar", infile: "foo/bar"} } - let(:command) { "sync" } + let(:opts) { {command: "sync", space_id: "foo", content_id: "bar", infile: "foo/bar"} } specify { expect(Commands::Sync).to receive(:call).with(subject) { nil } - subject.execute(command) + subject.execute } end describe "validate" do - let(:opts) { {space_id: "foo", content_dir: "/tmp"} } - let(:command) { "validate" } + let(:opts) { {command: "validate", space_id: "foo", content_dir: "/tmp"} } specify { expect(Commands::Validate).to receive(:call).with(subject) { nil } - subject.execute(command) + subject.execute } end describe "manifest" do - let(:opts) { {space_id: "foo"} } - let(:command) { "manifest" } + let(:opts) { {command: "manifest", space_id: "foo"} } specify { expect(Commands::DownloadManifest).to receive(:call).with(subject) { nil } - subject.execute(command) + subject.execute + } + end + + describe "storage" do + let(:opts) { {command: "storage", space_id: "foo"} } + specify { + expect(Commands::GetStorageReport).to receive(:call).with(subject) { nil } + subject.execute } end diff --git a/spec/unit/client_spec.rb b/spec/unit/client_spec.rb index 9381905..1b39601 100644 --- a/spec/unit/client_spec.rb +++ b/spec/unit/client_spec.rb @@ -329,5 +329,29 @@ module Duracloud .to raise_error(NotImplementedError) } end + + describe "get_storage_reports_by_space" do + specify { + stub = stub_request(:get, "https://example.com/durastore/report/space/foo") + subject.get_storage_reports_by_space("foo") + expect(stub).to have_been_requested + } + end + + describe "get_storage_reports_by_store" do + specify { + stub = stub_request(:get, "https://example.com/durastore/report/store") + subject.get_storage_reports_by_store + expect(stub).to have_been_requested + } + end + + describe "get_storage_reports_for_all_spaces_in_a_store" do + specify { + stub = stub_request(:get, "https://example.com/durastore/report/store/1499400000000") + subject.get_storage_reports_for_all_spaces_in_a_store(1499400000000) + expect(stub).to have_been_requested + } + end end end diff --git a/spec/unit/storage_report_spec.rb b/spec/unit/storage_report_spec.rb new file mode 100644 index 0000000..93380cb --- /dev/null +++ b/spec/unit/storage_report_spec.rb @@ -0,0 +1,15 @@ +module Duracloud + RSpec.describe StorageReport do + + subject { described_class.new("timestamp"=>1312588800000,"accountId"=>"account1","spaceId"=>"space1","storeId"=>"store1","byteCount"=>1000,"objectCount"=>10) } + + its(:space_id) { is_expected.to eq "space1" } + its(:store_id) { is_expected.to eq "store1" } + its(:account_id) { is_expected.to eq "account1" } + its(:byte_count) { is_expected.to eq 1000 } + its(:object_count) { is_expected.to eq 10 } + its(:timestamp) { is_expected.to eq 1312588800000 } + its(:time) { is_expected.to eq Time.at(1312588800000 / 1000.0) } + + end +end diff --git a/spec/unit/storage_reports_spec.rb b/spec/unit/storage_reports_spec.rb new file mode 100644 index 0000000..6174de1 --- /dev/null +++ b/spec/unit/storage_reports_spec.rb @@ -0,0 +1,45 @@ +module Duracloud + RSpec.describe StorageReports do + + describe ".by_space" do + subject { described_class.by_space("foo") } + specify { + stub_request(:get, "https://example.com/durastore/report/space/foo") + .to_return(body: '[ + {"timestamp":1312588800000,"accountId":"","spaceId":"","storeId":"","byteCount":1000,"objectCount":10}, + {"timestamp":1315008000000,"accountId":"","spaceId":"","storeId":"","byteCount":1000,"objectCount":10}, + {"timestamp":1315526400000,"accountId":"","spaceId":"","storeId":"","byteCount":1000,"objectCount":10} +]') + expect(subject.map(&:timestamp)).to eq [ 1312588800000, 1315008000000, 1315526400000 ] + } + end + + describe ".by_store" do + subject { described_class.by_store } + specify { + stub_request(:get, "https://example.com/durastore/report/store") + .to_return(body: '[ + {"timestamp":1312588800000,"accountId":"","storeId":"","byteCount":1000,"objectCount":10}, + {"timestamp":1315008000000,"accountId":"","storeId":"","byteCount":1000,"objectCount":10}, + {"timestamp":1315526400000,"accountId":"","storeId":"","byteCount":1000,"objectCount":10} +]') + expect(subject.map(&:timestamp)).to eq [ 1312588800000, 1315008000000, 1315526400000 ] + } + end + + describe ".for_all_spaces_in_a_store" do + subject { described_class.for_all_spaces_in_a_store(1499400000000) } + specify { + stub_request(:get, "https://example.com/durastore/report/store/1499400000000") + .to_return(body: '[ + {"timestamp":1312588800000,"accountId":"","spaceId":"","storeId":"","byteCount":1000,"objectCount":10}, + {"timestamp":1315008000000,"accountId":"","spaceId":"","storeId":"","byteCount":1000,"objectCount":10}, + {"timestamp":1315526400000,"accountId":"","spaceId":"","storeId":"","byteCount":1000,"objectCount":10} +]') + expect(subject.map(&:timestamp)).to eq [ 1312588800000, 1315008000000, 1315526400000 ] + } + end + + + end +end From 5faa05ca95f18da21e4e269f5fa5d7bc1897180e Mon Sep 17 00:00:00 2001 From: David Chandek-Stark Date: Fri, 28 Jul 2017 09:53:23 -0400 Subject: [PATCH 04/15] SyncValidation refactored to accept a work_dir in which to write files. The default behavior is to write to a temp directory which is removed on completion of the process. If a work_dir is provided, the files are not removed on completion. Added -w/--work-dir option to CLI. --- lib/duracloud/cli.rb | 9 ++- lib/duracloud/commands/command.rb | 4 +- lib/duracloud/commands/validate.rb | 2 +- lib/duracloud/sync_validation.rb | 117 ++++++++++++++++++++--------- 4 files changed, 90 insertions(+), 42 deletions(-) diff --git a/lib/duracloud/cli.rb b/lib/duracloud/cli.rb index 1f0eb15..800bb00 100644 --- a/lib/duracloud/cli.rb +++ b/lib/duracloud/cli.rb @@ -20,7 +20,7 @@ class CLI attr_accessor :command, :user, :password, :host, :port, :space_id, :store_id, :content_id, :content_type, :md5, - :content_dir, :format, :infile, + :content_dir, :format, :infile, :work_dir, :logging validates_presence_of :space_id, message: "-s/--space-id option is required.", unless: "command == 'storage'" @@ -101,7 +101,7 @@ def self.call(*args) options[:format] = Manifest::BAGIT_FORMAT end - opts.on("-d", "--content-dir CONTENT_DIR", + opts.on("-d", "--content-dir DIR", "Local content directory") do |v| options[:content_dir] = v end @@ -116,6 +116,11 @@ def self.call(*args) print_version exit end + + opts.on("-w", "--work-dir DIR", + "Working directory") do |v| + options[:work_dir] = v + end end command = args.shift if COMMANDS.include?(args.first) diff --git a/lib/duracloud/commands/command.rb b/lib/duracloud/commands/command.rb index 012abaf..4c1cb4a 100644 --- a/lib/duracloud/commands/command.rb +++ b/lib/duracloud/commands/command.rb @@ -3,8 +3,8 @@ module Duracloud::Commands class Command < SimpleDelegator - def self.call(command) - new(command).call + def self.call(cli) + new(cli).call end end diff --git a/lib/duracloud/commands/validate.rb b/lib/duracloud/commands/validate.rb index d898146..3719aec 100644 --- a/lib/duracloud/commands/validate.rb +++ b/lib/duracloud/commands/validate.rb @@ -4,7 +4,7 @@ module Duracloud::Commands class Validate < Command def call - Duracloud::SyncValidation.call(space_id: space_id, store_id: store_id, content_dir: content_dir) + Duracloud::SyncValidation.call(space_id: space_id, store_id: store_id, content_dir: content_dir, work_dir: work_dir) end end diff --git a/lib/duracloud/sync_validation.rb b/lib/duracloud/sync_validation.rb index 80b8009..11d89e1 100644 --- a/lib/duracloud/sync_validation.rb +++ b/lib/duracloud/sync_validation.rb @@ -1,6 +1,7 @@ require 'active_model' require 'tempfile' require 'csv' +require 'fileutils' module Duracloud class SyncValidation @@ -10,56 +11,98 @@ class SyncValidation MD5_CSV_OPTS = { col_sep: TWO_SPACES }.freeze MANIFEST_CSV_OPTS = { col_sep: "\t", headers: true, return_headers: false }.freeze - attr_accessor :space_id, :content_dir, :store_id + MISSING = "MISSING" + CHANGED = "CHANGED" + FOUND = "FOUND" + + attr_accessor :space_id, :content_dir, :store_id, :work_dir def self.call(*args) new(*args).call end + def in_work_dir + if work_dir + FileUtils.cd(work_dir) { yield } + else + Dir.mktmpdir("#{space_id}-validation-") do |tmpdir| + FileUtils.cd(tmpdir) { yield } + end + end + end + def call - Tempfile.open("#{space_id}-manifest") do |manifest| + in_work_dir do + download_manifest + convert_manifest + audit + end + end + + def download_manifest + File.open(manifest_filename, "w") do |manifest| Manifest.download(space_id, store_id) do |chunk| manifest.write(chunk) end - manifest.close + end + end + + def convert_manifest + File.open(md5_filename, "w") do |f| + CSV.foreach(manifest_filename, MANIFEST_CSV_OPTS) do |row| + f.puts [ row[2], row[1] ].join(TWO_SPACES) + end + end + end - # convert manifest into md5deep format - Tempfile.open("#{space_id}-md5") do |md5_list| - CSV.foreach(manifest.path, MANIFEST_CSV_OPTS) do |row| - md5_list.puts [ row[2], row[1] ].join(TWO_SPACES) - end - md5_list.close + def audit + outfile = File.join(FileUtils.pwd, audit_filename) + infile = File.join(FileUtils.pwd, md5_filename) + pid = spawn("md5deep", "-X", infile, "-l", "-r", ".", chdir: content_dir, out: outfile) + Process.wait(pid) + case $?.exitstatus + when 0 + true + when 1, 2 + recheck_failures + when 64, 128 + raise Error, "md5deep error." + else + raise Error, "Unknown error." + end + end - # run md5deep to find files not listed in the manifest - Tempfile.open("#{space_id}-audit") do |audit| - audit.close - pid = spawn("md5deep", "-X", md5_list.path, "-l", "-r", ".", chdir: content_dir, out: audit.path) - Process.wait(pid) - case $?.exitstatus - when 0 - true - when 1, 2 - failures = [] - CSV.foreach(audit.path, MD5_CSV_OPTS) do |md5, path| - content_id = path.sub(/^\.\//, "") - begin - if !Duracloud::Content.exist?(space_id: space_id, store_id: store_id, content_id: content_id, md5: md5) - failures << [ "MISSING", md5, content_id ].join("\t") - end - rescue MessageDigestError => e - failures << [ "CHANGED", md5, content_id ].join("\t") - end - end - STDOUT.puts failures - failures.empty? - when 64 - raise Error, "md5deep user error." - when 128 - raise Error, "md5deep internal error." - end - end + def recheck_failures + success = true + CSV($stdout, col_sep: "\t") do |output| + CSV.foreach(audit_filename, MD5_CSV_OPTS) do |md5, path| + content_id = path.sub(/^\.\//, "") + status = begin + if Duracloud::Content.exist?(space_id: space_id, store_id: store_id, content_id: content_id, md5: md5) + FOUND + else + MISSING + end + rescue MessageDigestError => e + CHANGED + end + output << [ status, md5, content_id ] + success &&= ( status == FOUND ) end end + success + end + + def manifest_filename + "#{space_id}-manifest.tsv" + end + + def md5_filename + "#{space_id}-md5.txt" + end + + def audit_filename + "#{space_id}-audit.txt" end end From 04ddc12c0e741f4d79ff625c6846aef12e08ec90 Mon Sep 17 00:00:00 2001 From: David Chandek-Stark Date: Fri, 28 Jul 2017 13:41:47 -0400 Subject: [PATCH 05/15] Adds `Duracloud::FastSyncValidation`. Fast validation only compares the local list of relative file paths to the list of content IDs from the DuraCloud space. MD5 checksums are not compared (md5deep is not used). The rechecking phase is the same, albeit without MD5 verification. --- README.md | 15 ++++- lib/duracloud.rb | 1 + lib/duracloud/cli.rb | 7 ++- lib/duracloud/commands/validate.rb | 3 +- lib/duracloud/fast_sync_validation.rb | 42 ++++++++++++++ lib/duracloud/sync_validation.rb | 84 ++++++++++++++++++++------- 6 files changed, 127 insertions(+), 25 deletions(-) create mode 100644 lib/duracloud/fast_sync_validation.rb diff --git a/README.md b/README.md index 2e9e6b7..3382c75 100644 --- a/README.md +++ b/README.md @@ -140,12 +140,25 @@ Assumptions: Process: - The space manifest is downloaded from DuraCloud and converted to the expected input format for `md5deep` (two columns: md5 hash and file path, separated by two spaces). - `md5deep` is run against the content directory in "non-matching" mode (-X) with the converted manifest as the list of "known hashes". -- Non-matching files from the `md5deep` run are re-checked individually by calling `Duracloud::Content.exist?`. This pass will account for content sync after `md5deep` started as well as files that have been chunked in DuraCloud. +- Non-matching files from the `md5deep` run are re-checked individually by calling `Duracloud::Content.exist?`. +This pass will account for content sync after `md5deep` started as well as files that have been chunked in DuraCloud. +Results of this re-check are printed to STDOUT (or, if using the `:work_dir` option, below, to a file). ```ruby Duracloud::SyncValidation.call(space_id: 'foo', content_dir: '/var/foo/bar') ``` +*Added in version 0.9.0* - `:work_dir` option to specify existing directory in which to write validation files. The default behavior is to +create a temporary directory which is deleted on completion of the process. If `:work_dir` is specified, no cleanup is performed. + +Files created in work directory: +- `{SPACE_ID}-manifest.tsv` (DuraCloud manifest as downloaded) +- `{SPACE_ID}-md5.txt` (Munged manifest for md5deep) +- `{SPACE_ID}-audit.txt` (Output of md5deep, empty if all files match) +- `{SPACE_ID}-recheck.txt` (Out of audit recheck, if necessary) + +*Added in version 0.9.0* - `Duracloud::FastSyncValidation`. This variant of sync validation does not compute local hashes but instead compares the local file list (generated with `find`) to the list of content IDs in the space manifest. Local misses are rechecked as in `SyncValidation` (but without MD5 comparison). + ### Content #### Create a new content item and store it in DuraCloud diff --git a/lib/duracloud.rb b/lib/duracloud.rb index b7a1e6f..9b20cc1 100644 --- a/lib/duracloud.rb +++ b/lib/duracloud.rb @@ -14,6 +14,7 @@ module Duracloud autoload :ContentManifest, "duracloud/content_manifest" autoload :DurastoreRequest, "duracloud/durastore_request" autoload :ErrorHandler, "duracloud/error_handler" + autoload :FastSyncValidation, "duracloud/fast_sync_validation" autoload :HasProperties, "duracloud/has_properties" autoload :Manifest, "duracloud/manifest" autoload :Persistence, "duracloud/persistence" diff --git a/lib/duracloud/cli.rb b/lib/duracloud/cli.rb index 800bb00..a8f53e9 100644 --- a/lib/duracloud/cli.rb +++ b/lib/duracloud/cli.rb @@ -20,7 +20,7 @@ class CLI attr_accessor :command, :user, :password, :host, :port, :space_id, :store_id, :content_id, :content_type, :md5, - :content_dir, :format, :infile, :work_dir, + :content_dir, :format, :infile, :work_dir, :fast, :logging validates_presence_of :space_id, message: "-s/--space-id option is required.", unless: "command == 'storage'" @@ -121,6 +121,11 @@ def self.call(*args) "Working directory") do |v| options[:work_dir] = v end + + opts.on("-F", "--[no-]fast", + "Use fast audit for sync validation") do |v| + options[:fast] = v + end end command = args.shift if COMMANDS.include?(args.first) diff --git a/lib/duracloud/commands/validate.rb b/lib/duracloud/commands/validate.rb index 3719aec..a2773a8 100644 --- a/lib/duracloud/commands/validate.rb +++ b/lib/duracloud/commands/validate.rb @@ -4,7 +4,8 @@ module Duracloud::Commands class Validate < Command def call - Duracloud::SyncValidation.call(space_id: space_id, store_id: store_id, content_dir: content_dir, work_dir: work_dir) + klass = fast ? Duracloud::FastSyncValidation : DuracloudSyncValidation + klass.call(space_id: space_id, store_id: store_id, content_dir: content_dir, work_dir: work_dir) end end diff --git a/lib/duracloud/fast_sync_validation.rb b/lib/duracloud/fast_sync_validation.rb new file mode 100644 index 0000000..b4ea79a --- /dev/null +++ b/lib/duracloud/fast_sync_validation.rb @@ -0,0 +1,42 @@ +module Duracloud + class FastSyncValidation < SyncValidation + + def convert_manifest + # content-id is the 2nd column of the manifest + system("cut -f 2 #{manifest_filename} | sort", out: converted_manifest_filename) + end + + def audit + find_files + if system("comm", "-23", find_filename, converted_manifest_filename, out: audit_filename) + File.empty?(audit_filename) || recheck + else + raise Error, "Error comparing #{find_filename} with #{converted_manifest_filename}." + end + end + + def find_files + # TODO handle exclude file? + outfile = File.join(FileUtils.pwd, find_filename) + # Using a separate command for sort so we get find results incrementally + system("find -L . -type f | sed -e 's|^\./||'", chdir: content_dir, out: outfile) && + system("sort", "-o", find_filename, find_filename) + end + + private + + def do_recheck + Enumerator.new do |e| + File.foreach(audit_filename) do |line| + content_id = line.chomp + e << check(content_id) + end + end + end + + def find_filename + filename("find.txt") + end + + end +end diff --git a/lib/duracloud/sync_validation.rb b/lib/duracloud/sync_validation.rb index 11d89e1..dddf21f 100644 --- a/lib/duracloud/sync_validation.rb +++ b/lib/duracloud/sync_validation.rb @@ -15,7 +15,7 @@ class SyncValidation CHANGED = "CHANGED" FOUND = "FOUND" - attr_accessor :space_id, :content_dir, :store_id, :work_dir + attr_accessor :space_id, :content_dir, :store_id, :work_dir, :fast def self.call(*args) new(*args).call @@ -48,7 +48,7 @@ def download_manifest end def convert_manifest - File.open(md5_filename, "w") do |f| + File.open(converted_manifest_filename, "w") do |f| CSV.foreach(manifest_filename, MANIFEST_CSV_OPTS) do |row| f.puts [ row[2], row[1] ].join(TWO_SPACES) end @@ -57,14 +57,14 @@ def convert_manifest def audit outfile = File.join(FileUtils.pwd, audit_filename) - infile = File.join(FileUtils.pwd, md5_filename) + infile = File.join(FileUtils.pwd, converted_manifest_filename) pid = spawn("md5deep", "-X", infile, "-l", "-r", ".", chdir: content_dir, out: outfile) Process.wait(pid) case $?.exitstatus when 0 true when 1, 2 - recheck_failures + recheck when 64, 128 raise Error, "md5deep error." else @@ -72,37 +72,77 @@ def audit end end - def recheck_failures + def recheck success = true - CSV($stdout, col_sep: "\t") do |output| + recheck_file do |csv| + do_recheck.each do |result| + csv << result.to_a + success &&= result.found? + end + end + success + end + + private + + CheckResult = Struct.new(:status, :md5, :content_id) do + def found? + status == FOUND + end + end + + def recheck_file + if work_dir + CSV.open(recheck_filename, "w", col_sep: "\t") { |csv| yield(csv) } + else + CSV($stdout, col_sep: "\t") { |csv| yield(csv) } + end + end + + def check(content_id, md5 = nil) + status = begin + exist?(content_id, md5) ? FOUND : MISSING + rescue MessageDigestError => e + CHANGED + end + CheckResult.new(status, md5 || "-", content_id) + end + + def exist?(content_id, md5 = nil) + Duracloud::Content.exist?(space_id: space_id, store_id: store_id, content_id: content_id, md5: md5) + end + + def do_recheck + Enumerator.new do |e| CSV.foreach(audit_filename, MD5_CSV_OPTS) do |md5, path| content_id = path.sub(/^\.\//, "") - status = begin - if Duracloud::Content.exist?(space_id: space_id, store_id: store_id, content_id: content_id, md5: md5) - FOUND - else - MISSING - end - rescue MessageDigestError => e - CHANGED - end - output << [ status, md5, content_id ] - success &&= ( status == FOUND ) + e << check(content_id, md5) end end - success + end + + def prefix + space_id + end + + def filename(suffix) + [ prefix, suffix ].join("-") end def manifest_filename - "#{space_id}-manifest.tsv" + filename("manifest.tsv") end - def md5_filename - "#{space_id}-md5.txt" + def converted_manifest_filename + filename("converted-manifest.txt") end def audit_filename - "#{space_id}-audit.txt" + filename("audit.txt") + end + + def recheck_filename + filename("recheck.txt") end end From b4685f60c27f09ab3d40762e5be4a45355bedf63 Mon Sep 17 00:00:00 2001 From: David Chandek-Stark Date: Mon, 28 Aug 2017 15:05:26 -0400 Subject: [PATCH 06/15] Added count, ids, and items commands. --- lib/duracloud/cli.rb | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/lib/duracloud/cli.rb b/lib/duracloud/cli.rb index a8f53e9..6278914 100644 --- a/lib/duracloud/cli.rb +++ b/lib/duracloud/cli.rb @@ -5,7 +5,7 @@ module Duracloud class CLI include ActiveModel::Model - COMMANDS = %w( sync validate manifest properties storage ) + COMMANDS = %w( sync validate manifest properties storage ids items count ) USAGE = <<-EOS Usage: duracloud [COMMAND] [options] @@ -19,7 +19,7 @@ class CLI attr_accessor :command, :user, :password, :host, :port, :space_id, :store_id, :content_id, - :content_type, :md5, + :content_type, :md5, :prefix, :content_dir, :format, :infile, :work_dir, :fast, :logging @@ -122,10 +122,15 @@ def self.call(*args) options[:work_dir] = v end - opts.on("-F", "--[no-]fast", + opts.on("-F", "--[no-]fast-audit", "Use fast audit for sync validation") do |v| options[:fast] = v end + + opts.on("-a", "--prefix PREFIX", + "Content prefix") do |v| + options[:prefix] = v + end end command = args.shift if COMMANDS.include?(args.first) @@ -148,6 +153,10 @@ def execute protected + def count + Commands::Count + end + def storage Commands::GetStorageReport end @@ -168,6 +177,14 @@ def properties Commands::GetProperties end + def ids + Commands::ListContentIds + end + + def items + Commands::ListItems + end + private def configure_client From 72e819485a838e9b1420f9432a93467868974d04 Mon Sep 17 00:00:00 2001 From: David Chandek-Stark Date: Mon, 28 Aug 2017 15:52:20 -0400 Subject: [PATCH 07/15] Added files that should have been included in commit 086783353d2839024b0d1cdaa38b04dbb921b543. --- lib/duracloud/commands/count.rb | 17 +++++++++++++++++ lib/duracloud/commands/list_content_ids.rb | 13 +++++++++++++ lib/duracloud/commands/list_items.rb | 18 ++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 lib/duracloud/commands/count.rb create mode 100644 lib/duracloud/commands/list_content_ids.rb create mode 100644 lib/duracloud/commands/list_items.rb diff --git a/lib/duracloud/commands/count.rb b/lib/duracloud/commands/count.rb new file mode 100644 index 0000000..46288d8 --- /dev/null +++ b/lib/duracloud/commands/count.rb @@ -0,0 +1,17 @@ +require_relative "command" + +module Duracloud::Commands + class Count < Command + + def call + space = Duracloud::Space.find(space_id, store_id) + count = if prefix || space.count == 1000 + space.content_ids(prefix: prefix).to_a.length + else + space.count + end + puts count + end + + end +end diff --git a/lib/duracloud/commands/list_content_ids.rb b/lib/duracloud/commands/list_content_ids.rb new file mode 100644 index 0000000..9be6d9f --- /dev/null +++ b/lib/duracloud/commands/list_content_ids.rb @@ -0,0 +1,13 @@ +require_relative "command" + +module Duracloud::Commands + class ListContentIds < Command + + def call + Duracloud::Space.content_ids(space_id, store_id: store_id, prefix: prefix).each do |id| + puts id + end + end + + end +end diff --git a/lib/duracloud/commands/list_items.rb b/lib/duracloud/commands/list_items.rb new file mode 100644 index 0000000..ee56d98 --- /dev/null +++ b/lib/duracloud/commands/list_items.rb @@ -0,0 +1,18 @@ +require_relative "command" +require "csv" + +module Duracloud::Commands + class ListItems < Command + + HEADERS = %i( content_id md5 size content_type modified ) + + def call + ::CSV.instance($stdout, headers: HEADERS, write_headers: true) do |csv| + Duracloud::Space.items(space_id, store_id: store_id, prefix: prefix).each do |item| + csv << HEADERS.map { |header| item.send(header) } + end + end + end + + end +end From 8e9da6f615cfff802f0c85446776ad7a4c2ea7a5 Mon Sep 17 00:00:00 2001 From: David Chandek-Stark Date: Mon, 28 Aug 2017 16:25:07 -0400 Subject: [PATCH 08/15] Sorted CLI accessor names --- lib/duracloud/cli.rb | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/duracloud/cli.rb b/lib/duracloud/cli.rb index 6278914..87789e1 100644 --- a/lib/duracloud/cli.rb +++ b/lib/duracloud/cli.rb @@ -17,11 +17,23 @@ class CLI EOS HELP = "Type 'duracloud -h/--help' for usage." - attr_accessor :command, :user, :password, :host, :port, - :space_id, :store_id, :content_id, - :content_type, :md5, :prefix, - :content_dir, :format, :infile, :work_dir, :fast, - :logging + attr_accessor :command, + :content_dir, + :content_id, + :content_type, + :fast, + :format, + :host, + :infile, + :logging, + :md5, + :password, + :port, + :prefix, + :space_id, + :store_id, + :user, + :work_dir validates_presence_of :space_id, message: "-s/--space-id option is required.", unless: "command == 'storage'" validates_inclusion_of :command, in: COMMANDS From 5447bffc19f13ec891b312873b3e2ac3b9db4bdd Mon Sep 17 00:00:00 2001 From: David Chandek-Stark Date: Mon, 28 Aug 2017 17:50:57 -0400 Subject: [PATCH 09/15] `Content#move` ensures content is verified in destination before deleting source. --- lib/duracloud/content.rb | 4 +++- spec/unit/content_spec.rb | 17 ++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/duracloud/content.rb b/lib/duracloud/content.rb index 9b8f4da..0f194de 100644 --- a/lib/duracloud/content.rb +++ b/lib/duracloud/content.rb @@ -79,6 +79,7 @@ def download(&block) # @return [Duracloud::Content] the copied content # The current instance still represents the original content. + # @raise [Duracloud::Error] def copy(**args) dest = args.except(:force) dest[:space_id] ||= space_id @@ -89,11 +90,12 @@ def copy(**args) end options = { storeID: dest[:store_id], headers: copy_headers } Client.copy_content(dest[:space_id], dest[:content_id], **options) - Content.new(dest.merge(md5: md5)) + Content.find(dest.merge(md5: md5)) end # @return [Duracloud::Content] the moved content # The current instance still represents the deleted content. + # @raise [Duracloud::Error] def move(**args) copied = copy(**args) delete diff --git a/spec/unit/content_spec.rb b/spec/unit/content_spec.rb index e6e5212..681c884 100644 --- a/spec/unit/content_spec.rb +++ b/spec/unit/content_spec.rb @@ -226,8 +226,8 @@ module Duracloud before do stub_request(:put, target) .with(headers: {'x-dura-meta-copy-source'=>'foo/bar'}) - stub_request(:head, target).to_return(status: 404) - stub_request(:head, "#{target}.dura-manifest").to_return(status: 404) + allow(described_class).to receive(:exist?).with(space_id: "spam", content_id: "eggs") { false } + stub_request(:head, target).to_return(status: 200) end specify { copied = subject.copy(space_id: "spam", content_id: "eggs") @@ -237,17 +237,16 @@ module Duracloud target = "https://example.com/durastore/foo/eggs" stub1 = stub_request(:put, target) .with(headers: {'x-dura-meta-copy-source'=>'foo/bar'}) - stub2 = stub_request(:head, target).to_return(status: 404) - stub3 = stub_request(:head, "#{target}.dura-manifest").to_return(status: 404) + allow(described_class).to receive(:exist?).with(space_id: "foo", content_id: "eggs") { false } + stub2 = stub_request(:head, target).to_return(status: 200) copied = subject.copy(content_id: "eggs") expect(copied).to be_a(Content) expect(stub1).to have_been_requested expect(stub2).to have_been_requested - expect(stub3).to have_been_requested end describe "when the target exists" do before do - stub_request(:head, target).to_return(status: 200) + allow(described_class).to receive(:exist?).with(space_id: "spam", content_id: "eggs") { true } end describe "and force argument is true" do it "overwrites the target" do @@ -267,16 +266,15 @@ module Duracloud subject { Content.new(space_id: "foo", content_id: "bar") } describe "when copy succeeds" do it "deletes the source" do + allow(described_class).to receive(:exist?).with(space_id: "spam", content_id: "eggs") { false } stub1 = stub_request(:put, target) .with(headers: {'x-dura-meta-copy-source'=>'foo/bar'}) - stub2 = stub_request(:head, target).to_return(status: 404) - stub2a = stub_request(:head, "#{target}.dura-manifest").to_return(status: 404) + stub2 = stub_request(:head, target).to_return(status: 200) stub3 = stub_request(:delete, "https://example.com/durastore/foo/bar") moved = subject.move(space_id: "spam", content_id: "eggs") expect(moved).to be_a(Content) expect(stub1).to have_been_requested expect(stub2).to have_been_requested - expect(stub2a).to have_been_requested expect(stub3).to have_been_requested end end @@ -289,6 +287,7 @@ module Duracloud end describe "when target exists" do before do + allow(described_class).to receive(:exist?).with(space_id: "spam", content_id: "eggs") { false } stub_request(:put, target) .with(headers: {'x-dura-meta-copy-source'=>'foo/bar'}) stub_request(:delete, "https://example.com/durastore/foo/bar") From a80cdf19db16f30fb35be71b22095e981f31d702 Mon Sep 17 00:00:00 2001 From: David Chandek-Stark Date: Mon, 28 Aug 2017 19:21:32 -0400 Subject: [PATCH 10/15] Revert to using WebMock, now with multiple responses. --- spec/unit/content_spec.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/spec/unit/content_spec.rb b/spec/unit/content_spec.rb index 681c884..b4e6a63 100644 --- a/spec/unit/content_spec.rb +++ b/spec/unit/content_spec.rb @@ -226,8 +226,8 @@ module Duracloud before do stub_request(:put, target) .with(headers: {'x-dura-meta-copy-source'=>'foo/bar'}) - allow(described_class).to receive(:exist?).with(space_id: "spam", content_id: "eggs") { false } - stub_request(:head, target).to_return(status: 200) + stub_request(:head, target).to_return({status: 404}, {status: 200}) + stub_request(:head, "#{target}.dura-manifest").to_return(status: 404) end specify { copied = subject.copy(space_id: "spam", content_id: "eggs") @@ -237,16 +237,17 @@ module Duracloud target = "https://example.com/durastore/foo/eggs" stub1 = stub_request(:put, target) .with(headers: {'x-dura-meta-copy-source'=>'foo/bar'}) - allow(described_class).to receive(:exist?).with(space_id: "foo", content_id: "eggs") { false } - stub2 = stub_request(:head, target).to_return(status: 200) + stub2 = stub_request(:head, target).to_return({status: 404}, {status: 200}) + stub3 = stub_request(:head, "#{target}.dura-manifest").to_return(status: 404) copied = subject.copy(content_id: "eggs") expect(copied).to be_a(Content) expect(stub1).to have_been_requested - expect(stub2).to have_been_requested + expect(stub2).to have_been_requested.twice + expect(stub3).to have_been_requested end describe "when the target exists" do before do - allow(described_class).to receive(:exist?).with(space_id: "spam", content_id: "eggs") { true } + stub_request(:head, target).to_return(status: 200) end describe "and force argument is true" do it "overwrites the target" do From 53e6ce9f8864e06741a6364264122804d478033a Mon Sep 17 00:00:00 2001 From: David Chandek-Stark Date: Tue, 29 Aug 2017 09:37:02 -0400 Subject: [PATCH 11/15] Content copy verifies MD5 of response. --- lib/duracloud/content.rb | 15 +++++++++++---- spec/unit/content_spec.rb | 30 ++++++++++++++++++++---------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/lib/duracloud/content.rb b/lib/duracloud/content.rb index 0f194de..902e6b6 100644 --- a/lib/duracloud/content.rb +++ b/lib/duracloud/content.rb @@ -83,14 +83,21 @@ def download(&block) def copy(**args) dest = args.except(:force) dest[:space_id] ||= space_id + dest[:store_id] ||= store_id dest[:content_id] ||= content_id - raise CopyError, "Destination is the same as the source." if dest == copy_source + if dest == copy_source + raise CopyError, "Destination is the same as the source." + end if !args[:force] && Content.exist?(**dest) - raise CopyError, "Destination exists and :false option is false." + raise CopyError, "Destination exists and `:force' option is false." end options = { storeID: dest[:store_id], headers: copy_headers } - Client.copy_content(dest[:space_id], dest[:content_id], **options) - Content.find(dest.merge(md5: md5)) + response = Client.copy_content(dest[:space_id], dest[:content_id], **options) + if md5 != response.md5 + raise CopyError, "Message digest of copy does not match source " \ + "(source: #{md5}; destination: #{response.md5})" + end + Content.new(dest.merge(md5: md5)) end # @return [Duracloud::Content] the moved content diff --git a/spec/unit/content_spec.rb b/spec/unit/content_spec.rb index b4e6a63..9607bec 100644 --- a/spec/unit/content_spec.rb +++ b/spec/unit/content_spec.rb @@ -221,12 +221,14 @@ module Duracloud end describe "#copy" do - subject { Content.new(space_id: "foo", content_id: "bar") } + subject { Content.new(space_id: "foo", content_id: "bar", md5: "08a008a01d498c404b0c30852b39d3b8") } let(:target) { "https://example.com/durastore/spam/eggs" } before do stub_request(:put, target) .with(headers: {'x-dura-meta-copy-source'=>'foo/bar'}) - stub_request(:head, target).to_return({status: 404}, {status: 200}) + .to_return(status: 201, + headers: {'Content-MD5'=>'08a008a01d498c404b0c30852b39d3b8'}) + stub_request(:head, target).to_return(status: 404) stub_request(:head, "#{target}.dura-manifest").to_return(status: 404) end specify { @@ -237,12 +239,14 @@ module Duracloud target = "https://example.com/durastore/foo/eggs" stub1 = stub_request(:put, target) .with(headers: {'x-dura-meta-copy-source'=>'foo/bar'}) - stub2 = stub_request(:head, target).to_return({status: 404}, {status: 200}) + .to_return(status: 201, + headers: {'Content-MD5'=>'08a008a01d498c404b0c30852b39d3b8'}) + stub2 = stub_request(:head, target).to_return(status: 404) stub3 = stub_request(:head, "#{target}.dura-manifest").to_return(status: 404) copied = subject.copy(content_id: "eggs") expect(copied).to be_a(Content) expect(stub1).to have_been_requested - expect(stub2).to have_been_requested.twice + expect(stub2).to have_been_requested expect(stub3).to have_been_requested end describe "when the target exists" do @@ -264,19 +268,21 @@ module Duracloud describe "#move" do let(:target) { "https://example.com/durastore/spam/eggs" } - subject { Content.new(space_id: "foo", content_id: "bar") } + subject { Content.new(space_id: "foo", content_id: "bar", md5: '08a008a01d498c404b0c30852b39d3b8') } describe "when copy succeeds" do it "deletes the source" do - allow(described_class).to receive(:exist?).with(space_id: "spam", content_id: "eggs") { false } + allow(described_class) + .to receive(:exist?) + .with(space_id: "spam", content_id: "eggs", store_id: nil) { false } stub1 = stub_request(:put, target) .with(headers: {'x-dura-meta-copy-source'=>'foo/bar'}) - stub2 = stub_request(:head, target).to_return(status: 200) - stub3 = stub_request(:delete, "https://example.com/durastore/foo/bar") + .to_return(status: 201, + headers: {'Content-MD5'=>'08a008a01d498c404b0c30852b39d3b8'}) + stub2 = stub_request(:delete, "https://example.com/durastore/foo/bar") moved = subject.move(space_id: "spam", content_id: "eggs") expect(moved).to be_a(Content) expect(stub1).to have_been_requested expect(stub2).to have_been_requested - expect(stub3).to have_been_requested end end describe "when copy fails" do @@ -288,9 +294,13 @@ module Duracloud end describe "when target exists" do before do - allow(described_class).to receive(:exist?).with(space_id: "spam", content_id: "eggs") { false } + allow(described_class) + .to receive(:exist?) + .with(space_id: "spam", content_id: "eggs", store_id: nil) { true } stub_request(:put, target) .with(headers: {'x-dura-meta-copy-source'=>'foo/bar'}) + .to_return(status: 201, + headers: {'Content-MD5'=>'08a008a01d498c404b0c30852b39d3b8'}) stub_request(:delete, "https://example.com/durastore/foo/bar") stub_request(:head, target).to_return(status: 200) allow(Content).to receive(:exist?).with(space_id: "spam", content_id: "eggs") { true } From 3e90fa00991449ec1ba66aabba14479808af9c4a Mon Sep 17 00:00:00 2001 From: David Chandek-Stark Date: Wed, 6 Sep 2017 17:41:33 -0400 Subject: [PATCH 12/15] Refactoring of commands. Adds `find` command to find multiple items. --- lib/duracloud.rb | 1 + lib/duracloud/cli.rb | 41 ++------------- lib/duracloud/commands.rb | 40 +++++++++++++++ lib/duracloud/commands/command.rb | 4 ++ lib/duracloud/commands/count.rb | 2 - lib/duracloud/commands/download_manifest.rb | 2 - lib/duracloud/commands/find.rb | 16 ++++++ lib/duracloud/commands/find_item.rb | 16 ++++++ lib/duracloud/commands/find_items.rb | 22 ++++++++ lib/duracloud/commands/find_space.rb | 12 +++++ lib/duracloud/commands/get_properties.rb | 27 ---------- lib/duracloud/commands/get_storage_report.rb | 2 - lib/duracloud/commands/list_content_ids.rb | 2 - lib/duracloud/commands/list_items.rb | 3 +- lib/duracloud/commands/sync.rb | 2 - lib/duracloud/commands/validate.rb | 2 - spec/unit/cli_spec.rb | 54 +++++++++++++++++--- 17 files changed, 162 insertions(+), 86 deletions(-) create mode 100644 lib/duracloud/commands.rb create mode 100644 lib/duracloud/commands/find.rb create mode 100644 lib/duracloud/commands/find_item.rb create mode 100644 lib/duracloud/commands/find_items.rb create mode 100644 lib/duracloud/commands/find_space.rb delete mode 100644 lib/duracloud/commands/get_properties.rb diff --git a/lib/duracloud.rb b/lib/duracloud.rb index 9b20cc1..28a88a1 100644 --- a/lib/duracloud.rb +++ b/lib/duracloud.rb @@ -8,6 +8,7 @@ module Duracloud autoload :ChunkedContent, "duracloud/chunked_content" autoload :Client, "duracloud/client" autoload :CLI, "duracloud/cli" + autoload :Commands, "duracloud/commands" autoload :Configuration, "duracloud/configuration" autoload :Connection, "duracloud/connection" autoload :Content, "duracloud/content" diff --git a/lib/duracloud/cli.rb b/lib/duracloud/cli.rb index 87789e1..492fad4 100644 --- a/lib/duracloud/cli.rb +++ b/lib/duracloud/cli.rb @@ -4,8 +4,9 @@ module Duracloud class CLI include ActiveModel::Model + include Commands - COMMANDS = %w( sync validate manifest properties storage ids items count ) + COMMANDS = Commands.public_instance_methods.map(&:to_s) USAGE = <<-EOS Usage: duracloud [COMMAND] [options] @@ -160,41 +161,7 @@ def self.call(*args) def execute configure_client - send(command).call(self) - end - - protected - - def count - Commands::Count - end - - def storage - Commands::GetStorageReport - end - - def sync - Commands::Sync - end - - def validate - Commands::Validate - end - - def manifest - Commands::DownloadManifest - end - - def properties - Commands::GetProperties - end - - def ids - Commands::ListContentIds - end - - def items - Commands::ListItems + send(command, self) end private @@ -212,5 +179,3 @@ def configure_client end end - -Dir[File.expand_path("../commands/*.rb", __FILE__)].each { |m| require m } diff --git a/lib/duracloud/commands.rb b/lib/duracloud/commands.rb new file mode 100644 index 0000000..949a756 --- /dev/null +++ b/lib/duracloud/commands.rb @@ -0,0 +1,40 @@ +module Duracloud + module Commands + + def find(cli) + Find.call(cli) + end + + def count(cli) + Count.call(cli) + end + + def get_storage_report(cli) + GetStorageReport.call(cli) + end + + def sync(cli) + Sync.call(cli) + end + + def validate(cli) + Validate.call(cli) + end + + def download_manifest(cli) + DownloadManifest.call(cli) + end + + def list_content_ids(cli) + ListContentIds.call(cli) + end + + def list_items(cli) + ListItems.call(cli) + end + + end +end + +require 'duracloud/commands/command' +Dir[File.expand_path("../commands/*.rb", __FILE__)].each { |m| require m } diff --git a/lib/duracloud/commands/command.rb b/lib/duracloud/commands/command.rb index 4c1cb4a..b9f8bf0 100644 --- a/lib/duracloud/commands/command.rb +++ b/lib/duracloud/commands/command.rb @@ -7,5 +7,9 @@ def self.call(cli) new(cli).call end + def cli + __getobj__ + end + end end diff --git a/lib/duracloud/commands/count.rb b/lib/duracloud/commands/count.rb index 46288d8..1b98622 100644 --- a/lib/duracloud/commands/count.rb +++ b/lib/duracloud/commands/count.rb @@ -1,5 +1,3 @@ -require_relative "command" - module Duracloud::Commands class Count < Command diff --git a/lib/duracloud/commands/download_manifest.rb b/lib/duracloud/commands/download_manifest.rb index c3dc43a..3edff71 100644 --- a/lib/duracloud/commands/download_manifest.rb +++ b/lib/duracloud/commands/download_manifest.rb @@ -1,5 +1,3 @@ -require_relative "command" - module Duracloud::Commands class DownloadManifest < Command diff --git a/lib/duracloud/commands/find.rb b/lib/duracloud/commands/find.rb new file mode 100644 index 0000000..804cbc3 --- /dev/null +++ b/lib/duracloud/commands/find.rb @@ -0,0 +1,16 @@ +module Duracloud::Commands + class Find < Command + + def call + delegate_to = if content_id + FindItem + elsif infile + FindItems + else + FindSpace + end + delegate_to.call(cli) + end + + end +end diff --git a/lib/duracloud/commands/find_item.rb b/lib/duracloud/commands/find_item.rb new file mode 100644 index 0000000..c6c0222 --- /dev/null +++ b/lib/duracloud/commands/find_item.rb @@ -0,0 +1,16 @@ +module Duracloud::Commands + class FindItem < Command + + def call + content = Duracloud::Content.find(space_id: space_id, store_id: store_id, content_id: content_id, md5: md5) + props = content.properties.dup + props.merge!("MD5" => content.md5, + "Size" => content.size, + "Chunked" => content.chunked?) + props.each do |k, v| + puts "#{k}: #{v}" + end + end + + end +end diff --git a/lib/duracloud/commands/find_items.rb b/lib/duracloud/commands/find_items.rb new file mode 100644 index 0000000..1c477ce --- /dev/null +++ b/lib/duracloud/commands/find_items.rb @@ -0,0 +1,22 @@ +require 'csv' + +module Duracloud::Commands + class FindItems < Command + + HEADERS = %i( content_id md5 size content_type modified ) + + def call + CSV.instance($stdout, headers: HEADERS, write_headers: true) do |csv| + CSV.foreach(infile, headers: false) do |row| + begin + item = Duracloud::Content.find(space_id: space_id, store_id: store_id, content_id: row[0], md5: row[1]) + csv << HEADERS.map { |header| item.send(header) } + rescue Duracloud::NotFoundError, Duracloud::MessageDigestError => e + $stderr.puts "ERROR: Content ID #{row[0]} -- #{e.message}" + end + end + end + end + + end +end diff --git a/lib/duracloud/commands/find_space.rb b/lib/duracloud/commands/find_space.rb new file mode 100644 index 0000000..a6e6c30 --- /dev/null +++ b/lib/duracloud/commands/find_space.rb @@ -0,0 +1,12 @@ +module Duracloud::Commands + class FindSpace < Command + + def call + space = Duracloud::Space.find(space_id, store_id) + space.properties.each do |k, v| + puts "#{k}: #{v}" + end + end + + end +end diff --git a/lib/duracloud/commands/get_properties.rb b/lib/duracloud/commands/get_properties.rb deleted file mode 100644 index 83932ca..0000000 --- a/lib/duracloud/commands/get_properties.rb +++ /dev/null @@ -1,27 +0,0 @@ -require_relative "command" - -module Duracloud::Commands - class GetProperties < Command - - def call - proplist = content_id ? content_properties : space_properties - puts proplist - end - - private - - def content_properties - content = Duracloud::Content.find(space_id: space_id, store_id: store_id, content_id: content_id, md5: md5) - proplist = content.properties.map { |k, v| "#{k}: #{v}" } - proplist << "MD5: #{content.md5}" - proplist << "Size: #{content.size} (#{content.human_size})" - proplist << "Chunked?: #{content.chunked?}" - end - - def space_properties - space = Duracloud::Space.find(space_id, store_id) - space.properties.map { |k, v| "#{k}: #{v}" } - end - - end -end diff --git a/lib/duracloud/commands/get_storage_report.rb b/lib/duracloud/commands/get_storage_report.rb index ba9c291..271a40f 100644 --- a/lib/duracloud/commands/get_storage_report.rb +++ b/lib/duracloud/commands/get_storage_report.rb @@ -1,5 +1,3 @@ -require_relative "command" - module Duracloud::Commands class GetStorageReport < Command diff --git a/lib/duracloud/commands/list_content_ids.rb b/lib/duracloud/commands/list_content_ids.rb index 9be6d9f..c8ddb51 100644 --- a/lib/duracloud/commands/list_content_ids.rb +++ b/lib/duracloud/commands/list_content_ids.rb @@ -1,5 +1,3 @@ -require_relative "command" - module Duracloud::Commands class ListContentIds < Command diff --git a/lib/duracloud/commands/list_items.rb b/lib/duracloud/commands/list_items.rb index ee56d98..633bb5c 100644 --- a/lib/duracloud/commands/list_items.rb +++ b/lib/duracloud/commands/list_items.rb @@ -1,4 +1,3 @@ -require_relative "command" require "csv" module Duracloud::Commands @@ -7,7 +6,7 @@ class ListItems < Command HEADERS = %i( content_id md5 size content_type modified ) def call - ::CSV.instance($stdout, headers: HEADERS, write_headers: true) do |csv| + CSV.instance($stdout, headers: HEADERS, write_headers: true) do |csv| Duracloud::Space.items(space_id, store_id: store_id, prefix: prefix).each do |item| csv << HEADERS.map { |header| item.send(header) } end diff --git a/lib/duracloud/commands/sync.rb b/lib/duracloud/commands/sync.rb index 7379f67..e8cddfd 100644 --- a/lib/duracloud/commands/sync.rb +++ b/lib/duracloud/commands/sync.rb @@ -1,5 +1,3 @@ -require_relative 'command' - module Duracloud::Commands class Sync < Command diff --git a/lib/duracloud/commands/validate.rb b/lib/duracloud/commands/validate.rb index a2773a8..05e4f6f 100644 --- a/lib/duracloud/commands/validate.rb +++ b/lib/duracloud/commands/validate.rb @@ -1,5 +1,3 @@ -require_relative "command" - module Duracloud::Commands class Validate < Command diff --git a/spec/unit/cli_spec.rb b/spec/unit/cli_spec.rb index 2604db0..3591800 100644 --- a/spec/unit/cli_spec.rb +++ b/spec/unit/cli_spec.rb @@ -3,10 +3,34 @@ module Duracloud subject { described_class.new(**opts) } - describe "properties" do - let(:opts) { {command: "properties", space_id: "foo", content_id: "bar"} } + describe "find item" do + let(:opts) { {command: "find", space_id: "foo", content_id: "bar"} } specify { - expect(Commands::GetProperties).to receive(:call).with(subject) { nil } + expect(Commands::FindItem).to receive(:call).with(subject) { nil } + subject.execute + } + end + + describe "find space" do + let(:opts) { {command: "find", space_id: "foo"} } + specify { + expect(Commands::FindSpace).to receive(:call).with(subject) { nil } + subject.execute + } + end + + describe "find items" do + let(:opts) { {command: "find", space_id: "foo", infile: "/foo/bar"} } + specify { + expect(Commands::FindItems).to receive(:call).with(subject) { nil } + subject.execute + } + end + + describe "count" do + let(:opts) { {command: "count", space_id: "foo"} } + specify { + expect(Commands::Count).to receive(:call).with(subject) { nil } subject.execute } end @@ -27,21 +51,37 @@ module Duracloud } end - describe "manifest" do - let(:opts) { {command: "manifest", space_id: "foo"} } + describe "download_manifest" do + let(:opts) { {command: "download_manifest", space_id: "foo"} } specify { expect(Commands::DownloadManifest).to receive(:call).with(subject) { nil } subject.execute } end - describe "storage" do - let(:opts) { {command: "storage", space_id: "foo"} } + describe "get_storage_report" do + let(:opts) { {command: "get_storage_report", space_id: "foo"} } specify { expect(Commands::GetStorageReport).to receive(:call).with(subject) { nil } subject.execute } end + describe "list content ids" do + let(:opts) { {command: "list_content_ids", space_id: "foo"} } + specify { + expect(Commands::ListContentIds).to receive(:call).with(subject) { nil } + subject.execute + } + end + + describe "list items" do + let(:opts) { {command: "list_items", space_id: "foo"} } + specify { + expect(Commands::ListItems).to receive(:call).with(subject) { nil } + subject.execute + } + end + end end From a92539268f4a9d99b925507faa7d7450cef62f15 Mon Sep 17 00:00:00 2001 From: David Chandek-Stark Date: Wed, 6 Sep 2017 18:31:17 -0400 Subject: [PATCH 13/15] Adds -M/--missing option to CLI for finding missing items. CommandOptions factored out of CLI. --- lib/duracloud.rb | 2 +- lib/duracloud/cli.rb | 105 +---------------- lib/duracloud/command_options.rb | 116 +++++++++++++++++++ lib/duracloud/commands/find.rb | 2 +- lib/duracloud/commands/find_missing_items.rb | 15 +++ 5 files changed, 136 insertions(+), 104 deletions(-) create mode 100644 lib/duracloud/command_options.rb create mode 100644 lib/duracloud/commands/find_missing_items.rb diff --git a/lib/duracloud.rb b/lib/duracloud.rb index 28a88a1..2e16fab 100644 --- a/lib/duracloud.rb +++ b/lib/duracloud.rb @@ -8,6 +8,7 @@ module Duracloud autoload :ChunkedContent, "duracloud/chunked_content" autoload :Client, "duracloud/client" autoload :CLI, "duracloud/cli" + autoload :CommandOptions, "duracloud/command_options" autoload :Commands, "duracloud/commands" autoload :Configuration, "duracloud/configuration" autoload :Connection, "duracloud/connection" @@ -16,7 +17,6 @@ module Duracloud autoload :DurastoreRequest, "duracloud/durastore_request" autoload :ErrorHandler, "duracloud/error_handler" autoload :FastSyncValidation, "duracloud/fast_sync_validation" - autoload :HasProperties, "duracloud/has_properties" autoload :Manifest, "duracloud/manifest" autoload :Persistence, "duracloud/persistence" autoload :Properties, "duracloud/properties" diff --git a/lib/duracloud/cli.rb b/lib/duracloud/cli.rb index 492fad4..773d1df 100644 --- a/lib/duracloud/cli.rb +++ b/lib/duracloud/cli.rb @@ -28,6 +28,7 @@ class CLI :infile, :logging, :md5, + :missing, :password, :port, :prefix, @@ -39,10 +40,6 @@ class CLI validates_presence_of :space_id, message: "-s/--space-id option is required.", unless: "command == 'storage'" validates_inclusion_of :command, in: COMMANDS - def self.print_version - puts "duracloud-client #{Duracloud::VERSION}" - end - def self.error!(exception) $stderr.puts exception.message if [ CommandError, OptionParser::ParseError ].include?(exception.class) @@ -52,104 +49,8 @@ def self.error!(exception) end def self.call(*args) - options = {} - - parser = OptionParser.new do |opts| - opts.banner = USAGE - - opts.on("-h", "--help", - "Prints help") do - print_version - puts opts - exit - end - - opts.on("-H", "--host HOST", - "DuraCloud host") do |v| - options[:host] = v - end - - opts.on("-P", "--port PORT", - "DuraCloud port") do |v| - options[:port] = v - end - - opts.on("-u", "--user USER", - "DuraCloud user") do |v| - options[:user] = v - end - - opts.on("-p", "--password PASSWORD", - "DuraCloud password") do |v| - options[:password] = v - end - - opts.on("-l", "--[no-]logging", - "Enable/disable logging to STDERR") do |v| - options[:logging] = v - end - - opts.on("-s", "--space-id SPACE_ID", - "DuraCloud space ID") do |v| - options[:space_id] = v - end - - opts.on("-i", "--store-id STORE_ID", - "DuraCloud store ID") do |v| - options[:store_id] = v - end - - opts.on("-c", "--content-id CONTENT_ID", - "DuraCloud content ID") do |v| - options[:content_id] = v - end - - opts.on("-m", "--md5 MD5", - "MD5 digest of content to store or retrieve") do |v| - options[:md5] = v - end - - opts.on("-b", "--bagit", - "Get manifest in BAGIT format (default is TSV)") do - options[:format] = Manifest::BAGIT_FORMAT - end - - opts.on("-d", "--content-dir DIR", - "Local content directory") do |v| - options[:content_dir] = v - end - - opts.on("-f", "--infile FILE", - "Input file") do |v| - options[:infile] = v - end - - opts.on("-v", "--version", - "Print version and exit") do |v| - print_version - exit - end - - opts.on("-w", "--work-dir DIR", - "Working directory") do |v| - options[:work_dir] = v - end - - opts.on("-F", "--[no-]fast-audit", - "Use fast audit for sync validation") do |v| - options[:fast] = v - end - - opts.on("-a", "--prefix PREFIX", - "Content prefix") do |v| - options[:prefix] = v - end - end - - command = args.shift if COMMANDS.include?(args.first) - parser.parse!(args) - - cli = new(options.merge(command: command)) + options = CommandOptions.new(*args) + cli = new(options) # .merge(command: command)) if cli.invalid? message = cli.errors.map { |k, v| "ERROR: #{v}" }.join("\n") raise CommandError, message diff --git a/lib/duracloud/command_options.rb b/lib/duracloud/command_options.rb new file mode 100644 index 0000000..6436028 --- /dev/null +++ b/lib/duracloud/command_options.rb @@ -0,0 +1,116 @@ +require 'optparse' +require 'hashie' + +module Duracloud + class CommandOptions < Hashie::Mash + + def initialize(*args) + super() + self.command = args.shift if CLI::COMMANDS.include?(args.first) + parser.parse!(args) + end + + def print_version + puts "duracloud-client #{Duracloud::VERSION}" + end + + def parser + OptionParser.new do |opts| + opts.banner = CLI::USAGE + + opts.on("-h", "--help", + "Prints help") do + print_version + puts opts + exit + end + + opts.on("-H", "--host HOST", + "DuraCloud host") do |v| + self.host = v + end + + opts.on("-P", "--port PORT", + "DuraCloud port") do |v| + self.port = v + end + + opts.on("-u", "--user USER", + "DuraCloud user") do |v| + self.user = v + end + + opts.on("-p", "--password PASSWORD", + "DuraCloud password") do |v| + self.password = v + end + + opts.on("-l", "--[no-]logging", + "Enable/disable logging to STDERR") do |v| + self.logging = v + end + + opts.on("-s", "--space-id SPACE_ID", + "DuraCloud space ID") do |v| + self.space_id = v + end + + opts.on("-i", "--store-id STORE_ID", + "DuraCloud store ID") do |v| + self.store_id = v + end + + opts.on("-c", "--content-id CONTENT_ID", + "DuraCloud content ID") do |v| + self.content_id = v + end + + opts.on("-m", "--md5 MD5", + "MD5 digest of content to store or retrieve") do |v| + self.md5 = v + end + + opts.on("-b", "--bagit", + "Get manifest in BAGIT format (default is TSV)") do + self.format = Manifest::BAGIT_FORMAT + end + + opts.on("-d", "--content-dir DIR", + "Local content directory") do |v| + self.content_dir = v + end + + opts.on("-f", "--infile FILE", + "Input file") do |v| + self.infile = v + end + + opts.on("-v", "--version", + "Print version and exit") do |v| + print_version + exit + end + + opts.on("-w", "--work-dir DIR", + "Working directory") do |v| + self.work_dir = v + end + + opts.on("-F", "--[no-]fast-audit", + "Use fast audit for sync validation") do |v| + self.fast = v + end + + opts.on("-a", "--prefix PREFIX", + "Content prefix") do |v| + self.prefix = v + end + + opts.on("-M", "--[no-]missing", "Find missing items") do |v| + self.missing = v + end + end + end + + end +end diff --git a/lib/duracloud/commands/find.rb b/lib/duracloud/commands/find.rb index 804cbc3..5bff3ff 100644 --- a/lib/duracloud/commands/find.rb +++ b/lib/duracloud/commands/find.rb @@ -5,7 +5,7 @@ def call delegate_to = if content_id FindItem elsif infile - FindItems + missing ? FindMissingItems : FindItems else FindSpace end diff --git a/lib/duracloud/commands/find_missing_items.rb b/lib/duracloud/commands/find_missing_items.rb new file mode 100644 index 0000000..a1f3474 --- /dev/null +++ b/lib/duracloud/commands/find_missing_items.rb @@ -0,0 +1,15 @@ +module Duracloud::Commands + class FindMissingItems < Command + + def call + CSV.instance($stdout, headers: false) do |csv| + CSV.foreach(infile, headers: false) do |row| + unless Duracloud::Content.exist?(space_id: space_id, store_id: store_id, content_id: row[0], md5: row[1]) + csv << row + end + end + end + end + + end +end From aad0a050870f8e46e6f04edb8c48378d08eeb738 Mon Sep 17 00:00:00 2001 From: David Chandek-Stark Date: Fri, 8 Sep 2017 12:51:23 -0400 Subject: [PATCH 14/15] Fixes bug in StorageReports. Adds --all-spaces option to CLI for getting storage for all spaces. --- lib/duracloud/cli.rb | 5 +++-- lib/duracloud/command_options.rb | 4 ++++ lib/duracloud/commands/get_storage_report.rb | 15 ++++++++------- .../commands/get_storage_report_for_all_spaces.rb | 12 ++++++++++++ .../commands/get_storage_report_for_space.rb | 10 ++++++++++ .../commands/get_storage_report_for_store.rb | 10 ++++++++++ lib/duracloud/storage_reports.rb | 2 +- 7 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 lib/duracloud/commands/get_storage_report_for_all_spaces.rb create mode 100644 lib/duracloud/commands/get_storage_report_for_space.rb create mode 100644 lib/duracloud/commands/get_storage_report_for_store.rb diff --git a/lib/duracloud/cli.rb b/lib/duracloud/cli.rb index 773d1df..454468f 100644 --- a/lib/duracloud/cli.rb +++ b/lib/duracloud/cli.rb @@ -18,7 +18,8 @@ class CLI EOS HELP = "Type 'duracloud -h/--help' for usage." - attr_accessor :command, + attr_accessor :all_spaces, + :command, :content_dir, :content_id, :content_type, @@ -37,7 +38,7 @@ class CLI :user, :work_dir - validates_presence_of :space_id, message: "-s/--space-id option is required.", unless: "command == 'storage'" + validates_presence_of :space_id, message: "-s/--space-id option is required.", unless: "command == 'get_storage_report'" validates_inclusion_of :command, in: COMMANDS def self.error!(exception) diff --git a/lib/duracloud/command_options.rb b/lib/duracloud/command_options.rb index 6436028..b8db165 100644 --- a/lib/duracloud/command_options.rb +++ b/lib/duracloud/command_options.rb @@ -109,6 +109,10 @@ def parser opts.on("-M", "--[no-]missing", "Find missing items") do |v| self.missing = v end + + opts.on("--[no-]all-spaces", "Get report for all spaces") do |v| + self.all_spaces = v + end end end diff --git a/lib/duracloud/commands/get_storage_report.rb b/lib/duracloud/commands/get_storage_report.rb index 271a40f..583cbaf 100644 --- a/lib/duracloud/commands/get_storage_report.rb +++ b/lib/duracloud/commands/get_storage_report.rb @@ -2,13 +2,14 @@ module Duracloud::Commands class GetStorageReport < Command def call - reports = if space_id - Duracloud::StorageReports.by_space(space_id, store_id: store_id) - else - Duracloud::StorageReports.by_store(store_id: store_id) - end - report = reports.last - puts report.to_s + delegate_to = if space_id + GetStorageReportForSpace + elsif all_spaces + GetStorageReportForAllSpaces + else + GetStorageReportForStore + end + delegate_to.call(cli) end end diff --git a/lib/duracloud/commands/get_storage_report_for_all_spaces.rb b/lib/duracloud/commands/get_storage_report_for_all_spaces.rb new file mode 100644 index 0000000..5c7fc8e --- /dev/null +++ b/lib/duracloud/commands/get_storage_report_for_all_spaces.rb @@ -0,0 +1,12 @@ +module Duracloud::Commands + class GetStorageReportForAllSpaces < Command + + def call + Duracloud::StorageReports.for_all_spaces_in_a_store(store_id: store_id).each do |report| + puts "-"*40 + puts report.to_s + end + end + + end +end diff --git a/lib/duracloud/commands/get_storage_report_for_space.rb b/lib/duracloud/commands/get_storage_report_for_space.rb new file mode 100644 index 0000000..eb7b483 --- /dev/null +++ b/lib/duracloud/commands/get_storage_report_for_space.rb @@ -0,0 +1,10 @@ +module Duracloud::Commands + class GetStorageReportForSpace < Command + + def call + reports = Duracloud::StorageReports.by_space(space_id, store_id: store_id) + puts reports.last.to_s + end + + end +end diff --git a/lib/duracloud/commands/get_storage_report_for_store.rb b/lib/duracloud/commands/get_storage_report_for_store.rb new file mode 100644 index 0000000..d14f357 --- /dev/null +++ b/lib/duracloud/commands/get_storage_report_for_store.rb @@ -0,0 +1,10 @@ +module Duracloud::Commands + class GetStorageReportForStore < Command + + def call + reports = Duracloud::StorageReports.by_store(store_id: store_id) + puts reports.last.to_s + end + + end +end diff --git a/lib/duracloud/storage_reports.rb b/lib/duracloud/storage_reports.rb index 94a19ec..a9a6510 100644 --- a/lib/duracloud/storage_reports.rb +++ b/lib/duracloud/storage_reports.rb @@ -23,7 +23,7 @@ def self.by_store(**query) end def self.for_all_spaces_in_a_store(epoch_ms = nil, **query) - epoch_ms ||= Time.now.to_i * 1000 + epoch_ms ||= (Time.now - (3600 * 24)).to_i * 1000 params = Params.new(query) response = Client.get_storage_reports_for_all_spaces_in_a_store(epoch_ms, **params) new(response) From 68ec923ab0da95cb3c381997cacdaf2e3526a8c1 Mon Sep 17 00:00:00 2001 From: David Chandek-Stark Date: Fri, 8 Sep 2017 12:54:29 -0400 Subject: [PATCH 15/15] Version 0.9.0 --- lib/duracloud/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/duracloud/version.rb b/lib/duracloud/version.rb index a3cc6f6..34aae28 100644 --- a/lib/duracloud/version.rb +++ b/lib/duracloud/version.rb @@ -1,3 +1,3 @@ module Duracloud - VERSION = "0.9.0.pre" + VERSION = "0.9.0" end