Skip to content

Commit

Permalink
issue #17, #30, schemas no longer deleted,collections after 24 hrs, a…
Browse files Browse the repository at this point in the history 
…ll caching/ref code removed
  • Loading branch information
@iterman
iterman committed Nov 15, 2012
1 parent 20b089f commit ca7e262
Show file tree
Hide file tree
Showing 5 changed files with 3 additions and 168 deletions.
21 changes: 0 additions & 21 deletions app.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,28 +75,7 @@ def do_at_exit(str1)
Dir.mkdir(schemas_dirname)
Logger.info "Schemas directory #{schemas_dirname} created"
do_at_exit($tempdir)
# "globals initialized in SchemaCatalog class- not in app.rb, $schema_references,$schema_references_permanent,$MD5toURL
end
=begin
def shutdown_stats
Logger.info "write_count=#{$write_count}";
Logger.info "iostats write_count=#{$iostats.writes}";
Logger.info "Shutdown Statistics"
Logger.info "Current"
Logger.info "Count\tDate\t\t\t\tMD5\t\t\t\t\tURL"
$schema_references.keys.each do |k|
val = $schema_references[k]
Logger.info "#{val}\t#{$MD5toRecord[k].last_modified.iso8601}\t#{k}\t#{$MD5toRecord[k].location}"
end
Logger.info "Cumulative"
Logger.info "Count\t\t\t\t\tMD5\t\t\t\t\tURL"
$schema_references_permanent.keys.each do |k|
val = $schema_references_permanent[k]
Logger.info "#{val}\t\t\t\t\t#{k}\t#{$MD5toRecord[k].location}"
end
end
=end

begin
load 'lib/app/helpers.rb'
Expand Down
23 changes: 0 additions & 23 deletions lib/app/gets.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,26 +63,3 @@
get '/status' do
[ 200, {'Content-Type' => 'application/xml'}, "<status/>\n" ]
end
=begin
get '/stats' do
stats = '<h1><b>DAITSS XMLResolution Statistics</b></h1>'
stats = stats << "<br><h3> running since: #{$iostats.startup_time}<h3>"
stats = stats << '<table border="1" style="background-color:magenta"><th><h3><b>Current</b></h3></th>'
stats = stats << "<tr><td><b>Collections</b><td>#{$iostats.collections}</td></tr>"
stats = stats << "<tr><td><b>Schemas/DTDs/PIs<td>#{$schema_references.size}</td></tr>"
stats = stats << "<tr><td><b>URL Reads</b></td><td>#{$iostats.url_reads}</td><tr>"
stats = stats << "<tr><td><b>URL Bytes Read</b></td><td>#{$iostats.url_bytes_read}</td><tr>"
stats = stats << "<tr><td><b>Storage Writes</b></td><td>#{$iostats.writes}</td><tr>"
stats = stats << "<tr><td><b>Bytes Written</b></td><td>#{$iostats.bytes_written}</td><tr>"
stats = stats << "<tr><td><b>Successes</b></td><td>#{$iostats.successes}</td><tr>"
stats = stats << "<tr><td><b>Redirects</b></td><td>#{$iostats.redirect_cases}</td><tr>"
stats = stats << "<tr><td><b>Unresolveds</b></td><td>#{$iostats.fails}</td><tr>"
stats = stats << '<tr><td><b>Ref Count</b></td><td><b>Date</b></td><td><b>MD5</b></td><td><b>Schema/DTD/PI</b></td></tr>'
$schema_references.keys.each do |k|
val = $schema_references[k]
stats = stats << '<tr>' << "<td>#{val}</td><td>#{$MD5toRecord[k].last_modified.iso8601}</td><td>#{k}</td><td>#{$MD5toRecord[k].location}</td></tr>"
end
stats = stats << '</th></table>'
end
end
=end
1 change: 0 additions & 1 deletion lib/app/puts.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
else
ResolverCollection.new settings.data_path, collection_id
status 201
# $iostats.collections += 1
"Collection #{collection_id} created.\n"
end

Expand Down
83 changes: 1 addition & 82 deletions lib/xmlresolution/resolver_collection.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,15 +314,6 @@ def tar
tarwriter = XmlResolution::TarWriter.new(io, { :uid => 80, :gid => 80, :username => 'daitss', :groupname => 'daitss' })
manifest_file(rs) { |path| tarwriter.write path, File.join(collection_name, 'manifest.xml') }
schemas(rs) { |localpath, url| tarwriter.write localpath, File.join(collection_name, url.gsub(':', '/').gsub(%r{//+}, '/')) }
schemas(rs) do |p,u|
s = p[p.length-32..p.length];
if $schema_references[s] != nil
$schema_references[s] = $schema_references[s] - 1
end
end
delete_schemas
delete_collection collection_name
delete_zero_refed_schemas

tarwriter.close # closes io as side effect

Expand All @@ -332,78 +323,6 @@ def tar
io.close
io.unlink
end
#always get rid of collection no matter what
def delete_collection collection_name
collection = File.join($tempdir,'collections',collection_name)
if File.exists?(collection)
FileUtils.rm_rf collection
end
end

=begin
def get_schema_reference_count collection
count = 0;
begin
Dir.open(collection) do |dir|
line = String.new
schema = String.new
file = nil
dir.each do |f|
file = f
next if File.directory?(f)
s = File.new(File.join(collection,f))
lines = s.lines()
lines.each do |l|
line = l
id = l[0..6]
if id == "SCHEMA "
schema = l[7..38]
count = count + $schema_references[schema]
end
end
end
end
rescue Exception
Logger.err("get_schema_reference_count Exception #{$!} at: #{$@[0]}")
Logger.err("line=#{line}")
Logger.err("collection=#{collection} count=#{count} schema=#{schema} file=#{file} Exception #{$!}")
Logger.err("schema_reference[#{schema}]=#{$schema_references[schema]}")
count = -1
end
count
end
=end

def delete_schemas
begin
schemaDir = File.join($tempdir,"schemas")
$schema_references.keys.each do |k|
val = $schema_references[k]
if val == 0
#$schema_references.delete(k)
schema = File.join(schemaDir,k)
if File.exists?(schema)
FileUtils.rm_rf schema
end # added
end
end
rescue
puts "delete_schemas rescue $!=#{$!}"
end
end

def delete_zero_refed_schemas
begin
$schema_references.keys.each do |k|
val = $schema_references[k]
if val == 0
$schema_references.delete(k)
end
end
rescue
puts "delete_zero_refed_schemas rescue $!=#{$!}"
end
end

# resolutions
#
Expand Down Expand Up @@ -453,7 +372,7 @@ def ResolverCollection.age_out_collections collections_pathname
next unless ResolverUtils.collection_name_ok? File.split(dir)[-1]

if (Time.now - File::stat(dir).mtime) > TOO_LONG_SINCE_LAST_MODIFIED
Logger.err("Collection #{dir} has not been modified since #{File::stat(dir).mtime} exceeding TOO_LONG_SINCE_MODIFIED #{TOO_LONG_SINCE_LAST_MODIFIED} seconds")
Logger.info("Collection #{dir} deleted, since has not been modified since #{File::stat(dir).mtime} exceeding TOO_LONG_SINCE_MODIFIED #{TOO_LONG_SINCE_LAST_MODIFIED} seconds")
FileUtils.rm_rf dir
end
end
Expand Down
43 changes: 2 additions & 41 deletions lib/xmlresolution/schema_catalog.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,25 +26,8 @@ module XmlResolution

class SchemaCatalog
$tempdir = Dir.mktmpdir
$schema_references = Hash.new
#begin
$schema_references_permanent = Hash.new
$MD5toURL = Hash.new
$MD5toRecord = Hash.new
$write_count = 0
ios = Struct.new("Iostats",:url_reads,:url_bytes_read,:writes,:bytes_written,:successes,:redirect_cases,:redirect_total,:fails,:collections,:startup_time)
$iostats = ios.new
$iostats.url_reads = 0
$iostats.url_bytes_read = 0
$iostats.writes = 0
$iostats.bytes_written = 0
$iostats.successes = 0
$iostats.redirect_cases = 0
$iostats.redirect_total = 0
$iostats.fails = 0
$iostats.collections = 0
$iostats.startup_time = Time.new.iso8601
#end
=begin
=end

# Be sure to keep the following more or less in sync with the Struct::SchemaReloaded used in the XmlResolverReloaded class;
# it is the data structure that we maintain in the SchemaCatalog.
Expand Down Expand Up @@ -196,29 +179,12 @@ def get_schema_record location, namespace
record.location = actual_location
record.retrieval_status = :success

if $schema_references[record.digest] == nil
$schema_references[record.digest] = 1
else
$schema_references[record.digest]=$schema_references[record.digest]+1
end
=begin statistics
$MD5toURL[record.digest] = record.location
$MD5toRecord[record.digest] = record
if $schema_references_permanent[record.digest] == nil
$schema_references_permanent[record.digest] = 1
else
$schema_references_permanent[record.digest]=$schema_references_permanent[record.digest]+1
end
=end

# file_recorded? has a side effect. gets a write lock
if not file_recorded?(record.localpath, record.last_modified, record.digest)
ResolverUtils.write_lock(record.localpath) do |fd|
fd.write response.body
fd.close
#$iostats.writes += 1
#$iostats.bytes_written += response.body.length
File.utime(File.atime(record.localpath), record.last_modified, record.localpath)
end
end
Expand Down Expand Up @@ -251,17 +217,12 @@ def fetch location, limit = 5

Net::HTTP::Proxy(@proxy_addr, @proxy_port).start(uri.host, uri.port) do |http|
response = http.get(uri.path)
#$iostats.url_reads += 1
case response
when Net::HTTPSuccess then
#$iostats.successes += 1
#$iostats.url_bytes_read += response.body.length
return [ response, location ]
when Net::HTTPRedirection then
#$iostats.redirect_cases += 1
fetch response['location'], limit - 1
else
#$iostats.fails += 1
response.error!
end
end
Expand Down

0 comments on commit ca7e262

Please sign in to comment.