Skip to content

Commit

Permalink
New evaluation method
Browse files Browse the repository at this point in the history
  • Loading branch information
Bas Wenneker committed Aug 27, 2010
1 parent 07817b6 commit 10caabf
Show file tree
Hide file tree
Showing 7 changed files with 328 additions and 30 deletions.
4 changes: 2 additions & 2 deletions app/controllers/eval_references_controller.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ class EvalReferencesController < ApplicationController
# GET /eval_references
# GET /eval_references.xml
def index
@eval_references = EvalReference.all(:conditions => {:referencetype => 'expert'})
@eval_references = EvalReference.all(:conditions => {:referencetype => 'expert', :status => '1'}, :order => "status ASC")

respond_to do |format|
format.html # index.html.erb
Expand Down Expand Up @@ -64,7 +64,7 @@ def update
respond_to do |format|
if @eval_reference.update_attributes(params[:eval_reference])
flash[:notice] = 'EvalReference was successfully updated.'
format.html { redirect_to(@eval_reference) }
format.html { redirect_to(:action => 'index') }
format.xml { head :ok }
else
format.html { render :action => "edit" }
Expand Down
250 changes: 243 additions & 7 deletions app/models/eval_reference.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,29 +37,49 @@ def from_bibmix_reference(reference)

def calcstats

parsed = EvalReference.all(:conditions => {:referencetype => 'parsed'})
experts = EvalReference.all(:conditions => {:referencetype => 'expert', :status => '1'})

attrs = get_reference_attrs

result = {}
precision = {}
attrs.each do |attr|
result[attr] = 0
result[attr] = 0
precision[attr] = 0
recall[attr] = 0
end
count = 0
parsed.each do |p|
enriched = EvalReference.find(:first, :conditions => {:referencetype => 'enriched', :citation => p.citation})
expert = EvalReference.find(:first, :conditions => {:referencetype => 'expert', :citation => p.citation})
experts.each do |expert|
enriched = EvalReference.find(:first, :conditions => {:referencetype => 'enriched', :citation => expert.citation})



attrs.each do |attr|

correct = !(enriched.has_attr(attr)^p.has_attr(attr)) && enriched[attr] == p[attr]
if enriched.has_attr(attr)
precision[attr] += 1
end

if enriched.has_attr(attr)
recall[attr] += 1
end

correct = !(enriched.has_attr(attr)^expert.has_attr(attr))

if expert[attr].nil? || expert[attr].empty?
correct = enriched[attr].nil? || enriched[attr].empty?
else
correct = enriched[attr] == expert[attr]
end


if correct
result[attr] += 1
end

end
end


count += 1
end

Expand All @@ -72,6 +92,222 @@ def calcstats

end

def calcstats2

address_relevant = 0
address_retrieved = 0
expert_results = EvalReference.all(:conditions => {:referencetype => 'expert', :status => '1'})

attrs = get_reference_attrs

relevant = {}
retrieved = {}
retrieved_expert = {}
attrs.each do |attr|
relevant[attr] = 0
retrieved[attr] = 0
retrieved_expert[attr] = 0
end

c = 0
expert_results.each do |expert|
enriched = EvalReference.find(:first, :conditions => {:referencetype => 'enriched', :citation => expert.citation})

attrs.each do |attr|
if enriched.has_attr(attr)
retrieved[attr] += 1

if expert.has_attr(attr) && enriched[attr] == expert[attr]
relevant[attr] += 1
end
end



if expert.has_attr(attr)
retrieved_expert[attr] += 1
end
end

c += 1
#address_retrieved = enriched.has_attr('address')
end

# puts address_retrieved, address_relevant, address_relevant/address_retrieved.to_f, address_relevant/c.to_f

puts c

attrs.each do |attr|
recall = relevant[attr].to_f/retrieved_expert[attr].to_f
precision = relevant[attr].to_f/retrieved[attr].to_f
puts "#{attr}:\t\t #{precision}\t\t#{recall}"
end

end

def calcstats3

expert_results = EvalReference.all(:conditions => {:referencetype => 'expert', :status => '1'})

attrs = get_reference_attrs

enriched_correct = {}
enriched_retrieved = {}
expert_retrieved = {}
attrs.each do |attr|
enriched_correct[attr] = 0
enriched_retrieved[attr] = 0
expert_retrieved[attr] = 0
end

c = 0
expert_results.each do |expert|
enriched = EvalReference.find(:first, :conditions => {:referencetype => 'enriched', :citation => expert.citation})

attrs.each do |attr|
# precision
if enriched.has_attr(attr)
if enriched[attr] == expert[attr]
enriched_correct[attr] += 1
end
enriched_retrieved[attr] += 1

end
# correct = false
# if expert[attr].nil? || expert[attr].empty?
# correct = enriched[attr].nil? || enriched[attr].empty?
# else
# correct = enriched[attr] == expert[attr]
# end

# if correct
# enriched_correct[attr] += 1
# end

if expert.has_attr(attr)
expert_retrieved[attr] += 1
end
end

c += 1

end


attrs.each do |attr|

precision = enriched_correct[attr]/c.to_f
recall = enriched_correct[attr]/expert_retrieved[attr].to_f
puts "#{attr}:\t\t #{precision}\t\t#{recall}"
end

end

def calcstats4

expert_results = EvalReference.all(:conditions => {:referencetype => 'expert', :status => '1'})

attrs = get_reference_attrs

tp, fp, tn, fn = {}, {}, {}, {}

attrs.each do |attr|
tp[attr] = 0
fp[attr] = 0
tn[attr] = 0
fn[attr] = 0
end

c = 0
expert_results.each do |expert|
enriched = EvalReference.find(:first, :conditions => {:referencetype => 'enriched', :citation => expert.citation})

attrs.each do |attr|
# precision
if enriched.has_attr(attr)
if enriched[attr] == expert[attr]
# True positive: Sick people correctly diagnosed as sick
# TP: gevonden en hetzelfde als expert
tp[attr] += 1
else
# False positive: Healthy people incorrectly identified as sick
# FP: gevonden en niet hetzelfde als expert
fp[attr] += 1
end
else

if !expert.has_attr(attr)
#tn: niets gevonden en expert ook niet
tn[attr] += 1
else
#fn: niets gevonden maar expert wel
fn[attr] += 1
end

end

#False negative: Sick people incorrectly identified as healthy.


# correct = false
# if expert[attr].nil? || expert[attr].empty?
# correct = enriched[attr].nil? || enriched[attr].empty?
# else
# correct = enriched[attr] == expert[attr]
# end

# if correct
# enriched_correct[attr] += 1
# end

# if expert.has_attr(attr)
# expert_retrieved[attr] += 1
# end
end

c += 1

end


puts "measure:\t\t Precision\t\tRecall\t\tF"

afp, afr, aff = 0, 0, 0

nanat = []
attrs.each do |attr|

precision = tp[attr].to_f/(tp[attr]+fp[attr])
recall = tp[attr].to_f/(tp[attr]+fn[attr])
f = (2*precision*recall)/(precision+recall)

if precision.nan? || recall.nan?
nanat << attr
next
end

# precision = 1 if precision.nan?
# recall = 1 if recall.nan?
# f = 1 if f.nan?

afp += precision
afr += recall
aff += f
#puts attr
precision = (precision*100).round(1)
recall = (recall*100).round(1)
f = (f*100).round(1)
puts "\hline \textbf{#{attr.capitalize}} & #{precision} & #{recall} & #{f} \\"
end
afp = ((afp/(attrs.size-nanat.size))*100).round(1)
afr = ((afr/(attrs.size-nanat.size))*100).round(1)
aff = ((aff/(attrs.size-nanat.size))*100).round(1)
puts "#################################"
puts "Totals\t\t #{afp}\t\t#{afr}\t\t#{aff}"
puts attrs.size, (attrs.size-nanat.size)
puts nanat.to_yaml
end

def has_attr(attr)
!self[attr].nil? && !self[attr].empty?
end
Expand Down
7 changes: 6 additions & 1 deletion app/views/eval_references/edit.html.erb
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
<h1>Editing eval_reference</h1>
<h1><%= @eval_reference.title %></h1>

<style>
td, tr {border:1px solid #ccc}
input {width:100%}
textarea{height:100px}
</style>
<a target="_blank" href="http://liinwww.ira.uka.de/csbib?query=<%= CGI.escape(@eval_reference.title) %>">CCSB</a>
<a target="_blank" href="http://scholar.google.nl/scholar?q=<%= CGI.escape(@eval_reference.title) %>&hl=nl&btnG=Zoeken&lr=">Goog Scholar</a>
<a target="_blank" href="http://portal.acm.org/results.cfm?query=<%= CGI.escape(@eval_reference.title) %>">ACM</a>
<a target="_blank" href="http://dblp.l3s.de/?q=<%= CGI.escape(@eval_reference.title) %>&search_opt=all&newQuery=yes&resTableName=query_result7ktqPy&synt_query_exp=full">DBLP</a>
<a target="_blank" href="http://www.bibsonomy.org/search/<%= CGI.escape(@eval_reference.title) %>">Bibsonomy</a>

<% form_for(@eval_reference) do |f| %>
<%= f.error_messages %>
Expand Down
8 changes: 7 additions & 1 deletion app/views/eval_references/index.html.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

<table>
<tr>
<th>xx</th>
<th>Referencetype</th>
<th>Address</th>
<th>Annotate</th>
Expand Down Expand Up @@ -30,7 +31,12 @@
</tr>

<% @eval_references.each do |eval_reference| %>
<tr>
<tr <%
if eval_reference.status != '' && eval_reference.status != '0' && eval_reference.status != nil
%>style="color:green"<%
end
%>>
<td><%= link_to 'Edit', edit_eval_reference_path(eval_reference) %></td>
<td><%=h eval_reference.referencetype %></td>
<td><%=h eval_reference.address %></td>
<td><%=h eval_reference.annotate %></td>
Expand Down
Loading

0 comments on commit 10caabf

Please sign in to comment.