Skip to content

Commit

Permalink
Merge branch 'seek-1.15' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
stuzart committed Jun 17, 2024
2 parents 8b8178c + 6a67035 commit 56d7375
Show file tree
Hide file tree
Showing 64 changed files with 111 additions and 33 deletions.
4 changes: 2 additions & 2 deletions app/controllers/models_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ def compare_versions
json = compare @blob1.filepath, @blob2.filepath, ["reportHtml", "crnJson", "json", "SBML"]
@crn = JSON.parse(json)["crnJson"]
@comparison_html = JSON.parse(json)["reportHtml"]
rescue StandardError => e
rescue Exception => e
raise e unless Rails.env.production?
flash.now[:error]="there was an error trying to compare the two versions - #{e.message}"
flash.now[:error]="there was an error trying to compare the two versions - #{e.message.gsub(/[\s\S]*STDERR:/,'')}"
end
else
flash.now[:error]="One of the version files could not be found, or you are not authorized to examine it"
Expand Down
2 changes: 1 addition & 1 deletion app/jobs/application_job.rb
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def queue_job(priority = nil, delay = default_delay)
end

def report_exception(exception, message = nil, data = {})
data.merge!({job_class: self.class.name})
data.merge!({job_class: self.class.name, arguments: arguments.inspect})
message ||= "Error executing job for #{self.class.name}"
Seek::Errors::ExceptionForwarder.send_notification(exception, data: data)
Rails.logger.error(message)
Expand Down
5 changes: 3 additions & 2 deletions app/models/git/repository.rb
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,14 @@ def remote_refs
refs[:branches] << h
end

git_base.tags.each do |tag|
git_base.tags.to_a.sort_by do |tag|
tag.target.time
end.reverse.each do |tag|
h = { name: tag.name, ref: tag.canonical_name, sha: tag.target.oid }
refs[:tags] << h
end

refs[:branches] = refs[:branches].sort_by { |x| [x[:default] ? 0 : 1, x[:name].downcase] }
refs[:tags] = refs[:tags].sort_by { |x| x[:name] }

refs
end
Expand Down
6 changes: 5 additions & 1 deletion lib/scrapers/github_scraper.rb
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ def create_resources(repositories)
end
tags = [tag]
else
tags = repo.remote_refs[:tags]&.map { |t| t[:name] } || []
tags = all_tags(repo)
if tags.empty?
output.puts " No tags found to register"
next
Expand Down Expand Up @@ -175,5 +175,9 @@ def latest_tag(repo)

tag
end

def all_tags(repo)
repo.remote_refs[:tags]&.map { |t| t[:name] } || []
end
end
end
9 changes: 9 additions & 0 deletions lib/scrapers/nfcore_scraper.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ class NfcoreScraper < GithubScraper

def list_repositories
repos = JSON.parse(RestClient.get('https://nf-co.re/pipelines.json'))['remote_workflows']
repos.reject! { |r| r['archived'] || r['disabled'] }
@nfcore_pipelines = {} # Store repo metadata from pipelines.json to fetch main branch name and topics later
repos.each do |r|
r['clone_url'] = "https://github.com/#{r['full_name']}.git"
Expand All @@ -22,6 +23,14 @@ def topics(repo)
@nfcore_pipelines.dig(repo.remote, 'topics') || []
end

def latest_tag(repo)
all_tags(repo).last
end

def all_tags(repo)
(@nfcore_pipelines.dig(repo.remote, 'releases') || []).sort_by { |t| Date.parse(t['published_at']) }.map { |t| t['tag_name'] } - ['dev']
end

def workflow_wizard(repo, tag)
GitWorkflowWizard.new(workflow_class: WorkflowClass.find_by_key('nextflow'),
params: {
Expand Down
8 changes: 8 additions & 0 deletions test/fixtures/files/mocking/nfcore_pipelines_trunc.json
Original file line number Diff line number Diff line change
Expand Up @@ -1653,6 +1653,14 @@
}
}
]
},
{
"name": "archived",
"archived": true
},
{
"name": "disabled",
"disabled": true
}
]
}
7 changes: 1 addition & 6 deletions test/fixtures/git/nf-core-rnaseq/_git/FETCH_HEAD
Original file line number Diff line number Diff line change
@@ -1,6 +1 @@
88562f8745302ec159679369326c6db5e4bd6e3b not-for-merge branch 'TEMPLATE' of https://github.com/nf-core/rnaseq.git
825345ea0c17b3962066a181fc5b5e405bfee5aa not-for-merge branch 'dev' of https://github.com/nf-core/rnaseq.git
3643a94411b65f42bce5357c5015603099556ad9 not-for-merge branch 'master' of https://github.com/nf-core/rnaseq.git
ccfb140d0f21d144e65498edc098f89c0d66d961 not-for-merge branch 'nf-core-template-merge-1.13.2' of https://github.com/nf-core/rnaseq.git
88562f8745302ec159679369326c6db5e4bd6e3b not-for-merge branch 'nf-core-template-merge-1.13.3' of https://github.com/nf-core/rnaseq.git
5681aa9012608ed67c67073ccee07b27586ee36a not-for-merge branch 'pytest-workflow' of https://github.com/nf-core/rnaseq.git
b89fac32650aacc86fcda9ee77e00612a1d77066 branch 'master' of https://github.com/nf-core/rnaseq
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/ORIG_HEAD
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3643a94411b65f42bce5357c5015603099556ad9
Binary file added test/fixtures/git/nf-core-rnaseq/_git/index
Binary file not shown.
2 changes: 2 additions & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/logs/HEAD
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
3643a94411b65f42bce5357c5015603099556ad9 3643a94411b65f42bce5357c5015603099556ad9 Finn Bacall <[email protected]> 1717586509 +0100 checkout: moving from master to master
3643a94411b65f42bce5357c5015603099556ad9 b89fac32650aacc86fcda9ee77e00612a1d77066 Finn Bacall <[email protected]> 1717586512 +0100 pull origin master: Fast-forward
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3643a94411b65f42bce5357c5015603099556ad9 b89fac32650aacc86fcda9ee77e00612a1d77066 Finn Bacall <[email protected]> 1717586512 +0100 pull origin master: Fast-forward
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
0000000000000000000000000000000000000000 e9f315d51e88f55398b21b4ff0f4d5e04a7c1dc4 Finn Bacall <[email protected]> 1616508146 +0000 fetch origin
e9f315d51e88f55398b21b4ff0f4d5e04a7c1dc4 88562f8745302ec159679369326c6db5e4bd6e3b Finn Bacall <[email protected]> 1617012765 +0100 fetch origin
88562f8745302ec159679369326c6db5e4bd6e3b eecf64e1f291a1aedbb23d96e95a3cdadd4640dc Finn Bacall <[email protected]> 1717582518 +0100 fetch: fast-forward
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0000000000000000000000000000000000000000 d2f00635cee40ea206e53d2e49a7119bc9482307 Finn Bacall <[email protected]> 1717582518 +0100 fetch: storing head
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0000000000000000000000000000000000000000 21fe86c80fa9e57144522944a01405f95c7e58b0 Finn Bacall <[email protected]> 1717582518 +0100 fetch: storing head
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
0000000000000000000000000000000000000000 825345ea0c17b3962066a181fc5b5e405bfee5aa Finn Bacall <[email protected]> 1616508146 +0000 fetch origin
825345ea0c17b3962066a181fc5b5e405bfee5aa 21095f381e70b4b73dc392c36318218044a8bc95 Finn Bacall <[email protected]> 1717582518 +0100 fetch: fast-forward
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0000000000000000000000000000000000000000 cdc8f53f3515b1a7e982006a1b2a282a9fdffafb Finn Bacall <[email protected]> 1717582518 +0100 fetch: storing head
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0000000000000000000000000000000000000000 31a9d622bbc4eeb285a2424c1af7938712b4f925 Finn Bacall <[email protected]> 1717582518 +0100 fetch: storing head
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0000000000000000000000000000000000000000 b5b728b9f36c650aea2245cfd893f7e2cfd8942a Finn Bacall <[email protected]> 1717582518 +0100 fetch: storing head
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0000000000000000000000000000000000000000 f018a0cf87c634a6a2eeb34d64a554a6fa852be0 Finn Bacall <[email protected]> 1717582518 +0100 fetch: storing head
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0000000000000000000000000000000000000000 5bef11b969acb2f2091eb6c84745580b5f70b527 Finn Bacall <[email protected]> 1717582518 +0100 fetch: storing head
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0000000000000000000000000000000000000000 5ce89ae977e4017c63f6887034c469dae54ca763 Finn Bacall <[email protected]> 1717582518 +0100 fetch: storing head
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
0000000000000000000000000000000000000000 3643a94411b65f42bce5357c5015603099556ad9 Finn Bacall <[email protected]> 1616508146 +0000 fetch origin
3643a94411b65f42bce5357c5015603099556ad9 b89fac32650aacc86fcda9ee77e00612a1d77066 Finn Bacall <[email protected]> 1717582518 +0100 fetch: fast-forward
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0000000000000000000000000000000000000000 6694d694f9b43fb59b73d79b04e1cdbaf3d67ce2 Finn Bacall <[email protected]> 1717582518 +0100 fetch: storing head
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0000000000000000000000000000000000000000 4ab3ecb3cfd734fcd149ee0a00a98b8da0c37ff2 Finn Bacall <[email protected]> 1717582518 +0100 fetch: storing head
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0000000000000000000000000000000000000000 b264499527af3b4c349f258d38c31759d7497c8e Finn Bacall <[email protected]> 1717582518 +0100 fetch: storing head
Binary file not shown.
Binary file not shown.
2 changes: 1 addition & 1 deletion test/fixtures/git/nf-core-rnaseq/_git/refs/heads/master
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3643a94411b65f42bce5357c5015603099556ad9
b89fac32650aacc86fcda9ee77e00612a1d77066
Original file line number Diff line number Diff line change
@@ -1 +1 @@
88562f8745302ec159679369326c6db5e4bd6e3b
eecf64e1f291a1aedbb23d96e95a3cdadd4640dc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
d2f00635cee40ea206e53d2e49a7119bc9482307
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
21fe86c80fa9e57144522944a01405f95c7e58b0
Original file line number Diff line number Diff line change
@@ -1 +1 @@
825345ea0c17b3962066a181fc5b5e405bfee5aa
21095f381e70b4b73dc392c36318218044a8bc95
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
cdc8f53f3515b1a7e982006a1b2a282a9fdffafb
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
31a9d622bbc4eeb285a2424c1af7938712b4f925
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b5b728b9f36c650aea2245cfd893f7e2cfd8942a
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
f018a0cf87c634a6a2eeb34d64a554a6fa852be0
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
5bef11b969acb2f2091eb6c84745580b5f70b527
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
5ce89ae977e4017c63f6887034c469dae54ca763
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3643a94411b65f42bce5357c5015603099556ad9
b89fac32650aacc86fcda9ee77e00612a1d77066
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
6694d694f9b43fb59b73d79b04e1cdbaf3d67ce2
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
4ab3ecb3cfd734fcd149ee0a00a98b8da0c37ff2
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b264499527af3b4c349f258d38c31759d7497c8e
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0fcbb0ac491ecb8a80ef879c4f3dad5f869021f9
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.10
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
adce7ce9abc8f6b338b4f53d0d988ff9a0fd52ff
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.10.1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
6e1e448f535ccf34d11cc691bb241cfd6e60a647
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.11.0
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
48fb9b4ea640f029f48c79283217d0f20661d38e
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.11.1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
287afcfe30a93de77e9b7cf70a1085f58c9525d8
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.11.2
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
5671b65af97fe78a2f9b4d05d850304918b1b86e
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.12.0
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3bec2331cac2b5ff88a1dc71a21fab6529b57a0f
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.13.0
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
14f9d26444e08da7b51ddcb1b8c4e0703edde375
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.13.1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b59e87a54eae60e02f9ae12e3b9c9c59959328d7
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.13.2
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
a10f41afa204538d5dcc89a5910c299d68f94f41
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.14.0
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b89fac32650aacc86fcda9ee77e00612a1d77066
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.2
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b3ff92bc54363faf17d820689a8e9074ffd99045
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.3
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8094c42add6dcdf69ce54dfdec957789c37ae903
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.4
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
964425e3fd8bfc3dc7bce43279a98d17a874d3f7
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.5
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
646723c70f04ee6d66391758b02822d4f0fe2966
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.6
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7106bd792b3fb04f9f09b4e737165fa4e736ea81
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.7
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e0dfce9af5c2299bcc2b8a74b6559ce055965455
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.8
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
6995330476244a6bffe55ddcbe50b8ed5cf6c2e2
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.8.1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
89bf536ce4faa98b4d50a8ec0a0343780bc62e0a
1 change: 1 addition & 0 deletions test/fixtures/git/nf-core-rnaseq/_git/refs/tags/3.9
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e049f51f0214b2aef7624b9dd496a404a7c34d14
11 changes: 6 additions & 5 deletions test/functional/git_repositories_controller_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,13 @@ class GitRepositoriesControllerTest < ActionController::TestCase
"ref" => "refs/remotes/origin/symlink",
"sha" => "728337a507db00b8b8ba9979330a4f53d6d43b18"}],
"tags" =>
[{ "name" => "v0.01",
"ref" => "refs/tags/v0.01",
"sha" => "3f2c23e92da3ccbc89d7893b4af6039e66bdaaaf" },
{ "name" => "v0.02",
[{ "name" => "v0.02",
"ref" => "refs/tags/v0.02",
"sha" => "94ae9926a824ebe809a9e9103cbdb1d5c5f98608" }] }
"sha" => "94ae9926a824ebe809a9e9103cbdb1d5c5f98608" },
{ "name" => "v0.01",
"ref" => "refs/tags/v0.01",
"sha" => "3f2c23e92da3ccbc89d7893b4af6039e66bdaaaf" }
]}

assert_equal expected, res
end
Expand Down
35 changes: 23 additions & 12 deletions test/integration/nfcore_scraper_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ class NfcoreScraperTest < ActionDispatch::IntegrationTest
assert_equal bot, wf.contributor
assert_equal [project], wf.projects
assert_equal 'nf-core/rnaseq', wf.title
assert_equal 'Nextflow RNA-Seq analysis pipeline, part of the nf-core community.', wf.description
assert_equal 'RNA sequencing analysis pipeline for gene/isoform quantification and extensive quality control.', wf.description
assert_equal 'MIT', wf.license
assert_equal 'nextflow.config', wf.main_workflow_path
assert_equal '3.0', wf.git_version.name
assert_equal '3.13.2', wf.git_version.name
assert_equal %w[rna rna-seq], wf.tags.sort
end
end
Expand Down Expand Up @@ -67,10 +67,10 @@ class NfcoreScraperTest < ActionDispatch::IntegrationTest
assert_equal bot, wf.contributor
assert_equal [project], wf.projects
assert_equal 'nf-core/rnaseq', wf.title
assert_equal 'Nextflow RNA-Seq analysis pipeline, part of the nf-core community.', wf.description
assert_equal 'RNA sequencing analysis pipeline for gene/isoform quantification and extensive quality control.', wf.description
assert_equal 'MIT', wf.license
assert_equal 'nextflow.config', wf.main_workflow_path
assert_equal '3.0', wf.git_version.name
assert_equal '3.13.2', wf.git_version.name
end
end
end
Expand All @@ -89,10 +89,10 @@ class NfcoreScraperTest < ActionDispatch::IntegrationTest
contributor: bot,
projects: [project],
source_link_url: 'https://github.com/nf-core/rnaseq',
git_version_attributes: { name: '3.0',
git_version_attributes: { name: '3.13.2',
git_repository_id: repos.first.id,
ref: 'refs/tags/3.0',
commit: '3643a94',
ref: 'refs/tags/3.13.2',
commit: 'a10f41a',
main_workflow_path: 'nextflow.config',
mutable: false })

Expand All @@ -117,16 +117,18 @@ class NfcoreScraperTest < ActionDispatch::IntegrationTest

repos = [FactoryBot.create(:nfcore_remote_repository)]
# These are the available remote Git tags in the above repo:
tags = ['1.0', '1.1', '1.2', '1.3', '1.4', '1.4.1', '1.4.2', '2.0', '3.0']
tags = ['1.0', '1.1', '1.2', '1.3', '1.4', '1.4.1', '1.4.2', '2.0', '3.0', '3.1', '3.2', '3.3', '3.4', '3.5',
'3.6', '3.7', '3.8', '3.8.1', '3.9', '3.10', '3.10.1', '3.11.0', '3.11.1', '3.11.2', '3.12.0', '3.13.1',
'3.13.0', '3.13.2']

existing = FactoryBot.create(:nfcore_git_workflow,
contributor: bot,
projects: [project],
source_link_url: 'https://github.com/nf-core/rnaseq',
git_version_attributes: { name: '3.0',
git_version_attributes: { name: '1.0',
git_repository_id: repos.first.id,
ref: 'refs/tags/3.0',
commit: '3643a94',
ref: 'refs/tags/1.0',
commit: '44f1525',
main_workflow_path: 'nextflow.config',
mutable: false })

Expand All @@ -140,14 +142,23 @@ class NfcoreScraperTest < ActionDispatch::IntegrationTest
refute scraped.empty?

assert_equal tags.count, existing.reload.versions.count
assert_equal tags.sort, existing.versions.map(&:name).sort
assert_equal tags, existing.versions.sort_by(&:created_at).map(&:name)
end
end
end
end
end
end

test 'does not list archived or disabled repositories' do
project = Scrapers::Util.bot_project(title: 'test')
bot = Scrapers::Util.bot_account
scraper = Scrapers::NfcoreScraper.new('test-123', project, bot, output: StringIO.new)
repos = scraper.send(:list_repositories)
assert_equal 1, repos.length
assert_includes repos.map { |r| r['name'] }, 'rnaseq'
end

private

def login_as(user)
Expand Down

0 comments on commit 56d7375

Please sign in to comment.