diff --git a/.gitignore b/.gitignore index ccf97ea0..0cd105fc 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ config/*.p12 config/*.json data/ projectFilesBackup/ +.bundle .ruby-version repo* *.turtle diff --git a/Gemfile b/Gemfile index f136281f..8b180c6e 100644 --- a/Gemfile +++ b/Gemfile @@ -24,10 +24,10 @@ gem 'sys-proctable' gem 'cube-ruby', require: 'cube' # NCBO -gem 'goo', github: 'ncbo/goo', branch: 'master' -gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'master' -gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'master' -gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' +gem 'goo', github: 'ncbo/goo', branch: 'develop' +gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'develop' +gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'develop' +gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop' group :test do gem 'email_spec' diff --git a/Gemfile.lock b/Gemfile.lock index aef96992..4e13127e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,13 +1,14 @@ GIT remote: https://github.com/ncbo/goo.git - revision: f01386fe55ffba39cbf5e0bf95c8e2dc18767711 - branch: master + revision: 33583fd1c1d72b449cd6bb91815e3422b8448983 + branch: develop specs: goo (0.0.2) addressable (~> 2.8) pry rdf (= 1.0.8) redis + request_store rest-client rsolr sparql-client @@ -15,8 +16,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: 63c986880aa88c9384043e6611a682434a14aba7 - branch: master + revision: 7531e10ad55ac66e925c099d7fc05a5a3ceae67e + branch: develop specs: ncbo_annotator (0.0.1) goo @@ -26,8 +27,8 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 4dc3b1123e871b53ba9eb51983ff69c5d2c75c18 - branch: master + revision: a3f6cf0493bb595011f98acf95efda33111c537b + branch: develop specs: ontologies_linked_data (0.0.1) activesupport @@ -46,8 +47,8 @@ GIT GIT remote: https://github.com/ncbo/sparql-client.git - revision: d418d56a6c9ff5692f925b45739a2a1c66bca851 - branch: master + revision: 1657f0dd69fd4b522d3549a6848670175f5e98cc + branch: develop specs: sparql-client (1.0.1) json_pure (>= 1.4) @@ -59,6 +60,8 @@ PATH specs: ncbo_cron (0.0.1) dante + faraday (~> 2) + faraday-follow_redirects (~> 0) goo google-analytics-data mlanett-redis-lock @@ -97,8 +100,10 @@ GEM base64 faraday-net_http (>= 2.0, < 3.1) ruby2_keywords (>= 0.0.4) + faraday-follow_redirects (0.3.0) + faraday (>= 1, < 3) faraday-net_http (3.0.2) - faraday-retry (2.2.0) + faraday-retry (2.2.1) faraday (~> 2.0) ffi (1.16.3) gapic-common (0.21.1) @@ -146,10 +151,10 @@ GEM multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.62.0) + grpc (1.63.0) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.62.0-x86_64-linux) + grpc (1.63.0-x86_64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) htmlentities (4.3.4) @@ -159,8 +164,8 @@ GEM httpclient (2.8.3) i18n (0.9.5) concurrent-ruby (~> 1.0) - json (2.7.1) - json_pure (2.7.1) + json (2.7.2) + json_pure (2.7.2) jwt (2.8.1) base64 launchy (2.5.2) @@ -174,7 +179,7 @@ GEM net-imap net-pop net-smtp - method_source (1.0.0) + method_source (1.1.0) mime-types (3.5.2) mime-types-data (~> 3.2015) mime-types-data (3.2024.0305) @@ -205,21 +210,23 @@ GEM pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.4) + public_suffix (5.0.5) rack (3.0.10) rack-test (2.1.0) rack (>= 1.3) - rake (13.1.0) + rake (13.2.1) rdf (1.0.8) addressable (>= 2.2) - redis (5.1.0) - redis-client (>= 0.17.0) - redis-client (0.21.1) + redis (5.2.0) + redis-client (>= 0.22.0) + redis-client (0.22.1) connection_pool representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) + request_store (1.7.0) + rack (>= 1.4) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) diff --git a/lib/ncbo_cron/obofoundry_sync.rb b/lib/ncbo_cron/obofoundry_sync.rb index 2437a9f3..23dd6e8d 100644 --- a/lib/ncbo_cron/obofoundry_sync.rb +++ b/lib/ncbo_cron/obofoundry_sync.rb @@ -1,7 +1,6 @@ -require 'base64' -require 'json' -require 'net/http' -require 'uri' +require 'faraday' +require 'faraday/follow_redirects' +require 'multi_json' module NcboCron module Models @@ -9,9 +8,6 @@ class OBOFoundrySync def initialize @logger = Logger.new(STDOUT) - @oauth_token = Base64.decode64(NcboCron.settings.git_repo_access_token) - @graphql_uri = URI.parse("https://api.github.com/graphql") - @request_options = { use_ssl: @graphql_uri.scheme == "https" } end def run @@ -19,24 +15,25 @@ def run map = get_ids_to_acronyms_map onts = get_obofoundry_ontologies - @logger.info("Found #{onts.size} OBO Library ontologies") + @logger.info("Found #{onts.size} OBO Foundry ontologies") - # Are any OBO Library ontologies missing from BioPortal? + # Are any OBO Foundry ontologies missing from BioPortal? missing_onts = [] - active_onts = onts.reject { |ont| ont.key?("is_obsolete") } + active_onts = onts.reject { |ont| ont.key?('is_obsolete') } + @logger.info("#{active_onts.size} OBO Foundry ontologies are currently active") active_onts.each do |ont| - if not map.key?(ont["id"]) + if not map.key?(ont['id']) missing_onts << ont - @logger.info("Missing OBO Library ontology: #{ont['title']} (#{ont['id']})") + @logger.info("Missing OBO Foundry ontology: #{ont['title']} (#{ont['id']})") end end - # Have any of the OBO Library ontologies that BioPortal hosts become obsolete? + # Have any of the OBO Foundry ontologies that BioPortal hosts become obsolete? obsolete_onts = [] - ids = active_onts.map{ |ont| ont["id"] } + ids = active_onts.map{ |ont| ont['id'] } obsolete_ids = map.keys - ids obsolete_ids.each do |id| - ont = onts.find{ |ont| ont["id"] == id } + ont = onts.find{ |ont| ont['id'] == id } @logger.info("Deprecated OBO Library ontology: #{ont['title']} (#{ont['id']})") obsolete_onts << ont end @@ -45,49 +42,19 @@ def run end def get_ids_to_acronyms_map - query = "query { - repository(name: \"ncbo.github.io\", owner: \"ncbo\") { - object(expression: \"master:oboids_to_bpacronyms.json\") { - ... on Blob { - text - } - } - } - }" - - response = issue_request(query) - JSON.parse(response) + response = Faraday.get('https://ncbo.github.io/oboids_to_bpacronyms.json') + MultiJson.load(response.body) end def get_obofoundry_ontologies - query = "query { - repository(name: \"OBOFoundry.github.io\", owner: \"OBOFoundry\") { - object(expression: \"master:registry/ontologies.jsonld\") { - ... on Blob { - text - } - } - } - }" - - response = issue_request(query) - ont_registry = JSON.parse(response) - ont_registry["ontologies"].to_a - end - - def issue_request(query) - request = Net::HTTP::Post.new(@graphql_uri) - request["Authorization"] = "bearer #{@oauth_token}" - request.body = JSON.dump({"query" => query}) - - response = Net::HTTP.start(@graphql_uri.hostname, @graphql_uri.port, @request_options) do |http| - http.request(request) + conn = Faraday.new do |faraday| + faraday.response :follow_redirects + faraday.adapter Faraday.default_adapter end + response = conn.get('http://purl.obolibrary.org/meta/ontologies.jsonld') - parsed = JSON.parse(response.body) - parsed.dig("data", "repository", "object", "text") + MultiJson.load(response.body)['ontologies'] end - end end end diff --git a/ncbo_cron.gemspec b/ncbo_cron.gemspec index c8faa03d..e48d7e7f 100644 --- a/ncbo_cron.gemspec +++ b/ncbo_cron.gemspec @@ -15,6 +15,8 @@ Gem::Specification.new do |gem| gem.require_paths = ["lib"] gem.add_dependency("dante") + gem.add_dependency('faraday', '~> 2') + gem.add_dependency('faraday-follow_redirects', '~> 0') gem.add_dependency("goo") gem.add_dependency("google-analytics-data") gem.add_dependency("mlanett-redis-lock")