diff --git a/Gemfile b/Gemfile index bcf5f137..8b180c6e 100644 --- a/Gemfile +++ b/Gemfile @@ -10,7 +10,6 @@ gem 'ffi' gem "google-apis-analytics_v3" gem 'google-analytics-data' -gem 'mail', '2.6.6' gem 'multi_json' gem 'oj', '~> 3.0' gem 'parseconfig' @@ -25,10 +24,10 @@ gem 'sys-proctable' gem 'cube-ruby', require: 'cube' # NCBO -gem 'goo', github: 'ncbo/goo', branch: 'master' -gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'master' -gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'master' -gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' +gem 'goo', github: 'ncbo/goo', branch: 'develop' +gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'develop' +gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'develop' +gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop' group :test do gem 'email_spec' @@ -36,4 +35,5 @@ group :test do gem 'simplecov' gem 'simplecov-cobertura' # for codecov.io gem 'test-unit-minitest' + gem 'webrick' end diff --git a/Gemfile.lock b/Gemfile.lock index 457464a9..6216881f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 75436fe8e387febc53e34ee31ff0e6dd837a9d3f - branch: master + revision: f00c8f812c4a1f5eaacb4a59b708d0b228dcd41a + branch: develop specs: goo (0.0.2) addressable (~> 2.8) @@ -15,8 +15,8 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: 63c986880aa88c9384043e6611a682434a14aba7 - branch: master + revision: fe0783c542479ed0a12da5663ef28e1673e80eba + branch: develop specs: ncbo_annotator (0.0.1) goo @@ -26,8 +26,8 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 0003905c5d101f4df17e35784c0d15d2f5d42110 - branch: master + revision: a973d62ca8be3a6cd86677d9bed24f361f580882 + branch: develop specs: ontologies_linked_data (0.0.1) activesupport @@ -46,8 +46,8 @@ GIT GIT remote: https://github.com/ncbo/sparql-client.git - revision: d418d56a6c9ff5692f925b45739a2a1c66bca851 - branch: master + revision: 55e7dbf858eb571c767bc67868f9af61663859cb + branch: develop specs: sparql-client (1.0.1) json_pure (>= 1.4) @@ -85,13 +85,14 @@ GEM connection_pool (2.4.1) cube-ruby (0.0.3) dante (0.2.0) + date (3.3.4) declarative (0.0.20) docile (1.4.0) domain_name (0.6.20240107) - email_spec (2.1.1) + email_spec (2.2.2) htmlentities (~> 4.3.3) launchy (~> 2.1) - mail (~> 2.6) + mail (~> 2.7) faraday (2.8.1) base64 faraday-net_http (>= 2.0, < 3.1) @@ -108,15 +109,15 @@ GEM googleapis-common-protos-types (>= 1.11.0, < 2.a) googleauth (~> 1.9) grpc (~> 1.59) - google-analytics-data (0.5.0) + google-analytics-data (0.6.0) google-analytics-data-v1beta (>= 0.11, < 2.a) google-cloud-core (~> 1.6) - google-analytics-data-v1beta (0.11.2) + google-analytics-data-v1beta (0.12.0) gapic-common (>= 0.21.1, < 2.a) google-cloud-errors (~> 1.0) - google-apis-analytics_v3 (0.14.0) - google-apis-core (>= 0.12.0, < 2.a) - google-apis-core (0.13.0) + google-apis-analytics_v3 (0.15.0) + google-apis-core (>= 0.14.0, < 2.a) + google-apis-core (0.14.1) addressable (~> 2.5, >= 2.5.1) googleauth (~> 1.9) httpclient (>= 2.8.1, < 3.a) @@ -124,35 +125,31 @@ GEM representable (~> 3.0) retriable (>= 2.0, < 4.a) rexml - google-cloud-core (1.6.1) + google-cloud-core (1.7.0) google-cloud-env (>= 1.0, < 3.a) google-cloud-errors (~> 1.0) google-cloud-env (2.1.1) faraday (>= 1.0, < 3.a) - google-cloud-errors (1.3.1) - google-protobuf (3.25.2) - google-protobuf (3.25.2-x86_64-darwin) - google-protobuf (3.25.2-x86_64-linux) - googleapis-common-protos (1.4.0) - google-protobuf (~> 3.14) - googleapis-common-protos-types (~> 1.2) - grpc (~> 1.27) - googleapis-common-protos-types (1.12.0) + google-cloud-errors (1.4.0) + google-protobuf (3.25.3) + google-protobuf (3.25.3-x86_64-linux) + googleapis-common-protos (1.5.0) google-protobuf (~> 3.18) - googleauth (1.10.0) + googleapis-common-protos-types (~> 1.7) + grpc (~> 1.41) + googleapis-common-protos-types (1.13.0) + google-protobuf (~> 3.18) + googleauth (1.11.0) faraday (>= 1.0, < 3.a) google-cloud-env (~> 2.1) jwt (>= 1.4, < 3.0) multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.61.0) - google-protobuf (~> 3.25) - googleapis-common-protos-types (~> 1.0) - grpc (1.61.0-x86_64-darwin) + grpc (1.62.0) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) - grpc (1.61.0-x86_64-linux) + grpc (1.62.0-x86_64-linux) google-protobuf (~> 3.25) googleapis-common-protos-types (~> 1.0) htmlentities (4.3.4) @@ -164,25 +161,38 @@ GEM concurrent-ruby (~> 1.0) json (2.7.1) json_pure (2.7.1) - jwt (2.7.1) + jwt (2.8.1) + base64 launchy (2.5.2) addressable (~> 2.8) - libxml-ruby (5.0.2) + libxml-ruby (5.0.3) logger (1.6.0) macaddr (1.7.2) systemu (~> 2.6.5) - mail (2.6.6) - mime-types (>= 1.16, < 4) + mail (2.8.1) + mini_mime (>= 0.1.1) + net-imap + net-pop + net-smtp method_source (1.0.0) mime-types (3.5.2) mime-types-data (~> 3.2015) - mime-types-data (3.2024.0206) + mime-types-data (3.2024.0305) mini_mime (1.1.5) minitest (4.7.5) mlanett-redis-lock (0.2.7) redis multi_json (1.15.0) net-http-persistent (2.9.4) + net-imap (0.4.10) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.2) + timeout + net-smtp (0.4.0.1) + net-protocol netrc (0.11.0) oj (3.16.3) bigdecimal (>= 3.0) @@ -196,7 +206,7 @@ GEM coderay (~> 1.1) method_source (~> 1.0) public_suffix (5.0.4) - rack (3.0.9) + rack (3.0.9.1) rack-test (2.1.0) rack (>= 1.3) rake (13.1.0) @@ -204,7 +214,7 @@ GEM addressable (>= 2.2) redis (5.1.0) redis-client (>= 0.17.0) - redis-client (0.20.0) + redis-client (0.21.0) connection_pool representable (3.2.0) declarative (< 0.1.0) @@ -244,16 +254,17 @@ GEM systemu (2.6.5) test-unit-minitest (0.9.1) minitest (~> 4.7) + timeout (0.4.1) trailblazer-option (0.1.2) tzinfo (2.0.6) concurrent-ruby (~> 1.0) uber (0.1.0) uuid (2.3.9) macaddr (~> 1.0) + webrick (1.8.1) PLATFORMS ruby - x86_64-darwin-21 x86_64-linux DEPENDENCIES @@ -263,7 +274,6 @@ DEPENDENCIES goo! google-analytics-data google-apis-analytics_v3 - mail (= 2.6.6) minitest (< 5.0) multi_json ncbo_annotator! @@ -281,6 +291,7 @@ DEPENDENCIES sparql-client! sys-proctable test-unit-minitest + webrick BUNDLED WITH 2.4.22 diff --git a/docker-compose.yml b/docker-compose.yml index af25ac24..44c9fda4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,9 +63,9 @@ services: GOO_BACKEND_NAME: ag GOO_PORT: 10035 GOO_HOST: agraph-ut - GOO_PATH_QUERY: /repositories/bioportal_test - GOO_PATH_DATA: /repositories/bioportal_test/statements - GOO_PATH_UPDATE: /repositories/bioportal_test/statements + GOO_PATH_QUERY: /repositories/ontoportal_test + GOO_PATH_DATA: /repositories/ontoportal_test/statements + GOO_PATH_UPDATE: /repositories/ontoportal_test/statements profiles: - agraph depends_on: @@ -93,7 +93,7 @@ services: - 4store solr-ut: - image: ontoportal/solr-ut:0.0.2 + image: ontoportal/solr-ut:0.1.0 healthcheck: test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"] start_period: 3s @@ -112,7 +112,7 @@ services: retries: 5 agraph-ut: - image: franzinc/agraph:v8.0.1 + image: franzinc/agraph:v8.1.0 platform: linux/amd64 environment: - AGRAPH_SUPER_USER=test @@ -122,12 +122,12 @@ services: # - 10035:10035 command: > bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start - ; agtool repos create bioportal_test + ; agtool repos create --supersede ontoportal_test ; agtool users add anonymous - ; agtool users grant anonymous root:bioportal_test:rw + ; agtool users grant anonymous root:ontoportal_test:rw ; tail -f /agraph/data/agraph.log" healthcheck: - test: ["CMD-SHELL", "agtool storage-report bioportal_test || exit 1"] + test: ["CMD-SHELL", "agtool storage-report ontoportal_test || exit 1"] start_period: 30s #AllegroGraph can take a loooooong time to start interval: 20s timeout: 10s diff --git a/lib/ncbo_cron/ontology_analytics.rb b/lib/ncbo_cron/ontology_analytics.rb index c5a4de00..3a0f25ee 100644 --- a/lib/ncbo_cron/ontology_analytics.rb +++ b/lib/ncbo_cron/ontology_analytics.rb @@ -158,15 +158,15 @@ def merge_and_fill_missing_data(ga4_data) ua_ga4_intersecting_month = Date.parse(GA4_START_DATE).month.to_s # add up hits for June of 2023 (the only intersecting month between UA and GA4) - ua_data.each do |acronym, _| - if ga4_data.has_key?(acronym) - if ga4_data[acronym][ua_ga4_intersecting_year].has_key?(ua_ga4_intersecting_month) - ua_data[acronym][ua_ga4_intersecting_year][ua_ga4_intersecting_month] += - ga4_data[acronym][ua_ga4_intersecting_year][ua_ga4_intersecting_month] - # delete data for June of 2023 from ga4_data to avoid overwriting when merging - ga4_data[acronym][ua_ga4_intersecting_year].delete(ua_ga4_intersecting_month) - end - end + ua_data.each do |acronym| + next unless ga4_data.key?(acronym) + next unless ga4_data[acronym].key?(ua_ga4_intersecting_year) + next unless ga4_data[acronym][ua_ga4_intersecting_year].key?(ua_ga4_intersecting_month) + + ua_data[acronym][ua_ga4_intersecting_year][ua_ga4_intersecting_month] += + ga4_data[acronym][ua_ga4_intersecting_year][ua_ga4_intersecting_month] + # delete data for June of 2023 from ga4_data to avoid overwriting when merging + ga4_data[acronym][ua_ga4_intersecting_year].delete(ua_ga4_intersecting_month) end end