Merge pull request #570 from Sage-Bionetworks/develop

Sprint 16-19 updates

.Rbuildignore

@@ -1,2 +1,3 @@
+^\.pre-commit-config\.yaml$
 ^renv$
 ^renv\.lock$

.github/workflows/docker_build.yml

@@ -8,13 +8,13 @@ on:
   push:
     tags:
       - '*beta*'
+      - '*rc*'
       - v[0-9]+.[0-9]+.[0-9]+
-    branches:
-      - 'dev*'
 
 env:
   REGISTRY: ghcr.io
   IMAGE_PATH: ghcr.io/${{ github.repository }}
+  DCA_VERSION: ${{ github.REF_NAME }}
 
 jobs:
   build-and-push-image:
@@ -51,4 +51,7 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            DCA_VERSION=${{ env.DCA_VERSION }}
+
 

.github/workflows/shinyapps_deploy.yml

@@ -1,16 +1,10 @@
 name: shiny-deploy
 
 on:
-  push:
+  pull_request:
     branches:
       - main
-      - 'dev*'
-    tags:
-      - v[0-9]+.[0-9]+.[0-9]+
-    paths-ignore:
-      - '.github/ISSUE_TEMPLATE/**'
-      - '**/*.md'
-      - '**/.gitignore'
+
 jobs:
   shiny-deploy:
     runs-on: ubuntu-latest

.github/workflows/test_schematic_api.yml

@@ -7,11 +7,9 @@
 name: test-schematic-api
 
 on:
-  push:
+  pull_request:
     branches:
-      - develop
-    paths-ignore:
-      - .github/ISSUE_TEMPLATE/**
+      - main
 
 jobs:
   test-schematic-rest-api:
@@ -89,5 +87,3 @@ jobs:
         run: |
           devtools::test()
 
-
-

.pre-commit-config.yaml

@@ -0,0 +1,73 @@
+# All available hooks: https://pre-commit.com/hooks.html
+# R specific hooks: https://github.com/lorenzwalthert/precommit
+repos:
+-   repo: https://github.com/lorenzwalthert/precommit
+    rev: v0.3.2.9019
+    hooks: 
+    -   id: style-files
+        args: [--style_pkg=styler, --style_fun=tidyverse_style]    
+    -   id: roxygenize
+    # codemeta must be above use-tidy-description when both are used
+    # -   id: codemeta-description-updated
+    -   id: use-tidy-description
+    -   id: spell-check
+        exclude: >
+          (?x)^(
+          .*\.[rR]|
+          .*\.feather|
+          .*\.jpeg|
+          .*\.pdf|
+          .*\.png|
+          .*\.py|
+          .*\.RData|
+          .*\.rds|
+          .*\.Rds|
+          .*\.Rproj|
+          .*\.sh|
+          (.*/|)\.gitignore|
+          (.*/|)\.gitlab-ci\.yml|
+          (.*/|)\.lintr|
+          (.*/|)\.pre-commit-.*|
+          (.*/|)\.Rbuildignore|
+          (.*/|)\.Renviron|
+          (.*/|)\.Rprofile|
+          (.*/|)\.travis\.yml|
+          (.*/|)appveyor\.yml|
+          (.*/|)NAMESPACE|
+          (.*/|)renv/settings\.dcf|
+          (.*/|)renv\.lock|
+          (.*/|)WORDLIST|
+          \.github/workflows/.*|
+          data/.*|
+          )$
+    -   id: lintr
+    -   id: readme-rmd-rendered
+    -   id: parsable-R
+    -   id: no-browser-statement
+    -   id: no-debug-statement
+    -   id: deps-in-desc
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks: 
+    -   id: check-added-large-files
+        args: ['--maxkb=200']
+    -   id: file-contents-sorter
+        files: '^\.Rbuildignore$'
+    -   id: end-of-file-fixer
+        exclude: '\.Rd'
+-   repo: https://github.com/pre-commit-ci/pre-commit-ci-config
+    rev: v1.5.1
+    hooks:
+    # Only reuiqred when https://pre-commit.ci is used for config validation
+    -   id: check-pre-commit-ci-config
+-   repo: local
+    hooks:
+    -   id: forbid-to-commit
+        name: Don't commit common R artifacts
+        entry: Cannot commit .Rhistory, .RData, .Rds or .rds.
+        language: fail
+        files: '\.(Rhistory|RData|Rds|rds)$'
+        # `exclude: <regex>` to allow committing specific files
+
+ci:
+    autoupdate_schedule: monthly

Dockerfile

@@ -1,5 +1,9 @@
 FROM ghcr.io/afwillia/shiny-base:release-update-node
-LABEL maintainer="Anthony [email protected]"
+
+# add version tag as a build argument
+ARG DCA_VERSION
+
+ENV DCA_VERSION=$DCA_VERSION
 
 USER root
 RUN apt-get update

R/schematic_rest_api.R

@@ -57,7 +57,8 @@ manifest_generate <- function(url="http://localhost:3001/v1/manifest/generate",
                               schema_url="https://raw.githubusercontent.com/ncihtan/data-models/main/HTAN.model.jsonld", #nolint
                               title, data_type,
                               use_annotations="false", dataset_id=NULL,
-                              asset_view, output_format, access_token = NULL) {
+                              asset_view, output_format, access_token = NULL,
+                              strict_validation = FALSE) {
 
   req <- httr::GET(url,
                    query = list(
@@ -68,7 +69,8 @@ manifest_generate <- function(url="http://localhost:3001/v1/manifest/generate",
                      dataset_id=dataset_id,
                      asset_view=asset_view,
                      output_format=output_format,
-                     access_token = access_token
+                     access_token = access_token,
+                     strict_validation = strict_validation
                    ))
 
   check_success(req)

dca_startup.sh

@@ -3,6 +3,7 @@
 # Pass environment variable to Shiny
 echo "" >> .Renviron
 echo R_CONFIG_ACTIVE=$R_CONFIG_ACTIVE >> .Renviron
+echo DCA_VERSION=$DCA_VERSION >> .Renviron
 
 # Now run the base start-up script
 ./startup.sh

functions/schematic_rest_api.R

@@ -57,7 +57,8 @@ manifest_generate <- function(url="http://localhost:3001/v1/manifest/generate",
                               schema_url="https://raw.githubusercontent.com/ncihtan/data-models/main/HTAN.model.jsonld", #nolint
                               title, data_type,
                               use_annotations="false", dataset_id=NULL,
-                              asset_view, output_format, access_token = NULL) {
+                              asset_view, output_format, access_token = NULL,
+                              strict_validation = FALSE) {
 
   req <- httr::GET(url,
                    query = list(
@@ -68,7 +69,8 @@ manifest_generate <- function(url="http://localhost:3001/v1/manifest/generate",
                      dataset_id=dataset_id,
                      asset_view=asset_view,
                      output_format=output_format,
-                     access_token = access_token
+                     access_token = access_token,
+                     strict_validation = strict_validation
                    ))
 
   check_success(req)
@@ -157,7 +159,7 @@ model_submit <- function(url="http://localhost:3001/v1/model/submit",
                          schema_url="https://raw.githubusercontent.com/ncihtan/data-models/main/HTAN.model.jsonld", #notlint
                          data_type, dataset_id, restrict_rules=FALSE, access_token, json_str=NULL, asset_view,
                          use_schema_label=TRUE, manifest_record_type="table_and_file", file_name,
-                         table_manipulation="replace") {
+                         table_manipulation="replace", hide_blanks=FALSE) {
   req <- httr::POST(url,
                     #add_headers(Authorization=paste0("Bearer ", pat)),
                     query=list(
@@ -170,7 +172,8 @@ model_submit <- function(url="http://localhost:3001/v1/model/submit",
                       asset_view=asset_view,
                       use_schema_label=use_schema_label,
                       manifest_record_type=manifest_record_type,
-                      table_manipulation=table_manipulation),
+                      table_manipulation=table_manipulation,
+                      hide_blanks=hide_blanks),
                     body=list(file_name=httr::upload_file(file_name))
                     #body=list(file_name=file_name)
   )

global.R

@@ -29,7 +29,7 @@ suppressPackageStartupMessages({
 # Set up futures/promises for asynchronous calls
 ncores <- availableCores()
 message(sprintf("Available cores: %s", ncores))
-plan(multisession, workers = ncores)
+plan(multicore, workers = ncores)
 
 # import R files
 source_files <- list.files(c("functions", "modules"), pattern = "*\\.R$", recursive = TRUE, full.names = TRUE)
@@ -61,8 +61,18 @@ if (dca_schematic_api == "rest") {
   Sys.getenv("DCA_API_PORT"),
   sep = ":")
   )
+
+  # Get Schematic version
+  get_schematic_version <- try(httr::GET(file.path(api_uri, "v1/version")), silent=TRUE)
+  if (inherits(get_schematic_version, "try-error")) {
+    schematic_version <- ""
+  } else if (httr::http_error(get_schematic_version)) {
+    schematic_version <- ""
+  } else schematic_version <- httr::content(get_schematic_version)
 }
 
+dca_version <- Sys.getenv("DCA_VERSION")
+
 dca_synapse_api <- Sys.getenv("DCA_SYNAPSE_PROJECT_API")
 
 # update port if running app locally

modules/DTable.R

@@ -57,7 +57,7 @@ DTableServer <- function(id, data, escape = TRUE,
         df <- df %>% formatStyle(1:ncol(data), border = "1px solid #ddd")
       }
 
-      output$table <- renderDT(df)
+      output$table <- renderDT(df, future = TRUE)
     }
   )
 }

modules/csvInfile.R

@@ -28,7 +28,10 @@ csvInfileServer <- function(id, na = c("", "NA"), colsAsCharacters = FALSE, keep
         }
 
         if (trimEmptyRows) {
-          infile <- infile %>% filter_all(any_vars(!is.na(.)))
+          # Originally used the following, but got C buffer error w/ 74+ columns
+          #infile <- infile %>% filter_all(any_vars(!is.na(.)))
+          keep_rows <- apply(infile, 1, function(x) !all(is.na(x)))
+          if (length(keep_rows) > 0) infile <- infile[which(keep_rows), ]
         }
 
         if (keepBlank) {

renv.lock

@@ -909,7 +909,7 @@
     },
     "renv": {
       "Package": "renv",
-      "Version": "1.0.0",
+      "Version": "0.17.3",
       "OS_type": null,
       "Repository": "CRAN",
       "Source": "Repository"

renv/activate.R

@@ -2,7 +2,7 @@
 local({
 
   # the requested version of renv
-  version <- "1.0.0"
+  version <- "0.17.3"
 
   # the project directory
   project <- getwd()