From 1073d8a804c43039f9f823f239790653e9cf38bc Mon Sep 17 00:00:00 2001 From: Robert Allaway Date: Thu, 24 Feb 2022 10:13:16 -0800 Subject: [PATCH 1/9] push to production with tag instead of branch To keep the list of branches clean, and to allow use of the GH "releases" feature which creates _tags_ rather than branches. --- .github/workflows/shinyapps_deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/shinyapps_deploy.yml b/.github/workflows/shinyapps_deploy.yml index 4e720be0..c19478b6 100644 --- a/.github/workflows/shinyapps_deploy.yml +++ b/.github/workflows/shinyapps_deploy.yml @@ -10,6 +10,7 @@ on: branches: - shiny-server-main - shiny-server-develop # add develop for testing for now + tags: - release* paths-ignore: - .github/ISSUE_TEMPLATE/** From 44337669aa38980446f01fbff151e8633e18e38b Mon Sep 17 00:00:00 2001 From: rrchai Date: Fri, 25 Feb 2022 22:34:56 +0000 Subject: [PATCH 2/9] update functions/metadata_model.py to enable array in data type of config.yml --- functions/metadata_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/metadata_model.py b/functions/metadata_model.py index 1a4a9176..8a729afc 100644 --- a/functions/metadata_model.py +++ b/functions/metadata_model.py @@ -8,6 +8,6 @@ inputMModelLocationType = CONFIG["model"]["input"]["file_type"] manifest_title = CONFIG["manifest"]["title"] -manifest_data_type = CONFIG["manifest"]["data_type"] +manifest_data_type = CONFIG["manifest"]["data_type"][0] metadata_model = MetadataModel(inputMModelLocation, inputMModelLocationType) From 8fd123b2392e765eda64fdecf8795b3999d719aa Mon Sep 17 00:00:00 2001 From: Rongrong Chai <73901500+rrchai@users.noreply.github.com> Date: Wed, 2 Mar 2022 15:54:07 -0500 Subject: [PATCH 3/9] Simplify the setup process regarding python dependencies and schematic config (#289) * update setup * add required to use_virtualenv * update-setuo * no need stringsAsFactors = FALSE * update workflow * update schematic config * test staging * ignore md * activate .venv each bash * update .gitignore * update branch name * use r function on base64 * update set_gh_secrets * update set_gh_secrets --- .github/schematic_config.yml | 41 --------------- .github/set_gh_secrets.R | 31 ----------- .github/workflows/shinyapps_deploy.yml | 35 ++++++------ .gitignore | 3 +- README.md | 8 +-- functions/metadata_model.py | 2 +- global.R | 2 +- requirements.txt | 61 --------------------- schematic_config.yml | 40 ++++++++++++++ set_gh_secrets.R | 73 ++++++++++++++++++++++++++ 10 files changed, 135 insertions(+), 161 deletions(-) delete mode 100644 .github/schematic_config.yml delete mode 100644 .github/set_gh_secrets.R delete mode 100644 requirements.txt create mode 100644 schematic_config.yml create mode 100644 set_gh_secrets.R diff --git a/.github/schematic_config.yml b/.github/schematic_config.yml deleted file mode 100644 index e8c23666..00000000 --- a/.github/schematic_config.yml +++ /dev/null @@ -1,41 +0,0 @@ -# During the github workflow to auto deploy the app -# This config file will be used to overwrite the config.yml in the schematic folder -# -# Please modify the configuration values based on your project - -# Do not change the 'definitions' section unless you know what you're doing -definitions: - synapse_config: '.synapseConfig' - creds_path: 'credentials.json' - token_pickle: 'token.pickle' - service_acct_creds: 'schematic_service_account_creds.json' - -synapse: - master_fileview: 'syn20446927' # fileview of project with datasets on Synapse - manifest_folder: 'manifests' # manifests will be downloaded to this folder - manifest_filename: 'synapse_storage_manifest.csv' # name of the manifest file in the project dataset - token_creds: 'syn23643259' # synapse ID of credentials.json file - service_acct_creds: 'syn25171627' # synapse ID of service_account_creds.json file - -manifest: - title: 'Patient Manifest' # title of metadata manifest file - data_type: 'Patient' # component or data type from the data model - -model: - input: - download_url: 'https://raw.githubusercontent.com/ncihtan/data-models/main/HTAN.model.jsonld' # url to download JSON-LD data model - location: 'data-models/HTAN.model.jsonld' # path to JSON-LD data model - file_type: 'local' # only type "local" is supported currently - -style: - google_manifest: - req_bg_color: - red: 0.9215 - green: 0.9725 - blue: 0.9803 - opt_bg_color: - red: 1.0 - green: 1.0 - blue: 0.9019 - master_template_id: '1LYS5qE4nV9jzcYw5sXwCza25slDfRA1CIg3cs-hCdpU' - strict_validation: true diff --git a/.github/set_gh_secrets.R b/.github/set_gh_secrets.R deleted file mode 100644 index df72ff74..00000000 --- a/.github/set_gh_secrets.R +++ /dev/null @@ -1,31 +0,0 @@ -# require github cli package - 'gh': https://cli.github.com/manual/ - -args <- commandArgs(trailingOnly = TRUE) - -repo <- basename(Sys.getenv("PWD")) - -if (repo %in% c("schematic", "data-models")) stop("You are not in the data curator folder !!!") - -secret_lists <- yaml::yaml.load_file(toString(args[1]))$definitions -message("Assuming all definition files are stored in the schematic folder ...") - -for (f in c("synapse_config", "creds_path", "service_acct_creds")) { - system( - sprintf( - "gh secret set %s < %s", - toupper(paste0("schematic_", f)), file.path("schematic", secret_lists[f]) - ) - ) -} - -# token.pickle is a binary file -# If you want to manually add to the secret: -# 1. copy below code snippet and run it in the terminal -# 2. copy content from 'token.pickle.b64' to your github secret -system( - " - less schematic/token.pickle | base64 --wrap=0 > schematic/token.pickle.b64; - gh secret set SCHEMATIC_TOKEN_PICKLE < schematic/token.pickle.b64; - " -) -unlink("schematic/token.pickle.b64") \ No newline at end of file diff --git a/.github/workflows/shinyapps_deploy.yml b/.github/workflows/shinyapps_deploy.yml index c19478b6..d6196125 100644 --- a/.github/workflows/shinyapps_deploy.yml +++ b/.github/workflows/shinyapps_deploy.yml @@ -8,12 +8,14 @@ name: shiny-deploy on: push: branches: - - shiny-server-main - - shiny-server-develop # add develop for testing for now + - main + - develop tags: - - release* + - v*.*.* paths-ignore: - - .github/ISSUE_TEMPLATE/** + - '.github/ISSUE_TEMPLATE/**' + - '**/*.md' + - '**/.gitignore' jobs: shiny-deploy: @@ -45,15 +47,10 @@ jobs: run: | R -f install-pkgs.R - - name: Install Python Packages Dependencies - shell: bash - run: | - source .venv/bin/activate - pip3 install -r requirements.txt - - name: Install Schematic shell: bash run: | + # has to activate each bash step source .venv/bin/activate # use 'poetry' to install schematic from the develop branch pip3 install poetry @@ -68,19 +65,17 @@ jobs: source .venv/bin/activate # download data model to the correct location R -e ' - config <- yaml::yaml.load_file(".github/schematic_config.yml"); + config <- yaml::yaml.load_file("schematic_config.yml"); url <- config$model$input$download_url; path <- config$model$input$location; system(sprintf("mkdir -p %s", dirname(path))); system(sprintf("wget %s -O %s", url, path)); ' - # overwrite the config.yml in schematic - mv -f .github/schematic_config.yml schematic/config.yml # write out configuration files using github secrets - echo "${{ secrets.SCHEMATIC_SYNAPSE_CONFIG }}" > schematic/.synapseConfig - echo "${{ secrets.SCHEMATIC_SERVICE_ACCT_CREDS }}" > schematic/schematic_service_account_creds.json - echo "${{ secrets.SCHEMATIC_CREDS_PATH }}" > schematic/credentials.json - echo "${{ secrets.SCHEMATIC_TOKEN_PICKLE }}" | base64 -d > schematic/token.pickle + echo "${{ secrets.SCHEMATIC_SYNAPSE_CONFIG }}" > .synapseConfig + echo "${{ secrets.SCHEMATIC_SERVICE_ACCT_CREDS }}" > schematic_service_account_creds.json + echo "${{ secrets.SCHEMATIC_CREDS_PATH }}" > credentials.json + echo "${{ secrets.SCHEMATIC_TOKEN_PICKLE }}" | base64 -d > token.pickle - name: zip virtual env shell: bash @@ -93,10 +88,12 @@ jobs: - name: Authorize and deploy app shell: Rscript {0} run: | - branch <- Sys.getenv("GITHUB_REF_NAME") + # if there is a tag, 'refName' will be tag name + refName <- Sys.getenv("GITHUB_REF_NAME") repo <- Sys.getenv("GITHUB_REPOSITORY") appName <- strsplit(repo, "/")[[1]][2] - if (!startsWith(branch, "release")) { + # if tag is v*.*.*, deploy to prod, otherwise to staging + if (!grepl("v[0-9]+.[0-9]+.[0-9]+", refName)) { appName <- paste(appName, "staging", sep = "-") } rsConnectUser <- "${{ secrets.RSCONNECT_USER }}" diff --git a/.gitignore b/.gitignore index dab873e6..4d97d450 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,8 @@ schematic/ **/token.pickle **/__pycache__ oauth_config.yml +schematic_config.yml .project .settings/** .venv -data-models/ +data-models/ \ No newline at end of file diff --git a/README.md b/README.md index abe759f6..aa523087 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,7 @@ Follow the steps below to make sure the _Data Curator App_ is fully setup to wor python -m venv .venv source .venv/bin/activate -4. Install required Python pacakges dependencies: - - pip install -r requirements.txt - -5. Install required R pacakges dependencies: +4. Install required R pacakges dependencies: R -f install-pkgs.R @@ -50,7 +46,7 @@ Follow the steps below to make sure the _Data Curator App_ is fully setup to wor poetry build pip install dist/schematicpy-1.0.0-py3-none-any.whl -3. Set up the `schematic` configuration. To do so, follow the instructions on the [schematic's documentation](https://sage-schematic.readthedocs.io/en/develop/index.html#package-installation-and-setup) +3. Modify the `schematic_config.yml` to set up schematic configuration. To do so, follow the instructions on the [schematic's documentation](https://sage-schematic.readthedocs.io/en/develop/index.html#package-installation-and-setup) ### Data Model Configuration diff --git a/functions/metadata_model.py b/functions/metadata_model.py index 8a729afc..16220cba 100644 --- a/functions/metadata_model.py +++ b/functions/metadata_model.py @@ -2,7 +2,7 @@ from schematic import CONFIG -config = CONFIG.load_config("schematic/config.yml") +config = CONFIG.load_config("schematic_config.yml") inputMModelLocation = CONFIG["model"]["input"]["location"] inputMModelLocationType = CONFIG["model"]["input"]["file_type"] diff --git a/global.R b/global.R index 8d76f7d7..3fc5ec00 100644 --- a/global.R +++ b/global.R @@ -26,7 +26,7 @@ system("chmod -R +x .venv") # Activate virtual env # Don't necessarily have to set `RETICULATE_PYTHON` env variable Sys.unsetenv("RETICULATE_PYTHON") -reticulate::use_virtualenv(file.path(getwd(), ".venv")) +reticulate::use_virtualenv(file.path(getwd(), ".venv"), required = TRUE) suppressPackageStartupMessages({ library(shiny) diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index ceddb075..00000000 --- a/requirements.txt +++ /dev/null @@ -1,61 +0,0 @@ -attrs==19.3.0 -cachetools==4.1.1 -certifi==2020.6.20 -cffi==1.14.0 -chardet==3.0.4 -click>=7.1.2 -click-log>=0.3.2 -cryptography==3.3.2 -decorator==4.4.2 -Deprecated==1.2.4 -entrypoints==0.3 -fastjsonschema==2.14.4 -google-api-core==1.26.3 -google-api-python-client>=1.12.8 -google-auth>=1.19.1 -google-auth-httplib2>=0.0.4 -google-auth-oauthlib>=0.4.4 -googleapis-common-protos==1.53.0 -graphviz>=0.16 -httplib2==0.19.0 -idna==2.10 -importlib-metadata==1.7.0 -inflection==0.5.1 -isodate==0.6.0 -jsonschema==3.2.0 -keyring==12.0.2 -keyrings.alt==3.1 -networkx==2.5.1 -numpy==1.21.0 -oauth2client==3.0.0 -oauthlib==3.1.0 -orderedset==2.0.1 -packaging==20.9 -pandas==1.2.3 -pip==21.1 -protobuf==3.15.7 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 -pycparser==2.20 -pygsheets==2.0.5 -pyparsing==2.4.7 -pyrsistent==0.16.0 -python-dateutil==2.8.1 -pytz==2020.1 -PyYAML==5.4.1 -rdflib==5.0.0 -requests>=2.24.0 -requests-oauthlib>=1.3.0 -rsa==4.7 -schematicpy==1.0.0 -SecretStorage==2.3.1 -setuptools==52.0.0 -six==1.15.0 -synapseclient==2.5.1 -tabletext==0.1 -toml==0.10.2 -uritemplate==3.0.1 -urllib3==1.26.5 -wheel==0.34.2 -wrapt==1.12.1 -zipp==3.1.0 \ No newline at end of file diff --git a/schematic_config.yml b/schematic_config.yml new file mode 100644 index 00000000..08547a01 --- /dev/null +++ b/schematic_config.yml @@ -0,0 +1,40 @@ +# Do not change the 'definitions' section unless you know what you're doing +definitions: + synapse_config: ".synapseConfig" + creds_path: "credentials.json" + token_pickle: "token.pickle" + service_acct_creds: "schematic_service_account_creds.json" + +synapse: + master_fileview: 'syn23643253' + manifest_folder: 'manifests' + manifest_filename: 'synapse_storage_manifest.csv' + token_creds: 'syn23643259' + service_acct_creds: 'syn25171627' + +manifest: + # if making many manifests, just include name prefix + title: 'example' + # to make all manifests enter only 'all manifests' + data_type: + - 'Biospecimen' + - 'Patient' + +model: + input: + download_url: 'https://raw.githubusercontent.com/Sage-Bionetworks/schematic/develop/tests/data/example.model.jsonld' # url to download JSON-LD data model + location: 'schematic/tests/data/example.model.jsonld' # path to JSON-LD data mode + file_type: 'local' + +style: + google_manifest: + req_bg_color: + red: 0.9215 + green: 0.9725 + blue: 0.9803 + opt_bg_color: + red: 1.0 + green: 1.0 + blue: 0.9019 + master_template_id: '1LYS5qE4nV9jzcYw5sXwCza25slDfRA1CIg3cs-hCdpU' + strict_validation: true diff --git a/set_gh_secrets.R b/set_gh_secrets.R new file mode 100644 index 00000000..58c12617 --- /dev/null +++ b/set_gh_secrets.R @@ -0,0 +1,73 @@ +# require github cli package - 'gh': https://cli.github.com/manual/ + +args <- commandArgs(trailingOnly = TRUE) + +repo <- basename(Sys.getenv("PWD")) + +# Read OAuth credential +oauth_client <- yaml::yaml.load_file(args[1]) +client_id <- toString(oauth_client$CLIENT_ID) +client_secret <- toString(oauth_client$CLIENT_SECRET) + +# Read credential files locations +secret_lists <- yaml::yaml.load_file(args[2])$definitions +# Change working directory to the directory of config file +setwd(dirname(args[2])) + +# Validation +# Ensure client credentials are not empty +if (is.null(client_id) || nchar(client_id) == 0) stop(args[1], " is missing CLIENT_ID") +if (is.null(client_secret) || nchar(client_secret) == 0) stop(args[1], " is missing CLIENT_SECRET") +# Ensure it is not schematic or data-model repos +if (!grepl("data[-|_]curator", repo)) stop("You are not in the data curator folder !!!") +# Ensure all credential files exist +diff <- setdiff(c("synapse_config", "creds_path", "token_pickle", "service_acct_creds"), names(secret_lists)) +if (length(diff) > 0) { + stop(paste(diff, "not found in the config file", collapse = "\n")) +} + +# Add OAuth secrets via gh CLI +system( + sprintf( + " + gh secret set OAUTH_CLIENT_ID --body %s; + gh secret set OAUTH_CLIENT_SECRET --body %s; + ", + sQuote(client_id), sQuote(client_secret) + ) +) + +# Set secret names SCHEMATIC_* +secret_names <- toupper(paste0("schematic_", names(secret_lists))) + +# Add schematic secrets via gh CLI +for (i in seq_along(secret_lists)) { + if (!file.exists(toString(secret_lists[[i]]))) stop(secret_lists[[i]], " not found") + if (secret_names[i] == "SCHEMATIC_TOKEN_PICKLE") { + # token.pickle is a binary file + # If you want to manually add to the secret: + # 1. encode token.pickle in the terminal: base64 -w0 (linux) or base64 -b0 (macOS) + # 2. copy encoded character string to your github secret + token_str <- base64enc::base64encode(secret_lists[[i]]) + system( + sprintf( + " + gh secret set %s --body %s; + ", + secret_names[i], sQuote(token_str) + ) + ) + } else { + system( + sprintf( + "gh secret set %s < %s", + secret_names[i], secret_lists[[i]] + ) + ) + } +} + +cat(" + <<<<<<<< Important >>>>>>>> +Please manually add 'REPO_PAT', 'RSCONNECT_USER', 'RSCONNECT_SECRET', 'RSCONNECT_TOKEN' +check 'shinyapps_deploy.md' for details\n") \ No newline at end of file From b8d710a4303e6ce144ff189aa566a4c16a9d2c8c Mon Sep 17 00:00:00 2001 From: rrchai Date: Wed, 2 Mar 2022 23:34:05 +0000 Subject: [PATCH 4/9] update README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index aa523087..64684e27 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ Follow the steps below to make sure the _Data Curator App_ is fully setup to wor ### Data Curator App Setup -1. Clone this repo (front-end) with one single branch (i.e., _shiny-server-main_): +1. Clone this repo (front-end) with one single branch (i.e., _main_): - git clone --single-branch --branch shiny-server-main https://github.com/Sage-Bionetworks/data_curator.git + git clone --single-branch --branch main https://github.com/Sage-Bionetworks/data_curator.git 2. Create and modify the configuration file ([How to obtain OAuth Credential](https://github.com/Sage-Bionetworks/data_curator#Authentication)): From 54455367ae91b843e09510bba93701d2274bf1ea Mon Sep 17 00:00:00 2001 From: Anthony Williams Date: Thu, 3 Mar 2022 13:14:22 -0500 Subject: [PATCH 5/9] Add documentation for the deployment to Shinyapps.io (#292) * Remove shinyapps_deploy.txt and put into PR * Add shinyapps.io deployment steps * change shiny deploy doc to markdown * Update docs for deloyment Co-authored-by: Anthony Williams Co-authored-by: rrchai Co-authored-by: Rongrong Chai <73901500+rrchai@users.noreply.github.com> --- shinyapps_deploy.md | 60 ++++++++++++++++++++++++++++++++++++++++++++ shinyapps_deploy.txt | 58 ------------------------------------------ 2 files changed, 60 insertions(+), 58 deletions(-) create mode 100644 shinyapps_deploy.md delete mode 100644 shinyapps_deploy.txt diff --git a/shinyapps_deploy.md b/shinyapps_deploy.md new file mode 100644 index 00000000..24f0042b --- /dev/null +++ b/shinyapps_deploy.md @@ -0,0 +1,60 @@ +# Deploying Data Curator to shinyapps.io + +Anthony Williams + + 02-23-2022 + +Deploying an app to shinyapps.io is done with the [rsconnect package](https://github.com/rstudio/rsconnect/). +This can be done manually or automated in a [GitHub action](.github/workflows/shinyapps_deploy.yml). + +The GitHub action installs data curator's system and python dependencies on an Ubuntu [Rstudio server image](https://hub.docker.com/r/rocker/rstudio), and then pushes +the entire directory to shinyapps.io using `rsconnect`. + +## Setup Configuration of OAuth and Schematic +- See ["Data Curator App Setup #2"](https://github.com/Sage-Bionetworks/data_curator#data-curator-app-setup) to set up `oauth_config.yml` schematic. +- See ["Schematic Setup #3"](https://github.com/Sage-Bionetworks/data_curator#schematic-setup) to set up `schematic_config.yml` + +## Access shinyapps.io +Ask IT for access to the shinyapps.io account info in the Shared-Sage LastPass folder. Make sure you can view the site password. To login, enter credentials on [shinyapps.io](https://www.shinyapps.io/admin/#/login). + +## Add GitHub Secrets +Add shinyapps.io and synapse OAuth and schematic info to +[GitHub secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets) +for the repo. +```bash +*OAUTH_CLIENT_ID # your OAuth client ID +*OAUTH_CLIENT_SECRET # your OAuth client secret +*SCHEMATIC_SYNAPSE_CONFIG # content of .synapseConfig +*SCHEMATIC_SERVICE_ACCT_CREDS # content of schematic_service_account_creds.json +*SCHEMATIC_CREDS_PATH # content of credentials.json +*SCHEMATIC_TOKEN_PICKLE # content of token.pickle +RSCONNECT_USER # see the shinyapps.io record in LastPass +RSCONNECT_TOKEN # see the shinyapps.io record in LastPass +RSCONNECT_SECRET # see the shinyapps.io record in LastPass +REPO_PAT # GitHub personal access token +``` + +To automatically add credentials with * to GH secrets via [GitHub CLI](https://cli.github.com/manual/index) (Note: you still need to manually add shinyapps.io's credentials and GH PAT): + +```bash +# follow instructions to configure GH CLI +gh auth login +# add secrets for OAuth and schematic +Rscript set_gh_secrets.R oauth_config.yml schematic_config.yml +``` + +## Configure the [GitHub workflow file](.github/workflows/shinyapps_deploy.yml) + +### Specify branches to push to which instances +There are two instances of data curator on shinyapps.io, "production" and "staging". By default, the action is configured to deploy the app to the "production" instance if there are pushes on tags named as semantic versions (e.g. `v1.0.0`). While pushing changes to branches named "main" or "develop" will auto-deploy app to the staging instance. + +### Install data curator dependencies +Currently, data curator requires several system dependencies to run on shinyapps.io in addition to schematic's python dependencies. libcurl4-openssl-dev + +Set up a python virtual environment, install the `develop` version of schematic with `poetry`, and install necessary R packages as instructed in the [README](https://github.com/Sage-Bionetworks/data_curator/blob/main/README.md). + +### Update schematic configurations +Use GitHub secrets to write out the credentials files for OAuth and schematic, which are required for the configuration of data curator. + +### Deploy the app with rsconnect +To configure `rsconnect`, use `rsconnect::setAccountInfo(user, token, secret)` with the GH secretes started with "RSCONNECT". To deploy the app to "production" or "staging" instances based on what is setup in the workflow using `rsconnect::deployApp()`. diff --git a/shinyapps_deploy.txt b/shinyapps_deploy.txt deleted file mode 100644 index ab3401c1..00000000 --- a/shinyapps_deploy.txt +++ /dev/null @@ -1,58 +0,0 @@ -# Deploying Data Curator to shinyapps.io -23FEB2022 -Anthony Williams - -Deploying an app to shinyapps.io is done with the [rsconnect package](https://github.com/rstudio/rsconnect/) -This can be done manually or automated in a [GitHub action](https://docs.github.com/en/actions/quickstart). - -The GitHub action installs data curator's system and python dependencies on an Ubuntu -[Rstudio server image](https://hub.docker.com/r/rocker/rstudio) then pushes -the entire directory to shinyapps.io using rsconnect. - -# Access shinyapps.io -Ask IT for access to the shinyapps.io account info in the Shared-Sage LastPass -folder. Make sure you can view the site password. To login, enter credentials on -https://www.shinyapps.io/admin/#/login. - -# Add GitHub Secrets -Add shinyapps.io and synapse OAuth and schematic info to -[GitHub secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets) -for the repo. -OAUTH_CLIENT_ID -OAUTH_CLIENT_SECRET -RSCONNECT_USER -RSCONNECT_TOKEN -RSCONNECT_SECRET -SCHEMATIC_SYNAPSE_CONFIG - .synapseConfig -SCHEMATIC_SERVICE_ACCT_CREDS - schematic_service_account_creds.json -SCHEMATIC_CREDS_PATH - credentials.json -SCHEMATIC_TOKEN_PICKLE - token.pickle - -# Configure the GitHub workflow file - -## Specify branches to push to which instances -data_curator/.github/workflows/shinyapps_deploy.yml. -There are two instances of data curator on shinyapps.io, release and staging. -The action is configured to push branches named "release*" to the -release instance, while other hard-coded branches, like shiny-server-main, are -pushed to the staging instance. - -## Install data curator dependencies -Currently, data curator requires several system dependencies to run on shinyapps.io -in addition to schematic's python dependencies. libcurl4-openssl-dev - -Set up a python virtual environment, install the devel version of schematic -with poetry, and install necessary R packages as instructed in the README. - -## Update schematic configurations -Schematic requires configuration for data curator, so use a prepopulated file. -Download a [schematic_config.yml file](https://github.com/Sage-Bionetworks/data_curator/blob/shiny-server-main/.github/schematic_config.yml) and update its fields with GitHub secret information. -SCHEMATIC_SYNAPSE_CONFIG - .synapseConfig -SCHEMATIC_SERVICE_ACCT_CREDS - schematic_service_account_creds.json -SCHEMATIC_CREDS_PATH - credentials.json -SCHEMATIC_TOKEN_PICKLE - token.pickle - -## Deploy the app with rsconnect -Set the RSCONNECT user, secret, and token and create a -config.yml file with GitHub secrets. Set the shinyapps.io instance to release -or staging based on the name of the branch. Deply with rsconnect::deployApp(). From e15fcbf09b78a05775e6072975f2006e1efc2851 Mon Sep 17 00:00:00 2001 From: Lauren Wolfe <30699094+lakikowolfe@users.noreply.github.com> Date: Tue, 8 Mar 2022 09:22:30 -0800 Subject: [PATCH 6/9] match config to example schema (#269) * match config to example schema * Remove shinyapps_deploy.txt and put into PR * remove main_fileview, change community to example Co-authored-by: Anthony Williams Co-authored-by: Rongrong Chai <73901500+rrchai@users.noreply.github.com> --- www/config.json | 43 ++++--------------------------------------- 1 file changed, 4 insertions(+), 39 deletions(-) diff --git a/www/config.json b/www/config.json index 39853b65..43b19924 100644 --- a/www/config.json +++ b/www/config.json @@ -1,43 +1,8 @@ { "manifest_schemas": [ - {"display_name": "scRNA-seq Level 1", "schema_name": "ScRNA-seqLevel1", "type": "assay"}, - {"display_name": "scRNA-seq Level 2", "schema_name": "ScRNA-seqLevel2", "type": "assay"}, - {"display_name": "scRNA-seq Level 3", "schema_name": "ScRNA-seqLevel3", "type": "assay"}, - {"display_name": "scRNA-seq Level 4", "schema_name": "ScRNA-seqLevel4", "type": "assay"}, - {"display_name": "scATAC-seq Level 1", "schema_name": "ScATAC-seqLevel1", "type": "assay"}, - {"display_name": "scATAC-seq Level 4", "schema_name": "ScATAC-seqLevel4", "type": "assay"}, - {"display_name": "Bulk RNA-seq Level 1", "schema_name": "BulkRNA-seqLevel1", "type": "assay"}, - {"display_name": "Bulk RNA-seq Level 2", "schema_name": "BulkRNA-seqLevel2", "type": "assay"}, - {"display_name": "Bulk RNA-seq Level 3", "schema_name": "BulkRNA-seqLevel3", "type": "assay"}, - {"display_name": "Bulk WES Level 1", "schema_name": "BulkWESLevel1", "type": "assay"}, - {"display_name": "Bulk WES Level 2", "schema_name": "BulkWESLevel2", "type": "assay"}, - {"display_name": "Bulk WES Level 3", "schema_name": "BulkWESLevel3", "type": "assay"}, - {"display_name": "Imaging Level 2", "schema_name": "ImagingLevel2", "type": "assay"}, - {"display_name": "Imaging Level 3", "schema_name": "ImagingLevel3", "type": "assay"}, - {"display_name": "Clinical Tier 1: Demographics", "schema_name": "Demographics", "type": "clinical"}, - {"display_name": "Clinical Tier 1: Diagnosis", "schema_name": "Diagnosis", "type": "clinical"}, - {"display_name": "Clinical Tier 1: FamilyHistory", "schema_name": "FamilyHistory", "type": "clinical"}, - {"display_name": "Clinical Tier 1: Exposure", "schema_name": "Exposure", "type": "clinical"}, - {"display_name": "Clinical Tier 1: FollowUp", "schema_name": "FollowUp", "type": "clinical"}, - {"display_name": "Clinical Tier 1: Molecular Test", "schema_name": "MolecularTest", "type": "clinical"}, - {"display_name": "Clinical Tier 1: Therapy", "schema_name": "Therapy", "type": "clinical"}, - {"display_name": "Clinical Tier 2", "schema_name": "ClinicalDataTier2", "type": "clinical"}, - {"display_name": "Clinical Tier 3: Acute Lymphoblastic Leukemia", "schema_name": "AcuteLymphoblasticLeukemiaTier3", "type": "clinical"}, - {"display_name": "Clinical Tier 3: Breast Precancer and Cancer ", "schema_name": "BreastCancerTier3", "type": "clinical"}, - {"display_name": "Clinical Tier 3: Colorectal Precancer and Cancer", "schema_name": "ColorectalCancerTier3", "type": "clinical"}, - {"display_name": "Clinical Tier 3: Lung Precancer and Cancer", "schema_name": "LungCancerTier3", "type": "clinical"}, - {"display_name": "Clinical Tier 3: Neuroblastoma and Glioma", "schema_name": "BrainCancerTier3", "type": "clinical"}, - {"display_name": "Clinical Tier 3: Melanoma", "schema_name": "MelanomaTier3", "type": "clinical"}, - {"display_name": "Clinical Tier 3: Ovarian Precancer and Cancer ", "schema_name": "OvarianCancerTier3", "type": "clinical"}, - {"display_name": "Clinical Tier 3: Pancreatic Precancer and Cancer", "schema_name": "PancreaticCancerTier3", "type": "clinical"}, - {"display_name": "Clinical Tier 3: Prostate Precancer and Cancer", "schema_name": "ProstateCancerTier3", "type": "clinical"}, - {"display_name": "Clinical Tier 3: Sarcoma", "schema_name": "SarcomaTier3", "type": "clinical"}, - {"display_name": "Biospecimen Tier 1 & 2", "schema_name": "Biospecimen", "type": "biospecimen"}, - {"display_name": "Other Assay (Minimal Metadata)", "schema_name": "OtherAssay", "type": "assay"}, - {"display_name": "SRRS Biospecimen Tier 1 & 2", "schema_name": "SRRSBiospecimen", "type": "biospecimen"}, - {"display_name": "SRRS Clinical Tier 2", "schema_name": "SRRSClinicalDataTier2", "type": "clinical"}, - {"display_name": "SRRS Imaging Level 2", "schema_name": "SRRSImagingLevel2", "type": "assay"} + {"display_name": "Bulk RNA-seq Assay", "schema_name": "BulkRNA-seqAssay", "type": "assay"}, + {"display_name": "Patient", "schema_name": "Patient", "type": "clinical"}, + {"display_name": "Biospecimen Tier 1 & 2", "schema_name": "Biospecimen", "type": "biospecimen"} ], - "main_fileview" : "syn20446927", - "community" : "HTAN" + "community" : "Example" } From 398a762dd530c0efb861f10e95a090df5723359f Mon Sep 17 00:00:00 2001 From: rrchai Date: Fri, 11 Mar 2022 16:53:19 +0000 Subject: [PATCH 7/9] add schematic and schema version --- www/config.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/www/config.json b/www/config.json index 43b19924..76215c9b 100644 --- a/www/config.json +++ b/www/config.json @@ -4,5 +4,7 @@ {"display_name": "Patient", "schema_name": "Patient", "type": "clinical"}, {"display_name": "Biospecimen Tier 1 & 2", "schema_name": "Biospecimen", "type": "biospecimen"} ], - "community" : "Example" + "community": "Example", + "schema": "v0.1.0", + "schematic_service": "v1" } From e076f9bbffd03e41d28f8c0df6e630a8384fd739 Mon Sep 17 00:00:00 2001 From: rrchai Date: Fri, 11 Mar 2022 18:39:36 +0000 Subject: [PATCH 8/9] quick update README about deployment and contributors --- README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 64684e27..e9519263 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,9 @@ Use the app configuration file `www/config.json` to adapt this app to your DCC. - `display_name` : The display name for the dropdown. (e.g. _scRNA-seq Level 1_) - `schema_name`: The name of the manifest in the JSON-LD schema (e.g. _ScRNA-seqLevel1_) - `type`: The type of manifest. As currently configured in `app.R`, will only display manifests of type _assay_. -- `main_fileview` : The Synapse ID of a fileview that is scoped to all files, folders, & projects in your community. (e.g. _syn20446927_) -- `community` : the abbreviated name of the community or project. (e.g. _HTAN_) +- `community` (optional): The abbreviated name of the community or project. (e.g. _HTAN_) +- `schema` (optional): The version of data model +- `schematic_service` (optional): The version of schematic service --- @@ -70,16 +71,21 @@ This utilizes a Synapse Authentication (OAuth) client (code motivated by [ShinyO --- +## Deployment +To deploy the app to shinyapps.io, please follow the instructions in the [shinyapps_deploy.md](./shinyapps_deploy.md). + ## Contributors Main contributors and developers: - [Rongrong Chai](https://github.com/rrchai) -- [Xengie Doan](https://github.com/xdoan) +- [Anthony Williams](https://github.com/afwillia) - [Milen Nikolov](https://github.com/milen-sage) -- [Sujay Patil](https://github.com/sujaypatil96) +- [Lauren Wolfe](https://github.com/lakikowolfe) - [Robert Allaway](https://github.com/allaway) - [Bruno Grande](https://github.com/BrunoGrandePhD) +- [Xengie Doan](https://github.com/xdoan) +- [Sujay Patil](https://github.com/sujaypatil96) From 4802eba3107d500145138b4f2c0a2f1ef7ee394a Mon Sep 17 00:00:00 2001 From: rrchai Date: Fri, 11 Mar 2022 18:41:03 +0000 Subject: [PATCH 9/9] specify R image version to 4.1.2 --- .github/workflows/shinyapps_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/shinyapps_deploy.yml b/.github/workflows/shinyapps_deploy.yml index d6196125..97f3f8d6 100644 --- a/.github/workflows/shinyapps_deploy.yml +++ b/.github/workflows/shinyapps_deploy.yml @@ -21,7 +21,7 @@ jobs: shiny-deploy: runs-on: ubuntu-latest # This image seems to be based on rocker/r-ver which in turn is based on debian - container: rocker/rstudio + container: rocker/rstudio:4.1.2 env: # This should not be necessary for installing from public repo's however remotes::install_github() fails without it. GITHUB_PAT: ${{ secrets.REPO_PAT }}