From 0e559c539fbd5f7dbf10729743aad377f3f9a63f Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Wed, 22 Nov 2023 02:38:51 +0100 Subject: [PATCH] Feature: add ontoportal bash script (#59) * add ontoportal bash script to run test and development servers * update README.md * update docker CI to work in production releases --- .env.sample | 1 + ...ocker-dev-release.yml => docker-image.yml} | 20 ++- .github/workflows/ruby-unit-tests.yml | 2 + .gitignore | 2 + Gemfile.lock | 16 +- README.md | 62 ++++++- bin/ontoportal | 166 ++++++++++++++++++ config/environments/config.rb.sample | 2 +- docker-compose.yml | 9 +- 9 files changed, 250 insertions(+), 30 deletions(-) create mode 100644 .env.sample rename .github/workflows/{docker-dev-release.yml => docker-image.yml} (64%) create mode 100755 bin/ontoportal diff --git a/.env.sample b/.env.sample new file mode 100644 index 00000000..719949b1 --- /dev/null +++ b/.env.sample @@ -0,0 +1 @@ +API_URL=http://localhost:9393 \ No newline at end of file diff --git a/.github/workflows/docker-dev-release.yml b/.github/workflows/docker-image.yml similarity index 64% rename from .github/workflows/docker-dev-release.yml rename to .github/workflows/docker-image.yml index dda0554d..0368c3ac 100644 --- a/.github/workflows/docker-dev-release.yml +++ b/.github/workflows/docker-image.yml @@ -6,7 +6,8 @@ on: - development - stage - test - + release: + types: [ published ] jobs: push_to_registry: name: Push Docker branch image to Docker Hub @@ -22,24 +23,33 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Log in to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v4 with: - images: agroportal/ontologies_api + images: | + agroportal/ontologies_api + ghcr.io/${{ github.repository }} - name: Build and push Docker image uses: docker/build-push-action@v4 with: context: . - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64, linux/arm64 build-args: | - RUBY_VERSION=2.7 + RUBY_VERSION=2.7.8 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml index 6b2c973d..e7b3524f 100644 --- a/.github/workflows/ruby-unit-tests.yml +++ b/.github/workflows/ruby-unit-tests.yml @@ -12,6 +12,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: copy-env-config + run: cp .env.sample .env - name: Build docker-compose run: docker-compose --profile 4store build #profile flag is set in order to build all containers in this step - name: Run unit tests diff --git a/.gitignore b/.gitignore index 886a220f..8b568832 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,5 @@ test/data/ontology_files/catalog-v001.xml create_permissions.log ontologies_api.iml + +.env diff --git a/Gemfile.lock b/Gemfile.lock index 8ff9e209..9875b7fc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 7d16b7b1ecc4da8bfd93db2a106737b06d84d6ff + revision: 4c7dfa80a8bb4a7d8cfb7ad1fc8a1a88e420e59e branch: development specs: ontologies_linked_data (0.0.1) @@ -109,7 +109,7 @@ GEM sshkit (>= 1.6.1, != 1.7.0) backports (3.24.1) base64 (0.2.0) - bcrypt (3.1.19) + bcrypt (3.1.20) bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) @@ -134,8 +134,7 @@ GEM date (3.3.4) declarative (0.0.20) docile (1.4.0) - domain_name (0.5.20190701) - unf (>= 0.0.5, < 1.0.0) + domain_name (0.6.20231109) ed25519 (1.3.0) faraday (1.10.3) faraday-em_http (~> 1.0) @@ -217,7 +216,7 @@ GEM multi_json (1.15.0) multipart-post (2.3.0) net-http-persistent (2.9.4) - net-imap (0.4.4) + net-imap (0.4.6) date net-protocol net-pop (0.1.2) @@ -242,7 +241,7 @@ GEM pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.3) + public_suffix (5.0.4) rack (1.6.13) rack-accept (0.4.5) rack (>= 0.4) @@ -321,7 +320,7 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.21.5) + sshkit (1.21.6) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) systemu (2.6.5) @@ -332,9 +331,6 @@ GEM tzinfo (2.0.6) concurrent-ruby (~> 1.0) uber (0.1.0) - unf (0.1.4) - unf_ext - unf_ext (0.0.8.2) unicorn (6.1.0) kgio (~> 2.6) raindrops (~> 0.7) diff --git a/README.md b/README.md index dfaa77ea..b4caa10a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,53 @@ ontologies_api provides a RESTful interface for accessing [BioPortal](https://bioportal.bioontology.org/) (an open repository of biomedical ontologies). Supported services include downloads, search, access to terms and concepts, text annotation, and much more. -## Prerequisites +# Run ontologies_api + +## Using OntoPortal api utilities script +### See help + +```bash +bin/ontoportal help +``` + +``` +Usage: bin/ontoportal {dev|test|run|help} [--reset-cache] [--api-url API_URL] [--api-key API_KEY] + dev : Start the Ontoportal API development server. + Example: bin/ontoportal dev --api-url http://localhost:9393 + Use --reset-cache to remove volumes: bin/ontoportal dev --reset-cache + test : Run tests. + run : Run a command in the Ontoportal API Docker container. + help : Show this help message. + +Description: + This script provides convenient commands for managing an Ontoportal API + application using Docker Compose. It includes options for starting the development server, + running tests, and executing commands within the Ontoportal API Docker container. + +Goals: + - Simplify common tasks related to Ontoportal API development using Docker. + - Provide a consistent and easy-to-use interface for common actions. + + +``` +### Configuration +``` +cp .env.sample .env +``` + +### Run dev +```bash +bin/ontoportal dev +``` + +### Run test with a local OntoPortal API +```bash +bin/ontoportal test +``` + + +## Manually +### Prerequisites - [Ruby 2.x](http://www.ruby-lang.org/en/downloads/) (most recent patch level) - [rbenv](https://github.com/sstephenson/rbenv) and [ruby-build](https://github.com/sstephenson/ruby-build) (optional) @@ -19,7 +65,7 @@ ontologies_api provides a RESTful interface for accessing [BioPortal](https://bi - [Solr](http://lucene.apache.org/solr/) - BioPortal indexes ontology class and property content using Solr (a Lucene-based server) -## Configuring Solr +### Configuring Solr To configure Solr for ontologies_api usage, modify the example project included with Solr by doing the following: @@ -46,22 +92,22 @@ To configure Solr for ontologies_api usage, modify the example project included # Edit the ontologieS_api/config/environments/{env}.rb file to point to your running instance: # http://localhost:8983/solr/NCBO1 -## Installing +### Installing -### Clone the repository +#### Clone the repository ``` $ git clone git@github.com:ncbo/ontologies_api.git $ cd ontologies_api ``` -### Install the dependencies +#### Install the dependencies ``` $ bundle install ``` -### Create an environment configuration file +#### Create an environment configuration file ``` $ cp config/environments/config.rb.sample config/environments/development.rb @@ -73,7 +119,7 @@ production.rb
development.rb
test.rb -### Run the unit tests (optional) +#### Run the unit tests (optional) Requires a configuration file for the test environment: @@ -87,7 +133,7 @@ Execute the suite of tests from the command line: $ bundle exec rake test ``` -### Run the application +#### Run the application ``` $ bundle exec rackup --port 9393 diff --git a/bin/ontoportal b/bin/ontoportal new file mode 100755 index 00000000..573b49c7 --- /dev/null +++ b/bin/ontoportal @@ -0,0 +1,166 @@ +#!/usr/bin/env bash + +# Function to display script usage information +show_help() { + echo "Usage: $0 {dev|test|run|help} [--reset-cache] [--api-url API_URL] [--api-key API_KEY]" + echo " dev : Start the Ontoportal API development server." + echo " Example: $0 dev --api-url http://localhost:9393" + echo " Use --reset-cache to remove volumes: $0 dev --reset-cache" + echo " test : Run tests." + echo " run : Run a command in the Ontoportal API Docker container." + echo " help : Show this help message." + echo + echo "Description:" + echo " This script provides convenient commands for managing an Ontoportal API" + echo " application using Docker Compose. It includes options for starting the development server," + echo " running tests, and executing commands within the Ontoportal API Docker container." + echo + echo "Goals:" + echo " - Simplify common tasks related to Ontoportal API development using Docker." + echo " - Provide a consistent and easy-to-use interface for common actions." +} +# Function to update or create the .env file with API_URL and API_KEY +update_env_file() { + local api_url="$1" + + # Update the .env file with the provided values + file_content=$(<.env) + + # Make changes to the variable + while IFS= read -r line; do + if [[ "$line" == "API_URL="* ]]; then + echo "API_URL=$api_url" + else + echo "$line" + fi + done <<< "$file_content" > .env +} + +# Function to create configuration files if they don't exist +create_config_files() { + if [ ! -f ".env" ]; then + echo "Creating .env file from env.sample" + cp .env.sample .env + fi + + if [ ! -f "config/environments/development.rb" ]; then + echo "Creating config/environments/development.rb file from config/environments/config.rb.sample" + cp config/bioportal_config_env.rb.sample config/bioportal_config_development.rb + fi +} + +# Function to handle the "dev" option +dev() { + echo "Starting Ontoportal API development server..." + + create_config_files + local reset_cache=false + local api_url="" + + + # Check for command line arguments + while [[ "$#" -gt 0 ]]; do + case $1 in + --reset-cache) + reset_cache=true + shift + ;; + --api-url) + api_url="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + show_help + exit 1 + ;; + esac + done + + + + # Check if arguments are provided + if [ -n "$api_url" ] ; then + # If arguments are provided, update the .env file + update_env_file "$api_url" + else + # If no arguments, fetch values from the .env file + source .env + api_url="$API_URL" + fi + + if [ -z "$api_url" ] ; then + echo "Error: Missing required arguments. Please provide both --api-url or update them in your .env" + exit 1 + fi + + # Check if --reset-cache is present and execute docker compose down --volumes + if [ "$reset_cache" = true ]; then + echo "Resetting cache. Running: docker compose down --volumes" + docker compose down --volumes + fi + + echo "Run: bundle exec api s -b 0.0.0.0 -p 3000" + docker compose run --rm -it --service-ports api bash -c "(bundle check || bundle install) && bundle exec rackup -o 0.0.0.0 --port 9393" +} + +# Function to handle the "test" option +test() { + + + local api_url="" + local test_path="" + local test_options="" + + # Check for command line arguments + while [ "$#" -gt 0 ]; do + case "$1" in + --api-url) + shift + api_url="$1" + ;; + *) + if [ -z "$test_path" ]; then + test_path="$1" + else + test_options="$test_options $1" + fi + ;; + esac + shift + done + + + + script="API_URL=$api_url bundle exec rake test TEST=\"$test_path\" TESTOPTS=\"$test_options\"" + echo "Running tests..." + echo "Run: $script" + + docker compose run --rm -it api bash -c "(bundle check || bundle install) && $script" +} + +# Function to handle the "run" option +run() { + echo "Run: $*" + docker compose run --rm -it api bash -c "$*" +} + +# Main script logic +case "$1" in + "run") + run "${@:2}" + ;; + "dev") + dev "${@:2}" + ;; + "test") + test "${@:2}" + ;; + "help") + show_help + ;; + *) + show_help + exit 1 + ;; +esac diff --git a/config/environments/config.rb.sample b/config/environments/config.rb.sample index 45ebf2be..8713b9f2 100644 --- a/config/environments/config.rb.sample +++ b/config/environments/config.rb.sample @@ -18,7 +18,7 @@ REDIS_PERSISTENT_HOST = ENV.include?("REDIS_PERSISTENT_HOST") ? ENV["REDIS_PERSI REDIS_PORT = ENV.include?("REDIS_PORT") ? ENV["REDIS_PORT"] : 6379 REPORT_PATH = ENV.include?("REPORT_PATH") ? ENV["REPORT_PATH"] : "./test/ontologies_report.json" REPOSITORY_FOLDER = ENV.include?("REPOSITORY_FOLDER") ? ENV["REPOSITORY_FOLDER"] : "./test/data/ontology_files/repo" -REST_URL_PREFIX = ENV.include?("REST_URL_PREFIX") ? ENV["REST_URL_PREFIX"] : "http://localhost:9393" +REST_URL_PREFIX = ENV.include?("REST_URL_PREFIX") ? ENV["REST_URL_PREFIX"] : ENV["API_URL"] || "http://localhost:9393" SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr/prop_search_core1" SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr/term_search_core1" diff --git a/docker-compose.yml b/docker-compose.yml index 5cb64963..f7325381 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,5 @@ x-app: &app - build: - context: . - args: - RUBY_VERSION: '2.7' - # Increase the version number in the image tag every time Dockerfile or its arguments is changed - image: ontologies_api:0.0.1 + image: agroportal/ontologies_api:development environment: &env BUNDLE_PATH: /srv/ontoportal/bundle # default bundle config resolves to /usr/local/bundle/config inside of the container @@ -39,6 +34,8 @@ x-app: &app services: api: <<: *app + env_file: + .env environment: <<: *env GOO_BACKEND_NAME: 4store