diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..3148574b --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,31 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/typescript-node +{ + "name": "Node.js & TypeScript", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/typescript-node:1-20-bookworm", + "features": { + "ghcr.io/devcontainers-extra/features/firebase-cli:2": { + "version": "13.20.0" + }, + "ghcr.io/devcontainers/features/java:1": { + "version": "23" + } + + }, + + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [4000,4500,5001,5002,8081,9001,9150,9099] + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "yarn install", + + // Configure tool-specific properties. + // "customizations": {}, + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/.firebaserc b/.firebaserc index 7536d832..56c94657 100644 --- a/.firebaserc +++ b/.firebaserc @@ -1,7 +1,7 @@ { "projects": { - "default": "cioos-metadata-form", - "dev": "cioos-metadata-form-dev" + "default": "cioos-metadata-form-8d942", + "dev": "cioos-metadata-form-dev-258dc" }, "targets": {}, "etags": {} diff --git a/.github/workflows/firebase-hosting-delete-channel.yml b/.github/workflows/firebase-hosting-delete-channel.yml index 948de945..924fd6d3 100644 --- a/.github/workflows/firebase-hosting-delete-channel.yml +++ b/.github/workflows/firebase-hosting-delete-channel.yml @@ -19,5 +19,5 @@ jobs: args: hosting:channel:delete pr${{github.event.number}}-${{steps.get-short-name.outputs.id}} --force env: FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }} - PROJECT_ID: cioos-metadata-form-dev + PROJECT_ID: cioos-metadata-form-dev-258dc GITHUB_AUTH: ${{ secrets.ISSUE_CREATOR_PAT }} \ No newline at end of file diff --git a/.github/workflows/firebase-hosting-pull-request.yml b/.github/workflows/firebase-hosting-pull-request.yml index 0b1d48b5..7053ac45 100644 --- a/.github/workflows/firebase-hosting-pull-request.yml +++ b/.github/workflows/firebase-hosting-pull-request.yml @@ -26,7 +26,7 @@ jobs: firebaseServiceAccount: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_CIOOS_METADATA_FORM }}' expires: 30d entryPoint: '.' - projectId: cioos-metadata-form-dev + projectId: cioos-metadata-form-dev-258dc env: FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }} GMAIL_USER: ${{ secrets.GMAIL_USER }} diff --git a/.gitignore b/.gitignore index ce230ee1..e325bf56 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,6 @@ firebase_to_xml/venv *.egg-info TODO -.devcontainer .vscode node_modules @@ -39,4 +38,6 @@ firebase-debug.log .Rproj.user yarn.lock -.env \ No newline at end of file +.env +.env.* +*.log \ No newline at end of file diff --git a/README.md b/README.md index 470bb5cc..29e302b7 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ This will start a hot-reloading dev server. Click on the link that it outputs to ## Monitoring -Monitoring of production site availability is done via the [cioos-upptime](https://github.com/cioos-siooc/cwatch-upptime) and notices are posted to the CIOOS cwatch-upptime slack channel. Error collection is performed by sentry and reported in the [cioos-metadata-entry-form](https://hakai-institute.sentry.io/projects/cioos-metadata-entry-form/) project. +Monitoring of production site availability is done via the [cioos-upptime](https://github.com/cioos-siooc/cwatch-upptime) and notices are posted to the CIOOS cwatch-upptime slack channel. Error collection is performed by sentry and reported in the [cioos-metadata-entry-form](https://cioos.sentry.io/projects/cioos-metadata-entry-form/) project. ### Running the Firebase emulator @@ -73,17 +73,24 @@ We use a GitHub Actions workflow named `firebase-deploy` for deploying Firebase ### Deploying to Development Project -To deploy updated Firebase functions to the "cioos-metadata-form-dev" development project, follow these steps: +To deploy updated Firebase functions to the "cioos-metadata-form-dev-258dc" development project, follow these steps: -1. **Ensure your local setup is linked to the correct Firebase project** by using the Firebase CLI to login and select the "cioos-metadata-form-dev" project. +1. **Login** to firebase + + ```bash + firebase login + ``` + +2. **Ensure your local setup is linked to the correct Firebase project** by using the Firebase CLI to login and select the "cioos-metadata-form-dev-258dc" project. ```bash - firebase use cioos-metadata-form-dev + firebase use cioos-metadata-form-dev-258dc ``` -2. **Make necessary changes to your Firebase functions.** +3. **Make necessary changes to your Firebase functions.** -3. **Deploy the changes by running the command:** +4. **Deploy the changes by running the command:** +From the `./firebase-functions/functions` directory run the command: ```bash firebase deploy --only functions @@ -103,7 +110,7 @@ The workflow utilizes the following secrets to create the virtual `.env` file fo - `GITHUB_AUTH` used to push to github pages branch and other github action type stuff - `REACT_APP_DEV_DEPLOYMENT` used to switch between development and production databases. Default False, set to True to use Dev database - `REACT_APP_GOOGLE_CLOUD_API_KEY` found at https://console.cloud.google.com/apis/credentials?project=cioos-metadata-form -- `REACT_APP_GOOGLE_CLOUD_API_KEY_DEV` found at https://console.cloud.google.com/apis/credentials?project=cioos-metadata-form-dev +- `REACT_APP_GOOGLE_CLOUD_API_KEY_DEV` found at https://console.cloud.google.com/apis/credentials?project=cioos-metadata-form-dev-258dc ### Using Parameterized Configuration in Firebase Functions @@ -137,12 +144,12 @@ Deploying Firebase Realtime Database security rules via the Firebase CLI is reco ### Define targets -This project has two databases: `cioos-metadata-form` (this is the default/main db for production) and `cioos-metadata-form-dev` (dev). +This project has two databases: `cioos-metadata-form-8d942` (this is the default/main db for production) and `cioos-metadata-form-dev-258dc` (dev). Use Firebase CLI targets to manage rules deployment: ```bash -firebase target:apply database prod cioos-metadata-form -firebase target:apply database dev cioos-metadata-form-dev +firebase target:apply database prod cioos-metadata-form-8d942 +firebase target:apply database dev cioos-metadata-form-dev-258dc ``` ### Configure firebase.json @@ -188,10 +195,10 @@ When hosting the application in a new place there are a couple of things to upda - You must add your new domain to the allowed list for authenication in firebase. https://console.firebase.google.com/u/0/project/cioos-metadata-form/authentication/settings - https://console.firebase.google.com/u/0/project/cioos-metadata-form-dev/authentication/settings + https://console.firebase.google.com/u/0/project/cioos-metadata-form-dev-258dc/authentication/settings - You have to allow your domain under Website restrictions for the firebase browser key https://console.cloud.google.com/apis/credentials/key/405d637a-efd4-48f5-95c6-f0af1d7f4889?project=cioos-metadata-form - https://console.cloud.google.com/apis/credentials/key/23d360a3-4b55-43f2-bc1c-b485371c0e07?project=cioos-metadata-form-dev + https://console.cloud.google.com/apis/credentials/key/23d360a3-4b55-43f2-bc1c-b485371c0e07?project=cioos-metadata-form-dev-258dc \ No newline at end of file diff --git a/cioos-records-update/.gitignore b/cioos-records-update/.gitignore index 5fb43a7e..e4f36960 100644 --- a/cioos-records-update/.gitignore +++ b/cioos-records-update/.gitignore @@ -4,3 +4,4 @@ xml key.json deploy.sh record.json +metadata \ No newline at end of file diff --git a/cioos-records-update/README.md b/cioos-records-update/README.md index a5d9b1b0..c6e0d1a3 100644 --- a/cioos-records-update/README.md +++ b/cioos-records-update/README.md @@ -1,8 +1,18 @@ # CIOOS Records Update -This is a basic Flask app to create XML from the metadata form +Section contains all the different compontents used to synchronized the cioos catalogues with the metadata-entry-form. + +Both services share the same volume of data where one generates and maintains the different forms and the second provides access to the forms. + +## metadata-entry-forms-web-api +A basic Flask app to create XML from the metadata form Requires `key.json` which contains a service account key json specifically one from [here](https://console.cloud.google.com/iam-admin/serviceaccounts/details/108735728189650899572/keys?authuser=0&hl=en&project=cioos-metadata-form) you can also generate one [here](https://console.firebase.google.com/u/0/project/cioos-metadata-form/settings/serviceaccounts/adminsdk) Alternatively the same service account key json string can be specified in the FIREBASE_SERVICE_ACCOUNT_KEY environment variable. The app will look for the environment variable first and fall back to the key.json file if ot found. + +## metadata-entry-forms-waf + +The metadata-entry-forms-waf service is used to retrieve the metadata forms generated by the metadata entry form. + diff --git a/cioos-records-update/app.py b/cioos-records-update/app.py index 9f9db53e..8644c556 100644 --- a/cioos-records-update/app.py +++ b/cioos-records-update/app.py @@ -28,7 +28,7 @@ # this is bind mounted to /var/www/html/dev/metadata xml_folder = "xml" -waf_url = "https://pac-dev1.cioos.org/dev/metadata/" +waf_url = "https://waf.forms.cioos.ca/metadata/" app = Flask(__name__) def delete_record(basename): @@ -63,6 +63,11 @@ def get_complete_path(status, region, basename,file_suffix): return filename +@app.route("/status", methods=["GET"]) +def status(): + return jsonify(message="OK") + + @app.route("/recordDelete") def recordDelete(): filenameToDelete = request.args.get("filename") diff --git a/cioos-records-update/docker-compose.yaml b/cioos-records-update/docker-compose.yaml index 52215f17..fe016a5e 100644 --- a/cioos-records-update/docker-compose.yaml +++ b/cioos-records-update/docker-compose.yaml @@ -1,11 +1,26 @@ -version: "3" services: web: + container_name: metadata-entry-forms-web-api build: . ports: - "8091:8000" volumes: - - /var/www/html/dev/metadata:/usr/src/app/xml + - cioos_metadata_form_files:/usr/src/app/xml environment: FLASK_ENV: production restart: always + waf: + image: nginx:alpine + container_name: metadata-entry-forms-waf + ports: + - "8092:80" + volumes: + - ./waf/nginx.conf:/etc/nginx/conf.d/default.conf + - ./waf/header.html:/usr/share/nginx/html/files/header.txt + - ./waf/footer.html:/usr/share/nginx/html/files/footer.txt + - cioos_metadata_form_files:/usr/share/nginx/html/metadata:ro + +volumes: + cioos_metadata_form_files: + external: true + name: cioos-metadata-form-files diff --git a/cioos-records-update/get_pac_dev1_metadata.sh b/cioos-records-update/get_pac_dev1_metadata.sh new file mode 100644 index 00000000..ffa437f7 --- /dev/null +++ b/cioos-records-update/get_pac_dev1_metadata.sh @@ -0,0 +1,2 @@ +#!/bin/bash +wget --no-parent -r "https://pac-dev1.cioos.org/dev/metadata" -A ".xml" -A ".yaml" diff --git a/cioos-records-update/save_metadata_to_volume.sh b/cioos-records-update/save_metadata_to_volume.sh new file mode 100644 index 00000000..0428e63a --- /dev/null +++ b/cioos-records-update/save_metadata_to_volume.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Check if the folder to be copied is provided as an argument +if [ -z "$1" ]; then + echo "Usage: $0 /path/to/local/folder" + exit 1 +fi + +# Define the folder to be copied and the volume name +FOLDER_TO_COPY=$1 +VOLUME_NAME="cioos-metadata-form-files" + +# if the volume exists remove its content +if docker volume inspect ${VOLUME_NAME} &> /dev/null; then + echo "Volume ${VOLUME_NAME} exists. Removing its content..." + docker run --rm -v ${VOLUME_NAME}:/data busybox rm -rf /data/* +fi + +# Create a temporary container with the volume mounted +TEMP_CONTAINER=$(docker run -d -v ${VOLUME_NAME}:/data busybox sleep 3600) + +# Copy the folder to the volume +docker cp ${FOLDER_TO_COPY} ${TEMP_CONTAINER}:/data + +# Stop and remove the temporary container +docker rm -f ${TEMP_CONTAINER} + +echo "Folder ${FOLDER_TO_COPY} has been copied to the Docker volume ${VOLUME_NAME}." \ No newline at end of file diff --git a/cioos-records-update/waf/footer.html b/cioos-records-update/waf/footer.html new file mode 100644 index 00000000..c2bf2e4d --- /dev/null +++ b/cioos-records-update/waf/footer.html @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/cioos-records-update/waf/header.html b/cioos-records-update/waf/header.html new file mode 100644 index 00000000..29a4c999 --- /dev/null +++ b/cioos-records-update/waf/header.html @@ -0,0 +1,36 @@ + + + + + + Directory Listing + + + +

CIOOS Metadata Forms

+ This page present the list of metadata forms generated by the cioos metadata forms . +