Migrate CIOOS Metadata

.devcontainer/devcontainer.json

@@ -0,0 +1,31 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/typescript-node
+{
+	"name": "Node.js & TypeScript",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/typescript-node:1-20-bookworm",
+	"features": {
+		"ghcr.io/devcontainers-extra/features/firebase-cli:2": {
+			"version": "13.20.0"
+		},
+		"ghcr.io/devcontainers/features/java:1": {
+			"version": "23"
+		}
+
+	},
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [4000,4500,5001,5002,8081,9001,9150,9099]
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "yarn install",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.firebaserc

@@ -1,7 +1,7 @@
 {
   "projects": {
-    "default": "cioos-metadata-form",
-    "dev": "cioos-metadata-form-dev"
+    "default": "cioos-metadata-form-8d942",
+    "dev": "cioos-metadata-form-dev-258dc"
   },
   "targets": {},
   "etags": {}

.github/workflows/firebase-hosting-delete-channel.yml

@@ -19,5 +19,5 @@ jobs:
           args: hosting:channel:delete pr${{github.event.number}}-${{steps.get-short-name.outputs.id}} --force
         env:
           FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}
-          PROJECT_ID: cioos-metadata-form-dev
+          PROJECT_ID: cioos-metadata-form-dev-258dc
           GITHUB_AUTH: ${{ secrets.ISSUE_CREATOR_PAT }}

.github/workflows/firebase-hosting-pull-request.yml

@@ -26,7 +26,7 @@ jobs:
           firebaseServiceAccount: '${{ secrets.FIREBASE_SERVICE_ACCOUNT_CIOOS_METADATA_FORM }}'
           expires: 30d
           entryPoint: '.'
-          projectId: cioos-metadata-form-dev
+          projectId: cioos-metadata-form-dev-258dc
         env:
           FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}
           GMAIL_USER: ${{ secrets.GMAIL_USER }}

.gitignore

@@ -27,7 +27,6 @@ firebase_to_xml/venv
 *.egg-info
 TODO
 
-.devcontainer
 .vscode
 
 node_modules
@@ -39,4 +38,6 @@ firebase-debug.log
 .Rproj.user
 
 yarn.lock
-.env
+.env
+.env.*
+*.log

README.md

@@ -27,7 +27,7 @@ This will start a hot-reloading dev server. Click on the link that it outputs to
 
 ## Monitoring
 
-Monitoring of production site availability is done via the [cioos-upptime](https://github.com/cioos-siooc/cwatch-upptime) and notices are posted to the CIOOS cwatch-upptime slack channel. Error collection is performed by sentry and reported in the [cioos-metadata-entry-form](https://hakai-institute.sentry.io/projects/cioos-metadata-entry-form/) project.
+Monitoring of production site availability is done via the [cioos-upptime](https://github.com/cioos-siooc/cwatch-upptime) and notices are posted to the CIOOS cwatch-upptime slack channel. Error collection is performed by sentry and reported in the [cioos-metadata-entry-form](https://cioos.sentry.io/projects/cioos-metadata-entry-form/) project.
 
 ### Running the Firebase emulator
 
@@ -73,17 +73,24 @@ We use a GitHub Actions workflow named `firebase-deploy` for deploying Firebase
 
 ### Deploying to Development Project
 
-To deploy updated Firebase functions to the "cioos-metadata-form-dev" development project, follow these steps:
+To deploy updated Firebase functions to the "cioos-metadata-form-dev-258dc" development project, follow these steps:
 
-1. **Ensure your local setup is linked to the correct Firebase project** by using the Firebase CLI to login and select the "cioos-metadata-form-dev" project.
+1. **Login** to firebase
+
+    ```bash
+    firebase login
+    ```
+
+2. **Ensure your local setup is linked to the correct Firebase project** by using the Firebase CLI to login and select the "cioos-metadata-form-dev-258dc" project.
 
     ```bash
-    firebase use cioos-metadata-form-dev
+    firebase use cioos-metadata-form-dev-258dc
     ```
 
-2. **Make necessary changes to your Firebase functions.**
+3. **Make necessary changes to your Firebase functions.**
 
-3. **Deploy the changes by running the command:**
+4. **Deploy the changes by running the command:**
+From the `./firebase-functions/functions` directory run the command:
 
     ```bash
     firebase deploy --only functions
@@ -103,7 +110,7 @@ The workflow utilizes the following secrets to create the virtual `.env` file fo
 - `GITHUB_AUTH` used to push to github pages branch and other github action type stuff
 - `REACT_APP_DEV_DEPLOYMENT` used to switch between development and production databases. Default False, set to True to use Dev database
 - `REACT_APP_GOOGLE_CLOUD_API_KEY` found at https://console.cloud.google.com/apis/credentials?project=cioos-metadata-form
-- `REACT_APP_GOOGLE_CLOUD_API_KEY_DEV` found at https://console.cloud.google.com/apis/credentials?project=cioos-metadata-form-dev
+- `REACT_APP_GOOGLE_CLOUD_API_KEY_DEV` found at https://console.cloud.google.com/apis/credentials?project=cioos-metadata-form-dev-258dc
 
 ### Using Parameterized Configuration in Firebase Functions
 
@@ -137,12 +144,12 @@ Deploying Firebase Realtime Database security rules via the Firebase CLI is reco
 
 ### Define targets
 
-This project has two databases: `cioos-metadata-form` (this is the default/main db for production) and `cioos-metadata-form-dev` (dev). 
+This project has two databases: `cioos-metadata-form-8d942` (this is the default/main db for production) and `cioos-metadata-form-dev-258dc` (dev). 
 Use Firebase CLI targets to manage rules deployment:
 
 ```bash
-firebase target:apply database prod cioos-metadata-form
-firebase target:apply database dev cioos-metadata-form-dev
+firebase target:apply database prod cioos-metadata-form-8d942
+firebase target:apply database dev cioos-metadata-form-dev-258dc
 ```
 
 ### Configure firebase.json
@@ -188,10 +195,10 @@ When hosting the application in a new place there are a couple of things to upda
 
 - You must add your new domain to the allowed list for authenication in firebase.
   https://console.firebase.google.com/u/0/project/cioos-metadata-form/authentication/settings
-  https://console.firebase.google.com/u/0/project/cioos-metadata-form-dev/authentication/settings
+  https://console.firebase.google.com/u/0/project/cioos-metadata-form-dev-258dc/authentication/settings
 
 - You have to allow your domain under Website restrictions for the firebase browser key
   https://console.cloud.google.com/apis/credentials/key/405d637a-efd4-48f5-95c6-f0af1d7f4889?project=cioos-metadata-form
-  https://console.cloud.google.com/apis/credentials/key/23d360a3-4b55-43f2-bc1c-b485371c0e07?project=cioos-metadata-form-dev
+  https://console.cloud.google.com/apis/credentials/key/23d360a3-4b55-43f2-bc1c-b485371c0e07?project=cioos-metadata-form-dev-258dc
 
 

cioos-records-update/.gitignore

@@ -4,3 +4,4 @@ xml
 key.json
 deploy.sh
 record.json
+metadata

cioos-records-update/README.md

@@ -1,8 +1,18 @@
 # CIOOS Records Update
 
-This is a basic Flask app to create XML from the metadata form
+Section contains all the different compontents used to synchronized the cioos catalogues with the metadata-entry-form.
+
+Both services share the same volume of data where one generates and maintains the different forms and the second provides access to the forms. 
+
+## metadata-entry-forms-web-api
+A basic Flask app to create XML from the metadata form
 
 Requires `key.json` which contains a service account key json
 specifically one from [here](https://console.cloud.google.com/iam-admin/serviceaccounts/details/108735728189650899572/keys?authuser=0&hl=en&project=cioos-metadata-form) you can also generate one [here](https://console.firebase.google.com/u/0/project/cioos-metadata-form/settings/serviceaccounts/adminsdk)
 
 Alternatively the same service account key json string can be specified in the FIREBASE_SERVICE_ACCOUNT_KEY environment variable. The app will look for the environment variable first and fall back to the key.json file if ot found.
+
+## metadata-entry-forms-waf
+
+The metadata-entry-forms-waf service is used to retrieve the metadata forms generated by the metadata entry form. 
+

cioos-records-update/app.py

@@ -28,7 +28,7 @@
 # this is bind mounted to /var/www/html/dev/metadata
 xml_folder = "xml"
 
-waf_url = "https://pac-dev1.cioos.org/dev/metadata/"
+waf_url = "https://waf.forms.cioos.ca/metadata/"
 app = Flask(__name__)
 
 def delete_record(basename):
@@ -63,6 +63,11 @@ def get_complete_path(status, region, basename,file_suffix):
     return filename
 
 
+@app.route("/status", methods=["GET"])
+def status():
+    return jsonify(message="OK")
+
+
 @app.route("/recordDelete")
 def recordDelete():
     filenameToDelete = request.args.get("filename")

cioos-records-update/docker-compose.yaml

@@ -1,11 +1,26 @@
-version: "3"
 services:
   web:
+    container_name: metadata-entry-forms-web-api
     build: .
     ports:
       - "8091:8000"
     volumes:
-      - /var/www/html/dev/metadata:/usr/src/app/xml
+      - cioos_metadata_form_files:/usr/src/app/xml
     environment:
       FLASK_ENV: production
     restart: always
+  waf:
+    image: nginx:alpine
+    container_name: metadata-entry-forms-waf
+    ports:
+      - "8092:80"
+    volumes:
+      - ./waf/nginx.conf:/etc/nginx/conf.d/default.conf
+      - ./waf/header.html:/usr/share/nginx/html/files/header.txt
+      - ./waf/footer.html:/usr/share/nginx/html/files/footer.txt
+      - cioos_metadata_form_files:/usr/share/nginx/html/metadata:ro
+
+volumes:
+  cioos_metadata_form_files:
+    external: true
+    name: cioos-metadata-form-files

cioos-records-update/get_pac_dev1_metadata.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+wget --no-parent -r "https://pac-dev1.cioos.org/dev/metadata" -A ".xml" -A ".yaml"

cioos-records-update/save_metadata_to_volume.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Check if the folder to be copied is provided as an argument
+if [ -z "$1" ]; then
+  echo "Usage: $0 /path/to/local/folder"
+  exit 1
+fi
+
+# Define the folder to be copied and the volume name
+FOLDER_TO_COPY=$1
+VOLUME_NAME="cioos-metadata-form-files"
+
+# if the volume exists remove its content
+if docker volume inspect ${VOLUME_NAME} &> /dev/null; then
+  echo "Volume ${VOLUME_NAME} exists. Removing its content..."
+  docker run --rm -v ${VOLUME_NAME}:/data busybox rm -rf /data/*
+fi
+
+# Create a temporary container with the volume mounted
+TEMP_CONTAINER=$(docker run -d -v ${VOLUME_NAME}:/data busybox sleep 3600)
+
+# Copy the folder to the volume
+docker cp ${FOLDER_TO_COPY} ${TEMP_CONTAINER}:/data
+
+# Stop and remove the temporary container
+docker rm -f ${TEMP_CONTAINER}
+
+echo "Folder ${FOLDER_TO_COPY} has been copied to the Docker volume ${VOLUME_NAME}."

cioos-records-update/waf/footer.html

@@ -0,0 +1,3 @@
+    </ul>
+</body>
+</html>

cioos-records-update/waf/header.html

@@ -0,0 +1,36 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Directory Listing</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            margin: 0;
+            padding: 20px;
+            background-color: #f4f4f4;
+        }
+        h1 {
+            color: #333;
+        }
+        ul {
+            list-style-type: none;
+            padding: 0;
+        }
+        li {
+            margin: 5px 0;
+        }
+        a {
+            text-decoration: none;
+            color: #007bff;
+        }
+        a:hover {
+            text-decoration: underline;
+        }
+    </style>
+</head>
+<body>
+    <h1>CIOOS Metadata Forms</h1>
+    This page present the list of metadata forms generated by the <a href="https://cioos-siooc.github.io/metadata-entry-form/#/en/region-select">cioos metadata forms</a> .
+    <ul>

cioos-records-update/waf/nginx.conf

@@ -0,0 +1,18 @@
+server {
+    listen 80;
+    server_name localhost;
+
+    location / {
+        alias /usr/share/nginx/html/;
+        autoindex on;
+        autoindex_exact_size off;
+        autoindex_localtime off;
+        index index.html;
+        add_before_body /files/header.txt;
+        add_after_body /files/footer.txt;
+        types {
+            text/plain yaml;
+            text/xml xml;
+        }
+    }
+} 

firebase-functions/.firebaserc

@@ -1,26 +1,26 @@
 {
   "projects": {
-    "default": "cioos-metadata-form",
-    "dev": "cioos-metadata-form-dev"
+    "default": "cioos-metadata-form-8d942",
+    "dev": "cioos-metadata-form-dev-258dc"
   },
   "targets": {
-    "cioos-metadata-form": {
+    "cioos-metadata-form-8d942": {
       "database": {
         "prod": [
-          "cioos-metadata-form"
+          "cioos-metadata-form-8d942"
         ],
         "dev": [
-          "cioos-metadata-form-dev"
+          "cioos-metadata-form-dev-258dc"
         ]
       }
     },
-    "development": {
+    "cioos-metadata-form-dev-258dc": {
       "database": {
         "prod": [
-          "cioos-metadata-form"
+          "cioos-metadata-form-258dc"
         ],
         "dev": [
-          "cioos-metadata-form-dev"
+          "cioos-metadata-form-dev-258dc"
         ]
       }
     }

firebase-functions/functions/updates.js

@@ -3,7 +3,7 @@ const functions = require("firebase-functions");
 const https = require("https");
 const axios = require("axios");
 
-const urlBaseDefault = "https://pac-dev1.cioos.org/cioos-xml/"
+const urlBaseDefault = "https://api.forms.cioos.ca/"
 
 function getRecordFilename(record) {
   return `${record.title[record.language].slice(

firebase_to_xml/.python-version

@@ -0,0 +1 @@
+3.12