Merge branch 'main' into sdm

.github/workflows/docker_ui.yml

@@ -5,35 +5,36 @@ on:
     branches:
       - main
     paths:
-      - 'ui/**'
+      - "ui/**"
 
 jobs:
   ui:
     runs-on: ubuntu-latest
     steps:
-    - name: Check out the repo
-      uses: actions/checkout@v3
-
-    - name: Log in to Docker Hub
-      uses: docker/login-action@v2
-      with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-
-    - name: Extract metadata (tags, labels) for Docker
-      id: meta
-      uses: docker/metadata-action@v4
-      with:
-        images: geobon/bon-in-a-box
-        tags: |
-          type=raw,value=ui
-    
-    - name: Build and push Docker image
-      uses: docker/build-push-action@v3
-      with:
-        context: ui
-        file: ui/Dockerfile.prod
-        push: true
-        tags: ${{ steps.meta.outputs.tags }}
-        labels: ${{ steps.meta.outputs.labels }}
-
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - run: echo REACT_APP_VIEWER_HOST=/viewer >> ui/.env
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: geobon/bon-in-a-box
+          tags: |
+            type=raw,value=ui
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ui
+          file: ui/Dockerfile.prod
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

README.md

@@ -7,7 +7,7 @@ A GEO BON project, born from a collaboration between Microsoft, McGill, Humbolt
 ## Contributing
 If you wish to contribute your indicator or EBV code, please let us know at [email protected].
 
-The recommended method is to setup an instance of BON in a Box somewhere you can easily play with the script files, using the local or remote setup below. You can create a fork to save your work. Make sure that the code is general, and will work when used with various parameters, such as in different regions around the globe. Once the integration of the new scripts or pipelines are complete, open a pull request to this repository. The pull request will be peer-reviewed before acceptation.
+The recommended method is to setup an instance of BON in a Box somewhere you can easily play with the script files, using the local or remote setup below. You can create a branch or fork to save your work. Make sure that the code is general, and will work when used with various parameters, such as in different regions around the globe. Once the integration of the new scripts or pipelines are complete, open a pull request to this repository. The pull request will be peer-reviewed before acceptation.
 
 ## Running the servers locally
 Prerequisites : 
@@ -73,6 +73,13 @@ When modifying pipelines in the /pipelines folder, servers do not need to be res
 3. Create a .env file on the server, as above.
 4. Take dockers down and up to load the .env file (this allows accessing GBIF, etc.)
 
+## Running a script or pipeline
+You have an instance of BON in a Box running, either [locally](#running-the-servers-locally) or [remotely](#running-the-servers-remotely), and you want to run your first script or pipeline.
+
+There is one page to run scripts, and one to run pipelines. Select the script or pipelines from the dropdown and fill the form.
+
+The form might ask you for a file. In order to provide a file that you own locally, upload or copy it to the `userdata` folder. You can then refer to it with a url as such: `userdata/myFile.shp`, or `userdata/myFolder/myFile.shp` if there are subfolders.
+
 ## Scripts
 The scripts perform the actual work behind the scenes. They are located in [/scripts folder](/scripts)
 

compose.dev.yml

@@ -3,23 +3,24 @@
 version: "3.7"
 
 services:
-## UI server to use when developing UI code
+  ## UI server to use when developing UI code
   ui:
     container_name: biab-ui
     build:
       context: ./ui
       dockerfile: Dockerfile.dev
     user: "node"
     volumes:
-      - './ui:/app:rw'
+      - "./ui:/app:rw"
     working_dir: "/app"
     command: sh -c "cd BonInABoxScriptService; npm run build; cd -; npm install; npm start;"
     expose:
-      - '3000'
+      - "3000"
     environment:
-      - CHOKIDAR_USEPOLLING=true 
+      - CHOKIDAR_USEPOLLING=true
       # https://github.com/facebook/create-react-app/issues/11779
       - WDS_SOCKET_PORT=0
+      - REACT_APP_VIEWER_HOST=${REACT_APP_VIEWER_HOST}
     depends_on:
       - script-server
       - tiler
@@ -31,8 +32,8 @@ services:
       - ui
       - openapi_swagger
 
-## In the DEV version, using a volume and a single docker. To rebuild without restarting everything, use
-## docker exec -it biab-script-server sh -c "cd /home/gradle/project/ && gradle build"
+  ## In the DEV version, using a volume and a single docker. To rebuild without restarting everything, use
+  ## docker exec -it biab-script-server sh -c "cd /home/gradle/project/ && gradle build"
   script-server:
     build:
       context: ./script-server
@@ -44,7 +45,7 @@ services:
     environment:
       DEV: true
 
-## Server to use when making changes to the OpenAPI specification.
+  ## Server to use when making changes to the OpenAPI specification.
   openapi_swagger:
     container_name: swagger_editor
     image: swaggerapi/swagger-editor

compose.yml

@@ -9,12 +9,14 @@ services:
     volumes:
       - ./scripts:/scripts:ro
       - ./pipelines:/pipelines:ro
+      - ./userdata:/userdata:ro
       - ./output:/output:rw
       - /var/run/docker.sock:/var/run/docker.sock
     expose:
       - "8080"
     environment:
       - SCRIPT_LOCATION=/scripts
+      - USERDATA_LOCATION=/userdata
       - PIPELINES_LOCATION=/pipelines
       - OUTPUT_LOCATION=/output
       - HOST_PATH=${PWD}
@@ -37,9 +39,11 @@ services:
     tty: true # Needed to keep the container alive, waiting for requests.
     volumes:
       - ./scripts:/scripts:ro
+      - ./userdata:/userdata:ro
       - ./output:/output:rw
     environment:
       - SCRIPT_LOCATION=/scripts
+      - USERDATA_LOCATION=/userdata
       - OUTPUT_LOCATION=/output
       - JUPYTERHUB_API_TOKEN=${JUPYTERHUB_API_TOKEN}
       - GBIF_USER=${GBIF_USER}
@@ -57,9 +61,11 @@ services:
     tty: true # Needed to keep the container alive, waiting for requests.
     volumes:
       - ./scripts:/scripts:ro
+      - ./userdata:/userdata:ro
       - ./output:/output:rw
     environment:
       - SCRIPT_LOCATION=/scripts
+      - USERDATA_LOCATION=/userdata
       - OUTPUT_LOCATION=/output
 
   http-gateway:
@@ -78,6 +84,7 @@ services:
     image: ghcr.io/developmentseed/titiler:latest
     volumes:
       - ./output:/output:ro
+      - ./userdata:/userdata:ro
     environment:
       - PORT=8000
       - WORKERS_PER_CORE=1

pipelines/SDM/SDM_BART.json

@@ -731,8 +731,8 @@
     "data>loadFromStac.yml@114|spatial_res": {
       "description": "Integer, spatial resolution of the rasters",
       "label": "spatial resolution",
-      "type": "int",
-      "example": 1000
+      "type": "float",
+      "example": 1000.0
     },
     "data>loadFromStac.yml@114|mask": {
       "description": "Shapefile, used to mask the output rasters",

pipelines/SDM/SDM_maxEnt.json

@@ -875,8 +875,8 @@
     "data>loadFromStac.yml@119|spatial_res": {
       "description": "Integer, spatial resolution of the rasters",
       "label": "spatial resolution",
-      "type": "int",
-      "example": 5000
+      "type": "float",
+      "example": 5000.0
     },
     "data>loadFromStac.yml@119|mask": {
       "description": "Shapefile, used to mask the output rasters",

pipelines/SDM/SDM_maxEnt_outputs.json

@@ -904,8 +904,8 @@
     "data>loadFromStac.yml@119|spatial_res": {
       "description": "Integer, spatial resolution of the rasters",
       "label": "spatial resolution",
-      "type": "int",
-      "example": 1000
+      "type": "float",
+      "example": 1000.0
     },
     "data>loadFromStac.yml@119|mask": {
       "description": "Shapefile, used to mask the output rasters",
@@ -922,7 +922,7 @@
   "outputs": {
     "pipeline@121": {
       "label": "Species name",
-      "description": "Species for which the SDM is generated",
+      "description": "Species for which the distribution model is generated",
       "weight": 1
     },
     "filtering>cleanCoordinates.yml@34|clean_presence": {
@@ -931,18 +931,18 @@
       "type": "text/tab-separated-values",
       "weight": 2
     },
-    "SDM>removeCollinearity.yml@97|rasters_selected": {
-      "description": "Environmental layers used as predictors in species distribution modeling",
-      "label": "Environmental predictors",
-      "type": "image/tiff;application=geotiff[]",
-      "weight": 3
-    },
     "data>heatmapFromSTAC.yml@67|raster": {
       "description": "Heatmap of GBIF occurences for plants used for bias correction",
       "label": "Density of GBIF occurrences",
       "type": "image/tiff;application=geotiff",
       "weight": 4
     },
+    "SDM>removeCollinearity.yml@97|rasters_selected": {
+      "description": "Environmental layers used as predictors in species distribution modeling",
+      "label": "Environmental predictors",
+      "type": "image/tiff;application=geotiff[]",
+      "weight": 3
+    },
     "SDM>runMaxent.yml@108|sdm_pred": {
       "description": "Model predictions from Maxent algorithm",
       "label": "Predictions",
@@ -956,5 +956,24 @@
       "type": "image/tiff;application=geotiff",
       "weight": 6
     }
+  },
+  "metadata": {
+    "name": "Species distribution modeling with Maxent",
+    "description": "This pipeline generates predictions for a species distribution model using the Maxent algorithm. Bias correction is achieved using random pseudo-absences throught the region of interest. A variance map to represent the prediction uncertainty is generated through bootstraping.",
+    "author": [
+      {
+        "name": "Sarah Valentin",
+        "identifier": "https://orcid.org/0000-0002-9028-681X"
+      },
+      {
+        "name": "Guillaume Larocque",
+        "identifier": "https://orcid.org/0000-0002-5967-9156"
+      },
+      {
+        "name": "François Rousseu"
+      }
+    ],
+    "license": "MIT",
+    "external_link": "https://github.com/GEO-BON/biab-2.0/blob/main/scripts/SDM/runMaxent.R"
   }
 }

runners/julia-dockerfile

@@ -1,6 +1,6 @@
-FROM julia:1.8.1
+FROM julia:1.9.3
 
 # Pre-compiling Julia dependencies
-RUN  julia -e 'pwd(); using Pkg; Pkg.add.(["BiodiversityObservationNetworks", "JSON", "SimpleSDMLayers", "NeutralLandscapes", "Plots"]); Pkg.instantiate();'
+RUN  julia -e 'pwd(); using Pkg; Pkg.add.(["SpeciesDistributionToolkit", "JSON", "CSV", "DataFrames", "StatsBase", "EvoTrees", "MultivariateStats" ]); Pkg.instantiate();'
 
-RUN date +"%Y-%m-%d %R" > /version.txt
+RUN date +"%Y-%m-%d %R" > /version.txt

script-server/src/main/kotlin/org/geobon/pipeline/RunContext.kt

@@ -3,6 +3,7 @@ package org.geobon.pipeline
 import com.google.gson.Gson
 import com.google.gson.GsonBuilder
 import com.google.gson.JsonParseException
+import com.google.gson.reflect.TypeToken
 import com.google.gson.stream.MalformedJsonException
 import org.geobon.utils.toMD5
 import java.io.File
@@ -20,7 +21,7 @@ data class RunContext(val runId: String, val inputs: String?) {
             // Unique to this script
             descriptionFile.relativeTo(scriptRoot).path.removeSuffix(".yml"),
             // Unique to these params
-            if (inputs.isNullOrEmpty()) "no_params" else inputs.toMD5()
+            if (inputs.isNullOrEmpty()) "no_params" else inputsToMd5(inputs)
         ).path,
         inputs
     )
@@ -67,5 +68,18 @@ data class RunContext(val runId: String, val inputs: String?) {
                 }
             }
             .create()
+
+        /**
+         * Makes sure the file gives the same hash, regardless of the key order.
+         */
+        fun inputsToMd5(jsonString: String): String {
+            val sorted = gson.fromJson<Map<String, Any>>(
+                jsonString,
+                object : TypeToken<Map<String, Any>>() {}.type
+            ).toSortedMap()
+
+            return gson.toJson(sorted).toMD5()
+        }
+
     }
 }

script-server/src/main/kotlin/org/geobon/server/plugins/Cache.kt

@@ -7,7 +7,7 @@ import java.io.File
 import java.io.IOException
 import kotlin.io.path.moveTo
 
-const val CACHE_VERSION = "2.0"
+const val CACHE_VERSION = "2.1"
 val CACHE_VERSION_FILE = File(outputRoot, "cacheVersion.txt")
 
 fun checkCacheVersion() {
@@ -24,7 +24,11 @@ fun checkCacheVersion() {
                 archivePath.mkdir()
                 topLevelFiles.forEach {
                     if (it.name != ".gitignore") {
-                        it.toPath().moveTo(File(archivePath, it.name).toPath())
+                        try {
+                            it.toPath().moveTo(File(archivePath, it.name).toPath())
+                        } catch (e:IOException) {
+                            logger.error("Failed to move ${it.toPath()}.\nGot ${e.message}")
+                        }
                     }
                 }
             }

script-server/src/main/kotlin/org/geobon/server/plugins/Routing.kt

@@ -7,14 +7,11 @@ import io.ktor.server.application.*
 import io.ktor.server.request.*
 import io.ktor.server.response.*
 import io.ktor.server.routing.*
-import kotlinx.coroutines.coroutineScope
-import kotlinx.coroutines.launch
 import org.geobon.pipeline.*
 import org.geobon.pipeline.Pipeline.Companion.createMiniPipelineFromScript
 import org.geobon.pipeline.Pipeline.Companion.createRootPipeline
 import org.geobon.pipeline.RunContext.Companion.scriptRoot
 import org.geobon.utils.runCommand
-import org.geobon.utils.toMD5
 import org.json.JSONObject
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
@@ -139,8 +136,8 @@ fun Application.configureRouting() {
 
             val withoutExtension = descriptionPath.removeSuffix(".json").removeSuffix(".yml")
 
-            // Unique   to this pipeline                                               and to these params
-            val runId = withoutExtension + FILE_SEPARATOR + inputFileContent.toMD5()
+            // Unique   to this pipeline                    and to these params
+            val runId = withoutExtension + FILE_SEPARATOR + RunContext.inputsToMd5(inputFileContent)
             val pipelineOutputFolder = File(outputRoot, runId.replace(FILE_SEPARATOR, '/'))
             logger.info("Pipeline: $descriptionPath\nFolder: $pipelineOutputFolder\nBody: $inputFileContent")
 

script-server/src/test/kotlin/org/geobon/pipeline/PipelineValidation.kt

@@ -37,7 +37,10 @@ internal class PipelineValidation {
                     pipelineJSON.optJSONObject(INPUTS)?.let { inputsSpec ->
                         inputsSpec.keySet().forEach { key ->
                             inputsSpec.optJSONObject(key)?.let { inputSpec ->
-                                fakeInputs.put(key, inputSpec.get(INPUTS__EXAMPLE))
+                                fakeInputs.put(
+                                    key,
+                                    inputSpec.opt(INPUTS__EXAMPLE) ?: JSONObject.NULL
+                                )
                             }
                         }
                     }