diff --git a/.github/workflows/docker_ui.yml b/.github/workflows/docker_ui.yml index 351b596b..ce7ea3ff 100644 --- a/.github/workflows/docker_ui.yml +++ b/.github/workflows/docker_ui.yml @@ -5,35 +5,36 @@ on: branches: - main paths: - - 'ui/**' + - "ui/**" jobs: ui: runs-on: ubuntu-latest steps: - - name: Check out the repo - uses: actions/checkout@v3 - - - name: Log in to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v4 - with: - images: geobon/bon-in-a-box - tags: | - type=raw,value=ui - - - name: Build and push Docker image - uses: docker/build-push-action@v3 - with: - context: ui - file: ui/Dockerfile.prod - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - run: echo REACT_APP_VIEWER_HOST=/viewer >> ui/.env + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + images: geobon/bon-in-a-box + tags: | + type=raw,value=ui + + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + context: ui + file: ui/Dockerfile.prod + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/README.md b/README.md index c266652e..f61619e0 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ A GEO BON project, born from a collaboration between Microsoft, McGill, Humbolt ## Contributing If you wish to contribute your indicator or EBV code, please let us know at web@geobon.org. -The recommended method is to setup an instance of BON in a Box somewhere you can easily play with the script files, using the local or remote setup below. You can create a fork to save your work. Make sure that the code is general, and will work when used with various parameters, such as in different regions around the globe. Once the integration of the new scripts or pipelines are complete, open a pull request to this repository. The pull request will be peer-reviewed before acceptation. +The recommended method is to setup an instance of BON in a Box somewhere you can easily play with the script files, using the local or remote setup below. You can create a branch or fork to save your work. Make sure that the code is general, and will work when used with various parameters, such as in different regions around the globe. Once the integration of the new scripts or pipelines are complete, open a pull request to this repository. The pull request will be peer-reviewed before acceptation. ## Running the servers locally Prerequisites : @@ -73,6 +73,13 @@ When modifying pipelines in the /pipelines folder, servers do not need to be res 3. Create a .env file on the server, as above. 4. Take dockers down and up to load the .env file (this allows accessing GBIF, etc.) +## Running a script or pipeline +You have an instance of BON in a Box running, either [locally](#running-the-servers-locally) or [remotely](#running-the-servers-remotely), and you want to run your first script or pipeline. + +There is one page to run scripts, and one to run pipelines. Select the script or pipelines from the dropdown and fill the form. + +The form might ask you for a file. In order to provide a file that you own locally, upload or copy it to the `userdata` folder. You can then refer to it with a url as such: `userdata/myFile.shp`, or `userdata/myFolder/myFile.shp` if there are subfolders. + ## Scripts The scripts perform the actual work behind the scenes. They are located in [/scripts folder](/scripts) diff --git a/compose.dev.yml b/compose.dev.yml index 999e9b32..519f2139 100755 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -3,7 +3,7 @@ version: "3.7" services: -## UI server to use when developing UI code + ## UI server to use when developing UI code ui: container_name: biab-ui build: @@ -11,15 +11,16 @@ services: dockerfile: Dockerfile.dev user: "node" volumes: - - './ui:/app:rw' + - "./ui:/app:rw" working_dir: "/app" command: sh -c "cd BonInABoxScriptService; npm run build; cd -; npm install; npm start;" expose: - - '3000' + - "3000" environment: - - CHOKIDAR_USEPOLLING=true + - CHOKIDAR_USEPOLLING=true # https://github.com/facebook/create-react-app/issues/11779 - WDS_SOCKET_PORT=0 + - REACT_APP_VIEWER_HOST=${REACT_APP_VIEWER_HOST} depends_on: - script-server - tiler @@ -31,8 +32,8 @@ services: - ui - openapi_swagger -## In the DEV version, using a volume and a single docker. To rebuild without restarting everything, use -## docker exec -it biab-script-server sh -c "cd /home/gradle/project/ && gradle build" + ## In the DEV version, using a volume and a single docker. To rebuild without restarting everything, use + ## docker exec -it biab-script-server sh -c "cd /home/gradle/project/ && gradle build" script-server: build: context: ./script-server @@ -44,7 +45,7 @@ services: environment: DEV: true -## Server to use when making changes to the OpenAPI specification. + ## Server to use when making changes to the OpenAPI specification. openapi_swagger: container_name: swagger_editor image: swaggerapi/swagger-editor diff --git a/compose.yml b/compose.yml index 89d0d27a..14ac20d7 100755 --- a/compose.yml +++ b/compose.yml @@ -9,12 +9,14 @@ services: volumes: - ./scripts:/scripts:ro - ./pipelines:/pipelines:ro + - ./userdata:/userdata:ro - ./output:/output:rw - /var/run/docker.sock:/var/run/docker.sock expose: - "8080" environment: - SCRIPT_LOCATION=/scripts + - USERDATA_LOCATION=/userdata - PIPELINES_LOCATION=/pipelines - OUTPUT_LOCATION=/output - HOST_PATH=${PWD} @@ -37,9 +39,11 @@ services: tty: true # Needed to keep the container alive, waiting for requests. volumes: - ./scripts:/scripts:ro + - ./userdata:/userdata:ro - ./output:/output:rw environment: - SCRIPT_LOCATION=/scripts + - USERDATA_LOCATION=/userdata - OUTPUT_LOCATION=/output - JUPYTERHUB_API_TOKEN=${JUPYTERHUB_API_TOKEN} - GBIF_USER=${GBIF_USER} @@ -57,9 +61,11 @@ services: tty: true # Needed to keep the container alive, waiting for requests. volumes: - ./scripts:/scripts:ro + - ./userdata:/userdata:ro - ./output:/output:rw environment: - SCRIPT_LOCATION=/scripts + - USERDATA_LOCATION=/userdata - OUTPUT_LOCATION=/output http-gateway: @@ -78,6 +84,7 @@ services: image: ghcr.io/developmentseed/titiler:latest volumes: - ./output:/output:ro + - ./userdata:/userdata:ro environment: - PORT=8000 - WORKERS_PER_CORE=1 diff --git a/pipelines/Block3/block3.json b/pipelines/Block3/block3.json new file mode 100644 index 00000000..cdee850e --- /dev/null +++ b/pipelines/Block3/block3.json @@ -0,0 +1,594 @@ +{ + "nodes": [ + { + "id": "1", + "type": "io", + "position": { + "x": 391.76898108164164, + "y": 573.1078098700178 + }, + "data": { + "descriptionFile": "data>loadFromStac.yml" + } + }, + { + "id": "2", + "type": "constant", + "position": { + "x": -347.9849165399878, + "y": -91.01105222521502 + }, + "dragHandle": ".dragHandle", + "data": { + "type": "text", + "value": "https://io.biodiversite-quebec.ca/stac/" + } + }, + { + "id": "3", + "type": "constant", + "position": { + "x": -312.14873695815743, + "y": 496.4338350821323 + }, + "dragHandle": ".dragHandle", + "data": { + "type": "float[]", + "value": [ + "-86.748047", + "33.72434", + "-52.207031", + "63.273182" + ] + } + }, + { + "id": "4", + "type": "constant", + "position": { + "x": -398.5944218132245, + "y": 939.8221162348001 + }, + "dragHandle": ".dragHandle", + "data": { + "type": "float", + "value": 0.008 + } + }, + { + "id": "6", + "type": "constant", + "position": { + "x": -369.5859975359332, + "y": 39.41973092263399 + }, + "dragHandle": ".dragHandle", + "data": { + "type": "text[]", + "value": [ + "chelsa-clim|bio1", + "chelsa-clim|bio2", + "chelsa-clim|bio3", + "chelsa-clim|bio4", + "chelsa-clim|bio5", + "chelsa-clim|bio6", + "chelsa-clim|bio7", + "chelsa-clim|bio8", + "chelsa-clim|bio9", + "chelsa-clim|bio10", + "chelsa-clim|bio11", + "chelsa-clim|bio12", + "chelsa-clim|bio13", + "chelsa-clim|bio14", + "chelsa-clim|bio15", + "chelsa-clim|bio16", + "chelsa-clim|bio17", + "chelsa-clim|bio18", + "chelsa-clim|bio19" + ] + } + }, + { + "id": "9", + "type": "io", + "position": { + "x": 353.6499938964844, + "y": 884 + }, + "data": { + "descriptionFile": "data>loadFromStac.yml" + } + }, + { + "id": "10", + "type": "constant", + "position": { + "x": -270.3500061035156, + "y": 775 + }, + "dragHandle": ".dragHandle", + "data": { + "type": "text[]", + "value": [ + "esacci-lc|esacci-lc-2020" + ] + } + }, + { + "id": "12", + "type": "output", + "position": { + "x": 943.6499938964844, + "y": 426 + }, + "data": { + "label": "Output" + } + }, + { + "id": "15", + "type": "output", + "position": { + "x": 838.6499938964844, + "y": 885 + }, + "data": { + "label": "Output" + } + }, + { + "id": "16", + "type": "io", + "position": { + "x": 1321.6499938964844, + "y": 685 + }, + "data": { + "descriptionFile": "Block3>climateUniqueness.yml" + } + }, + { + "id": "17", + "type": "output", + "position": { + "x": 1761.6499938964844, + "y": 682 + }, + "data": { + "label": "Output" + } + }, + { + "id": "21", + "type": "io", + "position": { + "x": 570.124027546199, + "y": -163.4163222373429 + }, + "data": { + "descriptionFile": "data>loadFromStac.yml" + } + }, + { + "id": "24", + "type": "constant", + "position": { + "x": -763.681948816768, + "y": 324.6128233177092 + }, + "dragHandle": ".dragHandle", + "data": { + "type": "text", + "value": "layer, current, change\nGBSTAC|chelsa-clim|bio1, 0.7, 0.6\nGBSTAC|chelsa-clim|bio2, 0.3, 0.4\n" + } + }, + { + "id": "25", + "type": "constant", + "position": { + "x": 57.14488835869702, + "y": -1050.2978267070198 + }, + "dragHandle": ".dragHandle", + "data": { + "type": "text[]", + "value": [ + "chelsa-clim-proj|bio1_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio2_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio3_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio4_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio5_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio6_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio7_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio8_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio9_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio10_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio11_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio12_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio13_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio14_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio15_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio16_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio17_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio18_2071-2100_gfdl-esm4_ssp370", + "chelsa-clim-proj|bio19_2071-2100_gfdl-esm4_ssp370" + ] + } + }, + { + "id": "30", + "type": "io", + "position": { + "x": 499.6499938964844, + "y": 1230 + }, + "data": { + "descriptionFile": "data>loadFromStac.yml" + } + }, + { + "id": "31", + "type": "constant", + "position": { + "x": -271.1670907550228, + "y": 1223.2499264553558 + }, + "dragHandle": ".dragHandle", + "data": { + "type": "text[]", + "value": [ + "accessibility_to_cities|accessibility" + ] + } + }, + { + "id": "33", + "type": "constant", + "position": { + "x": -613.3848707089167, + "y": 638.0014281579736 + }, + "dragHandle": ".dragHandle", + "data": { + "type": "text", + "value": "EPSG:4326" + } + }, + { + "id": "40", + "type": "constant", + "position": { + "x": 181.0333251953125, + "y": 220.6666717529297 + }, + "dragHandle": ".dragHandle", + "data": { + "type": "application/dbf" + } + }, + { + "id": "41", + "type": "io", + "position": { + "x": 1387.7113959370417, + "y": 133.94012486202405 + }, + "data": { + "descriptionFile": "Block3>climateVelocity.yml" + } + }, + { + "id": "42", + "type": "output", + "position": { + "x": 1743.3434534806934, + "y": 102.68967835196827 + }, + "data": { + "label": "Output" + } + } + ], + "edges": [ + { + "source": "2", + "sourceHandle": null, + "target": "1", + "targetHandle": "stac_url", + "id": "reactflow__edge-2-1stac_url" + }, + { + "source": "3", + "sourceHandle": null, + "target": "1", + "targetHandle": "bbox", + "id": "reactflow__edge-3-1bbox" + }, + { + "source": "4", + "sourceHandle": null, + "target": "1", + "targetHandle": "spatial_res", + "id": "reactflow__edge-4-1spatial_res" + }, + { + "source": "6", + "sourceHandle": null, + "target": "1", + "targetHandle": "collections_items", + "id": "reactflow__edge-6-1collections_items" + }, + { + "source": "3", + "sourceHandle": null, + "target": "9", + "targetHandle": "bbox", + "id": "reactflow__edge-3-9bbox" + }, + { + "source": "4", + "sourceHandle": null, + "target": "9", + "targetHandle": "spatial_res", + "id": "reactflow__edge-4-9spatial_res" + }, + { + "source": "10", + "sourceHandle": null, + "target": "9", + "targetHandle": "collections_items", + "id": "reactflow__edge-10-9collections_items" + }, + { + "source": "2", + "sourceHandle": null, + "target": "9", + "targetHandle": "stac_url", + "id": "reactflow__edge-2-9stac_url" + }, + { + "source": "1", + "sourceHandle": "rasters", + "target": "12", + "targetHandle": null, + "id": "reactflow__edge-1rasters-12" + }, + { + "source": "9", + "sourceHandle": "rasters", + "target": "15", + "targetHandle": null, + "id": "reactflow__edge-9rasters-15" + }, + { + "source": "1", + "sourceHandle": "rasters", + "target": "16", + "targetHandle": "layers", + "id": "reactflow__edge-1rasters-16layers" + }, + { + "source": "9", + "sourceHandle": "rasters", + "target": "16", + "targetHandle": "water_mask", + "id": "reactflow__edge-9rasters-16water_mask" + }, + { + "source": "16", + "sourceHandle": "climate_uniqueness", + "target": "17", + "targetHandle": null, + "id": "reactflow__edge-16climate_uniqueness-17" + }, + { + "source": "2", + "sourceHandle": null, + "target": "21", + "targetHandle": "stac_url", + "id": "reactflow__edge-2-21stac_url" + }, + { + "source": "3", + "sourceHandle": null, + "target": "21", + "targetHandle": "bbox", + "id": "reactflow__edge-3-21bbox" + }, + { + "source": "24", + "sourceHandle": null, + "target": "1", + "targetHandle": "weight_matrix_with_ids", + "id": "reactflow__edge-24-1weight_matrix_with_ids" + }, + { + "source": "24", + "sourceHandle": null, + "target": "21", + "targetHandle": "weight_matrix_with_ids", + "id": "reactflow__edge-24-21weight_matrix_with_ids" + }, + { + "source": "4", + "sourceHandle": null, + "target": "21", + "targetHandle": "spatial_res", + "id": "reactflow__edge-4-21spatial_res" + }, + { + "source": "25", + "sourceHandle": null, + "target": "21", + "targetHandle": "collections_items", + "id": "reactflow__edge-25-21collections_items" + }, + { + "source": "4", + "sourceHandle": null, + "target": "30", + "targetHandle": "spatial_res", + "id": "reactflow__edge-4-30spatial_res" + }, + { + "source": "3", + "sourceHandle": null, + "target": "30", + "targetHandle": "bbox", + "id": "reactflow__edge-3-30bbox" + }, + { + "source": "2", + "sourceHandle": null, + "target": "30", + "targetHandle": "stac_url", + "id": "reactflow__edge-2-30stac_url" + }, + { + "source": "24", + "sourceHandle": null, + "target": "30", + "targetHandle": "weight_matrix_with_ids", + "id": "reactflow__edge-24-30weight_matrix_with_ids" + }, + { + "source": "31", + "sourceHandle": null, + "target": "30", + "targetHandle": "collections_items", + "id": "reactflow__edge-31-30collections_items" + }, + { + "source": "33", + "sourceHandle": null, + "target": "21", + "targetHandle": "proj", + "id": "reactflow__edge-33-21proj" + }, + { + "source": "33", + "sourceHandle": null, + "target": "1", + "targetHandle": "proj", + "id": "reactflow__edge-33-1proj" + }, + { + "source": "33", + "sourceHandle": null, + "target": "9", + "targetHandle": "proj", + "id": "reactflow__edge-33-9proj" + }, + { + "source": "33", + "sourceHandle": null, + "target": "30", + "targetHandle": "proj", + "id": "reactflow__edge-33-30proj" + }, + { + "source": "40", + "sourceHandle": null, + "target": "30", + "targetHandle": "mask", + "id": "reactflow__edge-40-30mask" + }, + { + "source": "40", + "sourceHandle": null, + "target": "9", + "targetHandle": "mask", + "id": "reactflow__edge-40-9mask" + }, + { + "source": "40", + "sourceHandle": null, + "target": "1", + "targetHandle": "mask", + "id": "reactflow__edge-40-1mask" + }, + { + "source": "40", + "sourceHandle": null, + "target": "21", + "targetHandle": "mask", + "id": "reactflow__edge-40-21mask" + }, + { + "source": "21", + "sourceHandle": "rasters", + "target": "41", + "targetHandle": "end_layers", + "id": "reactflow__edge-21rasters-41end_layers" + }, + { + "source": "1", + "sourceHandle": "rasters", + "target": "41", + "targetHandle": "baseline_layers", + "id": "reactflow__edge-1rasters-41baseline_layers" + }, + { + "source": "9", + "sourceHandle": "rasters", + "target": "41", + "targetHandle": "landcover", + "id": "reactflow__edge-9rasters-41landcover" + }, + { + "source": "41", + "sourceHandle": "climate_velocity", + "target": "42", + "targetHandle": null, + "id": "reactflow__edge-41climate_velocity-42" + } + ], + "inputs": { + "data>loadFromStac.yml@9|weight_matrix_with_ids": { + "description": "Weight matrix used for Bon optimization. Vector of strings, collection name followed by '|' followed by item id, followed by weights. Cannot be used if collection_items is set.", + "label": "Weight matrix with ids", + "type": "text", + "example": "layer, current, change\nGBSTAC|chelsa-clim|bio1, 0.7, 0.6\nGBSTAC|chelsa-clim|bio2, 0.3, 0.4\n" + }, + "Block3>climateUniqueness.yml@16|k": { + "description": "the value of k for the k-means algorithm", + "label": "k", + "type": "int", + "example": 5 + } + }, + "outputs": { + "data>loadFromStac.yml@1|rasters": { + "description": "array of output raster paths", + "label": "rasters", + "type": "image/tiff;application=geotiff[]" + }, + "data>loadFromStac.yml@9|rasters": { + "description": "array of output raster paths", + "label": "rasters", + "type": "image/tiff;application=geotiff[]" + }, + "Block3>climateUniqueness.yml@16|climate_uniqueness": { + "description": "map", + "label": "climate uniqueness", + "type": "image/tiff;application=geotiff" + }, + "Block3>climateVelocity.yml@41|climate_velocity": { + "description": "map", + "label": "climate velocity", + "type": "image/tiff;application=geotiff" + } + }, + "metadata": { + "name": "pipeline", + "description": "da pipline", + "author": [ + { + "name": "Michael D. Catchen", + "identifier": null + } + ], + "license": "MIT", + "external_link": null + } + } \ No newline at end of file diff --git a/runners/julia-dockerfile b/runners/julia-dockerfile index baf0a6d9..2c778900 100644 --- a/runners/julia-dockerfile +++ b/runners/julia-dockerfile @@ -1,7 +1,7 @@ FROM julia:1.9.3 # Pre-compiling Julia dependencies -RUN julia -e 'pwd(); using Pkg; Pkg.add.(["SpeciesDistributionToolkit", "JSON", "CSV", "DataFrames", "StatsBase", "EvoTrees", "MultivariateStats" ]); Pkg.instantiate();' +RUN julia -e 'pwd(); using Pkg; Pkg.add.(["SpeciesDistributionToolkit", "Dates", "Clustering", "JSON", "CSV", "DataFrames", "StatsBase", "EvoTrees", "MultivariateStats" ]); Pkg.instantiate();' #COPY Project.toml /root/Project.toml #COPY instantiate.jl /root/instantiate.jl diff --git a/script-server/src/test/kotlin/org/geobon/pipeline/PipelineValidation.kt b/script-server/src/test/kotlin/org/geobon/pipeline/PipelineValidation.kt index f1ad73f7..89d56282 100644 --- a/script-server/src/test/kotlin/org/geobon/pipeline/PipelineValidation.kt +++ b/script-server/src/test/kotlin/org/geobon/pipeline/PipelineValidation.kt @@ -37,7 +37,10 @@ internal class PipelineValidation { pipelineJSON.optJSONObject(INPUTS)?.let { inputsSpec -> inputsSpec.keySet().forEach { key -> inputsSpec.optJSONObject(key)?.let { inputSpec -> - fakeInputs.put(key, inputSpec.get(INPUTS__EXAMPLE)) + fakeInputs.put( + key, + inputSpec.opt(INPUTS__EXAMPLE) ?: JSONObject.NULL + ) } } } diff --git a/scripts/Block3/climateUniqueness.jl b/scripts/Block3/climateUniqueness.jl new file mode 100644 index 00000000..1aed6783 --- /dev/null +++ b/scripts/Block3/climateUniqueness.jl @@ -0,0 +1,105 @@ +using SpeciesDistributionToolkit +using Clustering +using StatsBase +using CSV +using DataFrames +using MultivariateStats +using Dates +using JSON + +include("shared.jl") + +function convert_layers_to_features_matrix(layers, data_matrix, land_idx) + for (i,l) in enumerate(layers) + x = Float32.(vec(l.grid[land_idx])) + z = StatsBase.fit(ZScoreTransform, x) + data_matrix[i,:] .= StatsBase.transform(z, x) + end + data_matrix +end + +function fill_layer!(empty_layer, vec, land_idx) + empty_layer.grid .= nothing + for (i, idx) in enumerate(land_idx) + empty_layer.grid[idx] = vec[i] + end +end + +function pca_data_matrix(data_matrix) + pca = MultivariateStats.fit(PCA, data_matrix) + MultivariateStats.transform(pca, data_matrix) +end + +function make_pca_matrix(layers, data_matrix, land_idx) + pca_mat = pca_data_matrix(convert_layers_to_features_matrix(layers, data_matrix, land_idx)) +end + +function fill_pca_layers(layers, pca_mat, land_idx) + pca_layers = [convert(Float32, similar(layers[begin])) for l in 1:size(pca_mat, 1)] + for (i,pca_layer) in enumerate(pca_layers) + fill_layer!(pca_layer, pca_mat[i,:], land_idx) + end + pca_layers +end + +function kmeans_and_pca(layers, data_matrix, land_idx, k) + pca_mat = make_pca_matrix(layers, data_matrix, land_idx) + pca_layers = fill_pca_layers(layers, pca_mat, land_idx) + + km = Clustering.kmeans(pca_mat, k) + Λ = collect(eachcol(km.centers)) + + pca_layers, Λ +end + +function make_climate_uniqueness(k, land_idx, layers) + data_matrix = zeros(Float32, length(layers), length(land_idx)) + + pca_layers, Λ = kmeans_and_pca(layers, data_matrix, land_idx, k) + + uniqueness = similar(layers[begin]) + uniqueness.grid .= nothing + + for i in land_idx + env_vec = [pca_layer.grid[i] for pca_layer in pca_layers] + _, m = findmin(x-> sum((env_vec .- x).^2), Λ) + uniqueness.grid[i] = sum( (env_vec .- Λ[m]).^2 ) + end + return uniqueness +end + +function write_outputs(runtime_dir, uniqueness) + outpath = joinpath(runtime_dir, "uniqueness.tif") + SpeciesDistributionToolkit._write_geotiff(outpath, uniqueness; compress="COG") + + output_json_path = joinpath(runtime_dir, "output.json") + open(output_json_path, "w") do f + write(f, JSON.json(Dict( + :climate_uniqueness => outpath + ))) + end +end + + +function main() + runtime_dir = ARGS[1] + inputs = read_inputs_dict(runtime_dir) + + mask_path = inputs["water_mask"] + layer_paths = inputs["layers"] + k = inputs["k"] + + + OPEN_WATER_LABEL = 210 + lc = SimpleSDMPredictor(mask_path) + land_idx = findall(!isequal(OPEN_WATER_LABEL), lc.grid) + + + layers = SimpleSDMPredictor.(layer_paths) + + uniqueness = make_climate_uniqueness(k, land_idx, layers) + + write_outputs(runtime_dir, uniqueness) +end + +main() \ No newline at end of file diff --git a/scripts/Block3/climateUniqueness.yml b/scripts/Block3/climateUniqueness.yml new file mode 100644 index 00000000..768ebe34 --- /dev/null +++ b/scripts/Block3/climateUniqueness.yml @@ -0,0 +1,28 @@ +script: climateUniqueness.jl +name: Climate Uniqueness +description: "This script takes in multiple climate layers and computes a single layer where high values indicate a more rare combination of layers. Uniqueness is computed as distance to the nearest k-means center, where k is a user input." +author: + - name: Michael D. Catchen + identifier: https://orcid.org/0000-0002-6506-6487 +inputs: + k: + label: k + description: the value of k for the k-means algorithm + type: int + example: 5 + layers: + label: layers + description: a list of climate layers to use to compute uniqueness + type: image/tiff;application=geotiff[] + example: ["foo"] + water_mask: + label: water_mask + description: a raster with landcover values + type: image/tiff;application=geotiff[] + example: "/output/foobar/lc.tif" +outputs: + climate_uniqueness: + label: climate uniqueness + description: map + type: image/tiff;application=geotiff + diff --git a/scripts/Block3/climateVelocity.jl b/scripts/Block3/climateVelocity.jl new file mode 100644 index 00000000..279e9234 --- /dev/null +++ b/scripts/Block3/climateVelocity.jl @@ -0,0 +1,132 @@ +using SpeciesDistributionToolkit +using Clustering +using StatsBase +using CSV +using DataFrames +using MultivariateStats +using Dates +using JSON + +include("shared.jl") + +function read_climate(runtime_dir, inputs, water_idx) + baseline_paths = inputs["baseline_layers"] + end_paths = inputs["end_layers"] + + layers = [map(SimpleSDMPredictor, joinpath.(runtime_dir, x)) for x in [baseline_paths, end_paths]] + + for ls in layers + for l in ls + l.grid[water_idx] .= nothing + end + end + + layers +end + +function get_baseline_standardization(baseline_layers, land_idx) + [StatsBase.fit(ZScoreTransform, Float32.(vec(l.grid[land_idx]))) for l in baseline_layers] +end + +function standardize_layers!(zs, layers, land_idx) + for i in eachindex(zs) + v = StatsBase.transform(zs[i], Float32.(vec(layers[i].grid[land_idx]))) + layers[i].grid[land_idx] .= v + end +end + + +function convert_layers_to_features_matrix(layer_set, data_matrix, land_idx) + for (i,l) in enumerate(layer_set) + x = Float32.(vec(l.grid[land_idx])) + data_matrix[i,:] .= x + end + data_matrix +end + +function fill_layer!(empty_layer, vec, land_idx) + empty_layer.grid .= nothing + for (i, idx) in enumerate(land_idx) + empty_layer.grid[idx] = vec[i] + end +end + +function pca_data_matrix(data_matrix) + pca = MultivariateStats.fit(PCA, data_matrix) +end + +function apply_pca_to_layers(pca, layers, land_idx) + data_matrix = zeros(Float32, length(layers), length(land_idx)) + feat_mat = convert_layers_to_features_matrix(layers, data_matrix, land_idx) + + pca_mat = MultivariateStats.transform(pca, feat_mat) + + pca_layers = [similar(layers[begin]) for i in 1:size(pca_mat,1)] + for (i,l) in enumerate(pca_layers) + l.grid .= nothing + l.grid[land_idx] = pca_mat[i,:] + end + pca_layers +end + +function compute_velocity(climate_layers, land_idx) + delta(a,b) = vec(abs.((a - b).grid[land_idx])) + baseline, future = climate_layers[begin], climate_layers[end] + + velocity = similar(climate_layers[begin][begin]) + velocity.grid .= nothing + velocity.grid[land_idx] .= 0. + + for i in eachindex(baseline) + dl = delta(baseline[i], future[i]) + velocity.grid[land_idx] += dl + end + velocity +end + +function write_outputs(runtime_dir, velocity) + outpath = joinpath(runtime_dir, "velocity.tif") + SpeciesDistributionToolkit._write_geotiff(outpath, velocity; compress="COG") + + output_json_path = joinpath(runtime_dir, "output.json") + open(output_json_path, "w") do f + write(f, JSON.json(Dict( + :climate_velocity => outpath + ))) + end +end + +function main() + runtime_dir = ARGS[1] + inputs = read_inputs_dict(runtime_dir) + + lc_path = inputs["landcover"] + OPEN_WATER_LABEL = 210 + lc = SimpleSDMPredictor(lc_path) + land_idx = findall(x->!isnothing(x) && x != OPEN_WATER_LABEL, lc.grid) + + + water_idx = findall(isequal(OPEN_WATER_LABEL), lc.grid) + + @info "about to read climate" + climate_layers = read_climate(runtime_dir, inputs, water_idx) + @info "about to standardize" + zs = get_baseline_standardization(climate_layers[begin], land_idx) + for layers in climate_layers + standardize_layers!(zs, layers, land_idx) + end + + data_matrix = zeros(Float32, length(climate_layers[begin]), length(land_idx)) + data_matrix = convert_layers_to_features_matrix(climate_layers[begin], data_matrix, land_idx) + pca = pca_data_matrix(data_matrix) + + pca_layers = [apply_pca_to_layers(pca, layers, land_idx) for layers in climate_layers] + + + @info "about to compute velocity" + velocity = compute_velocity(pca_layers, land_idx) + write_outputs(runtime_dir, velocity) +end + +@info "hello?????" +main() \ No newline at end of file diff --git a/scripts/Block3/climateVelocity.yml b/scripts/Block3/climateVelocity.yml new file mode 100644 index 00000000..6832aea3 --- /dev/null +++ b/scripts/Block3/climateVelocity.yml @@ -0,0 +1,31 @@ +script: climateVelocity.jl +name: Climate Velocity +description: "This script takes in multiple climate layers and computes the +velocity (rate) of change of each layer standardized across the period +1970-2100, split into four intervals. (This is because by default, CHELSA is +provided that way)" +author: + - name: Michael D. Catchen + identifier: https://orcid.org/0000-0002-6506-6487 +inputs: + baseline_layers: + label: baseline_layers + description: a list of the baseline climate layers to use to compute velocity + type: image/tiff;application=geotiff[] + example: ["foo"] + end_layers: + label: end_layers + description: a list of the climate at the end of the period to compute + type: image/tiff;application=geotiff[] + example: "/output/foobar/lc.tif" + landcover: + label: landcover raster + description: land cover raster to mask out water + type: image/tiff;application=geotiff[] + example: "foo" +outputs: + climate_velocity: + label: climate velocity + description: map + type: image/tiff;application=geotiff + diff --git a/scripts/Block3/shared.jl b/scripts/Block3/shared.jl new file mode 100644 index 00000000..5f904463 --- /dev/null +++ b/scripts/Block3/shared.jl @@ -0,0 +1,6 @@ +function read_inputs_dict(runtime_dir) + filepath = joinpath(runtime_dir, "input.json") + output_dir = joinpath(runtime_dir, "data/") + isdir(output_dir) || mkdir(output_dir) + return JSON.parsefile(filepath) +end \ No newline at end of file diff --git a/scripts/Block3/weightLayers.jl b/scripts/Block3/weightLayers.jl new file mode 100644 index 00000000..31203e55 --- /dev/null +++ b/scripts/Block3/weightLayers.jl @@ -0,0 +1,38 @@ +using JSON +using CSV +using DataFrames +using SpeciesDistributionToolkit + + + +function write_outputs(runtime_dir, priority) + outpath = joinpath(runtime_dir, "priority.tif") + SpeciesDistributionToolkit._write_geotiff(outpath, priority; compress="COG") + + output_json_path = joinpath(runtime_dir, "output.json") + open(output_json_path, "w") do f + write(f, JSON.json(Dict(:priority_map=>outpath))) + end +end + +function main() + runtime_dir = ARGS[1] + inputs = read_inputs_dict(runtime_dir) + + β = inputs["weights"] + + uncert_path = inputs["uncertainty"] + uniqueness_path = inputs["climate_uniqueness"] + velocity_path = inputs["climate_velocity"] + access_path = inputs["accessability"] + + layer_paths = [uncert_path, uniqueness_path, velocity_path, access_path] + + layers = [rescale(SimpleSDMPredictor(joinpath(runtime_dir, layer_path)), (0,1)) for layer_path in layer_paths] + + priority = sum([β[i] .* layers[i].grid for i in eachindex(β)]) + + write_outputs(runtime_dir, priority) +end + +main() \ No newline at end of file diff --git a/scripts/Block3/weightLayers.yml b/scripts/Block3/weightLayers.yml new file mode 100644 index 00000000..01a7b78f --- /dev/null +++ b/scripts/Block3/weightLayers.yml @@ -0,0 +1,38 @@ +script: weightLayers.jl +name: Weight Layers +author: + - name: Michael D. Catchen + identifier: https://orcid.org/0000-0002-6506-6487 +description: "This script takes in multiple layers and computes a priority map based on a weighted average of those layers" +inputs: + uncertainty: + label: uncertainty + description: uncertainty in the inference of some value across space + type: image/tiff;application=geotiff + example: y2000_class210_binary.tif + climate_uniqueness: + label: climate uniqueness + description: foobar + type: image/tiff;application=geotiff + example: y2000_class210_binary.tif + climate_velocity: + label: climate velocity + description: foobar + type: image/tiff;application=geotiff + example: y2000_class210_binary.tif + accessibility: + label: accessibility + descrpition: foobar + type: image/tiff;application=geotiff[] + example: y2000_class210_binary.tif + weights: + label: weights + description: weight vector that sums to one. Weights correnspond to layers in this order, [uncertainty, uniqueness, velocity, accessability] + type: float[] + example: [0.3, 0.2, 0.4, 0.1] +outputs: + priority_map: + label: priority_map + description: Sampling priority map. + type: image/tiff;application=geotiff + diff --git a/scripts/helloWorld/helloR.R b/scripts/helloWorld/helloR.R index e24f51ad..e706b789 100644 --- a/scripts/helloWorld/helloR.R +++ b/scripts/helloWorld/helloR.R @@ -57,6 +57,7 @@ output <- list(#"error" = "Some error", # Use error key to stop the rest of the "some_csv_data" = some_csv_data, "some_tsv_data" = some_tsv_data, "some_picture" = example_jpg, + "userdata_available" = list.files(file.path(Sys.getenv("USERDATA_LOCATION"))), "undocumented_output" = "Some debug output") jsonData <- toJSON(output, indent=2) diff --git a/scripts/helloWorld/helloR.yml b/scripts/helloWorld/helloR.yml index b41edbfc..0bb84c3e 100644 --- a/scripts/helloWorld/helloR.yml +++ b/scripts/helloWorld/helloR.yml @@ -7,9 +7,9 @@ author: description: "This sample script shows how it works." external_link: https://github.com/GEO-BON/biab-2.0 inputs: - occurence: - label: Occurence - description: Occurrence data, some description here + raster: + label: Some raster + description: Some raster file, with some description here type: image/tiff;application=geotiff example: http://something-compatible.tiff intensity: @@ -71,6 +71,13 @@ outputs: description: Some picture/graph/etc that shows bla bla... type: image/jpg example: /path/to/some/example.jpg + userdata_available: + label: Available user data + description: This is just printing out the content of userdata folder, to show that you can use data uploaded there. + type: text[] + example: + - file1.txt + - file2.tiff references: - text: John Doe, The ins and outs of copy-pasting, BioScience, Volume 71, Issue 5, May 2021, Pages 448–451 doi: 10.1093/biosci/biab041 diff --git a/ui/Dockerfile.prod b/ui/Dockerfile.prod index d4cb5a14..0481a16d 100644 --- a/ui/Dockerfile.prod +++ b/ui/Dockerfile.prod @@ -15,6 +15,7 @@ RUN cd BonInABoxScriptService; npm run build; cd ..; \ # add app COPY . ./ +ENV REACT_APP_VIEWER_HOST=${REACT_APP_VIEWER_HOST} RUN npm run build CMD ["npm", "start"] diff --git a/ui/src/components/PipelineEditor/MetadataPane.js b/ui/src/components/PipelineEditor/MetadataPane.js index 73e17e43..90bd78ee 100644 --- a/ui/src/components/PipelineEditor/MetadataPane.js +++ b/ui/src/components/PipelineEditor/MetadataPane.js @@ -12,6 +12,11 @@ author: # 1 to many identifier: # Optional, full URL of a unique digital identifier such as an ORCID license: # Optional. If unspecified, the project's MIT license will apply. external_link: # Optional, link to a separate project, github repo, etc. +references: # 0 to many + - text: # plain text reference + doi: # link + - text: # plain text reference + doi: # link ` /** diff --git a/ui/src/components/PipelinePage.js b/ui/src/components/PipelinePage.js index 0f7e5bee..30fb7ff1 100644 --- a/ui/src/components/PipelinePage.js +++ b/ui/src/components/PipelinePage.js @@ -13,19 +13,21 @@ import errorImg from "../img/error.svg"; import warningImg from "../img/warning.svg"; import infoImg from "../img/info.svg"; import { LogViewer } from "./LogViewer"; -import { getFolderAndNameFromMetadata, GeneralDescription } from "./StepDescription"; -import { PipelineForm } from "./form/PipelineForm"; import { - getScript, - getScriptOutput, - getBreadcrumbs, -} from "../utils/IOId"; + getFolderAndNameFromMetadata, + GeneralDescription, +} from "./StepDescription"; +import { PipelineForm } from "./form/PipelineForm"; +import { getScript, getScriptOutput, getBreadcrumbs } from "../utils/IOId"; import { useParams } from "react-router-dom"; import { isEmptyObject } from "../utils/isEmptyObject"; import { InlineSpinner } from "./Spinner"; -const pipelineConfig = {extension: ".json", defaultFile: "helloWorld.json", }; -const scriptConfig = {extension: ".yml", defaultFile: "helloWorld>helloR.yml"}; +const pipelineConfig = { extension: ".json", defaultFile: "helloWorld.json" }; +const scriptConfig = { + extension: ".yml", + defaultFile: "helloWorld>helloR.yml", +}; const BonInABoxScriptService = require("bon_in_a_box_script_service"); export const api = new BonInABoxScriptService.DefaultApi(); @@ -39,7 +41,10 @@ function pipReducer(state, action) { }; } case "url": { - let selectionUrl = action.newDescriptionFile.substring(0, action.newDescriptionFile.lastIndexOf(".")); + let selectionUrl = action.newDescriptionFile.substring( + 0, + action.newDescriptionFile.lastIndexOf(".") + ); return { lastAction: "url", runHash: action.newHash, @@ -49,7 +54,7 @@ function pipReducer(state, action) { }; } case "reset": { - return pipInitialState({ runType: action.runType }) + return pipInitialState({ runType: action.runType }); } default: throw Error("Unknown action: " + action.type); @@ -57,20 +62,20 @@ function pipReducer(state, action) { } function pipInitialState(init) { - let config = init.runType === "pipeline" ? pipelineConfig : scriptConfig - let descriptionFile = config.defaultFile + let config = init.runType === "pipeline" ? pipelineConfig : scriptConfig; + let descriptionFile = config.defaultFile; let runHash = null; - let runId = null + let runId = null; let action = "reset"; - + if (init.selectionUrl) { action = "url"; - descriptionFile = init.selectionUrl + config.extension + descriptionFile = init.selectionUrl + config.extension; if (init.runHash) { runHash = init.runHash; - runId = init.selectionUrl + ">" + runHash + runId = init.selectionUrl + ">" + runHash; } } @@ -83,7 +88,7 @@ function pipInitialState(init) { }; } -export function PipelinePage({runType}) { +export function PipelinePage({ runType }) { const [stoppable, setStoppable] = useState(null); const [runningScripts, setRunningScripts] = useState(new Set()); const [resultsData, setResultsData] = useState(null); @@ -98,7 +103,7 @@ export function PipelinePage({runType}) { const { pipeline, runHash } = useParams(); const [pipStates, setPipStates] = useReducer( pipReducer, - {runType, selectionUrl: pipeline, runHash}, + { runType, selectionUrl: pipeline, runHash }, pipInitialState ); @@ -111,27 +116,31 @@ export function PipelinePage({runType}) { let timeout; function loadPipelineOutputs() { if (pipStates.runHash) { - api.getOutputFolders(runType, pipStates.runId, (error, data, response) => { - if (error) { - showHttpError(error, response); - } else { - let allOutputFoldersKnown = Object.values(data).every( - (val) => val !== "" - ); - if (!allOutputFoldersKnown) { - // try again later - timeout = setTimeout(loadPipelineOutputs, 1000); + api.getOutputFolders( + runType, + pipStates.runId, + (error, data, response) => { + if (error) { + showHttpError(error, response); + } else { + let allOutputFoldersKnown = Object.values(data).every( + (val) => val !== "" + ); + if (!allOutputFoldersKnown) { + // try again later + timeout = setTimeout(loadPipelineOutputs, 1000); + } + setResultsData(data); } - setResultsData(data); } - }); + ); } else { setResultsData(null); } } function loadPipelineMetadata(choice, setExamples = true) { - setHttpError(null) + setHttpError(null); var callback = function (error, data, response) { if (error) { showHttpError(error, response); @@ -156,7 +165,8 @@ export function PipelinePage({runType}) { } function loadPipelineInputs(pip, hash) { - var inputJson = "/output/" + pip.replaceAll('>','/') + "/" + hash + "/input.json"; + var inputJson = + "/output/" + pip.replaceAll(">", "/") + "/" + hash + "/input.json"; fetch(inputJson) .then((response) => { if (response.ok) { @@ -164,7 +174,7 @@ export function PipelinePage({runType}) { } // This has never ran. No inputs to load. - return false; + return false; }) .then((json) => { if (json) { @@ -181,7 +191,7 @@ export function PipelinePage({runType}) { useEffect(() => { setResultsData(null); - switch(pipStates.lastAction) { + switch (pipStates.lastAction) { case "reset": loadPipelineMetadata(pipStates.descriptionFile, true); break; @@ -200,7 +210,8 @@ export function PipelinePage({runType}) { useEffect(() => { // set by the route if (pipeline) { - let descriptionFile = pipeline + (runType === "pipeline" ? ".json" : ".yml") + let descriptionFile = + pipeline + (runType === "pipeline" ? ".json" : ".yml"); setPipStates({ type: "url", newDescriptionFile: descriptionFile, @@ -264,6 +275,7 @@ export function PipelinePage({runType}) { resultsData={resultsData} runningScripts={runningScripts} setRunningScripts={setRunningScripts} + pipeline={pipeline} runHash={runHash} isPipeline={runType === "pipeline"} /> @@ -277,17 +289,23 @@ function PipelineResults({ resultsData, runningScripts, setRunningScripts, + pipeline, runHash, isPipeline, }) { const [activeRenderer, setActiveRenderer] = useState({}); const [pipelineOutputResults, setPipelineOutputResults] = useState({}); + let viewerHost = null; + if (process.env.REACT_APP_VIEWER_HOST) { + viewerHost = process.env.REACT_APP_VIEWER_HOST; + } + useEffect(() => { - if(!isPipeline && !isEmptyObject(resultsData)) { - setActiveRenderer(Object.keys(resultsData)[0]) + if (!isPipeline && !isEmptyObject(resultsData)) { + setActiveRenderer(Object.keys(resultsData)[0]); } - }, [resultsData, isPipeline, setActiveRenderer]) + }, [resultsData, isPipeline, setActiveRenderer]); useEffect(() => { // Put outputResults at initial value @@ -306,49 +324,62 @@ function PipelineResults({ value={createContext(activeRenderer, setActiveRenderer)} >
Running...
; - inline = ( -Waiting for previous steps to complete.
; diff --git a/ui/src/components/form/PipelineForm.js b/ui/src/components/form/PipelineForm.js index 110f76f2..e9c70384 100644 --- a/ui/src/components/form/PipelineForm.js +++ b/ui/src/components/form/PipelineForm.js @@ -100,7 +100,7 @@ export function PipelineForm({ setInputFileContent={setInputFileContent} />