From 644611218ad21f52332977fd3d13e7bea0f7d6f3 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Mon, 23 Oct 2023 15:24:51 +0100 Subject: [PATCH 01/27] added back R to build automation --- .github/workflows/build_main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_main.yml b/.github/workflows/build_main.yml index 8d3509ac7..c98ffb3b2 100644 --- a/.github/workflows/build_main.yml +++ b/.github/workflows/build_main.yml @@ -26,5 +26,7 @@ jobs: uses: ./.github/actions/scala_build - name: build python uses: ./.github/actions/python_build + - name: build R + uses: ./.github/actions/R_build - name: upload artefacts uses: ./.github/actions/upload_artefacts \ No newline at end of file From 19a71ec8f1ede144731f4f6770a3c34de7fa19ae Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Mon, 23 Oct 2023 16:44:20 +0100 Subject: [PATCH 02/27] temporarily reduced min coverage --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index eb4d8d27d..6799812e7 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ ${mosaic.version} - 80 + 20 true 1.11 1.11 From 0c2fbe5e19263ca65aef920ba4c5eb6eac904b96 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 24 Oct 2023 09:34:05 +0100 Subject: [PATCH 03/27] temporarily reduced min coverage --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6799812e7..cda3f14bb 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ ${mosaic.version} - 20 + 0 true 1.11 1.11 From f6bd09c537f25efcb57b1947bef625473fbf7e45 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 24 Oct 2023 10:08:50 +0100 Subject: [PATCH 04/27] update R `libPath` --- R/build_r_package.R | 1 + 1 file changed, 1 insertion(+) diff --git a/R/build_r_package.R b/R/build_r_package.R index 807688f91..960672eb2 100644 --- a/R/build_r_package.R +++ b/R/build_r_package.R @@ -23,6 +23,7 @@ for(repo in names(mirror_status)){ } } +.libPaths(c("/usr/lib/R/site-library", .libPaths())) install.packages("pkgbuild", repos=repo) install.packages("roxygen2", repos=repo) install.packages("sparklyr", repos=repo) From af24a7ced6d9a4099879e4f72eefc682d3283b5d Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 24 Oct 2023 10:55:47 +0100 Subject: [PATCH 05/27] install dependencies from posit public binaries --- R/build_r_package.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/build_r_package.R b/R/build_r_package.R index 960672eb2..6e9887455 100644 --- a/R/build_r_package.R +++ b/R/build_r_package.R @@ -1,5 +1,6 @@ repos = c( - "https://cran.ma.imperial.ac.uk" = "https://cran.ma.imperial.ac.uk" + CRAN = "https://packagemanager.posit.co/cran/__linux__/focal/latest" + ,"https://cran.ma.imperial.ac.uk" = "https://cran.ma.imperial.ac.uk" ,"https://www.stats.bris.ac.uk/R" = "https://www.stats.bris.ac.uk/R" ,"https://cran.rstudio.com/" = "https://cran.rstudio.com/" ) From 05f3c1a5c06135f9bbeb4074b7fac9dbfb760146 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 24 Oct 2023 10:58:25 +0100 Subject: [PATCH 06/27] remove references to `RasterAPI` --- R/sparkR-mosaic/enableMosaic.R | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/R/sparkR-mosaic/enableMosaic.R b/R/sparkR-mosaic/enableMosaic.R index 40a5c7b32..2f81da4f8 100644 --- a/R/sparkR-mosaic/enableMosaic.R +++ b/R/sparkR-mosaic/enableMosaic.R @@ -17,14 +17,11 @@ enableMosaic <- function( geometryAPI="JTS" ,indexSystem="H3" - ,rasterAPI="GDAL" ){ geometry_api <- sparkR.callJStatic(x="com.databricks.labs.mosaic.core.geometry.api.GeometryAPI", methodName="apply", geometryAPI) indexing_system <- sparkR.callJStatic(x="com.databricks.labs.mosaic.core.index.IndexSystemFactory", methodName="getIndexSystem", indexSystem) - raster_api <- sparkR.callJStatic(x="com.databricks.labs.mosaic.core.raster.api.RasterAPI", methodName="apply", rasterAPI) - - mosaic_context <- sparkR.newJObject(x="com.databricks.labs.mosaic.functions.MosaicContext", indexing_system, geometry_api, raster_api) + mosaic_context <- sparkR.newJObject(x="com.databricks.labs.mosaic.functions.MosaicContext", indexing_system, geometry_api) functions <<- sparkR.callJMethod(mosaic_context, "functions") # register the sql functions for use in sql() commands sparkR.callJMethod(mosaic_context, "register") From 739c89db607d6fb3fb02dce0b32d8d0194dae4d9 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 24 Oct 2023 11:07:37 +0100 Subject: [PATCH 07/27] added caching for slow download of Spark --- .github/actions/r_build/action.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/actions/r_build/action.yml b/.github/actions/r_build/action.yml index 2fa260bb4..ea4ed1097 100644 --- a/.github/actions/r_build/action.yml +++ b/.github/actions/r_build/action.yml @@ -7,12 +7,21 @@ runs: shell: bash run: | sudo apt-get update && sudo apt-get install -y curl libcurl4-openssl-dev pkg-config libharfbuzz-dev libfribidi-dev - - name: Download and unpack Spark + - name: Create download location for Spark shell: bash run: | sudo mkdir -p /usr/spark-download/raw sudo mkdir -p /usr/spark-download/unzipped sudo chown -R $USER: /usr/spark-download/ + - name: Cache Spark download + id: cache-spark + uses: actions/cache@v3 + with: + path: /usr/spark-download/unzipped + - if: ${{ steps.cache-spark.outputs.cache-hit != 'true' }} + name: Download and unpack Spark + shell: bash + run: | wget -P /usr/spark-download/raw https://archive.apache.org/dist/spark/spark-3.2.1/spark-3.2.1-bin-hadoop2.7.tgz tar zxvf /usr/spark-download/raw/spark-3.2.1-bin-hadoop2.7.tgz -C /usr/spark-download/unzipped From c53995c8f49a27ba81b5d7173cdee6a3bfb7f82b Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 24 Oct 2023 11:15:24 +0100 Subject: [PATCH 08/27] added caching for slow download of Spark --- .github/actions/r_build/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/r_build/action.yml b/.github/actions/r_build/action.yml index ea4ed1097..31c9a2811 100644 --- a/.github/actions/r_build/action.yml +++ b/.github/actions/r_build/action.yml @@ -18,6 +18,7 @@ runs: uses: actions/cache@v3 with: path: /usr/spark-download/unzipped + key: r_build-spark - if: ${{ steps.cache-spark.outputs.cache-hit != 'true' }} name: Download and unpack Spark shell: bash From cb11c4bd605c4aae016901323493b598cc92b367 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 24 Oct 2023 11:57:08 +0100 Subject: [PATCH 09/27] updated user agent for installing from posit public binaries --- R/build_r_package.R | 3 +++ 1 file changed, 3 insertions(+) diff --git a/R/build_r_package.R b/R/build_r_package.R index 6e9887455..41ef1dbb6 100644 --- a/R/build_r_package.R +++ b/R/build_r_package.R @@ -5,6 +5,9 @@ repos = c( ,"https://cran.rstudio.com/" = "https://cran.rstudio.com/" ) +options(HTTPUserAgent = sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) +options(download.file.extra = sprintf("--header \"User-Agent: R (%s)\"", paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) + mirror_is_up <- function(x){ out <- tryCatch({ available.packages(contrib.url(x)) From 0c9610ef282bc5d776f31746ed7e9e5e72c0ee72 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 24 Oct 2023 13:00:53 +0100 Subject: [PATCH 10/27] isolate install.packages issue --- .github/actions/r_build/action.yml | 52 +++++++++++++++++------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/.github/actions/r_build/action.yml b/.github/actions/r_build/action.yml index 31c9a2811..f3ee78ce2 100644 --- a/.github/actions/r_build/action.yml +++ b/.github/actions/r_build/action.yml @@ -3,36 +3,44 @@ description: build mosaic R runs: using: "composite" steps: - - name: Setup R build environment - shell: bash - run: | - sudo apt-get update && sudo apt-get install -y curl libcurl4-openssl-dev pkg-config libharfbuzz-dev libfribidi-dev - - name: Create download location for Spark +# - name: Setup R build environment +# shell: bash +# run: | +# sudo apt-get update && sudo apt-get install -y curl libcurl4-openssl-dev pkg-config libharfbuzz-dev libfribidi-dev +# - name: Create download location for Spark +# shell: bash +# run: | +# sudo mkdir -p /usr/spark-download/raw +# sudo mkdir -p /usr/spark-download/unzipped +# sudo chown -R $USER: /usr/spark-download/ +# - name: Cache Spark download +# id: cache-spark +# uses: actions/cache@v3 +# with: +# path: /usr/spark-download/unzipped +# key: r_build-spark +# - if: ${{ steps.cache-spark.outputs.cache-hit != 'true' }} +# name: Download and unpack Spark +# shell: bash +# run: | +# wget -P /usr/spark-download/raw https://archive.apache.org/dist/spark/spark-3.2.1/spark-3.2.1-bin-hadoop2.7.tgz +# tar zxvf /usr/spark-download/raw/spark-3.2.1-bin-hadoop2.7.tgz -C /usr/spark-download/unzipped + + - name: Create R environment shell: bash run: | - sudo mkdir -p /usr/spark-download/raw - sudo mkdir -p /usr/spark-download/unzipped - sudo chown -R $USER: /usr/spark-download/ - - name: Cache Spark download - id: cache-spark + sudo mkdir -p /usr/lib/R/site-library + sudo chown -R $USER: /usr/lib/R/site-library + - name: Cache R packages download + id: cache-rlibs uses: actions/cache@v3 with: - path: /usr/spark-download/unzipped - key: r_build-spark - - if: ${{ steps.cache-spark.outputs.cache-hit != 'true' }} - name: Download and unpack Spark - shell: bash - run: | - wget -P /usr/spark-download/raw https://archive.apache.org/dist/spark/spark-3.2.1/spark-3.2.1-bin-hadoop2.7.tgz - tar zxvf /usr/spark-download/raw/spark-3.2.1-bin-hadoop2.7.tgz -C /usr/spark-download/unzipped - - + path: /usr/lib/R/site-library + key: r_build-rlibs - name: Build R package shell: bash run: | cd R - sudo mkdir -p /usr/lib/R/site-library - sudo chown -R $USER: /usr/lib/R/site-library Rscript --vanilla build_r_package.R - name: Test R package shell: bash From cdf762aa01ed5b3cc3e0a8d2f9280ca10be1e40f Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 24 Oct 2023 13:13:09 +0100 Subject: [PATCH 11/27] isolate install.packages issue --- R/build_r_package.R | 2 ++ 1 file changed, 2 insertions(+) diff --git a/R/build_r_package.R b/R/build_r_package.R index 41ef1dbb6..5c5d732c0 100644 --- a/R/build_r_package.R +++ b/R/build_r_package.R @@ -8,6 +8,8 @@ repos = c( options(HTTPUserAgent = sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) options(download.file.extra = sprintf("--header \"User-Agent: R (%s)\"", paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) +print(sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) + mirror_is_up <- function(x){ out <- tryCatch({ available.packages(contrib.url(x)) From 9fcca23d97faf7de4ffe3af3660733c5caad5896 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 24 Oct 2023 13:29:09 +0100 Subject: [PATCH 12/27] isolate install.packages issue --- .github/actions/r_build/action.yml | 3 +++ R/build_r_package.R | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/r_build/action.yml b/.github/actions/r_build/action.yml index f3ee78ce2..8ea187777 100644 --- a/.github/actions/r_build/action.yml +++ b/.github/actions/r_build/action.yml @@ -37,6 +37,9 @@ runs: with: path: /usr/lib/R/site-library key: r_build-rlibs + - name: Test binary download + shell: bash + run: curl -H "User-Agent: R/4.3.1 R (4.3.1 x86_64-pc-linux-gnu x86_64 linux-gnu)" -I https://packagemanager.posit.co/cran/__linux__/focal/latest/src/contrib/A3_1.0.0.tar.gz - name: Build R package shell: bash run: | diff --git a/R/build_r_package.R b/R/build_r_package.R index 5c5d732c0..653a2d1f0 100644 --- a/R/build_r_package.R +++ b/R/build_r_package.R @@ -1,9 +1,9 @@ repos = c( - CRAN = "https://packagemanager.posit.co/cran/__linux__/focal/latest" - ,"https://cran.ma.imperial.ac.uk" = "https://cran.ma.imperial.ac.uk" + "https://cran.ma.imperial.ac.uk" = "https://cran.ma.imperial.ac.uk" ,"https://www.stats.bris.ac.uk/R" = "https://www.stats.bris.ac.uk/R" ,"https://cran.rstudio.com/" = "https://cran.rstudio.com/" ) +# CRAN = "https://packagemanager.posit.co/cran/__linux__/focal/latest" options(HTTPUserAgent = sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) options(download.file.extra = sprintf("--header \"User-Agent: R (%s)\"", paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) From 51145fbae5ac8f6dc47effcd2e8ae66ddafc6649 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 24 Oct 2023 13:29:45 +0100 Subject: [PATCH 13/27] isolate install.packages issue --- .github/actions/r_build/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/r_build/action.yml b/.github/actions/r_build/action.yml index 8ea187777..73e55ef8b 100644 --- a/.github/actions/r_build/action.yml +++ b/.github/actions/r_build/action.yml @@ -39,7 +39,8 @@ runs: key: r_build-rlibs - name: Test binary download shell: bash - run: curl -H "User-Agent: R/4.3.1 R (4.3.1 x86_64-pc-linux-gnu x86_64 linux-gnu)" -I https://packagemanager.posit.co/cran/__linux__/focal/latest/src/contrib/A3_1.0.0.tar.gz + run: | + curl -H "User-Agent: R/4.3.1 R (4.3.1 x86_64-pc-linux-gnu x86_64 linux-gnu)" -I https://packagemanager.posit.co/cran/__linux__/focal/latest/src/contrib/A3_1.0.0.tar.gz - name: Build R package shell: bash run: | From 318ff64ed7666d846520c7212efa66dd340e7efd Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 24 Oct 2023 14:36:21 +0100 Subject: [PATCH 14/27] isolate install.packages issue --- .github/actions/r_build/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/r_build/action.yml b/.github/actions/r_build/action.yml index 73e55ef8b..12c0466e9 100644 --- a/.github/actions/r_build/action.yml +++ b/.github/actions/r_build/action.yml @@ -3,10 +3,10 @@ description: build mosaic R runs: using: "composite" steps: -# - name: Setup R build environment -# shell: bash -# run: | -# sudo apt-get update && sudo apt-get install -y curl libcurl4-openssl-dev pkg-config libharfbuzz-dev libfribidi-dev + - name: Setup R build environment + shell: bash + run: | + sudo apt-get update && sudo apt-get install -y curl libcurl4-openssl-dev pkg-config libharfbuzz-dev libfribidi-dev # - name: Create download location for Spark # shell: bash # run: | From 20c65f701f808dc5583d81955738c3473d11e365 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Tue, 24 Oct 2023 14:52:13 +0100 Subject: [PATCH 15/27] isolate install.packages issue --- .github/actions/r_build/action.yml | 47 +++++++++++++++--------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/.github/actions/r_build/action.yml b/.github/actions/r_build/action.yml index 12c0466e9..b3d7035d3 100644 --- a/.github/actions/r_build/action.yml +++ b/.github/actions/r_build/action.yml @@ -7,25 +7,24 @@ runs: shell: bash run: | sudo apt-get update && sudo apt-get install -y curl libcurl4-openssl-dev pkg-config libharfbuzz-dev libfribidi-dev -# - name: Create download location for Spark -# shell: bash -# run: | -# sudo mkdir -p /usr/spark-download/raw -# sudo mkdir -p /usr/spark-download/unzipped -# sudo chown -R $USER: /usr/spark-download/ -# - name: Cache Spark download -# id: cache-spark -# uses: actions/cache@v3 -# with: -# path: /usr/spark-download/unzipped -# key: r_build-spark -# - if: ${{ steps.cache-spark.outputs.cache-hit != 'true' }} -# name: Download and unpack Spark -# shell: bash -# run: | -# wget -P /usr/spark-download/raw https://archive.apache.org/dist/spark/spark-3.2.1/spark-3.2.1-bin-hadoop2.7.tgz -# tar zxvf /usr/spark-download/raw/spark-3.2.1-bin-hadoop2.7.tgz -C /usr/spark-download/unzipped - + - name: Create download location for Spark + shell: bash + run: | + sudo mkdir -p /usr/spark-download/raw + sudo mkdir -p /usr/spark-download/unzipped + sudo chown -R $USER: /usr/spark-download/ + - name: Cache Spark download + id: cache-spark + uses: actions/cache@v3 + with: + path: /usr/spark-download/unzipped + key: r_build-spark + - if: ${{ steps.cache-spark.outputs.cache-hit != 'true' }} + name: Download and unpack Spark + shell: bash + run: | + wget -P /usr/spark-download/raw https://archive.apache.org/dist/spark/spark-3.2.1/spark-3.2.1-bin-hadoop2.7.tgz + tar zxvf /usr/spark-download/raw/spark-3.2.1-bin-hadoop2.7.tgz -C /usr/spark-download/unzipped - name: Create R environment shell: bash run: | @@ -46,11 +45,11 @@ runs: run: | cd R Rscript --vanilla build_r_package.R - - name: Test R package - shell: bash - run: | - cd R/sparkR-mosaic - Rscript --vanilla tests.R +# - name: Test R package +# shell: bash +# run: | +# cd R/sparkR-mosaic +# Rscript --vanilla tests.R - name: Copy R artifacts to GH Actions run shell: bash run: | From adfbda3023923cd49d2319dd558b28c424c71280 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 25 Oct 2023 10:06:58 +0100 Subject: [PATCH 16/27] split out steps to enable caching of dependencies --- .github/actions/r_build/action.yml | 16 +++++++++-- R/build_r_package.R | 46 ------------------------------ R/generate_docs.R | 10 +++++++ R/install_deps.R | 36 +++++++++++++++++++++++ 4 files changed, 60 insertions(+), 48 deletions(-) create mode 100644 R/generate_docs.R create mode 100644 R/install_deps.R diff --git a/.github/actions/r_build/action.yml b/.github/actions/r_build/action.yml index b3d7035d3..841eb1cf6 100644 --- a/.github/actions/r_build/action.yml +++ b/.github/actions/r_build/action.yml @@ -36,10 +36,22 @@ runs: with: path: /usr/lib/R/site-library key: r_build-rlibs - - name: Test binary download + - if: ${{ steps.cache-rlibs.outputs.cache-hit != 'true' }} + name: Install R dependencies shell: bash run: | - curl -H "User-Agent: R/4.3.1 R (4.3.1 x86_64-pc-linux-gnu x86_64 linux-gnu)" -I https://packagemanager.posit.co/cran/__linux__/focal/latest/src/contrib/A3_1.0.0.tar.gz + cd R + Rscript --vanilla install_deps.R + - name: Generate R bindings + shell: bash + run: | + cd R + Rscript --vanilla generate_R_bindings.R ../src/main/scala/com/databricks/labs/mosaic/functions/MosaicContext.scala + - name: Build R docs + shell: bash + run: | + cd R + Rscript --vanilla generate_docs.R - name: Build R package shell: bash run: | diff --git a/R/build_r_package.R b/R/build_r_package.R index 653a2d1f0..ab82b99b1 100644 --- a/R/build_r_package.R +++ b/R/build_r_package.R @@ -1,38 +1,3 @@ -repos = c( - "https://cran.ma.imperial.ac.uk" = "https://cran.ma.imperial.ac.uk" - ,"https://www.stats.bris.ac.uk/R" = "https://www.stats.bris.ac.uk/R" - ,"https://cran.rstudio.com/" = "https://cran.rstudio.com/" -) -# CRAN = "https://packagemanager.posit.co/cran/__linux__/focal/latest" - -options(HTTPUserAgent = sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) -options(download.file.extra = sprintf("--header \"User-Agent: R (%s)\"", paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) - -print(sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) - -mirror_is_up <- function(x){ - out <- tryCatch({ - available.packages(contrib.url(x)) - } - ,error = function(cond){return(0)} - ,warning = function(cond){return(0)} - ,finally = function(cond){} - ) - return(length(out)) -} - -mirror_status = lapply(repos, mirror_is_up) -for(repo in names(mirror_status)){ - if (mirror_status[[repo]] > 1){ - repo <<- repo - break - } -} - -.libPaths(c("/usr/lib/R/site-library", .libPaths())) -install.packages("pkgbuild", repos=repo) -install.packages("roxygen2", repos=repo) -install.packages("sparklyr", repos=repo) spark_location <- "/usr/spark-download/unzipped/spark-3.2.1-bin-hadoop2.7" Sys.setenv(SPARK_HOME = spark_location) @@ -40,22 +5,11 @@ library(SparkR, lib.loc = c(file.path(spark_location, "R", "lib"))) library(pkgbuild) -library(roxygen2) library(sparklyr) build_mosaic_bindings <- function(){ - # build functions - scala_file_path <- "../src/main/scala/com/databricks/labs/mosaic/functions/MosaicContext.scala" - system_cmd <- paste0(c("Rscript --vanilla generate_R_bindings.R", scala_file_path), collapse = " ") - system(system_cmd) - - # build doc - roxygen2::roxygenize("sparkR-mosaic/sparkrMosaic") - roxygen2::roxygenize("sparklyr-mosaic/sparklyrMosaic") - - ## build package pkgbuild::build("sparkR-mosaic/sparkrMosaic") pkgbuild::build("sparklyr-mosaic/sparklyrMosaic") diff --git a/R/generate_docs.R b/R/generate_docs.R new file mode 100644 index 000000000..2ef419692 --- /dev/null +++ b/R/generate_docs.R @@ -0,0 +1,10 @@ +library(roxygen2) + +build_mosaic_docs <- function(){ + # build doc + roxygen2::roxygenize("sparkR-mosaic/sparkrMosaic") + roxygen2::roxygenize("sparklyr-mosaic/sparklyrMosaic") + +} + +build_mosaic_docs() \ No newline at end of file diff --git a/R/install_deps.R b/R/install_deps.R new file mode 100644 index 000000000..19654f985 --- /dev/null +++ b/R/install_deps.R @@ -0,0 +1,36 @@ +repos = c( + "https://cran.ma.imperial.ac.uk" = "https://cran.ma.imperial.ac.uk" + ,"https://www.stats.bris.ac.uk/R" = "https://www.stats.bris.ac.uk/R" + ,"https://cran.rstudio.com/" = "https://cran.rstudio.com/" +) +# CRAN = "https://packagemanager.posit.co/cran/__linux__/focal/latest" + +options(HTTPUserAgent = sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) +options(download.file.extra = sprintf("--header \"User-Agent: R (%s)\"", paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) + +print(sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) + +mirror_is_up <- function(x){ + out <- tryCatch({ + available.packages(contrib.url(x)) + } + ,error = function(cond){return(0)} + ,warning = function(cond){return(0)} + ,finally = function(cond){} + ) + return(length(out)) +} + +mirror_status = lapply(repos, mirror_is_up) +for(repo in names(mirror_status)){ + if (mirror_status[[repo]] > 1){ + repo <<- repo + break + } +} + +.libPaths(c("/usr/lib/R/site-library", .libPaths())) + +install.packages("pkgbuild", repos=repo) +install.packages("roxygen2", repos=repo) +install.packages("sparklyr", repos=repo) \ No newline at end of file From db84dcb691f2d0f88836159b628c622bd31073c7 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 25 Oct 2023 10:13:42 +0100 Subject: [PATCH 17/27] split out steps to enable caching of dependencies --- R/generate_docs.R | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/R/generate_docs.R b/R/generate_docs.R index 2ef419692..06b23e6fa 100644 --- a/R/generate_docs.R +++ b/R/generate_docs.R @@ -1,3 +1,7 @@ +spark_location <- "/usr/spark-download/unzipped/spark-3.2.1-bin-hadoop2.7" +Sys.setenv(SPARK_HOME = spark_location) + +library(SparkR, lib.loc = c(file.path(spark_location, "R", "lib"))) library(roxygen2) build_mosaic_docs <- function(){ From bd0f8d50191106d0a61e2e031e7cedc0dc5969fe Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 25 Oct 2023 10:21:57 +0100 Subject: [PATCH 18/27] add back R tests --- .github/actions/r_build/action.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/r_build/action.yml b/.github/actions/r_build/action.yml index 841eb1cf6..e2b611e35 100644 --- a/.github/actions/r_build/action.yml +++ b/.github/actions/r_build/action.yml @@ -57,11 +57,11 @@ runs: run: | cd R Rscript --vanilla build_r_package.R -# - name: Test R package -# shell: bash -# run: | -# cd R/sparkR-mosaic -# Rscript --vanilla tests.R + - name: Test R package + shell: bash + run: | + cd R/sparkR-mosaic + Rscript --vanilla tests.R - name: Copy R artifacts to GH Actions run shell: bash run: | From d5f18f4c2eda02926ab3cac4b4ab9f02beaebf2e Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 25 Oct 2023 11:56:03 +0100 Subject: [PATCH 19/27] fixed existing R tests --- .github/actions/r_build/action.yml | 2 +- R/.gitignore | 1 + R/install_deps.R | 6 +++--- R/sparkR-mosaic/sparkrMosaic/DESCRIPTION | 2 +- R/sparkR-mosaic/tests.R | 4 ++-- R/sparklyr-mosaic/sparklyrMosaic/DESCRIPTION | 2 +- 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/actions/r_build/action.yml b/.github/actions/r_build/action.yml index e2b611e35..9660e29ef 100644 --- a/.github/actions/r_build/action.yml +++ b/.github/actions/r_build/action.yml @@ -10,8 +10,8 @@ runs: - name: Create download location for Spark shell: bash run: | - sudo mkdir -p /usr/spark-download/raw sudo mkdir -p /usr/spark-download/unzipped + sudo mkdir -p /usr/spark-download/raw sudo chown -R $USER: /usr/spark-download/ - name: Cache Spark download id: cache-spark diff --git a/R/.gitignore b/R/.gitignore index bef411cb1..eb4bec116 100644 --- a/R/.gitignore +++ b/R/.gitignore @@ -1 +1,2 @@ **/.Rhistory +**/*.tar.gz diff --git a/R/install_deps.R b/R/install_deps.R index 19654f985..68cb8a355 100644 --- a/R/install_deps.R +++ b/R/install_deps.R @@ -31,6 +31,6 @@ for(repo in names(mirror_status)){ .libPaths(c("/usr/lib/R/site-library", .libPaths())) -install.packages("pkgbuild", repos=repo) -install.packages("roxygen2", repos=repo) -install.packages("sparklyr", repos=repo) \ No newline at end of file +install.packages("pkgbuild", repos="https://packagemanager.posit.co/cran/__linux__/bullseye/latest") +install.packages("roxygen2", repos="https://packagemanager.posit.co/cran/__linux__/bullseye/latest") +install.packages("sparklyr", repos="https://packagemanager.posit.co/cran/__linux__/bullseye/latest") \ No newline at end of file diff --git a/R/sparkR-mosaic/sparkrMosaic/DESCRIPTION b/R/sparkR-mosaic/sparkrMosaic/DESCRIPTION index 3fe2ab1a6..eb25a280a 100644 --- a/R/sparkR-mosaic/sparkrMosaic/DESCRIPTION +++ b/R/sparkR-mosaic/sparkrMosaic/DESCRIPTION @@ -8,7 +8,7 @@ Description: This package extends SparkR to bring the Databricks Mosaic for geos License: Databricks Encoding: UTF-8 Roxygen: list(markdown = TRUE) -RoxygenNote: 7.2.0 +RoxygenNote: 7.2.3 Collate: 'enableMosaic.R' 'generics.R' diff --git a/R/sparkR-mosaic/tests.R b/R/sparkR-mosaic/tests.R index 34b0c61f7..7cb4e9c50 100644 --- a/R/sparkR-mosaic/tests.R +++ b/R/sparkR-mosaic/tests.R @@ -66,7 +66,7 @@ sdf <- withColumn(sdf, "transformed_geom", st_transform(column("geom_with_srid") # Grid functions sdf <- withColumn(sdf, "grid_longlatascellid", grid_longlatascellid(lit(1), lit(1), lit(1L))) sdf <- withColumn(sdf, "grid_pointascellid", grid_pointascellid(column("point_wkt"), lit(1L))) -sdf <- withColumn(sdf, "grid_boundaryaswkb", grid_boundaryaswkb( SparkR::cast(lit(1), "long"))) +sdf <- withColumn(sdf, "grid_boundaryaswkb", grid_boundaryaswkb(column("grid_pointascellid"))) sdf <- withColumn(sdf, "grid_polyfill", grid_polyfill(column("wkt"), lit(1L))) sdf <- withColumn(sdf, "grid_tessellateexplode", grid_tessellateexplode(column("wkt"), lit(1L))) sdf <- withColumn(sdf, "grid_tessellate", grid_tessellate(column("wkt"), lit(1L))) @@ -74,7 +74,7 @@ sdf <- withColumn(sdf, "grid_tessellate", grid_tessellate(column("wkt"), lit(1L) # Deprecated sdf <- withColumn(sdf, "point_index_lonlat", point_index_lonlat(lit(1), lit(1), lit(1L))) sdf <- withColumn(sdf, "point_index_geom", point_index_geom(column("point_wkt"), lit(1L))) -sdf <- withColumn(sdf, "index_geometry", index_geometry( SparkR::cast(lit(1), "long"))) +sdf <- withColumn(sdf, "index_geometry", index_geometry(column("point_index_geom"))) sdf <- withColumn(sdf, "polyfill", polyfill(column("wkt"), lit(1L))) sdf <- withColumn(sdf, "mosaic_explode", mosaic_explode(column("wkt"), lit(1L))) sdf <- withColumn(sdf, "mosaicfill", mosaicfill(column("wkt"), lit(1L))) diff --git a/R/sparklyr-mosaic/sparklyrMosaic/DESCRIPTION b/R/sparklyr-mosaic/sparklyrMosaic/DESCRIPTION index 9bf65215a..0e5bcbb38 100644 --- a/R/sparklyr-mosaic/sparklyrMosaic/DESCRIPTION +++ b/R/sparklyr-mosaic/sparklyrMosaic/DESCRIPTION @@ -8,7 +8,7 @@ Description: This package extends sparklyr to bring the Databricks Mosaic for ge License: Databricks Encoding: UTF-8 Roxygen: list(markdown = TRUE) -RoxygenNote: 7.2.0 +RoxygenNote: 7.2.3 Collate: 'enableMosaic.R' 'sparkFunctions.R' From 1aa7bc24844704ab4e1b44fdf30bf67ec1453da8 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 25 Oct 2023 12:25:21 +0100 Subject: [PATCH 20/27] testing rlib actions --- .github/actions/r_build/action.yml | 35 ++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/.github/actions/r_build/action.yml b/.github/actions/r_build/action.yml index 9660e29ef..edf9caafe 100644 --- a/.github/actions/r_build/action.yml +++ b/.github/actions/r_build/action.yml @@ -30,18 +30,31 @@ runs: run: | sudo mkdir -p /usr/lib/R/site-library sudo chown -R $USER: /usr/lib/R/site-library - - name: Cache R packages download - id: cache-rlibs - uses: actions/cache@v3 +# - name: Cache R packages download +# id: cache-rlibs +# uses: actions/cache@v3 +# with: +# path: /usr/lib/R/site-library +# key: r_build-rlibs +# - if: ${{ steps.cache-rlibs.outputs.cache-hit != 'true' }} +# name: Install R dependencies +# shell: bash +# run: | +# cd R +# Rscript --vanilla install_deps.R + - name: Setup R + uses: r-lib/actions/setup-r@v2 with: - path: /usr/lib/R/site-library - key: r_build-rlibs - - if: ${{ steps.cache-rlibs.outputs.cache-hit != 'true' }} - name: Install R dependencies - shell: bash - run: | - cd R - Rscript --vanilla install_deps.R + r-version: ${{ matrix.R }} + use-public-rspm: true + - name: Install R dependencies + uses: r-lib/actions/setup-r-dependencies@v2 + with: + cache: true + extra-packages: | + any::pkgbuild + any::roxygen2 + any::sparklyr - name: Generate R bindings shell: bash run: | From 280f003bb9d17ae3a1c20fd5c57da9c51b60d96f Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 25 Oct 2023 12:27:53 +0100 Subject: [PATCH 21/27] corrected typo in `build_main.yml` --- .github/workflows/build_main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_main.yml b/.github/workflows/build_main.yml index c98ffb3b2..ac5cb0623 100644 --- a/.github/workflows/build_main.yml +++ b/.github/workflows/build_main.yml @@ -27,6 +27,6 @@ jobs: - name: build python uses: ./.github/actions/python_build - name: build R - uses: ./.github/actions/R_build + uses: ./.github/actions/r_build - name: upload artefacts uses: ./.github/actions/upload_artefacts \ No newline at end of file From 4a9e4c5202b4e432c382f28b55ea982211476e43 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 25 Oct 2023 12:47:34 +0100 Subject: [PATCH 22/27] testing `use-public-rspm` flag --- .github/actions/r_build/action.yml | 38 +++++++++++++++--------------- R/install_deps.R | 9 ++++--- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/.github/actions/r_build/action.yml b/.github/actions/r_build/action.yml index edf9caafe..3aea3d036 100644 --- a/.github/actions/r_build/action.yml +++ b/.github/actions/r_build/action.yml @@ -30,31 +30,31 @@ runs: run: | sudo mkdir -p /usr/lib/R/site-library sudo chown -R $USER: /usr/lib/R/site-library -# - name: Cache R packages download -# id: cache-rlibs -# uses: actions/cache@v3 -# with: -# path: /usr/lib/R/site-library -# key: r_build-rlibs -# - if: ${{ steps.cache-rlibs.outputs.cache-hit != 'true' }} -# name: Install R dependencies -# shell: bash -# run: | -# cd R -# Rscript --vanilla install_deps.R - name: Setup R uses: r-lib/actions/setup-r@v2 with: r-version: ${{ matrix.R }} use-public-rspm: true - - name: Install R dependencies - uses: r-lib/actions/setup-r-dependencies@v2 + - name: Cache R packages download + id: cache-rlibs + uses: actions/cache@v3 with: - cache: true - extra-packages: | - any::pkgbuild - any::roxygen2 - any::sparklyr + path: /usr/lib/R/site-library + key: r_build-rlibs +# - if: ${{ steps.cache-rlibs.outputs.cache-hit != 'true' }} + - name: Install R dependencies + shell: bash + run: | + cd R + Rscript --vanilla install_deps.R +# - name: Install R dependencies +# uses: r-lib/actions/setup-r-dependencies@v2 +# with: +# cache: true +# extra-packages: | +# any::pkgbuild +# any::roxygen2 +# any::sparklyr - name: Generate R bindings shell: bash run: | diff --git a/R/install_deps.R b/R/install_deps.R index 68cb8a355..0bfb6f6ed 100644 --- a/R/install_deps.R +++ b/R/install_deps.R @@ -31,6 +31,9 @@ for(repo in names(mirror_status)){ .libPaths(c("/usr/lib/R/site-library", .libPaths())) -install.packages("pkgbuild", repos="https://packagemanager.posit.co/cran/__linux__/bullseye/latest") -install.packages("roxygen2", repos="https://packagemanager.posit.co/cran/__linux__/bullseye/latest") -install.packages("sparklyr", repos="https://packagemanager.posit.co/cran/__linux__/bullseye/latest") \ No newline at end of file +# install.packages("pkgbuild", repos="https://packagemanager.posit.co/cran/__linux__/bullseye/latest") +# install.packages("roxygen2", repos="https://packagemanager.posit.co/cran/__linux__/bullseye/latest") +# install.packages("sparklyr", repos="https://packagemanager.posit.co/cran/__linux__/bullseye/latest") +install.packages("pkgbuild") +install.packages("roxygen2") +install.packages("sparklyr") \ No newline at end of file From ec256c8b89cacb01b1153a0716afe7616cb539a9 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 25 Oct 2023 12:56:42 +0100 Subject: [PATCH 23/27] testing `use-public-rspm` flag --- R/install_deps.R | 57 ++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/R/install_deps.R b/R/install_deps.R index 0bfb6f6ed..8024ac650 100644 --- a/R/install_deps.R +++ b/R/install_deps.R @@ -1,33 +1,34 @@ -repos = c( - "https://cran.ma.imperial.ac.uk" = "https://cran.ma.imperial.ac.uk" - ,"https://www.stats.bris.ac.uk/R" = "https://www.stats.bris.ac.uk/R" - ,"https://cran.rstudio.com/" = "https://cran.rstudio.com/" -) +# repos = c( +# "https://cran.ma.imperial.ac.uk" = "https://cran.ma.imperial.ac.uk" +# ,"https://www.stats.bris.ac.uk/R" = "https://www.stats.bris.ac.uk/R" +# ,"https://cran.rstudio.com/" = "https://cran.rstudio.com/" +# ) +options(repos = c(CRAN = "https://packagemanager.posit.co/cran/__linux__/focal/latest")) # CRAN = "https://packagemanager.posit.co/cran/__linux__/focal/latest" +# +# options(HTTPUserAgent = sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) +# options(download.file.extra = sprintf("--header \"User-Agent: R (%s)\"", paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) +# +# print(sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) -options(HTTPUserAgent = sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) -options(download.file.extra = sprintf("--header \"User-Agent: R (%s)\"", paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) - -print(sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) - -mirror_is_up <- function(x){ - out <- tryCatch({ - available.packages(contrib.url(x)) - } - ,error = function(cond){return(0)} - ,warning = function(cond){return(0)} - ,finally = function(cond){} - ) - return(length(out)) -} - -mirror_status = lapply(repos, mirror_is_up) -for(repo in names(mirror_status)){ - if (mirror_status[[repo]] > 1){ - repo <<- repo - break - } -} +# mirror_is_up <- function(x){ +# out <- tryCatch({ +# available.packages(contrib.url(x)) +# } +# ,error = function(cond){return(0)} +# ,warning = function(cond){return(0)} +# ,finally = function(cond){} +# ) +# return(length(out)) +# } +# +# mirror_status = lapply(repos, mirror_is_up) +# for(repo in names(mirror_status)){ +# if (mirror_status[[repo]] > 1){ +# repo <<- repo +# break +# } +# } .libPaths(c("/usr/lib/R/site-library", .libPaths())) From 0c9c598d76326f5572192f1cc2858bbb16f717c7 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 25 Oct 2023 13:35:13 +0100 Subject: [PATCH 24/27] testing `use-public-rspm` flag --- R/install_deps.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/install_deps.R b/R/install_deps.R index 8024ac650..8a473eb7c 100644 --- a/R/install_deps.R +++ b/R/install_deps.R @@ -30,7 +30,7 @@ options(repos = c(CRAN = "https://packagemanager.posit.co/cran/__linux__/focal/l # } # } -.libPaths(c("/usr/lib/R/site-library", .libPaths())) +# .libPaths(c("/usr/lib/R/site-library", .libPaths())) # install.packages("pkgbuild", repos="https://packagemanager.posit.co/cran/__linux__/bullseye/latest") # install.packages("roxygen2", repos="https://packagemanager.posit.co/cran/__linux__/bullseye/latest") From a6f2fc0f36bf042c5351953a18b0f794f591796b Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 25 Oct 2023 13:44:01 +0100 Subject: [PATCH 25/27] removed some cruft from the R build files --- .github/actions/r_build/action.yml | 15 ------------- R/install_deps.R | 35 ------------------------------ R/sparkR-mosaic/tests.R | 2 -- 3 files changed, 52 deletions(-) diff --git a/.github/actions/r_build/action.yml b/.github/actions/r_build/action.yml index 3aea3d036..e970f2fdb 100644 --- a/.github/actions/r_build/action.yml +++ b/.github/actions/r_build/action.yml @@ -35,26 +35,11 @@ runs: with: r-version: ${{ matrix.R }} use-public-rspm: true - - name: Cache R packages download - id: cache-rlibs - uses: actions/cache@v3 - with: - path: /usr/lib/R/site-library - key: r_build-rlibs -# - if: ${{ steps.cache-rlibs.outputs.cache-hit != 'true' }} - name: Install R dependencies shell: bash run: | cd R Rscript --vanilla install_deps.R -# - name: Install R dependencies -# uses: r-lib/actions/setup-r-dependencies@v2 -# with: -# cache: true -# extra-packages: | -# any::pkgbuild -# any::roxygen2 -# any::sparklyr - name: Generate R bindings shell: bash run: | diff --git a/R/install_deps.R b/R/install_deps.R index 8a473eb7c..d05207329 100644 --- a/R/install_deps.R +++ b/R/install_deps.R @@ -1,40 +1,5 @@ -# repos = c( -# "https://cran.ma.imperial.ac.uk" = "https://cran.ma.imperial.ac.uk" -# ,"https://www.stats.bris.ac.uk/R" = "https://www.stats.bris.ac.uk/R" -# ,"https://cran.rstudio.com/" = "https://cran.rstudio.com/" -# ) options(repos = c(CRAN = "https://packagemanager.posit.co/cran/__linux__/focal/latest")) -# CRAN = "https://packagemanager.posit.co/cran/__linux__/focal/latest" -# -# options(HTTPUserAgent = sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) -# options(download.file.extra = sprintf("--header \"User-Agent: R (%s)\"", paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) -# -# print(sprintf("R/%s R (%s)", getRversion(), paste(getRversion(), R.version["platform"], R.version["arch"], R.version["os"]))) -# mirror_is_up <- function(x){ -# out <- tryCatch({ -# available.packages(contrib.url(x)) -# } -# ,error = function(cond){return(0)} -# ,warning = function(cond){return(0)} -# ,finally = function(cond){} -# ) -# return(length(out)) -# } -# -# mirror_status = lapply(repos, mirror_is_up) -# for(repo in names(mirror_status)){ -# if (mirror_status[[repo]] > 1){ -# repo <<- repo -# break -# } -# } - -# .libPaths(c("/usr/lib/R/site-library", .libPaths())) - -# install.packages("pkgbuild", repos="https://packagemanager.posit.co/cran/__linux__/bullseye/latest") -# install.packages("roxygen2", repos="https://packagemanager.posit.co/cran/__linux__/bullseye/latest") -# install.packages("sparklyr", repos="https://packagemanager.posit.co/cran/__linux__/bullseye/latest") install.packages("pkgbuild") install.packages("roxygen2") install.packages("sparklyr") \ No newline at end of file diff --git a/R/sparkR-mosaic/tests.R b/R/sparkR-mosaic/tests.R index 7cb4e9c50..f4071c3d2 100644 --- a/R/sparkR-mosaic/tests.R +++ b/R/sparkR-mosaic/tests.R @@ -1,5 +1,3 @@ -repo<-"https://cran.ma.imperial.ac.uk/" - spark_location <- "/usr/spark-download/unzipped/spark-3.2.1-bin-hadoop2.7" Sys.setenv(SPARK_HOME = spark_location) library(SparkR, lib.loc = c(file.path(spark_location, "R", "lib"))) From 0f60feb37870fb660f243e2af085c1d909304d9f Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Wed, 25 Oct 2023 13:50:13 +0100 Subject: [PATCH 26/27] restored `minimum.coverage` in `pom.xml` --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cda3f14bb..eb4d8d27d 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ ${mosaic.version} - 0 + 80 true 1.11 1.11 From fc1cc74a24b8a9dd731f6e3d4678c3c2ad47d9fc Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Thu, 26 Oct 2023 14:16:54 +0100 Subject: [PATCH 27/27] skip coverage check if tests skipped --- .github/actions/scala_build/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/scala_build/action.yml b/.github/actions/scala_build/action.yml index 85b70451b..950ae2200 100644 --- a/.github/actions/scala_build/action.yml +++ b/.github/actions/scala_build/action.yml @@ -27,8 +27,9 @@ runs: - name: Build the scala JAR - skip tests is true if: inputs.skip_tests == 'true' shell: bash - run: sudo mvn -q clean install -DskipTests + run: sudo mvn -q clean install -DskipTests -Dscoverage.skip - name: Publish test coverage + if: inputs.skip_tests == 'false' uses: codecov/codecov-action@v1 - name: Copy Scala artifacts to GH Actions run shell: bash