Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Small area stats #597

Merged
merged 15 commits into from
Nov 13, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -148,9 +148,19 @@ case class MosaicRasterBandGDAL(band: Band, id: Int) {
*/
def computeMinMax: Seq[Double] = {
val minMaxVals = Array.fill[Double](2)(0)
Try(band.ComputeRasterMinMax(minMaxVals, 0))
.map(_ => minMaxVals.toSeq)
.getOrElse(Seq(Double.NaN, Double.NaN))
// will GDAL refuse to compute these stats?
if (band.GetXSize() == 1 || band.GetYSize() == 1) {
val validPixels = values.filter(_ != noDataValue)
if (validPixels.isEmpty) {
return Seq(Double.NaN, Double.NaN)
} else {
Seq(validPixels.min, validPixels.max)
}
} else {
Try(band.ComputeRasterMinMax(minMaxVals, 0))
.map(_ => minMaxVals.toSeq)
.getOrElse(Seq(Double.NaN, Double.NaN))
}
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ class RasterAsGridReader(sparkSession: SparkSession) extends MosaicDataFrameRead
} else {
lit(config("convertToFormat"))
}

val rasterToGridCombiner = getRasterToGridFunc(config("combiner"))

val loadedDf = retiledDf
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ import org.scalatest.matchers.should.Matchers._

trait RST_MedianBehaviors extends QueryTest {

def behavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {
def largeAreaBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {

val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
Expand Down Expand Up @@ -49,4 +50,38 @@ trait RST_MedianBehaviors extends QueryTest {

}

def smallAreaBehavior(indexSystem: IndexSystem, geometryAPI: GeometryAPI): Unit = {

val mc = MosaicContext.build(indexSystem, geometryAPI)
mc.register()
val sc = spark
import mc.functions._
import sc.implicits._

val testRegion = "Polygon ((-8151524 1216659, -8151061 1216659, -8151061 1217123, -8151524 1217123, -8151524 1216659))"

val rastersInMemory = spark.read
.format("gdal")
.option("raster_storage", "in-memory")
.load("src/test/resources/modis")

rastersInMemory.createOrReplaceTempView("source")

val df = rastersInMemory
.withColumn("tile", rst_clip($"tile", st_buffer(lit(testRegion), lit(-20))))
.withColumn("result", rst_median($"tile"))
.select("result")
.select(explode($"result").as("result"))

noException should be thrownBy spark.sql("""
|select rst_median(tile) from source
|""".stripMargin)

val result = df.as[Double].collect().max

result should be > 0.0

}


}
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,18 @@ class RST_MedianTest extends QueryTest with SharedSparkSessionGDAL with RST_Medi

// These tests are not index system nor geometry API specific.
// Only testing one pairing is sufficient.
test("Testing rst_median behavior with H3IndexSystem and JTS") {
test("Testing rst_median behavior with H3IndexSystem and JTS (tessellation case)") {
noCodegen {
assume(System.getProperty("os.name") == "Linux")
behavior(H3IndexSystem, JTS)
largeAreaBehavior(H3IndexSystem, JTS)
}
}

test("Testing rst_median behavior with H3IndexSystem and JTS (small area case)") {
noCodegen {
assume(System.getProperty("os.name") == "Linux")
smallAreaBehavior(H3IndexSystem, JTS)
}
}

}
Loading