-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #252 from wri/gtc-3055
GTC-3055 Added new GHG (green-house gas) analysis
- Loading branch information
Showing
29 changed files
with
1,096 additions
and
46 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
23 changes: 23 additions & 0 deletions
23
src/main/scala/org/globalforestwatch/features/GfwProFeatureExt.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| package org.globalforestwatch.features | ||
|
|
||
| // GfwPro Feature that includes commodity and yield columns for GHG analysis. | ||
|
|
||
| object GfwProFeatureExt extends Feature { | ||
| val listIdPos = 0 | ||
| val locationIdPos = 1 | ||
| val geomPos = 2 | ||
| val commodityPos = 3 | ||
| val yieldPos = 4 | ||
|
|
||
| val featureIdExpr = "list_id as listId, cast(location_id as int) as locationId, commodity as commodity, yield as yield" | ||
|
|
||
| def getFeatureId(i: Array[String], parsed: Boolean = false): FeatureId = { | ||
|
|
||
| val listId: String = i(0) | ||
| val locationId: Int = i(1).toInt | ||
| val commodity: String = i(2) | ||
| val yieldVal: Float = i(3).toFloat | ||
|
|
||
| GfwProFeatureExtId(listId, locationId, commodity, yieldVal) | ||
| } | ||
| } |
7 changes: 7 additions & 0 deletions
7
src/main/scala/org/globalforestwatch/features/GfwProFeatureExtId.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| package org.globalforestwatch.features | ||
|
|
||
| // GfwPro FeatureId that includes commodity and yield for GHG analysis. | ||
|
|
||
| case class GfwProFeatureExtId(listId: String, locationId: Int, commodity: String, yieldVal: Float) extends FeatureId { | ||
| override def toString: String = s"$listId, $locationId, $commodity, $yieldVal" | ||
| } |
17 changes: 17 additions & 0 deletions
17
src/main/scala/org/globalforestwatch/layers/MapspamYield.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| package org.globalforestwatch.layers | ||
|
|
||
| import org.globalforestwatch.grids.GridTile | ||
|
|
||
| /** Parameterized layer for all the various Mapspam commodity yield datasets. | ||
| * 'commodity' should be one of the four-letter uppercase Mapspam commodity codes, | ||
| * such as 'COCO' or 'OILP'. | ||
| */ | ||
| case class MapspamYield(commodity: String, gridTile: GridTile, kwargs: Map[String, Any]) | ||
| extends FloatLayer | ||
| with OptionalFLayer { | ||
|
|
||
| val datasetName = s"mapspam_yield_${commodity.toLowerCase()}" | ||
|
|
||
| val uri: String = | ||
| uriForGrid(gridTile, kwargs) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
72 changes: 72 additions & 0 deletions
72
src/main/scala/org/globalforestwatch/summarystats/ghg/GHGAnalysis.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| package org.globalforestwatch.summarystats.ghg | ||
|
|
||
| import cats.data.Validated.{Invalid, Valid} | ||
| import cats.implicits._ | ||
|
|
||
| import geotrellis.vector.{Feature, Geometry} | ||
| import org.locationtech.jts.geom.Geometry | ||
| import org.apache.spark.rdd.RDD | ||
| import org.apache.spark.sql.SparkSession | ||
| import org.globalforestwatch.summarystats.{Location, NoIntersectionError, SummaryAnalysis, ValidatedLocation} | ||
| import org.apache.spark.storage.StorageLevel | ||
| import org.globalforestwatch.ValidatedWorkflow | ||
|
|
||
| object GHGAnalysis extends SummaryAnalysis { | ||
|
|
||
| val name = "ghg" | ||
|
|
||
| /** Greenhouse gas analysis of input features in a TSV file. The TSV file contains | ||
| * the individual list items (location IDs >= 0) and optional merged ("dissolved") | ||
| * list geometries (location id -1). | ||
| * | ||
| * This function assumes that all features have already been split by 1x1 degree | ||
| * grid, so each location and merged list may have a single or multiple rows. | ||
| */ | ||
| def apply( | ||
| features: RDD[ValidatedLocation[Geometry]], | ||
| kwargs: Map[String, Any] | ||
| )(implicit spark: SparkSession): RDD[ValidatedLocation[GHGData]] = { | ||
| features.persist(StorageLevel.MEMORY_AND_DISK) | ||
|
|
||
| try { | ||
| val partialResult: RDD[ValidatedLocation[GHGData]] = { | ||
| ValidatedWorkflow(features) | ||
| .flatMap { locationGeometries => | ||
| val locationSummaries: RDD[ValidatedLocation[GHGSummary]] = { | ||
| val tmp = locationGeometries.map { case Location(id, geom) => Feature(geom, id) } | ||
|
|
||
| // This is where the main analysis happens, in ErrorSummaryRDD.apply(), | ||
| // which eventually calls into GHGSummary via runPolygonalSummary(). | ||
| GHGRDD(tmp, GHGGrid.blockTileGrid, kwargs) | ||
| } | ||
|
|
||
| // For all rows that didn't get an error from the GHG analysis, do the | ||
| // transformation from GHGSummary to GHGData | ||
| ValidatedWorkflow(locationSummaries).mapValid { summaries => | ||
| summaries | ||
| .mapValues { | ||
| case summary: GHGSummary => | ||
| val data = summary.toGHGData() | ||
| data | ||
| } | ||
| } | ||
| } | ||
| .unify | ||
| .persist(StorageLevel.MEMORY_AND_DISK) | ||
| } | ||
|
|
||
| // If a location has empty GHGData results, then the geometry | ||
| // must not have intersected the centroid of any pixels, so report the location | ||
| // as NoIntersectionError. | ||
| partialResult.map { | ||
| case Valid(Location(fid, data)) if data.equals(GHGData.empty) => | ||
| Invalid(Location(fid, NoIntersectionError)) | ||
| case data => data | ||
| } | ||
| } catch { | ||
| case e: StackOverflowError => | ||
| e.printStackTrace() | ||
| throw e | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.