Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add MSAVI index to backend #29

Open
wants to merge 27 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
8e9ef5e
feat: check that time range is valid for S2 harmonized collection
tomADC443 Nov 13, 2024
78d8aee
feat: add preprocessing step to remove unnecessary bands from images
tomADC443 Nov 13, 2024
3e35820
refactor: Aggregation and Filler DataFrame operations into two separa…
tomADC443 Nov 13, 2024
d06b41f
refactor: temporal utility functions (cleaner, more readable)
tomADC443 Nov 13, 2024
128b9c4
fix: different timezones (now always utc)
tomADC443 Nov 13, 2024
326e70a
feat: introduce static cache file
tomADC443 Nov 13, 2024
43f992c
refactor: delete emply file
wherop Nov 18, 2024
a2bc44f
refactor: delete emply file
wherop Nov 18, 2024
df6cc26
feat: refactor 'ndvi_router' to 'sat_index_router'; add MSAVI route t…
wherop Nov 19, 2024
084a689
Merge branch 'main' into feature/msavi-endpoint
wherop Nov 20, 2024
109c80e
refactor: rename image preprocessing file
wherop Nov 20, 2024
331fdea
refactor: rename gee/ndvi.py to gee/sat_index_info.py
wherop Nov 20, 2024
437f864
feat: add function to change index formula
wherop Nov 20, 2024
60295bb
feat: pass index_type from router to GEE
wherop Nov 20, 2024
05a751b
Merge branch 'main' into feature/msavi-endpoint
wherop Nov 21, 2024
b8828d4
refactor: add '__init__.py' to all python package directories
wherop Nov 21, 2024
b517f73
fix: update fastapi dependency reference to 'fastapi[standard]'
wherop Nov 21, 2024
d9c1ec3
fix: correct constants import error
wherop Nov 21, 2024
128650b
style: update route labels to be more descriptive
wherop Nov 21, 2024
d98b02e
feat: add index type to response meta; correct unit for MSAVI meta
wherop Nov 21, 2024
86e72fe
feat&docs: add comments and print statements to index service
wherop Nov 22, 2024
a5edee1
refactor: move NDVI cache to cache directory
wherop Nov 22, 2024
b9a52b1
feat: create MSAVI cache
wherop Nov 22, 2024
dcf318d
feat: add a rudamentary caching script (WIP)
wherop Nov 22, 2024
69767bc
feat: integrate MSAVI cache
wherop Nov 22, 2024
21b6288
fix(index service): handle requests where start or end date equals ca…
wherop Nov 25, 2024
82329e5
Merge branch 'main' into feature/msavi-endpoint
wherop Nov 26, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion backend/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@ earthengine-api
python-dotenv
pandas
plotly
fastapi
fastapi[standard]
uvicorn
544 changes: 544 additions & 0 deletions backend/src/cache/msavi_cache.py

Large diffs are not rendered by default.

File renamed without changes.
1 change: 1 addition & 0 deletions backend/src/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ class Unit(str, Enum):
MM = "mm"
PERCENT = "%"
M3 = "m³/m³"
DIMENSIONLESS = "Dimensionless"


class LocationPolygon(Enum):
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,35 +1,36 @@
from datetime import datetime, timezone

from fastapi import APIRouter, Query
from fastapi import Query
from fastapi.responses import JSONResponse
from src.constants import (
IndexType,
AggregationMethod,
LocationName,
TemporalResolution,
Unit,
)
from src.service import ndvi_service
from src.service import sat_index_service
from src.utils.temporal import get_optimistic_rounding
from src.validation.models import NDVIResponse
from src.validation.utils import (
validate_timestamp_in_range_of_S2_imagery,
validate_timestamp_start_date_before_end_date,
)

ndvi_router = APIRouter()


@ndvi_router.get("/ndvi", response_model=NDVIResponse)
async def get_temperature_data(
async def sat_index_controller(
sat_index_type: IndexType,
startDate: int = Query(...,

description="Start date as UNIX timestamp in seconds"),
endDate: int = Query(...,

description="End date as UNIX timestamp in seconds"),
location: LocationName = Query(..., description="Location name"),
temporalResolution: TemporalResolution = Query(
..., description="Temporal resolution"
),
aggregation: AggregationMethod = Query(...,

description="Aggregation method"),
):

Expand All @@ -42,7 +43,7 @@ async def get_temperature_data(
start_date_dt, end_date_dt, temporalResolution
)

data = ndvi_service(
data = sat_index_service(
location=location,
temporal_resolution=temporalResolution,
aggregation_method=aggregation,
Expand All @@ -62,4 +63,4 @@ async def get_temperature_data(
"data": data,
}

return JSONResponse(content=response)
return JSONResponse(content=response)
30 changes: 30 additions & 0 deletions backend/src/gee/caching_script.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
import json

content = [
{"timestamp": 1493251200, "value": 0.6265267304234295},
{"timestamp": 1494720000, "value": 0.68603163673333},
{"timestamp": 1494979200, "value": 0.755257128311451},
]
var_name = "msavi_daily_cache"
file_name = "../cache/temp_cache.py"
INDENT = " "

def combine_chache_with_update():
# update existing cache
return

def write_year(file, year, results):
file.write(f"{INDENT}# Year {year}\n")

def write_results_to_cache(results: list, var_name: str, file_name: str):
with open(file_name, "w") as file:
file.write(f"{var_name} = [\n")
for day in results:
file.write(f"{INDENT}{json.dumps(day)},\n")
file.write("]\n")
file.close()

return


# write_results_to_cache(content, var_name, file_name)
File renamed without changes.
47 changes: 33 additions & 14 deletions backend/src/gee/ndvi.py → backend/src/gee/sat_index_info.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,38 @@
import ee
from ..constants import IndexType

INDEX_FEATURE_LABEL = "indexing_value"
TIMESTAMP_FEATURE_LABEL = "start_of_day_timestamp"
INDEX_NULL_STRING = "NULL"


def calculate_mean_ndvi_GEE_SERVER(image: ee.Image, aoi: ee.Geometry.Polygon):
ndvi = image.normalizedDifference(["B8", "B4"]).rename("NDVI")
mean_ndvi = ndvi.reduceRegion(
def get_index_image_by_index_type(index_type: IndexType, image: ee.Image):
match index_type:
case IndexType.NDVI:
return image.normalizedDifference(["B8", "B4"]).rename("NDVI")
case IndexType.MSAVI:
return image.expression(
expression="((2 * NIR + 1) - ((2 * NIR + 1)**2 - 8 * (NIR - RED))**0.5) / 2",
opt_map={
"NIR": image.select("B4"),
"RED": image.select("B8"),
},
).rename("MSAVI")
case _:
return None


def calculate_mean_index_GEE_SERVER(image: ee.Image, aoi: ee.Geometry.Polygon, index_type: IndexType):
index_image = get_index_image_by_index_type(index_type, image)
mean_index = index_image.reduceRegion(
reducer=ee.Reducer.mean(), geometry=aoi, scale=10, maxPixels=1e8
).get("NDVI")
mean_ndvi = ee.Algorithms.If(
ee.Algorithms.IsEqual(mean_ndvi, None), INDEX_NULL_STRING, mean_ndvi
).get(index_type.value)

mean_index = ee.Algorithms.If(
ee.Algorithms.IsEqual(mean_index, None), INDEX_NULL_STRING, mean_index
)
return image.set(INDEX_FEATURE_LABEL, mean_ndvi)
return image.set(INDEX_FEATURE_LABEL, mean_index)



def calculate_start_of_day_timestamp_GEE_SERVER(image: ee.Image):
Expand All @@ -27,28 +46,28 @@ def calculate_start_of_day_timestamp_GEE_SERVER(image: ee.Image):
return image.set(TIMESTAMP_FEATURE_LABEL, start_of_day)


def get_ndvi_info(
image_collection: ee.ImageCollection, coordinates: ee.Geometry.Polygon
def get_sat_index_info(
image_collection: ee.ImageCollection, coordinates: ee.Geometry.Polygon, index_type: IndexType
):
aoi = ee.Geometry.Polygon(coordinates)

# Setting indexing values Server side
image_collection_with_ndvi = image_collection.map(
lambda img: calculate_mean_ndvi_GEE_SERVER(img, aoi)
image_collection_with_index = image_collection.map(
lambda img: calculate_mean_index_GEE_SERVER(img, aoi, index_type)
)

# Setting timestamps Server side
image_collection_with_timestamp_and_ndvi = image_collection_with_ndvi.map(
image_collection_with_timestamp_and_index = image_collection_with_index.map(
lambda img: calculate_start_of_day_timestamp_GEE_SERVER(img)
)

# Getting indexing values to the client side
index_value_list = image_collection_with_timestamp_and_ndvi.aggregate_array(
index_value_list = image_collection_with_timestamp_and_index.aggregate_array(
INDEX_FEATURE_LABEL
).getInfo()

# Getting timestamps to the client
timestamp_list = image_collection_with_timestamp_and_ndvi.aggregate_array(
timestamp_list = image_collection_with_timestamp_and_index.aggregate_array(
TIMESTAMP_FEATURE_LABEL
).getInfo()

Expand Down
5 changes: 3 additions & 2 deletions backend/src/main.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware

from src.routes.ndvi_router import ndvi_router
from src.routes.sat_index_router import sat_index_router

from .weather.router import router as weather_router

Expand All @@ -17,10 +17,11 @@
allow_headers=["*"],
)


@app.get("/")
def read_root():
return {"Hello": "World"}


app.include_router(weather_router, prefix="/weather", tags=["Weather Data"])
app.include_router(ndvi_router, prefix="/index", tags=["NDVI Data"])
app.include_router(sat_index_router, prefix="/index", tags=["Vegetation Indices"])
File renamed without changes.
126 changes: 126 additions & 0 deletions backend/src/routes/sat_index_router.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
from datetime import datetime, timezone

from fastapi import APIRouter, Query
from fastapi.responses import JSONResponse
from src.constants import (
AggregationMethod,
LocationName,
TemporalResolution,
Unit,
IndexType,
)
from src.service import sat_index_service
from src.utils.temporal import get_optimistic_rounding
from src.validation.models import NDVIResponse, MSAVIResponse
from src.validation.utils import (
validate_timestamp_in_range_of_S2_imagery,
validate_timestamp_start_date_before_end_date,
)

sat_index_router = APIRouter()


@sat_index_router.get("/ndvi", response_model=NDVIResponse)
async def get_ndvi_data(
startDate: int = Query(
...,
description="First date of requested date range in UNIX timestamp as seconds",
),
endDate: int = Query(
...,
description="Last date of requested date range in UNIX timestamp as seconds",
),
location: LocationName = Query(..., description="Name of the requested location"),
temporalResolution: TemporalResolution = Query(
...,
description="Time interval that a single data point should represent e.g. one month",
),
aggregation: AggregationMethod = Query(
...,
description="Method of aggregating available data into a single datapoint to represent the selected time interval e.g. mean average",
),
):
validate_timestamp_start_date_before_end_date(startDate, endDate)
validate_timestamp_in_range_of_S2_imagery(startDate, endDate)
start_date_dt = datetime.fromtimestamp(startDate, tz=timezone.utc)
end_date_dt = datetime.fromtimestamp(endDate, tz=timezone.utc)

rounded_start_date, rounded_end_date = get_optimistic_rounding(
start_date_dt, end_date_dt, temporalResolution
)

data = sat_index_service(
location=location,
temporal_resolution=temporalResolution,
aggregation_method=aggregation,
start_date=rounded_start_date,
end_date=rounded_end_date,
index_type=IndexType.NDVI,
)

response = {
"meta": {
"location": LocationName[location].value,
"startDate": int(rounded_start_date.timestamp()),
"endDate": int(rounded_end_date.timestamp()),
"temporalResolution": TemporalResolution[temporalResolution].value,
"aggregation": AggregationMethod[aggregation].value,
"unit": Unit.NORMALIZED_DIFFERENCE.value,
},
"data": data,
}

return JSONResponse(content=response)


@sat_index_router.get("/msavi", response_model=MSAVIResponse)
async def get_msavi_data(
startDate: int = Query(
...,
description="First date of requested date range in UNIX timestamp as seconds",
),
endDate: int = Query(
...,
description="Last date of requested date range in UNIX timestamp as seconds",
),
location: LocationName = Query(..., description="Name of the requested location"),
temporalResolution: TemporalResolution = Query(
...,
description="Time interval that a single data point should represent e.g. one month",
),
aggregation: AggregationMethod = Query(
...,
description="Method of aggregating available data into a single datapoint to represent the selected time interval e.g. mean average",
),
):
validate_timestamp_start_date_before_end_date(startDate, endDate)
validate_timestamp_in_range_of_S2_imagery(startDate, endDate)
start_date_dt = datetime.fromtimestamp(startDate, tz=timezone.utc)
end_date_dt = datetime.fromtimestamp(endDate, tz=timezone.utc)

rounded_start_date, rounded_end_date = get_optimistic_rounding(
start_date_dt, end_date_dt, temporalResolution
)

data = sat_index_service(
location=location,
temporal_resolution=temporalResolution,
aggregation_method=aggregation,
start_date=rounded_start_date,
end_date=rounded_end_date,
index_type=IndexType.MSAVI,
)

response = {
"meta": {
"location": LocationName[location].value,
"startDate": int(rounded_start_date.timestamp()),
"endDate": int(rounded_end_date.timestamp()),
"temporalResolution": TemporalResolution[temporalResolution].value,
"aggregation": AggregationMethod[aggregation].value,
"unit": Unit.DIMENSIONLESS.value,
},
"data": data,
}

return JSONResponse(content=response)
Loading