Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Release v0.7.1 #172

Merged
merged 7 commits into from
Dec 17, 2024
Merged

Release v0.7.1 #172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
219ac15
switch to ruff from flake8
AndrewPlayer3 Dec 16, 2024
dfd0271
ruff config
AndrewPlayer3 Dec 17, 2024
e2d27ac
ruff config to pyproject
AndrewPlayer3 Dec 17, 2024
0f36ec2
Merge branch 'add-ruff' of https://github.com/ASFHyP3/gis-services in…
AndrewPlayer3 Dec 17, 2024
cd06a61
update ruff src
jtherrmann Dec 17, 2024
68aec7a
delete ruff.toml
jtherrmann Dec 17, 2024
8ece4e7
Merge pull request #171 from ASFHyP3/add-ruff
AndrewPlayer3 Dec 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .github/workflows/static-analysis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ name: Static analysis
on: push

jobs:
call-flake8-workflow:
uses: ASFHyP3/actions/.github/workflows/[email protected]
with:
local_package_names: None

call-secrets-analysis-workflow:
uses: ASFHyP3/actions/.github/workflows/[email protected]
# Docs: https://github.com/ASFHyP3/actions
uses: ASFHyP3/actions/.github/workflows/[email protected]

call-ruff-workflow:
# Docs: https://github.com/ASFHyP3/actions
uses: ASFHyP3/actions/.github/workflows/[email protected]
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [0.7.1]
### Changed
- The [`static-analysis`](.github/workflows/static-analysis.yml) Github Actions workflow now uses `ruff` rather than `flake8` for linting.

## [0.7.0]
### Removed
- Deleted files related to PDC image services.
Expand Down
20 changes: 9 additions & 11 deletions image_services/egis/create_configs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from jinja2 import Environment, PackageLoader, StrictUndefined, select_autoescape


SEASONS = {
'JJA': {
'Season': 'summer',
Expand All @@ -22,21 +23,18 @@
'Season': 'spring',
'SeasonAbbrev': 'Mar/Apr/May',
'SeasonFull': 'March/April/May',
}
},
}


def make_configuration(data_type, polarization, season):
config = {
"project_name": "GSSICB",
"s3_prefix": "tiles/",
"s3_suffix": f"_{SEASONS[season]['Season']}_{polarization.lower()}_{data_type}.tif",
"dataset_name": f"{data_type}_{polarization.upper()}_{season}",
"raster_function_templates": [
"ScaledCoherence.rft.xml",
"UnscaledCoherence.rft.xml"
],
"default_raster_function_template": "UnscaledCoherence.rft.xml"
'project_name': 'GSSICB',
's3_prefix': 'tiles/',
's3_suffix': f"_{SEASONS[season]['Season']}_{polarization.lower()}_{data_type}.tif",
'dataset_name': f'{data_type}_{polarization.upper()}_{season}',
'raster_function_templates': ['ScaledCoherence.rft.xml', 'UnscaledCoherence.rft.xml'],
'default_raster_function_template': 'UnscaledCoherence.rft.xml',
}
return config

Expand All @@ -48,7 +46,7 @@ def make_metadata_fields(data_type, polarization, season):
'polarization': polarization,
'months_abbreviated': SEASONS[season]['SeasonAbbrev'],
'season': SEASONS[season]['Season'],
'months_full': SEASONS[season]['SeasonFull']
'months_full': SEASONS[season]['SeasonFull'],
}
return metadata

Expand Down
58 changes: 43 additions & 15 deletions image_services/egis/make_egis_services.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import boto3
from osgeo import gdal, osr


gdal.UseExceptions()
gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', 'EMPTY_DIR')

Expand Down Expand Up @@ -39,7 +40,7 @@
'Season': 'March/April/May',
'StartDate': '03/01/2020',
'EndDate': '05/31/2020',
}
},
}


Expand Down Expand Up @@ -69,7 +70,7 @@ def get_projection(srs_wkt: str) -> str:


def remove_prefix(raster_path, prefix):
return raster_path[len(prefix):]
return raster_path[len(prefix) :]


def get_raster_metadata(raster_path: str) -> dict:
Expand Down Expand Up @@ -135,9 +136,24 @@ def calculate_overview_fields(mosaic_dataset, local_path):
# This function calculates custom attribute values for the overview record
print('Calculating field values for overview record')
ds = os.path.join(local_path, mosaic_dataset)
ds_cursor = arcpy.da.UpdateCursor(ds, ['Tag', 'MinPS', 'Category', 'StartDate', 'EndDate', 'GroupName',
'Name', 'ProductType', 'Season', 'Polarization', 'Tile', 'DownloadURL',
'URLDisplay'])
ds_cursor = arcpy.da.UpdateCursor(
ds,
[
'Tag',
'MinPS',
'Category',
'StartDate',
'EndDate',
'GroupName',
'Name',
'ProductType',
'Season',
'Polarization',
'Tile',
'DownloadURL',
'URLDisplay',
],
)
if ds_cursor is not None:
print('Updating Overview Field Values')
for row in ds_cursor:
Expand Down Expand Up @@ -187,8 +203,9 @@ def main():

csv_file = os.path.join(args.working_directory, f'{config["project_name"]}_{config["dataset_name"]}.csv')

raster_function_template = ''.join([f'{template_directory / template};'
for template in config['raster_function_templates']])
raster_function_template = ''.join(
[f'{template_directory / template};' for template in config['raster_function_templates']]
)
if config['default_raster_function_template'] != 'None':
default_raster_function_template = str(template_directory / config['default_raster_function_template'])
else:
Expand All @@ -214,11 +231,13 @@ def main():
)

logging.info('Creating mosaic dataset')
mosaic_dataset = str(arcpy.management.CreateMosaicDataset(
in_workspace=geodatabase,
in_mosaicdataset_name=config['dataset_name'],
coordinate_system=3857,
))
mosaic_dataset = str(
arcpy.management.CreateMosaicDataset(
in_workspace=geodatabase,
in_mosaicdataset_name=config['dataset_name'],
coordinate_system=3857,
)
)

logging.info(f'Adding source rasters to {mosaic_dataset}')
arcpy.management.AddRastersToMosaicDataset(
Expand Down Expand Up @@ -274,8 +293,8 @@ def main():
cell_size=3,
metadata_level='BASIC',
transmission_fields='Name;StartDate;EndDate;MinPS;MaxPS;LowPS;HighPS;Date;ZOrder;Dataset_ID;CenterX;'
'CenterY;Tag;GroupName;StartDate;EndDate;ProductType;Season;Polarization;Tile;'
'DownloadURL;URLDisplay',
'CenterY;Tag;GroupName;StartDate;EndDate;ProductType;Season;Polarization;Tile;'
'DownloadURL;URLDisplay',
use_time='DISABLED',
start_time_field='StartDate',
end_time_field='EndDate',
Expand Down Expand Up @@ -314,7 +333,16 @@ def main():

os.environ['AWS_PROFILE'] = 'hyp3'
logging.info(f'Moving CRF to {s3_overview}')
subprocess.run(['aws', 's3', 'cp', local_overview, s3_overview.replace('/vsis3/', 's3://'), '--recursive'])
subprocess.run(
[
'aws',
's3',
'cp',
local_overview,
s3_overview.replace('/vsis3/', 's3://'),
'--recursive',
]
)

logging.info('Adding overview to mosaic dataset')
arcpy.management.AddRastersToMosaicDataset(
Expand Down
5 changes: 3 additions & 2 deletions image_services/egis/metadata/print_service_metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

from jinja2 import Environment, PackageLoader, StrictUndefined, select_autoescape


SEASONS = {
'JJA': {
'Season': 'summer',
Expand All @@ -23,7 +24,7 @@
'Season': 'spring',
'SeasonAbbrev': 'Mar/Apr/May',
'SeasonFull': 'March/April/May',
}
},
}


Expand Down Expand Up @@ -62,7 +63,7 @@ def main():
'polarization': polarization,
'months_abbreviated': SEASONS[season]['SeasonAbbrev'],
'season': SEASONS[season]['Season'],
'months_full': SEASONS[season]['SeasonFull']
'months_full': SEASONS[season]['SeasonFull'],
}

output_text = render_template(args.template, fields)
Expand Down
45 changes: 27 additions & 18 deletions image_services/glo_30_hand/make_hand_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
'Rasters from the glo-30-hand collection will be selected using this '
'filter before they are added to the mosaic dataset. The syntax is as '
'expected by the `filter` option of arcpy.management.AddRastersToMosaicDataset.'
)
),
)
args = parser.parse_args()

Expand Down Expand Up @@ -79,9 +79,18 @@
['Tile', '!Name!.split("_")[4] + !Name!.split("_")[6]'],
['Tag', '"GLO30_HAND"'],
['Dataset_ID', '"Global_30m_HAND"'],
['ProductName', '"GLO30_HAND_"+ !Name!.split("_")[4] + !Name!.split("_")[6]'],
['URLDisplay', '"GLO30_HAND_"+ !Name!.split("_")[4] + !Name!.split("_")[6]'],
['DownloadURL', '"https://glo-30-hand.s3.amazonaws.com/v1/2021/" + !Name! + ".tif"'],
[
'ProductName',
'"GLO30_HAND_"+ !Name!.split("_")[4] + !Name!.split("_")[6]',
],
[
'URLDisplay',
'"GLO30_HAND_"+ !Name!.split("_")[4] + !Name!.split("_")[6]',
],
[
'DownloadURL',
'"https://glo-30-hand.s3.amazonaws.com/v1/2021/" + !Name! + ".tif"',
],
['MaxPS', '610'],
],
)
Expand Down Expand Up @@ -132,7 +141,7 @@
cell_size=3,
metadata_level='BASIC',
transmission_fields='Name;MinPS;MaxPS;LowPS;HighPS;ZOrder;Dataset_ID;CenterX;CenterY;Tag;Tile;ProductName;'
'DownloadURL;URLDisplay',
'DownloadURL;URLDisplay',
use_time='DISABLED',
max_num_of_download_items=50,
max_num_of_records_returned=2000,
Expand Down Expand Up @@ -160,7 +169,7 @@
)

logging.info('aws s3 cp')
subprocess.run(['aws', 's3', 'cp', local_overview, s3_overview.replace("/vsis3/", "s3://"), '--recursive'])
subprocess.run(['aws', 's3', 'cp', local_overview, s3_overview.replace('/vsis3/', 's3://'), '--recursive'])

logging.info('AddRastersToMosaicDataset')
arcpy.management.AddRastersToMosaicDataset(
Expand Down Expand Up @@ -195,18 +204,18 @@
raster_or_mosaic_layer=mosaic_dataset,
out_sddraft=service_definition_draft.name,
service_name=dataset_name,
summary="Height Above Nearest Drainage (HAND) is a terrain model that normalizes topography to the "
"relative heights along the drainage network and is used to describe the relative soil "
"gravitational potentials or the local drainage potentials. Each pixel value represents the "
"vertical distance to the nearest drainage. The HAND data provides near-worldwide land coverage "
"at 30 meters and was produced from the 2021 release of the Copernicus GLO-30 Public DEM as "
"distributed in the Registry of Open Data on AWS (https://registry.opendata.aws/copernicus-dem/) "
"using the the ASF Tools Python Package ("
"https://hyp3-docs.asf.alaska.edu/tools/asf_tools_api/#asf_tools.hand.calculate) and the PySheds "
"Python library (https://github.com/mdbartos/pysheds). The HAND data are provided as a tiled set "
"of Cloud Optimized GeoTIFFs (COGs) with 30-meter (1 arcsecond) pixel spacing. The COGs are "
"organized into the same 1 degree by 1 degree grid tiles as the GLO-30 DEM, and individual tiles "
"are pixel-aligned to the corresponding COG DEM tile.",
summary='Height Above Nearest Drainage (HAND) is a terrain model that normalizes topography to the '
'relative heights along the drainage network and is used to describe the relative soil '
'gravitational potentials or the local drainage potentials. Each pixel value represents the '
'vertical distance to the nearest drainage. The HAND data provides near-worldwide land coverage '
'at 30 meters and was produced from the 2021 release of the Copernicus GLO-30 Public DEM as '
'distributed in the Registry of Open Data on AWS (https://registry.opendata.aws/copernicus-dem/) '
'using the the ASF Tools Python Package ('
'https://hyp3-docs.asf.alaska.edu/tools/asf_tools_api/#asf_tools.hand.calculate) and the PySheds '
'Python library (https://github.com/mdbartos/pysheds). The HAND data are provided as a tiled set '
'of Cloud Optimized GeoTIFFs (COGs) with 30-meter (1 arcsecond) pixel spacing. The COGs are '
'organized into the same 1 degree by 1 degree grid tiles as the GLO-30 DEM, and individual tiles '
'are pixel-aligned to the corresponding COG DEM tile.',
)

logging.info('StageService')
Expand Down
Loading
Loading