diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml
index 1d001cc5..3ce6f429 100644
--- a/.github/workflows/changelog.yml
+++ b/.github/workflows/changelog.yml
@@ -13,6 +13,6 @@ on:
jobs:
call-changelog-check-workflow:
- uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.10.0
+ uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.0
secrets:
USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml
index 8436b4ed..0b69efec 100644
--- a/.github/workflows/create-jira-issue.yml
+++ b/.github/workflows/create-jira-issue.yml
@@ -6,7 +6,7 @@ on:
jobs:
call-create-jira-issue-workflow:
- uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.10.0
+ uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.0
secrets:
JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml
index 48fc9e84..66ba502e 100644
--- a/.github/workflows/labeled-pr.yml
+++ b/.github/workflows/labeled-pr.yml
@@ -12,4 +12,4 @@ on:
jobs:
call-labeled-pr-check-workflow:
- uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.10.0
+ uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.0
diff --git a/.github/workflows/release-template-comment.yml b/.github/workflows/release-template-comment.yml
index 64197b00..cae89e6f 100644
--- a/.github/workflows/release-template-comment.yml
+++ b/.github/workflows/release-template-comment.yml
@@ -7,6 +7,6 @@ on:
jobs:
call-release-checklist-workflow:
- uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.10.0
+ uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.0
secrets:
USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 6f0d6ffc..c8fed248 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -7,7 +7,7 @@ on:
jobs:
call-release-workflow:
- uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.10.0
+ uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.0
with:
release_prefix: HyP3 autoRIFT
secrets:
diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml
index a142be7f..3b07f93b 100644
--- a/.github/workflows/static-analysis.yml
+++ b/.github/workflows/static-analysis.yml
@@ -4,10 +4,10 @@ on: push
jobs:
call-flake8-workflow:
- uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.10.0
+ uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.11.0
with:
local_package_names: hyp3_autorift
excludes: src/hyp3_autorift/vend
call-secrets-analysis-workflow:
- uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.10.0
+ uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.0
diff --git a/.github/workflows/tag-version.yml b/.github/workflows/tag-version.yml
index eeaf346f..4714c1f1 100644
--- a/.github/workflows/tag-version.yml
+++ b/.github/workflows/tag-version.yml
@@ -7,6 +7,6 @@ on:
jobs:
call-bump-version-workflow:
- uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.10.0
+ uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.0
secrets:
USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}
diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml
index 1e215924..cee1a1e2 100644
--- a/.github/workflows/test-and-build.yml
+++ b/.github/workflows/test-and-build.yml
@@ -12,18 +12,18 @@ on:
jobs:
call-pytest-workflow:
- uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.10.0
+ uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.11.0
with:
local_package_name: hyp3_autorift
python_versions: >-
["3.9"]
call-version-info-workflow:
- uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.10.0
+ uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.11.0
call-docker-ghcr-workflow:
needs: call-version-info-workflow
- uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.10.0
+ uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.11.0
with:
version_tag: ${{ needs.call-version-info-workflow.outputs.version_tag }}
secrets:
diff --git a/.trufflehog.txt b/.trufflehog.txt
deleted file mode 100644
index 4ed920df..00000000
--- a/.trufflehog.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-.*gitleaks.toml$
-.*hyp3_autorift/vend/workflow/.*
-.*hyp3_autorift/vend/README.md$
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a4c6b257..c8f67e6a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [0.15.0]
+### Added
+* `--publish-bucket` option has been added to the HyP3 entry point to additionally publish products an AWS bucket, such as the ITS_LIVE AWS Open Data bucket, `s3://its-live-data`.
+* `upload_file_to_s3_with_publish_access_keys` to perform S3 uploads using credentials from the `PUBLISH_ACCESS_KEY_ID` and `PUBLISH_SECRET_ACCESS_KEY` environment vairables.
## [0.14.1]
### Changed
diff --git a/src/hyp3_autorift/process.py b/src/hyp3_autorift/process.py
index 99ecaccb..de807959 100644
--- a/src/hyp3_autorift/process.py
+++ b/src/hyp3_autorift/process.py
@@ -27,7 +27,7 @@
from hyp3_autorift import geometry, image, io
from hyp3_autorift.crop import crop_netcdf_product
-from hyp3_autorift.utils import get_esa_credentials
+from hyp3_autorift.utils import get_esa_credentials, upload_file_to_s3_with_publish_access_keys
log = logging.getLogger(__name__)
@@ -48,6 +48,16 @@
DEFAULT_PARAMETER_FILE = '/vsicurl/http://its-live-data.s3.amazonaws.com/' \
'autorift_parameters/v001/autorift_landice_0120m.shp'
+PLATFORM_SHORTNAME_LONGNAME_MAPPING = {
+ 'S1': 'sentinel1',
+ 'S2': 'sentinel2',
+ 'L4': 'landsatOLI',
+ 'L5': 'landsatOLI',
+ 'L7': 'landsatOLI',
+ 'L8': 'landsatOLI',
+ 'L9': 'landsatOLI',
+}
+
def get_lc2_stac_json_key(scene_name: str) -> str:
platform = get_platform(scene_name)
@@ -319,6 +329,48 @@ def apply_landsat_filtering(reference_path: str, secondary_path: str) \
return reference_path, reference_zero_path, secondary_path, secondary_zero_path
+def get_lat_lon_from_ncfile(ncfile: Path) -> Tuple[float, float]:
+ with Dataset(ncfile) as ds:
+ var = ds.variables['img_pair_info']
+ return var.latitude, var.longitude
+
+
+def point_to_region(lat: float, lon: float) -> str:
+ """
+ Returns a string (for example, N78W124) of a region name based on
+ granule center point lat,lon
+ """
+ nw_hemisphere = 'N' if lat >= 0.0 else 'S'
+ ew_hemisphere = 'E' if lon >= 0.0 else 'W'
+
+ region_lat = int(10*np.trunc(np.abs(lat/10.0)))
+ if region_lat == 90: # if you are exactly at a pole, put in lat = 80 bin
+ region_lat = 80
+
+ region_lon = int(10*np.trunc(np.abs(lon/10.0)))
+
+ if region_lon >= 180: # if you are at the dateline, back off to the 170 bin
+ region_lon = 170
+
+ return f'{nw_hemisphere}{region_lat:02d}{ew_hemisphere}{region_lon:03d}'
+
+
+def get_opendata_prefix(file: Path):
+ # filenames have form GRANULE1_X_GRANULE2
+ scene = file.name.split('_X_')[0]
+
+ platform_shortname = get_platform(scene)
+ lat, lon = get_lat_lon_from_ncfile(file)
+ region = point_to_region(lat, lon)
+
+ return '/'.join([
+ 'velocity_image_pair',
+ PLATFORM_SHORTNAME_LONGNAME_MAPPING[platform_shortname],
+ 'v02',
+ region
+ ])
+
+
def process(
reference: str,
secondary: str,
@@ -520,6 +572,9 @@ def main():
)
parser.add_argument('--bucket', help='AWS bucket to upload product files to')
parser.add_argument('--bucket-prefix', default='', help='AWS prefix (location in bucket) to add to product files')
+ parser.add_argument('--publish-bucket', default='',
+ help='Additionally, publish products to this bucket. Necessary credentials must be provided '
+ 'via the `PUBLISH_ACCESS_KEY_ID` and `PUBLISH_SECRET_ACCESS_KEY` environment variables.')
parser.add_argument('--esa-username', default=None, help="Username for ESA's Copernicus Data Space Ecosystem")
parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem")
parser.add_argument('--parameter-file', default=DEFAULT_PARAMETER_FILE,
@@ -538,9 +593,15 @@ def main():
g1, g2 = sorted(args.granules, key=get_datetime)
product_file, browse_file = process(g1, g2, parameter_file=args.parameter_file, naming_scheme=args.naming_scheme)
+ thumbnail_file = create_thumbnail(browse_file)
if args.bucket:
upload_file_to_s3(product_file, args.bucket, args.bucket_prefix)
upload_file_to_s3(browse_file, args.bucket, args.bucket_prefix)
- thumbnail_file = create_thumbnail(browse_file)
upload_file_to_s3(thumbnail_file, args.bucket, args.bucket_prefix)
+
+ if args.publish_bucket:
+ prefix = get_opendata_prefix(product_file)
+ upload_file_to_s3_with_publish_access_keys(product_file, args.publish_bucket, prefix)
+ upload_file_to_s3_with_publish_access_keys(browse_file, args.publish_bucket, prefix)
+ upload_file_to_s3_with_publish_access_keys(thumbnail_file, args.publish_bucket, prefix)
diff --git a/src/hyp3_autorift/utils.py b/src/hyp3_autorift/utils.py
index 61845eb8..6bcb6244 100644
--- a/src/hyp3_autorift/utils.py
+++ b/src/hyp3_autorift/utils.py
@@ -1,9 +1,13 @@
+import logging
import netrc
import os
from pathlib import Path
from platform import system
from typing import Tuple
+import boto3
+from hyp3lib.aws import get_content_type, get_tag_set
+
ESA_HOST = 'dataspace.copernicus.eu'
@@ -28,3 +32,25 @@ def get_esa_credentials() -> Tuple[str, str]:
"Please provide Copernicus Data Space Ecosystem (CDSE) credentials via the "
"ESA_USERNAME and ESA_PASSWORD environment variables, or your netrc file."
)
+
+
+def upload_file_to_s3_with_publish_access_keys(path_to_file: Path, bucket: str, prefix: str = ''):
+ try:
+ access_key_id = os.environ['PUBLISH_ACCESS_KEY_ID']
+ access_key_secret = os.environ['PUBLISH_SECRET_ACCESS_KEY']
+ except KeyError:
+ raise ValueError(
+ 'Please provide S3 Bucket upload access key credentials via the '
+ 'PUBLISH_ACCESS_KEY_ID and PUBLISH_SECRET_ACCESS_KEY environment variables'
+ )
+
+ s3_client = boto3.client('s3', aws_access_key_id=access_key_id, aws_secret_access_key=access_key_secret)
+ key = str(Path(prefix) / path_to_file.name)
+ extra_args = {'ContentType': get_content_type(key)}
+
+ logging.info(f'Uploading s3://{bucket}/{key}')
+ s3_client.upload_file(str(path_to_file), bucket, key, extra_args)
+
+ tag_set = get_tag_set(path_to_file.name)
+
+ s3_client.put_object_tagging(Bucket=bucket, Key=key, Tagging=tag_set)
diff --git a/tests/data/LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc b/tests/data/LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc
new file mode 100644
index 00000000..487be3bc
Binary files /dev/null and b/tests/data/LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc differ
diff --git a/tests/test_process.py b/tests/test_process.py
index f5d6d5c6..d9040e33 100644
--- a/tests/test_process.py
+++ b/tests/test_process.py
@@ -1,5 +1,6 @@
import io
from datetime import datetime
+from pathlib import Path
from re import match
from unittest import mock
from unittest.mock import MagicMock, patch
@@ -415,3 +416,27 @@ def mock_apply_filter_function(scene, _):
process.apply_landsat_filtering('LT04', 'LE07')
assert process.apply_landsat_filtering('LT04', 'LT05') == ('LT04', None, 'LT05', None)
assert process.apply_landsat_filtering('LT04', 'LT04') == ('LT04', None, 'LT04', None)
+
+
+def test_point_to_prefix():
+ assert process.point_to_region(63.0, 128.0) == 'N60E120'
+ assert process.point_to_region(-63.0, 128.0) == 'S60E120'
+ assert process.point_to_region(63.0, -128.0) == 'N60W120'
+ assert process.point_to_region(-63.0, -128.0) == 'S60W120'
+ assert process.point_to_region(0.0, 128.0) == 'N00E120'
+ assert process.point_to_region(0.0, -128.0) == 'N00W120'
+ assert process.point_to_region(63.0, 0.0) == 'N60E000'
+ assert process.point_to_region(-63.0, 0.0) == 'S60E000'
+ assert process.point_to_region(0.0, 0.0) == 'N00E000'
+
+
+def test_get_lat_lon_from_ncfile():
+ file = Path('tests/data/'
+ 'LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc')
+ assert process.get_lat_lon_from_ncfile(file) == (-81.49, -128.28)
+
+
+def test_get_opendata_prefix():
+ file = Path('tests/data/'
+ 'LT05_L1GS_219121_19841206_20200918_02_T2_X_LT05_L1GS_226120_19850124_20200918_02_T2_G0120V02_P000.nc')
+ assert process.get_opendata_prefix(file) == 'velocity_image_pair/landsatOLI/v02/S80W120'
diff --git a/tests/test_utils.py b/tests/test_utils.py
index f3263d06..e640d411 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,6 +1,6 @@
import pytest
-from hyp3_autorift.utils import ESA_HOST, get_esa_credentials
+from hyp3_autorift.utils import ESA_HOST, get_esa_credentials, upload_file_to_s3_with_publish_access_keys
def test_get_esa_credentials_env(tmp_path, monkeypatch):
@@ -45,3 +45,19 @@ def test_get_esa_credentials_missing(tmp_path, monkeypatch):
msg = 'Please provide.*'
with pytest.raises(ValueError, match=msg):
get_esa_credentials()
+
+
+def test_upload_file_to_s3_credentials_missing(tmp_path, monkeypatch):
+ with monkeypatch.context() as m:
+ m.delenv('PUBLISH_ACCESS_KEY_ID', raising=False)
+ m.setenv('PUBLISH_SECRET_ACCESS_KEY', 'publish_access_key_secret')
+ msg = 'Please provide.*'
+ with pytest.raises(ValueError, match=msg):
+ upload_file_to_s3_with_publish_access_keys('file.zip', 'myBucket')
+
+ with monkeypatch.context() as m:
+ m.setenv('PUBLISH_ACCESS_KEY_ID', 'publish_access_key_id')
+ m.delenv('PUBLISH_SECRET_ACCESS_KEY', raising=False)
+ msg = 'Please provide.*'
+ with pytest.raises(ValueError, match=msg):
+ upload_file_to_s3_with_publish_access_keys('file.zip', 'myBucket')