Skip to content

Commit

Permalink
Merge pull request #48 from podaac/release/0.9.0
Browse files Browse the repository at this point in the history 
Release 0.9.0
  • Loading branch information
@jamesfwood
jamesfwood authored Feb 22, 2024
2 parents ea52d88 + 4892225 commit db8b6ac
Show file tree
Hide file tree
Showing 17 changed files with 904 additions and 743 deletions.
126 changes: 12 additions & 114 deletions .github/workflows/build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@ jobs:
with:
poetry-version: 1.3.2

- name: Set up QEMU
uses: docker/setup-qemu-action@v2

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1

#########################################################################
# Versioning (featuring weird gradle output work-arounds)
#########################################################################
Expand Down Expand Up @@ -267,7 +273,7 @@ jobs:
github.ref == 'refs/heads/develop' ||
github.ref == 'refs/heads/main' ||
startsWith(github.ref, 'refs/heads/release')
run: |
run: |
if [ -n "$(git status --porcelain)" ]; then
echo "changes=true" >> $GITHUB_ENV
echo "::set-output name=changes::true"
Expand Down Expand Up @@ -459,72 +465,16 @@ jobs:
uses: docker/build-push-action@v3
with:
context: .
file: ./docker/lambdaDockerfile
file: ./docker/lambdaDockerfileArm
push: true
pull: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/arm/v7
build-args: |
SOURCE=${{ env.pyproject_name }}==${{ env.the_version }}
## Build and publish to Service ECR
- name: Upload Docker image to Service ECR
if: |
github.ref == 'refs/heads/develop' ||
github.ref == 'refs/heads/main' ||
startsWith(github.ref, 'refs/heads/release')
uses: vitr/actions-build-and-upload-to-ecs@master
with:
access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
account_id: ${{ secrets[format('AWS_ACCOUNT_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
repo: podaac/tig
region: us-west-2
tags: ${{ env.the_version }}
create_repo: true
dockerfile: ./docker/lambdaDockerfile
extra_build_args: --build-arg SOURCE=${{ env.pyproject_name }}==${{ env.the_version }}


## Build and publish to Cumulus ECR
- name: Upload Docker image to Cumulus ECR
if: |
github.ref == 'refs/heads/develop' ||
github.ref == 'refs/heads/main' ||
startsWith(github.ref, 'refs/heads/release')
uses: vitr/actions-build-and-upload-to-ecs@master
with:
access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}
secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}
account_id: ${{ secrets[format('AWS_ACCOUNT_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}
repo: podaac/tig
region: us-west-2
tags: ${{ env.the_version }}
create_repo: true
dockerfile: ./docker/lambdaDockerfile
extra_build_args: --build-arg SOURCE=${{ env.pyproject_name }}==${{ env.the_version }}


## Build and publish to SWOT Cumulus ECR
- name: Upload Docker image to Cumulus ECR
if: |
github.ref == 'refs/heads/develop' ||
github.ref == 'refs/heads/main' ||
startsWith(github.ref, 'refs/heads/release')
uses: vitr/actions-build-and-upload-to-ecs@master
with:
access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }}
secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }}
account_id: ${{ secrets[format('AWS_ACCOUNT_ID_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }}
repo: podaac/tig
region: us-west-2
tags: ${{ env.the_version }}
create_repo: true
dockerfile: ./docker/lambdaDockerfile
extra_build_args: --build-arg SOURCE=${{ env.pyproject_name }}==${{ env.the_version }}


## Local tig docker builds

- name: Get Local Tig Build
Expand All @@ -544,68 +494,16 @@ jobs:
uses: docker/build-push-action@v3
with:
context: .
file: ./docker/lambdaDockerfile
file: ./docker/lambdaDockerfileArm
push: true
pull: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/arm/v7
build-args: |
DIST_PATH="dist/"
SOURCE=${{ env.local_tig }}
## Build and publish to Service ECR
- name: Upload Local TIG Docker image to Service ECR
if: |
github.event.head_commit.message == '/deploy sit' ||
github.event.head_commit.message == '/deploy uat'
uses: vitr/actions-build-and-upload-to-ecs@master
with:
access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
account_id: ${{ secrets[format('AWS_ACCOUNT_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
repo: podaac/tig
region: us-west-2
tags: ${{ steps.meta.outputs.version }}
create_repo: true
dockerfile: ./docker/lambdaDockerfile
extra_build_args: --build-arg DIST_PATH="dist/" --build-arg SOURCE=${{ env.local_tig }}

## Build and publish to Cumulus ECR
- name: Upload Local TIG Docker image to Cumulus ECR
if: |
github.event.head_commit.message == '/deploy sit' ||
github.event.head_commit.message == '/deploy uat' ||
github.event.head_commit.message == '/deploy sandbox'
uses: vitr/actions-build-and-upload-to-ecs@master
with:
access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}
secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}
account_id: ${{ secrets[format('AWS_ACCOUNT_ID_CUMULUS_{0}', env.TARGET_ENV_UPPERCASE)] }}
repo: podaac/tig
region: us-west-2
tags: ${{ steps.meta.outputs.version }}
create_repo: true
dockerfile: ./docker/lambdaDockerfile
extra_build_args: --build-arg DIST_PATH="dist/" --build-arg SOURCE=${{ env.local_tig }}

## Build and publish to SWOT Cumulus ECR
- name: Upload Local TIG Docker image to Cumulus ECR
if: |
github.event.head_commit.message == '/deploy sit' ||
github.event.head_commit.message == '/deploy uat' ||
github.event.head_commit.message == '/deploy sandbox'
uses: vitr/actions-build-and-upload-to-ecs@master
with:
access_key_id: ${{ secrets[format('AWS_ACCESS_KEY_ID_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }}
secret_access_key: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }}
account_id: ${{ secrets[format('AWS_ACCOUNT_ID_CUMULUS_SWOT_{0}', env.TARGET_ENV_UPPERCASE)] }}
repo: podaac/tig
region: us-west-2
tags: ${{ steps.meta.outputs.version }}
create_repo: true
dockerfile: ./docker/lambdaDockerfile
extra_build_args: --build-arg DIST_PATH="dist/" --build-arg SOURCE=${{ env.local_tig }}

# #########################################################################
# # Build and Publish Documentation
# #########################################################################
Expand Down Expand Up @@ -650,7 +548,7 @@ jobs:
ls -al bin/
which python3
python3 --version
python3 override.py https://github.com/podaac/tig/releases/download/${{ env.the_version }}/tig-terraform-${{ env.the_version }}.zip ${{ secrets[format('AWS_ACCOUNT_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}.dkr.ecr.us-west-2.amazonaws.com/podaac/tig:${{ steps.meta.outputs.version }}
python3 override.py https://github.com/podaac/tig/releases/download/${{ env.the_version }}/tig-terraform-${{ env.the_version }}.zip "ghcr.io/podaac/tig:${{ env.DOCKER_METADATA_OUTPUT_VERSION }}"
ls -al
echo "Show override contents"
cat override.tf.json
Expand Down
20 changes: 20 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Security


## [0.9.0]

### Added
- ** Add Arm Architecture **
- update tig to be able to run with arm architecture
- ** Add image uploading **
- update tig to upload tig image to ecr
- ** Add palette support to generating configuration **
- [issue/44] (https://github.com/podaac/hitide/issues/44): Add support to palette, ppd, fill_missing in csv in generating configuration
- ** Fix swot 2.0 alignment issue **
- Added way to use global grid for region if it is in hitide configurations
### Changed
### Deprecated
### Removed
### Fixed
- ** Add function to fill in missing pixels for certain collections **
- [issue/26](https://github.com/podaac/tig/issues/26): Fix images with missing pixels for ASCAT collections
### Security


## [0.8.0]

### Added
Expand Down
8 changes: 8 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,14 @@ latitude: latitude variable include the group if they're in a group defaults to
time: time variable include the group if they're in a group defaults to time
footprint_strategy: strategy to generate footprint will default to None options should be ["periodic", "linestring", "polar", "swot_linestring", "polarsides", "smap"]

### CSV Columns

variable: name of variable
min: min value for variable
max: max value for variable
palette (optional): the palette to be used for the variable
fill_missing (optional): if the generated images have missing pixel in images most likely resolution is to big, either lower resolution or we can fill in the pixels with surrounding pixel
ppd (optional): resolution of the variable, must be an integer


## How to load and use tig module
Expand Down
25 changes: 25 additions & 0 deletions docker/lambdaDockerfileArm
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#ARG FUNCTION_DIR="/function"

FROM public.ecr.aws/lambda/python:3.10-arm64

# Include global arg in this stage of the build
ARG SOURCE

RUN yum -q -y install gcc

# Create function directory
# RUN mkdir -p ${FUNCTION_DIR}
# WORKDIR ${FUNCTION_DIR}

# Install tig from artifactory
COPY $DIST_PATH $DIST_PATH

# install tig into working directory so we can call lambda
RUN pip3 install awslambdaric --target $LAMBDA_TASK_ROOT

RUN pip3 install --no-cache-dir --force --index-url https://pypi.org/simple/ --extra-index-url https://test.pypi.org/simple/ --target "${LAMBDA_TASK_ROOT}" $SOURCE

RUN rm -rf $DIST_PATH

ENTRYPOINT []
CMD ["podaac.lambda_handler.lambda_handler.handler"]
6 changes: 3 additions & 3 deletions example_vars.csv
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
variable,min,max
data_01/ku/ssha,-0.2,0.2
data_01/ku/swh_ocean,0,30
variable,min,max,palette,fill_missing,ppd
data_01/ku/ssha,-0.2,0.2,paletteMedspirationIndexed,TRUE,20
data_01/ku/swh_ocean,0,30,palette_AQUARIUS,FALSE,15
38 changes: 28 additions & 10 deletions podaac/tig/generate_hitide_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,9 @@ def generate_hitide_config(granule, dataset_id, include_image_variables, longitu
's2': '0:*,*:*'
}

vars_min_max = {}
vars_data = {}
if include_image_variables:
vars_min_max = read_min_max_csv(include_image_variables)
vars_data = read_min_max_csv(include_image_variables)

with nc.Dataset(granule, 'r') as dataset: # pylint: disable=no-member

Expand All @@ -115,28 +115,46 @@ def generate_hitide_config(granule, dataset_id, include_image_variables, longitu

try:
for data_var in data_var_names:
if vars_min_max and data_var in vars_min_max:
if vars_data and data_var in vars_data:
variable = dataset[data_var]

units = variable.units if 'units' in variable.ncattrs() else ''
long_name = variable.long_name if 'long_name' in variable.ncattrs() else ''

if data_var in vars_min_max:
min_max = vars_min_max[data_var]
min_val = float(min_max['min'])
max_val = float(min_max['max'])
palette = 'paletteMedspirationIndexed'
fill_missing = False
ppd = 16

if data_var in vars_data:
min_val = float(vars_data[data_var]['min'])
max_val = float(vars_data[data_var]['max'])
palette = vars_data[data_var].get('palette')
fill_missing = vars_data[data_var].get('fill_missing', False)
ppd = vars_data[data_var].get('ppd', 16)
else:
min_val = variable.valid_min if 'valid_min' in variable.ncattrs() else ''
max_val = variable.valid_max if 'valid_max' in variable.ncattrs() else ''

dataset_config['imgVariables'].append({
if not palette:
palette = 'paletteMedspirationIndexed'

dataset_dict = {
'id': data_var,
'title': long_name,
'units': units,
'min': min_val,
'max': max_val,
'palette': 'paletteMedspirationIndexed'
})
'palette': palette
}

if fill_missing:
fill_missing = fill_missing.lower().strip()
dataset_dict['fill_missing'] = fill_missing == "true"

if ppd != 16 and ppd.isdigit():
dataset_dict['ppd'] = int(ppd)

dataset_config['imgVariables'].append(dataset_dict)

except Exception as ex: # pylint: disable=broad-exception-caught
print(f"Error: Failed on variable {data_var}, exception: " + str(ex))
Expand Down
Loading

0 comments on commit db8b6ac

Please sign in to comment.