Merge branch 'develop' into gcp

.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    labels:
+      - "bumpless"

.github/workflows/bump-version.yml

@@ -0,0 +1,16 @@
+name: Tag New Version
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  call-bump-version-workflow:
+    uses: ASFHyP3/actions/.github/workflows/[email protected]
+    with:
+      user: forrest-bot
+      email: [email protected]
+    secrets:
+      USER_TOKEN: ${{ secrets.FORREST_BOT_PAK }}
+

.github/workflows/changelog-check.yml

@@ -0,0 +1,18 @@
+name: Changelog updated?
+
+on:
+  pull_request:
+    types:
+      - opened
+      - labeled
+      - unlabeled
+      - synchronize
+    branches:
+      - main
+      - develop
+
+jobs:
+  call-changelog-check-workflow:
+    uses: ASFHyP3/actions/.github/workflows/[email protected]
+    secrets:
+      USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/labeled-pr-check.yml

@@ -0,0 +1,15 @@
+name: Is PR labeled?
+
+on:
+  pull_request:
+    types:
+      - opened
+      - labeled
+      - unlabeled
+      - synchronize
+    branches:
+      - main
+
+jobs:
+  call-labeled-pr-check-workflow:
+    uses: ASFHyP3/actions/.github/workflows/[email protected]

.github/workflows/release-checklist-comment.yml

@@ -0,0 +1,16 @@
+name: Checklist comment
+
+on:
+  pull_request:
+    types:
+      - opened
+    branches:
+      - main
+
+jobs:
+  call-release-workflow:
+    uses: ASFHyP3/actions/.github/workflows/[email protected]
+    permissions:
+      pull-requests: write
+    secrets:
+      USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -0,0 +1,17 @@
+name: Create Release
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  call-release-workflow:
+    uses: ASFHyP3/actions/.github/workflows/[email protected]
+    with:
+      release_prefix: INDEX_SAFE
+      release_branch: main      # Optional; default shown
+      develop_branch: develop   # Optional; default shown
+      sync_pr_label: forrest-bot  # Optional; default shown
+    secrets:
+      USER_TOKEN: ${{ secrets.FORREST_BOT_PAK }}

.github/workflows/static-analysis.yml

@@ -0,0 +1,32 @@
+name: Static analysis
+
+on: [pull_request]
+
+jobs:
+  call-secrets-analysis-workflow:
+    # Docs: https://github.com/ASFHyP3/actions
+    uses: ASFHyP3/actions/.github/workflows/[email protected]
+
+  check-with-black:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: psf/black@stable
+        with:
+          options: "--check --diff --color"
+          src: "."
+
+  check-with-ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install ruff
+      - name: Run Ruff
+        run: ruff check --format=github . src/index_safe/*.py

CHANGELOG.md

@@ -0,0 +1,19 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
+and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+
+## [0.0.1]
+
+### Added
+* Initial version of project (with no changes), to merge a non-zero version into main
+
+## [0.0.0]
+
+### Added
+* Initial version of project
+

pyproject.toml

@@ -46,6 +46,11 @@ Homepage = "https://github.com/forrestfwilliams/indexed_safe"
 include-package-data = true
 zip-safe = false
 
+[tool.black]
+line-length = 120
+skip-string-normalization = true
+include = '\.pyx?$'
+
 [tool.ruff]
 line-length = 120
 

src/index_safe/benchmark.py

@@ -25,7 +25,7 @@ def download_requests(url, offset, length):
 
 
 def download_fsspec(url, offset, length):
-    base_fs = fsspec.filesystem('https', block_size = 5 * (2**20))
+    base_fs = fsspec.filesystem('https', block_size=5 * (2**20))
     with base_fs.open(url, 'rb') as f:
         f.seek(offset)
         body = f.read(length)

src/index_safe/burst.xsl

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+  <xsl:template match="/">
+    <burst>
+      <manifest>
+        <xsl:copy-of select="document(files/@manifest)/*"/>
+      </manifest>
+      <metadata>
+        <xsl:for-each select="/files/file">
+          <xsl:variable name="root" select="name(document(@name)/*)"/>
+            <xsl:element name="{$root}">
+              <xsl:attribute name="source_filename">
+                <xsl:value-of select="@label"/>
+              </xsl:attribute>
+              <xsl:copy-of select="document(@name)/*/adsHeader/swath"/>
+              <xsl:copy-of select="document(@name)/*/adsHeader/polarisation"/>
+              <content>
+                <xsl:copy-of select="document(@name)/*/*"/>
+              </content>
+            </xsl:element>
+        </xsl:for-each>
+      </metadata>
+    </burst>
+  </xsl:template>
+</xsl:stylesheet>

src/index_safe/create_index.py

@@ -232,7 +232,7 @@ def create_index(zipped_safe_path: str, zinfo: zipfile.ZipInfo, output_json: boo
     return bursts
 
 
-def save_xml_metadata_as_json(entries: utils.XmlMetadata, out_name: str) -> str:
+def save_xml_metadata_as_json(entries: Iterable[utils.XmlMetadata], out_name: str) -> str:
     """Save a list of XmlMetadata objects as a json.
 
     Args:
@@ -253,6 +253,28 @@ def save_xml_metadata_as_json(entries: utils.XmlMetadata, out_name: str) -> str:
     return out_name
 
 
+def get_indexes(zipped_safe_path: Path):
+    """Get indexes for XML and bursts in zipped SAFE.
+
+    Args:
+        zipped_safe_path: Path to zipped SAFE
+
+    Returns: tuple of lists of XmlMetadata and BurstMetadata objects
+
+    """
+    with zipfile.ZipFile(zipped_safe_path) as f:
+        tiffs = [x for x in f.infolist() if 'tiff' in Path(x.filename).name]
+        xmls = [x for x in f.infolist() if 'xml' in Path(x.filename).name]
+        xmls += [x for x in f.infolist() if 'manifest.safe' == Path(x.filename).name]
+
+    print('Reading XMLs...')
+    xml_metadatas = [create_xml_metadata(zipped_safe_path, x) for x in tqdm(xmls)]
+
+    print('Reading Bursts...')
+    burst_metadatas = dict(ChainMap(*[create_index(zipped_safe_path, x) for x in tqdm(tiffs)]))
+    return xml_metadatas, burst_metadatas
+
+
 def index_safe(slc_name: str, edl_token: str = None, working_dir='.', output_json: bool = True, keep: bool = True):
     """Create the index and other metadata needed to directly download
     and correctly format burst tiffs/metadata Sentinel-1 SAFE zip. Save
@@ -277,17 +299,9 @@ def index_safe(slc_name: str, edl_token: str = None, working_dir='.', output_jso
     else:
         print('SLC exists locally, skipping download')
 
-    with zipfile.ZipFile(zipped_safe_path) as f:
-        tiffs = [x for x in f.infolist() if 'tiff' in Path(x.filename).name]
-        xmls = [x for x in f.infolist() if 'xml' in Path(x.filename).name]
-        xmls += [x for x in f.infolist() if 'manifest.safe' == Path(x.filename).name]
-
-    print('Reading XMLs...')
-    xml_metadatas = [create_xml_metadata(zipped_safe_path, x) for x in tqdm(xmls)]
-    save_xml_metadata_as_json(xml_metadatas, absolute_dir / f'{slc_name}_metadata.json')
+    xml_metadatas, burst_metadatas = get_indexes(zipped_safe_path)
 
-    print('Reading Bursts...')
-    burst_metadatas = dict(ChainMap(*[create_index(zipped_safe_path, x) for x in tqdm(tiffs)]))
+    save_xml_metadata_as_json(xml_metadatas, str(absolute_dir / f'{slc_name}_metadata.json'))
 
     if output_json:
         for key, value in burst_metadatas.items():