Merge pull request #183 from hotosm/feature/hdx

Feature : HDX Exports

.github/workflows/Unit-Test.yml

@@ -31,10 +31,10 @@ jobs:
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 2
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v1
         with:
-          python-version: 3.8
+          python-version: 3.10.13
 
       - name: Insert sample db data
         run: |

.github/workflows/build.yml

@@ -33,10 +33,10 @@ jobs:
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 2
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v1
         with:
-          python-version: 3.8
+          python-version: 3.10.13
       - name: Install necessary dependecies for rawdata loading
         run: sudo apt-get update && sudo apt-get install osm2pgsql
       - name: check version

.github/workflows/code-check.yml

@@ -2,7 +2,7 @@ name: Code Check - Linting using flake8
 on:
   push:
     paths-ignore:
-      - 'infra/**'
+      - "infra/**"
     branches:
       - master
       - develop
@@ -16,10 +16,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.10.13
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/publish_mkdocs.yml

@@ -6,19 +6,19 @@ on:
       - master
     paths:
       # Only rebuild documentation when docs have changed
-      - 'docs/**'
-      - '.github/workflows/publish_mkdocs.yml'
+      - "docs/**"
+      - ".github/workflows/publish_mkdocs.yml"
 permissions:
   contents: write
 jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.10.13
           publish_branch: gh-pages
       - name: Install Dependencies
         run: |

API/api_worker.py

@@ -11,15 +11,15 @@
 import sozipfile.sozipfile as zipfile
 from celery import Celery
 
-from src.app import PolygonStats, RawData, S3FileTransfer
+from src.app import HDX, PolygonStats, RawData, S3FileTransfer
 from src.config import ALLOW_BIND_ZIP_FILTER
 from src.config import CELERY_BROKER_URL as celery_broker_uri
 from src.config import CELERY_RESULT_BACKEND as celery_backend
 from src.config import ENABLE_TILES
 from src.config import USE_S3_TO_UPLOAD as use_s3_to_upload
 from src.config import logger as logging
 from src.query_builder.builder import format_file_name_str
-from src.validation.models import RawDataOutputType
+from src.validation.models import DatasetConfig, RawDataOutputType
 
 celery = Celery("Raw Data API")
 celery.conf.broker_url = celery_broker_uri
@@ -105,8 +105,8 @@ def process_raw_data(self, params):
             readme_content += "Exported through Raw-data-api (https://github.com/hotosm/raw-data-api) using OpenStreetMap data.\n"
             readme_content += "Learn more about OpenStreetMap and its data usage policy : https://www.openstreetmap.org/about \n"
             if polygon_stats:
-                readme_content += f'{polygon_stats["summary"]["building"]}\n'
-                readme_content += f'{polygon_stats["summary"]["road"]}\n'
+                readme_content += f'{polygon_stats["summary"]["buildings"]}\n'
+                readme_content += f'{polygon_stats["summary"]["roads"]}\n'
                 readme_content += "Read about what this summary means: indicators: https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/indicators.md,metrics: https://github.com/hotosm/raw-data-api/tree/develop/docs/src/stats/metrics.md"
 
             zf.writestr("Readme.txt", readme_content)
@@ -186,6 +186,18 @@ def process_raw_data(self, params):
         raise ex
 
 
+@celery.task(bind=True, name="process_hdx_request")
+def process_hdx_request(self, params):
+    if not params.dataset:
+        params.dataset = DatasetConfig()
+    hdx_object = HDX(params)
+    try:
+        return hdx_object.process_hdx_tags()
+    except Exception as ex:
+        hdx_object.clean_resources()
+        raise ex
+
+
 def remove_file(path: str) -> None:
     """Used for removing temp file dir and its all content after zip file is delivered to user"""
     try: