From ac0e6f0d47e9983edfd0aef258f9c42e043703d0 Mon Sep 17 00:00:00 2001 From: Samapriya Roy Date: Sat, 4 Feb 2023 12:52:55 -0600 Subject: [PATCH] updated v0.6.2 - Removed call to shell - Now prints status of task at task creation - Overwrite option for both images and tables --- README.md | 5 +++++ docs/changelog.md | 5 +++++ geeup/__init__.py | 2 +- geeup/batch_uploader.py | 17 ++++++----------- geeup/tuploader.py | 39 +++++++++++++++------------------------ setup.py | 2 +- 6 files changed, 33 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index ecc935f..97f13f8 100644 --- a/README.md +++ b/README.md @@ -339,6 +339,11 @@ optional arguments: # Changelog +### 0.6.2 +- Removed call to shell +- Now prints status of task at task creation +- Overwrite option for both images and tables + ### 0.6.1 - Removed dependency on pipwin uses pipgeo instead - Removed dependency on beautifulsoup diff --git a/docs/changelog.md b/docs/changelog.md index ac209c3..ba33006 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,10 @@ # Changelog +### 0.6.2 +- Removed call to shell +- Now prints status of task at task creation +- Overwrite option for both images and tables + ### 0.6.1 - Removed dependency on pipwin uses pipgeo instead - Removed dependency on beautifulsoup diff --git a/geeup/__init__.py b/geeup/__init__.py index cc0ad40..a52d817 100644 --- a/geeup/__init__.py +++ b/geeup/__init__.py @@ -2,4 +2,4 @@ __author__ = "Samapriya Roy" __email__ = "samapriya.roy@gmail.com" -__version__ = "0.6.1" +__version__ = "0.6.2" diff --git a/geeup/batch_uploader.py b/geeup/batch_uploader.py index 5f6ea11..877adc3 100644 --- a/geeup/batch_uploader.py +++ b/geeup/batch_uploader.py @@ -240,21 +240,16 @@ def upload( if v.validate(asset_validate, schema) is False: print(v.errors) raise Exception - with open(os.path.join(lp, "data.json"), "w") as outfile: - json.dump(main_payload, outfile) + request_id = ee.data.newTaskId()[0] check_list = ['yes', 'y'] if overwrite is not None and overwrite.lower() in check_list: - output = subprocess.check_output( - f"earthengine upload image --manifest {os.path.join(lp, 'data.json')} -f", - shell=True - ) + output = ee.data.startIngestion( + request_id, main_payload, allow_overwrite=True) else: - output = subprocess.check_output( - f"earthengine upload image --manifest {os.path.join(lp, 'data.json')}", - shell=True - ) + output = ee.data.startIngestion( + request_id, main_payload, allow_overwrite=False) logging.info( - f"Ingesting {current_image_no+1} of {file_count} {str(os.path.basename(asset_full_path))} with Task Id: {output.decode('ascii').strip().split(' ')[-1]}" + f"Ingesting {current_image_no+1} of {file_count} {str(os.path.basename(asset_full_path))} with Task Id: {output['id']} & status {output['started']}" ) except Exception as error: print(error) diff --git a/geeup/tuploader.py b/geeup/tuploader.py index d2d060a..3de6cfa 100644 --- a/geeup/tuploader.py +++ b/geeup/tuploader.py @@ -1,6 +1,6 @@ __copyright__ = """ - Copyright 2022 Samapriya Roy + Copyright 2023 Samapriya Roy Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -269,22 +269,15 @@ def tabup(dirc, uname, destination, x, y, overwrite=None): if v.validate(asset_validate, schema) is False: print(v.errors) raise Exception - with open( - os.path.join(lp, "data.json"), "w" - ) as outfile: - json.dump(main_payload, outfile) + request_id = ee.data.newTaskId()[0] if overwrite is not None and overwrite.lower() in check_list: - output = subprocess.check_output( - f"earthengine upload table --manifest {os.path.join(lp, 'data.json')} -f", - shell=True - ) + output = ee.data.startTableIngestion( + request_id, main_payload, allow_overwrite=True) else: - output = subprocess.check_output( - f"earthengine upload table --manifest {os.path.join(lp, 'data.json')}", - shell=True - ) + output = ee.data.startTableIngestion( + request_id, main_payload, allow_overwrite=False) logging.info( - f"Ingesting {i+1} of {file_count} {str(os.path.basename(asset_full_path))} with Task Id: {output.decode('ascii').strip().split(' ')[-1]}" + f"Ingesting {i+1} of {file_count} {str(os.path.basename(asset_full_path))} with Task Id: {output['id']} & status {output['started']}" ) elif base_ext == ".csv": m = MultipartEncoder( @@ -337,17 +330,15 @@ def tabup(dirc, uname, destination, x, y, overwrite=None): if v.validate(asset_validate, schema) is False: print(v.errors) raise Exception - with open( - os.path.join(lp, "data.json"), "w" - ) as outfile: - json.dump(main_payload, outfile) - manifest_file = os.path.join(lp, "data.json") - output = subprocess.check_output( - f"earthengine upload table --manifest {manifest_file}", - shell=True, - ) + request_id = ee.data.newTaskId()[0] + if overwrite is not None and overwrite.lower() in check_list: + output = ee.data.startTableIngestion( + request_id, main_payload, allow_overwrite=True) + else: + output = ee.data.startTableIngestion( + request_id, main_payload, allow_overwrite=False) logging.info( - f"Ingesting {i+1} of {file_count} {str(os.path.basename(asset_full_path))} with Task Id: {output.decode('ascii').strip().split(' ')[-1]}" + f"Ingesting {i+1} of {file_count} {str(os.path.basename(asset_full_path))} with Task Id: {output['id']} & status {output['started']}" ) except Exception as error: print(error) diff --git a/setup.py b/setup.py index 7b8b22c..17949b0 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ def readme(): setuptools.setup( name="geeup", - version="0.6.1", + version="0.6.2", packages=find_packages(), url="https://github.com/samapriya/geeup", install_requires=[