diff --git a/README.md b/README.md index 207835c..1d9fa1e 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,13 @@ http://doi.org/10.5281/zenodo.3066243 * [Installation](#installation) * [Getting started](#getting-started) * [geeup Simple CLI for Earth Engine Uploads](#geeup-simple-cli-for-earth-engine-uploads) - * [selenium update](#selenium-update) + * [geeup init](#geeup-init) * [gee Quota](#gee-quota) + * [gee getmeta](#gee-getmeta) * [gee Zipshape](#gee-zipshape) * [gee upload](#gee-upload) * [gee seltabup](#gee-seltabup) + * [gee selsetup](#gee-selsetup) * [gee tasks](#gee-tasks) * [gee delete](#gee-delete) @@ -33,6 +35,21 @@ This assumes that you have native python & pip installed in your system, you can If you get no errors and you have python 2.7.14 or higher you should be good to go. Please note that I have tested this only on python 2.7.15, but it should run on Python 3. +**This command line tool is dependent on shapely and fiona and as such uses functionality from GDAL** +For installing GDAL in Ubuntu +``` +sudo add-apt-repository ppa:ubuntugis/ppa && sudo apt-get update +sudo apt-get install gdal-bin +sudo apt-get install python-gdal +``` +For Windows I found this [guide](https://webcache.googleusercontent.com/search?q=cache:UZWc-pnCgwsJ:https://sandbox.idre.ucla.edu/sandbox/tutorials/installing-gdal-for-windows+&cd=4&hl=en&ct=clnk&gl=us) from UCLA + +You can also install the unofficial binary for windows for [gdal here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#gdal) + +Also for Ubuntu Linux I saw that this is necessary before the install + +```sudo apt install libcurl4-openssl-dev libssl-dev``` + **This also needs earthengine cli to be [installed and authenticated on your system](https://developers.google.com/earth-engine/python_install_manual) and earthengine to be callable in your command line or terminal** To install **geeup: Simple CLI for Earth Engine Uploads** you can install using two methods. @@ -69,8 +86,11 @@ positional arguments: quota Print Earth Engine total quota and used quota zipshape Zips all shapefiles and subsidary files into individual zip files + getmeta Generates generalized metadata for all rasters in folder upload Batch Asset Uploader using Selenium seltabup Batch Table Uploader using Selenium. + selsetup Non headless setup for new google account, use if upload + throws errors tasks Queries current task status [completed,running,ready,failed,cancelled] delete Deletes collection and all items inside. Supports @@ -86,10 +106,10 @@ To obtain help for specific functionality, simply call it with _help_ switch, e. ## geeup Simple CLI for Earth Engine Uploads The tool is designed to handle batch uploading of images and tables(shapefiles). While there are image collection where you can batch upload imagery, for vector or shapefiles you have to batch upload them to a folder. -### selenium update +### geeup init **This is a key step since all upload function depends on this step, so make sure you run this**. This downloads selenium driver and places to your local directory for windows and Linux subsystems. This is the first step to use selenium supported upload. -``` geeup update``` +``` geeup init``` ### gee Quota Just a simple tool to print your earth engine quota quickly. @@ -116,6 +136,21 @@ Required named arguments.: files if present in input will be zipped and stored ``` +### gee getmeta +This script generates a generalized metadata using information parsed from gdalinfo and metadata properties. For now it generates metadata with image name, x and y dimension of images, the pixel resolution and the number of bands. + +``` +usage: geeup getmeta [-h] --input INPUT --metadata METADATA + +optional arguments: + -h, --help show this help message and exit + +Required named arguments.: + --input INPUT Path to the input directory with all raster files + --metadata METADATA Full path to export metadata.csv file + +``` + ### gee upload The script creates an Image Collection from GeoTIFFs in your local directory. By default, the image name in the collection is the same as the local directory name; with the optional parameter you can provide a different name. @@ -155,6 +190,11 @@ Required named arguments.: -u USER, --user USER Google account name (gmail address). ``` +### gee selsetup +Once in a while the geckodriver requires manual input before signing into the google earth engine, this tool will allow you to interact with the initialization of Google Earth Engine code editor window. It allows the user to specify the account they want to use, and should only be needed once. + +```geeup selsetup``` + ### gee tasks This script counts all currently running, ready, completed, failed and canceled tasks along with failed tasks. This tool is linked to your google earth engine account with which you initialized the earth engine client. This takes no argument. @@ -180,6 +220,15 @@ optional arguments: ``` # Changelog +### v0.2.5 + +- Now allows for downloading geckodriver for macos Fix to [Issue 10](https://github.com/samapriya/geeup/issues/10) +- Now includes a metadata tool to generate a generalized metadata for any raster to allow upload. +Fix to [Issue 7](https://github.com/samapriya/geeup/issues/7) +- Changed from geeup update to init to signify initialization +- Added selsetup this tool allows for setting up the gecko driver with your account incase there are issues uploading +- Better error handling for selenium driver download + ### v0.2.4 - Made general improvements diff --git a/geeup/__init__.py b/geeup/__init__.py index 5434312..9e78a0a 100644 --- a/geeup/__init__.py +++ b/geeup/__init__.py @@ -2,4 +2,4 @@ __author__ = 'Samapriya Roy' __email__ = 'samapriya.roy@gmail.com' -__version__ = '0.2.4' +__version__ = '0.2.5' diff --git a/geeup/geeup.py b/geeup/geeup.py index af721b6..3e9b5c9 100644 --- a/geeup/geeup.py +++ b/geeup/geeup.py @@ -25,6 +25,7 @@ from batch_remover import delete from sel_tuploader import seltabup from zipfiles import zipshape +from getmeta import getmeta from os.path import expanduser lpath=os.path.dirname(os.path.realpath(__file__)) sys.path.append(lpath) @@ -34,11 +35,19 @@ def update(): os.system("python sel-latest-win.py") elif str(platform.system()) =="Linux": os.system("python sel-latest-linux.py") + elif str(platform.system()) =="darwin": + os.system("python sel-latest-mac.py") else: print("Architecture not recognized") -def update_from_parser(args): +def init_from_parser(args): update() +def selsetup(): + os.system("python sel_setup.py") + +def selsetup_from_parser(args): + selsetup() + suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] def humansize(nbytes): i = 0 @@ -60,6 +69,9 @@ def quota_from_parser(args): def zipshape_from_parser(args): zipshape(directory=args.input,export=args.output) +def getmeta_from_parser(args): + getmeta(indir=args.input,mfile=args.metadata) + def _comma_separated_strings(string): """Parses an input consisting of comma-separated strings. Slightly modified version of: https://pypkg.com/pypi/earthengine-api/f/ee/cli/commands.py @@ -106,8 +118,11 @@ def main(args=None): subparsers = parser.add_subparsers() - parser_update=subparsers.add_parser('update',help='Updates Selenium drivers for firefox [windows or linux systems]') - parser_update.set_defaults(func=update_from_parser) + parser_init=subparsers.add_parser('init',help='Initializes the tool by downloading and updating selenium drivers for firefox') + parser_init.set_defaults(func=init_from_parser) + + parser_selsetup = subparsers.add_parser('selsetup', help='Non headless setup for new google account, use if upload throws errors') + parser_selsetup.set_defaults(func=selsetup_from_parser) parser_quota = subparsers.add_parser('quota', help='Print Earth Engine total quota and used quota') parser_quota.set_defaults(func=quota_from_parser) @@ -118,6 +133,12 @@ def main(args=None): required_named.add_argument('--output', help='Destination folder Full path where shp, shx, prj and dbf files if present in input will be zipped and stored', required=True) parser_zipshape.set_defaults(func=zipshape_from_parser) + parser_getmeta = subparsers.add_parser('getmeta', help='Creates a generalized metadata for rasters in folder') + required_named = parser_getmeta.add_argument_group('Required named arguments.') + required_named.add_argument('--input', help='Path to the input directory with all raster files', required=True) + required_named.add_argument('--metadata', help='Full path to export metadata.csv file', required=True) + parser_getmeta.set_defaults(func=getmeta_from_parser) + parser_upload = subparsers.add_parser('upload', help='Batch Asset Uploader using Selenium') required_named = parser_upload.add_argument_group('Required named arguments.') required_named.add_argument('--source', help='Path to the directory with images for upload.', required=True) diff --git a/geeup/getmeta.py b/geeup/getmeta.py new file mode 100644 index 0000000..7d23705 --- /dev/null +++ b/geeup/getmeta.py @@ -0,0 +1,38 @@ +from __future__ import print_function +try: + from osgeo import gdal +except ImportError: + import gdal +import os +import math +import csv + + +def getmeta(indir,mfile): + i=1 + flength=len([name for name in os.listdir(indir) if name.endswith('.tif')]) + with open(mfile,'w') as csvfile: + writer=csv.DictWriter(csvfile,fieldnames=["id_no", "xsize", "ysize", "pixel_resolution","num_bands"], delimiter=',') + writer.writeheader() + for filename in os.listdir(indir): + if filename.endswith('.tif'): + gtif = gdal.Open(os.path.join(indir,filename)) + try: + print("Processed: "+str(i)+ ' of '+str(flength), end='\r') + fname=(os.path.basename(gtif.GetDescription()).split('.')[0]) + xsize=(gtif.RasterXSize) + ysize=(gtif.RasterYSize) + ulx, xres, xskew, uly, yskew, yres = gtif.GetGeoTransform() + stepper = 10.0 ** 2 + res=(math.trunc(stepper * xres) / stepper) + bsize=(gtif.RasterCount) + with open(mfile,'a') as csvfile: + writer=csv.writer(csvfile,delimiter=',',lineterminator='\n') + writer.writerow([fname,xsize,ysize,res,bsize]) + csvfile.close() + i=i+1 + except Exception as e: + print(e) + i=i+1 +# getmeta(indir=r'C:\planet_demo\dbwater\water_mask_2017_v1', +# mfile=r'C:\planet_demo\rmeta.csv') diff --git a/geeup/sel-latest-mac.py b/geeup/sel-latest-mac.py new file mode 100644 index 0000000..7359745 --- /dev/null +++ b/geeup/sel-latest-mac.py @@ -0,0 +1,33 @@ +from bs4 import BeautifulSoup +import requests,csv,zipfile,os,platform,tarfile +from pathlib import Path +from pySmartDL import SmartDL +sysinfo=platform.machine()[-2:] +#comb="win"+str(sysinfo)+".zip" +comb="macos.tar.gz" +directory=os.path.dirname(os.path.realpath(__file__)) +os.chdir(os.path.dirname(os.path.realpath(__file__))) +def geckodown(directory): + source=requests.get("https://github.com/mozilla/geckodriver/releases/latest").text + soup=BeautifulSoup(source.encode("utf-8"),'lxml') + vr=str(soup.title.text.encode("utf-8")).split(' ')[1] + container="https://github.com/mozilla/geckodriver/releases/download/"+vr+"/geckodriver-"+vr+'-'+comb + print("Downloading from: "+str(container)) + try: + url = container + dest = directory + obj = SmartDL(url, dest) + obj.start() + path=obj.get_dest() + print(os.path.join(directory,'geckodriver-'+vr+'-linux64.zip')) + filepath=os.path.join(directory,'geckodriver-'+vr+'-'+comb) + if (filepath.endswith("tar.gz")): + tar = tarfile.open(filepath,'r:*') + tar.extractall(directory) + tar.close() + #print "Extracted in Current Directory" + print("Use selenium driver path as "+os.path.join(directory,"geckodriver")) + except Exception as e: + print('Issues updating with error '+str(e)) + +geckodown(directory=directory) diff --git a/geeup/sel_setup.py b/geeup/sel_setup.py new file mode 100644 index 0000000..6a630ea --- /dev/null +++ b/geeup/sel_setup.py @@ -0,0 +1,41 @@ +import requests +import time +import os +import getpass +from selenium import webdriver + + +pathway=os.path.dirname(os.path.realpath(__file__)) +def authenticate(): + authorization_url="https://code.earthengine.google.com" + uname=str(raw_input("Enter your Username: ")) + passw=str(getpass.getpass("Enter your Password: ")) + driver = webdriver.Firefox(executable_path=os.path.join(pathway,"geckodriver.exe")) + driver.get(authorization_url) + time.sleep(2) + try: + username = driver.find_element_by_xpath('//*[@id="identifierId"]') + username.send_keys(uname) + driver.find_element_by_id("identifierNext").click() + time.sleep(5) + passw=driver.find_element_by_name("password").send_keys(passw) + driver.find_element_by_id("passwordNext").click() + time.sleep(5) + driver.find_element_by_xpath("//div[@id='view_container']/form/div[2]/div/div/div/ul/li/div/div[2]/p").click() + time.sleep(5) + driver.find_element_by_xpath("//div[@id='submit_approve_access']/content/span").click() + time.sleep(5) + driver.find_element_by_xpath("(.//*[normalize-space(text()) and normalize-space(.)='terms of service'])[1]/following::span[2]").click() + time.sleep(3) + driver.find_element_by_id("profileIdentifier").click() + time.sleep(2) + driver.find_element_by_xpath("(.//*[normalize-space(text()) and normalize-space(.)='Earth Engine Code Editor'])[1]/following::div[13]").click() + except Exception as e: + pass + cookies = driver.get_cookies() + s = requests.Session() + for cookie in cookies: + s.cookies.set(cookie['name'], cookie['value']) + print('\n'+'Selenium Setup complete with Google Profile') + driver.close() +authenticate() diff --git a/setup.py b/setup.py index 432b2f8..5b7e186 100644 --- a/setup.py +++ b/setup.py @@ -18,10 +18,10 @@ def readme(): return f.read() setuptools.setup( name='geeup', - version='0.2.2', + version='0.2.5', packages=find_packages(), url='https://github.com/samapriya/geeup', - install_requires=['earthengine_api >= 0.1.87','requests >= 2.10.0','retrying >= 1.3.3','beautifulsoup4 >= 4.5.1', + install_requires=['earthengine_api >= 0.1.175','requests >= 2.10.0','retrying >= 1.3.3','beautifulsoup4 >= 4.5.1', 'pandas>=0.23.0','psutil>=5.4.5','requests_toolbelt >= 0.7.0','pytest >= 3.0.0','future >= 0.16.0', 'google-cloud-storage >= 1.1.1','selenium>=3.13.0','pySmartDL==1.2.5;python_version<"3.4"', 'pySmartDL>=1.3.1;python_version>"3.4"','pathlib>=1.0.1','lxml>=4.1.1','oauth2client>=4.1.3'],