Skip to content

Commit

Permalink
fixes for #135 #128
Browse files Browse the repository at this point in the history 
For #135, addressing_style can be added into the .aws/credentials file, see below credential sample that was tested with minio server to use local storage systems #128

[minio]
aws_access_key_id = AKIAIOSFODNN7EXAMPLE
aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
aws_region = us-east-1
aws_endpoint_url=http://localhost:9000
addressing_style = path
  • Loading branch information
@Chamlika
Chamlika committed Sep 19, 2022
1 parent dc510df commit 25c3ae7
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 12 deletions.
41 changes: 29 additions & 12 deletions OptimizeRasters.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
# ------------------------------------------------------------------------------
# Name: OptimizeRasters.py
# Description: Optimizes rasters via gdal_translate/gdaladdo
# Version: 20220426
# Version: 20220919
# Requirements: Python
# Required Arguments: -input -output
# Optional Arguments: -mode -cache -config -quality -prec -pyramids
Expand Down Expand Up @@ -76,10 +76,11 @@ def getBooleanValue(value):
if (sys.version_info[0] < 3):
import ConfigParser
from urllib import urlopen, urlencode
from urlparse import urlparse
else:
import configparser as ConfigParser
from urllib.request import urlopen
from urllib.parse import urlencode
from urllib.parse import urlencode, urlparse
# ends

# enum error codes
Expand Down Expand Up @@ -320,6 +321,8 @@ def validateCredentials(self):
awsCredentials = ConfigParser.RawConfigParser()
rootPath = '.aws'
AwsEndpoint = 'aws_endpoint_url'
AwsAddressingStyle = 'addressing_style'
useAddrStyle = 'virtual'
if (self._credentialProfile):
if (self._storageType == self.TypeAlibaba):
rootPath = '.OptimizeRasters/Alibaba'
Expand All @@ -330,11 +333,15 @@ def validateCredentials(self):
return False
endPoint = awsCredentials.get(self._credentialProfile, AwsEndpoint) if awsCredentials.has_option(
self._credentialProfile, AwsEndpoint) else None
useAddrStyle = awsCredentials.get(self._credentialProfile, AwsAddressingStyle) if awsCredentials.has_option(
self._credentialProfile, AwsAddressingStyle) else useAddrStyle
if (AwsEndpoint in self._properties):
endPoint = self._properties[AwsEndpoint]
if (AwsAddressingStyle in self._properties):
useAddrStyle = self._properties[AwsAddressingStyle]
useAlibaba = endPoint and endPoint.lower().find(SigAlibaba) != -1
con = session.resource('s3', endpoint_url=endPoint, config=botocore.config.Config(
s3={'addressing_style': 'virtual'}))
s3={'addressing_style': useAddrStyle}))
# this will throw if credentials are invalid.
[self._availableBuckets.append(i.name)
for i in con.buckets.all()]
Expand Down Expand Up @@ -3239,18 +3246,25 @@ def init(self, remote_path, s3_key, s3_secret, direction):
endpointURL = None
AWSEndpointURL = 'aws_endpoint_url'
AWSRegion = 'region'
AWSAddressingStyle = 'addressing_style'
useAddrStyle = 'virtual'
SessionProfile = 'profiles'
region = DefS3Region
if (_profile_name and
SessionProfile in session._session.full_config and
_profile_name in session._session.full_config[SessionProfile]):
if (AWSEndpointURL in session._session.full_config[SessionProfile][_profile_name]):
endpointURL = session._session.full_config[
SessionProfile][_profile_name][AWSEndpointURL]
_profile = session._session.full_config[SessionProfile][_profile_name]
if (AWSAddressingStyle in _profile):
useAddrStyle = _profile[AWSAddressingStyle]
if (AWSEndpointURL in _profile):
endpointURL = _profile[AWSEndpointURL]
self._base.message('Using {} endpoint> {}'.format(
'output' if direction == CS3STORAGE_OUT else 'input', endpointURL))
if (AWSRegion in session._session.full_config[SessionProfile][_profile_name]):
region = session._session.full_config[SessionProfile][_profile_name][AWSRegion]
resp = urlparse(endpointURL)
os.environ['AWS_S3_ENDPOINT'] = resp.netloc
os.environ['AWS_HTTPS'] = 'false' # for GDAL to work with custom aws endpoints
if (AWSRegion in _profile):
region = _profile[AWSRegion]
profileCredentials = session.get_credentials()
if (profileCredentials):
# initialize access_key, secret_key using the profile.
Expand All @@ -3269,7 +3283,7 @@ def init(self, remote_path, s3_key, s3_secret, direction):
).secret_key
self.m_user_config.setValue('{}oss'.format(
'in' if direction == CS3STORAGE_IN else 'out'), useAlibaba)
bucketCon = session.client('s3')
bucketCon = session.client('s3', endpoint_url=endpointURL)
region = DefS3Region
try:
loc = bucketCon.get_bucket_location(Bucket=self.m_bucketname)[
Expand All @@ -3279,8 +3293,11 @@ def init(self, remote_path, s3_key, s3_secret, direction):
except Exception as e:
self._base.message(
'get/bucket/region ({})'.format(str(e)), self._base.const_warning_text)
if ('AWS_VIRTUAL_HOSTING' in os.environ): # AWS env overrides the .aws/credential entry
useAddrStyle = 'path' if not self._base.getBooleanValue(os.environ['AWS_VIRTUAL_HOSTING']) else 'virtual'
os.environ['AWS_VIRTUAL_HOSTING'] = 'false' if useAddrStyle == 'path' else 'true' # env must be set for GDAL
self.con = session.resource('s3', region, endpoint_url=endpointURL if endpointURL else None, config=botocore.config.Config(
s3={'addressing_style': 'virtual'}))
s3={'addressing_style': useAddrStyle}))
if (self._isBucketPublic):
self.con.meta.client.meta.events.register(
'choose-signer.s3.*', botocore.handlers.disable_signing)
Expand Down Expand Up @@ -5357,8 +5374,8 @@ def makedirs(filepath):


class Application(object):
__program_ver__ = 'v2.0.6i'
__program_date__ = '20220426'
__program_ver__ = 'v2.0.6j'
__program_date__ = '20220919'
__program_name__ = 'OptimizeRasters.py {}/{}'.format(
__program_ver__, __program_date__)
__program_desc__ = 'Convert raster formats to a valid output format through GDAL_Translate.\n' + \
Expand Down
Binary file modified Setup/OptimizeRastersToolsSetup.exe
View file
Binary file not shown.

0 comments on commit 25c3ae7

Please sign in to comment.